From 541bf419233a70285a54c894645e88606e4d848d Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Mon, 16 Mar 2020 15:43:54 -0700 Subject: [PATCH 01/85] vrepl: where clause filtering WIP Signed-off-by: Sugu Sougoumarane --- .../tabletserver/vstreamer/planbuilder.go | 165 +++++++++++++++--- 1 file changed, 140 insertions(+), 25 deletions(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index c084327c877..6441cc01c2b 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -36,10 +36,30 @@ import ( // Plan represents the plan for a table. type Plan struct { Table *Table + + // Filters is the list of filters to be applied to the columns + // of the table. + Filters []Filter + // ColExprs is the list of column expressions to be sent // in the stream. ColExprs []ColExpr +} + +type Opcode int + +const ( + Equal = Opcode(iota) + VindexMatch +) + +// Filter contains opcodes for filtering. +type Filter struct { + Opcode Opcode + ColNum int + Value sqltypes.Value + // Parameters for VindexMatch. // Vindex, VindexColumns and KeyRange, if set, will be used // to filter the row. // VindexColumns contains the column numbers of the table, @@ -90,13 +110,24 @@ func (plan *Plan) fields() []*querypb.Field { // filter filters the row against the plan. It returns false if the row did not match. // If the row matched, it returns the columns to be sent. func (plan *Plan) filter(values []sqltypes.Value) (bool, []sqltypes.Value, error) { - if plan.Vindex != nil { - ksid, err := getKeyspaceID(values, plan.Vindex, plan.VindexColumns) - if err != nil { - return false, nil, err - } - if !key.KeyRangeContains(plan.KeyRange, ksid) { - return false, nil, nil + for _, filter := range plan.Filters { + switch filter.Opcode { + case Equal: + result, err := sqltypes.NullsafeCompare(values[filter.ColNum], filter.Value) + if err != nil { + return false, nil, err + } + if result != 0 { + return false, nil, nil + } + case VindexMatch: + ksid, err := getKeyspaceID(values, filter.Vindex, filter.VindexColumns) + if err != nil { + return false, nil, err + } + if !key.KeyRangeContains(filter.KeyRange, ksid) { + return false, nil, nil + } } } @@ -242,8 +273,11 @@ func buildREPlan(ti *Table, vschema *localVSchema, filter string) (*Plan, error) if err != nil { return nil, err } - plan.Vindex = cv.Vindex - plan.VindexColumns, err = buildVindexColumns(plan.Table, cv.Columns) + whereFilter := Filter{ + Opcode: VindexMatch, + Vindex: cv.Vindex, + } + whereFilter.VindexColumns, err = buildVindexColumns(plan.Table, cv.Columns) if err != nil { return nil, err } @@ -256,7 +290,8 @@ func buildREPlan(ti *Table, vschema *localVSchema, filter string) (*Plan, error) if len(keyranges) != 1 { return nil, fmt.Errorf("error parsing keyrange: %v", filter) } - plan.KeyRange = keyranges[0] + whereFilter.KeyRange = keyranges[0] + plan.Filters = append(plan.Filters, whereFilter) return plan, nil } @@ -274,6 +309,9 @@ func buildTablePlan(ti *Table, vschema *localVSchema, query string) (*Plan, erro plan := &Plan{ Table: ti, } + if err := plan.analyzeWhere(vschema, sel.Where); err != nil { + return nil, err + } if err := plan.analyzeExprs(vschema, sel.SelectExprs); err != nil { return nil, err } @@ -282,16 +320,6 @@ func buildTablePlan(ti *Table, vschema *localVSchema, query string) (*Plan, erro return plan, nil } - funcExpr, ok := sel.Where.Expr.(*sqlparser.FuncExpr) - if !ok { - return nil, fmt.Errorf("unsupported where clause: %v", sqlparser.String(sel.Where)) - } - if !funcExpr.Name.EqualString("in_keyrange") { - return nil, fmt.Errorf("unsupported where clause: %v", sqlparser.String(sel.Where)) - } - if err := plan.analyzeInKeyRange(vschema, funcExpr.Exprs); err != nil { - return nil, err - } return plan, nil } @@ -318,6 +346,89 @@ func analyzeSelect(query string) (sel *sqlparser.Select, fromTable sqlparser.Tab return sel, fromTable, nil } +func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) error { + if where == nil { + return nil + } + exprs := splitAndExpression(nil, where.Expr) + for _, expr := range exprs { + switch expr := expr.(type) { + case *sqlparser.ComparisonExpr: + qualifiedName, ok := expr.Left.(*sqlparser.ColName) + if !ok { + return fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + } + if !qualifiedName.Qualifier.IsEmpty() { + return fmt.Errorf("unsupported qualifier for column: %v", sqlparser.String(qualifiedName)) + } + colnum, err := findColumn(plan.Table, qualifiedName.Name) + if err != nil { + return err + } + val, ok := expr.Right.(*sqlparser.SQLVal) + if !ok { + return fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + } + if val.Type != sqlparser.IntVal { + return fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + } + pv, err := sqlparser.NewPlanValue(val) + if err != nil { + return err + } + resolved, err := pv.ResolveValue(nil) + if err != nil { + return err + } + plan.Filters = append(plan.Filters, Filter{ + Opcode: Equal, + ColNum: colnum, + Value: resolved, + }) + case *sqlparser.FuncExpr: + if !expr.Name.EqualString("in_keyrange") { + return fmt.Errorf("unsupported constraint: %v", sqlparser.String(expr)) + } + if err := plan.analyzeInKeyRange(vschema, expr.Exprs); err != nil { + return err + } + default: + return fmt.Errorf("unsupported constraint: %v", sqlparser.String(expr)) + } + } + return nil +} + +// splitAndExpression breaks up the Expr into AND-separated conditions +// and appends them to filters, which can be shuffled and recombined +// as needed. +func splitAndExpression(filters []sqlparser.Expr, node sqlparser.Expr) []sqlparser.Expr { + if node == nil { + return filters + } + switch node := node.(type) { + case *sqlparser.AndExpr: + filters = splitAndExpression(filters, node.Left) + return splitAndExpression(filters, node.Right) + case *sqlparser.ParenExpr: + // If the inner expression is AndExpr, then we can remove + // the parenthesis because they are unnecessary. + if node, ok := node.Expr.(*sqlparser.AndExpr); ok { + return splitAndExpression(filters, node) + } + } + return append(filters, node) +} + +// skipParenthesis skips the parenthesis (if any) of an expression and +// returns the innermost unparenthesized expression. +func skipParenthesis(node sqlparser.Expr) sqlparser.Expr { + if node, ok := node.(*sqlparser.ParenExpr); ok { + return skipParenthesis(node.Expr) + } + return node +} + func (plan *Plan) analyzeExprs(vschema *localVSchema, selExprs sqlparser.SelectExprs) error { if _, ok := selExprs[0].(*sqlparser.StarExpr); !ok { for _, expr := range selExprs { @@ -396,6 +507,9 @@ func (plan *Plan) analyzeExpr(vschema *localVSchema, selExpr sqlparser.SelectExp func (plan *Plan) analyzeInKeyRange(vschema *localVSchema, exprs sqlparser.SelectExprs) error { var colnames []sqlparser.ColIdent var krExpr sqlparser.SelectExpr + whereFilter := Filter{ + Opcode: VindexMatch, + } switch { case len(exprs) == 1: cv, err := vschema.FindColVindex(plan.Table.Name) @@ -403,7 +517,7 @@ func (plan *Plan) analyzeInKeyRange(vschema *localVSchema, exprs sqlparser.Selec return err } colnames = cv.Columns - plan.Vindex = cv.Vindex + whereFilter.Vindex = cv.Vindex krExpr = exprs[0] case len(exprs) >= 3: for _, expr := range exprs[:len(exprs)-2] { @@ -425,11 +539,11 @@ func (plan *Plan) analyzeInKeyRange(vschema *localVSchema, exprs sqlparser.Selec if err != nil { return err } - plan.Vindex, err = vschema.FindOrCreateVindex(vtype) + whereFilter.Vindex, err = vschema.FindOrCreateVindex(vtype) if err != nil { return err } - if !plan.Vindex.IsUnique() { + if !whereFilter.Vindex.IsUnique() { return fmt.Errorf("vindex must be Unique to be used for VReplication: %s", vtype) } @@ -438,7 +552,7 @@ func (plan *Plan) analyzeInKeyRange(vschema *localVSchema, exprs sqlparser.Selec return fmt.Errorf("unexpected in_keyrange parameters: %v", sqlparser.String(exprs)) } var err error - plan.VindexColumns, err = buildVindexColumns(plan.Table, colnames) + whereFilter.VindexColumns, err = buildVindexColumns(plan.Table, colnames) if err != nil { return err } @@ -453,7 +567,8 @@ func (plan *Plan) analyzeInKeyRange(vschema *localVSchema, exprs sqlparser.Selec if len(keyranges) != 1 { return fmt.Errorf("unexpected in_keyrange parameter: %v", sqlparser.String(krExpr)) } - plan.KeyRange = keyranges[0] + whereFilter.KeyRange = keyranges[0] + plan.Filters = append(plan.Filters, whereFilter) return nil } From 48d380ea47ac40ea2c2f8a954212385df105abe1 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Wed, 18 Mar 2020 23:14:25 +0100 Subject: [PATCH 02/85] Update test cases for where clause Signed-off-by: Rohit Nayak --- .../tabletserver/vstreamer/planbuilder.go | 13 ++-- .../vstreamer/planbuilder_test.go | 77 +++++++++++++++---- 2 files changed, 71 insertions(+), 19 deletions(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index 6441cc01c2b..c6474d1b8ec 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -37,13 +37,14 @@ import ( type Plan struct { Table *Table + // ColExprs is the list of column expressions to be sent + // in the stream. + ColExprs []ColExpr + // Filters is the list of filters to be applied to the columns // of the table. Filters []Filter - // ColExprs is the list of column expressions to be sent - // in the stream. - ColExprs []ColExpr } type Opcode int @@ -381,9 +382,9 @@ func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) er return err } plan.Filters = append(plan.Filters, Filter{ - Opcode: Equal, - ColNum: colnum, - Value: resolved, + Opcode: Equal, + ColNum: colnum, + Value: resolved, }) case *sqlparser.FuncExpr: if !expr.Name.EqualString("in_keyrange") { diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go index 0ff9bfea5fa..e9ac90e98fa 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go @@ -245,7 +245,14 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("val"), Type: sqltypes.VarBinary, }}, - VindexColumns: []int{0}, + Filters: []Filter {{ + Opcode: VindexMatch, + ColNum: 0, + Value: sqltypes.NULL, + Vindex: nil, + VindexColumns: []int{0}, + KeyRange: nil, + }}, }, }, { inTable: t1, @@ -302,7 +309,14 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - VindexColumns: []int{0}, + Filters: []Filter {{ + Opcode: VindexMatch, + ColNum: 0, + Value: sqltypes.NULL, + Vindex: nil, + VindexColumns: []int{0}, + KeyRange: nil, + }}, }, }, { inTable: t1, @@ -317,7 +331,36 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - VindexColumns: []int{0}, + Filters: []Filter {{ + Opcode: VindexMatch, + ColNum: 0, + Value: sqltypes.NULL, + Vindex: nil, + VindexColumns: []int{0}, + KeyRange: nil, + }}, + }, + },{ + inTable: t1, + inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select val, id from t1 where id = 1"}, + outPlan: &Plan{ + ColExprs: []ColExpr{{ + ColNum: 1, + Alias: sqlparser.NewColIdent("val"), + Type: sqltypes.VarBinary, + }, { + ColNum: 0, + Alias: sqlparser.NewColIdent("id"), + Type: sqltypes.Int64, + }}, + Filters: []Filter {{ + Opcode: Equal, + ColNum: 0, + Value: sqltypes.NewInt64(1), + Vindex: nil, + VindexColumns: nil, + KeyRange: nil, + }}, }, }, { inTable: t2, @@ -335,7 +378,14 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - VindexColumns: []int{0, 1}, + Filters: []Filter {{ + Opcode: VindexMatch, + ColNum: 0, + Value: sqltypes.NULL, + Vindex: nil, + VindexColumns: []int{0,1}, + KeyRange: nil, + }}, }, }, { inTable: regional, @@ -400,14 +450,10 @@ func TestPlanbuilder(t *testing.T) { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select *, id from t1"}, outErr: `unsupported: *, id`, - }, { - inTable: t1, - inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val from t1 where id=1"}, - outErr: `unsupported where clause: where id = 1`, - }, { + },{ inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val from t1 where max(id)"}, - outErr: `unsupported where clause: where max(id)`, + outErr: `unsupported constraint: max(id)`, }, { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val from t1 where in_keyrange(id)"}, @@ -463,15 +509,20 @@ func TestPlanbuilder(t *testing.T) { inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val from t1 where in_keyrange(id, 1+1, '-80')"}, outErr: `unsupported: 1 + 1`, }} - for _, tcase := range testcases { plan, err := buildPlan(tcase.inTable, testLocalVSchema, &binlogdatapb.Filter{ Rules: []*binlogdatapb.Rule{tcase.inRule}, }) if plan != nil { plan.Table = nil - plan.Vindex = nil - plan.KeyRange = nil + for ind, _ := range plan.Filters { + plan.Filters[ind].KeyRange = nil + if plan.Filters[ind]. + Opcode == VindexMatch { + plan.Filters[ind].Value = sqltypes.NULL + } + plan.Filters[ind].Vindex = nil + } if !reflect.DeepEqual(tcase.outPlan, plan) { t.Errorf("Plan(%v, %v):\n%v, want\n%v", tcase.inTable, tcase.inRule, plan, tcase.outPlan) } From 073d3333380ac35b994dca3294bf0f5e02b89142 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 19 Mar 2020 18:30:54 +0100 Subject: [PATCH 03/85] Test case for int filter Signed-off-by: Rohit Nayak --- .../tabletserver/vstreamer/planbuilder.go | 19 +++---- .../vstreamer/planbuilder_test.go | 18 +++---- .../tabletserver/vstreamer/vstreamer_test.go | 53 +++++++++++++++++++ 3 files changed, 68 insertions(+), 22 deletions(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index c6474d1b8ec..918c2d421b5 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -44,13 +44,15 @@ type Plan struct { // Filters is the list of filters to be applied to the columns // of the table. Filters []Filter - } +// Opcode enumerates the operators supported in a where clause type Opcode int const ( + // Equal is used to filter an integer column on a specific value Equal = Opcode(iota) + // VindexMatch is used for an in_keyrange() construct VindexMatch ) @@ -382,9 +384,9 @@ func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) er return err } plan.Filters = append(plan.Filters, Filter{ - Opcode: Equal, - ColNum: colnum, - Value: resolved, + Opcode: Equal, + ColNum: colnum, + Value: resolved, }) case *sqlparser.FuncExpr: if !expr.Name.EqualString("in_keyrange") { @@ -421,15 +423,6 @@ func splitAndExpression(filters []sqlparser.Expr, node sqlparser.Expr) []sqlpars return append(filters, node) } -// skipParenthesis skips the parenthesis (if any) of an expression and -// returns the innermost unparenthesized expression. -func skipParenthesis(node sqlparser.Expr) sqlparser.Expr { - if node, ok := node.(*sqlparser.ParenExpr); ok { - return skipParenthesis(node.Expr) - } - return node -} - func (plan *Plan) analyzeExprs(vschema *localVSchema, selExprs sqlparser.SelectExprs) error { if _, ok := selExprs[0].(*sqlparser.StarExpr); !ok { for _, expr := range selExprs { diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go index e9ac90e98fa..fa0a44ec1b0 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go @@ -245,7 +245,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("val"), Type: sqltypes.VarBinary, }}, - Filters: []Filter {{ + Filters: []Filter{{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, @@ -309,7 +309,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter {{ + Filters: []Filter{{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, @@ -331,7 +331,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter {{ + Filters: []Filter{{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, @@ -340,7 +340,7 @@ func TestPlanbuilder(t *testing.T) { KeyRange: nil, }}, }, - },{ + }, { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select val, id from t1 where id = 1"}, outPlan: &Plan{ @@ -353,7 +353,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter {{ + Filters: []Filter{{ Opcode: Equal, ColNum: 0, Value: sqltypes.NewInt64(1), @@ -378,12 +378,12 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter {{ + Filters: []Filter{{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, Vindex: nil, - VindexColumns: []int{0,1}, + VindexColumns: []int{0, 1}, KeyRange: nil, }}, }, @@ -450,7 +450,7 @@ func TestPlanbuilder(t *testing.T) { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select *, id from t1"}, outErr: `unsupported: *, id`, - },{ + }, { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val from t1 where max(id)"}, outErr: `unsupported constraint: max(id)`, @@ -515,7 +515,7 @@ func TestPlanbuilder(t *testing.T) { }) if plan != nil { plan.Table = nil - for ind, _ := range plan.Filters { + for ind := range plan.Filters { plan.Filters[ind].KeyRange = nil if plan.Filters[ind]. Opcode == VindexMatch { diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go index 5b9edf50860..572c07da983 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go @@ -37,6 +37,59 @@ type testcase struct { output [][]string } +func TestFilteredInt(t *testing.T) { + if testing.Short() { + t.Skip() + } + + execStatements(t, []string{ + "create table t1(id1 int, id2 int, val varbinary(128), primary key(id1))", + }) + defer execStatements(t, []string{ + "drop table t1", + }) + engine.se.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select id1, val from t1 where id2 = 200", + }}, + } + + testcases := []testcase{{ + input: []string{ + "begin", + "insert into t1 values (1, 100, 'aaa')", + "insert into t1 values (2, 200, 'bbb')", + "insert into t1 values (3, 100, 'ccc')", + "insert into t1 values (4, 200, 'ddd')", + "insert into t1 values (5, 200, 'eee')", + "update t1 set val = 'newddd' where id1 = 4", + "update t1 set id2 = 200 where id1 = 1", + "update t1 set id2 = 100 where id1 = 2", + "update t1 set id2 = 100 where id1 = 1", + "update t1 set id2 = 200 where id1 = 2", + "commit", + }, + output: [][]string{{ + `begin`, + `type:FIELD field_event: fields: > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: after: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `gtid`, + `commit`, + }}, + }} + runCases(t, filter, testcases, "") +} + func TestStatements(t *testing.T) { if testing.Short() { t.Skip() From c141cc0fe57247c3a7254fc3c377483881865205 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 19 Mar 2020 23:09:20 +0100 Subject: [PATCH 04/85] Added rowstreamer test for filtered vreplication Signed-off-by: Rohit Nayak --- .../vstreamer/rowstreamer_test.go | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go index ab294562dde..63fb44a3168 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go @@ -210,6 +210,36 @@ func TestStreamRowsKeyRange(t *testing.T) { checkStream(t, "select * from t1 where in_keyrange('-80')", nil, wantQuery, wantStream) } +func TestStreamRowsFilterInt(t *testing.T) { + if testing.Short() { + t.Skip() + } + + if err := env.SetVSchema(shardedVSchema); err != nil { + t.Fatal(err) + } + defer env.SetVSchema("{}") + + execStatements(t, []string{ + "create table t1(id1 int, id2 int, val varbinary(128), primary key(id1))", + "insert into t1 values (1, 100, 'aaa'), (2, 200, 'bbb'), (3, 200, 'ccc'), (4, 100, 'ddd'), (5, 200, 'eee')", + }) + defer execStatements(t, []string{ + "drop table t1", + }) + engine.se.Reload(context.Background()) + + time.Sleep(1 * time.Second) + + // Only the first row should be returned, but lastpk should be 6. + wantStream := []string{ + `fields: fields: pkfields: `, + `rows: rows: lastpk: `, + } + wantQuery := "select id1, id2, val from t1 order by id1" + checkStream(t, "select id1, val from t1 where id2 = 100", nil, wantQuery, wantStream) +} + func TestStreamRowsMultiPacket(t *testing.T) { if testing.Short() { t.Skip() From b601056d2a7a93b724a2391810b3705834096bdc Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Fri, 20 Mar 2020 11:05:37 +0100 Subject: [PATCH 05/85] Added support for varbinary, tests for both int/varbinary Signed-off-by: Rohit Nayak --- .../tabletserver/vstreamer/planbuilder.go | 25 ++++++++- .../vstreamer/rowstreamer_test.go | 30 +++++++++++ .../tabletserver/vstreamer/vstreamer.go | 4 +- .../tabletserver/vstreamer/vstreamer_test.go | 53 +++++++++++++++++++ 4 files changed, 108 insertions(+), 4 deletions(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index 918c2d421b5..2d4d43819f7 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -110,13 +110,33 @@ func (plan *Plan) fields() []*querypb.Field { return fields } +func resolveValue(val sqltypes.Value) (sqltypes.Value, error) { + pv, err := sqlparser.NewPlanValue(sqlparser.NewStrVal(val.Raw())) + if err != nil { + return sqltypes.NULL, err + } + resolved, err := pv.ResolveValue(nil) + if err != nil { + return sqltypes.NULL, err + } + return resolved, nil +} + // filter filters the row against the plan. It returns false if the row did not match. // If the row matched, it returns the columns to be sent. func (plan *Plan) filter(values []sqltypes.Value) (bool, []sqltypes.Value, error) { for _, filter := range plan.Filters { switch filter.Opcode { case Equal: - result, err := sqltypes.NullsafeCompare(values[filter.ColNum], filter.Value) + filterValue, err := resolveValue(filter.Value) + if err != nil { + return false, nil, err + } + colValue, err := resolveValue(values[filter.ColNum]) + if err != nil { + return false, nil, err + } + result, err := sqltypes.NullsafeCompare(colValue, filterValue) if err != nil { return false, nil, err } @@ -372,7 +392,8 @@ func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) er if !ok { return fmt.Errorf("unexpected: %v", sqlparser.String(expr)) } - if val.Type != sqlparser.IntVal { + //StrVal is varbinary, we do not support varchar since we would have to implement all collation types + if val.Type != sqlparser.IntVal && val.Type != sqlparser.StrVal { return fmt.Errorf("unexpected: %v", sqlparser.String(expr)) } pv, err := sqlparser.NewPlanValue(val) diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go index 63fb44a3168..8b2c4a39007 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go @@ -240,6 +240,36 @@ func TestStreamRowsFilterInt(t *testing.T) { checkStream(t, "select id1, val from t1 where id2 = 100", nil, wantQuery, wantStream) } +func TestStreamRowsFilterVarBinary(t *testing.T) { + if testing.Short() { + t.Skip() + } + + if err := env.SetVSchema(shardedVSchema); err != nil { + t.Fatal(err) + } + defer env.SetVSchema("{}") + + execStatements(t, []string{ + "create table t1(id1 int, val varbinary(128), primary key(id1))", + "insert into t1 values (1,'kepler'), (2, 'newton'), (3, 'newton'), (4, 'kepler'), (5, 'newton'), (6, 'kepler')", + }) + defer execStatements(t, []string{ + "drop table t1", + }) + engine.se.Reload(context.Background()) + + time.Sleep(1 * time.Second) + + // Only the first row should be returned, but lastpk should be 6. + wantStream := []string{ + `fields: fields: pkfields: `, + `rows: rows: rows: lastpk: `, + } + wantQuery := "select id1, val from t1 order by id1" + checkStream(t, "select id1, val from t1 where val = 'newton'", nil, wantQuery, wantStream) +} + func TestStreamRowsMultiPacket(t *testing.T) { if testing.Short() { t.Skip() diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 7f99791be36..08c5ea5aa64 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -545,7 +545,7 @@ func (vs *vstreamer) buildJournalPlan(id uint64, tm *mysql.TableMap) error { return err } if len(st.Columns) < len(tm.Types) { - return fmt.Errorf("cannot determine table columns for %s: event has %v, schema as %v", tm.Name, tm.Types, st.Columns) + return fmt.Errorf("cannot determine table columns for %s: event has %v, schema has %v", tm.Name, tm.Types, st.Columns) } table := &Table{ Name: "_vt.resharding_journal", @@ -620,7 +620,7 @@ func (vs *vstreamer) buildTableColumns(id uint64, tm *mysql.TableMap) ([]schema. if len(st.Columns) < len(tm.Types) { if vs.filter.FieldEventMode == binlogdatapb.Filter_ERR_ON_MISMATCH { - return nil, fmt.Errorf("cannot determine table columns for %s: event has %v, schema as %v", tm.Name, tm.Types, st.Columns) + return nil, fmt.Errorf("cannot determine table columns for %s: event has %v, schema has %v", tm.Name, tm.Types, st.Columns) } return cols, nil } diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go index 572c07da983..c693b9efc2e 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go @@ -37,6 +37,59 @@ type testcase struct { output [][]string } +func TestFilteredVarBinary(t *testing.T) { + if testing.Short() { + t.Skip() + } + + execStatements(t, []string{ + "create table t1(id1 int, val varbinary(128), primary key(id1))", + }) + defer execStatements(t, []string{ + "drop table t1", + }) + engine.se.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select id1, val from t1 where val = 'newton'", + }}, + } + + testcases := []testcase{{ + input: []string{ + "begin", + "insert into t1 values (1, 'kepler')", + "insert into t1 values (2, 'newton')", + "insert into t1 values (3, 'newton')", + "insert into t1 values (4, 'kepler')", + "insert into t1 values (5, 'newton')", + "update t1 set val = 'newton' where id1 = 1", + "update t1 set val = 'kepler' where id1 = 2", + "update t1 set val = 'newton' where id1 = 2", + "update t1 set val = 'kepler' where id1 = 1", + "delete from t1 where id1 in (2,3)", + "commit", + }, + output: [][]string{{ + `begin`, + `type:FIELD field_event: fields: > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > row_changes: > > `, + `gtid`, + `commit`, + }}, + }} + runCases(t, filter, testcases, "") +} + func TestFilteredInt(t *testing.T) { if testing.Short() { t.Skip() From 8f4277e232effeb8fa138cbfb84c9945c83b258e Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sat, 21 Mar 2020 00:07:55 +0100 Subject: [PATCH 06/85] Handle varbinary correctly Signed-off-by: Rohit Nayak --- go/mysql/binlog_event_rbr.go | 5 +++-- .../tabletserver/vstreamer/planbuilder.go | 22 +------------------ .../vstreamer/rowstreamer_test.go | 2 -- .../tabletserver/vstreamer/vstreamer.go | 1 + 4 files changed, 5 insertions(+), 25 deletions(-) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index e29e3f0fabe..92866f63859 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -447,15 +447,16 @@ func CellValue(data []byte, pos int, typ byte, metadata uint16, styp querypb.Typ return sqltypes.MakeTrusted(querypb.Type_DATETIME, []byte(fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second))), 8, nil case TypeVarchar, TypeVarString: + // We trust that styp is compatible with the column type // Length is encoded in 1 or 2 bytes. if metadata > 255 { l := int(uint64(data[pos]) | uint64(data[pos+1])<<8) - return sqltypes.MakeTrusted(querypb.Type_VARCHAR, + return sqltypes.MakeTrusted(styp, data[pos+2:pos+2+l]), l + 2, nil } l := int(data[pos]) - return sqltypes.MakeTrusted(querypb.Type_VARCHAR, + return sqltypes.MakeTrusted(styp, data[pos+1:pos+1+l]), l + 1, nil case TypeBit: // The contents is just the bytes, quoted. diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index 2d4d43819f7..48de92a03d3 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -110,33 +110,13 @@ func (plan *Plan) fields() []*querypb.Field { return fields } -func resolveValue(val sqltypes.Value) (sqltypes.Value, error) { - pv, err := sqlparser.NewPlanValue(sqlparser.NewStrVal(val.Raw())) - if err != nil { - return sqltypes.NULL, err - } - resolved, err := pv.ResolveValue(nil) - if err != nil { - return sqltypes.NULL, err - } - return resolved, nil -} - // filter filters the row against the plan. It returns false if the row did not match. // If the row matched, it returns the columns to be sent. func (plan *Plan) filter(values []sqltypes.Value) (bool, []sqltypes.Value, error) { for _, filter := range plan.Filters { switch filter.Opcode { case Equal: - filterValue, err := resolveValue(filter.Value) - if err != nil { - return false, nil, err - } - colValue, err := resolveValue(values[filter.ColNum]) - if err != nil { - return false, nil, err - } - result, err := sqltypes.NullsafeCompare(colValue, filterValue) + result, err := sqltypes.NullsafeCompare(values[filter.ColNum], filter.Value) if err != nil { return false, nil, err } diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go index 8b2c4a39007..01056c893e3 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go @@ -231,7 +231,6 @@ func TestStreamRowsFilterInt(t *testing.T) { time.Sleep(1 * time.Second) - // Only the first row should be returned, but lastpk should be 6. wantStream := []string{ `fields: fields: pkfields: `, `rows: rows: lastpk: `, @@ -261,7 +260,6 @@ func TestStreamRowsFilterVarBinary(t *testing.T) { time.Sleep(1 * time.Second) - // Only the first row should be returned, but lastpk should be 6. wantStream := []string{ `fields: fields: pkfields: `, `rows: rows: rows: lastpk: `, diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 08c5ea5aa64..e7025f9b794 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -630,6 +630,7 @@ func (vs *vstreamer) buildTableColumns(id uint64, tm *mysql.TableMap) ([]schema. if !sqltypes.AreTypesEquivalent(cols[i].Type, st.Columns[i].Type) { return cols, nil } + cols[i].Type = st.Columns[i].Type } // Columns should be truncated to match those in tm. From cd940475b1463e2adb19bd603e68eeb63575c26e Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sat, 21 Mar 2020 20:07:42 +0100 Subject: [PATCH 07/85] Vreplication and multiple where clause tests Signed-off-by: Rohit Nayak --- examples/are-you-alive/go.mod | 2 + .../vreplication/vplayer_test.go | 38 +++++++++++++++ .../tabletserver/vstreamer/vstreamer_test.go | 46 +++++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/examples/are-you-alive/go.mod b/examples/are-you-alive/go.mod index 9bb69e1307b..5f3d04ddccd 100644 --- a/examples/are-you-alive/go.mod +++ b/examples/are-you-alive/go.mod @@ -4,6 +4,8 @@ go 1.12 require ( github.com/go-sql-driver/mysql v1.5.0 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/prometheus/client_golang v1.4.1 github.com/sirupsen/logrus v1.4.2 ) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go index dd95e70912a..5abc68b0b3c 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go @@ -135,6 +135,10 @@ func TestPlayerFilters(t *testing.T) { "create table no(id int, val varbinary(128), primary key(id))", "create table nopk(id int, val varbinary(128))", fmt.Sprintf("create table %s.nopk(id int, val varbinary(128))", vrepldb), + "create table src4(id1 int, id2 int, val varbinary(128), primary key(id1))", + fmt.Sprintf("create table %s.dst4(id1 int, val varbinary(128), primary key(id1))", vrepldb), + "create table src5(id1 int, id2 int, val varbinary(128), primary key(id1))", + fmt.Sprintf("create table %s.dst5(id1 int, val varbinary(128), primary key(id1))", vrepldb), }) defer execStatements(t, []string{ "drop table src1", @@ -148,6 +152,10 @@ func TestPlayerFilters(t *testing.T) { "drop table no", "drop table nopk", fmt.Sprintf("drop table %s.nopk", vrepldb), + "drop table src4", + fmt.Sprintf("drop table %s.dst4", vrepldb), + "drop table src5", + fmt.Sprintf("drop table %s.dst5", vrepldb), }) env.SchemaEngine.Reload(context.Background()) @@ -165,6 +173,12 @@ func TestPlayerFilters(t *testing.T) { Match: "/yes", }, { Match: "/nopk", + }, { + Match: "dst4", + Filter: "select id1, val from src4 where id2 = 100", + }, { + Match: "dst5", + Filter: "select id1, val from src5 where val = 'abc'", }}, } bls := &binlogdatapb.BinlogSource{ @@ -386,6 +400,30 @@ func TestPlayerFilters(t *testing.T) { }, table: "nopk", data: [][]string{}, + }, { + // filter by int + input: "insert into src4 values (1,100,'aaa'),(2,200,'bbb'),(3,100,'ccc')", + output: []string{ + "begin", + "insert into dst4(id1,val) values (1,'aaa')", + "insert into dst4(id1,val) values (3,'ccc')", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst4", + data: [][]string{{"1", "aaa"}, {"3", "ccc"}}, + }, { + // filter by int + input: "insert into src5 values (1,100,'abc'),(2,200,'xyz'),(3,100,'xyz'),(4,300,'abc'),(5,200,'xyz')", + output: []string{ + "begin", + "insert into dst5(id1,val) values (1,'abc')", + "insert into dst5(id1,val) values (4,'abc')", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst5", + data: [][]string{{"1", "abc"}, {"4", "abc"}}, }} for _, tcase := range testcases { diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go index c693b9efc2e..98cb8e2ddc1 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go @@ -1241,6 +1241,52 @@ func TestNoFutureGTID(t *testing.T) { } } +func TestFilteredMultipleWhere(t *testing.T) { + if testing.Short() { + t.Skip() + } + + execStatements(t, []string{ + "create table t1(id1 int, id2 int, id3 int, val varbinary(128), primary key(id1))", + }) + defer execStatements(t, []string{ + "drop table t1", + }) + engine.se.Reload(context.Background()) + + setVSchema(t, shardedVSchema) + defer env.SetVSchema("{}") + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select id1, val from t1 where in_keyrange('-80') and id2 = 200 and id3 = 1000 and val = 'newton'", + }}, + } + + testcases := []testcase{{ + input: []string{ + "begin", + "insert into t1 values (1, 100, 1000, 'kepler')", + "insert into t1 values (2, 200, 1000, 'newton')", + "insert into t1 values (3, 100, 2000, 'kepler')", + "insert into t1 values (128, 200, 1000, 'newton')", + "insert into t1 values (5, 200, 2000, 'kepler')", + "insert into t1 values (129, 200, 1000, 'kepler')", + "commit", + }, + output: [][]string{{ + `begin`, + `type:FIELD field_event: fields: > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `gtid`, + `commit`, + }}, + }} + runCases(t, filter, testcases, "") +} + func runCases(t *testing.T, filter *binlogdatapb.Filter, testcases []testcase, position string) { t.Helper() ctx, cancel := context.WithCancel(context.Background()) From 9bcd0619d639f7be5c0167de60351d5f41066398 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sat, 21 Mar 2020 21:31:43 +0100 Subject: [PATCH 08/85] Only use styp if not null Signed-off-by: Rohit Nayak --- go/mysql/binlog_event_rbr.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index 92866f63859..01a5102e8b6 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -333,6 +333,7 @@ func printTimestamp(v uint32) *bytes.Buffer { // many bytes it takes. It only uses the querypb.Type value for the // signed flag. func CellValue(data []byte, pos int, typ byte, metadata uint16, styp querypb.Type) (sqltypes.Value, int, error) { + fmt.Printf("CellValue typ %v styp %v\n", typ, styp) switch typ { case TypeTiny: if sqltypes.IsSigned(styp) { @@ -449,14 +450,20 @@ func CellValue(data []byte, pos int, typ byte, metadata uint16, styp querypb.Typ case TypeVarchar, TypeVarString: // We trust that styp is compatible with the column type // Length is encoded in 1 or 2 bytes. + var typeToUse querypb.Type + if styp != querypb.Type_NULL_TYPE { + typeToUse = styp + } else { + typeToUse = querypb.Type_VARCHAR + } if metadata > 255 { l := int(uint64(data[pos]) | uint64(data[pos+1])<<8) - return sqltypes.MakeTrusted(styp, + return sqltypes.MakeTrusted(typeToUse, data[pos+2:pos+2+l]), l + 2, nil } l := int(data[pos]) - return sqltypes.MakeTrusted(styp, + return sqltypes.MakeTrusted(typeToUse, data[pos+1:pos+1+l]), l + 1, nil case TypeBit: // The contents is just the bytes, quoted. From 491ec3766571cd171ed50507ec6102d588875b7a Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sat, 21 Mar 2020 22:07:41 +0100 Subject: [PATCH 09/85] Removed extraneous log Signed-off-by: Rohit Nayak --- go/mysql/binlog_event_rbr.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index 01a5102e8b6..5d7d2411c3e 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -333,7 +333,6 @@ func printTimestamp(v uint32) *bytes.Buffer { // many bytes it takes. It only uses the querypb.Type value for the // signed flag. func CellValue(data []byte, pos int, typ byte, metadata uint16, styp querypb.Type) (sqltypes.Value, int, error) { - fmt.Printf("CellValue typ %v styp %v\n", typ, styp) switch typ { case TypeTiny: if sqltypes.IsSigned(styp) { From 319854469b91ad57b36b10983ef399a6a36b071c Mon Sep 17 00:00:00 2001 From: deepthi Date: Mon, 23 Mar 2020 13:34:45 -0700 Subject: [PATCH 10/85] add option to pass in flags to xtrabackup prepare phase Signed-off-by: deepthi --- go/test/endtoend/backup/vtctlbackup/backup_utils.go | 5 +++++ go/vt/mysqlctl/xtrabackupengine.go | 12 ++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index aeb1d53365a..a7e858abf0f 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -125,6 +125,11 @@ func LaunchCluster(setupType int, streamMode string, stripes int) (int, error) { "-xtrabackup_backup_flags", fmt.Sprintf("--password=%s", dbPassword), } + // if streamMode is xbstream, add some additional args to test other xtrabackup flags + if streamMode == "xbstream" { + xtrabackupArgs = append(xtrabackupArgs, "-xtrabackup_prepare_flags", fmt.Sprintf("--use-memory=100M")) + } + commonTabletArg = append(commonTabletArg, xtrabackupArgs...) } diff --git a/go/vt/mysqlctl/xtrabackupengine.go b/go/vt/mysqlctl/xtrabackupengine.go index 6572064fa86..a7c57c93365 100644 --- a/go/vt/mysqlctl/xtrabackupengine.go +++ b/go/vt/mysqlctl/xtrabackupengine.go @@ -49,9 +49,11 @@ type XtrabackupEngine struct { var ( // path where backup engine program is located xtrabackupEnginePath = flag.String("xtrabackup_root_path", "", "directory location of the xtrabackup executable, e.g., /usr/bin") - // flags to pass through to backup engine + // flags to pass through to backup phase xtrabackupBackupFlags = flag.String("xtrabackup_backup_flags", "", "flags to pass to backup command. These should be space separated and will be added to the end of the command") - // flags to pass through to restore phase + // flags to pass through to prepare phase of restore + xtrabackupPrepareFlags = flag.String("xtrabackup_prepare_flags", "", "flags to pass to prepare command. These should be space separated and will be added to the end of the command") + // flags to pass through to extract phase of restore xbstreamRestoreFlags = flag.String("xbstream_restore_flags", "", "flags to pass to xbstream command during restore. These should be space separated and will be added to the end of the command. These need to match the ones used for backup e.g. --compress / --decompress, --encrypt / --decrypt") // streaming mode xtrabackupStreamMode = flag.String("xtrabackup_stream_mode", "tar", "which mode to use if streaming, valid values are tar and xbstream") @@ -427,6 +429,9 @@ func (be *XtrabackupEngine) restoreFromBackup(ctx context.Context, cnf *Mycnf, b "--prepare", "--target-dir=" + tempDir, } + if *xtrabackupPrepareFlags != "" { + flagsToExec = append(flagsToExec, strings.Fields(*xtrabackupPrepareFlags)...) + } prepareCmd := exec.CommandContext(ctx, restoreProgram, flagsToExec...) prepareOut, err := prepareCmd.StdoutPipe() if err != nil { @@ -575,11 +580,10 @@ func (be *XtrabackupEngine) extractFiles(ctx context.Context, logger logutil.Log case xbstream: // now extract the files by running xbstream xbstreamProgram := xbstream - flagsToExec := []string{} + flagsToExec := []string{"-C", tempDir, "-xv"} if *xbstreamRestoreFlags != "" { flagsToExec = append(flagsToExec, strings.Fields(*xbstreamRestoreFlags)...) } - flagsToExec = append(flagsToExec, "-C", tempDir, "-xv") xbstreamCmd := exec.CommandContext(ctx, xbstreamProgram, flagsToExec...) logger.Infof("Executing xbstream cmd: %v %v", xbstreamProgram, flagsToExec) xbstreamCmd.Stdin = reader From d2a0754b891400b18d4d3b7aa54d2ba80bb1b891 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Mon, 16 Mar 2020 15:43:54 -0700 Subject: [PATCH 11/85] vrepl: where clause filtering WIP Signed-off-by: Sugu Sougoumarane --- .../tabletserver/vstreamer/planbuilder.go | 165 +++++++++++++++--- 1 file changed, 140 insertions(+), 25 deletions(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index c084327c877..6441cc01c2b 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -36,10 +36,30 @@ import ( // Plan represents the plan for a table. type Plan struct { Table *Table + + // Filters is the list of filters to be applied to the columns + // of the table. + Filters []Filter + // ColExprs is the list of column expressions to be sent // in the stream. ColExprs []ColExpr +} + +type Opcode int + +const ( + Equal = Opcode(iota) + VindexMatch +) + +// Filter contains opcodes for filtering. +type Filter struct { + Opcode Opcode + ColNum int + Value sqltypes.Value + // Parameters for VindexMatch. // Vindex, VindexColumns and KeyRange, if set, will be used // to filter the row. // VindexColumns contains the column numbers of the table, @@ -90,13 +110,24 @@ func (plan *Plan) fields() []*querypb.Field { // filter filters the row against the plan. It returns false if the row did not match. // If the row matched, it returns the columns to be sent. func (plan *Plan) filter(values []sqltypes.Value) (bool, []sqltypes.Value, error) { - if plan.Vindex != nil { - ksid, err := getKeyspaceID(values, plan.Vindex, plan.VindexColumns) - if err != nil { - return false, nil, err - } - if !key.KeyRangeContains(plan.KeyRange, ksid) { - return false, nil, nil + for _, filter := range plan.Filters { + switch filter.Opcode { + case Equal: + result, err := sqltypes.NullsafeCompare(values[filter.ColNum], filter.Value) + if err != nil { + return false, nil, err + } + if result != 0 { + return false, nil, nil + } + case VindexMatch: + ksid, err := getKeyspaceID(values, filter.Vindex, filter.VindexColumns) + if err != nil { + return false, nil, err + } + if !key.KeyRangeContains(filter.KeyRange, ksid) { + return false, nil, nil + } } } @@ -242,8 +273,11 @@ func buildREPlan(ti *Table, vschema *localVSchema, filter string) (*Plan, error) if err != nil { return nil, err } - plan.Vindex = cv.Vindex - plan.VindexColumns, err = buildVindexColumns(plan.Table, cv.Columns) + whereFilter := Filter{ + Opcode: VindexMatch, + Vindex: cv.Vindex, + } + whereFilter.VindexColumns, err = buildVindexColumns(plan.Table, cv.Columns) if err != nil { return nil, err } @@ -256,7 +290,8 @@ func buildREPlan(ti *Table, vschema *localVSchema, filter string) (*Plan, error) if len(keyranges) != 1 { return nil, fmt.Errorf("error parsing keyrange: %v", filter) } - plan.KeyRange = keyranges[0] + whereFilter.KeyRange = keyranges[0] + plan.Filters = append(plan.Filters, whereFilter) return plan, nil } @@ -274,6 +309,9 @@ func buildTablePlan(ti *Table, vschema *localVSchema, query string) (*Plan, erro plan := &Plan{ Table: ti, } + if err := plan.analyzeWhere(vschema, sel.Where); err != nil { + return nil, err + } if err := plan.analyzeExprs(vschema, sel.SelectExprs); err != nil { return nil, err } @@ -282,16 +320,6 @@ func buildTablePlan(ti *Table, vschema *localVSchema, query string) (*Plan, erro return plan, nil } - funcExpr, ok := sel.Where.Expr.(*sqlparser.FuncExpr) - if !ok { - return nil, fmt.Errorf("unsupported where clause: %v", sqlparser.String(sel.Where)) - } - if !funcExpr.Name.EqualString("in_keyrange") { - return nil, fmt.Errorf("unsupported where clause: %v", sqlparser.String(sel.Where)) - } - if err := plan.analyzeInKeyRange(vschema, funcExpr.Exprs); err != nil { - return nil, err - } return plan, nil } @@ -318,6 +346,89 @@ func analyzeSelect(query string) (sel *sqlparser.Select, fromTable sqlparser.Tab return sel, fromTable, nil } +func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) error { + if where == nil { + return nil + } + exprs := splitAndExpression(nil, where.Expr) + for _, expr := range exprs { + switch expr := expr.(type) { + case *sqlparser.ComparisonExpr: + qualifiedName, ok := expr.Left.(*sqlparser.ColName) + if !ok { + return fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + } + if !qualifiedName.Qualifier.IsEmpty() { + return fmt.Errorf("unsupported qualifier for column: %v", sqlparser.String(qualifiedName)) + } + colnum, err := findColumn(plan.Table, qualifiedName.Name) + if err != nil { + return err + } + val, ok := expr.Right.(*sqlparser.SQLVal) + if !ok { + return fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + } + if val.Type != sqlparser.IntVal { + return fmt.Errorf("unexpected: %v", sqlparser.String(expr)) + } + pv, err := sqlparser.NewPlanValue(val) + if err != nil { + return err + } + resolved, err := pv.ResolveValue(nil) + if err != nil { + return err + } + plan.Filters = append(plan.Filters, Filter{ + Opcode: Equal, + ColNum: colnum, + Value: resolved, + }) + case *sqlparser.FuncExpr: + if !expr.Name.EqualString("in_keyrange") { + return fmt.Errorf("unsupported constraint: %v", sqlparser.String(expr)) + } + if err := plan.analyzeInKeyRange(vschema, expr.Exprs); err != nil { + return err + } + default: + return fmt.Errorf("unsupported constraint: %v", sqlparser.String(expr)) + } + } + return nil +} + +// splitAndExpression breaks up the Expr into AND-separated conditions +// and appends them to filters, which can be shuffled and recombined +// as needed. +func splitAndExpression(filters []sqlparser.Expr, node sqlparser.Expr) []sqlparser.Expr { + if node == nil { + return filters + } + switch node := node.(type) { + case *sqlparser.AndExpr: + filters = splitAndExpression(filters, node.Left) + return splitAndExpression(filters, node.Right) + case *sqlparser.ParenExpr: + // If the inner expression is AndExpr, then we can remove + // the parenthesis because they are unnecessary. + if node, ok := node.Expr.(*sqlparser.AndExpr); ok { + return splitAndExpression(filters, node) + } + } + return append(filters, node) +} + +// skipParenthesis skips the parenthesis (if any) of an expression and +// returns the innermost unparenthesized expression. +func skipParenthesis(node sqlparser.Expr) sqlparser.Expr { + if node, ok := node.(*sqlparser.ParenExpr); ok { + return skipParenthesis(node.Expr) + } + return node +} + func (plan *Plan) analyzeExprs(vschema *localVSchema, selExprs sqlparser.SelectExprs) error { if _, ok := selExprs[0].(*sqlparser.StarExpr); !ok { for _, expr := range selExprs { @@ -396,6 +507,9 @@ func (plan *Plan) analyzeExpr(vschema *localVSchema, selExpr sqlparser.SelectExp func (plan *Plan) analyzeInKeyRange(vschema *localVSchema, exprs sqlparser.SelectExprs) error { var colnames []sqlparser.ColIdent var krExpr sqlparser.SelectExpr + whereFilter := Filter{ + Opcode: VindexMatch, + } switch { case len(exprs) == 1: cv, err := vschema.FindColVindex(plan.Table.Name) @@ -403,7 +517,7 @@ func (plan *Plan) analyzeInKeyRange(vschema *localVSchema, exprs sqlparser.Selec return err } colnames = cv.Columns - plan.Vindex = cv.Vindex + whereFilter.Vindex = cv.Vindex krExpr = exprs[0] case len(exprs) >= 3: for _, expr := range exprs[:len(exprs)-2] { @@ -425,11 +539,11 @@ func (plan *Plan) analyzeInKeyRange(vschema *localVSchema, exprs sqlparser.Selec if err != nil { return err } - plan.Vindex, err = vschema.FindOrCreateVindex(vtype) + whereFilter.Vindex, err = vschema.FindOrCreateVindex(vtype) if err != nil { return err } - if !plan.Vindex.IsUnique() { + if !whereFilter.Vindex.IsUnique() { return fmt.Errorf("vindex must be Unique to be used for VReplication: %s", vtype) } @@ -438,7 +552,7 @@ func (plan *Plan) analyzeInKeyRange(vschema *localVSchema, exprs sqlparser.Selec return fmt.Errorf("unexpected in_keyrange parameters: %v", sqlparser.String(exprs)) } var err error - plan.VindexColumns, err = buildVindexColumns(plan.Table, colnames) + whereFilter.VindexColumns, err = buildVindexColumns(plan.Table, colnames) if err != nil { return err } @@ -453,7 +567,8 @@ func (plan *Plan) analyzeInKeyRange(vschema *localVSchema, exprs sqlparser.Selec if len(keyranges) != 1 { return fmt.Errorf("unexpected in_keyrange parameter: %v", sqlparser.String(krExpr)) } - plan.KeyRange = keyranges[0] + whereFilter.KeyRange = keyranges[0] + plan.Filters = append(plan.Filters, whereFilter) return nil } From b683ef8c4105287c414a7bf0e654e98467b63dac Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Wed, 18 Mar 2020 23:14:25 +0100 Subject: [PATCH 12/85] Update test cases for where clause Signed-off-by: Rohit Nayak --- .../tabletserver/vstreamer/planbuilder.go | 13 ++-- .../vstreamer/planbuilder_test.go | 77 +++++++++++++++---- 2 files changed, 71 insertions(+), 19 deletions(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index 6441cc01c2b..c6474d1b8ec 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -37,13 +37,14 @@ import ( type Plan struct { Table *Table + // ColExprs is the list of column expressions to be sent + // in the stream. + ColExprs []ColExpr + // Filters is the list of filters to be applied to the columns // of the table. Filters []Filter - // ColExprs is the list of column expressions to be sent - // in the stream. - ColExprs []ColExpr } type Opcode int @@ -381,9 +382,9 @@ func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) er return err } plan.Filters = append(plan.Filters, Filter{ - Opcode: Equal, - ColNum: colnum, - Value: resolved, + Opcode: Equal, + ColNum: colnum, + Value: resolved, }) case *sqlparser.FuncExpr: if !expr.Name.EqualString("in_keyrange") { diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go index 0ff9bfea5fa..e9ac90e98fa 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go @@ -245,7 +245,14 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("val"), Type: sqltypes.VarBinary, }}, - VindexColumns: []int{0}, + Filters: []Filter {{ + Opcode: VindexMatch, + ColNum: 0, + Value: sqltypes.NULL, + Vindex: nil, + VindexColumns: []int{0}, + KeyRange: nil, + }}, }, }, { inTable: t1, @@ -302,7 +309,14 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - VindexColumns: []int{0}, + Filters: []Filter {{ + Opcode: VindexMatch, + ColNum: 0, + Value: sqltypes.NULL, + Vindex: nil, + VindexColumns: []int{0}, + KeyRange: nil, + }}, }, }, { inTable: t1, @@ -317,7 +331,36 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - VindexColumns: []int{0}, + Filters: []Filter {{ + Opcode: VindexMatch, + ColNum: 0, + Value: sqltypes.NULL, + Vindex: nil, + VindexColumns: []int{0}, + KeyRange: nil, + }}, + }, + },{ + inTable: t1, + inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select val, id from t1 where id = 1"}, + outPlan: &Plan{ + ColExprs: []ColExpr{{ + ColNum: 1, + Alias: sqlparser.NewColIdent("val"), + Type: sqltypes.VarBinary, + }, { + ColNum: 0, + Alias: sqlparser.NewColIdent("id"), + Type: sqltypes.Int64, + }}, + Filters: []Filter {{ + Opcode: Equal, + ColNum: 0, + Value: sqltypes.NewInt64(1), + Vindex: nil, + VindexColumns: nil, + KeyRange: nil, + }}, }, }, { inTable: t2, @@ -335,7 +378,14 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - VindexColumns: []int{0, 1}, + Filters: []Filter {{ + Opcode: VindexMatch, + ColNum: 0, + Value: sqltypes.NULL, + Vindex: nil, + VindexColumns: []int{0,1}, + KeyRange: nil, + }}, }, }, { inTable: regional, @@ -400,14 +450,10 @@ func TestPlanbuilder(t *testing.T) { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select *, id from t1"}, outErr: `unsupported: *, id`, - }, { - inTable: t1, - inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val from t1 where id=1"}, - outErr: `unsupported where clause: where id = 1`, - }, { + },{ inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val from t1 where max(id)"}, - outErr: `unsupported where clause: where max(id)`, + outErr: `unsupported constraint: max(id)`, }, { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val from t1 where in_keyrange(id)"}, @@ -463,15 +509,20 @@ func TestPlanbuilder(t *testing.T) { inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val from t1 where in_keyrange(id, 1+1, '-80')"}, outErr: `unsupported: 1 + 1`, }} - for _, tcase := range testcases { plan, err := buildPlan(tcase.inTable, testLocalVSchema, &binlogdatapb.Filter{ Rules: []*binlogdatapb.Rule{tcase.inRule}, }) if plan != nil { plan.Table = nil - plan.Vindex = nil - plan.KeyRange = nil + for ind, _ := range plan.Filters { + plan.Filters[ind].KeyRange = nil + if plan.Filters[ind]. + Opcode == VindexMatch { + plan.Filters[ind].Value = sqltypes.NULL + } + plan.Filters[ind].Vindex = nil + } if !reflect.DeepEqual(tcase.outPlan, plan) { t.Errorf("Plan(%v, %v):\n%v, want\n%v", tcase.inTable, tcase.inRule, plan, tcase.outPlan) } From 62c5d693b03957791d9b1dc5b14b7388427da628 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 19 Mar 2020 18:30:54 +0100 Subject: [PATCH 13/85] Test case for int filter Signed-off-by: Rohit Nayak --- .../tabletserver/vstreamer/planbuilder.go | 19 +++---- .../vstreamer/planbuilder_test.go | 18 +++---- .../tabletserver/vstreamer/vstreamer_test.go | 53 +++++++++++++++++++ 3 files changed, 68 insertions(+), 22 deletions(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index c6474d1b8ec..918c2d421b5 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -44,13 +44,15 @@ type Plan struct { // Filters is the list of filters to be applied to the columns // of the table. Filters []Filter - } +// Opcode enumerates the operators supported in a where clause type Opcode int const ( + // Equal is used to filter an integer column on a specific value Equal = Opcode(iota) + // VindexMatch is used for an in_keyrange() construct VindexMatch ) @@ -382,9 +384,9 @@ func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) er return err } plan.Filters = append(plan.Filters, Filter{ - Opcode: Equal, - ColNum: colnum, - Value: resolved, + Opcode: Equal, + ColNum: colnum, + Value: resolved, }) case *sqlparser.FuncExpr: if !expr.Name.EqualString("in_keyrange") { @@ -421,15 +423,6 @@ func splitAndExpression(filters []sqlparser.Expr, node sqlparser.Expr) []sqlpars return append(filters, node) } -// skipParenthesis skips the parenthesis (if any) of an expression and -// returns the innermost unparenthesized expression. -func skipParenthesis(node sqlparser.Expr) sqlparser.Expr { - if node, ok := node.(*sqlparser.ParenExpr); ok { - return skipParenthesis(node.Expr) - } - return node -} - func (plan *Plan) analyzeExprs(vschema *localVSchema, selExprs sqlparser.SelectExprs) error { if _, ok := selExprs[0].(*sqlparser.StarExpr); !ok { for _, expr := range selExprs { diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go index e9ac90e98fa..fa0a44ec1b0 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go @@ -245,7 +245,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("val"), Type: sqltypes.VarBinary, }}, - Filters: []Filter {{ + Filters: []Filter{{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, @@ -309,7 +309,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter {{ + Filters: []Filter{{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, @@ -331,7 +331,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter {{ + Filters: []Filter{{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, @@ -340,7 +340,7 @@ func TestPlanbuilder(t *testing.T) { KeyRange: nil, }}, }, - },{ + }, { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select val, id from t1 where id = 1"}, outPlan: &Plan{ @@ -353,7 +353,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter {{ + Filters: []Filter{{ Opcode: Equal, ColNum: 0, Value: sqltypes.NewInt64(1), @@ -378,12 +378,12 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter {{ + Filters: []Filter{{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, Vindex: nil, - VindexColumns: []int{0,1}, + VindexColumns: []int{0, 1}, KeyRange: nil, }}, }, @@ -450,7 +450,7 @@ func TestPlanbuilder(t *testing.T) { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select *, id from t1"}, outErr: `unsupported: *, id`, - },{ + }, { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val from t1 where max(id)"}, outErr: `unsupported constraint: max(id)`, @@ -515,7 +515,7 @@ func TestPlanbuilder(t *testing.T) { }) if plan != nil { plan.Table = nil - for ind, _ := range plan.Filters { + for ind := range plan.Filters { plan.Filters[ind].KeyRange = nil if plan.Filters[ind]. Opcode == VindexMatch { diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go index 5b9edf50860..572c07da983 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go @@ -37,6 +37,59 @@ type testcase struct { output [][]string } +func TestFilteredInt(t *testing.T) { + if testing.Short() { + t.Skip() + } + + execStatements(t, []string{ + "create table t1(id1 int, id2 int, val varbinary(128), primary key(id1))", + }) + defer execStatements(t, []string{ + "drop table t1", + }) + engine.se.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select id1, val from t1 where id2 = 200", + }}, + } + + testcases := []testcase{{ + input: []string{ + "begin", + "insert into t1 values (1, 100, 'aaa')", + "insert into t1 values (2, 200, 'bbb')", + "insert into t1 values (3, 100, 'ccc')", + "insert into t1 values (4, 200, 'ddd')", + "insert into t1 values (5, 200, 'eee')", + "update t1 set val = 'newddd' where id1 = 4", + "update t1 set id2 = 200 where id1 = 1", + "update t1 set id2 = 100 where id1 = 2", + "update t1 set id2 = 100 where id1 = 1", + "update t1 set id2 = 200 where id1 = 2", + "commit", + }, + output: [][]string{{ + `begin`, + `type:FIELD field_event: fields: > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: after: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `gtid`, + `commit`, + }}, + }} + runCases(t, filter, testcases, "") +} + func TestStatements(t *testing.T) { if testing.Short() { t.Skip() From bd5af6ea1e5ac337a35649a137686eb982cd7975 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 19 Mar 2020 23:09:20 +0100 Subject: [PATCH 14/85] Added rowstreamer test for filtered vreplication Signed-off-by: Rohit Nayak --- .../vstreamer/rowstreamer_test.go | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go index ab294562dde..63fb44a3168 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go @@ -210,6 +210,36 @@ func TestStreamRowsKeyRange(t *testing.T) { checkStream(t, "select * from t1 where in_keyrange('-80')", nil, wantQuery, wantStream) } +func TestStreamRowsFilterInt(t *testing.T) { + if testing.Short() { + t.Skip() + } + + if err := env.SetVSchema(shardedVSchema); err != nil { + t.Fatal(err) + } + defer env.SetVSchema("{}") + + execStatements(t, []string{ + "create table t1(id1 int, id2 int, val varbinary(128), primary key(id1))", + "insert into t1 values (1, 100, 'aaa'), (2, 200, 'bbb'), (3, 200, 'ccc'), (4, 100, 'ddd'), (5, 200, 'eee')", + }) + defer execStatements(t, []string{ + "drop table t1", + }) + engine.se.Reload(context.Background()) + + time.Sleep(1 * time.Second) + + // Only the first row should be returned, but lastpk should be 6. + wantStream := []string{ + `fields: fields: pkfields: `, + `rows: rows: lastpk: `, + } + wantQuery := "select id1, id2, val from t1 order by id1" + checkStream(t, "select id1, val from t1 where id2 = 100", nil, wantQuery, wantStream) +} + func TestStreamRowsMultiPacket(t *testing.T) { if testing.Short() { t.Skip() From 47bc7b2187a6bf35881be12a916ebe1ee29a81b8 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Fri, 20 Mar 2020 11:05:37 +0100 Subject: [PATCH 15/85] Added support for varbinary, tests for both int/varbinary Signed-off-by: Rohit Nayak --- .../tabletserver/vstreamer/planbuilder.go | 25 ++++++++- .../vstreamer/rowstreamer_test.go | 30 +++++++++++ .../tabletserver/vstreamer/vstreamer.go | 4 +- .../tabletserver/vstreamer/vstreamer_test.go | 53 +++++++++++++++++++ 4 files changed, 108 insertions(+), 4 deletions(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index 918c2d421b5..2d4d43819f7 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -110,13 +110,33 @@ func (plan *Plan) fields() []*querypb.Field { return fields } +func resolveValue(val sqltypes.Value) (sqltypes.Value, error) { + pv, err := sqlparser.NewPlanValue(sqlparser.NewStrVal(val.Raw())) + if err != nil { + return sqltypes.NULL, err + } + resolved, err := pv.ResolveValue(nil) + if err != nil { + return sqltypes.NULL, err + } + return resolved, nil +} + // filter filters the row against the plan. It returns false if the row did not match. // If the row matched, it returns the columns to be sent. func (plan *Plan) filter(values []sqltypes.Value) (bool, []sqltypes.Value, error) { for _, filter := range plan.Filters { switch filter.Opcode { case Equal: - result, err := sqltypes.NullsafeCompare(values[filter.ColNum], filter.Value) + filterValue, err := resolveValue(filter.Value) + if err != nil { + return false, nil, err + } + colValue, err := resolveValue(values[filter.ColNum]) + if err != nil { + return false, nil, err + } + result, err := sqltypes.NullsafeCompare(colValue, filterValue) if err != nil { return false, nil, err } @@ -372,7 +392,8 @@ func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) er if !ok { return fmt.Errorf("unexpected: %v", sqlparser.String(expr)) } - if val.Type != sqlparser.IntVal { + //StrVal is varbinary, we do not support varchar since we would have to implement all collation types + if val.Type != sqlparser.IntVal && val.Type != sqlparser.StrVal { return fmt.Errorf("unexpected: %v", sqlparser.String(expr)) } pv, err := sqlparser.NewPlanValue(val) diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go index 63fb44a3168..8b2c4a39007 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go @@ -240,6 +240,36 @@ func TestStreamRowsFilterInt(t *testing.T) { checkStream(t, "select id1, val from t1 where id2 = 100", nil, wantQuery, wantStream) } +func TestStreamRowsFilterVarBinary(t *testing.T) { + if testing.Short() { + t.Skip() + } + + if err := env.SetVSchema(shardedVSchema); err != nil { + t.Fatal(err) + } + defer env.SetVSchema("{}") + + execStatements(t, []string{ + "create table t1(id1 int, val varbinary(128), primary key(id1))", + "insert into t1 values (1,'kepler'), (2, 'newton'), (3, 'newton'), (4, 'kepler'), (5, 'newton'), (6, 'kepler')", + }) + defer execStatements(t, []string{ + "drop table t1", + }) + engine.se.Reload(context.Background()) + + time.Sleep(1 * time.Second) + + // Only the first row should be returned, but lastpk should be 6. + wantStream := []string{ + `fields: fields: pkfields: `, + `rows: rows: rows: lastpk: `, + } + wantQuery := "select id1, val from t1 order by id1" + checkStream(t, "select id1, val from t1 where val = 'newton'", nil, wantQuery, wantStream) +} + func TestStreamRowsMultiPacket(t *testing.T) { if testing.Short() { t.Skip() diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 7f99791be36..08c5ea5aa64 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -545,7 +545,7 @@ func (vs *vstreamer) buildJournalPlan(id uint64, tm *mysql.TableMap) error { return err } if len(st.Columns) < len(tm.Types) { - return fmt.Errorf("cannot determine table columns for %s: event has %v, schema as %v", tm.Name, tm.Types, st.Columns) + return fmt.Errorf("cannot determine table columns for %s: event has %v, schema has %v", tm.Name, tm.Types, st.Columns) } table := &Table{ Name: "_vt.resharding_journal", @@ -620,7 +620,7 @@ func (vs *vstreamer) buildTableColumns(id uint64, tm *mysql.TableMap) ([]schema. if len(st.Columns) < len(tm.Types) { if vs.filter.FieldEventMode == binlogdatapb.Filter_ERR_ON_MISMATCH { - return nil, fmt.Errorf("cannot determine table columns for %s: event has %v, schema as %v", tm.Name, tm.Types, st.Columns) + return nil, fmt.Errorf("cannot determine table columns for %s: event has %v, schema has %v", tm.Name, tm.Types, st.Columns) } return cols, nil } diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go index 572c07da983..c693b9efc2e 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go @@ -37,6 +37,59 @@ type testcase struct { output [][]string } +func TestFilteredVarBinary(t *testing.T) { + if testing.Short() { + t.Skip() + } + + execStatements(t, []string{ + "create table t1(id1 int, val varbinary(128), primary key(id1))", + }) + defer execStatements(t, []string{ + "drop table t1", + }) + engine.se.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select id1, val from t1 where val = 'newton'", + }}, + } + + testcases := []testcase{{ + input: []string{ + "begin", + "insert into t1 values (1, 'kepler')", + "insert into t1 values (2, 'newton')", + "insert into t1 values (3, 'newton')", + "insert into t1 values (4, 'kepler')", + "insert into t1 values (5, 'newton')", + "update t1 set val = 'newton' where id1 = 1", + "update t1 set val = 'kepler' where id1 = 2", + "update t1 set val = 'newton' where id1 = 2", + "update t1 set val = 'kepler' where id1 = 1", + "delete from t1 where id1 in (2,3)", + "commit", + }, + output: [][]string{{ + `begin`, + `type:FIELD field_event: fields: > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > row_changes: > > `, + `gtid`, + `commit`, + }}, + }} + runCases(t, filter, testcases, "") +} + func TestFilteredInt(t *testing.T) { if testing.Short() { t.Skip() From cdded8887c36f4e153eb485f0ea9a7eb7730b741 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sat, 21 Mar 2020 00:07:55 +0100 Subject: [PATCH 16/85] Handle varbinary correctly Signed-off-by: Rohit Nayak --- go/mysql/binlog_event_rbr.go | 5 +++-- .../tabletserver/vstreamer/planbuilder.go | 22 +------------------ .../vstreamer/rowstreamer_test.go | 2 -- .../tabletserver/vstreamer/vstreamer.go | 1 + 4 files changed, 5 insertions(+), 25 deletions(-) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index e29e3f0fabe..92866f63859 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -447,15 +447,16 @@ func CellValue(data []byte, pos int, typ byte, metadata uint16, styp querypb.Typ return sqltypes.MakeTrusted(querypb.Type_DATETIME, []byte(fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second))), 8, nil case TypeVarchar, TypeVarString: + // We trust that styp is compatible with the column type // Length is encoded in 1 or 2 bytes. if metadata > 255 { l := int(uint64(data[pos]) | uint64(data[pos+1])<<8) - return sqltypes.MakeTrusted(querypb.Type_VARCHAR, + return sqltypes.MakeTrusted(styp, data[pos+2:pos+2+l]), l + 2, nil } l := int(data[pos]) - return sqltypes.MakeTrusted(querypb.Type_VARCHAR, + return sqltypes.MakeTrusted(styp, data[pos+1:pos+1+l]), l + 1, nil case TypeBit: // The contents is just the bytes, quoted. diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index 2d4d43819f7..48de92a03d3 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -110,33 +110,13 @@ func (plan *Plan) fields() []*querypb.Field { return fields } -func resolveValue(val sqltypes.Value) (sqltypes.Value, error) { - pv, err := sqlparser.NewPlanValue(sqlparser.NewStrVal(val.Raw())) - if err != nil { - return sqltypes.NULL, err - } - resolved, err := pv.ResolveValue(nil) - if err != nil { - return sqltypes.NULL, err - } - return resolved, nil -} - // filter filters the row against the plan. It returns false if the row did not match. // If the row matched, it returns the columns to be sent. func (plan *Plan) filter(values []sqltypes.Value) (bool, []sqltypes.Value, error) { for _, filter := range plan.Filters { switch filter.Opcode { case Equal: - filterValue, err := resolveValue(filter.Value) - if err != nil { - return false, nil, err - } - colValue, err := resolveValue(values[filter.ColNum]) - if err != nil { - return false, nil, err - } - result, err := sqltypes.NullsafeCompare(colValue, filterValue) + result, err := sqltypes.NullsafeCompare(values[filter.ColNum], filter.Value) if err != nil { return false, nil, err } diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go index 8b2c4a39007..01056c893e3 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go @@ -231,7 +231,6 @@ func TestStreamRowsFilterInt(t *testing.T) { time.Sleep(1 * time.Second) - // Only the first row should be returned, but lastpk should be 6. wantStream := []string{ `fields: fields: pkfields: `, `rows: rows: lastpk: `, @@ -261,7 +260,6 @@ func TestStreamRowsFilterVarBinary(t *testing.T) { time.Sleep(1 * time.Second) - // Only the first row should be returned, but lastpk should be 6. wantStream := []string{ `fields: fields: pkfields: `, `rows: rows: rows: lastpk: `, diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 08c5ea5aa64..e7025f9b794 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -630,6 +630,7 @@ func (vs *vstreamer) buildTableColumns(id uint64, tm *mysql.TableMap) ([]schema. if !sqltypes.AreTypesEquivalent(cols[i].Type, st.Columns[i].Type) { return cols, nil } + cols[i].Type = st.Columns[i].Type } // Columns should be truncated to match those in tm. From 1ef8d29f08abeefc3f151fd79c396ba8dbd5cdcb Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sat, 21 Mar 2020 20:07:42 +0100 Subject: [PATCH 17/85] Vreplication and multiple where clause tests Signed-off-by: Rohit Nayak --- examples/are-you-alive/go.mod | 2 + .../vreplication/vplayer_test.go | 38 +++++++++++++++ .../tabletserver/vstreamer/vstreamer_test.go | 46 +++++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/examples/are-you-alive/go.mod b/examples/are-you-alive/go.mod index 9bb69e1307b..5f3d04ddccd 100644 --- a/examples/are-you-alive/go.mod +++ b/examples/are-you-alive/go.mod @@ -4,6 +4,8 @@ go 1.12 require ( github.com/go-sql-driver/mysql v1.5.0 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/prometheus/client_golang v1.4.1 github.com/sirupsen/logrus v1.4.2 ) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go index dd95e70912a..5abc68b0b3c 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go @@ -135,6 +135,10 @@ func TestPlayerFilters(t *testing.T) { "create table no(id int, val varbinary(128), primary key(id))", "create table nopk(id int, val varbinary(128))", fmt.Sprintf("create table %s.nopk(id int, val varbinary(128))", vrepldb), + "create table src4(id1 int, id2 int, val varbinary(128), primary key(id1))", + fmt.Sprintf("create table %s.dst4(id1 int, val varbinary(128), primary key(id1))", vrepldb), + "create table src5(id1 int, id2 int, val varbinary(128), primary key(id1))", + fmt.Sprintf("create table %s.dst5(id1 int, val varbinary(128), primary key(id1))", vrepldb), }) defer execStatements(t, []string{ "drop table src1", @@ -148,6 +152,10 @@ func TestPlayerFilters(t *testing.T) { "drop table no", "drop table nopk", fmt.Sprintf("drop table %s.nopk", vrepldb), + "drop table src4", + fmt.Sprintf("drop table %s.dst4", vrepldb), + "drop table src5", + fmt.Sprintf("drop table %s.dst5", vrepldb), }) env.SchemaEngine.Reload(context.Background()) @@ -165,6 +173,12 @@ func TestPlayerFilters(t *testing.T) { Match: "/yes", }, { Match: "/nopk", + }, { + Match: "dst4", + Filter: "select id1, val from src4 where id2 = 100", + }, { + Match: "dst5", + Filter: "select id1, val from src5 where val = 'abc'", }}, } bls := &binlogdatapb.BinlogSource{ @@ -386,6 +400,30 @@ func TestPlayerFilters(t *testing.T) { }, table: "nopk", data: [][]string{}, + }, { + // filter by int + input: "insert into src4 values (1,100,'aaa'),(2,200,'bbb'),(3,100,'ccc')", + output: []string{ + "begin", + "insert into dst4(id1,val) values (1,'aaa')", + "insert into dst4(id1,val) values (3,'ccc')", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst4", + data: [][]string{{"1", "aaa"}, {"3", "ccc"}}, + }, { + // filter by int + input: "insert into src5 values (1,100,'abc'),(2,200,'xyz'),(3,100,'xyz'),(4,300,'abc'),(5,200,'xyz')", + output: []string{ + "begin", + "insert into dst5(id1,val) values (1,'abc')", + "insert into dst5(id1,val) values (4,'abc')", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst5", + data: [][]string{{"1", "abc"}, {"4", "abc"}}, }} for _, tcase := range testcases { diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go index c693b9efc2e..98cb8e2ddc1 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go @@ -1241,6 +1241,52 @@ func TestNoFutureGTID(t *testing.T) { } } +func TestFilteredMultipleWhere(t *testing.T) { + if testing.Short() { + t.Skip() + } + + execStatements(t, []string{ + "create table t1(id1 int, id2 int, id3 int, val varbinary(128), primary key(id1))", + }) + defer execStatements(t, []string{ + "drop table t1", + }) + engine.se.Reload(context.Background()) + + setVSchema(t, shardedVSchema) + defer env.SetVSchema("{}") + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select id1, val from t1 where in_keyrange('-80') and id2 = 200 and id3 = 1000 and val = 'newton'", + }}, + } + + testcases := []testcase{{ + input: []string{ + "begin", + "insert into t1 values (1, 100, 1000, 'kepler')", + "insert into t1 values (2, 200, 1000, 'newton')", + "insert into t1 values (3, 100, 2000, 'kepler')", + "insert into t1 values (128, 200, 1000, 'newton')", + "insert into t1 values (5, 200, 2000, 'kepler')", + "insert into t1 values (129, 200, 1000, 'kepler')", + "commit", + }, + output: [][]string{{ + `begin`, + `type:FIELD field_event: fields: > `, + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `gtid`, + `commit`, + }}, + }} + runCases(t, filter, testcases, "") +} + func runCases(t *testing.T, filter *binlogdatapb.Filter, testcases []testcase, position string) { t.Helper() ctx, cancel := context.WithCancel(context.Background()) From 9445b2880fe94f7f7cc1ecd8bcedf36f986d8fa2 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sat, 21 Mar 2020 21:31:43 +0100 Subject: [PATCH 18/85] Only use styp if not null Signed-off-by: Rohit Nayak --- go/mysql/binlog_event_rbr.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index 92866f63859..01a5102e8b6 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -333,6 +333,7 @@ func printTimestamp(v uint32) *bytes.Buffer { // many bytes it takes. It only uses the querypb.Type value for the // signed flag. func CellValue(data []byte, pos int, typ byte, metadata uint16, styp querypb.Type) (sqltypes.Value, int, error) { + fmt.Printf("CellValue typ %v styp %v\n", typ, styp) switch typ { case TypeTiny: if sqltypes.IsSigned(styp) { @@ -449,14 +450,20 @@ func CellValue(data []byte, pos int, typ byte, metadata uint16, styp querypb.Typ case TypeVarchar, TypeVarString: // We trust that styp is compatible with the column type // Length is encoded in 1 or 2 bytes. + var typeToUse querypb.Type + if styp != querypb.Type_NULL_TYPE { + typeToUse = styp + } else { + typeToUse = querypb.Type_VARCHAR + } if metadata > 255 { l := int(uint64(data[pos]) | uint64(data[pos+1])<<8) - return sqltypes.MakeTrusted(styp, + return sqltypes.MakeTrusted(typeToUse, data[pos+2:pos+2+l]), l + 2, nil } l := int(data[pos]) - return sqltypes.MakeTrusted(styp, + return sqltypes.MakeTrusted(typeToUse, data[pos+1:pos+1+l]), l + 1, nil case TypeBit: // The contents is just the bytes, quoted. From 1457bd9496414cc781ea820a4d2da80dddac2a77 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sat, 21 Mar 2020 22:07:41 +0100 Subject: [PATCH 19/85] Removed extraneous log Signed-off-by: Rohit Nayak --- go/mysql/binlog_event_rbr.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index 01a5102e8b6..5d7d2411c3e 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -333,7 +333,6 @@ func printTimestamp(v uint32) *bytes.Buffer { // many bytes it takes. It only uses the querypb.Type value for the // signed flag. func CellValue(data []byte, pos int, typ byte, metadata uint16, styp querypb.Type) (sqltypes.Value, int, error) { - fmt.Printf("CellValue typ %v styp %v\n", typ, styp) switch typ { case TypeTiny: if sqltypes.IsSigned(styp) { From 30e03022d793af92277f36620b0b691943f29f6a Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Mon, 16 Mar 2020 15:43:54 -0700 Subject: [PATCH 20/85] vrepl: where clause filtering WIP Signed-off-by: Sugu Sougoumarane --- examples/are-you-alive/go.mod | 2 - examples/local/x/env.sh | 72 +++++++++++++++++++ examples/local/x/scripts/etcd-down.sh | 22 ++++++ examples/local/x/scripts/etcd-up.sh | 51 +++++++++++++ examples/local/x/scripts/k3s-down.sh | 30 ++++++++ examples/local/x/scripts/k3s-up.sh | 49 +++++++++++++ examples/local/x/scripts/mysqlctl-down.sh | 23 ++++++ examples/local/x/scripts/mysqlctl-up.sh | 42 +++++++++++ examples/local/x/scripts/vtctld-down.sh | 22 ++++++ examples/local/x/scripts/vtctld-up.sh | 38 ++++++++++ examples/local/x/scripts/vtgate-down.sh | 23 ++++++ examples/local/x/scripts/vtgate-up.sh | 58 +++++++++++++++ examples/local/x/scripts/vttablet-down.sh | 30 ++++++++ examples/local/x/scripts/vttablet-up.sh | 69 ++++++++++++++++++ examples/local/x/scripts/zk-down.sh | 26 +++++++ examples/local/x/scripts/zk-up.sh | 62 ++++++++++++++++ .../tabletserver/vstreamer/planbuilder.go | 9 +-- 17 files changed, 622 insertions(+), 6 deletions(-) create mode 100644 examples/local/x/env.sh create mode 100755 examples/local/x/scripts/etcd-down.sh create mode 100755 examples/local/x/scripts/etcd-up.sh create mode 100755 examples/local/x/scripts/k3s-down.sh create mode 100755 examples/local/x/scripts/k3s-up.sh create mode 100755 examples/local/x/scripts/mysqlctl-down.sh create mode 100755 examples/local/x/scripts/mysqlctl-up.sh create mode 100755 examples/local/x/scripts/vtctld-down.sh create mode 100755 examples/local/x/scripts/vtctld-up.sh create mode 100755 examples/local/x/scripts/vtgate-down.sh create mode 100755 examples/local/x/scripts/vtgate-up.sh create mode 100755 examples/local/x/scripts/vttablet-down.sh create mode 100755 examples/local/x/scripts/vttablet-up.sh create mode 100755 examples/local/x/scripts/zk-down.sh create mode 100755 examples/local/x/scripts/zk-up.sh diff --git a/examples/are-you-alive/go.mod b/examples/are-you-alive/go.mod index 5f3d04ddccd..9bb69e1307b 100644 --- a/examples/are-you-alive/go.mod +++ b/examples/are-you-alive/go.mod @@ -4,8 +4,6 @@ go 1.12 require ( github.com/go-sql-driver/mysql v1.5.0 - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect github.com/prometheus/client_golang v1.4.1 github.com/sirupsen/logrus v1.4.2 ) diff --git a/examples/local/x/env.sh b/examples/local/x/env.sh new file mode 100644 index 00000000000..1d21fafc98f --- /dev/null +++ b/examples/local/x/env.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +hostname=`hostname -f` +vtctld_web_port=15000 +export VTDATAROOT="${VTDATAROOT:-${PWD}/vtdataroot}" + +function fail() { + echo "ERROR: $1" + exit 1 +} + +if [[ $EUID -eq 0 ]]; then + fail "This script refuses to be run as root. Please switch to a regular user." +fi + +# mysqld might be in /usr/sbin which will not be in the default PATH +PATH="/usr/sbin:$PATH" +for binary in mysqld etcd etcdctl curl vtctlclient vttablet vtgate vtctld vtctl mysqlctl; do + command -v "$binary" > /dev/null || fail "${binary} is not installed in PATH. See https://vitess.io/docs/get-started/local/ for install instructions." +done; + +if [ "${TOPO}" = "zk2" ]; then + # Each ZooKeeper server needs a list of all servers in the quorum. + # Since we're running them all locally, we need to give them unique ports. + # In a real deployment, these should be on different machines, and their + # respective hostnames should be given. + zkcfg=(\ + "1@$hostname:28881:38881:21811" \ + "2@$hostname:28882:38882:21812" \ + "3@$hostname:28883:38883:21813" \ + ) + printf -v zkcfg ",%s" "${zkcfg[@]}" + zkcfg=${zkcfg:1} + + zkids='1 2 3' + + # Set topology environment parameters. + ZK_SERVER="localhost:21811,localhost:21812,localhost:21813" + # shellcheck disable=SC2034 + TOPOLOGY_FLAGS="-topo_implementation zk2 -topo_global_server_address ${ZK_SERVER} -topo_global_root /vitess/global" +elif [ "${TOPO}" = "k8s" ]; then + # Set topology environment parameters. + K8S_ADDR="localhost" + K8S_PORT="8443" + K8S_KUBECONFIG=$VTDATAROOT/tmp/k8s.kubeconfig + # shellcheck disable=SC2034 + TOPOLOGY_FLAGS="-topo_implementation k8s -topo_k8s_kubeconfig ${K8S_KUBECONFIG} -topo_global_server_address ${K8S_ADDR}:${K8S_PORT} -topo_global_root /vitess/global" +else + ETCD_SERVER="localhost:2379" + TOPOLOGY_FLAGS="-topo_implementation etcd2 -topo_global_server_address $ETCD_SERVER -topo_global_root /vitess/global" + + mkdir -p "${VTDATAROOT}/etcd" +fi + +# Create a tmp dir +mkdir -p "${VTDATAROOT}/tmp" diff --git a/examples/local/x/scripts/etcd-down.sh b/examples/local/x/scripts/etcd-down.sh new file mode 100755 index 00000000000..018af7432a3 --- /dev/null +++ b/examples/local/x/scripts/etcd-down.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops the etcd servers started by etcd-up.sh. + +source ./env.sh + +echo "Stopping etcd..." +kill -9 `cat $VTDATAROOT/tmp/etcd.pid` diff --git a/examples/local/x/scripts/etcd-up.sh b/examples/local/x/scripts/etcd-up.sh new file mode 100755 index 00000000000..d8c0a869a7e --- /dev/null +++ b/examples/local/x/scripts/etcd-up.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that creates a quorum of ZooKeeper servers. + +source ./env.sh + +cell=${CELL:-'test'} +export ETCDCTL_API=2 + +# Check that etcd is not already running +curl "http://${ETCD_SERVER}" > /dev/null 2>&1 && fail "etcd is already running. Exiting." + +etcd --enable-v2=true --data-dir "${VTDATAROOT}/etcd/" --listen-client-urls "http://${ETCD_SERVER}" --advertise-client-urls "http://${ETCD_SERVER}" > "${VTDATAROOT}"/tmp/etcd.out 2>&1 & +PID=$! +echo $PID > "${VTDATAROOT}/tmp/etcd.pid" +sleep 5 + +echo "add /vitess/global" +etcdctl --endpoints "http://${ETCD_SERVER}" mkdir /vitess/global & + +echo "add /vitess/$cell" +etcdctl --endpoints "http://${ETCD_SERVER}" mkdir /vitess/$cell & + +# And also add the CellInfo description for the cell. +# If the node already exists, it's fine, means we used existing data. +echo "add $cell CellInfo" +set +e +# shellcheck disable=SC2086 +vtctl $TOPOLOGY_FLAGS AddCellInfo \ + -root /vitess/$cell \ + -server_address "${ETCD_SERVER}" \ + $cell +set -e + +echo "etcd start done..." + + diff --git a/examples/local/x/scripts/k3s-down.sh b/examples/local/x/scripts/k3s-down.sh new file mode 100755 index 00000000000..590dc604e3e --- /dev/null +++ b/examples/local/x/scripts/k3s-down.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops the k3s server started by k3s-up.sh. + +set -e + +# shellcheck source=./env.sh +# shellcheck disable=SC1091 +source ./env.sh + +# Stop K3s server. +echo "Stopping k3s server..." + +pid=`cat $VTDATAROOT/tmp/k3s.pid` +echo "Stopping k3s..." +kill -9 $pid diff --git a/examples/local/x/scripts/k3s-up.sh b/examples/local/x/scripts/k3s-up.sh new file mode 100755 index 00000000000..3b77600018e --- /dev/null +++ b/examples/local/x/scripts/k3s-up.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that creates a Kubernetes api for topo use by running k3s + +set -e +cell=${CELL:-'test'} + +script_root=$(dirname "${BASH_SOURCE[0]}") + +# shellcheck source=./env.sh +# shellcheck disable=SC1091 +source ./env.sh + +k3s server --disable-agent --data-dir "${VTDATAROOT}/k3s/" --https-listen-port "${K8S_PORT}" --write-kubeconfig "${K8S_KUBECONFIG}" > "${VTDATAROOT}"/tmp/k3s.out 2>&1 & +PID=$! +echo $PID > "${VTDATAROOT}/tmp/k3s.pid" +disown -a +echo "Waiting for k3s server to start" +sleep 15 + +# Use k3s built-in kubectl with custom config +KUBECTL="k3s kubectl --kubeconfig=${K8S_KUBECONFIG}" + +# Create the CRD for vitesstopologynodes +$KUBECTL create -f ../../go/vt/topo/k8stopo/VitessTopoNodes-crd.yaml + +# Add the CellInfo description for the cell +set +e +echo "add $cell CellInfo" +vtctl $TOPOLOGY_FLAGS AddCellInfo \ + -root /vitess/$cell \ + $cell +set -e + +echo "k3s start done..." diff --git a/examples/local/x/scripts/mysqlctl-down.sh b/examples/local/x/scripts/mysqlctl-down.sh new file mode 100755 index 00000000000..9e5491d1d4c --- /dev/null +++ b/examples/local/x/scripts/mysqlctl-down.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops the mysqld and vttablet instances +# created by vttablet-up.sh + +source ./env.sh + +mysqlctl -tablet_uid $TABLET_UID shutdown + diff --git a/examples/local/x/scripts/mysqlctl-up.sh b/examples/local/x/scripts/mysqlctl-up.sh new file mode 100755 index 00000000000..e4c9f58f819 --- /dev/null +++ b/examples/local/x/scripts/mysqlctl-up.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that creates a single shard vttablet deployment. + +source ./env.sh + +cell=${CELL:-'test'} +uid=$TABLET_UID +mysql_port=$[17000 + $uid] +printf -v alias '%s-%010d' $cell $uid +printf -v tablet_dir 'vt_%010d' $uid + +mkdir -p $VTDATAROOT/backups + +echo "Starting MySQL for tablet $alias..." +action="init" + +if [ -d $VTDATAROOT/$tablet_dir ]; then + echo "Resuming from existing vttablet dir:" + echo " $VTDATAROOT/$tablet_dir" + action='start' +fi + +mysqlctl \ + -log_dir $VTDATAROOT/tmp \ + -tablet_uid $uid \ + -mysql_port $mysql_port \ + $action diff --git a/examples/local/x/scripts/vtctld-down.sh b/examples/local/x/scripts/vtctld-down.sh new file mode 100755 index 00000000000..d96fa3b927f --- /dev/null +++ b/examples/local/x/scripts/vtctld-down.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops vtctld. + +source ./env.sh + +echo "Stopping vtctld..." +kill -9 `cat $VTDATAROOT/tmp/vtctld.pid` diff --git a/examples/local/x/scripts/vtctld-up.sh b/examples/local/x/scripts/vtctld-up.sh new file mode 100755 index 00000000000..662a234ae48 --- /dev/null +++ b/examples/local/x/scripts/vtctld-up.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that starts vtctld. + +source ./env.sh + +cell=${CELL:-'test'} +grpc_port=15999 + +echo "Starting vtctld..." +# shellcheck disable=SC2086 +vtctld \ + $TOPOLOGY_FLAGS \ + -cell $cell \ + -workflow_manager_init \ + -workflow_manager_use_election \ + -service_map 'grpc-vtctl' \ + -backup_storage_implementation file \ + -file_backup_storage_root $VTDATAROOT/backups \ + -log_dir $VTDATAROOT/tmp \ + -port $vtctld_web_port \ + -grpc_port $grpc_port \ + -pid_file $VTDATAROOT/tmp/vtctld.pid \ + > $VTDATAROOT/tmp/vtctld.out 2>&1 & diff --git a/examples/local/x/scripts/vtgate-down.sh b/examples/local/x/scripts/vtgate-down.sh new file mode 100755 index 00000000000..9da0a7179df --- /dev/null +++ b/examples/local/x/scripts/vtgate-down.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops the instance started by vtgate-up.sh. + +source ./env.sh + +# Stop vtgate. +echo "Stopping vtgate..." +kill `cat $VTDATAROOT/tmp/vtgate.pid` diff --git a/examples/local/x/scripts/vtgate-up.sh b/examples/local/x/scripts/vtgate-up.sh new file mode 100755 index 00000000000..e3fe2d17f82 --- /dev/null +++ b/examples/local/x/scripts/vtgate-up.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that starts a single vtgate. + +source ./env.sh + +cell=${CELL:-'test'} +web_port=15001 +grpc_port=15991 +mysql_server_port=15306 +mysql_server_socket_path="/tmp/mysql.sock" + +# Start vtgate. +# shellcheck disable=SC2086 +vtgate \ + $TOPOLOGY_FLAGS \ + -log_dir $VTDATAROOT/tmp \ + -log_queries_to_file $VTDATAROOT/tmp/vtgate_querylog.txt \ + -port $web_port \ + -grpc_port $grpc_port \ + -mysql_server_port $mysql_server_port \ + -mysql_server_socket_path $mysql_server_socket_path \ + -cell $cell \ + -cells_to_watch $cell \ + -tablet_types_to_wait MASTER,REPLICA \ + -gateway_implementation discoverygateway \ + -service_map 'grpc-vtgateservice' \ + -pid_file $VTDATAROOT/tmp/vtgate.pid \ + -mysql_auth_server_impl none \ + > $VTDATAROOT/tmp/vtgate.out 2>&1 & + +# Block waiting for vtgate to be listening +# Not the same as healthy + +echo "Waiting for vtgate to be up..." +while true; do + curl -I "http://$hostname:$web_port/debug/status" >/dev/null 2>&1 && break + sleep 0.1 +done; +echo "vtgate is up!" + +echo "Access vtgate at http://$hostname:$web_port/debug/status" + +disown -a diff --git a/examples/local/x/scripts/vttablet-down.sh b/examples/local/x/scripts/vttablet-down.sh new file mode 100755 index 00000000000..47b881b9793 --- /dev/null +++ b/examples/local/x/scripts/vttablet-down.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops the mysqld and vttablet instances +# created by vttablet-up.sh + +source ./env.sh + +printf -v tablet_dir 'vt_%010d' $TABLET_UID +pid=`cat $VTDATAROOT/$tablet_dir/vttablet.pid` + +kill $pid + +# Wait for vttablet to die. +while ps -p $pid > /dev/null; do sleep 1; done + + diff --git a/examples/local/x/scripts/vttablet-up.sh b/examples/local/x/scripts/vttablet-up.sh new file mode 100755 index 00000000000..c3f66fa806d --- /dev/null +++ b/examples/local/x/scripts/vttablet-up.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ./env.sh + +cell=${CELL:-'test'} +keyspace=${KEYSPACE:-'test_keyspace'} +shard=${SHARD:-'0'} +uid=$TABLET_UID +mysql_port=$[17000 + $uid] +port=$[15000 + $uid] +grpc_port=$[16000 + $uid] +printf -v alias '%s-%010d' $cell $uid +printf -v tablet_dir 'vt_%010d' $uid +tablet_hostname='' +printf -v tablet_logfile 'vttablet_%010d_querylog.txt' $uid + +tablet_type=replica +if [[ "${uid: -1}" -gt 1 ]]; then + tablet_type=rdonly +fi + +echo "Starting vttablet for $alias..." +# shellcheck disable=SC2086 +vttablet \ + $TOPOLOGY_FLAGS \ + -log_dir $VTDATAROOT/tmp \ + -log_queries_to_file $VTDATAROOT/tmp/$tablet_logfile \ + -tablet-path $alias \ + -tablet_hostname "$tablet_hostname" \ + -init_keyspace $keyspace \ + -init_shard $shard \ + -init_tablet_type $tablet_type \ + -health_check_interval 5s \ + -enable_semi_sync \ + -enable_replication_reporter \ + -backup_storage_implementation file \ + -file_backup_storage_root $VTDATAROOT/backups \ + -restore_from_backup \ + -port $port \ + -grpc_port $grpc_port \ + -service_map 'grpc-queryservice,grpc-tabletmanager,grpc-updatestream' \ + -pid_file $VTDATAROOT/$tablet_dir/vttablet.pid \ + -vtctld_addr http://$hostname:$vtctld_web_port/ \ + > $VTDATAROOT/$tablet_dir/vttablet.out 2>&1 & + +# Block waiting for the tablet to be listening +# Not the same as healthy + +for i in $(seq 0 300); do + curl -I "http://$hostname:$port/debug/status" >/dev/null 2>&1 && break + sleep 0.1 +done + +# check one last time +curl -I "http://$hostname:$port/debug/status" || fail "tablet could not be started!" diff --git a/examples/local/x/scripts/zk-down.sh b/examples/local/x/scripts/zk-down.sh new file mode 100755 index 00000000000..18dd7933bc9 --- /dev/null +++ b/examples/local/x/scripts/zk-down.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops the ZooKeeper servers started by zk-up.sh. + +source ./env.sh + +# Stop ZooKeeper servers. +echo "Stopping zk servers..." +for zkid in $zkids; do + zkctl -zk.myid $zkid -zk.cfg $zkcfg -log_dir $VTDATAROOT/tmp shutdown +done + diff --git a/examples/local/x/scripts/zk-up.sh b/examples/local/x/scripts/zk-up.sh new file mode 100755 index 00000000000..6671d1063a6 --- /dev/null +++ b/examples/local/x/scripts/zk-up.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that creates a quorum of ZooKeeper servers. + +source ./env.sh + +cell=${CELL:-'test'} + +# Start ZooKeeper servers. +# The "zkctl init" command won't return until the server is able to contact its +# peers, so we need to start them all in the background and then wait for them. +echo "Starting zk servers..." +for zkid in $zkids; do + action='init' + printf -v zkdir 'zk_%03d' $zkid + if [ -f $VTDATAROOT/$zkdir/myid ]; then + echo "Resuming from existing ZK data dir:" + echo " $VTDATAROOT/$zkdir" + action='start' + fi + zkctl -zk.myid $zkid -zk.cfg $zkcfg -log_dir $VTDATAROOT/tmp $action \ + > $VTDATAROOT/tmp/zkctl_$zkid.out 2>&1 & + pids[$zkid]=$! +done + +# Wait for all the zkctl commands to return. +echo "Waiting for zk servers to be ready..." + +for zkid in $zkids; do + if ! wait ${pids[$zkid]}; then + echo "ZK server number $zkid failed to start. See log:" + echo " $VTDATAROOT/tmp/zkctl_$zkid.out" + fi +done + +echo "Started zk servers." + +# Add the CellInfo description for the $CELL cell. +# If the node already exists, it's fine, means we used existing data. +set +e +# shellcheck disable=SC2086 +vtctl $TOPOLOGY_FLAGS AddCellInfo \ + -root /vitess/$cell \ + -server_address $ZK_SERVER \ + $cell +set -e + +echo "Configured zk servers." diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index 48de92a03d3..ed6a56b4ca1 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -37,15 +37,16 @@ import ( type Plan struct { Table *Table - // ColExprs is the list of column expressions to be sent - // in the stream. - ColExprs []ColExpr - // Filters is the list of filters to be applied to the columns // of the table. Filters []Filter + + // ColExprs is the list of column expressions to be sent + // in the stream. + ColExprs []ColExpr } + // Opcode enumerates the operators supported in a where clause type Opcode int From 186247f9a23b767678f2153c2eab33d21e2bd05b Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Mon, 23 Mar 2020 16:38:37 -0700 Subject: [PATCH 21/85] deprecation: delete v2 api code and tests Signed-off-by: Sugu Sougoumarane --- go/cmd/vtgate/vtgate.go | 5 - .../vtgateclienttest/goclienttest/callerid.go | 44 - go/cmd/vtgateclienttest/goclienttest/echo.go | 303 +-- .../vtgateclienttest/goclienttest/errors.go | 138 -- go/cmd/vtgateclienttest/goclienttest/main.go | 1 - .../vtgateclienttest/goclienttest/success.go | 67 - go/cmd/vtgateclienttest/services/callerid.go | 82 - go/cmd/vtgateclienttest/services/echo.go | 200 -- go/cmd/vtgateclienttest/services/errors.go | 157 -- go/cmd/vtgateclienttest/services/fallback.go | 64 - go/cmd/vtgateclienttest/services/success.go | 58 - go/cmd/vtgateclienttest/services/terminal.go | 64 - go/vt/proto/vtgateservice/vtgateservice.pb.go | 818 +----- go/vt/vitessdriver/fakeserver_test.go | 83 - go/vt/vtctl/query.go | 128 - go/vt/vtctl/vtctl.go | 2 +- go/vt/vtgate/fakerpcvtgateconn/conn.go | 160 +- go/vt/vtgate/grpcvtgateconn/conn.go | 326 +-- go/vt/vtgate/grpcvtgateconn/conn_rpc_test.go | 2 +- go/vt/vtgate/grpcvtgateconn/suite_test.go | 2160 ++-------------- go/vt/vtgate/grpcvtgateservice/server.go | 275 -- go/vt/vtgate/vtgate.go | 601 +---- go/vt/vtgate/vtgate_test.go | 2202 ++--------------- go/vt/vtgate/vtgateconn/vtgateconn.go | 227 -- go/vt/vtgate/vtgateservice/interface.go | 25 - proto/vtgateservice.proto | 72 - 26 files changed, 393 insertions(+), 7871 deletions(-) delete mode 100644 go/cmd/vtgateclienttest/goclienttest/success.go diff --git a/go/cmd/vtgate/vtgate.go b/go/cmd/vtgate/vtgate.go index ad8148fe429..55779162e35 100644 --- a/go/cmd/vtgate/vtgate.go +++ b/go/cmd/vtgate/vtgate.go @@ -47,8 +47,6 @@ var ( var resilientServer *srvtopo.ResilientServer var healthCheck discovery.HealthCheck -var initFakeZK func() - func init() { rand.Seed(time.Now().UnixNano()) servenv.RegisterDefaultFlags() @@ -60,9 +58,6 @@ func main() { servenv.ParseFlags("vtgate") servenv.Init() - if initFakeZK != nil { - initFakeZK() - } ts := topo.Open() defer ts.Close() diff --git a/go/cmd/vtgateclienttest/goclienttest/callerid.go b/go/cmd/vtgateclienttest/goclienttest/callerid.go index 98aff6ed552..7d2b4718d7f 100644 --- a/go/cmd/vtgateclienttest/goclienttest/callerid.go +++ b/go/cmd/vtgateclienttest/goclienttest/callerid.go @@ -25,10 +25,6 @@ import ( "vitess.io/vitess/go/cmd/vtgateclienttest/services" "vitess.io/vitess/go/vt/callerid" "vitess.io/vitess/go/vt/vtgate/vtgateconn" - - querypb "vitess.io/vitess/go/vt/proto/query" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" ) // testCallerID adds a caller ID to a context, and makes sure the server @@ -49,49 +45,9 @@ func testCallerID(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgateconn _, err = session.Execute(ctx, query, nil) checkCallerIDError(t, "Execute", err) - _, err = conn.ExecuteShards(ctx, query, "", nil, nil, topodatapb.TabletType_MASTER, nil) - checkCallerIDError(t, "ExecuteShards", err) - - _, err = conn.ExecuteKeyspaceIds(ctx, query, "", nil, nil, topodatapb.TabletType_MASTER, nil) - checkCallerIDError(t, "ExecuteKeyspaceIds", err) - - _, err = conn.ExecuteKeyRanges(ctx, query, "", nil, nil, topodatapb.TabletType_MASTER, nil) - checkCallerIDError(t, "ExecuteKeyRanges", err) - - _, err = conn.ExecuteEntityIds(ctx, query, "", "", nil, nil, topodatapb.TabletType_MASTER, nil) - checkCallerIDError(t, "ExecuteEntityIds", err) - - // test ExecuteBatch calls forward the callerID - _, err = conn.ExecuteBatchShards(ctx, []*vtgatepb.BoundShardQuery{ - { - Query: &querypb.BoundQuery{ - Sql: query, - }, - }, - }, topodatapb.TabletType_MASTER, false, nil) - checkCallerIDError(t, "ExecuteBatchShards", err) - - _, err = conn.ExecuteBatchKeyspaceIds(ctx, []*vtgatepb.BoundKeyspaceIdQuery{ - { - Query: &querypb.BoundQuery{ - Sql: query, - }, - }, - }, topodatapb.TabletType_MASTER, false, nil) - checkCallerIDError(t, "ExecuteBatchKeyspaceIds", err) - // test StreamExecute calls forward the callerID err = getStreamError(session.StreamExecute(ctx, query, nil)) checkCallerIDError(t, "StreamExecute", err) - - err = getStreamError(conn.StreamExecuteShards(ctx, query, "", nil, nil, topodatapb.TabletType_MASTER, nil)) - checkCallerIDError(t, "StreamExecuteShards", err) - - err = getStreamError(conn.StreamExecuteKeyspaceIds(ctx, query, "", nil, nil, topodatapb.TabletType_MASTER, nil)) - checkCallerIDError(t, "StreamExecuteKeyspaceIds", err) - - err = getStreamError(conn.StreamExecuteKeyRanges(ctx, query, "", nil, nil, topodatapb.TabletType_MASTER, nil)) - checkCallerIDError(t, "StreamExecuteKeyRanges", err) } func checkCallerIDError(t *testing.T, name string, err error) { diff --git a/go/cmd/vtgateclienttest/goclienttest/echo.go b/go/cmd/vtgateclienttest/goclienttest/echo.go index 0e8f4c1d89d..eedbb301bda 100644 --- a/go/cmd/vtgateclienttest/goclienttest/echo.go +++ b/go/cmd/vtgateclienttest/goclienttest/echo.go @@ -26,52 +26,12 @@ import ( "vitess.io/vitess/go/vt/vtgate/vtgateconn" querypb "vitess.io/vitess/go/vt/proto/query" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" ) var ( echoPrefix = "echo://" - query = "test query with unicode: \u6211\u80fd\u541e\u4e0b\u73bb\u7483\u800c\u4e0d\u50b7\u8eab\u9ad4" - keyspace = "test_keyspace" - - shards = []string{"-80", "80-"} - shardsEcho = "[-80 80-]" - - keyspaceIDs = [][]byte{ - {1, 2, 3, 4}, - {5, 6, 7, 8}, - } - keyspaceIDsEcho = "[[1 2 3 4] [5 6 7 8]]" - - keyRanges = []*topodatapb.KeyRange{ - {Start: []byte{1, 2, 3, 4}, End: []byte{5, 6, 7, 8}}, - } - keyRangeZeroEcho = "start:\"\\001\\002\\003\\004\" end:\"\\005\\006\\007\\010\" " - keyRangesEcho = "[" + keyRangeZeroEcho + "]" - - entityKeyspaceIDs = []*vtgatepb.ExecuteEntityIdsRequest_EntityId{ - { - KeyspaceId: []byte{1, 2, 3}, - Type: sqltypes.Int64, - Value: []byte("123"), - }, - { - KeyspaceId: []byte{4, 5, 6}, - Type: sqltypes.Float64, - Value: []byte("2"), - }, - { - KeyspaceId: []byte{7, 8, 9}, - Type: sqltypes.VarBinary, - Value: []byte{1, 2, 3}, - }, - } - entityKeyspaceIDsEcho = "[type:INT64 value:\"123\" keyspace_id:\"\\001\\002\\003\" type:FLOAT64 value:\"2\" keyspace_id:\"\\004\\005\\006\" type:VARBINARY value:\"\\001\\002\\003\" keyspace_id:\"\\007\\010\\t\" ]" - - tabletType = topodatapb.TabletType_REPLICA - tabletTypeEcho = topodatapb.TabletType_name[int32(tabletType)] + query = "test query with unicode: \u6211\u80fd\u541e\u4e0b\u73bb\u7483\u800c\u4e0d\u50b7\u8eab\u9ad4" bindVars = map[string]*querypb.BindVariable{ "int": sqltypes.Int64BindVariable(123), @@ -80,22 +40,14 @@ var ( } bindVarsEcho = "map[bytes:type:VARBINARY value:\"\\001\\002\\003\" float:type:FLOAT64 value:\"2.1\" int:type:INT64 value:\"123\" ]" - sessionEcho = "in_transaction:true " - callerID = callerid.NewEffectiveCallerID("test_principal", "test_component", "test_subcomponent") callerIDEcho = "principal:\"test_principal\" component:\"test_component\" subcomponent:\"test_subcomponent\" " - - options = &querypb.ExecuteOptions{ - IncludedFields: querypb.ExecuteOptions_TYPE_ONLY, - } - optionsEcho = "included_fields:TYPE_ONLY " ) // testEcho exercises the test cases provided by the "echo" service. func testEcho(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgateconn.VTGateSession) { testEchoExecute(t, conn, session) testEchoStreamExecute(t, conn, session) - testEchoTransactionExecute(t, conn) } func testEchoExecute(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgateconn.VTGateSession) { @@ -110,95 +62,6 @@ func testEchoExecute(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgatec "query": echoPrefix + query, "bindVars": bindVarsEcho, }) - - qr, err = conn.ExecuteShards(ctx, echoPrefix+query, keyspace, shards, bindVars, tabletType, options) - checkEcho(t, "ExecuteShards", qr, err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "keyspace": keyspace, - "shards": shardsEcho, - "bindVars": bindVarsEcho, - "tabletType": tabletTypeEcho, - "options": optionsEcho, - }) - - qr, err = conn.ExecuteKeyspaceIds(ctx, echoPrefix+query, keyspace, keyspaceIDs, bindVars, tabletType, options) - checkEcho(t, "ExecuteKeyspaceIds", qr, err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "keyspace": keyspace, - "keyspaceIds": keyspaceIDsEcho, - "bindVars": bindVarsEcho, - "tabletType": tabletTypeEcho, - "options": optionsEcho, - }) - - qr, err = conn.ExecuteKeyRanges(ctx, echoPrefix+query, keyspace, keyRanges, bindVars, tabletType, options) - checkEcho(t, "ExecuteKeyRanges", qr, err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "keyspace": keyspace, - "keyRanges": keyRangesEcho, - "bindVars": bindVarsEcho, - "tabletType": tabletTypeEcho, - "options": optionsEcho, - }) - - qr, err = conn.ExecuteEntityIds(ctx, echoPrefix+query, keyspace, "column1", entityKeyspaceIDs, bindVars, tabletType, options) - checkEcho(t, "ExecuteEntityIds", qr, err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "keyspace": keyspace, - "entityColumnName": "column1", - "entityIds": entityKeyspaceIDsEcho, - "bindVars": bindVarsEcho, - "tabletType": tabletTypeEcho, - "options": optionsEcho, - }) - - var qrs []sqltypes.Result - - qrs, err = conn.ExecuteBatchShards(ctx, []*vtgatepb.BoundShardQuery{ - { - Query: &querypb.BoundQuery{ - Sql: echoPrefix + query, - BindVariables: bindVars, - }, - Keyspace: keyspace, - Shards: shards, - }, - }, tabletType, true, options) - checkEcho(t, "ExecuteBatchShards", &qrs[0], err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "keyspace": keyspace, - "shards": shardsEcho, - "bindVars": bindVarsEcho, - "tabletType": tabletTypeEcho, - "asTransaction": "true", - "options": optionsEcho, - }) - - qrs, err = conn.ExecuteBatchKeyspaceIds(ctx, []*vtgatepb.BoundKeyspaceIdQuery{ - { - Query: &querypb.BoundQuery{ - Sql: echoPrefix + query, - BindVariables: bindVars, - }, - Keyspace: keyspace, - KeyspaceIds: keyspaceIDs, - }, - }, tabletType, true, options) - checkEcho(t, "ExecuteBatchKeyspaceIds", &qrs[0], err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "keyspace": keyspace, - "keyspaceIds": keyspaceIDsEcho, - "bindVars": bindVarsEcho, - "tabletType": tabletTypeEcho, - "asTransaction": "true", - "options": optionsEcho, - }) } func testEchoStreamExecute(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgateconn.VTGateSession) { @@ -218,170 +81,6 @@ func testEchoStreamExecute(t *testing.T, conn *vtgateconn.VTGateConn, session *v "query": echoPrefix + query, "bindVars": bindVarsEcho, }) - - stream, err = conn.StreamExecuteShards(ctx, echoPrefix+query, keyspace, shards, bindVars, tabletType, options) - if err != nil { - t.Fatal(err) - } - qr, err = stream.Recv() - checkEcho(t, "StreamExecuteShards", qr, err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "keyspace": keyspace, - "shards": shardsEcho, - "bindVars": bindVarsEcho, - "tabletType": tabletTypeEcho, - "options": optionsEcho, - }) - - stream, err = conn.StreamExecuteKeyspaceIds(ctx, echoPrefix+query, keyspace, keyspaceIDs, bindVars, tabletType, options) - if err != nil { - t.Fatal(err) - } - qr, err = stream.Recv() - checkEcho(t, "StreamExecuteKeyspaceIds", qr, err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "keyspace": keyspace, - "keyspaceIds": keyspaceIDsEcho, - "bindVars": bindVarsEcho, - "tabletType": tabletTypeEcho, - "options": optionsEcho, - }) - - stream, err = conn.StreamExecuteKeyRanges(ctx, echoPrefix+query, keyspace, keyRanges, bindVars, tabletType, options) - if err != nil { - t.Fatal(err) - } - qr, err = stream.Recv() - checkEcho(t, "StreamExecuteKeyRanges", qr, err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "keyspace": keyspace, - "keyRanges": keyRangesEcho, - "bindVars": bindVarsEcho, - "tabletType": tabletTypeEcho, - "options": optionsEcho, - }) -} - -func testEchoTransactionExecute(t *testing.T, conn *vtgateconn.VTGateConn) { - var qr *sqltypes.Result - var err error - - ctx := callerid.NewContext(context.Background(), callerID, nil) - - tx, err := conn.Begin(ctx) - if err != nil { - t.Fatalf("Begin error: %v", err) - } - - qr, err = tx.ExecuteShards(ctx, echoPrefix+query, keyspace, shards, bindVars, tabletType, options) - checkEcho(t, "ExecuteShards", qr, err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "keyspace": keyspace, - "shards": shardsEcho, - "bindVars": bindVarsEcho, - "tabletType": tabletTypeEcho, - "session": sessionEcho, - "notInTransaction": "false", - "options": optionsEcho, - }) - - qr, err = tx.ExecuteKeyspaceIds(ctx, echoPrefix+query, keyspace, keyspaceIDs, bindVars, tabletType, options) - checkEcho(t, "ExecuteKeyspaceIds", qr, err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "keyspace": keyspace, - "keyspaceIds": keyspaceIDsEcho, - "bindVars": bindVarsEcho, - "tabletType": tabletTypeEcho, - "session": sessionEcho, - "notInTransaction": "false", - "options": optionsEcho, - }) - - qr, err = tx.ExecuteKeyRanges(ctx, echoPrefix+query, keyspace, keyRanges, bindVars, tabletType, options) - checkEcho(t, "ExecuteKeyRanges", qr, err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "keyspace": keyspace, - "keyRanges": keyRangesEcho, - "bindVars": bindVarsEcho, - "tabletType": tabletTypeEcho, - "session": sessionEcho, - "notInTransaction": "false", - "options": optionsEcho, - }) - - qr, err = tx.ExecuteEntityIds(ctx, echoPrefix+query, keyspace, "column1", entityKeyspaceIDs, bindVars, tabletType, options) - checkEcho(t, "ExecuteEntityIds", qr, err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "keyspace": keyspace, - "entityColumnName": "column1", - "entityIds": entityKeyspaceIDsEcho, - "bindVars": bindVarsEcho, - "tabletType": tabletTypeEcho, - "session": sessionEcho, - "notInTransaction": "false", - "options": optionsEcho, - }) - - if err := tx.Rollback(ctx); err != nil { - t.Fatalf("Rollback error: %v", err) - } - tx, err = conn.Begin(ctx) - if err != nil { - t.Fatalf("Begin (again) error: %v", err) - } - - var qrs []sqltypes.Result - - qrs, err = tx.ExecuteBatchShards(ctx, []*vtgatepb.BoundShardQuery{ - { - Query: &querypb.BoundQuery{ - Sql: echoPrefix + query, - BindVariables: bindVars, - }, - Keyspace: keyspace, - Shards: shards, - }, - }, tabletType, options) - checkEcho(t, "ExecuteBatchShards", &qrs[0], err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "keyspace": keyspace, - "shards": shardsEcho, - "bindVars": bindVarsEcho, - "tabletType": tabletTypeEcho, - "session": sessionEcho, - "asTransaction": "false", - "options": optionsEcho, - }) - - qrs, err = tx.ExecuteBatchKeyspaceIds(ctx, []*vtgatepb.BoundKeyspaceIdQuery{ - { - Query: &querypb.BoundQuery{ - Sql: echoPrefix + query, - BindVariables: bindVars, - }, - Keyspace: keyspace, - KeyspaceIds: keyspaceIDs, - }, - }, tabletType, options) - checkEcho(t, "ExecuteBatchKeyspaceIds", &qrs[0], err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "keyspace": keyspace, - "keyspaceIds": keyspaceIDsEcho, - "bindVars": bindVarsEcho, - "tabletType": tabletTypeEcho, - "session": sessionEcho, - "asTransaction": "false", - "options": optionsEcho, - }) } // getEcho extracts the echoed field values from a query result. diff --git a/go/cmd/vtgateclienttest/goclienttest/errors.go b/go/cmd/vtgateclienttest/goclienttest/errors.go index 93327e12dae..063b561cfca 100644 --- a/go/cmd/vtgateclienttest/goclienttest/errors.go +++ b/go/cmd/vtgateclienttest/goclienttest/errors.go @@ -27,8 +27,6 @@ import ( "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/vtgateconn" - querypb "vitess.io/vitess/go/vt/proto/query" - vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) @@ -51,7 +49,6 @@ var ( func testErrors(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgateconn.VTGateSession) { testExecuteErrors(t, conn, session) testStreamExecuteErrors(t, conn, session) - testTransactionExecuteErrors(t, conn) } func testExecuteErrors(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgateconn.VTGateSession) { @@ -61,48 +58,6 @@ func testExecuteErrors(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgat _, err := session.Execute(ctx, query, bindVars) return err }) - checkExecuteErrors(t, func(query string) error { - _, err := conn.ExecuteShards(ctx, query, keyspace, shards, bindVars, tabletType, nil) - return err - }) - checkExecuteErrors(t, func(query string) error { - _, err := conn.ExecuteKeyspaceIds(ctx, query, keyspace, keyspaceIDs, bindVars, tabletType, nil) - return err - }) - checkExecuteErrors(t, func(query string) error { - _, err := conn.ExecuteKeyRanges(ctx, query, keyspace, keyRanges, bindVars, tabletType, nil) - return err - }) - checkExecuteErrors(t, func(query string) error { - _, err := conn.ExecuteEntityIds(ctx, query, keyspace, "column1", entityKeyspaceIDs, bindVars, tabletType, nil) - return err - }) - checkExecuteErrors(t, func(query string) error { - _, err := conn.ExecuteBatchShards(ctx, []*vtgatepb.BoundShardQuery{ - { - Query: &querypb.BoundQuery{ - Sql: query, - BindVariables: bindVars, - }, - Keyspace: keyspace, - Shards: shards, - }, - }, tabletType, true, nil) - return err - }) - checkExecuteErrors(t, func(query string) error { - _, err := conn.ExecuteBatchKeyspaceIds(ctx, []*vtgatepb.BoundKeyspaceIdQuery{ - { - Query: &querypb.BoundQuery{ - Sql: query, - BindVariables: bindVars, - }, - Keyspace: keyspace, - KeyspaceIds: keyspaceIDs, - }, - }, tabletType, true, nil) - return err - }) } func testStreamExecuteErrors(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgateconn.VTGateSession) { @@ -111,62 +66,6 @@ func testStreamExecuteErrors(t *testing.T, conn *vtgateconn.VTGateConn, session checkStreamExecuteErrors(t, func(query string) error { return getStreamError(session.StreamExecute(ctx, query, bindVars)) }) - checkStreamExecuteErrors(t, func(query string) error { - return getStreamError(conn.StreamExecuteShards(ctx, query, keyspace, shards, bindVars, tabletType, nil)) - }) - checkStreamExecuteErrors(t, func(query string) error { - return getStreamError(conn.StreamExecuteKeyspaceIds(ctx, query, keyspace, keyspaceIDs, bindVars, tabletType, nil)) - }) - checkStreamExecuteErrors(t, func(query string) error { - return getStreamError(conn.StreamExecuteKeyRanges(ctx, query, keyspace, keyRanges, bindVars, tabletType, nil)) - }) -} - -func testTransactionExecuteErrors(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := context.Background() - - checkTransactionExecuteErrors(t, conn, func(tx *vtgateconn.VTGateTx, query string) error { - _, err := tx.ExecuteShards(ctx, query, keyspace, shards, bindVars, tabletType, nil) - return err - }) - checkTransactionExecuteErrors(t, conn, func(tx *vtgateconn.VTGateTx, query string) error { - _, err := tx.ExecuteKeyspaceIds(ctx, query, keyspace, keyspaceIDs, bindVars, tabletType, nil) - return err - }) - checkTransactionExecuteErrors(t, conn, func(tx *vtgateconn.VTGateTx, query string) error { - _, err := tx.ExecuteKeyRanges(ctx, query, keyspace, keyRanges, bindVars, tabletType, nil) - return err - }) - checkTransactionExecuteErrors(t, conn, func(tx *vtgateconn.VTGateTx, query string) error { - _, err := tx.ExecuteEntityIds(ctx, query, keyspace, "column1", entityKeyspaceIDs, bindVars, tabletType, nil) - return err - }) - checkTransactionExecuteErrors(t, conn, func(tx *vtgateconn.VTGateTx, query string) error { - _, err := tx.ExecuteBatchShards(ctx, []*vtgatepb.BoundShardQuery{ - { - Query: &querypb.BoundQuery{ - Sql: query, - BindVariables: bindVars, - }, - Keyspace: keyspace, - Shards: shards, - }, - }, tabletType, nil) - return err - }) - checkTransactionExecuteErrors(t, conn, func(tx *vtgateconn.VTGateTx, query string) error { - _, err := tx.ExecuteBatchKeyspaceIds(ctx, []*vtgatepb.BoundKeyspaceIdQuery{ - { - Query: &querypb.BoundQuery{ - Sql: query, - BindVariables: bindVars, - }, - Keyspace: keyspace, - KeyspaceIds: keyspaceIDs, - }, - }, tabletType, nil) - return err - }) } func getStreamError(stream sqltypes.ResultStream, err error) error { @@ -203,43 +102,6 @@ func checkStreamExecuteErrors(t *testing.T, execute func(string) error) { } } -func checkTransactionExecuteErrors(t *testing.T, conn *vtgateconn.VTGateConn, execute func(tx *vtgateconn.VTGateTx, query string) error) { - ctx := context.Background() - - for errStr, errCode := range executeErrors { - query := errorPrefix + errStr - tx, err := conn.Begin(ctx) - if err != nil { - t.Errorf("[%v] Begin error: %v", query, err) - } - checkError(t, execute(tx, query), query, errStr, errCode) - - // Partial error where server doesn't close the session. - query = partialErrorPrefix + errStr - tx, err = conn.Begin(ctx) - if err != nil { - t.Errorf("[%v] Begin error: %v", query, err) - } - checkError(t, execute(tx, query), query, errStr, errCode) - // The transaction should still be usable now. - if err := tx.Rollback(ctx); err != nil { - t.Errorf("[%v] Rollback error: %v", query, err) - } - - // Partial error where server closes the session. - tx, err = conn.Begin(ctx) - if err != nil { - t.Errorf("[%v] Begin error: %v", query, err) - } - query = partialErrorPrefix + errStr + "/close transaction" - checkError(t, execute(tx, query), query, errStr, errCode) - // The transaction should be unusable now. - if tx.Rollback(ctx) == nil { - t.Errorf("[%v] expected Rollback error, got nil", query) - } - } -} - func checkError(t *testing.T, err error, query, errStr string, errCode vtrpcpb.Code) { if err == nil { t.Errorf("[%v] expected error, got nil", query) diff --git a/go/cmd/vtgateclienttest/goclienttest/main.go b/go/cmd/vtgateclienttest/goclienttest/main.go index ab33a400164..d8fc0a48ead 100644 --- a/go/cmd/vtgateclienttest/goclienttest/main.go +++ b/go/cmd/vtgateclienttest/goclienttest/main.go @@ -46,7 +46,6 @@ func TestGoClient(t *testing.T, protocol, addr string) { testCallerID(t, conn, session) testEcho(t, conn, session) testErrors(t, conn, session) - testSuccess(t, conn) // and clean up conn.Close() diff --git a/go/cmd/vtgateclienttest/goclienttest/success.go b/go/cmd/vtgateclienttest/goclienttest/success.go deleted file mode 100644 index 5bbe7439aa6..00000000000 --- a/go/cmd/vtgateclienttest/goclienttest/success.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package goclienttest - -import ( - "testing" - - "golang.org/x/net/context" - - "github.com/golang/protobuf/proto" - "vitess.io/vitess/go/vt/vtgate/vtgateconn" - - topodatapb "vitess.io/vitess/go/vt/proto/topodata" -) - -// testSuccess exercises the test cases provided by the "success" service. -func testSuccess(t *testing.T, conn *vtgateconn.VTGateConn) { - testGetSrvKeyspace(t, conn) -} - -func testGetSrvKeyspace(t *testing.T, conn *vtgateconn.VTGateConn) { - want := &topodatapb.SrvKeyspace{ - Partitions: []*topodatapb.SrvKeyspace_KeyspacePartition{ - { - ServedType: topodatapb.TabletType_REPLICA, - ShardReferences: []*topodatapb.ShardReference{ - { - Name: "shard0", - KeyRange: &topodatapb.KeyRange{ - Start: []byte{0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - End: []byte{0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - }, - }, - }, - }, - }, - ShardingColumnName: "sharding_column_name", - ShardingColumnType: topodatapb.KeyspaceIdType_UINT64, - ServedFrom: []*topodatapb.SrvKeyspace_ServedFrom{ - { - TabletType: topodatapb.TabletType_MASTER, - Keyspace: "other_keyspace", - }, - }, - } - got, err := conn.GetSrvKeyspace(context.Background(), "big") - if err != nil { - t.Fatalf("GetSrvKeyspace error: %v", err) - } - if !proto.Equal(got, want) { - t.Errorf("GetSrvKeyspace() = %v, want %v", proto.MarshalTextString(got), proto.MarshalTextString(want)) - } -} diff --git a/go/cmd/vtgateclienttest/services/callerid.go b/go/cmd/vtgateclienttest/services/callerid.go index ec8a8a16fc5..99d68781b46 100644 --- a/go/cmd/vtgateclienttest/services/callerid.go +++ b/go/cmd/vtgateclienttest/services/callerid.go @@ -29,7 +29,6 @@ import ( "vitess.io/vitess/go/vt/vtgate/vtgateservice" querypb "vitess.io/vitess/go/vt/proto/query" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) @@ -99,84 +98,3 @@ func (c *callerIDClient) StreamExecute(ctx context.Context, session *vtgatepb.Se } return c.fallbackClient.StreamExecute(ctx, session, sql, bindVariables, callback) } - -func (c *callerIDClient) ExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if ok, err := c.checkCallerID(ctx, sql); ok { - return nil, err - } - return c.fallbackClient.ExecuteShards(ctx, sql, bindVariables, keyspace, shards, tabletType, session, notInTransaction, options) -} - -func (c *callerIDClient) ExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if ok, err := c.checkCallerID(ctx, sql); ok { - return nil, err - } - return c.fallbackClient.ExecuteKeyspaceIds(ctx, sql, bindVariables, keyspace, keyspaceIds, tabletType, session, notInTransaction, options) -} - -func (c *callerIDClient) ExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if ok, err := c.checkCallerID(ctx, sql); ok { - return nil, err - } - return c.fallbackClient.ExecuteKeyRanges(ctx, sql, bindVariables, keyspace, keyRanges, tabletType, session, notInTransaction, options) -} - -func (c *callerIDClient) ExecuteEntityIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, entityColumnName string, entityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if ok, err := c.checkCallerID(ctx, sql); ok { - return nil, err - } - return c.fallbackClient.ExecuteEntityIds(ctx, sql, bindVariables, keyspace, entityColumnName, entityKeyspaceIDs, tabletType, session, notInTransaction, options) -} - -func (c *callerIDClient) ExecuteBatchShards(ctx context.Context, queries []*vtgatepb.BoundShardQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - if len(queries) == 1 { - if ok, err := c.checkCallerID(ctx, queries[0].Query.Sql); ok { - return nil, err - } - } - return c.fallbackClient.ExecuteBatchShards(ctx, queries, tabletType, asTransaction, session, options) -} - -func (c *callerIDClient) ExecuteBatchKeyspaceIds(ctx context.Context, queries []*vtgatepb.BoundKeyspaceIdQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - if len(queries) == 1 { - if ok, err := c.checkCallerID(ctx, queries[0].Query.Sql); ok { - return nil, err - } - } - return c.fallbackClient.ExecuteBatchKeyspaceIds(ctx, queries, tabletType, asTransaction, session, options) -} - -func (c *callerIDClient) StreamExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - if ok, err := c.checkCallerID(ctx, sql); ok { - return err - } - return c.fallbackClient.StreamExecuteShards(ctx, sql, bindVariables, keyspace, shards, tabletType, options, callback) -} - -func (c *callerIDClient) StreamExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - if ok, err := c.checkCallerID(ctx, sql); ok { - return err - } - return c.fallbackClient.StreamExecuteKeyspaceIds(ctx, sql, bindVariables, keyspace, keyspaceIds, tabletType, options, callback) -} - -func (c *callerIDClient) StreamExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - if ok, err := c.checkCallerID(ctx, sql); ok { - return err - } - return c.fallbackClient.StreamExecuteKeyRanges(ctx, sql, bindVariables, keyspace, keyRanges, tabletType, options, callback) -} - -func (c *callerIDClient) MessageStream(ctx context.Context, keyspace string, shard string, keyRange *topodatapb.KeyRange, name string, callback func(*sqltypes.Result) error) error { - if ok, err := c.checkCallerID(ctx, name); ok { - return err - } - return c.fallback.MessageStream(ctx, keyspace, shard, keyRange, name, callback) -} - -func (c *callerIDClient) MessageAck(ctx context.Context, keyspace string, name string, ids []*querypb.Value) (int64, error) { - if ok, err := c.checkCallerID(ctx, name); ok { - return 0, err - } - return c.fallback.MessageAck(ctx, keyspace, name, ids) -} diff --git a/go/cmd/vtgateclienttest/services/echo.go b/go/cmd/vtgateclienttest/services/echo.go index f200d07bdab..138215cbb5e 100644 --- a/go/cmd/vtgateclienttest/services/echo.go +++ b/go/cmd/vtgateclienttest/services/echo.go @@ -130,206 +130,6 @@ func (c *echoClient) ExecuteBatch(ctx context.Context, session *vtgatepb.Session return c.fallbackClient.ExecuteBatch(ctx, session, sqlList, bindVariablesList) } -func (c *echoClient) StreamExecute(ctx context.Context, session *vtgatepb.Session, sql string, bindVariables map[string]*querypb.BindVariable, callback func(*sqltypes.Result) error) error { - if strings.HasPrefix(sql, EchoPrefix) { - callback(echoQueryResult(map[string]interface{}{ - "callerId": callerid.EffectiveCallerIDFromContext(ctx), - "query": sql, - "bindVars": bindVariables, - "session": session, - })) - return nil - } - return c.fallbackClient.StreamExecute(ctx, session, sql, bindVariables, callback) -} - -func (c *echoClient) ExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if strings.HasPrefix(sql, EchoPrefix) { - return echoQueryResult(map[string]interface{}{ - "callerId": callerid.EffectiveCallerIDFromContext(ctx), - "query": sql, - "bindVars": bindVariables, - "keyspace": keyspace, - "shards": shards, - "tabletType": tabletType, - "session": session, - "notInTransaction": notInTransaction, - "options": options, - }), nil - } - return c.fallbackClient.ExecuteShards(ctx, sql, bindVariables, keyspace, shards, tabletType, session, notInTransaction, options) -} - -func (c *echoClient) ExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if strings.HasPrefix(sql, EchoPrefix) { - return echoQueryResult(map[string]interface{}{ - "callerId": callerid.EffectiveCallerIDFromContext(ctx), - "query": sql, - "bindVars": bindVariables, - "keyspace": keyspace, - "keyspaceIds": keyspaceIds, - "tabletType": tabletType, - "session": session, - "notInTransaction": notInTransaction, - "options": options, - }), nil - } - return c.fallbackClient.ExecuteKeyspaceIds(ctx, sql, bindVariables, keyspace, keyspaceIds, tabletType, session, notInTransaction, options) -} - -func (c *echoClient) ExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if strings.HasPrefix(sql, EchoPrefix) { - return echoQueryResult(map[string]interface{}{ - "callerId": callerid.EffectiveCallerIDFromContext(ctx), - "query": sql, - "bindVars": bindVariables, - "keyspace": keyspace, - "keyRanges": keyRanges, - "tabletType": tabletType, - "session": session, - "notInTransaction": notInTransaction, - "options": options, - }), nil - } - return c.fallbackClient.ExecuteKeyRanges(ctx, sql, bindVariables, keyspace, keyRanges, tabletType, session, notInTransaction, options) -} - -func (c *echoClient) ExecuteEntityIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, entityColumnName string, entityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if strings.HasPrefix(sql, EchoPrefix) { - return echoQueryResult(map[string]interface{}{ - "callerId": callerid.EffectiveCallerIDFromContext(ctx), - "query": sql, - "bindVars": bindVariables, - "keyspace": keyspace, - "entityColumnName": entityColumnName, - "entityIds": entityKeyspaceIDs, - "tabletType": tabletType, - "session": session, - "notInTransaction": notInTransaction, - "options": options, - }), nil - } - return c.fallbackClient.ExecuteEntityIds(ctx, sql, bindVariables, keyspace, entityColumnName, entityKeyspaceIDs, tabletType, session, notInTransaction, options) -} - -func (c *echoClient) ExecuteBatchShards(ctx context.Context, queries []*vtgatepb.BoundShardQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - if len(queries) > 0 && strings.HasPrefix(queries[0].Query.Sql, EchoPrefix) { - var result []sqltypes.Result - for _, query := range queries { - result = append(result, *echoQueryResult(map[string]interface{}{ - "callerId": callerid.EffectiveCallerIDFromContext(ctx), - "query": query.Query.Sql, - "bindVars": query.Query.BindVariables, - "keyspace": query.Keyspace, - "shards": query.Shards, - "tabletType": tabletType, - "session": session, - "asTransaction": asTransaction, - "options": options, - })) - } - return result, nil - } - return c.fallbackClient.ExecuteBatchShards(ctx, queries, tabletType, asTransaction, session, options) -} - -func (c *echoClient) ExecuteBatchKeyspaceIds(ctx context.Context, queries []*vtgatepb.BoundKeyspaceIdQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - if len(queries) > 0 && strings.HasPrefix(queries[0].Query.Sql, EchoPrefix) { - var result []sqltypes.Result - for _, query := range queries { - result = append(result, *echoQueryResult(map[string]interface{}{ - "callerId": callerid.EffectiveCallerIDFromContext(ctx), - "query": query.Query.Sql, - "bindVars": query.Query.BindVariables, - "keyspace": query.Keyspace, - "keyspaceIds": query.KeyspaceIds, - "tabletType": tabletType, - "session": session, - "asTransaction": asTransaction, - "options": options, - })) - } - return result, nil - } - return c.fallbackClient.ExecuteBatchKeyspaceIds(ctx, queries, tabletType, asTransaction, session, options) -} - -func (c *echoClient) StreamExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - if strings.HasPrefix(sql, EchoPrefix) { - callback(echoQueryResult(map[string]interface{}{ - "callerId": callerid.EffectiveCallerIDFromContext(ctx), - "query": sql, - "bindVars": bindVariables, - "keyspace": keyspace, - "shards": shards, - "tabletType": tabletType, - "options": options, - })) - return nil - } - return c.fallbackClient.StreamExecuteShards(ctx, sql, bindVariables, keyspace, shards, tabletType, options, callback) -} - -func (c *echoClient) StreamExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - if strings.HasPrefix(sql, EchoPrefix) { - callback(echoQueryResult(map[string]interface{}{ - "callerId": callerid.EffectiveCallerIDFromContext(ctx), - "query": sql, - "bindVars": bindVariables, - "keyspace": keyspace, - "keyspaceIds": keyspaceIds, - "tabletType": tabletType, - "options": options, - })) - return nil - } - return c.fallbackClient.StreamExecuteKeyspaceIds(ctx, sql, bindVariables, keyspace, keyspaceIds, tabletType, options, callback) -} - -func (c *echoClient) StreamExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - if strings.HasPrefix(sql, EchoPrefix) { - callback(echoQueryResult(map[string]interface{}{ - "callerId": callerid.EffectiveCallerIDFromContext(ctx), - "query": sql, - "bindVars": bindVariables, - "keyspace": keyspace, - "keyRanges": keyRanges, - "tabletType": tabletType, - "options": options, - })) - return nil - } - return c.fallbackClient.StreamExecuteKeyRanges(ctx, sql, bindVariables, keyspace, keyRanges, tabletType, options, callback) -} - -func (c *echoClient) MessageStream(ctx context.Context, keyspace string, shard string, keyRange *topodatapb.KeyRange, name string, callback func(*sqltypes.Result) error) error { - if strings.HasPrefix(name, EchoPrefix) { - callback(echoQueryResult(map[string]interface{}{ - "callerId": callerid.EffectiveCallerIDFromContext(ctx), - "keyspace": keyspace, - "shard": shard, - "keyRange": keyRange, - "name": name, - })) - return nil - } - return c.fallbackClient.MessageStream(ctx, keyspace, shard, keyRange, name, callback) -} - -func (c *echoClient) MessageAck(ctx context.Context, keyspace string, name string, ids []*querypb.Value) (int64, error) { - if strings.HasPrefix(name, EchoPrefix) { - return int64(len(ids)), nil - } - return c.fallback.MessageAck(ctx, keyspace, name, ids) -} - -func (c *echoClient) MessageAckKeyspaceIds(ctx context.Context, keyspace string, name string, idKeyspaceIDs []*vtgatepb.IdKeyspaceId) (int64, error) { - if strings.HasPrefix(name, EchoPrefix) { - return int64(len(idKeyspaceIDs)), nil - } - return c.fallback.MessageAckKeyspaceIds(ctx, keyspace, name, idKeyspaceIDs) -} - func (c *echoClient) VStream(ctx context.Context, tabletType topodatapb.TabletType, vgtid *binlogdatapb.VGtid, filter *binlogdatapb.Filter, callback func([]*binlogdatapb.VEvent) error) error { if strings.HasPrefix(vgtid.ShardGtids[0].Shard, EchoPrefix) { _ = callback([]*binlogdatapb.VEvent{ diff --git a/go/cmd/vtgateclienttest/services/errors.go b/go/cmd/vtgateclienttest/services/errors.go index e5626cc4fba..ad2203b7509 100644 --- a/go/cmd/vtgateclienttest/services/errors.go +++ b/go/cmd/vtgateclienttest/services/errors.go @@ -22,12 +22,10 @@ import ( "golang.org/x/net/context" "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/vt/callerid" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/vtgateservice" querypb "vitess.io/vitess/go/vt/proto/query" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) @@ -141,158 +139,3 @@ func (c *errorClient) StreamExecute(ctx context.Context, session *vtgatepb.Sessi } return c.fallbackClient.StreamExecute(ctx, session, sql, bindVariables, callback) } - -func (c *errorClient) ExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if err := requestToPartialError(sql, session); err != nil { - return nil, err - } - if err := requestToError(sql); err != nil { - return nil, err - } - return c.fallbackClient.ExecuteShards(ctx, sql, bindVariables, keyspace, shards, tabletType, session, notInTransaction, options) -} - -func (c *errorClient) ExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if err := requestToPartialError(sql, session); err != nil { - return nil, err - } - if err := requestToError(sql); err != nil { - return nil, err - } - return c.fallbackClient.ExecuteKeyspaceIds(ctx, sql, bindVariables, keyspace, keyspaceIds, tabletType, session, notInTransaction, options) -} - -func (c *errorClient) ExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if err := requestToPartialError(sql, session); err != nil { - return nil, err - } - if err := requestToError(sql); err != nil { - return nil, err - } - return c.fallbackClient.ExecuteKeyRanges(ctx, sql, bindVariables, keyspace, keyRanges, tabletType, session, notInTransaction, options) -} - -func (c *errorClient) ExecuteEntityIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, entityColumnName string, entityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if err := requestToPartialError(sql, session); err != nil { - return nil, err - } - if err := requestToError(sql); err != nil { - return nil, err - } - return c.fallbackClient.ExecuteEntityIds(ctx, sql, bindVariables, keyspace, entityColumnName, entityKeyspaceIDs, tabletType, session, notInTransaction, options) -} - -func (c *errorClient) ExecuteBatchShards(ctx context.Context, queries []*vtgatepb.BoundShardQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - if len(queries) == 1 { - if err := requestToPartialError(queries[0].Query.Sql, session); err != nil { - return nil, err - } - if err := requestToError(queries[0].Query.Sql); err != nil { - return nil, err - } - } - return c.fallbackClient.ExecuteBatchShards(ctx, queries, tabletType, asTransaction, session, options) -} - -func (c *errorClient) ExecuteBatchKeyspaceIds(ctx context.Context, queries []*vtgatepb.BoundKeyspaceIdQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - if len(queries) == 1 { - if err := requestToPartialError(queries[0].Query.Sql, session); err != nil { - return nil, err - } - if err := requestToError(queries[0].Query.Sql); err != nil { - return nil, err - } - } - return c.fallbackClient.ExecuteBatchKeyspaceIds(ctx, queries, tabletType, asTransaction, session, options) -} - -func (c *errorClient) StreamExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - if err := requestToError(sql); err != nil { - return err - } - return c.fallbackClient.StreamExecuteShards(ctx, sql, bindVariables, keyspace, shards, tabletType, options, callback) -} - -func (c *errorClient) StreamExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - if err := requestToError(sql); err != nil { - return err - } - return c.fallbackClient.StreamExecuteKeyspaceIds(ctx, sql, bindVariables, keyspace, keyspaceIds, tabletType, options, callback) -} - -func (c *errorClient) StreamExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - if err := requestToError(sql); err != nil { - return err - } - return c.fallbackClient.StreamExecuteKeyRanges(ctx, sql, bindVariables, keyspace, keyRanges, tabletType, options, callback) -} - -func (c *errorClient) Begin(ctx context.Context, singledb bool) (*vtgatepb.Session, error) { - // The client sends the error request through the callerid, as there are no other parameters - cid := callerid.EffectiveCallerIDFromContext(ctx) - request := callerid.GetPrincipal(cid) - if err := requestToError(request); err != nil { - return nil, err - } - return c.fallbackClient.Begin(ctx, singledb) -} - -func (c *errorClient) Commit(ctx context.Context, twopc bool, session *vtgatepb.Session) error { - // The client sends the error request through the callerid, as there are no other parameters - cid := callerid.EffectiveCallerIDFromContext(ctx) - request := callerid.GetPrincipal(cid) - if err := requestToError(request); err != nil { - return err - } - return c.fallbackClient.Commit(ctx, twopc, session) -} - -func (c *errorClient) Rollback(ctx context.Context, session *vtgatepb.Session) error { - // The client sends the error request through the callerid, as there are no other parameters - cid := callerid.EffectiveCallerIDFromContext(ctx) - request := callerid.GetPrincipal(cid) - if err := requestToError(request); err != nil { - return err - } - return c.fallbackClient.Rollback(ctx, session) -} - -func (c *errorClient) MessageStream(ctx context.Context, keyspace string, shard string, keyRange *topodatapb.KeyRange, name string, callback func(*sqltypes.Result) error) error { - cid := callerid.EffectiveCallerIDFromContext(ctx) - request := callerid.GetPrincipal(cid) - if err := requestToError(request); err != nil { - return err - } - if err := requestToError(name); err != nil { - return err - } - return c.fallback.MessageStream(ctx, keyspace, shard, keyRange, name, callback) -} - -func (c *errorClient) MessageAck(ctx context.Context, keyspace string, name string, ids []*querypb.Value) (int64, error) { - cid := callerid.EffectiveCallerIDFromContext(ctx) - request := callerid.GetPrincipal(cid) - if err := requestToError(request); err != nil { - return 0, err - } - if err := requestToError(name); err != nil { - return 0, err - } - return c.fallback.MessageAck(ctx, keyspace, name, ids) -} - -func (c *errorClient) MessageAckKeyspaceIds(ctx context.Context, keyspace string, name string, idKeyspaceIDs []*vtgatepb.IdKeyspaceId) (int64, error) { - cid := callerid.EffectiveCallerIDFromContext(ctx) - request := callerid.GetPrincipal(cid) - if err := requestToError(request); err != nil { - return 0, err - } - return c.fallback.MessageAckKeyspaceIds(ctx, keyspace, name, idKeyspaceIDs) -} - -func (c *errorClient) GetSrvKeyspace(ctx context.Context, keyspace string) (*topodatapb.SrvKeyspace, error) { - if err := requestToError(keyspace); err != nil { - return nil, err - } - return c.fallbackClient.GetSrvKeyspace(ctx, keyspace) -} diff --git a/go/cmd/vtgateclienttest/services/fallback.go b/go/cmd/vtgateclienttest/services/fallback.go index 9196030b1a7..1f8b8032afb 100644 --- a/go/cmd/vtgateclienttest/services/fallback.go +++ b/go/cmd/vtgateclienttest/services/fallback.go @@ -52,74 +52,10 @@ func (c fallbackClient) StreamExecute(ctx context.Context, session *vtgatepb.Ses return c.fallback.StreamExecute(ctx, session, sql, bindVariables, callback) } -func (c fallbackClient) ExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - return c.fallback.ExecuteShards(ctx, sql, bindVariables, keyspace, shards, tabletType, session, notInTransaction, options) -} - -func (c fallbackClient) ExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - return c.fallback.ExecuteKeyspaceIds(ctx, sql, bindVariables, keyspace, keyspaceIds, tabletType, session, notInTransaction, options) -} - -func (c fallbackClient) ExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - return c.fallback.ExecuteKeyRanges(ctx, sql, bindVariables, keyspace, keyRanges, tabletType, session, notInTransaction, options) -} - -func (c fallbackClient) ExecuteEntityIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, entityColumnName string, entityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - return c.fallback.ExecuteEntityIds(ctx, sql, bindVariables, keyspace, entityColumnName, entityKeyspaceIDs, tabletType, session, notInTransaction, options) -} - -func (c fallbackClient) ExecuteBatchShards(ctx context.Context, queries []*vtgatepb.BoundShardQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - return c.fallback.ExecuteBatchShards(ctx, queries, tabletType, asTransaction, session, options) -} - -func (c fallbackClient) ExecuteBatchKeyspaceIds(ctx context.Context, queries []*vtgatepb.BoundKeyspaceIdQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - return c.fallback.ExecuteBatchKeyspaceIds(ctx, queries, tabletType, asTransaction, session, options) -} - -func (c fallbackClient) StreamExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - return c.fallback.StreamExecuteShards(ctx, sql, bindVariables, keyspace, shards, tabletType, options, callback) -} - -func (c fallbackClient) StreamExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - return c.fallback.StreamExecuteKeyspaceIds(ctx, sql, bindVariables, keyspace, keyspaceIds, tabletType, options, callback) -} - -func (c fallbackClient) StreamExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - return c.fallback.StreamExecuteKeyRanges(ctx, sql, bindVariables, keyspace, keyRanges, tabletType, options, callback) -} - -func (c fallbackClient) Begin(ctx context.Context, singledb bool) (*vtgatepb.Session, error) { - return c.fallback.Begin(ctx, singledb) -} - -func (c fallbackClient) Commit(ctx context.Context, twopc bool, session *vtgatepb.Session) error { - return c.fallback.Commit(ctx, twopc, session) -} - -func (c fallbackClient) Rollback(ctx context.Context, session *vtgatepb.Session) error { - return c.fallback.Rollback(ctx, session) -} - func (c fallbackClient) ResolveTransaction(ctx context.Context, dtid string) error { return c.fallback.ResolveTransaction(ctx, dtid) } -func (c fallbackClient) MessageStream(ctx context.Context, keyspace string, shard string, keyRange *topodatapb.KeyRange, name string, callback func(*sqltypes.Result) error) error { - return c.fallback.MessageStream(ctx, keyspace, shard, keyRange, name, callback) -} - -func (c fallbackClient) MessageAck(ctx context.Context, keyspace string, name string, ids []*querypb.Value) (int64, error) { - return c.fallback.MessageAck(ctx, keyspace, name, ids) -} - -func (c fallbackClient) MessageAckKeyspaceIds(ctx context.Context, keyspace string, name string, idKeyspaceIDs []*vtgatepb.IdKeyspaceId) (int64, error) { - return c.fallback.MessageAckKeyspaceIds(ctx, keyspace, name, idKeyspaceIDs) -} - -func (c fallbackClient) GetSrvKeyspace(ctx context.Context, keyspace string) (*topodatapb.SrvKeyspace, error) { - return c.fallback.GetSrvKeyspace(ctx, keyspace) -} - func (c fallbackClient) VStream(ctx context.Context, tabletType topodatapb.TabletType, vgtid *binlogdatapb.VGtid, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { return c.fallback.VStream(ctx, tabletType, vgtid, filter, send) } diff --git a/go/cmd/vtgateclienttest/services/success.go b/go/cmd/vtgateclienttest/services/success.go index 0a78cbdde5d..88ad19d3363 100644 --- a/go/cmd/vtgateclienttest/services/success.go +++ b/go/cmd/vtgateclienttest/services/success.go @@ -17,12 +17,7 @@ limitations under the License. package services import ( - "golang.org/x/net/context" - "vitess.io/vitess/go/vt/vtgate/vtgateservice" - - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" ) // successClient implements vtgateservice.VTGateService @@ -37,56 +32,3 @@ func newSuccessClient(fallback vtgateservice.VTGateService) *successClient { fallbackClient: newFallbackClient(fallback), } } - -func (c *successClient) Begin(ctx context.Context, singledb bool) (*vtgatepb.Session, error) { - return &vtgatepb.Session{ - InTransaction: true, - }, nil -} - -func (c *successClient) Commit(ctx context.Context, twopc bool, session *vtgatepb.Session) error { - if session != nil && session.InTransaction { - return nil - } - return c.fallback.Commit(ctx, twopc, session) -} - -func (c *successClient) Rollback(ctx context.Context, session *vtgatepb.Session) error { - if session != nil && session.InTransaction { - return nil - } - return c.fallback.Rollback(ctx, session) -} - -func (c *successClient) GetSrvKeyspace(ctx context.Context, keyspace string) (*topodatapb.SrvKeyspace, error) { - if keyspace == "big" { - return &topodatapb.SrvKeyspace{ - Partitions: []*topodatapb.SrvKeyspace_KeyspacePartition{ - { - ServedType: topodatapb.TabletType_REPLICA, - ShardReferences: []*topodatapb.ShardReference{ - { - Name: "shard0", - KeyRange: &topodatapb.KeyRange{ - Start: []byte{0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - End: []byte{0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - }, - }, - }, - }, - }, - ShardingColumnName: "sharding_column_name", - ShardingColumnType: topodatapb.KeyspaceIdType_UINT64, - ServedFrom: []*topodatapb.SrvKeyspace_ServedFrom{ - { - TabletType: topodatapb.TabletType_MASTER, - Keyspace: "other_keyspace", - }, - }, - }, nil - } - if keyspace == "small" { - return &topodatapb.SrvKeyspace{}, nil - } - return c.fallback.GetSrvKeyspace(ctx, keyspace) -} diff --git a/go/cmd/vtgateclienttest/services/terminal.go b/go/cmd/vtgateclienttest/services/terminal.go index 44c80e26e7d..43a104b7a71 100644 --- a/go/cmd/vtgateclienttest/services/terminal.go +++ b/go/cmd/vtgateclienttest/services/terminal.go @@ -61,74 +61,10 @@ func (c *terminalClient) StreamExecute(ctx context.Context, session *vtgatepb.Se return errTerminal } -func (c *terminalClient) ExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - return nil, errTerminal -} - -func (c *terminalClient) ExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - return nil, errTerminal -} - -func (c *terminalClient) ExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - return nil, errTerminal -} - -func (c *terminalClient) ExecuteEntityIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, entityColumnName string, entityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - return nil, errTerminal -} - -func (c *terminalClient) ExecuteBatchShards(ctx context.Context, queries []*vtgatepb.BoundShardQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - return nil, errTerminal -} - -func (c *terminalClient) ExecuteBatchKeyspaceIds(ctx context.Context, queries []*vtgatepb.BoundKeyspaceIdQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - return nil, errTerminal -} - -func (c *terminalClient) StreamExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - return errTerminal -} - -func (c *terminalClient) StreamExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - return errTerminal -} - -func (c *terminalClient) StreamExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - return errTerminal -} - -func (c *terminalClient) Begin(ctx context.Context, singledb bool) (*vtgatepb.Session, error) { - return nil, errTerminal -} - -func (c *terminalClient) Commit(ctx context.Context, twopc bool, session *vtgatepb.Session) error { - return errTerminal -} - -func (c *terminalClient) Rollback(ctx context.Context, session *vtgatepb.Session) error { - return errTerminal -} - func (c *terminalClient) ResolveTransaction(ctx context.Context, dtid string) error { return errTerminal } -func (c *terminalClient) MessageStream(ctx context.Context, keyspace string, shard string, keyRange *topodatapb.KeyRange, name string, callback func(*sqltypes.Result) error) error { - return errTerminal -} - -func (c *terminalClient) MessageAck(ctx context.Context, keyspace string, name string, ids []*querypb.Value) (int64, error) { - return 0, errTerminal -} - -func (c *terminalClient) MessageAckKeyspaceIds(ctx context.Context, keyspace string, name string, idKeyspaceIDs []*vtgatepb.IdKeyspaceId) (int64, error) { - return 0, errTerminal -} - -func (c *terminalClient) GetSrvKeyspace(ctx context.Context, keyspace string) (*topodatapb.SrvKeyspace, error) { - return nil, errTerminal -} - func (c *terminalClient) VStream(ctx context.Context, tabletType topodatapb.TabletType, vgtid *binlogdatapb.VGtid, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { return errTerminal } diff --git a/go/vt/proto/vtgateservice/vtgateservice.pb.go b/go/vt/proto/vtgateservice/vtgateservice.pb.go index 7f8211a5737..47083723c94 100644 --- a/go/vt/proto/vtgateservice/vtgateservice.pb.go +++ b/go/vt/proto/vtgateservice/vtgateservice.pb.go @@ -12,7 +12,6 @@ import ( grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" - query "vitess.io/vitess/go/vt/proto/query" vtgate "vitess.io/vitess/go/vt/proto/vtgate" ) @@ -30,42 +29,23 @@ const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package func init() { proto.RegisterFile("vtgateservice.proto", fileDescriptor_601ae27c95081e0f) } var fileDescriptor_601ae27c95081e0f = []byte{ - // 556 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x95, 0xdb, 0x6b, 0x13, 0x41, - 0x14, 0xc6, 0xf5, 0xc1, 0x54, 0x8e, 0x8d, 0xc8, 0xb4, 0x4d, 0xda, 0x78, 0x6d, 0xd4, 0x56, 0x7c, - 0x48, 0x44, 0x41, 0x10, 0x44, 0x49, 0xb4, 0x88, 0x14, 0x6f, 0x89, 0x44, 0x28, 0xf8, 0x30, 0xd9, - 0x1e, 0xb6, 0x4b, 0x92, 0x9d, 0x64, 0x67, 0xb2, 0x98, 0xff, 0xda, 0x3f, 0x41, 0xdc, 0xb9, 0xec, - 0xcc, 0xec, 0x24, 0x79, 0xcb, 0x7c, 0xdf, 0x77, 0x7e, 0x3b, 0x73, 0x72, 0x98, 0x81, 0xbd, 0x5c, - 0xc4, 0x54, 0x20, 0xc7, 0x2c, 0x4f, 0x22, 0xec, 0xcc, 0x33, 0x26, 0x18, 0xa9, 0x3b, 0x62, 0x6b, - 0x57, 0x2e, 0xa5, 0xd9, 0xba, 0xb5, 0x58, 0x62, 0xb6, 0x92, 0x8b, 0x97, 0x7f, 0xeb, 0x50, 0x1b, - 0x25, 0x02, 0x39, 0x27, 0x6f, 0x61, 0xe7, 0xec, 0x0f, 0x46, 0x4b, 0x81, 0xa4, 0xd1, 0x51, 0x15, - 0x4a, 0x18, 0xe0, 0x62, 0x89, 0x5c, 0xb4, 0x9a, 0x15, 0x9d, 0xcf, 0x59, 0xca, 0xb1, 0x7d, 0x8d, - 0x9c, 0xc3, 0xae, 0x12, 0xfb, 0x54, 0x44, 0x57, 0xe4, 0xae, 0x17, 0x2d, 0x54, 0xcd, 0xb9, 0x17, - 0x36, 0x0d, 0xec, 0x3b, 0xd4, 0x87, 0x22, 0x43, 0x3a, 0xd3, 0x1b, 0x32, 0x05, 0x8e, 0xac, 0x71, - 0xf7, 0xd7, 0xb8, 0x9a, 0xf7, 0xe2, 0x3a, 0xf9, 0x0a, 0x75, 0x25, 0x0f, 0xaf, 0x68, 0x76, 0xc9, - 0x89, 0xbf, 0x05, 0x29, 0x57, 0x88, 0x9e, 0x6b, 0x76, 0xf8, 0x1b, 0x88, 0xb2, 0xce, 0x71, 0xc5, - 0xe7, 0x34, 0xc2, 0xcf, 0x97, 0x9c, 0x1c, 0x7b, 0x65, 0x96, 0xa7, 0xc9, 0xed, 0x4d, 0x11, 0x83, - 0xff, 0x05, 0x77, 0x4a, 0x7f, 0x40, 0xd3, 0x18, 0x39, 0x79, 0x58, 0xad, 0x94, 0x8e, 0x46, 0x3f, - 0x5a, 0x1f, 0x08, 0x80, 0xcf, 0x52, 0x91, 0x88, 0xd5, 0xff, 0x5d, 0xfb, 0x60, 0xe3, 0xac, 0x03, - 0x5b, 0x81, 0x40, 0x43, 0x8a, 0x3f, 0x53, 0x75, 0xf9, 0x38, 0xf4, 0x47, 0xbb, 0xad, 0x6e, 0x6f, - 0x8a, 0x18, 0xfc, 0x14, 0x9a, 0xb6, 0x6f, 0x37, 0xfd, 0x24, 0x04, 0x08, 0x74, 0xfe, 0x74, 0x6b, - 0xce, 0x7c, 0x6d, 0x0c, 0x7b, 0xce, 0x28, 0xa9, 0xd3, 0xb4, 0x83, 0x73, 0xe6, 0x1e, 0xe7, 0xf1, - 0xc6, 0x8c, 0x35, 0x91, 0x0b, 0x38, 0x74, 0x22, 0xf6, 0x91, 0x4e, 0x83, 0x90, 0xc0, 0x99, 0x9e, - 0x6d, 0x0f, 0x5a, 0x9f, 0x9c, 0x40, 0xc3, 0xcf, 0xa9, 0xd9, 0x7a, 0xba, 0x8e, 0xe3, 0x4e, 0xd8, - 0xc9, 0xb6, 0x98, 0xf5, 0xb1, 0xd7, 0x70, 0xa3, 0x8f, 0x71, 0x92, 0x92, 0x7d, 0x5d, 0x54, 0x2c, - 0x35, 0xea, 0xc0, 0x53, 0x4d, 0xef, 0xdf, 0x40, 0xed, 0x03, 0x9b, 0xcd, 0x12, 0x41, 0x4c, 0x44, - 0xae, 0x75, 0x65, 0xc3, 0x97, 0x4d, 0xe9, 0x7b, 0xb8, 0x39, 0x60, 0xd3, 0xe9, 0x98, 0x46, 0x13, - 0x62, 0xae, 0x2a, 0xad, 0xe8, 0xf2, 0xc3, 0xaa, 0x61, 0x0f, 0xf1, 0x00, 0x39, 0x9b, 0xe6, 0xf8, - 0x33, 0xa3, 0x29, 0xa7, 0x91, 0x48, 0x58, 0x5a, 0x0e, 0x71, 0xd5, 0xab, 0x0c, 0x71, 0x28, 0x62, - 0xf0, 0xdf, 0xa0, 0xfe, 0x05, 0x39, 0xa7, 0x31, 0xca, 0xfe, 0x95, 0x97, 0x90, 0x23, 0x97, 0xb7, - 0xa4, 0xbc, 0xa9, 0x3d, 0xd3, 0xea, 0xf1, 0x47, 0x00, 0x65, 0xf6, 0xa2, 0x09, 0x39, 0xf2, 0x68, - 0xbd, 0xf2, 0xd0, 0x47, 0x2e, 0xaa, 0xe7, 0x9c, 0xfa, 0x02, 0x0e, 0x4a, 0xdd, 0x1e, 0xc3, 0x27, - 0x55, 0x60, 0x60, 0x06, 0x37, 0xb2, 0x7f, 0xc0, 0xed, 0x4f, 0x28, 0x86, 0x59, 0xae, 0x0b, 0x89, - 0xb9, 0x5a, 0x5d, 0x5d, 0xd3, 0x1e, 0xac, 0xb3, 0x0d, 0xf2, 0x1d, 0xec, 0x8c, 0x54, 0xff, 0xcc, - 0x28, 0x8c, 0xdc, 0xce, 0x35, 0x2b, 0x7a, 0xd9, 0xb4, 0x7e, 0x1f, 0xf6, 0x13, 0xd6, 0xc9, 0x8b, - 0x47, 0x4f, 0xbe, 0x82, 0x9d, 0x38, 0x9b, 0x47, 0x17, 0xcf, 0x95, 0x94, 0xb0, 0xae, 0xfc, 0xd5, - 0x8d, 0x59, 0x37, 0x17, 0xdd, 0x22, 0xd2, 0x75, 0x5e, 0xd4, 0x71, 0xad, 0x10, 0x5f, 0xfd, 0x0b, - 0x00, 0x00, 0xff, 0xff, 0x96, 0x13, 0x38, 0x44, 0x7e, 0x07, 0x00, 0x00, + // 247 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x91, 0x3f, 0x4b, 0x03, 0x41, + 0x10, 0xc5, 0x15, 0x21, 0x81, 0x25, 0x69, 0x46, 0x51, 0x88, 0x5a, 0x98, 0xd2, 0xe2, 0x56, 0xb4, + 0x15, 0x8b, 0x03, 0x2b, 0x1b, 0x89, 0x92, 0x42, 0xb0, 0x58, 0x97, 0xe1, 0x5c, 0xd0, 0x9b, 0x73, + 0x67, 0xb2, 0xf8, 0x01, 0xfc, 0xe0, 0xc2, 0xed, 0x9f, 0x70, 0x9e, 0xda, 0xdd, 0xfd, 0xde, 0x9b, + 0xb7, 0xc3, 0x3c, 0xb5, 0x1f, 0xa4, 0x31, 0x82, 0x8c, 0x3e, 0x38, 0x8b, 0x55, 0xe7, 0x49, 0x08, + 0xe6, 0x03, 0xb8, 0x98, 0xc5, 0xdf, 0x28, 0x5e, 0x7e, 0xed, 0xa9, 0xc9, 0xda, 0x09, 0x32, 0xc3, + 0xb5, 0x9a, 0xde, 0x7e, 0xa2, 0xdd, 0x08, 0xc2, 0x61, 0x95, 0x4c, 0x09, 0xac, 0xf0, 0x63, 0x83, + 0x2c, 0x8b, 0xa3, 0x11, 0xe7, 0x8e, 0x5a, 0xc6, 0xe5, 0x0e, 0xdc, 0xa9, 0x59, 0x82, 0xb5, 0x11, + 0xfb, 0x0a, 0xc7, 0x3f, 0xac, 0x3d, 0xcd, 0x39, 0x27, 0xbf, 0x8b, 0x25, 0xec, 0x5e, 0xcd, 0x1f, + 0xc4, 0xa3, 0x79, 0xcf, 0x0b, 0x95, 0x81, 0x01, 0xce, 0x71, 0xa7, 0x7f, 0xa8, 0x39, 0xef, 0x62, + 0x17, 0x9e, 0x15, 0xac, 0x90, 0xe9, 0x2d, 0xe0, 0xa3, 0x37, 0x2d, 0x1b, 0x2b, 0x8e, 0x5a, 0x38, + 0xcb, 0x83, 0x63, 0x2d, 0x67, 0x2f, 0xff, 0xb3, 0x94, 0x85, 0x6f, 0xd4, 0x74, 0x1d, 0x1f, 0xdf, + 0xde, 0x2e, 0x81, 0xd1, 0xed, 0x0a, 0xdf, 0xae, 0x57, 0xd7, 0xea, 0xc0, 0x51, 0x15, 0xfa, 0x22, + 0x62, 0x33, 0x55, 0xe3, 0x3b, 0xfb, 0x74, 0x9e, 0x90, 0x23, 0x1d, 0xbf, 0x74, 0x43, 0x3a, 0x88, + 0xee, 0x2d, 0x7a, 0x50, 0xec, 0xcb, 0xa4, 0x87, 0x57, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xd5, + 0x14, 0x87, 0x30, 0x05, 0x02, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -96,64 +76,9 @@ type VitessClient interface { // Use this method if the query returns a large number of rows. // API group: v3 StreamExecute(ctx context.Context, in *vtgate.StreamExecuteRequest, opts ...grpc.CallOption) (Vitess_StreamExecuteClient, error) - // ExecuteShards executes the query on the specified shards. - // API group: Custom Sharding - ExecuteShards(ctx context.Context, in *vtgate.ExecuteShardsRequest, opts ...grpc.CallOption) (*vtgate.ExecuteShardsResponse, error) - // ExecuteKeyspaceIds executes the query based on the specified keyspace ids. - // API group: Range-based Sharding - ExecuteKeyspaceIds(ctx context.Context, in *vtgate.ExecuteKeyspaceIdsRequest, opts ...grpc.CallOption) (*vtgate.ExecuteKeyspaceIdsResponse, error) - // ExecuteKeyRanges executes the query based on the specified key ranges. - // API group: Range-based Sharding - ExecuteKeyRanges(ctx context.Context, in *vtgate.ExecuteKeyRangesRequest, opts ...grpc.CallOption) (*vtgate.ExecuteKeyRangesResponse, error) - // ExecuteEntityIds executes the query based on the specified external id to keyspace id map. - // API group: Range-based Sharding - ExecuteEntityIds(ctx context.Context, in *vtgate.ExecuteEntityIdsRequest, opts ...grpc.CallOption) (*vtgate.ExecuteEntityIdsResponse, error) - // ExecuteBatchShards executes the list of queries on the specified shards. - // API group: Custom Sharding - ExecuteBatchShards(ctx context.Context, in *vtgate.ExecuteBatchShardsRequest, opts ...grpc.CallOption) (*vtgate.ExecuteBatchShardsResponse, error) - // ExecuteBatchKeyspaceIds executes the list of queries based on the specified keyspace ids. - // API group: Range-based Sharding - ExecuteBatchKeyspaceIds(ctx context.Context, in *vtgate.ExecuteBatchKeyspaceIdsRequest, opts ...grpc.CallOption) (*vtgate.ExecuteBatchKeyspaceIdsResponse, error) - // StreamExecuteShards executes a streaming query based on shards. - // Use this method if the query returns a large number of rows. - // API group: Custom Sharding - StreamExecuteShards(ctx context.Context, in *vtgate.StreamExecuteShardsRequest, opts ...grpc.CallOption) (Vitess_StreamExecuteShardsClient, error) - // StreamExecuteKeyspaceIds executes a streaming query based on keyspace ids. - // Use this method if the query returns a large number of rows. - // API group: Range-based Sharding - StreamExecuteKeyspaceIds(ctx context.Context, in *vtgate.StreamExecuteKeyspaceIdsRequest, opts ...grpc.CallOption) (Vitess_StreamExecuteKeyspaceIdsClient, error) - // StreamExecuteKeyRanges executes a streaming query based on key ranges. - // Use this method if the query returns a large number of rows. - // API group: Range-based Sharding - StreamExecuteKeyRanges(ctx context.Context, in *vtgate.StreamExecuteKeyRangesRequest, opts ...grpc.CallOption) (Vitess_StreamExecuteKeyRangesClient, error) - // Begin a transaction. - // API group: Transactions - Begin(ctx context.Context, in *vtgate.BeginRequest, opts ...grpc.CallOption) (*vtgate.BeginResponse, error) - // Commit a transaction. - // API group: Transactions - Commit(ctx context.Context, in *vtgate.CommitRequest, opts ...grpc.CallOption) (*vtgate.CommitResponse, error) - // Rollback a transaction. - // API group: Transactions - Rollback(ctx context.Context, in *vtgate.RollbackRequest, opts ...grpc.CallOption) (*vtgate.RollbackResponse, error) // ResolveTransaction resolves a transaction. // API group: Transactions ResolveTransaction(ctx context.Context, in *vtgate.ResolveTransactionRequest, opts ...grpc.CallOption) (*vtgate.ResolveTransactionResponse, error) - // MessageStream streams messages from a message table. - MessageStream(ctx context.Context, in *vtgate.MessageStreamRequest, opts ...grpc.CallOption) (Vitess_MessageStreamClient, error) - // MessageAck acks messages for a table. - MessageAck(ctx context.Context, in *vtgate.MessageAckRequest, opts ...grpc.CallOption) (*query.MessageAckResponse, error) - // MessageAckKeyspaceIds routes Message Acks using the associated - // keyspace ids. - MessageAckKeyspaceIds(ctx context.Context, in *vtgate.MessageAckKeyspaceIdsRequest, opts ...grpc.CallOption) (*query.MessageAckResponse, error) - // GetSrvKeyspace returns a SrvKeyspace object (as seen by this vtgate). - // This method is provided as a convenient way for clients to take a - // look at the sharding configuration for a Keyspace. Looking at the - // sharding information should not be used for routing queries (as the - // information may change, use the Execute calls for that). - // It is convenient for monitoring applications for instance, or if - // using custom sharding. - // API group: Topology - GetSrvKeyspace(ctx context.Context, in *vtgate.GetSrvKeyspaceRequest, opts ...grpc.CallOption) (*vtgate.GetSrvKeyspaceResponse, error) // VStream streams binlog events from the requested sources. VStream(ctx context.Context, in *vtgate.VStreamRequest, opts ...grpc.CallOption) (Vitess_VStreamClient, error) } @@ -216,183 +141,6 @@ func (x *vitessStreamExecuteClient) Recv() (*vtgate.StreamExecuteResponse, error return m, nil } -func (c *vitessClient) ExecuteShards(ctx context.Context, in *vtgate.ExecuteShardsRequest, opts ...grpc.CallOption) (*vtgate.ExecuteShardsResponse, error) { - out := new(vtgate.ExecuteShardsResponse) - err := c.cc.Invoke(ctx, "/vtgateservice.Vitess/ExecuteShards", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *vitessClient) ExecuteKeyspaceIds(ctx context.Context, in *vtgate.ExecuteKeyspaceIdsRequest, opts ...grpc.CallOption) (*vtgate.ExecuteKeyspaceIdsResponse, error) { - out := new(vtgate.ExecuteKeyspaceIdsResponse) - err := c.cc.Invoke(ctx, "/vtgateservice.Vitess/ExecuteKeyspaceIds", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *vitessClient) ExecuteKeyRanges(ctx context.Context, in *vtgate.ExecuteKeyRangesRequest, opts ...grpc.CallOption) (*vtgate.ExecuteKeyRangesResponse, error) { - out := new(vtgate.ExecuteKeyRangesResponse) - err := c.cc.Invoke(ctx, "/vtgateservice.Vitess/ExecuteKeyRanges", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *vitessClient) ExecuteEntityIds(ctx context.Context, in *vtgate.ExecuteEntityIdsRequest, opts ...grpc.CallOption) (*vtgate.ExecuteEntityIdsResponse, error) { - out := new(vtgate.ExecuteEntityIdsResponse) - err := c.cc.Invoke(ctx, "/vtgateservice.Vitess/ExecuteEntityIds", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *vitessClient) ExecuteBatchShards(ctx context.Context, in *vtgate.ExecuteBatchShardsRequest, opts ...grpc.CallOption) (*vtgate.ExecuteBatchShardsResponse, error) { - out := new(vtgate.ExecuteBatchShardsResponse) - err := c.cc.Invoke(ctx, "/vtgateservice.Vitess/ExecuteBatchShards", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *vitessClient) ExecuteBatchKeyspaceIds(ctx context.Context, in *vtgate.ExecuteBatchKeyspaceIdsRequest, opts ...grpc.CallOption) (*vtgate.ExecuteBatchKeyspaceIdsResponse, error) { - out := new(vtgate.ExecuteBatchKeyspaceIdsResponse) - err := c.cc.Invoke(ctx, "/vtgateservice.Vitess/ExecuteBatchKeyspaceIds", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *vitessClient) StreamExecuteShards(ctx context.Context, in *vtgate.StreamExecuteShardsRequest, opts ...grpc.CallOption) (Vitess_StreamExecuteShardsClient, error) { - stream, err := c.cc.NewStream(ctx, &_Vitess_serviceDesc.Streams[1], "/vtgateservice.Vitess/StreamExecuteShards", opts...) - if err != nil { - return nil, err - } - x := &vitessStreamExecuteShardsClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type Vitess_StreamExecuteShardsClient interface { - Recv() (*vtgate.StreamExecuteShardsResponse, error) - grpc.ClientStream -} - -type vitessStreamExecuteShardsClient struct { - grpc.ClientStream -} - -func (x *vitessStreamExecuteShardsClient) Recv() (*vtgate.StreamExecuteShardsResponse, error) { - m := new(vtgate.StreamExecuteShardsResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *vitessClient) StreamExecuteKeyspaceIds(ctx context.Context, in *vtgate.StreamExecuteKeyspaceIdsRequest, opts ...grpc.CallOption) (Vitess_StreamExecuteKeyspaceIdsClient, error) { - stream, err := c.cc.NewStream(ctx, &_Vitess_serviceDesc.Streams[2], "/vtgateservice.Vitess/StreamExecuteKeyspaceIds", opts...) - if err != nil { - return nil, err - } - x := &vitessStreamExecuteKeyspaceIdsClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type Vitess_StreamExecuteKeyspaceIdsClient interface { - Recv() (*vtgate.StreamExecuteKeyspaceIdsResponse, error) - grpc.ClientStream -} - -type vitessStreamExecuteKeyspaceIdsClient struct { - grpc.ClientStream -} - -func (x *vitessStreamExecuteKeyspaceIdsClient) Recv() (*vtgate.StreamExecuteKeyspaceIdsResponse, error) { - m := new(vtgate.StreamExecuteKeyspaceIdsResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *vitessClient) StreamExecuteKeyRanges(ctx context.Context, in *vtgate.StreamExecuteKeyRangesRequest, opts ...grpc.CallOption) (Vitess_StreamExecuteKeyRangesClient, error) { - stream, err := c.cc.NewStream(ctx, &_Vitess_serviceDesc.Streams[3], "/vtgateservice.Vitess/StreamExecuteKeyRanges", opts...) - if err != nil { - return nil, err - } - x := &vitessStreamExecuteKeyRangesClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type Vitess_StreamExecuteKeyRangesClient interface { - Recv() (*vtgate.StreamExecuteKeyRangesResponse, error) - grpc.ClientStream -} - -type vitessStreamExecuteKeyRangesClient struct { - grpc.ClientStream -} - -func (x *vitessStreamExecuteKeyRangesClient) Recv() (*vtgate.StreamExecuteKeyRangesResponse, error) { - m := new(vtgate.StreamExecuteKeyRangesResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *vitessClient) Begin(ctx context.Context, in *vtgate.BeginRequest, opts ...grpc.CallOption) (*vtgate.BeginResponse, error) { - out := new(vtgate.BeginResponse) - err := c.cc.Invoke(ctx, "/vtgateservice.Vitess/Begin", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *vitessClient) Commit(ctx context.Context, in *vtgate.CommitRequest, opts ...grpc.CallOption) (*vtgate.CommitResponse, error) { - out := new(vtgate.CommitResponse) - err := c.cc.Invoke(ctx, "/vtgateservice.Vitess/Commit", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *vitessClient) Rollback(ctx context.Context, in *vtgate.RollbackRequest, opts ...grpc.CallOption) (*vtgate.RollbackResponse, error) { - out := new(vtgate.RollbackResponse) - err := c.cc.Invoke(ctx, "/vtgateservice.Vitess/Rollback", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *vitessClient) ResolveTransaction(ctx context.Context, in *vtgate.ResolveTransactionRequest, opts ...grpc.CallOption) (*vtgate.ResolveTransactionResponse, error) { out := new(vtgate.ResolveTransactionResponse) err := c.cc.Invoke(ctx, "/vtgateservice.Vitess/ResolveTransaction", in, out, opts...) @@ -402,67 +150,8 @@ func (c *vitessClient) ResolveTransaction(ctx context.Context, in *vtgate.Resolv return out, nil } -func (c *vitessClient) MessageStream(ctx context.Context, in *vtgate.MessageStreamRequest, opts ...grpc.CallOption) (Vitess_MessageStreamClient, error) { - stream, err := c.cc.NewStream(ctx, &_Vitess_serviceDesc.Streams[4], "/vtgateservice.Vitess/MessageStream", opts...) - if err != nil { - return nil, err - } - x := &vitessMessageStreamClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type Vitess_MessageStreamClient interface { - Recv() (*query.MessageStreamResponse, error) - grpc.ClientStream -} - -type vitessMessageStreamClient struct { - grpc.ClientStream -} - -func (x *vitessMessageStreamClient) Recv() (*query.MessageStreamResponse, error) { - m := new(query.MessageStreamResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *vitessClient) MessageAck(ctx context.Context, in *vtgate.MessageAckRequest, opts ...grpc.CallOption) (*query.MessageAckResponse, error) { - out := new(query.MessageAckResponse) - err := c.cc.Invoke(ctx, "/vtgateservice.Vitess/MessageAck", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *vitessClient) MessageAckKeyspaceIds(ctx context.Context, in *vtgate.MessageAckKeyspaceIdsRequest, opts ...grpc.CallOption) (*query.MessageAckResponse, error) { - out := new(query.MessageAckResponse) - err := c.cc.Invoke(ctx, "/vtgateservice.Vitess/MessageAckKeyspaceIds", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *vitessClient) GetSrvKeyspace(ctx context.Context, in *vtgate.GetSrvKeyspaceRequest, opts ...grpc.CallOption) (*vtgate.GetSrvKeyspaceResponse, error) { - out := new(vtgate.GetSrvKeyspaceResponse) - err := c.cc.Invoke(ctx, "/vtgateservice.Vitess/GetSrvKeyspace", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *vitessClient) VStream(ctx context.Context, in *vtgate.VStreamRequest, opts ...grpc.CallOption) (Vitess_VStreamClient, error) { - stream, err := c.cc.NewStream(ctx, &_Vitess_serviceDesc.Streams[5], "/vtgateservice.Vitess/VStream", opts...) + stream, err := c.cc.NewStream(ctx, &_Vitess_serviceDesc.Streams[1], "/vtgateservice.Vitess/VStream", opts...) if err != nil { return nil, err } @@ -511,64 +200,9 @@ type VitessServer interface { // Use this method if the query returns a large number of rows. // API group: v3 StreamExecute(*vtgate.StreamExecuteRequest, Vitess_StreamExecuteServer) error - // ExecuteShards executes the query on the specified shards. - // API group: Custom Sharding - ExecuteShards(context.Context, *vtgate.ExecuteShardsRequest) (*vtgate.ExecuteShardsResponse, error) - // ExecuteKeyspaceIds executes the query based on the specified keyspace ids. - // API group: Range-based Sharding - ExecuteKeyspaceIds(context.Context, *vtgate.ExecuteKeyspaceIdsRequest) (*vtgate.ExecuteKeyspaceIdsResponse, error) - // ExecuteKeyRanges executes the query based on the specified key ranges. - // API group: Range-based Sharding - ExecuteKeyRanges(context.Context, *vtgate.ExecuteKeyRangesRequest) (*vtgate.ExecuteKeyRangesResponse, error) - // ExecuteEntityIds executes the query based on the specified external id to keyspace id map. - // API group: Range-based Sharding - ExecuteEntityIds(context.Context, *vtgate.ExecuteEntityIdsRequest) (*vtgate.ExecuteEntityIdsResponse, error) - // ExecuteBatchShards executes the list of queries on the specified shards. - // API group: Custom Sharding - ExecuteBatchShards(context.Context, *vtgate.ExecuteBatchShardsRequest) (*vtgate.ExecuteBatchShardsResponse, error) - // ExecuteBatchKeyspaceIds executes the list of queries based on the specified keyspace ids. - // API group: Range-based Sharding - ExecuteBatchKeyspaceIds(context.Context, *vtgate.ExecuteBatchKeyspaceIdsRequest) (*vtgate.ExecuteBatchKeyspaceIdsResponse, error) - // StreamExecuteShards executes a streaming query based on shards. - // Use this method if the query returns a large number of rows. - // API group: Custom Sharding - StreamExecuteShards(*vtgate.StreamExecuteShardsRequest, Vitess_StreamExecuteShardsServer) error - // StreamExecuteKeyspaceIds executes a streaming query based on keyspace ids. - // Use this method if the query returns a large number of rows. - // API group: Range-based Sharding - StreamExecuteKeyspaceIds(*vtgate.StreamExecuteKeyspaceIdsRequest, Vitess_StreamExecuteKeyspaceIdsServer) error - // StreamExecuteKeyRanges executes a streaming query based on key ranges. - // Use this method if the query returns a large number of rows. - // API group: Range-based Sharding - StreamExecuteKeyRanges(*vtgate.StreamExecuteKeyRangesRequest, Vitess_StreamExecuteKeyRangesServer) error - // Begin a transaction. - // API group: Transactions - Begin(context.Context, *vtgate.BeginRequest) (*vtgate.BeginResponse, error) - // Commit a transaction. - // API group: Transactions - Commit(context.Context, *vtgate.CommitRequest) (*vtgate.CommitResponse, error) - // Rollback a transaction. - // API group: Transactions - Rollback(context.Context, *vtgate.RollbackRequest) (*vtgate.RollbackResponse, error) // ResolveTransaction resolves a transaction. // API group: Transactions ResolveTransaction(context.Context, *vtgate.ResolveTransactionRequest) (*vtgate.ResolveTransactionResponse, error) - // MessageStream streams messages from a message table. - MessageStream(*vtgate.MessageStreamRequest, Vitess_MessageStreamServer) error - // MessageAck acks messages for a table. - MessageAck(context.Context, *vtgate.MessageAckRequest) (*query.MessageAckResponse, error) - // MessageAckKeyspaceIds routes Message Acks using the associated - // keyspace ids. - MessageAckKeyspaceIds(context.Context, *vtgate.MessageAckKeyspaceIdsRequest) (*query.MessageAckResponse, error) - // GetSrvKeyspace returns a SrvKeyspace object (as seen by this vtgate). - // This method is provided as a convenient way for clients to take a - // look at the sharding configuration for a Keyspace. Looking at the - // sharding information should not be used for routing queries (as the - // information may change, use the Execute calls for that). - // It is convenient for monitoring applications for instance, or if - // using custom sharding. - // API group: Topology - GetSrvKeyspace(context.Context, *vtgate.GetSrvKeyspaceRequest) (*vtgate.GetSrvKeyspaceResponse, error) // VStream streams binlog events from the requested sources. VStream(*vtgate.VStreamRequest, Vitess_VStreamServer) error } @@ -586,57 +220,9 @@ func (*UnimplementedVitessServer) ExecuteBatch(ctx context.Context, req *vtgate. func (*UnimplementedVitessServer) StreamExecute(req *vtgate.StreamExecuteRequest, srv Vitess_StreamExecuteServer) error { return status.Errorf(codes.Unimplemented, "method StreamExecute not implemented") } -func (*UnimplementedVitessServer) ExecuteShards(ctx context.Context, req *vtgate.ExecuteShardsRequest) (*vtgate.ExecuteShardsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ExecuteShards not implemented") -} -func (*UnimplementedVitessServer) ExecuteKeyspaceIds(ctx context.Context, req *vtgate.ExecuteKeyspaceIdsRequest) (*vtgate.ExecuteKeyspaceIdsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ExecuteKeyspaceIds not implemented") -} -func (*UnimplementedVitessServer) ExecuteKeyRanges(ctx context.Context, req *vtgate.ExecuteKeyRangesRequest) (*vtgate.ExecuteKeyRangesResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ExecuteKeyRanges not implemented") -} -func (*UnimplementedVitessServer) ExecuteEntityIds(ctx context.Context, req *vtgate.ExecuteEntityIdsRequest) (*vtgate.ExecuteEntityIdsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ExecuteEntityIds not implemented") -} -func (*UnimplementedVitessServer) ExecuteBatchShards(ctx context.Context, req *vtgate.ExecuteBatchShardsRequest) (*vtgate.ExecuteBatchShardsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ExecuteBatchShards not implemented") -} -func (*UnimplementedVitessServer) ExecuteBatchKeyspaceIds(ctx context.Context, req *vtgate.ExecuteBatchKeyspaceIdsRequest) (*vtgate.ExecuteBatchKeyspaceIdsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ExecuteBatchKeyspaceIds not implemented") -} -func (*UnimplementedVitessServer) StreamExecuteShards(req *vtgate.StreamExecuteShardsRequest, srv Vitess_StreamExecuteShardsServer) error { - return status.Errorf(codes.Unimplemented, "method StreamExecuteShards not implemented") -} -func (*UnimplementedVitessServer) StreamExecuteKeyspaceIds(req *vtgate.StreamExecuteKeyspaceIdsRequest, srv Vitess_StreamExecuteKeyspaceIdsServer) error { - return status.Errorf(codes.Unimplemented, "method StreamExecuteKeyspaceIds not implemented") -} -func (*UnimplementedVitessServer) StreamExecuteKeyRanges(req *vtgate.StreamExecuteKeyRangesRequest, srv Vitess_StreamExecuteKeyRangesServer) error { - return status.Errorf(codes.Unimplemented, "method StreamExecuteKeyRanges not implemented") -} -func (*UnimplementedVitessServer) Begin(ctx context.Context, req *vtgate.BeginRequest) (*vtgate.BeginResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Begin not implemented") -} -func (*UnimplementedVitessServer) Commit(ctx context.Context, req *vtgate.CommitRequest) (*vtgate.CommitResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Commit not implemented") -} -func (*UnimplementedVitessServer) Rollback(ctx context.Context, req *vtgate.RollbackRequest) (*vtgate.RollbackResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Rollback not implemented") -} func (*UnimplementedVitessServer) ResolveTransaction(ctx context.Context, req *vtgate.ResolveTransactionRequest) (*vtgate.ResolveTransactionResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ResolveTransaction not implemented") } -func (*UnimplementedVitessServer) MessageStream(req *vtgate.MessageStreamRequest, srv Vitess_MessageStreamServer) error { - return status.Errorf(codes.Unimplemented, "method MessageStream not implemented") -} -func (*UnimplementedVitessServer) MessageAck(ctx context.Context, req *vtgate.MessageAckRequest) (*query.MessageAckResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method MessageAck not implemented") -} -func (*UnimplementedVitessServer) MessageAckKeyspaceIds(ctx context.Context, req *vtgate.MessageAckKeyspaceIdsRequest) (*query.MessageAckResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method MessageAckKeyspaceIds not implemented") -} -func (*UnimplementedVitessServer) GetSrvKeyspace(ctx context.Context, req *vtgate.GetSrvKeyspaceRequest) (*vtgate.GetSrvKeyspaceResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetSrvKeyspace not implemented") -} func (*UnimplementedVitessServer) VStream(req *vtgate.VStreamRequest, srv Vitess_VStreamServer) error { return status.Errorf(codes.Unimplemented, "method VStream not implemented") } @@ -702,231 +288,6 @@ func (x *vitessStreamExecuteServer) Send(m *vtgate.StreamExecuteResponse) error return x.ServerStream.SendMsg(m) } -func _Vitess_ExecuteShards_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(vtgate.ExecuteShardsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VitessServer).ExecuteShards(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/vtgateservice.Vitess/ExecuteShards", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VitessServer).ExecuteShards(ctx, req.(*vtgate.ExecuteShardsRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Vitess_ExecuteKeyspaceIds_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(vtgate.ExecuteKeyspaceIdsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VitessServer).ExecuteKeyspaceIds(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/vtgateservice.Vitess/ExecuteKeyspaceIds", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VitessServer).ExecuteKeyspaceIds(ctx, req.(*vtgate.ExecuteKeyspaceIdsRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Vitess_ExecuteKeyRanges_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(vtgate.ExecuteKeyRangesRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VitessServer).ExecuteKeyRanges(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/vtgateservice.Vitess/ExecuteKeyRanges", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VitessServer).ExecuteKeyRanges(ctx, req.(*vtgate.ExecuteKeyRangesRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Vitess_ExecuteEntityIds_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(vtgate.ExecuteEntityIdsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VitessServer).ExecuteEntityIds(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/vtgateservice.Vitess/ExecuteEntityIds", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VitessServer).ExecuteEntityIds(ctx, req.(*vtgate.ExecuteEntityIdsRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Vitess_ExecuteBatchShards_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(vtgate.ExecuteBatchShardsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VitessServer).ExecuteBatchShards(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/vtgateservice.Vitess/ExecuteBatchShards", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VitessServer).ExecuteBatchShards(ctx, req.(*vtgate.ExecuteBatchShardsRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Vitess_ExecuteBatchKeyspaceIds_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(vtgate.ExecuteBatchKeyspaceIdsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VitessServer).ExecuteBatchKeyspaceIds(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/vtgateservice.Vitess/ExecuteBatchKeyspaceIds", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VitessServer).ExecuteBatchKeyspaceIds(ctx, req.(*vtgate.ExecuteBatchKeyspaceIdsRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Vitess_StreamExecuteShards_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(vtgate.StreamExecuteShardsRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(VitessServer).StreamExecuteShards(m, &vitessStreamExecuteShardsServer{stream}) -} - -type Vitess_StreamExecuteShardsServer interface { - Send(*vtgate.StreamExecuteShardsResponse) error - grpc.ServerStream -} - -type vitessStreamExecuteShardsServer struct { - grpc.ServerStream -} - -func (x *vitessStreamExecuteShardsServer) Send(m *vtgate.StreamExecuteShardsResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _Vitess_StreamExecuteKeyspaceIds_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(vtgate.StreamExecuteKeyspaceIdsRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(VitessServer).StreamExecuteKeyspaceIds(m, &vitessStreamExecuteKeyspaceIdsServer{stream}) -} - -type Vitess_StreamExecuteKeyspaceIdsServer interface { - Send(*vtgate.StreamExecuteKeyspaceIdsResponse) error - grpc.ServerStream -} - -type vitessStreamExecuteKeyspaceIdsServer struct { - grpc.ServerStream -} - -func (x *vitessStreamExecuteKeyspaceIdsServer) Send(m *vtgate.StreamExecuteKeyspaceIdsResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _Vitess_StreamExecuteKeyRanges_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(vtgate.StreamExecuteKeyRangesRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(VitessServer).StreamExecuteKeyRanges(m, &vitessStreamExecuteKeyRangesServer{stream}) -} - -type Vitess_StreamExecuteKeyRangesServer interface { - Send(*vtgate.StreamExecuteKeyRangesResponse) error - grpc.ServerStream -} - -type vitessStreamExecuteKeyRangesServer struct { - grpc.ServerStream -} - -func (x *vitessStreamExecuteKeyRangesServer) Send(m *vtgate.StreamExecuteKeyRangesResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _Vitess_Begin_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(vtgate.BeginRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VitessServer).Begin(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/vtgateservice.Vitess/Begin", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VitessServer).Begin(ctx, req.(*vtgate.BeginRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Vitess_Commit_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(vtgate.CommitRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VitessServer).Commit(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/vtgateservice.Vitess/Commit", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VitessServer).Commit(ctx, req.(*vtgate.CommitRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Vitess_Rollback_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(vtgate.RollbackRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VitessServer).Rollback(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/vtgateservice.Vitess/Rollback", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VitessServer).Rollback(ctx, req.(*vtgate.RollbackRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _Vitess_ResolveTransaction_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(vtgate.ResolveTransactionRequest) if err := dec(in); err != nil { @@ -945,81 +306,6 @@ func _Vitess_ResolveTransaction_Handler(srv interface{}, ctx context.Context, de return interceptor(ctx, in, info, handler) } -func _Vitess_MessageStream_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(vtgate.MessageStreamRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(VitessServer).MessageStream(m, &vitessMessageStreamServer{stream}) -} - -type Vitess_MessageStreamServer interface { - Send(*query.MessageStreamResponse) error - grpc.ServerStream -} - -type vitessMessageStreamServer struct { - grpc.ServerStream -} - -func (x *vitessMessageStreamServer) Send(m *query.MessageStreamResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _Vitess_MessageAck_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(vtgate.MessageAckRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VitessServer).MessageAck(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/vtgateservice.Vitess/MessageAck", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VitessServer).MessageAck(ctx, req.(*vtgate.MessageAckRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Vitess_MessageAckKeyspaceIds_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(vtgate.MessageAckKeyspaceIdsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VitessServer).MessageAckKeyspaceIds(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/vtgateservice.Vitess/MessageAckKeyspaceIds", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VitessServer).MessageAckKeyspaceIds(ctx, req.(*vtgate.MessageAckKeyspaceIdsRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Vitess_GetSrvKeyspace_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(vtgate.GetSrvKeyspaceRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VitessServer).GetSrvKeyspace(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/vtgateservice.Vitess/GetSrvKeyspace", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VitessServer).GetSrvKeyspace(ctx, req.(*vtgate.GetSrvKeyspaceRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _Vitess_VStream_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(vtgate.VStreamRequest) if err := stream.RecvMsg(m); err != nil { @@ -1053,58 +339,10 @@ var _Vitess_serviceDesc = grpc.ServiceDesc{ MethodName: "ExecuteBatch", Handler: _Vitess_ExecuteBatch_Handler, }, - { - MethodName: "ExecuteShards", - Handler: _Vitess_ExecuteShards_Handler, - }, - { - MethodName: "ExecuteKeyspaceIds", - Handler: _Vitess_ExecuteKeyspaceIds_Handler, - }, - { - MethodName: "ExecuteKeyRanges", - Handler: _Vitess_ExecuteKeyRanges_Handler, - }, - { - MethodName: "ExecuteEntityIds", - Handler: _Vitess_ExecuteEntityIds_Handler, - }, - { - MethodName: "ExecuteBatchShards", - Handler: _Vitess_ExecuteBatchShards_Handler, - }, - { - MethodName: "ExecuteBatchKeyspaceIds", - Handler: _Vitess_ExecuteBatchKeyspaceIds_Handler, - }, - { - MethodName: "Begin", - Handler: _Vitess_Begin_Handler, - }, - { - MethodName: "Commit", - Handler: _Vitess_Commit_Handler, - }, - { - MethodName: "Rollback", - Handler: _Vitess_Rollback_Handler, - }, { MethodName: "ResolveTransaction", Handler: _Vitess_ResolveTransaction_Handler, }, - { - MethodName: "MessageAck", - Handler: _Vitess_MessageAck_Handler, - }, - { - MethodName: "MessageAckKeyspaceIds", - Handler: _Vitess_MessageAckKeyspaceIds_Handler, - }, - { - MethodName: "GetSrvKeyspace", - Handler: _Vitess_GetSrvKeyspace_Handler, - }, }, Streams: []grpc.StreamDesc{ { @@ -1112,26 +350,6 @@ var _Vitess_serviceDesc = grpc.ServiceDesc{ Handler: _Vitess_StreamExecute_Handler, ServerStreams: true, }, - { - StreamName: "StreamExecuteShards", - Handler: _Vitess_StreamExecuteShards_Handler, - ServerStreams: true, - }, - { - StreamName: "StreamExecuteKeyspaceIds", - Handler: _Vitess_StreamExecuteKeyspaceIds_Handler, - ServerStreams: true, - }, - { - StreamName: "StreamExecuteKeyRanges", - Handler: _Vitess_StreamExecuteKeyRanges_Handler, - ServerStreams: true, - }, - { - StreamName: "MessageStream", - Handler: _Vitess_MessageStream_Handler, - ServerStreams: true, - }, { StreamName: "VStream", Handler: _Vitess_VStream_Handler, diff --git a/go/vt/vitessdriver/fakeserver_test.go b/go/vt/vitessdriver/fakeserver_test.go index 73c0731caa2..13af932516d 100644 --- a/go/vt/vitessdriver/fakeserver_test.go +++ b/go/vt/vitessdriver/fakeserver_test.go @@ -131,72 +131,6 @@ func (f *fakeVTGateService) StreamExecute(ctx context.Context, session *vtgatepb return nil } -// ExecuteShards is part of the VTGateService interface -func (f *fakeVTGateService) ExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - return nil, nil -} - -// ExecuteKeyspaceIds is part of the VTGateService interface -func (f *fakeVTGateService) ExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - return nil, nil -} - -// ExecuteKeyRanges is part of the VTGateService interface -func (f *fakeVTGateService) ExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - return nil, nil -} - -// ExecuteEntityIds is part of the VTGateService interface -func (f *fakeVTGateService) ExecuteEntityIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, entityColumnName string, entityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - return nil, nil -} - -// ExecuteBatchShard is part of the VTGateService interface -func (f *fakeVTGateService) ExecuteBatchShards(ctx context.Context, queries []*vtgatepb.BoundShardQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - return nil, nil -} - -// ExecuteBatchKeyspaceIds is part of the VTGateService interface -func (f *fakeVTGateService) ExecuteBatchKeyspaceIds(ctx context.Context, queries []*vtgatepb.BoundKeyspaceIdQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - return nil, nil -} - -// StreamExecuteShards is part of the VTGateService interface -func (f *fakeVTGateService) StreamExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - return nil -} - -// StreamExecuteKeyspaceIds is part of the VTGateService interface -func (f *fakeVTGateService) StreamExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - return nil -} - -// StreamExecuteKeyRanges is part of the VTGateService interface -func (f *fakeVTGateService) StreamExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - return nil -} - -// Begin is part of the VTGateService interface -func (f *fakeVTGateService) Begin(ctx context.Context, singledb bool) (*vtgatepb.Session, error) { - return session1, nil -} - -// Commit is part of the VTGateService interface -func (f *fakeVTGateService) Commit(ctx context.Context, twopc bool, session *vtgatepb.Session) error { - if !proto.Equal(session, session2) { - return errors.New("commit: session mismatch") - } - return nil -} - -// Rollback is part of the VTGateService interface -func (f *fakeVTGateService) Rollback(ctx context.Context, session *vtgatepb.Session) error { - if !proto.Equal(session, session2) { - return errors.New("rollback: session mismatch") - } - return nil -} - // ResolveTransaction is part of the VTGateService interface func (f *fakeVTGateService) ResolveTransaction(ctx context.Context, dtid string) error { if dtid != dtid2 { @@ -205,23 +139,6 @@ func (f *fakeVTGateService) ResolveTransaction(ctx context.Context, dtid string) return nil } -func (f *fakeVTGateService) MessageStream(ctx context.Context, keyspace string, shard string, keyRange *topodatapb.KeyRange, name string, callback func(*sqltypes.Result) error) error { - return nil -} - -func (f *fakeVTGateService) MessageAck(ctx context.Context, keyspace string, name string, ids []*querypb.Value) (int64, error) { - return 0, nil -} - -func (f *fakeVTGateService) MessageAckKeyspaceIds(ctx context.Context, keyspace string, name string, idKeyspaceIDs []*vtgatepb.IdKeyspaceId) (int64, error) { - return 0, nil -} - -// GetSrvKeyspace is part of the VTGateService interface -func (f *fakeVTGateService) GetSrvKeyspace(ctx context.Context, keyspace string) (*topodatapb.SrvKeyspace, error) { - return &topodatapb.SrvKeyspace{}, nil -} - func (f *fakeVTGateService) VStream(ctx context.Context, tabletType topodatapb.TabletType, vgtid *binlogdatapb.VGtid, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { return nil } diff --git a/go/vt/vtctl/query.go b/go/vt/vtctl/query.go index b5b2846c73b..e6cd28b7c04 100644 --- a/go/vt/vtctl/query.go +++ b/go/vt/vtctl/query.go @@ -17,14 +17,12 @@ limitations under the License. package vtctl import ( - "encoding/hex" "encoding/json" "errors" "flag" "fmt" "io" "strconv" - "strings" "github.com/golang/protobuf/proto" "github.com/olekukonko/tablewriter" @@ -39,7 +37,6 @@ import ( "vitess.io/vitess/go/vt/wrangler" querypb "vitess.io/vitess/go/vt/proto/query" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) // This file contains the query command group for vtctl. @@ -59,16 +56,6 @@ func init() { commandVtGateExecute, "-server [-bind_variables ] [-keyspace ] [-tablet_type ] [-options ] [-json] ", "Executes the given SQL query with the provided bound variables against the vtgate server."}) - addCommand(queriesGroupName, command{ - "VtGateExecuteShards", - commandVtGateExecuteShards, - "-server -keyspace -shards ,,... [-bind_variables ] [-tablet_type ] [-options ] [-json] ", - "Executes the given SQL query with the provided bound variables against the vtgate server. It is routed to the provided shards."}) - addCommand(queriesGroupName, command{ - "VtGateExecuteKeyspaceIds", - commandVtGateExecuteKeyspaceIds, - "-server -keyspace -keyspace_ids ,,... [-bind_variables ] [-tablet_type ] [-options ] [-json] ", - "Executes the given SQL query with the provided bound variables against the vtgate server. It is routed to the shards that contain the provided keyspace ids."}) // VtTablet commands addCommand(queriesGroupName, command{ @@ -194,121 +181,6 @@ func commandVtGateExecute(ctx context.Context, wr *wrangler.Wrangler, subFlags * return nil } -func commandVtGateExecuteShards(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { - if !*enableQueries { - return fmt.Errorf("query commands are disabled (set the -enable_queries flag to enable)") - } - - server := subFlags.String("server", "", "VtGate server to connect to") - bindVariables := newBindvars(subFlags) - tabletType := subFlags.String("tablet_type", "master", "tablet type to query") - keyspace := subFlags.String("keyspace", "", "keyspace to send query to") - shardsStr := subFlags.String("shards", "", "comma-separated list of shards to send query to") - options := subFlags.String("options", "", "execute options values as a text encoded proto of the ExecuteOptions structure") - json := subFlags.Bool("json", false, "Output JSON instead of human-readable table") - - if err := subFlags.Parse(args); err != nil { - return err - } - if subFlags.NArg() != 1 { - return fmt.Errorf("the argument is required for the VtGateExecuteShards command") - } - t, err := parseTabletType(*tabletType, []topodatapb.TabletType{topodatapb.TabletType_MASTER, topodatapb.TabletType_REPLICA, topodatapb.TabletType_RDONLY}) - if err != nil { - return err - } - var shards []string - if *shardsStr != "" { - shards = strings.Split(*shardsStr, ",") - } - executeOptions, err := parseExecuteOptions(*options) - if err != nil { - return err - } - - vtgateConn, err := vtgateconn.Dial(ctx, *server) - if err != nil { - return fmt.Errorf("error connecting to vtgate '%v': %v", *server, err) - } - defer vtgateConn.Close() - - bindVars, err := sqltypes.BuildBindVariables(*bindVariables) - if err != nil { - return fmt.Errorf("BuildBindVariables failed: %v", err) - } - - qr, err := vtgateConn.ExecuteShards(ctx, subFlags.Arg(0), *keyspace, shards, bindVars, t, executeOptions) - if err != nil { - return fmt.Errorf("ExecuteShards failed: %v", err) - } - if *json { - return printJSON(wr.Logger(), qr) - } - printQueryResult(loggerWriter{wr.Logger()}, qr) - return nil -} - -func commandVtGateExecuteKeyspaceIds(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { - if !*enableQueries { - return fmt.Errorf("query commands are disabled (set the -enable_queries flag to enable)") - } - - server := subFlags.String("server", "", "VtGate server to connect to") - bindVariables := newBindvars(subFlags) - tabletType := subFlags.String("tablet_type", "master", "tablet type to query") - keyspace := subFlags.String("keyspace", "", "keyspace to send query to") - keyspaceIDsStr := subFlags.String("keyspace_ids", "", "comma-separated list of keyspace ids (in hex) that will map into shards to send query to") - options := subFlags.String("options", "", "execute options values as a text encoded proto of the ExecuteOptions structure") - json := subFlags.Bool("json", false, "Output JSON instead of human-readable table") - - if err := subFlags.Parse(args); err != nil { - return err - } - if subFlags.NArg() != 1 { - return fmt.Errorf("the argument is required for the VtGateExecuteKeyspaceIds command") - } - t, err := parseTabletType(*tabletType, []topodatapb.TabletType{topodatapb.TabletType_MASTER, topodatapb.TabletType_REPLICA, topodatapb.TabletType_RDONLY}) - if err != nil { - return err - } - var keyspaceIDs [][]byte - if *keyspaceIDsStr != "" { - keyspaceIDHexs := strings.Split(*keyspaceIDsStr, ",") - keyspaceIDs = make([][]byte, len(keyspaceIDHexs)) - for i, keyspaceIDHex := range keyspaceIDHexs { - keyspaceIDs[i], err = hex.DecodeString(keyspaceIDHex) - if err != nil { - return fmt.Errorf("cannot hex-decode value %v '%v': %v", i, keyspaceIDHex, err) - } - } - } - executeOptions, err := parseExecuteOptions(*options) - if err != nil { - return err - } - - vtgateConn, err := vtgateconn.Dial(ctx, *server) - if err != nil { - return fmt.Errorf("error connecting to vtgate '%v': %v", *server, err) - } - defer vtgateConn.Close() - - bindVars, err := sqltypes.BuildBindVariables(*bindVariables) - if err != nil { - return fmt.Errorf("BuildBindVariables failed: %v", err) - } - - qr, err := vtgateConn.ExecuteKeyspaceIds(ctx, subFlags.Arg(0), *keyspace, keyspaceIDs, bindVars, t, executeOptions) - if err != nil { - return fmt.Errorf("ExecuteKeyspaceIds failed: %v", err) - } - if *json { - return printJSON(wr.Logger(), qr) - } - printQueryResult(loggerWriter{wr.Logger()}, qr) - return nil -} - func commandVtTabletExecute(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { if !*enableQueries { return fmt.Errorf("query commands are disabled (set the -enable_queries flag to enable)") diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 3268d9e95d7..06cf73aa26c 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -2555,7 +2555,7 @@ func commandApplyVSchema(ctx context.Context, wr *wrangler.Wrangler, subFlags *f if _, err := wr.TopoServer().GetKeyspace(ctx, keyspace); err != nil { if strings.Contains(err.Error(), "node doesn't exist") { - return fmt.Errorf("keyspace(%s) doesn't exist, check if the keyspace is initialized.\n", keyspace) + return fmt.Errorf("keyspace(%s) doesn't exist, check if the keyspace is initialized", keyspace) } return err } diff --git a/go/vt/vtgate/fakerpcvtgateconn/conn.go b/go/vt/vtgate/fakerpcvtgateconn/conn.go index 0ca9b5f42dc..7492520432a 100644 --- a/go/vt/vtgate/fakerpcvtgateconn/conn.go +++ b/go/vt/vtgate/fakerpcvtgateconn/conn.go @@ -20,13 +20,10 @@ limitations under the License. package fakerpcvtgateconn import ( - "errors" "fmt" "io" "math/rand" "reflect" - "sort" - "strings" "golang.org/x/net/context" "vitess.io/vitess/go/sqltypes" @@ -45,22 +42,10 @@ type queryExecute struct { Session *vtgatepb.Session } -// queryExecuteShards contains all the fields we use to test ExecuteShards -type queryExecuteShards struct { - SQL string - BindVariables map[string]*querypb.BindVariable - Keyspace string - Shards []string - TabletType topodatapb.TabletType - Session *vtgatepb.Session - NotInTransaction bool -} - type queryResponse struct { - execQuery *queryExecute - shardQuery *queryExecuteShards - reply *sqltypes.Result - err error + execQuery *queryExecute + reply *sqltypes.Result + err error } // FakeVTGateConn provides a fake implementation of vtgateconn.Impl @@ -98,30 +83,6 @@ func (conn *FakeVTGateConn) AddQuery( } } -// AddShardQuery adds a shard query and expected result. -func (conn *FakeVTGateConn) AddShardQuery( - sql string, - bindVariables map[string]*querypb.BindVariable, - keyspace string, - shards []string, - tabletType topodatapb.TabletType, - session *vtgatepb.Session, - notInTransaction bool, - expectedResult *sqltypes.Result) { - conn.execMap[getShardQueryKey(sql, shards)] = &queryResponse{ - shardQuery: &queryExecuteShards{ - SQL: sql, - BindVariables: bindVariables, - Keyspace: keyspace, - Shards: shards, - TabletType: tabletType, - Session: session, - NotInTransaction: notInTransaction, - }, - reply: expectedResult, - } -} - // Execute please see vtgateconn.Impl.Execute func (conn *FakeVTGateConn) Execute(ctx context.Context, session *vtgatepb.Session, sql string, bindVars map[string]*querypb.BindVariable) (*vtgatepb.Session, *sqltypes.Result, error) { response, ok := conn.execMap[sql] @@ -184,60 +145,6 @@ func (conn *FakeVTGateConn) StreamExecute(ctx context.Context, session *vtgatepb return &streamExecuteAdapter{resultChan}, nil } -// ExecuteShards please see vtgateconn.Impl.ExecuteShard -func (conn *FakeVTGateConn) ExecuteShards(ctx context.Context, sql string, keyspace string, shards []string, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, *sqltypes.Result, error) { - var s *vtgatepb.Session - if session != nil { - s = session - } - response, ok := conn.execMap[getShardQueryKey(sql, shards)] - if !ok { - return nil, nil, fmt.Errorf("no match for: %s", sql) - } - query := &queryExecuteShards{ - SQL: sql, - BindVariables: bindVars, - TabletType: tabletType, - Keyspace: keyspace, - Shards: shards, - Session: s, - } - if !reflect.DeepEqual(query, response.shardQuery) { - return nil, nil, fmt.Errorf( - "ExecuteShards: %+v, want %+v", query, response.shardQuery) - } - reply := *response.reply - if s != nil { - s = newSession(true, keyspace, shards, tabletType) - } - return s, &reply, nil -} - -// ExecuteKeyspaceIds please see vtgateconn.Impl.ExecuteKeyspaceIds -func (conn *FakeVTGateConn) ExecuteKeyspaceIds(ctx context.Context, query string, keyspace string, keyspaceIds [][]byte, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, *sqltypes.Result, error) { - panic("not implemented") -} - -// ExecuteKeyRanges please see vtgateconn.Impl.ExecuteKeyRanges -func (conn *FakeVTGateConn) ExecuteKeyRanges(ctx context.Context, query string, keyspace string, keyRanges []*topodatapb.KeyRange, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, *sqltypes.Result, error) { - panic("not implemented") -} - -// ExecuteEntityIds please see vtgateconn.Impl.ExecuteEntityIds -func (conn *FakeVTGateConn) ExecuteEntityIds(ctx context.Context, query string, keyspace string, entityColumnName string, entityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, *sqltypes.Result, error) { - panic("not implemented") -} - -// ExecuteBatchShards please see vtgateconn.Impl.ExecuteBatchShards -func (conn *FakeVTGateConn) ExecuteBatchShards(ctx context.Context, queries []*vtgatepb.BoundShardQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, []sqltypes.Result, error) { - panic("not implemented") -} - -// ExecuteBatchKeyspaceIds please see vtgateconn.Impl.ExecuteBatchKeyspaceIds -func (conn *FakeVTGateConn) ExecuteBatchKeyspaceIds(ctx context.Context, queries []*vtgatepb.BoundKeyspaceIdQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, []sqltypes.Result, error) { - panic("not implemented") -} - type streamExecuteAdapter struct { c chan *sqltypes.Result } @@ -250,67 +157,11 @@ func (a *streamExecuteAdapter) Recv() (*sqltypes.Result, error) { return r, nil } -// StreamExecuteShards please see vtgateconn.Impl.StreamExecuteShards -func (conn *FakeVTGateConn) StreamExecuteShards(ctx context.Context, query string, keyspace string, shards []string, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (sqltypes.ResultStream, error) { - panic("not implemented") -} - -// StreamExecuteKeyRanges please see vtgateconn.Impl.StreamExecuteKeyRanges -func (conn *FakeVTGateConn) StreamExecuteKeyRanges(ctx context.Context, query string, keyspace string, keyRanges []*topodatapb.KeyRange, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (sqltypes.ResultStream, error) { - panic("not implemented") -} - -// StreamExecuteKeyspaceIds please see vtgateconn.Impl.StreamExecuteKeyspaceIds -func (conn *FakeVTGateConn) StreamExecuteKeyspaceIds(ctx context.Context, query string, keyspace string, keyspaceIds [][]byte, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (sqltypes.ResultStream, error) { - panic("not implemented") -} - -// Begin please see vtgateconn.Impl.Begin -func (conn *FakeVTGateConn) Begin(ctx context.Context, singledb bool) (*vtgatepb.Session, error) { - return &vtgatepb.Session{ - InTransaction: true, - SingleDb: singledb, - }, nil -} - -// Commit please see vtgateconn.Impl.Commit -func (conn *FakeVTGateConn) Commit(ctx context.Context, session *vtgatepb.Session, twopc bool) error { - if session == nil { - return errors.New("commit: not in transaction") - } - return nil -} - -// Rollback please see vtgateconn.Impl.Rollback -func (conn *FakeVTGateConn) Rollback(ctx context.Context, session *vtgatepb.Session) error { - return nil -} - // ResolveTransaction please see vtgateconn.Impl.ResolveTransaction func (conn *FakeVTGateConn) ResolveTransaction(ctx context.Context, dtid string) error { return nil } -// MessageStream is part of the vtgate service API. -func (conn *FakeVTGateConn) MessageStream(ctx context.Context, keyspace string, shard string, keyRange *topodatapb.KeyRange, name string, callback func(*sqltypes.Result) error) error { - panic("not implemented") -} - -// MessageAck is part of the vtgate service API. -func (conn *FakeVTGateConn) MessageAck(ctx context.Context, keyspace string, name string, ids []*querypb.Value) (int64, error) { - panic("not implemented") -} - -// MessageAckKeyspaceIds is part of the vtgate service API. -func (conn *FakeVTGateConn) MessageAckKeyspaceIds(ctx context.Context, keyspace string, name string, idKeyspaceIDs []*vtgatepb.IdKeyspaceId) (int64, error) { - panic("not implemented") -} - -// GetSrvKeyspace please see vtgateconn.Impl.GetSrvKeyspace -func (conn *FakeVTGateConn) GetSrvKeyspace(ctx context.Context, keyspace string) (*topodatapb.SrvKeyspace, error) { - return nil, fmt.Errorf("NYI") -} - // VStream streams binlog events. func (conn *FakeVTGateConn) VStream(ctx context.Context, tabletType topodatapb.TabletType, vgtid *binlogdatapb.VGtid, filter *binlogdatapb.Filter) (vtgateconn.VStreamReader, error) { return nil, fmt.Errorf("NYI") @@ -320,11 +171,6 @@ func (conn *FakeVTGateConn) VStream(ctx context.Context, tabletType topodatapb.T func (conn *FakeVTGateConn) Close() { } -func getShardQueryKey(sql string, shards []string) string { - sort.Strings(shards) - return fmt.Sprintf("%s-%s", sql, strings.Join(shards, ":")) -} - func newSession( inTransaction bool, keyspace string, diff --git a/go/vt/vtgate/grpcvtgateconn/conn.go b/go/vt/vtgate/grpcvtgateconn/conn.go index 8d4293d03f0..e5f8abc54cc 100644 --- a/go/vt/vtgate/grpcvtgateconn/conn.go +++ b/go/vt/vtgate/grpcvtgateconn/conn.go @@ -19,7 +19,6 @@ package grpcvtgateconn import ( "flag" - "io" "google.golang.org/grpc" @@ -123,165 +122,6 @@ func (conn *vtgateConn) ExecuteBatch(ctx context.Context, session *vtgatepb.Sess return response.Session, sqltypes.Proto3ToQueryReponses(response.Results), nil } -func (conn *vtgateConn) StreamExecute(ctx context.Context, session *vtgatepb.Session, query string, bindVars map[string]*querypb.BindVariable) (sqltypes.ResultStream, error) { - req := &vtgatepb.StreamExecuteRequest{ - CallerId: callerid.EffectiveCallerIDFromContext(ctx), - Query: &querypb.BoundQuery{ - Sql: query, - BindVariables: bindVars, - }, - Session: session, - } - stream, err := conn.c.StreamExecute(ctx, req) - if err != nil { - return nil, vterrors.FromGRPC(err) - } - return &streamExecuteAdapter{ - recv: func() (*querypb.QueryResult, error) { - ser, err := stream.Recv() - if err != nil { - return nil, err - } - return ser.Result, nil - }, - }, nil -} - -func (conn *vtgateConn) ExecuteShards(ctx context.Context, query string, keyspace string, shards []string, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, *sqltypes.Result, error) { - request := &vtgatepb.ExecuteShardsRequest{ - CallerId: callerid.EffectiveCallerIDFromContext(ctx), - Query: &querypb.BoundQuery{ - Sql: query, - BindVariables: bindVars, - }, - Session: session, - Keyspace: keyspace, - Shards: shards, - TabletType: tabletType, - Options: options, - } - response, err := conn.c.ExecuteShards(ctx, request) - if err != nil { - return session, nil, vterrors.FromGRPC(err) - } - if response.Error != nil { - return response.Session, nil, vterrors.FromVTRPC(response.Error) - } - return response.Session, sqltypes.Proto3ToResult(response.Result), nil -} - -func (conn *vtgateConn) ExecuteKeyspaceIds(ctx context.Context, query string, keyspace string, keyspaceIds [][]byte, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, *sqltypes.Result, error) { - request := &vtgatepb.ExecuteKeyspaceIdsRequest{ - CallerId: callerid.EffectiveCallerIDFromContext(ctx), - Query: &querypb.BoundQuery{ - Sql: query, - BindVariables: bindVars, - }, - Session: session, - Keyspace: keyspace, - KeyspaceIds: keyspaceIds, - TabletType: tabletType, - Options: options, - } - response, err := conn.c.ExecuteKeyspaceIds(ctx, request) - if err != nil { - return session, nil, vterrors.FromGRPC(err) - } - if response.Error != nil { - return response.Session, nil, vterrors.FromVTRPC(response.Error) - } - return response.Session, sqltypes.Proto3ToResult(response.Result), nil -} - -func (conn *vtgateConn) ExecuteKeyRanges(ctx context.Context, query string, keyspace string, keyRanges []*topodatapb.KeyRange, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, *sqltypes.Result, error) { - request := &vtgatepb.ExecuteKeyRangesRequest{ - CallerId: callerid.EffectiveCallerIDFromContext(ctx), - Query: &querypb.BoundQuery{ - Sql: query, - BindVariables: bindVars, - }, - Session: session, - Keyspace: keyspace, - KeyRanges: keyRanges, - TabletType: tabletType, - Options: options, - } - response, err := conn.c.ExecuteKeyRanges(ctx, request) - if err != nil { - return session, nil, vterrors.FromGRPC(err) - } - if response.Error != nil { - return response.Session, nil, vterrors.FromVTRPC(response.Error) - } - return response.Session, sqltypes.Proto3ToResult(response.Result), nil -} - -func (conn *vtgateConn) ExecuteEntityIds(ctx context.Context, query string, keyspace string, entityColumnName string, entityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, *sqltypes.Result, error) { - request := &vtgatepb.ExecuteEntityIdsRequest{ - CallerId: callerid.EffectiveCallerIDFromContext(ctx), - Query: &querypb.BoundQuery{ - Sql: query, - BindVariables: bindVars, - }, - Session: session, - Keyspace: keyspace, - EntityColumnName: entityColumnName, - EntityKeyspaceIds: entityKeyspaceIDs, - TabletType: tabletType, - Options: options, - } - response, err := conn.c.ExecuteEntityIds(ctx, request) - if err != nil { - return session, nil, vterrors.FromGRPC(err) - } - if response.Error != nil { - return response.Session, nil, vterrors.FromVTRPC(response.Error) - } - return response.Session, sqltypes.Proto3ToResult(response.Result), nil -} - -func (conn *vtgateConn) ExecuteBatchShards(ctx context.Context, queries []*vtgatepb.BoundShardQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, []sqltypes.Result, error) { - request := &vtgatepb.ExecuteBatchShardsRequest{ - CallerId: callerid.EffectiveCallerIDFromContext(ctx), - Session: session, - Queries: queries, - TabletType: tabletType, - AsTransaction: asTransaction, - Options: options, - } - response, err := conn.c.ExecuteBatchShards(ctx, request) - if err != nil { - return session, nil, vterrors.FromGRPC(err) - } - if response.Error != nil { - return response.Session, nil, vterrors.FromVTRPC(response.Error) - } - return response.Session, sqltypes.Proto3ToResults(response.Results), nil -} - -func (conn *vtgateConn) ExecuteBatchKeyspaceIds(ctx context.Context, queries []*vtgatepb.BoundKeyspaceIdQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, []sqltypes.Result, error) { - var s *vtgatepb.Session - if session != nil { - s = session - } - request := &vtgatepb.ExecuteBatchKeyspaceIdsRequest{ - CallerId: callerid.EffectiveCallerIDFromContext(ctx), - Session: s, - Queries: queries, - TabletType: tabletType, - AsTransaction: asTransaction, - Options: options, - } - response, err := conn.c.ExecuteBatchKeyspaceIds(ctx, request) - if err != nil { - return session, nil, vterrors.FromGRPC(err) - } - if response.Error != nil { - return response.Session, nil, vterrors.FromVTRPC(response.Error) - } - return response.Session, sqltypes.Proto3ToResults(response.Results), nil -} - type streamExecuteAdapter struct { recv func() (*querypb.QueryResult, error) fields []*querypb.Field @@ -298,73 +138,16 @@ func (a *streamExecuteAdapter) Recv() (*sqltypes.Result, error) { return sqltypes.CustomProto3ToResult(a.fields, qr), nil } -func (conn *vtgateConn) StreamExecuteShards(ctx context.Context, query string, keyspace string, shards []string, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (sqltypes.ResultStream, error) { - req := &vtgatepb.StreamExecuteShardsRequest{ - CallerId: callerid.EffectiveCallerIDFromContext(ctx), - Query: &querypb.BoundQuery{ - Sql: query, - BindVariables: bindVars, - }, - Keyspace: keyspace, - Shards: shards, - TabletType: tabletType, - Options: options, - } - stream, err := conn.c.StreamExecuteShards(ctx, req) - if err != nil { - return nil, vterrors.FromGRPC(err) - } - return &streamExecuteAdapter{ - recv: func() (*querypb.QueryResult, error) { - ser, err := stream.Recv() - if err != nil { - return nil, err - } - return ser.Result, nil - }, - }, nil -} - -func (conn *vtgateConn) StreamExecuteKeyRanges(ctx context.Context, query string, keyspace string, keyRanges []*topodatapb.KeyRange, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (sqltypes.ResultStream, error) { - req := &vtgatepb.StreamExecuteKeyRangesRequest{ - CallerId: callerid.EffectiveCallerIDFromContext(ctx), - Query: &querypb.BoundQuery{ - Sql: query, - BindVariables: bindVars, - }, - Keyspace: keyspace, - KeyRanges: keyRanges, - TabletType: tabletType, - Options: options, - } - stream, err := conn.c.StreamExecuteKeyRanges(ctx, req) - if err != nil { - return nil, vterrors.FromGRPC(err) - } - return &streamExecuteAdapter{ - recv: func() (*querypb.QueryResult, error) { - ser, err := stream.Recv() - if err != nil { - return nil, err - } - return ser.Result, nil - }, - }, nil -} - -func (conn *vtgateConn) StreamExecuteKeyspaceIds(ctx context.Context, query string, keyspace string, keyspaceIds [][]byte, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (sqltypes.ResultStream, error) { - req := &vtgatepb.StreamExecuteKeyspaceIdsRequest{ +func (conn *vtgateConn) StreamExecute(ctx context.Context, session *vtgatepb.Session, query string, bindVars map[string]*querypb.BindVariable) (sqltypes.ResultStream, error) { + req := &vtgatepb.StreamExecuteRequest{ CallerId: callerid.EffectiveCallerIDFromContext(ctx), Query: &querypb.BoundQuery{ Sql: query, BindVariables: bindVars, }, - Keyspace: keyspace, - KeyspaceIds: keyspaceIds, - TabletType: tabletType, - Options: options, + Session: session, } - stream, err := conn.c.StreamExecuteKeyspaceIds(ctx, req) + stream, err := conn.c.StreamExecute(ctx, req) if err != nil { return nil, vterrors.FromGRPC(err) } @@ -379,37 +162,6 @@ func (conn *vtgateConn) StreamExecuteKeyspaceIds(ctx context.Context, query stri }, nil } -func (conn *vtgateConn) Begin(ctx context.Context, singledb bool) (*vtgatepb.Session, error) { - request := &vtgatepb.BeginRequest{ - CallerId: callerid.EffectiveCallerIDFromContext(ctx), - SingleDb: singledb, - } - response, err := conn.c.Begin(ctx, request) - if err != nil { - return nil, vterrors.FromGRPC(err) - } - return response.Session, nil -} - -func (conn *vtgateConn) Commit(ctx context.Context, session *vtgatepb.Session, twopc bool) error { - request := &vtgatepb.CommitRequest{ - CallerId: callerid.EffectiveCallerIDFromContext(ctx), - Session: session, - Atomic: twopc, - } - _, err := conn.c.Commit(ctx, request) - return vterrors.FromGRPC(err) -} - -func (conn *vtgateConn) Rollback(ctx context.Context, session *vtgatepb.Session) error { - request := &vtgatepb.RollbackRequest{ - CallerId: callerid.EffectiveCallerIDFromContext(ctx), - Session: session, - } - _, err := conn.c.Rollback(ctx, request) - return vterrors.FromGRPC(err) -} - func (conn *vtgateConn) ResolveTransaction(ctx context.Context, dtid string) error { request := &vtgatepb.ResolveTransactionRequest{ CallerId: callerid.EffectiveCallerIDFromContext(ctx), @@ -419,76 +171,6 @@ func (conn *vtgateConn) ResolveTransaction(ctx context.Context, dtid string) err return vterrors.FromGRPC(err) } -func (conn *vtgateConn) MessageStream(ctx context.Context, keyspace string, shard string, keyRange *topodatapb.KeyRange, name string, callback func(*sqltypes.Result) error) error { - request := &vtgatepb.MessageStreamRequest{ - CallerId: callerid.EffectiveCallerIDFromContext(ctx), - Keyspace: keyspace, - Shard: shard, - KeyRange: keyRange, - Name: name, - } - stream, err := conn.c.MessageStream(ctx, request) - if err != nil { - return vterrors.FromGRPC(err) - } - var fields []*querypb.Field - for { - r, err := stream.Recv() - if err != nil { - if err != io.EOF { - return vterrors.FromGRPC(err) - } - return nil - } - if fields == nil { - fields = r.Result.Fields - } - err = callback(sqltypes.CustomProto3ToResult(fields, r.Result)) - if err != nil { - return err - } - } -} - -func (conn *vtgateConn) MessageAck(ctx context.Context, keyspace string, name string, ids []*querypb.Value) (int64, error) { - request := &vtgatepb.MessageAckRequest{ - CallerId: callerid.EffectiveCallerIDFromContext(ctx), - Keyspace: keyspace, - Name: name, - Ids: ids, - } - r, err := conn.c.MessageAck(ctx, request) - if err != nil { - return 0, vterrors.FromGRPC(err) - } - return int64(r.Result.RowsAffected), nil -} - -func (conn *vtgateConn) MessageAckKeyspaceIds(ctx context.Context, keyspace string, name string, idKeyspaceIDs []*vtgatepb.IdKeyspaceId) (int64, error) { - request := &vtgatepb.MessageAckKeyspaceIdsRequest{ - CallerId: callerid.EffectiveCallerIDFromContext(ctx), - Keyspace: keyspace, - Name: name, - IdKeyspaceIds: idKeyspaceIDs, - } - r, err := conn.c.MessageAckKeyspaceIds(ctx, request) - if err != nil { - return 0, vterrors.FromGRPC(err) - } - return int64(r.Result.RowsAffected), nil -} - -func (conn *vtgateConn) GetSrvKeyspace(ctx context.Context, keyspace string) (*topodatapb.SrvKeyspace, error) { - request := &vtgatepb.GetSrvKeyspaceRequest{ - Keyspace: keyspace, - } - response, err := conn.c.GetSrvKeyspace(ctx, request) - if err != nil { - return nil, vterrors.FromGRPC(err) - } - return response.SrvKeyspace, nil -} - type vstreamAdapter struct { stream vtgateservicepb.Vitess_VStreamClient } diff --git a/go/vt/vtgate/grpcvtgateconn/conn_rpc_test.go b/go/vt/vtgate/grpcvtgateconn/conn_rpc_test.go index b9f0e3cb42b..81af4c55e24 100644 --- a/go/vt/vtgate/grpcvtgateconn/conn_rpc_test.go +++ b/go/vt/vtgate/grpcvtgateconn/conn_rpc_test.go @@ -145,7 +145,7 @@ func TestGRPCVTGateConnAuth(t *testing.T) { RegisterTestDialProtocol(client) conn, _ := vtgateconn.DialProtocol(context.Background(), "test", "") // run the test suite - _, err = conn.Begin(context.Background()) + _, err = conn.Session("", nil).Execute(context.Background(), "select * from t", nil) want := "rpc error: code = Unauthenticated desc = username and password must be provided" if err == nil || err.Error() != want { t.Errorf("expected auth failure:\n%v, want\n%s", err, want) diff --git a/go/vt/vtgate/grpcvtgateconn/suite_test.go b/go/vt/vtgate/grpcvtgateconn/suite_test.go index d762191ad20..c3f41ee8303 100644 --- a/go/vt/vtgate/grpcvtgateconn/suite_test.go +++ b/go/vt/vtgate/grpcvtgateconn/suite_test.go @@ -25,7 +25,6 @@ import ( "errors" "fmt" "io" - "reflect" "strings" "testing" @@ -52,10 +51,8 @@ type fakeVTGateService struct { t *testing.T panics bool hasError bool - // If True, calls to Begin will always succeed. This is necessary so that - // we can test subsequent calls in the transaction (e.g., Commit, Rollback). - forceBeginSuccess bool - errorWait chan struct{} + + errorWait chan struct{} } const ( @@ -199,1658 +196,265 @@ func (f *fakeVTGateService) StreamExecute(ctx context.Context, session *vtgatepb return nil } -// queryExecuteShards contains all the fields we use to test ExecuteShards -type queryExecuteShards struct { - SQL string - BindVariables map[string]*querypb.BindVariable - Keyspace string - Shards []string - TabletType topodatapb.TabletType - Session *vtgatepb.Session - NotInTransaction bool -} - -func (q *queryExecuteShards) equal(q2 *queryExecuteShards) bool { - return q.SQL == q2.SQL && - sqltypes.BindVariablesEqual(q.BindVariables, q2.BindVariables) && - q.Keyspace == q2.Keyspace && - reflect.DeepEqual(q.Shards, q2.Shards) && - q.TabletType == q2.TabletType && - proto.Equal(q.Session, q2.Session) && - q.NotInTransaction == q2.NotInTransaction -} - -// ExecuteShards is part of the VTGateService interface -func (f *fakeVTGateService) ExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { +// ResolveTransaction is part of the VTGateService interface +func (f *fakeVTGateService) ResolveTransaction(ctx context.Context, dtid string) error { if f.hasError { - return nil, errTestVtGateError + return errTestVtGateError } if f.panics { panic(fmt.Errorf("test forced panic")) } - f.checkCallerID(ctx, "ExecuteShards") - if !proto.Equal(options, testExecuteOptions) { - f.t.Errorf("wrong Execute options, got %+v, want %+v", options, testExecuteOptions) - } - execCase, ok := execMap[sql] - if !ok { - return nil, fmt.Errorf("no match for: %s", sql) - } - query := &queryExecuteShards{ - SQL: sql, - BindVariables: bindVariables, - TabletType: tabletType, - Keyspace: keyspace, - Shards: shards, - Session: session, - NotInTransaction: notInTransaction, - } - if !query.equal(execCase.shardQuery) { - f.t.Errorf("ExecuteShards: %+v, want %+v", query, execCase.shardQuery) - return nil, nil - } - if execCase.outSession != nil { - *session = *execCase.outSession + f.checkCallerID(ctx, "ResolveTransaction") + if dtid != dtid2 { + return errors.New("ResolveTransaction: dtid mismatch") } - return execCase.result, nil -} - -// queryExecuteKeyspaceIds contains all the fields we use to test -// ExecuteKeyspaceIds -type queryExecuteKeyspaceIds struct { - SQL string - BindVariables map[string]*querypb.BindVariable - Keyspace string - KeyspaceIds [][]byte - TabletType topodatapb.TabletType - Session *vtgatepb.Session - NotInTransaction bool + return nil } -func (q *queryExecuteKeyspaceIds) equal(q2 *queryExecuteKeyspaceIds) bool { - return q.SQL == q2.SQL && - sqltypes.BindVariablesEqual(q.BindVariables, q2.BindVariables) && - q.Keyspace == q2.Keyspace && - reflect.DeepEqual(q.KeyspaceIds, q2.KeyspaceIds) && - q.TabletType == q2.TabletType && - proto.Equal(q.Session, q2.Session) && - q.NotInTransaction == q2.NotInTransaction +func (f *fakeVTGateService) VStream(ctx context.Context, tabletType topodatapb.TabletType, vgtid *binlogdatapb.VGtid, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { + panic("unimplemented") } -// ExecuteKeyspaceIds is part of the VTGateService interface -func (f *fakeVTGateService) ExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if f.hasError { - return nil, errTestVtGateError - } - if f.panics { - panic(fmt.Errorf("test forced panic")) - } - f.checkCallerID(ctx, "ExecuteKeyspaceIds") - if !proto.Equal(options, testExecuteOptions) { - f.t.Errorf("wrong Execute options, got %+v, want %+v", options, testExecuteOptions) - } - execCase, ok := execMap[sql] - if !ok { - return nil, fmt.Errorf("no match for: %s", sql) - } - query := &queryExecuteKeyspaceIds{ - SQL: sql, - BindVariables: bindVariables, - TabletType: tabletType, - Keyspace: keyspace, - KeyspaceIds: keyspaceIds, - Session: session, - NotInTransaction: notInTransaction, - } - if !query.equal(execCase.keyspaceIDQuery) { - f.t.Errorf("ExecuteKeyspaceIds: %+v, want %+v", query, execCase.keyspaceIDQuery) - return nil, nil - } - if execCase.outSession != nil { - *session = *execCase.outSession +// CreateFakeServer returns the fake server for the tests +func CreateFakeServer(t *testing.T) vtgateservice.VTGateService { + return &fakeVTGateService{ + t: t, + panics: false, + errorWait: make(chan struct{}), } - return execCase.result, nil } -// queryExecuteKeyRanges contains all the fields we use to test ExecuteKeyRanges -type queryExecuteKeyRanges struct { - SQL string - BindVariables map[string]*querypb.BindVariable - Keyspace string - KeyRanges []*topodatapb.KeyRange - TabletType topodatapb.TabletType - Session *vtgatepb.Session - NotInTransaction bool +// RegisterTestDialProtocol registers a vtgateconn implementation under the "test" protocol +func RegisterTestDialProtocol(impl vtgateconn.Impl) { + vtgateconn.RegisterDialer("test", func(ctx context.Context, address string) (vtgateconn.Impl, error) { + return impl, nil + }) } -func (q *queryExecuteKeyRanges) equal(q2 *queryExecuteKeyRanges) bool { - if q.SQL != q2.SQL || - !sqltypes.BindVariablesEqual(q.BindVariables, q2.BindVariables) || - q.Keyspace != q2.Keyspace || - len(q.KeyRanges) != len(q2.KeyRanges) || - q.TabletType != q2.TabletType || - !proto.Equal(q.Session, q2.Session) || - q.NotInTransaction != q2.NotInTransaction { - return false - } - for i, kr := range q.KeyRanges { - if !proto.Equal(kr, q2.KeyRanges[i]) { - return false - } +// HandlePanic is part of the VTGateService interface +func (f *fakeVTGateService) HandlePanic(err *error) { + if x := recover(); x != nil { + // gRPC 0.13 chokes when you return a streaming error that contains newlines. + *err = fmt.Errorf("uncaught panic: %v, %s", x, + strings.Replace(string(tb.Stack(4)), "\n", ";", -1)) } - return true } -// ExecuteKeyRanges is part of the VTGateService interface -func (f *fakeVTGateService) ExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if f.hasError { - return nil, errTestVtGateError - } - if f.panics { - panic(fmt.Errorf("test forced panic")) - } - f.checkCallerID(ctx, "ExecuteKeyRanges") - if !proto.Equal(options, testExecuteOptions) { - f.t.Errorf("wrong Execute options, got %+v, want %+v", options, testExecuteOptions) - } - execCase, ok := execMap[sql] - if !ok { - return nil, fmt.Errorf("no match for: %s", sql) - } - query := &queryExecuteKeyRanges{ - SQL: sql, - BindVariables: bindVariables, - TabletType: tabletType, - Keyspace: keyspace, - KeyRanges: keyRanges, - Session: session, - NotInTransaction: notInTransaction, - } - if !query.equal(execCase.keyRangeQuery) { - f.t.Errorf("ExecuteKeyRanges: %+v, want %+v", query, execCase.keyRangeQuery) - return nil, nil - } - if execCase.outSession != nil { - *session = *execCase.outSession +// RunTests runs all the tests +func RunTests(t *testing.T, impl vtgateconn.Impl, fakeServer vtgateservice.VTGateService) { + vtgateconn.RegisterDialer("test", func(ctx context.Context, address string) (vtgateconn.Impl, error) { + return impl, nil + }) + conn, err := vtgateconn.DialProtocol(context.Background(), "test", "") + if err != nil { + t.Fatalf("Got err: %v from vtgateconn.DialProtocol", err) } - return execCase.result, nil -} + session := conn.Session("connection_ks@rdonly", testExecuteOptions) + + fs := fakeServer.(*fakeVTGateService) + + testExecute(t, session) + testStreamExecute(t, session) + testExecuteBatch(t, session) -// queryExecuteEntityIds contains all the fields we use to test ExecuteEntityIds -type queryExecuteEntityIds struct { - SQL string - BindVariables map[string]*querypb.BindVariable - Keyspace string - EntityColumnName string - EntityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId - TabletType topodatapb.TabletType - Session *vtgatepb.Session - NotInTransaction bool + // force a panic at every call, then test that works + fs.panics = true + testExecutePanic(t, session) + testExecuteBatchPanic(t, session) + testStreamExecutePanic(t, session) + fs.panics = false } -func (q *queryExecuteEntityIds) equal(q2 *queryExecuteEntityIds) bool { - if q.SQL != q2.SQL || - !sqltypes.BindVariablesEqual(q.BindVariables, q2.BindVariables) || - q.Keyspace != q2.Keyspace || - q.EntityColumnName != q2.EntityColumnName || - len(q.EntityKeyspaceIDs) != len(q2.EntityKeyspaceIDs) || - q.TabletType != q2.TabletType || - !proto.Equal(q.Session, q2.Session) || - q.NotInTransaction != q2.NotInTransaction { - return false - } - for i, e := range q.EntityKeyspaceIDs { - if !proto.Equal(e, q2.EntityKeyspaceIDs[i]) { - return false - } +// RunErrorTests runs all the tests that expect errors +func RunErrorTests(t *testing.T, fakeServer vtgateservice.VTGateService) { + conn, err := vtgateconn.DialProtocol(context.Background(), "test", "") + if err != nil { + t.Fatalf("Got err: %v from vtgateconn.DialProtocol", err) } - return true + session := conn.Session("connection_ks@rdonly", testExecuteOptions) + + fs := fakeServer.(*fakeVTGateService) + + // return an error for every call, make sure they're handled properly + fs.hasError = true + testExecuteError(t, session, fs) + testExecuteBatchError(t, session, fs) + testStreamExecuteError(t, session, fs) + fs.hasError = false } -// ExecuteEntityIds is part of the VTGateService interface -func (f *fakeVTGateService) ExecuteEntityIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, entityColumnName string, entityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if f.hasError { - return nil, errTestVtGateError - } - if f.panics { - panic(fmt.Errorf("test forced panic")) +func expectPanic(t *testing.T, err error) { + expected1 := "test forced panic" + expected2 := "uncaught panic" + if err == nil || !strings.Contains(err.Error(), expected1) || !strings.Contains(err.Error(), expected2) { + t.Fatalf("Expected a panic error with '%v' or '%v' but got: %v", expected1, expected2, err) } - f.checkCallerID(ctx, "ExecuteEntityIds") - if !proto.Equal(options, testExecuteOptions) { - f.t.Errorf("wrong Execute options, got %+v, want %+v", options, testExecuteOptions) +} + +// Verifies the returned error has the properties that we expect. +func verifyError(t *testing.T, err error, method string) { + if err == nil { + t.Errorf("%s was expecting an error, didn't get one", method) + return } - execCase, ok := execMap[sql] - if !ok { - return nil, fmt.Errorf("no match for: %s", sql) + // verify error code + code := vterrors.Code(err) + if code != expectedCode { + t.Errorf("Unexpected error code from %s: got %v, wanted %v", method, code, expectedCode) } - query := &queryExecuteEntityIds{ - SQL: sql, - BindVariables: bindVariables, - TabletType: tabletType, - Keyspace: keyspace, - EntityColumnName: entityColumnName, - EntityKeyspaceIDs: entityKeyspaceIDs, - Session: session, - NotInTransaction: notInTransaction, + verifyErrorString(t, err, method) +} + +func verifyErrorString(t *testing.T, err error, method string) { + if err == nil { + t.Errorf("%s was expecting an error, didn't get one", method) + return } - if len(query.EntityKeyspaceIDs) == 1 && len(query.EntityKeyspaceIDs[0].Value) == 0 { - query.EntityKeyspaceIDs[0].Value = nil + + if !strings.Contains(err.Error(), expectedErrMatch) { + t.Errorf("Unexpected error from %s: got %v, wanted err containing: %v", method, err, errTestVtGateError.Error()) } - if !query.equal(execCase.entityIdsQuery) { - f.t.Errorf("ExecuteEntityIds: %+v, want %+v", query, execCase.entityIdsQuery) - return nil, nil +} + +func testExecute(t *testing.T, session *vtgateconn.VTGateSession) { + ctx := newContext() + execCase := execMap["request1"] + qr, err := session.Execute(ctx, execCase.execQuery.SQL, execCase.execQuery.BindVariables) + require.NoError(t, err) + if !qr.Equal(execCase.result) { + t.Errorf("Unexpected result from Execute: got\n%#v want\n%#v", qr, execCase.result) } - if execCase.outSession != nil { - *session = *execCase.outSession + + _, err = session.Execute(ctx, "none", nil) + want := "no match for: none" + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("none request: %v, want %v", err, want) } - return execCase.result, nil } -// queryExecuteBatchShards contains all the fields we use to test -// ExecuteBatchShards -type queryExecuteBatchShards struct { - Queries []*vtgatepb.BoundShardQuery - TabletType topodatapb.TabletType - AsTransaction bool - Session *vtgatepb.Session +func testExecuteError(t *testing.T, session *vtgateconn.VTGateSession, fake *fakeVTGateService) { + ctx := newContext() + execCase := execMap["errorRequst"] + + _, err := session.Execute(ctx, execCase.execQuery.SQL, execCase.execQuery.BindVariables) + verifyError(t, err, "Execute") } -func (q *queryExecuteBatchShards) equal(q2 *queryExecuteBatchShards) bool { - if len(q.Queries) != len(q2.Queries) || - q.TabletType != q2.TabletType || - q.AsTransaction != q2.AsTransaction || - !proto.Equal(q.Session, q2.Session) { - return false - } - for i, bsq := range q.Queries { - if !proto.Equal(bsq, q2.Queries[i]) { - return false - } - } - return true +func testExecutePanic(t *testing.T, session *vtgateconn.VTGateSession) { + ctx := newContext() + execCase := execMap["request1"] + _, err := session.Execute(ctx, execCase.execQuery.SQL, execCase.execQuery.BindVariables) + expectPanic(t, err) } -// ExecuteBatchShards is part of the VTGateService interface -func (f *fakeVTGateService) ExecuteBatchShards(ctx context.Context, queries []*vtgatepb.BoundShardQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - if f.hasError { - return nil, errTestVtGateError - } - if f.panics { - panic(fmt.Errorf("test forced panic")) - } - f.checkCallerID(ctx, "ExecuteBatchShards") - if !proto.Equal(options, testExecuteOptions) { - f.t.Errorf("wrong Execute options, got %+v, want %+v", options, testExecuteOptions) - } - execCase, ok := execMap[queries[0].Query.Sql] - if !ok { - return nil, fmt.Errorf("no match for: %s", queries[0].Query.Sql) - } - query := &queryExecuteBatchShards{ - Queries: queries, - TabletType: tabletType, - AsTransaction: asTransaction, - Session: session, - } - if !query.equal(execCase.batchQueryShard) { - f.t.Errorf("ExecuteBatchShards: %+v, want %+v", query, execCase.batchQueryShard) - return nil, nil - } - if execCase.outSession != nil { - *session = *execCase.outSession +func testExecuteBatch(t *testing.T, session *vtgateconn.VTGateSession) { + ctx := newContext() + execCase := execMap["request1"] + qr, err := session.ExecuteBatch(ctx, []string{execCase.execQuery.SQL}, []map[string]*querypb.BindVariable{execCase.execQuery.BindVariables}) + require.NoError(t, err) + if !qr[0].QueryResult.Equal(execCase.result) { + t.Errorf("Unexpected result from Execute: got\n%#v want\n%#v", qr, execCase.result) } - if execCase.result != nil { - return []sqltypes.Result{*execCase.result}, nil + + _, err = session.ExecuteBatch(ctx, []string{"none"}, nil) + want := "no match for: none" + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("none request: %v, want %v", err, want) } - return nil, nil } -// queryExecuteBatchKeyspaceIds contains all the fields we use to test -// ExecuteBatchKeyspaceIds -type queryExecuteBatchKeyspaceIds struct { - Queries []*vtgatepb.BoundKeyspaceIdQuery - TabletType topodatapb.TabletType - AsTransaction bool - Session *vtgatepb.Session +func testExecuteBatchError(t *testing.T, session *vtgateconn.VTGateSession, fake *fakeVTGateService) { + ctx := newContext() + execCase := execMap["errorRequst"] + + _, err := session.ExecuteBatch(ctx, []string{execCase.execQuery.SQL}, []map[string]*querypb.BindVariable{execCase.execQuery.BindVariables}) + verifyError(t, err, "ExecuteBatch") } -func (q *queryExecuteBatchKeyspaceIds) equal(q2 *queryExecuteBatchKeyspaceIds) bool { - if len(q.Queries) != len(q2.Queries) || - q.TabletType != q2.TabletType || - q.AsTransaction != q2.AsTransaction || - !proto.Equal(q.Session, q2.Session) { - return false - } - for i, bsq := range q.Queries { - if !proto.Equal(bsq, q2.Queries[i]) { - return false - } - } - return true +func testExecuteBatchPanic(t *testing.T, session *vtgateconn.VTGateSession) { + ctx := newContext() + execCase := execMap["request1"] + _, err := session.ExecuteBatch(ctx, []string{execCase.execQuery.SQL}, []map[string]*querypb.BindVariable{execCase.execQuery.BindVariables}) + expectPanic(t, err) } -// ExecuteBatchKeyspaceIds is part of the VTGateService interface -func (f *fakeVTGateService) ExecuteBatchKeyspaceIds(ctx context.Context, queries []*vtgatepb.BoundKeyspaceIdQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - if f.hasError { - return nil, errTestVtGateError - } - if f.panics { - panic(fmt.Errorf("test forced panic")) +func testStreamExecute(t *testing.T, session *vtgateconn.VTGateSession) { + ctx := newContext() + execCase := execMap["request1"] + stream, err := session.StreamExecute(ctx, execCase.execQuery.SQL, execCase.execQuery.BindVariables) + if err != nil { + t.Fatal(err) } - f.checkCallerID(ctx, "ExecuteBatchKeyspaceIds") - if !proto.Equal(options, testExecuteOptions) { - f.t.Errorf("wrong Execute options, got %+v, want %+v", options, testExecuteOptions) + var qr sqltypes.Result + for { + packet, err := stream.Recv() + if err != nil { + if err != io.EOF { + t.Error(err) + } + break + } + if len(packet.Fields) != 0 { + qr.Fields = packet.Fields + } + if len(packet.Rows) != 0 { + qr.Rows = append(qr.Rows, packet.Rows...) + } } - execCase, ok := execMap[queries[0].Query.Sql] - if !ok { - return nil, fmt.Errorf("no match for: %s", queries[0].Query.Sql) + wantResult := *execCase.result + wantResult.RowsAffected = 0 + wantResult.InsertID = 0 + if !qr.Equal(&wantResult) { + t.Errorf("Unexpected result from StreamExecute: got %+v want %+v", qr, wantResult) } - query := &queryExecuteBatchKeyspaceIds{ - Queries: queries, - TabletType: tabletType, - AsTransaction: asTransaction, - Session: session, + + stream, err = session.StreamExecute(ctx, "none", nil) + if err != nil { + t.Fatal(err) } - if !query.equal(execCase.keyspaceIDBatchQuery) { - f.t.Errorf("ExecuteBatchKeyspaceIds: %+v, want %+v", query, execCase.keyspaceIDBatchQuery) - return nil, nil + _, err = stream.Recv() + want := "no match for: none" + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("none request: %v, want %v", err, want) } - if execCase.outSession != nil { - *session = *execCase.outSession - } - if execCase.result != nil { - return []sqltypes.Result{*execCase.result}, nil - } - return nil, nil -} - -// StreamExecuteShards is part of the VTGateService interface -func (f *fakeVTGateService) StreamExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - if f.panics { - panic(fmt.Errorf("test forced panic")) - } - f.checkCallerID(ctx, "StreamExecuteShards") - if !proto.Equal(options, testExecuteOptions) { - f.t.Errorf("wrong Execute options, got %+v, want %+v", options, testExecuteOptions) - } - execCase, ok := execMap[sql] - if !ok { - return fmt.Errorf("no match for: %s", sql) - } - query := &queryExecuteShards{ - SQL: sql, - BindVariables: bindVariables, - Keyspace: keyspace, - Shards: shards, - TabletType: tabletType, - } - if !query.equal(execCase.shardQuery) { - f.t.Errorf("Execute: %+v, want %+v", query, execCase.shardQuery) - return nil - } - if execCase.result != nil { - result := &sqltypes.Result{ - Fields: execCase.result.Fields, - } - if err := callback(result); err != nil { - return err - } - if f.hasError { - // wait until the client has the response, since all streaming implementation may not - // send previous messages if an error has been triggered. - <-f.errorWait - f.errorWait = make(chan struct{}) // for next test - return errTestVtGateError - } - for _, row := range execCase.result.Rows { - result := &sqltypes.Result{ - Rows: [][]sqltypes.Value{row}, - } - if err := callback(result); err != nil { - return err - } - } - } - return nil -} - -// StreamExecuteKeyspaceIds is part of the VTGateService interface -func (f *fakeVTGateService) StreamExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - if f.panics { - panic(fmt.Errorf("test forced panic")) - } - f.checkCallerID(ctx, "StreamExecuteKeyspaceIds") - if !proto.Equal(options, testExecuteOptions) { - f.t.Errorf("wrong Execute options, got %+v, want %+v", options, testExecuteOptions) - } - execCase, ok := execMap[sql] - if !ok { - return fmt.Errorf("no match for: %s", sql) - } - query := &queryExecuteKeyspaceIds{ - SQL: sql, - BindVariables: bindVariables, - Keyspace: keyspace, - KeyspaceIds: keyspaceIds, - TabletType: tabletType, - } - if !query.equal(execCase.keyspaceIDQuery) { - f.t.Errorf("StreamExecuteKeyspaceIds: %+v, want %+v", query, execCase.keyspaceIDQuery) - return nil - } - if execCase.result != nil { - result := &sqltypes.Result{ - Fields: execCase.result.Fields, - } - if err := callback(result); err != nil { - return err - } - if f.hasError { - // wait until the client has the response, since all streaming implementation may not - // send previous messages if an error has been triggered. - <-f.errorWait - f.errorWait = make(chan struct{}) // for next test - return errTestVtGateError - } - for _, row := range execCase.result.Rows { - result := &sqltypes.Result{ - Rows: [][]sqltypes.Value{row}, - } - if err := callback(result); err != nil { - return err - } - } - } - return nil -} - -// StreamExecuteKeyRanges is part of the VTGateService interface -func (f *fakeVTGateService) StreamExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - if f.panics { - panic(fmt.Errorf("test forced panic")) - } - f.checkCallerID(ctx, "StreamExecuteKeyRanges") - if !proto.Equal(options, testExecuteOptions) { - f.t.Errorf("wrong Execute options, got %+v, want %+v", options, testExecuteOptions) - } - execCase, ok := execMap[sql] - if !ok { - return fmt.Errorf("no match for: %s", sql) - } - query := &queryExecuteKeyRanges{ - SQL: sql, - BindVariables: bindVariables, - Keyspace: keyspace, - KeyRanges: keyRanges, - TabletType: tabletType, - } - if !query.equal(execCase.keyRangeQuery) { - f.t.Errorf("StreamExecuteKeyRanges: %+v, want %+v", query, execCase.keyRangeQuery) - return nil - } - if execCase.result != nil { - result := &sqltypes.Result{ - Fields: execCase.result.Fields, - } - if err := callback(result); err != nil { - return err - } - if f.hasError { - // wait until the client has the response, since all streaming implementation may not - // send previous messages if an error has been triggered. - <-f.errorWait - f.errorWait = make(chan struct{}) // for next test - return errTestVtGateError - } - for _, row := range execCase.result.Rows { - result := &sqltypes.Result{ - Rows: [][]sqltypes.Value{row}, - } - if err := callback(result); err != nil { - return err - } - } - } - return nil -} - -// Begin is part of the VTGateService interface -func (f *fakeVTGateService) Begin(ctx context.Context, singledb bool) (*vtgatepb.Session, error) { - f.checkCallerID(ctx, "Begin") - switch { - case f.forceBeginSuccess: - case f.hasError: - return nil, errTestVtGateError - case f.panics: - panic(fmt.Errorf("test forced panic")) - default: - } - return session1, nil -} - -// Commit is part of the VTGateService interface -func (f *fakeVTGateService) Commit(ctx context.Context, twopc bool, inSession *vtgatepb.Session) error { - f.checkCallerID(ctx, "Commit") - if f.hasError { - return errTestVtGateError - } - if f.panics { - panic(fmt.Errorf("test forced panic")) - } - if !proto.Equal(inSession, session2) { - return errors.New("commit: session mismatch") - } - return nil -} - -// Rollback is part of the VTGateService interface -func (f *fakeVTGateService) Rollback(ctx context.Context, inSession *vtgatepb.Session) error { - if f.hasError { - return errTestVtGateError - } - if f.panics { - panic(fmt.Errorf("test forced panic")) - } - f.checkCallerID(ctx, "Rollback") - if !proto.Equal(inSession, session2) { - return errors.New("rollback: session mismatch") - } - return nil -} - -// ResolveTransaction is part of the VTGateService interface -func (f *fakeVTGateService) ResolveTransaction(ctx context.Context, dtid string) error { - if f.hasError { - return errTestVtGateError - } - if f.panics { - panic(fmt.Errorf("test forced panic")) - } - f.checkCallerID(ctx, "ResolveTransaction") - if dtid != dtid2 { - return errors.New("ResolveTransaction: dtid mismatch") - } - return nil -} - -func (f *fakeVTGateService) MessageStream(ctx context.Context, keyspace string, shard string, keyRange *topodatapb.KeyRange, name string, callback func(*sqltypes.Result) error) error { - if f.hasError { - return errTestVtGateError - } - if f.panics { - panic(fmt.Errorf("test forced panic")) - } - f.checkCallerID(ctx, "ResolveTransaction") - if name != messageName { - return errors.New("MessageStream name mismatch") - } - callback(messageStreamResult) - return nil -} - -func (f *fakeVTGateService) MessageAck(ctx context.Context, keyspace string, name string, ids []*querypb.Value) (int64, error) { - if f.hasError { - return 0, errTestVtGateError - } - if f.panics { - panic(fmt.Errorf("test forced panic")) - } - f.checkCallerID(ctx, "ResolveTransaction") - if !sqltypes.Proto3ValuesEqual(ids, messageids) { - return 0, errors.New("MessageAck ids mismatch") - } - return messageAckRowsAffected, nil -} - -func (f *fakeVTGateService) MessageAckKeyspaceIds(ctx context.Context, keyspace string, name string, idKeyspaceIDs []*vtgatepb.IdKeyspaceId) (int64, error) { - if f.hasError { - return 0, errTestVtGateError - } - if f.panics { - panic(fmt.Errorf("test forced panic")) - } - f.checkCallerID(ctx, "ResolveTransaction") - msg1 := &vtgatepb.MessageAckKeyspaceIdsRequest{ - IdKeyspaceIds: idKeyspaceIDs, - } - msg2 := &vtgatepb.MessageAckKeyspaceIdsRequest{ - IdKeyspaceIds: testIDKeyspaceIDs, - } - if !proto.Equal(msg1, msg2) { - return 0, errors.New("MessageAck ids mismatch") - } - return messageAckRowsAffected, nil -} - -// GetSrvKeyspace is part of the VTGateService interface -func (f *fakeVTGateService) GetSrvKeyspace(ctx context.Context, keyspace string) (*topodatapb.SrvKeyspace, error) { - if f.hasError { - return nil, errTestVtGateError - } - if f.panics { - panic(fmt.Errorf("test forced panic")) - } - if keyspace != getSrvKeyspaceKeyspace { - f.t.Errorf("GetSrvKeyspace has wrong input: got %v wanted %v", keyspace, getSrvKeyspaceKeyspace) - } - return getSrvKeyspaceResult, nil -} - -func (f *fakeVTGateService) VStream(ctx context.Context, tabletType topodatapb.TabletType, vgtid *binlogdatapb.VGtid, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { - panic("unimplemented") -} - -// CreateFakeServer returns the fake server for the tests -func CreateFakeServer(t *testing.T) vtgateservice.VTGateService { - return &fakeVTGateService{ - t: t, - panics: false, - errorWait: make(chan struct{}), - } -} - -// RegisterTestDialProtocol registers a vtgateconn implementation under the "test" protocol -func RegisterTestDialProtocol(impl vtgateconn.Impl) { - vtgateconn.RegisterDialer("test", func(ctx context.Context, address string) (vtgateconn.Impl, error) { - return impl, nil - }) -} - -// HandlePanic is part of the VTGateService interface -func (f *fakeVTGateService) HandlePanic(err *error) { - if x := recover(); x != nil { - // gRPC 0.13 chokes when you return a streaming error that contains newlines. - *err = fmt.Errorf("uncaught panic: %v, %s", x, - strings.Replace(string(tb.Stack(4)), "\n", ";", -1)) - } -} - -// RunTests runs all the tests -func RunTests(t *testing.T, impl vtgateconn.Impl, fakeServer vtgateservice.VTGateService) { - vtgateconn.RegisterDialer("test", func(ctx context.Context, address string) (vtgateconn.Impl, error) { - return impl, nil - }) - conn, err := vtgateconn.DialProtocol(context.Background(), "test", "") - if err != nil { - t.Fatalf("Got err: %v from vtgateconn.DialProtocol", err) - } - session := conn.Session("connection_ks@rdonly", testExecuteOptions) - - fs := fakeServer.(*fakeVTGateService) - - testBegin(t, conn) - testExecute(t, session) - testExecuteBatch(t, session) - testExecuteShards(t, conn) - testExecuteKeyspaceIds(t, conn) - testExecuteKeyRanges(t, conn) - testExecuteEntityIds(t, conn) - testExecuteBatchShards(t, conn) - testExecuteBatchKeyspaceIds(t, conn) - testStreamExecute(t, session) - testStreamExecuteShards(t, conn) - testStreamExecuteKeyRanges(t, conn) - testStreamExecuteKeyspaceIds(t, conn) - testTxPass(t, conn) - testResolveTransaction(t, conn) - testTxFail(t, conn) - testMessageStream(t, conn) - testMessageAck(t, conn) - testMessageAckKeyspaceIds(t, conn) - testGetSrvKeyspace(t, conn) - - // force a panic at every call, then test that works - fs.panics = true - testBeginPanic(t, conn) - testCommitPanic(t, conn, fs) - testRollbackPanic(t, conn, fs) - testResolveTransactionPanic(t, conn, fs) - testExecutePanic(t, session) - testExecuteBatchPanic(t, session) - testExecuteShardsPanic(t, conn) - testExecuteKeyspaceIdsPanic(t, conn) - testExecuteKeyRangesPanic(t, conn) - testExecuteEntityIdsPanic(t, conn) - testExecuteBatchShardsPanic(t, conn) - testExecuteBatchKeyspaceIdsPanic(t, conn) - testStreamExecutePanic(t, session) - testStreamExecuteShardsPanic(t, conn) - testStreamExecuteKeyRangesPanic(t, conn) - testStreamExecuteKeyspaceIdsPanic(t, conn) - testMessageStreamPanic(t, conn) - testMessageAckPanic(t, conn) - testMessageAckKeyspaceIdsPanic(t, conn) - testGetSrvKeyspacePanic(t, conn) - fs.panics = false -} - -// RunErrorTests runs all the tests that expect errors -func RunErrorTests(t *testing.T, fakeServer vtgateservice.VTGateService) { - conn, err := vtgateconn.DialProtocol(context.Background(), "test", "") - if err != nil { - t.Fatalf("Got err: %v from vtgateconn.DialProtocol", err) - } - session := conn.Session("connection_ks@rdonly", testExecuteOptions) - - fs := fakeServer.(*fakeVTGateService) - - // return an error for every call, make sure they're handled properly - fs.hasError = true - testBeginError(t, conn) - testCommitError(t, conn, fs) - testRollbackError(t, conn, fs) - testResolveTransactionError(t, conn, fs) - testExecuteError(t, session, fs) - testExecuteBatchError(t, session, fs) - testExecuteShardsError(t, conn, fs) - testExecuteKeyspaceIdsError(t, conn, fs) - testExecuteKeyRangesError(t, conn, fs) - testExecuteEntityIdsError(t, conn, fs) - testExecuteBatchShardsError(t, conn, fs) - testExecuteBatchKeyspaceIdsError(t, conn, fs) - testStreamExecuteError(t, session, fs) - testStreamExecuteShardsError(t, conn, fs) - testStreamExecuteKeyRangesError(t, conn, fs) - testStreamExecuteKeyspaceIdsError(t, conn, fs) - testMessageStreamError(t, conn) - testMessageAckError(t, conn) - testMessageAckKeyspaceIdsError(t, conn) - testGetSrvKeyspaceError(t, conn) - fs.hasError = false -} - -func expectPanic(t *testing.T, err error) { - expected1 := "test forced panic" - expected2 := "uncaught panic" - if err == nil || !strings.Contains(err.Error(), expected1) || !strings.Contains(err.Error(), expected2) { - t.Fatalf("Expected a panic error with '%v' or '%v' but got: %v", expected1, expected2, err) - } -} - -// Verifies the returned error has the properties that we expect. -func verifyError(t *testing.T, err error, method string) { - if err == nil { - t.Errorf("%s was expecting an error, didn't get one", method) - return - } - // verify error code - code := vterrors.Code(err) - if code != expectedCode { - t.Errorf("Unexpected error code from %s: got %v, wanted %v", method, code, expectedCode) - } - verifyErrorString(t, err, method) -} - -func verifyErrorString(t *testing.T, err error, method string) { - if err == nil { - t.Errorf("%s was expecting an error, didn't get one", method) - return - } - - if !strings.Contains(err.Error(), expectedErrMatch) { - t.Errorf("Unexpected error from %s: got %v, wanted err containing: %v", method, err, errTestVtGateError.Error()) - } -} - -func testBegin(t *testing.T, conn *vtgateconn.VTGateConn) { - _, err := conn.Begin(newContext()) - require.NoError(t, err) -} - -func testExecute(t *testing.T, session *vtgateconn.VTGateSession) { - ctx := newContext() - execCase := execMap["request1"] - qr, err := session.Execute(ctx, execCase.execQuery.SQL, execCase.execQuery.BindVariables) - require.NoError(t, err) - if !qr.Equal(execCase.result) { - t.Errorf("Unexpected result from Execute: got\n%#v want\n%#v", qr, execCase.result) - } - - _, err = session.Execute(ctx, "none", nil) - want := "no match for: none" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("none request: %v, want %v", err, want) - } -} - -func testExecuteError(t *testing.T, session *vtgateconn.VTGateSession, fake *fakeVTGateService) { - ctx := newContext() - execCase := execMap["errorRequst"] - - _, err := session.Execute(ctx, execCase.execQuery.SQL, execCase.execQuery.BindVariables) - verifyError(t, err, "Execute") -} - -func testExecutePanic(t *testing.T, session *vtgateconn.VTGateSession) { - ctx := newContext() - execCase := execMap["request1"] - _, err := session.Execute(ctx, execCase.execQuery.SQL, execCase.execQuery.BindVariables) - expectPanic(t, err) -} - -func testExecuteBatch(t *testing.T, session *vtgateconn.VTGateSession) { - ctx := newContext() - execCase := execMap["request1"] - qr, err := session.ExecuteBatch(ctx, []string{execCase.execQuery.SQL}, []map[string]*querypb.BindVariable{execCase.execQuery.BindVariables}) - require.NoError(t, err) - if !qr[0].QueryResult.Equal(execCase.result) { - t.Errorf("Unexpected result from Execute: got\n%#v want\n%#v", qr, execCase.result) - } - - _, err = session.ExecuteBatch(ctx, []string{"none"}, nil) - want := "no match for: none" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("none request: %v, want %v", err, want) - } -} - -func testExecuteBatchError(t *testing.T, session *vtgateconn.VTGateSession, fake *fakeVTGateService) { - ctx := newContext() - execCase := execMap["errorRequst"] - - _, err := session.ExecuteBatch(ctx, []string{execCase.execQuery.SQL}, []map[string]*querypb.BindVariable{execCase.execQuery.BindVariables}) - verifyError(t, err, "ExecuteBatch") -} - -func testExecuteBatchPanic(t *testing.T, session *vtgateconn.VTGateSession) { - ctx := newContext() - execCase := execMap["request1"] - _, err := session.ExecuteBatch(ctx, []string{execCase.execQuery.SQL}, []map[string]*querypb.BindVariable{execCase.execQuery.BindVariables}) - expectPanic(t, err) -} - -func testExecuteShards(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - qr, err := conn.ExecuteShards(ctx, execCase.shardQuery.SQL, execCase.shardQuery.Keyspace, execCase.shardQuery.Shards, execCase.shardQuery.BindVariables, execCase.shardQuery.TabletType, testExecuteOptions) - require.NoError(t, err) - if !qr.Equal(execCase.result) { - t.Errorf("Unexpected result from Execute: got %+v want %+v", qr, execCase.result) - } - - _, err = conn.ExecuteShards(ctx, "none", "", []string{}, nil, topodatapb.TabletType_RDONLY, testExecuteOptions) - want := "no match for: none" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("none request: %v, want %v", err, want) - } -} - -func testExecuteShardsError(t *testing.T, conn *vtgateconn.VTGateConn, fake *fakeVTGateService) { - ctx := newContext() - execCase := execMap["errorRequst"] - - _, err := conn.ExecuteShards(ctx, execCase.shardQuery.SQL, execCase.shardQuery.Keyspace, execCase.shardQuery.Shards, execCase.shardQuery.BindVariables, execCase.shardQuery.TabletType, testExecuteOptions) - verifyError(t, err, "ExecuteShards") -} - -func testExecuteShardsPanic(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - _, err := conn.ExecuteShards(ctx, execCase.execQuery.SQL, "ks", []string{"1", "2"}, execCase.shardQuery.BindVariables, execCase.shardQuery.TabletType, testExecuteOptions) - expectPanic(t, err) -} - -func testExecuteKeyspaceIds(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - qr, err := conn.ExecuteKeyspaceIds(ctx, execCase.keyspaceIDQuery.SQL, execCase.keyspaceIDQuery.Keyspace, execCase.keyspaceIDQuery.KeyspaceIds, execCase.keyspaceIDQuery.BindVariables, execCase.keyspaceIDQuery.TabletType, testExecuteOptions) - require.NoError(t, err) - if !qr.Equal(execCase.result) { - t.Errorf("Unexpected result from Execute: got %+v want %+v", qr, execCase.result) - } - - _, err = conn.ExecuteKeyspaceIds(ctx, "none", "", [][]byte{}, nil, topodatapb.TabletType_REPLICA, testExecuteOptions) - want := "no match for: none" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("none request: %v, want %v", err, want) - } -} - -func testExecuteKeyspaceIdsError(t *testing.T, conn *vtgateconn.VTGateConn, fake *fakeVTGateService) { - ctx := newContext() - execCase := execMap["errorRequst"] - - _, err := conn.ExecuteKeyspaceIds(ctx, execCase.keyspaceIDQuery.SQL, execCase.keyspaceIDQuery.Keyspace, execCase.keyspaceIDQuery.KeyspaceIds, execCase.keyspaceIDQuery.BindVariables, execCase.keyspaceIDQuery.TabletType, testExecuteOptions) - verifyError(t, err, "ExecuteKeyspaceIds") -} - -func testExecuteKeyspaceIdsPanic(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - _, err := conn.ExecuteKeyspaceIds(ctx, execCase.keyspaceIDQuery.SQL, execCase.keyspaceIDQuery.Keyspace, execCase.keyspaceIDQuery.KeyspaceIds, execCase.keyspaceIDQuery.BindVariables, execCase.keyspaceIDQuery.TabletType, testExecuteOptions) - expectPanic(t, err) -} - -func testExecuteKeyRanges(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - qr, err := conn.ExecuteKeyRanges(ctx, execCase.keyRangeQuery.SQL, execCase.keyRangeQuery.Keyspace, execCase.keyRangeQuery.KeyRanges, execCase.keyRangeQuery.BindVariables, execCase.keyRangeQuery.TabletType, testExecuteOptions) - require.NoError(t, err) - if !qr.Equal(execCase.result) { - t.Errorf("Unexpected result from Execute: got %+v want %+v", qr, execCase.result) - } - - _, err = conn.ExecuteKeyRanges(ctx, "none", "", []*topodatapb.KeyRange{}, nil, topodatapb.TabletType_REPLICA, testExecuteOptions) - want := "no match for: none" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("none request: %v, want %v", err, want) - } -} - -func testExecuteKeyRangesError(t *testing.T, conn *vtgateconn.VTGateConn, fake *fakeVTGateService) { - ctx := newContext() - execCase := execMap["errorRequst"] - - _, err := conn.ExecuteKeyRanges(ctx, execCase.keyRangeQuery.SQL, execCase.keyRangeQuery.Keyspace, execCase.keyRangeQuery.KeyRanges, execCase.keyRangeQuery.BindVariables, execCase.keyRangeQuery.TabletType, testExecuteOptions) - verifyError(t, err, "ExecuteKeyRanges") -} - -func testExecuteKeyRangesPanic(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - _, err := conn.ExecuteKeyRanges(ctx, execCase.keyRangeQuery.SQL, execCase.keyRangeQuery.Keyspace, execCase.keyRangeQuery.KeyRanges, execCase.keyRangeQuery.BindVariables, execCase.keyRangeQuery.TabletType, testExecuteOptions) - expectPanic(t, err) -} - -func testExecuteEntityIds(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - qr, err := conn.ExecuteEntityIds(ctx, execCase.entityIdsQuery.SQL, execCase.entityIdsQuery.Keyspace, execCase.entityIdsQuery.EntityColumnName, execCase.entityIdsQuery.EntityKeyspaceIDs, execCase.entityIdsQuery.BindVariables, execCase.entityIdsQuery.TabletType, testExecuteOptions) - require.NoError(t, err) - if !qr.Equal(execCase.result) { - t.Errorf("Unexpected result from Execute: got %+v want %+v", qr, execCase.result) - } - - _, err = conn.ExecuteEntityIds(ctx, "none", "", "", []*vtgatepb.ExecuteEntityIdsRequest_EntityId{}, nil, topodatapb.TabletType_REPLICA, testExecuteOptions) - want := "no match for: none" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("none request: %v, want %v", err, want) - } -} - -func testExecuteEntityIdsError(t *testing.T, conn *vtgateconn.VTGateConn, fake *fakeVTGateService) { - ctx := newContext() - execCase := execMap["errorRequst"] - - _, err := conn.ExecuteEntityIds(ctx, execCase.entityIdsQuery.SQL, execCase.entityIdsQuery.Keyspace, execCase.entityIdsQuery.EntityColumnName, execCase.entityIdsQuery.EntityKeyspaceIDs, execCase.entityIdsQuery.BindVariables, execCase.entityIdsQuery.TabletType, testExecuteOptions) - verifyError(t, err, "ExecuteEntityIds") -} - -func testExecuteEntityIdsPanic(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - _, err := conn.ExecuteEntityIds(ctx, execCase.entityIdsQuery.SQL, execCase.entityIdsQuery.Keyspace, execCase.entityIdsQuery.EntityColumnName, execCase.entityIdsQuery.EntityKeyspaceIDs, execCase.entityIdsQuery.BindVariables, execCase.entityIdsQuery.TabletType, testExecuteOptions) - expectPanic(t, err) -} - -func testExecuteBatchShards(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - ql, err := conn.ExecuteBatchShards(ctx, execCase.batchQueryShard.Queries, execCase.batchQueryShard.TabletType, execCase.batchQueryShard.AsTransaction, testExecuteOptions) - require.NoError(t, err) - if !ql[0].Equal(execCase.result) { - t.Errorf("Unexpected result from Execute: got %+v want %+v", ql, execCase.result) - } - - _, err = conn.ExecuteBatchShards(ctx, []*vtgatepb.BoundShardQuery{ - {Query: &querypb.BoundQuery{Sql: "none"}}}, - topodatapb.TabletType_REPLICA, true, testExecuteOptions) - want := "no match for: none" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("none request: %v, want %v", err, want) - } -} - -func testExecuteBatchShardsError(t *testing.T, conn *vtgateconn.VTGateConn, fake *fakeVTGateService) { - ctx := newContext() - execCase := execMap["errorRequst"] - - _, err := conn.ExecuteBatchShards(ctx, execCase.batchQueryShard.Queries, execCase.batchQueryShard.TabletType, execCase.batchQueryShard.AsTransaction, testExecuteOptions) - verifyError(t, err, "ExecuteBatchShards") -} - -func testExecuteBatchShardsPanic(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - _, err := conn.ExecuteBatchShards(ctx, execCase.batchQueryShard.Queries, execCase.batchQueryShard.TabletType, execCase.batchQueryShard.AsTransaction, testExecuteOptions) - expectPanic(t, err) -} - -func testExecuteBatchKeyspaceIds(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - ql, err := conn.ExecuteBatchKeyspaceIds(ctx, execCase.keyspaceIDBatchQuery.Queries, execCase.keyspaceIDBatchQuery.TabletType, execCase.batchQueryShard.AsTransaction, testExecuteOptions) - require.NoError(t, err) - if !ql[0].Equal(execCase.result) { - t.Errorf("Unexpected result from Execute: got %+v want %+v", ql, execCase.result) - } - - _, err = conn.ExecuteBatchKeyspaceIds(ctx, []*vtgatepb.BoundKeyspaceIdQuery{ - {Query: &querypb.BoundQuery{Sql: "none"}, KeyspaceIds: [][]byte{}}}, - topodatapb.TabletType_REPLICA, false, testExecuteOptions) - want := "no match for: none" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("none request: %v, want %v", err, want) - } -} - -func testExecuteBatchKeyspaceIdsError(t *testing.T, conn *vtgateconn.VTGateConn, fake *fakeVTGateService) { - ctx := newContext() - execCase := execMap["errorRequst"] - - _, err := conn.ExecuteBatchKeyspaceIds(ctx, execCase.keyspaceIDBatchQuery.Queries, execCase.keyspaceIDBatchQuery.TabletType, execCase.keyspaceIDBatchQuery.AsTransaction, testExecuteOptions) - verifyError(t, err, "ExecuteBatchKeyspaceIds") -} - -func testExecuteBatchKeyspaceIdsPanic(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - _, err := conn.ExecuteBatchKeyspaceIds(ctx, execCase.keyspaceIDBatchQuery.Queries, execCase.keyspaceIDBatchQuery.TabletType, execCase.keyspaceIDBatchQuery.AsTransaction, testExecuteOptions) - expectPanic(t, err) -} - -func testStreamExecute(t *testing.T, session *vtgateconn.VTGateSession) { - ctx := newContext() - execCase := execMap["request1"] - stream, err := session.StreamExecute(ctx, execCase.execQuery.SQL, execCase.execQuery.BindVariables) - if err != nil { - t.Fatal(err) - } - var qr sqltypes.Result - for { - packet, err := stream.Recv() - if err != nil { - if err != io.EOF { - t.Error(err) - } - break - } - if len(packet.Fields) != 0 { - qr.Fields = packet.Fields - } - if len(packet.Rows) != 0 { - qr.Rows = append(qr.Rows, packet.Rows...) - } - } - wantResult := *execCase.result - wantResult.RowsAffected = 0 - wantResult.InsertID = 0 - if !qr.Equal(&wantResult) { - t.Errorf("Unexpected result from StreamExecute: got %+v want %+v", qr, wantResult) - } - - stream, err = session.StreamExecute(ctx, "none", nil) - if err != nil { - t.Fatal(err) - } - _, err = stream.Recv() - want := "no match for: none" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("none request: %v, want %v", err, want) - } -} - -func testStreamExecuteError(t *testing.T, session *vtgateconn.VTGateSession, fake *fakeVTGateService) { - ctx := newContext() - execCase := execMap["request1"] - stream, err := session.StreamExecute(ctx, execCase.execQuery.SQL, execCase.execQuery.BindVariables) - if err != nil { - t.Fatalf("StreamExecute failed: %v", err) - } - qr, err := stream.Recv() - if err != nil { - t.Fatalf("StreamExecute failed: cannot read result1: %v", err) - } - - if !qr.Equal(&streamResultFields) { - t.Errorf("Unexpected result from StreamExecute: got %#v want %#v", qr, &streamResultFields) - } - // signal to the server that the first result has been received - close(fake.errorWait) - // After 1 result, we expect to get an error (no more results). - _, err = stream.Recv() - if err == nil { - t.Fatalf("StreamExecute channel wasn't closed") - } - verifyError(t, err, "StreamExecute") -} - -func testStreamExecutePanic(t *testing.T, session *vtgateconn.VTGateSession) { - ctx := newContext() - execCase := execMap["request1"] - stream, err := session.StreamExecute(ctx, execCase.execQuery.SQL, execCase.execQuery.BindVariables) - if err != nil { - t.Fatal(err) - } - _, err = stream.Recv() - if err == nil { - t.Fatalf("Received packets instead of panic?") - } - expectPanic(t, err) -} - -func testStreamExecuteShards(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - stream, err := conn.StreamExecuteShards(ctx, execCase.shardQuery.SQL, execCase.shardQuery.Keyspace, execCase.shardQuery.Shards, execCase.shardQuery.BindVariables, execCase.shardQuery.TabletType, testExecuteOptions) - if err != nil { - t.Fatal(err) - } - var qr sqltypes.Result - for { - packet, err := stream.Recv() - if err != nil { - if err != io.EOF { - t.Error(err) - } - break - } - if len(packet.Fields) != 0 { - qr.Fields = packet.Fields - } - if len(packet.Rows) != 0 { - qr.Rows = append(qr.Rows, packet.Rows...) - } - } - wantResult := *execCase.result - wantResult.RowsAffected = 0 - wantResult.InsertID = 0 - if !qr.Equal(&wantResult) { - t.Errorf("Unexpected result from StreamExecuteShards: got %+v want %+v", qr, wantResult) - } - - stream, err = conn.StreamExecuteShards(ctx, "none", "", []string{}, nil, topodatapb.TabletType_REPLICA, testExecuteOptions) - if err != nil { - t.Fatal(err) - } - _, err = stream.Recv() - want := "no match for: none" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("none request: %v, want %v", err, want) - } -} - -func testStreamExecuteShardsError(t *testing.T, conn *vtgateconn.VTGateConn, fake *fakeVTGateService) { - ctx := newContext() - execCase := execMap["request1"] - stream, err := conn.StreamExecuteShards(ctx, execCase.shardQuery.SQL, execCase.shardQuery.Keyspace, execCase.shardQuery.Shards, execCase.shardQuery.BindVariables, execCase.shardQuery.TabletType, testExecuteOptions) - if err != nil { - t.Fatalf("StreamExecuteShards failed: %v", err) - } - qr, err := stream.Recv() - if err != nil { - t.Fatalf("StreamExecuteShards failed: cannot read result1: %v", err) - } - - if !qr.Equal(&streamResultFields) { - t.Errorf("Unexpected result from StreamExecuteShards: got %#v want %#v", qr, &streamResultFields) - } - // signal to the server that the first result has been received - close(fake.errorWait) - // After 1 result, we expect to get an error (no more results). - _, err = stream.Recv() - if err == nil { - t.Fatalf("StreamExecuteShards channel wasn't closed") - } - verifyError(t, err, "StreamExecuteShards") -} - -func testStreamExecuteShardsPanic(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - stream, err := conn.StreamExecuteShards(ctx, execCase.shardQuery.SQL, execCase.shardQuery.Keyspace, execCase.shardQuery.Shards, execCase.shardQuery.BindVariables, execCase.shardQuery.TabletType, testExecuteOptions) - if err != nil { - t.Fatal(err) - } - _, err = stream.Recv() - if err == nil { - t.Fatalf("Received packets instead of panic?") - } - expectPanic(t, err) -} - -func testStreamExecuteKeyRanges(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - stream, err := conn.StreamExecuteKeyRanges(ctx, execCase.keyRangeQuery.SQL, execCase.keyRangeQuery.Keyspace, execCase.keyRangeQuery.KeyRanges, execCase.keyRangeQuery.BindVariables, execCase.keyRangeQuery.TabletType, testExecuteOptions) - if err != nil { - t.Fatal(err) - } - var qr sqltypes.Result - for { - packet, err := stream.Recv() - if err != nil { - if err != io.EOF { - t.Error(err) - } - break - } - if len(packet.Fields) != 0 { - qr.Fields = packet.Fields - } - if len(packet.Rows) != 0 { - qr.Rows = append(qr.Rows, packet.Rows...) - } - } - wantResult := *execCase.result - wantResult.RowsAffected = 0 - wantResult.InsertID = 0 - if !qr.Equal(&wantResult) { - t.Errorf("Unexpected result from StreamExecuteKeyRanges: got %+v want %+v", qr, wantResult) - } - - stream, err = conn.StreamExecuteKeyRanges(ctx, "none", "", []*topodatapb.KeyRange{}, nil, topodatapb.TabletType_REPLICA, testExecuteOptions) - if err != nil { - t.Fatal(err) - } - _, err = stream.Recv() - want := "no match for: none" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("none request: %v, want %v", err, want) - } -} - -func testStreamExecuteKeyRangesError(t *testing.T, conn *vtgateconn.VTGateConn, fake *fakeVTGateService) { - ctx := newContext() - execCase := execMap["request1"] - stream, err := conn.StreamExecuteKeyRanges(ctx, execCase.keyRangeQuery.SQL, execCase.keyRangeQuery.Keyspace, execCase.keyRangeQuery.KeyRanges, execCase.keyRangeQuery.BindVariables, execCase.keyRangeQuery.TabletType, testExecuteOptions) - if err != nil { - t.Fatalf("StreamExecuteKeyRanges failed: %v", err) - } - qr, err := stream.Recv() - if err != nil { - t.Fatalf("StreamExecuteKeyRanges failed: cannot read result1: %v", err) - } - - if !qr.Equal(&streamResultFields) { - t.Errorf("Unexpected result from StreamExecuteKeyRanges: got %#v want %#v", qr, &streamResultFields) - } - // signal to the server that the first result has been received - close(fake.errorWait) - // After 1 result, we expect to get an error (no more results). - _, err = stream.Recv() - if err == nil { - t.Fatalf("StreamExecuteKeyRanges channel wasn't closed") - } - verifyError(t, err, "StreamExecuteKeyRanges") -} - -func testStreamExecuteKeyRangesPanic(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - stream, err := conn.StreamExecuteKeyRanges(ctx, execCase.keyRangeQuery.SQL, execCase.keyRangeQuery.Keyspace, execCase.keyRangeQuery.KeyRanges, execCase.keyRangeQuery.BindVariables, execCase.keyRangeQuery.TabletType, testExecuteOptions) - if err != nil { - t.Fatal(err) - } - _, err = stream.Recv() - if err == nil { - t.Fatalf("Received packets instead of panic?") - } - expectPanic(t, err) -} - -func testStreamExecuteKeyspaceIds(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - stream, err := conn.StreamExecuteKeyspaceIds(ctx, execCase.keyspaceIDQuery.SQL, execCase.keyspaceIDQuery.Keyspace, execCase.keyspaceIDQuery.KeyspaceIds, execCase.keyspaceIDQuery.BindVariables, execCase.keyspaceIDQuery.TabletType, testExecuteOptions) - if err != nil { - t.Fatal(err) - } - var qr sqltypes.Result - for { - packet, err := stream.Recv() - if err != nil { - if err != io.EOF { - t.Error(err) - } - break - } - if len(packet.Fields) != 0 { - qr.Fields = packet.Fields - } - if len(packet.Rows) != 0 { - qr.Rows = append(qr.Rows, packet.Rows...) - } - } - wantResult := *execCase.result - wantResult.RowsAffected = 0 - wantResult.InsertID = 0 - if !qr.Equal(&wantResult) { - t.Errorf("Unexpected result from StreamExecuteKeyspaceIds: got %+v want %+v", qr, wantResult) - } - - stream, err = conn.StreamExecuteKeyspaceIds(ctx, "none", "", [][]byte{}, nil, topodatapb.TabletType_REPLICA, testExecuteOptions) - if err != nil { - t.Fatal(err) - } - _, err = stream.Recv() - want := "no match for: none" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("none request: %v, want %v", err, want) - } -} - -func testStreamExecuteKeyspaceIdsError(t *testing.T, conn *vtgateconn.VTGateConn, fake *fakeVTGateService) { - ctx := newContext() - execCase := execMap["request1"] - stream, err := conn.StreamExecuteKeyspaceIds(ctx, execCase.keyspaceIDQuery.SQL, execCase.keyspaceIDQuery.Keyspace, execCase.keyspaceIDQuery.KeyspaceIds, execCase.keyspaceIDQuery.BindVariables, execCase.keyspaceIDQuery.TabletType, testExecuteOptions) - if err != nil { - t.Fatalf("StreamExecuteKeyspaceIds failed: %v", err) - } - qr, err := stream.Recv() - if err != nil { - t.Fatalf("StreamExecuteKeyspaceIds failed: cannot read result1: %v", err) - } - - if !qr.Equal(&streamResultFields) { - t.Errorf("Unexpected result from StreamExecuteKeyspaceIds: got %#v want %#v", qr, &streamResultFields) - } - // signal to the server that the first result has been received - close(fake.errorWait) - // After 1 result, we expect to get an error (no more results). - _, err = stream.Recv() - if err == nil { - t.Fatalf("StreamExecuteKeyspaceIds channel wasn't closed") - } - verifyError(t, err, "StreamExecuteKeyspaceIds") -} - -func testStreamExecuteKeyspaceIdsPanic(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["request1"] - stream, err := conn.StreamExecuteKeyspaceIds(ctx, execCase.keyspaceIDQuery.SQL, execCase.keyspaceIDQuery.Keyspace, execCase.keyspaceIDQuery.KeyspaceIds, execCase.keyspaceIDQuery.BindVariables, execCase.keyspaceIDQuery.TabletType, testExecuteOptions) - if err != nil { - t.Fatal(err) - } - _, err = stream.Recv() - if err == nil { - t.Fatalf("Received packets instead of panic?") - } - expectPanic(t, err) -} - -func testTxPass(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - execCase := execMap["txRequest"] - - // ExecuteShards - tx, err := conn.Begin(ctx) - require.NoError(t, err) - _, err = tx.ExecuteShards(ctx, execCase.shardQuery.SQL, execCase.shardQuery.Keyspace, execCase.shardQuery.Shards, execCase.shardQuery.BindVariables, execCase.shardQuery.TabletType, testExecuteOptions) - require.NoError(t, err) - err = tx.Rollback(ctx) - require.NoError(t, err) - - // ExecuteKeyspaceIds - tx, err = conn.Begin(ctx) - require.NoError(t, err) - _, err = tx.ExecuteKeyspaceIds(ctx, execCase.keyspaceIDQuery.SQL, execCase.keyspaceIDQuery.Keyspace, execCase.keyspaceIDQuery.KeyspaceIds, execCase.keyspaceIDQuery.BindVariables, execCase.keyspaceIDQuery.TabletType, testExecuteOptions) - require.NoError(t, err) - err = tx.Rollback(ctx) - require.NoError(t, err) - - // ExecuteKeyRanges - tx, err = conn.Begin(ctx) - require.NoError(t, err) - _, err = tx.ExecuteKeyRanges(ctx, execCase.keyRangeQuery.SQL, execCase.keyRangeQuery.Keyspace, execCase.keyRangeQuery.KeyRanges, execCase.keyRangeQuery.BindVariables, execCase.keyRangeQuery.TabletType, testExecuteOptions) - require.NoError(t, err) - err = tx.Rollback(ctx) - require.NoError(t, err) - - // ExecuteEntityIds - tx, err = conn.Begin(ctx) - require.NoError(t, err) - _, err = tx.ExecuteEntityIds(ctx, execCase.entityIdsQuery.SQL, execCase.entityIdsQuery.Keyspace, execCase.entityIdsQuery.EntityColumnName, execCase.entityIdsQuery.EntityKeyspaceIDs, execCase.entityIdsQuery.BindVariables, execCase.entityIdsQuery.TabletType, testExecuteOptions) - require.NoError(t, err) - err = tx.Rollback(ctx) - require.NoError(t, err) - - // ExecuteBatchShards - tx, err = conn.Begin(ctx) - require.NoError(t, err) - _, err = tx.ExecuteBatchShards(ctx, execCase.batchQueryShard.Queries, execCase.batchQueryShard.TabletType, testExecuteOptions) - require.NoError(t, err) - err = tx.Rollback(ctx) - require.NoError(t, err) - - // ExecuteBatchKeyspaceIds - tx, err = conn.Begin(ctx) - require.NoError(t, err) - _, err = tx.ExecuteBatchKeyspaceIds(ctx, execCase.keyspaceIDBatchQuery.Queries, execCase.keyspaceIDBatchQuery.TabletType, testExecuteOptions) - require.NoError(t, err) - err = tx.Rollback(ctx) - require.NoError(t, err) -} - -func testResolveTransaction(t *testing.T, conn *vtgateconn.VTGateConn) { - if err := conn.ResolveTransaction(newContext(), dtid2); err != nil { - t.Error(err) - } -} - -func testBeginError(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - _, err := conn.Begin(ctx) - verifyError(t, err, "Begin") -} - -func testCommitError(t *testing.T, conn *vtgateconn.VTGateConn, fake *fakeVTGateService) { - ctx := newContext() - - fake.forceBeginSuccess = true - tx, err := conn.Begin(ctx) - fake.forceBeginSuccess = false - - require.NoError(t, err) - err = tx.Commit(ctx) - verifyError(t, err, "Commit") } -func testRollbackError(t *testing.T, conn *vtgateconn.VTGateConn, fake *fakeVTGateService) { - ctx := newContext() - - fake.forceBeginSuccess = true - tx, err := conn.Begin(ctx) - fake.forceBeginSuccess = false - - require.NoError(t, err) - err = tx.Rollback(ctx) - verifyError(t, err, "Rollback") -} - -func testResolveTransactionError(t *testing.T, conn *vtgateconn.VTGateConn, fake *fakeVTGateService) { - err := conn.ResolveTransaction(newContext(), "") - verifyError(t, err, "ResolveTransaction") -} - -func testBeginPanic(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - _, err := conn.Begin(ctx) - expectPanic(t, err) -} - -func testCommitPanic(t *testing.T, conn *vtgateconn.VTGateConn, fake *fakeVTGateService) { - ctx := newContext() - - fake.forceBeginSuccess = true - tx, err := conn.Begin(ctx) - fake.forceBeginSuccess = false - - require.NoError(t, err) - err = tx.Commit(ctx) - expectPanic(t, err) -} - -func testRollbackPanic(t *testing.T, conn *vtgateconn.VTGateConn, fake *fakeVTGateService) { - ctx := newContext() - - fake.forceBeginSuccess = true - tx, err := conn.Begin(ctx) - fake.forceBeginSuccess = false - - require.NoError(t, err) - err = tx.Rollback(ctx) - expectPanic(t, err) -} - -func testResolveTransactionPanic(t *testing.T, conn *vtgateconn.VTGateConn, fake *fakeVTGateService) { - err := conn.ResolveTransaction(newContext(), "") - expectPanic(t, err) -} - -func testTxFail(t *testing.T, conn *vtgateconn.VTGateConn) { +func testStreamExecuteError(t *testing.T, session *vtgateconn.VTGateSession, fake *fakeVTGateService) { ctx := newContext() - tx, err := conn.Begin(ctx) - require.NoError(t, err) - err = tx.Commit(ctx) - want := "commit: session mismatch" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("Commit: %v, want %v", err, want) - } - - _, err = tx.ExecuteShards(ctx, "", "", nil, nil, topodatapb.TabletType_REPLICA, testExecuteOptions) - want = "executeShards: not in transaction" - if err == nil || err.Error() != want { - t.Errorf("ExecuteShards: %v, want %v", err, want) - } - - _, err = tx.ExecuteKeyspaceIds(ctx, "", "", nil, nil, topodatapb.TabletType_REPLICA, testExecuteOptions) - want = "executeKeyspaceIds: not in transaction" - if err == nil || err.Error() != want { - t.Errorf("ExecuteShards: %v, want %v", err, want) - } - - _, err = tx.ExecuteKeyRanges(ctx, "", "", nil, nil, topodatapb.TabletType_REPLICA, testExecuteOptions) - want = "executeKeyRanges: not in transaction" - if err == nil || err.Error() != want { - t.Errorf("ExecuteShards: %v, want %v", err, want) - } - - _, err = tx.ExecuteEntityIds(ctx, "", "", "", nil, nil, topodatapb.TabletType_REPLICA, testExecuteOptions) - want = "executeEntityIds: not in transaction" - if err == nil || err.Error() != want { - t.Errorf("ExecuteShards: %v, want %v", err, want) - } - - _, err = tx.ExecuteBatchShards(ctx, nil, topodatapb.TabletType_REPLICA, testExecuteOptions) - want = "executeBatchShards: not in transaction" - if err == nil || err.Error() != want { - t.Errorf("ExecuteShards: %v, want %v", err, want) - } - - _, err = tx.ExecuteBatchKeyspaceIds(ctx, nil, topodatapb.TabletType_REPLICA, testExecuteOptions) - want = "executeBatchKeyspaceIds: not in transaction" - if err == nil || err.Error() != want { - t.Errorf("ExecuteShards: %v, want %v", err, want) - } - - err = tx.Commit(ctx) - want = "commit: not in transaction" - if err == nil || err.Error() != want { - t.Errorf("Commit: %v, want %v", err, want) + execCase := execMap["request1"] + stream, err := session.StreamExecute(ctx, execCase.execQuery.SQL, execCase.execQuery.BindVariables) + if err != nil { + t.Fatalf("StreamExecute failed: %v", err) } - - err = tx.Rollback(ctx) - require.NoError(t, err) - - tx, err = conn.Begin(ctx) - require.NoError(t, err) - err = tx.Rollback(ctx) - want = "rollback: session mismatch" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("Rollback: %v, want %v", err, want) + qr, err := stream.Recv() + if err != nil { + t.Fatalf("StreamExecute failed: cannot read result1: %v", err) } -} - -func testMessageStream(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - err := conn.MessageStream(ctx, "", "", nil, messageName, func(qr *sqltypes.Result) error { - if !qr.Equal(messageStreamResult) { - t.Errorf("reply: %v, want %v", qr, messageStreamResult) - } - return nil - }) - require.NoError(t, err) -} - -func testMessageStreamError(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - err := conn.MessageStream(ctx, "", "", nil, messageName, func(qr *sqltypes.Result) error { - return nil - }) - verifyError(t, err, "MessageStream") -} - -func testMessageStreamPanic(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - err := conn.MessageStream(ctx, "", "", nil, messageName, func(qr *sqltypes.Result) error { - return nil - }) - expectPanic(t, err) -} -func testMessageAck(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - got, err := conn.MessageAck(ctx, "", messageName, messageids) - if got != messageAckRowsAffected { - t.Errorf("MessageAck: %d, want %d", got, messageAckRowsAffected) + if !qr.Equal(&streamResultFields) { + t.Errorf("Unexpected result from StreamExecute: got %#v want %#v", qr, &streamResultFields) } - require.NoError(t, err) -} - -func testMessageAckError(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - _, err := conn.MessageAck(ctx, "", messageName, messageids) - verifyError(t, err, "MessageAck") -} - -func testMessageAckPanic(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - _, err := conn.MessageAck(ctx, "", messageName, messageids) - expectPanic(t, err) -} - -func testMessageAckKeyspaceIds(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - got, err := conn.MessageAckKeyspaceIds(ctx, "", messageName, testIDKeyspaceIDs) - if got != messageAckRowsAffected { - t.Errorf("MessageAckKeyspaceIds: %d, want %d", got, messageAckRowsAffected) + // signal to the server that the first result has been received + close(fake.errorWait) + // After 1 result, we expect to get an error (no more results). + _, err = stream.Recv() + if err == nil { + t.Fatalf("StreamExecute channel wasn't closed") } - require.NoError(t, err) -} - -func testMessageAckKeyspaceIdsError(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - _, err := conn.MessageAckKeyspaceIds(ctx, "", messageName, testIDKeyspaceIDs) - verifyError(t, err, "MessageAckKeyspaceIds") -} - -func testMessageAckKeyspaceIdsPanic(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - _, err := conn.MessageAckKeyspaceIds(ctx, "", messageName, testIDKeyspaceIDs) - expectPanic(t, err) + verifyError(t, err, "StreamExecute") } -func testGetSrvKeyspace(t *testing.T, conn *vtgateconn.VTGateConn) { +func testStreamExecutePanic(t *testing.T, session *vtgateconn.VTGateSession) { ctx := newContext() - sk, err := conn.GetSrvKeyspace(ctx, getSrvKeyspaceKeyspace) + execCase := execMap["request1"] + stream, err := session.StreamExecute(ctx, execCase.execQuery.SQL, execCase.execQuery.BindVariables) if err != nil { - t.Fatalf("GetSrvKeyspace failed: %v", err) + t.Fatal(err) } - if !proto.Equal(sk, getSrvKeyspaceResult) { - t.Errorf("GetSrvKeyspace returned wrong result: got %+v wanted %+v", sk, getSrvKeyspaceResult) + _, err = stream.Recv() + if err == nil { + t.Fatalf("Received packets instead of panic?") } -} - -func testGetSrvKeyspaceError(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - _, err := conn.GetSrvKeyspace(ctx, getSrvKeyspaceKeyspace) - verifyErrorString(t, err, "GetSrvKeyspace") -} - -func testGetSrvKeyspacePanic(t *testing.T, conn *vtgateconn.VTGateConn) { - ctx := newContext() - _, err := conn.GetSrvKeyspace(ctx, getSrvKeyspaceKeyspace) expectPanic(t, err) } @@ -1865,16 +469,10 @@ var testExecuteOptions = &querypb.ExecuteOptions{ } var execMap = map[string]struct { - execQuery *queryExecute - shardQuery *queryExecuteShards - keyspaceIDQuery *queryExecuteKeyspaceIds - keyRangeQuery *queryExecuteKeyRanges - entityIdsQuery *queryExecuteEntityIds - batchQueryShard *queryExecuteBatchShards - keyspaceIDBatchQuery *queryExecuteBatchKeyspaceIds - result *sqltypes.Result - outSession *vtgatepb.Session - err error + execQuery *queryExecute + result *sqltypes.Result + outSession *vtgatepb.Session + err error }{ "request1": { execQuery: &queryExecute{ @@ -1888,98 +486,7 @@ var execMap = map[string]struct { Autocommit: true, }, }, - shardQuery: &queryExecuteShards{ - SQL: "request1", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(0), - }, - Keyspace: "ks", - Shards: []string{"1", "2"}, - TabletType: topodatapb.TabletType_RDONLY, - Session: nil, - }, - keyspaceIDQuery: &queryExecuteKeyspaceIds{ - SQL: "request1", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(0), - }, - Keyspace: "ks", - KeyspaceIds: [][]byte{ - {0x61}, - }, - TabletType: topodatapb.TabletType_RDONLY, - Session: nil, - }, - keyRangeQuery: &queryExecuteKeyRanges{ - SQL: "request1", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(0), - }, - Keyspace: "ks", - KeyRanges: []*topodatapb.KeyRange{ - { - Start: []byte{0x72}, - End: []byte{0x90}, - }, - }, - TabletType: topodatapb.TabletType_RDONLY, - Session: nil, - }, - entityIdsQuery: &queryExecuteEntityIds{ - SQL: "request1", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(0), - }, - Keyspace: "ks", - EntityColumnName: "column", - EntityKeyspaceIDs: []*vtgatepb.ExecuteEntityIdsRequest_EntityId{ - { - Type: sqltypes.VarBinary, - Value: []byte{105, 100, 49}, - KeyspaceId: []byte{0x6B}, - }, - }, - TabletType: topodatapb.TabletType_RDONLY, - Session: nil, - }, - batchQueryShard: &queryExecuteBatchShards{ - Queries: []*vtgatepb.BoundShardQuery{ - { - Query: &querypb.BoundQuery{ - Sql: "request1", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(11143), - }, - }, - Keyspace: "ks", - Shards: []string{"-80", "80-"}, - }, - }, - TabletType: topodatapb.TabletType_RDONLY, - AsTransaction: true, - Session: nil, - }, - keyspaceIDBatchQuery: &queryExecuteBatchKeyspaceIds{ - Queries: []*vtgatepb.BoundKeyspaceIdQuery{ - { - Query: &querypb.BoundQuery{ - Sql: "request1", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(11143), - }, - }, - Keyspace: "ks", - KeyspaceIds: [][]byte{ - {'k', 'i', '1'}, - }, - }, - }, - TabletType: topodatapb.TabletType_RDONLY, - AsTransaction: true, - Session: nil, - }, - result: &result1, - outSession: nil, + result: &result1, }, "errorRequst": { execQuery: &queryExecute{ @@ -1992,197 +499,6 @@ var execMap = map[string]struct { Options: testExecuteOptions, }, }, - shardQuery: &queryExecuteShards{ - SQL: "errorRequst", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(0), - }, - TabletType: topodatapb.TabletType_RDONLY, - Keyspace: "", - Shards: []string{"s1", "s2"}, - Session: nil, - }, - keyspaceIDQuery: &queryExecuteKeyspaceIds{ - SQL: "errorRequst", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(0), - }, - Keyspace: "ks", - KeyspaceIds: [][]byte{ - {0x61}, - }, - TabletType: topodatapb.TabletType_RDONLY, - Session: nil, - }, - keyRangeQuery: &queryExecuteKeyRanges{ - SQL: "errorRequst", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(0), - }, - Keyspace: "ks", - KeyRanges: []*topodatapb.KeyRange{ - { - Start: []byte{0x73}, - End: []byte{0x99}, - }, - }, - TabletType: topodatapb.TabletType_RDONLY, - Session: nil, - }, - entityIdsQuery: &queryExecuteEntityIds{ - SQL: "errorRequst", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(0), - }, - Keyspace: "ks", - EntityColumnName: "column", - EntityKeyspaceIDs: []*vtgatepb.ExecuteEntityIdsRequest_EntityId{ - { - Type: sqltypes.VarBinary, - Value: []byte{105, 100, 49}, - KeyspaceId: []byte{0x6B}, - }, - }, - TabletType: topodatapb.TabletType_RDONLY, - Session: nil, - }, - batchQueryShard: &queryExecuteBatchShards{ - Queries: []*vtgatepb.BoundShardQuery{ - { - Query: &querypb.BoundQuery{ - Sql: "errorRequst", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(11143), - }, - }, - Keyspace: "ks", - Shards: []string{"-80", "80-"}, - }, - }, - TabletType: topodatapb.TabletType_RDONLY, - AsTransaction: false, - Session: nil, - }, - keyspaceIDBatchQuery: &queryExecuteBatchKeyspaceIds{ - Queries: []*vtgatepb.BoundKeyspaceIdQuery{ - { - Query: &querypb.BoundQuery{ - Sql: "errorRequst", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(11143), - }, - }, - Keyspace: "ks", - KeyspaceIds: [][]byte{ - {'k', 'i', '1'}, - }, - }, - }, - TabletType: topodatapb.TabletType_RDONLY, - AsTransaction: false, - Session: nil, - }, - result: nil, - outSession: nil, - }, - "txRequest": { - execQuery: &queryExecute{ - SQL: "txRequest", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(0), - }, - Session: session1, - }, - shardQuery: &queryExecuteShards{ - SQL: "txRequest", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(0), - }, - TabletType: topodatapb.TabletType_MASTER, - Keyspace: "", - Shards: []string{"s1", "s2"}, - Session: session1, - }, - keyspaceIDQuery: &queryExecuteKeyspaceIds{ - SQL: "txRequest", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(0), - }, - Keyspace: "ks", - KeyspaceIds: [][]byte{ - {0x61}, - }, - TabletType: topodatapb.TabletType_RDONLY, - Session: session1, - }, - keyRangeQuery: &queryExecuteKeyRanges{ - SQL: "txRequest", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(0), - }, - Keyspace: "ks", - KeyRanges: []*topodatapb.KeyRange{ - { - Start: []byte{0x23}, - End: []byte{0x66}, - }, - }, - TabletType: topodatapb.TabletType_RDONLY, - Session: session1, - }, - entityIdsQuery: &queryExecuteEntityIds{ - SQL: "txRequest", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(0), - }, - Keyspace: "ks", - EntityColumnName: "column", - EntityKeyspaceIDs: []*vtgatepb.ExecuteEntityIdsRequest_EntityId{ - { - Type: sqltypes.Int64, - Value: []byte("-12345"), - KeyspaceId: []byte{0x6B}, - }, - }, - TabletType: topodatapb.TabletType_RDONLY, - Session: session1, - }, - batchQueryShard: &queryExecuteBatchShards{ - Queries: []*vtgatepb.BoundShardQuery{ - { - Query: &querypb.BoundQuery{ - Sql: "txRequest", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(11143), - }, - }, - Keyspace: "ks", - Shards: []string{"-80", "80-"}, - }, - }, - TabletType: topodatapb.TabletType_RDONLY, - Session: session1, - }, - keyspaceIDBatchQuery: &queryExecuteBatchKeyspaceIds{ - Queries: []*vtgatepb.BoundKeyspaceIdQuery{ - { - Query: &querypb.BoundQuery{ - Sql: "txRequest", - BindVariables: map[string]*querypb.BindVariable{ - "bind1": sqltypes.Int64BindVariable(11143), - }, - }, - Keyspace: "ks", - KeyspaceIds: [][]byte{ - {'k', 'i', '1'}, - }, - }, - }, - TabletType: topodatapb.TabletType_RDONLY, - Session: session1, - }, - result: nil, - outSession: session2, }, } @@ -2217,78 +533,4 @@ var streamResultFields = sqltypes.Result{ Rows: [][]sqltypes.Value{}, } -var session1 = &vtgatepb.Session{ - InTransaction: true, - TargetString: "connection_ks@rdonly", - Options: testExecuteOptions, -} - -var session2 = &vtgatepb.Session{ - InTransaction: true, - ShardSessions: []*vtgatepb.Session_ShardSession{ - { - Target: &querypb.Target{ - Keyspace: "ks", - Shard: "1", - TabletType: topodatapb.TabletType_MASTER, - }, - TransactionId: 1, - }, - }, - TargetString: "connection_ks@rdonly", - Options: testExecuteOptions, -} - var dtid2 = "aa" - -var getSrvKeyspaceKeyspace = "test_keyspace" - -var getSrvKeyspaceResult = &topodatapb.SrvKeyspace{ - Partitions: []*topodatapb.SrvKeyspace_KeyspacePartition{ - { - ServedType: topodatapb.TabletType_REPLICA, - ShardReferences: []*topodatapb.ShardReference{ - { - Name: "shard0", - KeyRange: &topodatapb.KeyRange{ - Start: []byte{'s'}, - End: []byte{'e'}, - }, - }, - }, - }, - }, - ShardingColumnName: "sharding_column_name", - ShardingColumnType: topodatapb.KeyspaceIdType_UINT64, - ServedFrom: []*topodatapb.SrvKeyspace_ServedFrom{ - { - TabletType: topodatapb.TabletType_MASTER, - Keyspace: "other_keyspace", - }, - }, -} - -var messageName = "vitess_message" -var messageStreamResult = &sqltypes.Result{ - Fields: []*querypb.Field{{ - Name: "id", - Type: sqltypes.VarBinary, - }, { - Name: "message", - Type: sqltypes.VarBinary, - }}, - Rows: [][]sqltypes.Value{{ - sqltypes.NewVarBinary("2"), - sqltypes.NewVarBinary("3"), - }}, -} -var messageids = []*querypb.Value{ - sqltypes.ValueToProto(sqltypes.NewVarBinary("1")), - sqltypes.ValueToProto(sqltypes.NewVarBinary("3")), -} -var messageAckRowsAffected = int64(1) - -var testIDKeyspaceIDs = []*vtgatepb.IdKeyspaceId{{ - Id: sqltypes.ValueToProto(sqltypes.NewVarBinary("1")), - KeyspaceId: []byte{0x6B}, -}} diff --git a/go/vt/vtgate/grpcvtgateservice/server.go b/go/vt/vtgate/grpcvtgateservice/server.go index cbda8f0c5f1..16f14635fc6 100644 --- a/go/vt/vtgate/grpcvtgateservice/server.go +++ b/go/vt/vtgate/grpcvtgateservice/server.go @@ -177,224 +177,6 @@ func (vtg *VTGate) StreamExecute(request *vtgatepb.StreamExecuteRequest, stream return vterrors.ToGRPC(vtgErr) } -// ExecuteShards is the RPC version of vtgateservice.VTGateService method -func (vtg *VTGate) ExecuteShards(ctx context.Context, request *vtgatepb.ExecuteShardsRequest) (response *vtgatepb.ExecuteShardsResponse, err error) { - defer vtg.server.HandlePanic(&err) - ctx = withCallerIDContext(ctx, request.CallerId) - result, err := vtg.server.ExecuteShards(ctx, - request.Query.Sql, - request.Query.BindVariables, - request.Keyspace, - request.Shards, - request.TabletType, - request.Session, - request.NotInTransaction, - request.Options) - return &vtgatepb.ExecuteShardsResponse{ - Result: sqltypes.ResultToProto3(result), - Session: request.Session, - Error: vterrors.ToVTRPC(err), - }, nil -} - -// ExecuteKeyspaceIds is the RPC version of vtgateservice.VTGateService method -func (vtg *VTGate) ExecuteKeyspaceIds(ctx context.Context, request *vtgatepb.ExecuteKeyspaceIdsRequest) (response *vtgatepb.ExecuteKeyspaceIdsResponse, err error) { - defer vtg.server.HandlePanic(&err) - ctx = withCallerIDContext(ctx, request.CallerId) - result, err := vtg.server.ExecuteKeyspaceIds(ctx, - request.Query.Sql, - request.Query.BindVariables, - request.Keyspace, - request.KeyspaceIds, - request.TabletType, - request.Session, - request.NotInTransaction, - request.Options) - return &vtgatepb.ExecuteKeyspaceIdsResponse{ - Result: sqltypes.ResultToProto3(result), - Session: request.Session, - Error: vterrors.ToVTRPC(err), - }, nil -} - -// ExecuteKeyRanges is the RPC version of vtgateservice.VTGateService method -func (vtg *VTGate) ExecuteKeyRanges(ctx context.Context, request *vtgatepb.ExecuteKeyRangesRequest) (response *vtgatepb.ExecuteKeyRangesResponse, err error) { - defer vtg.server.HandlePanic(&err) - ctx = withCallerIDContext(ctx, request.CallerId) - result, err := vtg.server.ExecuteKeyRanges(ctx, - request.Query.Sql, - request.Query.BindVariables, - request.Keyspace, - request.KeyRanges, - request.TabletType, - request.Session, - request.NotInTransaction, - request.Options) - return &vtgatepb.ExecuteKeyRangesResponse{ - Result: sqltypes.ResultToProto3(result), - Session: request.Session, - Error: vterrors.ToVTRPC(err), - }, nil -} - -// ExecuteEntityIds is the RPC version of vtgateservice.VTGateService method -func (vtg *VTGate) ExecuteEntityIds(ctx context.Context, request *vtgatepb.ExecuteEntityIdsRequest) (response *vtgatepb.ExecuteEntityIdsResponse, err error) { - defer vtg.server.HandlePanic(&err) - ctx = withCallerIDContext(ctx, request.CallerId) - result, err := vtg.server.ExecuteEntityIds(ctx, - request.Query.Sql, - request.Query.BindVariables, - request.Keyspace, - request.EntityColumnName, - request.EntityKeyspaceIds, - request.TabletType, - request.Session, - request.NotInTransaction, - request.Options) - return &vtgatepb.ExecuteEntityIdsResponse{ - Result: sqltypes.ResultToProto3(result), - Session: request.Session, - Error: vterrors.ToVTRPC(err), - }, nil -} - -// ExecuteBatchShards is the RPC version of vtgateservice.VTGateService method -func (vtg *VTGate) ExecuteBatchShards(ctx context.Context, request *vtgatepb.ExecuteBatchShardsRequest) (response *vtgatepb.ExecuteBatchShardsResponse, err error) { - defer vtg.server.HandlePanic(&err) - ctx = withCallerIDContext(ctx, request.CallerId) - result, err := vtg.server.ExecuteBatchShards(ctx, - request.Queries, - request.TabletType, - request.AsTransaction, - request.Session, - request.Options) - return &vtgatepb.ExecuteBatchShardsResponse{ - Results: sqltypes.ResultsToProto3(result), - Session: request.Session, - Error: vterrors.ToVTRPC(err), - }, nil -} - -// ExecuteBatchKeyspaceIds is the RPC version of -// vtgateservice.VTGateService method -func (vtg *VTGate) ExecuteBatchKeyspaceIds(ctx context.Context, request *vtgatepb.ExecuteBatchKeyspaceIdsRequest) (response *vtgatepb.ExecuteBatchKeyspaceIdsResponse, err error) { - defer vtg.server.HandlePanic(&err) - ctx = withCallerIDContext(ctx, request.CallerId) - result, err := vtg.server.ExecuteBatchKeyspaceIds(ctx, - request.Queries, - request.TabletType, - request.AsTransaction, - request.Session, - request.Options) - return &vtgatepb.ExecuteBatchKeyspaceIdsResponse{ - Results: sqltypes.ResultsToProto3(result), - Session: request.Session, - Error: vterrors.ToVTRPC(err), - }, nil -} - -// StreamExecuteShards is the RPC version of vtgateservice.VTGateService method -func (vtg *VTGate) StreamExecuteShards(request *vtgatepb.StreamExecuteShardsRequest, stream vtgateservicepb.Vitess_StreamExecuteShardsServer) (err error) { - defer vtg.server.HandlePanic(&err) - ctx := withCallerIDContext(stream.Context(), request.CallerId) - vtgErr := vtg.server.StreamExecuteShards(ctx, - request.Query.Sql, - request.Query.BindVariables, - request.Keyspace, - request.Shards, - request.TabletType, - request.Options, - func(value *sqltypes.Result) error { - // Send is not safe to call concurrently, but vtgate - // guarantees that it's not. - return stream.Send(&vtgatepb.StreamExecuteShardsResponse{ - Result: sqltypes.ResultToProto3(value), - }) - }) - return vterrors.ToGRPC(vtgErr) -} - -// StreamExecuteKeyspaceIds is the RPC version of -// vtgateservice.VTGateService method -func (vtg *VTGate) StreamExecuteKeyspaceIds(request *vtgatepb.StreamExecuteKeyspaceIdsRequest, stream vtgateservicepb.Vitess_StreamExecuteKeyspaceIdsServer) (err error) { - defer vtg.server.HandlePanic(&err) - ctx := withCallerIDContext(stream.Context(), request.CallerId) - vtgErr := vtg.server.StreamExecuteKeyspaceIds(ctx, - request.Query.Sql, - request.Query.BindVariables, - request.Keyspace, - request.KeyspaceIds, - request.TabletType, - request.Options, - func(value *sqltypes.Result) error { - // Send is not safe to call concurrently, but vtgate - // guarantees that it's not. - return stream.Send(&vtgatepb.StreamExecuteKeyspaceIdsResponse{ - Result: sqltypes.ResultToProto3(value), - }) - }) - return vterrors.ToGRPC(vtgErr) -} - -// StreamExecuteKeyRanges is the RPC version of -// vtgateservice.VTGateService method -func (vtg *VTGate) StreamExecuteKeyRanges(request *vtgatepb.StreamExecuteKeyRangesRequest, stream vtgateservicepb.Vitess_StreamExecuteKeyRangesServer) (err error) { - defer vtg.server.HandlePanic(&err) - ctx := withCallerIDContext(stream.Context(), request.CallerId) - vtgErr := vtg.server.StreamExecuteKeyRanges(ctx, - request.Query.Sql, - request.Query.BindVariables, - request.Keyspace, - request.KeyRanges, - request.TabletType, - request.Options, - func(value *sqltypes.Result) error { - // Send is not safe to call concurrently, but vtgate - // guarantees that it's not. - return stream.Send(&vtgatepb.StreamExecuteKeyRangesResponse{ - Result: sqltypes.ResultToProto3(value), - }) - }) - return vterrors.ToGRPC(vtgErr) -} - -// Begin is the RPC version of vtgateservice.VTGateService method -func (vtg *VTGate) Begin(ctx context.Context, request *vtgatepb.BeginRequest) (response *vtgatepb.BeginResponse, err error) { - defer vtg.server.HandlePanic(&err) - ctx = withCallerIDContext(ctx, request.CallerId) - session, vtgErr := vtg.server.Begin(ctx, request.SingleDb) - if vtgErr == nil { - return &vtgatepb.BeginResponse{ - Session: session, - }, nil - } - return nil, vterrors.ToGRPC(vtgErr) -} - -// Commit is the RPC version of vtgateservice.VTGateService method -func (vtg *VTGate) Commit(ctx context.Context, request *vtgatepb.CommitRequest) (response *vtgatepb.CommitResponse, err error) { - defer vtg.server.HandlePanic(&err) - ctx = withCallerIDContext(ctx, request.CallerId) - vtgErr := vtg.server.Commit(ctx, request.Atomic, request.Session) - response = &vtgatepb.CommitResponse{} - if vtgErr == nil { - return response, nil - } - return nil, vterrors.ToGRPC(vtgErr) -} - -// Rollback is the RPC version of vtgateservice.VTGateService method -func (vtg *VTGate) Rollback(ctx context.Context, request *vtgatepb.RollbackRequest) (response *vtgatepb.RollbackResponse, err error) { - defer vtg.server.HandlePanic(&err) - ctx = withCallerIDContext(ctx, request.CallerId) - vtgErr := vtg.server.Rollback(ctx, request.Session) - response = &vtgatepb.RollbackResponse{} - if vtgErr == nil { - return response, nil - } - return nil, vterrors.ToGRPC(vtgErr) -} - // ResolveTransaction is the RPC version of vtgateservice.VTGateService method func (vtg *VTGate) ResolveTransaction(ctx context.Context, request *vtgatepb.ResolveTransactionRequest) (response *vtgatepb.ResolveTransactionResponse, err error) { defer vtg.server.HandlePanic(&err) @@ -407,63 +189,6 @@ func (vtg *VTGate) ResolveTransaction(ctx context.Context, request *vtgatepb.Res return nil, vterrors.ToGRPC(vtgErr) } -// MessageStream is the RPC version of vtgateservice.VTGateService method -func (vtg *VTGate) MessageStream(request *vtgatepb.MessageStreamRequest, stream vtgateservicepb.Vitess_MessageStreamServer) (err error) { - defer vtg.server.HandlePanic(&err) - ctx := withCallerIDContext(stream.Context(), request.CallerId) - vtgErr := vtg.server.MessageStream(ctx, request.Keyspace, request.Shard, request.KeyRange, request.Name, func(qr *sqltypes.Result) error { - // Send is not safe to call concurrently, but vtgate - // guarantees that it's not. - return stream.Send(&querypb.MessageStreamResponse{ - Result: sqltypes.ResultToProto3(qr), - }) - }) - return vterrors.ToGRPC(vtgErr) -} - -// MessageAck is the RPC version of vtgateservice.VTGateService method -func (vtg *VTGate) MessageAck(ctx context.Context, request *vtgatepb.MessageAckRequest) (response *querypb.MessageAckResponse, err error) { - defer vtg.server.HandlePanic(&err) - ctx = withCallerIDContext(ctx, request.CallerId) - count, vtgErr := vtg.server.MessageAck(ctx, request.Keyspace, request.Name, request.Ids) - if vtgErr != nil { - return nil, vterrors.ToGRPC(vtgErr) - } - return &querypb.MessageAckResponse{ - Result: &querypb.QueryResult{ - RowsAffected: uint64(count), - }, - }, nil -} - -// MessageAckKeyspaceIds routes Message Acks using the associated -// keyspace ids. -func (vtg *VTGate) MessageAckKeyspaceIds(ctx context.Context, request *vtgatepb.MessageAckKeyspaceIdsRequest) (response *querypb.MessageAckResponse, err error) { - defer vtg.server.HandlePanic(&err) - ctx = withCallerIDContext(ctx, request.CallerId) - count, vtgErr := vtg.server.MessageAckKeyspaceIds(ctx, request.Keyspace, request.Name, request.IdKeyspaceIds) - if vtgErr != nil { - return nil, vterrors.ToGRPC(vtgErr) - } - return &querypb.MessageAckResponse{ - Result: &querypb.QueryResult{ - RowsAffected: uint64(count), - }, - }, nil -} - -// GetSrvKeyspace is the RPC version of vtgateservice.VTGateService method -func (vtg *VTGate) GetSrvKeyspace(ctx context.Context, request *vtgatepb.GetSrvKeyspaceRequest) (response *vtgatepb.GetSrvKeyspaceResponse, err error) { - defer vtg.server.HandlePanic(&err) - sk, vtgErr := vtg.server.GetSrvKeyspace(ctx, request.Keyspace) - if vtgErr != nil { - return nil, vterrors.ToGRPC(vtgErr) - } - return &vtgatepb.GetSrvKeyspaceResponse{ - SrvKeyspace: sk, - }, nil -} - // VStream is the RPC version of vtgateservice.VTGateService method func (vtg *VTGate) VStream(request *vtgatepb.VStreamRequest, stream vtgateservicepb.Vitess_VStreamServer) (err error) { defer vtg.server.HandlePanic(&err) diff --git a/go/vt/vtgate/vtgate.go b/go/vt/vtgate/vtgate.go index 620f486230a..e518768768b 100644 --- a/go/vt/vtgate/vtgate.go +++ b/go/vt/vtgate/vtgate.go @@ -35,7 +35,6 @@ import ( "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/servenv" - "vitess.io/vitess/go/vt/sqlannotation" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/topo/topoproto" @@ -115,18 +114,8 @@ type VTGate struct { rowsReturned *stats.CountersWithMultiLabels // the throttled loggers for all errors, one per API entry - logExecute *logutil.ThrottledLogger - logStreamExecute *logutil.ThrottledLogger - logExecuteShards *logutil.ThrottledLogger - logExecuteKeyspaceIds *logutil.ThrottledLogger - logExecuteKeyRanges *logutil.ThrottledLogger - logExecuteEntityIds *logutil.ThrottledLogger - logExecuteBatchShards *logutil.ThrottledLogger - logExecuteBatchKeyspaceIds *logutil.ThrottledLogger - logStreamExecuteKeyspaceIds *logutil.ThrottledLogger - logStreamExecuteKeyRanges *logutil.ThrottledLogger - logStreamExecuteShards *logutil.ThrottledLogger - logMessageStream *logutil.ThrottledLogger + logExecute *logutil.ThrottledLogger + logStreamExecute *logutil.ThrottledLogger } // RegisterVTGate defines the type of registration mechanism. @@ -187,18 +176,8 @@ func Init(ctx context.Context, hc discovery.HealthCheck, serv srvtopo.Server, ce "Rows returned through the VTgate API", []string{"Operation", "Keyspace", "DbType"}), - logExecute: logutil.NewThrottledLogger("Execute", 5*time.Second), - logStreamExecute: logutil.NewThrottledLogger("StreamExecute", 5*time.Second), - logExecuteShards: logutil.NewThrottledLogger("ExecuteShards", 5*time.Second), - logExecuteKeyspaceIds: logutil.NewThrottledLogger("ExecuteKeyspaceIds", 5*time.Second), - logExecuteKeyRanges: logutil.NewThrottledLogger("ExecuteKeyRanges", 5*time.Second), - logExecuteEntityIds: logutil.NewThrottledLogger("ExecuteEntityIds", 5*time.Second), - logExecuteBatchShards: logutil.NewThrottledLogger("ExecuteBatchShards", 5*time.Second), - logExecuteBatchKeyspaceIds: logutil.NewThrottledLogger("ExecuteBatchKeyspaceIds", 5*time.Second), - logStreamExecuteKeyspaceIds: logutil.NewThrottledLogger("StreamExecuteKeyspaceIds", 5*time.Second), - logStreamExecuteKeyRanges: logutil.NewThrottledLogger("StreamExecuteKeyRanges", 5*time.Second), - logStreamExecuteShards: logutil.NewThrottledLogger("StreamExecuteShards", 5*time.Second), - logMessageStream: logutil.NewThrottledLogger("MessageStream", 5*time.Second), + logExecute: logutil.NewThrottledLogger("Execute", 5*time.Second), + logStreamExecute: logutil.NewThrottledLogger("StreamExecute", 5*time.Second), } errorCounts = stats.NewCountersWithMultiLabels("VtgateApiErrorCounts", "Vtgate API error counts per error type", []string{"Operation", "Keyspace", "DbType", "Code"}) @@ -371,446 +350,6 @@ handleError: return nil } -// ExecuteShards executes a non-streaming query on the specified shards. -// This is a legacy function. -func (vtg *VTGate) ExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - startTime := time.Now() - ltt := topoproto.TabletTypeLString(tabletType) - statsKey := []string{"ExecuteShards", keyspace, ltt} - defer vtg.timings.Record(statsKey, startTime) - - var qr *sqltypes.Result - var err error - - if bvErr := sqltypes.ValidateBindVariables(bindVariables); bvErr != nil { - err = vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "%v", bvErr) - goto handleError - } - - sql = sqlannotation.AnnotateIfDML(sql, nil) - - qr, err = vtg.resolver.Execute( - ctx, - sql, - bindVariables, - keyspace, - tabletType, - key.DestinationShards(shards), - NewSafeSession(session), - notInTransaction, - options, - nil, - false, /* autocommit */ - ) - if err == nil { - vtg.rowsReturned.Add(statsKey, int64(len(qr.Rows))) - return qr, nil - } - -handleError: - query := map[string]interface{}{ - "Sql": sql, - "BindVariables": bindVariables, - "Keyspace": keyspace, - "Shards": shards, - "TabletType": ltt, - "Session": session, - "NotInTransaction": notInTransaction, - "Options": options, - } - err = recordAndAnnotateError(err, statsKey, query, vtg.logExecuteShards) - return nil, err -} - -// ExecuteKeyspaceIds executes a non-streaming query based on the specified keyspace ids. This is a legacy function. -func (vtg *VTGate) ExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - startTime := time.Now() - ltt := topoproto.TabletTypeLString(tabletType) - statsKey := []string{"ExecuteKeyspaceIds", keyspace, ltt} - defer vtg.timings.Record(statsKey, startTime) - - var qr *sqltypes.Result - var err error - - if bvErr := sqltypes.ValidateBindVariables(bindVariables); bvErr != nil { - err = vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "%v", bvErr) - goto handleError - } - - sql = sqlannotation.AnnotateIfDML(sql, keyspaceIds) - if sqlparser.IsDML(sql) && len(keyspaceIds) > 1 { - err = vterrors.New(vtrpcpb.Code_INVALID_ARGUMENT, "DML should not span multiple keyspace_ids") - goto handleError - } - - qr, err = vtg.resolver.Execute(ctx, sql, bindVariables, keyspace, tabletType, key.DestinationKeyspaceIDs(keyspaceIds), NewSafeSession(session), notInTransaction, options, nil /* LogStats */, false /* autocommit */) - if err == nil { - vtg.rowsReturned.Add(statsKey, int64(len(qr.Rows))) - return qr, nil - } - -handleError: - query := map[string]interface{}{ - "Sql": sql, - "BindVariables": bindVariables, - "Keyspace": keyspace, - "KeyspaceIds": keyspaceIds, - "TabletType": ltt, - "Session": session, - "NotInTransaction": notInTransaction, - "Options": options, - } - err = recordAndAnnotateError(err, statsKey, query, vtg.logExecuteKeyspaceIds) - return nil, err -} - -// ExecuteKeyRanges executes a non-streaming query based on the specified keyranges. This is a legacy function. -func (vtg *VTGate) ExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - startTime := time.Now() - ltt := topoproto.TabletTypeLString(tabletType) - statsKey := []string{"ExecuteKeyRanges", keyspace, ltt} - defer vtg.timings.Record(statsKey, startTime) - - var qr *sqltypes.Result - var err error - - if bvErr := sqltypes.ValidateBindVariables(bindVariables); bvErr != nil { - err = vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "%v", bvErr) - goto handleError - } - - sql = sqlannotation.AnnotateIfDML(sql, nil) - - qr, err = vtg.resolver.Execute(ctx, sql, bindVariables, keyspace, tabletType, key.DestinationKeyRanges(keyRanges), NewSafeSession(session), notInTransaction, options, nil /* LogStats */, false /* autocommit */) - if err == nil { - vtg.rowsReturned.Add(statsKey, int64(len(qr.Rows))) - return qr, nil - } - -handleError: - query := map[string]interface{}{ - "Sql": sql, - "BindVariables": bindVariables, - "Keyspace": keyspace, - "KeyRanges": keyRanges, - "TabletType": ltt, - "Session": session, - "NotInTransaction": notInTransaction, - "Options": options, - } - err = recordAndAnnotateError(err, statsKey, query, vtg.logExecuteKeyRanges) - return nil, err -} - -// ExecuteEntityIds excutes a non-streaming query based on given KeyspaceId map. This is a legacy function. -func (vtg *VTGate) ExecuteEntityIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, entityColumnName string, entityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - startTime := time.Now() - ltt := topoproto.TabletTypeLString(tabletType) - statsKey := []string{"ExecuteEntityIds", keyspace, ltt} - defer vtg.timings.Record(statsKey, startTime) - - var qr *sqltypes.Result - var err error - - if bvErr := sqltypes.ValidateBindVariables(bindVariables); bvErr != nil { - err = vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "%v", bvErr) - goto handleError - } - - sql = sqlannotation.AnnotateIfDML(sql, nil) - - qr, err = vtg.resolver.ExecuteEntityIds(ctx, sql, bindVariables, keyspace, entityColumnName, entityKeyspaceIDs, tabletType, session, notInTransaction, options) - if err == nil { - vtg.rowsReturned.Add(statsKey, int64(len(qr.Rows))) - return qr, nil - } - -handleError: - query := map[string]interface{}{ - "Sql": sql, - "BindVariables": bindVariables, - "Keyspace": keyspace, - "EntityColumnName": entityColumnName, - "EntityKeyspaceIDs": entityKeyspaceIDs, - "TabletType": ltt, - "Session": session, - "NotInTransaction": notInTransaction, - "Options": options, - } - err = recordAndAnnotateError(err, statsKey, query, vtg.logExecuteEntityIds) - return nil, err -} - -// ExecuteBatchShards executes a group of queries on the specified shards. This is a legacy function. -func (vtg *VTGate) ExecuteBatchShards(ctx context.Context, queries []*vtgatepb.BoundShardQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - startTime := time.Now() - ltt := topoproto.TabletTypeLString(tabletType) - statsKey := []string{"ExecuteBatchShards", unambiguousKeyspaceBSQ(queries), ltt} - defer vtg.timings.Record(statsKey, startTime) - - var qrs []sqltypes.Result - var err error - - for _, query := range queries { - if bvErr := sqltypes.ValidateBindVariables(query.Query.BindVariables); bvErr != nil { - err = vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "%v", bvErr) - goto handleError - } - } - - annotateBoundShardQueriesAsUnfriendly(queries) - - qrs, err = vtg.resolver.ExecuteBatch( - ctx, - tabletType, - asTransaction, - session, - options, - func() (*scatterBatchRequest, error) { - return boundShardQueriesToScatterBatchRequest(ctx, vtg.resolver.resolver, queries, tabletType) - }) - if err == nil { - var rowCount int64 - for _, qr := range qrs { - rowCount += int64(len(qr.Rows)) - } - vtg.rowsReturned.Add(statsKey, rowCount) - return qrs, nil - } - -handleError: - query := map[string]interface{}{ - "Queries": queries, - "TabletType": ltt, - "AsTransaction": asTransaction, - "Session": session, - "Options": options, - } - err = recordAndAnnotateError(err, statsKey, query, vtg.logExecuteBatchShards) - return nil, err -} - -// ExecuteBatchKeyspaceIds executes a group of queries based on the specified keyspace ids. This is a legacy function. -func (vtg *VTGate) ExecuteBatchKeyspaceIds(ctx context.Context, queries []*vtgatepb.BoundKeyspaceIdQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - startTime := time.Now() - ltt := topoproto.TabletTypeLString(tabletType) - statsKey := []string{"ExecuteBatchKeyspaceIds", unambiguousKeyspaceBKSIQ(queries), ltt} - defer vtg.timings.Record(statsKey, startTime) - - var qrs []sqltypes.Result - var err error - - for _, query := range queries { - if bvErr := sqltypes.ValidateBindVariables(query.Query.BindVariables); bvErr != nil { - err = vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "%v", bvErr) - goto handleError - } - } - - annotateBoundKeyspaceIDQueries(queries) - - qrs, err = vtg.resolver.ExecuteBatch( - ctx, - tabletType, - asTransaction, - session, - options, - func() (*scatterBatchRequest, error) { - return boundKeyspaceIDQueriesToScatterBatchRequest(ctx, vtg.resolver.resolver, queries, tabletType) - }) - if err == nil { - var rowCount int64 - for _, qr := range qrs { - rowCount += int64(len(qr.Rows)) - } - vtg.rowsReturned.Add(statsKey, rowCount) - return qrs, nil - } - -handleError: - query := map[string]interface{}{ - "Queries": queries, - "TabletType": ltt, - "AsTransaction": asTransaction, - "Session": session, - "Options": options, - } - err = recordAndAnnotateError(err, statsKey, query, vtg.logExecuteBatchKeyspaceIds) - return nil, err -} - -// StreamExecuteKeyspaceIds executes a streaming query on the specified KeyspaceIds. -// The KeyspaceIds are resolved to shards using the serving graph. -// This function currently temporarily enforces the restriction of executing on -// one shard since it cannot merge-sort the results to guarantee ordering of -// response which is needed for checkpointing. -// The api supports supplying multiple KeyspaceIds to make it future proof. This is a legacy function. -// Note we guarantee the callback will not be called concurrently -// by multiple go routines. -func (vtg *VTGate) StreamExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - startTime := time.Now() - ltt := topoproto.TabletTypeLString(tabletType) - statsKey := []string{"StreamExecuteKeyspaceIds", keyspace, ltt} - defer vtg.timings.Record(statsKey, startTime) - - var err error - - if bvErr := sqltypes.ValidateBindVariables(bindVariables); bvErr != nil { - err = vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "%v", bvErr) - goto handleError - } - - err = vtg.resolver.StreamExecute( - ctx, - sql, - bindVariables, - keyspace, - tabletType, - key.DestinationKeyspaceIDs(keyspaceIds), - options, - func(reply *sqltypes.Result) error { - vtg.rowsReturned.Add(statsKey, int64(len(reply.Rows))) - return callback(reply) - }) - -handleError: - if err != nil { - query := map[string]interface{}{ - "Sql": sql, - "BindVariables": bindVariables, - "Keyspace": keyspace, - "KeyspaceIds": keyspaceIds, - "TabletType": ltt, - "Options": options, - } - return recordAndAnnotateError(err, statsKey, query, vtg.logStreamExecuteKeyspaceIds) - } - return nil -} - -// StreamExecuteKeyRanges executes a streaming query on the specified KeyRanges. -// The KeyRanges are resolved to shards using the serving graph. -// This function currently temporarily enforces the restriction of executing on -// one shard since it cannot merge-sort the results to guarantee ordering of -// response which is needed for checkpointing. -// The api supports supplying multiple keyranges to make it future proof. This is a legacy function. -// Note we guarantee the callback will not be called concurrently -// by multiple go routines. -func (vtg *VTGate) StreamExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - startTime := time.Now() - ltt := topoproto.TabletTypeLString(tabletType) - statsKey := []string{"StreamExecuteKeyRanges", keyspace, ltt} - defer vtg.timings.Record(statsKey, startTime) - - var err error - - if bvErr := sqltypes.ValidateBindVariables(bindVariables); bvErr != nil { - err = vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "%v", bvErr) - goto handleError - } - - err = vtg.resolver.StreamExecute( - ctx, - sql, - bindVariables, - keyspace, - tabletType, - key.DestinationKeyRanges(keyRanges), - options, - func(reply *sqltypes.Result) error { - vtg.rowsReturned.Add(statsKey, int64(len(reply.Rows))) - return callback(reply) - }) - -handleError: - if err != nil { - query := map[string]interface{}{ - "Sql": sql, - "BindVariables": bindVariables, - "Keyspace": keyspace, - "KeyRanges": keyRanges, - "TabletType": ltt, - "Options": options, - } - return recordAndAnnotateError(err, statsKey, query, vtg.logStreamExecuteKeyRanges) - } - return nil -} - -// StreamExecuteShards executes a streaming query on the specified shards. This is a legacy function. -// Note we guarantee the callback will not be called concurrently -// by multiple go routines. -func (vtg *VTGate) StreamExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error { - startTime := time.Now() - ltt := topoproto.TabletTypeLString(tabletType) - statsKey := []string{"StreamExecuteShards", keyspace, ltt} - defer vtg.timings.Record(statsKey, startTime) - - var err error - - if bvErr := sqltypes.ValidateBindVariables(bindVariables); bvErr != nil { - err = vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "%v", bvErr) - goto handleError - } - - err = vtg.resolver.StreamExecute( - ctx, - sql, - bindVariables, - keyspace, - tabletType, - key.DestinationShards(shards), - options, - func(reply *sqltypes.Result) error { - vtg.rowsReturned.Add(statsKey, int64(len(reply.Rows))) - return callback(reply) - }) - -handleError: - if err != nil { - query := map[string]interface{}{ - "Sql": sql, - "BindVariables": bindVariables, - "Keyspace": keyspace, - "Shards": shards, - "TabletType": ltt, - "Options": options, - } - return recordAndAnnotateError(err, statsKey, query, vtg.logStreamExecuteShards) - } - return nil -} - -// Begin begins a transaction. This is a legacy function. -func (vtg *VTGate) Begin(ctx context.Context, singledb bool) (*vtgatepb.Session, error) { - if !singledb && vtg.txConn.mode == vtgatepb.TransactionMode_SINGLE { - return nil, vterrors.New(vtrpcpb.Code_INVALID_ARGUMENT, "multi-db transaction disallowed") - } - return &vtgatepb.Session{ - InTransaction: true, - SingleDb: singledb, - }, nil -} - -// Commit commits a transaction. This is a legacy function. -func (vtg *VTGate) Commit(ctx context.Context, twopc bool, session *vtgatepb.Session) error { - if session == nil { - return formatError(vterrors.New(vtrpcpb.Code_INVALID_ARGUMENT, "cannot commit: empty session")) - } - if !session.InTransaction { - return formatError(vterrors.New(vtrpcpb.Code_ABORTED, "cannot commit: not in transaction")) - } - if twopc { - session.TransactionMode = vtgatepb.TransactionMode_TWOPC - } - return formatError(vtg.txConn.Commit(ctx, NewSafeSession(session))) -} - -// Rollback rolls back a transaction. This is a legacy function. -func (vtg *VTGate) Rollback(ctx context.Context, session *vtgatepb.Session) error { - return formatError(vtg.txConn.Rollback(ctx, NewSafeSession(session))) -} - // ResolveTransaction resolves the specified 2PC transaction. func (vtg *VTGate) ResolveTransaction(ctx context.Context, dtid string) error { return formatError(vtg.txConn.Resolve(ctx, dtid)) @@ -844,80 +383,6 @@ handleError: return session, nil, err } -// GetSrvKeyspace is part of the vtgate service API. -func (vtg *VTGate) GetSrvKeyspace(ctx context.Context, keyspace string) (*topodatapb.SrvKeyspace, error) { - return vtg.resolver.toposerv.GetSrvKeyspace(ctx, vtg.resolver.cell, keyspace) -} - -// MessageStream is part of the vtgate service API. This is a V2 level API -// that's sent to the Resolver. -// Note we guarantee the callback will not be called concurrently -// by multiple go routines. -func (vtg *VTGate) MessageStream(ctx context.Context, keyspace string, shard string, keyRange *topodatapb.KeyRange, name string, callback func(*sqltypes.Result) error) error { - startTime := time.Now() - ltt := topoproto.TabletTypeLString(topodatapb.TabletType_MASTER) - statsKey := []string{"MessageStream", keyspace, ltt} - defer vtg.timings.Record(statsKey, startTime) - - err := vtg.executor.MessageStream( - ctx, - keyspace, - shard, - keyRange, - name, - callback, - ) - if err != nil { - request := map[string]interface{}{ - "Keyspace": keyspace, - "Shard": shard, - "KeyRange": keyRange, - "TabletType": ltt, - "MessageName": name, - } - recordAndAnnotateError(err, statsKey, request, vtg.logMessageStream) - } - return formatError(err) -} - -// MessageAck is part of the vtgate service API. This is a V3 level API that's sent -// to the executor. The table name will be resolved using V3 rules, and the routing -// will make use of vindexes for sharded keyspaces. -// TODO(sougou): Deprecate this in favor of an SQL statement. -func (vtg *VTGate) MessageAck(ctx context.Context, keyspace string, name string, ids []*querypb.Value) (int64, error) { - startTime := time.Now() - ltt := topoproto.TabletTypeLString(topodatapb.TabletType_MASTER) - statsKey := []string{"MessageAck", keyspace, ltt} - defer vtg.timings.Record(statsKey, startTime) - - for _, id := range ids { - if _, err := sqltypes.NewValue(id.Type, id.Value); err != nil { - return 0, formatError(err) - } - } - - count, err := vtg.executor.MessageAck(ctx, keyspace, name, ids) - return count, formatError(err) -} - -// MessageAckKeyspaceIds is part of the vtgate service API. It routes -// message acks based on the associated keyspace ids. -func (vtg *VTGate) MessageAckKeyspaceIds(ctx context.Context, keyspace string, name string, idKeyspaceIDs []*vtgatepb.IdKeyspaceId) (int64, error) { - startTime := time.Now() - ltt := topoproto.TabletTypeLString(topodatapb.TabletType_MASTER) - statsKey := []string{"MessageAckKeyspaceIds", keyspace, ltt} - defer vtg.timings.Record(statsKey, startTime) - - for _, idKeyspaceID := range idKeyspaceIDs { - if _, err := sqltypes.NewValue(idKeyspaceID.Id.Type, idKeyspaceID.Id.Value); err != nil { - return 0, formatError(err) - } - } - - count, err := vtg.resolver.MessageAckKeyspaceIds(ctx, keyspace, name, idKeyspaceIDs) - return count, formatError(err) -} - // VStream streams binlog events. func (vtg *VTGate) VStream(ctx context.Context, tabletType topodatapb.TabletType, vgtid *binlogdatapb.VGtid, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { return vtg.vsm.VStream(ctx, tabletType, vgtid, filter, send) @@ -990,61 +455,3 @@ func (vtg *VTGate) HandlePanic(err *error) { errorCounts.Add([]string{"Panic", "Unknown", "Unknown", vtrpcpb.Code_INTERNAL.String()}, 1) } } - -// Helper function used in ExecuteBatchKeyspaceIds -func annotateBoundKeyspaceIDQueries(queries []*vtgatepb.BoundKeyspaceIdQuery) { - for i, q := range queries { - queries[i].Query.Sql = sqlannotation.AnnotateIfDML(q.Query.Sql, q.KeyspaceIds) - } -} - -// Helper function used in ExecuteBatchShards -func annotateBoundShardQueriesAsUnfriendly(queries []*vtgatepb.BoundShardQuery) { - for i, q := range queries { - queries[i].Query.Sql = sqlannotation.AnnotateIfDML(q.Query.Sql, nil) - } -} - -// unambiguousKeyspaceBKSIQ is a helper function used in the -// ExecuteBatchKeyspaceIds method to determine the "keyspace" label for the -// stats reporting. -// If all queries target the same keyspace, it returns that keyspace. -// Otherwise it returns an empty string. -func unambiguousKeyspaceBKSIQ(queries []*vtgatepb.BoundKeyspaceIdQuery) string { - switch len(queries) { - case 0: - return "" - case 1: - return queries[0].Keyspace - default: - keyspace := queries[0].Keyspace - for _, q := range queries[1:] { - if q.Keyspace != keyspace { - // Request targets at least two different keyspaces. - return "" - } - } - return keyspace - } -} - -// unambiguousKeyspaceBSQ is the same as unambiguousKeyspaceBKSIQ but for the -// ExecuteBatchShards method. We are intentionally duplicating the code here and -// do not try to generalize it because this may be less performant. -func unambiguousKeyspaceBSQ(queries []*vtgatepb.BoundShardQuery) string { - switch len(queries) { - case 0: - return "" - case 1: - return queries[0].Keyspace - default: - keyspace := queries[0].Keyspace - for _, q := range queries[1:] { - if q.Keyspace != keyspace { - // Request targets at least two different keyspaces. - return "" - } - } - return keyspace - } -} diff --git a/go/vt/vtgate/vtgate_test.go b/go/vt/vtgate/vtgate_test.go index 40929264352..72d30c25851 100644 --- a/go/vt/vtgate/vtgate_test.go +++ b/go/vt/vtgate/vtgate_test.go @@ -17,18 +17,13 @@ limitations under the License. package vtgate import ( - "encoding/hex" - "fmt" - "io" "reflect" "strings" "testing" - "time" "golang.org/x/net/context" "github.com/golang/protobuf/proto" - "github.com/stretchr/testify/require" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/discovery" "vitess.io/vitess/go/vt/vterrors" @@ -86,127 +81,6 @@ func init() { initMySQLProtocol() } -func TestVTGateBegin(t *testing.T) { - save := rpcVTGate.txConn.mode - defer func() { - rpcVTGate.txConn.mode = save - }() - - rpcVTGate.txConn.mode = vtgatepb.TransactionMode_SINGLE - got, err := rpcVTGate.Begin(context.Background(), true) - require.NoError(t, err) - wantSession := &vtgatepb.Session{ - InTransaction: true, - SingleDb: true, - } - if !proto.Equal(got, wantSession) { - t.Errorf("Begin(single): %v, want %v", got, wantSession) - } - - _, err = rpcVTGate.Begin(context.Background(), false) - wantErr := "multi-db transaction disallowed" - if err == nil || err.Error() != wantErr { - t.Errorf("Begin(multi): %v, want %s", err, wantErr) - } - - rpcVTGate.txConn.mode = vtgatepb.TransactionMode_MULTI - got, err = rpcVTGate.Begin(context.Background(), true) - require.NoError(t, err) - wantSession = &vtgatepb.Session{ - InTransaction: true, - SingleDb: true, - } - if !proto.Equal(got, wantSession) { - t.Errorf("Begin(single): %v, want %v", got, wantSession) - } - - got, err = rpcVTGate.Begin(context.Background(), false) - require.NoError(t, err) - wantSession = &vtgatepb.Session{ - InTransaction: true, - } - if !proto.Equal(got, wantSession) { - t.Errorf("Begin(single): %v, want %v", got, wantSession) - } - - rpcVTGate.txConn.mode = vtgatepb.TransactionMode_TWOPC - got, err = rpcVTGate.Begin(context.Background(), true) - require.NoError(t, err) - wantSession = &vtgatepb.Session{ - InTransaction: true, - SingleDb: true, - } - if !proto.Equal(got, wantSession) { - t.Errorf("Begin(single): %v, want %v", got, wantSession) - } - - got, err = rpcVTGate.Begin(context.Background(), false) - require.NoError(t, err) - wantSession = &vtgatepb.Session{ - InTransaction: true, - } - if !proto.Equal(got, wantSession) { - t.Errorf("Begin(single): %v, want %v", got, wantSession) - } -} - -func TestVTGateCommit(t *testing.T) { - save := rpcVTGate.txConn.mode - defer func() { - rpcVTGate.txConn.mode = save - }() - - session := &vtgatepb.Session{ - InTransaction: true, - } - - rpcVTGate.txConn.mode = vtgatepb.TransactionMode_SINGLE - err := rpcVTGate.Commit(context.Background(), true, session) - wantErr := "vtgate: : 2pc transaction disallowed" - if err == nil || err.Error() != wantErr { - t.Errorf("Begin(multi): %v, want %s", err, wantErr) - } - - session = &vtgatepb.Session{ - InTransaction: true, - } - err = rpcVTGate.Commit(context.Background(), false, session) - require.NoError(t, err) - - rpcVTGate.txConn.mode = vtgatepb.TransactionMode_MULTI - session = &vtgatepb.Session{ - InTransaction: true, - } - err = rpcVTGate.Commit(context.Background(), true, session) - if err == nil || err.Error() != wantErr { - t.Errorf("Begin(multi): %v, want %s", err, wantErr) - } - - session = &vtgatepb.Session{ - InTransaction: true, - } - err = rpcVTGate.Commit(context.Background(), false, session) - require.NoError(t, err) - - rpcVTGate.txConn.mode = vtgatepb.TransactionMode_TWOPC - session = &vtgatepb.Session{ - InTransaction: true, - } - err = rpcVTGate.Commit(context.Background(), true, session) - require.NoError(t, err) - - session = &vtgatepb.Session{ - InTransaction: true, - } - err = rpcVTGate.Commit(context.Background(), false, session) - require.NoError(t, err) -} - -func TestVTGateRollbackNil(t *testing.T) { - err := rpcVTGate.Rollback(context.Background(), nil) - require.NoError(t, err) -} - func TestVTGateExecute(t *testing.T) { createSandbox(KsTestUnsharded) hcVTGateTest.Reset() @@ -220,1970 +94,201 @@ func TestVTGateExecute(t *testing.T) { }, "select id from t1", nil, - ) - if err != nil { - t.Errorf("want nil, got %v", err) - } - if !reflect.DeepEqual(sandboxconn.SingleRowResult, qr) { - t.Errorf("want \n%+v, got \n%+v", sandboxconn.SingleRowResult, qr) - } - if !proto.Equal(sbc.Options[0], executeOptions) { - t.Errorf("got ExecuteOptions \n%+v, want \n%+v", sbc.Options[0], executeOptions) - } - - session, err := rpcVTGate.Begin(context.Background(), false) - require.NoError(t, err) - if !session.InTransaction { - t.Errorf("want true, got false") - } - rpcVTGate.Execute( - context.Background(), - session, - "select id from t1", - nil, - ) - wantSession := &vtgatepb.Session{ - InTransaction: true, - ShardSessions: []*vtgatepb.Session_ShardSession{{ - Target: &querypb.Target{ - Keyspace: KsTestUnsharded, - Shard: "0", - TabletType: topodatapb.TabletType_MASTER, - }, - TransactionId: 1, - }}, - FoundRows: 1, - } - if !proto.Equal(wantSession, session) { - t.Errorf("want \n%+v, got \n%+v", wantSession, session) - } - - rpcVTGate.Commit(context.Background(), false, session) - if commitCount := sbc.CommitCount.Get(); commitCount != 1 { - t.Errorf("want 1, got %d", commitCount) - } - - session, _ = rpcVTGate.Begin(context.Background(), false) - rpcVTGate.Execute( - context.Background(), - session, - "select id from t1", - nil, - ) - rpcVTGate.Rollback(context.Background(), session) - if sbc.RollbackCount.Get() != 1 { - t.Errorf("want 1, got %d", sbc.RollbackCount.Get()) - } -} - -func TestVTGateExecuteWithKeyspaceShard(t *testing.T) { - createSandbox(KsTestUnsharded) - hcVTGateTest.Reset() - hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, KsTestUnsharded, "0", topodatapb.TabletType_MASTER, true, 1, nil) - - // Valid keyspace. - _, qr, err := rpcVTGate.Execute( - context.Background(), - &vtgatepb.Session{ - TargetString: KsTestUnsharded, - }, - "select id from none", - nil, - ) - if err != nil { - t.Errorf("want nil, got %v", err) - } - if !reflect.DeepEqual(sandboxconn.SingleRowResult, qr) { - t.Errorf("want \n%+v, got \n%+v", sandboxconn.SingleRowResult, qr) - } - - // Invalid keyspace. - _, _, err = rpcVTGate.Execute( - context.Background(), - &vtgatepb.Session{ - TargetString: "invalid_keyspace", - }, - "select id from none", - nil, - ) - want := "vtgate: : keyspace invalid_keyspace not found in vschema" - if err == nil || err.Error() != want { - t.Errorf("Execute: %v, want %s", err, want) - } - - // Valid keyspace/shard. - _, qr, err = rpcVTGate.Execute( - context.Background(), - &vtgatepb.Session{ - TargetString: KsTestUnsharded + ":0@master", - }, - "select id from none", - nil, - ) - if err != nil { - t.Errorf("want nil, got %v", err) - } - if !reflect.DeepEqual(sandboxconn.SingleRowResult, qr) { - t.Errorf("want \n%+v, got \n%+v", sandboxconn.SingleRowResult, qr) - } - - // Invalid keyspace/shard. - _, _, err = rpcVTGate.Execute( - context.Background(), - &vtgatepb.Session{ - TargetString: KsTestUnsharded + ":noshard@master", - }, - "select id from none", - nil, - ) - want = "TestUnsharded.noshard.master: no valid tablet" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("Execute: %v, want %s", err, want) - } -} -func TestVTGateExecuteShards(t *testing.T) { - ks := "TestVTGateExecuteShards" - shard := "0" - createSandbox(ks) - hcVTGateTest.Reset() - sbc := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, shard, topodatapb.TabletType_REPLICA, true, 1, nil) - qr, err := rpcVTGate.ExecuteShards(context.Background(), - "query", - nil, - ks, - []string{shard}, - topodatapb.TabletType_REPLICA, - nil, - false, - executeOptions) - if err != nil { - t.Errorf("want nil, got %v", err) - } - if !reflect.DeepEqual(sandboxconn.SingleRowResult, qr) { - t.Errorf("want \n%+v, got \n%+v", sandboxconn.SingleRowResult, qr) - } - if !proto.Equal(sbc.Options[0], executeOptions) { - t.Errorf("got ExecuteOptions \n%+v, want \n%+v", sbc.Options[0], executeOptions) - } - - session, _ := rpcVTGate.Begin(context.Background(), false) - if !session.InTransaction { - t.Errorf("want true, got false") - } - rpcVTGate.ExecuteShards(context.Background(), - "query", - nil, - ks, - []string{shard}, - topodatapb.TabletType_REPLICA, - session, - false, - nil) - wantSession := &vtgatepb.Session{ - InTransaction: true, - ShardSessions: []*vtgatepb.Session_ShardSession{{ - Target: &querypb.Target{ - Keyspace: ks, - Shard: shard, - TabletType: topodatapb.TabletType_REPLICA, - }, - TransactionId: 1, - }}, - } - if !proto.Equal(wantSession, session) { - t.Errorf("want \n%+v, got \n%+v", wantSession, session) - } - - rpcVTGate.Commit(context.Background(), false, session) - if commitCount := sbc.CommitCount.Get(); commitCount != 1 { - t.Errorf("want 1, got %d", commitCount) - } - - session, _ = rpcVTGate.Begin(context.Background(), false) - rpcVTGate.ExecuteShards(context.Background(), - "query", - nil, - ks, - []string{shard}, - topodatapb.TabletType_REPLICA, - session, - false, - nil) - rpcVTGate.Rollback(context.Background(), session) - if sbc.RollbackCount.Get() != 1 { - t.Errorf("want 1, got %d", sbc.RollbackCount.Get()) - } -} - -func TestVTGateExecuteKeyspaceIds(t *testing.T) { - ks := "TestVTGateExecuteKeyspaceIds" - shard1 := "-20" - shard2 := "20-40" - createSandbox(ks) - hcVTGateTest.Reset() - sbc1 := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, shard1, topodatapb.TabletType_MASTER, true, 1, nil) - hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1002, ks, shard2, topodatapb.TabletType_MASTER, true, 1, nil) - // Test for successful execution - qr, err := rpcVTGate.ExecuteKeyspaceIds(context.Background(), - "query", - nil, - ks, - [][]byte{{0x10}}, - topodatapb.TabletType_MASTER, - nil, - false, - executeOptions) - if err != nil { - t.Errorf("want nil, got %v", err) - } - if !reflect.DeepEqual(sandboxconn.SingleRowResult, qr) { - t.Errorf("want \n%+v, got \n%+v", sandboxconn.SingleRowResult, qr) - } - if execCount := sbc1.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v\n", execCount) - } - if !proto.Equal(sbc1.Options[0], executeOptions) { - t.Errorf("got ExecuteOptions \n%+v, want \n%+v", sbc1.Options[0], executeOptions) - } - // Test for successful execution in transaction - session, _ := rpcVTGate.Begin(context.Background(), false) - if !session.InTransaction { - t.Errorf("want true, got false") - } - rpcVTGate.ExecuteKeyspaceIds(context.Background(), - "query", - nil, - ks, - [][]byte{{0x10}}, - topodatapb.TabletType_MASTER, - session, - false, - nil) - wantSession := &vtgatepb.Session{ - InTransaction: true, - ShardSessions: []*vtgatepb.Session_ShardSession{{ - Target: &querypb.Target{ - Keyspace: ks, - Shard: shard1, - TabletType: topodatapb.TabletType_MASTER, - }, - TransactionId: 1, - }}, - } - if !proto.Equal(wantSession, session) { - t.Errorf("want \n%+v, got \n%+v", wantSession, session) - } - rpcVTGate.Commit(context.Background(), false, session) - if commitCount := sbc1.CommitCount.Get(); commitCount != 1 { - t.Errorf("want 1, got %d", commitCount) - } - // Test for multiple shards - qr, err = rpcVTGate.ExecuteKeyspaceIds(context.Background(), - "query", - nil, - ks, - [][]byte{{0x10}, {0x30}}, - topodatapb.TabletType_MASTER, - session, - false, - nil) - if err != nil { - t.Fatalf("ExecuteKeyspaceIds failed: %v", err) - } - if qr.RowsAffected != 2 { - t.Errorf("want 2, got %v", qr.RowsAffected) - } - // Test for multiple shards for DML - _, err = rpcVTGate.ExecuteKeyspaceIds(context.Background(), - "update table set a = b", - nil, - ks, - [][]byte{{0x10}, {0x30}}, - topodatapb.TabletType_MASTER, - session, - false, - nil) - errStr := "DML should not span multiple keyspace_ids" - if err == nil || !strings.Contains(err.Error(), errStr) { - t.Errorf("want '%v', got '%v'", errStr, err) - } -} - -func TestVTGateExecuteKeyRanges(t *testing.T) { - ks := "TestVTGateExecuteKeyRanges" - shard1 := "-20" - shard2 := "20-40" - createSandbox(ks) - hcVTGateTest.Reset() - sbc1 := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, shard1, topodatapb.TabletType_MASTER, true, 1, nil) - hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1002, ks, shard2, topodatapb.TabletType_MASTER, true, 1, nil) - // Test for successful execution - qr, err := rpcVTGate.ExecuteKeyRanges(context.Background(), - "query", - nil, - ks, - []*topodatapb.KeyRange{{End: []byte{0x20}}}, - topodatapb.TabletType_MASTER, - nil, - false, - executeOptions) - if err != nil { - t.Errorf("want nil, got %v", err) - } - if !reflect.DeepEqual(sandboxconn.SingleRowResult, qr) { - t.Errorf("want \n%+v, got \n%+v", sandboxconn.SingleRowResult, qr) - } - if execCount := sbc1.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v\n", execCount) - } - if !proto.Equal(sbc1.Options[0], executeOptions) { - t.Errorf("got ExecuteOptions \n%+v, want \n%+v", sbc1.Options[0], executeOptions) - } - // Test for successful execution in transaction - session, _ := rpcVTGate.Begin(context.Background(), false) - if !session.InTransaction { - t.Errorf("want true, got false") - } - _, err = rpcVTGate.ExecuteKeyRanges(context.Background(), - "query", - nil, - ks, - []*topodatapb.KeyRange{{End: []byte{0x20}}}, - topodatapb.TabletType_MASTER, - session, - false, - nil) - if err != nil { - t.Errorf("want nil, got %v", err) - } - wantSession := &vtgatepb.Session{ - InTransaction: true, - ShardSessions: []*vtgatepb.Session_ShardSession{{ - Target: &querypb.Target{ - Keyspace: ks, - Shard: shard1, - TabletType: topodatapb.TabletType_MASTER, - }, - TransactionId: 1, - }}, - } - if !proto.Equal(wantSession, session) { - t.Errorf("want \n%+v, got \n%+v", wantSession, session) - } - rpcVTGate.Commit(context.Background(), false, session) - if commitCount := sbc1.CommitCount.Get(); commitCount != 1 { - t.Errorf("want 1, got %v", commitCount) - } - // Test for multiple shards - qr, err = rpcVTGate.ExecuteKeyRanges(context.Background(), "query", - nil, - ks, - []*topodatapb.KeyRange{{Start: []byte{0x10}, End: []byte{0x30}}}, - topodatapb.TabletType_MASTER, - nil, - false, - nil) - if err != nil { - t.Fatalf("ExecuteKeyRanges failed: %v", err) - } - if qr.RowsAffected != 2 { - t.Errorf("want 2, got %v", qr.RowsAffected) - } -} - -func TestVTGateExecuteEntityIds(t *testing.T) { - ks := "TestVTGateExecuteEntityIds" - shard1 := "-20" - shard2 := "20-40" - createSandbox(ks) - hcVTGateTest.Reset() - sbc1 := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, shard1, topodatapb.TabletType_MASTER, true, 1, nil) - hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1002, ks, shard2, topodatapb.TabletType_MASTER, true, 1, nil) - // Test for successful execution - qr, err := rpcVTGate.ExecuteEntityIds(context.Background(), - "query", - nil, - ks, - "kid", - []*vtgatepb.ExecuteEntityIdsRequest_EntityId{ - { - Type: sqltypes.VarBinary, - Value: []byte("id1"), - KeyspaceId: []byte{0x10}, - }, - }, - topodatapb.TabletType_MASTER, - nil, - false, - executeOptions) - if err != nil { - t.Errorf("want nil, got %v", err) - } - if !reflect.DeepEqual(sandboxconn.SingleRowResult, qr) { - t.Errorf("want \n%+v, got \n%+v", sandboxconn.SingleRowResult, qr) - } - if execCount := sbc1.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v\n", execCount) - } - if !proto.Equal(sbc1.Options[0], executeOptions) { - t.Errorf("got ExecuteOptions \n%+v, want \n%+v", sbc1.Options[0], executeOptions) - } - // Test for successful execution in transaction - session, _ := rpcVTGate.Begin(context.Background(), false) - if !session.InTransaction { - t.Errorf("want true, got false") - } - rpcVTGate.ExecuteEntityIds(context.Background(), - "query", - nil, - ks, - "kid", - []*vtgatepb.ExecuteEntityIdsRequest_EntityId{ - { - Type: sqltypes.VarBinary, - Value: []byte("id1"), - KeyspaceId: []byte{0x10}, - }, - }, - topodatapb.TabletType_MASTER, - session, - false, - nil) - wantSession := &vtgatepb.Session{ - InTransaction: true, - ShardSessions: []*vtgatepb.Session_ShardSession{{ - Target: &querypb.Target{ - Keyspace: ks, - Shard: shard1, - TabletType: topodatapb.TabletType_MASTER, - }, - TransactionId: 1, - }}, - } - if !proto.Equal(wantSession, session) { - t.Errorf("want \n%+v, got \n%+v", wantSession, session) - } - rpcVTGate.Commit(context.Background(), false, session) - if commitCount := sbc1.CommitCount.Get(); commitCount != 1 { - t.Errorf("want 1, got %d", commitCount) - } - - // Test for multiple shards - qr, err = rpcVTGate.ExecuteEntityIds(context.Background(), "query", - nil, - ks, - "kid", - []*vtgatepb.ExecuteEntityIdsRequest_EntityId{ - { - Type: sqltypes.VarBinary, - Value: []byte("id1"), - KeyspaceId: []byte{0x10}, - }, - { - Type: sqltypes.VarBinary, - Value: []byte("id2"), - KeyspaceId: []byte{0x30}, - }, - }, - topodatapb.TabletType_MASTER, - nil, - false, - nil) - if err != nil { - t.Fatalf("ExecuteEntityIds failed: %v", err) - } - if qr.RowsAffected != 2 { - t.Errorf("want 2, got %v", qr.RowsAffected) - } -} - -func TestVTGateExecuteBatch(t *testing.T) { - // TODO(sougou): using masterSession as global has some bugs - // which requires us to reset it here. This needs to be fixed. - masterSession = &vtgatepb.Session{ - TargetString: "@master", - } - - createSandbox(KsTestUnsharded) - hcVTGateTest.Reset() - sbc := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, KsTestUnsharded, "0", topodatapb.TabletType_MASTER, true, 1, nil) - - startCommit := sbc.CommitCount.Get() - startRollback := sbc.RollbackCount.Get() - - sqlList := []string{ - "begin", - "select id from t1", - "begin", - "select id from t1", - "commit", - "select id from t1", - "commit", - "begin", - "select id from t1", - "rollback", - "select id from t1", - "rollback", - "begin", - "select id from t1", - } - - session, qrl, err := rpcVTGate.ExecuteBatch(context.Background(), masterSession, sqlList, nil) - if err != nil { - t.Fatal(err) - } - // Spot-check one result. - qr := qrl[3].QueryResult - if !reflect.DeepEqual(sandboxconn.SingleRowResult, qr) { - t.Errorf("want \n%+v, got \n%+v", sandboxconn.SingleRowResult, qr) - } - if len(session.ShardSessions) != 1 { - t.Errorf("want 1, got %d", len(session.ShardSessions)) - } - if got, want := sbc.CommitCount.Get(), startCommit+3; got != want { - t.Errorf("got %d, want %d", got, want) - } - if got, want := sbc.RollbackCount.Get(), startRollback+2; got != want { - t.Errorf("got %d, want %d", got, want) - } -} - -func TestVTGateExecuteBatchShards(t *testing.T) { - ks := "TestVTGateExecuteBatchShards" - createSandbox(ks) - shard1 := "-20" - shard2 := "20-40" - hcVTGateTest.Reset() - sbc1 := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, shard1, topodatapb.TabletType_MASTER, true, 1, nil) - hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1002, ks, shard2, topodatapb.TabletType_MASTER, true, 1, nil) - qrl, err := rpcVTGate.ExecuteBatchShards(context.Background(), - []*vtgatepb.BoundShardQuery{{ - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: ks, - Shards: []string{shard1, shard2}, - }, { - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: ks, - Shards: []string{shard1, shard2}, - }}, - topodatapb.TabletType_MASTER, - false, - nil, - executeOptions) - if err != nil { - t.Fatalf("want nil, got %v", err) - } - if len(qrl) != 2 { - t.Errorf("want 2, got %v", len(qrl)) - } - if qrl[0].RowsAffected != 2 { - t.Errorf("want 2, got %v", qrl[0].RowsAffected) - } - if !proto.Equal(sbc1.Options[0], executeOptions) { - t.Errorf("got ExecuteOptions \n%+v, want \n%+v", sbc1.Options[0], executeOptions) - } - - session, _ := rpcVTGate.Begin(context.Background(), false) - rpcVTGate.ExecuteBatchShards(context.Background(), - []*vtgatepb.BoundShardQuery{{ - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: ks, - Shards: []string{shard1, shard2}, - }, { - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: ks, - Shards: []string{shard1, shard2}, - }}, - topodatapb.TabletType_MASTER, - false, - session, - nil) - if len(session.ShardSessions) != 2 { - t.Errorf("want 2, got %d", len(session.ShardSessions)) - } - - timingsCount := rpcVTGate.timings.Counts()["ExecuteBatchShards.TestVTGateExecuteBatchShards.master"] - if got, want := timingsCount, int64(2); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } -} - -func TestVTGateExecuteBatchKeyspaceIds(t *testing.T) { - ks := "TestVTGateExecuteBatchKeyspaceIds" - shard1 := "-20" - shard2 := "20-40" - createSandbox(ks) - hcVTGateTest.Reset() - sbc1 := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, shard1, topodatapb.TabletType_MASTER, true, 1, nil) - hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1002, ks, shard2, topodatapb.TabletType_MASTER, true, 1, nil) - kid10 := []byte{0x10} - kid30 := []byte{0x30} - qrl, err := rpcVTGate.ExecuteBatchKeyspaceIds(context.Background(), - []*vtgatepb.BoundKeyspaceIdQuery{{ - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: ks, - KeyspaceIds: [][]byte{kid10, kid30}, - }, { - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: ks, - KeyspaceIds: [][]byte{kid10, kid30}, - }}, - topodatapb.TabletType_MASTER, - false, - nil, - executeOptions) - if err != nil { - t.Fatalf("want nil, got %v", err) - } - if len(qrl) != 2 { - t.Errorf("want 2, got %v", len(qrl)) - } - if qrl[0].RowsAffected != 2 { - t.Errorf("want 2, got %v", qrl[0].RowsAffected) - } - if !proto.Equal(sbc1.Options[0], executeOptions) { - t.Errorf("got ExecuteOptions \n%+v, want \n%+v", sbc1.Options[0], executeOptions) - } - - session, _ := rpcVTGate.Begin(context.Background(), false) - rpcVTGate.ExecuteBatchKeyspaceIds(context.Background(), - []*vtgatepb.BoundKeyspaceIdQuery{{ - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: ks, - KeyspaceIds: [][]byte{kid10, kid30}, - }, { - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: ks, - KeyspaceIds: [][]byte{kid10, kid30}, - }}, - topodatapb.TabletType_MASTER, - false, - session, - nil) - if len(session.ShardSessions) != 2 { - t.Errorf("want 2, got %d", len(session.ShardSessions)) - } - - timingsCount := rpcVTGate.timings.Counts()["ExecuteBatchKeyspaceIds.TestVTGateExecuteBatchKeyspaceIds.master"] - if got, want := timingsCount, int64(2); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } -} - -func TestVTGateStreamExecute(t *testing.T) { - ks := KsTestUnsharded - shard := "0" - createSandbox(ks) - hcVTGateTest.Reset() - sbc := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, shard, topodatapb.TabletType_MASTER, true, 1, nil) - var qrs []*sqltypes.Result - err := rpcVTGate.StreamExecute( - context.Background(), - &vtgatepb.Session{ - TargetString: "@master", - Options: executeOptions, - }, - "select id from t1", - nil, - func(r *sqltypes.Result) error { - qrs = append(qrs, r) - return nil - }, - ) - if err != nil { - t.Errorf("want nil, got %v", err) - } - want := []*sqltypes.Result{{ - Fields: sandboxconn.StreamRowResult.Fields, - }, { - Rows: sandboxconn.StreamRowResult.Rows, - }} - if !reflect.DeepEqual(want, qrs) { - t.Errorf("want \n%+v, got \n%+v", want, qrs) - } - if !proto.Equal(sbc.Options[0], executeOptions) { - t.Errorf("got ExecuteOptions \n%+v, want \n%+v", sbc.Options[0], executeOptions) - } -} - -func TestVTGateStreamExecuteKeyspaceShard(t *testing.T) { - ks := KsTestUnsharded - shard := "0" - createSandbox(ks) - hcVTGateTest.Reset() - sbc := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, shard, topodatapb.TabletType_MASTER, true, 1, nil) - var qrs []*sqltypes.Result - err := rpcVTGate.StreamExecute( - context.Background(), - &vtgatepb.Session{ - TargetString: ks + "/" + shard + "@master", - Options: executeOptions, - }, - "random statement", - nil, - func(r *sqltypes.Result) error { - qrs = append(qrs, r) - return nil - }, - ) - if err != nil { - t.Errorf("want nil, got %v", err) - } - want := []*sqltypes.Result{sandboxconn.SingleRowResult} - if !reflect.DeepEqual(want, qrs) { - t.Errorf("want \n%+v, got \n%+v", want, qrs) - } - if !proto.Equal(sbc.Options[0], executeOptions) { - t.Errorf("got ExecuteOptions \n%+v, want \n%+v", sbc.Options[0], executeOptions) - } -} - -func TestVTGateStreamExecuteKeyspaceIds(t *testing.T) { - ks := "TestVTGateStreamExecuteKeyspaceIds" - shard1 := "-20" - shard2 := "20-40" - createSandbox(ks) - hcVTGateTest.Reset() - sbc1 := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, shard1, topodatapb.TabletType_MASTER, true, 1, nil) - hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1002, ks, shard2, topodatapb.TabletType_MASTER, true, 1, nil) - // Test for successful execution - var qrs []*sqltypes.Result - err := rpcVTGate.StreamExecuteKeyspaceIds(context.Background(), - "query", - nil, - ks, - [][]byte{{0x10}}, - topodatapb.TabletType_MASTER, - executeOptions, - func(r *sqltypes.Result) error { - qrs = append(qrs, r) - return nil - }) - if err != nil { - t.Errorf("want nil, got %v", err) - } - want := []*sqltypes.Result{sandboxconn.SingleRowResult} - if !reflect.DeepEqual(want, qrs) { - t.Errorf("want \n%+v, got \n%+v", want, qrs) - } - if !proto.Equal(sbc1.Options[0], executeOptions) { - t.Errorf("got ExecuteOptions \n%+v, want \n%+v", sbc1.Options[0], executeOptions) - } - - // Test for successful execution - multiple keyspaceids in single shard - qrs = nil - err = rpcVTGate.StreamExecuteKeyspaceIds(context.Background(), - "query", - nil, - ks, - [][]byte{{0x10}, {0x15}}, - topodatapb.TabletType_MASTER, - nil, - func(r *sqltypes.Result) error { - qrs = append(qrs, r) - return nil - }) - if err != nil { - t.Errorf("want nil, got %v", err) - } - want = []*sqltypes.Result{sandboxconn.SingleRowResult} - if !reflect.DeepEqual(want, qrs) { - t.Errorf("want \n%+v, got \n%+v", want, qrs) - } - // Test for successful execution - multiple keyspaceids in multiple shards - err = rpcVTGate.StreamExecuteKeyspaceIds(context.Background(), - "query", - nil, - ks, - [][]byte{{0x10}, {0x30}}, - topodatapb.TabletType_MASTER, - nil, - func(r *sqltypes.Result) error { - qrs = append(qrs, r) - return nil - }) - if err != nil { - t.Errorf("want nil, got %v", err) - } -} - -func TestVTGateStreamExecuteKeyRanges(t *testing.T) { - ks := "TestVTGateStreamExecuteKeyRanges" - shard1 := "-20" - shard2 := "20-40" - createSandbox(ks) - hcVTGateTest.Reset() - sbc1 := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, shard1, topodatapb.TabletType_MASTER, true, 1, nil) - hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1002, ks, shard2, topodatapb.TabletType_MASTER, true, 1, nil) - // Test for successful execution - var qrs []*sqltypes.Result - err := rpcVTGate.StreamExecuteKeyRanges(context.Background(), - "query", - nil, - ks, - []*topodatapb.KeyRange{{End: []byte{0x20}}}, - topodatapb.TabletType_MASTER, - executeOptions, - func(r *sqltypes.Result) error { - qrs = append(qrs, r) - return nil - }) - if err != nil { - t.Errorf("want nil, got %v", err) - } - want := []*sqltypes.Result{sandboxconn.SingleRowResult} - if !reflect.DeepEqual(want, qrs) { - t.Errorf("want \n%+v, got \n%+v", want, qrs) - } - if !proto.Equal(sbc1.Options[0], executeOptions) { - t.Errorf("got ExecuteOptions \n%+v, want \n%+v", sbc1.Options[0], executeOptions) - } - - // Test for successful execution - multiple shards - err = rpcVTGate.StreamExecuteKeyRanges(context.Background(), - "query", - nil, - ks, - []*topodatapb.KeyRange{{Start: []byte{0x10}, End: []byte{0x40}}}, - topodatapb.TabletType_MASTER, - nil, - func(r *sqltypes.Result) error { - qrs = append(qrs, r) - return nil - }) - if err != nil { - t.Errorf("want nil, got %v", err) - } -} - -func TestVTGateStreamExecuteShards(t *testing.T) { - ks := "TestVTGateStreamExecuteShards" - shard := "0" - createSandbox(ks) - hcVTGateTest.Reset() - sbc := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, shard, topodatapb.TabletType_MASTER, true, 1, nil) - // Test for successful execution - var qrs []*sqltypes.Result - err := rpcVTGate.StreamExecuteShards(context.Background(), - "query", - nil, - ks, - []string{shard}, - topodatapb.TabletType_MASTER, - executeOptions, - func(r *sqltypes.Result) error { - qrs = append(qrs, r) - return nil - }) - if err != nil { - t.Errorf("want nil, got %v", err) - } - want := []*sqltypes.Result{sandboxconn.SingleRowResult} - if !reflect.DeepEqual(want, qrs) { - t.Errorf("want \n%+v, got \n%+v", want, qrs) - } - if !proto.Equal(sbc.Options[0], executeOptions) { - t.Errorf("got ExecuteOptions \n%+v, want \n%+v", sbc.Options[0], executeOptions) - } -} - -func TestVTGateMessageStreamSharded(t *testing.T) { - ks := "TestVTGateMessageStreamSharded" - createSandbox(ks) - hcVTGateTest.Reset() - _ = hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_MASTER, true, 1, nil) - _ = hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1002, ks, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) - ch := make(chan *sqltypes.Result) - done := make(chan struct{}) - ctx, cancel := context.WithCancel(context.Background()) - go func() { - kr := &topodatapb.KeyRange{End: []byte{0x40}} - err := rpcVTGate.MessageStream(ctx, ks, "", kr, "msg", func(qr *sqltypes.Result) error { - select { - case <-ctx.Done(): - return io.EOF - case ch <- qr: - } - return nil - }) - if err != nil { - t.Error(err) - } - close(done) - }() - // We should get two messages. - <-ch - got := <-ch - if !reflect.DeepEqual(got, sandboxconn.SingleRowResult) { - t.Errorf("MessageStream: %v, want %v", got, sandboxconn.SingleRowResult) - } - // Once we cancel, the function should return. - cancel() - <-done - - // Test error case. - kr := &topodatapb.KeyRange{End: []byte{0x30}} - err := rpcVTGate.MessageStream(context.Background(), ks, "", kr, "msg", func(qr *sqltypes.Result) error { - ch <- qr - return nil - }) - want := "keyrange -30 does not exactly match shards" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MessageStream: '%v', must contain '%s'", err, want) - } -} - -func TestVTGateMessageStreamUnsharded(t *testing.T) { - ks := KsTestUnsharded - createSandbox(ks) - hcVTGateTest.Reset() - _ = hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, "0", topodatapb.TabletType_MASTER, true, 1, nil) - ch := make(chan *sqltypes.Result) - done := make(chan struct{}) - ctx, cancel := context.WithCancel(context.Background()) - go func() { - err := rpcVTGate.MessageStream(ctx, ks, "0", nil, "msg", func(qr *sqltypes.Result) error { - select { - case <-ctx.Done(): - return io.EOF - case ch <- qr: - } - return nil - }) - if err != nil { - t.Error(err) - } - close(done) - }() - got := <-ch - if !reflect.DeepEqual(got, sandboxconn.SingleRowResult) { - t.Errorf("MessageStream: %v, want %v", got, sandboxconn.SingleRowResult) - } - // Function should return after cancel. - cancel() - <-done -} - -func TestVTGateMessageStreamRetry(t *testing.T) { - *messageStreamGracePeriod = 5 * time.Second - defer func() { - *messageStreamGracePeriod = 30 * time.Second - }() - ks := KsTestUnsharded - createSandbox(ks) - hcVTGateTest.Reset() - _ = hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, "0", topodatapb.TabletType_MASTER, true, 1, nil) - ch := make(chan *sqltypes.Result) - done := make(chan struct{}) - ctx, cancel := context.WithCancel(context.Background()) - go func() { - err := rpcVTGate.MessageStream(ctx, ks, "0", nil, "msg", func(qr *sqltypes.Result) error { - select { - case <-ctx.Done(): - return io.EOF - case ch <- qr: - } - return nil - }) - if err != nil { - t.Error(err) - } - close(done) - }() - <-ch - - // By default, will end the stream after the first message, - // which should make vtgate wait for 1s (5s/5) and retry. - start := time.Now() - <-ch - duration := time.Since(start).Round(time.Second) - if duration < 1*time.Second || duration > 2*time.Second { - t.Errorf("Retry duration should be around 1 second: %v", duration) - } - // Function should return after cancel. - cancel() - <-done -} - -func TestVTGateMessageStreamUnavailable(t *testing.T) { - *messageStreamGracePeriod = 5 * time.Second - defer func() { - *messageStreamGracePeriod = 30 * time.Second - }() - ks := KsTestUnsharded - createSandbox(ks) - hcVTGateTest.Reset() - tablet := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, "0", topodatapb.TabletType_MASTER, true, 1, nil) - // Unavailable error should cause vtgate to wait 1s and retry. - tablet.MustFailCodes[vtrpcpb.Code_UNAVAILABLE] = 1 - ch := make(chan *sqltypes.Result) - done := make(chan struct{}) - ctx, cancel := context.WithCancel(context.Background()) - go func() { - err := rpcVTGate.MessageStream(ctx, ks, "0", nil, "msg", func(qr *sqltypes.Result) error { - select { - case <-ctx.Done(): - return io.EOF - case ch <- qr: - } - return nil - }) - if err != nil { - t.Error(err) - } - close(done) - }() - - // Verify the 1s delay. - start := time.Now() - <-ch - duration := time.Since(start) - if duration < 1*time.Second || duration > 2*time.Second { - t.Errorf("Retry duration should be around 1 second: %v", duration) - } - // Function should return after cancel. - cancel() - <-done -} - -func TestVTGateMessageStreamGracePeriod(t *testing.T) { - *messageStreamGracePeriod = 1 * time.Second - defer func() { - *messageStreamGracePeriod = 30 * time.Second - }() - ks := KsTestUnsharded - createSandbox(ks) - hcVTGateTest.Reset() - tablet := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, "0", topodatapb.TabletType_MASTER, true, 1, nil) - // tablet should return no results for at least 5 calls for it to exceed the grace period. - tablet.SetResults([]*sqltypes.Result{ - nil, - nil, - nil, - nil, - nil, - nil, - nil, - nil, - }) - start := time.Now() - err := rpcVTGate.MessageStream(context.Background(), ks, "0", nil, "msg", func(qr *sqltypes.Result) error { - return nil - }) - want := "has repeatedly failed for longer than 1s" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MessageStream err: %v, must contain %s", err, want) - } - duration := time.Since(start) - if duration < 1*time.Second || duration > 2*time.Second { - t.Errorf("Retry duration should be around 1 second: %v", duration) - } -} - -func TestVTGateMessageStreamFail(t *testing.T) { - ks := KsTestUnsharded - createSandbox(ks) - hcVTGateTest.Reset() - tablet := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, "0", topodatapb.TabletType_MASTER, true, 1, nil) - // tablet should fail immediately if the error is not EOF or UNAVAILABLE. - tablet.MustFailCodes[vtrpcpb.Code_RESOURCE_EXHAUSTED] = 1 - err := rpcVTGate.MessageStream(context.Background(), ks, "0", nil, "msg", func(qr *sqltypes.Result) error { - return nil - }) - want := "RESOURCE_EXHAUSTED error" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MessageStream err: %v, must contain %s", err, want) - } -} - -func TestVTGateMessageAck(t *testing.T) { - ks := KsTestUnsharded - createSandbox(ks) - hcVTGateTest.Reset() - sbc := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, "0", topodatapb.TabletType_MASTER, true, 1, nil) - ids := []*querypb.Value{{ - Type: sqltypes.VarChar, - Value: []byte("1"), - }, { - Type: sqltypes.VarChar, - Value: []byte("2"), - }} - count, err := rpcVTGate.MessageAck(context.Background(), ks, "msg", ids) - require.NoError(t, err) - if count != 2 { - t.Errorf("MessageAck: %d, want 2", count) - } - if !sqltypes.Proto3ValuesEqual(sbc.MessageIDs, ids) { - t.Errorf("sbc1.MessageIDs: %v, want %v", sbc.MessageIDs, ids) - } -} - -func TestVTGateMessageAckKeyspaceIds(t *testing.T) { - ks := KsTestUnsharded - createSandbox(ks) - hcVTGateTest.Reset() - sbc := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, "0", topodatapb.TabletType_MASTER, true, 1, nil) - idKeyspaceIDs := []*vtgatepb.IdKeyspaceId{ - { - Id: &querypb.Value{ - Type: sqltypes.VarChar, - Value: []byte("1"), - }, - }, - { - Id: &querypb.Value{ - Type: sqltypes.VarChar, - Value: []byte("2"), - }, - }, - } - count, err := rpcVTGate.MessageAckKeyspaceIds(context.Background(), ks, "msg", idKeyspaceIDs) - require.NoError(t, err) - if count != 2 { - t.Errorf("MessageAck: %d, want 2", count) - } - wantids := []*querypb.Value{{ - Type: sqltypes.VarChar, - Value: []byte("1"), - }, { - Type: sqltypes.VarChar, - Value: []byte("2"), - }} - if !sqltypes.Proto3ValuesEqual(sbc.MessageIDs, wantids) { - t.Errorf("sbc1.MessageIDs: %v, want %v", sbc.MessageIDs, wantids) - } -} - -func TestVTGateBindVarError(t *testing.T) { - ks := KsTestUnsharded - createSandbox(ks) - hcVTGateTest.Reset() - ctx := context.Background() - session := &vtgatepb.Session{} - bindVars := map[string]*querypb.BindVariable{ - "v": { - Type: querypb.Type_EXPRESSION, - Value: []byte("1"), - }, - } - want := "v: invalid type specified for MakeValue: EXPRESSION" - - tcases := []struct { - name string - f func() error - }{{ - name: "Execute", - f: func() error { - _, _, err := rpcVTGate.Execute(ctx, session, "", bindVars) - return err - }, - }, { - name: "ExecuteBatch", - f: func() error { - _, _, err := rpcVTGate.ExecuteBatch(ctx, session, []string{""}, []map[string]*querypb.BindVariable{bindVars}) - return err - }, - }, { - name: "StreamExecute", - f: func() error { - return rpcVTGate.StreamExecute(ctx, session, "", bindVars, func(_ *sqltypes.Result) error { return nil }) - }, - }, { - name: "ExecuteShards", - f: func() error { - _, err := rpcVTGate.ExecuteShards(ctx, "", bindVars, "", []string{""}, topodatapb.TabletType_MASTER, session, false, nil) - return err - }, - }, { - name: "ExecuteKeyspaceIds", - f: func() error { - _, err := rpcVTGate.ExecuteKeyspaceIds(ctx, "", bindVars, "", [][]byte{}, topodatapb.TabletType_MASTER, session, false, nil) - return err - }, - }, { - name: "ExecuteKeyRanges", - f: func() error { - _, err := rpcVTGate.ExecuteKeyRanges(ctx, "", bindVars, "", []*topodatapb.KeyRange{}, topodatapb.TabletType_MASTER, session, false, nil) - return err - }, - }, { - name: "ExecuteEntityIds", - f: func() error { - _, err := rpcVTGate.ExecuteEntityIds(ctx, "", bindVars, "", "", []*vtgatepb.ExecuteEntityIdsRequest_EntityId{}, topodatapb.TabletType_MASTER, session, false, nil) - return err - }, - }, { - name: "ExecuteBatchShards", - f: func() error { - _, err := rpcVTGate.ExecuteBatchShards(ctx, - []*vtgatepb.BoundShardQuery{{ - Query: &querypb.BoundQuery{ - BindVariables: bindVars, - }, - }}, - topodatapb.TabletType_MASTER, false, session, nil) - return err - }, - }, { - name: "ExecuteBatchKeyspaceIds", - f: func() error { - _, err := rpcVTGate.ExecuteBatchKeyspaceIds(ctx, - []*vtgatepb.BoundKeyspaceIdQuery{{ - Query: &querypb.BoundQuery{ - BindVariables: bindVars, - }, - }}, - topodatapb.TabletType_MASTER, false, session, nil) - return err - }, - }, { - name: "StreamExecuteKeyspaceIds", - f: func() error { - return rpcVTGate.StreamExecuteKeyspaceIds(ctx, "", bindVars, "", [][]byte{}, topodatapb.TabletType_MASTER, nil, func(_ *sqltypes.Result) error { return nil }) - }, - }, { - name: "StreamExecuteKeyRanges", - f: func() error { - return rpcVTGate.StreamExecuteKeyRanges(ctx, "", bindVars, "", []*topodatapb.KeyRange{}, topodatapb.TabletType_MASTER, nil, func(_ *sqltypes.Result) error { return nil }) - }, - }, { - name: "StreamExecuteShards", - f: func() error { - return rpcVTGate.StreamExecuteShards(ctx, "", bindVars, "", []string{}, topodatapb.TabletType_MASTER, nil, func(_ *sqltypes.Result) error { return nil }) - }, - }} - for _, tcase := range tcases { - if err := tcase.f(); err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("%v error: %v, must contain %s", tcase.name, err, want) - } - } -} - -// Functions for testing -// keyspace_id and 'filtered_replication_unfriendly' -// annotations. -func TestAnnotatingExecuteKeyspaceIds(t *testing.T) { - keyspace, shards := setUpSandboxWithTwoShards("TestAnnotatingExecuteKeyspaceIds") - - _, err := rpcVTGate.ExecuteKeyspaceIds( - context.Background(), - "INSERT INTO table () VALUES();", - nil, - keyspace, - [][]byte{{0x10}}, - topodatapb.TabletType_MASTER, - nil, - false, - nil) - if err != nil { - t.Fatalf("want nil, got %v", err) - } - - verifyQueryAnnotatedWithKeyspaceID(t, []byte{0x10}, shards[0]) -} - -func TestAnnotatingExecuteKeyspaceIdsMultipleIds(t *testing.T) { - keyspace, shards := setUpSandboxWithTwoShards("TestAnnotatingExecuteKeyspaceIdsMultipleIds") - - _, err := rpcVTGate.ExecuteKeyspaceIds( - context.Background(), - "INSERT INTO table () VALUES();", - nil, - keyspace, - [][]byte{{0x10}, {0x15}}, - topodatapb.TabletType_MASTER, - nil, - false, - nil) - if err == nil || !strings.Contains(err.Error(), "DML should not span multiple keyspace_ids") { - t.Fatalf("want specific error, got %v", err) - } - - // Currently, there's logic in resolver.go for rejecting - // multiple-ids DML's so we expect 0 queries here. - verifyNumQueries(t, 0, shards[0].Queries) -} - -func TestAnnotatingExecuteKeyRanges(t *testing.T) { - keyspace, shards := setUpSandboxWithTwoShards("TestAnnotatingExecuteKeyRanges") - - _, err := rpcVTGate.ExecuteKeyRanges( - context.Background(), - "UPDATE table SET col1=1 WHERE col2>3;", - nil, - keyspace, - []*topodatapb.KeyRange{{Start: []byte{0x10}, End: []byte{0x40}}}, - topodatapb.TabletType_MASTER, - nil, - false, - nil) - if err != nil { - t.Fatalf("want nil, got %v", err) - } - - // Keyrange spans both shards. - verifyQueryAnnotatedAsUnfriendly(t, shards[0]) - verifyQueryAnnotatedAsUnfriendly(t, shards[1]) -} - -func TestAnnotatingExecuteEntityIds(t *testing.T) { - keyspace, shards := setUpSandboxWithTwoShards("TestAnnotatingExecuteEntityIds") - - _, err := rpcVTGate.ExecuteEntityIds( - context.Background(), - "INSERT INTO table () VALUES();", - nil, - keyspace, - "entity_column_name", - []*vtgatepb.ExecuteEntityIdsRequest_EntityId{ - { - Type: sqltypes.Int64, - Value: []byte("0"), - KeyspaceId: []byte{0x10}, // First shard. - }, - { - Type: sqltypes.Int64, - Value: []byte("1"), - KeyspaceId: []byte{0x25}, // Second shard. - }, - }, - topodatapb.TabletType_MASTER, - nil, - false, - nil) - if err != nil { - t.Fatalf("want nil, got %v", err) - } - - verifyQueryAnnotatedAsUnfriendly(t, shards[0]) - verifyQueryAnnotatedAsUnfriendly(t, shards[1]) -} - -func TestAnnotatingExecuteShards(t *testing.T) { - keyspace, shards := setUpSandboxWithTwoShards("TestAnnotatingExecuteShards") - _, err := rpcVTGate.ExecuteShards( - context.Background(), - "INSERT INTO table () VALUES();", - nil, - keyspace, - []string{"20-40"}, - topodatapb.TabletType_MASTER, - nil, - false, - nil) - if err != nil { - t.Fatalf("want nil, got %v", err) - } - - verifyQueryAnnotatedAsUnfriendly(t, shards[1]) -} - -func TestAnnotatingExecuteBatchKeyspaceIds(t *testing.T) { - keyspace, shards := setUpSandboxWithTwoShards("TestAnnotatingExecuteBatchKeyspaceIds") - _, err := rpcVTGate.ExecuteBatchKeyspaceIds( - context.Background(), - []*vtgatepb.BoundKeyspaceIdQuery{ - { - Query: &querypb.BoundQuery{ - Sql: "INSERT INTO table () VALUES();", - }, - Keyspace: keyspace, - KeyspaceIds: [][]byte{{0x10}}, - }, - { - Query: &querypb.BoundQuery{ - Sql: "UPDATE table SET col1=1 WHERE col2>3;", - }, - Keyspace: keyspace, - KeyspaceIds: [][]byte{{0x15}}, - }, - { - Query: &querypb.BoundQuery{ - Sql: "DELETE FROM table WHERE col1==4;", - }, - Keyspace: keyspace, - KeyspaceIds: [][]byte{{0x25}}, - }, - }, - topodatapb.TabletType_MASTER, - false, - nil, - nil) - if err != nil { - t.Fatalf("want nil, got %v", err) - } - - verifyBatchQueryAnnotatedWithKeyspaceIds( - t, - [][]byte{{0x10}, {0x15}}, - shards[0]) - verifyBatchQueryAnnotatedWithKeyspaceIds( - t, - [][]byte{{0x25}}, - shards[1]) -} - -func TestAnnotatingExecuteBatchKeyspaceIdsMultipleIds(t *testing.T) { - keyspace, shards := setUpSandboxWithTwoShards("TestAnnotatingExecuteBatchKeyspaceIdsMultipleIds") - _, err := rpcVTGate.ExecuteBatchKeyspaceIds( - context.Background(), - []*vtgatepb.BoundKeyspaceIdQuery{ - { - Query: &querypb.BoundQuery{ - Sql: "INSERT INTO table () VALUES();", - }, - Keyspace: keyspace, - KeyspaceIds: [][]byte{ - {0x10}, - {0x15}, - }, - }, - }, - topodatapb.TabletType_MASTER, - false, - nil, - nil) - if err != nil { - t.Fatalf("want nil, got %v", err) - } - - verifyBatchQueryAnnotatedAsUnfriendly( - t, - 1, // expectedNumQueries - shards[0]) -} - -func TestAnnotatingExecuteBatchShards(t *testing.T) { - keyspace, shards := setUpSandboxWithTwoShards("TestAnnotatingExecuteBatchShards") - - _, err := rpcVTGate.ExecuteBatchShards( - context.Background(), - []*vtgatepb.BoundShardQuery{ - { - Query: &querypb.BoundQuery{ - Sql: "INSERT INTO table () VALUES();", - }, - Keyspace: keyspace, - Shards: []string{"-20", "20-40"}, - }, - { - Query: &querypb.BoundQuery{ - Sql: "UPDATE table SET col1=1 WHERE col2>3;", - }, - Keyspace: keyspace, - Shards: []string{"-20"}, - }, - { - Query: &querypb.BoundQuery{ - Sql: "UPDATE table SET col1=1 WHERE col2>3;", - }, - Keyspace: keyspace, - Shards: []string{"20-40"}, - }, - { - Query: &querypb.BoundQuery{ - Sql: "DELETE FROM table WHERE col1==4;", - }, - Keyspace: keyspace, - Shards: []string{"20-40"}, - }, - }, - topodatapb.TabletType_MASTER, - false, - nil, - nil) - if err != nil { - t.Fatalf("want nil, got %v", err) - } - - verifyBatchQueryAnnotatedAsUnfriendly( - t, - 2, // expectedNumQueries - shards[0]) - verifyBatchQueryAnnotatedAsUnfriendly( - t, - 3, // expectedNumQueries - shards[1]) -} - -// TODO(erez): Add testing annotations of vtgate.Execute (V3) - -// Sets up a sandbox with two shards: -// the first named "-20" for the -20 keyrange, and -// the second named "20-40" for the 20-40 keyrange. -// It returns the created shards and as a convenience the given -// keyspace. -// -// NOTE: You should not call this method multiple times with -// the same 'keyspace' parameter: "shardGateway" caches connections -// for a keyspace, and may re-send queries to the shards created in -// a previous call to this method. -func setUpSandboxWithTwoShards(keyspace string) (string, []*sandboxconn.SandboxConn) { - shards := []*sandboxconn.SandboxConn{{}, {}} - createSandbox(keyspace) - hcVTGateTest.Reset() - shards[0] = hcVTGateTest.AddTestTablet("aa", "-20", 1, keyspace, "-20", topodatapb.TabletType_MASTER, true, 1, nil) - shards[1] = hcVTGateTest.AddTestTablet("aa", "20-40", 1, keyspace, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) - return keyspace, shards -} - -// Verifies that 'shard' was sent exactly one query and that it -// was annotated with 'expectedKeyspaceID' -func verifyQueryAnnotatedWithKeyspaceID(t *testing.T, expectedKeyspaceID []byte, shard *sandboxconn.SandboxConn) { - if !verifyNumQueries(t, 1, shard.Queries) { - return - } - verifyBoundQueryAnnotatedWithKeyspaceID(t, expectedKeyspaceID, shard.Queries[0]) -} - -// Verifies that 'shard' was sent exactly one query and that it -// was annotated as unfriendly. -func verifyQueryAnnotatedAsUnfriendly(t *testing.T, shard *sandboxconn.SandboxConn) { - if !verifyNumQueries(t, 1, shard.Queries) { - return - } - verifyBoundQueryAnnotatedAsUnfriendly(t, shard.Queries[0]) -} - -// Verifies 'queries' has exactly 'expectedNumQueries' elements. -// Returns true if verification succeeds. -func verifyNumQueries(t *testing.T, expectedNumQueries int, queries []*querypb.BoundQuery) bool { - numElements := len(queries) - if numElements != expectedNumQueries { - t.Errorf("want %v queries, got: %v (queries: %v)", expectedNumQueries, numElements, queries) - return false - } - return true -} - -// Verifies 'batchQueries' has exactly 'expectedNumQueries' elements. -// Returns true if verification succeeds. -func verifyNumBatchQueries(t *testing.T, expectedNumQueries int, batchQueries [][]*querypb.BoundQuery) bool { - numElements := len(batchQueries) - if numElements != expectedNumQueries { - t.Errorf("want %v batch queries, got: %v (batch queries: %v)", expectedNumQueries, numElements, batchQueries) - return false - } - return true -} - -func verifyBoundQueryAnnotatedWithKeyspaceID(t *testing.T, expectedKeyspaceID []byte, query *querypb.BoundQuery) { - verifyBoundQueryAnnotatedWithComment( - t, - "/* vtgate:: keyspace_id:"+hex.EncodeToString(expectedKeyspaceID)+" */", - query) -} - -func verifyBoundQueryAnnotatedAsUnfriendly(t *testing.T, query *querypb.BoundQuery) { - verifyBoundQueryAnnotatedWithComment( - t, - "/* vtgate:: filtered_replication_unfriendly */", - query) -} - -func verifyBoundQueryAnnotatedWithComment(t *testing.T, expectedComment string, query *querypb.BoundQuery) { - if !strings.Contains(query.Sql, expectedComment) { - t.Errorf("want query '%v' to be annotated with '%v'", query.Sql, expectedComment) - } -} - -// Verifies that 'shard' was sent exactly one batch-query and that its -// (single) queries are annotated with the elements of expectedKeyspaceIDs -// in order. -func verifyBatchQueryAnnotatedWithKeyspaceIds(t *testing.T, expectedKeyspaceIDs [][]byte, shard *sandboxconn.SandboxConn) { - if !verifyNumBatchQueries(t, 1, shard.BatchQueries) { - return - } - verifyBoundQueriesAnnotatedWithKeyspaceIds(t, expectedKeyspaceIDs, shard.BatchQueries[0]) -} - -// Verifies that 'shard' was sent exactly one batch-query and that its -// (single) queries are annotated as unfriendly. -func verifyBatchQueryAnnotatedAsUnfriendly(t *testing.T, expectedNumQueries int, shard *sandboxconn.SandboxConn) { - if !verifyNumBatchQueries(t, 1, shard.BatchQueries) { - return - } - verifyBoundQueriesAnnotatedAsUnfriendly(t, expectedNumQueries, shard.BatchQueries[0]) -} - -func verifyBoundQueriesAnnotatedWithKeyspaceIds(t *testing.T, expectedKeyspaceIDs [][]byte, queries []*querypb.BoundQuery) { - if !verifyNumQueries(t, len(expectedKeyspaceIDs), queries) { - return - } - for i := range queries { - verifyBoundQueryAnnotatedWithKeyspaceID(t, expectedKeyspaceIDs[i], queries[i]) - } -} - -func verifyBoundQueriesAnnotatedAsUnfriendly(t *testing.T, expectedNumQueries int, queries []*querypb.BoundQuery) { - if !verifyNumQueries(t, expectedNumQueries, queries) { - return - } - for i := range queries { - verifyBoundQueryAnnotatedAsUnfriendly(t, queries[i]) - } -} - -func testErrorPropagation(t *testing.T, sbcs []*sandboxconn.SandboxConn, before func(sbc *sandboxconn.SandboxConn), after func(sbc *sandboxconn.SandboxConn), expected vtrpcpb.Code) { - - // Execute - for _, sbc := range sbcs { - before(sbc) - } - _, _, err := rpcVTGate.Execute( - context.Background(), - masterSession, - "select id from t1", - nil, - ) - if err == nil { - t.Errorf("error %v not propagated for Execute", expected) - } else { - ec := vterrors.Code(err) - if ec != expected { - t.Errorf("unexpected error, got code %v err %v, want %v", ec, err, expected) - } - } - for _, sbc := range sbcs { - after(sbc) - } - - // ExecuteShards - for _, sbc := range sbcs { - before(sbc) - } - _, err = rpcVTGate.ExecuteShards(context.Background(), - "query", - nil, - KsTestUnsharded, - []string{"0"}, - topodatapb.TabletType_MASTER, - nil, - false, - executeOptions) - if err == nil { - t.Errorf("error %v not propagated for ExecuteShards", expected) - } else { - ec := vterrors.Code(err) - if ec != expected { - t.Errorf("unexpected error, got %v want %v: %v", ec, expected, err) - } - } - for _, sbc := range sbcs { - after(sbc) - } - - // ExecuteKeyspaceIds - for _, sbc := range sbcs { - before(sbc) - } - _, err = rpcVTGate.ExecuteKeyspaceIds(context.Background(), - "query", - nil, - KsTestUnsharded, - [][]byte{{0x10}}, - topodatapb.TabletType_MASTER, - nil, - false, - executeOptions) - if err == nil { - t.Errorf("error %v not propagated for ExecuteKeyspaceIds", expected) - } else { - ec := vterrors.Code(err) - if ec != expected { - t.Errorf("unexpected error, got %v want %v: %v", ec, expected, err) - } + ) + if err != nil { + t.Errorf("want nil, got %v", err) } - for _, sbc := range sbcs { - after(sbc) + if !reflect.DeepEqual(sandboxconn.SingleRowResult, qr) { + t.Errorf("want \n%+v, got \n%+v", sandboxconn.SingleRowResult, qr) } - - // ExecuteKeyRanges - for _, sbc := range sbcs { - before(sbc) + if !proto.Equal(sbc.Options[0], executeOptions) { + t.Errorf("got ExecuteOptions \n%+v, want \n%+v", sbc.Options[0], executeOptions) } - _, err = rpcVTGate.ExecuteKeyRanges(context.Background(), - "query", - nil, - KsTestUnsharded, - []*topodatapb.KeyRange{{End: []byte{0x20}}}, - topodatapb.TabletType_MASTER, +} + +func TestVTGateExecuteWithKeyspaceShard(t *testing.T) { + createSandbox(KsTestUnsharded) + hcVTGateTest.Reset() + hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, KsTestUnsharded, "0", topodatapb.TabletType_MASTER, true, 1, nil) + + // Valid keyspace. + _, qr, err := rpcVTGate.Execute( + context.Background(), + &vtgatepb.Session{ + TargetString: KsTestUnsharded, + }, + "select id from none", nil, - false, - executeOptions) - if err == nil { - t.Errorf("error %v not propagated for ExecuteKeyRanges", expected) - } else { - ec := vterrors.Code(err) - if ec != expected { - t.Errorf("unexpected error, got %v want %v: %v", ec, expected, err) - } + ) + if err != nil { + t.Errorf("want nil, got %v", err) } - for _, sbc := range sbcs { - after(sbc) + if !reflect.DeepEqual(sandboxconn.SingleRowResult, qr) { + t.Errorf("want \n%+v, got \n%+v", sandboxconn.SingleRowResult, qr) } - // ExecuteEntityIds - for _, sbc := range sbcs { - before(sbc) - } - _, err = rpcVTGate.ExecuteEntityIds(context.Background(), - "query", - nil, - KsTestUnsharded, - "kid", - []*vtgatepb.ExecuteEntityIdsRequest_EntityId{ - { - Type: sqltypes.VarBinary, - Value: []byte("id1"), - KeyspaceId: []byte{0x10}, - }, + // Invalid keyspace. + _, _, err = rpcVTGate.Execute( + context.Background(), + &vtgatepb.Session{ + TargetString: "invalid_keyspace", }, - topodatapb.TabletType_MASTER, + "select id from none", nil, - false, - executeOptions) - if err == nil { - t.Errorf("error %v not propagated for ExecuteEntityIds", expected) - } else { - ec := vterrors.Code(err) - if ec != expected { - t.Errorf("unexpected error, got %v want %v: %v", ec, expected, err) - } - } - for _, sbc := range sbcs { - after(sbc) + ) + want := "vtgate: : keyspace invalid_keyspace not found in vschema" + if err == nil || err.Error() != want { + t.Errorf("Execute: %v, want %s", err, want) } - // ExecuteBatchShards - for _, sbc := range sbcs { - before(sbc) - } - _, err = rpcVTGate.ExecuteBatchShards(context.Background(), - []*vtgatepb.BoundShardQuery{{ - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: KsTestUnsharded, - Shards: []string{"0", "0"}, - }, { - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: KsTestUnsharded, - Shards: []string{"0", "0"}, - }}, - topodatapb.TabletType_MASTER, - false, + // Valid keyspace/shard. + _, qr, err = rpcVTGate.Execute( + context.Background(), + &vtgatepb.Session{ + TargetString: KsTestUnsharded + ":0@master", + }, + "select id from none", nil, - executeOptions) - if err == nil { - t.Errorf("error %v not propagated for ExecuteBatchShards", expected) - } else { - ec := vterrors.Code(err) - if ec != expected { - t.Errorf("unexpected error, got %v want %v: %v", ec, expected, err) - } - } - statsKey := fmt.Sprintf("%s.%s.master.%v", "ExecuteBatchShards", KsTestUnsharded, vterrors.Code(err)) - if got, want := errorCounts.Counts()[statsKey], int64(1); got != want { - t.Errorf("errorCounts not increased for '%s': got = %v, want = %v", statsKey, got, want) + ) + if err != nil { + t.Errorf("want nil, got %v", err) } - for _, sbc := range sbcs { - after(sbc) + if !reflect.DeepEqual(sandboxconn.SingleRowResult, qr) { + t.Errorf("want \n%+v, got \n%+v", sandboxconn.SingleRowResult, qr) } - // ExecuteBatchKeyspaceIds - for _, sbc := range sbcs { - before(sbc) - } - kid10 := []byte{0x10} - kid30 := []byte{0x30} - _, err = rpcVTGate.ExecuteBatchKeyspaceIds(context.Background(), - []*vtgatepb.BoundKeyspaceIdQuery{{ - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: KsTestUnsharded, - KeyspaceIds: [][]byte{kid10, kid30}, - }, { - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: KsTestUnsharded, - KeyspaceIds: [][]byte{kid10, kid30}, - }}, - topodatapb.TabletType_MASTER, - false, + // Invalid keyspace/shard. + _, _, err = rpcVTGate.Execute( + context.Background(), + &vtgatepb.Session{ + TargetString: KsTestUnsharded + ":noshard@master", + }, + "select id from none", nil, - executeOptions) - if err == nil { - t.Errorf("error %v not propagated for ExecuteBatchKeyspaceIds", expected) - } else { - ec := vterrors.Code(err) - if ec != expected { - t.Errorf("unexpected error, got %v want %v: %v", ec, expected, err) - } - } - statsKey = fmt.Sprintf("%s.%s.master.%v", "ExecuteBatchKeyspaceIds", KsTestUnsharded, vterrors.Code(err)) - if got, want := errorCounts.Counts()[statsKey], int64(1); got != want { - t.Errorf("errorCounts not increased for '%s': got = %v, want = %v", statsKey, got, want) - } - for _, sbc := range sbcs { - after(sbc) + ) + want = "TestUnsharded.noshard.master: no valid tablet" + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("Execute: %v, want %s", err, want) } +} - // StreamExecute - for _, sbc := range sbcs { - before(sbc) - } - err = rpcVTGate.StreamExecute( +func TestVTGateStreamExecute(t *testing.T) { + ks := KsTestUnsharded + shard := "0" + createSandbox(ks) + hcVTGateTest.Reset() + sbc := hcVTGateTest.AddTestTablet("aa", "1.1.1.1", 1001, ks, shard, topodatapb.TabletType_MASTER, true, 1, nil) + var qrs []*sqltypes.Result + err := rpcVTGate.StreamExecute( context.Background(), - masterSession, + &vtgatepb.Session{ + TargetString: "@master", + Options: executeOptions, + }, "select id from t1", nil, func(r *sqltypes.Result) error { + qrs = append(qrs, r) return nil }, ) - if err == nil { - t.Errorf("error %v not propagated for StreamExecute", expected) - } else { - ec := vterrors.Code(err) - if ec != expected { - t.Errorf("unexpected error, got %v want %v: %v", ec, expected, err) - } + if err != nil { + t.Errorf("want nil, got %v", err) } - for _, sbc := range sbcs { - after(sbc) + want := []*sqltypes.Result{{ + Fields: sandboxconn.StreamRowResult.Fields, + }, { + Rows: sandboxconn.StreamRowResult.Rows, + }} + if !reflect.DeepEqual(want, qrs) { + t.Errorf("want \n%+v, got \n%+v", want, qrs) + } + if !proto.Equal(sbc.Options[0], executeOptions) { + t.Errorf("got ExecuteOptions \n%+v, want \n%+v", sbc.Options[0], executeOptions) } +} - // StreamExecuteShards - for _, sbc := range sbcs { - before(sbc) +func TestVTGateBindVarError(t *testing.T) { + ks := KsTestUnsharded + createSandbox(ks) + hcVTGateTest.Reset() + ctx := context.Background() + session := &vtgatepb.Session{} + bindVars := map[string]*querypb.BindVariable{ + "v": { + Type: querypb.Type_EXPRESSION, + Value: []byte("1"), + }, } - err = rpcVTGate.StreamExecuteShards(context.Background(), - "query", - nil, - KsTestUnsharded, - []string{"0"}, - topodatapb.TabletType_MASTER, - executeOptions, - func(r *sqltypes.Result) error { - return nil - }) - if err == nil { - t.Errorf("error %v not propagated for StreamExecuteShards", expected) - } else { - ec := vterrors.Code(err) - if ec != expected { - t.Errorf("unexpected error, got %v want %v: %v", ec, expected, err) + want := "v: invalid type specified for MakeValue: EXPRESSION" + + tcases := []struct { + name string + f func() error + }{{ + name: "Execute", + f: func() error { + _, _, err := rpcVTGate.Execute(ctx, session, "", bindVars) + return err + }, + }, { + name: "ExecuteBatch", + f: func() error { + _, _, err := rpcVTGate.ExecuteBatch(ctx, session, []string{""}, []map[string]*querypb.BindVariable{bindVars}) + return err + }, + }, { + name: "StreamExecute", + f: func() error { + return rpcVTGate.StreamExecute(ctx, session, "", bindVars, func(_ *sqltypes.Result) error { return nil }) + }, + }} + for _, tcase := range tcases { + if err := tcase.f(); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%v error: %v, must contain %s", tcase.name, err, want) } } - for _, sbc := range sbcs { - after(sbc) - } +} - // StreamExecuteKeyspaceIds +func testErrorPropagation(t *testing.T, sbcs []*sandboxconn.SandboxConn, before func(sbc *sandboxconn.SandboxConn), after func(sbc *sandboxconn.SandboxConn), expected vtrpcpb.Code) { + + // Execute for _, sbc := range sbcs { before(sbc) } - err = rpcVTGate.StreamExecuteKeyspaceIds(context.Background(), - "query", + _, _, err := rpcVTGate.Execute( + context.Background(), + masterSession, + "select id from t1", nil, - KsTestUnsharded, - [][]byte{{0x10}}, - topodatapb.TabletType_MASTER, - executeOptions, - func(r *sqltypes.Result) error { - return nil - }) + ) if err == nil { - t.Errorf("error %v not propagated for StreamExecuteKeyspaceIds", expected) + t.Errorf("error %v not propagated for Execute", expected) } else { ec := vterrors.Code(err) if ec != expected { - t.Errorf("unexpected error, got %v want %v: %v", ec, expected, err) + t.Errorf("unexpected error, got code %v err %v, want %v", ec, err, expected) } } for _, sbc := range sbcs { after(sbc) } - // StreamExecuteKeyRanges + // StreamExecute for _, sbc := range sbcs { before(sbc) } - err = rpcVTGate.StreamExecuteKeyRanges(context.Background(), - "query", + err = rpcVTGate.StreamExecute( + context.Background(), + masterSession, + "select id from t1", nil, - KsTestUnsharded, - []*topodatapb.KeyRange{{End: []byte{0x20}}}, - topodatapb.TabletType_MASTER, - executeOptions, func(r *sqltypes.Result) error { return nil - }) - if err == nil { - t.Errorf("error %v not propagated for StreamExecuteKeyRanges", expected) - } else { - ec := vterrors.Code(err) - if ec != expected { - t.Errorf("unexpected error, got %v want %v: %v", ec, expected, err) - } - } - for _, sbc := range sbcs { - after(sbc) - } - - // Begin is skipped, it doesn't end up going to the tablet. - - // Commit - for _, sbc := range sbcs { - before(sbc) - } - session := &vtgatepb.Session{ - InTransaction: true, - ShardSessions: []*vtgatepb.Session_ShardSession{{ - Target: &querypb.Target{ - Keyspace: KsTestUnsharded, - Shard: "0", - TabletType: topodatapb.TabletType_MASTER, - }, - TransactionId: 1, - }}, - } - err = rpcVTGate.Commit(context.Background(), false, session) + }, + ) if err == nil { - t.Errorf("error %v not propagated for Commit", expected) + t.Errorf("error %v not propagated for StreamExecute", expected) } else { ec := vterrors.Code(err) if ec != expected { @@ -2193,8 +298,6 @@ func testErrorPropagation(t *testing.T, sbcs []*sandboxconn.SandboxConn, before for _, sbc := range sbcs { after(sbc) } - - // Rollback is skipped, it doesn't forward errors. } // TestErrorPropagation tests an error returned by sandboxconn is @@ -2294,7 +397,12 @@ func TestErrorIssuesRollback(t *testing.T) { // Start a transaction, send one statement. // Simulate an error that should trigger a rollback: // vtrpcpb.Code_ABORTED case. - session, err := rpcVTGate.Begin(context.Background(), false) + session, _, err := rpcVTGate.Execute( + context.Background(), + &vtgatepb.Session{}, + "begin", + nil, + ) if err != nil { t.Fatalf("cannot start a transaction: %v", err) } @@ -2329,7 +437,12 @@ func TestErrorIssuesRollback(t *testing.T) { // Start a transaction, send one statement. // Simulate an error that should trigger a rollback: // vtrpcpb.ErrorCode_RESOURCE_EXHAUSTED case. - session, err = rpcVTGate.Begin(context.Background(), false) + session, _, err = rpcVTGate.Execute( + context.Background(), + &vtgatepb.Session{}, + "begin", + nil, + ) if err != nil { t.Fatalf("cannot start a transaction: %v", err) } @@ -2364,7 +477,12 @@ func TestErrorIssuesRollback(t *testing.T) { // Start a transaction, send one statement. // Simulate an error that should *not* trigger a rollback: // vtrpcpb.Code_ALREADY_EXISTS case. - session, err = rpcVTGate.Begin(context.Background(), false) + session, _, err = rpcVTGate.Execute( + context.Background(), + &vtgatepb.Session{}, + "begin", + nil, + ) if err != nil { t.Fatalf("cannot start a transaction: %v", err) } diff --git a/go/vt/vtgate/vtgateconn/vtgateconn.go b/go/vt/vtgate/vtgateconn/vtgateconn.go index 7c764343384..87ce3620ad4 100644 --- a/go/vt/vtgate/vtgateconn/vtgateconn.go +++ b/go/vt/vtgate/vtgateconn/vtgateconn.go @@ -56,115 +56,17 @@ func (conn *VTGateConn) Session(targetString string, options *querypb.ExecuteOpt } } -// ExecuteShards executes a non-streaming query for multiple shards on vtgate. -func (conn *VTGateConn) ExecuteShards(ctx context.Context, query string, keyspace string, shards []string, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - _, res, err := conn.impl.ExecuteShards(ctx, query, keyspace, shards, bindVars, tabletType, nil, options) - return res, err -} - -// ExecuteKeyspaceIds executes a non-streaming query for multiple keyspace_ids. -func (conn *VTGateConn) ExecuteKeyspaceIds(ctx context.Context, query string, keyspace string, keyspaceIds [][]byte, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - _, res, err := conn.impl.ExecuteKeyspaceIds(ctx, query, keyspace, keyspaceIds, bindVars, tabletType, nil, options) - return res, err -} - -// ExecuteKeyRanges executes a non-streaming query on a key range. -func (conn *VTGateConn) ExecuteKeyRanges(ctx context.Context, query string, keyspace string, keyRanges []*topodatapb.KeyRange, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - _, res, err := conn.impl.ExecuteKeyRanges(ctx, query, keyspace, keyRanges, bindVars, tabletType, nil, options) - return res, err -} - -// ExecuteEntityIds executes a non-streaming query for multiple entities. -func (conn *VTGateConn) ExecuteEntityIds(ctx context.Context, query string, keyspace string, entityColumnName string, entityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - _, res, err := conn.impl.ExecuteEntityIds(ctx, query, keyspace, entityColumnName, entityKeyspaceIDs, bindVars, tabletType, nil, options) - return res, err -} - -// ExecuteBatchShards executes a set of non-streaming queries for multiple shards. -// If "asTransaction" is true, vtgate will automatically create a transaction -// (per shard) that encloses all the batch queries. -func (conn *VTGateConn) ExecuteBatchShards(ctx context.Context, queries []*vtgatepb.BoundShardQuery, tabletType topodatapb.TabletType, asTransaction bool, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - _, res, err := conn.impl.ExecuteBatchShards(ctx, queries, tabletType, asTransaction, nil, options) - return res, err -} - -// ExecuteBatchKeyspaceIds executes a set of non-streaming queries for multiple keyspace ids. -// If "asTransaction" is true, vtgate will automatically create a transaction -// (per shard) that encloses all the batch queries. -func (conn *VTGateConn) ExecuteBatchKeyspaceIds(ctx context.Context, queries []*vtgatepb.BoundKeyspaceIdQuery, tabletType topodatapb.TabletType, asTransaction bool, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - _, res, err := conn.impl.ExecuteBatchKeyspaceIds(ctx, queries, tabletType, asTransaction, nil, options) - return res, err -} - -// StreamExecuteShards executes a streaming query on vtgate, on a set -// of shards. It returns a ResultStream and an error. First check the -// error. Then you can pull values from the ResultStream until io.EOF, -// or another error. -func (conn *VTGateConn) StreamExecuteShards(ctx context.Context, query string, keyspace string, shards []string, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (sqltypes.ResultStream, error) { - return conn.impl.StreamExecuteShards(ctx, query, keyspace, shards, bindVars, tabletType, options) -} - -// StreamExecuteKeyRanges executes a streaming query on vtgate, on a -// set of keyranges. It returns a ResultStream and an error. First check the -// error. Then you can pull values from the ResultStream until io.EOF, -// or another error. -func (conn *VTGateConn) StreamExecuteKeyRanges(ctx context.Context, query string, keyspace string, keyRanges []*topodatapb.KeyRange, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (sqltypes.ResultStream, error) { - return conn.impl.StreamExecuteKeyRanges(ctx, query, keyspace, keyRanges, bindVars, tabletType, options) -} - -// StreamExecuteKeyspaceIds executes a streaming query on vtgate, for -// the given keyspaceIds. It returns a ResultStream and an error. First check the -// error. Then you can pull values from the ResultStream until io.EOF, -// or another error. -func (conn *VTGateConn) StreamExecuteKeyspaceIds(ctx context.Context, query string, keyspace string, keyspaceIds [][]byte, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (sqltypes.ResultStream, error) { - return conn.impl.StreamExecuteKeyspaceIds(ctx, query, keyspace, keyspaceIds, bindVars, tabletType, options) -} - // ResolveTransaction resolves the 2pc transaction. func (conn *VTGateConn) ResolveTransaction(ctx context.Context, dtid string) error { return conn.impl.ResolveTransaction(ctx, dtid) } -// MessageStream streams messages. -func (conn *VTGateConn) MessageStream(ctx context.Context, keyspace string, shard string, keyRange *topodatapb.KeyRange, name string, callback func(*sqltypes.Result) error) error { - return conn.impl.MessageStream(ctx, keyspace, shard, keyRange, name, callback) -} - -// MessageAck acks messages. -func (conn *VTGateConn) MessageAck(ctx context.Context, keyspace string, name string, ids []*querypb.Value) (int64, error) { - return conn.impl.MessageAck(ctx, keyspace, name, ids) -} - -// MessageAckKeyspaceIds is part of the vtgate service API. It routes -// message acks based on the associated keyspace ids. -func (conn *VTGateConn) MessageAckKeyspaceIds(ctx context.Context, keyspace string, name string, idKeyspaceIDs []*vtgatepb.IdKeyspaceId) (int64, error) { - return conn.impl.MessageAckKeyspaceIds(ctx, keyspace, name, idKeyspaceIDs) -} - -// Begin starts a transaction and returns a VTGateTX. -func (conn *VTGateConn) Begin(ctx context.Context) (*VTGateTx, error) { - session, err := conn.impl.Begin(ctx, false /* singledb */) - if err != nil { - return nil, err - } - - return &VTGateTx{ - conn: conn, - session: session, - }, nil -} - // Close must be called for releasing resources. func (conn *VTGateConn) Close() { conn.impl.Close() conn.impl = nil } -// GetSrvKeyspace returns a topo.SrvKeyspace object. -func (conn *VTGateConn) GetSrvKeyspace(ctx context.Context, keyspace string) (*topodatapb.SrvKeyspace, error) { - return conn.impl.GetSrvKeyspace(ctx, keyspace) -} - // VStreamReader is returned by VStream. type VStreamReader interface { // Recv returns the next result on the stream. @@ -213,93 +115,6 @@ func (sn *VTGateSession) StreamExecute(ctx context.Context, query string, bindVa return sn.impl.StreamExecute(ctx, sn.session, query, bindVars) } -// VTGateTx defines an ongoing transaction. -// It should not be concurrently used across goroutines. -type VTGateTx struct { - conn *VTGateConn - session *vtgatepb.Session -} - -// ExecuteShards executes a query for multiple shards on vtgate within the current transaction. -func (tx *VTGateTx) ExecuteShards(ctx context.Context, query string, keyspace string, shards []string, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if tx.session == nil { - return nil, fmt.Errorf("executeShards: not in transaction") - } - session, res, err := tx.conn.impl.ExecuteShards(ctx, query, keyspace, shards, bindVars, tabletType, tx.session, options) - tx.session = session - return res, err -} - -// ExecuteKeyspaceIds executes a non-streaming query for multiple keyspace_ids. -func (tx *VTGateTx) ExecuteKeyspaceIds(ctx context.Context, query string, keyspace string, keyspaceIds [][]byte, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if tx.session == nil { - return nil, fmt.Errorf("executeKeyspaceIds: not in transaction") - } - session, res, err := tx.conn.impl.ExecuteKeyspaceIds(ctx, query, keyspace, keyspaceIds, bindVars, tabletType, tx.session, options) - tx.session = session - return res, err -} - -// ExecuteKeyRanges executes a non-streaming query on a key range. -func (tx *VTGateTx) ExecuteKeyRanges(ctx context.Context, query string, keyspace string, keyRanges []*topodatapb.KeyRange, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if tx.session == nil { - return nil, fmt.Errorf("executeKeyRanges: not in transaction") - } - session, res, err := tx.conn.impl.ExecuteKeyRanges(ctx, query, keyspace, keyRanges, bindVars, tabletType, tx.session, options) - tx.session = session - return res, err -} - -// ExecuteEntityIds executes a non-streaming query for multiple entities. -func (tx *VTGateTx) ExecuteEntityIds(ctx context.Context, query string, keyspace string, entityColumnName string, entityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (*sqltypes.Result, error) { - if tx.session == nil { - return nil, fmt.Errorf("executeEntityIds: not in transaction") - } - session, res, err := tx.conn.impl.ExecuteEntityIds(ctx, query, keyspace, entityColumnName, entityKeyspaceIDs, bindVars, tabletType, tx.session, options) - tx.session = session - return res, err -} - -// ExecuteBatchShards executes a set of non-streaming queries for multiple shards. -func (tx *VTGateTx) ExecuteBatchShards(ctx context.Context, queries []*vtgatepb.BoundShardQuery, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - if tx.session == nil { - return nil, fmt.Errorf("executeBatchShards: not in transaction") - } - session, res, err := tx.conn.impl.ExecuteBatchShards(ctx, queries, tabletType, false /* asTransaction */, tx.session, options) - tx.session = session - return res, err -} - -// ExecuteBatchKeyspaceIds executes a set of non-streaming queries for multiple keyspace ids. -func (tx *VTGateTx) ExecuteBatchKeyspaceIds(ctx context.Context, queries []*vtgatepb.BoundKeyspaceIdQuery, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) { - if tx.session == nil { - return nil, fmt.Errorf("executeBatchKeyspaceIds: not in transaction") - } - session, res, err := tx.conn.impl.ExecuteBatchKeyspaceIds(ctx, queries, tabletType, false /* asTransaction */, tx.session, options) - tx.session = session - return res, err -} - -// Commit commits the current transaction. -func (tx *VTGateTx) Commit(ctx context.Context) error { - if tx.session == nil { - return fmt.Errorf("commit: not in transaction") - } - err := tx.conn.impl.Commit(ctx, tx.session, false /* twopc */) - tx.session = nil - return err -} - -// Rollback rolls back the current transaction. -func (tx *VTGateTx) Rollback(ctx context.Context) error { - if tx.session == nil { - return nil - } - err := tx.conn.impl.Rollback(ctx, tx.session) - tx.session = nil - return err -} - // // The rest of this file is for the protocol implementations. // @@ -316,51 +131,9 @@ type Impl interface { // StreamExecute executes a streaming query on vtgate. This is a V3 function. StreamExecute(ctx context.Context, session *vtgatepb.Session, query string, bindVars map[string]*querypb.BindVariable) (sqltypes.ResultStream, error) - // ExecuteShards executes a non-streaming query for multiple shards on vtgate. This is a legacy function. - ExecuteShards(ctx context.Context, query string, keyspace string, shards []string, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, *sqltypes.Result, error) - - // ExecuteKeyspaceIds executes a non-streaming query for multiple keyspace_ids. This is a legacy function. - ExecuteKeyspaceIds(ctx context.Context, query string, keyspace string, keyspaceIds [][]byte, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, *sqltypes.Result, error) - - // ExecuteKeyRanges executes a non-streaming query on a key range. This is a legacy function. - ExecuteKeyRanges(ctx context.Context, query string, keyspace string, keyRanges []*topodatapb.KeyRange, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, *sqltypes.Result, error) - - // ExecuteEntityIds executes a non-streaming query for multiple entities. This is a legacy function. - ExecuteEntityIds(ctx context.Context, query string, keyspace string, entityColumnName string, entityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, *sqltypes.Result, error) - - // ExecuteBatchShards executes a set of non-streaming queries for multiple shards. This is a legacy function. - ExecuteBatchShards(ctx context.Context, queries []*vtgatepb.BoundShardQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, []sqltypes.Result, error) - - // ExecuteBatchKeyspaceIds executes a set of non-streaming queries for multiple keyspace ids. This is a legacy function. - ExecuteBatchKeyspaceIds(ctx context.Context, queries []*vtgatepb.BoundKeyspaceIdQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) (*vtgatepb.Session, []sqltypes.Result, error) - - // StreamExecuteShards executes a streaming query on vtgate, on a set of shards. This is a legacy function. - StreamExecuteShards(ctx context.Context, query string, keyspace string, shards []string, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (sqltypes.ResultStream, error) - - // StreamExecuteKeyRanges executes a streaming query on vtgate, on a set of keyranges. This is a legacy function. - StreamExecuteKeyRanges(ctx context.Context, query string, keyspace string, keyRanges []*topodatapb.KeyRange, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (sqltypes.ResultStream, error) - - // StreamExecuteKeyspaceIds executes a streaming query on vtgate, for the given keyspaceIds. This is a legacy function. - StreamExecuteKeyspaceIds(ctx context.Context, query string, keyspace string, keyspaceIds [][]byte, bindVars map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions) (sqltypes.ResultStream, error) - - // Begin starts a transaction and returns a VTGateTX. This is a legacy function. - Begin(ctx context.Context, singledb bool) (*vtgatepb.Session, error) - // Commit commits the current transaction. This is a legacy function. - Commit(ctx context.Context, session *vtgatepb.Session, twopc bool) error - // Rollback rolls back the current transaction. This is a legacy function. - Rollback(ctx context.Context, session *vtgatepb.Session) error - // ResolveTransaction resolves the specified 2pc transaction. ResolveTransaction(ctx context.Context, dtid string) error - // Messaging functions. - MessageStream(ctx context.Context, keyspace string, shard string, keyRange *topodatapb.KeyRange, name string, callback func(*sqltypes.Result) error) error - MessageAck(ctx context.Context, keyspace string, name string, ids []*querypb.Value) (int64, error) - MessageAckKeyspaceIds(ctx context.Context, keyspace string, name string, idKeyspaceIDs []*vtgatepb.IdKeyspaceId) (int64, error) - - // GetSrvKeyspace returns a topo.SrvKeyspace. - GetSrvKeyspace(ctx context.Context, keyspace string) (*topodatapb.SrvKeyspace, error) - // VStream streams binlogevents VStream(ctx context.Context, tabletType topodatapb.TabletType, vgtid *binlogdatapb.VGtid, filter *binlogdatapb.Filter) (VStreamReader, error) diff --git a/go/vt/vtgate/vtgateservice/interface.go b/go/vt/vtgate/vtgateservice/interface.go index 83bcb2919f1..6f0cabed28e 100644 --- a/go/vt/vtgate/vtgateservice/interface.go +++ b/go/vt/vtgate/vtgateservice/interface.go @@ -36,31 +36,9 @@ type VTGateService interface { ExecuteBatch(ctx context.Context, session *vtgatepb.Session, sqlList []string, bindVariablesList []map[string]*querypb.BindVariable) (*vtgatepb.Session, []sqltypes.QueryResponse, error) StreamExecute(ctx context.Context, session *vtgatepb.Session, sql string, bindVariables map[string]*querypb.BindVariable, callback func(*sqltypes.Result) error) error - // Legacy API - ExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) - ExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) - ExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) - ExecuteEntityIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, entityColumnName string, entityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId, tabletType topodatapb.TabletType, session *vtgatepb.Session, notInTransaction bool, options *querypb.ExecuteOptions) (*sqltypes.Result, error) - ExecuteBatchShards(ctx context.Context, queries []*vtgatepb.BoundShardQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) - ExecuteBatchKeyspaceIds(ctx context.Context, queries []*vtgatepb.BoundKeyspaceIdQuery, tabletType topodatapb.TabletType, asTransaction bool, session *vtgatepb.Session, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) - StreamExecuteShards(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, shards []string, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error - StreamExecuteKeyspaceIds(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyspaceIds [][]byte, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error - StreamExecuteKeyRanges(ctx context.Context, sql string, bindVariables map[string]*querypb.BindVariable, keyspace string, keyRanges []*topodatapb.KeyRange, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(*sqltypes.Result) error) error - Begin(ctx context.Context, singledb bool) (*vtgatepb.Session, error) - Commit(ctx context.Context, twopc bool, session *vtgatepb.Session) error - Rollback(ctx context.Context, session *vtgatepb.Session) error - // 2PC support ResolveTransaction(ctx context.Context, dtid string) error - // Messaging - MessageStream(ctx context.Context, keyspace string, shard string, keyRange *topodatapb.KeyRange, name string, callback func(*sqltypes.Result) error) error - MessageAck(ctx context.Context, keyspace string, name string, ids []*querypb.Value) (int64, error) - MessageAckKeyspaceIds(ctx context.Context, keyspace string, name string, idKeyspaceIDs []*vtgatepb.IdKeyspaceId) (int64, error) - - // Topology support - GetSrvKeyspace(ctx context.Context, keyspace string) (*topodatapb.SrvKeyspace, error) - // Update Stream methods VStream(ctx context.Context, tabletType topodatapb.TabletType, vgtid *binlogdatapb.VGtid, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error @@ -68,6 +46,3 @@ type VTGateService interface { // RPC implementation method, before calling any of the previous methods HandlePanic(err *error) } - -// Command to generate a mock for this interface with mockgen. -//go:generate mockgen -source $GOFILE -destination vtgateservice_testing/mock_vtgateservice.go -package vtgateservice_testing diff --git a/proto/vtgateservice.proto b/proto/vtgateservice.proto index 22ea1e9e7c0..894b969792c 100644 --- a/proto/vtgateservice.proto +++ b/proto/vtgateservice.proto @@ -25,7 +25,6 @@ option java_package="io.vitess.proto.grpc"; package vtgateservice; import "vtgate.proto"; -import "query.proto"; // Vitess is the main service to access a Vitess cluster. It is the API that vtgate // exposes to serve all queries. @@ -49,81 +48,10 @@ service Vitess { // API group: v3 rpc StreamExecute(vtgate.StreamExecuteRequest) returns (stream vtgate.StreamExecuteResponse) {}; - // ExecuteShards executes the query on the specified shards. - // API group: Custom Sharding - rpc ExecuteShards(vtgate.ExecuteShardsRequest) returns (vtgate.ExecuteShardsResponse) {}; - - // ExecuteKeyspaceIds executes the query based on the specified keyspace ids. - // API group: Range-based Sharding - rpc ExecuteKeyspaceIds(vtgate.ExecuteKeyspaceIdsRequest) returns (vtgate.ExecuteKeyspaceIdsResponse) {}; - - // ExecuteKeyRanges executes the query based on the specified key ranges. - // API group: Range-based Sharding - rpc ExecuteKeyRanges(vtgate.ExecuteKeyRangesRequest) returns (vtgate.ExecuteKeyRangesResponse) {}; - - // ExecuteEntityIds executes the query based on the specified external id to keyspace id map. - // API group: Range-based Sharding - rpc ExecuteEntityIds(vtgate.ExecuteEntityIdsRequest) returns (vtgate.ExecuteEntityIdsResponse) {}; - - // ExecuteBatchShards executes the list of queries on the specified shards. - // API group: Custom Sharding - rpc ExecuteBatchShards(vtgate.ExecuteBatchShardsRequest) returns (vtgate.ExecuteBatchShardsResponse) {}; - - // ExecuteBatchKeyspaceIds executes the list of queries based on the specified keyspace ids. - // API group: Range-based Sharding - rpc ExecuteBatchKeyspaceIds(vtgate.ExecuteBatchKeyspaceIdsRequest) returns (vtgate.ExecuteBatchKeyspaceIdsResponse) {}; - - // StreamExecuteShards executes a streaming query based on shards. - // Use this method if the query returns a large number of rows. - // API group: Custom Sharding - rpc StreamExecuteShards(vtgate.StreamExecuteShardsRequest) returns (stream vtgate.StreamExecuteShardsResponse) {}; - - // StreamExecuteKeyspaceIds executes a streaming query based on keyspace ids. - // Use this method if the query returns a large number of rows. - // API group: Range-based Sharding - rpc StreamExecuteKeyspaceIds(vtgate.StreamExecuteKeyspaceIdsRequest) returns (stream vtgate.StreamExecuteKeyspaceIdsResponse) {}; - - // StreamExecuteKeyRanges executes a streaming query based on key ranges. - // Use this method if the query returns a large number of rows. - // API group: Range-based Sharding - rpc StreamExecuteKeyRanges(vtgate.StreamExecuteKeyRangesRequest) returns (stream vtgate.StreamExecuteKeyRangesResponse) {}; - - // Begin a transaction. - // API group: Transactions - rpc Begin(vtgate.BeginRequest) returns (vtgate.BeginResponse) {}; - - // Commit a transaction. - // API group: Transactions - rpc Commit(vtgate.CommitRequest) returns (vtgate.CommitResponse) {}; - - // Rollback a transaction. - // API group: Transactions - rpc Rollback(vtgate.RollbackRequest) returns (vtgate.RollbackResponse) {}; - // ResolveTransaction resolves a transaction. // API group: Transactions rpc ResolveTransaction(vtgate.ResolveTransactionRequest) returns (vtgate.ResolveTransactionResponse) {}; - // MessageStream streams messages from a message table. - rpc MessageStream(vtgate.MessageStreamRequest) returns (stream query.MessageStreamResponse) {}; - - // MessageAck acks messages for a table. - rpc MessageAck(vtgate.MessageAckRequest) returns (query.MessageAckResponse) {}; - - // MessageAckKeyspaceIds routes Message Acks using the associated - // keyspace ids. - rpc MessageAckKeyspaceIds(vtgate.MessageAckKeyspaceIdsRequest) returns (query.MessageAckResponse) {}; - - // GetSrvKeyspace returns a SrvKeyspace object (as seen by this vtgate). - // This method is provided as a convenient way for clients to take a - // look at the sharding configuration for a Keyspace. Looking at the - // sharding information should not be used for routing queries (as the - // information may change, use the Execute calls for that). - // It is convenient for monitoring applications for instance, or if - // using custom sharding. - // API group: Topology - rpc GetSrvKeyspace(vtgate.GetSrvKeyspaceRequest) returns (vtgate.GetSrvKeyspaceResponse) {}; - // VStream streams binlog events from the requested sources. rpc VStream(vtgate.VStreamRequest) returns (stream vtgate.VStreamResponse) {}; } From 1e40496a5bedbba13e8521db11eb9bc6b15e08a3 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Wed, 18 Mar 2020 23:14:25 +0100 Subject: [PATCH 22/85] Update test cases for where clause Signed-off-by: Rohit Nayak --- .../tabletserver/vstreamer/planbuilder.go | 13 +++++++------ .../tabletserver/vstreamer/planbuilder_test.go | 18 +++++++++--------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index ed6a56b4ca1..d501fe768a6 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -37,13 +37,14 @@ import ( type Plan struct { Table *Table + // ColExprs is the list of column expressions to be sent + // in the stream. + ColExprs []ColExpr + // Filters is the list of filters to be applied to the columns // of the table. Filters []Filter - // ColExprs is the list of column expressions to be sent - // in the stream. - ColExprs []ColExpr } @@ -386,9 +387,9 @@ func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) er return err } plan.Filters = append(plan.Filters, Filter{ - Opcode: Equal, - ColNum: colnum, - Value: resolved, + Opcode: Equal, + ColNum: colnum, + Value: resolved, }) case *sqlparser.FuncExpr: if !expr.Name.EqualString("in_keyrange") { diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go index fa0a44ec1b0..e9ac90e98fa 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go @@ -245,7 +245,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("val"), Type: sqltypes.VarBinary, }}, - Filters: []Filter{{ + Filters: []Filter {{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, @@ -309,7 +309,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter{{ + Filters: []Filter {{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, @@ -331,7 +331,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter{{ + Filters: []Filter {{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, @@ -340,7 +340,7 @@ func TestPlanbuilder(t *testing.T) { KeyRange: nil, }}, }, - }, { + },{ inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select val, id from t1 where id = 1"}, outPlan: &Plan{ @@ -353,7 +353,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter{{ + Filters: []Filter {{ Opcode: Equal, ColNum: 0, Value: sqltypes.NewInt64(1), @@ -378,12 +378,12 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter{{ + Filters: []Filter {{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, Vindex: nil, - VindexColumns: []int{0, 1}, + VindexColumns: []int{0,1}, KeyRange: nil, }}, }, @@ -450,7 +450,7 @@ func TestPlanbuilder(t *testing.T) { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select *, id from t1"}, outErr: `unsupported: *, id`, - }, { + },{ inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val from t1 where max(id)"}, outErr: `unsupported constraint: max(id)`, @@ -515,7 +515,7 @@ func TestPlanbuilder(t *testing.T) { }) if plan != nil { plan.Table = nil - for ind := range plan.Filters { + for ind, _ := range plan.Filters { plan.Filters[ind].KeyRange = nil if plan.Filters[ind]. Opcode == VindexMatch { From a80b389056129406b852c69bf2eeec5631823efe Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 19 Mar 2020 18:30:54 +0100 Subject: [PATCH 23/85] Test case for int filter Signed-off-by: Rohit Nayak --- .../tabletserver/vstreamer/planbuilder.go | 8 +++----- .../tabletserver/vstreamer/planbuilder_test.go | 18 +++++++++--------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index d501fe768a6..48de92a03d3 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -44,10 +44,8 @@ type Plan struct { // Filters is the list of filters to be applied to the columns // of the table. Filters []Filter - } - // Opcode enumerates the operators supported in a where clause type Opcode int @@ -387,9 +385,9 @@ func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) er return err } plan.Filters = append(plan.Filters, Filter{ - Opcode: Equal, - ColNum: colnum, - Value: resolved, + Opcode: Equal, + ColNum: colnum, + Value: resolved, }) case *sqlparser.FuncExpr: if !expr.Name.EqualString("in_keyrange") { diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go index e9ac90e98fa..fa0a44ec1b0 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go @@ -245,7 +245,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("val"), Type: sqltypes.VarBinary, }}, - Filters: []Filter {{ + Filters: []Filter{{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, @@ -309,7 +309,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter {{ + Filters: []Filter{{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, @@ -331,7 +331,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter {{ + Filters: []Filter{{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, @@ -340,7 +340,7 @@ func TestPlanbuilder(t *testing.T) { KeyRange: nil, }}, }, - },{ + }, { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select val, id from t1 where id = 1"}, outPlan: &Plan{ @@ -353,7 +353,7 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter {{ + Filters: []Filter{{ Opcode: Equal, ColNum: 0, Value: sqltypes.NewInt64(1), @@ -378,12 +378,12 @@ func TestPlanbuilder(t *testing.T) { Alias: sqlparser.NewColIdent("id"), Type: sqltypes.Int64, }}, - Filters: []Filter {{ + Filters: []Filter{{ Opcode: VindexMatch, ColNum: 0, Value: sqltypes.NULL, Vindex: nil, - VindexColumns: []int{0,1}, + VindexColumns: []int{0, 1}, KeyRange: nil, }}, }, @@ -450,7 +450,7 @@ func TestPlanbuilder(t *testing.T) { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select *, id from t1"}, outErr: `unsupported: *, id`, - },{ + }, { inTable: t1, inRule: &binlogdatapb.Rule{Match: "t1", Filter: "select id, val from t1 where max(id)"}, outErr: `unsupported constraint: max(id)`, @@ -515,7 +515,7 @@ func TestPlanbuilder(t *testing.T) { }) if plan != nil { plan.Table = nil - for ind, _ := range plan.Filters { + for ind := range plan.Filters { plan.Filters[ind].KeyRange = nil if plan.Filters[ind]. Opcode == VindexMatch { From 25cd62343b75583408cd73e375507b32feb85202 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Fri, 20 Mar 2020 11:05:37 +0100 Subject: [PATCH 24/85] Added support for varbinary, tests for both int/varbinary Signed-off-by: Rohit Nayak --- .../tabletserver/vstreamer/planbuilder.go | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index 48de92a03d3..2d4d43819f7 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -110,13 +110,33 @@ func (plan *Plan) fields() []*querypb.Field { return fields } +func resolveValue(val sqltypes.Value) (sqltypes.Value, error) { + pv, err := sqlparser.NewPlanValue(sqlparser.NewStrVal(val.Raw())) + if err != nil { + return sqltypes.NULL, err + } + resolved, err := pv.ResolveValue(nil) + if err != nil { + return sqltypes.NULL, err + } + return resolved, nil +} + // filter filters the row against the plan. It returns false if the row did not match. // If the row matched, it returns the columns to be sent. func (plan *Plan) filter(values []sqltypes.Value) (bool, []sqltypes.Value, error) { for _, filter := range plan.Filters { switch filter.Opcode { case Equal: - result, err := sqltypes.NullsafeCompare(values[filter.ColNum], filter.Value) + filterValue, err := resolveValue(filter.Value) + if err != nil { + return false, nil, err + } + colValue, err := resolveValue(values[filter.ColNum]) + if err != nil { + return false, nil, err + } + result, err := sqltypes.NullsafeCompare(colValue, filterValue) if err != nil { return false, nil, err } From c7c7e3f40bce19a619f5f856a2313c101908f114 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sat, 21 Mar 2020 00:07:55 +0100 Subject: [PATCH 25/85] Handle varbinary correctly Signed-off-by: Rohit Nayak --- .../tabletserver/vstreamer/planbuilder.go | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index 2d4d43819f7..48de92a03d3 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -110,33 +110,13 @@ func (plan *Plan) fields() []*querypb.Field { return fields } -func resolveValue(val sqltypes.Value) (sqltypes.Value, error) { - pv, err := sqlparser.NewPlanValue(sqlparser.NewStrVal(val.Raw())) - if err != nil { - return sqltypes.NULL, err - } - resolved, err := pv.ResolveValue(nil) - if err != nil { - return sqltypes.NULL, err - } - return resolved, nil -} - // filter filters the row against the plan. It returns false if the row did not match. // If the row matched, it returns the columns to be sent. func (plan *Plan) filter(values []sqltypes.Value) (bool, []sqltypes.Value, error) { for _, filter := range plan.Filters { switch filter.Opcode { case Equal: - filterValue, err := resolveValue(filter.Value) - if err != nil { - return false, nil, err - } - colValue, err := resolveValue(values[filter.ColNum]) - if err != nil { - return false, nil, err - } - result, err := sqltypes.NullsafeCompare(colValue, filterValue) + result, err := sqltypes.NullsafeCompare(values[filter.ColNum], filter.Value) if err != nil { return false, nil, err } From f20ad6364a5686381775afa2a2f9bde10f191a0f Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sat, 21 Mar 2020 20:07:42 +0100 Subject: [PATCH 26/85] Vreplication and multiple where clause tests Signed-off-by: Rohit Nayak --- examples/are-you-alive/go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/are-you-alive/go.mod b/examples/are-you-alive/go.mod index 9bb69e1307b..5f3d04ddccd 100644 --- a/examples/are-you-alive/go.mod +++ b/examples/are-you-alive/go.mod @@ -4,6 +4,8 @@ go 1.12 require ( github.com/go-sql-driver/mysql v1.5.0 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/prometheus/client_golang v1.4.1 github.com/sirupsen/logrus v1.4.2 ) From 1d5eab1031e78bdfee4d1c8d06336824e48402c6 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sat, 21 Mar 2020 21:31:43 +0100 Subject: [PATCH 27/85] Only use styp if not null Signed-off-by: Rohit Nayak --- go/mysql/binlog_event_rbr.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index 5d7d2411c3e..01a5102e8b6 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -333,6 +333,7 @@ func printTimestamp(v uint32) *bytes.Buffer { // many bytes it takes. It only uses the querypb.Type value for the // signed flag. func CellValue(data []byte, pos int, typ byte, metadata uint16, styp querypb.Type) (sqltypes.Value, int, error) { + fmt.Printf("CellValue typ %v styp %v\n", typ, styp) switch typ { case TypeTiny: if sqltypes.IsSigned(styp) { From 3caa0ea90f3094f54229d6b279bd8d6a8d2132d7 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sat, 21 Mar 2020 22:07:41 +0100 Subject: [PATCH 28/85] Removed extraneous log Signed-off-by: Rohit Nayak --- go/mysql/binlog_event_rbr.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index 01a5102e8b6..5d7d2411c3e 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -333,7 +333,6 @@ func printTimestamp(v uint32) *bytes.Buffer { // many bytes it takes. It only uses the querypb.Type value for the // signed flag. func CellValue(data []byte, pos int, typ byte, metadata uint16, styp querypb.Type) (sqltypes.Value, int, error) { - fmt.Printf("CellValue typ %v styp %v\n", typ, styp) switch typ { case TypeTiny: if sqltypes.IsSigned(styp) { From 93852a412f6eb851a22e4d9ff53c1f7d127a4489 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Mon, 23 Mar 2020 22:14:13 -0700 Subject: [PATCH 29/85] deprecation: delete v2 java code Signed-off-by: Sugu Sougoumarane --- go/cmd/vtgateclienttest/services/echo.go | 13 + .../main/java/io/vitess/client/RpcClient.java | 166 ------- .../io/vitess/client/VTGateBlockingConn.java | 101 ---- .../io/vitess/client/VTGateBlockingTx.java | 146 ------ .../java/io/vitess/client/VTGateConn.java | 319 ------------ .../main/java/io/vitess/client/VTGateTx.java | 456 ------------------ .../java/io/vitess/client/RpcClientTest.java | 431 +---------------- .../test/java/io/vitess/client/TestUtil.java | 21 - .../io/vitess/client/grpc/GrpcClient.java | 139 ------ .../client/grpc/GrpcClientStaticAuthTest.java | 4 +- 10 files changed, 16 insertions(+), 1780 deletions(-) delete mode 100644 java/client/src/main/java/io/vitess/client/VTGateBlockingTx.java delete mode 100644 java/client/src/main/java/io/vitess/client/VTGateTx.java diff --git a/go/cmd/vtgateclienttest/services/echo.go b/go/cmd/vtgateclienttest/services/echo.go index 138215cbb5e..4323defe961 100644 --- a/go/cmd/vtgateclienttest/services/echo.go +++ b/go/cmd/vtgateclienttest/services/echo.go @@ -110,6 +110,19 @@ func (c *echoClient) Execute(ctx context.Context, session *vtgatepb.Session, sql return c.fallbackClient.Execute(ctx, session, sql, bindVariables) } +func (c *echoClient) StreamExecute(ctx context.Context, session *vtgatepb.Session, sql string, bindVariables map[string]*querypb.BindVariable, callback func(*sqltypes.Result) error) error { + if strings.HasPrefix(sql, EchoPrefix) { + callback(echoQueryResult(map[string]interface{}{ + "callerId": callerid.EffectiveCallerIDFromContext(ctx), + "query": sql, + "bindVars": bindVariables, + "session": session, + })) + return nil + } + return c.fallbackClient.StreamExecute(ctx, session, sql, bindVariables, callback) +} + func (c *echoClient) ExecuteBatch(ctx context.Context, session *vtgatepb.Session, sqlList []string, bindVariablesList []map[string]*querypb.BindVariable) (*vtgatepb.Session, []sqltypes.QueryResponse, error) { if len(sqlList) > 0 && strings.HasPrefix(sqlList[0], EchoPrefix) { var queryResponse []sqltypes.QueryResponse diff --git a/java/client/src/main/java/io/vitess/client/RpcClient.java b/java/client/src/main/java/io/vitess/client/RpcClient.java index eb8da35a96c..45cbf0c2cef 100644 --- a/java/client/src/main/java/io/vitess/client/RpcClient.java +++ b/java/client/src/main/java/io/vitess/client/RpcClient.java @@ -20,32 +20,9 @@ import io.vitess.proto.Query.QueryResult; import io.vitess.proto.Vtgate; -import io.vitess.proto.Vtgate.BeginRequest; -import io.vitess.proto.Vtgate.BeginResponse; -import io.vitess.proto.Vtgate.CommitRequest; -import io.vitess.proto.Vtgate.CommitResponse; -import io.vitess.proto.Vtgate.ExecuteBatchKeyspaceIdsRequest; -import io.vitess.proto.Vtgate.ExecuteBatchKeyspaceIdsResponse; -import io.vitess.proto.Vtgate.ExecuteBatchShardsRequest; -import io.vitess.proto.Vtgate.ExecuteBatchShardsResponse; -import io.vitess.proto.Vtgate.ExecuteEntityIdsRequest; -import io.vitess.proto.Vtgate.ExecuteEntityIdsResponse; -import io.vitess.proto.Vtgate.ExecuteKeyRangesRequest; -import io.vitess.proto.Vtgate.ExecuteKeyRangesResponse; -import io.vitess.proto.Vtgate.ExecuteKeyspaceIdsRequest; -import io.vitess.proto.Vtgate.ExecuteKeyspaceIdsResponse; import io.vitess.proto.Vtgate.ExecuteRequest; import io.vitess.proto.Vtgate.ExecuteResponse; -import io.vitess.proto.Vtgate.ExecuteShardsRequest; -import io.vitess.proto.Vtgate.ExecuteShardsResponse; -import io.vitess.proto.Vtgate.GetSrvKeyspaceRequest; -import io.vitess.proto.Vtgate.GetSrvKeyspaceResponse; -import io.vitess.proto.Vtgate.RollbackRequest; -import io.vitess.proto.Vtgate.RollbackResponse; -import io.vitess.proto.Vtgate.StreamExecuteKeyRangesRequest; -import io.vitess.proto.Vtgate.StreamExecuteKeyspaceIdsRequest; import io.vitess.proto.Vtgate.StreamExecuteRequest; -import io.vitess.proto.Vtgate.StreamExecuteShardsRequest; import io.vitess.proto.Vtgate.VStreamRequest; import io.vitess.proto.Vtgate.VStreamResponse; @@ -67,46 +44,6 @@ public interface RpcClient extends Closeable { ListenableFuture execute(Context ctx, ExecuteRequest request) throws SQLException; - /** - * Sends a single query to a set of shards. - * - *

See the - * proto - * definition for canonical documentation on this VTGate API. - */ - ListenableFuture executeShards(Context ctx, ExecuteShardsRequest request) - throws SQLException; - - /** - * Sends a query with a set of keyspace IDs. - * - *

See the - * proto - * definition for canonical documentation on this VTGate API. - */ - ListenableFuture executeKeyspaceIds( - Context ctx, ExecuteKeyspaceIdsRequest request) throws SQLException; - - /** - * Sends a query with a set of key ranges. - * - *

See the - * proto - * definition for canonical documentation on this VTGate API. - */ - ListenableFuture executeKeyRanges( - Context ctx, ExecuteKeyRangesRequest request) throws SQLException; - - /** - * Sends a query with a set of entity IDs. - * - *

See the - * proto - * definition for canonical documentation on this VTGate API. - */ - ListenableFuture executeEntityIds( - Context ctx, ExecuteEntityIdsRequest request) throws SQLException; - /** * Sends a list of queries using the VTGate V3 API. * @@ -118,26 +55,6 @@ ListenableFuture executeBatch(Context ctx, Vtgate.ExecuteBatchRequest request) throws SQLException; - /** - * Sends a list of queries to a set of shards. - * - *

See the - * proto - * definition for canonical documentation on this VTGate API. - */ - ListenableFuture executeBatchShards( - Context ctx, ExecuteBatchShardsRequest request) throws SQLException; - - /** - * Sends a list of queries with keyspace ids as bind variables. - * - *

See the - * proto - * definition for canonical documentation on this VTGate API. - */ - ListenableFuture executeBatchKeyspaceIds( - Context ctx, ExecuteBatchKeyspaceIdsRequest request) throws SQLException; - /** * Starts stream queries with the VTGate V3 API. * @@ -153,89 +70,6 @@ ListenableFuture executeBatchKeyspaceIds( StreamIterator streamExecute(Context ctx, StreamExecuteRequest request) throws SQLException; - /** - * Starts stream queries with multiple shards. - * - *

Note: Streaming queries are not asynchronous, because they typically shouldn't - * be used from a latency-critical serving path anyway. This method will return as soon as the - * request is initiated, but StreamIterator methods will block until the next chunk of results is - * received from the server. - * - *

See the - * proto - * definition for canonical documentation on this VTGate API. - */ - StreamIterator streamExecuteShards(Context ctx, StreamExecuteShardsRequest request) - throws SQLException; - - /** - * Starts a list of stream queries with keyspace ids as bind variables. - * - *

Note: Streaming queries are not asynchronous, because they typically shouldn't - * be used from a latency-critical serving path anyway. This method will return as soon as the - * request is initiated, but StreamIterator methods will block until the next chunk of results is - * received from the server. - * - *

See the - * proto - * definition for canonical documentation on this VTGate API. - */ - StreamIterator streamExecuteKeyspaceIds( - Context ctx, StreamExecuteKeyspaceIdsRequest request) throws SQLException; - - /** - * Starts stream query with a set of key ranges. - * - *

Note: Streaming queries are not asynchronous, because they typically shouldn't - * be used from a latency-critical serving path anyway. This method will return as soon as the - * request is initiated, but StreamIterator methods will block until the next chunk of results is - * received from the server. - * - *

See the - * proto - * definition for canonical documentation on this VTGate API. - */ - StreamIterator streamExecuteKeyRanges( - Context ctx, StreamExecuteKeyRangesRequest request) throws SQLException; - - /** - * Starts a transaction. - * - *

See the - * proto - * definition for canonical documentation on this VTGate API. - */ - ListenableFuture begin(Context ctx, BeginRequest request) throws SQLException; - - /** - * Commits a transaction. - * - *

See the - * proto - * definition for canonical documentation on this VTGate API. - */ - ListenableFuture commit(Context ctx, CommitRequest request) throws SQLException; - - /** - * Rolls back a pending transaction. - * - *

See the - * proto - * definition for canonical documentation on this VTGate API. - */ - ListenableFuture rollback(Context ctx, RollbackRequest request) - throws SQLException; - - /** - * Returns a list of serving keyspaces. - * - *

See the - * proto - * definition for canonical documentation on this VTGate API. - */ - ListenableFuture getSrvKeyspace( - Context ctx, GetSrvKeyspaceRequest request) throws SQLException; - /** * Starts streaming the vstream binlog events. * diff --git a/java/client/src/main/java/io/vitess/client/VTGateBlockingConn.java b/java/client/src/main/java/io/vitess/client/VTGateBlockingConn.java index 0473da5000a..7c261c679f4 100644 --- a/java/client/src/main/java/io/vitess/client/VTGateBlockingConn.java +++ b/java/client/src/main/java/io/vitess/client/VTGateBlockingConn.java @@ -19,11 +19,7 @@ import io.vitess.client.cursor.Cursor; import io.vitess.client.cursor.CursorWithError; import io.vitess.proto.Query; -import io.vitess.proto.Topodata.KeyRange; -import io.vitess.proto.Topodata.SrvKeyspace; import io.vitess.proto.Topodata.TabletType; -import io.vitess.proto.Vtgate.BoundKeyspaceIdQuery; -import io.vitess.proto.Vtgate.BoundShardQuery; import java.io.Closeable; import java.io.IOException; @@ -75,39 +71,6 @@ public Cursor execute(Context ctx, String query, Map bindVars, Tablet return conn.execute(ctx, query, bindVars, tabletType, includedFields).checkedGet(); } - public Cursor executeShards(Context ctx, String query, String keyspace, Iterable shards, - Map bindVars, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) throws SQLException { - return conn.executeShards(ctx, query, keyspace, shards, bindVars, tabletType, includedFields) - .checkedGet(); - } - - public Cursor executeKeyspaceIds(Context ctx, String query, String keyspace, - Iterable keyspaceIds, Map bindVars, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return conn - .executeKeyspaceIds(ctx, query, keyspace, keyspaceIds, bindVars, tabletType, includedFields) - .checkedGet(); - } - - public Cursor executeKeyRanges(Context ctx, String query, String keyspace, - Iterable keyRanges, Map bindVars, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return conn - .executeKeyRanges(ctx, query, keyspace, keyRanges, bindVars, tabletType, includedFields) - .checkedGet(); - } - - public Cursor executeEntityIds(Context ctx, String query, String keyspace, - String entityColumnName, Map entityKeyspaceIds, Map bindVars, - TabletType tabletType, Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return conn.executeEntityIds(ctx, query, keyspace, entityColumnName, entityKeyspaceIds, - bindVars, tabletType, includedFields).checkedGet(); - } - public List executeBatch(Context ctx, ArrayList queryList, @Nullable ArrayList> bindVarsList, TabletType tabletType, Query.ExecuteOptions.IncludedFields includedFields) throws SQLException { @@ -123,76 +86,12 @@ public List executeBatch(Context ctx, ArrayList queryLi .checkedGet(); } - /** - * Execute multiple keyspace ID queries as a batch. - * - * @param asTransaction If true, automatically create a transaction (per shard) that encloses - * all the batch queries. - */ - public List executeBatchShards(Context ctx, Iterable queries, - TabletType tabletType, boolean asTransaction, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return conn.executeBatchShards(ctx, queries, tabletType, asTransaction, includedFields) - .checkedGet(); - } - - /** - * Execute multiple keyspace ID queries as a batch. - * - * @param asTransaction If true, automatically create a transaction (per shard) that encloses - * all the batch queries. - */ - public List executeBatchKeyspaceIds(Context ctx, - Iterable queries, TabletType tabletType, - boolean asTransaction, Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return conn.executeBatchKeyspaceIds(ctx, queries, tabletType, asTransaction, includedFields) - .checkedGet(); - } - public Cursor streamExecute(Context ctx, String query, Map bindVars, TabletType tabletType, Query.ExecuteOptions.IncludedFields includedFields) throws SQLException { return conn.streamExecute(ctx, query, bindVars, tabletType, includedFields); } - public Cursor streamExecuteShards(Context ctx, String query, String keyspace, - Iterable shards, Map bindVars, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return conn - .streamExecuteShards(ctx, query, keyspace, shards, bindVars, tabletType, includedFields); - } - - public Cursor streamExecuteKeyspaceIds(Context ctx, String query, String keyspace, - Iterable keyspaceIds, Map bindVars, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return conn.streamExecuteKeyspaceIds(ctx, query, keyspace, keyspaceIds, bindVars, tabletType, - includedFields); - } - - public Cursor streamExecuteKeyRanges(Context ctx, String query, String keyspace, - Iterable keyRanges, Map bindVars, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return conn.streamExecuteKeyRanges(ctx, query, keyspace, keyRanges, bindVars, tabletType, - includedFields); - } - - public VTGateBlockingTx begin(Context ctx) throws SQLException { - return begin(ctx, false); - } - - public VTGateBlockingTx begin(Context ctx, boolean singleDB) throws SQLException { - return new VTGateBlockingTx(conn.begin(ctx, singleDB).checkedGet()); - } - - public SrvKeyspace getSrvKeyspace(Context ctx, String keyspace) throws SQLException { - return conn.getSrvKeyspace(ctx, keyspace).checkedGet(); - } - @Override public void close() throws IOException { conn.close(); diff --git a/java/client/src/main/java/io/vitess/client/VTGateBlockingTx.java b/java/client/src/main/java/io/vitess/client/VTGateBlockingTx.java deleted file mode 100644 index 87ecf482dff..00000000000 --- a/java/client/src/main/java/io/vitess/client/VTGateBlockingTx.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright 2019 The Vitess Authors. - - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - - * http://www.apache.org/licenses/LICENSE-2.0 - - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.vitess.client; - -import io.vitess.client.cursor.Cursor; -import io.vitess.client.cursor.CursorWithError; -import io.vitess.proto.Query; -import io.vitess.proto.Topodata.KeyRange; -import io.vitess.proto.Topodata.TabletType; -import io.vitess.proto.Vtgate.BoundKeyspaceIdQuery; -import io.vitess.proto.Vtgate.BoundShardQuery; - -import java.sql.SQLException; -import java.util.List; -import java.util.Map; - -import javax.annotation.Nullable; - -/** - * A synchronous wrapper around a VTGate transaction. - * - *

This is a wrapper around the asynchronous {@link VTGateTx} class - * that converts all methods to synchronous. - */ -@Deprecated -public class VTGateBlockingTx { - - private final VTGateTx tx; - - /** - * Wraps an existing {@link VTGateTx} in a synchronous API. - */ - public VTGateBlockingTx(VTGateTx tx) { - this.tx = tx; - } - - public Cursor execute(Context ctx, String query, Map bindVars, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return tx.execute(ctx, query, bindVars, tabletType, includedFields).checkedGet(); - } - - public Cursor executeShards( - Context ctx, - String query, - String keyspace, - Iterable shards, - Map bindVars, - TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return tx.executeShards(ctx, query, keyspace, shards, bindVars, tabletType, includedFields) - .checkedGet(); - } - - public Cursor executeKeyspaceIds( - Context ctx, - String query, - String keyspace, - Iterable keyspaceIds, - Map bindVars, - TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return tx - .executeKeyspaceIds(ctx, query, keyspace, keyspaceIds, bindVars, tabletType, includedFields) - .checkedGet(); - } - - public Cursor executeKeyRanges( - Context ctx, - String query, - String keyspace, - Iterable keyRanges, - Map bindVars, - TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return tx - .executeKeyRanges(ctx, query, keyspace, keyRanges, bindVars, tabletType, includedFields) - .checkedGet(); - } - - public Cursor executeEntityIds( - Context ctx, - String query, - String keyspace, - String entityColumnName, - Map entityKeyspaceIds, - Map bindVars, - TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return tx.executeEntityIds( - ctx, query, keyspace, entityColumnName, entityKeyspaceIds, bindVars, tabletType, - includedFields) - .checkedGet(); - } - - public List executeBatch(Context ctx, List queryList, - @Nullable List> bindVarsList, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return tx.executeBatch(ctx, queryList, bindVarsList, tabletType, includedFields).checkedGet(); - } - - public List executeBatchShards( - Context ctx, Iterable queries, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return tx.executeBatchShards(ctx, queries, tabletType, includedFields).checkedGet(); - } - - public List executeBatchKeyspaceIds( - Context ctx, Iterable queries, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return tx.executeBatchKeyspaceIds(ctx, queries, tabletType, includedFields).checkedGet(); - } - - public void commit(Context ctx) throws SQLException { - commit(ctx, false); - } - - public void commit(Context ctx, boolean atomic) throws SQLException { - tx.commit(ctx, atomic).checkedGet(); - } - - public void rollback(Context ctx) throws SQLException { - tx.rollback(ctx).checkedGet(); - } -} diff --git a/java/client/src/main/java/io/vitess/client/VTGateConn.java b/java/client/src/main/java/io/vitess/client/VTGateConn.java index bf1eb296823..16225911693 100644 --- a/java/client/src/main/java/io/vitess/client/VTGateConn.java +++ b/java/client/src/main/java/io/vitess/client/VTGateConn.java @@ -30,34 +30,11 @@ import io.vitess.client.cursor.SimpleCursor; import io.vitess.client.cursor.StreamCursor; import io.vitess.proto.Query; -import io.vitess.proto.Topodata.KeyRange; -import io.vitess.proto.Topodata.SrvKeyspace; import io.vitess.proto.Topodata.TabletType; import io.vitess.proto.Vtgate; -import io.vitess.proto.Vtgate.BeginRequest; -import io.vitess.proto.Vtgate.BeginResponse; -import io.vitess.proto.Vtgate.BoundKeyspaceIdQuery; -import io.vitess.proto.Vtgate.BoundShardQuery; -import io.vitess.proto.Vtgate.ExecuteBatchKeyspaceIdsRequest; -import io.vitess.proto.Vtgate.ExecuteBatchKeyspaceIdsResponse; -import io.vitess.proto.Vtgate.ExecuteBatchShardsRequest; -import io.vitess.proto.Vtgate.ExecuteBatchShardsResponse; -import io.vitess.proto.Vtgate.ExecuteEntityIdsRequest; -import io.vitess.proto.Vtgate.ExecuteEntityIdsResponse; -import io.vitess.proto.Vtgate.ExecuteKeyRangesRequest; -import io.vitess.proto.Vtgate.ExecuteKeyRangesResponse; -import io.vitess.proto.Vtgate.ExecuteKeyspaceIdsRequest; -import io.vitess.proto.Vtgate.ExecuteKeyspaceIdsResponse; import io.vitess.proto.Vtgate.ExecuteRequest; import io.vitess.proto.Vtgate.ExecuteResponse; -import io.vitess.proto.Vtgate.ExecuteShardsRequest; -import io.vitess.proto.Vtgate.ExecuteShardsResponse; -import io.vitess.proto.Vtgate.GetSrvKeyspaceRequest; -import io.vitess.proto.Vtgate.GetSrvKeyspaceResponse; -import io.vitess.proto.Vtgate.StreamExecuteKeyRangesRequest; -import io.vitess.proto.Vtgate.StreamExecuteKeyspaceIdsRequest; import io.vitess.proto.Vtgate.StreamExecuteRequest; -import io.vitess.proto.Vtgate.StreamExecuteShardsRequest; import java.io.Closeable; import java.io.IOException; @@ -141,130 +118,6 @@ public ListenableFuture apply(ExecuteResponse response) throws Exception directExecutor())); } - public SQLFuture executeShards(Context ctx, String query, String keyspace, - Iterable shards, @Nullable Map bindVars, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - ExecuteShardsRequest.Builder requestBuilder = - ExecuteShardsRequest.newBuilder() - .setQuery(Proto.bindQuery(checkNotNull(query), bindVars)) - .setKeyspace(checkNotNull(keyspace)) - .addAllShards(checkNotNull(shards)) - .setTabletType(checkNotNull(tabletType)) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - return new SQLFuture( - transformAsync( - client.executeShards(ctx, requestBuilder.build()), - new AsyncFunction() { - @Override - public ListenableFuture apply(ExecuteShardsResponse response) - throws Exception { - Proto.checkError(response.getError()); - return Futures.immediateFuture(new SimpleCursor(response.getResult())); - } - }, - directExecutor())); - } - - public SQLFuture executeKeyspaceIds(Context ctx, String query, String keyspace, - Iterable keyspaceIds, @Nullable Map bindVars, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - ExecuteKeyspaceIdsRequest.Builder requestBuilder = ExecuteKeyspaceIdsRequest.newBuilder() - .setQuery(Proto.bindQuery(checkNotNull(query), bindVars)) - .setKeyspace(checkNotNull(keyspace)) - .addAllKeyspaceIds( - Iterables.transform(checkNotNull(keyspaceIds), Proto.BYTE_ARRAY_TO_BYTE_STRING)) - .setTabletType(checkNotNull(tabletType)) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - return new SQLFuture( - transformAsync( - client.executeKeyspaceIds(ctx, requestBuilder.build()), - new AsyncFunction() { - @Override - public ListenableFuture apply(ExecuteKeyspaceIdsResponse response) - throws Exception { - Proto.checkError(response.getError()); - return Futures.immediateFuture(new SimpleCursor(response.getResult())); - } - }, - directExecutor())); - } - - public SQLFuture executeKeyRanges(Context ctx, String query, String keyspace, - Iterable keyRanges, @Nullable Map bindVars, - TabletType tabletType, Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - ExecuteKeyRangesRequest.Builder requestBuilder = ExecuteKeyRangesRequest.newBuilder() - .setQuery(Proto.bindQuery(checkNotNull(query), bindVars)) - .setKeyspace(checkNotNull(keyspace)) - .addAllKeyRanges(checkNotNull(keyRanges)) - .setTabletType(checkNotNull(tabletType)) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - return new SQLFuture( - transformAsync( - client.executeKeyRanges(ctx, requestBuilder.build()), - new AsyncFunction() { - @Override - public ListenableFuture apply(ExecuteKeyRangesResponse response) - throws Exception { - Proto.checkError(response.getError()); - return Futures.immediateFuture(new SimpleCursor(response.getResult())); - } - }, - directExecutor())); - } - - public SQLFuture executeEntityIds(Context ctx, String query, String keyspace, - String entityColumnName, Map entityKeyspaceIds, @Nullable Map bindVars, - TabletType tabletType, Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - ExecuteEntityIdsRequest.Builder requestBuilder = ExecuteEntityIdsRequest.newBuilder() - .setQuery(Proto.bindQuery(checkNotNull(query), bindVars)) - .setKeyspace(checkNotNull(keyspace)) - .setEntityColumnName(checkNotNull(entityColumnName)) - .addAllEntityKeyspaceIds(Iterables - .transform(entityKeyspaceIds.entrySet(), Proto.MAP_ENTRY_TO_ENTITY_KEYSPACE_ID)) - .setTabletType(checkNotNull(tabletType)) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - return new SQLFuture( - transformAsync( - client.executeEntityIds(ctx, requestBuilder.build()), - new AsyncFunction() { - @Override - public ListenableFuture apply(ExecuteEntityIdsResponse response) - throws Exception { - Proto.checkError(response.getError()); - return Futures.immediateFuture(new SimpleCursor(response.getResult())); - } - }, - directExecutor())); - } - public SQLFuture> executeBatch(Context ctx, List queryList, @Nullable List> bindVarsList, TabletType tabletType, Query.ExecuteOptions.IncludedFields includedFields) throws SQLException { @@ -315,80 +168,6 @@ public ListenableFuture> apply( directExecutor())); } - /** - * Execute multiple keyspace ID queries as a batch. - * - * @param asTransaction If true, automatically create a transaction (per shard) that encloses - * all the batch queries. - */ - public SQLFuture> executeBatchShards(Context ctx, - Iterable queries, TabletType tabletType, boolean asTransaction, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - ExecuteBatchShardsRequest.Builder requestBuilder = - ExecuteBatchShardsRequest.newBuilder() - .addAllQueries(checkNotNull(queries)) - .setTabletType(checkNotNull(tabletType)) - .setAsTransaction(asTransaction) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - return new SQLFuture>( - transformAsync( - client.executeBatchShards(ctx, requestBuilder.build()), - new AsyncFunction>() { - @Override - public ListenableFuture> apply(ExecuteBatchShardsResponse response) - throws Exception { - Proto.checkError(response.getError()); - return Futures.>immediateFuture( - Proto.toCursorList(response.getResultsList())); - } - }, - directExecutor())); - } - - /** - * Execute multiple keyspace ID queries as a batch. - * - * @param asTransaction If true, automatically create a transaction (per shard) that encloses - * all the batch queries. - */ - public SQLFuture> executeBatchKeyspaceIds(Context ctx, - Iterable queries, TabletType tabletType, - boolean asTransaction, Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - ExecuteBatchKeyspaceIdsRequest.Builder requestBuilder = - ExecuteBatchKeyspaceIdsRequest.newBuilder() - .addAllQueries(checkNotNull(queries)) - .setTabletType(checkNotNull(tabletType)) - .setAsTransaction(asTransaction) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - return new SQLFuture>( - transformAsync( - client.executeBatchKeyspaceIds(ctx, requestBuilder.build()), - new AsyncFunction>() { - @Override - public ListenableFuture> apply(ExecuteBatchKeyspaceIdsResponse response) - throws Exception { - Proto.checkError(response.getError()); - return Futures.>immediateFuture( - Proto.toCursorList(response.getResultsList())); - } - }, - directExecutor())); - } - public Cursor streamExecute(Context ctx, String query, @Nullable Map bindVars, TabletType tabletType, Query.ExecuteOptions.IncludedFields includedFields) throws SQLException { @@ -407,104 +186,6 @@ public Cursor streamExecute(Context ctx, String query, @Nullable Map return new StreamCursor(client.streamExecute(ctx, requestBuilder.build())); } - public Cursor streamExecuteShards(Context ctx, String query, String keyspace, - Iterable shards, @Nullable Map bindVars, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - StreamExecuteShardsRequest.Builder requestBuilder = StreamExecuteShardsRequest.newBuilder() - .setQuery(Proto.bindQuery(checkNotNull(query), bindVars)) - .setKeyspace(checkNotNull(keyspace)) - .addAllShards(checkNotNull(shards)) - .setTabletType(checkNotNull(tabletType)) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - return new StreamCursor(client.streamExecuteShards(ctx, requestBuilder.build())); - } - - public Cursor streamExecuteKeyspaceIds(Context ctx, String query, String keyspace, - Iterable keyspaceIds, @Nullable Map bindVars, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - StreamExecuteKeyspaceIdsRequest.Builder requestBuilder = StreamExecuteKeyspaceIdsRequest - .newBuilder() - .setQuery(Proto.bindQuery(checkNotNull(query), bindVars)) - .setKeyspace(checkNotNull(keyspace)) - .addAllKeyspaceIds( - Iterables.transform(checkNotNull(keyspaceIds), Proto.BYTE_ARRAY_TO_BYTE_STRING)) - .setTabletType(checkNotNull(tabletType)) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - return new StreamCursor(client.streamExecuteKeyspaceIds(ctx, requestBuilder.build())); - } - - public Cursor streamExecuteKeyRanges(Context ctx, String query, String keyspace, - Iterable keyRanges, @Nullable Map bindVars, - TabletType tabletType, Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - StreamExecuteKeyRangesRequest.Builder requestBuilder = StreamExecuteKeyRangesRequest - .newBuilder() - .setQuery(Proto.bindQuery(checkNotNull(query), bindVars)) - .setKeyspace(checkNotNull(keyspace)) - .addAllKeyRanges(checkNotNull(keyRanges)) - .setTabletType(checkNotNull(tabletType)) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - return new StreamCursor(client.streamExecuteKeyRanges(ctx, requestBuilder.build())); - } - - public SQLFuture begin(Context ctx) throws SQLException { - return begin(ctx, false); - } - - public SQLFuture begin(Context ctx, boolean singleDB) throws SQLException { - BeginRequest.Builder requestBuilder = BeginRequest.newBuilder().setSingleDb(singleDB); - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - return new SQLFuture( - transformAsync( - client.begin(ctx, requestBuilder.build()), - new AsyncFunction() { - @Override - public ListenableFuture apply(BeginResponse response) throws Exception { - return Futures.immediateFuture( - new VTGateTx(client, response.getSession(), keyspace)); - } - }, - directExecutor())); - } - - public SQLFuture getSrvKeyspace(Context ctx, String keyspace) throws SQLException { - GetSrvKeyspaceRequest.Builder requestBuilder = - GetSrvKeyspaceRequest.newBuilder().setKeyspace(checkNotNull(keyspace)); - return new SQLFuture( - transformAsync( - client.getSrvKeyspace(ctx, requestBuilder.build()), - new AsyncFunction() { - @Override - public ListenableFuture apply(GetSrvKeyspaceResponse response) - throws Exception { - return Futures.immediateFuture(response.getSrvKeyspace()); - } - }, - directExecutor())); - } - @Override public void close() throws IOException { client.close(); diff --git a/java/client/src/main/java/io/vitess/client/VTGateTx.java b/java/client/src/main/java/io/vitess/client/VTGateTx.java deleted file mode 100644 index 52b20ec169d..00000000000 --- a/java/client/src/main/java/io/vitess/client/VTGateTx.java +++ /dev/null @@ -1,456 +0,0 @@ -/* - * Copyright 2019 The Vitess Authors. - - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - - * http://www.apache.org/licenses/LICENSE-2.0 - - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.vitess.client; - -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.util.concurrent.Futures.transformAsync; -import static com.google.common.util.concurrent.MoreExecutors.directExecutor; - -import com.google.common.collect.Iterables; -import com.google.common.util.concurrent.AsyncFunction; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; - -import io.vitess.client.cursor.Cursor; -import io.vitess.client.cursor.CursorWithError; -import io.vitess.client.cursor.SimpleCursor; -import io.vitess.proto.Query; -import io.vitess.proto.Topodata.KeyRange; -import io.vitess.proto.Topodata.TabletType; -import io.vitess.proto.Vtgate; -import io.vitess.proto.Vtgate.BoundKeyspaceIdQuery; -import io.vitess.proto.Vtgate.BoundShardQuery; -import io.vitess.proto.Vtgate.CommitRequest; -import io.vitess.proto.Vtgate.CommitResponse; -import io.vitess.proto.Vtgate.ExecuteBatchKeyspaceIdsRequest; -import io.vitess.proto.Vtgate.ExecuteBatchKeyspaceIdsResponse; -import io.vitess.proto.Vtgate.ExecuteBatchShardsRequest; -import io.vitess.proto.Vtgate.ExecuteBatchShardsResponse; -import io.vitess.proto.Vtgate.ExecuteEntityIdsRequest; -import io.vitess.proto.Vtgate.ExecuteEntityIdsResponse; -import io.vitess.proto.Vtgate.ExecuteKeyRangesRequest; -import io.vitess.proto.Vtgate.ExecuteKeyRangesResponse; -import io.vitess.proto.Vtgate.ExecuteKeyspaceIdsRequest; -import io.vitess.proto.Vtgate.ExecuteKeyspaceIdsResponse; -import io.vitess.proto.Vtgate.ExecuteRequest; -import io.vitess.proto.Vtgate.ExecuteResponse; -import io.vitess.proto.Vtgate.ExecuteShardsRequest; -import io.vitess.proto.Vtgate.ExecuteShardsResponse; -import io.vitess.proto.Vtgate.RollbackRequest; -import io.vitess.proto.Vtgate.RollbackResponse; -import io.vitess.proto.Vtgate.Session; - -import java.sql.SQLDataException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import javax.annotation.Nullable; - -/** - * An asynchronous VTGate transaction session. - * - *

Because {@code VTGateTx} manages a session cookie, only one operation can be in flight at a - * time on a given instance. The methods are {@code synchronized} only because the session cookie is - * updated asynchronously when the RPC response comes back. - * - *

After calling any method that returns a {@link SQLFuture}, you must wait for that future to - * complete before calling any other methods on that {@code VTGateTx} instance. An {@link - * IllegalStateException} will be thrown if this constraint is violated. - * - *

All operations on {@code VTGateTx} are asynchronous, including those whose ultimate return - * type is {@link Void}, such as {@link #commit(Context)} and {@link #rollback(Context)}. You must - * still wait for the futures returned by these methods to complete and check the error on them - * (such as by calling {@code checkedGet()} before you can assume the operation has finished - * successfully. - * - *

If you prefer a synchronous API, you can use {@link VTGateBlockingConn#begin(Context)}, which - * returns a {@link VTGateBlockingTx} instead. - */ -@Deprecated -public class VTGateTx { - - private final RpcClient client; - private final String keyspace; - private Session session; - private SQLFuture lastCall; - - VTGateTx(RpcClient client, Session session, String keyspace) { - this.client = checkNotNull(client); - this.keyspace = checkNotNull(keyspace); - setSession(checkNotNull(session)); - } - - public synchronized SQLFuture execute(Context ctx, String query, Map bindVars, - TabletType tabletType, Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - checkCallIsAllowed("execute"); - ExecuteRequest.Builder requestBuilder = - ExecuteRequest.newBuilder() - .setQuery(Proto.bindQuery(query, bindVars)) - .setKeyspaceShard(keyspace) - .setTabletType(tabletType) - .setSession(session) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - SQLFuture call = - new SQLFuture<>( - transformAsync( - client.execute(ctx, requestBuilder.build()), - new AsyncFunction() { - @Override - public ListenableFuture apply(ExecuteResponse response) throws Exception { - setSession(response.getSession()); - Proto.checkError(response.getError()); - return Futures.immediateFuture(new SimpleCursor(response.getResult())); - } - }, - directExecutor())); - lastCall = call; - return call; - } - - public synchronized SQLFuture executeShards(Context ctx, String query, String keyspace, - Iterable shards, Map bindVars, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) throws SQLException { - checkCallIsAllowed("executeShards"); - ExecuteShardsRequest.Builder requestBuilder = ExecuteShardsRequest.newBuilder() - .setQuery(Proto.bindQuery(query, bindVars)) - .setKeyspace(keyspace) - .addAllShards(shards) - .setTabletType(tabletType) - .setSession(session) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - SQLFuture call = - new SQLFuture<>( - transformAsync( - client.executeShards(ctx, requestBuilder.build()), - new AsyncFunction() { - @Override - public ListenableFuture apply(ExecuteShardsResponse response) - throws Exception { - setSession(response.getSession()); - Proto.checkError(response.getError()); - return Futures.immediateFuture(new SimpleCursor(response.getResult())); - } - }, - directExecutor())); - lastCall = call; - return call; - } - - public synchronized SQLFuture executeKeyspaceIds(Context ctx, String query, - String keyspace, Iterable keyspaceIds, Map bindVars, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - checkCallIsAllowed("executeKeyspaceIds"); - ExecuteKeyspaceIdsRequest.Builder requestBuilder = ExecuteKeyspaceIdsRequest.newBuilder() - .setQuery(Proto.bindQuery(query, bindVars)) - .setKeyspace(keyspace) - .addAllKeyspaceIds(Iterables.transform(keyspaceIds, Proto.BYTE_ARRAY_TO_BYTE_STRING)) - .setTabletType(tabletType) - .setSession(session) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - SQLFuture call = - new SQLFuture<>( - transformAsync( - client.executeKeyspaceIds(ctx, requestBuilder.build()), - new AsyncFunction() { - @Override - public ListenableFuture apply(ExecuteKeyspaceIdsResponse response) - throws Exception { - setSession(response.getSession()); - Proto.checkError(response.getError()); - return Futures.immediateFuture(new SimpleCursor(response.getResult())); - } - }, - directExecutor())); - lastCall = call; - return call; - } - - public synchronized SQLFuture executeKeyRanges(Context ctx, String query, String keyspace, - Iterable keyRanges, Map bindVars, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - checkCallIsAllowed("executeKeyRanges"); - ExecuteKeyRangesRequest.Builder requestBuilder = ExecuteKeyRangesRequest.newBuilder() - .setQuery(Proto.bindQuery(query, bindVars)) - .setKeyspace(keyspace) - .addAllKeyRanges(keyRanges) - .setTabletType(tabletType) - .setSession(session) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - SQLFuture call = - new SQLFuture<>( - transformAsync( - client.executeKeyRanges(ctx, requestBuilder.build()), - new AsyncFunction() { - @Override - public ListenableFuture apply(ExecuteKeyRangesResponse response) - throws Exception { - setSession(response.getSession()); - Proto.checkError(response.getError()); - return Futures.immediateFuture(new SimpleCursor(response.getResult())); - } - }, - directExecutor())); - lastCall = call; - return call; - } - - public synchronized SQLFuture executeEntityIds(Context ctx, String query, String keyspace, - String entityColumnName, Map entityKeyspaceIds, Map bindVars, - TabletType tabletType, Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - checkCallIsAllowed("executeEntityIds"); - ExecuteEntityIdsRequest.Builder requestBuilder = ExecuteEntityIdsRequest.newBuilder() - .setQuery(Proto.bindQuery(query, bindVars)) - .setKeyspace(keyspace) - .setEntityColumnName(entityColumnName) - .addAllEntityKeyspaceIds(Iterables - .transform(entityKeyspaceIds.entrySet(), Proto.MAP_ENTRY_TO_ENTITY_KEYSPACE_ID)) - .setTabletType(tabletType) - .setSession(session) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - SQLFuture call = - new SQLFuture<>( - transformAsync( - client.executeEntityIds(ctx, requestBuilder.build()), - new AsyncFunction() { - @Override - public ListenableFuture apply(ExecuteEntityIdsResponse response) - throws Exception { - setSession(response.getSession()); - Proto.checkError(response.getError()); - return Futures.immediateFuture(new SimpleCursor(response.getResult())); - } - }, - directExecutor())); - lastCall = call; - return call; - } - - public SQLFuture> executeBatch(Context ctx, List queryList, - @Nullable List> bindVarsList, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - List queries = new ArrayList<>(); - - if (null != bindVarsList && bindVarsList.size() != queryList.size()) { - throw new SQLDataException( - "Size of SQL Query list does not match the bind variables list"); - } - - for (int i = 0; i < queryList.size(); ++i) { - queries.add(i, Proto.bindQuery(checkNotNull(queryList.get(i)), - bindVarsList == null ? null : bindVarsList.get(i))); - } - - Vtgate.ExecuteBatchRequest.Builder requestBuilder = - Vtgate.ExecuteBatchRequest.newBuilder() - .addAllQueries(checkNotNull(queries)) - .setKeyspaceShard(keyspace) - .setTabletType(checkNotNull(tabletType)) - .setSession(session) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - return new SQLFuture<>( - transformAsync( - client.executeBatch(ctx, requestBuilder.build()), - new AsyncFunction>() { - @Override - public ListenableFuture> apply( - Vtgate.ExecuteBatchResponse response) throws Exception { - setSession(response.getSession()); - Proto.checkError(response.getError()); - return Futures.immediateFuture( - Proto.fromQueryResponsesToCursorList(response.getResultsList())); - } - }, - directExecutor())); - } - - public synchronized SQLFuture> executeBatchShards(Context ctx, - Iterable queries, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) throws SQLException { - checkCallIsAllowed("executeBatchShards"); - ExecuteBatchShardsRequest.Builder requestBuilder = ExecuteBatchShardsRequest.newBuilder() - .addAllQueries(queries) - .setTabletType(tabletType) - .setSession(session) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - SQLFuture> call = - new SQLFuture<>( - transformAsync( - client.executeBatchShards(ctx, requestBuilder.build()), - new AsyncFunction>() { - @Override - public ListenableFuture> apply(ExecuteBatchShardsResponse response) - throws Exception { - setSession(response.getSession()); - Proto.checkError(response.getError()); - return Futures.>immediateFuture( - Proto.toCursorList(response.getResultsList())); - } - }, - directExecutor())); - lastCall = call; - return call; - } - - public synchronized SQLFuture> executeBatchKeyspaceIds(Context ctx, - Iterable queries, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) throws SQLException { - checkCallIsAllowed("executeBatchKeyspaceIds"); - ExecuteBatchKeyspaceIdsRequest.Builder requestBuilder = ExecuteBatchKeyspaceIdsRequest - .newBuilder() - .addAllQueries(queries) - .setTabletType(tabletType) - .setSession(session) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - SQLFuture> call = - new SQLFuture<>( - transformAsync( - client.executeBatchKeyspaceIds(ctx, requestBuilder.build()), - new AsyncFunction>() { - @Override - public ListenableFuture> apply( - ExecuteBatchKeyspaceIdsResponse response) throws Exception { - setSession(response.getSession()); - Proto.checkError(response.getError()); - return Futures.>immediateFuture( - Proto.toCursorList(response.getResultsList())); - } - }, - directExecutor())); - lastCall = call; - return call; - } - - public synchronized SQLFuture commit(Context ctx) throws SQLException { - return commit(ctx, false); - } - - public synchronized SQLFuture commit(Context ctx, boolean atomic) throws SQLException { - checkCallIsAllowed("commit"); - CommitRequest.Builder requestBuilder = CommitRequest.newBuilder().setSession(session) - .setAtomic(atomic); - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - SQLFuture call = - new SQLFuture<>( - transformAsync( - client.commit(ctx, requestBuilder.build()), - new AsyncFunction() { - @Override - public ListenableFuture apply(CommitResponse response) throws Exception { - setSession(null); - return Futures.immediateFuture(null); - } - }, - directExecutor())); - lastCall = call; - return call; - } - - public synchronized SQLFuture rollback(Context ctx) throws SQLException { - checkCallIsAllowed("rollback"); - RollbackRequest.Builder requestBuilder = RollbackRequest.newBuilder().setSession(session); - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - SQLFuture call = - new SQLFuture<>( - transformAsync( - client.rollback(ctx, requestBuilder.build()), - new AsyncFunction() { - @Override - public ListenableFuture apply(RollbackResponse response) throws Exception { - setSession(null); - return Futures.immediateFuture(null); - } - }, - directExecutor())); - lastCall = call; - return call; - } - - protected synchronized void checkCallIsAllowed(String call) throws SQLException { - // Calls are not allowed to overlap. - if (lastCall != null && !lastCall.isDone()) { - throw new IllegalStateException("Can't call " + call - + "() on a VTGateTx instance until the last asynchronous call is done."); - } - // All calls must occur within a valid transaction. - if (session == null || !session.getInTransaction()) { - throw new SQLDataException("Can't perform " + call + "() while not in transaction."); - } - } - - protected synchronized void setSession(Session session) { - this.session = session; - } -} diff --git a/java/client/src/test/java/io/vitess/client/RpcClientTest.java b/java/client/src/test/java/io/vitess/client/RpcClientTest.java index 4f59f3c177a..7ace70e33ee 100644 --- a/java/client/src/test/java/io/vitess/client/RpcClientTest.java +++ b/java/client/src/test/java/io/vitess/client/RpcClientTest.java @@ -31,11 +31,6 @@ import io.vitess.client.cursor.Row; import io.vitess.proto.Query; import io.vitess.proto.Query.Field; -import io.vitess.proto.Topodata.KeyRange; -import io.vitess.proto.Topodata.KeyspaceIdType; -import io.vitess.proto.Topodata.ShardReference; -import io.vitess.proto.Topodata.SrvKeyspace; -import io.vitess.proto.Topodata.SrvKeyspace.KeyspacePartition; import io.vitess.proto.Topodata.TabletType; import io.vitess.proto.Vtgate.VStreamRequest; import io.vitess.proto.Vtgate.VStreamResponse; @@ -116,28 +111,6 @@ public void setUp() throws SQLException, InterruptedException { private static final String QUERY = "test query with unicode: \u6211\u80fd\u541e\u4e0b\u73bb\u7483\u800c\u4e0d\u50b7\u8eab\u9ad4"; private static final String KEYSPACE = "test_keyspace"; - private static final List SHARDS = Arrays.asList("-80", "80-"); - private static final String SHARDS_ECHO = "[-80 80-]"; - - private static final List KEYSPACE_IDS = - Arrays.asList(new byte[]{1, 2, 3, 4}, new byte[]{5, 6, 7, 8}); - private static final String KEYSPACE_IDS_ECHO = "[[1 2 3 4] [5 6 7 8]]"; - - private static final List KEY_RANGES = - Arrays.asList(KeyRange.newBuilder().setStart(ByteString.copyFrom(new byte[]{1, 2, 3, 4})) - .setEnd(ByteString.copyFrom(new byte[]{5, 6, 7, 8})).build()); - private static final String KEY_RANGES_ECHO = - "[start:\"\\001\\002\\003\\004\" end:\"\\005\\006\\007\\010\" ]"; - - private static final ImmutableMap ENTITY_KEYSPACE_IDS = - new ImmutableMap.Builder() - .put(new byte[]{1, 2, 3}, 123) - .put(new byte[]{4, 5, 6}, 2.5) - .put(new byte[]{7, 8, 9}, new byte[]{1, 2, 3}) - .build(); - private static final String ENTITY_KEYSPACE_IDS_ECHO = - "[type:INT64 value:\"123\" keyspace_id:\"\\001\\002\\003\" type:FLOAT64 value:\"2.5\" keyspace_id:\"\\004\\005\\006\" type:VARBINARY value:\"\\001\\002\\003\" keyspace_id:\"\\007\\010\\t\" ]"; - private static final TabletType TABLET_TYPE = TabletType.REPLICA; private static final String TABLET_TYPE_ECHO = TABLET_TYPE.toString(); private static final Query.ExecuteOptions.IncludedFields ALL_FIELDS = Query.ExecuteOptions.IncludedFields.ALL; @@ -152,12 +125,8 @@ public void setUp() throws SQLException, InterruptedException { private static final String BIND_VARS_ECHO = "map[bytes:type:VARBINARY value:\"\\001\\002\\003\" float:type:FLOAT64 value:\"2.5\" int:type:INT64 value:\"123\" ]"; - private static final String SESSION_ECHO = "in_transaction:true "; - private static final String NONTX_V3_SESSION_ECHO = "autocommit:true target_string:\"test_keyspace@replica\" options: "; - private static final String V3_SESSION_ECHO = "in_transaction:true target_string:\"test_keyspace@replica\" options: "; - private static final CallerID CALLER_ID = CallerID.newBuilder().setPrincipal("test_principal") .setComponent("test_component").setSubcomponent("test_subcomponent").build(); private static final String CALLER_ID_ECHO = @@ -210,7 +179,7 @@ private void waitForVtgateclienttest() throws SQLException, InterruptedException while (DateTime.now().isBefore(deadline)) { try { ctx = Context.getDefault().withDeadlineAfter(Duration.standardSeconds(30)); - conn.getSrvKeyspace(ctx, "small"); + conn.execute(ctx, ECHO_PREFIX + QUERY, BIND_VARS, TABLET_TYPE, ALL_FIELDS); // RPC succeeded. Stop testing. break; } catch (SQLTransientException e) { @@ -245,73 +214,6 @@ public void testEchoExecute() throws Exception { Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); Assert.assertEquals(NONTX_V3_SESSION_ECHO, echo.get("session")); - - echo = getEcho( - conn.executeShards(ctx, ECHO_PREFIX + QUERY, KEYSPACE, SHARDS, BIND_VARS, TABLET_TYPE, - ALL_FIELDS)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(KEYSPACE, echo.get("keyspace")); - Assert.assertEquals(SHARDS_ECHO, echo.get("shards")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(TABLET_TYPE_ECHO, echo.get("tabletType")); - Assert.assertEquals(OPTIONS_ALL_FIELDS_ECHO, echo.get("options")); - - echo = getEcho(conn.executeKeyspaceIds(ctx, ECHO_PREFIX + QUERY, KEYSPACE, KEYSPACE_IDS, - BIND_VARS, TABLET_TYPE, ALL_FIELDS)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(KEYSPACE, echo.get("keyspace")); - Assert.assertEquals(KEYSPACE_IDS_ECHO, echo.get("keyspaceIds")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(TABLET_TYPE_ECHO, echo.get("tabletType")); - Assert.assertEquals(OPTIONS_ALL_FIELDS_ECHO, echo.get("options")); - - echo = getEcho(conn.executeKeyRanges(ctx, ECHO_PREFIX + QUERY, KEYSPACE, KEY_RANGES, BIND_VARS, - TABLET_TYPE, ALL_FIELDS)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(KEYSPACE, echo.get("keyspace")); - Assert.assertEquals(KEY_RANGES_ECHO, echo.get("keyRanges")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(TABLET_TYPE_ECHO, echo.get("tabletType")); - Assert.assertEquals(OPTIONS_ALL_FIELDS_ECHO, echo.get("options")); - - echo = getEcho(conn.executeEntityIds(ctx, ECHO_PREFIX + QUERY, KEYSPACE, "column1", - ENTITY_KEYSPACE_IDS, BIND_VARS, TABLET_TYPE, ALL_FIELDS)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(KEYSPACE, echo.get("keyspace")); - Assert.assertEquals("column1", echo.get("entityColumnName")); - Assert.assertEquals(ENTITY_KEYSPACE_IDS_ECHO, echo.get("entityIds")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(TABLET_TYPE_ECHO, echo.get("tabletType")); - Assert.assertEquals(OPTIONS_ALL_FIELDS_ECHO, echo.get("options")); - - echo = getEcho(conn.executeBatchShards(ctx, - Arrays.asList(Proto.bindShardQuery(KEYSPACE, SHARDS, ECHO_PREFIX + QUERY, BIND_VARS)), - TABLET_TYPE, true, ALL_FIELDS).get(0)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(KEYSPACE, echo.get("keyspace")); - Assert.assertEquals(SHARDS_ECHO, echo.get("shards")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(TABLET_TYPE_ECHO, echo.get("tabletType")); - Assert.assertEquals("true", echo.get("asTransaction")); - Assert.assertEquals(OPTIONS_ALL_FIELDS_ECHO, echo.get("options")); - - echo = getEcho(conn.executeBatchKeyspaceIds(ctx, - Arrays.asList( - Proto.bindKeyspaceIdQuery(KEYSPACE, KEYSPACE_IDS, ECHO_PREFIX + QUERY, BIND_VARS)), - TABLET_TYPE, true, ALL_FIELDS).get(0)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(KEYSPACE, echo.get("keyspace")); - Assert.assertEquals(KEYSPACE_IDS_ECHO, echo.get("keyspaceIds")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(TABLET_TYPE_ECHO, echo.get("tabletType")); - Assert.assertEquals("true", echo.get("asTransaction")); - Assert.assertEquals(OPTIONS_ALL_FIELDS_ECHO, echo.get("options")); } @Test @@ -324,154 +226,6 @@ public void testEchoStreamExecute() throws Exception { Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); Assert.assertEquals(NONTX_V3_SESSION_ECHO, echo.get("session")); - - echo = getEcho(conn.streamExecuteShards(ctx, ECHO_PREFIX + QUERY, KEYSPACE, SHARDS, BIND_VARS, - TABLET_TYPE, ALL_FIELDS)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(KEYSPACE, echo.get("keyspace")); - Assert.assertEquals(SHARDS_ECHO, echo.get("shards")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(TABLET_TYPE_ECHO, echo.get("tabletType")); - Assert.assertEquals(OPTIONS_ALL_FIELDS_ECHO, echo.get("options")); - - echo = getEcho(conn.streamExecuteKeyspaceIds(ctx, ECHO_PREFIX + QUERY, KEYSPACE, KEYSPACE_IDS, - BIND_VARS, TABLET_TYPE, ALL_FIELDS)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(KEYSPACE, echo.get("keyspace")); - Assert.assertEquals(KEYSPACE_IDS_ECHO, echo.get("keyspaceIds")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(TABLET_TYPE_ECHO, echo.get("tabletType")); - Assert.assertEquals(OPTIONS_ALL_FIELDS_ECHO, echo.get("options")); - - echo = getEcho(conn.streamExecuteKeyRanges(ctx, ECHO_PREFIX + QUERY, KEYSPACE, KEY_RANGES, - BIND_VARS, TABLET_TYPE, ALL_FIELDS)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(KEYSPACE, echo.get("keyspace")); - Assert.assertEquals(KEY_RANGES_ECHO, echo.get("keyRanges")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(TABLET_TYPE_ECHO, echo.get("tabletType")); - Assert.assertEquals(OPTIONS_ALL_FIELDS_ECHO, echo.get("options")); - } - - @Test - public void testEchoTransactionExecute() throws Exception { - Map echo; - - VTGateBlockingTx tx = conn.begin(ctx); - - echo = getEcho(tx.execute(ctx, ECHO_PREFIX + QUERY, BIND_VARS, TABLET_TYPE, ALL_FIELDS)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(V3_SESSION_ECHO, echo.get("session")); - - // V3 returns additional session artifacts that V2 - // doesn't care about. So, start with a new session - // before testing V2 functionality. - tx.rollback(ctx); - tx = conn.begin(ctx); - - echo = getEcho( - tx.executeShards(ctx, ECHO_PREFIX + QUERY, KEYSPACE, SHARDS, BIND_VARS, TABLET_TYPE, - ALL_FIELDS)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(KEYSPACE, echo.get("keyspace")); - Assert.assertEquals(SHARDS_ECHO, echo.get("shards")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(TABLET_TYPE_ECHO, echo.get("tabletType")); - Assert.assertEquals(SESSION_ECHO, echo.get("session")); - Assert.assertEquals("false", echo.get("notInTransaction")); - Assert.assertEquals(OPTIONS_ALL_FIELDS_ECHO, echo.get("options")); - - echo = getEcho(tx.executeKeyspaceIds(ctx, ECHO_PREFIX + QUERY, KEYSPACE, KEYSPACE_IDS, - BIND_VARS, TABLET_TYPE, ALL_FIELDS)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(KEYSPACE, echo.get("keyspace")); - Assert.assertEquals(KEYSPACE_IDS_ECHO, echo.get("keyspaceIds")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(TABLET_TYPE_ECHO, echo.get("tabletType")); - Assert.assertEquals(SESSION_ECHO, echo.get("session")); - Assert.assertEquals("false", echo.get("notInTransaction")); - Assert.assertEquals(OPTIONS_ALL_FIELDS_ECHO, echo.get("options")); - - echo = getEcho(tx.executeKeyRanges(ctx, ECHO_PREFIX + QUERY, KEYSPACE, KEY_RANGES, BIND_VARS, - TABLET_TYPE, ALL_FIELDS)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(KEYSPACE, echo.get("keyspace")); - Assert.assertEquals(KEY_RANGES_ECHO, echo.get("keyRanges")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(TABLET_TYPE_ECHO, echo.get("tabletType")); - Assert.assertEquals(SESSION_ECHO, echo.get("session")); - Assert.assertEquals("false", echo.get("notInTransaction")); - Assert.assertEquals(OPTIONS_ALL_FIELDS_ECHO, echo.get("options")); - - echo = getEcho(tx.executeEntityIds(ctx, ECHO_PREFIX + QUERY, KEYSPACE, "column1", - ENTITY_KEYSPACE_IDS, BIND_VARS, TABLET_TYPE, ALL_FIELDS)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(KEYSPACE, echo.get("keyspace")); - Assert.assertEquals("column1", echo.get("entityColumnName")); - Assert.assertEquals(ENTITY_KEYSPACE_IDS_ECHO, echo.get("entityIds")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(TABLET_TYPE_ECHO, echo.get("tabletType")); - Assert.assertEquals(SESSION_ECHO, echo.get("session")); - Assert.assertEquals("false", echo.get("notInTransaction")); - Assert.assertEquals(OPTIONS_ALL_FIELDS_ECHO, echo.get("options")); - - tx.rollback(ctx); - tx = conn.begin(ctx); - - echo = getEcho(tx.executeBatchShards(ctx, - Arrays.asList(Proto.bindShardQuery(KEYSPACE, SHARDS, ECHO_PREFIX + QUERY, BIND_VARS)), - TABLET_TYPE, ALL_FIELDS).get(0)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(KEYSPACE, echo.get("keyspace")); - Assert.assertEquals(SHARDS_ECHO, echo.get("shards")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(TABLET_TYPE_ECHO, echo.get("tabletType")); - Assert.assertEquals(SESSION_ECHO, echo.get("session")); - Assert.assertEquals("false", echo.get("asTransaction")); - Assert.assertEquals(OPTIONS_ALL_FIELDS_ECHO, echo.get("options")); - - echo = getEcho(tx.executeBatchKeyspaceIds(ctx, - Arrays.asList( - Proto.bindKeyspaceIdQuery(KEYSPACE, KEYSPACE_IDS, ECHO_PREFIX + QUERY, BIND_VARS)), - TABLET_TYPE, ALL_FIELDS).get(0)); - Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); - Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); - Assert.assertEquals(KEYSPACE, echo.get("keyspace")); - Assert.assertEquals(KEYSPACE_IDS_ECHO, echo.get("keyspaceIds")); - Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); - Assert.assertEquals(TABLET_TYPE_ECHO, echo.get("tabletType")); - Assert.assertEquals(SESSION_ECHO, echo.get("session")); - Assert.assertEquals("false", echo.get("asTransaction")); - Assert.assertEquals(OPTIONS_ALL_FIELDS_ECHO, echo.get("options")); - - tx.commit(ctx); - } - - @Test - public void testGetSrvKeyspace() throws Exception { - SrvKeyspace expected = SrvKeyspace.newBuilder() - .addPartitions(KeyspacePartition.newBuilder().setServedType(TabletType.REPLICA) - .addShardReferences(ShardReference.newBuilder().setName("shard0").setKeyRange(KeyRange - .newBuilder().setStart(ByteString.copyFrom(new byte[]{0x40, 0, 0, 0, 0, 0, 0, 0})) - .setEnd(ByteString.copyFrom(new byte[]{(byte) 0x80, 0, 0, 0, 0, 0, 0, 0})).build()) - .build()) - .build()) - .setShardingColumnName("sharding_column_name") - .setShardingColumnType(KeyspaceIdType.UINT64).addServedFrom(SrvKeyspace.ServedFrom - .newBuilder().setTabletType(TabletType.MASTER).setKeyspace("other_keyspace").build()) - .build(); - SrvKeyspace actual = conn.getSrvKeyspace(ctx, "big"); - Assert.assertEquals(expected, actual); } abstract static class Executable { @@ -512,72 +266,6 @@ void checkStreamExecuteErrors(Executable exe) { checkExecuteErrors(exe, false); } - abstract static class TransactionExecutable { - - abstract void execute(VTGateBlockingTx tx, String query) throws Exception; - } - - void checkTransactionExecuteErrors(TransactionExecutable exe) throws Exception { - for (String error : EXECUTE_ERRORS.keySet()) { - Class cls = EXECUTE_ERRORS.get(error); - - try { - VTGateBlockingTx tx = conn.begin(ctx); - String query = ERROR_PREFIX + error; - exe.execute(tx, query); - Assert.fail("no exception thrown for " + query); - } catch (Exception e) { - Assert.assertEquals(cls, e.getClass()); - - if (error.equals("integrity error")) { - // The mock test server sends back errno:1062 sqlstate:23000 for this case. - // Make sure these values get properly extracted by the client. - SQLException sqlException = (SQLException) e; - Assert.assertEquals(1062, sqlException.getErrorCode()); - Assert.assertEquals("23000", sqlException.getSQLState()); - } - } - - // Don't close the transaction on partial error. - VTGateBlockingTx tx = conn.begin(ctx); - try { - String query = PARTIAL_ERROR_PREFIX + error; - exe.execute(tx, query); - Assert.fail("no exception thrown for " + query); - } catch (Exception e) { - Assert.assertEquals(cls, e.getClass()); - - if (error.equals("integrity error")) { - // The mock test server sends back errno:1062 sqlstate:23000 for this case. - // Make sure these values get properly extracted by the client. - SQLException sqlException = (SQLException) e; - Assert.assertEquals(1062, sqlException.getErrorCode()); - Assert.assertEquals("23000", sqlException.getSQLState()); - } - } - // The transaction should still be usable now. - tx.rollback(ctx); - - // Close the transaction on partial error. - tx = conn.begin(ctx); - try { - String query = PARTIAL_ERROR_PREFIX + error + "/close transaction"; - exe.execute(tx, query); - Assert.fail("no exception thrown for " + query); - } catch (Exception e) { - Assert.assertEquals(cls, e.getClass()); - } - // The transaction should be unusable now. - try { - tx.rollback(ctx); - Assert.fail("no exception thrown for rollback() after closed transaction"); - } catch (Exception e) { - Assert.assertEquals(SQLDataException.class, e.getClass()); - Assert.assertEquals(true, e.getMessage().contains("not in transaction")); - } - } - } - @Test public void testExecuteErrors() throws Exception { checkExecuteErrors(new Executable() { @@ -586,48 +274,6 @@ void execute(String query) throws Exception { conn.execute(ctx, query, BIND_VARS, TABLET_TYPE, ALL_FIELDS); } }); - checkExecuteErrors(new Executable() { - @Override - void execute(String query) throws Exception { - conn.executeShards(ctx, query, KEYSPACE, SHARDS, BIND_VARS, TABLET_TYPE, ALL_FIELDS); - } - }); - checkExecuteErrors(new Executable() { - @Override - void execute(String query) throws Exception { - conn.executeKeyspaceIds(ctx, query, KEYSPACE, KEYSPACE_IDS, BIND_VARS, TABLET_TYPE, - ALL_FIELDS); - } - }); - checkExecuteErrors(new Executable() { - @Override - void execute(String query) throws Exception { - conn.executeKeyRanges(ctx, query, KEYSPACE, KEY_RANGES, BIND_VARS, TABLET_TYPE, ALL_FIELDS); - } - }); - checkExecuteErrors(new Executable() { - @Override - void execute(String query) throws Exception { - conn.executeEntityIds(ctx, query, KEYSPACE, "column1", ENTITY_KEYSPACE_IDS, BIND_VARS, - TABLET_TYPE, ALL_FIELDS); - } - }); - checkExecuteErrors(new Executable() { - @Override - void execute(String query) throws Exception { - conn.executeBatchShards(ctx, - Arrays.asList(Proto.bindShardQuery(KEYSPACE, SHARDS, query, BIND_VARS)), TABLET_TYPE, - true, ALL_FIELDS); - } - }); - checkExecuteErrors(new Executable() { - @Override - void execute(String query) throws Exception { - conn.executeBatchKeyspaceIds(ctx, - Arrays.asList(Proto.bindKeyspaceIdQuery(KEYSPACE, KEYSPACE_IDS, query, BIND_VARS)), - TABLET_TYPE, true, ALL_FIELDS); - } - }); } @Test @@ -638,81 +284,6 @@ void execute(String query) throws Exception { conn.streamExecute(ctx, query, BIND_VARS, TABLET_TYPE, ALL_FIELDS).next(); } }); - checkStreamExecuteErrors(new Executable() { - @Override - void execute(String query) throws Exception { - conn.streamExecuteShards(ctx, query, KEYSPACE, SHARDS, BIND_VARS, TABLET_TYPE, ALL_FIELDS) - .next(); - } - }); - checkStreamExecuteErrors(new Executable() { - @Override - void execute(String query) throws Exception { - conn.streamExecuteKeyspaceIds(ctx, query, KEYSPACE, KEYSPACE_IDS, BIND_VARS, TABLET_TYPE, - ALL_FIELDS) - .next(); - } - }); - checkStreamExecuteErrors(new Executable() { - @Override - void execute(String query) throws Exception { - conn.streamExecuteKeyRanges(ctx, query, KEYSPACE, KEY_RANGES, BIND_VARS, TABLET_TYPE, - ALL_FIELDS) - .next(); - } - }); - } - - @Test - public void testTransactionExecuteErrors() throws Exception { - checkTransactionExecuteErrors(new TransactionExecutable() { - @Override - void execute(VTGateBlockingTx tx, String query) throws Exception { - tx.execute(ctx, query, BIND_VARS, TABLET_TYPE, ALL_FIELDS); - } - }); - checkTransactionExecuteErrors(new TransactionExecutable() { - @Override - void execute(VTGateBlockingTx tx, String query) throws Exception { - tx.executeShards(ctx, query, KEYSPACE, SHARDS, BIND_VARS, TABLET_TYPE, ALL_FIELDS); - } - }); - checkTransactionExecuteErrors(new TransactionExecutable() { - @Override - void execute(VTGateBlockingTx tx, String query) throws Exception { - tx.executeKeyspaceIds(ctx, query, KEYSPACE, KEYSPACE_IDS, BIND_VARS, TABLET_TYPE, - ALL_FIELDS); - } - }); - checkTransactionExecuteErrors(new TransactionExecutable() { - @Override - void execute(VTGateBlockingTx tx, String query) throws Exception { - tx.executeKeyRanges(ctx, query, KEYSPACE, KEY_RANGES, BIND_VARS, TABLET_TYPE, ALL_FIELDS); - } - }); - checkTransactionExecuteErrors(new TransactionExecutable() { - @Override - void execute(VTGateBlockingTx tx, String query) throws Exception { - tx.executeEntityIds(ctx, query, KEYSPACE, "column1", ENTITY_KEYSPACE_IDS, BIND_VARS, - TABLET_TYPE, ALL_FIELDS); - } - }); - checkTransactionExecuteErrors(new TransactionExecutable() { - @Override - void execute(VTGateBlockingTx tx, String query) throws Exception { - tx.executeBatchShards(ctx, - Arrays.asList(Proto.bindShardQuery(KEYSPACE, SHARDS, query, BIND_VARS)), TABLET_TYPE, - ALL_FIELDS); - } - }); - checkTransactionExecuteErrors(new TransactionExecutable() { - @Override - void execute(VTGateBlockingTx tx, String query) throws Exception { - tx.executeBatchKeyspaceIds(ctx, - Arrays.asList(Proto.bindKeyspaceIdQuery(KEYSPACE, KEYSPACE_IDS, query, BIND_VARS)), - TABLET_TYPE, ALL_FIELDS); - } - }); } @Test diff --git a/java/client/src/test/java/io/vitess/client/TestUtil.java b/java/client/src/test/java/io/vitess/client/TestUtil.java index d9d13b41d1d..59e83b40ff3 100644 --- a/java/client/src/test/java/io/vitess/client/TestUtil.java +++ b/java/client/src/test/java/io/vitess/client/TestUtil.java @@ -122,25 +122,4 @@ public static VTGateBlockingConn getBlockingConn(TestEnv testEnv) { getRpcClientFactory().create(ctx, "localhost:" + testEnv.getPort()), testEnv.getKeyspace()); } - - public static void insertRows(TestEnv testEnv, int startId, int count) throws Exception { - try (VTGateBlockingConn conn = getBlockingConn(testEnv)) { - // Deadline for the overall insert loop - Context ctx = Context.getDefault().withDeadlineAfter(Duration.millis(5000)); - - VTGateBlockingTx tx = conn.begin(ctx); - String insertSql = "insert into vtgate_test " - + "(id, name, age, percent) values (:id, :name, :age, :percent)"; - Map bindVars = new HashMap<>(); - for (int id = startId; id - startId < count; id++) { - bindVars.put("id", id); - bindVars.put("name", "name_" + id); - bindVars.put("age", id % 10); - bindVars.put("percent", id / 100.0); - tx.execute(ctx, insertSql, bindVars, TabletType.MASTER, - Query.ExecuteOptions.IncludedFields.ALL); - } - tx.commit(ctx); - } - } } diff --git a/java/grpc-client/src/main/java/io/vitess/client/grpc/GrpcClient.java b/java/grpc-client/src/main/java/io/vitess/client/grpc/GrpcClient.java index 9396846880a..46d829a34d8 100644 --- a/java/grpc-client/src/main/java/io/vitess/client/grpc/GrpcClient.java +++ b/java/grpc-client/src/main/java/io/vitess/client/grpc/GrpcClient.java @@ -31,36 +31,10 @@ import io.vitess.client.StreamIterator; import io.vitess.proto.Query.QueryResult; import io.vitess.proto.Vtgate; -import io.vitess.proto.Vtgate.BeginRequest; -import io.vitess.proto.Vtgate.BeginResponse; -import io.vitess.proto.Vtgate.CommitRequest; -import io.vitess.proto.Vtgate.CommitResponse; -import io.vitess.proto.Vtgate.ExecuteBatchKeyspaceIdsRequest; -import io.vitess.proto.Vtgate.ExecuteBatchKeyspaceIdsResponse; -import io.vitess.proto.Vtgate.ExecuteBatchShardsRequest; -import io.vitess.proto.Vtgate.ExecuteBatchShardsResponse; -import io.vitess.proto.Vtgate.ExecuteEntityIdsRequest; -import io.vitess.proto.Vtgate.ExecuteEntityIdsResponse; -import io.vitess.proto.Vtgate.ExecuteKeyRangesRequest; -import io.vitess.proto.Vtgate.ExecuteKeyRangesResponse; -import io.vitess.proto.Vtgate.ExecuteKeyspaceIdsRequest; -import io.vitess.proto.Vtgate.ExecuteKeyspaceIdsResponse; import io.vitess.proto.Vtgate.ExecuteRequest; import io.vitess.proto.Vtgate.ExecuteResponse; -import io.vitess.proto.Vtgate.ExecuteShardsRequest; -import io.vitess.proto.Vtgate.ExecuteShardsResponse; -import io.vitess.proto.Vtgate.GetSrvKeyspaceRequest; -import io.vitess.proto.Vtgate.GetSrvKeyspaceResponse; -import io.vitess.proto.Vtgate.RollbackRequest; -import io.vitess.proto.Vtgate.RollbackResponse; -import io.vitess.proto.Vtgate.StreamExecuteKeyRangesRequest; -import io.vitess.proto.Vtgate.StreamExecuteKeyRangesResponse; -import io.vitess.proto.Vtgate.StreamExecuteKeyspaceIdsRequest; -import io.vitess.proto.Vtgate.StreamExecuteKeyspaceIdsResponse; import io.vitess.proto.Vtgate.StreamExecuteRequest; import io.vitess.proto.Vtgate.StreamExecuteResponse; -import io.vitess.proto.Vtgate.StreamExecuteShardsRequest; -import io.vitess.proto.Vtgate.StreamExecuteShardsResponse; import io.vitess.proto.Vtgate.VStreamResponse; import io.vitess.proto.grpc.VitessGrpc; import io.vitess.proto.grpc.VitessGrpc.VitessFutureStub; @@ -141,34 +115,6 @@ public ListenableFuture execute(Context ctx, ExecuteRequest req new ExceptionConverter(), MoreExecutors.directExecutor()); } - @Override - public ListenableFuture executeShards(Context ctx, - ExecuteShardsRequest request) throws SQLException { - return Futures.catchingAsync(getFutureStub(ctx).executeShards(request), Exception.class, - new ExceptionConverter(), MoreExecutors.directExecutor()); - } - - @Override - public ListenableFuture executeKeyspaceIds(Context ctx, - ExecuteKeyspaceIdsRequest request) throws SQLException { - return Futures.catchingAsync(getFutureStub(ctx).executeKeyspaceIds(request), Exception.class, - new ExceptionConverter(), MoreExecutors.directExecutor()); - } - - @Override - public ListenableFuture executeKeyRanges(Context ctx, - ExecuteKeyRangesRequest request) throws SQLException { - return Futures.catchingAsync(getFutureStub(ctx).executeKeyRanges(request), Exception.class, - new ExceptionConverter(), MoreExecutors.directExecutor()); - } - - @Override - public ListenableFuture executeEntityIds(Context ctx, - ExecuteEntityIdsRequest request) throws SQLException { - return Futures.catchingAsync(getFutureStub(ctx).executeEntityIds(request), Exception.class, - new ExceptionConverter(), MoreExecutors.directExecutor()); - } - @Override public ListenableFuture executeBatch(Context ctx, Vtgate.ExecuteBatchRequest request) throws SQLException { @@ -176,21 +122,6 @@ public ListenableFuture executeBatch(Context ctx, new ExceptionConverter(), MoreExecutors.directExecutor()); } - @Override - public ListenableFuture executeBatchShards(Context ctx, - ExecuteBatchShardsRequest request) throws SQLException { - return Futures.catchingAsync(getFutureStub(ctx).executeBatchShards(request), Exception.class, - new ExceptionConverter(), MoreExecutors.directExecutor()); - } - - @Override - public ListenableFuture executeBatchKeyspaceIds(Context ctx, - ExecuteBatchKeyspaceIdsRequest request) throws SQLException { - return Futures.catchingAsync(getFutureStub(ctx).executeBatchKeyspaceIds(request), - Exception.class, new ExceptionConverter(), - MoreExecutors.directExecutor()); - } - @Override public StreamIterator streamExecute(Context ctx, StreamExecuteRequest request) throws SQLException { @@ -205,76 +136,6 @@ QueryResult getResult(StreamExecuteResponse response) throws SQLException { return adapter; } - @Override - public StreamIterator streamExecuteShards(Context ctx, - StreamExecuteShardsRequest request) throws SQLException { - GrpcStreamAdapter adapter = - new GrpcStreamAdapter() { - @Override - QueryResult getResult(StreamExecuteShardsResponse response) throws SQLException { - return response.getResult(); - } - }; - getAsyncStub(ctx).streamExecuteShards(request, adapter); - return adapter; - } - - @Override - public StreamIterator streamExecuteKeyspaceIds(Context ctx, - StreamExecuteKeyspaceIdsRequest request) throws SQLException { - GrpcStreamAdapter adapter = - new GrpcStreamAdapter() { - @Override - QueryResult getResult(StreamExecuteKeyspaceIdsResponse response) throws SQLException { - return response.getResult(); - } - }; - getAsyncStub(ctx).streamExecuteKeyspaceIds(request, adapter); - return adapter; - } - - @Override - public StreamIterator streamExecuteKeyRanges(Context ctx, - StreamExecuteKeyRangesRequest request) throws SQLException { - GrpcStreamAdapter adapter = - new GrpcStreamAdapter() { - @Override - QueryResult getResult(StreamExecuteKeyRangesResponse response) throws SQLException { - return response.getResult(); - } - }; - getAsyncStub(ctx).streamExecuteKeyRanges(request, adapter); - return adapter; - } - - @Override - public ListenableFuture begin(Context ctx, BeginRequest request) - throws SQLException { - return Futures.catchingAsync(getFutureStub(ctx).begin(request), Exception.class, - new ExceptionConverter(), MoreExecutors.directExecutor()); - } - - @Override - public ListenableFuture commit(Context ctx, CommitRequest request) - throws SQLException { - return Futures.catchingAsync(getFutureStub(ctx).commit(request), Exception.class, - new ExceptionConverter(), MoreExecutors.directExecutor()); - } - - @Override - public ListenableFuture rollback(Context ctx, RollbackRequest request) - throws SQLException { - return Futures.catchingAsync(getFutureStub(ctx).rollback(request), Exception.class, - new ExceptionConverter(), MoreExecutors.directExecutor()); - } - - @Override - public ListenableFuture getSrvKeyspace(Context ctx, - GetSrvKeyspaceRequest request) throws SQLException { - return Futures.catchingAsync(getFutureStub(ctx).getSrvKeyspace(request), Exception.class, - new ExceptionConverter(), MoreExecutors.directExecutor()); - } - @Override public StreamIterator getVStream(Context ctx, Vtgate.VStreamRequest vstreamRequest) { diff --git a/java/grpc-client/src/test/java/io/client/grpc/GrpcClientStaticAuthTest.java b/java/grpc-client/src/test/java/io/client/grpc/GrpcClientStaticAuthTest.java index 3666b489d2f..41720557dfa 100644 --- a/java/grpc-client/src/test/java/io/client/grpc/GrpcClientStaticAuthTest.java +++ b/java/grpc-client/src/test/java/io/client/grpc/GrpcClientStaticAuthTest.java @@ -25,7 +25,7 @@ import io.vitess.client.RpcClientTest; import io.vitess.client.grpc.GrpcClientFactory; import io.vitess.client.grpc.StaticAuthCredentials; -import io.vitess.proto.Vtgate.GetSrvKeyspaceRequest; +import io.vitess.proto.Vtgate.ExecuteRequest; import org.joda.time.Duration; @@ -92,7 +92,7 @@ public void testWrongPassword() throws Exception { .setCallCredentials(new StaticAuthCredentials("test-username", "WRONG-password")) .create(Context.getDefault(), "localhost:" + port); try { - client.getSrvKeyspace(Context.getDefault(), GetSrvKeyspaceRequest.getDefaultInstance()).get(); + client.execute(Context.getDefault(), ExecuteRequest.getDefaultInstance()).get(); Assert.fail(); } catch (ExecutionException e) { StatusRuntimeException cause = (StatusRuntimeException) Throwables.getRootCause(e); From f1add91d5db1c6981168513ce7f1b39ae6c2a90e Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Mon, 23 Mar 2020 22:40:16 -0700 Subject: [PATCH 30/85] deprecation: delete obsolete files Signed-off-by: Sugu Sougoumarane --- .../vtgateclienttest/goclienttest/callerid.go | 61 --------- go/cmd/vtgateclienttest/goclienttest/echo.go | 118 ------------------ .../vtgateclienttest/goclienttest/errors.go | 113 ----------------- go/cmd/vtgateclienttest/goclienttest/main.go | 52 -------- .../grpcclienttest/grpc_goclient_test.go | 48 ------- .../vtgateclienttest/grpcclienttest/import.go | 22 ---- go/cmd/vtgateclienttest/services/services.go | 1 - go/cmd/vtgateclienttest/services/success.go | 34 ----- 8 files changed, 449 deletions(-) delete mode 100644 go/cmd/vtgateclienttest/goclienttest/callerid.go delete mode 100644 go/cmd/vtgateclienttest/goclienttest/echo.go delete mode 100644 go/cmd/vtgateclienttest/goclienttest/errors.go delete mode 100644 go/cmd/vtgateclienttest/goclienttest/main.go delete mode 100644 go/cmd/vtgateclienttest/grpcclienttest/grpc_goclient_test.go delete mode 100644 go/cmd/vtgateclienttest/grpcclienttest/import.go delete mode 100644 go/cmd/vtgateclienttest/services/success.go diff --git a/go/cmd/vtgateclienttest/goclienttest/callerid.go b/go/cmd/vtgateclienttest/goclienttest/callerid.go deleted file mode 100644 index 7d2b4718d7f..00000000000 --- a/go/cmd/vtgateclienttest/goclienttest/callerid.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package goclienttest - -import ( - "encoding/json" - "strings" - "testing" - - "golang.org/x/net/context" - "vitess.io/vitess/go/cmd/vtgateclienttest/services" - "vitess.io/vitess/go/vt/callerid" - "vitess.io/vitess/go/vt/vtgate/vtgateconn" -) - -// testCallerID adds a caller ID to a context, and makes sure the server -// gets it. -func testCallerID(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgateconn.VTGateSession) { - ctx := context.Background() - callerID := callerid.NewEffectiveCallerID("test_principal", "test_component", "test_subcomponent") - ctx = callerid.NewContext(ctx, callerID, nil) - - data, err := json.Marshal(callerID) - if err != nil { - t.Errorf("failed to marshal callerid: %v", err) - return - } - query := services.CallerIDPrefix + string(data) - - // test Execute calls forward the callerID - _, err = session.Execute(ctx, query, nil) - checkCallerIDError(t, "Execute", err) - - // test StreamExecute calls forward the callerID - err = getStreamError(session.StreamExecute(ctx, query, nil)) - checkCallerIDError(t, "StreamExecute", err) -} - -func checkCallerIDError(t *testing.T, name string, err error) { - if err == nil { - t.Errorf("callerid: got no error for %v", name) - return - } - if !strings.Contains(err.Error(), "SUCCESS: ") { - t.Errorf("failed to pass callerid for %v: %v", name, err) - } -} diff --git a/go/cmd/vtgateclienttest/goclienttest/echo.go b/go/cmd/vtgateclienttest/goclienttest/echo.go deleted file mode 100644 index eedbb301bda..00000000000 --- a/go/cmd/vtgateclienttest/goclienttest/echo.go +++ /dev/null @@ -1,118 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package goclienttest - -import ( - "testing" - - "golang.org/x/net/context" - - "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/vt/callerid" - "vitess.io/vitess/go/vt/vtgate/vtgateconn" - - querypb "vitess.io/vitess/go/vt/proto/query" -) - -var ( - echoPrefix = "echo://" - - query = "test query with unicode: \u6211\u80fd\u541e\u4e0b\u73bb\u7483\u800c\u4e0d\u50b7\u8eab\u9ad4" - - bindVars = map[string]*querypb.BindVariable{ - "int": sqltypes.Int64BindVariable(123), - "float": sqltypes.Float64BindVariable(2.1), - "bytes": sqltypes.BytesBindVariable([]byte{1, 2, 3}), - } - bindVarsEcho = "map[bytes:type:VARBINARY value:\"\\001\\002\\003\" float:type:FLOAT64 value:\"2.1\" int:type:INT64 value:\"123\" ]" - - callerID = callerid.NewEffectiveCallerID("test_principal", "test_component", "test_subcomponent") - callerIDEcho = "principal:\"test_principal\" component:\"test_component\" subcomponent:\"test_subcomponent\" " -) - -// testEcho exercises the test cases provided by the "echo" service. -func testEcho(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgateconn.VTGateSession) { - testEchoExecute(t, conn, session) - testEchoStreamExecute(t, conn, session) -} - -func testEchoExecute(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgateconn.VTGateSession) { - var qr *sqltypes.Result - var err error - - ctx := callerid.NewContext(context.Background(), callerID, nil) - - qr, err = session.Execute(ctx, echoPrefix+query, bindVars) - checkEcho(t, "Execute", qr, err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "bindVars": bindVarsEcho, - }) -} - -func testEchoStreamExecute(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgateconn.VTGateSession) { - var stream sqltypes.ResultStream - var err error - var qr *sqltypes.Result - - ctx := callerid.NewContext(context.Background(), callerID, nil) - - stream, err = session.StreamExecute(ctx, echoPrefix+query, bindVars) - if err != nil { - t.Fatal(err) - } - qr, err = stream.Recv() - checkEcho(t, "StreamExecute", qr, err, map[string]string{ - "callerId": callerIDEcho, - "query": echoPrefix + query, - "bindVars": bindVarsEcho, - }) -} - -// getEcho extracts the echoed field values from a query result. -func getEcho(qr *sqltypes.Result) map[string]sqltypes.Value { - values := map[string]sqltypes.Value{} - if qr == nil { - return values - } - for i, field := range qr.Fields { - values[field.Name] = qr.Rows[0][i] - } - return values -} - -// checkEcho verifies that the values present in 'want' are equal to those in -// 'got'. Note that extra values in 'got' are fine. -func checkEcho(t *testing.T, name string, qr *sqltypes.Result, err error, want map[string]string) { - if err != nil { - t.Fatalf("%v error: %v", name, err) - } - got := getEcho(qr) - for k, v := range want { - if got[k].ToString() != v { - t.Errorf("%v: %v = \n%q, want \n%q", name, k, got[k], v) - } - } - - // Check NULL and empty string. - if !got["null"].IsNull() { - t.Errorf("MySQL NULL value wasn't preserved") - } - if !got["emptyString"].IsQuoted() || got["emptyString"].ToString() != "" { - t.Errorf("Empty string value wasn't preserved: %#v", got) - } -} diff --git a/go/cmd/vtgateclienttest/goclienttest/errors.go b/go/cmd/vtgateclienttest/goclienttest/errors.go deleted file mode 100644 index 063b561cfca..00000000000 --- a/go/cmd/vtgateclienttest/goclienttest/errors.go +++ /dev/null @@ -1,113 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package goclienttest - -import ( - "io" - "testing" - - "golang.org/x/net/context" - - "vitess.io/vitess/go/sqltypes" - - "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/vt/vtgate/vtgateconn" - - vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" -) - -var ( - errorPrefix = "error://" - partialErrorPrefix = "partialerror://" - - executeErrors = map[string]vtrpcpb.Code{ - "bad input": vtrpcpb.Code_INVALID_ARGUMENT, - "deadline exceeded": vtrpcpb.Code_DEADLINE_EXCEEDED, - "integrity error": vtrpcpb.Code_ALREADY_EXISTS, - "transient error": vtrpcpb.Code_UNAVAILABLE, - "unauthenticated": vtrpcpb.Code_UNAUTHENTICATED, - "aborted": vtrpcpb.Code_ABORTED, - "unknown error": vtrpcpb.Code_UNKNOWN, - } -) - -// testErrors exercises the test cases provided by the "errors" service. -func testErrors(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgateconn.VTGateSession) { - testExecuteErrors(t, conn, session) - testStreamExecuteErrors(t, conn, session) -} - -func testExecuteErrors(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgateconn.VTGateSession) { - ctx := context.Background() - - checkExecuteErrors(t, func(query string) error { - _, err := session.Execute(ctx, query, bindVars) - return err - }) -} - -func testStreamExecuteErrors(t *testing.T, conn *vtgateconn.VTGateConn, session *vtgateconn.VTGateSession) { - ctx := context.Background() - - checkStreamExecuteErrors(t, func(query string) error { - return getStreamError(session.StreamExecute(ctx, query, bindVars)) - }) -} - -func getStreamError(stream sqltypes.ResultStream, err error) error { - if err != nil { - return err - } - for { - _, err := stream.Recv() - switch err { - case nil: - // keep going - case io.EOF: - return nil - default: - return err - } - } -} - -func checkExecuteErrors(t *testing.T, execute func(string) error) { - for errStr, errCode := range executeErrors { - query := errorPrefix + errStr - checkError(t, execute(query), query, errStr, errCode) - - query = partialErrorPrefix + errStr - checkError(t, execute(query), query, errStr, errCode) - } -} - -func checkStreamExecuteErrors(t *testing.T, execute func(string) error) { - for errStr, errCode := range executeErrors { - query := errorPrefix + errStr - checkError(t, execute(query), query, errStr, errCode) - } -} - -func checkError(t *testing.T, err error, query, errStr string, errCode vtrpcpb.Code) { - if err == nil { - t.Errorf("[%v] expected error, got nil", query) - return - } - if got, want := vterrors.Code(err), errCode; got != want { - t.Errorf("[%v] error code = %v, want %v", query, got, want) - } -} diff --git a/go/cmd/vtgateclienttest/goclienttest/main.go b/go/cmd/vtgateclienttest/goclienttest/main.go deleted file mode 100644 index d8fc0a48ead..00000000000 --- a/go/cmd/vtgateclienttest/goclienttest/main.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package goclienttest - -import ( - "testing" - - "golang.org/x/net/context" - "vitess.io/vitess/go/vt/vtgate/vtgateconn" -) - -const connectionKeyspace = "conn_ks" - -// This file contains the reference test for clients. It tests -// all the corner cases of the API, and makes sure the go client -// is full featured. -// -// It can be used as a template by other languages for their test suites. -// -// TODO(team) add more unit test cases. - -// TestGoClient runs the test suite for the provided client -func TestGoClient(t *testing.T, protocol, addr string) { - // Create a client connecting to the server - ctx := context.Background() - conn, err := vtgateconn.DialProtocol(ctx, protocol, addr) - session := conn.Session(connectionKeyspace, nil) - if err != nil { - t.Fatalf("dial failed: %v", err) - } - - testCallerID(t, conn, session) - testEcho(t, conn, session) - testErrors(t, conn, session) - - // and clean up - conn.Close() -} diff --git a/go/cmd/vtgateclienttest/grpcclienttest/grpc_goclient_test.go b/go/cmd/vtgateclienttest/grpcclienttest/grpc_goclient_test.go deleted file mode 100644 index f9ef8110ec9..00000000000 --- a/go/cmd/vtgateclienttest/grpcclienttest/grpc_goclient_test.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package grpcclienttest - -import ( - "net" - "testing" - - "google.golang.org/grpc" - - "vitess.io/vitess/go/cmd/vtgateclienttest/goclienttest" - "vitess.io/vitess/go/cmd/vtgateclienttest/services" - "vitess.io/vitess/go/vt/vtgate/grpcvtgateservice" -) - -// TestGRPCGoClient tests the go client using gRPC -func TestGRPCGoClient(t *testing.T) { - service := services.CreateServices() - - // listen on a random port - listener, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatalf("Cannot listen: %v", err) - } - defer listener.Close() - - // Create a gRPC server and listen on the port - server := grpc.NewServer() - grpcvtgateservice.RegisterForTest(server, service) - go server.Serve(listener) - - // and run the test suite - goclienttest.TestGoClient(t, "grpc", listener.Addr().String()) -} diff --git a/go/cmd/vtgateclienttest/grpcclienttest/import.go b/go/cmd/vtgateclienttest/grpcclienttest/import.go deleted file mode 100644 index b0bccb047f8..00000000000 --- a/go/cmd/vtgateclienttest/grpcclienttest/import.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package grpcclienttest - -import ( - // import the grpc client, it will register itself - _ "vitess.io/vitess/go/vt/vtgate/grpcvtgateconn" -) diff --git a/go/cmd/vtgateclienttest/services/services.go b/go/cmd/vtgateclienttest/services/services.go index f43c1456741..999c000e736 100644 --- a/go/cmd/vtgateclienttest/services/services.go +++ b/go/cmd/vtgateclienttest/services/services.go @@ -25,7 +25,6 @@ import ( func CreateServices() vtgateservice.VTGateService { var s vtgateservice.VTGateService s = newTerminalClient() - s = newSuccessClient(s) s = newErrorClient(s) s = newCallerIDClient(s) s = newEchoClient(s) diff --git a/go/cmd/vtgateclienttest/services/success.go b/go/cmd/vtgateclienttest/services/success.go deleted file mode 100644 index 88ad19d3363..00000000000 --- a/go/cmd/vtgateclienttest/services/success.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package services - -import ( - "vitess.io/vitess/go/vt/vtgate/vtgateservice" -) - -// successClient implements vtgateservice.VTGateService -// and returns specific values. It is meant to test all possible success cases, -// and make sure all clients handle any corner case correctly. -type successClient struct { - fallbackClient -} - -func newSuccessClient(fallback vtgateservice.VTGateService) *successClient { - return &successClient{ - fallbackClient: newFallbackClient(fallback), - } -} From 8c9d13ae1a2d73c92f2a33f0f5486aa9f6509391 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Tue, 24 Mar 2020 09:21:34 -0700 Subject: [PATCH 31/85] deprecation: fix endtoend verticalsplit Signed-off-by: Sugu Sougoumarane --- .../verticalsplit/vertical_split_test.go | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/go/test/endtoend/sharding/verticalsplit/vertical_split_test.go b/go/test/endtoend/sharding/verticalsplit/vertical_split_test.go index 594df68c590..e8624315b77 100644 --- a/go/test/endtoend/sharding/verticalsplit/vertical_split_test.go +++ b/go/test/endtoend/sharding/verticalsplit/vertical_split_test.go @@ -407,6 +407,7 @@ func TestVerticalSplit(t *testing.T) { // then serve master from the destination shards err = clusterInstance.VtctlclientProcess.ExecuteCommand("MigrateServedFrom", "destination_keyspace/0", "master") + require.NoError(t, err) checkSrvKeyspaceServedFrom(t, cellj, destinationKeyspace, "", *clusterInstance) checkBlacklistedTables(t, sourceMasterTablet, sourceKeyspace, []string{"/moving/", "view1"}) checkBlacklistedTables(t, sourceReplicaTablet, sourceKeyspace, []string{"/moving/", "view1"}) @@ -460,8 +461,10 @@ func verifyVtctlSetShardTabletControl(t *testing.T) { require.Nil(t, err) shardJSON, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetShard", "source_keyspace/0") + require.NoError(t, err) var shardJSONData topodata.Shard err = json2.Unmarshal([]byte(shardJSON), &shardJSONData) + require.NoError(t, err) assert.Empty(t, shardJSONData.TabletControls) } @@ -483,24 +486,24 @@ func checkStats(t *testing.T) { resultMap, err := clusterInstance.VtgateProcess.GetVars() require.Nil(t, err) - resultVtTabletCall, _ := resultMap["VttabletCall"] - resultVtTabletCallMap, _ := resultVtTabletCall.(map[string]interface{}) - resultHistograms, _ := resultVtTabletCallMap["Histograms"] - resultHistogramsMap, _ := resultHistograms.(map[string]interface{}) - resultTablet, _ := resultHistogramsMap["Execute.source_keyspace.0.replica"] - resultTableMap, _ := resultTablet.(map[string]interface{}) + resultVtTabletCall := resultMap["VttabletCall"] + resultVtTabletCallMap := resultVtTabletCall.(map[string]interface{}) + resultHistograms := resultVtTabletCallMap["Histograms"] + resultHistogramsMap := resultHistograms.(map[string]interface{}) + resultTablet := resultHistogramsMap["Execute.source_keyspace.0.replica"] + resultTableMap := resultTablet.(map[string]interface{}) resultCountStr := fmt.Sprintf("%v", reflect.ValueOf(resultTableMap["Count"])) assert.Equal(t, "2", resultCountStr, fmt.Sprintf("unexpected value for VttabletCall(Execute.source_keyspace.0.replica) inside %s", resultCountStr)) // Verify master reads done by self._check_client_conn_redirection(). - resultVtgateAPI, _ := resultMap["VtgateApi"] - resultVtgateAPIMap, _ := resultVtgateAPI.(map[string]interface{}) - resultAPIHistograms, _ := resultVtgateAPIMap["Histograms"] - resultAPIHistogramsMap, _ := resultAPIHistograms.(map[string]interface{}) - resultTabletDestination, _ := resultAPIHistogramsMap["ExecuteKeyRanges.destination_keyspace.master"] - resultTabletDestinationMap, _ := resultTabletDestination.(map[string]interface{}) + resultVtgateAPI := resultMap["VtgateApi"] + resultVtgateAPIMap := resultVtgateAPI.(map[string]interface{}) + resultAPIHistograms := resultVtgateAPIMap["Histograms"] + resultAPIHistogramsMap := resultAPIHistograms.(map[string]interface{}) + resultTabletDestination := resultAPIHistogramsMap["Execute.destination_keyspace.master"] + resultTabletDestinationMap := resultTabletDestination.(map[string]interface{}) resultCountStrDestination := fmt.Sprintf("%v", reflect.ValueOf(resultTabletDestinationMap["Count"])) - assert.Equal(t, "6", resultCountStrDestination, fmt.Sprintf("unexpected value for VtgateApi(ExecuteKeyRanges.destination_keyspace.master) inside %s)", resultCountStrDestination)) + assert.Equal(t, "6", resultCountStrDestination, fmt.Sprintf("unexpected value for VtgateApi(Execute.destination_keyspace.master) inside %s)", resultCountStrDestination)) assert.Empty(t, resultMap["VtgateApiErrorCounts"]) @@ -527,15 +530,11 @@ func insertInitialValues(t *testing.T, conn *mysql.Conn, sourceMasterTablet clus } func checkClientConnRedirectionExecuteKeyrange(ctx context.Context, t *testing.T, conn *vtgateconn.VTGateConn, keyspace string, servedFromDbTypes []topodata.TabletType, movedTables []string) { - var testKeyRange = &topodata.KeyRange{ - Start: []byte{}, - End: []byte{}, - } - keyRanges := []*topodata.KeyRange{testKeyRange} // check that the ServedFrom indirection worked correctly. - for _, tableType := range servedFromDbTypes { + for _, tabletType := range servedFromDbTypes { + session := conn.Session(fmt.Sprintf("%s@%v", keyspace, tabletType), nil) for _, table := range movedTables { - _, err := conn.ExecuteKeyRanges(ctx, fmt.Sprintf("select * from %s", table), keyspace, keyRanges, nil, tableType, nil) + _, err := session.Execute(ctx, fmt.Sprintf("select * from %s", table), nil) require.Nil(t, err) } } @@ -667,7 +666,7 @@ func initializeCluster() (int, error) { for i := 0; i < 4; i++ { // instantiate vttablet object with reserved ports tabletUID := clusterInstance.GetAndReserveTabletUID() - var tablet *cluster.Vttablet = nil + var tablet *cluster.Vttablet if i == 0 { tablet = clusterInstance.GetVttabletInstance("replica", tabletUID, cellj) } else if i == 1 { @@ -677,11 +676,11 @@ func initializeCluster() (int, error) { } // Start Mysqlctl process tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, clusterInstance.TmpDirectory) - if proc, err := tablet.MysqlctlProcess.StartProcess(); err != nil { + proc, err := tablet.MysqlctlProcess.StartProcess() + if err != nil { return 1, err - } else { - mysqlProcesses = append(mysqlProcesses, proc) } + mysqlProcesses = append(mysqlProcesses, proc) // start vttablet process tablet.VttabletProcess = cluster.VttabletProcessInstance(tablet.HTTPPort, tablet.GrpcPort, From 320a50da8b7f08a105db20e24706c2a765e1813e Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Tue, 24 Mar 2020 19:31:28 +0100 Subject: [PATCH 32/85] Fixed merge issue Signed-off-by: Rohit Nayak --- go/vt/vttablet/tabletserver/vstreamer/vstreamer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 04032f31bce..411f3cff8bb 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -637,7 +637,6 @@ func (vs *vstreamer) buildTableColumns(id uint64, tm *mysql.TableMap) ([]*queryp if !sqltypes.AreTypesEquivalent(fields[i].Type, st.Fields[i].Type) { return fields, nil } - cols[i].Type = st.Columns[i].Type } // Columns should be truncated to match those in tm. From e5642dc20ad989de4b4fc761bad11f446a9847cb Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Tue, 24 Mar 2020 20:24:24 +0100 Subject: [PATCH 33/85] varchar type can have only varchar or varbinary column types Signed-off-by: Rohit Nayak --- go/mysql/binlog_event_rbr.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index 5d7d2411c3e..f1765f46f8a 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -449,11 +449,9 @@ func CellValue(data []byte, pos int, typ byte, metadata uint16, styp querypb.Typ case TypeVarchar, TypeVarString: // We trust that styp is compatible with the column type // Length is encoded in 1 or 2 bytes. - var typeToUse querypb.Type - if styp != querypb.Type_NULL_TYPE { + typeToUse := querypb.Type_VARCHAR + if styp == querypb.Type_VARBINARY { typeToUse = styp - } else { - typeToUse = querypb.Type_VARCHAR } if metadata > 255 { l := int(uint64(data[pos]) | From cbc17442c9a209a548ead35d1b44fc36adf29c93 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Tue, 24 Mar 2020 13:03:30 -0700 Subject: [PATCH 34/85] deprecation: fix endtoend message Signed-off-by: Sugu Sougoumarane --- .../endtoend/messaging/cluster_ops_test.go | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/go/test/endtoend/messaging/cluster_ops_test.go b/go/test/endtoend/messaging/cluster_ops_test.go index 4396a92d8c4..32a06f19fd7 100644 --- a/go/test/endtoend/messaging/cluster_ops_test.go +++ b/go/test/endtoend/messaging/cluster_ops_test.go @@ -30,6 +30,7 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/vtgate/vtgateconn" @@ -100,7 +101,7 @@ func TestRepareting(t *testing.T) { assert.Equal(t, 0, getClientCount(shard0Replica)) assert.Equal(t, 1, getClientCount(shard1Master)) - _, err = stream.MessageAck(ctx, userKeyspace, name, keyRange(3)) + _, err = session.Execute(context.Background(), "update "+name+" set time_acked = 1, time_next = null where id in (3) and time_acked is null", nil) require.Nil(t, err) } @@ -149,7 +150,7 @@ func TestConnection(t *testing.T) { _, err = stream.Next() require.Nil(t, err) - _, err = stream.MessageAck(ctx, userKeyspace, name, keyRange(2, 5)) + _, err = session.Execute(context.Background(), "update "+name+" set time_acked = 1, time_next = null where id in (2, 5) and time_acked is null", nil) require.Nil(t, err) // After closing one stream, ensure vttablets have dropped it. stream.Close() @@ -197,9 +198,9 @@ func testMessaging(t *testing.T, name, ks string) { resMap = make(map[string]string) // validate message ack with id 4 - count, err := stream.MessageAck(ctx, ks, name, keyRange(4)) + qr, err := session.Execute(context.Background(), "update "+name+" set time_acked = 1, time_next = null where id in (4) and time_acked is null", nil) require.Nil(t, err) - assert.Equal(t, int64(1), count) + assert.Equal(t, uint64(1), qr.RowsAffected) res, err = stream.Next() require.Nil(t, err) for _, row := range res.Rows { @@ -215,9 +216,9 @@ func testMessaging(t *testing.T, name, ks string) { assert.Equal(t, "hello world 1", resMap["1"]) // validate message ack with 1 and 4, only 1 should be ack - count, err = stream.MessageAck(ctx, ks, name, keyRange(1, 4)) + qr, err = session.Execute(context.Background(), "update "+name+" set time_acked = 1, time_next = null where id in (1, 4) and time_acked is null", nil) require.Nil(t, err) - assert.Equal(t, int64(1), count) + assert.Equal(t, uint64(1), qr.RowsAffected) } func validateField(t *testing.T, field *query.Field, name string, _type query.Type) { @@ -249,12 +250,26 @@ func VtgateGrpcConn(ctx context.Context, cluster *cluster.LocalProcessCluster) ( // MessageStream strarts the stream for the corresponding connection. func (stream *VTGateStream) MessageStream(ks, shard string, keyRange *topodata.KeyRange, name string) (*sqltypes.Result, error) { // start message stream which send received message to the respChan - go stream.VTGateConn.MessageStream(stream.ctx, ks, shard, keyRange, name, func(s *sqltypes.Result) error { - stream.respChan <- s - return nil - }) - // wait for field details - return stream.Next() + session := stream.Session("@master", nil) + resultStream, err := session.StreamExecute(stream.ctx, fmt.Sprintf("stream * from %s", name), nil) + if err != nil { + return nil, err + } + qr, err := resultStream.Recv() + if err != nil { + return nil, err + } + go func() { + for { + qr, err := resultStream.Recv() + if err != nil { + log.Infof("Message stream ended: %v", err) + return + } + stream.respChan <- qr + } + }() + return qr, nil } // Next reads the new msg available in stream. @@ -312,18 +327,3 @@ func getVar(vttablet *cluster.Vttablet) (map[string]interface{}, error) { } return nil, nil } - -// keyRange created keyRange array for correponding ids -func keyRange(s ...int) []*query.Value { - out := make([]*query.Value, 0, len(s)) - - for _, v := range s { - q := new(query.Value) - q.Type = query.Type_INT64 - q.Value = []byte(fmt.Sprint(v)) - - out = append(out, q) - } - - return out -} From a8ce1a0ba0a5957b973923e30d9157b3a6d6ff81 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Tue, 24 Mar 2020 21:54:07 +0100 Subject: [PATCH 35/85] Also treat binary/blob types as varbinary Signed-off-by: Rohit Nayak --- examples/are-you-alive/go.mod | 2 -- go/mysql/binlog_event_rbr.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/are-you-alive/go.mod b/examples/are-you-alive/go.mod index 5f3d04ddccd..9bb69e1307b 100644 --- a/examples/are-you-alive/go.mod +++ b/examples/are-you-alive/go.mod @@ -4,8 +4,6 @@ go 1.12 require ( github.com/go-sql-driver/mysql v1.5.0 - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect github.com/prometheus/client_golang v1.4.1 github.com/sirupsen/logrus v1.4.2 ) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index f1765f46f8a..3a18bc74682 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -450,7 +450,7 @@ func CellValue(data []byte, pos int, typ byte, metadata uint16, styp querypb.Typ // We trust that styp is compatible with the column type // Length is encoded in 1 or 2 bytes. typeToUse := querypb.Type_VARCHAR - if styp == querypb.Type_VARBINARY { + if styp == querypb.Type_VARBINARY || styp == querypb.Type_BINARY || styp == querypb.Type_BLOB{ typeToUse = styp } if metadata > 255 { From fd3374a538d2651217d6f8aed7c75e3e5b07f9ba Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Tue, 24 Mar 2020 21:56:04 +0100 Subject: [PATCH 36/85] Also treat binary/blob types as varbinary, gofmt-ed ... Signed-off-by: Rohit Nayak --- go/mysql/binlog_event_rbr.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index 3a18bc74682..89e87c19091 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -450,7 +450,7 @@ func CellValue(data []byte, pos int, typ byte, metadata uint16, styp querypb.Typ // We trust that styp is compatible with the column type // Length is encoded in 1 or 2 bytes. typeToUse := querypb.Type_VARCHAR - if styp == querypb.Type_VARBINARY || styp == querypb.Type_BINARY || styp == querypb.Type_BLOB{ + if styp == querypb.Type_VARBINARY || styp == querypb.Type_BINARY || styp == querypb.Type_BLOB { typeToUse = styp } if metadata > 255 { From 449c3f6e5dc4b183ef03802d59ffb3613ae415ee Mon Sep 17 00:00:00 2001 From: deepthi Date: Tue, 24 Mar 2020 15:29:23 -0700 Subject: [PATCH 37/85] Add back logging of reserve port because it sometimes hangs in CI. Use vt/log instead of prometheus/common/log Signed-off-by: deepthi --- .../transform/backup_transform_utils.go | 8 +++---- go/test/endtoend/cluster/cluster_process.go | 4 +++- go/test/endtoend/cluster/mysqlctl_process.go | 2 +- go/test/endtoend/cluster/vtbackup_process.go | 2 -- .../sharding/resharding/resharding_base.go | 24 +++++++++---------- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/go/test/endtoend/backup/transform/backup_transform_utils.go b/go/test/endtoend/backup/transform/backup_transform_utils.go index 90d6f8827bc..99b8346bd60 100644 --- a/go/test/endtoend/backup/transform/backup_transform_utils.go +++ b/go/test/endtoend/backup/transform/backup_transform_utils.go @@ -266,7 +266,7 @@ func TestBackupTransformImpl(t *testing.T) { } -// TestBackupTransformError validate backup with test_backup_error +// TestBackupTransformErrorImpl validate backup with test_backup_error // backup_storage_hook, which should fail. func TestBackupTransformErrorImpl(t *testing.T) { // restart the replica with transform hook parameter @@ -309,13 +309,13 @@ func validateManifestFile(t *testing.T, backupLocation string) { require.Nilf(t, err, "error while parsing MANIFEST %v", err) // validate manifest - transformHook, _ := manifest["TransformHook"] + transformHook := manifest["TransformHook"] require.Equalf(t, "test_backup_transform", transformHook, "invalid transformHook in MANIFEST") - skipCompress, _ := manifest["SkipCompress"] + skipCompress := manifest["SkipCompress"] assert.Equalf(t, skipCompress, true, "invalid value of skipCompress") // validate backup files - fielEntries, _ := manifest["FileEntries"] + fielEntries := manifest["FileEntries"] fileArr, ok := fielEntries.([]interface{}) require.True(t, ok) for i := range fileArr { diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go index c907a66cc5a..7fa2ccee13b 100644 --- a/go/test/endtoend/cluster/cluster_process.go +++ b/go/test/endtoend/cluster/cluster_process.go @@ -344,7 +344,7 @@ func (cluster *LocalProcessCluster) StartKeyspace(keyspace Keyspace, shardNames // the required services (ex topo, vtgate, mysql and vttablet) func (cluster *LocalProcessCluster) LaunchCluster(keyspace *Keyspace, shards []Shard) (err error) { - log.Infof("Starting keyspace : %v", keyspace.Name) + log.Infof("Starting keyspace: %v", keyspace.Name) // Create Keyspace err = cluster.VtctlProcess.CreateKeyspace(keyspace.Name) @@ -577,6 +577,7 @@ func (cluster *LocalProcessCluster) GetAndReservePort() int { } for { cluster.nextPortForProcess = cluster.nextPortForProcess + 1 + log.Infof("Attempting to reserve port: %v", cluster.nextPortForProcess) ln, err := net.Listen("tcp", fmt.Sprintf(":%v", cluster.nextPortForProcess)) if err != nil { @@ -584,6 +585,7 @@ func (cluster *LocalProcessCluster) GetAndReservePort() int { continue } + log.Infof("Port %v is available, reserving..", cluster.nextPortForProcess) ln.Close() break } diff --git a/go/test/endtoend/cluster/mysqlctl_process.go b/go/test/endtoend/cluster/mysqlctl_process.go index b619c4fe78b..adcffc4062e 100644 --- a/go/test/endtoend/cluster/mysqlctl_process.go +++ b/go/test/endtoend/cluster/mysqlctl_process.go @@ -86,7 +86,7 @@ func (mysqlctl *MysqlctlProcess) StartProcess() (*exec.Cmd, error) { "-init_db_sql_file", mysqlctl.InitDBFile) } tmpProcess.Args = append(tmpProcess.Args, "start") - + log.Infof("Starting mysqlctl with command: %v", tmpProcess.Args) return tmpProcess, tmpProcess.Start() } diff --git a/go/test/endtoend/cluster/vtbackup_process.go b/go/test/endtoend/cluster/vtbackup_process.go index 3d105876f92..d211aa35862 100644 --- a/go/test/endtoend/cluster/vtbackup_process.go +++ b/go/test/endtoend/cluster/vtbackup_process.go @@ -47,8 +47,6 @@ type VtbackupProcess struct { ExtraArgs []string initialBackup bool initDBfile string - dbPassword string - dbName string proc *exec.Cmd exit chan error diff --git a/go/test/endtoend/sharding/resharding/resharding_base.go b/go/test/endtoend/sharding/resharding/resharding_base.go index 2f6142d2233..b79b17ff5a3 100644 --- a/go/test/endtoend/sharding/resharding/resharding_base.go +++ b/go/test/endtoend/sharding/resharding/resharding_base.go @@ -26,12 +26,12 @@ import ( "testing" "time" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/mysql" - "github.com/prometheus/common/log" - "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/test/endtoend/sharding" querypb "vitess.io/vitess/go/vt/proto/query" @@ -255,7 +255,7 @@ func TestResharding(t *testing.T, useVarbinaryShardingKeyType bool) { var mysqlCtlProcessList []*exec.Cmd for _, shard := range clusterInstance.Keyspaces[0].Shards { for _, tablet := range shard.Vttablets { - log.Infof("Starting MySql for tablet %v", tablet.Alias) + log.Infof("Starting mysql for tablet %v", tablet.Alias) if proc, err := tablet.MysqlctlProcess.StartProcess(); err != nil { t.Fatal(err) } else { @@ -548,9 +548,9 @@ func TestResharding(t *testing.T, useVarbinaryShardingKeyType bool) { // testing filtered replication: insert a bunch of data on shard 1, check we get most of it after a few seconds, // wait for binlog server timeout, check we get all of it. - log.Debug("Inserting lots of data on source shard") + log.Info("Inserting lots of data on source shard") insertLots(100, 0, *shard1Master, tableName, fixedParentID, keyspaceName) - log.Debug("Executing MultiValue Insert Queries") + log.Info("Executing MultiValue Insert Queries") execMultiShardDmls(t, keyspaceName) // Checking 100 percent of data is sent quickly @@ -574,7 +574,7 @@ func TestResharding(t *testing.T, useVarbinaryShardingKeyType bool) { clusterInstance.VtworkerProcess.Cell = cell1 // Compare using SplitDiff - log.Debug("Running vtworker SplitDiff") + log.Info("Running vtworker SplitDiff") err = clusterInstance.VtworkerProcess.ExecuteVtworkerCommand(clusterInstance.GetAndReservePort(), clusterInstance.GetAndReservePort(), "--use_v3_resharding_mode=true", @@ -585,7 +585,7 @@ func TestResharding(t *testing.T, useVarbinaryShardingKeyType bool) { require.Nil(t, err) // Compare using MultiSplitDiff - log.Debug("Running vtworker MultiSplitDiff") + log.Info("Running vtworker MultiSplitDiff") err = clusterInstance.VtworkerProcess.ExecuteVtworkerCommand(clusterInstance.GetAndReservePort(), clusterInstance.GetAndReservePort(), "--use_v3_resharding_mode=true", @@ -618,9 +618,9 @@ func TestResharding(t *testing.T, useVarbinaryShardingKeyType bool) { require.Nil(t, err) // test data goes through again - log.Debug("Inserting lots of data on source shard") + log.Info("Inserting lots of data on source shard") insertLots(100, 100, *shard1Master, tableName, fixedParentID, keyspaceName) - log.Debug("Checking 100 percent of data was sent quickly") + log.Info("Checking 100 percent of data was sent quickly") assert.True(t, checkLotsTimeout(t, 100, 100, tableName, keyspaceName, shardingKeyType)) sharding.CheckBinlogServerVars(t, *shard1Replica2, 80, 80, false) @@ -640,7 +640,7 @@ func TestResharding(t *testing.T, useVarbinaryShardingKeyType bool) { "VtTabletStreamHealth", "-count", "1", master.Alias) require.Nil(t, err) - log.Debug("Got health: ", streamHealth) + log.Info("Got health: ", streamHealth) var streamHealthResponse querypb.StreamHealthResponse err = json.Unmarshal([]byte(streamHealth), &streamHealthResponse) @@ -788,7 +788,7 @@ func TestResharding(t *testing.T, useVarbinaryShardingKeyType bool) { assert.True(t, checkLotsTimeout(t, 100, 200, tableName, keyspaceName, shardingKeyType)) // Compare using SplitDiff - log.Debug("Running vtworker SplitDiff") + log.Info("Running vtworker SplitDiff") err = clusterInstance.VtworkerProcess.ExecuteVtworkerCommand(clusterInstance.GetAndReservePort(), clusterInstance.GetAndReservePort(), "--use_v3_resharding_mode=true", @@ -799,7 +799,7 @@ func TestResharding(t *testing.T, useVarbinaryShardingKeyType bool) { require.Nil(t, err) // Compare using MultiSplitDiff - log.Debug("Running vtworker MultiSplitDiff") + log.Info("Running vtworker MultiSplitDiff") err = clusterInstance.VtworkerProcess.ExecuteVtworkerCommand(clusterInstance.GetAndReservePort(), clusterInstance.GetAndReservePort(), "--use_v3_resharding_mode=true", From 3f94f05794456101b5aaf87e23da5b8238ec344b Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Tue, 24 Mar 2020 15:46:27 -0700 Subject: [PATCH 38/85] deprecation: dead code in resolver, etc Signed-off-by: Sugu Sougoumarane --- go/vt/vtgate/executor.go | 70 --- go/vt/vtgate/executor_framework_test.go | 7 + go/vt/vtgate/executor_test.go | 52 -- go/vt/vtgate/resolver.go | 215 -------- go/vt/vtgate/resolver_test.go | 673 ------------------------ go/vt/vtgate/sandbox_test.go | 7 - go/vt/vtgate/scatter_conn.go | 203 ------- go/vt/vtgate/scatter_conn_test.go | 23 - 8 files changed, 7 insertions(+), 1243 deletions(-) delete mode 100644 go/vt/vtgate/resolver_test.go diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 71aae611bca..f4d1a02d2cd 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -1334,76 +1334,6 @@ func (e *Executor) MessageStream(ctx context.Context, keyspace string, shard str return formatError(err) } -// MessageAck acks messages. -// FIXME(alainjobart) the keyspace field here is not used for routing, -// but just for finding the table in the VSchema. If we don't find the -// table in the VSchema, we could just assume it's sharded (which would work -// for unsharded as well) and route it to the provided keyspace. -func (e *Executor) MessageAck(ctx context.Context, keyspace, name string, ids []*querypb.Value) (int64, error) { - table, err := e.VSchema().FindTable(keyspace, name) - if err != nil { - return 0, err - } - - var rss []*srvtopo.ResolvedShard - var rssValues [][]*querypb.Value - if table.Keyspace.Sharded { - // TODO(sougou): Change this to use Session. - vcursor := newVCursorImpl( - ctx, - NewSafeSession(&vtgatepb.Session{}), - table.Keyspace.Name, - topodatapb.TabletType_MASTER, - sqlparser.MarginComments{}, - e, - nil, - ) - - // convert []*querypb.Value to []sqltypes.Value for calling Map. - values := make([]sqltypes.Value, 0, len(ids)) - for _, id := range ids { - values = append(values, sqltypes.ProtoToValue(id)) - } - // We always use the (unique) primary vindex. The ID must be the - // primary vindex for message tables. - single, ok := table.ColumnVindexes[0].Vindex.(vindexes.SingleColumn) - if !ok { - return 0, fmt.Errorf("multi-column vindexes not supported") - } - destinations, err := single.Map(vcursor, values) - if err != nil { - return 0, err - } - rss, rssValues, err = e.resolver.resolver.ResolveDestinations(ctx, table.Keyspace.Name, topodatapb.TabletType_MASTER, ids, destinations) - if err != nil { - return 0, err - } - } else { - // All ids go into the first shard, so we only resolve - // one destination, and put all IDs in there. - rss, err = e.resolver.resolver.ResolveDestination(ctx, table.Keyspace.Name, topodatapb.TabletType_MASTER, key.DestinationAnyShard{}) - if err != nil { - return 0, err - } - rssValues = [][]*querypb.Value{ids} - } - return e.scatterConn.MessageAck(ctx, rss, rssValues, name) -} - -// IsKeyspaceRangeBasedSharded returns true if the keyspace in the vschema is -// marked as sharded. -func (e *Executor) IsKeyspaceRangeBasedSharded(keyspace string) bool { - vschema := e.VSchema() - ks, ok := vschema.Keyspaces[keyspace] - if !ok { - return false - } - if ks.Keyspace == nil { - return false - } - return ks.Keyspace.Sharded -} - // VSchema returns the VSchema. func (e *Executor) VSchema() *vindexes.VSchema { e.mu.Lock() diff --git a/go/vt/vtgate/executor_framework_test.go b/go/vt/vtgate/executor_framework_test.go index 632de9c0bcc..1b788d2c206 100644 --- a/go/vt/vtgate/executor_framework_test.go +++ b/go/vt/vtgate/executor_framework_test.go @@ -29,6 +29,7 @@ import ( "vitess.io/vitess/go/streamlog" "vitess.io/vitess/go/vt/discovery" "vitess.io/vitess/go/vt/key" + "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/vtgate/vindexes" "vitess.io/vitess/go/vt/vttablet/sandboxconn" @@ -555,3 +556,9 @@ func testQueryLog(t *testing.T, logChan chan interface{}, method, stmtType, sql return logStats } + +func newTestResolver(hc discovery.HealthCheck, serv srvtopo.Server, cell string) *Resolver { + sc := newTestScatterConn(hc, serv, cell) + srvResolver := srvtopo.NewResolver(serv, sc.gateway, cell) + return NewResolver(srvResolver, serv, cell, sc) +} diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index 6e3ed78bf16..a33ae3adc7b 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -1934,58 +1934,6 @@ func TestExecutorUnrecognized(t *testing.T) { } } -func TestExecutorMessageAckSharded(t *testing.T) { - executor, sbc1, sbc2, _ := createExecutorEnv() - - // Constant in IN clause is just a number, not a bind variable. - ids := []*querypb.Value{{ - Type: sqltypes.VarChar, - Value: []byte("1"), - }} - count, err := executor.MessageAck(context.Background(), "", "user", ids) - require.NoError(t, err) - if count != 1 { - t.Errorf("count: %d, want 1", count) - } - if !reflect.DeepEqual(sbc1.MessageIDs, ids) { - t.Errorf("sbc1.MessageIDs: %v, want %v", sbc1.MessageIDs, ids) - } - if sbc2.MessageIDs != nil { - t.Errorf("sbc2.MessageIDs: %+v, want nil\n", sbc2.MessageIDs) - } - - // Constants in IN clause are just numbers, not bind variables. - // They result in two different MessageIDs on two shards. - sbc1.MessageIDs = nil - sbc2.MessageIDs = nil - ids = []*querypb.Value{{ - Type: sqltypes.VarChar, - Value: []byte("1"), - }, { - Type: sqltypes.VarChar, - Value: []byte("3"), - }} - count, err = executor.MessageAck(context.Background(), "", "user", ids) - require.NoError(t, err) - if count != 2 { - t.Errorf("count: %d, want 2", count) - } - wantids := []*querypb.Value{{ - Type: sqltypes.VarChar, - Value: []byte("1"), - }} - if !reflect.DeepEqual(sbc1.MessageIDs, wantids) { - t.Errorf("sbc1.MessageIDs: %+v, want %+v\n", sbc1.MessageIDs, wantids) - } - wantids = []*querypb.Value{{ - Type: sqltypes.VarChar, - Value: []byte("3"), - }} - if !reflect.DeepEqual(sbc2.MessageIDs, wantids) { - t.Errorf("sbc2.MessageIDs: %+v, want %+v\n", sbc2.MessageIDs, wantids) - } -} - // TestVSchemaStats makes sure the building and displaying of the // VSchemaStats works. func TestVSchemaStats(t *testing.T) { diff --git a/go/vt/vtgate/resolver.go b/go/vt/vtgate/resolver.go index 6dc66af2708..fa3a6aa6f26 100644 --- a/go/vt/vtgate/resolver.go +++ b/go/vt/vtgate/resolver.go @@ -17,12 +17,6 @@ limitations under the License. package vtgate import ( - "bytes" - "fmt" - "reflect" - "sort" - "strings" - "golang.org/x/net/context" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" @@ -32,17 +26,9 @@ import ( querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" - vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) -var ( - sqlListIdentifier = []byte("::") - inOperator = []byte(" in ") - kwAnd = []byte(" and ") - kwWhere = []byte(" where ") -) - // Resolver is the layer to resolve KeyspaceIds and KeyRanges // to shards. It will try to re-resolve shards if ScatterConn // returns retryable error, which may imply horizontal or vertical @@ -125,119 +111,6 @@ func (res *Resolver) Execute( } } -// ExecuteEntityIds executes a non-streaming query based on given KeyspaceId map. -// It retries query if new keyspace/shards are re-resolved after a retryable error. -func (res *Resolver) ExecuteEntityIds( - ctx context.Context, - sql string, - bindVariables map[string]*querypb.BindVariable, - keyspace string, - entityColumnName string, - entityKeyspaceIDs []*vtgatepb.ExecuteEntityIdsRequest_EntityId, - tabletType topodatapb.TabletType, - session *vtgatepb.Session, - notInTransaction bool, - options *querypb.ExecuteOptions, -) (*sqltypes.Result, error) { - // Unpack the entityKeyspaceIDs into []ids and []Destination - ids := make([]*querypb.Value, len(entityKeyspaceIDs)) - destinations := make([]key.Destination, len(entityKeyspaceIDs)) - for i, eki := range entityKeyspaceIDs { - ids[i] = &querypb.Value{ - Type: eki.Type, - Value: eki.Value, - } - destinations[i] = key.DestinationKeyspaceID(eki.KeyspaceId) - } - - rss, values, err := res.resolver.ResolveDestinations( - ctx, - keyspace, - tabletType, - ids, - destinations) - if err != nil { - return nil, err - } - for { - sqls, bindVars := buildEntityIds(values, sql, entityColumnName, bindVariables) - qr, err := res.scatterConn.ExecuteEntityIds( - ctx, - rss, - sqls, - bindVars, - tabletType, - NewSafeSession(session), - notInTransaction, - options) - if isRetryableError(err) { - newRss, newValues, err := res.resolver.ResolveDestinations( - ctx, - keyspace, - tabletType, - ids, - destinations) - if err != nil { - return nil, err - } - if !srvtopo.ResolvedShardsEqual(rss, newRss) || !srvtopo.ValuesEqual(values, newValues) { - // Retry if resharding happened. - rss = newRss - values = newValues - continue - } - } - if err != nil { - return nil, err - } - return qr, err - } -} - -// ExecuteBatch executes a group of queries based on shards resolved by given func. -// It retries query if new keyspace/shards are re-resolved after a retryable error. -func (res *Resolver) ExecuteBatch( - ctx context.Context, - tabletType topodatapb.TabletType, - asTransaction bool, - session *vtgatepb.Session, - options *querypb.ExecuteOptions, - buildBatchRequest func() (*scatterBatchRequest, error), -) ([]sqltypes.Result, error) { - batchRequest, err := buildBatchRequest() - if err != nil { - return nil, err - } - for { - qrs, err := res.scatterConn.ExecuteBatch( - ctx, - batchRequest, - tabletType, - asTransaction, - NewSafeSession(session), - options) - // Don't retry transactional requests. - if asTransaction { - return qrs, err - } - // If lower level retries failed, check if there was a resharding event - // and retry again if needed. - if isRetryableError(err) { - newBatchRequest, buildErr := buildBatchRequest() - if buildErr != nil { - return nil, buildErr - } - // Use reflect to see if the request has changed. - if reflect.DeepEqual(*batchRequest, *newBatchRequest) { - return qrs, err - } - batchRequest = newBatchRequest - continue - } - return qrs, err - } -} - // StreamExecute executes a streaming query on shards resolved by given func. // This function currently temporarily enforces the restriction of executing on // one shard since it cannot merge-sort the results to guarantee ordering of @@ -289,95 +162,7 @@ func (res *Resolver) MessageStream(ctx context.Context, keyspace string, shard s return res.scatterConn.MessageStream(ctx, rss, name, callback) } -// MessageAckKeyspaceIds routes message acks based on the associated keyspace ids. -func (res *Resolver) MessageAckKeyspaceIds(ctx context.Context, keyspace, name string, idKeyspaceIDs []*vtgatepb.IdKeyspaceId) (int64, error) { - ids := make([]*querypb.Value, len(idKeyspaceIDs)) - ksids := make([]key.Destination, len(idKeyspaceIDs)) - for i, iki := range idKeyspaceIDs { - ids[i] = iki.Id - ksids[i] = key.DestinationKeyspaceID(iki.KeyspaceId) - } - - rss, values, err := res.resolver.ResolveDestinations(ctx, keyspace, topodatapb.TabletType_MASTER, ids, ksids) - if err != nil { - return 0, err - } - - return res.scatterConn.MessageAck(ctx, rss, values, name) -} - // GetGatewayCacheStatus returns a displayable version of the Gateway cache. func (res *Resolver) GetGatewayCacheStatus() gateway.TabletCacheStatusList { return res.scatterConn.GetGatewayCacheStatus() } - -// StrsEquals compares contents of two string slices. -func StrsEquals(a, b []string) bool { - if len(a) != len(b) { - return false - } - sort.Strings(a) - sort.Strings(b) - for i, v := range a { - if v != b[i] { - return false - } - } - return true -} - -// buildEntityIds populates SQL and BindVariables. -func buildEntityIds(values [][]*querypb.Value, qSQL, entityColName string, qBindVars map[string]*querypb.BindVariable) ([]string, []map[string]*querypb.BindVariable) { - sqls := make([]string, len(values)) - bindVars := make([]map[string]*querypb.BindVariable, len(values)) - for i, val := range values { - var b bytes.Buffer - b.Write([]byte(entityColName)) - bindVariables := make(map[string]*querypb.BindVariable) - for k, v := range qBindVars { - bindVariables[k] = v - } - bvName := fmt.Sprintf("%v_entity_ids", entityColName) - bindVariables[bvName] = &querypb.BindVariable{ - Type: querypb.Type_TUPLE, - Values: val, - } - b.Write(inOperator) - b.Write(sqlListIdentifier) - b.Write([]byte(bvName)) - sqls[i] = insertSQLClause(qSQL, b.String()) - bindVars[i] = bindVariables - } - return sqls, bindVars -} - -func insertSQLClause(querySQL, clause string) string { - // get first index of any additional clause: group by, order by, limit, for update, sql end if nothing - // insert clause into the index position - sql := strings.ToLower(querySQL) - idxExtra := len(sql) - if idxGroupBy := strings.Index(sql, " group by"); idxGroupBy > 0 && idxGroupBy < idxExtra { - idxExtra = idxGroupBy - } - if idxOrderBy := strings.Index(sql, " order by"); idxOrderBy > 0 && idxOrderBy < idxExtra { - idxExtra = idxOrderBy - } - if idxLimit := strings.Index(sql, " limit"); idxLimit > 0 && idxLimit < idxExtra { - idxExtra = idxLimit - } - if idxForUpdate := strings.Index(sql, " for update"); idxForUpdate > 0 && idxForUpdate < idxExtra { - idxExtra = idxForUpdate - } - var b bytes.Buffer - b.Write([]byte(querySQL[:idxExtra])) - if strings.Contains(sql, "where") { - b.Write(kwAnd) - } else { - b.Write(kwWhere) - } - b.Write([]byte(clause)) - if idxExtra < len(sql) { - b.Write([]byte(querySQL[idxExtra:])) - } - return b.String() -} diff --git a/go/vt/vtgate/resolver_test.go b/go/vt/vtgate/resolver_test.go deleted file mode 100644 index 71a43cbbc0a..00000000000 --- a/go/vt/vtgate/resolver_test.go +++ /dev/null @@ -1,673 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package vtgate provides query routing rpc services -// for vttablets. -package vtgate - -import ( - "fmt" - "reflect" - "sort" - "strings" - "testing" - - "github.com/stretchr/testify/require" - "golang.org/x/net/context" - "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/vt/discovery" - "vitess.io/vitess/go/vt/key" - "vitess.io/vitess/go/vt/srvtopo" - - querypb "vitess.io/vitess/go/vt/proto/query" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" - vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" -) - -// This file uses the sandbox_test framework. - -func TestResolverExecuteKeyspaceIds(t *testing.T) { - testResolverGeneric(t, "TestResolverExecuteKeyspaceIds", func(res *Resolver) (*sqltypes.Result, error) { - return res.Execute(context.Background(), - "query", - nil, - "TestResolverExecuteKeyspaceIds", - topodatapb.TabletType_MASTER, - key.DestinationKeyspaceIDs([][]byte{{0x10}, {0x25}}), - NewSafeSession(&vtgatepb.Session{}), - false, - nil, - nil, - false, - ) - }) -} - -func TestResolverExecuteKeyRanges(t *testing.T) { - testResolverGeneric(t, "TestResolverExecuteKeyRanges", func(res *Resolver) (*sqltypes.Result, error) { - return res.Execute(context.Background(), - "query", - nil, - "TestResolverExecuteKeyRanges", - topodatapb.TabletType_MASTER, - key.DestinationKeyRanges([]*topodatapb.KeyRange{{Start: []byte{0x10}, End: []byte{0x25}}}), - NewSafeSession(&vtgatepb.Session{}), - false, - nil, - nil, - false, - ) - }) -} - -func TestResolverExecuteEntityIds(t *testing.T) { - testResolverGeneric(t, "TestResolverExecuteEntityIds", func(res *Resolver) (*sqltypes.Result, error) { - return res.ExecuteEntityIds(context.Background(), - "query", - nil, - "TestResolverExecuteEntityIds", - "col", - []*vtgatepb.ExecuteEntityIdsRequest_EntityId{ - { - Type: sqltypes.Int64, - Value: []byte("0"), - KeyspaceId: []byte{0x10}, - }, - { - Type: sqltypes.VarBinary, - Value: []byte("1"), - KeyspaceId: []byte{0x25}, - }, - }, - topodatapb.TabletType_MASTER, - nil, - false, - nil) - }) -} - -func TestResolverExecuteBatchKeyspaceIds(t *testing.T) { - testResolverGeneric(t, "TestResolverExecuteBatchKeyspaceIds", func(res *Resolver) (*sqltypes.Result, error) { - queries := []*vtgatepb.BoundKeyspaceIdQuery{{ - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: "TestResolverExecuteBatchKeyspaceIds", - KeyspaceIds: [][]byte{ - {0x10}, - {0x25}, - }, - }} - qrs, err := res.ExecuteBatch(context.Background(), - topodatapb.TabletType_MASTER, - false, - nil, - nil, - func() (*scatterBatchRequest, error) { - return boundKeyspaceIDQueriesToScatterBatchRequest(context.Background(), res.resolver, queries, topodatapb.TabletType_MASTER) - }) - if err != nil { - return nil, err - } - return &qrs[0], err - }) -} - -func TestResolverStreamExecuteKeyspaceIds(t *testing.T) { - keyspace := "TestResolverStreamExecuteKeyspaceIds" - testResolverStreamGeneric(t, keyspace, func(res *Resolver) (*sqltypes.Result, error) { - qr := new(sqltypes.Result) - err := res.StreamExecute(context.Background(), - "query", - nil, - keyspace, - topodatapb.TabletType_MASTER, - key.DestinationKeyspaceIDs([][]byte{{0x10}, {0x15}}), - nil, - func(r *sqltypes.Result) error { - qr.AppendResult(r) - return nil - }) - return qr, err - }) - testResolverStreamGeneric(t, keyspace, func(res *Resolver) (*sqltypes.Result, error) { - qr := new(sqltypes.Result) - err := res.StreamExecute(context.Background(), - "query", - nil, - keyspace, - topodatapb.TabletType_MASTER, - key.DestinationKeyspaceIDs([][]byte{{0x10}, {0x15}, {0x25}}), - nil, - func(r *sqltypes.Result) error { - qr.AppendResult(r) - return nil - }) - return qr, err - }) -} - -func TestResolverStreamExecuteKeyRanges(t *testing.T) { - keyspace := "TestResolverStreamExecuteKeyRanges" - // streaming a single shard - testResolverStreamGeneric(t, keyspace, func(res *Resolver) (*sqltypes.Result, error) { - qr := new(sqltypes.Result) - err := res.StreamExecute(context.Background(), - "query", - nil, - keyspace, - topodatapb.TabletType_MASTER, - key.DestinationKeyRanges([]*topodatapb.KeyRange{{Start: []byte{0x10}, End: []byte{0x15}}}), - nil, - func(r *sqltypes.Result) error { - qr.AppendResult(r) - return nil - }) - return qr, err - }) - // streaming multiple shards - testResolverStreamGeneric(t, keyspace, func(res *Resolver) (*sqltypes.Result, error) { - qr := new(sqltypes.Result) - err := res.StreamExecute(context.Background(), - "query", - nil, - keyspace, - topodatapb.TabletType_MASTER, - key.DestinationKeyRanges([]*topodatapb.KeyRange{{Start: []byte{0x10}, End: []byte{0x25}}}), - nil, - func(r *sqltypes.Result) error { - qr.AppendResult(r) - return nil - }) - return qr, err - }) -} - -func testResolverGeneric(t *testing.T, name string, action func(res *Resolver) (*sqltypes.Result, error)) { - t.Run("successful execute", func(t *testing.T) { - createSandbox(name) - hc := discovery.NewFakeHealthCheck() - res := newTestResolver(hc, new(sandboxTopo), "aa") - sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) - sbc1 := hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) - - _, err := action(res) - if err != nil { - t.Errorf("want nil, got %v", err) - } - if execCount := sbc0.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v", execCount) - } - if execCount := sbc1.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v", execCount) - } - }) - - t.Run("non-retryable failure", func(t *testing.T) { - s := createSandbox(name) - hc := discovery.NewFakeHealthCheck() - res := newTestResolver(hc, new(sandboxTopo), "aa") - sbc0 := hc.AddTestTablet("aa", "-20", 1, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) - sbc1 := hc.AddTestTablet("aa", "20-40", 1, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) - sbc0.MustFailCodes[vtrpcpb.Code_INVALID_ARGUMENT] = 1 - sbc1.MustFailCodes[vtrpcpb.Code_INTERNAL] = 1 - _, err := action(res) - want := []string{ - fmt.Sprintf("target: %s.-20.master, used tablet: aa-0 (-20): INVALID_ARGUMENT error", name), - fmt.Sprintf("target: %s.20-40.master, used tablet: aa-0 (20-40): INTERNAL error", name), - } - if err == nil { - t.Errorf("want\n%v\ngot\n%v", want, err) - } else { - got := strings.Split(err.Error(), "\n") - sort.Strings(got) - if !reflect.DeepEqual(want, got) { - t.Errorf("want\n%v\ngot\n%v", want, got) - } - } - // Ensure that we tried only once - if execCount := sbc0.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v", execCount) - } - if execCount := sbc1.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v", execCount) - } - // Ensure that we tried topo only once when mapping KeyspaceId/KeyRange to shards - if s.SrvKeyspaceCounter != 1 { - t.Errorf("want 1, got %v", s.SrvKeyspaceCounter) - } - - }) - - t.Run("retryable failure, no sharding event", func(t *testing.T) { - s := createSandbox(name) - hc := discovery.NewFakeHealthCheck() - res := newTestResolver(hc, new(sandboxTopo), "aa") - sbc0 := hc.AddTestTablet("aa", "-20", 1, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) - sbc1 := hc.AddTestTablet("aa", "20-40", 1, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) - sbc0.MustFailCodes[vtrpcpb.Code_FAILED_PRECONDITION] = 1 - sbc1.MustFailCodes[vtrpcpb.Code_FAILED_PRECONDITION] = 1 - _, err := action(res) - want := []string{ - fmt.Sprintf("target: %s.-20.master, used tablet: aa-0 (-20): FAILED_PRECONDITION error", name), - fmt.Sprintf("target: %s.20-40.master, used tablet: aa-0 (20-40): FAILED_PRECONDITION error", name), - } - if err == nil { - t.Errorf("want\n%v\ngot\n%v", want, err) - } else { - got := strings.Split(err.Error(), "\n") - sort.Strings(got) - if !reflect.DeepEqual(want, got) { - t.Errorf("want\n%v\ngot\n%v", want, got) - } - } - // Ensure that we tried only once. - if execCount := sbc0.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v", execCount) - } - if execCount := sbc1.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v", execCount) - } - // Ensure that we tried topo only twice. - if s.SrvKeyspaceCounter != 2 { - t.Errorf("want 2, got %v", s.SrvKeyspaceCounter) - } - }) - - t.Run("no failure, initial vertical resharding", func(t *testing.T) { - s := createSandbox(name) - hc := discovery.NewFakeHealthCheck() - res := newTestResolver(hc, new(sandboxTopo), "aa") - addSandboxServedFrom(name, name+"ServedFrom0") - sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) - sbc1 := hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) - s0 := createSandbox(name + "ServedFrom0") // make sure we have a fresh copy - s0.ShardSpec = "-80-" - sbc2 := hc.AddTestTablet("aa", "1.1.1.1", 1003, name+"ServedFrom0", "-80", topodatapb.TabletType_MASTER, true, 1, nil) - _, err := action(res) - if err != nil { - t.Errorf("want nil, got %v", err) - } - // Ensure original keyspace is not used. - if execCount := sbc0.ExecCount.Get(); execCount != 0 { - t.Errorf("want 0, got %v", execCount) - } - if execCount := sbc1.ExecCount.Get(); execCount != 0 { - t.Errorf("want 0, got %v", execCount) - } - // Ensure redirected keyspace is accessed once. - if execCount := sbc2.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v", execCount) - } - // Ensure that we tried each keyspace only once. - if s.SrvKeyspaceCounter != 1 { - t.Errorf("want 1, got %v", s.SrvKeyspaceCounter) - } - if s0.SrvKeyspaceCounter != 1 { - t.Errorf("want 1, got %v", s0.SrvKeyspaceCounter) - } - s0.Reset() - }) - - // - t.Run("retryable failure, vertical resharding", func(t *testing.T) { - s := createSandbox(name) - hc := discovery.NewFakeHealthCheck() - res := newTestResolver(hc, new(sandboxTopo), "aa") - - sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) - sbc1 := hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) - sbc1.MustFailCodes[vtrpcpb.Code_FAILED_PRECONDITION] = 1 - i := 0 - s.SrvKeyspaceCallback = func() { - if i == 1 { - addSandboxServedFrom(name, name+"ServedFrom") - hc.Reset() - hc.AddTestTablet("aa", "1.1.1.1", 1001, name+"ServedFrom", "-20", topodatapb.TabletType_MASTER, true, 1, nil) - hc.AddTestTablet("aa", "1.1.1.1", 1002, name+"ServedFrom", "20-40", topodatapb.TabletType_MASTER, true, 1, nil) - } - i++ - } - _, err := action(res) - if err != nil { - t.Errorf("want nil, got %v", err) - } - // Ensure that we tried only once on the original conn. - if execCount := sbc0.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v", execCount) - } - if execCount := sbc1.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v", execCount) - } - // Ensure that we tried topo only 3 times. - if s.SrvKeyspaceCounter != 3 { - t.Errorf("want 3, got %v", s.SrvKeyspaceCounter) - } - }) - - t.Run("retryable failure, horizontal resharding", func(t *testing.T) { - s := createSandbox(name) - hc := discovery.NewFakeHealthCheck() - res := newTestResolver(hc, new(sandboxTopo), "aa") - - s.Reset() - hc.Reset() - sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) - sbc1 := hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) - sbc1.MustFailCodes[vtrpcpb.Code_FAILED_PRECONDITION] = 1 - i := 0 - s.SrvKeyspaceCallback = func() { - if i == 1 { - s.ShardSpec = "-20-30-40-60-80-a0-c0-e0-" - hc.Reset() - hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) - hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-30", topodatapb.TabletType_MASTER, true, 1, nil) - } - i++ - } - _, err := action(res) - if err != nil { - t.Errorf("want nil, got %v", err) - } - // Ensure that we tried only once on original guy. - if execCount := sbc0.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v", execCount) - } - if execCount := sbc1.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v", execCount) - } - // Ensure that we tried topo only twice. - if s.SrvKeyspaceCounter != 2 { - t.Errorf("want 2, got %v", s.SrvKeyspaceCounter) - } - }) -} - -func testResolverStreamGeneric(t *testing.T, name string, action func(res *Resolver) (*sqltypes.Result, error)) { - t.Run("successful execute", func(t *testing.T) { - createSandbox(name) - hc := discovery.NewFakeHealthCheck() - res := newTestResolver(hc, new(sandboxTopo), "aa") - sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) - hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) - _, err := action(res) - if err != nil { - t.Errorf("want nil, got %v", err) - } - if execCount := sbc0.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v", execCount) - } - - }) - - t.Run("failure", func(t *testing.T) { - s := createSandbox(name) - hc := discovery.NewFakeHealthCheck() - res := newTestResolver(hc, new(sandboxTopo), "aa") - sbc0 := hc.AddTestTablet("aa", "-20", 1, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) - hc.AddTestTablet("aa", "20-40", 1, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) - sbc0.MustFailCodes[vtrpcpb.Code_INTERNAL] = 1 - _, err := action(res) - want := fmt.Sprintf("target: %s.-20.master, used tablet: aa-0 (-20): INTERNAL error", name) - if err == nil || err.Error() != want { - t.Errorf("want\n%s\ngot\n%v", want, err) - } - // Ensure that we tried only once. - if execCount := sbc0.ExecCount.Get(); execCount != 1 { - t.Errorf("want 1, got %v", execCount) - } - // Ensure that we tried topo only once - if s.SrvKeyspaceCounter != 1 { - t.Errorf("want 1, got %v", s.SrvKeyspaceCounter) - } - }) -} - -func TestResolverMessageAckSharded(t *testing.T) { - name := "TestResolverMessageAckSharded" - _ = createSandbox(name) - hc := discovery.NewFakeHealthCheck() - res := newTestResolver(hc, new(sandboxTopo), "aa") - sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) - sbc1 := hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) - - sbc0.MessageIDs = nil - sbc1.MessageIDs = nil - idKeyspaceIDs := []*vtgatepb.IdKeyspaceId{ - { - Id: &querypb.Value{ - Type: sqltypes.VarChar, - Value: []byte("1"), - }, - KeyspaceId: []byte{0x10}, - }, - { - Id: &querypb.Value{ - Type: sqltypes.VarChar, - Value: []byte("3"), - }, - KeyspaceId: []byte{0x30}, - }, - } - count, err := res.MessageAckKeyspaceIds(context.Background(), name, "user", idKeyspaceIDs) - require.NoError(t, err) - if count != 2 { - t.Errorf("count: %d, want 2", count) - } - wantids := []*querypb.Value{{ - Type: sqltypes.VarChar, - Value: []byte("1"), - }} - if !sqltypes.Proto3ValuesEqual(sbc0.MessageIDs, wantids) { - t.Errorf("sbc0.MessageIDs: %+v, want %+v\n", sbc0.MessageIDs, wantids) - } - wantids = []*querypb.Value{{ - Type: sqltypes.VarChar, - Value: []byte("3"), - }} - if !sqltypes.Proto3ValuesEqual(sbc1.MessageIDs, wantids) { - t.Errorf("sbc1.MessageIDs: %+v, want %+v\n", sbc1.MessageIDs, wantids) - } -} - -func TestResolverMessageAckUnsharded(t *testing.T) { - createSandbox(KsTestUnsharded) - hc := discovery.NewFakeHealthCheck() - res := newTestResolver(hc, new(sandboxTopo), "aa") - sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, KsTestUnsharded, "0", topodatapb.TabletType_MASTER, true, 1, nil) - - sbc0.MessageIDs = nil - idKeyspaceIDs := []*vtgatepb.IdKeyspaceId{ - { - Id: &querypb.Value{ - Type: sqltypes.VarChar, - Value: []byte("1"), - }, - }, - { - Id: &querypb.Value{ - Type: sqltypes.VarChar, - Value: []byte("3"), - }, - }, - } - count, err := res.MessageAckKeyspaceIds(context.Background(), KsTestUnsharded, "user", idKeyspaceIDs) - require.NoError(t, err) - if count != 2 { - t.Errorf("count: %d, want 2", count) - } - wantids := []*querypb.Value{{ - Type: sqltypes.VarChar, - Value: []byte("1"), - }, { - Type: sqltypes.VarChar, - Value: []byte("3"), - }} - if !sqltypes.Proto3ValuesEqual(sbc0.MessageIDs, wantids) { - t.Errorf("sbc0.MessageIDs: %+v, want %+v\n", sbc0.MessageIDs, wantids) - } -} - -func TestResolverInsertSqlClause(t *testing.T) { - clause := "col in (:col1, :col2)" - tests := [][]string{ - { - "select a from table", - "select a from table where " + clause}, - { - "select a from table where id = 1", - "select a from table where id = 1 and " + clause}, - { - "select a from table group by a", - "select a from table where " + clause + " group by a"}, - { - "select a from table where id = 1 order by a limit 10", - "select a from table where id = 1 and " + clause + " order by a limit 10"}, - { - "select a from table where id = 1 for update", - "select a from table where id = 1 and " + clause + " for update"}, - } - for _, test := range tests { - got := insertSQLClause(test[0], clause) - if got != test[1] { - t.Errorf("want '%v', got '%v'", test[1], got) - } - } -} - -func TestResolverBuildEntityIds(t *testing.T) { - values := [][]*querypb.Value{ - { - { - Type: querypb.Type_VARCHAR, - Value: []byte("0"), - }, - { - Type: querypb.Type_INT64, - Value: []byte("1"), - }, - }, - { - { - Type: querypb.Type_VARCHAR, - Value: []byte("2"), - }, - }, - } - sql := "select a from table where id=:id" - entityColName := "uid" - bindVar := map[string]*querypb.BindVariable{ - "id": sqltypes.Int64BindVariable(10), - } - sqls, bindVars := buildEntityIds(values, sql, entityColName, bindVar) - wantSqls := []string{ - "select a from table where id=:id and uid in ::uid_entity_ids", - "select a from table where id=:id and uid in ::uid_entity_ids", - } - wantBindVars := []map[string]*querypb.BindVariable{ - {"id": sqltypes.Int64BindVariable(10), "uid_entity_ids": sqltypes.TestBindVariable([]interface{}{"0", 1})}, - {"id": sqltypes.Int64BindVariable(10), "uid_entity_ids": sqltypes.TestBindVariable([]interface{}{"2"})}, - } - if !reflect.DeepEqual(wantSqls, sqls) { - t.Errorf("want %+v, got %+v", wantSqls, sqls) - } - if !reflect.DeepEqual(wantBindVars, bindVars) { - t.Errorf("want\n%+v, got\n%+v", wantBindVars, bindVars) - } -} - -func TestResolverExecBatchReresolve(t *testing.T) { - keyspace := "TestResolverExecBatchReresolve" - createSandbox(keyspace) - hc := discovery.NewFakeHealthCheck() - res := newTestResolver(hc, new(sandboxTopo), "aa") - - sbc := hc.AddTestTablet("aa", "0", 1, keyspace, "0", topodatapb.TabletType_MASTER, true, 1, nil) - sbc.MustFailCodes[vtrpcpb.Code_FAILED_PRECONDITION] = 20 - - callcount := 0 - buildBatchRequest := func() (*scatterBatchRequest, error) { - callcount++ - queries := []*vtgatepb.BoundShardQuery{{ - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: keyspace, - Shards: []string{"0"}, - }} - return boundShardQueriesToScatterBatchRequest(context.Background(), res.resolver, queries, topodatapb.TabletType_MASTER) - } - - _, err := res.ExecuteBatch(context.Background(), topodatapb.TabletType_MASTER, false, nil, nil, buildBatchRequest) - want := "target: TestResolverExecBatchReresolve.0.master, used tablet: aa-0 (0): FAILED_PRECONDITION error" - if err == nil || err.Error() != want { - t.Errorf("want %s, got %v", want, err) - } - // Ensure scatter tried a re-resolve - if callcount != 2 { - t.Errorf("want 2, got %v", callcount) - } - if count := sbc.AsTransactionCount.Get(); count != 0 { - t.Errorf("want 0, got %v", count) - } -} - -func TestResolverExecBatchAsTransaction(t *testing.T) { - keyspace := "TestResolverExecBatchAsTransaction" - createSandbox(keyspace) - hc := discovery.NewFakeHealthCheck() - res := newTestResolver(hc, new(sandboxTopo), "aa") - - sbc := hc.AddTestTablet("aa", "0", 1, keyspace, "0", topodatapb.TabletType_MASTER, true, 1, nil) - sbc.MustFailCodes[vtrpcpb.Code_INTERNAL] = 20 - - callcount := 0 - buildBatchRequest := func() (*scatterBatchRequest, error) { - callcount++ - queries := []*vtgatepb.BoundShardQuery{{ - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: keyspace, - Shards: []string{"0"}, - }} - return boundShardQueriesToScatterBatchRequest(context.Background(), res.resolver, queries, topodatapb.TabletType_MASTER) - } - - _, err := res.ExecuteBatch(context.Background(), topodatapb.TabletType_MASTER, true, nil, nil, buildBatchRequest) - want := "target: TestResolverExecBatchAsTransaction.0.master, used tablet: aa-0 (0): INTERNAL error" - if err == nil || err.Error() != want { - t.Errorf("want %v, got %v", want, err) - } - // Ensure scatter did not re-resolve - if callcount != 1 { - t.Errorf("want 1, got %v", callcount) - } - if count := sbc.AsTransactionCount.Get(); count != 1 { - t.Errorf("want 1, got %v", count) - return - } -} - -func newTestResolver(hc discovery.HealthCheck, serv srvtopo.Server, cell string) *Resolver { - sc := newTestScatterConn(hc, serv, cell) - srvResolver := srvtopo.NewResolver(serv, sc.gateway, cell) - return NewResolver(srvResolver, serv, cell, sc) -} diff --git a/go/vt/vtgate/sandbox_test.go b/go/vt/vtgate/sandbox_test.go index c97f1255daf..a8ca3ae21bc 100644 --- a/go/vt/vtgate/sandbox_test.go +++ b/go/vt/vtgate/sandbox_test.go @@ -89,13 +89,6 @@ func getSandboxSrvVSchema() *vschemapb.SrvVSchema { return result } -func addSandboxServedFrom(keyspace, servedFrom string) { - sandboxMu.Lock() - defer sandboxMu.Unlock() - ksToSandbox[keyspace].KeyspaceServedFrom = servedFrom - ksToSandbox[servedFrom] = ksToSandbox[keyspace] -} - type sandbox struct { // Use sandmu to access the variables below sandmu sync.Mutex diff --git a/go/vt/vtgate/scatter_conn.go b/go/vt/vtgate/scatter_conn.go index 53ecdce9bbd..5d175751add 100644 --- a/go/vt/vtgate/scatter_conn.go +++ b/go/vt/vtgate/scatter_conn.go @@ -28,7 +28,6 @@ import ( "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/concurrency" "vitess.io/vitess/go/vt/discovery" - "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vterrors" @@ -254,191 +253,6 @@ func (stc *ScatterConn) executeAutocommit(ctx context.Context, rs *srvtopo.Resol return &qrs[0], nil } -// ExecuteEntityIds executes queries that are shard specific. -func (stc *ScatterConn) ExecuteEntityIds( - ctx context.Context, - rss []*srvtopo.ResolvedShard, - sqls []string, - bindVars []map[string]*querypb.BindVariable, - tabletType topodatapb.TabletType, - session *SafeSession, - notInTransaction bool, - options *querypb.ExecuteOptions, -) (*sqltypes.Result, error) { - - // mu protects qr - var mu sync.Mutex - qr := new(sqltypes.Result) - - allErrors := stc.multiGoTransaction( - ctx, - "ExecuteEntityIds", - rss, - tabletType, - session, - notInTransaction, - func(rs *srvtopo.ResolvedShard, i int, shouldBegin bool, transactionID int64) (int64, error) { - var innerqr *sqltypes.Result - var err error - - if shouldBegin { - innerqr, transactionID, err = rs.QueryService.BeginExecute(ctx, rs.Target, sqls[i], bindVars[i], options) - } else { - innerqr, err = rs.QueryService.Execute(ctx, rs.Target, sqls[i], bindVars[i], transactionID, options) - } - if err != nil { - return transactionID, err - } - - mu.Lock() - defer mu.Unlock() - qr.AppendResult(innerqr) - return transactionID, nil - }) - - return qr, allErrors.AggrError(vterrors.Aggregate) -} - -// scatterBatchRequest needs to be built to perform a scatter batch query. -// A VTGate batch request will get translated into a different set of batches -// for each keyspace:shard, and those results will map to different positions in the -// results list. The length specifies the total length of the final results -// list. In each request variable, the resultIndexes specifies the position -// for each result from the shard. -type scatterBatchRequest struct { - // length is the total number of queries we have. - length int - // requests maps the 'keyspace:shard' key to the structure below. - requests map[string]*shardBatchRequest -} - -type shardBatchRequest struct { - // rs is the ResolvedShard to send the queries to. - rs *srvtopo.ResolvedShard - // queries are the queries to send to that ResolvedShard. - queries []*querypb.BoundQuery - // resultIndexes describes the index of the query and its results - // into the full original query array. - resultIndexes []int -} - -func boundShardQueriesToScatterBatchRequest(ctx context.Context, resolver *srvtopo.Resolver, boundQueries []*vtgatepb.BoundShardQuery, tabletType topodatapb.TabletType) (*scatterBatchRequest, error) { - requests := &scatterBatchRequest{ - length: len(boundQueries), - requests: make(map[string]*shardBatchRequest), - } - for i, boundQuery := range boundQueries { - rss, err := resolver.ResolveDestination(ctx, boundQuery.Keyspace, tabletType, key.DestinationShards(boundQuery.Shards)) - if err != nil { - return nil, err - } - - for _, rs := range rss { - key := rs.Target.Keyspace + ":" + rs.Target.Shard - request := requests.requests[key] - if request == nil { - request = &shardBatchRequest{ - rs: rs, - } - requests.requests[key] = request - } - request.queries = append(request.queries, boundQuery.Query) - request.resultIndexes = append(request.resultIndexes, i) - } - } - return requests, nil -} - -func boundKeyspaceIDQueriesToScatterBatchRequest(ctx context.Context, resolver *srvtopo.Resolver, boundQueries []*vtgatepb.BoundKeyspaceIdQuery, tabletType topodatapb.TabletType) (*scatterBatchRequest, error) { - requests := &scatterBatchRequest{ - length: len(boundQueries), - requests: make(map[string]*shardBatchRequest), - } - for i, boundQuery := range boundQueries { - rss, err := resolver.ResolveDestination(ctx, boundQuery.Keyspace, tabletType, key.DestinationKeyspaceIDs(boundQuery.KeyspaceIds)) - if err != nil { - return nil, err - } - - for _, rs := range rss { - key := rs.Target.Keyspace + ":" + rs.Target.Shard - request := requests.requests[key] - if request == nil { - request = &shardBatchRequest{ - rs: rs, - } - requests.requests[key] = request - } - request.queries = append(request.queries, boundQuery.Query) - request.resultIndexes = append(request.resultIndexes, i) - } - } - return requests, nil -} - -// ExecuteBatch executes a batch of non-streaming queries on the specified shards. -func (stc *ScatterConn) ExecuteBatch( - ctx context.Context, - batchRequest *scatterBatchRequest, - tabletType topodatapb.TabletType, - asTransaction bool, - session *SafeSession, - options *querypb.ExecuteOptions) (qrs []sqltypes.Result, err error) { - - allErrors := new(concurrency.AllErrorRecorder) - - results := make([]sqltypes.Result, batchRequest.length) - var resMutex sync.Mutex - - var wg sync.WaitGroup - for _, req := range batchRequest.requests { - wg.Add(1) - go func(req *shardBatchRequest) { - defer wg.Done() - var err error - startTime, statsKey := stc.startAction("ExecuteBatch", req.rs.Target) - defer stc.endAction(startTime, allErrors, statsKey, &err, session) - - shouldBegin, transactionID := transactionInfo(req.rs.Target, session, false) - var innerqrs []sqltypes.Result - if shouldBegin { - innerqrs, transactionID, err = req.rs.QueryService.BeginExecuteBatch(ctx, req.rs.Target, req.queries, asTransaction, options) - if transactionID != 0 { - if appendErr := session.Append(&vtgatepb.Session_ShardSession{ - Target: req.rs.Target, - TransactionId: transactionID, - }, stc.txConn.mode); appendErr != nil { - err = appendErr - } - } - if err != nil { - return - } - } else { - innerqrs, err = req.rs.QueryService.ExecuteBatch(ctx, req.rs.Target, req.queries, asTransaction, transactionID, options) - if err != nil { - return - } - } - - resMutex.Lock() - defer resMutex.Unlock() - for i, result := range innerqrs { - results[req.resultIndexes[i]].AppendResult(&result) - } - }(req) - } - wg.Wait() - - if session.MustRollback() { - stc.txConn.Rollback(ctx, session) - } - if allErrors.HasErrors() { - return nil, allErrors.AggrError(vterrors.Aggregate) - } - return results, nil -} - func (stc *ScatterConn) processOneStreamingResult(mu *sync.Mutex, fieldSent *bool, qr *sqltypes.Result, callback func(*sqltypes.Result) error) error { mu.Lock() defer mu.Unlock() @@ -597,23 +411,6 @@ func (stc *ScatterConn) MessageStream(ctx context.Context, rss []*srvtopo.Resolv return allErrors.AggrError(vterrors.Aggregate) } -// MessageAck acks messages across multiple shards. -func (stc *ScatterConn) MessageAck(ctx context.Context, rss []*srvtopo.ResolvedShard, values [][]*querypb.Value, name string) (int64, error) { - var mu sync.Mutex - var totalCount int64 - allErrors := stc.multiGo(ctx, "MessageAck", rss, topodatapb.TabletType_MASTER, func(rs *srvtopo.ResolvedShard, i int) error { - count, err := rs.QueryService.MessageAck(ctx, rs.Target, name, values[i]) - if err != nil { - return err - } - mu.Lock() - totalCount += count - mu.Unlock() - return nil - }) - return totalCount, allErrors.AggrError(vterrors.Aggregate) -} - // Close closes the underlying Gateway. func (stc *ScatterConn) Close() error { return stc.gateway.Close(context.Background()) diff --git a/go/vt/vtgate/scatter_conn_test.go b/go/vt/vtgate/scatter_conn_test.go index 407b0969be6..7566a63b327 100644 --- a/go/vt/vtgate/scatter_conn_test.go +++ b/go/vt/vtgate/scatter_conn_test.go @@ -74,29 +74,6 @@ func TestScatterConnExecuteMulti(t *testing.T) { }) } -func TestScatterConnExecuteBatch(t *testing.T) { - testScatterConnGeneric(t, "TestScatterConnExecuteBatch", func(sc *ScatterConn, shards []string) (*sqltypes.Result, error) { - queries := []*vtgatepb.BoundShardQuery{{ - Query: &querypb.BoundQuery{ - Sql: "query", - BindVariables: nil, - }, - Keyspace: "TestScatterConnExecuteBatch", - Shards: shards, - }} - res := srvtopo.NewResolver(&sandboxTopo{}, sc.gateway, "aa") - scatterRequest, err := boundShardQueriesToScatterBatchRequest(context.Background(), res, queries, topodatapb.TabletType_REPLICA) - if err != nil { - return nil, err - } - qrs, err := sc.ExecuteBatch(context.Background(), scatterRequest, topodatapb.TabletType_REPLICA, false, NewSafeSession(nil), nil) - if err != nil { - return nil, err - } - return &qrs[0], err - }) -} - func TestScatterConnStreamExecute(t *testing.T) { testScatterConnGeneric(t, "TestScatterConnStreamExecute", func(sc *ScatterConn, shards []string) (*sqltypes.Result, error) { res := srvtopo.NewResolver(&sandboxTopo{}, sc.gateway, "aa") From 050b037affb34f30b5323841f24c7bd238193631 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Tue, 24 Mar 2020 21:52:55 -0700 Subject: [PATCH 39/85] deprecation: delete v2 protos Signed-off-by: Sugu Sougoumarane --- go/vt/proto/vtgate/vtgate.pb.go | 2746 ++--------------- .../src/main/java/io/vitess/client/Proto.java | 60 - .../java/io/vitess/client/EntityIdTest.java | 125 - proto/vtgate.proto | 515 +--- 4 files changed, 244 insertions(+), 3202 deletions(-) delete mode 100644 java/client/src/test/java/io/vitess/client/EntityIdTest.java diff --git a/go/vt/proto/vtgate/vtgate.pb.go b/go/vt/proto/vtgate/vtgate.pb.go index 10786f95d07..c54b7f1e596 100644 --- a/go/vt/proto/vtgate/vtgate.pb.go +++ b/go/vt/proto/vtgate/vtgate.pb.go @@ -322,7 +322,6 @@ type ExecuteRequest struct { // These values are deprecated. Use session instead. // TODO(sougou): remove in 3.1 TabletType topodata.TabletType `protobuf:"varint,4,opt,name=tablet_type,json=tabletType,proto3,enum=topodata.TabletType" json:"tablet_type,omitempty"` - NotInTransaction bool `protobuf:"varint,5,opt,name=not_in_transaction,json=notInTransaction,proto3" json:"not_in_transaction,omitempty"` KeyspaceShard string `protobuf:"bytes,6,opt,name=keyspace_shard,json=keyspaceShard,proto3" json:"keyspace_shard,omitempty"` Options *query.ExecuteOptions `protobuf:"bytes,7,opt,name=options,proto3" json:"options,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -383,13 +382,6 @@ func (m *ExecuteRequest) GetTabletType() topodata.TabletType { return topodata.TabletType_UNKNOWN } -func (m *ExecuteRequest) GetNotInTransaction() bool { - if m != nil { - return m.NotInTransaction - } - return false -} - func (m *ExecuteRequest) GetKeyspaceShard() string { if m != nil { return m.KeyspaceShard @@ -465,2455 +457,362 @@ func (m *ExecuteResponse) GetResult() *query.QueryResult { return nil } -// ExecuteShardsRequest is the payload to ExecuteShards. -type ExecuteShardsRequest struct { +// ExecuteBatchRequest is the payload to ExecuteBatch. +type ExecuteBatchRequest struct { // caller_id identifies the caller. This is the effective caller ID, // set by the application to further identify the caller. CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // session carries the current transaction data. It is returned by Begin. - // Do not fill it in if outside of a transaction. + // session carries the session state. Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` - // query is the query and bind variables to execute. - Query *query.BoundQuery `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` - // keyspace to target the query to. - Keyspace string `protobuf:"bytes,4,opt,name=keyspace,proto3" json:"keyspace,omitempty"` - // shards to target the query to. A DML can only target one shard. - Shards []string `protobuf:"bytes,5,rep,name=shards,proto3" json:"shards,omitempty"` - // tablet_type is the type of tablets that this query is targeted to. - TabletType topodata.TabletType `protobuf:"varint,6,opt,name=tablet_type,json=tabletType,proto3,enum=topodata.TabletType" json:"tablet_type,omitempty"` - // not_in_transaction is deprecated. - NotInTransaction bool `protobuf:"varint,7,opt,name=not_in_transaction,json=notInTransaction,proto3" json:"not_in_transaction,omitempty"` - // options - Options *query.ExecuteOptions `protobuf:"bytes,8,opt,name=options,proto3" json:"options,omitempty"` + // queries is a list of query and bind variables to execute. + Queries []*query.BoundQuery `protobuf:"bytes,3,rep,name=queries,proto3" json:"queries,omitempty"` + // These values are deprecated. Use session instead. + // TODO(sougou): remove in 3.1 + TabletType topodata.TabletType `protobuf:"varint,4,opt,name=tablet_type,json=tabletType,proto3,enum=topodata.TabletType" json:"tablet_type,omitempty"` + AsTransaction bool `protobuf:"varint,5,opt,name=as_transaction,json=asTransaction,proto3" json:"as_transaction,omitempty"` + KeyspaceShard string `protobuf:"bytes,6,opt,name=keyspace_shard,json=keyspaceShard,proto3" json:"keyspace_shard,omitempty"` + Options *query.ExecuteOptions `protobuf:"bytes,7,opt,name=options,proto3" json:"options,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *ExecuteShardsRequest) Reset() { *m = ExecuteShardsRequest{} } -func (m *ExecuteShardsRequest) String() string { return proto.CompactTextString(m) } -func (*ExecuteShardsRequest) ProtoMessage() {} -func (*ExecuteShardsRequest) Descriptor() ([]byte, []int) { +func (m *ExecuteBatchRequest) Reset() { *m = ExecuteBatchRequest{} } +func (m *ExecuteBatchRequest) String() string { return proto.CompactTextString(m) } +func (*ExecuteBatchRequest) ProtoMessage() {} +func (*ExecuteBatchRequest) Descriptor() ([]byte, []int) { return fileDescriptor_aab96496ceaf1ebb, []int{3} } -func (m *ExecuteShardsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExecuteShardsRequest.Unmarshal(m, b) +func (m *ExecuteBatchRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ExecuteBatchRequest.Unmarshal(m, b) } -func (m *ExecuteShardsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExecuteShardsRequest.Marshal(b, m, deterministic) +func (m *ExecuteBatchRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ExecuteBatchRequest.Marshal(b, m, deterministic) } -func (m *ExecuteShardsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExecuteShardsRequest.Merge(m, src) +func (m *ExecuteBatchRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ExecuteBatchRequest.Merge(m, src) } -func (m *ExecuteShardsRequest) XXX_Size() int { - return xxx_messageInfo_ExecuteShardsRequest.Size(m) +func (m *ExecuteBatchRequest) XXX_Size() int { + return xxx_messageInfo_ExecuteBatchRequest.Size(m) } -func (m *ExecuteShardsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ExecuteShardsRequest.DiscardUnknown(m) +func (m *ExecuteBatchRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ExecuteBatchRequest.DiscardUnknown(m) } -var xxx_messageInfo_ExecuteShardsRequest proto.InternalMessageInfo +var xxx_messageInfo_ExecuteBatchRequest proto.InternalMessageInfo -func (m *ExecuteShardsRequest) GetCallerId() *vtrpc.CallerID { +func (m *ExecuteBatchRequest) GetCallerId() *vtrpc.CallerID { if m != nil { return m.CallerId } return nil } -func (m *ExecuteShardsRequest) GetSession() *Session { +func (m *ExecuteBatchRequest) GetSession() *Session { if m != nil { return m.Session } return nil } -func (m *ExecuteShardsRequest) GetQuery() *query.BoundQuery { +func (m *ExecuteBatchRequest) GetQueries() []*query.BoundQuery { if m != nil { - return m.Query + return m.Queries } return nil } -func (m *ExecuteShardsRequest) GetKeyspace() string { - if m != nil { - return m.Keyspace - } - return "" -} - -func (m *ExecuteShardsRequest) GetShards() []string { +func (m *ExecuteBatchRequest) GetTabletType() topodata.TabletType { if m != nil { - return m.Shards + return m.TabletType } - return nil + return topodata.TabletType_UNKNOWN } -func (m *ExecuteShardsRequest) GetTabletType() topodata.TabletType { +func (m *ExecuteBatchRequest) GetAsTransaction() bool { if m != nil { - return m.TabletType + return m.AsTransaction } - return topodata.TabletType_UNKNOWN + return false } -func (m *ExecuteShardsRequest) GetNotInTransaction() bool { +func (m *ExecuteBatchRequest) GetKeyspaceShard() string { if m != nil { - return m.NotInTransaction + return m.KeyspaceShard } - return false + return "" } -func (m *ExecuteShardsRequest) GetOptions() *query.ExecuteOptions { +func (m *ExecuteBatchRequest) GetOptions() *query.ExecuteOptions { if m != nil { return m.Options } return nil } -// ExecuteShardsResponse is the returned value from ExecuteShards. -type ExecuteShardsResponse struct { +// ExecuteBatchResponse is the returned value from ExecuteBatch. +type ExecuteBatchResponse struct { // error contains an application level error if necessary. Note the // session may have changed, even when an error is returned (for // instance if a database integrity error happened). Error *vtrpc.RPCError `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` - // session is the updated session information (only returned inside a transaction). + // session is the updated session information. Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` - // result contains the query result, only set if error is unset. - Result *query.QueryResult `protobuf:"bytes,3,opt,name=result,proto3" json:"result,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + // results contains the query results, only set if application level error is unset. + Results []*query.ResultWithError `protobuf:"bytes,3,rep,name=results,proto3" json:"results,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *ExecuteShardsResponse) Reset() { *m = ExecuteShardsResponse{} } -func (m *ExecuteShardsResponse) String() string { return proto.CompactTextString(m) } -func (*ExecuteShardsResponse) ProtoMessage() {} -func (*ExecuteShardsResponse) Descriptor() ([]byte, []int) { +func (m *ExecuteBatchResponse) Reset() { *m = ExecuteBatchResponse{} } +func (m *ExecuteBatchResponse) String() string { return proto.CompactTextString(m) } +func (*ExecuteBatchResponse) ProtoMessage() {} +func (*ExecuteBatchResponse) Descriptor() ([]byte, []int) { return fileDescriptor_aab96496ceaf1ebb, []int{4} } -func (m *ExecuteShardsResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExecuteShardsResponse.Unmarshal(m, b) +func (m *ExecuteBatchResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ExecuteBatchResponse.Unmarshal(m, b) } -func (m *ExecuteShardsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExecuteShardsResponse.Marshal(b, m, deterministic) +func (m *ExecuteBatchResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ExecuteBatchResponse.Marshal(b, m, deterministic) } -func (m *ExecuteShardsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExecuteShardsResponse.Merge(m, src) +func (m *ExecuteBatchResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ExecuteBatchResponse.Merge(m, src) } -func (m *ExecuteShardsResponse) XXX_Size() int { - return xxx_messageInfo_ExecuteShardsResponse.Size(m) +func (m *ExecuteBatchResponse) XXX_Size() int { + return xxx_messageInfo_ExecuteBatchResponse.Size(m) } -func (m *ExecuteShardsResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ExecuteShardsResponse.DiscardUnknown(m) +func (m *ExecuteBatchResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ExecuteBatchResponse.DiscardUnknown(m) } -var xxx_messageInfo_ExecuteShardsResponse proto.InternalMessageInfo +var xxx_messageInfo_ExecuteBatchResponse proto.InternalMessageInfo -func (m *ExecuteShardsResponse) GetError() *vtrpc.RPCError { +func (m *ExecuteBatchResponse) GetError() *vtrpc.RPCError { if m != nil { return m.Error } return nil } -func (m *ExecuteShardsResponse) GetSession() *Session { +func (m *ExecuteBatchResponse) GetSession() *Session { if m != nil { return m.Session } return nil } -func (m *ExecuteShardsResponse) GetResult() *query.QueryResult { +func (m *ExecuteBatchResponse) GetResults() []*query.ResultWithError { if m != nil { - return m.Result + return m.Results } return nil } -// ExecuteKeyspaceIdsRequest is the payload to ExecuteKeyspaceIds. -type ExecuteKeyspaceIdsRequest struct { +// StreamExecuteRequest is the payload to StreamExecute. +type StreamExecuteRequest struct { // caller_id identifies the caller. This is the effective caller ID, // set by the application to further identify the caller. CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // session carries the current transaction data. It is returned by Begin. - // Do not fill it in if outside of a transaction. - Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` // query is the query and bind variables to execute. - Query *query.BoundQuery `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` - // keyspace to target the query to. - Keyspace string `protobuf:"bytes,4,opt,name=keyspace,proto3" json:"keyspace,omitempty"` - // keyspace_ids contains the list of keyspace_ids affected by this query. - // Will be used to find the shards to send the query to. - KeyspaceIds [][]byte `protobuf:"bytes,5,rep,name=keyspace_ids,json=keyspaceIds,proto3" json:"keyspace_ids,omitempty"` - // tablet_type is the type of tablets that this query is targeted to. - TabletType topodata.TabletType `protobuf:"varint,6,opt,name=tablet_type,json=tabletType,proto3,enum=topodata.TabletType" json:"tablet_type,omitempty"` - // not_in_transaction is deprecated. - NotInTransaction bool `protobuf:"varint,7,opt,name=not_in_transaction,json=notInTransaction,proto3" json:"not_in_transaction,omitempty"` - // options - Options *query.ExecuteOptions `protobuf:"bytes,8,opt,name=options,proto3" json:"options,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Query *query.BoundQuery `protobuf:"bytes,2,opt,name=query,proto3" json:"query,omitempty"` + // These values are deprecated. Use session instead. + // TODO(sougou): remove in 3.1 + TabletType topodata.TabletType `protobuf:"varint,3,opt,name=tablet_type,json=tabletType,proto3,enum=topodata.TabletType" json:"tablet_type,omitempty"` + KeyspaceShard string `protobuf:"bytes,4,opt,name=keyspace_shard,json=keyspaceShard,proto3" json:"keyspace_shard,omitempty"` + Options *query.ExecuteOptions `protobuf:"bytes,5,opt,name=options,proto3" json:"options,omitempty"` + // session carries the session state. + Session *Session `protobuf:"bytes,6,opt,name=session,proto3" json:"session,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *ExecuteKeyspaceIdsRequest) Reset() { *m = ExecuteKeyspaceIdsRequest{} } -func (m *ExecuteKeyspaceIdsRequest) String() string { return proto.CompactTextString(m) } -func (*ExecuteKeyspaceIdsRequest) ProtoMessage() {} -func (*ExecuteKeyspaceIdsRequest) Descriptor() ([]byte, []int) { +func (m *StreamExecuteRequest) Reset() { *m = StreamExecuteRequest{} } +func (m *StreamExecuteRequest) String() string { return proto.CompactTextString(m) } +func (*StreamExecuteRequest) ProtoMessage() {} +func (*StreamExecuteRequest) Descriptor() ([]byte, []int) { return fileDescriptor_aab96496ceaf1ebb, []int{5} } -func (m *ExecuteKeyspaceIdsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExecuteKeyspaceIdsRequest.Unmarshal(m, b) +func (m *StreamExecuteRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_StreamExecuteRequest.Unmarshal(m, b) } -func (m *ExecuteKeyspaceIdsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExecuteKeyspaceIdsRequest.Marshal(b, m, deterministic) +func (m *StreamExecuteRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_StreamExecuteRequest.Marshal(b, m, deterministic) } -func (m *ExecuteKeyspaceIdsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExecuteKeyspaceIdsRequest.Merge(m, src) +func (m *StreamExecuteRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_StreamExecuteRequest.Merge(m, src) } -func (m *ExecuteKeyspaceIdsRequest) XXX_Size() int { - return xxx_messageInfo_ExecuteKeyspaceIdsRequest.Size(m) +func (m *StreamExecuteRequest) XXX_Size() int { + return xxx_messageInfo_StreamExecuteRequest.Size(m) } -func (m *ExecuteKeyspaceIdsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ExecuteKeyspaceIdsRequest.DiscardUnknown(m) +func (m *StreamExecuteRequest) XXX_DiscardUnknown() { + xxx_messageInfo_StreamExecuteRequest.DiscardUnknown(m) } -var xxx_messageInfo_ExecuteKeyspaceIdsRequest proto.InternalMessageInfo +var xxx_messageInfo_StreamExecuteRequest proto.InternalMessageInfo -func (m *ExecuteKeyspaceIdsRequest) GetCallerId() *vtrpc.CallerID { +func (m *StreamExecuteRequest) GetCallerId() *vtrpc.CallerID { if m != nil { return m.CallerId } return nil } -func (m *ExecuteKeyspaceIdsRequest) GetSession() *Session { +func (m *StreamExecuteRequest) GetQuery() *query.BoundQuery { if m != nil { - return m.Session + return m.Query } return nil } -func (m *ExecuteKeyspaceIdsRequest) GetQuery() *query.BoundQuery { +func (m *StreamExecuteRequest) GetTabletType() topodata.TabletType { if m != nil { - return m.Query + return m.TabletType } - return nil + return topodata.TabletType_UNKNOWN } -func (m *ExecuteKeyspaceIdsRequest) GetKeyspace() string { +func (m *StreamExecuteRequest) GetKeyspaceShard() string { if m != nil { - return m.Keyspace + return m.KeyspaceShard } return "" } -func (m *ExecuteKeyspaceIdsRequest) GetKeyspaceIds() [][]byte { +func (m *StreamExecuteRequest) GetOptions() *query.ExecuteOptions { if m != nil { - return m.KeyspaceIds + return m.Options } return nil } -func (m *ExecuteKeyspaceIdsRequest) GetTabletType() topodata.TabletType { - if m != nil { - return m.TabletType - } - return topodata.TabletType_UNKNOWN -} - -func (m *ExecuteKeyspaceIdsRequest) GetNotInTransaction() bool { - if m != nil { - return m.NotInTransaction - } - return false -} - -func (m *ExecuteKeyspaceIdsRequest) GetOptions() *query.ExecuteOptions { +func (m *StreamExecuteRequest) GetSession() *Session { if m != nil { - return m.Options + return m.Session } return nil } -// ExecuteKeyspaceIdsResponse is the returned value from ExecuteKeyspaceIds. -type ExecuteKeyspaceIdsResponse struct { - // error contains an application level error if necessary. Note the - // session may have changed, even when an error is returned (for - // instance if a database integrity error happened). - Error *vtrpc.RPCError `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` - // session is the updated session information (only returned inside a transaction). - Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` - // result contains the query result, only set if error is unset. - Result *query.QueryResult `protobuf:"bytes,3,opt,name=result,proto3" json:"result,omitempty"` +// StreamExecuteResponse is the returned value from StreamExecute. +// The session is currently not returned because StreamExecute is +// not expected to modify it. +type StreamExecuteResponse struct { + // result contains the result data. + // The first value contains only Fields information. + // The next values contain the actual rows, a few values per result. + Result *query.QueryResult `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *ExecuteKeyspaceIdsResponse) Reset() { *m = ExecuteKeyspaceIdsResponse{} } -func (m *ExecuteKeyspaceIdsResponse) String() string { return proto.CompactTextString(m) } -func (*ExecuteKeyspaceIdsResponse) ProtoMessage() {} -func (*ExecuteKeyspaceIdsResponse) Descriptor() ([]byte, []int) { +func (m *StreamExecuteResponse) Reset() { *m = StreamExecuteResponse{} } +func (m *StreamExecuteResponse) String() string { return proto.CompactTextString(m) } +func (*StreamExecuteResponse) ProtoMessage() {} +func (*StreamExecuteResponse) Descriptor() ([]byte, []int) { return fileDescriptor_aab96496ceaf1ebb, []int{6} } -func (m *ExecuteKeyspaceIdsResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExecuteKeyspaceIdsResponse.Unmarshal(m, b) -} -func (m *ExecuteKeyspaceIdsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExecuteKeyspaceIdsResponse.Marshal(b, m, deterministic) +func (m *StreamExecuteResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_StreamExecuteResponse.Unmarshal(m, b) } -func (m *ExecuteKeyspaceIdsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExecuteKeyspaceIdsResponse.Merge(m, src) +func (m *StreamExecuteResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_StreamExecuteResponse.Marshal(b, m, deterministic) } -func (m *ExecuteKeyspaceIdsResponse) XXX_Size() int { - return xxx_messageInfo_ExecuteKeyspaceIdsResponse.Size(m) +func (m *StreamExecuteResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_StreamExecuteResponse.Merge(m, src) } -func (m *ExecuteKeyspaceIdsResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ExecuteKeyspaceIdsResponse.DiscardUnknown(m) +func (m *StreamExecuteResponse) XXX_Size() int { + return xxx_messageInfo_StreamExecuteResponse.Size(m) } - -var xxx_messageInfo_ExecuteKeyspaceIdsResponse proto.InternalMessageInfo - -func (m *ExecuteKeyspaceIdsResponse) GetError() *vtrpc.RPCError { - if m != nil { - return m.Error - } - return nil +func (m *StreamExecuteResponse) XXX_DiscardUnknown() { + xxx_messageInfo_StreamExecuteResponse.DiscardUnknown(m) } -func (m *ExecuteKeyspaceIdsResponse) GetSession() *Session { - if m != nil { - return m.Session - } - return nil -} +var xxx_messageInfo_StreamExecuteResponse proto.InternalMessageInfo -func (m *ExecuteKeyspaceIdsResponse) GetResult() *query.QueryResult { +func (m *StreamExecuteResponse) GetResult() *query.QueryResult { if m != nil { return m.Result } return nil } -// ExecuteKeyRangesRequest is the payload to ExecuteKeyRanges. -type ExecuteKeyRangesRequest struct { +// ResolveTransactionRequest is the payload to ResolveTransaction. +type ResolveTransactionRequest struct { // caller_id identifies the caller. This is the effective caller ID, // set by the application to further identify the caller. CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // session carries the current transaction data. It is returned by Begin. - // Do not fill it in if outside of a transaction. - Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` - // query is the query and bind variables to execute. - Query *query.BoundQuery `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` - // keyspace to target the query to - Keyspace string `protobuf:"bytes,4,opt,name=keyspace,proto3" json:"keyspace,omitempty"` - // key_ranges contains the list of key ranges affected by this query. - // Will be used to find the shards to send the query to. - KeyRanges []*topodata.KeyRange `protobuf:"bytes,5,rep,name=key_ranges,json=keyRanges,proto3" json:"key_ranges,omitempty"` - // tablet_type is the type of tablets that this query is targeted to. - TabletType topodata.TabletType `protobuf:"varint,6,opt,name=tablet_type,json=tabletType,proto3,enum=topodata.TabletType" json:"tablet_type,omitempty"` - // not_in_transaction is deprecated. - NotInTransaction bool `protobuf:"varint,7,opt,name=not_in_transaction,json=notInTransaction,proto3" json:"not_in_transaction,omitempty"` - // options - Options *query.ExecuteOptions `protobuf:"bytes,8,opt,name=options,proto3" json:"options,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + // dtid is the dtid of the transaction to be resolved. + Dtid string `protobuf:"bytes,2,opt,name=dtid,proto3" json:"dtid,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *ExecuteKeyRangesRequest) Reset() { *m = ExecuteKeyRangesRequest{} } -func (m *ExecuteKeyRangesRequest) String() string { return proto.CompactTextString(m) } -func (*ExecuteKeyRangesRequest) ProtoMessage() {} -func (*ExecuteKeyRangesRequest) Descriptor() ([]byte, []int) { +func (m *ResolveTransactionRequest) Reset() { *m = ResolveTransactionRequest{} } +func (m *ResolveTransactionRequest) String() string { return proto.CompactTextString(m) } +func (*ResolveTransactionRequest) ProtoMessage() {} +func (*ResolveTransactionRequest) Descriptor() ([]byte, []int) { return fileDescriptor_aab96496ceaf1ebb, []int{7} } -func (m *ExecuteKeyRangesRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExecuteKeyRangesRequest.Unmarshal(m, b) +func (m *ResolveTransactionRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ResolveTransactionRequest.Unmarshal(m, b) } -func (m *ExecuteKeyRangesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExecuteKeyRangesRequest.Marshal(b, m, deterministic) +func (m *ResolveTransactionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ResolveTransactionRequest.Marshal(b, m, deterministic) } -func (m *ExecuteKeyRangesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExecuteKeyRangesRequest.Merge(m, src) +func (m *ResolveTransactionRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ResolveTransactionRequest.Merge(m, src) } -func (m *ExecuteKeyRangesRequest) XXX_Size() int { - return xxx_messageInfo_ExecuteKeyRangesRequest.Size(m) +func (m *ResolveTransactionRequest) XXX_Size() int { + return xxx_messageInfo_ResolveTransactionRequest.Size(m) } -func (m *ExecuteKeyRangesRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ExecuteKeyRangesRequest.DiscardUnknown(m) +func (m *ResolveTransactionRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ResolveTransactionRequest.DiscardUnknown(m) } -var xxx_messageInfo_ExecuteKeyRangesRequest proto.InternalMessageInfo +var xxx_messageInfo_ResolveTransactionRequest proto.InternalMessageInfo -func (m *ExecuteKeyRangesRequest) GetCallerId() *vtrpc.CallerID { +func (m *ResolveTransactionRequest) GetCallerId() *vtrpc.CallerID { if m != nil { return m.CallerId } return nil } -func (m *ExecuteKeyRangesRequest) GetSession() *Session { +func (m *ResolveTransactionRequest) GetDtid() string { if m != nil { - return m.Session + return m.Dtid } - return nil + return "" } -func (m *ExecuteKeyRangesRequest) GetQuery() *query.BoundQuery { - if m != nil { - return m.Query - } - return nil +// ResolveTransactionResponse is the returned value from Rollback. +type ResolveTransactionResponse struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *ExecuteKeyRangesRequest) GetKeyspace() string { - if m != nil { - return m.Keyspace - } - return "" +func (m *ResolveTransactionResponse) Reset() { *m = ResolveTransactionResponse{} } +func (m *ResolveTransactionResponse) String() string { return proto.CompactTextString(m) } +func (*ResolveTransactionResponse) ProtoMessage() {} +func (*ResolveTransactionResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_aab96496ceaf1ebb, []int{8} } -func (m *ExecuteKeyRangesRequest) GetKeyRanges() []*topodata.KeyRange { - if m != nil { - return m.KeyRanges - } - return nil +func (m *ResolveTransactionResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ResolveTransactionResponse.Unmarshal(m, b) } - -func (m *ExecuteKeyRangesRequest) GetTabletType() topodata.TabletType { - if m != nil { - return m.TabletType - } - return topodata.TabletType_UNKNOWN -} - -func (m *ExecuteKeyRangesRequest) GetNotInTransaction() bool { - if m != nil { - return m.NotInTransaction - } - return false -} - -func (m *ExecuteKeyRangesRequest) GetOptions() *query.ExecuteOptions { - if m != nil { - return m.Options - } - return nil -} - -// ExecuteKeyRangesResponse is the returned value from ExecuteKeyRanges. -type ExecuteKeyRangesResponse struct { - // error contains an application level error if necessary. Note the - // session may have changed, even when an error is returned (for - // instance if a database integrity error happened). - Error *vtrpc.RPCError `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` - // session is the updated session information (only returned inside a transaction). - Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` - // result contains the query result, only set if error is unset. - Result *query.QueryResult `protobuf:"bytes,3,opt,name=result,proto3" json:"result,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ExecuteKeyRangesResponse) Reset() { *m = ExecuteKeyRangesResponse{} } -func (m *ExecuteKeyRangesResponse) String() string { return proto.CompactTextString(m) } -func (*ExecuteKeyRangesResponse) ProtoMessage() {} -func (*ExecuteKeyRangesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{8} -} - -func (m *ExecuteKeyRangesResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExecuteKeyRangesResponse.Unmarshal(m, b) -} -func (m *ExecuteKeyRangesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExecuteKeyRangesResponse.Marshal(b, m, deterministic) -} -func (m *ExecuteKeyRangesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExecuteKeyRangesResponse.Merge(m, src) -} -func (m *ExecuteKeyRangesResponse) XXX_Size() int { - return xxx_messageInfo_ExecuteKeyRangesResponse.Size(m) -} -func (m *ExecuteKeyRangesResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ExecuteKeyRangesResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_ExecuteKeyRangesResponse proto.InternalMessageInfo - -func (m *ExecuteKeyRangesResponse) GetError() *vtrpc.RPCError { - if m != nil { - return m.Error - } - return nil -} - -func (m *ExecuteKeyRangesResponse) GetSession() *Session { - if m != nil { - return m.Session - } - return nil -} - -func (m *ExecuteKeyRangesResponse) GetResult() *query.QueryResult { - if m != nil { - return m.Result - } - return nil -} - -// ExecuteEntityIdsRequest is the payload to ExecuteEntityIds. -type ExecuteEntityIdsRequest struct { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // session carries the current transaction data. It is returned by Begin. - // Do not fill it in if outside of a transaction. - Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` - // query is the query and bind variables to execute. - Query *query.BoundQuery `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` - // keyspace to target the query to. - Keyspace string `protobuf:"bytes,4,opt,name=keyspace,proto3" json:"keyspace,omitempty"` - // entity_column_name is the column name to use. - EntityColumnName string `protobuf:"bytes,5,opt,name=entity_column_name,json=entityColumnName,proto3" json:"entity_column_name,omitempty"` - // entity_keyspace_ids are pairs of entity_column_name values - // associated with its corresponding keyspace_id. - EntityKeyspaceIds []*ExecuteEntityIdsRequest_EntityId `protobuf:"bytes,6,rep,name=entity_keyspace_ids,json=entityKeyspaceIds,proto3" json:"entity_keyspace_ids,omitempty"` - // tablet_type is the type of tablets that this query is targeted to. - TabletType topodata.TabletType `protobuf:"varint,7,opt,name=tablet_type,json=tabletType,proto3,enum=topodata.TabletType" json:"tablet_type,omitempty"` - // not_in_transaction is deprecated. - NotInTransaction bool `protobuf:"varint,8,opt,name=not_in_transaction,json=notInTransaction,proto3" json:"not_in_transaction,omitempty"` - // options - Options *query.ExecuteOptions `protobuf:"bytes,9,opt,name=options,proto3" json:"options,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ExecuteEntityIdsRequest) Reset() { *m = ExecuteEntityIdsRequest{} } -func (m *ExecuteEntityIdsRequest) String() string { return proto.CompactTextString(m) } -func (*ExecuteEntityIdsRequest) ProtoMessage() {} -func (*ExecuteEntityIdsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{9} -} - -func (m *ExecuteEntityIdsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExecuteEntityIdsRequest.Unmarshal(m, b) -} -func (m *ExecuteEntityIdsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExecuteEntityIdsRequest.Marshal(b, m, deterministic) -} -func (m *ExecuteEntityIdsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExecuteEntityIdsRequest.Merge(m, src) -} -func (m *ExecuteEntityIdsRequest) XXX_Size() int { - return xxx_messageInfo_ExecuteEntityIdsRequest.Size(m) -} -func (m *ExecuteEntityIdsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ExecuteEntityIdsRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_ExecuteEntityIdsRequest proto.InternalMessageInfo - -func (m *ExecuteEntityIdsRequest) GetCallerId() *vtrpc.CallerID { - if m != nil { - return m.CallerId - } - return nil -} - -func (m *ExecuteEntityIdsRequest) GetSession() *Session { - if m != nil { - return m.Session - } - return nil -} - -func (m *ExecuteEntityIdsRequest) GetQuery() *query.BoundQuery { - if m != nil { - return m.Query - } - return nil -} - -func (m *ExecuteEntityIdsRequest) GetKeyspace() string { - if m != nil { - return m.Keyspace - } - return "" -} - -func (m *ExecuteEntityIdsRequest) GetEntityColumnName() string { - if m != nil { - return m.EntityColumnName - } - return "" -} - -func (m *ExecuteEntityIdsRequest) GetEntityKeyspaceIds() []*ExecuteEntityIdsRequest_EntityId { - if m != nil { - return m.EntityKeyspaceIds - } - return nil -} - -func (m *ExecuteEntityIdsRequest) GetTabletType() topodata.TabletType { - if m != nil { - return m.TabletType - } - return topodata.TabletType_UNKNOWN -} - -func (m *ExecuteEntityIdsRequest) GetNotInTransaction() bool { - if m != nil { - return m.NotInTransaction - } - return false -} - -func (m *ExecuteEntityIdsRequest) GetOptions() *query.ExecuteOptions { - if m != nil { - return m.Options - } - return nil -} - -type ExecuteEntityIdsRequest_EntityId struct { - // type is the type of the entity's value. Can be NULL_TYPE. - Type query.Type `protobuf:"varint,1,opt,name=type,proto3,enum=query.Type" json:"type,omitempty"` - // value is the value for the entity. Not set if type is NULL_TYPE. - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - // keyspace_id is the associated keyspace_id for the entity. - KeyspaceId []byte `protobuf:"bytes,3,opt,name=keyspace_id,json=keyspaceId,proto3" json:"keyspace_id,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ExecuteEntityIdsRequest_EntityId) Reset() { *m = ExecuteEntityIdsRequest_EntityId{} } -func (m *ExecuteEntityIdsRequest_EntityId) String() string { return proto.CompactTextString(m) } -func (*ExecuteEntityIdsRequest_EntityId) ProtoMessage() {} -func (*ExecuteEntityIdsRequest_EntityId) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{9, 0} -} - -func (m *ExecuteEntityIdsRequest_EntityId) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExecuteEntityIdsRequest_EntityId.Unmarshal(m, b) -} -func (m *ExecuteEntityIdsRequest_EntityId) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExecuteEntityIdsRequest_EntityId.Marshal(b, m, deterministic) -} -func (m *ExecuteEntityIdsRequest_EntityId) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExecuteEntityIdsRequest_EntityId.Merge(m, src) -} -func (m *ExecuteEntityIdsRequest_EntityId) XXX_Size() int { - return xxx_messageInfo_ExecuteEntityIdsRequest_EntityId.Size(m) -} -func (m *ExecuteEntityIdsRequest_EntityId) XXX_DiscardUnknown() { - xxx_messageInfo_ExecuteEntityIdsRequest_EntityId.DiscardUnknown(m) -} - -var xxx_messageInfo_ExecuteEntityIdsRequest_EntityId proto.InternalMessageInfo - -func (m *ExecuteEntityIdsRequest_EntityId) GetType() query.Type { - if m != nil { - return m.Type - } - return query.Type_NULL_TYPE -} - -func (m *ExecuteEntityIdsRequest_EntityId) GetValue() []byte { - if m != nil { - return m.Value - } - return nil -} - -func (m *ExecuteEntityIdsRequest_EntityId) GetKeyspaceId() []byte { - if m != nil { - return m.KeyspaceId - } - return nil -} - -// ExecuteEntityIdsResponse is the returned value from ExecuteEntityIds. -type ExecuteEntityIdsResponse struct { - // error contains an application level error if necessary. Note the - // session may have changed, even when an error is returned (for - // instance if a database integrity error happened). - Error *vtrpc.RPCError `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` - // session is the updated session information (only returned inside a transaction). - Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` - // result contains the query result, only set if error is unset. - Result *query.QueryResult `protobuf:"bytes,3,opt,name=result,proto3" json:"result,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ExecuteEntityIdsResponse) Reset() { *m = ExecuteEntityIdsResponse{} } -func (m *ExecuteEntityIdsResponse) String() string { return proto.CompactTextString(m) } -func (*ExecuteEntityIdsResponse) ProtoMessage() {} -func (*ExecuteEntityIdsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{10} -} - -func (m *ExecuteEntityIdsResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExecuteEntityIdsResponse.Unmarshal(m, b) -} -func (m *ExecuteEntityIdsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExecuteEntityIdsResponse.Marshal(b, m, deterministic) -} -func (m *ExecuteEntityIdsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExecuteEntityIdsResponse.Merge(m, src) -} -func (m *ExecuteEntityIdsResponse) XXX_Size() int { - return xxx_messageInfo_ExecuteEntityIdsResponse.Size(m) -} -func (m *ExecuteEntityIdsResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ExecuteEntityIdsResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_ExecuteEntityIdsResponse proto.InternalMessageInfo - -func (m *ExecuteEntityIdsResponse) GetError() *vtrpc.RPCError { - if m != nil { - return m.Error - } - return nil -} - -func (m *ExecuteEntityIdsResponse) GetSession() *Session { - if m != nil { - return m.Session - } - return nil -} - -func (m *ExecuteEntityIdsResponse) GetResult() *query.QueryResult { - if m != nil { - return m.Result - } - return nil -} - -// ExecuteBatchRequest is the payload to ExecuteBatch. -type ExecuteBatchRequest struct { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // session carries the session state. - Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` - // queries is a list of query and bind variables to execute. - Queries []*query.BoundQuery `protobuf:"bytes,3,rep,name=queries,proto3" json:"queries,omitempty"` - // These values are deprecated. Use session instead. - // TODO(sougou): remove in 3.1 - TabletType topodata.TabletType `protobuf:"varint,4,opt,name=tablet_type,json=tabletType,proto3,enum=topodata.TabletType" json:"tablet_type,omitempty"` - AsTransaction bool `protobuf:"varint,5,opt,name=as_transaction,json=asTransaction,proto3" json:"as_transaction,omitempty"` - KeyspaceShard string `protobuf:"bytes,6,opt,name=keyspace_shard,json=keyspaceShard,proto3" json:"keyspace_shard,omitempty"` - Options *query.ExecuteOptions `protobuf:"bytes,7,opt,name=options,proto3" json:"options,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ExecuteBatchRequest) Reset() { *m = ExecuteBatchRequest{} } -func (m *ExecuteBatchRequest) String() string { return proto.CompactTextString(m) } -func (*ExecuteBatchRequest) ProtoMessage() {} -func (*ExecuteBatchRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{11} -} - -func (m *ExecuteBatchRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExecuteBatchRequest.Unmarshal(m, b) -} -func (m *ExecuteBatchRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExecuteBatchRequest.Marshal(b, m, deterministic) -} -func (m *ExecuteBatchRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExecuteBatchRequest.Merge(m, src) -} -func (m *ExecuteBatchRequest) XXX_Size() int { - return xxx_messageInfo_ExecuteBatchRequest.Size(m) -} -func (m *ExecuteBatchRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ExecuteBatchRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_ExecuteBatchRequest proto.InternalMessageInfo - -func (m *ExecuteBatchRequest) GetCallerId() *vtrpc.CallerID { - if m != nil { - return m.CallerId - } - return nil -} - -func (m *ExecuteBatchRequest) GetSession() *Session { - if m != nil { - return m.Session - } - return nil -} - -func (m *ExecuteBatchRequest) GetQueries() []*query.BoundQuery { - if m != nil { - return m.Queries - } - return nil -} - -func (m *ExecuteBatchRequest) GetTabletType() topodata.TabletType { - if m != nil { - return m.TabletType - } - return topodata.TabletType_UNKNOWN -} - -func (m *ExecuteBatchRequest) GetAsTransaction() bool { - if m != nil { - return m.AsTransaction - } - return false -} - -func (m *ExecuteBatchRequest) GetKeyspaceShard() string { - if m != nil { - return m.KeyspaceShard - } - return "" -} - -func (m *ExecuteBatchRequest) GetOptions() *query.ExecuteOptions { - if m != nil { - return m.Options - } - return nil -} - -// ExecuteBatchResponse is the returned value from ExecuteBatch. -type ExecuteBatchResponse struct { - // error contains an application level error if necessary. Note the - // session may have changed, even when an error is returned (for - // instance if a database integrity error happened). - Error *vtrpc.RPCError `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` - // session is the updated session information. - Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` - // results contains the query results, only set if application level error is unset. - Results []*query.ResultWithError `protobuf:"bytes,3,rep,name=results,proto3" json:"results,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ExecuteBatchResponse) Reset() { *m = ExecuteBatchResponse{} } -func (m *ExecuteBatchResponse) String() string { return proto.CompactTextString(m) } -func (*ExecuteBatchResponse) ProtoMessage() {} -func (*ExecuteBatchResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{12} -} - -func (m *ExecuteBatchResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExecuteBatchResponse.Unmarshal(m, b) -} -func (m *ExecuteBatchResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExecuteBatchResponse.Marshal(b, m, deterministic) -} -func (m *ExecuteBatchResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExecuteBatchResponse.Merge(m, src) -} -func (m *ExecuteBatchResponse) XXX_Size() int { - return xxx_messageInfo_ExecuteBatchResponse.Size(m) -} -func (m *ExecuteBatchResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ExecuteBatchResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_ExecuteBatchResponse proto.InternalMessageInfo - -func (m *ExecuteBatchResponse) GetError() *vtrpc.RPCError { - if m != nil { - return m.Error - } - return nil -} - -func (m *ExecuteBatchResponse) GetSession() *Session { - if m != nil { - return m.Session - } - return nil -} - -func (m *ExecuteBatchResponse) GetResults() []*query.ResultWithError { - if m != nil { - return m.Results - } - return nil -} - -// BoundShardQuery represents a single query request for the -// specified list of shards. This is used in a list for -// ExecuteBatchShardsRequest. -type BoundShardQuery struct { - // query is the query and bind variables to execute. - Query *query.BoundQuery `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` - // keyspace to target the query to. - Keyspace string `protobuf:"bytes,2,opt,name=keyspace,proto3" json:"keyspace,omitempty"` - // shards to target the query to. A DML can only target one shard. - Shards []string `protobuf:"bytes,3,rep,name=shards,proto3" json:"shards,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *BoundShardQuery) Reset() { *m = BoundShardQuery{} } -func (m *BoundShardQuery) String() string { return proto.CompactTextString(m) } -func (*BoundShardQuery) ProtoMessage() {} -func (*BoundShardQuery) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{13} -} - -func (m *BoundShardQuery) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_BoundShardQuery.Unmarshal(m, b) -} -func (m *BoundShardQuery) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_BoundShardQuery.Marshal(b, m, deterministic) -} -func (m *BoundShardQuery) XXX_Merge(src proto.Message) { - xxx_messageInfo_BoundShardQuery.Merge(m, src) -} -func (m *BoundShardQuery) XXX_Size() int { - return xxx_messageInfo_BoundShardQuery.Size(m) -} -func (m *BoundShardQuery) XXX_DiscardUnknown() { - xxx_messageInfo_BoundShardQuery.DiscardUnknown(m) -} - -var xxx_messageInfo_BoundShardQuery proto.InternalMessageInfo - -func (m *BoundShardQuery) GetQuery() *query.BoundQuery { - if m != nil { - return m.Query - } - return nil -} - -func (m *BoundShardQuery) GetKeyspace() string { - if m != nil { - return m.Keyspace - } - return "" -} - -func (m *BoundShardQuery) GetShards() []string { - if m != nil { - return m.Shards - } - return nil -} - -// ExecuteBatchShardsRequest is the payload to ExecuteBatchShards -type ExecuteBatchShardsRequest struct { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // session carries the current transaction data. It is returned by Begin. - // Do not fill it in if outside of a transaction. - Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` - // queries carries all the queries to execute. - Queries []*BoundShardQuery `protobuf:"bytes,3,rep,name=queries,proto3" json:"queries,omitempty"` - // tablet_type is the type of tablets that this query is targeted to. - TabletType topodata.TabletType `protobuf:"varint,4,opt,name=tablet_type,json=tabletType,proto3,enum=topodata.TabletType" json:"tablet_type,omitempty"` - // as_transaction will execute the queries in this batch in a single transaction per shard, created for this purpose. - // (this can be seen as adding a 'begin' before and 'commit' after the queries). - // Only makes sense if tablet_type is master. If set, the Session is ignored. - AsTransaction bool `protobuf:"varint,5,opt,name=as_transaction,json=asTransaction,proto3" json:"as_transaction,omitempty"` - // options - Options *query.ExecuteOptions `protobuf:"bytes,6,opt,name=options,proto3" json:"options,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ExecuteBatchShardsRequest) Reset() { *m = ExecuteBatchShardsRequest{} } -func (m *ExecuteBatchShardsRequest) String() string { return proto.CompactTextString(m) } -func (*ExecuteBatchShardsRequest) ProtoMessage() {} -func (*ExecuteBatchShardsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{14} -} - -func (m *ExecuteBatchShardsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExecuteBatchShardsRequest.Unmarshal(m, b) -} -func (m *ExecuteBatchShardsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExecuteBatchShardsRequest.Marshal(b, m, deterministic) -} -func (m *ExecuteBatchShardsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExecuteBatchShardsRequest.Merge(m, src) -} -func (m *ExecuteBatchShardsRequest) XXX_Size() int { - return xxx_messageInfo_ExecuteBatchShardsRequest.Size(m) -} -func (m *ExecuteBatchShardsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ExecuteBatchShardsRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_ExecuteBatchShardsRequest proto.InternalMessageInfo - -func (m *ExecuteBatchShardsRequest) GetCallerId() *vtrpc.CallerID { - if m != nil { - return m.CallerId - } - return nil -} - -func (m *ExecuteBatchShardsRequest) GetSession() *Session { - if m != nil { - return m.Session - } - return nil -} - -func (m *ExecuteBatchShardsRequest) GetQueries() []*BoundShardQuery { - if m != nil { - return m.Queries - } - return nil -} - -func (m *ExecuteBatchShardsRequest) GetTabletType() topodata.TabletType { - if m != nil { - return m.TabletType - } - return topodata.TabletType_UNKNOWN -} - -func (m *ExecuteBatchShardsRequest) GetAsTransaction() bool { - if m != nil { - return m.AsTransaction - } - return false -} - -func (m *ExecuteBatchShardsRequest) GetOptions() *query.ExecuteOptions { - if m != nil { - return m.Options - } - return nil -} - -// ExecuteBatchShardsResponse is the returned value from ExecuteBatchShards. -type ExecuteBatchShardsResponse struct { - // error contains an application level error if necessary. Note the - // session may have changed, even when an error is returned (for - // instance if a database integrity error happened). - Error *vtrpc.RPCError `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` - // session is the updated session information (only returned inside a transaction). - Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` - // result contains the query result, only set if error is unset. - Results []*query.QueryResult `protobuf:"bytes,3,rep,name=results,proto3" json:"results,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ExecuteBatchShardsResponse) Reset() { *m = ExecuteBatchShardsResponse{} } -func (m *ExecuteBatchShardsResponse) String() string { return proto.CompactTextString(m) } -func (*ExecuteBatchShardsResponse) ProtoMessage() {} -func (*ExecuteBatchShardsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{15} -} - -func (m *ExecuteBatchShardsResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExecuteBatchShardsResponse.Unmarshal(m, b) -} -func (m *ExecuteBatchShardsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExecuteBatchShardsResponse.Marshal(b, m, deterministic) -} -func (m *ExecuteBatchShardsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExecuteBatchShardsResponse.Merge(m, src) -} -func (m *ExecuteBatchShardsResponse) XXX_Size() int { - return xxx_messageInfo_ExecuteBatchShardsResponse.Size(m) -} -func (m *ExecuteBatchShardsResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ExecuteBatchShardsResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_ExecuteBatchShardsResponse proto.InternalMessageInfo - -func (m *ExecuteBatchShardsResponse) GetError() *vtrpc.RPCError { - if m != nil { - return m.Error - } - return nil -} - -func (m *ExecuteBatchShardsResponse) GetSession() *Session { - if m != nil { - return m.Session - } - return nil -} - -func (m *ExecuteBatchShardsResponse) GetResults() []*query.QueryResult { - if m != nil { - return m.Results - } - return nil -} - -// BoundKeyspaceIdQuery represents a single query request for the -// specified list of keyspace ids. This is used in a list for -// ExecuteBatchKeyspaceIdsRequest. -type BoundKeyspaceIdQuery struct { - // query is the query and bind variables to execute. - Query *query.BoundQuery `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` - // keyspace to target the query to. - Keyspace string `protobuf:"bytes,2,opt,name=keyspace,proto3" json:"keyspace,omitempty"` - // keyspace_ids contains the list of keyspace_ids affected by this query. - // Will be used to find the shards to send the query to. - KeyspaceIds [][]byte `protobuf:"bytes,3,rep,name=keyspace_ids,json=keyspaceIds,proto3" json:"keyspace_ids,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *BoundKeyspaceIdQuery) Reset() { *m = BoundKeyspaceIdQuery{} } -func (m *BoundKeyspaceIdQuery) String() string { return proto.CompactTextString(m) } -func (*BoundKeyspaceIdQuery) ProtoMessage() {} -func (*BoundKeyspaceIdQuery) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{16} -} - -func (m *BoundKeyspaceIdQuery) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_BoundKeyspaceIdQuery.Unmarshal(m, b) -} -func (m *BoundKeyspaceIdQuery) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_BoundKeyspaceIdQuery.Marshal(b, m, deterministic) -} -func (m *BoundKeyspaceIdQuery) XXX_Merge(src proto.Message) { - xxx_messageInfo_BoundKeyspaceIdQuery.Merge(m, src) -} -func (m *BoundKeyspaceIdQuery) XXX_Size() int { - return xxx_messageInfo_BoundKeyspaceIdQuery.Size(m) -} -func (m *BoundKeyspaceIdQuery) XXX_DiscardUnknown() { - xxx_messageInfo_BoundKeyspaceIdQuery.DiscardUnknown(m) -} - -var xxx_messageInfo_BoundKeyspaceIdQuery proto.InternalMessageInfo - -func (m *BoundKeyspaceIdQuery) GetQuery() *query.BoundQuery { - if m != nil { - return m.Query - } - return nil -} - -func (m *BoundKeyspaceIdQuery) GetKeyspace() string { - if m != nil { - return m.Keyspace - } - return "" -} - -func (m *BoundKeyspaceIdQuery) GetKeyspaceIds() [][]byte { - if m != nil { - return m.KeyspaceIds - } - return nil -} - -// ExecuteBatchKeyspaceIdsRequest is the payload to ExecuteBatchKeyspaceId. -type ExecuteBatchKeyspaceIdsRequest struct { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // session carries the current transaction data. It is returned by Begin. - // Do not fill it in if outside of a transaction. - Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` - Queries []*BoundKeyspaceIdQuery `protobuf:"bytes,3,rep,name=queries,proto3" json:"queries,omitempty"` - // tablet_type is the type of tablets that this query is targeted to. - TabletType topodata.TabletType `protobuf:"varint,4,opt,name=tablet_type,json=tabletType,proto3,enum=topodata.TabletType" json:"tablet_type,omitempty"` - // as_transaction will execute the queries in this batch in a single transaction per shard, created for this purpose. - // (this can be seen as adding a 'begin' before and 'commit' after the queries). - // Only makes sense if tablet_type is master. If set, the Session is ignored. - AsTransaction bool `protobuf:"varint,5,opt,name=as_transaction,json=asTransaction,proto3" json:"as_transaction,omitempty"` - // options - Options *query.ExecuteOptions `protobuf:"bytes,6,opt,name=options,proto3" json:"options,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ExecuteBatchKeyspaceIdsRequest) Reset() { *m = ExecuteBatchKeyspaceIdsRequest{} } -func (m *ExecuteBatchKeyspaceIdsRequest) String() string { return proto.CompactTextString(m) } -func (*ExecuteBatchKeyspaceIdsRequest) ProtoMessage() {} -func (*ExecuteBatchKeyspaceIdsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{17} -} - -func (m *ExecuteBatchKeyspaceIdsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExecuteBatchKeyspaceIdsRequest.Unmarshal(m, b) -} -func (m *ExecuteBatchKeyspaceIdsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExecuteBatchKeyspaceIdsRequest.Marshal(b, m, deterministic) -} -func (m *ExecuteBatchKeyspaceIdsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExecuteBatchKeyspaceIdsRequest.Merge(m, src) -} -func (m *ExecuteBatchKeyspaceIdsRequest) XXX_Size() int { - return xxx_messageInfo_ExecuteBatchKeyspaceIdsRequest.Size(m) -} -func (m *ExecuteBatchKeyspaceIdsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ExecuteBatchKeyspaceIdsRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_ExecuteBatchKeyspaceIdsRequest proto.InternalMessageInfo - -func (m *ExecuteBatchKeyspaceIdsRequest) GetCallerId() *vtrpc.CallerID { - if m != nil { - return m.CallerId - } - return nil -} - -func (m *ExecuteBatchKeyspaceIdsRequest) GetSession() *Session { - if m != nil { - return m.Session - } - return nil -} - -func (m *ExecuteBatchKeyspaceIdsRequest) GetQueries() []*BoundKeyspaceIdQuery { - if m != nil { - return m.Queries - } - return nil -} - -func (m *ExecuteBatchKeyspaceIdsRequest) GetTabletType() topodata.TabletType { - if m != nil { - return m.TabletType - } - return topodata.TabletType_UNKNOWN -} - -func (m *ExecuteBatchKeyspaceIdsRequest) GetAsTransaction() bool { - if m != nil { - return m.AsTransaction - } - return false -} - -func (m *ExecuteBatchKeyspaceIdsRequest) GetOptions() *query.ExecuteOptions { - if m != nil { - return m.Options - } - return nil -} - -// ExecuteBatchKeyspaceIdsResponse is the returned value from ExecuteBatchKeyspaceId. -type ExecuteBatchKeyspaceIdsResponse struct { - // error contains an application level error if necessary. Note the - // session may have changed, even when an error is returned (for - // instance if a database integrity error happened). - Error *vtrpc.RPCError `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` - // session is the updated session information (only returned inside a transaction). - Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` - // result contains the query result, only set if error is unset. - Results []*query.QueryResult `protobuf:"bytes,3,rep,name=results,proto3" json:"results,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ExecuteBatchKeyspaceIdsResponse) Reset() { *m = ExecuteBatchKeyspaceIdsResponse{} } -func (m *ExecuteBatchKeyspaceIdsResponse) String() string { return proto.CompactTextString(m) } -func (*ExecuteBatchKeyspaceIdsResponse) ProtoMessage() {} -func (*ExecuteBatchKeyspaceIdsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{18} -} - -func (m *ExecuteBatchKeyspaceIdsResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExecuteBatchKeyspaceIdsResponse.Unmarshal(m, b) -} -func (m *ExecuteBatchKeyspaceIdsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExecuteBatchKeyspaceIdsResponse.Marshal(b, m, deterministic) -} -func (m *ExecuteBatchKeyspaceIdsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExecuteBatchKeyspaceIdsResponse.Merge(m, src) -} -func (m *ExecuteBatchKeyspaceIdsResponse) XXX_Size() int { - return xxx_messageInfo_ExecuteBatchKeyspaceIdsResponse.Size(m) -} -func (m *ExecuteBatchKeyspaceIdsResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ExecuteBatchKeyspaceIdsResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_ExecuteBatchKeyspaceIdsResponse proto.InternalMessageInfo - -func (m *ExecuteBatchKeyspaceIdsResponse) GetError() *vtrpc.RPCError { - if m != nil { - return m.Error - } - return nil -} - -func (m *ExecuteBatchKeyspaceIdsResponse) GetSession() *Session { - if m != nil { - return m.Session - } - return nil -} - -func (m *ExecuteBatchKeyspaceIdsResponse) GetResults() []*query.QueryResult { - if m != nil { - return m.Results - } - return nil -} - -// StreamExecuteRequest is the payload to StreamExecute. -type StreamExecuteRequest struct { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // query is the query and bind variables to execute. - Query *query.BoundQuery `protobuf:"bytes,2,opt,name=query,proto3" json:"query,omitempty"` - // These values are deprecated. Use session instead. - // TODO(sougou): remove in 3.1 - TabletType topodata.TabletType `protobuf:"varint,3,opt,name=tablet_type,json=tabletType,proto3,enum=topodata.TabletType" json:"tablet_type,omitempty"` - KeyspaceShard string `protobuf:"bytes,4,opt,name=keyspace_shard,json=keyspaceShard,proto3" json:"keyspace_shard,omitempty"` - Options *query.ExecuteOptions `protobuf:"bytes,5,opt,name=options,proto3" json:"options,omitempty"` - // session carries the session state. - Session *Session `protobuf:"bytes,6,opt,name=session,proto3" json:"session,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamExecuteRequest) Reset() { *m = StreamExecuteRequest{} } -func (m *StreamExecuteRequest) String() string { return proto.CompactTextString(m) } -func (*StreamExecuteRequest) ProtoMessage() {} -func (*StreamExecuteRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{19} -} - -func (m *StreamExecuteRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamExecuteRequest.Unmarshal(m, b) -} -func (m *StreamExecuteRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamExecuteRequest.Marshal(b, m, deterministic) -} -func (m *StreamExecuteRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamExecuteRequest.Merge(m, src) -} -func (m *StreamExecuteRequest) XXX_Size() int { - return xxx_messageInfo_StreamExecuteRequest.Size(m) -} -func (m *StreamExecuteRequest) XXX_DiscardUnknown() { - xxx_messageInfo_StreamExecuteRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamExecuteRequest proto.InternalMessageInfo - -func (m *StreamExecuteRequest) GetCallerId() *vtrpc.CallerID { - if m != nil { - return m.CallerId - } - return nil -} - -func (m *StreamExecuteRequest) GetQuery() *query.BoundQuery { - if m != nil { - return m.Query - } - return nil -} - -func (m *StreamExecuteRequest) GetTabletType() topodata.TabletType { - if m != nil { - return m.TabletType - } - return topodata.TabletType_UNKNOWN -} - -func (m *StreamExecuteRequest) GetKeyspaceShard() string { - if m != nil { - return m.KeyspaceShard - } - return "" -} - -func (m *StreamExecuteRequest) GetOptions() *query.ExecuteOptions { - if m != nil { - return m.Options - } - return nil -} - -func (m *StreamExecuteRequest) GetSession() *Session { - if m != nil { - return m.Session - } - return nil -} - -// StreamExecuteResponse is the returned value from StreamExecute. -// The session is currently not returned because StreamExecute is -// not expected to modify it. -type StreamExecuteResponse struct { - // result contains the result data. - // The first value contains only Fields information. - // The next values contain the actual rows, a few values per result. - Result *query.QueryResult `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamExecuteResponse) Reset() { *m = StreamExecuteResponse{} } -func (m *StreamExecuteResponse) String() string { return proto.CompactTextString(m) } -func (*StreamExecuteResponse) ProtoMessage() {} -func (*StreamExecuteResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{20} -} - -func (m *StreamExecuteResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamExecuteResponse.Unmarshal(m, b) -} -func (m *StreamExecuteResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamExecuteResponse.Marshal(b, m, deterministic) -} -func (m *StreamExecuteResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamExecuteResponse.Merge(m, src) -} -func (m *StreamExecuteResponse) XXX_Size() int { - return xxx_messageInfo_StreamExecuteResponse.Size(m) -} -func (m *StreamExecuteResponse) XXX_DiscardUnknown() { - xxx_messageInfo_StreamExecuteResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamExecuteResponse proto.InternalMessageInfo - -func (m *StreamExecuteResponse) GetResult() *query.QueryResult { - if m != nil { - return m.Result - } - return nil -} - -// StreamExecuteShardsRequest is the payload to StreamExecuteShards. -type StreamExecuteShardsRequest struct { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // query is the query and bind variables to execute. - Query *query.BoundQuery `protobuf:"bytes,2,opt,name=query,proto3" json:"query,omitempty"` - // keyspace to target the query to. - Keyspace string `protobuf:"bytes,3,opt,name=keyspace,proto3" json:"keyspace,omitempty"` - // shards to target the query to. - Shards []string `protobuf:"bytes,4,rep,name=shards,proto3" json:"shards,omitempty"` - // tablet_type is the type of tablets that this query is targeted to. - TabletType topodata.TabletType `protobuf:"varint,5,opt,name=tablet_type,json=tabletType,proto3,enum=topodata.TabletType" json:"tablet_type,omitempty"` - // options - Options *query.ExecuteOptions `protobuf:"bytes,6,opt,name=options,proto3" json:"options,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamExecuteShardsRequest) Reset() { *m = StreamExecuteShardsRequest{} } -func (m *StreamExecuteShardsRequest) String() string { return proto.CompactTextString(m) } -func (*StreamExecuteShardsRequest) ProtoMessage() {} -func (*StreamExecuteShardsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{21} -} - -func (m *StreamExecuteShardsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamExecuteShardsRequest.Unmarshal(m, b) -} -func (m *StreamExecuteShardsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamExecuteShardsRequest.Marshal(b, m, deterministic) -} -func (m *StreamExecuteShardsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamExecuteShardsRequest.Merge(m, src) -} -func (m *StreamExecuteShardsRequest) XXX_Size() int { - return xxx_messageInfo_StreamExecuteShardsRequest.Size(m) -} -func (m *StreamExecuteShardsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_StreamExecuteShardsRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamExecuteShardsRequest proto.InternalMessageInfo - -func (m *StreamExecuteShardsRequest) GetCallerId() *vtrpc.CallerID { - if m != nil { - return m.CallerId - } - return nil -} - -func (m *StreamExecuteShardsRequest) GetQuery() *query.BoundQuery { - if m != nil { - return m.Query - } - return nil -} - -func (m *StreamExecuteShardsRequest) GetKeyspace() string { - if m != nil { - return m.Keyspace - } - return "" -} - -func (m *StreamExecuteShardsRequest) GetShards() []string { - if m != nil { - return m.Shards - } - return nil -} - -func (m *StreamExecuteShardsRequest) GetTabletType() topodata.TabletType { - if m != nil { - return m.TabletType - } - return topodata.TabletType_UNKNOWN -} - -func (m *StreamExecuteShardsRequest) GetOptions() *query.ExecuteOptions { - if m != nil { - return m.Options - } - return nil -} - -// StreamExecuteShardsResponse is the returned value from StreamExecuteShards. -type StreamExecuteShardsResponse struct { - // result contains the result data. - // The first value contains only Fields information. - // The next values contain the actual rows, a few values per result. - Result *query.QueryResult `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamExecuteShardsResponse) Reset() { *m = StreamExecuteShardsResponse{} } -func (m *StreamExecuteShardsResponse) String() string { return proto.CompactTextString(m) } -func (*StreamExecuteShardsResponse) ProtoMessage() {} -func (*StreamExecuteShardsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{22} -} - -func (m *StreamExecuteShardsResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamExecuteShardsResponse.Unmarshal(m, b) -} -func (m *StreamExecuteShardsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamExecuteShardsResponse.Marshal(b, m, deterministic) -} -func (m *StreamExecuteShardsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamExecuteShardsResponse.Merge(m, src) -} -func (m *StreamExecuteShardsResponse) XXX_Size() int { - return xxx_messageInfo_StreamExecuteShardsResponse.Size(m) -} -func (m *StreamExecuteShardsResponse) XXX_DiscardUnknown() { - xxx_messageInfo_StreamExecuteShardsResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamExecuteShardsResponse proto.InternalMessageInfo - -func (m *StreamExecuteShardsResponse) GetResult() *query.QueryResult { - if m != nil { - return m.Result - } - return nil -} - -// StreamExecuteKeyspaceIdsRequest is the payload to StreamExecuteKeyspaceIds. -type StreamExecuteKeyspaceIdsRequest struct { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // query is the query and bind variables to execute. - Query *query.BoundQuery `protobuf:"bytes,2,opt,name=query,proto3" json:"query,omitempty"` - // keyspace to target the query to. - Keyspace string `protobuf:"bytes,3,opt,name=keyspace,proto3" json:"keyspace,omitempty"` - // keyspace_ids contains the list of keyspace_ids affected by this query. - // Will be used to find the shards to send the query to. - KeyspaceIds [][]byte `protobuf:"bytes,4,rep,name=keyspace_ids,json=keyspaceIds,proto3" json:"keyspace_ids,omitempty"` - // tablet_type is the type of tablets that this query is targeted to. - TabletType topodata.TabletType `protobuf:"varint,5,opt,name=tablet_type,json=tabletType,proto3,enum=topodata.TabletType" json:"tablet_type,omitempty"` - // options - Options *query.ExecuteOptions `protobuf:"bytes,6,opt,name=options,proto3" json:"options,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamExecuteKeyspaceIdsRequest) Reset() { *m = StreamExecuteKeyspaceIdsRequest{} } -func (m *StreamExecuteKeyspaceIdsRequest) String() string { return proto.CompactTextString(m) } -func (*StreamExecuteKeyspaceIdsRequest) ProtoMessage() {} -func (*StreamExecuteKeyspaceIdsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{23} -} - -func (m *StreamExecuteKeyspaceIdsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamExecuteKeyspaceIdsRequest.Unmarshal(m, b) -} -func (m *StreamExecuteKeyspaceIdsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamExecuteKeyspaceIdsRequest.Marshal(b, m, deterministic) -} -func (m *StreamExecuteKeyspaceIdsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamExecuteKeyspaceIdsRequest.Merge(m, src) -} -func (m *StreamExecuteKeyspaceIdsRequest) XXX_Size() int { - return xxx_messageInfo_StreamExecuteKeyspaceIdsRequest.Size(m) -} -func (m *StreamExecuteKeyspaceIdsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_StreamExecuteKeyspaceIdsRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamExecuteKeyspaceIdsRequest proto.InternalMessageInfo - -func (m *StreamExecuteKeyspaceIdsRequest) GetCallerId() *vtrpc.CallerID { - if m != nil { - return m.CallerId - } - return nil -} - -func (m *StreamExecuteKeyspaceIdsRequest) GetQuery() *query.BoundQuery { - if m != nil { - return m.Query - } - return nil -} - -func (m *StreamExecuteKeyspaceIdsRequest) GetKeyspace() string { - if m != nil { - return m.Keyspace - } - return "" -} - -func (m *StreamExecuteKeyspaceIdsRequest) GetKeyspaceIds() [][]byte { - if m != nil { - return m.KeyspaceIds - } - return nil -} - -func (m *StreamExecuteKeyspaceIdsRequest) GetTabletType() topodata.TabletType { - if m != nil { - return m.TabletType - } - return topodata.TabletType_UNKNOWN -} - -func (m *StreamExecuteKeyspaceIdsRequest) GetOptions() *query.ExecuteOptions { - if m != nil { - return m.Options - } - return nil -} - -// StreamExecuteKeyspaceIdsResponse is the returned value from StreamExecuteKeyspaceIds. -type StreamExecuteKeyspaceIdsResponse struct { - // result contains the result data. - // The first value contains only Fields information. - // The next values contain the actual rows, a few values per result. - Result *query.QueryResult `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamExecuteKeyspaceIdsResponse) Reset() { *m = StreamExecuteKeyspaceIdsResponse{} } -func (m *StreamExecuteKeyspaceIdsResponse) String() string { return proto.CompactTextString(m) } -func (*StreamExecuteKeyspaceIdsResponse) ProtoMessage() {} -func (*StreamExecuteKeyspaceIdsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{24} -} - -func (m *StreamExecuteKeyspaceIdsResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamExecuteKeyspaceIdsResponse.Unmarshal(m, b) -} -func (m *StreamExecuteKeyspaceIdsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamExecuteKeyspaceIdsResponse.Marshal(b, m, deterministic) -} -func (m *StreamExecuteKeyspaceIdsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamExecuteKeyspaceIdsResponse.Merge(m, src) -} -func (m *StreamExecuteKeyspaceIdsResponse) XXX_Size() int { - return xxx_messageInfo_StreamExecuteKeyspaceIdsResponse.Size(m) -} -func (m *StreamExecuteKeyspaceIdsResponse) XXX_DiscardUnknown() { - xxx_messageInfo_StreamExecuteKeyspaceIdsResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamExecuteKeyspaceIdsResponse proto.InternalMessageInfo - -func (m *StreamExecuteKeyspaceIdsResponse) GetResult() *query.QueryResult { - if m != nil { - return m.Result - } - return nil -} - -// StreamExecuteKeyRangesRequest is the payload to StreamExecuteKeyRanges. -type StreamExecuteKeyRangesRequest struct { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // query is the query and bind variables to execute. - Query *query.BoundQuery `protobuf:"bytes,2,opt,name=query,proto3" json:"query,omitempty"` - // keyspace to target the query to. - Keyspace string `protobuf:"bytes,3,opt,name=keyspace,proto3" json:"keyspace,omitempty"` - // key_ranges contains the list of key ranges affected by this query. - // Will be used to find the shards to send the query to. - KeyRanges []*topodata.KeyRange `protobuf:"bytes,4,rep,name=key_ranges,json=keyRanges,proto3" json:"key_ranges,omitempty"` - // tablet_type is the type of tablets that this query is targeted to. - TabletType topodata.TabletType `protobuf:"varint,5,opt,name=tablet_type,json=tabletType,proto3,enum=topodata.TabletType" json:"tablet_type,omitempty"` - // options - Options *query.ExecuteOptions `protobuf:"bytes,6,opt,name=options,proto3" json:"options,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamExecuteKeyRangesRequest) Reset() { *m = StreamExecuteKeyRangesRequest{} } -func (m *StreamExecuteKeyRangesRequest) String() string { return proto.CompactTextString(m) } -func (*StreamExecuteKeyRangesRequest) ProtoMessage() {} -func (*StreamExecuteKeyRangesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{25} -} - -func (m *StreamExecuteKeyRangesRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamExecuteKeyRangesRequest.Unmarshal(m, b) -} -func (m *StreamExecuteKeyRangesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamExecuteKeyRangesRequest.Marshal(b, m, deterministic) -} -func (m *StreamExecuteKeyRangesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamExecuteKeyRangesRequest.Merge(m, src) -} -func (m *StreamExecuteKeyRangesRequest) XXX_Size() int { - return xxx_messageInfo_StreamExecuteKeyRangesRequest.Size(m) -} -func (m *StreamExecuteKeyRangesRequest) XXX_DiscardUnknown() { - xxx_messageInfo_StreamExecuteKeyRangesRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamExecuteKeyRangesRequest proto.InternalMessageInfo - -func (m *StreamExecuteKeyRangesRequest) GetCallerId() *vtrpc.CallerID { - if m != nil { - return m.CallerId - } - return nil -} - -func (m *StreamExecuteKeyRangesRequest) GetQuery() *query.BoundQuery { - if m != nil { - return m.Query - } - return nil -} - -func (m *StreamExecuteKeyRangesRequest) GetKeyspace() string { - if m != nil { - return m.Keyspace - } - return "" -} - -func (m *StreamExecuteKeyRangesRequest) GetKeyRanges() []*topodata.KeyRange { - if m != nil { - return m.KeyRanges - } - return nil -} - -func (m *StreamExecuteKeyRangesRequest) GetTabletType() topodata.TabletType { - if m != nil { - return m.TabletType - } - return topodata.TabletType_UNKNOWN -} - -func (m *StreamExecuteKeyRangesRequest) GetOptions() *query.ExecuteOptions { - if m != nil { - return m.Options - } - return nil -} - -// StreamExecuteKeyRangesResponse is the returned value from StreamExecuteKeyRanges. -type StreamExecuteKeyRangesResponse struct { - // result contains the result data. - // The first value contains only Fields information. - // The next values contain the actual rows, a few values per result. - Result *query.QueryResult `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamExecuteKeyRangesResponse) Reset() { *m = StreamExecuteKeyRangesResponse{} } -func (m *StreamExecuteKeyRangesResponse) String() string { return proto.CompactTextString(m) } -func (*StreamExecuteKeyRangesResponse) ProtoMessage() {} -func (*StreamExecuteKeyRangesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{26} -} - -func (m *StreamExecuteKeyRangesResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamExecuteKeyRangesResponse.Unmarshal(m, b) -} -func (m *StreamExecuteKeyRangesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamExecuteKeyRangesResponse.Marshal(b, m, deterministic) -} -func (m *StreamExecuteKeyRangesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamExecuteKeyRangesResponse.Merge(m, src) -} -func (m *StreamExecuteKeyRangesResponse) XXX_Size() int { - return xxx_messageInfo_StreamExecuteKeyRangesResponse.Size(m) -} -func (m *StreamExecuteKeyRangesResponse) XXX_DiscardUnknown() { - xxx_messageInfo_StreamExecuteKeyRangesResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamExecuteKeyRangesResponse proto.InternalMessageInfo - -func (m *StreamExecuteKeyRangesResponse) GetResult() *query.QueryResult { - if m != nil { - return m.Result - } - return nil -} - -// BeginRequest is the payload to Begin. -type BeginRequest struct { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // single_db is deprecated. Use transaction_mode instead. - // The value specifies if the transaction should be restricted - // to a single database. - // TODO(sougou): remove in 3.1 - SingleDb bool `protobuf:"varint,2,opt,name=single_db,json=singleDb,proto3" json:"single_db,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *BeginRequest) Reset() { *m = BeginRequest{} } -func (m *BeginRequest) String() string { return proto.CompactTextString(m) } -func (*BeginRequest) ProtoMessage() {} -func (*BeginRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{27} -} - -func (m *BeginRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_BeginRequest.Unmarshal(m, b) -} -func (m *BeginRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_BeginRequest.Marshal(b, m, deterministic) -} -func (m *BeginRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_BeginRequest.Merge(m, src) -} -func (m *BeginRequest) XXX_Size() int { - return xxx_messageInfo_BeginRequest.Size(m) -} -func (m *BeginRequest) XXX_DiscardUnknown() { - xxx_messageInfo_BeginRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_BeginRequest proto.InternalMessageInfo - -func (m *BeginRequest) GetCallerId() *vtrpc.CallerID { - if m != nil { - return m.CallerId - } - return nil -} - -func (m *BeginRequest) GetSingleDb() bool { - if m != nil { - return m.SingleDb - } - return false -} - -// BeginResponse is the returned value from Begin. -type BeginResponse struct { - // session is the initial session information to use for subsequent queries. - Session *Session `protobuf:"bytes,1,opt,name=session,proto3" json:"session,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *BeginResponse) Reset() { *m = BeginResponse{} } -func (m *BeginResponse) String() string { return proto.CompactTextString(m) } -func (*BeginResponse) ProtoMessage() {} -func (*BeginResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{28} -} - -func (m *BeginResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_BeginResponse.Unmarshal(m, b) -} -func (m *BeginResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_BeginResponse.Marshal(b, m, deterministic) -} -func (m *BeginResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_BeginResponse.Merge(m, src) -} -func (m *BeginResponse) XXX_Size() int { - return xxx_messageInfo_BeginResponse.Size(m) -} -func (m *BeginResponse) XXX_DiscardUnknown() { - xxx_messageInfo_BeginResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_BeginResponse proto.InternalMessageInfo - -func (m *BeginResponse) GetSession() *Session { - if m != nil { - return m.Session - } - return nil -} - -// CommitRequest is the payload to Commit. -type CommitRequest struct { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // session carries the current transaction data to commit. - Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` - // atomic is deprecated. Use transaction_mode instead. - // The value specifies if the commit should go through the - // 2PC workflow to ensure atomicity. - // TODO(sougou): remove in 3.1 - Atomic bool `protobuf:"varint,3,opt,name=atomic,proto3" json:"atomic,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *CommitRequest) Reset() { *m = CommitRequest{} } -func (m *CommitRequest) String() string { return proto.CompactTextString(m) } -func (*CommitRequest) ProtoMessage() {} -func (*CommitRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{29} -} - -func (m *CommitRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_CommitRequest.Unmarshal(m, b) -} -func (m *CommitRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_CommitRequest.Marshal(b, m, deterministic) -} -func (m *CommitRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CommitRequest.Merge(m, src) -} -func (m *CommitRequest) XXX_Size() int { - return xxx_messageInfo_CommitRequest.Size(m) -} -func (m *CommitRequest) XXX_DiscardUnknown() { - xxx_messageInfo_CommitRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_CommitRequest proto.InternalMessageInfo - -func (m *CommitRequest) GetCallerId() *vtrpc.CallerID { - if m != nil { - return m.CallerId - } - return nil -} - -func (m *CommitRequest) GetSession() *Session { - if m != nil { - return m.Session - } - return nil -} - -func (m *CommitRequest) GetAtomic() bool { - if m != nil { - return m.Atomic - } - return false -} - -// CommitResponse is the returned value from Commit. -type CommitResponse struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *CommitResponse) Reset() { *m = CommitResponse{} } -func (m *CommitResponse) String() string { return proto.CompactTextString(m) } -func (*CommitResponse) ProtoMessage() {} -func (*CommitResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{30} -} - -func (m *CommitResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_CommitResponse.Unmarshal(m, b) -} -func (m *CommitResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_CommitResponse.Marshal(b, m, deterministic) -} -func (m *CommitResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CommitResponse.Merge(m, src) -} -func (m *CommitResponse) XXX_Size() int { - return xxx_messageInfo_CommitResponse.Size(m) -} -func (m *CommitResponse) XXX_DiscardUnknown() { - xxx_messageInfo_CommitResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_CommitResponse proto.InternalMessageInfo - -// RollbackRequest is the payload to Rollback. -type RollbackRequest struct { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // session carries the current transaction data to rollback. - Session *Session `protobuf:"bytes,2,opt,name=session,proto3" json:"session,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *RollbackRequest) Reset() { *m = RollbackRequest{} } -func (m *RollbackRequest) String() string { return proto.CompactTextString(m) } -func (*RollbackRequest) ProtoMessage() {} -func (*RollbackRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{31} -} - -func (m *RollbackRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RollbackRequest.Unmarshal(m, b) -} -func (m *RollbackRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RollbackRequest.Marshal(b, m, deterministic) -} -func (m *RollbackRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_RollbackRequest.Merge(m, src) -} -func (m *RollbackRequest) XXX_Size() int { - return xxx_messageInfo_RollbackRequest.Size(m) -} -func (m *RollbackRequest) XXX_DiscardUnknown() { - xxx_messageInfo_RollbackRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_RollbackRequest proto.InternalMessageInfo - -func (m *RollbackRequest) GetCallerId() *vtrpc.CallerID { - if m != nil { - return m.CallerId - } - return nil -} - -func (m *RollbackRequest) GetSession() *Session { - if m != nil { - return m.Session - } - return nil -} - -// RollbackResponse is the returned value from Rollback. -type RollbackResponse struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *RollbackResponse) Reset() { *m = RollbackResponse{} } -func (m *RollbackResponse) String() string { return proto.CompactTextString(m) } -func (*RollbackResponse) ProtoMessage() {} -func (*RollbackResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{32} -} - -func (m *RollbackResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RollbackResponse.Unmarshal(m, b) -} -func (m *RollbackResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RollbackResponse.Marshal(b, m, deterministic) -} -func (m *RollbackResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_RollbackResponse.Merge(m, src) -} -func (m *RollbackResponse) XXX_Size() int { - return xxx_messageInfo_RollbackResponse.Size(m) -} -func (m *RollbackResponse) XXX_DiscardUnknown() { - xxx_messageInfo_RollbackResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_RollbackResponse proto.InternalMessageInfo - -// ResolveTransactionRequest is the payload to ResolveTransaction. -type ResolveTransactionRequest struct { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // dtid is the dtid of the transaction to be resolved. - Dtid string `protobuf:"bytes,2,opt,name=dtid,proto3" json:"dtid,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ResolveTransactionRequest) Reset() { *m = ResolveTransactionRequest{} } -func (m *ResolveTransactionRequest) String() string { return proto.CompactTextString(m) } -func (*ResolveTransactionRequest) ProtoMessage() {} -func (*ResolveTransactionRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{33} -} - -func (m *ResolveTransactionRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ResolveTransactionRequest.Unmarshal(m, b) -} -func (m *ResolveTransactionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ResolveTransactionRequest.Marshal(b, m, deterministic) -} -func (m *ResolveTransactionRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResolveTransactionRequest.Merge(m, src) -} -func (m *ResolveTransactionRequest) XXX_Size() int { - return xxx_messageInfo_ResolveTransactionRequest.Size(m) -} -func (m *ResolveTransactionRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ResolveTransactionRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_ResolveTransactionRequest proto.InternalMessageInfo - -func (m *ResolveTransactionRequest) GetCallerId() *vtrpc.CallerID { - if m != nil { - return m.CallerId - } - return nil -} - -func (m *ResolveTransactionRequest) GetDtid() string { - if m != nil { - return m.Dtid - } - return "" -} - -// MessageStreamRequest is the request payload for MessageStream. -type MessageStreamRequest struct { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // keyspace to target the query to. - Keyspace string `protobuf:"bytes,2,opt,name=keyspace,proto3" json:"keyspace,omitempty"` - // shard to target the query to, for unsharded keyspaces. - Shard string `protobuf:"bytes,3,opt,name=shard,proto3" json:"shard,omitempty"` - // KeyRange to target the query to, for sharded keyspaces. - KeyRange *topodata.KeyRange `protobuf:"bytes,4,opt,name=key_range,json=keyRange,proto3" json:"key_range,omitempty"` - // name is the message table name. - Name string `protobuf:"bytes,5,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *MessageStreamRequest) Reset() { *m = MessageStreamRequest{} } -func (m *MessageStreamRequest) String() string { return proto.CompactTextString(m) } -func (*MessageStreamRequest) ProtoMessage() {} -func (*MessageStreamRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{34} -} - -func (m *MessageStreamRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_MessageStreamRequest.Unmarshal(m, b) -} -func (m *MessageStreamRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_MessageStreamRequest.Marshal(b, m, deterministic) -} -func (m *MessageStreamRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_MessageStreamRequest.Merge(m, src) -} -func (m *MessageStreamRequest) XXX_Size() int { - return xxx_messageInfo_MessageStreamRequest.Size(m) -} -func (m *MessageStreamRequest) XXX_DiscardUnknown() { - xxx_messageInfo_MessageStreamRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_MessageStreamRequest proto.InternalMessageInfo - -func (m *MessageStreamRequest) GetCallerId() *vtrpc.CallerID { - if m != nil { - return m.CallerId - } - return nil -} - -func (m *MessageStreamRequest) GetKeyspace() string { - if m != nil { - return m.Keyspace - } - return "" -} - -func (m *MessageStreamRequest) GetShard() string { - if m != nil { - return m.Shard - } - return "" -} - -func (m *MessageStreamRequest) GetKeyRange() *topodata.KeyRange { - if m != nil { - return m.KeyRange - } - return nil -} - -func (m *MessageStreamRequest) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -// MessageAckRequest is the request payload for MessageAck. -type MessageAckRequest struct { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // keyspace to target the message to. - Keyspace string `protobuf:"bytes,2,opt,name=keyspace,proto3" json:"keyspace,omitempty"` - // name is the message table name. - Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` - // ids is the list of ids to ack. - Ids []*query.Value `protobuf:"bytes,4,rep,name=ids,proto3" json:"ids,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *MessageAckRequest) Reset() { *m = MessageAckRequest{} } -func (m *MessageAckRequest) String() string { return proto.CompactTextString(m) } -func (*MessageAckRequest) ProtoMessage() {} -func (*MessageAckRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{35} -} - -func (m *MessageAckRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_MessageAckRequest.Unmarshal(m, b) -} -func (m *MessageAckRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_MessageAckRequest.Marshal(b, m, deterministic) -} -func (m *MessageAckRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_MessageAckRequest.Merge(m, src) -} -func (m *MessageAckRequest) XXX_Size() int { - return xxx_messageInfo_MessageAckRequest.Size(m) -} -func (m *MessageAckRequest) XXX_DiscardUnknown() { - xxx_messageInfo_MessageAckRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_MessageAckRequest proto.InternalMessageInfo - -func (m *MessageAckRequest) GetCallerId() *vtrpc.CallerID { - if m != nil { - return m.CallerId - } - return nil -} - -func (m *MessageAckRequest) GetKeyspace() string { - if m != nil { - return m.Keyspace - } - return "" -} - -func (m *MessageAckRequest) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -func (m *MessageAckRequest) GetIds() []*query.Value { - if m != nil { - return m.Ids - } - return nil -} - -// IdKeyspaceId represents an id and keyspace_id pair. -// The kesypace_id represents the routing info for id. -type IdKeyspaceId struct { - // id represents the message id. - Id *query.Value `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - // keyspace_id is the associated keyspace_id for the id. - KeyspaceId []byte `protobuf:"bytes,2,opt,name=keyspace_id,json=keyspaceId,proto3" json:"keyspace_id,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *IdKeyspaceId) Reset() { *m = IdKeyspaceId{} } -func (m *IdKeyspaceId) String() string { return proto.CompactTextString(m) } -func (*IdKeyspaceId) ProtoMessage() {} -func (*IdKeyspaceId) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{36} -} - -func (m *IdKeyspaceId) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_IdKeyspaceId.Unmarshal(m, b) -} -func (m *IdKeyspaceId) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_IdKeyspaceId.Marshal(b, m, deterministic) -} -func (m *IdKeyspaceId) XXX_Merge(src proto.Message) { - xxx_messageInfo_IdKeyspaceId.Merge(m, src) -} -func (m *IdKeyspaceId) XXX_Size() int { - return xxx_messageInfo_IdKeyspaceId.Size(m) -} -func (m *IdKeyspaceId) XXX_DiscardUnknown() { - xxx_messageInfo_IdKeyspaceId.DiscardUnknown(m) -} - -var xxx_messageInfo_IdKeyspaceId proto.InternalMessageInfo - -func (m *IdKeyspaceId) GetId() *query.Value { - if m != nil { - return m.Id - } - return nil -} - -func (m *IdKeyspaceId) GetKeyspaceId() []byte { - if m != nil { - return m.KeyspaceId - } - return nil -} - -// MessageAckKeyspaceIdsRequest is the payload to MessageAckKeyspaceIds. -type MessageAckKeyspaceIdsRequest struct { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` - // Optional keyspace for message table. - Keyspace string `protobuf:"bytes,2,opt,name=keyspace,proto3" json:"keyspace,omitempty"` - // name is the message table name. - Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` - IdKeyspaceIds []*IdKeyspaceId `protobuf:"bytes,4,rep,name=id_keyspace_ids,json=idKeyspaceIds,proto3" json:"id_keyspace_ids,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *MessageAckKeyspaceIdsRequest) Reset() { *m = MessageAckKeyspaceIdsRequest{} } -func (m *MessageAckKeyspaceIdsRequest) String() string { return proto.CompactTextString(m) } -func (*MessageAckKeyspaceIdsRequest) ProtoMessage() {} -func (*MessageAckKeyspaceIdsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{37} -} - -func (m *MessageAckKeyspaceIdsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_MessageAckKeyspaceIdsRequest.Unmarshal(m, b) -} -func (m *MessageAckKeyspaceIdsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_MessageAckKeyspaceIdsRequest.Marshal(b, m, deterministic) -} -func (m *MessageAckKeyspaceIdsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_MessageAckKeyspaceIdsRequest.Merge(m, src) -} -func (m *MessageAckKeyspaceIdsRequest) XXX_Size() int { - return xxx_messageInfo_MessageAckKeyspaceIdsRequest.Size(m) -} -func (m *MessageAckKeyspaceIdsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_MessageAckKeyspaceIdsRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_MessageAckKeyspaceIdsRequest proto.InternalMessageInfo - -func (m *MessageAckKeyspaceIdsRequest) GetCallerId() *vtrpc.CallerID { - if m != nil { - return m.CallerId - } - return nil -} - -func (m *MessageAckKeyspaceIdsRequest) GetKeyspace() string { - if m != nil { - return m.Keyspace - } - return "" -} - -func (m *MessageAckKeyspaceIdsRequest) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -func (m *MessageAckKeyspaceIdsRequest) GetIdKeyspaceIds() []*IdKeyspaceId { - if m != nil { - return m.IdKeyspaceIds - } - return nil -} - -// ResolveTransactionResponse is the returned value from Rollback. -type ResolveTransactionResponse struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ResolveTransactionResponse) Reset() { *m = ResolveTransactionResponse{} } -func (m *ResolveTransactionResponse) String() string { return proto.CompactTextString(m) } -func (*ResolveTransactionResponse) ProtoMessage() {} -func (*ResolveTransactionResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{38} -} - -func (m *ResolveTransactionResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ResolveTransactionResponse.Unmarshal(m, b) -} -func (m *ResolveTransactionResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ResolveTransactionResponse.Marshal(b, m, deterministic) +func (m *ResolveTransactionResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ResolveTransactionResponse.Marshal(b, m, deterministic) } func (m *ResolveTransactionResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_ResolveTransactionResponse.Merge(m, src) @@ -2927,88 +826,6 @@ func (m *ResolveTransactionResponse) XXX_DiscardUnknown() { var xxx_messageInfo_ResolveTransactionResponse proto.InternalMessageInfo -// GetSrvKeyspaceRequest is the payload to GetSrvKeyspace. -type GetSrvKeyspaceRequest struct { - // keyspace name to fetch. - Keyspace string `protobuf:"bytes,1,opt,name=keyspace,proto3" json:"keyspace,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *GetSrvKeyspaceRequest) Reset() { *m = GetSrvKeyspaceRequest{} } -func (m *GetSrvKeyspaceRequest) String() string { return proto.CompactTextString(m) } -func (*GetSrvKeyspaceRequest) ProtoMessage() {} -func (*GetSrvKeyspaceRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{39} -} - -func (m *GetSrvKeyspaceRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetSrvKeyspaceRequest.Unmarshal(m, b) -} -func (m *GetSrvKeyspaceRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetSrvKeyspaceRequest.Marshal(b, m, deterministic) -} -func (m *GetSrvKeyspaceRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetSrvKeyspaceRequest.Merge(m, src) -} -func (m *GetSrvKeyspaceRequest) XXX_Size() int { - return xxx_messageInfo_GetSrvKeyspaceRequest.Size(m) -} -func (m *GetSrvKeyspaceRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetSrvKeyspaceRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_GetSrvKeyspaceRequest proto.InternalMessageInfo - -func (m *GetSrvKeyspaceRequest) GetKeyspace() string { - if m != nil { - return m.Keyspace - } - return "" -} - -// GetSrvKeyspaceResponse is the returned value from GetSrvKeyspace. -type GetSrvKeyspaceResponse struct { - // srv_keyspace is the topology object for the SrvKeyspace. - SrvKeyspace *topodata.SrvKeyspace `protobuf:"bytes,1,opt,name=srv_keyspace,json=srvKeyspace,proto3" json:"srv_keyspace,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *GetSrvKeyspaceResponse) Reset() { *m = GetSrvKeyspaceResponse{} } -func (m *GetSrvKeyspaceResponse) String() string { return proto.CompactTextString(m) } -func (*GetSrvKeyspaceResponse) ProtoMessage() {} -func (*GetSrvKeyspaceResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{40} -} - -func (m *GetSrvKeyspaceResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetSrvKeyspaceResponse.Unmarshal(m, b) -} -func (m *GetSrvKeyspaceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetSrvKeyspaceResponse.Marshal(b, m, deterministic) -} -func (m *GetSrvKeyspaceResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetSrvKeyspaceResponse.Merge(m, src) -} -func (m *GetSrvKeyspaceResponse) XXX_Size() int { - return xxx_messageInfo_GetSrvKeyspaceResponse.Size(m) -} -func (m *GetSrvKeyspaceResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetSrvKeyspaceResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_GetSrvKeyspaceResponse proto.InternalMessageInfo - -func (m *GetSrvKeyspaceResponse) GetSrvKeyspace() *topodata.SrvKeyspace { - if m != nil { - return m.SrvKeyspace - } - return nil -} - // VStreamRequest is the payload for VStream. type VStreamRequest struct { CallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` @@ -3027,7 +844,7 @@ func (m *VStreamRequest) Reset() { *m = VStreamRequest{} } func (m *VStreamRequest) String() string { return proto.CompactTextString(m) } func (*VStreamRequest) ProtoMessage() {} func (*VStreamRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{41} + return fileDescriptor_aab96496ceaf1ebb, []int{9} } func (m *VStreamRequest) XXX_Unmarshal(b []byte) error { @@ -3088,7 +905,7 @@ func (m *VStreamResponse) Reset() { *m = VStreamResponse{} } func (m *VStreamResponse) String() string { return proto.CompactTextString(m) } func (*VStreamResponse) ProtoMessage() {} func (*VStreamResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_aab96496ceaf1ebb, []int{42} + return fileDescriptor_aab96496ceaf1ebb, []int{10} } func (m *VStreamResponse) XXX_Unmarshal(b []byte) error { @@ -3124,45 +941,12 @@ func init() { proto.RegisterType((*Session_ShardSession)(nil), "vtgate.Session.ShardSession") proto.RegisterType((*ExecuteRequest)(nil), "vtgate.ExecuteRequest") proto.RegisterType((*ExecuteResponse)(nil), "vtgate.ExecuteResponse") - proto.RegisterType((*ExecuteShardsRequest)(nil), "vtgate.ExecuteShardsRequest") - proto.RegisterType((*ExecuteShardsResponse)(nil), "vtgate.ExecuteShardsResponse") - proto.RegisterType((*ExecuteKeyspaceIdsRequest)(nil), "vtgate.ExecuteKeyspaceIdsRequest") - proto.RegisterType((*ExecuteKeyspaceIdsResponse)(nil), "vtgate.ExecuteKeyspaceIdsResponse") - proto.RegisterType((*ExecuteKeyRangesRequest)(nil), "vtgate.ExecuteKeyRangesRequest") - proto.RegisterType((*ExecuteKeyRangesResponse)(nil), "vtgate.ExecuteKeyRangesResponse") - proto.RegisterType((*ExecuteEntityIdsRequest)(nil), "vtgate.ExecuteEntityIdsRequest") - proto.RegisterType((*ExecuteEntityIdsRequest_EntityId)(nil), "vtgate.ExecuteEntityIdsRequest.EntityId") - proto.RegisterType((*ExecuteEntityIdsResponse)(nil), "vtgate.ExecuteEntityIdsResponse") proto.RegisterType((*ExecuteBatchRequest)(nil), "vtgate.ExecuteBatchRequest") proto.RegisterType((*ExecuteBatchResponse)(nil), "vtgate.ExecuteBatchResponse") - proto.RegisterType((*BoundShardQuery)(nil), "vtgate.BoundShardQuery") - proto.RegisterType((*ExecuteBatchShardsRequest)(nil), "vtgate.ExecuteBatchShardsRequest") - proto.RegisterType((*ExecuteBatchShardsResponse)(nil), "vtgate.ExecuteBatchShardsResponse") - proto.RegisterType((*BoundKeyspaceIdQuery)(nil), "vtgate.BoundKeyspaceIdQuery") - proto.RegisterType((*ExecuteBatchKeyspaceIdsRequest)(nil), "vtgate.ExecuteBatchKeyspaceIdsRequest") - proto.RegisterType((*ExecuteBatchKeyspaceIdsResponse)(nil), "vtgate.ExecuteBatchKeyspaceIdsResponse") proto.RegisterType((*StreamExecuteRequest)(nil), "vtgate.StreamExecuteRequest") proto.RegisterType((*StreamExecuteResponse)(nil), "vtgate.StreamExecuteResponse") - proto.RegisterType((*StreamExecuteShardsRequest)(nil), "vtgate.StreamExecuteShardsRequest") - proto.RegisterType((*StreamExecuteShardsResponse)(nil), "vtgate.StreamExecuteShardsResponse") - proto.RegisterType((*StreamExecuteKeyspaceIdsRequest)(nil), "vtgate.StreamExecuteKeyspaceIdsRequest") - proto.RegisterType((*StreamExecuteKeyspaceIdsResponse)(nil), "vtgate.StreamExecuteKeyspaceIdsResponse") - proto.RegisterType((*StreamExecuteKeyRangesRequest)(nil), "vtgate.StreamExecuteKeyRangesRequest") - proto.RegisterType((*StreamExecuteKeyRangesResponse)(nil), "vtgate.StreamExecuteKeyRangesResponse") - proto.RegisterType((*BeginRequest)(nil), "vtgate.BeginRequest") - proto.RegisterType((*BeginResponse)(nil), "vtgate.BeginResponse") - proto.RegisterType((*CommitRequest)(nil), "vtgate.CommitRequest") - proto.RegisterType((*CommitResponse)(nil), "vtgate.CommitResponse") - proto.RegisterType((*RollbackRequest)(nil), "vtgate.RollbackRequest") - proto.RegisterType((*RollbackResponse)(nil), "vtgate.RollbackResponse") proto.RegisterType((*ResolveTransactionRequest)(nil), "vtgate.ResolveTransactionRequest") - proto.RegisterType((*MessageStreamRequest)(nil), "vtgate.MessageStreamRequest") - proto.RegisterType((*MessageAckRequest)(nil), "vtgate.MessageAckRequest") - proto.RegisterType((*IdKeyspaceId)(nil), "vtgate.IdKeyspaceId") - proto.RegisterType((*MessageAckKeyspaceIdsRequest)(nil), "vtgate.MessageAckKeyspaceIdsRequest") proto.RegisterType((*ResolveTransactionResponse)(nil), "vtgate.ResolveTransactionResponse") - proto.RegisterType((*GetSrvKeyspaceRequest)(nil), "vtgate.GetSrvKeyspaceRequest") - proto.RegisterType((*GetSrvKeyspaceResponse)(nil), "vtgate.GetSrvKeyspaceResponse") proto.RegisterType((*VStreamRequest)(nil), "vtgate.VStreamRequest") proto.RegisterType((*VStreamResponse)(nil), "vtgate.VStreamResponse") } @@ -3170,120 +954,72 @@ func init() { func init() { proto.RegisterFile("vtgate.proto", fileDescriptor_aab96496ceaf1ebb) } var fileDescriptor_aab96496ceaf1ebb = []byte{ - // 1835 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x5a, 0xdd, 0x6e, 0x1b, 0xc7, - 0x15, 0xce, 0xee, 0xf2, 0xf7, 0x2c, 0xff, 0x3c, 0xa6, 0x15, 0x9a, 0x51, 0x6c, 0x76, 0x53, 0x21, - 0xb4, 0x6b, 0x48, 0x8d, 0x82, 0x06, 0x41, 0x90, 0xa2, 0xb0, 0x28, 0xc5, 0x20, 0x62, 0xfd, 0x74, - 0x44, 0xdb, 0x6d, 0x91, 0x62, 0xb1, 0xe2, 0x8e, 0xe9, 0xad, 0xa8, 0x5d, 0x66, 0x67, 0x48, 0x57, - 0x37, 0x45, 0xde, 0x20, 0xe8, 0x45, 0x81, 0xc2, 0x28, 0x50, 0xf4, 0xaa, 0x57, 0xbd, 0x2d, 0xd0, - 0xf6, 0xa6, 0xd7, 0xbd, 0x29, 0xfa, 0x0a, 0xed, 0x1b, 0xf4, 0x09, 0x82, 0x9d, 0x99, 0xfd, 0xe1, - 0x5a, 0xb4, 0x28, 0xca, 0x32, 0xe8, 0x1b, 0x61, 0x67, 0xce, 0xec, 0xcc, 0x37, 0xdf, 0xf9, 0xce, - 0x99, 0xc3, 0x59, 0x41, 0x69, 0xc2, 0x06, 0x16, 0x23, 0xeb, 0x23, 0xdf, 0x63, 0x1e, 0xca, 0x89, - 0x56, 0xb3, 0x76, 0xe4, 0xb8, 0x43, 0x6f, 0x60, 0x5b, 0xcc, 0x12, 0x96, 0xa6, 0xfe, 0xf5, 0x98, - 0xf8, 0xa7, 0xb2, 0x51, 0x61, 0xde, 0xc8, 0x4b, 0x1a, 0x27, 0xcc, 0x1f, 0xf5, 0x45, 0xc3, 0xf8, - 0x5f, 0x0e, 0xf2, 0x87, 0x84, 0x52, 0xc7, 0x73, 0xd1, 0x1a, 0x54, 0x1c, 0xd7, 0x64, 0xbe, 0xe5, - 0x52, 0xab, 0xcf, 0x1c, 0xcf, 0x6d, 0x28, 0x2d, 0xa5, 0x5d, 0xc0, 0x65, 0xc7, 0xed, 0xc5, 0x9d, - 0xa8, 0x03, 0x15, 0xfa, 0xcc, 0xf2, 0x6d, 0x93, 0x8a, 0xf7, 0x68, 0x43, 0x6d, 0x69, 0x6d, 0x7d, - 0x73, 0x75, 0x5d, 0xa2, 0x93, 0xf3, 0xad, 0x1f, 0x06, 0xa3, 0x64, 0x03, 0x97, 0x69, 0xa2, 0x45, - 0xd1, 0x7b, 0x50, 0xa4, 0x8e, 0x3b, 0x18, 0x12, 0xd3, 0x3e, 0x6a, 0x68, 0x7c, 0x99, 0x82, 0xe8, - 0xd8, 0x3e, 0x42, 0xb7, 0x00, 0xac, 0x31, 0xf3, 0xfa, 0xde, 0xc9, 0x89, 0xc3, 0x1a, 0x19, 0x6e, - 0x4d, 0xf4, 0xa0, 0x0f, 0xa0, 0xcc, 0x2c, 0x7f, 0x40, 0x98, 0x49, 0x99, 0xef, 0xb8, 0x83, 0x46, - 0xb6, 0xa5, 0xb4, 0x8b, 0xb8, 0x24, 0x3a, 0x0f, 0x79, 0x1f, 0xda, 0x80, 0xbc, 0x37, 0x62, 0x1c, - 0x5f, 0xae, 0xa5, 0xb4, 0xf5, 0xcd, 0x1b, 0xeb, 0x82, 0x95, 0x9d, 0x5f, 0x93, 0xfe, 0x98, 0x91, - 0x7d, 0x61, 0xc4, 0xe1, 0x28, 0xb4, 0x05, 0xb5, 0xc4, 0xde, 0xcd, 0x13, 0xcf, 0x26, 0x8d, 0x7c, - 0x4b, 0x69, 0x57, 0x36, 0xdf, 0x0d, 0x77, 0x96, 0xa0, 0x61, 0xd7, 0xb3, 0x09, 0xae, 0xb2, 0xe9, - 0x0e, 0xb4, 0x01, 0x85, 0xe7, 0x96, 0xef, 0x3a, 0xee, 0x80, 0x36, 0x0a, 0x9c, 0x95, 0xeb, 0x72, - 0xd5, 0x9f, 0x06, 0x7f, 0x9f, 0x08, 0x1b, 0x8e, 0x06, 0xa1, 0x9f, 0x40, 0x69, 0xe4, 0x93, 0x98, - 0xca, 0xe2, 0x1c, 0x54, 0xea, 0x23, 0x9f, 0x44, 0x44, 0xde, 0x87, 0xf2, 0xc8, 0xa3, 0x2c, 0x9e, - 0x01, 0xe6, 0x98, 0xa1, 0x14, 0xbc, 0x12, 0x4d, 0xf1, 0x7d, 0xa8, 0x0c, 0x2d, 0xca, 0x4c, 0xc7, - 0xa5, 0xc4, 0x67, 0xa6, 0x63, 0x37, 0xf4, 0x96, 0xd2, 0xce, 0xe0, 0x52, 0xd0, 0xdb, 0xe5, 0x9d, - 0x5d, 0x1b, 0xbd, 0x0f, 0xf0, 0xd4, 0x1b, 0xbb, 0xb6, 0xe9, 0x7b, 0xcf, 0x69, 0xa3, 0xc4, 0x47, - 0x14, 0x79, 0x0f, 0xf6, 0x9e, 0x53, 0x64, 0xc2, 0xca, 0x98, 0x12, 0xdf, 0xb4, 0xc9, 0x53, 0xc7, - 0x25, 0xb6, 0x39, 0xb1, 0x7c, 0xc7, 0x3a, 0x1a, 0x12, 0xda, 0x28, 0x73, 0x40, 0x77, 0xd2, 0x80, - 0x1e, 0x51, 0xe2, 0x6f, 0x8b, 0xc1, 0x8f, 0xc3, 0xb1, 0x3b, 0x2e, 0xf3, 0x4f, 0x71, 0x7d, 0x7c, - 0x86, 0xa9, 0xf9, 0x15, 0x94, 0x92, 0x7b, 0x40, 0x6b, 0x90, 0x13, 0xfe, 0xe6, 0x2a, 0xd5, 0x37, - 0xcb, 0x92, 0xe8, 0x1e, 0xef, 0xc4, 0xd2, 0x18, 0x88, 0x3a, 0xe9, 0x55, 0xc7, 0x6e, 0xa8, 0x2d, - 0xa5, 0xad, 0xe1, 0x72, 0xa2, 0xb7, 0x6b, 0x37, 0xbf, 0x82, 0x9b, 0x33, 0x01, 0xa1, 0x1a, 0x68, - 0xc7, 0xe4, 0x94, 0xaf, 0x53, 0xc4, 0xc1, 0x23, 0xba, 0x03, 0xd9, 0x89, 0x35, 0x1c, 0x13, 0x3e, - 0x59, 0xec, 0xe4, 0x2d, 0xc7, 0x8d, 0xde, 0xc5, 0x62, 0xc4, 0x67, 0xea, 0xa7, 0x8a, 0xf1, 0x6f, - 0x15, 0x2a, 0x52, 0x76, 0x98, 0x7c, 0x3d, 0x26, 0x94, 0xa1, 0x7b, 0x50, 0xec, 0x5b, 0xc3, 0x21, - 0xf1, 0x03, 0x48, 0x62, 0x07, 0xd5, 0x75, 0x11, 0x99, 0x1d, 0xde, 0xdf, 0xdd, 0xc6, 0x05, 0x31, - 0xa2, 0x6b, 0xa3, 0x3b, 0x90, 0x97, 0x0e, 0x96, 0x2b, 0x56, 0x53, 0x74, 0xe2, 0xd0, 0x8e, 0x3e, - 0x84, 0x2c, 0x07, 0xc3, 0xa3, 0x4a, 0xdf, 0xbc, 0x16, 0x42, 0x0b, 0x3c, 0xc5, 0x45, 0x88, 0x85, - 0x1d, 0xfd, 0x08, 0x74, 0x16, 0x00, 0x65, 0x26, 0x3b, 0x1d, 0x11, 0x1e, 0x66, 0x95, 0xcd, 0xfa, - 0x7a, 0x94, 0x2d, 0x7a, 0xdc, 0xd8, 0x3b, 0x1d, 0x11, 0x0c, 0x2c, 0x7a, 0x46, 0xf7, 0x00, 0xb9, - 0x5e, 0x20, 0x96, 0xa9, 0x4c, 0x91, 0xe5, 0x41, 0x5a, 0x73, 0x3d, 0xd6, 0x9d, 0x4a, 0x16, 0x6b, - 0x50, 0x39, 0x26, 0xa7, 0x74, 0x64, 0xf5, 0x89, 0xc9, 0x33, 0x00, 0x0f, 0xc6, 0x22, 0x2e, 0x87, - 0xbd, 0xdc, 0xa7, 0xc9, 0x60, 0xcd, 0xcf, 0x13, 0xac, 0xc6, 0xb7, 0x0a, 0x54, 0x23, 0x46, 0xe9, - 0xc8, 0x73, 0x29, 0x41, 0x6b, 0x90, 0x25, 0xbe, 0xef, 0xf9, 0x29, 0x3a, 0xf1, 0x41, 0x67, 0x27, - 0xe8, 0xc6, 0xc2, 0x7a, 0x11, 0x2e, 0xef, 0x42, 0xce, 0x27, 0x74, 0x3c, 0x64, 0x92, 0x4c, 0x94, - 0x0c, 0x66, 0xcc, 0x2d, 0x58, 0x8e, 0x30, 0xfe, 0xab, 0x42, 0x5d, 0x22, 0xe2, 0x7b, 0xa2, 0xcb, - 0xe3, 0xe9, 0x26, 0x14, 0x42, 0xba, 0xb9, 0x9b, 0x8b, 0x38, 0x6a, 0xa3, 0x15, 0xc8, 0x71, 0xbf, - 0xd0, 0x46, 0xb6, 0xa5, 0xb5, 0x8b, 0x58, 0xb6, 0xd2, 0xea, 0xc8, 0x5d, 0x4a, 0x1d, 0xf9, 0x19, - 0xea, 0x48, 0xb8, 0xbd, 0x30, 0x97, 0xdb, 0x7f, 0xa7, 0xc0, 0x8d, 0x14, 0xc9, 0x4b, 0xe1, 0xfc, - 0xff, 0xab, 0x70, 0x53, 0xe2, 0xfa, 0x52, 0x32, 0xdb, 0x7d, 0x5b, 0x14, 0xf0, 0x3d, 0x28, 0x45, - 0x21, 0xea, 0x48, 0x1d, 0x94, 0xb0, 0x7e, 0x1c, 0xef, 0x63, 0x49, 0xc5, 0xf0, 0x42, 0x81, 0xe6, - 0x59, 0xa4, 0x2f, 0x85, 0x22, 0xbe, 0xd1, 0xe0, 0xdd, 0x18, 0x1c, 0xb6, 0xdc, 0x01, 0x79, 0x4b, - 0xf4, 0xf0, 0x11, 0xc0, 0x31, 0x39, 0x35, 0x7d, 0x0e, 0x99, 0xab, 0x21, 0xd8, 0x69, 0xe4, 0xeb, - 0x70, 0x37, 0xb8, 0x78, 0x1c, 0xee, 0x6b, 0x49, 0xf5, 0xf1, 0x7b, 0x05, 0x1a, 0x2f, 0xbb, 0x60, - 0x29, 0xd4, 0xf1, 0xb7, 0x4c, 0xa4, 0x8e, 0x1d, 0x97, 0x39, 0xec, 0xf4, 0xad, 0xc9, 0x16, 0xf7, - 0x00, 0x11, 0x8e, 0xd8, 0xec, 0x7b, 0xc3, 0xf1, 0x89, 0x6b, 0xba, 0xd6, 0x09, 0x91, 0x05, 0x78, - 0x4d, 0x58, 0x3a, 0xdc, 0xb0, 0x67, 0x9d, 0x10, 0xf4, 0x33, 0xb8, 0x2e, 0x47, 0x4f, 0xa5, 0x98, - 0x1c, 0x17, 0x55, 0x3b, 0x44, 0x3a, 0x83, 0x89, 0xf5, 0xb0, 0x03, 0x5f, 0x13, 0x93, 0x7c, 0x39, - 0x3b, 0x25, 0xe5, 0x2f, 0x25, 0xb9, 0xc2, 0xf9, 0x92, 0x2b, 0xce, 0x23, 0xb9, 0xe6, 0x11, 0x14, - 0x42, 0xd0, 0xe8, 0x36, 0x64, 0x38, 0x34, 0x85, 0x43, 0xd3, 0xc3, 0xf2, 0x34, 0x40, 0xc4, 0x0d, - 0xa8, 0x9e, 0x2c, 0x22, 0x4b, 0xb2, 0x5e, 0x44, 0xb7, 0x41, 0x4f, 0x70, 0xc5, 0x7d, 0x55, 0xc2, - 0x10, 0x67, 0xe3, 0xa4, 0xac, 0x13, 0x8c, 0x2d, 0x85, 0xac, 0xff, 0xa3, 0xc2, 0x75, 0x09, 0x6d, - 0xcb, 0x62, 0xfd, 0x67, 0x57, 0x2e, 0xe9, 0x1f, 0x40, 0x3e, 0x40, 0xe3, 0x10, 0xda, 0xd0, 0xb8, - 0xa6, 0xce, 0x10, 0x75, 0x38, 0x62, 0xd1, 0x82, 0x77, 0x0d, 0x2a, 0x16, 0x3d, 0xa3, 0xd8, 0x2d, - 0x5b, 0xf4, 0x4d, 0x54, 0xba, 0x2f, 0x94, 0xa8, 0xae, 0x94, 0x9c, 0x5e, 0x99, 0xab, 0x7f, 0x08, - 0x79, 0xe1, 0xc8, 0x90, 0xcd, 0x15, 0x89, 0x4d, 0xb8, 0xf9, 0x89, 0xc3, 0x9e, 0x89, 0xa9, 0xc3, - 0x61, 0x86, 0x0b, 0x55, 0xce, 0x34, 0xdf, 0x1b, 0xa7, 0x3b, 0xce, 0x32, 0xca, 0x05, 0xb2, 0x8c, - 0x3a, 0xb3, 0x2a, 0xd5, 0x92, 0x55, 0xa9, 0xf1, 0xd7, 0xb8, 0xce, 0xe2, 0x64, 0xbc, 0xa1, 0x4a, - 0xfb, 0xa3, 0xb4, 0xcc, 0xa2, 0x1b, 0x81, 0xd4, 0xee, 0xdf, 0x94, 0xd8, 0x2e, 0x7a, 0xb9, 0x61, - 0xfc, 0x21, 0xae, 0x95, 0xa6, 0x88, 0xbb, 0x32, 0x2d, 0xdd, 0x4b, 0x6b, 0xe9, 0xac, 0xbc, 0x11, - 0xe9, 0xe8, 0x37, 0x50, 0xe7, 0x4c, 0xc6, 0x19, 0xfe, 0x35, 0x8a, 0x29, 0x5d, 0xe0, 0x6a, 0x2f, - 0x15, 0xb8, 0xc6, 0x3f, 0x55, 0xb8, 0x95, 0xa4, 0xe7, 0x4d, 0x16, 0xf1, 0x9f, 0xa4, 0xc5, 0xb5, - 0x3a, 0x25, 0xae, 0x14, 0x25, 0x4b, 0xab, 0xb0, 0x3f, 0x29, 0x70, 0x7b, 0x26, 0x85, 0x4b, 0x22, - 0xb3, 0x3f, 0xab, 0x50, 0x3f, 0x64, 0x3e, 0xb1, 0x4e, 0x2e, 0x75, 0x1b, 0x13, 0xa9, 0x52, 0xbd, - 0xd8, 0x15, 0x8b, 0x36, 0xbf, 0x8b, 0x52, 0x47, 0x49, 0xe6, 0x9c, 0xa3, 0x24, 0x3b, 0xd7, 0x0d, - 0x67, 0x82, 0xd7, 0xdc, 0xab, 0x79, 0x35, 0x3a, 0x70, 0x23, 0x45, 0x94, 0x74, 0x61, 0x5c, 0x0e, - 0x28, 0xe7, 0x96, 0x03, 0xdf, 0xaa, 0xd0, 0x9c, 0x9a, 0xe5, 0x32, 0xe9, 0x7a, 0x6e, 0xd2, 0x93, - 0xa9, 0x40, 0x9b, 0x79, 0xae, 0x64, 0x5e, 0x75, 0xdb, 0x91, 0x9d, 0xd3, 0x51, 0x17, 0x0e, 0x92, - 0x2e, 0xbc, 0x77, 0x26, 0x21, 0x0b, 0x90, 0xfb, 0x47, 0x15, 0x6e, 0x4f, 0xcd, 0x75, 0xe9, 0x9c, - 0xf5, 0x5a, 0x18, 0x4e, 0x27, 0xdb, 0xcc, 0xb9, 0xb7, 0x09, 0x57, 0x46, 0xf6, 0x1e, 0xb4, 0x66, - 0x13, 0xb4, 0x00, 0xe3, 0x7f, 0x51, 0xe1, 0xfd, 0xf4, 0x84, 0x97, 0xf9, 0x61, 0xff, 0x5a, 0xf8, - 0x9e, 0xfe, 0xb5, 0x9e, 0x59, 0xe0, 0xd7, 0xfa, 0x95, 0xf1, 0xff, 0x10, 0x6e, 0xcd, 0xa2, 0x6b, - 0x01, 0xf6, 0x7f, 0x0e, 0xa5, 0x2d, 0x32, 0x70, 0xdc, 0xc5, 0xb8, 0x9e, 0xfa, 0xde, 0xa4, 0x4e, - 0x7f, 0x6f, 0x32, 0x3e, 0x83, 0xb2, 0x9c, 0x5a, 0xe2, 0x4a, 0x24, 0x4a, 0xe5, 0x9c, 0x44, 0xf9, - 0x8d, 0x02, 0xe5, 0x0e, 0xff, 0x2c, 0x75, 0xe5, 0x85, 0xc2, 0x0a, 0xe4, 0x2c, 0xe6, 0x9d, 0x38, - 0x7d, 0xf9, 0xc1, 0x4c, 0xb6, 0x8c, 0x1a, 0x54, 0x42, 0x04, 0x02, 0xbf, 0xf1, 0x2b, 0xa8, 0x62, - 0x6f, 0x38, 0x3c, 0xb2, 0xfa, 0xc7, 0x57, 0x8d, 0xca, 0x40, 0x50, 0x8b, 0xd7, 0x92, 0xeb, 0xff, - 0x12, 0x6e, 0x62, 0x42, 0xbd, 0xe1, 0x84, 0x24, 0x4a, 0x8a, 0xc5, 0x90, 0x20, 0xc8, 0xd8, 0x4c, - 0x7e, 0xb5, 0x29, 0x62, 0xfe, 0x6c, 0xfc, 0x43, 0x81, 0xfa, 0x2e, 0xa1, 0xd4, 0x1a, 0x10, 0x21, - 0xb0, 0xc5, 0xa6, 0x7e, 0x55, 0xcd, 0x58, 0x87, 0xac, 0x38, 0x79, 0x45, 0xbc, 0x89, 0x06, 0xda, - 0x80, 0x62, 0x14, 0x6c, 0xfc, 0x4c, 0x3e, 0x3b, 0xd6, 0x0a, 0x61, 0xac, 0x05, 0xe8, 0x13, 0xf7, - 0x23, 0xfc, 0xd9, 0xf8, 0xad, 0x02, 0xd7, 0x24, 0xfa, 0xfb, 0x8b, 0xfa, 0xe7, 0x55, 0xd0, 0xc3, - 0x35, 0xb5, 0x78, 0x4d, 0x74, 0x0b, 0xb4, 0x30, 0x19, 0xeb, 0x9b, 0x25, 0x19, 0x65, 0x8f, 0xad, - 0xe1, 0x98, 0xe0, 0xc0, 0x60, 0xec, 0x42, 0xa9, 0x9b, 0xa8, 0x34, 0xd1, 0x2a, 0xa8, 0x11, 0x8c, - 0xe9, 0xe1, 0xaa, 0x63, 0xa7, 0xaf, 0x28, 0xd4, 0x97, 0xae, 0x28, 0xfe, 0xae, 0xc0, 0x6a, 0xbc, - 0xc5, 0x4b, 0x1f, 0x4c, 0x17, 0xdd, 0xed, 0xe7, 0x50, 0x75, 0x6c, 0xf3, 0xa5, 0x63, 0x48, 0xdf, - 0xac, 0x87, 0x2a, 0x4e, 0x6e, 0x16, 0x97, 0x9d, 0x44, 0x8b, 0x1a, 0xab, 0xd0, 0x3c, 0x4b, 0xbc, - 0x52, 0xda, 0x1f, 0xc3, 0x8d, 0x07, 0x84, 0x1d, 0xfa, 0x93, 0xf0, 0x95, 0x70, 0x4b, 0x49, 0x90, - 0xca, 0x34, 0x48, 0x03, 0xc3, 0x4a, 0xfa, 0x25, 0x99, 0x69, 0x3e, 0x85, 0x12, 0xf5, 0x27, 0xe6, - 0xd4, 0x9b, 0x41, 0x66, 0x8d, 0x44, 0x95, 0x7c, 0x49, 0xa7, 0x71, 0xc3, 0xf8, 0x97, 0x02, 0x95, - 0xc7, 0x97, 0x91, 0x7f, 0xea, 0x18, 0x50, 0xe7, 0x3c, 0x06, 0x3e, 0x84, 0xec, 0x64, 0xc0, 0xe4, - 0xcd, 0x54, 0x70, 0x6a, 0x25, 0xfe, 0xfb, 0xe0, 0xf1, 0x03, 0xe6, 0xd8, 0x58, 0xd8, 0x83, 0xe4, - 0xfe, 0xd4, 0x19, 0x32, 0xe2, 0x47, 0x91, 0x92, 0x18, 0xf9, 0x05, 0xb7, 0x60, 0x39, 0xc2, 0xf8, - 0x31, 0x54, 0xa3, 0xbd, 0xc4, 0x67, 0x03, 0x99, 0x10, 0x97, 0xd1, 0x86, 0x22, 0x0f, 0xb5, 0xe4, - 0x42, 0x3b, 0x81, 0x09, 0xcb, 0x11, 0x77, 0xb7, 0xa1, 0x9a, 0xfa, 0x34, 0x8f, 0xaa, 0xa0, 0x3f, - 0xda, 0x3b, 0x3c, 0xd8, 0xe9, 0x74, 0xbf, 0xe8, 0xee, 0x6c, 0xd7, 0xde, 0x41, 0x00, 0xb9, 0xc3, - 0xee, 0xde, 0x83, 0x87, 0x3b, 0x35, 0x05, 0x15, 0x21, 0xbb, 0xfb, 0xe8, 0x61, 0xaf, 0x5b, 0x53, - 0x83, 0xc7, 0xde, 0x93, 0xfd, 0x83, 0x4e, 0x4d, 0xbb, 0xfb, 0x39, 0xe8, 0x22, 0x8f, 0xee, 0xfb, - 0x36, 0xf1, 0x83, 0x17, 0xf6, 0xf6, 0xf1, 0xee, 0xfd, 0x87, 0xb5, 0x77, 0x50, 0x1e, 0xb4, 0x03, - 0x1c, 0xbc, 0x59, 0x80, 0xcc, 0xc1, 0xfe, 0x61, 0xaf, 0xa6, 0xa2, 0x0a, 0xc0, 0xfd, 0x47, 0xbd, - 0xfd, 0xce, 0xfe, 0xee, 0x6e, 0xb7, 0x57, 0xd3, 0xb6, 0x3e, 0x81, 0xaa, 0xe3, 0xad, 0x4f, 0x1c, - 0x46, 0x28, 0x15, 0xff, 0x5c, 0xf1, 0x8b, 0x0f, 0x64, 0xcb, 0xf1, 0x36, 0xc4, 0xd3, 0xc6, 0xc0, - 0xdb, 0x98, 0xb0, 0x0d, 0x6e, 0xdd, 0x10, 0x52, 0x3c, 0xca, 0xf1, 0xd6, 0xc7, 0xdf, 0x05, 0x00, - 0x00, 0xff, 0xff, 0xb3, 0x65, 0x97, 0x24, 0xdc, 0x21, 0x00, 0x00, + // 1058 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x56, 0xef, 0x6e, 0x23, 0xb5, + 0x17, 0xdd, 0xc9, 0xff, 0xdc, 0xfc, 0x9b, 0x9f, 0xb7, 0xbb, 0xbf, 0xd9, 0xb2, 0xa0, 0x28, 0xcb, + 0x6a, 0xd3, 0x82, 0x12, 0x14, 0x04, 0x42, 0x08, 0x84, 0xda, 0x34, 0xbb, 0x0a, 0x6a, 0x9b, 0xe2, + 0xa4, 0xad, 0x84, 0x16, 0x8d, 0xa6, 0x19, 0x37, 0xb5, 0x36, 0x1d, 0xcf, 0xda, 0x4e, 0x4a, 0x9e, + 0x82, 0xef, 0xbc, 0x00, 0xef, 0xc2, 0x37, 0x9e, 0x81, 0x17, 0x41, 0xb6, 0x67, 0x92, 0x69, 0x54, + 0xd8, 0x6e, 0x57, 0xfd, 0x12, 0xd9, 0xf7, 0x5c, 0xdb, 0xd7, 0xe7, 0xdc, 0xe3, 0x0c, 0x94, 0xe7, + 0x72, 0xe2, 0x49, 0xd2, 0x0a, 0x39, 0x93, 0x0c, 0xe5, 0xcc, 0x6c, 0xd3, 0x3e, 0xa3, 0xc1, 0x94, + 0x4d, 0x7c, 0x4f, 0x7a, 0x06, 0xd9, 0x2c, 0xbd, 0x9d, 0x11, 0xbe, 0x88, 0x26, 0x55, 0xc9, 0x42, + 0x96, 0x04, 0xe7, 0x92, 0x87, 0x63, 0x33, 0x69, 0xfc, 0x9d, 0x83, 0xfc, 0x90, 0x08, 0x41, 0x59, + 0x80, 0x9e, 0x43, 0x95, 0x06, 0xae, 0xe4, 0x5e, 0x20, 0xbc, 0xb1, 0xa4, 0x2c, 0x70, 0xac, 0xba, + 0xd5, 0x2c, 0xe0, 0x0a, 0x0d, 0x46, 0xab, 0x20, 0xea, 0x42, 0x55, 0x5c, 0x78, 0xdc, 0x77, 0x85, + 0x59, 0x27, 0x9c, 0x54, 0x3d, 0xdd, 0x2c, 0x75, 0x9e, 0xb6, 0xa2, 0xea, 0xa2, 0xfd, 0x5a, 0x43, + 0x95, 0x15, 0x4d, 0x70, 0x45, 0x24, 0x66, 0x02, 0x7d, 0x04, 0x45, 0x41, 0x83, 0xc9, 0x94, 0xb8, + 0xfe, 0x99, 0x93, 0xd6, 0xc7, 0x14, 0x4c, 0x60, 0xef, 0x0c, 0x7d, 0x02, 0xe0, 0xcd, 0x24, 0x1b, + 0xb3, 0xcb, 0x4b, 0x2a, 0x9d, 0x8c, 0x46, 0x13, 0x11, 0xf4, 0x0c, 0x2a, 0xd2, 0xe3, 0x13, 0x22, + 0x5d, 0x21, 0x39, 0x0d, 0x26, 0x4e, 0xb6, 0x6e, 0x35, 0x8b, 0xb8, 0x6c, 0x82, 0x43, 0x1d, 0x43, + 0x6d, 0xc8, 0xb3, 0x50, 0xea, 0xfa, 0x72, 0x75, 0xab, 0x59, 0xea, 0x3c, 0x6a, 0x19, 0x56, 0x7a, + 0xbf, 0x92, 0xf1, 0x4c, 0x92, 0x81, 0x01, 0x71, 0x9c, 0x85, 0x76, 0xc1, 0x4e, 0xdc, 0xdd, 0xbd, + 0x64, 0x3e, 0x71, 0xf2, 0x75, 0xab, 0x59, 0xed, 0xfc, 0x3f, 0xbe, 0x59, 0x82, 0x86, 0x03, 0xe6, + 0x13, 0x5c, 0x93, 0xd7, 0x03, 0xa8, 0x0d, 0x85, 0x2b, 0x8f, 0x07, 0x34, 0x98, 0x08, 0xa7, 0xa0, + 0x59, 0x79, 0x18, 0x9d, 0xfa, 0x93, 0xfa, 0x3d, 0x35, 0x18, 0x5e, 0x26, 0xa1, 0x1f, 0xa0, 0x1c, + 0x72, 0xb2, 0xa2, 0xb2, 0x78, 0x0b, 0x2a, 0x4b, 0x21, 0x27, 0x4b, 0x22, 0x77, 0xa0, 0x12, 0x32, + 0x21, 0x57, 0x3b, 0xc0, 0x2d, 0x76, 0x28, 0xab, 0x25, 0xcb, 0x2d, 0x3e, 0x85, 0xea, 0xd4, 0x13, + 0xd2, 0xa5, 0x81, 0x20, 0x5c, 0xba, 0xd4, 0x77, 0x4a, 0x75, 0xab, 0x99, 0xc1, 0x65, 0x15, 0xed, + 0xeb, 0x60, 0xdf, 0x47, 0x1f, 0x03, 0x9c, 0xb3, 0x59, 0xe0, 0xbb, 0x9c, 0x5d, 0x09, 0xa7, 0xac, + 0x33, 0x8a, 0x3a, 0x82, 0xd9, 0x95, 0x40, 0x2e, 0x3c, 0x9e, 0x09, 0xc2, 0x5d, 0x9f, 0x9c, 0xd3, + 0x80, 0xf8, 0xee, 0xdc, 0xe3, 0xd4, 0x3b, 0x9b, 0x12, 0xe1, 0x54, 0x74, 0x41, 0x5b, 0xeb, 0x05, + 0x1d, 0x0b, 0xc2, 0xf7, 0x4c, 0xf2, 0x49, 0x9c, 0xdb, 0x0b, 0x24, 0x5f, 0xe0, 0x8d, 0xd9, 0x0d, + 0xd0, 0xe6, 0x6b, 0x28, 0x27, 0xef, 0x80, 0x9e, 0x43, 0xce, 0xe8, 0xad, 0xbb, 0xb4, 0xd4, 0xa9, + 0x44, 0x44, 0x8f, 0x74, 0x10, 0x47, 0xa0, 0x6a, 0xea, 0xa4, 0xaa, 0xd4, 0x77, 0x52, 0x75, 0xab, + 0x99, 0xc6, 0x95, 0x44, 0xb4, 0xef, 0x6f, 0xbe, 0x86, 0x27, 0xff, 0x5a, 0x10, 0xb2, 0x21, 0xfd, + 0x86, 0x2c, 0xf4, 0x39, 0x45, 0xac, 0x86, 0x68, 0x0b, 0xb2, 0x73, 0x6f, 0x3a, 0x23, 0x7a, 0xb3, + 0x95, 0xc8, 0xbb, 0x34, 0x58, 0xae, 0xc5, 0x26, 0xe3, 0xdb, 0xd4, 0x37, 0x56, 0xe3, 0x8f, 0x14, + 0x54, 0xa3, 0xb6, 0xc3, 0xe4, 0xed, 0x8c, 0x08, 0x89, 0x3e, 0x87, 0xe2, 0xd8, 0x9b, 0x4e, 0x09, + 0x57, 0x25, 0x99, 0x1b, 0xd4, 0x5a, 0xc6, 0x99, 0x5d, 0x1d, 0xef, 0xef, 0xe1, 0x82, 0xc9, 0xe8, + 0xfb, 0x68, 0x0b, 0xf2, 0x91, 0xc0, 0xd1, 0x89, 0xb5, 0x35, 0x3a, 0x71, 0x8c, 0xa3, 0x17, 0x90, + 0xd5, 0xc5, 0x68, 0x57, 0x95, 0x3a, 0xff, 0x8b, 0x4b, 0x53, 0x4a, 0xe9, 0x26, 0xc4, 0x06, 0x47, + 0x5f, 0x41, 0x49, 0xaa, 0x42, 0xa5, 0x2b, 0x17, 0x21, 0xd1, 0x36, 0xab, 0x76, 0x36, 0x5a, 0xcb, + 0xd7, 0x62, 0xa4, 0xc1, 0xd1, 0x22, 0x24, 0x18, 0xe4, 0x72, 0xac, 0x08, 0x7d, 0x43, 0x16, 0x22, + 0xf4, 0xc6, 0xc4, 0xd5, 0x9e, 0xd6, 0xf6, 0x2a, 0xe2, 0x4a, 0x1c, 0xd5, 0x2a, 0x25, 0xed, 0x97, + 0xbf, 0x8d, 0xfd, 0x7e, 0xcc, 0x14, 0xb2, 0x76, 0xae, 0xf1, 0x9b, 0x05, 0xb5, 0x25, 0x53, 0x22, + 0x64, 0x81, 0x50, 0x27, 0x66, 0x09, 0xe7, 0x8c, 0xaf, 0xd1, 0x84, 0x8f, 0xba, 0x3d, 0x15, 0xc6, + 0x06, 0x7d, 0x1f, 0x8e, 0xb6, 0x21, 0xc7, 0x89, 0x98, 0x4d, 0x65, 0x44, 0x12, 0x4a, 0x9a, 0x14, + 0x6b, 0x04, 0x47, 0x19, 0x8d, 0xbf, 0x52, 0xf0, 0x30, 0xaa, 0x68, 0xd7, 0x93, 0xe3, 0x8b, 0x7b, + 0x17, 0xf0, 0x33, 0xc8, 0xab, 0x6a, 0x28, 0x11, 0x4e, 0x5a, 0x5b, 0xe7, 0x06, 0x09, 0xe3, 0x8c, + 0x0f, 0x10, 0xd1, 0x13, 0xd7, 0x9e, 0xfa, 0xac, 0x79, 0xea, 0x3d, 0x91, 0x7c, 0xea, 0xef, 0x49, + 0xeb, 0xc6, 0xef, 0x16, 0x6c, 0x5c, 0xe7, 0xf4, 0xde, 0xa4, 0xfe, 0x02, 0xf2, 0x46, 0xc8, 0x98, + 0xcd, 0xc7, 0x51, 0x6d, 0x46, 0xe6, 0x53, 0x2a, 0x2f, 0xcc, 0xd6, 0x71, 0x9a, 0x32, 0xeb, 0xc6, + 0x50, 0x72, 0xe2, 0x5d, 0x7e, 0x90, 0x65, 0x97, 0x3e, 0x4c, 0xbd, 0x9f, 0x0f, 0xd3, 0x77, 0xf6, + 0x61, 0xe6, 0x1d, 0xda, 0x64, 0x6f, 0xf5, 0x37, 0x98, 0xe0, 0x36, 0xf7, 0xdf, 0xdc, 0x36, 0xba, + 0xf0, 0x68, 0x8d, 0xa8, 0x48, 0xc6, 0x95, 0xbf, 0xac, 0x77, 0xfa, 0xeb, 0x17, 0x78, 0x82, 0x89, + 0x60, 0xd3, 0x39, 0x49, 0x74, 0xde, 0xdd, 0x28, 0x47, 0x90, 0xf1, 0x65, 0xf4, 0xc2, 0x17, 0xb1, + 0x1e, 0x37, 0x9e, 0xc2, 0xe6, 0x4d, 0xdb, 0x9b, 0x42, 0x1b, 0x7f, 0x5a, 0x50, 0x3d, 0x31, 0x77, + 0xb8, 0xdb, 0x91, 0x6b, 0xe2, 0xa5, 0x6e, 0x29, 0xde, 0x0b, 0xc8, 0xce, 0x27, 0xaa, 0xd4, 0xf8, + 0x91, 0x4e, 0x7c, 0xc2, 0x9d, 0xbc, 0x92, 0xd4, 0xc7, 0x06, 0x57, 0x4c, 0x9e, 0xd3, 0xa9, 0x24, + 0x5c, 0xab, 0xab, 0x98, 0x4c, 0x64, 0xbe, 0xd4, 0x08, 0x8e, 0x32, 0x1a, 0xdf, 0x43, 0x6d, 0x79, + 0x97, 0x95, 0x10, 0x64, 0x4e, 0x02, 0x29, 0x1c, 0x4b, 0x37, 0xff, 0xb5, 0xe5, 0x27, 0x3d, 0x05, + 0xe1, 0x28, 0x63, 0x7b, 0x0f, 0x6a, 0x6b, 0xdf, 0x37, 0xa8, 0x06, 0xa5, 0xe3, 0xc3, 0xe1, 0x51, + 0xaf, 0xdb, 0x7f, 0xd9, 0xef, 0xed, 0xd9, 0x0f, 0x10, 0x40, 0x6e, 0xd8, 0x3f, 0x7c, 0xb5, 0xdf, + 0xb3, 0x2d, 0x54, 0x84, 0xec, 0xc1, 0xf1, 0xfe, 0xa8, 0x6f, 0xa7, 0xd4, 0x70, 0x74, 0x3a, 0x38, + 0xea, 0xda, 0xe9, 0xed, 0xef, 0xa0, 0xd4, 0xd5, 0x5f, 0x69, 0x03, 0xee, 0x13, 0xae, 0x16, 0x1c, + 0x0e, 0xf0, 0xc1, 0xce, 0xbe, 0xfd, 0x00, 0xe5, 0x21, 0x7d, 0x84, 0xd5, 0xca, 0x02, 0x64, 0x8e, + 0x06, 0xc3, 0x91, 0x9d, 0x42, 0x55, 0x80, 0x9d, 0xe3, 0xd1, 0xa0, 0x3b, 0x38, 0x38, 0xe8, 0x8f, + 0xec, 0xf4, 0xee, 0xd7, 0x50, 0xa3, 0xac, 0x35, 0xa7, 0x92, 0x08, 0x61, 0xbe, 0x50, 0x7f, 0x7e, + 0x16, 0xcd, 0x28, 0x6b, 0x9b, 0x51, 0x7b, 0xc2, 0xda, 0x73, 0xd9, 0xd6, 0x68, 0xdb, 0xb4, 0xe6, + 0x59, 0x4e, 0xcf, 0xbe, 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0x4a, 0x19, 0x5b, 0xd5, 0x21, 0x0b, + 0x00, 0x00, } diff --git a/java/client/src/main/java/io/vitess/client/Proto.java b/java/client/src/main/java/io/vitess/client/Proto.java index 64042d2926d..f72cdde8c2c 100644 --- a/java/client/src/main/java/io/vitess/client/Proto.java +++ b/java/client/src/main/java/io/vitess/client/Proto.java @@ -29,9 +29,6 @@ import io.vitess.proto.Query.BindVariable; import io.vitess.proto.Query.BoundQuery; import io.vitess.proto.Query.QueryResult; -import io.vitess.proto.Vtgate.BoundKeyspaceIdQuery; -import io.vitess.proto.Vtgate.BoundShardQuery; -import io.vitess.proto.Vtgate.ExecuteEntityIdsRequest.EntityId; import io.vitess.proto.Vtrpc.RPCError; import java.math.BigDecimal; @@ -62,13 +59,6 @@ public ByteString apply(byte[] from) { return ByteString.copyFrom(from); } }; - public static final Function, EntityId> MAP_ENTRY_TO_ENTITY_KEYSPACE_ID = - new Function, EntityId>() { - @Override - public EntityId apply(Map.Entry entry) { - return buildEntityId(entry.getKey(), entry.getValue()); - } - }; private static final int MAX_DECIMAL_UNIT = 30; /** @@ -209,13 +199,6 @@ public static BindVariable buildBindVariable(Object value) { return builder.build(); } - public static EntityId buildEntityId(byte[] keyspaceId, Object value) { - TypedValue tval = new TypedValue(value); - - return EntityId.newBuilder().setKeyspaceId(ByteString.copyFrom(keyspaceId)).setType(tval.type) - .setValue(tval.value).build(); - } - /** * bindQuery creates a BoundQuery from query and vars. */ @@ -229,49 +212,6 @@ public static BoundQuery bindQuery(String query, Map vars) { return boundQueryBuilder.build(); } - /** - * bindShardQuery creates a BoundShardQuery. - */ - public static BoundShardQuery bindShardQuery(String keyspace, Iterable shards, - BoundQuery query) { - return BoundShardQuery.newBuilder().setKeyspace(keyspace).addAllShards(shards).setQuery(query) - .build(); - } - - /** - * bindShardQuery creates a BoundShardQuery. - */ - public static BoundShardQuery bindShardQuery(String keyspace, Iterable shards, - String query, Map vars) { - return bindShardQuery(keyspace, shards, bindQuery(query, vars)); - } - - /** - * bindKeyspaceIdQuery creates a BoundKeyspaceIdQuery. - */ - public static BoundKeyspaceIdQuery bindKeyspaceIdQuery(String keyspace, - Iterable keyspaceIds, BoundQuery query) { - return BoundKeyspaceIdQuery.newBuilder().setKeyspace(keyspace) - .addAllKeyspaceIds(Iterables.transform(keyspaceIds, BYTE_ARRAY_TO_BYTE_STRING)) - .setQuery(query).build(); - } - - /** - * bindKeyspaceIdQuery creates a BoundKeyspaceIdQuery. - */ - public static BoundKeyspaceIdQuery bindKeyspaceIdQuery(String keyspace, - Iterable keyspaceIds, String query, Map vars) { - return bindKeyspaceIdQuery(keyspace, keyspaceIds, bindQuery(query, vars)); - } - - public static List toCursorList(List queryResults) { - ImmutableList.Builder builder = new ImmutableList.Builder(); - for (QueryResult queryResult : queryResults) { - builder.add(new SimpleCursor(queryResult)); - } - return builder.build(); - } - public static List fromQueryResponsesToCursorList( List resultWithErrorList) { ImmutableList.Builder builder = new ImmutableList.Builder(); diff --git a/java/client/src/test/java/io/vitess/client/EntityIdTest.java b/java/client/src/test/java/io/vitess/client/EntityIdTest.java deleted file mode 100644 index b614fcff2c0..00000000000 --- a/java/client/src/test/java/io/vitess/client/EntityIdTest.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2019 The Vitess Authors. - - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - - * http://www.apache.org/licenses/LICENSE-2.0 - - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.vitess.client; - -import static org.junit.Assert.assertEquals; - -import com.google.common.primitives.UnsignedLong; -import com.google.protobuf.ByteString; - -import io.vitess.proto.Query; -import io.vitess.proto.Vtgate.ExecuteEntityIdsRequest.EntityId; - -import java.util.Arrays; -import java.util.Collection; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; - -@RunWith(value = Parameterized.class) -public class EntityIdTest { - - private Object input; - private EntityId expected; - - private static final ByteString KEYSPACE_ID = ByteString.copyFrom(new byte[]{1, 2, 3}); - - @Parameters - public static Collection testParams() { - Object[][] params = { - // SQL NULL - { - null, - EntityId.newBuilder().setKeyspaceId(KEYSPACE_ID).setType(Query.Type.NULL_TYPE).build() - }, - // String - { - "hello world", - EntityId.newBuilder() - .setKeyspaceId(KEYSPACE_ID) - .setType(Query.Type.VARCHAR) - .setValue(ByteString.copyFromUtf8("hello world")) - .build() - }, - // Bytes - { - new byte[]{1, 2, 3}, - EntityId.newBuilder() - .setKeyspaceId(KEYSPACE_ID) - .setType(Query.Type.VARBINARY) - .setValue(ByteString.copyFrom(new byte[]{1, 2, 3})) - .build() - }, - // Int - { - 123, - EntityId.newBuilder() - .setKeyspaceId(KEYSPACE_ID) - .setType(Query.Type.INT64) - .setValue(ByteString.copyFromUtf8("123")) - .build() - }, - { - 123L, - EntityId.newBuilder() - .setKeyspaceId(KEYSPACE_ID) - .setType(Query.Type.INT64) - .setValue(ByteString.copyFromUtf8("123")) - .build() - }, - // Uint - { - UnsignedLong.fromLongBits(-1), - EntityId.newBuilder() - .setKeyspaceId(KEYSPACE_ID) - .setType(Query.Type.UINT64) - .setValue(ByteString.copyFromUtf8("18446744073709551615")) - .build() - }, - // Float - { - 1.23f, - EntityId.newBuilder() - .setKeyspaceId(KEYSPACE_ID) - .setType(Query.Type.FLOAT64) - .setValue(ByteString.copyFromUtf8("1.23")) - .build() - }, - { - 1.23, - EntityId.newBuilder() - .setKeyspaceId(KEYSPACE_ID) - .setType(Query.Type.FLOAT64) - .setValue(ByteString.copyFromUtf8("1.23")) - .build() - }, - }; - return Arrays.asList(params); - } - - public EntityIdTest(Object input, EntityId expected) { - this.input = input; - this.expected = expected; - } - - @Test - public void testBuildEntityId() { - assertEquals(expected, Proto.buildEntityId(KEYSPACE_ID.toByteArray(), input)); - } -} diff --git a/proto/vtgate.proto b/proto/vtgate.proto index 6b187530502..aa8b552531e 100644 --- a/proto/vtgate.proto +++ b/proto/vtgate.proto @@ -118,6 +118,9 @@ message Session { // ExecuteRequest is the payload to Execute. message ExecuteRequest { + // Deprecated: + // bool not_in_transaction = 5; + reserved 5; // caller_id identifies the caller. This is the effective caller ID, // set by the application to further identify the caller. vtrpc.CallerID caller_id = 1; @@ -131,7 +134,6 @@ message ExecuteRequest { // These values are deprecated. Use session instead. // TODO(sougou): remove in 3.1 topodata.TabletType tablet_type = 4; - bool not_in_transaction = 5; string keyspace_shard = 6; query.ExecuteOptions options = 7; } @@ -150,195 +152,6 @@ message ExecuteResponse { query.QueryResult result = 3; } -// ExecuteShardsRequest is the payload to ExecuteShards. -message ExecuteShardsRequest { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - vtrpc.CallerID caller_id = 1; - - // session carries the current transaction data. It is returned by Begin. - // Do not fill it in if outside of a transaction. - Session session = 2; - - // query is the query and bind variables to execute. - query.BoundQuery query = 3; - - // keyspace to target the query to. - string keyspace = 4; - - // shards to target the query to. A DML can only target one shard. - repeated string shards = 5; - - // tablet_type is the type of tablets that this query is targeted to. - topodata.TabletType tablet_type = 6; - - // not_in_transaction is deprecated. - bool not_in_transaction = 7; - - // options - query.ExecuteOptions options = 8; -} - -// ExecuteShardsResponse is the returned value from ExecuteShards. -message ExecuteShardsResponse { - // error contains an application level error if necessary. Note the - // session may have changed, even when an error is returned (for - // instance if a database integrity error happened). - vtrpc.RPCError error = 1; - - // session is the updated session information (only returned inside a transaction). - Session session = 2; - - // result contains the query result, only set if error is unset. - query.QueryResult result = 3; -} - -// ExecuteKeyspaceIdsRequest is the payload to ExecuteKeyspaceIds. -message ExecuteKeyspaceIdsRequest { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - vtrpc.CallerID caller_id = 1; - - // session carries the current transaction data. It is returned by Begin. - // Do not fill it in if outside of a transaction. - Session session = 2; - - // query is the query and bind variables to execute. - query.BoundQuery query = 3; - - // keyspace to target the query to. - string keyspace = 4; - - // keyspace_ids contains the list of keyspace_ids affected by this query. - // Will be used to find the shards to send the query to. - repeated bytes keyspace_ids = 5; - - // tablet_type is the type of tablets that this query is targeted to. - topodata.TabletType tablet_type = 6; - - // not_in_transaction is deprecated. - bool not_in_transaction = 7; - - // options - query.ExecuteOptions options = 8; -} - -// ExecuteKeyspaceIdsResponse is the returned value from ExecuteKeyspaceIds. -message ExecuteKeyspaceIdsResponse { - // error contains an application level error if necessary. Note the - // session may have changed, even when an error is returned (for - // instance if a database integrity error happened). - vtrpc.RPCError error = 1; - - // session is the updated session information (only returned inside a transaction). - Session session = 2; - - // result contains the query result, only set if error is unset. - query.QueryResult result = 3; -} - -// ExecuteKeyRangesRequest is the payload to ExecuteKeyRanges. -message ExecuteKeyRangesRequest { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - vtrpc.CallerID caller_id = 1; - - // session carries the current transaction data. It is returned by Begin. - // Do not fill it in if outside of a transaction. - Session session = 2; - - // query is the query and bind variables to execute. - query.BoundQuery query = 3; - - // keyspace to target the query to - string keyspace = 4; - - // key_ranges contains the list of key ranges affected by this query. - // Will be used to find the shards to send the query to. - repeated topodata.KeyRange key_ranges = 5; - - // tablet_type is the type of tablets that this query is targeted to. - topodata.TabletType tablet_type = 6; - - // not_in_transaction is deprecated. - bool not_in_transaction = 7; - - // options - query.ExecuteOptions options = 8; -} - -// ExecuteKeyRangesResponse is the returned value from ExecuteKeyRanges. -message ExecuteKeyRangesResponse { - // error contains an application level error if necessary. Note the - // session may have changed, even when an error is returned (for - // instance if a database integrity error happened). - vtrpc.RPCError error = 1; - - // session is the updated session information (only returned inside a transaction). - Session session = 2; - - // result contains the query result, only set if error is unset. - query.QueryResult result = 3; -} - -// ExecuteEntityIdsRequest is the payload to ExecuteEntityIds. -message ExecuteEntityIdsRequest { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - vtrpc.CallerID caller_id = 1; - - // session carries the current transaction data. It is returned by Begin. - // Do not fill it in if outside of a transaction. - Session session = 2; - - // query is the query and bind variables to execute. - query.BoundQuery query = 3; - - // keyspace to target the query to. - string keyspace = 4; - - // entity_column_name is the column name to use. - string entity_column_name = 5; - - message EntityId { - // type is the type of the entity's value. Can be NULL_TYPE. - query.Type type = 1; - - // value is the value for the entity. Not set if type is NULL_TYPE. - bytes value = 2; - - // keyspace_id is the associated keyspace_id for the entity. - bytes keyspace_id = 3; - } - - // entity_keyspace_ids are pairs of entity_column_name values - // associated with its corresponding keyspace_id. - repeated EntityId entity_keyspace_ids = 6; - - // tablet_type is the type of tablets that this query is targeted to. - topodata.TabletType tablet_type = 7; - - // not_in_transaction is deprecated. - bool not_in_transaction = 8; - - // options - query.ExecuteOptions options = 9; -} - -// ExecuteEntityIdsResponse is the returned value from ExecuteEntityIds. -message ExecuteEntityIdsResponse { - // error contains an application level error if necessary. Note the - // session may have changed, even when an error is returned (for - // instance if a database integrity error happened). - vtrpc.RPCError error = 1; - - // session is the updated session information (only returned inside a transaction). - Session session = 2; - - // result contains the query result, only set if error is unset. - query.QueryResult result = 3; -} - // ExecuteBatchRequest is the payload to ExecuteBatch. message ExecuteBatchRequest { // caller_id identifies the caller. This is the effective caller ID, @@ -374,112 +187,6 @@ message ExecuteBatchResponse { repeated query.ResultWithError results = 3; } -// BoundShardQuery represents a single query request for the -// specified list of shards. This is used in a list for -// ExecuteBatchShardsRequest. -message BoundShardQuery { - // query is the query and bind variables to execute. - query.BoundQuery query = 1; - - // keyspace to target the query to. - string keyspace = 2; - - // shards to target the query to. A DML can only target one shard. - repeated string shards = 3; -} - -// ExecuteBatchShardsRequest is the payload to ExecuteBatchShards -message ExecuteBatchShardsRequest { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - vtrpc.CallerID caller_id = 1; - - // session carries the current transaction data. It is returned by Begin. - // Do not fill it in if outside of a transaction. - Session session = 2; - - // queries carries all the queries to execute. - repeated BoundShardQuery queries = 3; - - // tablet_type is the type of tablets that this query is targeted to. - topodata.TabletType tablet_type = 4; - - // as_transaction will execute the queries in this batch in a single transaction per shard, created for this purpose. - // (this can be seen as adding a 'begin' before and 'commit' after the queries). - // Only makes sense if tablet_type is master. If set, the Session is ignored. - bool as_transaction = 5; - - // options - query.ExecuteOptions options = 6; -} - -// ExecuteBatchShardsResponse is the returned value from ExecuteBatchShards. -message ExecuteBatchShardsResponse { - // error contains an application level error if necessary. Note the - // session may have changed, even when an error is returned (for - // instance if a database integrity error happened). - vtrpc.RPCError error = 1; - - // session is the updated session information (only returned inside a transaction). - Session session = 2; - - // result contains the query result, only set if error is unset. - repeated query.QueryResult results = 3; -} - -// BoundKeyspaceIdQuery represents a single query request for the -// specified list of keyspace ids. This is used in a list for -// ExecuteBatchKeyspaceIdsRequest. -message BoundKeyspaceIdQuery { - // query is the query and bind variables to execute. - query.BoundQuery query = 1; - - // keyspace to target the query to. - string keyspace = 2; - - // keyspace_ids contains the list of keyspace_ids affected by this query. - // Will be used to find the shards to send the query to. - repeated bytes keyspace_ids = 3; -} - -// ExecuteBatchKeyspaceIdsRequest is the payload to ExecuteBatchKeyspaceId. -message ExecuteBatchKeyspaceIdsRequest { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - vtrpc.CallerID caller_id = 1; - - // session carries the current transaction data. It is returned by Begin. - // Do not fill it in if outside of a transaction. - Session session = 2; - - repeated BoundKeyspaceIdQuery queries = 3; - - // tablet_type is the type of tablets that this query is targeted to. - topodata.TabletType tablet_type = 4; - - // as_transaction will execute the queries in this batch in a single transaction per shard, created for this purpose. - // (this can be seen as adding a 'begin' before and 'commit' after the queries). - // Only makes sense if tablet_type is master. If set, the Session is ignored. - bool as_transaction = 5; - - // options - query.ExecuteOptions options = 6; -} - -// ExecuteBatchKeyspaceIdsResponse is the returned value from ExecuteBatchKeyspaceId. -message ExecuteBatchKeyspaceIdsResponse { - // error contains an application level error if necessary. Note the - // session may have changed, even when an error is returned (for - // instance if a database integrity error happened). - vtrpc.RPCError error = 1; - - // session is the updated session information (only returned inside a transaction). - Session session = 2; - - // result contains the query result, only set if error is unset. - repeated query.QueryResult results = 3; -} - // StreamExecuteRequest is the payload to StreamExecute. message StreamExecuteRequest { // caller_id identifies the caller. This is the effective caller ID, @@ -509,151 +216,6 @@ message StreamExecuteResponse { query.QueryResult result = 1; } -// StreamExecuteShardsRequest is the payload to StreamExecuteShards. -message StreamExecuteShardsRequest { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - vtrpc.CallerID caller_id = 1; - - // query is the query and bind variables to execute. - query.BoundQuery query = 2; - - // keyspace to target the query to. - string keyspace = 3; - - // shards to target the query to. - repeated string shards = 4; - - // tablet_type is the type of tablets that this query is targeted to. - topodata.TabletType tablet_type = 5; - - // options - query.ExecuteOptions options = 6; -} - -// StreamExecuteShardsResponse is the returned value from StreamExecuteShards. -message StreamExecuteShardsResponse { - // result contains the result data. - // The first value contains only Fields information. - // The next values contain the actual rows, a few values per result. - query.QueryResult result = 1; -} - -// StreamExecuteKeyspaceIdsRequest is the payload to StreamExecuteKeyspaceIds. -message StreamExecuteKeyspaceIdsRequest { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - vtrpc.CallerID caller_id = 1; - - // query is the query and bind variables to execute. - query.BoundQuery query = 2; - - // keyspace to target the query to. - string keyspace = 3; - - // keyspace_ids contains the list of keyspace_ids affected by this query. - // Will be used to find the shards to send the query to. - repeated bytes keyspace_ids = 4; - - // tablet_type is the type of tablets that this query is targeted to. - topodata.TabletType tablet_type = 5; - - // options - query.ExecuteOptions options = 6; -} - -// StreamExecuteKeyspaceIdsResponse is the returned value from StreamExecuteKeyspaceIds. -message StreamExecuteKeyspaceIdsResponse { - // result contains the result data. - // The first value contains only Fields information. - // The next values contain the actual rows, a few values per result. - query.QueryResult result = 1; -} - -// StreamExecuteKeyRangesRequest is the payload to StreamExecuteKeyRanges. -message StreamExecuteKeyRangesRequest { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - vtrpc.CallerID caller_id = 1; - - // query is the query and bind variables to execute. - query.BoundQuery query = 2; - - // keyspace to target the query to. - string keyspace = 3; - - // key_ranges contains the list of key ranges affected by this query. - // Will be used to find the shards to send the query to. - repeated topodata.KeyRange key_ranges = 4; - - // tablet_type is the type of tablets that this query is targeted to. - topodata.TabletType tablet_type = 5; - - // options - query.ExecuteOptions options = 6; -} - -// StreamExecuteKeyRangesResponse is the returned value from StreamExecuteKeyRanges. -message StreamExecuteKeyRangesResponse { - // result contains the result data. - // The first value contains only Fields information. - // The next values contain the actual rows, a few values per result. - query.QueryResult result = 1; -} - -// BeginRequest is the payload to Begin. -message BeginRequest { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - vtrpc.CallerID caller_id = 1; - - // single_db is deprecated. Use transaction_mode instead. - // The value specifies if the transaction should be restricted - // to a single database. - // TODO(sougou): remove in 3.1 - bool single_db = 2; -} - -// BeginResponse is the returned value from Begin. -message BeginResponse { - // session is the initial session information to use for subsequent queries. - Session session = 1; -} - -// CommitRequest is the payload to Commit. -message CommitRequest { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - vtrpc.CallerID caller_id = 1; - - // session carries the current transaction data to commit. - Session session = 2; - - // atomic is deprecated. Use transaction_mode instead. - // The value specifies if the commit should go through the - // 2PC workflow to ensure atomicity. - // TODO(sougou): remove in 3.1 - bool atomic = 3; -} - -// CommitResponse is the returned value from Commit. -message CommitResponse { -} - -// RollbackRequest is the payload to Rollback. -message RollbackRequest { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - vtrpc.CallerID caller_id = 1; - - // session carries the current transaction data to rollback. - Session session = 2; -} - -// RollbackResponse is the returned value from Rollback. -message RollbackResponse { -} - // ResolveTransactionRequest is the payload to ResolveTransaction. message ResolveTransactionRequest { // caller_id identifies the caller. This is the effective caller ID, @@ -664,81 +226,10 @@ message ResolveTransactionRequest { string dtid = 2; } -// MessageStreamRequest is the request payload for MessageStream. -message MessageStreamRequest { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - vtrpc.CallerID caller_id = 1; - - // keyspace to target the query to. - string keyspace = 2; - - // shard to target the query to, for unsharded keyspaces. - string shard = 3; - - // KeyRange to target the query to, for sharded keyspaces. - topodata.KeyRange key_range = 4; - - // name is the message table name. - string name = 5; -} - -// MessageAckRequest is the request payload for MessageAck. -message MessageAckRequest { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - vtrpc.CallerID caller_id = 1; - - // keyspace to target the message to. - string keyspace = 2; - - // name is the message table name. - string name = 3; - // ids is the list of ids to ack. - repeated query.Value ids = 4; -} - -// IdKeyspaceId represents an id and keyspace_id pair. -// The kesypace_id represents the routing info for id. -message IdKeyspaceId { - // id represents the message id. - query.Value id = 1; - - // keyspace_id is the associated keyspace_id for the id. - bytes keyspace_id = 2; -} - -// MessageAckKeyspaceIdsRequest is the payload to MessageAckKeyspaceIds. -message MessageAckKeyspaceIdsRequest { - // caller_id identifies the caller. This is the effective caller ID, - // set by the application to further identify the caller. - vtrpc.CallerID caller_id = 1; - - // Optional keyspace for message table. - string keyspace = 2; - - // name is the message table name. - string name = 3; - - repeated IdKeyspaceId id_keyspace_ids = 4; -} - // ResolveTransactionResponse is the returned value from Rollback. message ResolveTransactionResponse { } -// GetSrvKeyspaceRequest is the payload to GetSrvKeyspace. -message GetSrvKeyspaceRequest { - // keyspace name to fetch. - string keyspace = 1; -} - -// GetSrvKeyspaceResponse is the returned value from GetSrvKeyspace. -message GetSrvKeyspaceResponse { - // srv_keyspace is the topology object for the SrvKeyspace. - topodata.SrvKeyspace srv_keyspace = 1; -} - // VStreamRequest is the payload for VStream. message VStreamRequest { vtrpc.CallerID caller_id = 1; From 47a08131748bc895416c5686b5315782b45e9914 Mon Sep 17 00:00:00 2001 From: Derek Perkins Date: Wed, 25 Mar 2020 00:33:27 -0600 Subject: [PATCH 40/85] messaging: add new priority field Signed-off-by: Derek Perkins --- go/test/endtoend/messaging/main_test.go | 6 ++-- go/test/endtoend/messaging/message_test.go | 6 ++-- go/vt/vttablet/tabletserver/messager/cache.go | 5 +-- .../tabletserver/messager/message_manager.go | 31 ++++++++++++------- .../messager/message_manager_test.go | 1 + .../tabletserver/query_executor_test.go | 3 ++ .../tabletserver/schema/engine_test.go | 3 ++ .../tabletserver/schema/load_table.go | 2 ++ .../tabletserver/schema/load_table_test.go | 8 +++-- .../schema/schematest/schematest.go | 3 ++ 10 files changed, 48 insertions(+), 20 deletions(-) diff --git a/go/test/endtoend/messaging/main_test.go b/go/test/endtoend/messaging/main_test.go index 0281a2ac129..36d2bf97517 100644 --- a/go/test/endtoend/messaging/main_test.go +++ b/go/test/endtoend/messaging/main_test.go @@ -42,22 +42,24 @@ var ( lookupKeyspace = "lookup" createShardedMessage = `create table sharded_message( id bigint, + priority bigint default 0, time_next bigint default 0, epoch bigint, time_acked bigint, message varchar(128), primary key(id), - index next_idx(time_next, epoch), + index next_idx(priority, time_next), index ack_idx(time_acked) ) comment 'vitess_message,vt_ack_wait=1,vt_purge_after=3,vt_batch_size=2,vt_cache_size=10,vt_poller_interval=1'` createUnshardedMessage = `create table unsharded_message( id bigint, + priority bigint default 0, time_next bigint default 0, epoch bigint, time_acked bigint, message varchar(128), primary key(id), - index next_idx(time_next, epoch), + index next_idx(priority, time_next), index ack_idx(time_acked) ) comment 'vitess_message,vt_ack_wait=1,vt_purge_after=3,vt_batch_size=2,vt_cache_size=10,vt_poller_interval=1'` userVschema = `{ diff --git a/go/test/endtoend/messaging/message_test.go b/go/test/endtoend/messaging/message_test.go index fe3851afeea..72312594067 100644 --- a/go/test/endtoend/messaging/message_test.go +++ b/go/test/endtoend/messaging/message_test.go @@ -31,12 +31,13 @@ import ( var createMessage = `create table vitess_message( id bigint, + priority bigint default 0, time_next bigint default 0, epoch bigint, time_acked bigint, message varchar(128), primary key(id), - index next_idx(time_next, epoch), + index next_idx(priority, time_next), index ack_idx(time_acked)) comment 'vitess_message,vt_ack_wait=1,vt_purge_after=3,vt_batch_size=2,vt_cache_size=10,vt_poller_interval=1'` @@ -141,13 +142,14 @@ func TestMessage(t *testing.T) { var createThreeColMessage = `create table vitess_message3( id bigint, + priority bigint default 0, time_next bigint default 0, epoch bigint, time_acked bigint, msg1 varchar(128), msg2 bigint, primary key(id), - index next_idx(time_next, epoch), + index next_idx(priority, time_next), index ack_idx(time_acked)) comment 'vitess_message,vt_ack_wait=1,vt_purge_after=3,vt_batch_size=2,vt_cache_size=10,vt_poller_interval=1'` diff --git a/go/vt/vttablet/tabletserver/messager/cache.go b/go/vt/vttablet/tabletserver/messager/cache.go index f2c1653c30e..0de059b7d32 100644 --- a/go/vt/vttablet/tabletserver/messager/cache.go +++ b/go/vt/vttablet/tabletserver/messager/cache.go @@ -28,6 +28,7 @@ import ( // MessageRow represents a message row. // The first column in Row is always the "id". type MessageRow struct { + Priority int64 TimeNext int64 Epoch int64 TimeAcked int64 @@ -47,8 +48,8 @@ func (mh messageHeap) Len() int { func (mh messageHeap) Less(i, j int) bool { // Lower epoch is more important. // If epochs match, newer messages are more important. - return mh[i].Epoch < mh[j].Epoch || - (mh[i].Epoch == mh[j].Epoch && mh[i].TimeNext > mh[j].TimeNext) + return mh[i].Priority < mh[j].Priority || + (mh[i].Priority == mh[j].Priority && mh[i].TimeNext > mh[j].TimeNext) } func (mh messageHeap) Swap(i, j int) { diff --git a/go/vt/vttablet/tabletserver/messager/message_manager.go b/go/vt/vttablet/tabletserver/messager/message_manager.go index 51943d3bbdf..f418d8557e9 100644 --- a/go/vt/vttablet/tabletserver/messager/message_manager.go +++ b/go/vt/vttablet/tabletserver/messager/message_manager.go @@ -203,11 +203,11 @@ type messageManager struct { // The goroutine must in turn defer on Done. wg sync.WaitGroup - vsFilter *binlogdatapb.Filter - readByTimeNext *sqlparser.ParsedQuery - ackQuery *sqlparser.ParsedQuery - postponeQuery *sqlparser.ParsedQuery - purgeQuery *sqlparser.ParsedQuery + vsFilter *binlogdatapb.Filter + readByPriorityAndTimeNext *sqlparser.ParsedQuery + ackQuery *sqlparser.ParsedQuery + postponeQuery *sqlparser.ParsedQuery + purgeQuery *sqlparser.ParsedQuery } // newMessageManager creates a new message manager. @@ -233,15 +233,15 @@ func newMessageManager(tsv TabletService, vs VStreamer, table *schema.Table, pos mm.cond.L = &mm.mu columnList := buildSelectColumnList(table) - vsQuery := fmt.Sprintf("select time_next, epoch, time_acked, %s from %v", columnList, mm.name) + vsQuery := fmt.Sprintf("select priority, time_next, epoch, time_acked, %s from %v", columnList, mm.name) mm.vsFilter = &binlogdatapb.Filter{ Rules: []*binlogdatapb.Rule{{ Match: table.Name.String(), Filter: vsQuery, }}, } - mm.readByTimeNext = sqlparser.BuildParsedQuery( - "select time_next, epoch, time_acked, %s from %v where time_next < %a order by time_next desc limit %a", + mm.readByPriorityAndTimeNext = sqlparser.BuildParsedQuery( + "select priority, time_next, epoch, time_acked, %s from %v where time_next < %a order by priority, time_next desc limit %a", columnList, mm.name, ":time_next", ":max") mm.ackQuery = sqlparser.BuildParsedQuery( "update %v set time_acked = %a, time_next = null where id in %a and time_acked is null", @@ -808,22 +808,29 @@ func (mm *messageManager) GeneratePurgeQuery(timeCutoff int64) (string, map[stri // BuildMessageRow builds a MessageRow for a db row. func BuildMessageRow(row []sqltypes.Value) (*MessageRow, error) { - mr := &MessageRow{Row: row[3:]} + mr := &MessageRow{Row: row[4:]} if !row[0].IsNull() { v, err := sqltypes.ToInt64(row[0]) if err != nil { return nil, err } - mr.TimeNext = v + mr.Priority = v } if !row[1].IsNull() { + v, err := sqltypes.ToInt64(row[0]) + if err != nil { + return nil, err + } + mr.TimeNext = v + } + if !row[2].IsNull() { v, err := sqltypes.ToInt64(row[1]) if err != nil { return nil, err } mr.Epoch = v } - if !row[2].IsNull() { + if !row[3].IsNull() { v, err := sqltypes.ToInt64(row[2]) if err != nil { return nil, err @@ -840,7 +847,7 @@ func (mm *messageManager) receiverCount() int { } func (mm *messageManager) readPending(ctx context.Context, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { - query, err := mm.readByTimeNext.GenerateQuery(bindVars, nil) + query, err := mm.readByPriorityAndTimeNext.GenerateQuery(bindVars, nil) if err != nil { tabletenv.InternalErrors.Add("Messages", 1) log.Errorf("Error reading rows from message table: %v", err) diff --git a/go/vt/vttablet/tabletserver/messager/message_manager_test.go b/go/vt/vttablet/tabletserver/messager/message_manager_test.go index 51f29bfe967..92ca5bf147b 100644 --- a/go/vt/vttablet/tabletserver/messager/message_manager_test.go +++ b/go/vt/vttablet/tabletserver/messager/message_manager_test.go @@ -73,6 +73,7 @@ func newMMTable() *schema.Table { func newMMRow(id int64) *querypb.Row { return sqltypes.RowToProto3([]sqltypes.Value{ + sqltypes.NewInt64(1), sqltypes.NewInt64(1), sqltypes.NewInt64(0), sqltypes.NULL, diff --git a/go/vt/vttablet/tabletserver/query_executor_test.go b/go/vt/vttablet/tabletserver/query_executor_test.go index b4ae36ad2f5..15cda647e35 100644 --- a/go/vt/vttablet/tabletserver/query_executor_test.go +++ b/go/vt/vttablet/tabletserver/query_executor_test.go @@ -1234,6 +1234,9 @@ func getQueryExecutorSupportedQueries(testTableHasMultipleUniqueKeys bool) map[s Fields: []*querypb.Field{{ Name: "id", Type: sqltypes.Int64, + }, { + Name: "priority", + Type: sqltypes.Int64, }, { Name: "time_next", Type: sqltypes.Int64, diff --git a/go/vt/vttablet/tabletserver/schema/engine_test.go b/go/vt/vttablet/tabletserver/schema/engine_test.go index 75efd20e3ca..2af7a9dc2a3 100644 --- a/go/vt/vttablet/tabletserver/schema/engine_test.go +++ b/go/vt/vttablet/tabletserver/schema/engine_test.go @@ -375,6 +375,9 @@ func initialSchema() map[string]*Table { Fields: []*querypb.Field{{ Name: "id", Type: sqltypes.Int64, + }, { + Name: "priority", + Type: sqltypes.Int64, }, { Name: "time_next", Type: sqltypes.Int64, diff --git a/go/vt/vttablet/tabletserver/schema/load_table.go b/go/vt/vttablet/tabletserver/schema/load_table.go index 45466d00a76..e46868e115c 100644 --- a/go/vt/vttablet/tabletserver/schema/load_table.go +++ b/go/vt/vttablet/tabletserver/schema/load_table.go @@ -58,6 +58,7 @@ func fetchColumns(ta *Table, conn *connpool.DBConn, sqlTableName string) error { func loadMessageInfo(ta *Table, comment string) error { hiddenCols := map[string]struct{}{ + "priority": {}, "time_next": {}, "epoch": {}, "time_acked": {}, @@ -65,6 +66,7 @@ func loadMessageInfo(ta *Table, comment string) error { requiredCols := []string{ "id", + "priority", "time_next", "epoch", "time_acked", diff --git a/go/vt/vttablet/tabletserver/schema/load_table_test.go b/go/vt/vttablet/tabletserver/schema/load_table_test.go index aae90320fac..02c9af95e18 100644 --- a/go/vt/vttablet/tabletserver/schema/load_table_test.go +++ b/go/vt/vttablet/tabletserver/schema/load_table_test.go @@ -97,6 +97,9 @@ func TestLoadTableMessage(t *testing.T) { Fields: []*querypb.Field{{ Name: "id", Type: sqltypes.Int64, + }, { + Name: "priority", + Type: sqltypes.Int64, }, { Name: "time_next", Type: sqltypes.Int64, @@ -178,13 +181,14 @@ func getTestLoadTableQueries() map[string]*sqltypes.Result { } func getMessageTableQueries() map[string]*sqltypes.Result { - // id is intentionally after the message column to ensure that the - // loader still makes it the first one. return map[string]*sqltypes.Result{ "select * from test_table where 1 != 1": { Fields: []*querypb.Field{{ Name: "id", Type: sqltypes.Int64, + }, { + Name: "priority", + Type: sqltypes.Int64, }, { Name: "time_next", Type: sqltypes.Int64, diff --git a/go/vt/vttablet/tabletserver/schema/schematest/schematest.go b/go/vt/vttablet/tabletserver/schema/schematest/schematest.go index d0458e25eee..64395121714 100644 --- a/go/vt/vttablet/tabletserver/schema/schematest/schematest.go +++ b/go/vt/vttablet/tabletserver/schema/schematest/schematest.go @@ -118,6 +118,9 @@ func Queries() map[string]*sqltypes.Result { Fields: []*querypb.Field{{ Name: "id", Type: sqltypes.Int64, + }, { + Name: "priority", + Type: sqltypes.Int64, }, { Name: "time_next", Type: sqltypes.Int64, From 64485a31b388cbbdf3abc533e281dae8828906c5 Mon Sep 17 00:00:00 2001 From: Derek Perkins Date: Wed, 25 Mar 2020 01:10:42 -0600 Subject: [PATCH 41/85] messaging: fix tests Signed-off-by: Derek Perkins --- go/vt/vttablet/tabletserver/messager/cache_test.go | 5 +++++ go/vt/vttablet/tabletserver/messager/message_manager_test.go | 1 + 2 files changed, 6 insertions(+) diff --git a/go/vt/vttablet/tabletserver/messager/cache_test.go b/go/vt/vttablet/tabletserver/messager/cache_test.go index 41277d80f23..a4dd23ab732 100644 --- a/go/vt/vttablet/tabletserver/messager/cache_test.go +++ b/go/vt/vttablet/tabletserver/messager/cache_test.go @@ -26,6 +26,7 @@ import ( func TestMessagerCacheOrder(t *testing.T) { mc := newCache(10) if !mc.Add(&MessageRow{ + Priority: 1, TimeNext: 1, Epoch: 0, Row: []sqltypes.Value{sqltypes.NewVarBinary("row01")}, @@ -33,6 +34,7 @@ func TestMessagerCacheOrder(t *testing.T) { t.Fatal("Add returned false") } if !mc.Add(&MessageRow{ + Priority: 1, TimeNext: 2, Epoch: 0, Row: []sqltypes.Value{sqltypes.NewVarBinary("row02")}, @@ -40,6 +42,7 @@ func TestMessagerCacheOrder(t *testing.T) { t.Fatal("Add returned false") } if !mc.Add(&MessageRow{ + Priority: 2, TimeNext: 2, Epoch: 1, Row: []sqltypes.Value{sqltypes.NewVarBinary("row12")}, @@ -47,6 +50,7 @@ func TestMessagerCacheOrder(t *testing.T) { t.Fatal("Add returned false") } if !mc.Add(&MessageRow{ + Priority: 2, TimeNext: 1, Epoch: 1, Row: []sqltypes.Value{sqltypes.NewVarBinary("row11")}, @@ -54,6 +58,7 @@ func TestMessagerCacheOrder(t *testing.T) { t.Fatal("Add returned false") } if !mc.Add(&MessageRow{ + Priority: 1, TimeNext: 3, Epoch: 0, Row: []sqltypes.Value{sqltypes.NewVarBinary("row03")}, diff --git a/go/vt/vttablet/tabletserver/messager/message_manager_test.go b/go/vt/vttablet/tabletserver/messager/message_manager_test.go index 92ca5bf147b..1cee1b698f5 100644 --- a/go/vt/vttablet/tabletserver/messager/message_manager_test.go +++ b/go/vt/vttablet/tabletserver/messager/message_manager_test.go @@ -52,6 +52,7 @@ var ( {Type: sqltypes.Int64}, {Type: sqltypes.Int64}, {Type: sqltypes.Int64}, + {Type: sqltypes.Int64}, {Type: sqltypes.VarBinary}, } ) From 5753a96734acbd68dbfe1023bc4c543bcb5c15f6 Mon Sep 17 00:00:00 2001 From: Derek Perkins Date: Wed, 25 Mar 2020 08:11:03 -0600 Subject: [PATCH 42/85] messaging: more test fixes Signed-off-by: Derek Perkins --- .../tabletserver/planbuilder/testdata/schema_test.json | 7 ++----- go/vt/vttablet/tabletserver/tabletserver_flaky_test.go | 3 +++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go/vt/vttablet/tabletserver/planbuilder/testdata/schema_test.json b/go/vt/vttablet/tabletserver/planbuilder/testdata/schema_test.json index 1a23e93eea4..00243d4b08b 100644 --- a/go/vt/vttablet/tabletserver/planbuilder/testdata/schema_test.json +++ b/go/vt/vttablet/tabletserver/planbuilder/testdata/schema_test.json @@ -313,10 +313,10 @@ "Name": "msg", "Columns": [ { - "Name": "time_scheduled" + "Name": "id" }, { - "Name": "id" + "Name": "priority" }, { "Name": "time_next" @@ -324,9 +324,6 @@ { "Name": "epoch" }, - { - "Name": "time_created" - }, { "Name": "time_acked" }, diff --git a/go/vt/vttablet/tabletserver/tabletserver_flaky_test.go b/go/vt/vttablet/tabletserver/tabletserver_flaky_test.go index 68b7e34cfee..193684f4fe4 100644 --- a/go/vt/vttablet/tabletserver/tabletserver_flaky_test.go +++ b/go/vt/vttablet/tabletserver/tabletserver_flaky_test.go @@ -2591,6 +2591,9 @@ func getSupportedQueries() map[string]*sqltypes.Result { Fields: []*querypb.Field{{ Name: "id", Type: sqltypes.Int64, + }, { + Name: "priority", + Type: sqltypes.Int64, }, { Name: "time_next", Type: sqltypes.Int64, From 98117d50fe0a6edcac0c35778edd01b5aac501ec Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Wed, 25 Mar 2020 15:36:00 +0100 Subject: [PATCH 43/85] Renamed resharding workflows Signed-off-by: Rohit Nayak --- .../vreplication/vreplication_test.go | 102 ++++++------ go/vt/vtctl/vtctl.go | 24 +-- go/vt/wrangler/materializer.go | 4 +- go/vt/wrangler/materializer_test.go | 4 +- go/vt/wrangler/stream_migrater_test.go | 92 +++++------ .../{migrater.go => traffic_switcher.go} | 20 +-- ...v_test.go => traffic_switcher_env_test.go} | 0 ...rater_test.go => traffic_switcher_test.go} | 146 +++++++++--------- 8 files changed, 196 insertions(+), 196 deletions(-) rename go/vt/wrangler/{migrater.go => traffic_switcher.go} (97%) rename go/vt/wrangler/{migrater_env_test.go => traffic_switcher_env_test.go} (100%) rename go/vt/wrangler/{migrater_test.go => traffic_switcher_test.go} (91%) diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index f3f86d62074..19100ee561d 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -117,20 +117,20 @@ func shardCustomer(t *testing.T, testReverse bool) { t.Fatal(err) } - if err := vc.VtctlClient.ExecuteCommand("Migrate", "-cell="+cell.Name, "-workflow=p2c", + if err := vc.VtctlClient.ExecuteCommand("MoveTables", "-cell="+cell.Name, "-workflow=p2c", "-tablet_types="+"replica,rdonly", "product", "customer", "customer"); err != nil { - t.Fatalf("Migrate command failed with %+v\n", err) + t.Fatalf("MoveTables command failed with %+v\n", err) } customerTab1 := vc.Cells[cell.Name].Keyspaces["customer"].Shards["-80"].Tablets["zone1-200"].Vttablet customerTab2 := vc.Cells[cell.Name].Keyspaces["customer"].Shards["80-"].Tablets["zone1-300"].Vttablet if vc.WaitForVReplicationToCatchup(customerTab1, "p2c", "vt_customer", 1*time.Second) != nil { - t.Fatal("Migrate timed out for customer.p2c -80") + t.Fatal("MoveTables timed out for customer.p2c -80") } if vc.WaitForVReplicationToCatchup(customerTab2, "p2c", "vt_customer", 1*time.Second) != nil { - t.Fatal("Migrate timed out for customer.p2c 80-") + t.Fatal("MoveTables timed out for customer.p2c 80-") } productTab := vc.Cells[cell.Name].Keyspaces["product"].Shards["0"].Tablets["zone1-100"].Vttablet @@ -141,17 +141,17 @@ func shardCustomer(t *testing.T, testReverse bool) { matchInsertQuery1 := "insert into customer(cid, name) values (:vtg1, :vtg2)" assert.True(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "product", insertQuery1, matchInsertQuery1)) vdiff(t, "customer.p2c") - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateReads", "-cells="+cell.Name, "-tablet_type=rdonly", "customer.p2c"); err != nil { - t.Fatalf("MigrateReads error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchReads", "-cells="+cell.Name, "-tablet_type=rdonly", "customer.p2c"); err != nil { + t.Fatalf("SwitchReads error: %s\n", output) } - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateReads", "-cells="+cell.Name, "-tablet_type=replica", "customer.p2c"); err != nil { - t.Fatalf("MigrateReads error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchReads", "-cells="+cell.Name, "-tablet_type=replica", "customer.p2c"); err != nil { + t.Fatalf("SwitchReads error: %s\n", output) } assert.False(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTabReplica, "customer", query, query)) assert.True(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "customer", query, query)) - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateWrites", "customer.p2c"); err != nil { - t.Fatalf("MigrateWrites error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchWrites", "customer.p2c"); err != nil { + t.Fatalf("SwitchWrites error: %s\n", output) } insertQuery2 := "insert into customer(name) values('tempCustomer2')" matchInsertQuery2 := "insert into customer(name, cid) values (:vtg1, :_cid0)" @@ -163,14 +163,14 @@ func shardCustomer(t *testing.T, testReverse bool) { if testReverse { //Reverse Replicate - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateReads", "-cells="+cell.Name, "-tablet_type=rdonly", "product.p2c_reverse"); err != nil { - t.Fatalf("MigrateReads error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchReads", "-cells="+cell.Name, "-tablet_type=rdonly", "product.p2c_reverse"); err != nil { + t.Fatalf("SwitchReads error: %s\n", output) } - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateReads", "-cells="+cell.Name, "-tablet_type=replica", "product.p2c_reverse"); err != nil { - t.Fatalf("MigrateReads error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchReads", "-cells="+cell.Name, "-tablet_type=replica", "product.p2c_reverse"); err != nil { + t.Fatalf("SwitchReads error: %s\n", output) } - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateWrites", "product.p2c_reverse"); err != nil { - t.Fatalf("MigrateWrites error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchWrites", "product.p2c_reverse"); err != nil { + t.Fatalf("SwitchWrites error: %s\n", output) } insertQuery1 = "insert into customer(cid, name) values(1002, 'tempCustomer5')" assert.True(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "product", insertQuery1, matchInsertQuery1)) @@ -180,14 +180,14 @@ func shardCustomer(t *testing.T, testReverse bool) { assert.False(t, validateThatQueryExecutesOnTablet(t, vtgateConn, customerTab2, "customer", insertQuery1, matchInsertQuery1)) //Go forward again - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateReads", "-cells="+cell.Name, "-tablet_type=rdonly", "customer.p2c"); err != nil { - t.Fatalf("MigrateReads error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchReads", "-cells="+cell.Name, "-tablet_type=rdonly", "customer.p2c"); err != nil { + t.Fatalf("SwitchReads error: %s\n", output) } - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateReads", "-cells="+cell.Name, "-tablet_type=replica", "customer.p2c"); err != nil { - t.Fatalf("MigrateReads error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchReads", "-cells="+cell.Name, "-tablet_type=replica", "customer.p2c"); err != nil { + t.Fatalf("SwitchReads error: %s\n", output) } - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateWrites", "customer.p2c"); err != nil { - t.Fatalf("MigrateWrites error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchWrites", "customer.p2c"); err != nil { + t.Fatalf("SwitchWrites error: %s\n", output) } insertQuery2 = "insert into customer(name) values('tempCustomer8')" //ID 103, hence due to reverse_bits in shard 80- assert.False(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "customer", insertQuery2, matchInsertQuery2)) @@ -274,7 +274,7 @@ func reshard(t *testing.T, ksName string, tableName string, workflow string, sou if strings.Index(targetShards, ","+tab.Shard+",") >= 0 { fmt.Printf("Waiting for vrepl to catch up on %s since it IS a target shard\n", tab.Shard) if vc.WaitForVReplicationToCatchup(tab, workflow, "vt_"+ksName, 3*time.Second) != nil { - t.Fatal("Migrate timed out") + t.Fatal("Reshard timed out") } } else { fmt.Printf("Not waiting for vrepl to catch up on %s since it is NOT a target shard\n", tab.Shard) @@ -282,14 +282,14 @@ func reshard(t *testing.T, ksName string, tableName string, workflow string, sou } } vdiff(t, ksWorkflow) - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateReads", "-cells="+cell.Name, "-tablet_type=rdonly", ksWorkflow); err != nil { - t.Fatalf("MigrateReads error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchReads", "-cells="+cell.Name, "-tablet_type=rdonly", ksWorkflow); err != nil { + t.Fatalf("SwitchReads error: %s\n", output) } - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateReads", "-cells="+cell.Name, "-tablet_type=replica", ksWorkflow); err != nil { - t.Fatalf("MigrateReads error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchReads", "-cells="+cell.Name, "-tablet_type=replica", ksWorkflow); err != nil { + t.Fatalf("SwitchReads error: %s\n", output) } - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateWrites", ksWorkflow); err != nil { - t.Fatalf("MigrateWrites error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchWrites", ksWorkflow); err != nil { + t.Fatalf("SwitchWrites error: %s\n", output) } if counts != nil { for tabletName, count := range counts { @@ -305,29 +305,29 @@ func shardOrders(t *testing.T) { if err := vc.VtctlClient.ExecuteCommand("ApplyVSchema", "-vschema", ordersVSchema, "customer"); err != nil { t.Fatal(err) } - if err := vc.VtctlClient.ExecuteCommand("Migrate", "-cell="+cell.Name, "-workflow=o2c", + if err := vc.VtctlClient.ExecuteCommand("MoveTables", "-cell="+cell.Name, "-workflow=o2c", "-tablet_types="+"replica,rdonly", "product", "customer", "orders"); err != nil { t.Fatal(err) } customerTab1 := vc.Cells[cell.Name].Keyspaces["customer"].Shards["-80"].Tablets["zone1-200"].Vttablet customerTab2 := vc.Cells[cell.Name].Keyspaces["customer"].Shards["80-"].Tablets["zone1-300"].Vttablet if vc.WaitForVReplicationToCatchup(customerTab1, "o2c", "vt_customer", 1*time.Second) != nil { - assert.Fail(t, "Migrate timed out for customer.o2c -80") + assert.Fail(t, "MoveTables timed out for customer.o2c -80") } if vc.WaitForVReplicationToCatchup(customerTab2, "o2c", "vt_customer", 1*time.Second) != nil { - assert.Fail(t, "Migrate timed out for customer.o2c 80-") + assert.Fail(t, "MoveTables timed out for customer.o2c 80-") } vdiff(t, "customer.o2c") - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateReads", "-cells="+cell.Name, "-tablet_type=rdonly", "customer.o2c"); err != nil { - t.Fatalf("MigrateReads error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchReads", "-cells="+cell.Name, "-tablet_type=rdonly", "customer.o2c"); err != nil { + t.Fatalf("SwitchReads error: %s\n", output) } - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateReads", "-cells="+cell.Name, "-tablet_type=replica", "customer.o2c"); err != nil { - t.Fatalf("MigrateReads error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchReads", "-cells="+cell.Name, "-tablet_type=replica", "customer.o2c"); err != nil { + t.Fatalf("SwitchReads error: %s\n", output) } - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateWrites", "customer.o2c"); err != nil { - t.Fatalf("MigrateWrites error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchWrites", "customer.o2c"); err != nil { + t.Fatalf("SwitchWrites error: %s\n", output) } assert.Empty(t, validateCountInTablet(t, customerTab1, "customer", "orders", 1)) @@ -346,7 +346,7 @@ func shardMerchant(t *testing.T) { if err := vtgate.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.master", "merchant", "80-"), 1); err != nil { t.Fatal(err) } - if err := vc.VtctlClient.ExecuteCommand("Migrate", "-cell="+cell.Name, "-workflow=p2m", + if err := vc.VtctlClient.ExecuteCommand("MoveTables", "-cell="+cell.Name, "-workflow=p2m", "-tablet_types="+"replica,rdonly", "product", "merchant", "merchant"); err != nil { t.Fatal(err) } @@ -354,22 +354,22 @@ func shardMerchant(t *testing.T) { merchantTab1 := vc.Cells[cell.Name].Keyspaces["merchant"].Shards["-80"].Tablets["zone1-400"].Vttablet merchantTab2 := vc.Cells[cell.Name].Keyspaces["merchant"].Shards["80-"].Tablets["zone1-500"].Vttablet if vc.WaitForVReplicationToCatchup(merchantTab1, "p2m", "vt_merchant", 1*time.Second) != nil { - t.Fatal("Migrate timed out for merchant.p2m -80") + t.Fatal("MoveTables timed out for merchant.p2m -80") } if vc.WaitForVReplicationToCatchup(merchantTab2, "p2m", "vt_merchant", 1*time.Second) != nil { - t.Fatal("Migrate timed out for merchant.p2m 80-") + t.Fatal("MoveTables timed out for merchant.p2m 80-") } vdiff(t, "merchant.p2m") - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateReads", "-cells="+cell.Name, "-tablet_type=rdonly", "merchant.p2m"); err != nil { - t.Fatalf("MigrateReads error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchReads", "-cells="+cell.Name, "-tablet_type=rdonly", "merchant.p2m"); err != nil { + t.Fatalf("SwitchReads error: %s\n", output) } - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateReads", "-cells="+cell.Name, "-tablet_type=replica", "merchant.p2m"); err != nil { - t.Fatalf("MigrateReads error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchReads", "-cells="+cell.Name, "-tablet_type=replica", "merchant.p2m"); err != nil { + t.Fatalf("SwitchReads error: %s\n", output) } - if output, err := vc.VtctlClient.ExecuteCommandWithOutput("MigrateWrites", "merchant.p2m"); err != nil { - t.Fatalf("MigrateWrites error: %s\n", output) + if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchWrites", "merchant.p2m"); err != nil { + t.Fatalf("SwitchWrites error: %s\n", output) } assert.Empty(t, validateCountInTablet(t, merchantTab1, "merchant", "merchant", 1)) @@ -410,7 +410,7 @@ func materializeProduct(t *testing.T) { customerTablets := vc.getVttabletsInKeyspace(t, cell, "customer", "master") for _, tab := range customerTablets { if vc.WaitForVReplicationToCatchup(tab, workflow, "vt_customer", 3*time.Second) != nil { - t.Fatal("Migrate timed out") + t.Fatal("Materialize timed out") } } for _, tab := range customerTablets { @@ -427,7 +427,7 @@ func materializeSales(t *testing.T) { } productTab := vc.Cells[cell.Name].Keyspaces["product"].Shards["0"].Tablets["zone1-100"].Vttablet if vc.WaitForVReplicationToCatchup(productTab, "sales", "vt_product", 3*time.Second) != nil { - assert.Fail(t, "Migrate timed out for product.sales -80") + assert.Fail(t, "Materialize timed out for product.sales -80") } assert.Empty(t, validateCount(t, vtgateConn, "product", "sales", 2)) @@ -444,7 +444,7 @@ func materializeMerchantSales(t *testing.T) { merchantTablets := vc.getVttabletsInKeyspace(t, cell, "merchant", "master") for _, tab := range merchantTablets { if vc.WaitForVReplicationToCatchup(tab, workflow, "vt_merchant", 1*time.Second) != nil { - t.Fatal("Migrate timed out") + t.Fatal("Materialize timed out") } } assert.Empty(t, validateCountInTablet(t, merchantTablets["zone1-400"], "merchant", "msales", 1)) @@ -465,7 +465,7 @@ func materializeMerchantOrders(t *testing.T) { merchantTablets := vc.getVttabletsInKeyspace(t, cell, "merchant", "master") for _, tab := range merchantTablets { if vc.WaitForVReplicationToCatchup(tab, workflow, "vt_merchant", 1*time.Second) != nil { - t.Fatal("Migrate timed out") + t.Fatal("Materialize timed out") } } assert.Empty(t, validateCountInTablet(t, merchantTablets["zone1-400"], "merchant", "morders", 2)) diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 40b74e22f71..3f625419057 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -311,9 +311,9 @@ var commands = []commandGroup{ {"Reshard", commandReshard, "[-skip_schema_copy] ", "Start a Resharding process. Example: Reshard ks.workflow001 '0' '-80,80-'"}, - {"Migrate", commandMigrate, + {"MoveTables", commandMoveTables, "[-cell=] [-tablet_types=] -workflow= ", - `Start a table(s) migration, table_specs is a list of tables or the tables section of the vschema for the target keyspace. Example: '{"t1":{"column_vindexes": [{""column": "id1", "name": "hash"}]}, "t2":{"column_vindexes": [{""column": "id2", "name": "hash"}]}}`}, + `Move table(s) to another keyspace, table_specs is a list of tables or the tables section of the vschema for the target keyspace. Example: '{"t1":{"column_vindexes": [{""column": "id1", "name": "hash"}]}, "t2":{"column_vindexes": [{""column": "id2", "name": "hash"}]}}`}, {"CreateLookupVindex", commandCreateLookupVindex, "[-cell=] [-tablet_types=] ", `Create and backfill a lookup vindex. the json_spec must contain the vindex and colvindex specs for the new lookup.`}, @@ -338,12 +338,12 @@ var commands = []commandGroup{ {"MigrateServedFrom", commandMigrateServedFrom, "[-cells=c1,c2,...] [-reverse] ", "Makes the serve the given type. This command also rebuilds the serving graph."}, - {"MigrateReads", commandMigrateReads, + {"SwitchReads", commandSwitchReads, "[-cells=c1,c2,...] [-reverse] -tablet_type={replica|rdonly} ", - "Migrate read traffic for the specified workflow."}, - {"MigrateWrites", commandMigrateWrites, + "Switch read traffic for the specified workflow."}, + {"SwitchWrites", commandSwitchWrites, "[-filtered_replication_wait_time=30s] [-cancel] [-reverse_replication=false] ", - "Migrate write traffic for the specified workflow."}, + "Switch write traffic for the specified workflow."}, {"CancelResharding", commandCancelResharding, "", "Permanently cancels a resharding in progress. All resharding related metadata will be deleted."}, @@ -1817,7 +1817,7 @@ func commandReshard(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.F return wr.Reshard(ctx, keyspace, workflow, source, target, *skipSchemaCopy) } -func commandMigrate(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { +func commandMoveTables(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { workflow := subFlags.String("workflow", "", "Workflow name. Will be used to later migrate traffic.") cell := subFlags.String("cell", "", "Cell to replicate from.") tabletTypes := subFlags.String("tablet_types", "", "Source tablet types to replicate from.") @@ -1833,7 +1833,7 @@ func commandMigrate(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.F source := subFlags.Arg(0) target := subFlags.Arg(1) tableSpecs := subFlags.Arg(2) - return wr.Migrate(ctx, *workflow, source, target, tableSpecs, *cell, *tabletTypes) + return wr.MoveTables(ctx, *workflow, source, target, tableSpecs, *cell, *tabletTypes) } func commandCreateLookupVindex(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -1991,7 +1991,7 @@ func commandMigrateServedFrom(ctx context.Context, wr *wrangler.Wrangler, subFla return wr.MigrateServedFrom(ctx, keyspace, shard, servedType, cells, *reverse, *filteredReplicationWaitTime) } -func commandMigrateReads(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { +func commandSwitchReads(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { reverse := subFlags.Bool("reverse", false, "Moves the served tablet type backward instead of forward.") cellsStr := subFlags.String("cells", "", "Specifies a comma-separated list of cells to update") tabletType := subFlags.String("tablet_type", "", "Tablet type (replica or rdonly)") @@ -2022,10 +2022,10 @@ func commandMigrateReads(ctx context.Context, wr *wrangler.Wrangler, subFlags *f return err } - return wr.MigrateReads(ctx, keyspace, workflow, servedType, cells, direction) + return wr.SwitchReads(ctx, keyspace, workflow, servedType, cells, direction) } -func commandMigrateWrites(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { +func commandSwitchWrites(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { filteredReplicationWaitTime := subFlags.Duration("filtered_replication_wait_time", 30*time.Second, "Specifies the maximum time to wait, in seconds, for filtered replication to catch up on master migrations. The migration will be aborted on timeout.") reverseReplication := subFlags.Bool("reverse_replication", true, "Also reverse the replication") cancelMigrate := subFlags.Bool("cancel", false, "Cancel the failed migration and serve from source") @@ -2041,7 +2041,7 @@ func commandMigrateWrites(ctx context.Context, wr *wrangler.Wrangler, subFlags * return err } - journalID, err := wr.MigrateWrites(ctx, keyspace, workflow, *filteredReplicationWaitTime, *cancelMigrate, *reverseReplication) + journalID, err := wr.SwitchWrites(ctx, keyspace, workflow, *filteredReplicationWaitTime, *cancelMigrate, *reverseReplication) if err != nil { return err } diff --git a/go/vt/wrangler/materializer.go b/go/vt/wrangler/materializer.go index 3d455e0e283..43213bd7e4d 100644 --- a/go/vt/wrangler/materializer.go +++ b/go/vt/wrangler/materializer.go @@ -48,8 +48,8 @@ type materializer struct { targetShards []*topo.ShardInfo } -// Migrate initiates a table migration. -func (wr *Wrangler) Migrate(ctx context.Context, workflow, sourceKeyspace, targetKeyspace, tableSpecs, cell, tabletTypes string) error { +// MoveTables initiates moving table(s) over to another keyspace +func (wr *Wrangler) MoveTables(ctx context.Context, workflow, sourceKeyspace, targetKeyspace, tableSpecs, cell, tabletTypes string) error { var tables []string var vschema *vschemapb.Keyspace if strings.HasPrefix(tableSpecs, "{") { diff --git a/go/vt/wrangler/materializer_test.go b/go/vt/wrangler/materializer_test.go index f3587020b29..85370056e31 100644 --- a/go/vt/wrangler/materializer_test.go +++ b/go/vt/wrangler/materializer_test.go @@ -54,7 +54,7 @@ func TestMigrateTables(t *testing.T) { env.tmc.expectVRQuery(200, mzUpdateQuery, &sqltypes.Result{}) ctx := context.Background() - err := env.wr.Migrate(ctx, "workflow", "sourceks", "targetks", "t1", "", "") + err := env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", "t1", "", "") assert.NoError(t, err) vschema, err := env.wr.ts.GetSrvVSchema(ctx, env.cell) assert.NoError(t, err) @@ -86,7 +86,7 @@ func TestMigrateVSchema(t *testing.T) { env.tmc.expectVRQuery(200, mzUpdateQuery, &sqltypes.Result{}) ctx := context.Background() - err := env.wr.Migrate(ctx, "workflow", "sourceks", "targetks", `{"t1":{}}`, "", "") + err := env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", `{"t1":{}}`, "", "") assert.NoError(t, err) vschema, err := env.wr.ts.GetSrvVSchema(ctx, env.cell) assert.NoError(t, err) diff --git a/go/vt/wrangler/stream_migrater_test.go b/go/vt/wrangler/stream_migrater_test.go index 3c6ef0b1d8c..2d6a729d251 100644 --- a/go/vt/wrangler/stream_migrater_test.go +++ b/go/vt/wrangler/stream_migrater_test.go @@ -40,11 +40,11 @@ func TestStreamMigrateMainflow(t *testing.T) { defer tme.stopTablets(t) // Migrate reads - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -163,7 +163,7 @@ func TestStreamMigrateMainflow(t *testing.T) { tme.expectStartReverseVReplication() tme.expectDeleteTargetVReplication() - if _, err := tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true); err != nil { + if _, err := tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true); err != nil { t.Fatal(err) } @@ -186,11 +186,11 @@ func TestStreamMigrateTwoStreams(t *testing.T) { defer tme.stopTablets(t) // Migrate reads - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -330,7 +330,7 @@ func TestStreamMigrateTwoStreams(t *testing.T) { tme.expectStartReverseVReplication() tme.expectDeleteTargetVReplication() - if _, err := tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true); err != nil { + if _, err := tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true); err != nil { t.Fatal(err) } @@ -353,11 +353,11 @@ func TestStreamMigrateOneToMany(t *testing.T) { defer tme.stopTablets(t) // Migrate reads - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -462,7 +462,7 @@ func TestStreamMigrateOneToMany(t *testing.T) { tme.expectStartReverseVReplication() tme.expectDeleteTargetVReplication() - if _, err := tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true); err != nil { + if _, err := tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true); err != nil { t.Fatal(err) } @@ -484,11 +484,11 @@ func TestStreamMigrateManyToOne(t *testing.T) { defer tme.stopTablets(t) // Migrate reads - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -596,7 +596,7 @@ func TestStreamMigrateManyToOne(t *testing.T) { tme.expectStartReverseVReplication() tme.expectDeleteTargetVReplication() - if _, err := tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true); err != nil { + if _, err := tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true); err != nil { t.Fatal(err) } @@ -617,11 +617,11 @@ func TestStreamMigrateSyncSuccess(t *testing.T) { defer tme.stopTablets(t) // Migrate reads - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -784,7 +784,7 @@ func TestStreamMigrateSyncSuccess(t *testing.T) { tme.expectStartReverseVReplication() tme.expectDeleteTargetVReplication() - if _, err := tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true); err != nil { + if _, err := tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true); err != nil { t.Fatal(err) } @@ -807,11 +807,11 @@ func TestStreamMigrateSyncFail(t *testing.T) { defer tme.stopTablets(t) // Migrate reads - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -913,10 +913,10 @@ func TestStreamMigrateSyncFail(t *testing.T) { tme.expectCancelMigration() - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) want := "does not match" if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MigrateWrites err: %v, want %s", err, want) + t.Errorf("SwitchWrites err: %v, want %s", err, want) } verifyQueries(t, tme.allDBClients) } @@ -927,11 +927,11 @@ func TestStreamMigrateCancel(t *testing.T) { defer tme.stopTablets(t) // Migrate reads - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -1005,10 +1005,10 @@ func TestStreamMigrateCancel(t *testing.T) { } cancelMigration() - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) want := "intentionally failed" if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MigrateWrites err: %v, want %s", err, want) + t.Errorf("SwitchWrites err: %v, want %s", err, want) } checkServedTypes(t, tme.ts, "ks:-40", 1) @@ -1030,11 +1030,11 @@ func TestStreamMigrateStoppedStreams(t *testing.T) { defer tme.stopTablets(t) // Migrate reads - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -1073,10 +1073,10 @@ func TestStreamMigrateStoppedStreams(t *testing.T) { } stopStreams() - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) want := "cannot migrate until all streams are running: 0" if err == nil || err.Error() != want { - t.Errorf("MigrateWrites err: %v, want %v", err, want) + t.Errorf("SwitchWrites err: %v, want %v", err, want) } verifyQueries(t, tme.allDBClients) } @@ -1087,11 +1087,11 @@ func TestStreamMigrateCancelWithStoppedStreams(t *testing.T) { defer tme.stopTablets(t) // Migrate reads - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -1139,7 +1139,7 @@ func TestStreamMigrateCancelWithStoppedStreams(t *testing.T) { tme.expectCancelMigration() - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, true, false) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, true, false) if err != nil { t.Fatal(err) } @@ -1152,11 +1152,11 @@ func TestStreamMigrateStillCopying(t *testing.T) { defer tme.stopTablets(t) // Migrate reads - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -1199,10 +1199,10 @@ func TestStreamMigrateStillCopying(t *testing.T) { } stopStreams() - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) want := "cannot migrate while vreplication streams in source shards are still copying: 0" if err == nil || err.Error() != want { - t.Errorf("MigrateWrites err: %v, want %v", err, want) + t.Errorf("SwitchWrites err: %v, want %v", err, want) } verifyQueries(t, tme.allDBClients) } @@ -1213,11 +1213,11 @@ func TestStreamMigrateEmptyWorflow(t *testing.T) { defer tme.stopTablets(t) // Migrate reads - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -1259,10 +1259,10 @@ func TestStreamMigrateEmptyWorflow(t *testing.T) { } stopStreams() - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) want := "VReplication streams must have named workflows for migration: shard: ks:0, stream: 1" if err == nil || err.Error() != want { - t.Errorf("MigrateWrites err: %v, want %v", err, want) + t.Errorf("SwitchWrites err: %v, want %v", err, want) } verifyQueries(t, tme.allDBClients) } @@ -1273,11 +1273,11 @@ func TestStreamMigrateDupWorflow(t *testing.T) { defer tme.stopTablets(t) // Migrate reads - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -1319,10 +1319,10 @@ func TestStreamMigrateDupWorflow(t *testing.T) { } stopStreams() - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) want := "VReplication stream has the same workflow name as the resharding workflow: shard: ks:0, stream: 1" if err == nil || err.Error() != want { - t.Errorf("MigrateWrites err: %v, want %v", err, want) + t.Errorf("SwitchWrites err: %v, want %v", err, want) } verifyQueries(t, tme.allDBClients) } @@ -1334,11 +1334,11 @@ func TestStreamMigrateStreamsMismatch(t *testing.T) { defer tme.stopTablets(t) // Migrate reads - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -1390,10 +1390,10 @@ func TestStreamMigrateStreamsMismatch(t *testing.T) { } stopStreams() - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) want := "streams are mismatched across source shards" if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MigrateWrites err: %v, must contain %v", err, want) + t.Errorf("SwitchWrites err: %v, must contain %v", err, want) } verifyQueries(t, tme.allDBClients) } diff --git a/go/vt/wrangler/migrater.go b/go/vt/wrangler/traffic_switcher.go similarity index 97% rename from go/vt/wrangler/migrater.go rename to go/vt/wrangler/traffic_switcher.go index e87552d2cb5..90089b3287a 100644 --- a/go/vt/wrangler/migrater.go +++ b/go/vt/wrangler/traffic_switcher.go @@ -100,8 +100,8 @@ type miSource struct { journaled bool } -// MigrateReads is a generic way of migrating read traffic for a resharding workflow. -func (wr *Wrangler) MigrateReads(ctx context.Context, targetKeyspace, workflow string, servedType topodatapb.TabletType, cells []string, direction MigrateDirection) error { +// SwitchReads is a generic way of migrating read traffic for a resharding workflow. +func (wr *Wrangler) SwitchReads(ctx context.Context, targetKeyspace, workflow string, servedType topodatapb.TabletType, cells []string, direction MigrateDirection) error { if servedType != topodatapb.TabletType_REPLICA && servedType != topodatapb.TabletType_RDONLY { return fmt.Errorf("tablet type must be REPLICA or RDONLY: %v", servedType) } @@ -111,7 +111,7 @@ func (wr *Wrangler) MigrateReads(ctx context.Context, targetKeyspace, workflow s return err } if mi.frozen { - return fmt.Errorf("cannot migrate reads while MigrateWrites is in progress") + return fmt.Errorf("cannot migrate reads while SwitchWrites is in progress") } if err := mi.validate(ctx, false /* isWrite */); err != nil { mi.wr.Logger().Errorf("validate failed: %v", err) @@ -119,7 +119,7 @@ func (wr *Wrangler) MigrateReads(ctx context.Context, targetKeyspace, workflow s } // For reads, locking the source keyspace is sufficient. - ctx, unlock, lockErr := wr.ts.LockKeyspace(ctx, mi.sourceKeyspace, "MigrateReads") + ctx, unlock, lockErr := wr.ts.LockKeyspace(ctx, mi.sourceKeyspace, "SwitchReads") if lockErr != nil { mi.wr.Logger().Errorf("LockKeyspace failed: %v", lockErr) return lockErr @@ -140,8 +140,8 @@ func (wr *Wrangler) MigrateReads(ctx context.Context, targetKeyspace, workflow s return nil } -// MigrateWrites is a generic way of migrating write traffic for a resharding workflow. -func (wr *Wrangler) MigrateWrites(ctx context.Context, targetKeyspace, workflow string, filteredReplicationWaitTime time.Duration, cancelMigrate, reverseReplication bool) (journalID int64, err error) { +// SwitchWrites is a generic way of migrating write traffic for a resharding workflow. +func (wr *Wrangler) SwitchWrites(ctx context.Context, targetKeyspace, workflow string, filteredReplicationWaitTime time.Duration, cancelMigrate, reverseReplication bool) (journalID int64, err error) { mi, err := wr.buildMigrater(ctx, targetKeyspace, workflow) if err != nil { wr.Logger().Errorf("buildMigrater failed: %v", err) @@ -163,14 +163,14 @@ func (wr *Wrangler) MigrateWrites(ctx context.Context, targetKeyspace, workflow } // Need to lock both source and target keyspaces. - ctx, sourceUnlock, lockErr := wr.ts.LockKeyspace(ctx, mi.sourceKeyspace, "MigrateWrites") + ctx, sourceUnlock, lockErr := wr.ts.LockKeyspace(ctx, mi.sourceKeyspace, "SwitchWrites") if lockErr != nil { mi.wr.Logger().Errorf("LockKeyspace failed: %v", lockErr) return 0, lockErr } defer sourceUnlock(&err) if mi.targetKeyspace != mi.sourceKeyspace { - tctx, targetUnlock, lockErr := wr.ts.LockKeyspace(ctx, mi.targetKeyspace, "MigrateWrites") + tctx, targetUnlock, lockErr := wr.ts.LockKeyspace(ctx, mi.targetKeyspace, "SwitchWrites") if lockErr != nil { mi.wr.Logger().Errorf("LockKeyspace failed: %v", lockErr) return 0, lockErr @@ -854,7 +854,7 @@ func (mi *migrater) changeTableRouting(ctx context.Context) error { // We assume that the following rules were setup when the targets were created: // table -> sourceKeyspace.table // targetKeyspace.table -> sourceKeyspace.table - // Additionally, MigrateReads would have added rules like this: + // Additionally, SwitchReads would have added rules like this: // table@replica -> targetKeyspace.table // targetKeyspace.table@replica -> targetKeyspace.table // After this step, only the following rules will be left: @@ -913,7 +913,7 @@ func (mi *migrater) startReverseVReplication(ctx context.Context) error { } func (mi *migrater) deleteTargetVReplication(ctx context.Context) error { - // Mark target streams as frozen before deleting. If MigrateWrites gets + // Mark target streams as frozen before deleting. If SwitchWrites gets // re-invoked after a freeze, it will skip all the previous steps and // jump directly here for the final cleanup. err := mi.forAllTargets(func(target *miTarget) error { diff --git a/go/vt/wrangler/migrater_env_test.go b/go/vt/wrangler/traffic_switcher_env_test.go similarity index 100% rename from go/vt/wrangler/migrater_env_test.go rename to go/vt/wrangler/traffic_switcher_env_test.go diff --git a/go/vt/wrangler/migrater_test.go b/go/vt/wrangler/traffic_switcher_test.go similarity index 91% rename from go/vt/wrangler/migrater_test.go rename to go/vt/wrangler/traffic_switcher_test.go index 042d5ad09f2..dbe56392b0e 100644 --- a/go/vt/wrangler/migrater_test.go +++ b/go/vt/wrangler/traffic_switcher_test.go @@ -77,7 +77,7 @@ func TestTableMigrateMainflow(t *testing.T) { //------------------------------------------------------------------------------------------------------------------- // Single cell RDONLY migration. - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, []string{"cell1"}, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, []string{"cell1"}, DirectionForward) if err != nil { t.Fatal(err) } @@ -107,7 +107,7 @@ func TestTableMigrateMainflow(t *testing.T) { // So, adding routes for replica and deploying to cell2 will also cause // cell2 to migrate rdonly. This is a quirk that can be fixed later if necessary. // TODO(sougou): check if it's worth fixing, or clearly document the quirk. - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, []string{"cell2"}, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, []string{"cell2"}, DirectionForward) if err != nil { t.Fatal(err) } @@ -145,7 +145,7 @@ func TestTableMigrateMainflow(t *testing.T) { //------------------------------------------------------------------------------------------------------------------- // Single cell backward REPLICA migration. - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, []string{"cell2"}, DirectionBackward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, []string{"cell2"}, DirectionBackward) if err != nil { t.Fatal(err) } @@ -165,7 +165,7 @@ func TestTableMigrateMainflow(t *testing.T) { //------------------------------------------------------------------------------------------------------------------- // Migrate all REPLICA. - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -191,7 +191,7 @@ func TestTableMigrateMainflow(t *testing.T) { //------------------------------------------------------------------------------------------------------------------- // All cells RDONLY backward migration. - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionBackward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionBackward) if err != nil { t.Fatal(err) } @@ -211,7 +211,7 @@ func TestTableMigrateMainflow(t *testing.T) { //------------------------------------------------------------------------------------------------------------------- // All cells RDONLY backward migration. - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionBackward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionBackward) if err != nil { t.Fatal(err) } @@ -224,32 +224,32 @@ func TestTableMigrateMainflow(t *testing.T) { verifyQueries(t, tme.allDBClients) //------------------------------------------------------------------------------------------------------------------- - // Can't migrate master with MigrateReads. - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_MASTER, nil, DirectionForward) + // Can't migrate master with SwitchReads. + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_MASTER, nil, DirectionForward) want := "tablet type must be REPLICA or RDONLY: MASTER" if err == nil || err.Error() != want { - t.Errorf("MigrateReads(master) err: %v, want %v", err, want) + t.Errorf("SwitchReads(master) err: %v, want %v", err, want) } verifyQueries(t, tme.allDBClients) //------------------------------------------------------------------------------------------------------------------- // Can't migrate writes if REPLICA and RDONLY have not fully migrated yet. - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) want = "missing tablet type specific routing, read-only traffic must be migrated before migrating writes" if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MigrateWrites err: %v, want %v", err, want) + t.Errorf("SwitchWrites err: %v, want %v", err, want) } verifyQueries(t, tme.allDBClients) //------------------------------------------------------------------------------------------------------------------- - // Test MigrateWrites cancelation on failure. + // Test SwitchWrites cancelation on failure. // Migrate all the reads first. - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -300,10 +300,10 @@ func TestTableMigrateMainflow(t *testing.T) { } cancelMigration() - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 0*time.Second, false, true) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 0*time.Second, false, true) want = "DeadlineExceeded" if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MigrateWrites(0 timeout) err: %v, must contain %v", err, want) + t.Errorf("SwitchWrites(0 timeout) err: %v, must contain %v", err, want) } verifyQueries(t, tme.allDBClients) checkRouting(t, tme.wr, map[string][]string{ @@ -330,7 +330,7 @@ func TestTableMigrateMainflow(t *testing.T) { checkBlacklist(t, tme.ts, "ks2:80-", nil) //------------------------------------------------------------------------------------------------------------------- - // Test successful MigrateWrites. + // Test successful SwitchWrites. checkJournals() @@ -415,7 +415,7 @@ func TestTableMigrateMainflow(t *testing.T) { } deleteTargetVReplication() - journalID, err := tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) + journalID, err := tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) if err != nil { t.Fatal(err) } @@ -452,7 +452,7 @@ func TestShardMigrateMainflow(t *testing.T) { //------------------------------------------------------------------------------------------------------------------- // Single cell RDONLY migration. - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, []string{"cell1"}, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, []string{"cell1"}, DirectionForward) if err != nil { t.Fatal(err) } @@ -468,7 +468,7 @@ func TestShardMigrateMainflow(t *testing.T) { //------------------------------------------------------------------------------------------------------------------- // Other cell REPLICA migration. - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, []string{"cell2"}, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, []string{"cell2"}, DirectionForward) if err != nil { t.Fatal(err) } @@ -484,7 +484,7 @@ func TestShardMigrateMainflow(t *testing.T) { //------------------------------------------------------------------------------------------------------------------- // Single cell backward REPLICA migration. - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, []string{"cell2"}, DirectionBackward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, []string{"cell2"}, DirectionBackward) if err != nil { t.Fatal(err) } @@ -503,7 +503,7 @@ func TestShardMigrateMainflow(t *testing.T) { // This is an extra step that does not exist in the tables test. // The per-cell migration mechanism is different for tables. So, this // extra step is needed to bring things in sync. - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -515,7 +515,7 @@ func TestShardMigrateMainflow(t *testing.T) { //------------------------------------------------------------------------------------------------------------------- // Migrate all REPLICA. - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -527,7 +527,7 @@ func TestShardMigrateMainflow(t *testing.T) { //------------------------------------------------------------------------------------------------------------------- // All cells RDONLY backward migration. - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionBackward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionBackward) if err != nil { t.Fatal(err) } @@ -538,28 +538,28 @@ func TestShardMigrateMainflow(t *testing.T) { verifyQueries(t, tme.allDBClients) //------------------------------------------------------------------------------------------------------------------- - // Can't migrate master with MigrateReads. - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_MASTER, nil, DirectionForward) + // Can't migrate master with SwitchReads. + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_MASTER, nil, DirectionForward) want := "tablet type must be REPLICA or RDONLY: MASTER" if err == nil || err.Error() != want { - t.Errorf("MigrateReads(master) err: %v, want %v", err, want) + t.Errorf("SwitchReads(master) err: %v, want %v", err, want) } verifyQueries(t, tme.allDBClients) //------------------------------------------------------------------------------------------------------------------- // Can't migrate writes if REPLICA and RDONLY have not fully migrated yet. - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) want = "cannot migrate MASTER away" if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MigrateWrites err: %v, want %v", err, want) + t.Errorf("SwitchWrites err: %v, want %v", err, want) } verifyQueries(t, tme.allDBClients) //------------------------------------------------------------------------------------------------------------------- - // Test MigrateWrites cancelation on failure. + // Test SwitchWrites cancelation on failure. // Migrate all the reads first. - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -610,10 +610,10 @@ func TestShardMigrateMainflow(t *testing.T) { } cancelMigration() - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 0*time.Second, false, true) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 0*time.Second, false, true) want = "DeadlineExceeded" if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MigrateWrites(0 timeout) err: %v, must contain %v", err, want) + t.Errorf("SwitchWrites(0 timeout) err: %v, must contain %v", err, want) } verifyQueries(t, tme.allDBClients) @@ -628,7 +628,7 @@ func TestShardMigrateMainflow(t *testing.T) { checkIsMasterServing(t, tme.ts, "ks:80-", false) //------------------------------------------------------------------------------------------------------------------- - // Test successful MigrateWrites. + // Test successful SwitchWrites. checkJournals() stopStreams() @@ -707,7 +707,7 @@ func TestShardMigrateMainflow(t *testing.T) { } deleteTargetVReplication() - journalID, err := tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) + journalID, err := tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) if err != nil { t.Fatal(err) } @@ -733,11 +733,11 @@ func TestTableMigrateOneToMany(t *testing.T) { tme := newTestTableMigraterCustom(ctx, t, []string{"0"}, []string{"-80", "80-"}, "select * %s") defer tme.stopTablets(t) - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -804,7 +804,7 @@ func TestTableMigrateOneToMany(t *testing.T) { } deleteTargetVReplication() - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false) if err != nil { t.Fatal(err) } @@ -816,11 +816,11 @@ func TestTableMigrateManyToOne(t *testing.T) { tme := newTestTableMigraterCustom(ctx, t, []string{"-80", "80-"}, []string{"0"}, "select * %s") defer tme.stopTablets(t) - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -889,7 +889,7 @@ func TestTableMigrateManyToOne(t *testing.T) { } deleteTargetVReplication() - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false) if err != nil { t.Fatal(err) } @@ -903,11 +903,11 @@ func TestMigrateFailJournal(t *testing.T) { tme := newTestTableMigrater(ctx, t) defer tme.stopTablets(t) - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -976,10 +976,10 @@ func TestMigrateFailJournal(t *testing.T) { tme.dbSourceClients[0].addQueryRE("insert into _vt.resharding_journal", nil, errors.New("journaling intentionally failed")) tme.dbSourceClients[1].addQueryRE("insert into _vt.resharding_journal", nil, errors.New("journaling intentionally failed")) - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) want := "journaling intentionally failed" if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MigrateWrites(0 timeout) err: %v, must contain %v", err, want) + t.Errorf("SwitchWrites(0 timeout) err: %v, must contain %v", err, want) } // Verify that cancel didn't happen. @@ -999,11 +999,11 @@ func TestTableMigrateJournalExists(t *testing.T) { tme := newTestTableMigrater(ctx, t) defer tme.stopTablets(t) - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -1043,7 +1043,7 @@ func TestTableMigrateJournalExists(t *testing.T) { tme.dbTargetClients[1].addQuery("delete from _vt.vreplication where id in (1, 2)", &sqltypes.Result{}, nil) tme.dbTargetClients[1].addQuery("delete from _vt.copy_state where vrepl_id in (1, 2)", &sqltypes.Result{}, nil) - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) if err != nil { t.Fatal(err) } @@ -1070,11 +1070,11 @@ func TestShardMigrateJournalExists(t *testing.T) { tme := newTestShardMigrater(ctx, t, []string{"-40", "40-"}, []string{"-80", "80-"}) defer tme.stopTablets(t) - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -1113,7 +1113,7 @@ func TestShardMigrateJournalExists(t *testing.T) { tme.dbTargetClients[1].addQuery("delete from _vt.vreplication where id in (2)", &sqltypes.Result{}, nil) tme.dbTargetClients[1].addQuery("delete from _vt.copy_state where vrepl_id in (2)", &sqltypes.Result{}, nil) - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) if err != nil { t.Fatal(err) } @@ -1136,11 +1136,11 @@ func TestTableMigrateCancel(t *testing.T) { tme := newTestTableMigrater(ctx, t) defer tme.stopTablets(t) - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -1173,7 +1173,7 @@ func TestTableMigrateCancel(t *testing.T) { } cancelMigration() - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, true, false) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, true, false) if err != nil { t.Fatal(err) } @@ -1185,11 +1185,11 @@ func TestTableMigrateNoReverse(t *testing.T) { tme := newTestTableMigrater(ctx, t) defer tme.stopTablets(t) - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -1278,7 +1278,7 @@ func TestTableMigrateNoReverse(t *testing.T) { } deleteTargetVReplication() - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false) if err != nil { t.Fatal(err) } @@ -1290,12 +1290,12 @@ func TestMigrateFrozen(t *testing.T) { tme := newTestTableMigrater(ctx, t) defer tme.stopTablets(t) - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) } - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) } @@ -1317,10 +1317,10 @@ func TestMigrateFrozen(t *testing.T) { ), nil) tme.dbTargetClients[1].addQuery(vreplQueryks2, &sqltypes.Result{}, nil) - err = tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) - want := "cannot migrate reads while MigrateWrites is in progress" + err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) + want := "cannot migrate reads while SwitchWrites is in progress" if err == nil || err.Error() != want { - t.Errorf("MigrateReads(frozen) err: %v, want %v", err, want) + t.Errorf("SwitchReads(frozen) err: %v, want %v", err, want) } tme.dbTargetClients[0].addQuery(vreplQueryks2, sqltypes.MakeTestResult(sqltypes.MakeTestFields( @@ -1341,7 +1341,7 @@ func TestMigrateFrozen(t *testing.T) { } deleteTargetVReplication() - _, err = tme.wr.MigrateWrites(ctx, tme.targetKeyspace, "test", 0*time.Second, false, true) + _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 0*time.Second, false, true) if err != nil { t.Fatal(err) } @@ -1356,10 +1356,10 @@ func TestMigrateNoStreamsFound(t *testing.T) { tme.dbTargetClients[0].addQuery(vreplQueryks2, &sqltypes.Result{}, nil) tme.dbTargetClients[1].addQuery(vreplQueryks2, &sqltypes.Result{}, nil) - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) want := "no streams found in keyspace ks2 for: test" if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MigrateReads: %v, must contain %v", err, want) + t.Errorf("SwitchReads: %v, must contain %v", err, want) } } @@ -1387,10 +1387,10 @@ func TestMigrateDistinctSources(t *testing.T) { fmt.Sprintf("1|%v|", bls), ), nil) - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) want := "source keyspaces are mismatched across streams" if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MigrateReads: %v, must contain %v", err, want) + t.Errorf("SwitchReads: %v, must contain %v", err, want) } } @@ -1416,10 +1416,10 @@ func TestMigrateMismatchedTables(t *testing.T) { nil, ) - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) want := "table lists are mismatched across streams" if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MigrateReads: %v, must contain %v", err, want) + t.Errorf("SwitchReads: %v, must contain %v", err, want) } } @@ -1430,10 +1430,10 @@ func TestTableMigrateAllShardsNotPresent(t *testing.T) { tme.dbTargetClients[0].addQuery(vreplQueryks2, &sqltypes.Result{}, nil) - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) want := "mismatched shards for keyspace" if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MigrateReads: %v, must contain %v", err, want) + t.Errorf("SwitchReads: %v, must contain %v", err, want) } } @@ -1484,10 +1484,10 @@ func TestMigrateNoTableWildcards(t *testing.T) { fmt.Sprintf("1|%v|", bls3), ), nil) - err := tme.wr.MigrateReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) + err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) want := "cannot migrate streams with wild card table names" if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("MigrateReads: %v, must contain %v", err, want) + t.Errorf("SwitchReads: %v, must contain %v", err, want) } } From 17dbe4521d02c727c32fc497be90b4f961396d6e Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Wed, 25 Mar 2020 15:51:11 +0100 Subject: [PATCH 44/85] Incorporated lint suggestions Signed-off-by: Rohit Nayak --- go/test/endtoend/vreplication/cluster.go | 2 +- go/test/endtoend/vreplication/helper.go | 9 ++------- .../endtoend/vreplication/vreplication_test.go | 16 +++++++--------- go/vt/vtctl/vtctl.go | 2 +- 4 files changed, 11 insertions(+), 18 deletions(-) diff --git a/go/test/endtoend/vreplication/cluster.go b/go/test/endtoend/vreplication/cluster.go index e28f004b41a..4f838506538 100644 --- a/go/test/endtoend/vreplication/cluster.go +++ b/go/test/endtoend/vreplication/cluster.go @@ -418,7 +418,7 @@ func (vc *VitessCluster) getVttabletsInKeyspace(t *testing.T, cell *Cell, ksName tablets := make(map[string]*cluster.VttabletProcess) for _, shard := range keyspace.Shards { for _, tablet := range shard.Tablets { - if tablet.Vttablet.GetTabletStatus() == "SERVING" && strings.ToLower(tablet.Vttablet.VreplicationTabletType) == strings.ToLower(tabletType) { + if tablet.Vttablet.GetTabletStatus() == "SERVING" && strings.EqualFold(tablet.Vttablet.VreplicationTabletType, tabletType) { fmt.Printf("Serving status of tablet %s is %s, %s\n", tablet.Name, tablet.Vttablet.ServingStatus, tablet.Vttablet.GetTabletStatus()) tablets[tablet.Name] = tablet.Vttablet } diff --git a/go/test/endtoend/vreplication/helper.go b/go/test/endtoend/vreplication/helper.go index d0bde84a397..160bafcb27f 100644 --- a/go/test/endtoend/vreplication/helper.go +++ b/go/test/endtoend/vreplication/helper.go @@ -110,10 +110,7 @@ func validateThatQueryExecutesOnTablet(t *testing.T, conn *mysql.Conn, tablet *c qr := execVtgateQuery(t, conn, ksName, query) assert.NotNil(t, qr) newCount := getQueryCount(tablet.QueryzURL, matchQuery) - if newCount == count+1 { - return true - } - return false + return newCount == count+1 } func getQueryCount(url string, query string) int { @@ -173,9 +170,7 @@ func getQueryCount(url string, query string) int { foundQuery := re.ReplaceAllLiteralString(row[queryIndex], "") cleanQuery := re.ReplaceAllLiteralString(query, "") if foundQuery == cleanQuery { - count, err = strconv.Atoi(row[countIndex]) - } else { - //fmt.Printf(">> %s %s %d %d\n", foundQuery, cleanQuery, len(foundQuery), len(cleanQuery)) + count, _ = strconv.Atoi(row[countIndex]) } } return count diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index 19100ee561d..4f96c8a3270 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -97,7 +97,7 @@ func insertMoreCustomers(t *testing.T, numCustomers int) { sql := "insert into customer (name) values " i := 0 for i < numCustomers { - i += 1 + i++ sql += fmt.Sprintf("('customer%d')", i) if i != numCustomers { sql += "," @@ -271,7 +271,7 @@ func reshard(t *testing.T, ksName string, tableName string, workflow string, sou tablets := vc.getVttabletsInKeyspace(t, cell, ksName, "master") targetShards = "," + targetShards + "," for _, tab := range tablets { - if strings.Index(targetShards, ","+tab.Shard+",") >= 0 { + if strings.Contains(targetShards, ","+tab.Shard+",") { fmt.Printf("Waiting for vrepl to catch up on %s since it IS a target shard\n", tab.Shard) if vc.WaitForVReplicationToCatchup(tab, workflow, "vt_"+ksName, 3*time.Second) != nil { t.Fatal("Reshard timed out") @@ -291,13 +291,11 @@ func reshard(t *testing.T, ksName string, tableName string, workflow string, sou if output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchWrites", ksWorkflow); err != nil { t.Fatalf("SwitchWrites error: %s\n", output) } - if counts != nil { - for tabletName, count := range counts { - if tablets[tabletName] == nil { - continue - } - assert.Empty(t, validateCountInTablet(t, tablets[tabletName], ksName, tableName, count)) + for tabletName, count := range counts { + if tablets[tabletName] == nil { + continue } + assert.Empty(t, validateCountInTablet(t, tablets[tabletName], ksName, tableName, count)) } } @@ -394,7 +392,7 @@ func vdiff(t *testing.T, workflow string) { assert.True(t, len(diffReports) > 0) for key, diffReport := range diffReports { if diffReport.ProcessedRows != diffReport.MatchingRows { - fmt.Errorf("vdiff error for %d : %#v\n", key, diffReport) + t.Errorf("vdiff error for %d : %#v\n", key, diffReport) } } } diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 3f625419057..dc279d0274b 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -2556,7 +2556,7 @@ func commandApplyVSchema(ctx context.Context, wr *wrangler.Wrangler, subFlags *f if _, err := wr.TopoServer().GetKeyspace(ctx, keyspace); err != nil { if strings.Contains(err.Error(), "node doesn't exist") { - return fmt.Errorf("keyspace(%s) doesn't exist, check if the keyspace is initialized.\n", keyspace) + return fmt.Errorf("keyspace(%s) doesn't exist, check if the keyspace is initialized", keyspace) } return err } From 54405e5c6362dfaf72f286ab5418527550591421 Mon Sep 17 00:00:00 2001 From: Derek Perkins Date: Wed, 25 Mar 2020 08:55:08 -0600 Subject: [PATCH 45/85] messaging: desc time_next index + typo fix Signed-off-by: Derek Perkins --- go/test/endtoend/messaging/cluster_ops_test.go | 4 ++-- go/test/endtoend/messaging/main_test.go | 4 ++-- go/test/endtoend/messaging/message_test.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go/test/endtoend/messaging/cluster_ops_test.go b/go/test/endtoend/messaging/cluster_ops_test.go index 4396a92d8c4..53558f5b4d2 100644 --- a/go/test/endtoend/messaging/cluster_ops_test.go +++ b/go/test/endtoend/messaging/cluster_ops_test.go @@ -45,8 +45,8 @@ func TestUnsharded(t *testing.T) { testMessaging(t, "unsharded_message", lookupKeyspace) } -// TestRepareting checks the client connection count after reparenting. -func TestRepareting(t *testing.T) { +// TestReparenting checks the client connection count after reparenting. +func TestReparenting(t *testing.T) { defer cluster.PanicHandler(t) name := "sharded_message" diff --git a/go/test/endtoend/messaging/main_test.go b/go/test/endtoend/messaging/main_test.go index 36d2bf97517..eb311fb39c8 100644 --- a/go/test/endtoend/messaging/main_test.go +++ b/go/test/endtoend/messaging/main_test.go @@ -48,7 +48,7 @@ var ( time_acked bigint, message varchar(128), primary key(id), - index next_idx(priority, time_next), + index next_idx(priority, time_next desc), index ack_idx(time_acked) ) comment 'vitess_message,vt_ack_wait=1,vt_purge_after=3,vt_batch_size=2,vt_cache_size=10,vt_poller_interval=1'` createUnshardedMessage = `create table unsharded_message( @@ -59,7 +59,7 @@ var ( time_acked bigint, message varchar(128), primary key(id), - index next_idx(priority, time_next), + index next_idx(priority, time_next desc), index ack_idx(time_acked) ) comment 'vitess_message,vt_ack_wait=1,vt_purge_after=3,vt_batch_size=2,vt_cache_size=10,vt_poller_interval=1'` userVschema = `{ diff --git a/go/test/endtoend/messaging/message_test.go b/go/test/endtoend/messaging/message_test.go index 72312594067..1183621dbf0 100644 --- a/go/test/endtoend/messaging/message_test.go +++ b/go/test/endtoend/messaging/message_test.go @@ -37,7 +37,7 @@ var createMessage = `create table vitess_message( time_acked bigint, message varchar(128), primary key(id), - index next_idx(priority, time_next), + index next_idx(priority, time_next desc), index ack_idx(time_acked)) comment 'vitess_message,vt_ack_wait=1,vt_purge_after=3,vt_batch_size=2,vt_cache_size=10,vt_poller_interval=1'` @@ -149,7 +149,7 @@ var createThreeColMessage = `create table vitess_message3( msg1 varchar(128), msg2 bigint, primary key(id), - index next_idx(priority, time_next), + index next_idx(priority, time_next desc), index ack_idx(time_acked)) comment 'vitess_message,vt_ack_wait=1,vt_purge_after=3,vt_batch_size=2,vt_cache_size=10,vt_poller_interval=1'` From 76f9135baa6194c890009b6dd2c67d1f4e87d434 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Wed, 25 Mar 2020 22:11:10 +0100 Subject: [PATCH 46/85] Fix race Signed-off-by: Rohit Nayak --- go/vt/discovery/healthcheck.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index b282b846d7b..f000ffeb3b4 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -44,6 +44,7 @@ import ( "hash/crc32" "html/template" "net/http" + "reflect" "sort" "strings" "sync" @@ -492,7 +493,7 @@ func (hc *HealthCheckImpl) finalizeConn(hcc *healthCheckConn) { // safe to simply get Err() value here and assign to LastError. hcc.tabletStats.LastError = hcc.ctx.Err() hc.updateHealth(hcc.tabletStats.Copy(), nil) - if hcc.conn != nil { + if hcc.conn != nil && !reflect.ValueOf(hcc.conn).IsNil() { // Don't use hcc.ctx because it's already closed. // Use a separate context, and add a timeout to prevent unbounded waits. ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -610,7 +611,7 @@ func (hcc *healthCheckConn) setServingState(serving bool, reason string) { // stream streams healthcheck responses to callback. func (hcc *healthCheckConn) stream(ctx context.Context, hc *HealthCheckImpl, callback func(*querypb.StreamHealthResponse) error) { - if hcc.conn == nil { + if hcc.conn == nil || reflect.ValueOf(hcc.conn).IsNil() { conn, err := tabletconn.GetDialer()(hcc.tabletStats.Tablet, grpcclient.FailFast(true)) if err != nil { hcc.tabletStats.LastError = err From cbf484d12e247d7d98adda409c5e4955a0623471 Mon Sep 17 00:00:00 2001 From: Yuvraj <10830562+evalsocket@users.noreply.github.com> Date: Thu, 26 Mar 2020 02:42:03 +0530 Subject: [PATCH 47/85] Added Timeout for github workflow (#5956) * Added Timeout for github workflow Signed-off-by: Yuvraj * Added Timeout for github workflow Signed-off-by: Yuvraj * update master Signed-off-by: Yuvraj --- .github/workflows/cluster_endtoend.yml | 1 + .github/workflows/e2e_race.yml | 1 + .github/workflows/endtoend.yml | 1 + .github/workflows/local_example.yml | 1 + .github/workflows/unit.yml | 1 + .github/workflows/unit_race.yml | 1 + 6 files changed, 6 insertions(+) diff --git a/.github/workflows/cluster_endtoend.yml b/.github/workflows/cluster_endtoend.yml index a5cb98fd7d1..401a147edb1 100644 --- a/.github/workflows/cluster_endtoend.yml +++ b/.github/workflows/cluster_endtoend.yml @@ -40,6 +40,7 @@ jobs: fi - name: sharded cluster_endtoend + timeout-minutes: 30 run: | source build.env eatmydata -- go run test.go -docker=false -print-log -follow -shard ${{matrix.name}} diff --git a/.github/workflows/e2e_race.yml b/.github/workflows/e2e_race.yml index 3909974b1ce..5484bf81372 100644 --- a/.github/workflows/e2e_race.yml +++ b/.github/workflows/e2e_race.yml @@ -30,5 +30,6 @@ jobs: make minimaltools - name: e2e_race + timeout-minutes: 30 run: | make e2e_test_race diff --git a/.github/workflows/endtoend.yml b/.github/workflows/endtoend.yml index 37f2c866343..5e039acfda2 100644 --- a/.github/workflows/endtoend.yml +++ b/.github/workflows/endtoend.yml @@ -34,5 +34,6 @@ jobs: make build - name: endtoend + timeout-minutes: 30 run: | tools/e2e_test_runner.sh diff --git a/.github/workflows/local_example.yml b/.github/workflows/local_example.yml index a1b2de262ca..47d3c8c29aa 100644 --- a/.github/workflows/local_example.yml +++ b/.github/workflows/local_example.yml @@ -43,6 +43,7 @@ jobs: make build - name: local_example + timeout-minutes: 30 run: | export TOPO=${{matrix.topo}} if [ ${{matrix.os}} = "macos-latest" ]; then diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 72f851beb82..5929e7f3366 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -85,5 +85,6 @@ jobs: make tools - name: unit + timeout-minutes: 30 run: | eatmydata -- make unit_test diff --git a/.github/workflows/unit_race.yml b/.github/workflows/unit_race.yml index 96219e5991c..09f1d996f5d 100644 --- a/.github/workflows/unit_race.yml +++ b/.github/workflows/unit_race.yml @@ -34,5 +34,6 @@ jobs: make tools - name: unit_race + timeout-minutes: 30 run: | eatmydata -- make unit_test_race From 2aaf63d86412a9bab7706dc90eeb4c63976cd1b1 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Tue, 24 Mar 2020 21:30:56 -0700 Subject: [PATCH 48/85] deprecation: delete keyspace id annotations Signed-off-by: Sugu Sougoumarane --- go/vt/binlog/keyrange_filter.go | 125 +--------------- go/vt/sqlannotation/sqlannotation.go | 172 ---------------------- go/vt/sqlannotation/sqlannotation_test.go | 109 -------------- go/vt/vtgate/autocommit_test.go | 38 ++--- go/vt/vtgate/engine/delete.go | 7 +- go/vt/vtgate/engine/delete_test.go | 8 +- go/vt/vtgate/engine/insert.go | 4 - go/vt/vtgate/engine/insert_test.go | 42 +++--- go/vt/vtgate/engine/update.go | 7 +- go/vt/vtgate/engine/update_test.go | 8 +- go/vt/vtgate/executor.go | 2 - go/vt/vtgate/executor_dml_test.go | 82 +++++------ go/vt/vtgate/vschema_stats.go | 2 - go/vt/vtgate/vtgate.go | 9 +- go/vt/vtgate/vtgateconn/vtgateconn.go | 6 +- 15 files changed, 101 insertions(+), 520 deletions(-) delete mode 100644 go/vt/sqlannotation/sqlannotation.go delete mode 100644 go/vt/sqlannotation/sqlannotation_test.go diff --git a/go/vt/binlog/keyrange_filter.go b/go/vt/binlog/keyrange_filter.go index f441f4112ed..3808d2e38f5 100644 --- a/go/vt/binlog/keyrange_filter.go +++ b/go/vt/binlog/keyrange_filter.go @@ -19,12 +19,6 @@ package binlog import ( "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/sqlannotation" - - "errors" - "fmt" - - "vitess.io/vitess/go/vt/sqlparser" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" @@ -51,54 +45,14 @@ func KeyRangeFilterFunc(keyrange *topodatapb.KeyRange, callback func(*binlogdata binlogdatapb.BinlogTransaction_Statement_BL_DELETE: // Handle RBR case first. if statement.KeyspaceID != nil { - if !key.KeyRangeContains(keyrange, statement.KeyspaceID) { - // Skip keyspace ids that don't belong to the destination shard. - continue - } - filtered = append(filtered, statement.Statement) - matched = true - continue - } - - // SBR case. - keyspaceIDS, err := sqlannotation.ExtractKeyspaceIDS(string(statement.Statement.Sql)) - if err != nil { - if statement.Statement.Category == binlogdatapb.BinlogTransaction_Statement_BL_INSERT { - // TODO(erez): Stop filtered-replication here, and alert. - logExtractKeySpaceIDError(err) - continue - } - // If no keyspace IDs are found, we replicate to all targets. - // This is safe for UPDATE and DELETE because vttablet rewrites queries to - // include the primary key and the query will only affect the shards that - // have the rows. - filtered = append(filtered, statement.Statement) - matched = true - continue - } - if len(keyspaceIDS) == 1 { - if !key.KeyRangeContains(keyrange, keyspaceIDS[0]) { - // Skip keyspace ids that don't belong to the destination shard. - continue - } - filtered = append(filtered, statement.Statement) - matched = true - continue + updateStreamErrors.Add("KeyRangeStream", 1) + log.Errorf("SBR mode unsupported for streaming: %s", statement.Statement.Sql) } - query, err := getValidRangeQuery(string(statement.Statement.Sql), keyspaceIDS, keyrange) - if err != nil { - log.Errorf("Error parsing statement (%s). Got %v", string(statement.Statement.Sql), err) + if !key.KeyRangeContains(keyrange, statement.KeyspaceID) { + // Skip keyspace ids that don't belong to the destination shard. continue } - if query == "" { - continue - } - splitStatement := &binlogdatapb.BinlogTransaction_Statement{ - Category: statement.Statement.Category, - Charset: statement.Statement.Charset, - Sql: []byte(query), - } - filtered = append(filtered, splitStatement) + filtered = append(filtered, statement.Statement) matched = true case binlogdatapb.BinlogTransaction_Statement_BL_UNRECOGNIZED: updateStreamErrors.Add("KeyRangeStream", 1) @@ -115,72 +69,3 @@ func KeyRangeFilterFunc(keyrange *topodatapb.KeyRange, callback func(*binlogdata return callback(trans) } } - -func getValidRangeQuery(sql string, keyspaceIDs [][]byte, keyrange *topodatapb.KeyRange) (query string, err error) { - statement, err := sqlparser.Parse(sql) - _, marginComments := sqlparser.SplitMarginComments(sql) - if err != nil { - return "", err - } - - switch statement := statement.(type) { - case *sqlparser.Insert: - query, err := generateSingleInsertQuery(statement, keyspaceIDs, marginComments, keyrange) - if err != nil { - return "", err - } - return query, nil - default: - return "", errors.New("unsupported construct ") - } -} - -func generateSingleInsertQuery(ins *sqlparser.Insert, keyspaceIDs [][]byte, marginComments sqlparser.MarginComments, keyrange *topodatapb.KeyRange) (query string, err error) { - switch rows := ins.Rows.(type) { - case *sqlparser.Select, *sqlparser.Union: - return "", errors.New("unsupported: insert into select") - case sqlparser.Values: - var values sqlparser.Values - if len(rows) != len(keyspaceIDs) { - return "", fmt.Errorf("length of values tuples %v doesn't match with length of keyspaceids %v", len(values), len(keyspaceIDs)) - } - queryBuf := sqlparser.NewTrackedBuffer(nil) - queryBuf.WriteString(marginComments.Leading) - for rowNum, val := range rows { - if key.KeyRangeContains(keyrange, keyspaceIDs[rowNum]) { - values = append(values, val) - } - } - if len(values) == 0 { - return "", nil - } - ins.Rows = values - ins.Format(queryBuf) - queryBuf.WriteString(marginComments.Trailing) - return queryBuf.String(), nil - - default: - return "", errors.New("unexpected construct in insert") - } -} - -func logExtractKeySpaceIDError(err error) { - extractErr, ok := err.(*sqlannotation.ExtractKeySpaceIDError) - if !ok { - log.Fatalf("Expected sqlannotation.ExtractKeySpaceIDError. Got: %v", err) - } - switch extractErr.Kind { - case sqlannotation.ExtractKeySpaceIDParseError: - log.Errorf( - "Error parsing keyspace id annotation. Skipping statement. (%s)", extractErr.Message) - updateStreamErrors.Add("ExtractKeySpaceIDParseError", 1) - case sqlannotation.ExtractKeySpaceIDReplicationUnfriendlyError: - log.Errorf( - "Found replication unfriendly statement. (%s). "+ - "Filtered replication should abort, but we're currently just skipping the statement.", - extractErr.Message) - updateStreamErrors.Add("ExtractKeySpaceIDReplicationUnfriendlyError", 1) - default: - log.Fatalf("Unexpected extractErr.Kind. (%v)", extractErr) - } -} diff --git a/go/vt/sqlannotation/sqlannotation.go b/go/vt/sqlannotation/sqlannotation.go deleted file mode 100644 index 5ddc16f9b0c..00000000000 --- a/go/vt/sqlannotation/sqlannotation.go +++ /dev/null @@ -1,172 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package sqlannotation provides functions -// for annotating DML statements with keyspace-id -// comments and parsing them. These annotations -// are used during filtered-replication to route -// the DML statement to the correct shard. -// TODO(erez): Move the code for the "_stream" annotations -// from vttablet to here. -package sqlannotation - -import ( - "encoding/hex" - "fmt" - "log" - "strings" - - "bytes" - - "vitess.io/vitess/go/stats" - "vitess.io/vitess/go/vt/sqlparser" -) - -const ( - filteredReplicationUnfriendlyAnnotation = "/* vtgate:: filtered_replication_unfriendly */" -) - -var ( - filteredReplicationUnfriendlyStatementsCount = stats.NewCounter("FilteredReplicationUnfriendlyStatementsCount", "Count of unfriendly statements found in filtered replication") -) - -// AnnotateIfDML annotates 'sql' based on 'keyspaceIDs' -// -// If 'sql' is not a DML statement no annotation is added. -// If 'sql' is a DML statement and contains exactly one keyspaceID -// it is used to annotate 'sql' -// Otherwise 'sql' is annotated as replication-unfriendly. -func AnnotateIfDML(sql string, keyspaceIDs [][]byte) string { - if !sqlparser.IsDML(sql) { - return sql - } - if len(keyspaceIDs) == 1 { - return AddKeyspaceIDs(sql, keyspaceIDs, "") - } - filteredReplicationUnfriendlyStatementsCount.Add(1) - return sql + filteredReplicationUnfriendlyAnnotation -} - -// AddKeyspaceIDs returns a copy of 'sql' annotated -// with the given keyspace id. It also appends the -// additional marginComments, if any. -func AddKeyspaceIDs(sql string, keyspaceIDs [][]byte, marginComments string) string { - encodedIDs := make([][]byte, len(keyspaceIDs)) - for i, src := range keyspaceIDs { - encodedIDs[i] = make([]byte, hex.EncodedLen(len(src))) - hex.Encode(encodedIDs[i], src) - } - return fmt.Sprintf("%s /* vtgate:: keyspace_id:%s */%s", - sql, bytes.Join(encodedIDs, []byte(",")), marginComments) -} - -// ExtractKeyspaceIDS parses the annotation of the given statement and tries -// to extract the keyspace id. -// If a keyspace-id comment exists 'keyspaceID' is set to the parsed keyspace id -// and err is set to nil; otherwise, if a filtered-replication-unfriendly comment exists -// or some other parsing error occurred, keyspaceID is set to nil and err is set to a non-nil -// error value. -func ExtractKeyspaceIDS(sql string) (keyspaceIDs [][]byte, err error) { - _, comments := sqlparser.SplitMarginComments(sql) - keyspaceIDString, hasKeyspaceID := extractStringBetween(comments.Trailing, "/* vtgate:: keyspace_id:", " ") - hasUnfriendlyAnnotation := strings.Contains(sql, filteredReplicationUnfriendlyAnnotation) - if !hasKeyspaceID { - if hasUnfriendlyAnnotation { - return nil, &ExtractKeySpaceIDError{ - Kind: ExtractKeySpaceIDReplicationUnfriendlyError, - Message: fmt.Sprintf("Statement: %v", sql), - } - } - // No annotations. - return nil, &ExtractKeySpaceIDError{ - Kind: ExtractKeySpaceIDParseError, - Message: fmt.Sprintf("No annotation found in '%v'", sql), - } - } - if hasUnfriendlyAnnotation { - return nil, &ExtractKeySpaceIDError{ - Kind: ExtractKeySpaceIDParseError, - Message: fmt.Sprintf("Conflicting annotations in statement '%v'", sql), - } - } - ksidStr := strings.Split(keyspaceIDString, ",") - keyspaceIDs = make([][]byte, len(ksidStr)) - for row, ksid := range ksidStr { - err = nil - keyspaceIDs[row], err = hex.DecodeString(ksid) - if err != nil { - keyspaceIDs[row] = nil - err = &ExtractKeySpaceIDError{ - Kind: ExtractKeySpaceIDParseError, - Message: fmt.Sprintf( - "Error parsing keyspace id value in statement: %v (%v)", sql, err), - } - } - } - return -} - -// Extracts the string from source contained between the leftmost instance of -// 'leftDelim' and the next instance of 'rightDelim'. If there is no next instance -// of 'rightDelim', returns the string contained between the end of the leftmost instance -// of 'leftDelim' to the end of 'source'. If 'leftDelim' does not appear in 'source', -// sets 'found' to false and 'match' to the empty string, otherwise 'found' is set to true -// and 'match' is set to the extracted string. -func extractStringBetween(source string, leftDelim string, rightDelim string) (match string, found bool) { - leftDelimStart := strings.Index(source, leftDelim) - if leftDelimStart == -1 { - found = false - match = "" - return - } - found = true - matchStart := leftDelimStart + len(leftDelim) - matchEnd := strings.Index(source[matchStart:], rightDelim) - if matchEnd != -1 { - match = source[matchStart : matchStart+matchEnd] - return - } - match = source[matchStart:] - return -} - -// ExtractKeySpaceIDError is the error type returned -// from ExtractKeySpaceID -// Kind is a numeric code for the error (see constants below) -// and Message is an error message string. -type ExtractKeySpaceIDError struct { - Kind int - Message string -} - -// Possible values for ExtractKeySpaceIDError.Kind -const ( - ExtractKeySpaceIDParseError = iota - ExtractKeySpaceIDReplicationUnfriendlyError = iota -) - -func (err ExtractKeySpaceIDError) Error() string { - switch err.Kind { - case ExtractKeySpaceIDParseError: - return fmt.Sprintf("Parse-Error. %v", err.Message) - case ExtractKeySpaceIDReplicationUnfriendlyError: - return fmt.Sprintf( - "Statement is filtered-replication-unfriendly. %v", err.Message) - default: - log.Fatalf("Unknown error type: %v", err) - return "" // Unreachable. - } -} diff --git a/go/vt/sqlannotation/sqlannotation_test.go b/go/vt/sqlannotation/sqlannotation_test.go deleted file mode 100644 index c7c332ee00a..00000000000 --- a/go/vt/sqlannotation/sqlannotation_test.go +++ /dev/null @@ -1,109 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sqlannotation - -import ( - "reflect" - "strings" - "testing" -) - -func TestExtractKeyspaceIDKeyspaceID(t *testing.T) { - keyspaceIDS, err := ExtractKeyspaceIDS("DML /* vtgate:: keyspace_id:25AF,25BF */") - if err != nil { - t.Errorf("want nil, got: %v", err) - } - if !reflect.DeepEqual(keyspaceIDS[0], []byte{0x25, 0xAF}) { - t.Errorf("want: %v, got: %v", []byte{0x25, 0xAF}, keyspaceIDS[0]) - } - - if !reflect.DeepEqual(keyspaceIDS[1], []byte{0x25, 0xBF}) { - t.Errorf("want: %v, got: %v", []byte{0x25, 0xBF}, keyspaceIDS[1]) - } -} - -func TestExtractKeyspaceIDUnfriendly(t *testing.T) { - _, err := ExtractKeyspaceIDS("DML /* vtgate:: filtered_replication_unfriendly */") - extErr, ok := err.(*ExtractKeySpaceIDError) - if !ok { - t.Fatalf("want a type *ExtractKeySpaceIDError, got: %v", err) - } - if extErr.Kind != ExtractKeySpaceIDReplicationUnfriendlyError { - t.Errorf("want ExtractKeySpaceIDReplicationUnfriendlyError got: %v", err) - } -} - -func TestExtractKeyspaceIDParseError(t *testing.T) { - verifyParseError(t, "DML /* vtgate:: filtered_replication_unfriendly */ /* vtgate:: keyspace_id:25AF */") - verifyParseError(t, "DML /* vtgate:: filtered_replication_unfriendl") - verifyParseError(t, "DML /* vtgate:: keyspace_id:25A */") - verifyParseError(t, "DML /* vtgate:: keyspace_id:25AFG */") - verifyParseError(t, "DML") -} - -func verifyParseError(t *testing.T, sql string) { - _, err := ExtractKeyspaceIDS(sql) - extErr, ok := err.(*ExtractKeySpaceIDError) - if !ok { - t.Fatalf("want a type *ExtractKeySpaceIDError, got: %v", err) - } - if extErr.Kind != ExtractKeySpaceIDParseError { - t.Errorf("want ExtractKeySpaceIDParseError got: %v", err) - } -} - -func BenchmarkExtractKeyspaceIDKeyspaceID(b *testing.B) { - for i := 0; i < b.N; i++ { - ExtractKeyspaceIDS("DML /* vtgate:: keyspace_id:25AF */") - } -} - -func BenchmarkNativeExtractKeyspaceIDKeyspaceID(b *testing.B) { - for i := 0; i < b.N; i++ { - ExtractKeyspaceIDS("DML /* vtgate:: keyspace_id:25AF */") - } -} - -func BenchmarkExtractKeySpaceIDReplicationUnfriendly(b *testing.B) { - for i := 0; i < b.N; i++ { - ExtractKeyspaceIDS("DML /* vtgate:: filtered_replication_unfriendly */") - } -} - -func BenchmarkExtractKeySpaceIDNothing(b *testing.B) { - for i := 0; i < b.N; i++ { - ExtractKeyspaceIDS("DML") - } -} - -func TestAddKeyspaceIDs(t *testing.T) { - ksid1 := []byte{0x37} - ksid2 := []byte{0x29} - ksids := make([][]byte, 0) - ksids = append(ksids, ksid1) - ksids = append(ksids, ksid2) - - sql := AddKeyspaceIDs("DML", [][]byte{ksid1}, "trailing_comments") - if !strings.EqualFold(sql, "DML /* vtgate:: keyspace_id:37 */trailing_comments") { - t.Errorf("want: %s, got: %s", "DML /* vtgate:: keyspace_id:37 */trailing_comments", sql) - } - - sql = AddKeyspaceIDs("DML", ksids, "trailing_comments") - if !strings.EqualFold(sql, "DML /* vtgate:: keyspace_id:37,29 */trailing_comments") { - t.Errorf("want: %s, got: %s", "DML /* vtgate:: keyspace_id:37,29 */trailing_comments", sql) - } -} diff --git a/go/vt/vtgate/autocommit_test.go b/go/vt/vtgate/autocommit_test.go index 1ca4750cfb9..1188cab4802 100644 --- a/go/vt/vtgate/autocommit_test.go +++ b/go/vt/vtgate/autocommit_test.go @@ -42,7 +42,7 @@ func TestAutocommitUpdateSharded(t *testing.T) { require.NoError(t, err) testBatchQuery(t, "sbc1", sbc1, &querypb.BoundQuery{ - Sql: "update user set a = 2 where id = 1 /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "update user set a = 2 where id = 1", BindVariables: map[string]*querypb.BindVariable{}, }) testAsTransactionCount(t, "sbc1", sbc1, 1) @@ -70,7 +70,7 @@ func TestAutocommitUpdateLookup(t *testing.T) { testCommitCount(t, "sbclookup", sbclookup, 1) testQueries(t, "sbc1", sbc1, []*querypb.BoundQuery{{ - Sql: "update music set a = 2 where id = 2 /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "update music set a = 2 where id = 2", BindVariables: map[string]*querypb.BindVariable{}, }}) testAsTransactionCount(t, "sbc1", sbc1, 0) @@ -111,7 +111,7 @@ func TestAutocommitUpdateVindexChange(t *testing.T) { Sql: "select id, name, lastname from user2 where id = 1 for update", BindVariables: map[string]*querypb.BindVariable{}, }, { - Sql: "update user2 set name = 'myname', lastname = 'mylastname' where id = 1 /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "update user2 set name = 'myname', lastname = 'mylastname' where id = 1", BindVariables: map[string]*querypb.BindVariable{}, }}) testAsTransactionCount(t, "sbc", sbc, 0) @@ -126,7 +126,7 @@ func TestAutocommitDeleteSharded(t *testing.T) { require.NoError(t, err) testBatchQuery(t, "sbc1", sbc1, &querypb.BoundQuery{ - Sql: "delete from user_extra where user_id = 1 /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "delete from user_extra where user_id = 1", BindVariables: map[string]*querypb.BindVariable{}, }) testAsTransactionCount(t, "sbc1", sbc1, 1) @@ -168,7 +168,7 @@ func TestAutocommitDeleteLookup(t *testing.T) { Sql: "select user_id, id from music where id = 1 for update", BindVariables: map[string]*querypb.BindVariable{}, }, { - Sql: "delete from music where id = 1 /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "delete from music where id = 1", BindVariables: map[string]*querypb.BindVariable{}, }}) testAsTransactionCount(t, "sbc1", sbc1, 0) @@ -183,7 +183,7 @@ func TestAutocommitDeleteMultiShard(t *testing.T) { require.NoError(t, err) testQueries(t, "sbc1", sbc1, []*querypb.BoundQuery{{ - Sql: "delete from user_extra where user_id in (1, 2)/* vtgate:: filtered_replication_unfriendly */", + Sql: "delete from user_extra where user_id in (1, 2)", BindVariables: map[string]*querypb.BindVariable{}, }}) testBatchQuery(t, "sbc1", sbc1, nil) @@ -191,7 +191,7 @@ func TestAutocommitDeleteMultiShard(t *testing.T) { testCommitCount(t, "sbc1", sbc1, 1) testQueries(t, "sbc2", sbc2, []*querypb.BoundQuery{{ - Sql: "delete from user_extra where user_id in (1, 2)/* vtgate:: filtered_replication_unfriendly */", + Sql: "delete from user_extra where user_id in (1, 2)", BindVariables: map[string]*querypb.BindVariable{}, }}) testBatchQuery(t, "sbc2", sbc2, nil) @@ -207,14 +207,14 @@ func TestAutocommitDeleteMultiShardAutoCommit(t *testing.T) { require.NoError(t, err) testBatchQuery(t, "sbc1", sbc1, &querypb.BoundQuery{ - Sql: "delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from user_extra where user_id in (1, 2)/* vtgate:: filtered_replication_unfriendly */", + Sql: "delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from user_extra where user_id in (1, 2)", BindVariables: map[string]*querypb.BindVariable{}, }) testAsTransactionCount(t, "sbc1", sbc1, 1) testCommitCount(t, "sbc1", sbc1, 0) testBatchQuery(t, "sbc2", sbc2, &querypb.BoundQuery{ - Sql: "delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from user_extra where user_id in (1, 2)/* vtgate:: filtered_replication_unfriendly */", + Sql: "delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from user_extra where user_id in (1, 2)", BindVariables: map[string]*querypb.BindVariable{}, }) testAsTransactionCount(t, "sbc2", sbc2, 1) @@ -229,7 +229,7 @@ func TestAutocommitInsertSharded(t *testing.T) { require.NoError(t, err) testBatchQuery(t, "sbc1", sbc1, &querypb.BoundQuery{ - Sql: "insert into user_extra(user_id, v) values (:_user_id0, 2) /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "insert into user_extra(user_id, v) values (:_user_id0, 2)", BindVariables: map[string]*querypb.BindVariable{ "_user_id0": sqltypes.Int64BindVariable(1), }, @@ -260,7 +260,7 @@ func TestAutocommitInsertLookup(t *testing.T) { testCommitCount(t, "sbclookup", sbclookup, 1) testQueries(t, "sbc1", sbc1, []*querypb.BoundQuery{{ - Sql: "insert into user(id, v, name) values (:_Id0, 2, :_name0) /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "insert into user(id, v, name) values (:_Id0, 2, :_name0)", BindVariables: map[string]*querypb.BindVariable{ "_Id0": sqltypes.Int64BindVariable(1), "_name0": sqltypes.BytesBindVariable([]byte("myname")), @@ -279,7 +279,7 @@ func TestAutocommitInsertMultishardAutoCommit(t *testing.T) { require.NoError(t, err) testBatchQuery(t, "sbc1", sbc1, &querypb.BoundQuery{ - Sql: "insert /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ into user_extra(user_id, v) values (:_user_id0, 2) /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "insert /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ into user_extra(user_id, v) values (:_user_id0, 2)", BindVariables: map[string]*querypb.BindVariable{ "_user_id0": sqltypes.Int64BindVariable(1), "_user_id1": sqltypes.Int64BindVariable(3), @@ -289,7 +289,7 @@ func TestAutocommitInsertMultishardAutoCommit(t *testing.T) { testCommitCount(t, "sbc1", sbc1, 0) testBatchQuery(t, "sbc2", sbc2, &querypb.BoundQuery{ - Sql: "insert /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ into user_extra(user_id, v) values (:_user_id1, 4) /* vtgate:: keyspace_id:4eb190c9a2fa169c */", + Sql: "insert /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ into user_extra(user_id, v) values (:_user_id1, 4)", BindVariables: map[string]*querypb.BindVariable{ "_user_id0": sqltypes.Int64BindVariable(1), "_user_id1": sqltypes.Int64BindVariable(3), @@ -311,7 +311,7 @@ func TestAutocommitInsertMultishardAutoCommit(t *testing.T) { testCommitCount(t, "sbc1", sbc1, 0) testBatchQuery(t, "sbc2", sbc2, &querypb.BoundQuery{ - Sql: "insert /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ into user_extra(user_id, v) values (:_user_id1, 4) /* vtgate:: keyspace_id:4eb190c9a2fa169c */", + Sql: "insert /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ into user_extra(user_id, v) values (:_user_id1, 4)", BindVariables: map[string]*querypb.BindVariable{ "_user_id0": sqltypes.Int64BindVariable(1), "_user_id1": sqltypes.Int64BindVariable(3), @@ -329,7 +329,7 @@ func TestAutocommitInsertMultishard(t *testing.T) { require.NoError(t, err) testQueries(t, "sbc1", sbc1, []*querypb.BoundQuery{{ - Sql: "insert into user_extra(user_id, v) values (:_user_id0, 2) /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "insert into user_extra(user_id, v) values (:_user_id0, 2)", BindVariables: map[string]*querypb.BindVariable{ "_user_id0": sqltypes.Int64BindVariable(1), "_user_id1": sqltypes.Int64BindVariable(3), @@ -339,7 +339,7 @@ func TestAutocommitInsertMultishard(t *testing.T) { testCommitCount(t, "sbc1", sbc1, 1) testQueries(t, "sbc2", sbc2, []*querypb.BoundQuery{{ - Sql: "insert into user_extra(user_id, v) values (:_user_id1, 4) /* vtgate:: keyspace_id:4eb190c9a2fa169c */", + Sql: "insert into user_extra(user_id, v) values (:_user_id1, 4)", BindVariables: map[string]*querypb.BindVariable{ "_user_id0": sqltypes.Int64BindVariable(1), "_user_id1": sqltypes.Int64BindVariable(3), @@ -386,7 +386,7 @@ func TestAutocommitTransactionStarted(t *testing.T) { require.NoError(t, err) testQueries(t, "sbc1", sbc1, []*querypb.BoundQuery{{ - Sql: "update user set a = 2 where id = 1 /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "update user set a = 2 where id = 1", BindVariables: map[string]*querypb.BindVariable{}, }}) testAsTransactionCount(t, "sbc1", sbc1, 0) @@ -408,7 +408,7 @@ func TestAutocommitDirectTarget(t *testing.T) { require.NoError(t, err) testBatchQuery(t, "sbclookup", sbclookup, &querypb.BoundQuery{ - Sql: sql + "/* vtgate:: filtered_replication_unfriendly */", + Sql: sql, BindVariables: map[string]*querypb.BindVariable{}, }) testAsTransactionCount(t, "sbclookup", sbclookup, 1) @@ -430,7 +430,7 @@ func TestAutocommitDirectRangeTarget(t *testing.T) { require.NoError(t, err) testQueries(t, "sbc1", sbc1, []*querypb.BoundQuery{{ - Sql: sql + "/* vtgate:: filtered_replication_unfriendly */", + Sql: sql, BindVariables: map[string]*querypb.BindVariable{}, }}) testAsTransactionCount(t, "sbc1", sbc1, 0) diff --git a/go/vt/vtgate/engine/delete.go b/go/vt/vtgate/engine/delete.go index 663e0a9c362..c1bf66db083 100644 --- a/go/vt/vtgate/engine/delete.go +++ b/go/vt/vtgate/engine/delete.go @@ -23,7 +23,6 @@ import ( "vitess.io/vitess/go/jsonutil" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" - "vitess.io/vitess/go/vt/sqlannotation" "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/vindexes" @@ -167,8 +166,7 @@ func (del *Delete) execDeleteEqual(vcursor VCursor, bindVars map[string]*querypb return nil, vterrors.Wrap(err, "execDeleteEqual") } } - rewritten := sqlannotation.AddKeyspaceIDs(del.Query, [][]byte{ksid}, "") - return execShard(vcursor, rewritten, bindVars, rs, true /* isDML */, true /* canAutocommit */) + return execShard(vcursor, del.Query, bindVars, rs, true /* isDML */, true /* canAutocommit */) } // deleteVindexEntries performs an delete if table owns vindex. @@ -220,10 +218,9 @@ func (del *Delete) execDeleteByDestination(vcursor VCursor, bindVars map[string] } queries := make([]*querypb.BoundQuery, len(rss)) - sql := sqlannotation.AnnotateIfDML(del.Query, nil) for i := range rss { queries[i] = &querypb.BoundQuery{ - Sql: sql, + Sql: del.Query, BindVariables: bindVars, } } diff --git a/go/vt/vtgate/engine/delete_test.go b/go/vt/vtgate/engine/delete_test.go index 8daa679da03..5045d06fcb8 100644 --- a/go/vt/vtgate/engine/delete_test.go +++ b/go/vt/vtgate/engine/delete_test.go @@ -80,7 +80,7 @@ func TestDeleteEqual(t *testing.T) { } vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationKeyspaceID(166b40b44aba4bd6)`, - `ExecuteMultiShard ks.-20: dummy_delete /* vtgate:: keyspace_id:166b40b44aba4bd6 */ {} true true`, + `ExecuteMultiShard ks.-20: dummy_delete {} true true`, }) // Failure case @@ -184,7 +184,7 @@ func TestDeleteOwnedVindex(t *testing.T) { `Execute delete from lkp2 where from1 = :from1 and from2 = :from2 and toc = :toc from1: type:INT64 value:"4" from2: type:INT64 value:"5" toc: type:VARBINARY value:"\026k@\264J\272K\326" true`, `Execute delete from lkp1 where from = :from and toc = :toc from: type:INT64 value:"6" toc: type:VARBINARY value:"\026k@\264J\272K\326" true`, // Finally, the actual delete, which is also sent to -20, same route as the subquery. - `ExecuteMultiShard sharded.-20: dummy_delete /* vtgate:: keyspace_id:166b40b44aba4bd6 */ {} true true`, + `ExecuteMultiShard sharded.-20: dummy_delete {} true true`, }) // No rows changing @@ -201,7 +201,7 @@ func TestDeleteOwnedVindex(t *testing.T) { // It gets used to perform the subquery to fetch the changing column values. `ExecuteMultiShard sharded.-20: dummy_subquery {} false false`, // Subquery returns no rows. So, no vindexes are deleted. We still pass-through the original delete. - `ExecuteMultiShard sharded.-20: dummy_delete /* vtgate:: keyspace_id:166b40b44aba4bd6 */ {} true true`, + `ExecuteMultiShard sharded.-20: dummy_delete {} true true`, }) // Delete can affect multiple rows @@ -233,7 +233,7 @@ func TestDeleteOwnedVindex(t *testing.T) { `Execute delete from lkp2 where from1 = :from1 and from2 = :from2 and toc = :toc from1: type:INT64 value:"7" from2: type:INT64 value:"8" toc: type:VARBINARY value:"\026k@\264J\272K\326" true`, `Execute delete from lkp1 where from = :from and toc = :toc from: type:INT64 value:"9" toc: type:VARBINARY value:"\026k@\264J\272K\326" true`, // Send the DML. - `ExecuteMultiShard sharded.-20: dummy_delete /* vtgate:: keyspace_id:166b40b44aba4bd6 */ {} true true`, + `ExecuteMultiShard sharded.-20: dummy_delete {} true true`, }) } diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index a15c2c0e668..625a4656231 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -26,7 +26,6 @@ import ( "vitess.io/vitess/go/jsonutil" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" - "vitess.io/vitess/go/vt/sqlannotation" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/vterrors" @@ -453,17 +452,14 @@ func (ins *Insert) getInsertShardedRoute(vcursor VCursor, bindVars map[string]*q queries := make([]*querypb.BoundQuery, len(rss)) for i := range rss { - var ksids [][]byte var mids []string for _, indexValue := range indexesPerRss[i] { index, _ := strconv.ParseInt(string(indexValue.Value), 0, 64) if keyspaceIDs[index] != nil { - ksids = append(ksids, keyspaceIDs[index]) mids = append(mids, ins.Mid[index]) } } rewritten := ins.Prefix + strings.Join(mids, ",") + ins.Suffix - rewritten = sqlannotation.AddKeyspaceIDs(rewritten, ksids, "") queries[i] = &querypb.BoundQuery{ Sql: rewritten, BindVariables: bindVars, diff --git a/go/vt/vtgate/engine/insert_test.go b/go/vt/vtgate/engine/insert_test.go index 7989ae64f3b..3c46e5510af 100644 --- a/go/vt/vtgate/engine/insert_test.go +++ b/go/vt/vtgate/engine/insert_test.go @@ -177,7 +177,7 @@ func TestInsertShardedSimple(t *testing.T) { `ResolveDestinations sharded [value:"0" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6)`, // Row 2 will go to -20, rows 1 & 3 will go to 20- `ExecuteMultiShard ` + - `sharded.20-: prefix mid1 suffix /* vtgate:: keyspace_id:166b40b44aba4bd6 */ {_id0: type:INT64 value:"1" } ` + + `sharded.20-: prefix mid1 suffix {_id0: type:INT64 value:"1" } ` + `true true`, }) @@ -216,8 +216,8 @@ func TestInsertShardedSimple(t *testing.T) { `ResolveDestinations sharded [value:"0" value:"1" value:"2" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(4eb190c9a2fa169c)`, // Row 2 will go to -20, rows 1 & 3 will go to 20- `ExecuteMultiShard ` + - `sharded.20-: prefix mid1, mid3 suffix /* vtgate:: keyspace_id:166b40b44aba4bd6,4eb190c9a2fa169c */ {_id0: type:INT64 value:"1" _id1: type:INT64 value:"2" _id2: type:INT64 value:"3" } ` + - `sharded.-20: prefix mid2 suffix /* vtgate:: keyspace_id:06e7ea22ce92708f */ {_id0: type:INT64 value:"1" _id1: type:INT64 value:"2" _id2: type:INT64 value:"3" } ` + + `sharded.20-: prefix mid1, mid3 suffix {_id0: type:INT64 value:"1" _id1: type:INT64 value:"2" _id2: type:INT64 value:"3" } ` + + `sharded.-20: prefix mid2 suffix {_id0: type:INT64 value:"1" _id1: type:INT64 value:"2" _id2: type:INT64 value:"3" } ` + `true false`, }) @@ -258,8 +258,8 @@ func TestInsertShardedSimple(t *testing.T) { `ResolveDestinations sharded [value:"0" value:"1" value:"2" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(4eb190c9a2fa169c)`, // Row 2 will go to -20, rows 1 & 3 will go to 20- `ExecuteMultiShard ` + - `sharded.20-: prefix mid1, mid3 suffix /* vtgate:: keyspace_id:166b40b44aba4bd6,4eb190c9a2fa169c */ {_id0: type:INT64 value:"1" _id1: type:INT64 value:"2" _id2: type:INT64 value:"3" } ` + - `sharded.-20: prefix mid2 suffix /* vtgate:: keyspace_id:06e7ea22ce92708f */ {_id0: type:INT64 value:"1" _id1: type:INT64 value:"2" _id2: type:INT64 value:"3" } ` + + `sharded.20-: prefix mid1, mid3 suffix {_id0: type:INT64 value:"1" _id1: type:INT64 value:"2" _id2: type:INT64 value:"3" } ` + + `sharded.-20: prefix mid2 suffix {_id0: type:INT64 value:"1" _id1: type:INT64 value:"2" _id2: type:INT64 value:"3" } ` + `true true`, }) } @@ -410,10 +410,10 @@ func TestInsertShardedGenerate(t *testing.T) { `ResolveDestinations sharded [value:"0" value:"1" value:"2" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(4eb190c9a2fa169c)`, // Row 2 will go to -20, rows 1 & 3 will go to 20- `ExecuteMultiShard ` + - `sharded.20-: prefix mid1, mid3 suffix /* vtgate:: keyspace_id:166b40b44aba4bd6,4eb190c9a2fa169c */ ` + + `sharded.20-: prefix mid1, mid3 suffix ` + `{__seq0: type:INT64 value:"1" __seq1: type:INT64 value:"2" __seq2: type:INT64 value:"2" ` + `_id0: type:INT64 value:"1" _id1: type:INT64 value:"2" _id2: type:INT64 value:"3" } ` + - `sharded.-20: prefix mid2 suffix /* vtgate:: keyspace_id:06e7ea22ce92708f */ ` + + `sharded.-20: prefix mid2 suffix ` + `{__seq0: type:INT64 value:"1" __seq1: type:INT64 value:"2" __seq2: type:INT64 value:"2" ` + `_id0: type:INT64 value:"1" _id1: type:INT64 value:"2" _id2: type:INT64 value:"3" } ` + `true false`, @@ -548,12 +548,12 @@ func TestInsertShardedOwned(t *testing.T) { // Based on shardForKsid, values returned will be 20-, -20, 20-. `ResolveDestinations sharded [value:"0" value:"1" value:"2" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(4eb190c9a2fa169c)`, `ExecuteMultiShard ` + - `sharded.20-: prefix mid1, mid3 suffix /* vtgate:: keyspace_id:166b40b44aba4bd6,4eb190c9a2fa169c */ ` + + `sharded.20-: prefix mid1, mid3 suffix ` + `{_c10: type:INT64 value:"4" _c11: type:INT64 value:"5" _c12: type:INT64 value:"6" ` + `_c20: type:INT64 value:"7" _c21: type:INT64 value:"8" _c22: type:INT64 value:"9" ` + `_c30: type:INT64 value:"10" _c31: type:INT64 value:"11" _c32: type:INT64 value:"12" ` + `_id0: type:INT64 value:"1" _id1: type:INT64 value:"2" _id2: type:INT64 value:"3" } ` + - `sharded.-20: prefix mid2 suffix /* vtgate:: keyspace_id:06e7ea22ce92708f */ ` + + `sharded.-20: prefix mid2 suffix ` + `{_c10: type:INT64 value:"4" _c11: type:INT64 value:"5" _c12: type:INT64 value:"6" ` + `_c20: type:INT64 value:"7" _c21: type:INT64 value:"8" _c22: type:INT64 value:"9" ` + `_c30: type:INT64 value:"10" _c31: type:INT64 value:"11" _c32: type:INT64 value:"12" ` + @@ -639,7 +639,7 @@ func TestInsertShardedOwnedWithNull(t *testing.T) { `Execute insert into lkp1(from, toc) values(:from0, :toc0) from0: toc0: type:VARBINARY ` + `value:"\026k@\264J\272K\326" true`, `ResolveDestinations sharded [value:"0" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6)`, - `ExecuteMultiShard sharded.20-: prefix mid1 suffix /* vtgate:: keyspace_id:166b40b44aba4bd6 */ ` + + `ExecuteMultiShard sharded.20-: prefix mid1 suffix ` + `{_c30: _id0: type:INT64 value:"1" } true true`, }) } @@ -736,10 +736,10 @@ func TestInsertShardedGeo(t *testing.T) { `id0: type:INT64 value:"1" id1: type:INT64 value:"1" ` + `keyspace_id0: type:VARBINARY value:"\001\026k@\264J\272K\326" keyspace_id1: type:VARBINARY value:"\377\026k@\264J\272K\326" true`, `ResolveDestinations sharded [value:"0" value:"1" ] Destinations:DestinationKeyspaceID(01166b40b44aba4bd6),DestinationKeyspaceID(ff166b40b44aba4bd6)`, - `ExecuteMultiShard sharded.20-: prefix mid1 suffix /* vtgate:: keyspace_id:01166b40b44aba4bd6 */ ` + + `ExecuteMultiShard sharded.20-: prefix mid1 suffix ` + `{_id0: type:INT64 value:"1" _id1: type:INT64 value:"1" ` + `_region0: type:INT64 value:"1" _region1: type:INT64 value:"255" } ` + - `sharded.-20: prefix mid2 suffix /* vtgate:: keyspace_id:ff166b40b44aba4bd6 */ ` + + `sharded.-20: prefix mid2 suffix ` + `{_id0: type:INT64 value:"1" _id1: type:INT64 value:"1" ` + `_region0: type:INT64 value:"1" _region1: type:INT64 value:"255" } ` + `true false`, @@ -923,12 +923,12 @@ func TestInsertShardedIgnoreOwned(t *testing.T) { `ResolveDestinations sharded [value:"0" value:"3" ] Destinations:DestinationKeyspaceID(00),DestinationKeyspaceID(00)`, // Bind vars for rows 2 & 3 may be missing because they were not sent. `ExecuteMultiShard ` + - `sharded.20-: prefix mid1 suffix /* vtgate:: keyspace_id:00 */ ` + + `sharded.20-: prefix mid1 suffix ` + `{_c10: type:INT64 value:"5" _c13: type:INT64 value:"8" ` + `_c20: type:INT64 value:"9" _c23: type:INT64 value:"12" ` + `_c30: type:INT64 value:"13" _c33: type:INT64 value:"16" ` + `_id0: type:INT64 value:"1" _id3: type:INT64 value:"4" } ` + - `sharded.-20: prefix mid4 suffix /* vtgate:: keyspace_id:00 */ ` + + `sharded.-20: prefix mid4 suffix ` + `{_c10: type:INT64 value:"5" _c13: type:INT64 value:"8" ` + `_c20: type:INT64 value:"9" _c23: type:INT64 value:"12" ` + `_c30: type:INT64 value:"13" _c33: type:INT64 value:"16" ` + @@ -1027,7 +1027,7 @@ func TestInsertShardedIgnoreOwnedWithNull(t *testing.T) { `value:"\026k@\264J\272K\326" true`, `Execute select from from lkp1 where from = :from and toc = :toc from: toc: type:VARBINARY value:"\026k@\264J\272K\326" false`, `ResolveDestinations sharded [value:"0" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6)`, - `ExecuteMultiShard sharded.-20: prefix mid1 suffix /* vtgate:: keyspace_id:166b40b44aba4bd6 */ ` + + `ExecuteMultiShard sharded.-20: prefix mid1 suffix ` + `{_c30: _id0: type:INT64 value:"1" } true true`, }) } @@ -1173,12 +1173,12 @@ func TestInsertShardedUnownedVerify(t *testing.T) { // Based on shardForKsid, values returned will be 20-, -20, 20-. `ResolveDestinations sharded [value:"0" value:"1" value:"2" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(4eb190c9a2fa169c)`, `ExecuteMultiShard ` + - `sharded.20-: prefix mid1, mid3 suffix /* vtgate:: keyspace_id:166b40b44aba4bd6,4eb190c9a2fa169c */ ` + + `sharded.20-: prefix mid1, mid3 suffix ` + `{_c10: type:INT64 value:"4" _c11: type:INT64 value:"5" _c12: type:INT64 value:"6" ` + `_c20: type:INT64 value:"7" _c21: type:INT64 value:"8" _c22: type:INT64 value:"9" ` + `_c30: type:INT64 value:"10" _c31: type:INT64 value:"11" _c32: type:INT64 value:"12" ` + `_id0: type:INT64 value:"1" _id1: type:INT64 value:"2" _id2: type:INT64 value:"3" } ` + - `sharded.-20: prefix mid2 suffix /* vtgate:: keyspace_id:06e7ea22ce92708f */ ` + + `sharded.-20: prefix mid2 suffix ` + `{_c10: type:INT64 value:"4" _c11: type:INT64 value:"5" _c12: type:INT64 value:"6" ` + `_c20: type:INT64 value:"7" _c21: type:INT64 value:"8" _c22: type:INT64 value:"9" ` + `_c30: type:INT64 value:"10" _c31: type:INT64 value:"11" _c32: type:INT64 value:"12" ` + @@ -1291,10 +1291,10 @@ func TestInsertShardedIgnoreUnownedVerify(t *testing.T) { // Based on shardForKsid, values returned will be 20-, -20. `ResolveDestinations sharded [value:"0" value:"2" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(4eb190c9a2fa169c)`, `ExecuteMultiShard ` + - `sharded.20-: prefix mid1 suffix /* vtgate:: keyspace_id:166b40b44aba4bd6 */ ` + + `sharded.20-: prefix mid1 suffix ` + `{_c30: type:INT64 value:"10" _c32: type:INT64 value:"12" ` + `_id0: type:INT64 value:"1" _id2: type:INT64 value:"3" } ` + - `sharded.-20: prefix mid3 suffix /* vtgate:: keyspace_id:4eb190c9a2fa169c */ ` + + `sharded.-20: prefix mid3 suffix ` + `{_c30: type:INT64 value:"10" _c32: type:INT64 value:"12" ` + `_id0: type:INT64 value:"1" _id2: type:INT64 value:"3" } ` + `true false`, @@ -1499,12 +1499,12 @@ func TestInsertShardedUnownedReverseMap(t *testing.T) { vc.ExpectLog(t, []string{ `ResolveDestinations sharded [value:"0" value:"1" value:"2" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(4eb190c9a2fa169c)`, `ExecuteMultiShard ` + - `sharded.20-: prefix mid1, mid3 suffix /* vtgate:: keyspace_id:166b40b44aba4bd6,4eb190c9a2fa169c */ ` + + `sharded.20-: prefix mid1, mid3 suffix ` + `{_c10: type:UINT64 value:"1" _c11: type:UINT64 value:"2" _c12: type:UINT64 value:"3" ` + `_c20: _c21: _c22: ` + `_c30: type:UINT64 value:"1" _c31: type:UINT64 value:"2" _c32: type:UINT64 value:"3" ` + `_id0: type:INT64 value:"1" _id1: type:INT64 value:"2" _id2: type:INT64 value:"3" } ` + - `sharded.-20: prefix mid2 suffix /* vtgate:: keyspace_id:06e7ea22ce92708f */ ` + + `sharded.-20: prefix mid2 suffix ` + `{_c10: type:UINT64 value:"1" _c11: type:UINT64 value:"2" _c12: type:UINT64 value:"3" ` + `_c20: _c21: _c22: ` + `_c30: type:UINT64 value:"1" _c31: type:UINT64 value:"2" _c32: type:UINT64 value:"3" ` + diff --git a/go/vt/vtgate/engine/update.go b/go/vt/vtgate/engine/update.go index c5c1c63f48b..565f86e1e14 100644 --- a/go/vt/vtgate/engine/update.go +++ b/go/vt/vtgate/engine/update.go @@ -23,7 +23,6 @@ import ( "vitess.io/vitess/go/jsonutil" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" - "vitess.io/vitess/go/vt/sqlannotation" "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/vindexes" @@ -174,8 +173,7 @@ func (upd *Update) execUpdateEqual(vcursor VCursor, bindVars map[string]*querypb return nil, vterrors.Wrap(err, "execUpdateEqual") } } - rewritten := sqlannotation.AddKeyspaceIDs(upd.Query, [][]byte{ksid}, "") - return execShard(vcursor, rewritten, bindVars, rs, true /* isDML */, true /* canAutocommit */) + return execShard(vcursor, upd.Query, bindVars, rs, true /* isDML */, true /* canAutocommit */) } // updateVindexEntries performs an update when a vindex is being modified @@ -247,10 +245,9 @@ func (upd *Update) execUpdateByDestination(vcursor VCursor, bindVars map[string] } queries := make([]*querypb.BoundQuery, len(rss)) - sql := sqlannotation.AnnotateIfDML(upd.Query, nil) for i := range rss { queries[i] = &querypb.BoundQuery{ - Sql: sql, + Sql: upd.Query, BindVariables: bindVars, } } diff --git a/go/vt/vtgate/engine/update_test.go b/go/vt/vtgate/engine/update_test.go index 356c9c10745..d73e8bf1688 100644 --- a/go/vt/vtgate/engine/update_test.go +++ b/go/vt/vtgate/engine/update_test.go @@ -79,7 +79,7 @@ func TestUpdateEqual(t *testing.T) { require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationKeyspaceID(166b40b44aba4bd6)`, - `ExecuteMultiShard ks.-20: dummy_update /* vtgate:: keyspace_id:166b40b44aba4bd6 */ {} true true`, + `ExecuteMultiShard ks.-20: dummy_update {} true true`, }) // Failure case @@ -241,7 +241,7 @@ func TestUpdateEqualChangedVindex(t *testing.T) { `Execute delete from lkp1 where from = :from and toc = :toc from: type:INT64 value:"6" toc: type:VARBINARY value:"\026k@\264J\272K\326" true`, `Execute insert into lkp1(from, toc) values(:from0, :toc0) from0: type:INT64 value:"3" toc0: type:VARBINARY value:"\026k@\264J\272K\326" true`, // Finally, the actual update, which is also sent to -20, same route as the subquery. - `ExecuteMultiShard sharded.-20: dummy_update /* vtgate:: keyspace_id:166b40b44aba4bd6 */ {} true true`, + `ExecuteMultiShard sharded.-20: dummy_update {} true true`, }) // No rows changing @@ -256,7 +256,7 @@ func TestUpdateEqualChangedVindex(t *testing.T) { // It gets used to perform the subquery to fetch the changing column values. `ExecuteMultiShard sharded.-20: dummy_subquery {} false false`, // Subquery returns no rows. So, no vindexes are updated. We still pass-through the original update. - `ExecuteMultiShard sharded.-20: dummy_update /* vtgate:: keyspace_id:166b40b44aba4bd6 */ {} true true`, + `ExecuteMultiShard sharded.-20: dummy_update {} true true`, }) // Failure case: multiple rows changing. @@ -291,7 +291,7 @@ func TestUpdateEqualChangedVindex(t *testing.T) { `Execute delete from lkp1 where from = :from and toc = :toc from: type:INT64 value:"9" toc: type:VARBINARY value:"\026k@\264J\272K\326" true`, `Execute insert into lkp1(from, toc) values(:from0, :toc0) from0: type:INT64 value:"3" toc0: type:VARBINARY value:"\026k@\264J\272K\326" true`, // Finally, the actual update, which is also sent to -20, same route as the subquery. - `ExecuteMultiShard sharded.-20: dummy_update /* vtgate:: keyspace_id:166b40b44aba4bd6 */ {} true true`, + `ExecuteMultiShard sharded.-20: dummy_update {} true true`, }) } diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index f4d1a02d2cd..02f7da1353b 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -39,7 +39,6 @@ import ( "vitess.io/vitess/go/vt/callerid" "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/sqlannotation" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/topo/topoproto" @@ -282,7 +281,6 @@ func (e *Executor) handleExec(ctx context.Context, safeSession *SafeSession, sql } execStart := time.Now() - sql = sqlannotation.AnnotateIfDML(sql, nil) if e.normalize { query, comments := sqlparser.SplitMarginComments(sql) stmt, err := sqlparser.Parse(query) diff --git a/go/vt/vtgate/executor_dml_test.go b/go/vt/vtgate/executor_dml_test.go index c87e1646c69..d8f2a3cc62b 100644 --- a/go/vt/vtgate/executor_dml_test.go +++ b/go/vt/vtgate/executor_dml_test.go @@ -44,7 +44,7 @@ func TestUpdateEqual(t *testing.T) { _, err := executorExec(executor, "update user set a=2 where id = 1", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "update user set a = 2 where id = 1 /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "update user set a = 2 where id = 1", BindVariables: map[string]*querypb.BindVariable{}, }} if !reflect.DeepEqual(sbc1.Queries, wantQueries) { @@ -59,7 +59,7 @@ func TestUpdateEqual(t *testing.T) { _, err = executorExec(executor, "update user set a=2 where id = 3", nil) require.NoError(t, err) wantQueries = []*querypb.BoundQuery{{ - Sql: "update user set a = 2 where id = 3 /* vtgate:: keyspace_id:4eb190c9a2fa169c */", + Sql: "update user set a = 2 where id = 3", BindVariables: map[string]*querypb.BindVariable{}, }} if !reflect.DeepEqual(sbc2.Queries, wantQueries) { @@ -109,7 +109,7 @@ func TestUpdateEqual(t *testing.T) { BindVariables: map[string]*querypb.BindVariable{}, }, { - Sql: "update user2 set name = 'myname', lastname = 'mylastname' where id = 1 /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "update user2 set name = 'myname', lastname = 'mylastname' where id = 1", BindVariables: map[string]*querypb.BindVariable{}, }, } @@ -220,7 +220,7 @@ func TestUpdateMultiOwned(t *testing.T) { Sql: "select id, a, b, c, d, e, f from user where id = 1 for update", BindVariables: map[string]*querypb.BindVariable{}, }, { - Sql: "update user set a = 1, b = 2, f = 4, e = 3 where id = 1 /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "update user set a = 1, b = 2, f = 4, e = 3 where id = 1", BindVariables: map[string]*querypb.BindVariable{}, }} if !reflect.DeepEqual(sbc1.Queries, wantQueries) { @@ -271,7 +271,7 @@ func TestUpdateComments(t *testing.T) { _, err := executorExec(executor, "update user set a=2 where id = 1 /* trailing */", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "update user set a = 2 where id = 1 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ /* trailing */", + Sql: "update user set a = 2 where id = 1 /* trailing */", BindVariables: map[string]*querypb.BindVariable{}, }} if !reflect.DeepEqual(sbc1.Queries, wantQueries) { @@ -289,7 +289,7 @@ func TestUpdateNormalize(t *testing.T) { _, err := executorExec(executor, "/* leading */ update user set a=2 where id = 1 /* trailing */", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "/* leading */ update user set a = :vtg1 where id = :vtg2 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ /* trailing */", + Sql: "/* leading */ update user set a = :vtg1 where id = :vtg2 /* trailing */", BindVariables: map[string]*querypb.BindVariable{ "vtg1": sqltypes.TestBindVariable(int64(2)), "vtg2": sqltypes.TestBindVariable(int64(1)), @@ -308,7 +308,7 @@ func TestUpdateNormalize(t *testing.T) { _, err = executorExec(executor, "/* leading */ update user set a=2 where id = 1 /* trailing */", nil) require.NoError(t, err) wantQueries = []*querypb.BoundQuery{{ - Sql: "/* leading */ update user set a = :vtg1 where id = :vtg2 /* trailing *//* vtgate:: filtered_replication_unfriendly */", + Sql: "/* leading */ update user set a = :vtg1 where id = :vtg2 /* trailing */", BindVariables: map[string]*querypb.BindVariable{ "vtg1": sqltypes.TestBindVariable(int64(2)), "vtg2": sqltypes.TestBindVariable(int64(1)), @@ -345,7 +345,7 @@ func TestDeleteEqual(t *testing.T) { Sql: "select Id, name from user where id = 1 for update", BindVariables: map[string]*querypb.BindVariable{}, }, { - Sql: "delete from user where id = 1 /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "delete from user where id = 1", BindVariables: map[string]*querypb.BindVariable{}, }} if !reflect.DeepEqual(sbc.Queries, wantQueries) { @@ -372,7 +372,7 @@ func TestDeleteEqual(t *testing.T) { Sql: "select Id, name from user where id = 1 for update", BindVariables: map[string]*querypb.BindVariable{}, }, { - Sql: "delete from user where id = 1 /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "delete from user where id = 1", BindVariables: map[string]*querypb.BindVariable{}, }} if !reflect.DeepEqual(sbc.Queries, wantQueries) { @@ -406,7 +406,7 @@ func TestDeleteEqual(t *testing.T) { _, err = executorExec(executor, "delete from user_extra where user_id = 1", nil) require.NoError(t, err) wantQueries = []*querypb.BoundQuery{{ - Sql: "delete from user_extra where user_id = 1 /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "delete from user_extra where user_id = 1", BindVariables: map[string]*querypb.BindVariable{}, }} if !reflect.DeepEqual(sbc.Queries, wantQueries) { @@ -431,7 +431,7 @@ func TestDeleteEqual(t *testing.T) { BindVariables: map[string]*querypb.BindVariable{}, }, { - Sql: "delete from user2 where id = 1 /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "delete from user2 where id = 1", BindVariables: map[string]*querypb.BindVariable{}, }, } @@ -461,7 +461,7 @@ func TestUpdateScatter(t *testing.T) { require.NoError(t, err) // Queries get annotatted. wantQueries := []*querypb.BoundQuery{{ - Sql: "update user_extra set col = 2/* vtgate:: filtered_replication_unfriendly */", + Sql: "update user_extra set col = 2", BindVariables: map[string]*querypb.BindVariable{}, }} if !reflect.DeepEqual(sbc1.Queries, wantQueries) { @@ -478,7 +478,7 @@ func TestDeleteScatter(t *testing.T) { require.NoError(t, err) // Queries get annotatted. wantQueries := []*querypb.BoundQuery{{ - Sql: "delete from user_extra/* vtgate:: filtered_replication_unfriendly */", + Sql: "delete from user_extra", BindVariables: map[string]*querypb.BindVariable{}, }} if !reflect.DeepEqual(sbc1.Queries, wantQueries) { @@ -496,7 +496,7 @@ func TestDeleteByDestination(t *testing.T) { require.NoError(t, err) // Queries get annotatted. wantQueries := []*querypb.BoundQuery{{ - Sql: "delete from user_extra limit 10/* vtgate:: filtered_replication_unfriendly */", + Sql: "delete from user_extra limit 10", BindVariables: map[string]*querypb.BindVariable{}, }} if !reflect.DeepEqual(sbc1.Queries, wantQueries) { @@ -528,7 +528,7 @@ func TestDeleteComments(t *testing.T) { Sql: "select Id, name from user where id = 1 for update /* trailing */", BindVariables: map[string]*querypb.BindVariable{}, }, { - Sql: "delete from user where id = 1 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ /* trailing */", + Sql: "delete from user where id = 1 /* trailing */", BindVariables: map[string]*querypb.BindVariable{}, }} if !reflect.DeepEqual(sbc.Queries, wantQueries) { @@ -556,7 +556,7 @@ func TestInsertSharded(t *testing.T) { _, err := executorExec(executor, "insert into user(id, v, name) values (1, 2, 'myname')", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "insert into user(id, v, name) values (:_Id0, 2, :_name0) /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "insert into user(id, v, name) values (:_Id0, 2, :_name0)", BindVariables: map[string]*querypb.BindVariable{ "_Id0": sqltypes.Int64BindVariable(1), "_name0": sqltypes.BytesBindVariable([]byte("myname")), @@ -588,7 +588,7 @@ func TestInsertSharded(t *testing.T) { _, err = executorExec(executor, "insert into user(id, v, name) values (3, 2, 'myname2')", nil) require.NoError(t, err) wantQueries = []*querypb.BoundQuery{{ - Sql: "insert into user(id, v, name) values (:_Id0, 2, :_name0) /* vtgate:: keyspace_id:4eb190c9a2fa169c */", + Sql: "insert into user(id, v, name) values (:_Id0, 2, :_name0)", BindVariables: map[string]*querypb.BindVariable{ "_Id0": sqltypes.Int64BindVariable(3), "__seq0": sqltypes.Int64BindVariable(3), @@ -616,7 +616,7 @@ func TestInsertSharded(t *testing.T) { _, err = executorExec(executor, "insert into user2(id, name, lastname) values (2, 'myname', 'mylastname')", nil) require.NoError(t, err) wantQueries = []*querypb.BoundQuery{{ - Sql: "insert into user2(id, name, lastname) values (:_id0, :_name0, :_lastname0) /* vtgate:: keyspace_id:06e7ea22ce92708f */", + Sql: "insert into user2(id, name, lastname) values (:_id0, :_name0, :_lastname0)", BindVariables: map[string]*querypb.BindVariable{ "_id0": sqltypes.Int64BindVariable(2), "_name0": sqltypes.BytesBindVariable([]byte("myname")), @@ -690,7 +690,7 @@ func TestInsertShardedAutocommitLookup(t *testing.T) { _, err := executorExec(executor, "insert into user(id, v, name) values (1, 2, 'myname')", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "insert into user(id, v, name) values (:_Id0, 2, :_name0) /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "insert into user(id, v, name) values (:_Id0, 2, :_name0)", BindVariables: map[string]*querypb.BindVariable{ "_Id0": sqltypes.Int64BindVariable(1), "_name0": sqltypes.BytesBindVariable([]byte("myname")), @@ -758,7 +758,7 @@ func TestInsertShardedIgnore(t *testing.T) { _, err := executorExec(executor, query, nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "insert ignore into insert_ignore_test(pv, owned, verify) values (:_pv0, :_owned0, :_verify0),(:_pv4, :_owned4, :_verify4) /* vtgate:: keyspace_id:166b40b44aba4bd6,166b40b44aba4bd6 */", + Sql: "insert ignore into insert_ignore_test(pv, owned, verify) values (:_pv0, :_owned0, :_verify0),(:_pv4, :_owned4, :_verify4)", BindVariables: map[string]*querypb.BindVariable{ "_pv0": sqltypes.Int64BindVariable(1), "_pv4": sqltypes.Int64BindVariable(5), @@ -775,7 +775,7 @@ func TestInsertShardedIgnore(t *testing.T) { t.Errorf("sbc1.Queries:\n%+v, want\n%+v\n", sbc1.Queries, wantQueries) } wantQueries = []*querypb.BoundQuery{{ - Sql: "insert ignore into insert_ignore_test(pv, owned, verify) values (:_pv5, :_owned5, :_verify5) /* vtgate:: keyspace_id:4eb190c9a2fa169c */", + Sql: "insert ignore into insert_ignore_test(pv, owned, verify) values (:_pv5, :_owned5, :_verify5)", BindVariables: map[string]*querypb.BindVariable{ "_pv0": sqltypes.Int64BindVariable(1), "_pv4": sqltypes.Int64BindVariable(5), @@ -908,7 +908,7 @@ func TestInsertOnDupKey(t *testing.T) { _, err := executorExec(executor, query, nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "insert into insert_ignore_test(pv, owned, verify) values (:_pv0, :_owned0, :_verify0) on duplicate key update col = 2 /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "insert into insert_ignore_test(pv, owned, verify) values (:_pv0, :_owned0, :_verify0) on duplicate key update col = 2", BindVariables: map[string]*querypb.BindVariable{ "_pv0": sqltypes.Int64BindVariable(1), "_owned0": sqltypes.Int64BindVariable(1), @@ -950,7 +950,7 @@ func TestInsertComments(t *testing.T) { _, err := executorExec(executor, "insert into user(id, v, name) values (1, 2, 'myname') /* trailing */", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "insert into user(id, v, name) values (:_Id0, 2, :_name0) /* vtgate:: keyspace_id:166b40b44aba4bd6 */ /* trailing */", + Sql: "insert into user(id, v, name) values (:_Id0, 2, :_name0) /* trailing */", BindVariables: map[string]*querypb.BindVariable{ "_Id0": sqltypes.Int64BindVariable(1), "_name0": sqltypes.BytesBindVariable([]byte("myname")), @@ -988,7 +988,7 @@ func TestInsertGeneratorSharded(t *testing.T) { result, err := executorExec(executor, "insert into user(v, name) values (2, 'myname')", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "insert into user(v, name, id) values (2, :_name0, :_Id0) /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "insert into user(v, name, id) values (2, :_name0, :_Id0)", BindVariables: map[string]*querypb.BindVariable{ "_Id0": sqltypes.Int64BindVariable(1), "__seq0": sqltypes.Int64BindVariable(1), @@ -1033,7 +1033,7 @@ func TestInsertAutoincSharded(t *testing.T) { result, err := executorExec(router, "insert into user_extra(user_id) values (2)", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "insert into user_extra(user_id) values (:_user_id0) /* vtgate:: keyspace_id:06e7ea22ce92708f */", + Sql: "insert into user_extra(user_id) values (:_user_id0)", BindVariables: map[string]*querypb.BindVariable{ "_user_id0": sqltypes.Int64BindVariable(2), }, @@ -1103,7 +1103,7 @@ func TestInsertLookupOwned(t *testing.T) { _, err := executorExec(executor, "insert into music(user_id, id) values (2, 3)", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "insert into music(user_id, id) values (:_user_id0, :_id0) /* vtgate:: keyspace_id:06e7ea22ce92708f */", + Sql: "insert into music(user_id, id) values (:_user_id0, :_id0)", BindVariables: map[string]*querypb.BindVariable{ "_user_id0": sqltypes.Int64BindVariable(2), "_id0": sqltypes.Int64BindVariable(3), @@ -1138,7 +1138,7 @@ func TestInsertLookupOwnedGenerator(t *testing.T) { result, err := executorExec(executor, "insert into music(user_id) values (2)", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "insert into music(user_id, id) values (:_user_id0, :_id0) /* vtgate:: keyspace_id:06e7ea22ce92708f */", + Sql: "insert into music(user_id, id) values (:_user_id0, :_id0)", BindVariables: map[string]*querypb.BindVariable{ "_user_id0": sqltypes.Int64BindVariable(2), "_id0": sqltypes.Int64BindVariable(4), @@ -1174,7 +1174,7 @@ func TestInsertLookupUnowned(t *testing.T) { _, err := executorExec(executor, "insert into music_extra(user_id, music_id) values (2, 3)", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "insert into music_extra(user_id, music_id) values (:_user_id0, :_music_id0) /* vtgate:: keyspace_id:06e7ea22ce92708f */", + Sql: "insert into music_extra(user_id, music_id) values (:_user_id0, :_music_id0)", BindVariables: map[string]*querypb.BindVariable{ "_user_id0": sqltypes.Int64BindVariable(2), "_music_id0": sqltypes.Int64BindVariable(3), @@ -1201,7 +1201,7 @@ func TestInsertLookupUnownedUnsupplied(t *testing.T) { _, err := executorExec(executor, "insert into music_extra_reversed(music_id) values (3)", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "insert into music_extra_reversed(music_id, user_id) values (:_music_id0, :_user_id0) /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "insert into music_extra_reversed(music_id, user_id) values (:_music_id0, :_user_id0)", BindVariables: map[string]*querypb.BindVariable{ "_user_id0": sqltypes.Uint64BindVariable(1), "_music_id0": sqltypes.Int64BindVariable(3), @@ -1270,7 +1270,7 @@ func TestMultiInsertSharded(t *testing.T) { _, err := executorExec(executor, "insert into user(id, v, name) values (1, 1, 'myname1'),(3, 3, 'myname3')", nil) require.NoError(t, err) wantQueries1 := []*querypb.BoundQuery{{ - Sql: "insert into user(id, v, name) values (:_Id0, 1, :_name0) /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "insert into user(id, v, name) values (:_Id0, 1, :_name0)", BindVariables: map[string]*querypb.BindVariable{ "_Id0": sqltypes.Int64BindVariable(1), "_name0": sqltypes.BytesBindVariable([]byte("myname1")), @@ -1282,7 +1282,7 @@ func TestMultiInsertSharded(t *testing.T) { }} wantQueries2 := []*querypb.BoundQuery{{ - Sql: "insert into user(id, v, name) values (:_Id1, 3, :_name1) /* vtgate:: keyspace_id:4eb190c9a2fa169c */", + Sql: "insert into user(id, v, name) values (:_Id1, 3, :_name1)", BindVariables: map[string]*querypb.BindVariable{ "_Id0": sqltypes.Int64BindVariable(1), "_name0": sqltypes.BytesBindVariable([]byte("myname1")), @@ -1319,7 +1319,7 @@ func TestMultiInsertSharded(t *testing.T) { _, err = executorExec(executor, "insert into user(id, v, name) values (1, 1, 'myname1'),(2, 2, 'myname2')", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "insert into user(id, v, name) values (:_Id0, 1, :_name0),(:_Id1, 2, :_name1) /* vtgate:: keyspace_id:166b40b44aba4bd6,06e7ea22ce92708f */", + Sql: "insert into user(id, v, name) values (:_Id0, 1, :_name0),(:_Id1, 2, :_name1)", BindVariables: map[string]*querypb.BindVariable{ "_Id0": sqltypes.Int64BindVariable(1), "__seq0": sqltypes.Int64BindVariable(1), @@ -1356,7 +1356,7 @@ func TestMultiInsertSharded(t *testing.T) { _, err = executorExec(executor, "insert into user2(id, name, lastname) values (2, 'myname', 'mylastname'), (3, 'myname2', 'mylastname2')", nil) require.NoError(t, err) wantQueries = []*querypb.BoundQuery{{ - Sql: "insert into user2(id, name, lastname) values (:_id0, :_name0, :_lastname0) /* vtgate:: keyspace_id:06e7ea22ce92708f */", + Sql: "insert into user2(id, name, lastname) values (:_id0, :_name0, :_lastname0)", BindVariables: map[string]*querypb.BindVariable{ "_id0": sqltypes.Int64BindVariable(2), "_name0": sqltypes.BytesBindVariable([]byte("myname")), @@ -1398,7 +1398,7 @@ func TestMultiInsertGenerator(t *testing.T) { result, err := executorExec(executor, "insert into music(user_id, name) values (:u, 'myname1'),(:u, 'myname2')", map[string]*querypb.BindVariable{"u": sqltypes.Int64BindVariable(2)}) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "insert into music(user_id, name, id) values (:_user_id0, 'myname1', :_id0),(:_user_id1, 'myname2', :_id1) /* vtgate:: keyspace_id:06e7ea22ce92708f,06e7ea22ce92708f */", + Sql: "insert into music(user_id, name, id) values (:_user_id0, 'myname1', :_id0),(:_user_id1, 'myname2', :_id1)", BindVariables: map[string]*querypb.BindVariable{ "u": sqltypes.Int64BindVariable(2), "_id0": sqltypes.Int64BindVariable(1), @@ -1447,7 +1447,7 @@ func TestMultiInsertGeneratorSparse(t *testing.T) { result, err := executorExec(executor, "insert into music(id, user_id, name) values (NULL, :u, 'myname1'),(2, :u, 'myname2'), (NULL, :u, 'myname3')", map[string]*querypb.BindVariable{"u": sqltypes.Int64BindVariable(2)}) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "insert into music(id, user_id, name) values (:_id0, :_user_id0, 'myname1'),(:_id1, :_user_id1, 'myname2'),(:_id2, :_user_id2, 'myname3') /* vtgate:: keyspace_id:06e7ea22ce92708f,06e7ea22ce92708f,06e7ea22ce92708f */", + Sql: "insert into music(id, user_id, name) values (:_id0, :_user_id0, 'myname1'),(:_id1, :_user_id1, 'myname2'),(:_id2, :_user_id2, 'myname3')", BindVariables: map[string]*querypb.BindVariable{ "u": sqltypes.Int64BindVariable(2), "_id0": sqltypes.Int64BindVariable(1), @@ -1532,7 +1532,7 @@ func TestKeyDestRangeQuery(t *testing.T) { require.NoError(t, err) sql := "DELETE FROM sharded_user_msgs LIMIT 1000" wantQueries := []*querypb.BoundQuery{{ - Sql: sql + "/* vtgate:: filtered_replication_unfriendly */", + Sql: sql, BindVariables: map[string]*querypb.BindVariable{}, }} @@ -1587,7 +1587,7 @@ func TestKeyDestRangeQuery(t *testing.T) { sql = "UPDATE sharded_user_msgs set message='test' LIMIT 1" wantQueries = []*querypb.BoundQuery{{ - Sql: sql + "/* vtgate:: filtered_replication_unfriendly */", + Sql: sql, BindVariables: map[string]*querypb.BindVariable{}, }} @@ -1622,7 +1622,7 @@ func TestKeyShardDestQuery(t *testing.T) { require.NoError(t, err) sql := "DELETE FROM sharded_user_msgs LIMIT 1000" wantQueries := []*querypb.BoundQuery{{ - Sql: sql + "/* vtgate:: filtered_replication_unfriendly */", + Sql: sql, BindVariables: map[string]*querypb.BindVariable{}, }} @@ -1667,7 +1667,7 @@ func TestKeyShardDestQuery(t *testing.T) { sql = "UPDATE sharded_user_msgs set message='test' LIMIT 1" wantQueries = []*querypb.BoundQuery{{ - Sql: sql + "/* vtgate:: filtered_replication_unfriendly */", + Sql: sql, BindVariables: map[string]*querypb.BindVariable{}, }} @@ -1690,7 +1690,7 @@ func TestKeyShardDestQuery(t *testing.T) { _, err = executorExec(executor, sql, nil) wantQueries = []*querypb.BoundQuery{{ - Sql: sql + "/* vtgate:: filtered_replication_unfriendly */", + Sql: sql, BindVariables: map[string]*querypb.BindVariable{}, }} require.NoError(t, err) @@ -1785,7 +1785,7 @@ func TestUpdateLastInsertID(t *testing.T) { _, err := executorExec(executor, sql, map[string]*querypb.BindVariable{}) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "update user set a = :__lastInsertId where id = :vtg1 /* vtgate:: keyspace_id:166b40b44aba4bd6 */", + Sql: "update user set a = :__lastInsertId where id = :vtg1", BindVariables: map[string]*querypb.BindVariable{ "__lastInsertId": sqltypes.Uint64BindVariable(43), "vtg1": sqltypes.Int64BindVariable(1)}, diff --git a/go/vt/vtgate/vschema_stats.go b/go/vt/vtgate/vschema_stats.go index 8e7126ca1d6..111976bc33d 100644 --- a/go/vt/vtgate/vschema_stats.go +++ b/go/vt/vtgate/vschema_stats.go @@ -16,8 +16,6 @@ limitations under the License. package vtgate -// This is a V3 file. Do not intermix with V2. - import ( "sort" diff --git a/go/vt/vtgate/vtgate.go b/go/vt/vtgate/vtgate.go index e518768768b..d5bbb255e21 100644 --- a/go/vt/vtgate/vtgate.go +++ b/go/vt/vtgate/vtgate.go @@ -91,16 +91,9 @@ var ( // VTGate is the rpc interface to vtgate. Only one instance // can be created. It implements vtgateservice.VTGateService -// VTGate exposes multiple generations of interfaces. The V3 -// interface is the latest one, which is capable of processing -// queries with no additional hints. V2 functions require -// the keyspace id or keyrange to be specified. V1 functions -// require shard info. V0 functions are informational that -// return topo information. Often, 'V2' or 'legacy' is used -// to refer to all legacy versions of the API (V2, V1 and V0). +// VTGate exposes multiple generations of interfaces. type VTGate struct { // Dependency: executor->resolver->scatterConn->txConn->gateway. - // VTGate still needs resolver and txConn to support legacy functions. executor *Executor resolver *Resolver vsm *vstreamManager diff --git a/go/vt/vtgate/vtgateconn/vtgateconn.go b/go/vt/vtgate/vtgateconn/vtgateconn.go index 87ce3620ad4..a1ec7a492be 100644 --- a/go/vt/vtgate/vtgateconn/vtgateconn.go +++ b/go/vt/vtgate/vtgateconn/vtgateconn.go @@ -36,10 +36,8 @@ var ( ) // VTGateConn is the client API object to talk to vtgate. -// It is constructed using the Dial method. It supports -// legacy V2 APIs. It can be used concurrently. To access -// V3 functionality, use the Session function to create a -// VTGateSession objects. +// It can support multiple simultaneous sessions. +// It is constructed using the Dial method. type VTGateConn struct { impl Impl } From 19ba263158fbd090c5016ab667ba030df3ab575b Mon Sep 17 00:00:00 2001 From: Saif Alharthi Date: Wed, 25 Mar 2020 15:05:40 -0700 Subject: [PATCH 49/85] Fix prepared statements results parsing for signed values Signed-off-by: Saif Alharthi --- go/mysql/query.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/mysql/query.go b/go/mysql/query.go index 8d340a1e6cd..ef37f6eb5ae 100644 --- a/go/mysql/query.go +++ b/go/mysql/query.go @@ -609,7 +609,7 @@ func (c *Conn) parseStmtArgs(data []byte, typ querypb.Type, pos int) (sqltypes.V return sqltypes.NULL, pos, true case sqltypes.Int8: val, pos, ok := readByte(data, pos) - return sqltypes.NewInt64(int64(val)), pos, ok + return sqltypes.NewInt64(int64(int8(val))), pos, ok case sqltypes.Uint8: val, pos, ok := readByte(data, pos) return sqltypes.NewUint64(uint64(val)), pos, ok @@ -618,13 +618,13 @@ func (c *Conn) parseStmtArgs(data []byte, typ querypb.Type, pos int) (sqltypes.V return sqltypes.NewUint64(uint64(val)), pos, ok case sqltypes.Int16, sqltypes.Year: val, pos, ok := readUint16(data, pos) - return sqltypes.NewInt64(int64(val)), pos, ok + return sqltypes.NewInt64(int64(int16(val))), pos, ok case sqltypes.Uint24, sqltypes.Uint32: val, pos, ok := readUint32(data, pos) return sqltypes.NewUint64(uint64(val)), pos, ok case sqltypes.Int24, sqltypes.Int32: val, pos, ok := readUint32(data, pos) - return sqltypes.NewInt64(int64(val)), pos, ok + return sqltypes.NewInt64(int64(int32(val))), pos, ok case sqltypes.Float32: val, pos, ok := readUint32(data, pos) return sqltypes.NewFloat64(float64(math.Float32frombits(uint32(val)))), pos, ok From 368fb6c06e32fe9d7adbc6f533e468eccb0a5b38 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Wed, 25 Mar 2020 15:32:13 -0700 Subject: [PATCH 50/85] deprecation: delete obsolete binlog test Signed-off-by: Sugu Sougoumarane --- go/vt/binlog/keyrange_filter_test.go | 202 --------------------------- go/vt/binlog/tables_filter_test.go | 10 ++ 2 files changed, 10 insertions(+), 202 deletions(-) delete mode 100644 go/vt/binlog/keyrange_filter_test.go diff --git a/go/vt/binlog/keyrange_filter_test.go b/go/vt/binlog/keyrange_filter_test.go deleted file mode 100644 index ee92fb0ddaf..00000000000 --- a/go/vt/binlog/keyrange_filter_test.go +++ /dev/null @@ -1,202 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package binlog - -import ( - "fmt" - "testing" - - binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" - querypb "vitess.io/vitess/go/vt/proto/query" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" -) - -var testKeyRange = &topodatapb.KeyRange{ - Start: []byte{}, - End: []byte{0x10}, -} - -func TestKeyRangeFilterPass(t *testing.T) { - statements := []FullBinlogStatement{ - { - Statement: &binlogdatapb.BinlogTransaction_Statement{ - Category: binlogdatapb.BinlogTransaction_Statement_BL_SET, - Sql: []byte("set1"), - }, - }, - { - Statement: &binlogdatapb.BinlogTransaction_Statement{ - Category: binlogdatapb.BinlogTransaction_Statement_BL_INSERT, - Sql: []byte("insert into tbl(col1, col2) values(1, a) /* vtgate:: keyspace_id:02 */"), - }, - }, - { - Statement: &binlogdatapb.BinlogTransaction_Statement{ - Category: binlogdatapb.BinlogTransaction_Statement_BL_INSERT, - Sql: []byte("insert into tbl(col1, col2, col3) values(1, 2, 3),(4, 5, 6) /* vtgate:: keyspace_id:01,02 *//*trailing_comments */"), - }, - }, - { - Statement: &binlogdatapb.BinlogTransaction_Statement{ - Category: binlogdatapb.BinlogTransaction_Statement_BL_INSERT, - Sql: []byte("insert into tbl(col1, col2, col3) values(1, 2, 3),(4, 5, 6) /* vtgate:: keyspace_id:01,20 *//*trailing_comments */"), - }, - }, - { - Statement: &binlogdatapb.BinlogTransaction_Statement{ - Category: binlogdatapb.BinlogTransaction_Statement_BL_INSERT, - Sql: []byte("insert into tbl(col1, col2, col3) values(1, 2, 3),(4, 5, 6) /* vtgate:: keyspace_id:10,20 *//*trailing_comments */"), - }, - }, - { - Statement: &binlogdatapb.BinlogTransaction_Statement{ - Category: binlogdatapb.BinlogTransaction_Statement_BL_UPDATE, - Sql: []byte("update tbl set col1=1"), - }, - }, - { - Statement: &binlogdatapb.BinlogTransaction_Statement{ - Category: binlogdatapb.BinlogTransaction_Statement_BL_DELETE, - Sql: []byte("delete from tbl where col1=1"), - }, - }, - } - eventToken := &querypb.EventToken{ - Position: "MariaDB/0-41983-1", - } - var got string - f := KeyRangeFilterFunc(testKeyRange, func(reply *binlogdatapb.BinlogTransaction) error { - got = bltToString(reply) - return nil - }) - f(eventToken, statements) - want := `statement: <6, "set1"> statement: <7, "insert into tbl(col1, col2) values(1, a) /* vtgate:: keyspace_id:02 */"> statement: <7, "insert into tbl(col1, col2, col3) values (1, 2, 3), (4, 5, 6) /* vtgate:: keyspace_id:01,02 *//*trailing_comments */"> statement: <7, "insert into tbl(col1, col2, col3) values (1, 2, 3) /* vtgate:: keyspace_id:01,20 *//*trailing_comments */"> statement: <8, "update tbl set col1=1"> statement: <9, "delete from tbl where col1=1"> position: "MariaDB/0-41983-1" ` - if want != got { - t.Errorf("want\n%s, got\n%s", want, got) - } -} - -func TestKeyRangeFilterSkip(t *testing.T) { - statements := []FullBinlogStatement{ - { - Statement: &binlogdatapb.BinlogTransaction_Statement{ - Category: binlogdatapb.BinlogTransaction_Statement_BL_SET, - Sql: []byte("set1"), - }, - }, - { - Statement: &binlogdatapb.BinlogTransaction_Statement{ - Category: binlogdatapb.BinlogTransaction_Statement_BL_INSERT, - Sql: []byte("dml1 /* vtgate:: keyspace_id:20 */"), - }, - }, - } - eventToken := &querypb.EventToken{ - Position: "MariaDB/0-41983-1", - } - var got string - f := KeyRangeFilterFunc(testKeyRange, func(reply *binlogdatapb.BinlogTransaction) error { - got = bltToString(reply) - return nil - }) - f(eventToken, statements) - want := `position: "MariaDB/0-41983-1" ` - if want != got { - t.Errorf("want %s, got %s", want, got) - } -} - -func TestKeyRangeFilterDDL(t *testing.T) { - statements := []FullBinlogStatement{ - { - Statement: &binlogdatapb.BinlogTransaction_Statement{ - Category: binlogdatapb.BinlogTransaction_Statement_BL_SET, - Sql: []byte("set1"), - }, - }, - { - Statement: &binlogdatapb.BinlogTransaction_Statement{ - Category: binlogdatapb.BinlogTransaction_Statement_BL_DDL, - Sql: []byte("ddl"), - }, - }, - } - eventToken := &querypb.EventToken{ - Position: "MariaDB/0-41983-1", - } - var got string - f := KeyRangeFilterFunc(testKeyRange, func(reply *binlogdatapb.BinlogTransaction) error { - got = bltToString(reply) - return nil - }) - f(eventToken, statements) - want := `position: "MariaDB/0-41983-1" ` - if want != got { - t.Errorf("want %s, got %s", want, got) - } -} - -func TestKeyRangeFilterMalformed(t *testing.T) { - statements := []FullBinlogStatement{ - { - Statement: &binlogdatapb.BinlogTransaction_Statement{ - Category: binlogdatapb.BinlogTransaction_Statement_BL_SET, - Sql: []byte("set1"), - }, - }, - { - Statement: &binlogdatapb.BinlogTransaction_Statement{ - Category: binlogdatapb.BinlogTransaction_Statement_BL_INSERT, - Sql: []byte("ddl"), - }, - }, - { - Statement: &binlogdatapb.BinlogTransaction_Statement{ - Category: binlogdatapb.BinlogTransaction_Statement_BL_INSERT, - Sql: []byte("dml1 /* vtgate:: keyspace_id:20*/"), - }, - }, - { - Statement: &binlogdatapb.BinlogTransaction_Statement{ - Category: binlogdatapb.BinlogTransaction_Statement_BL_INSERT, - Sql: []byte("dml1 /* vtgate:: keyspace_id:2 */"), // Odd-length hex string. - }, - }, - } - eventToken := &querypb.EventToken{ - Position: "MariaDB/0-41983-1", - } - var got string - f := KeyRangeFilterFunc(testKeyRange, func(reply *binlogdatapb.BinlogTransaction) error { - got = bltToString(reply) - return nil - }) - f(eventToken, statements) - want := `position: "MariaDB/0-41983-1" ` - if want != got { - t.Errorf("want %s, got %s", want, got) - } -} - -func bltToString(tx *binlogdatapb.BinlogTransaction) string { - result := "" - for _, statement := range tx.Statements { - result += fmt.Sprintf("statement: <%d, \"%s\"> ", statement.Category, string(statement.Sql)) - } - result += fmt.Sprintf("position: \"%v\" ", tx.EventToken.Position) - return result -} diff --git a/go/vt/binlog/tables_filter_test.go b/go/vt/binlog/tables_filter_test.go index 6f849143e0f..aa4fa36b77e 100644 --- a/go/vt/binlog/tables_filter_test.go +++ b/go/vt/binlog/tables_filter_test.go @@ -17,6 +17,7 @@ limitations under the License. package binlog import ( + "fmt" "testing" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" @@ -159,3 +160,12 @@ func TestTablesFilterMalformed(t *testing.T) { t.Errorf("want %s, got %s", want, got) } } + +func bltToString(tx *binlogdatapb.BinlogTransaction) string { + result := "" + for _, statement := range tx.Statements { + result += fmt.Sprintf("statement: <%d, \"%s\"> ", statement.Category, string(statement.Sql)) + } + result += fmt.Sprintf("position: \"%v\" ", tx.EventToken.Position) + return result +} From 065ba2061a4aab6970b9a4b3d9c2fcd38100e3c1 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Wed, 25 Mar 2020 15:38:35 -0700 Subject: [PATCH 51/85] deprecation: fix vtexplain tests Signed-off-by: Sugu Sougoumarane --- .../multi-output/deletesharded-output.txt | 28 +++++++-------- .../multi-output/insertsharded-output.txt | 28 +++++++-------- .../testdata/multi-output/options-output.txt | 4 +-- .../testdata/multi-output/target-output.txt | 2 +- .../multi-output/updatesharded-output.txt | 36 +++++++++---------- 5 files changed, 49 insertions(+), 49 deletions(-) diff --git a/go/vt/vtexplain/testdata/multi-output/deletesharded-output.txt b/go/vt/vtexplain/testdata/multi-output/deletesharded-output.txt index ed6ca3710c8..1c07d72904a 100644 --- a/go/vt/vtexplain/testdata/multi-output/deletesharded-output.txt +++ b/go/vt/vtexplain/testdata/multi-output/deletesharded-output.txt @@ -2,14 +2,14 @@ delete from music_extra where id=1 1 ks_sharded/-40: begin -1 ks_sharded/-40: delete from music_extra where id = 1 limit 10001 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ +1 ks_sharded/-40: delete from music_extra where id = 1 limit 10001 1 ks_sharded/-40: commit ---------------------------------------------------------------------- delete from music_extra where id=1 and extra='abc' 1 ks_sharded/-40: begin -1 ks_sharded/-40: delete from music_extra where id = 1 and extra = 'abc' limit 10001 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ +1 ks_sharded/-40: delete from music_extra where id = 1 and extra = 'abc' limit 10001 1 ks_sharded/-40: commit ---------------------------------------------------------------------- @@ -18,8 +18,8 @@ delete from user where id=1 1 ks_sharded/-40: begin 1 ks_sharded/-40: select id, name from user where id = 1 limit 10001 for update 2 ks_sharded/40-80: begin -2 ks_sharded/40-80: delete from name_user_map where name = 'name_val_2' and user_id = 1 limit 10001 /* vtgate:: keyspace_id:73004f940e97faf0a1b54ec5586a090e */ -3 ks_sharded/-40: delete from user where id = 1 limit 10001 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ +2 ks_sharded/40-80: delete from name_user_map where name = 'name_val_2' and user_id = 1 limit 10001 +3 ks_sharded/-40: delete from user where id = 1 limit 10001 4 ks_sharded/-40: commit 5 ks_sharded/40-80: commit @@ -31,8 +31,8 @@ delete from user where name='billy' 2 ks_sharded/-40: begin 2 ks_sharded/-40: select id, name from user where name = 'billy' limit 10001 for update 3 ks_sharded/40-80: begin -3 ks_sharded/40-80: delete from name_user_map where name = 'name_val_2' and user_id = 1 limit 10001 /* vtgate:: keyspace_id:73004f940e97faf0a1b54ec5586a090e */ -4 ks_sharded/-40: delete from user where name = 'billy' limit 10001 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ +3 ks_sharded/40-80: delete from name_user_map where name = 'name_val_2' and user_id = 1 limit 10001 +4 ks_sharded/-40: delete from user where name = 'billy' limit 10001 5 ks_sharded/c0-: commit 6 ks_sharded/-40: commit 7 ks_sharded/40-80: commit @@ -41,32 +41,32 @@ delete from user where name='billy' delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra='abc' 1 ks_sharded/-40: begin -1 ks_sharded/-40: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10001/* vtgate:: filtered_replication_unfriendly */ +1 ks_sharded/-40: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10001 1 ks_sharded/-40: commit 1 ks_sharded/40-80: begin -1 ks_sharded/40-80: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10001/* vtgate:: filtered_replication_unfriendly */ +1 ks_sharded/40-80: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10001 1 ks_sharded/40-80: commit 1 ks_sharded/80-c0: begin -1 ks_sharded/80-c0: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10001/* vtgate:: filtered_replication_unfriendly */ +1 ks_sharded/80-c0: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10001 1 ks_sharded/80-c0: commit 1 ks_sharded/c0-: begin -1 ks_sharded/c0-: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10001/* vtgate:: filtered_replication_unfriendly */ +1 ks_sharded/c0-: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10001 1 ks_sharded/c0-: commit ---------------------------------------------------------------------- delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from `ks_sharded[-]`.music_extra where extra='abc' LIMIT 10 1 ks_sharded/-40: begin -1 ks_sharded/-40: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10/* vtgate:: filtered_replication_unfriendly */ +1 ks_sharded/-40: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10 1 ks_sharded/-40: commit 1 ks_sharded/40-80: begin -1 ks_sharded/40-80: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10/* vtgate:: filtered_replication_unfriendly */ +1 ks_sharded/40-80: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10 1 ks_sharded/40-80: commit 1 ks_sharded/80-c0: begin -1 ks_sharded/80-c0: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10/* vtgate:: filtered_replication_unfriendly */ +1 ks_sharded/80-c0: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10 1 ks_sharded/80-c0: commit 1 ks_sharded/c0-: begin -1 ks_sharded/c0-: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10/* vtgate:: filtered_replication_unfriendly */ +1 ks_sharded/c0-: delete /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ from music_extra where extra = 'abc' limit 10 1 ks_sharded/c0-: commit ---------------------------------------------------------------------- diff --git a/go/vt/vtexplain/testdata/multi-output/insertsharded-output.txt b/go/vt/vtexplain/testdata/multi-output/insertsharded-output.txt index 2f972965b95..1de476b4d5d 100644 --- a/go/vt/vtexplain/testdata/multi-output/insertsharded-output.txt +++ b/go/vt/vtexplain/testdata/multi-output/insertsharded-output.txt @@ -2,9 +2,9 @@ insert into user (id, name) values(1, 'alice') 1 ks_sharded/40-80: begin -1 ks_sharded/40-80: insert into name_user_map(name, user_id) values ('alice', 1) /* vtgate:: keyspace_id:475e26c086f437f36bd72ecd883504a7 */ +1 ks_sharded/40-80: insert into name_user_map(name, user_id) values ('alice', 1) 2 ks_sharded/-40: begin -2 ks_sharded/-40: insert into user(id, name) values (1, 'alice') /* vtgate:: keyspace_id:166b40b44aba4bd6 */ +2 ks_sharded/-40: insert into user(id, name) values (1, 'alice') 3 ks_sharded/40-80: commit 4 ks_sharded/-40: commit @@ -12,9 +12,9 @@ insert into user (id, name) values(1, 'alice') insert into user (id, name) values(2, 'bob') 1 ks_sharded/c0-: begin -1 ks_sharded/c0-: insert into name_user_map(name, user_id) values ('bob', 2) /* vtgate:: keyspace_id:da8a82595aa28154c17717955ffeed8b */ +1 ks_sharded/c0-: insert into name_user_map(name, user_id) values ('bob', 2) 2 ks_sharded/-40: begin -2 ks_sharded/-40: insert into user(id, name) values (2, 'bob') /* vtgate:: keyspace_id:06e7ea22ce92708f */ +2 ks_sharded/-40: insert into user(id, name) values (2, 'bob') 3 ks_sharded/c0-: commit 4 ks_sharded/-40: commit @@ -22,10 +22,10 @@ insert into user (id, name) values(2, 'bob') insert ignore into user (id, name) values(2, 'bob') 1 ks_sharded/c0-: begin -1 ks_sharded/c0-: insert ignore into name_user_map(name, user_id) values ('bob', 2) /* vtgate:: keyspace_id:da8a82595aa28154c17717955ffeed8b */ +1 ks_sharded/c0-: insert ignore into name_user_map(name, user_id) values ('bob', 2) 2 ks_sharded/c0-: select name from name_user_map where name = 'bob' and user_id = 2 limit 10001 3 ks_sharded/-40: begin -3 ks_sharded/-40: insert ignore into user(id, name) values (2, 'bob') /* vtgate:: keyspace_id:06e7ea22ce92708f */ +3 ks_sharded/-40: insert ignore into user(id, name) values (2, 'bob') 4 ks_sharded/c0-: commit 5 ks_sharded/-40: commit @@ -33,10 +33,10 @@ insert ignore into user (id, name) values(2, 'bob') insert ignore into user (id, name, nickname) values(2, 'bob', 'bob') 1 ks_sharded/c0-: begin -1 ks_sharded/c0-: insert ignore into name_user_map(name, user_id) values ('bob', 2) /* vtgate:: keyspace_id:da8a82595aa28154c17717955ffeed8b */ +1 ks_sharded/c0-: insert ignore into name_user_map(name, user_id) values ('bob', 2) 2 ks_sharded/c0-: select name from name_user_map where name = 'bob' and user_id = 2 limit 10001 3 ks_sharded/-40: begin -3 ks_sharded/-40: insert ignore into user(id, name, nickname) values (2, 'bob', 'bob') /* vtgate:: keyspace_id:06e7ea22ce92708f */ +3 ks_sharded/-40: insert ignore into user(id, name, nickname) values (2, 'bob', 'bob') 4 ks_sharded/c0-: commit 5 ks_sharded/-40: commit @@ -44,10 +44,10 @@ insert ignore into user (id, name, nickname) values(2, 'bob', 'bob') insert into user (id, name, nickname) values(2, 'bob', 'bobby') on duplicate key update nickname='bobby' 1 ks_sharded/c0-: begin -1 ks_sharded/c0-: insert ignore into name_user_map(name, user_id) values ('bob', 2) /* vtgate:: keyspace_id:da8a82595aa28154c17717955ffeed8b */ +1 ks_sharded/c0-: insert ignore into name_user_map(name, user_id) values ('bob', 2) 2 ks_sharded/c0-: select name from name_user_map where name = 'bob' and user_id = 2 limit 10001 3 ks_sharded/-40: begin -3 ks_sharded/-40: insert into user(id, name, nickname) values (2, 'bob', 'bobby') on duplicate key update nickname = 'bobby' /* vtgate:: keyspace_id:06e7ea22ce92708f */ +3 ks_sharded/-40: insert into user(id, name, nickname) values (2, 'bob', 'bobby') on duplicate key update nickname = 'bobby' 4 ks_sharded/c0-: commit 5 ks_sharded/-40: commit @@ -55,10 +55,10 @@ insert into user (id, name, nickname) values(2, 'bob', 'bobby') on duplicate key insert into user (id, name, nickname, address) values(2, 'bob', 'bobby', '123 main st') on duplicate key update nickname=values(nickname), address=values(address) 1 ks_sharded/c0-: begin -1 ks_sharded/c0-: insert ignore into name_user_map(name, user_id) values ('bob', 2) /* vtgate:: keyspace_id:da8a82595aa28154c17717955ffeed8b */ +1 ks_sharded/c0-: insert ignore into name_user_map(name, user_id) values ('bob', 2) 2 ks_sharded/c0-: select name from name_user_map where name = 'bob' and user_id = 2 limit 10001 3 ks_sharded/-40: begin -3 ks_sharded/-40: insert into user(id, name, nickname, address) values (2, 'bob', 'bobby', '123 main st') on duplicate key update nickname = values(nickname), address = values(address) /* vtgate:: keyspace_id:06e7ea22ce92708f */ +3 ks_sharded/-40: insert into user(id, name, nickname, address) values (2, 'bob', 'bobby', '123 main st') on duplicate key update nickname = values(nickname), address = values(address) 4 ks_sharded/c0-: commit 5 ks_sharded/-40: commit @@ -66,10 +66,10 @@ insert into user (id, name, nickname, address) values(2, 'bob', 'bobby', '123 ma insert /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ into music_extra (id, extra) values (1, 'a'), (2, 'b'), (3, 'c') 1 ks_sharded/-40: begin -1 ks_sharded/-40: insert /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ into music_extra(id, extra) values (1, 'a'), (2, 'b') /* vtgate:: keyspace_id:166b40b44aba4bd6,06e7ea22ce92708f */ +1 ks_sharded/-40: insert /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ into music_extra(id, extra) values (1, 'a'), (2, 'b') 1 ks_sharded/-40: commit 1 ks_sharded/40-80: begin -1 ks_sharded/40-80: insert /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ into music_extra(id, extra) values (3, 'c') /* vtgate:: keyspace_id:4eb190c9a2fa169c */ +1 ks_sharded/40-80: insert /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ into music_extra(id, extra) values (3, 'c') 1 ks_sharded/40-80: commit ---------------------------------------------------------------------- diff --git a/go/vt/vtexplain/testdata/multi-output/options-output.txt b/go/vt/vtexplain/testdata/multi-output/options-output.txt index fa6f2610656..9784cf89477 100644 --- a/go/vt/vtexplain/testdata/multi-output/options-output.txt +++ b/go/vt/vtexplain/testdata/multi-output/options-output.txt @@ -17,9 +17,9 @@ select * from user where id in (1,2,3,4,5,6,7,8) insert into user (id, name) values (2, 'bob') 1 ks_sharded/c0-: begin -1 ks_sharded/c0-: insert into name_user_map(name, user_id) values ('bob', 2) /* vtgate:: keyspace_id:da8a82595aa28154c17717955ffeed8b */ +1 ks_sharded/c0-: insert into name_user_map(name, user_id) values ('bob', 2) 2 ks_sharded/-40: begin -2 ks_sharded/-40: insert into user(id, name) values (2, 'bob') /* vtgate:: keyspace_id:06e7ea22ce92708f */ +2 ks_sharded/-40: insert into user(id, name) values (2, 'bob') 3 ks_sharded/c0-: commit 4 ks_sharded/-40: commit diff --git a/go/vt/vtexplain/testdata/multi-output/target-output.txt b/go/vt/vtexplain/testdata/multi-output/target-output.txt index 10ce1dddf1e..abf70f5b57d 100644 --- a/go/vt/vtexplain/testdata/multi-output/target-output.txt +++ b/go/vt/vtexplain/testdata/multi-output/target-output.txt @@ -12,7 +12,7 @@ select * from user where id in (1,2,3,4,5,6,7,8) insert into user (id, name) values (2, 'bob') 1 ks_sharded/40-80: begin -1 ks_sharded/40-80: insert into user(id, name) values (2, 'bob')/* vtgate:: filtered_replication_unfriendly */ +1 ks_sharded/40-80: insert into user(id, name) values (2, 'bob') 1 ks_sharded/40-80: commit ---------------------------------------------------------------------- diff --git a/go/vt/vtexplain/testdata/multi-output/updatesharded-output.txt b/go/vt/vtexplain/testdata/multi-output/updatesharded-output.txt index 95b03cc4327..682802ac696 100644 --- a/go/vt/vtexplain/testdata/multi-output/updatesharded-output.txt +++ b/go/vt/vtexplain/testdata/multi-output/updatesharded-output.txt @@ -2,7 +2,7 @@ update user set nickname='alice' where id=1 1 ks_sharded/-40: begin -1 ks_sharded/-40: update user set nickname = 'alice' where id = 1 limit 10001 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ +1 ks_sharded/-40: update user set nickname = 'alice' where id = 1 limit 10001 1 ks_sharded/-40: commit ---------------------------------------------------------------------- @@ -11,7 +11,7 @@ update user set nickname='alice' where name='alice' 1 ks_sharded/40-80: begin 1 ks_sharded/40-80: select user_id from name_user_map where name = 'alice' limit 10001 2 ks_sharded/-40: begin -2 ks_sharded/-40: update user set nickname = 'alice' where name = 'alice' limit 10001 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ +2 ks_sharded/-40: update user set nickname = 'alice' where name = 'alice' limit 10001 3 ks_sharded/40-80: commit 4 ks_sharded/-40: commit @@ -19,7 +19,7 @@ update user set nickname='alice' where name='alice' update user set pet='fido' where id=1 1 ks_sharded/-40: begin -1 ks_sharded/-40: update user set pet = 'fido' where id = 1 limit 10001 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ +1 ks_sharded/-40: update user set pet = 'fido' where id = 1 limit 10001 1 ks_sharded/-40: commit ---------------------------------------------------------------------- @@ -28,10 +28,10 @@ update user set name='alicia' where id=1 1 ks_sharded/-40: begin 1 ks_sharded/-40: select id, name from user where id = 1 limit 10001 for update 2 ks_sharded/40-80: begin -2 ks_sharded/40-80: delete from name_user_map where name = 'name_val_2' and user_id = 1 limit 10001 /* vtgate:: keyspace_id:73004f940e97faf0a1b54ec5586a090e */ +2 ks_sharded/40-80: delete from name_user_map where name = 'name_val_2' and user_id = 1 limit 10001 3 ks_sharded/c0-: begin -3 ks_sharded/c0-: insert into name_user_map(name, user_id) values ('alicia', 1) /* vtgate:: keyspace_id:e2821261367fbee90bb5cf72955146c6 */ -4 ks_sharded/-40: update user set name = 'alicia' where id = 1 limit 10001 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ +3 ks_sharded/c0-: insert into name_user_map(name, user_id) values ('alicia', 1) +4 ks_sharded/-40: update user set name = 'alicia' where id = 1 limit 10001 5 ks_sharded/-40: commit 6 ks_sharded/40-80: commit 7 ks_sharded/c0-: commit @@ -43,10 +43,10 @@ update user set name='alicia' where name='alice' 1 ks_sharded/40-80: select user_id from name_user_map where name = 'alice' limit 10001 2 ks_sharded/-40: begin 2 ks_sharded/-40: select id, name from user where name = 'alice' limit 10001 for update -3 ks_sharded/40-80: delete from name_user_map where name = 'name_val_2' and user_id = 1 limit 10001 /* vtgate:: keyspace_id:73004f940e97faf0a1b54ec5586a090e */ +3 ks_sharded/40-80: delete from name_user_map where name = 'name_val_2' and user_id = 1 limit 10001 4 ks_sharded/c0-: begin -4 ks_sharded/c0-: insert into name_user_map(name, user_id) values ('alicia', 1) /* vtgate:: keyspace_id:e2821261367fbee90bb5cf72955146c6 */ -5 ks_sharded/-40: update user set name = 'alicia' where name = 'alice' limit 10001 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ +4 ks_sharded/c0-: insert into name_user_map(name, user_id) values ('alicia', 1) +5 ks_sharded/-40: update user set name = 'alicia' where name = 'alice' limit 10001 6 ks_sharded/40-80: commit 7 ks_sharded/-40: commit 8 ks_sharded/c0-: commit @@ -55,16 +55,16 @@ update user set name='alicia' where name='alice' update /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ name_info set has_nickname=1 where nickname != '' 1 ks_sharded/-40: begin -1 ks_sharded/-40: update /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ name_info set has_nickname = 1 where nickname != '' limit 10001/* vtgate:: filtered_replication_unfriendly */ +1 ks_sharded/-40: update /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ name_info set has_nickname = 1 where nickname != '' limit 10001 1 ks_sharded/-40: commit 1 ks_sharded/40-80: begin -1 ks_sharded/40-80: update /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ name_info set has_nickname = 1 where nickname != '' limit 10001/* vtgate:: filtered_replication_unfriendly */ +1 ks_sharded/40-80: update /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ name_info set has_nickname = 1 where nickname != '' limit 10001 1 ks_sharded/40-80: commit 1 ks_sharded/80-c0: begin -1 ks_sharded/80-c0: update /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ name_info set has_nickname = 1 where nickname != '' limit 10001/* vtgate:: filtered_replication_unfriendly */ +1 ks_sharded/80-c0: update /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ name_info set has_nickname = 1 where nickname != '' limit 10001 1 ks_sharded/80-c0: commit 1 ks_sharded/c0-: begin -1 ks_sharded/c0-: update /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ name_info set has_nickname = 1 where nickname != '' limit 10001/* vtgate:: filtered_replication_unfriendly */ +1 ks_sharded/c0-: update /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ name_info set has_nickname = 1 where nickname != '' limit 10001 1 ks_sharded/c0-: commit ---------------------------------------------------------------------- @@ -73,7 +73,7 @@ update user set pet='rover' where name='alice' 1 ks_sharded/40-80: begin 1 ks_sharded/40-80: select user_id from name_user_map where name = 'alice' limit 10001 2 ks_sharded/-40: begin -2 ks_sharded/-40: update user set pet = 'rover' where name = 'alice' limit 10001 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ +2 ks_sharded/-40: update user set pet = 'rover' where name = 'alice' limit 10001 3 ks_sharded/40-80: commit 4 ks_sharded/-40: commit @@ -85,12 +85,12 @@ begin update user set nickname='alice' where id=1 1 ks_sharded/-40: begin -1 ks_sharded/-40: update user set nickname = 'alice' where id = 1 limit 10001 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ +1 ks_sharded/-40: update user set nickname = 'alice' where id = 1 limit 10001 ---------------------------------------------------------------------- update user set nickname='bob' where id=1 -2 ks_sharded/-40: update user set nickname = 'bob' where id = 1 limit 10001 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ +2 ks_sharded/-40: update user set nickname = 'bob' where id = 1 limit 10001 ---------------------------------------------------------------------- commit @@ -105,13 +105,13 @@ begin update user set nickname='alice' where id=1 1 ks_sharded/-40: begin -1 ks_sharded/-40: update user set nickname = 'alice' where id = 1 limit 10001 /* vtgate:: keyspace_id:166b40b44aba4bd6 */ +1 ks_sharded/-40: update user set nickname = 'alice' where id = 1 limit 10001 ---------------------------------------------------------------------- update user set nickname='bob' where id=3 2 ks_sharded/40-80: begin -2 ks_sharded/40-80: update user set nickname = 'bob' where id = 3 limit 10001 /* vtgate:: keyspace_id:4eb190c9a2fa169c */ +2 ks_sharded/40-80: update user set nickname = 'bob' where id = 3 limit 10001 ---------------------------------------------------------------------- commit From 066093f202e7b7f208888886389b4d322226e831 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Wed, 25 Mar 2020 17:15:53 -0700 Subject: [PATCH 52/85] deprecation: address review comments Signed-off-by: Sugu Sougoumarane --- go/vt/binlog/keyrange_filter.go | 2 -- go/vt/vtgate/vtgateconn/vtgateconn.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/go/vt/binlog/keyrange_filter.go b/go/vt/binlog/keyrange_filter.go index 3808d2e38f5..5600dd837c1 100644 --- a/go/vt/binlog/keyrange_filter.go +++ b/go/vt/binlog/keyrange_filter.go @@ -43,13 +43,11 @@ func KeyRangeFilterFunc(keyrange *topodatapb.KeyRange, callback func(*binlogdata case binlogdatapb.BinlogTransaction_Statement_BL_INSERT, binlogdatapb.BinlogTransaction_Statement_BL_UPDATE, binlogdatapb.BinlogTransaction_Statement_BL_DELETE: - // Handle RBR case first. if statement.KeyspaceID != nil { updateStreamErrors.Add("KeyRangeStream", 1) log.Errorf("SBR mode unsupported for streaming: %s", statement.Statement.Sql) } if !key.KeyRangeContains(keyrange, statement.KeyspaceID) { - // Skip keyspace ids that don't belong to the destination shard. continue } filtered = append(filtered, statement.Statement) diff --git a/go/vt/vtgate/vtgateconn/vtgateconn.go b/go/vt/vtgate/vtgateconn/vtgateconn.go index a1ec7a492be..8d9ebd2f6e6 100644 --- a/go/vt/vtgate/vtgateconn/vtgateconn.go +++ b/go/vt/vtgate/vtgateconn/vtgateconn.go @@ -36,7 +36,7 @@ var ( ) // VTGateConn is the client API object to talk to vtgate. -// It can support multiple simultaneous sessions. +// It can support concurrent sessions. // It is constructed using the Dial method. type VTGateConn struct { impl Impl From 822453824890bf253e2e6c4065617914fb5e235e Mon Sep 17 00:00:00 2001 From: Saif Alharthi Date: Wed, 25 Mar 2020 19:56:01 -0700 Subject: [PATCH 53/85] Modified prepared statement test to support limits of integer variables Signed-off-by: Saif Alharthi --- go/test/endtoend/preparestmt/main_test.go | 20 ++++++++++++++++ .../endtoend/preparestmt/stmt_methods_test.go | 24 ++++++++++++++++--- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/go/test/endtoend/preparestmt/main_test.go b/go/test/endtoend/preparestmt/main_test.go index 1fa24bf62a3..d00ec84da65 100644 --- a/go/test/endtoend/preparestmt/main_test.go +++ b/go/test/endtoend/preparestmt/main_test.go @@ -137,6 +137,26 @@ var ( json_col JSON, text_col TEXT, data longblob, + tinyint_min TINYINT, + tinyint_max TINYINT, + tinyint_pos TINYINT, + tinyint_neg TINYINT, + smallint_min SMALLINT, + smallint_max SMALLINT, + smallint_pos SMALLINT, + smallint_neg SMALLINT, + medint_min MEDIUMINT, + medint_max MEDIUMINT, + medint_pos MEDIUMINT, + medint_neg MEDIUMINT, + int_min INT, + int_max INT, + int_pos INT, + int_neg INT, + bigint_min BIGINT, + bigint_max BIGINT, + bigint_pos BIGINT, + bigint_neg BIGINT, primary key (id) ) Engine=InnoDB` ) diff --git a/go/test/endtoend/preparestmt/stmt_methods_test.go b/go/test/endtoend/preparestmt/stmt_methods_test.go index 7d4b84e9922..2016293823e 100644 --- a/go/test/endtoend/preparestmt/stmt_methods_test.go +++ b/go/test/endtoend/preparestmt/stmt_methods_test.go @@ -44,7 +44,8 @@ func TestInsertUpdateDelete(t *testing.T) { defer dbo.Close() // prepare insert statement insertStmt := `insert into ` + tableName + ` values( ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);` + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);` textValue := fake.FullName() largeComment := fake.Paragraph() @@ -62,6 +63,11 @@ func TestInsertUpdateDelete(t *testing.T) { time.Now(), time.Date(2009, 5, 5, 0, 0, 0, 50000, time.UTC), 1, 1, 1, 1, 1, 1, 1, 1, 1, jsonExample, textValue, largeComment, + -128, 127, 1, -1, + -32768, 32767, 1, -1, + -8388608, 8388607, 1, -1, + -2147483648, 2147483647, 1, -1, + -(1 << 63), (1 << 63) - 1, 1, -1, } exec(t, dbo, insertStmt, insertValue...) @@ -118,8 +124,15 @@ func TestAutoIncColumns(t *testing.T) { msg,keyspace_id,tinyint_unsigned,bool_signed,smallint_unsigned, mediumint_unsigned,int_unsigned,float_unsigned,double_unsigned, decimal_unsigned,t_date,t_datetime,t_datetime_micros,t_time,t_timestamp,c8,c16,c24, - c32,c40,c48,c56,c63,c64,json_col,text_col,data) VALUES (?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);` + c32,c40,c48,c56,c63,c64,json_col,text_col,data, + tinyint_min,tinyint_max,tinyint_pos,tinyint_neg, + smallint_min,smallint_max,smallint_pos,smallint_neg, + medint_min,medint_max,medint_pos,medint_neg, + int_min,int_max,int_pos,int_neg, + bigint_min,bigint_max,bigint_pos,bigint_neg +) VALUES (?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);` insertValue := []interface{}{ "21", 0, 127, 1, 32767, 8388607, 2147483647, 2.55, 64.9, 55.5, @@ -129,6 +142,11 @@ func TestAutoIncColumns(t *testing.T) { time.Now(), time.Date(2009, 5, 5, 0, 0, 0, 50000, time.UTC), 1, 1, 1, 1, 1, 1, 1, 1, 1, jsonExample, fake.DomainName(), fake.Paragraph(), + -128, 127, 1, -1, + -32768, 32767, 1, -1, + -8388608, 8388607, 1, -1, + -2147483648, 2147483647, 1, -1, + -(1 << 63), (1 << 63) - 1, 1, -1, } exec(t, dbo, insertStmt, insertValue...) From 2391ddbfd2e8b69d1cfa7c4461be46512722e8b5 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 24 Mar 2020 10:17:51 +0100 Subject: [PATCH 54/85] Cleaning of vcursor & executor Signed-off-by: Andres Taylor --- go/vt/vtgate/executor.go | 16 +++++++++++++--- go/vt/vtgate/executor_test.go | 16 ++++++++-------- go/vt/vtgate/vcursor_impl.go | 33 ++++++++++++++++++++++++--------- 3 files changed, 45 insertions(+), 20 deletions(-) diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 02f7da1353b..cdc75c46211 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -312,7 +312,7 @@ func (e *Executor) handleExec(ctx context.Context, safeSession *SafeSession, sql // V3 mode. query, comments := sqlparser.SplitMarginComments(sql) - vcursor := newVCursorImpl(ctx, safeSession, destKeyspace, destTabletType, comments, e, logStats) + vcursor := newVCursorImpl(ctx, safeSession, destKeyspace, destTabletType, comments, e, logStats, e.VSchema(), e.resolver.resolver) plan, err := e.getPlan( vcursor, query, @@ -1217,7 +1217,7 @@ func (e *Executor) StreamExecute(ctx context.Context, method string, safeSession bindVars = make(map[string]*querypb.BindVariable) } query, comments := sqlparser.SplitMarginComments(sql) - vcursor := newVCursorImpl(ctx, safeSession, target.Keyspace, target.TabletType, comments, e, logStats) + vcursor := newVCursorImpl(ctx, safeSession, target.Keyspace, target.TabletType, comments, e, logStats, e.VSchema(), e.resolver.resolver) // check if this is a stream statement for messaging // TODO: support keyRange syntax @@ -1664,7 +1664,7 @@ func (e *Executor) prepare(ctx context.Context, safeSession *SafeSession, sql st func (e *Executor) handlePrepare(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, destKeyspace string, destTabletType topodatapb.TabletType, logStats *LogStats) ([]*querypb.Field, error) { // V3 mode. query, comments := sqlparser.SplitMarginComments(sql) - vcursor := newVCursorImpl(ctx, safeSession, destKeyspace, destTabletType, comments, e, logStats) + vcursor := newVCursorImpl(ctx, safeSession, destKeyspace, destTabletType, comments, e, logStats, e.VSchema(), e.resolver.resolver) plan, err := e.getPlan( vcursor, query, @@ -1695,3 +1695,13 @@ func (e *Executor) handlePrepare(ctx context.Context, safeSession *SafeSession, return qr.Fields, err } + +// ExecuteMultiShard implements the IExecutor interface +func (e *Executor) ExecuteMultiShard(ctx context.Context, rss []*srvtopo.ResolvedShard, queries []*querypb.BoundQuery, tabletType topodatapb.TabletType, session *SafeSession, notInTransaction bool, autocommit bool) (qr *sqltypes.Result, errs []error) { + return e.scatterConn.ExecuteMultiShard(ctx, rss, queries, tabletType, session, notInTransaction, autocommit) +} + +// StreamExecuteMulti implements the IExecutor interface +func (e *Executor) StreamExecuteMulti(ctx context.Context, query string, rss []*srvtopo.ResolvedShard, vars []map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(reply *sqltypes.Result) error) error { + return e.scatterConn.StreamExecuteMulti(ctx, query, rss, vars, tabletType, options, callback) +} diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index a33ae3adc7b..e9b8a67ac75 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -1959,8 +1959,8 @@ func TestVSchemaStats(t *testing.T) { func TestGetPlanUnnormalized(t *testing.T) { r, _, _, _ := createExecutorEnv() - emptyvc := newVCursorImpl(context.Background(), nil, "", 0, makeComments(""), r, nil) - unshardedvc := newVCursorImpl(context.Background(), nil, KsTestUnsharded, 0, makeComments(""), r, nil) + emptyvc := newVCursorImpl(context.Background(), nil, "", 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) + unshardedvc := newVCursorImpl(context.Background(), nil, KsTestUnsharded, 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) logStats1 := NewLogStats(context.Background(), "Test", "", nil) query1 := "select * from music_user_map where id = 1" @@ -2015,7 +2015,7 @@ func TestGetPlanUnnormalized(t *testing.T) { func TestGetPlanCacheUnnormalized(t *testing.T) { r, _, _, _ := createExecutorEnv() - emptyvc := newVCursorImpl(context.Background(), nil, "", 0, makeComments(""), r, nil) + emptyvc := newVCursorImpl(context.Background(), nil, "", 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) query1 := "select * from music_user_map where id = 1" logStats1 := NewLogStats(context.Background(), "Test", "", nil) _, err := r.getPlan(emptyvc, query1, makeComments(" /* comment */"), map[string]*querypb.BindVariable{}, true /* skipQueryPlanCache */, logStats1) @@ -2040,7 +2040,7 @@ func TestGetPlanCacheUnnormalized(t *testing.T) { // Skip cache using directive r, _, _, _ = createExecutorEnv() - unshardedvc := newVCursorImpl(context.Background(), nil, KsTestUnsharded, 0, makeComments(""), r, nil) + unshardedvc := newVCursorImpl(context.Background(), nil, KsTestUnsharded, 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) query1 = "insert /*vt+ SKIP_QUERY_PLAN_CACHE=1 */ into user(id) values (1), (2)" logStats1 = NewLogStats(context.Background(), "Test", "", nil) @@ -2061,7 +2061,7 @@ func TestGetPlanCacheUnnormalized(t *testing.T) { func TestGetPlanCacheNormalized(t *testing.T) { r, _, _, _ := createExecutorEnv() r.normalize = true - emptyvc := newVCursorImpl(context.Background(), nil, "", 0, makeComments(""), r, nil) + emptyvc := newVCursorImpl(context.Background(), nil, "", 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) query1 := "select * from music_user_map where id = 1" logStats1 := NewLogStats(context.Background(), "Test", "", nil) _, err := r.getPlan(emptyvc, query1, makeComments(" /* comment */"), map[string]*querypb.BindVariable{}, true /* skipQueryPlanCache */, logStats1) @@ -2086,7 +2086,7 @@ func TestGetPlanCacheNormalized(t *testing.T) { // Skip cache using directive r, _, _, _ = createExecutorEnv() r.normalize = true - unshardedvc := newVCursorImpl(context.Background(), nil, KsTestUnsharded, 0, makeComments(""), r, nil) + unshardedvc := newVCursorImpl(context.Background(), nil, KsTestUnsharded, 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) query1 = "insert /*vt+ SKIP_QUERY_PLAN_CACHE=1 */ into user(id) values (1), (2)" logStats1 = NewLogStats(context.Background(), "Test", "", nil) @@ -2107,8 +2107,8 @@ func TestGetPlanCacheNormalized(t *testing.T) { func TestGetPlanNormalized(t *testing.T) { r, _, _, _ := createExecutorEnv() r.normalize = true - emptyvc := newVCursorImpl(context.Background(), nil, "", 0, makeComments(""), r, nil) - unshardedvc := newVCursorImpl(context.Background(), nil, KsTestUnsharded, 0, makeComments(""), r, nil) + emptyvc := newVCursorImpl(context.Background(), nil, "", 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) + unshardedvc := newVCursorImpl(context.Background(), nil, KsTestUnsharded, 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) query1 := "select * from music_user_map where id = 1" query2 := "select * from music_user_map where id = 2" diff --git a/go/vt/vtgate/vcursor_impl.go b/go/vt/vtgate/vcursor_impl.go index 4d6ef1cb5e8..bba6596c4b3 100644 --- a/go/vt/vtgate/vcursor_impl.go +++ b/go/vt/vtgate/vcursor_impl.go @@ -37,6 +37,17 @@ import ( ) var _ engine.VCursor = (*vcursorImpl)(nil) +var _ iExecute = (*Executor)(nil) + +// vcursor_impl needs these facilities to be able to be able to execute queries for vindexes +type iExecute interface { + Execute(ctx context.Context, method string, session *SafeSession, s string, vars map[string]*querypb.BindVariable) (*sqltypes.Result, error) + ExecuteMultiShard(ctx context.Context, rss []*srvtopo.ResolvedShard, queries []*querypb.BoundQuery, tabletType topodatapb.TabletType, session *SafeSession, notInTransaction bool, autocommit bool) (qr *sqltypes.Result, errs []error) + StreamExecuteMulti(ctx context.Context, s string, rss []*srvtopo.ResolvedShard, vars []map[string]*querypb.BindVariable, tabletType topodatapb.TabletType, options *querypb.ExecuteOptions, callback func(reply *sqltypes.Result) error) error + + // TODO: remove when resolver is gone + ParseDestinationTarget(targetString string) (string, topodatapb.TabletType, key.Destination, error) +} // vcursorImpl implements the VCursor functionality used by dependent // packages to call back into VTGate. @@ -46,19 +57,21 @@ type vcursorImpl struct { keyspace string tabletType topodatapb.TabletType marginComments sqlparser.MarginComments - executor *Executor + executor iExecute + resolver *srvtopo.Resolver logStats *LogStats // hasPartialDML is set to true if any DML was successfully // executed. If there was a subsequent failure, the transaction // must be forced to rollback. hasPartialDML bool + vschema *vindexes.VSchema } // newVcursorImpl creates a vcursorImpl. Before creating this object, you have to separate out any marginComments that came with // the query and supply it here. Trailing comments are typically sent by the application for various reasons, // including as identifying markers. So, they have to be added back to all queries that are executed // on behalf of the original query. -func newVCursorImpl(ctx context.Context, safeSession *SafeSession, keyspace string, tabletType topodatapb.TabletType, marginComments sqlparser.MarginComments, executor *Executor, logStats *LogStats) *vcursorImpl { +func newVCursorImpl(ctx context.Context, safeSession *SafeSession, keyspace string, tabletType topodatapb.TabletType, marginComments sqlparser.MarginComments, executor *Executor, logStats *LogStats, vschema *vindexes.VSchema, resolver *srvtopo.Resolver) *vcursorImpl { return &vcursorImpl{ ctx: ctx, safeSession: safeSession, @@ -67,6 +80,8 @@ func newVCursorImpl(ctx context.Context, safeSession *SafeSession, keyspace stri marginComments: marginComments, executor: executor, logStats: logStats, + vschema: vschema, + resolver: resolver, } } @@ -102,7 +117,7 @@ func (vc *vcursorImpl) FindTable(name sqlparser.TableName) (*vindexes.Table, str if destKeyspace == "" { destKeyspace = vc.keyspace } - table, err := vc.executor.VSchema().FindTable(destKeyspace, name.Name.String()) + table, err := vc.vschema.FindTable(destKeyspace, name.Name.String()) if err != nil { return nil, "", destTabletType, nil, err } @@ -118,7 +133,7 @@ func (vc *vcursorImpl) FindTablesOrVindex(name sqlparser.TableName) ([]*vindexes if destKeyspace == "" { destKeyspace = vc.keyspace } - tables, vindex, err := vc.executor.VSchema().FindTablesOrVindex(destKeyspace, name.Name.String(), vc.tabletType) + tables, vindex, err := vc.vschema.FindTablesOrVindex(destKeyspace, name.Name.String(), vc.tabletType) if err != nil { return nil, nil, "", destTabletType, nil, err } @@ -132,7 +147,7 @@ func (vc *vcursorImpl) DefaultKeyspace() (*vindexes.Keyspace, error) { if vc.keyspace == "" { return nil, errNoKeyspace } - ks, ok := vc.executor.VSchema().Keyspaces[vc.keyspace] + ks, ok := vc.vschema.Keyspaces[vc.keyspace] if !ok { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "keyspace %s not found in vschema", vc.keyspace) } @@ -165,7 +180,7 @@ func (vc *vcursorImpl) Execute(method string, query string, bindVars map[string] // ExecuteMultiShard is part of the engine.VCursor interface. func (vc *vcursorImpl) ExecuteMultiShard(rss []*srvtopo.ResolvedShard, queries []*querypb.BoundQuery, isDML, autocommit bool) (*sqltypes.Result, []error) { atomic.AddUint32(&vc.logStats.ShardQueries, uint32(len(queries))) - qr, errs := vc.executor.scatterConn.ExecuteMultiShard(vc.ctx, rss, commentedShardQueries(queries, vc.marginComments), vc.tabletType, vc.safeSession, false, autocommit) + qr, errs := vc.executor.ExecuteMultiShard(vc.ctx, rss, commentedShardQueries(queries, vc.marginComments), vc.tabletType, vc.safeSession, false, autocommit) if errs == nil && isDML { vc.hasPartialDML = true @@ -189,14 +204,14 @@ func (vc *vcursorImpl) ExecuteStandalone(query string, bindVars map[string]*quer } // The autocommit flag is always set to false because we currently don't // execute DMLs through ExecuteStandalone. - qr, errs := vc.executor.scatterConn.ExecuteMultiShard(vc.ctx, rss, bqs, vc.tabletType, NewAutocommitSession(vc.safeSession.Session), false, false /* autocommit */) + qr, errs := vc.executor.ExecuteMultiShard(vc.ctx, rss, bqs, vc.tabletType, NewAutocommitSession(vc.safeSession.Session), false, false /* autocommit */) return qr, vterrors.Aggregate(errs) } // StreamExeculteMulti is the streaming version of ExecuteMultiShard. func (vc *vcursorImpl) StreamExecuteMulti(query string, rss []*srvtopo.ResolvedShard, bindVars []map[string]*querypb.BindVariable, callback func(reply *sqltypes.Result) error) error { atomic.AddUint32(&vc.logStats.ShardQueries, uint32(len(rss))) - return vc.executor.scatterConn.StreamExecuteMulti(vc.ctx, vc.marginComments.Leading+query+vc.marginComments.Trailing, rss, bindVars, vc.tabletType, vc.safeSession.Options, callback) + return vc.executor.StreamExecuteMulti(vc.ctx, vc.marginComments.Leading+query+vc.marginComments.Trailing, rss, bindVars, vc.tabletType, vc.safeSession.Options, callback) } // ExecuteKeyspaceID is part of the engine.VCursor interface. @@ -222,7 +237,7 @@ func (vc *vcursorImpl) ExecuteKeyspaceID(keyspace string, ksid []byte, query str } func (vc *vcursorImpl) ResolveDestinations(keyspace string, ids []*querypb.Value, destinations []key.Destination) ([]*srvtopo.ResolvedShard, [][]*querypb.Value, error) { - return vc.executor.resolver.resolver.ResolveDestinations(vc.ctx, keyspace, vc.tabletType, ids, destinations) + return vc.resolver.ResolveDestinations(vc.ctx, keyspace, vc.tabletType, ids, destinations) } func commentedShardQueries(shardQueries []*querypb.BoundQuery, marginComments sqlparser.MarginComments) []*querypb.BoundQuery { From 1a1fe683b9d6be423f86f76d30fc4ba9acc10c85 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 24 Mar 2020 10:20:55 +0100 Subject: [PATCH 55/85] Remove unused argument Signed-off-by: Andres Taylor --- go/vt/vtexplain/vtexplain_vtgate.go | 2 +- go/vt/vtgate/executor.go | 2 +- go/vt/vtgate/executor_framework_test.go | 4 ++-- go/vt/vtgate/executor_select_test.go | 22 +++++++++++----------- go/vt/vtgate/executor_stream_test.go | 2 +- go/vt/vtgate/vtgate.go | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/go/vt/vtexplain/vtexplain_vtgate.go b/go/vt/vtexplain/vtexplain_vtgate.go index 158baa5ddd3..5a86c06e840 100644 --- a/go/vt/vtexplain/vtexplain_vtgate.go +++ b/go/vt/vtexplain/vtexplain_vtgate.go @@ -67,7 +67,7 @@ func initVtgateExecutor(vSchemaStr string, opts *Options) error { streamSize := 10 queryPlanCacheSize := int64(10) - vtgateExecutor = vtgate.NewExecutor(context.Background(), explainTopo, vtexplainCell, "", resolver, opts.Normalize, streamSize, queryPlanCacheSize) + vtgateExecutor = vtgate.NewExecutor(context.Background(), explainTopo, vtexplainCell, resolver, opts.Normalize, streamSize, queryPlanCacheSize) return nil } diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index cdc75c46211..8fe11144c75 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -104,7 +104,7 @@ const pathScatterStats = "/debug/scatter_stats" const pathVSchema = "/debug/vschema" // NewExecutor creates a new Executor. -func NewExecutor(ctx context.Context, serv srvtopo.Server, cell, statsName string, resolver *Resolver, normalize bool, streamSize int, queryPlanCacheSize int64) *Executor { +func NewExecutor(ctx context.Context, serv srvtopo.Server, cell string, resolver *Resolver, normalize bool, streamSize int, queryPlanCacheSize int64) *Executor { e := &Executor{ serv: serv, cell: cell, diff --git a/go/vt/vtgate/executor_framework_test.go b/go/vt/vtgate/executor_framework_test.go index 1b788d2c206..60d97108e96 100644 --- a/go/vt/vtgate/executor_framework_test.go +++ b/go/vt/vtgate/executor_framework_test.go @@ -361,7 +361,7 @@ func createExecutorEnv() (executor *Executor, sbc1, sbc2, sbclookup *sandboxconn getSandbox(KsTestUnsharded).VSchema = unshardedVSchema - executor = NewExecutor(context.Background(), serv, cell, "", resolver, false, testBufferSize, testCacheSize) + executor = NewExecutor(context.Background(), serv, cell, resolver, false, testBufferSize, testCacheSize) key.AnyShardPicker = DestinationAnyShardPickerFirstShard{} return executor, sbc1, sbc2, sbclookup } @@ -380,7 +380,7 @@ func createCustomExecutor(vschema string) (executor *Executor, sbc1, sbc2, sbclo sbclookup = hc.AddTestTablet(cell, "0", 1, KsTestUnsharded, "0", topodatapb.TabletType_MASTER, true, 1, nil) getSandbox(KsTestUnsharded).VSchema = unshardedVSchema - executor = NewExecutor(context.Background(), serv, cell, "", resolver, false, testBufferSize, testCacheSize) + executor = NewExecutor(context.Background(), serv, cell, resolver, false, testBufferSize, testCacheSize) return executor, sbc1, sbc2, sbclookup } diff --git a/go/vt/vtgate/executor_select_test.go b/go/vt/vtgate/executor_select_test.go index eec59509766..502680c4945 100644 --- a/go/vt/vtgate/executor_select_test.go +++ b/go/vt/vtgate/executor_select_test.go @@ -864,7 +864,7 @@ func TestSelectScatter(t *testing.T) { sbc := hc.AddTestTablet(cell, shard, 1, "TestExecutor", shard, topodatapb.TabletType_MASTER, true, 1, nil) conns = append(conns, sbc) } - executor := NewExecutor(context.Background(), serv, cell, "", resolver, false, testBufferSize, testCacheSize) + executor := NewExecutor(context.Background(), serv, cell, resolver, false, testBufferSize, testCacheSize) logChan := QueryLogger.Subscribe("Test") defer QueryLogger.Unsubscribe(logChan) @@ -898,7 +898,7 @@ func TestSelectScatterPartial(t *testing.T) { conns = append(conns, sbc) } - executor := NewExecutor(context.Background(), serv, cell, "", resolver, false, testBufferSize, testCacheSize) + executor := NewExecutor(context.Background(), serv, cell, resolver, false, testBufferSize, testCacheSize) logChan := QueryLogger.Subscribe("Test") defer QueryLogger.Unsubscribe(logChan) @@ -955,7 +955,7 @@ func TestStreamSelectScatter(t *testing.T) { for _, shard := range shards { _ = hc.AddTestTablet(cell, shard, 1, "TestExecutor", shard, topodatapb.TabletType_MASTER, true, 1, nil) } - executor := NewExecutor(context.Background(), serv, cell, "", resolver, false, testBufferSize, testCacheSize) + executor := NewExecutor(context.Background(), serv, cell, resolver, false, testBufferSize, testCacheSize) sql := "select id from user" result, err := executorStream(executor, sql) @@ -1009,7 +1009,7 @@ func TestSelectScatterOrderBy(t *testing.T) { }}) conns = append(conns, sbc) } - executor := NewExecutor(context.Background(), serv, cell, "", resolver, false, testBufferSize, testCacheSize) + executor := NewExecutor(context.Background(), serv, cell, resolver, false, testBufferSize, testCacheSize) query := "select col1, col2 from user order by col2 desc" gotResult, err := executorExec(executor, query, nil) @@ -1080,7 +1080,7 @@ func TestSelectScatterOrderByVarChar(t *testing.T) { }}) conns = append(conns, sbc) } - executor := NewExecutor(context.Background(), serv, cell, "", resolver, false, testBufferSize, testCacheSize) + executor := NewExecutor(context.Background(), serv, cell, resolver, false, testBufferSize, testCacheSize) query := "select col1, textcol from user order by textcol desc" gotResult, err := executorExec(executor, query, nil) @@ -1146,7 +1146,7 @@ func TestStreamSelectScatterOrderBy(t *testing.T) { }}) conns = append(conns, sbc) } - executor := NewExecutor(context.Background(), serv, cell, "", resolver, false, testBufferSize, testCacheSize) + executor := NewExecutor(context.Background(), serv, cell, resolver, false, testBufferSize, testCacheSize) query := "select id, col from user order by col desc" gotResult, err := executorStream(executor, query) @@ -1208,7 +1208,7 @@ func TestStreamSelectScatterOrderByVarChar(t *testing.T) { }}) conns = append(conns, sbc) } - executor := NewExecutor(context.Background(), serv, cell, "", resolver, false, testBufferSize, testCacheSize) + executor := NewExecutor(context.Background(), serv, cell, resolver, false, testBufferSize, testCacheSize) query := "select id, textcol from user order by textcol desc" gotResult, err := executorStream(executor, query) @@ -1270,7 +1270,7 @@ func TestSelectScatterAggregate(t *testing.T) { }}) conns = append(conns, sbc) } - executor := NewExecutor(context.Background(), serv, cell, "", resolver, false, testBufferSize, testCacheSize) + executor := NewExecutor(context.Background(), serv, cell, resolver, false, testBufferSize, testCacheSize) query := "select col, sum(foo) from user group by col" gotResult, err := executorExec(executor, query, nil) @@ -1333,7 +1333,7 @@ func TestStreamSelectScatterAggregate(t *testing.T) { }}) conns = append(conns, sbc) } - executor := NewExecutor(context.Background(), serv, cell, "", resolver, false, testBufferSize, testCacheSize) + executor := NewExecutor(context.Background(), serv, cell, resolver, false, testBufferSize, testCacheSize) query := "select col, sum(foo) from user group by col" gotResult, err := executorStream(executor, query) @@ -1396,7 +1396,7 @@ func TestSelectScatterLimit(t *testing.T) { }}) conns = append(conns, sbc) } - executor := NewExecutor(context.Background(), serv, cell, "", resolver, false, testBufferSize, testCacheSize) + executor := NewExecutor(context.Background(), serv, cell, resolver, false, testBufferSize, testCacheSize) query := "select col1, col2 from user order by col2 desc limit 3" gotResult, err := executorExec(executor, query, nil) @@ -1468,7 +1468,7 @@ func TestStreamSelectScatterLimit(t *testing.T) { }}) conns = append(conns, sbc) } - executor := NewExecutor(context.Background(), serv, cell, "", resolver, false, testBufferSize, testCacheSize) + executor := NewExecutor(context.Background(), serv, cell, resolver, false, testBufferSize, testCacheSize) query := "select col1, col2 from user order by col2 desc limit 3" gotResult, err := executorStream(executor, query) diff --git a/go/vt/vtgate/executor_stream_test.go b/go/vt/vtgate/executor_stream_test.go index 4ed3cf526a8..ba708f4b724 100644 --- a/go/vt/vtgate/executor_stream_test.go +++ b/go/vt/vtgate/executor_stream_test.go @@ -57,7 +57,7 @@ func TestStreamSQLSharded(t *testing.T) { for _, shard := range shards { _ = hc.AddTestTablet(cell, shard, 1, "TestExecutor", shard, topodatapb.TabletType_MASTER, true, 1, nil) } - executor := NewExecutor(context.Background(), serv, cell, "", resolver, false, testBufferSize, testCacheSize) + executor := NewExecutor(context.Background(), serv, cell, resolver, false, testBufferSize, testCacheSize) sql := "stream * from sharded_user_msgs" result, err := executorStreamMessages(executor, sql) diff --git a/go/vt/vtgate/vtgate.go b/go/vt/vtgate/vtgate.go index d5bbb255e21..2b0f4dc21f2 100644 --- a/go/vt/vtgate/vtgate.go +++ b/go/vt/vtgate/vtgate.go @@ -155,7 +155,7 @@ func Init(ctx context.Context, hc discovery.HealthCheck, serv srvtopo.Server, ce vsm := newVStreamManager(srvResolver, serv, cell) rpcVTGate = &VTGate{ - executor: NewExecutor(ctx, serv, cell, "VTGateExecutor", resolver, *normalizeQueries, *streamBufferSize, *queryPlanCacheSize), + executor: NewExecutor(ctx, serv, cell, resolver, *normalizeQueries, *streamBufferSize, *queryPlanCacheSize), resolver: resolver, vsm: vsm, txConn: tc, From 500278ff3858db7e7eb1d1cc5c859070eafbe01d Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 24 Mar 2020 10:22:37 +0100 Subject: [PATCH 56/85] Remove unused arguments Signed-off-by: Andres Taylor --- go/vt/vtgate/executor.go | 41 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 8fe11144c75..49f4efea8e8 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -248,17 +248,17 @@ func (e *Executor) execute(ctx context.Context, safeSession *SafeSession, sql st case *sqlparser.DDL: return e.handleDDL(ctx, safeSession, sql, bindVars, dest, destKeyspace, destTabletType, logStats) case *sqlparser.Begin: - return e.handleBegin(ctx, safeSession, sql, bindVars, destTabletType, logStats) + return e.handleBegin(ctx, safeSession, destTabletType, logStats) case *sqlparser.Commit: - return e.handleCommit(ctx, safeSession, sql, bindVars, logStats) + return e.handleCommit(ctx, safeSession, logStats) case *sqlparser.Rollback: - return e.handleRollback(ctx, safeSession, sql, bindVars, logStats) + return e.handleRollback(ctx, safeSession, logStats) case *sqlparser.Set: - return e.handleSet(ctx, safeSession, sql, bindVars, logStats) + return e.handleSet(ctx, safeSession, sql, logStats) case *sqlparser.Show: return e.handleShow(ctx, safeSession, sql, bindVars, dest, destKeyspace, destTabletType, logStats) case *sqlparser.Use: - return e.handleUse(ctx, safeSession, sql, bindVars, specStmt) + return e.handleUse(safeSession, specStmt) case *sqlparser.OtherAdmin, *sqlparser.OtherRead: return e.handleOther(ctx, safeSession, sql, bindVars, dest, destKeyspace, destTabletType, logStats) } @@ -418,7 +418,7 @@ func (e *Executor) handleDDL(ctx context.Context, safeSession *SafeSession, sql sqlparser.AddSequenceStr, sqlparser.AddAutoIncStr: - err := e.handleVSchemaDDL(ctx, safeSession, dest, destKeyspace, destTabletType, ddl, logStats) + err := e.handleVSchemaDDL(ctx, destKeyspace, ddl) logStats.ExecuteTime = time.Since(execStart) return &sqltypes.Result{}, err default: @@ -444,7 +444,7 @@ func (e *Executor) handleDDL(ctx context.Context, safeSession *SafeSession, sql return result, err } -func (e *Executor) handleVSchemaDDL(ctx context.Context, safeSession *SafeSession, dest key.Destination, destKeyspace string, destTabletType topodatapb.TabletType, ddl *sqlparser.DDL, logStats *LogStats) error { +func (e *Executor) handleVSchemaDDL(ctx context.Context, destKeyspace string, ddl *sqlparser.DDL) error { vschema := e.vm.GetCurrentSrvVschema() if vschema == nil { return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "vschema not loaded") @@ -480,7 +480,7 @@ func (e *Executor) handleVSchemaDDL(ctx context.Context, safeSession *SafeSessio return e.vm.UpdateVSchema(ctx, ksName, vschema) } -func (e *Executor) handleBegin(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, destTabletType topodatapb.TabletType, logStats *LogStats) (*sqltypes.Result, error) { +func (e *Executor) handleBegin(ctx context.Context, safeSession *SafeSession, destTabletType topodatapb.TabletType, logStats *LogStats) (*sqltypes.Result, error) { if destTabletType != topodatapb.TabletType_MASTER { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "transactions are supported only for master tablet types, current type: %v", destTabletType) } @@ -494,7 +494,7 @@ func (e *Executor) handleBegin(ctx context.Context, safeSession *SafeSession, sq return &sqltypes.Result{}, err } -func (e *Executor) handleCommit(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, logStats *LogStats) (*sqltypes.Result, error) { +func (e *Executor) handleCommit(ctx context.Context, safeSession *SafeSession, logStats *LogStats) (*sqltypes.Result, error) { execStart := time.Now() logStats.PlanTime = execStart.Sub(logStats.StartTime) logStats.ShardQueries = uint32(len(safeSession.ShardSessions)) @@ -505,7 +505,7 @@ func (e *Executor) handleCommit(ctx context.Context, safeSession *SafeSession, s return &sqltypes.Result{}, err } -func (e *Executor) handleRollback(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, logStats *LogStats) (*sqltypes.Result, error) { +func (e *Executor) handleRollback(ctx context.Context, safeSession *SafeSession, logStats *LogStats) (*sqltypes.Result, error) { execStart := time.Now() logStats.PlanTime = execStart.Sub(logStats.StartTime) logStats.ShardQueries = uint32(len(safeSession.ShardSessions)) @@ -515,7 +515,7 @@ func (e *Executor) handleRollback(ctx context.Context, safeSession *SafeSession, return &sqltypes.Result{}, err } -func (e *Executor) handleSet(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, logStats *LogStats) (*sqltypes.Result, error) { +func (e *Executor) handleSet(ctx context.Context, safeSession *SafeSession, sql string, logStats *LogStats) (*sqltypes.Result, error) { vals, scope, err := sqlparser.ExtractSetValues(sql) execStart := time.Now() logStats.PlanTime = execStart.Sub(logStats.StartTime) @@ -536,7 +536,7 @@ func (e *Executor) handleSet(ctx context.Context, safeSession *SafeSession, sql case sqlparser.GlobalStr: return &sqltypes.Result{}, vterrors.New(vtrpcpb.Code_INVALID_ARGUMENT, "unsupported in set: global") case sqlparser.VitessMetadataStr: - return e.handleSetVitessMetadata(ctx, safeSession, k, v) + return e.handleSetVitessMetadata(ctx, k, v) case sqlparser.VariableStr: err := handleSetUserDefinedVariables(safeSession, k, v) if err != nil { @@ -735,7 +735,7 @@ func handleSetUserDefinedVariables(session *SafeSession, k sqlparser.SetKey, v i return nil } -func (e *Executor) handleSetVitessMetadata(ctx context.Context, session *SafeSession, k sqlparser.SetKey, v interface{}) (*sqltypes.Result, error) { +func (e *Executor) handleSetVitessMetadata(ctx context.Context, k sqlparser.SetKey, v interface{}) (*sqltypes.Result, error) { //TODO(kalfonso): move to its own acl check and consolidate into an acl component that can handle multiple operations (vschema, metadata) allowed := vschemaacl.Authorized(callerid.ImmediateCallerIDFromContext(ctx)) if !allowed { @@ -766,7 +766,7 @@ func (e *Executor) handleSetVitessMetadata(ctx context.Context, session *SafeSes return &sqltypes.Result{RowsAffected: 1}, nil } -func (e *Executor) handleShowVitessMetadata(ctx context.Context, session *SafeSession, opt *sqlparser.ShowTablesOpt) (*sqltypes.Result, error) { +func (e *Executor) handleShowVitessMetadata(ctx context.Context, opt *sqlparser.ShowTablesOpt) (*sqltypes.Result, error) { ts, err := e.serv.GetTopoServer() if err != nil { return nil, err @@ -833,7 +833,7 @@ func (e *Executor) handleShow(ctx context.Context, safeSession *SafeSession, sql switch strings.ToLower(show.Type) { case sqlparser.KeywordString(sqlparser.COLLATION), sqlparser.KeywordString(sqlparser.VARIABLES): if show.Scope == sqlparser.VitessMetadataStr { - return e.handleShowVitessMetadata(ctx, safeSession, show.ShowTablesOpt) + return e.handleShowVitessMetadata(ctx, show.ShowTablesOpt) } if destKeyspace == "" { @@ -1160,7 +1160,7 @@ func (e *Executor) handleShow(ctx context.Context, safeSession *SafeSession, sql return e.handleOther(ctx, safeSession, sql, bindVars, dest, destKeyspace, destTabletType, logStats) } -func (e *Executor) handleUse(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, use *sqlparser.Use) (*sqltypes.Result, error) { +func (e *Executor) handleUse(safeSession *SafeSession, use *sqlparser.Use) (*sqltypes.Result, error) { destKeyspace, destTabletType, _, err := e.ParseDestinationTarget(use.DBName.String()) if err != nil { return nil, err @@ -1222,7 +1222,7 @@ func (e *Executor) StreamExecute(ctx context.Context, method string, safeSession // check if this is a stream statement for messaging // TODO: support keyRange syntax if logStats.StmtType == sqlparser.StmtStream.String() { - return e.handleMessageStream(ctx, safeSession, sql, target, callback, vcursor, logStats) + return e.handleMessageStream(ctx, sql, target, callback, vcursor, logStats) } plan, err := e.getPlan( @@ -1289,7 +1289,7 @@ func (e *Executor) StreamExecute(ctx context.Context, method string, safeSession } // handleMessageStream executes queries of the form 'stream * from t' -func (e *Executor) handleMessageStream(ctx context.Context, safeSession *SafeSession, sql string, target querypb.Target, callback func(*sqltypes.Result) error, vcursor *vcursorImpl, logStats *LogStats) error { +func (e *Executor) handleMessageStream(ctx context.Context, sql string, target querypb.Target, callback func(*sqltypes.Result) error, vcursor *vcursorImpl, logStats *LogStats) error { stmt, err := sqlparser.Parse(sql) if err != nil { logStats.Error = err @@ -1656,7 +1656,10 @@ func (e *Executor) prepare(ctx context.Context, safeSession *SafeSession, sql st return nil, nil case sqlparser.StmtShow: res, err := e.handleShow(ctx, safeSession, sql, bindVars, dest, destKeyspace, destTabletType, logStats) - return res.Fields, err + if err != nil { + return nil, err + } + return res.Fields, nil } return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "unrecognized statement: %s", sql) } From cc2861b821306028a42c734064473cfb7f6fb8bc Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 26 Mar 2020 07:53:31 +0530 Subject: [PATCH 57/85] deprecaded old java clients Signed-off-by: Harshit Gangal --- .../io/vitess/client/VTGateBlockingConn.java | 99 --------- .../java/io/vitess/client/VTGateConn.java | 193 ------------------ .../java/io/vitess/client/cursor/Cursor.java | 4 +- .../java/io/vitess/client/RpcClientTest.java | 26 +-- .../test/java/io/vitess/client/TestUtil.java | 7 +- .../io/vitess/jdbc/VitessVTGateManager.java | 4 +- 6 files changed, 18 insertions(+), 315 deletions(-) delete mode 100644 java/client/src/main/java/io/vitess/client/VTGateBlockingConn.java delete mode 100644 java/client/src/main/java/io/vitess/client/VTGateConn.java diff --git a/java/client/src/main/java/io/vitess/client/VTGateBlockingConn.java b/java/client/src/main/java/io/vitess/client/VTGateBlockingConn.java deleted file mode 100644 index 7c261c679f4..00000000000 --- a/java/client/src/main/java/io/vitess/client/VTGateBlockingConn.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2019 The Vitess Authors. - - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - - * http://www.apache.org/licenses/LICENSE-2.0 - - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.vitess.client; - -import io.vitess.client.cursor.Cursor; -import io.vitess.client.cursor.CursorWithError; -import io.vitess.proto.Query; -import io.vitess.proto.Topodata.TabletType; - -import java.io.Closeable; -import java.io.IOException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import javax.annotation.Nullable; - -/** - * A synchronous wrapper around a VTGate connection. - * - *

- * This is a wrapper around the asynchronous {@link VTGateConn} class that converts all methods to - * synchronous. - */ -@Deprecated -public class VTGateBlockingConn implements Closeable { - - private final VTGateConn conn; - - /** - * Creates a new {@link VTGateConn} with the given {@link RpcClient} and wraps it in a synchronous - * API. - */ - public VTGateBlockingConn(RpcClient client) { - conn = new VTGateConn(client); - } - - /** - * Creates a new {@link VTGateConn} with the given {@link RpcClient} and wraps it in a synchronous - * API. - */ - public VTGateBlockingConn(RpcClient client, String keyspace) { - conn = new VTGateConn(client, keyspace); - } - - /** - * Wraps an existing {@link VTGateConn} in a synchronous API. - */ - public VTGateBlockingConn(VTGateConn conn) { - this.conn = conn; - } - - public Cursor execute(Context ctx, String query, Map bindVars, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return conn.execute(ctx, query, bindVars, tabletType, includedFields).checkedGet(); - } - - public List executeBatch(Context ctx, ArrayList queryList, - @Nullable ArrayList> bindVarsList, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) throws SQLException { - return executeBatch(ctx, queryList, bindVarsList, tabletType, false, includedFields); - } - - public List executeBatch(Context ctx, ArrayList queryList, - @Nullable ArrayList> bindVarsList, TabletType tabletType, - boolean asTransaction, Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return conn - .executeBatch(ctx, queryList, bindVarsList, tabletType, asTransaction, includedFields) - .checkedGet(); - } - - public Cursor streamExecute(Context ctx, String query, Map bindVars, - TabletType tabletType, Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - return conn.streamExecute(ctx, query, bindVars, tabletType, includedFields); - } - - @Override - public void close() throws IOException { - conn.close(); - } -} diff --git a/java/client/src/main/java/io/vitess/client/VTGateConn.java b/java/client/src/main/java/io/vitess/client/VTGateConn.java deleted file mode 100644 index 16225911693..00000000000 --- a/java/client/src/main/java/io/vitess/client/VTGateConn.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Copyright 2019 The Vitess Authors. - - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - - * http://www.apache.org/licenses/LICENSE-2.0 - - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.vitess.client; - -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.util.concurrent.Futures.transformAsync; -import static com.google.common.util.concurrent.MoreExecutors.directExecutor; - -import com.google.common.collect.Iterables; -import com.google.common.util.concurrent.AsyncFunction; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; - -import io.vitess.client.cursor.Cursor; -import io.vitess.client.cursor.CursorWithError; -import io.vitess.client.cursor.SimpleCursor; -import io.vitess.client.cursor.StreamCursor; -import io.vitess.proto.Query; -import io.vitess.proto.Topodata.TabletType; -import io.vitess.proto.Vtgate; -import io.vitess.proto.Vtgate.ExecuteRequest; -import io.vitess.proto.Vtgate.ExecuteResponse; -import io.vitess.proto.Vtgate.StreamExecuteRequest; - -import java.io.Closeable; -import java.io.IOException; -import java.sql.SQLDataException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import javax.annotation.Nullable; - -/** - * An asynchronous VTGate connection. - * - *

- * See the VitessClientExample - * for a usage example. - * - *

- * All non-streaming calls on {@code VTGateConn} are asynchronous. Use {@link VTGateBlockingConn} if - * you want synchronous calls. - */ -@Deprecated -public final class VTGateConn implements Closeable { - - private final RpcClient client; - private final String keyspace; - - /** - * Creates a VTGateConn with no default keyspace. - * - *

- * In this mode, methods like {@code execute()} and {@code streamExecute()} that don't have a - * per-call {@code keyspace} parameter will use VSchema to resolve the keyspace for any unprefixed - * table names. Note that this only works if the table name is unique across all keyspaces. - */ - public VTGateConn(RpcClient client) { - this.client = checkNotNull(client); - this.keyspace = ""; - } - - /** - * Creates a VTGateConn with a default keyspace. - * - *

- * The given {@code keyspace} will be used as the connection-wide default for {@code execute()} - * and {@code streamExecute()} calls, since those do not specify the keyspace for each call. Like - * the connection-wide default database of a MySQL connection, individual queries can still refer - * to other keyspaces by prefixing table names. For example: {@code "SELECT ... FROM - * keyspace.table ..."} - */ - public VTGateConn(RpcClient client, String keyspace) { - this.client = checkNotNull(client); - this.keyspace = checkNotNull(keyspace); - } - - public SQLFuture execute(Context ctx, String query, @Nullable Map bindVars, - TabletType tabletType, Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - ExecuteRequest.Builder requestBuilder = ExecuteRequest.newBuilder() - .setQuery(Proto.bindQuery(checkNotNull(query), bindVars)) - .setKeyspaceShard(keyspace) - .setTabletType(checkNotNull(tabletType)) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - return new SQLFuture( - transformAsync( - client.execute(ctx, requestBuilder.build()), - new AsyncFunction() { - @Override - public ListenableFuture apply(ExecuteResponse response) throws Exception { - Proto.checkError(response.getError()); - return Futures.immediateFuture(new SimpleCursor(response.getResult())); - } - }, - directExecutor())); - } - - public SQLFuture> executeBatch(Context ctx, List queryList, - @Nullable List> bindVarsList, TabletType tabletType, - Query.ExecuteOptions.IncludedFields includedFields) throws SQLException { - return executeBatch(ctx, queryList, bindVarsList, tabletType, false, includedFields); - } - - public SQLFuture> executeBatch(Context ctx, List queryList, - @Nullable List> bindVarsList, TabletType tabletType, - boolean asTransaction, Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - List queries = new ArrayList<>(); - - if (null != bindVarsList && bindVarsList.size() != queryList.size()) { - throw new SQLDataException( - "Size of SQL Query list does not match the bind variables list"); - } - - for (int i = 0; i < queryList.size(); ++i) { - queries.add(i, Proto.bindQuery(checkNotNull(queryList.get(i)), - bindVarsList == null ? null : bindVarsList.get(i))); - } - - Vtgate.ExecuteBatchRequest.Builder requestBuilder = - Vtgate.ExecuteBatchRequest.newBuilder() - .addAllQueries(checkNotNull(queries)) - .setKeyspaceShard(keyspace) - .setTabletType(checkNotNull(tabletType)) - .setAsTransaction(asTransaction) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - return new SQLFuture<>( - transformAsync( - client.executeBatch(ctx, requestBuilder.build()), - new AsyncFunction>() { - @Override - public ListenableFuture> apply( - Vtgate.ExecuteBatchResponse response) throws Exception { - Proto.checkError(response.getError()); - return Futures.immediateFuture( - Proto.fromQueryResponsesToCursorList(response.getResultsList())); - } - }, - directExecutor())); - } - - public Cursor streamExecute(Context ctx, String query, @Nullable Map bindVars, - TabletType tabletType, Query.ExecuteOptions.IncludedFields includedFields) - throws SQLException { - StreamExecuteRequest.Builder requestBuilder = - StreamExecuteRequest.newBuilder() - .setQuery(Proto.bindQuery(checkNotNull(query), bindVars)) - .setKeyspaceShard(keyspace) - .setTabletType(checkNotNull(tabletType)) - .setOptions(Query.ExecuteOptions.newBuilder() - .setIncludedFields(includedFields)); - - if (ctx.getCallerId() != null) { - requestBuilder.setCallerId(ctx.getCallerId()); - } - - return new StreamCursor(client.streamExecute(ctx, requestBuilder.build())); - } - - @Override - public void close() throws IOException { - client.close(); - } -} diff --git a/java/client/src/main/java/io/vitess/client/cursor/Cursor.java b/java/client/src/main/java/io/vitess/client/cursor/Cursor.java index 54218487184..37c98aa99ea 100644 --- a/java/client/src/main/java/io/vitess/client/cursor/Cursor.java +++ b/java/client/src/main/java/io/vitess/client/cursor/Cursor.java @@ -42,8 +42,8 @@ * should be called to free resources when done, regardless of whether all the rows were processed. * *

Each individual {@code Cursor} is not thread-safe; it must be protected if used concurrently. - * However, two cursors from the same {@link io.vitess.client.VTGateConn VTGateConn} can be accessed - * concurrently without additional synchronization. + * However, two cursors from the same {@link io.vitess.client.VTGateConnection VTGateConnection} + * can be accessed concurrently without additional synchronization. */ @NotThreadSafe public abstract class Cursor implements AutoCloseable { diff --git a/java/client/src/test/java/io/vitess/client/RpcClientTest.java b/java/client/src/test/java/io/vitess/client/RpcClientTest.java index 7ace70e33ee..24137c3e5b6 100644 --- a/java/client/src/test/java/io/vitess/client/RpcClientTest.java +++ b/java/client/src/test/java/io/vitess/client/RpcClientTest.java @@ -17,9 +17,7 @@ package io.vitess.client; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.protobuf.ByteString; import binlogdata.Binlogdata.FieldEvent; import binlogdata.Binlogdata.RowEvent; @@ -37,7 +35,6 @@ import io.vitess.proto.Vtrpc.CallerID; import java.nio.charset.StandardCharsets; -import java.sql.SQLDataException; import java.sql.SQLException; import java.sql.SQLIntegrityConstraintViolationException; import java.sql.SQLInvalidAuthorizationSpecException; @@ -46,7 +43,6 @@ import java.sql.SQLSyntaxErrorException; import java.sql.SQLTimeoutException; import java.sql.SQLTransientException; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -73,7 +69,8 @@ public abstract class RpcClientTest { private static boolean ready; private Context ctx; - private VTGateBlockingConn conn; + private VTGateBlockingConnection conn; + private VTSession session; @BeforeClass public static void resetReady() throws Exception { @@ -82,8 +79,9 @@ public static void resetReady() throws Exception { @Before public void setUp() throws SQLException, InterruptedException { - // Test VTGateConn via the synchronous VTGateBlockingConn wrapper. - conn = new VTGateBlockingConn(client, KEYSPACE); + // Test VTGateConnection via the synchronous VTGateBlockingConnection wrapper. + conn = new VTGateBlockingConnection(client); + session = new VTSession(KEYSPACE+"@"+TABLET_TYPE, Query.ExecuteOptions.newBuilder().setIncludedFields(ALL_FIELDS).build()); waitForVtgateclienttest(); @@ -125,7 +123,7 @@ public void setUp() throws SQLException, InterruptedException { private static final String BIND_VARS_ECHO = "map[bytes:type:VARBINARY value:\"\\001\\002\\003\" float:type:FLOAT64 value:\"2.5\" int:type:INT64 value:\"123\" ]"; - private static final String NONTX_V3_SESSION_ECHO = "autocommit:true target_string:\"test_keyspace@replica\" options: "; + private static final String NONTX_V3_SESSION_ECHO = "autocommit:true target_string:\"test_keyspace@REPLICA\" options: "; private static final CallerID CALLER_ID = CallerID.newBuilder().setPrincipal("test_principal") .setComponent("test_component").setSubcomponent("test_subcomponent").build(); @@ -179,7 +177,7 @@ private void waitForVtgateclienttest() throws SQLException, InterruptedException while (DateTime.now().isBefore(deadline)) { try { ctx = Context.getDefault().withDeadlineAfter(Duration.standardSeconds(30)); - conn.execute(ctx, ECHO_PREFIX + QUERY, BIND_VARS, TABLET_TYPE, ALL_FIELDS); + conn.execute(ctx, ECHO_PREFIX + QUERY, BIND_VARS, session); // RPC succeeded. Stop testing. break; } catch (SQLTransientException e) { @@ -208,8 +206,7 @@ private void waitForVtgateclienttest() throws SQLException, InterruptedException @Test public void testEchoExecute() throws Exception { Map echo; - - echo = getEcho(conn.execute(ctx, ECHO_PREFIX + QUERY, BIND_VARS, TABLET_TYPE, ALL_FIELDS)); + echo = getEcho(conn.execute(ctx, ECHO_PREFIX + QUERY, BIND_VARS, session)); Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); @@ -219,9 +216,8 @@ public void testEchoExecute() throws Exception { @Test public void testEchoStreamExecute() throws Exception { Map echo; - echo = getEcho( - conn.streamExecute(ctx, ECHO_PREFIX + QUERY, BIND_VARS, TABLET_TYPE, ALL_FIELDS)); + conn.streamExecute(ctx, ECHO_PREFIX + QUERY, BIND_VARS, session)); Assert.assertEquals(CALLER_ID_ECHO, echo.get("callerId")); Assert.assertEquals(ECHO_PREFIX + QUERY, echo.get("query")); Assert.assertEquals(BIND_VARS_ECHO, echo.get("bindVars")); @@ -271,7 +267,7 @@ public void testExecuteErrors() throws Exception { checkExecuteErrors(new Executable() { @Override void execute(String query) throws Exception { - conn.execute(ctx, query, BIND_VARS, TABLET_TYPE, ALL_FIELDS); + conn.execute(ctx, query, BIND_VARS, session); } }); } @@ -281,7 +277,7 @@ public void testStreamExecuteErrors() throws Exception { checkStreamExecuteErrors(new Executable() { @Override void execute(String query) throws Exception { - conn.streamExecute(ctx, query, BIND_VARS, TABLET_TYPE, ALL_FIELDS).next(); + conn.streamExecute(ctx, query, BIND_VARS, session).next(); } }); } diff --git a/java/client/src/test/java/io/vitess/client/TestUtil.java b/java/client/src/test/java/io/vitess/client/TestUtil.java index 59e83b40ff3..3174f25b54d 100644 --- a/java/client/src/test/java/io/vitess/client/TestUtil.java +++ b/java/client/src/test/java/io/vitess/client/TestUtil.java @@ -115,11 +115,10 @@ public static RpcClientFactory getRpcClientFactory() { } } - public static VTGateBlockingConn getBlockingConn(TestEnv testEnv) { + public static VTGateBlockingConnection getBlockingConn(TestEnv testEnv) { // Dial timeout Context ctx = Context.getDefault().withDeadlineAfter(Duration.millis(5000)); - return new VTGateBlockingConn( - getRpcClientFactory().create(ctx, "localhost:" + testEnv.getPort()), - testEnv.getKeyspace()); + return new VTGateBlockingConnection( + getRpcClientFactory().create(ctx, "localhost:" + testEnv.getPort())); } } diff --git a/java/jdbc/src/main/java/io/vitess/jdbc/VitessVTGateManager.java b/java/jdbc/src/main/java/io/vitess/jdbc/VitessVTGateManager.java index d19b0fcec9f..4550120be73 100644 --- a/java/jdbc/src/main/java/io/vitess/jdbc/VitessVTGateManager.java +++ b/java/jdbc/src/main/java/io/vitess/jdbc/VitessVTGateManager.java @@ -47,7 +47,7 @@ public class VitessVTGateManager { private static Logger logger = LogManager.getLogger(VitessVTGateManager.class); /* - Current implementation have one VTGateConn for ip-port-username combination + Current implementation have one VTGateConnection for ip-port-username combination */ private static ConcurrentHashMap vtGateConnHashMap = new ConcurrentHashMap<>(); @@ -125,7 +125,7 @@ private static String getIdentifer(String hostname, int port, String userIdentif } /** - * Create VTGateConn and update vtGateConnHashMap. + * Create {@link VTGateConnection} and update vtGateConnHashMap. */ private static void updateVtGateConnHashMap(String identifier, VitessJDBCUrl.HostInfo hostInfo, VitessConnection connection) { From e744688b6a5c00a563784bfea71403a694a3c38b Mon Sep 17 00:00:00 2001 From: Saif Al-Harthi Date: Wed, 25 Mar 2020 20:18:16 -0700 Subject: [PATCH 58/85] Fix transaction_mode parsing (#5963) * Fix transaction_mode parsing and program exit if presented with an invalid transaction_mode option Signed-off-by: Saif Alharthi --- go/vt/vtgate/vtgate.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/go/vt/vtgate/vtgate.go b/go/vt/vtgate/vtgate.go index d5bbb255e21..faf05af3082 100644 --- a/go/vt/vtgate/vtgate.go +++ b/go/vt/vtgate/vtgate.go @@ -22,6 +22,8 @@ import ( "flag" "fmt" "net/http" + "os" + "strings" "time" "golang.org/x/net/context" @@ -62,19 +64,21 @@ var ( ) func getTxMode() vtgatepb.TransactionMode { - switch *transactionMode { - case "SINGLE": + switch strings.ToLower(*transactionMode) { + case "single": log.Infof("Transaction mode: '%s'", *transactionMode) return vtgatepb.TransactionMode_SINGLE - case "MULTI": + case "multi": log.Infof("Transaction mode: '%s'", *transactionMode) return vtgatepb.TransactionMode_MULTI - case "TWOPC": + case "twopc": log.Infof("Transaction mode: '%s'", *transactionMode) return vtgatepb.TransactionMode_TWOPC default: - log.Warningf("Unrecognized transactionMode '%s'. Continuing with default 'MULTI'", *transactionMode) - return vtgatepb.TransactionMode_MULTI + fmt.Printf("Invalid option: %v\n", *transactionMode) + fmt.Println("Usage: -transaction_mode {SINGLE | MULTI | TWOPC}") + os.Exit(1) + return -1 } } From 41737c693f97a4a2000fa8b9bc553d392de65d75 Mon Sep 17 00:00:00 2001 From: Saif Alharthi Date: Wed, 25 Mar 2020 20:38:11 -0700 Subject: [PATCH 59/85] Modify test for readability Signed-off-by: Saif Alharthi --- go/test/endtoend/preparestmt/stmt_methods_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go/test/endtoend/preparestmt/stmt_methods_test.go b/go/test/endtoend/preparestmt/stmt_methods_test.go index 2016293823e..b18ea1883a0 100644 --- a/go/test/endtoend/preparestmt/stmt_methods_test.go +++ b/go/test/endtoend/preparestmt/stmt_methods_test.go @@ -142,10 +142,10 @@ func TestAutoIncColumns(t *testing.T) { time.Now(), time.Date(2009, 5, 5, 0, 0, 0, 50000, time.UTC), 1, 1, 1, 1, 1, 1, 1, 1, 1, jsonExample, fake.DomainName(), fake.Paragraph(), - -128, 127, 1, -1, - -32768, 32767, 1, -1, - -8388608, 8388607, 1, -1, - -2147483648, 2147483647, 1, -1, + -(1 << 7), (1 << 7) - 1, 1, -1, + -(1 << 15), (1 << 15) - 1, 1, -1, + -(1 << 23), (1 << 23) - 1, 1, -1, + -(1 << 31), (1 << 31) - 1, 1, -1, -(1 << 63), (1 << 63) - 1, 1, -1, } From d1adbfbb0229047ececb2859aed6f9725302ec70 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 26 Mar 2020 13:54:27 +0100 Subject: [PATCH 60/85] Fixing race in https://github.com/vitessio/vitess/issues/5970 properly. Check for an invalid tablet was missing in test case Signed-off-by: Rohit Nayak --- go/vt/discovery/healthcheck.go | 5 ++--- go/vt/wrangler/vdiff_env_test.go | 5 ++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index f000ffeb3b4..b282b846d7b 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -44,7 +44,6 @@ import ( "hash/crc32" "html/template" "net/http" - "reflect" "sort" "strings" "sync" @@ -493,7 +492,7 @@ func (hc *HealthCheckImpl) finalizeConn(hcc *healthCheckConn) { // safe to simply get Err() value here and assign to LastError. hcc.tabletStats.LastError = hcc.ctx.Err() hc.updateHealth(hcc.tabletStats.Copy(), nil) - if hcc.conn != nil && !reflect.ValueOf(hcc.conn).IsNil() { + if hcc.conn != nil { // Don't use hcc.ctx because it's already closed. // Use a separate context, and add a timeout to prevent unbounded waits. ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -611,7 +610,7 @@ func (hcc *healthCheckConn) setServingState(serving bool, reason string) { // stream streams healthcheck responses to callback. func (hcc *healthCheckConn) stream(ctx context.Context, hc *HealthCheckImpl, callback func(*querypb.StreamHealthResponse) error) { - if hcc.conn == nil || reflect.ValueOf(hcc.conn).IsNil() { + if hcc.conn == nil { conn, err := tabletconn.GetDialer()(hcc.tabletStats.Tablet, grpcclient.FailFast(true)) if err != nil { hcc.tabletStats.LastError = err diff --git a/go/vt/wrangler/vdiff_env_test.go b/go/vt/wrangler/vdiff_env_test.go index 44f4f9b85ee..95c780e4ebd 100644 --- a/go/vt/wrangler/vdiff_env_test.go +++ b/go/vt/wrangler/vdiff_env_test.go @@ -68,7 +68,10 @@ func init() { tabletconn.RegisterDialer("VDiffTest", func(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (queryservice.QueryService, error) { vdiffEnv.mu.Lock() defer vdiffEnv.mu.Unlock() - return vdiffEnv.tablets[int(tablet.Alias.Uid)], nil + if qs, ok := vdiffEnv.tablets[int(tablet.Alias.Uid)]; ok { + return qs, nil + } + return nil, fmt.Errorf("tablet %d not found", tablet.Alias.Uid) }) } From 297165241127e365d93dcad6aeae3dc4d95e61af Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 26 Mar 2020 22:04:24 +0100 Subject: [PATCH 61/85] Deleted extraneous files Signed-off-by: Rohit Nayak --- examples/local/x/env.sh | 72 ----------------------- examples/local/x/scripts/etcd-down.sh | 22 ------- examples/local/x/scripts/etcd-up.sh | 51 ---------------- examples/local/x/scripts/k3s-down.sh | 30 ---------- examples/local/x/scripts/k3s-up.sh | 49 --------------- examples/local/x/scripts/mysqlctl-down.sh | 23 -------- examples/local/x/scripts/mysqlctl-up.sh | 42 ------------- examples/local/x/scripts/vtctld-down.sh | 22 ------- examples/local/x/scripts/vtctld-up.sh | 38 ------------ examples/local/x/scripts/vtgate-down.sh | 23 -------- examples/local/x/scripts/vtgate-up.sh | 58 ------------------ examples/local/x/scripts/vttablet-down.sh | 30 ---------- examples/local/x/scripts/vttablet-up.sh | 69 ---------------------- examples/local/x/scripts/zk-down.sh | 26 -------- examples/local/x/scripts/zk-up.sh | 62 ------------------- 15 files changed, 617 deletions(-) delete mode 100644 examples/local/x/env.sh delete mode 100755 examples/local/x/scripts/etcd-down.sh delete mode 100755 examples/local/x/scripts/etcd-up.sh delete mode 100755 examples/local/x/scripts/k3s-down.sh delete mode 100755 examples/local/x/scripts/k3s-up.sh delete mode 100755 examples/local/x/scripts/mysqlctl-down.sh delete mode 100755 examples/local/x/scripts/mysqlctl-up.sh delete mode 100755 examples/local/x/scripts/vtctld-down.sh delete mode 100755 examples/local/x/scripts/vtctld-up.sh delete mode 100755 examples/local/x/scripts/vtgate-down.sh delete mode 100755 examples/local/x/scripts/vtgate-up.sh delete mode 100755 examples/local/x/scripts/vttablet-down.sh delete mode 100755 examples/local/x/scripts/vttablet-up.sh delete mode 100755 examples/local/x/scripts/zk-down.sh delete mode 100755 examples/local/x/scripts/zk-up.sh diff --git a/examples/local/x/env.sh b/examples/local/x/env.sh deleted file mode 100644 index 1d21fafc98f..00000000000 --- a/examples/local/x/env.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -hostname=`hostname -f` -vtctld_web_port=15000 -export VTDATAROOT="${VTDATAROOT:-${PWD}/vtdataroot}" - -function fail() { - echo "ERROR: $1" - exit 1 -} - -if [[ $EUID -eq 0 ]]; then - fail "This script refuses to be run as root. Please switch to a regular user." -fi - -# mysqld might be in /usr/sbin which will not be in the default PATH -PATH="/usr/sbin:$PATH" -for binary in mysqld etcd etcdctl curl vtctlclient vttablet vtgate vtctld vtctl mysqlctl; do - command -v "$binary" > /dev/null || fail "${binary} is not installed in PATH. See https://vitess.io/docs/get-started/local/ for install instructions." -done; - -if [ "${TOPO}" = "zk2" ]; then - # Each ZooKeeper server needs a list of all servers in the quorum. - # Since we're running them all locally, we need to give them unique ports. - # In a real deployment, these should be on different machines, and their - # respective hostnames should be given. - zkcfg=(\ - "1@$hostname:28881:38881:21811" \ - "2@$hostname:28882:38882:21812" \ - "3@$hostname:28883:38883:21813" \ - ) - printf -v zkcfg ",%s" "${zkcfg[@]}" - zkcfg=${zkcfg:1} - - zkids='1 2 3' - - # Set topology environment parameters. - ZK_SERVER="localhost:21811,localhost:21812,localhost:21813" - # shellcheck disable=SC2034 - TOPOLOGY_FLAGS="-topo_implementation zk2 -topo_global_server_address ${ZK_SERVER} -topo_global_root /vitess/global" -elif [ "${TOPO}" = "k8s" ]; then - # Set topology environment parameters. - K8S_ADDR="localhost" - K8S_PORT="8443" - K8S_KUBECONFIG=$VTDATAROOT/tmp/k8s.kubeconfig - # shellcheck disable=SC2034 - TOPOLOGY_FLAGS="-topo_implementation k8s -topo_k8s_kubeconfig ${K8S_KUBECONFIG} -topo_global_server_address ${K8S_ADDR}:${K8S_PORT} -topo_global_root /vitess/global" -else - ETCD_SERVER="localhost:2379" - TOPOLOGY_FLAGS="-topo_implementation etcd2 -topo_global_server_address $ETCD_SERVER -topo_global_root /vitess/global" - - mkdir -p "${VTDATAROOT}/etcd" -fi - -# Create a tmp dir -mkdir -p "${VTDATAROOT}/tmp" diff --git a/examples/local/x/scripts/etcd-down.sh b/examples/local/x/scripts/etcd-down.sh deleted file mode 100755 index 018af7432a3..00000000000 --- a/examples/local/x/scripts/etcd-down.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is an example script that stops the etcd servers started by etcd-up.sh. - -source ./env.sh - -echo "Stopping etcd..." -kill -9 `cat $VTDATAROOT/tmp/etcd.pid` diff --git a/examples/local/x/scripts/etcd-up.sh b/examples/local/x/scripts/etcd-up.sh deleted file mode 100755 index d8c0a869a7e..00000000000 --- a/examples/local/x/scripts/etcd-up.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is an example script that creates a quorum of ZooKeeper servers. - -source ./env.sh - -cell=${CELL:-'test'} -export ETCDCTL_API=2 - -# Check that etcd is not already running -curl "http://${ETCD_SERVER}" > /dev/null 2>&1 && fail "etcd is already running. Exiting." - -etcd --enable-v2=true --data-dir "${VTDATAROOT}/etcd/" --listen-client-urls "http://${ETCD_SERVER}" --advertise-client-urls "http://${ETCD_SERVER}" > "${VTDATAROOT}"/tmp/etcd.out 2>&1 & -PID=$! -echo $PID > "${VTDATAROOT}/tmp/etcd.pid" -sleep 5 - -echo "add /vitess/global" -etcdctl --endpoints "http://${ETCD_SERVER}" mkdir /vitess/global & - -echo "add /vitess/$cell" -etcdctl --endpoints "http://${ETCD_SERVER}" mkdir /vitess/$cell & - -# And also add the CellInfo description for the cell. -# If the node already exists, it's fine, means we used existing data. -echo "add $cell CellInfo" -set +e -# shellcheck disable=SC2086 -vtctl $TOPOLOGY_FLAGS AddCellInfo \ - -root /vitess/$cell \ - -server_address "${ETCD_SERVER}" \ - $cell -set -e - -echo "etcd start done..." - - diff --git a/examples/local/x/scripts/k3s-down.sh b/examples/local/x/scripts/k3s-down.sh deleted file mode 100755 index 590dc604e3e..00000000000 --- a/examples/local/x/scripts/k3s-down.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is an example script that stops the k3s server started by k3s-up.sh. - -set -e - -# shellcheck source=./env.sh -# shellcheck disable=SC1091 -source ./env.sh - -# Stop K3s server. -echo "Stopping k3s server..." - -pid=`cat $VTDATAROOT/tmp/k3s.pid` -echo "Stopping k3s..." -kill -9 $pid diff --git a/examples/local/x/scripts/k3s-up.sh b/examples/local/x/scripts/k3s-up.sh deleted file mode 100755 index 3b77600018e..00000000000 --- a/examples/local/x/scripts/k3s-up.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is an example script that creates a Kubernetes api for topo use by running k3s - -set -e -cell=${CELL:-'test'} - -script_root=$(dirname "${BASH_SOURCE[0]}") - -# shellcheck source=./env.sh -# shellcheck disable=SC1091 -source ./env.sh - -k3s server --disable-agent --data-dir "${VTDATAROOT}/k3s/" --https-listen-port "${K8S_PORT}" --write-kubeconfig "${K8S_KUBECONFIG}" > "${VTDATAROOT}"/tmp/k3s.out 2>&1 & -PID=$! -echo $PID > "${VTDATAROOT}/tmp/k3s.pid" -disown -a -echo "Waiting for k3s server to start" -sleep 15 - -# Use k3s built-in kubectl with custom config -KUBECTL="k3s kubectl --kubeconfig=${K8S_KUBECONFIG}" - -# Create the CRD for vitesstopologynodes -$KUBECTL create -f ../../go/vt/topo/k8stopo/VitessTopoNodes-crd.yaml - -# Add the CellInfo description for the cell -set +e -echo "add $cell CellInfo" -vtctl $TOPOLOGY_FLAGS AddCellInfo \ - -root /vitess/$cell \ - $cell -set -e - -echo "k3s start done..." diff --git a/examples/local/x/scripts/mysqlctl-down.sh b/examples/local/x/scripts/mysqlctl-down.sh deleted file mode 100755 index 9e5491d1d4c..00000000000 --- a/examples/local/x/scripts/mysqlctl-down.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is an example script that stops the mysqld and vttablet instances -# created by vttablet-up.sh - -source ./env.sh - -mysqlctl -tablet_uid $TABLET_UID shutdown - diff --git a/examples/local/x/scripts/mysqlctl-up.sh b/examples/local/x/scripts/mysqlctl-up.sh deleted file mode 100755 index e4c9f58f819..00000000000 --- a/examples/local/x/scripts/mysqlctl-up.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is an example script that creates a single shard vttablet deployment. - -source ./env.sh - -cell=${CELL:-'test'} -uid=$TABLET_UID -mysql_port=$[17000 + $uid] -printf -v alias '%s-%010d' $cell $uid -printf -v tablet_dir 'vt_%010d' $uid - -mkdir -p $VTDATAROOT/backups - -echo "Starting MySQL for tablet $alias..." -action="init" - -if [ -d $VTDATAROOT/$tablet_dir ]; then - echo "Resuming from existing vttablet dir:" - echo " $VTDATAROOT/$tablet_dir" - action='start' -fi - -mysqlctl \ - -log_dir $VTDATAROOT/tmp \ - -tablet_uid $uid \ - -mysql_port $mysql_port \ - $action diff --git a/examples/local/x/scripts/vtctld-down.sh b/examples/local/x/scripts/vtctld-down.sh deleted file mode 100755 index d96fa3b927f..00000000000 --- a/examples/local/x/scripts/vtctld-down.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is an example script that stops vtctld. - -source ./env.sh - -echo "Stopping vtctld..." -kill -9 `cat $VTDATAROOT/tmp/vtctld.pid` diff --git a/examples/local/x/scripts/vtctld-up.sh b/examples/local/x/scripts/vtctld-up.sh deleted file mode 100755 index 662a234ae48..00000000000 --- a/examples/local/x/scripts/vtctld-up.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is an example script that starts vtctld. - -source ./env.sh - -cell=${CELL:-'test'} -grpc_port=15999 - -echo "Starting vtctld..." -# shellcheck disable=SC2086 -vtctld \ - $TOPOLOGY_FLAGS \ - -cell $cell \ - -workflow_manager_init \ - -workflow_manager_use_election \ - -service_map 'grpc-vtctl' \ - -backup_storage_implementation file \ - -file_backup_storage_root $VTDATAROOT/backups \ - -log_dir $VTDATAROOT/tmp \ - -port $vtctld_web_port \ - -grpc_port $grpc_port \ - -pid_file $VTDATAROOT/tmp/vtctld.pid \ - > $VTDATAROOT/tmp/vtctld.out 2>&1 & diff --git a/examples/local/x/scripts/vtgate-down.sh b/examples/local/x/scripts/vtgate-down.sh deleted file mode 100755 index 9da0a7179df..00000000000 --- a/examples/local/x/scripts/vtgate-down.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is an example script that stops the instance started by vtgate-up.sh. - -source ./env.sh - -# Stop vtgate. -echo "Stopping vtgate..." -kill `cat $VTDATAROOT/tmp/vtgate.pid` diff --git a/examples/local/x/scripts/vtgate-up.sh b/examples/local/x/scripts/vtgate-up.sh deleted file mode 100755 index e3fe2d17f82..00000000000 --- a/examples/local/x/scripts/vtgate-up.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is an example script that starts a single vtgate. - -source ./env.sh - -cell=${CELL:-'test'} -web_port=15001 -grpc_port=15991 -mysql_server_port=15306 -mysql_server_socket_path="/tmp/mysql.sock" - -# Start vtgate. -# shellcheck disable=SC2086 -vtgate \ - $TOPOLOGY_FLAGS \ - -log_dir $VTDATAROOT/tmp \ - -log_queries_to_file $VTDATAROOT/tmp/vtgate_querylog.txt \ - -port $web_port \ - -grpc_port $grpc_port \ - -mysql_server_port $mysql_server_port \ - -mysql_server_socket_path $mysql_server_socket_path \ - -cell $cell \ - -cells_to_watch $cell \ - -tablet_types_to_wait MASTER,REPLICA \ - -gateway_implementation discoverygateway \ - -service_map 'grpc-vtgateservice' \ - -pid_file $VTDATAROOT/tmp/vtgate.pid \ - -mysql_auth_server_impl none \ - > $VTDATAROOT/tmp/vtgate.out 2>&1 & - -# Block waiting for vtgate to be listening -# Not the same as healthy - -echo "Waiting for vtgate to be up..." -while true; do - curl -I "http://$hostname:$web_port/debug/status" >/dev/null 2>&1 && break - sleep 0.1 -done; -echo "vtgate is up!" - -echo "Access vtgate at http://$hostname:$web_port/debug/status" - -disown -a diff --git a/examples/local/x/scripts/vttablet-down.sh b/examples/local/x/scripts/vttablet-down.sh deleted file mode 100755 index 47b881b9793..00000000000 --- a/examples/local/x/scripts/vttablet-down.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is an example script that stops the mysqld and vttablet instances -# created by vttablet-up.sh - -source ./env.sh - -printf -v tablet_dir 'vt_%010d' $TABLET_UID -pid=`cat $VTDATAROOT/$tablet_dir/vttablet.pid` - -kill $pid - -# Wait for vttablet to die. -while ps -p $pid > /dev/null; do sleep 1; done - - diff --git a/examples/local/x/scripts/vttablet-up.sh b/examples/local/x/scripts/vttablet-up.sh deleted file mode 100755 index c3f66fa806d..00000000000 --- a/examples/local/x/scripts/vttablet-up.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source ./env.sh - -cell=${CELL:-'test'} -keyspace=${KEYSPACE:-'test_keyspace'} -shard=${SHARD:-'0'} -uid=$TABLET_UID -mysql_port=$[17000 + $uid] -port=$[15000 + $uid] -grpc_port=$[16000 + $uid] -printf -v alias '%s-%010d' $cell $uid -printf -v tablet_dir 'vt_%010d' $uid -tablet_hostname='' -printf -v tablet_logfile 'vttablet_%010d_querylog.txt' $uid - -tablet_type=replica -if [[ "${uid: -1}" -gt 1 ]]; then - tablet_type=rdonly -fi - -echo "Starting vttablet for $alias..." -# shellcheck disable=SC2086 -vttablet \ - $TOPOLOGY_FLAGS \ - -log_dir $VTDATAROOT/tmp \ - -log_queries_to_file $VTDATAROOT/tmp/$tablet_logfile \ - -tablet-path $alias \ - -tablet_hostname "$tablet_hostname" \ - -init_keyspace $keyspace \ - -init_shard $shard \ - -init_tablet_type $tablet_type \ - -health_check_interval 5s \ - -enable_semi_sync \ - -enable_replication_reporter \ - -backup_storage_implementation file \ - -file_backup_storage_root $VTDATAROOT/backups \ - -restore_from_backup \ - -port $port \ - -grpc_port $grpc_port \ - -service_map 'grpc-queryservice,grpc-tabletmanager,grpc-updatestream' \ - -pid_file $VTDATAROOT/$tablet_dir/vttablet.pid \ - -vtctld_addr http://$hostname:$vtctld_web_port/ \ - > $VTDATAROOT/$tablet_dir/vttablet.out 2>&1 & - -# Block waiting for the tablet to be listening -# Not the same as healthy - -for i in $(seq 0 300); do - curl -I "http://$hostname:$port/debug/status" >/dev/null 2>&1 && break - sleep 0.1 -done - -# check one last time -curl -I "http://$hostname:$port/debug/status" || fail "tablet could not be started!" diff --git a/examples/local/x/scripts/zk-down.sh b/examples/local/x/scripts/zk-down.sh deleted file mode 100755 index 18dd7933bc9..00000000000 --- a/examples/local/x/scripts/zk-down.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is an example script that stops the ZooKeeper servers started by zk-up.sh. - -source ./env.sh - -# Stop ZooKeeper servers. -echo "Stopping zk servers..." -for zkid in $zkids; do - zkctl -zk.myid $zkid -zk.cfg $zkcfg -log_dir $VTDATAROOT/tmp shutdown -done - diff --git a/examples/local/x/scripts/zk-up.sh b/examples/local/x/scripts/zk-up.sh deleted file mode 100755 index 6671d1063a6..00000000000 --- a/examples/local/x/scripts/zk-up.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is an example script that creates a quorum of ZooKeeper servers. - -source ./env.sh - -cell=${CELL:-'test'} - -# Start ZooKeeper servers. -# The "zkctl init" command won't return until the server is able to contact its -# peers, so we need to start them all in the background and then wait for them. -echo "Starting zk servers..." -for zkid in $zkids; do - action='init' - printf -v zkdir 'zk_%03d' $zkid - if [ -f $VTDATAROOT/$zkdir/myid ]; then - echo "Resuming from existing ZK data dir:" - echo " $VTDATAROOT/$zkdir" - action='start' - fi - zkctl -zk.myid $zkid -zk.cfg $zkcfg -log_dir $VTDATAROOT/tmp $action \ - > $VTDATAROOT/tmp/zkctl_$zkid.out 2>&1 & - pids[$zkid]=$! -done - -# Wait for all the zkctl commands to return. -echo "Waiting for zk servers to be ready..." - -for zkid in $zkids; do - if ! wait ${pids[$zkid]}; then - echo "ZK server number $zkid failed to start. See log:" - echo " $VTDATAROOT/tmp/zkctl_$zkid.out" - fi -done - -echo "Started zk servers." - -# Add the CellInfo description for the $CELL cell. -# If the node already exists, it's fine, means we used existing data. -set +e -# shellcheck disable=SC2086 -vtctl $TOPOLOGY_FLAGS AddCellInfo \ - -root /vitess/$cell \ - -server_address $ZK_SERVER \ - $cell -set -e - -echo "Configured zk servers." From de992802166b77c2a996639769eecf18d3c05087 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 26 Mar 2020 23:16:38 +0100 Subject: [PATCH 62/85] Check if tablet exists before registering dialer Signed-off-by: Rohit Nayak --- go/vt/discovery/healthcheck_flaky_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/go/vt/discovery/healthcheck_flaky_test.go b/go/vt/discovery/healthcheck_flaky_test.go index 29afa001aac..58fa12ca0c3 100644 --- a/go/vt/discovery/healthcheck_flaky_test.go +++ b/go/vt/discovery/healthcheck_flaky_test.go @@ -697,7 +697,10 @@ func createFixedHealthConn(tablet *topodatapb.Tablet, fixedResult *querypb.Strea func discoveryDialer(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (queryservice.QueryService, error) { key := TabletToMapKey(tablet) - return connMap[key], nil + if qs, ok := connMap[key]; ok { + return qs, nil + } + return nil, fmt.Errorf("tablet %v not found", key) } // StreamHealth implements queryservice.QueryService. From 576fe0ac02d4bea425d8488fd32e98624ed3e742 Mon Sep 17 00:00:00 2001 From: deepthi Date: Thu, 26 Mar 2020 15:49:20 -0700 Subject: [PATCH 63/85] Remove Gateway interface and use concrete DiscoveryGateway type everywhere. Deprecate gateway_implementation flag Signed-off-by: deepthi --- go/cmd/vtgate/status.go | 3 +- go/vt/vtexplain/vtexplain_vtgate.go | 4 +- .../vtgate/{gateway => }/discoverygateway.go | 87 +++++++++---- .../{gateway => }/discoverygateway_test.go | 37 +++--- go/vt/vtgate/gateway/gateway.go | 122 ------------------ go/vt/vtgate/gateway/shard_error.go | 40 ------ .../vtgate/gatewaytest/grpc_discovery_test.go | 8 +- go/vt/vtgate/gatewaytest/suite.go | 11 +- go/vt/vtgate/resolver.go | 8 +- go/vt/vtgate/scatter_conn.go | 14 +- go/vt/vtgate/scatter_conn_test.go | 8 +- go/vt/vtgate/{gateway => }/status.go | 2 +- go/vt/vtgate/{gateway => }/status_test.go | 3 +- go/vt/vtgate/tx_conn.go | 8 +- go/vt/vtgate/vstream_manager_test.go | 3 +- go/vt/vtgate/vtgate.go | 17 ++- 16 files changed, 121 insertions(+), 254 deletions(-) rename go/vt/vtgate/{gateway => }/discoverygateway.go (79%) rename go/vt/vtgate/{gateway => }/discoverygateway_test.go (90%) delete mode 100644 go/vt/vtgate/gateway/gateway.go delete mode 100644 go/vt/vtgate/gateway/shard_error.go rename go/vt/vtgate/{gateway => }/status.go (99%) rename go/vt/vtgate/{gateway => }/status_test.go (99%) diff --git a/go/cmd/vtgate/status.go b/go/cmd/vtgate/status.go index 0aa168e06d3..c1bb461afb1 100644 --- a/go/cmd/vtgate/status.go +++ b/go/cmd/vtgate/status.go @@ -22,7 +22,6 @@ import ( "vitess.io/vitess/go/vt/srvtopo" _ "vitess.io/vitess/go/vt/status" "vitess.io/vitess/go/vt/vtgate" - "vitess.io/vitess/go/vt/vtgate/gateway" ) // For use by plugins which wish to avoid racing when registering status page parts. @@ -39,7 +38,7 @@ func addStatusParts(vtg *vtgate.VTGate) { servenv.AddStatusPart("Topology Cache", srvtopo.TopoTemplate, func() interface{} { return resilientServer.CacheStatus() }) - servenv.AddStatusPart("Gateway Status", gateway.StatusTemplate, func() interface{} { + servenv.AddStatusPart("Gateway Status", vtgate.StatusTemplate, func() interface{} { return vtg.GetGatewayCacheStatus() }) servenv.AddStatusPart("Health Check Cache", discovery.HealthCheckTemplate, func() interface{} { diff --git a/go/vt/vtexplain/vtexplain_vtgate.go b/go/vt/vtexplain/vtexplain_vtgate.go index 5a86c06e840..14ed0d21b6c 100644 --- a/go/vt/vtexplain/vtexplain_vtgate.go +++ b/go/vt/vtexplain/vtexplain_vtgate.go @@ -32,8 +32,6 @@ import ( "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/vtgate" "vitess.io/vitess/go/vt/vtgate/engine" - "vitess.io/vitess/go/vt/vtgate/gateway" - "vitess.io/vitess/go/vt/vttablet/queryservice" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -74,7 +72,7 @@ func initVtgateExecutor(vSchemaStr string, opts *Options) error { func newFakeResolver(opts *Options, hc discovery.HealthCheck, serv srvtopo.Server, cell string) *vtgate.Resolver { ctx := context.Background() - gw := gateway.GetCreator()(ctx, hc, serv, cell, 3) + gw := vtgate.New(ctx, hc, serv, cell, 3) gw.WaitForTablets(ctx, []topodatapb.TabletType{topodatapb.TabletType_REPLICA}) txMode := vtgatepb.TransactionMode_MULTI diff --git a/go/vt/vtgate/gateway/discoverygateway.go b/go/vt/vtgate/discoverygateway.go similarity index 79% rename from go/vt/vtgate/gateway/discoverygateway.go rename to go/vt/vtgate/discoverygateway.go index e8f2240ee40..2eded2f4f02 100644 --- a/go/vt/vtgate/gateway/discoverygateway.go +++ b/go/vt/vtgate/discoverygateway.go @@ -14,18 +14,18 @@ See the License for the specific language governing permissions and limitations under the License. */ -package gateway +package vtgate import ( "flag" "fmt" + "golang.org/x/net/context" "math/rand" "sort" "strings" "sync" "time" - - "golang.org/x/net/context" + "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/flagutil" "vitess.io/vitess/go/stats" @@ -44,25 +44,28 @@ import ( ) var ( + _ = flag.String("gateway_implementation", "discoverygateway", "Deprecated") + initialTabletTimeout = flag.Duration("gateway_initial_tablet_timeout", 30*time.Second, "At startup, the gateway will wait up to that duration to get one tablet per keyspace/shard/tablettype") + // KeyspacesToWatch - if provided this specifies which keyspaces should be + // visible to a vtgate. By default the vtgate will allow access to any + // keyspace. cellsToWatch = flag.String("cells_to_watch", "", "comma-separated list of cells for watching tablets") - tabletFilters flagutil.StringListValue refreshInterval = flag.Duration("tablet_refresh_interval", 1*time.Minute, "tablet refresh interval") refreshKnownTablets = flag.Bool("tablet_refresh_known_tablets", true, "tablet refresh reloads the tablet address/port map from topo in case it changes") topoReadConcurrency = flag.Int("topo_read_concurrency", 32, "concurrent topo reads") + allowedTabletTypes []topodatapb.TabletType -) -const ( - gatewayImplementationDiscovery = "discoverygateway" + tabletFilters flagutil.StringListValue + KeyspacesToWatch flagutil.StringListValue ) - func init() { + flag.Var(&KeyspacesToWatch, "keyspaces_to_watch", "Specifies which keyspaces this vtgate should have access to while routing queries or accessing the vschema") flag.Var(&tabletFilters, "tablet_filters", "Specifies a comma-separated list of 'keyspace|shard_name or keyrange' values to filter the tablets to watch") topoproto.TabletTypeListVar(&allowedTabletTypes, "allowed_tablet_types", "Specifies the tablet types this vtgate is allowed to route queries to") - RegisterCreator(gatewayImplementationDiscovery, createDiscoveryGateway) } -type discoveryGateway struct { +type DiscoveryGateway struct { queryservice.QueryService hc discovery.HealthCheck tsc *discovery.TabletStatsCache @@ -84,7 +87,7 @@ type discoveryGateway struct { buffer *buffer.Buffer } -func createDiscoveryGateway(ctx context.Context, hc discovery.HealthCheck, serv srvtopo.Server, cell string, retryCount int) Gateway { +func New(ctx context.Context, hc discovery.HealthCheck, serv srvtopo.Server, cell string, retryCount int) *DiscoveryGateway { var topoServer *topo.Server if serv != nil { var err error @@ -94,7 +97,7 @@ func createDiscoveryGateway(ctx context.Context, hc discovery.HealthCheck, serv } } - dg := &discoveryGateway{ + dg := &DiscoveryGateway{ hc: hc, tsc: discovery.NewTabletStatsCacheDoNotSetListener(topoServer, cell), srvTopoServer: serv, @@ -138,7 +141,7 @@ func createDiscoveryGateway(ctx context.Context, hc discovery.HealthCheck, serv // RegisterStats registers the stats to export the lag since the last refresh // and the checksum of the topology -func (dg *discoveryGateway) RegisterStats() { +func (dg *DiscoveryGateway) RegisterStats() { stats.NewGaugeDurationFunc( "TopologyWatcherMaxRefreshLag", "maximum time since the topology watcher refreshed a cell", @@ -154,7 +157,7 @@ func (dg *discoveryGateway) RegisterStats() { // topologyWatcherMaxRefreshLag returns the maximum lag since the watched // cells were refreshed from the topo server -func (dg *discoveryGateway) topologyWatcherMaxRefreshLag() time.Duration { +func (dg *DiscoveryGateway) topologyWatcherMaxRefreshLag() time.Duration { var lag time.Duration for _, tw := range dg.tabletsWatchers { cellLag := tw.RefreshLag() @@ -166,7 +169,7 @@ func (dg *discoveryGateway) topologyWatcherMaxRefreshLag() time.Duration { } // topologyWatcherChecksum returns a checksum of the topology watcher state -func (dg *discoveryGateway) topologyWatcherChecksum() int64 { +func (dg *DiscoveryGateway) topologyWatcherChecksum() int64 { var checksum int64 for _, tw := range dg.tabletsWatchers { checksum = checksum ^ int64(tw.TopoChecksum()) @@ -176,7 +179,7 @@ func (dg *discoveryGateway) topologyWatcherChecksum() int64 { // StatsUpdate forwards HealthCheck updates to TabletStatsCache and MasterBuffer. // It is part of the discovery.HealthCheckStatsListener interface. -func (dg *discoveryGateway) StatsUpdate(ts *discovery.TabletStats) { +func (dg *DiscoveryGateway) StatsUpdate(ts *discovery.TabletStats) { dg.tsc.StatsUpdate(ts) if ts.Target.TabletType == topodatapb.TabletType_MASTER { @@ -185,7 +188,7 @@ func (dg *discoveryGateway) StatsUpdate(ts *discovery.TabletStats) { } // WaitForTablets is part of the gateway.Gateway interface. -func (dg *discoveryGateway) WaitForTablets(ctx context.Context, tabletTypesToWait []topodatapb.TabletType) error { +func (dg *DiscoveryGateway) WaitForTablets(ctx context.Context, tabletTypesToWait []topodatapb.TabletType) error { // Skip waiting for tablets if we are not told to do so. if len(tabletTypesToWait) == 0 { return nil @@ -202,7 +205,7 @@ func (dg *discoveryGateway) WaitForTablets(ctx context.Context, tabletTypesToWai // Close shuts down underlying connections. // This function hides the inner implementation. -func (dg *discoveryGateway) Close(ctx context.Context) error { +func (dg *DiscoveryGateway) Close(ctx context.Context) error { dg.buffer.Shutdown() for _, ctw := range dg.tabletsWatchers { ctw.Stop() @@ -212,7 +215,7 @@ func (dg *discoveryGateway) Close(ctx context.Context) error { // CacheStatus returns a list of TabletCacheStatus per // keyspace/shard/tablet_type. -func (dg *discoveryGateway) CacheStatus() TabletCacheStatusList { +func (dg *DiscoveryGateway) CacheStatus() TabletCacheStatusList { dg.mu.RLock() res := make(TabletCacheStatusList, 0, len(dg.statusAggregators)) for _, aggr := range dg.statusAggregators { @@ -228,7 +231,7 @@ func (dg *discoveryGateway) CacheStatus() TabletCacheStatusList { // the middle of a transaction. While returning the error check if it maybe a result of // a resharding event, and set the re-resolve bit and let the upper layers // re-resolve and retry. -func (dg *discoveryGateway) withRetry(ctx context.Context, target *querypb.Target, unused queryservice.QueryService, name string, inTransaction bool, inner func(ctx context.Context, target *querypb.Target, conn queryservice.QueryService) (bool, error)) error { +func (dg *DiscoveryGateway) withRetry(ctx context.Context, target *querypb.Target, unused queryservice.QueryService, name string, inTransaction bool, inner func(ctx context.Context, target *querypb.Target, conn queryservice.QueryService) (bool, error)) error { var tabletLastUsed *topodatapb.Tablet var err error invalidTablets := make(map[string]bool) @@ -367,13 +370,13 @@ func nextTablet(cell string, tablets []discovery.TabletStats, offset, length int return -1 } -func (dg *discoveryGateway) updateStats(target *querypb.Target, startTime time.Time, err error) { +func (dg *DiscoveryGateway) updateStats(target *querypb.Target, startTime time.Time, err error) { elapsed := time.Since(startTime) aggr := dg.getStatsAggregator(target) aggr.UpdateQueryInfo("", target.TabletType, elapsed, err != nil) } -func (dg *discoveryGateway) getStatsAggregator(target *querypb.Target) *TabletStatusAggregator { +func (dg *DiscoveryGateway) getStatsAggregator(target *querypb.Target) *TabletStatusAggregator { key := fmt.Sprintf("%v/%v/%v", target.Keyspace, target.Shard, target.TabletType.String()) // get existing aggregator @@ -394,3 +397,43 @@ func (dg *discoveryGateway) getStatsAggregator(target *querypb.Target) *TabletSt dg.statusAggregators[key] = aggr return aggr } + +// WaitForTablets is a helper method to wait for the provided tablets, +// up until the *initialTabletTimeout. It will log what it is doing. +// Note it has the same name as the DiscoveryGateway's method, as it +// just calls it. +func WaitForTablets(gw *DiscoveryGateway, tabletTypesToWait []topodatapb.TabletType) error { + log.Infof("Gateway waiting for serving tablets of types %v ...", tabletTypesToWait) + ctx, cancel := context.WithTimeout(context.Background(), *initialTabletTimeout) + defer cancel() + + err := gw.WaitForTablets(ctx, tabletTypesToWait) + switch err { + case nil: + // Log so we know everything is fine. + log.Infof("Waiting for tablets completed") + case context.DeadlineExceeded: + // In this scenario, we were able to reach the + // topology service, but some tablets may not be + // ready. We just warn and keep going. + log.Warningf("Timeout waiting for all keyspaces / shards to have healthy tablets of types %v, may be in degraded mode", tabletTypesToWait) + err = nil + default: + // Nothing to do here, the caller will log.Fatalf. + } + return err +} + +// NewShardError returns a new error with the shard info amended. +func NewShardError(in error, target *querypb.Target, tablet *topodatapb.Tablet) error { + if in == nil { + return nil + } + if tablet != nil { + return vterrors.Wrapf(in, "target: %s.%s.%s, used tablet: %s", target.Keyspace, target.Shard, topoproto.TabletTypeLString(target.TabletType), topotools.TabletIdent(tablet)) + } + if target != nil { + return vterrors.Wrapf(in, "target: %s.%s.%s", target.Keyspace, target.Shard, topoproto.TabletTypeLString(target.TabletType)) + } + return in +} diff --git a/go/vt/vtgate/gateway/discoverygateway_test.go b/go/vt/vtgate/discoverygateway_test.go similarity index 90% rename from go/vt/vtgate/gateway/discoverygateway_test.go rename to go/vt/vtgate/discoverygateway_test.go index d6da59c34a9..7d2171055d6 100644 --- a/go/vt/vtgate/gateway/discoverygateway_test.go +++ b/go/vt/vtgate/discoverygateway_test.go @@ -14,13 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -package gateway +package vtgate import ( "fmt" "strings" "testing" - "golang.org/x/net/context" "vitess.io/vitess/go/sqltypes" @@ -37,23 +36,23 @@ import ( ) func TestDiscoveryGatewayExecute(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg Gateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { _, err := dg.Execute(context.Background(), target, "query", nil, 0, nil) return err }) - testDiscoveryGatewayTransact(t, func(dg Gateway, target *querypb.Target) error { + testDiscoveryGatewayTransact(t, func(dg *DiscoveryGateway, target *querypb.Target) error { _, err := dg.Execute(context.Background(), target, "query", nil, 1, nil) return err }) } func TestDiscoveryGatewayExecuteBatch(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg Gateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { queries := []*querypb.BoundQuery{{Sql: "query", BindVariables: nil}} _, err := dg.ExecuteBatch(context.Background(), target, queries, false, 0, nil) return err }) - testDiscoveryGatewayTransact(t, func(dg Gateway, target *querypb.Target) error { + testDiscoveryGatewayTransact(t, func(dg *DiscoveryGateway, target *querypb.Target) error { queries := []*querypb.BoundQuery{{Sql: "query", BindVariables: nil}} _, err := dg.ExecuteBatch(context.Background(), target, queries, false, 1, nil) return err @@ -61,7 +60,7 @@ func TestDiscoveryGatewayExecuteBatch(t *testing.T) { } func TestDiscoveryGatewayExecuteStream(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg Gateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { err := dg.StreamExecute(context.Background(), target, "query", nil, 0, nil, func(qr *sqltypes.Result) error { return nil }) @@ -70,33 +69,33 @@ func TestDiscoveryGatewayExecuteStream(t *testing.T) { } func TestDiscoveryGatewayBegin(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg Gateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { _, err := dg.Begin(context.Background(), target, nil) return err }) } func TestDiscoveryGatewayCommit(t *testing.T) { - testDiscoveryGatewayTransact(t, func(dg Gateway, target *querypb.Target) error { + testDiscoveryGatewayTransact(t, func(dg *DiscoveryGateway, target *querypb.Target) error { return dg.Commit(context.Background(), target, 1) }) } func TestDiscoveryGatewayRollback(t *testing.T) { - testDiscoveryGatewayTransact(t, func(dg Gateway, target *querypb.Target) error { + testDiscoveryGatewayTransact(t, func(dg *DiscoveryGateway, target *querypb.Target) error { return dg.Rollback(context.Background(), target, 1) }) } func TestDiscoveryGatewayBeginExecute(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg Gateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { _, _, err := dg.BeginExecute(context.Background(), target, "query", nil, nil) return err }) } func TestDiscoveryGatewayBeginExecuteBatch(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg Gateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { queries := []*querypb.BoundQuery{{Sql: "query", BindVariables: nil}} _, _, err := dg.BeginExecuteBatch(context.Background(), target, queries, false, nil) return err @@ -107,7 +106,7 @@ func TestDiscoveryGatewayGetTablets(t *testing.T) { keyspace := "ks" shard := "0" hc := discovery.NewFakeHealthCheck() - dg := createDiscoveryGateway(context.Background(), hc, nil, "local", 2).(*discoveryGateway) + dg := New(context.Background(), hc, nil, "local", 2) // replica should only use local ones hc.Reset() @@ -215,7 +214,7 @@ func TestDiscoveryGatewayGetTabletsInRegion(t *testing.T) { Cells: []string{"local-west", "local-east"}, } - dg := createDiscoveryGateway(context.Background(), hc, srvTopo, "local-west", 2).(*discoveryGateway) + dg := New(context.Background(), hc, srvTopo, "local-west", 2) ts.CreateCellsAlias(context.Background(), "local", cellsAlias) @@ -245,7 +244,7 @@ func TestDiscoveryGatewayGetTabletsWithRegion(t *testing.T) { Cells: []string{"local-west", "local-east"}, } - dg := createDiscoveryGateway(context.Background(), hc, srvTopo, "local", 2).(*discoveryGateway) + dg := New(context.Background(), hc, srvTopo, "local", 2) ts.CreateCellsAlias(context.Background(), "local", cellsAlias) @@ -264,7 +263,7 @@ func TestDiscoveryGatewayGetTabletsWithRegion(t *testing.T) { } } -func testDiscoveryGatewayGeneric(t *testing.T, f func(dg Gateway, target *querypb.Target) error) { +func testDiscoveryGatewayGeneric(t *testing.T, f func(dg *DiscoveryGateway, target *querypb.Target) error) { keyspace := "ks" shard := "0" tabletType := topodatapb.TabletType_REPLICA @@ -274,7 +273,7 @@ func testDiscoveryGatewayGeneric(t *testing.T, f func(dg Gateway, target *queryp TabletType: tabletType, } hc := discovery.NewFakeHealthCheck() - dg := createDiscoveryGateway(context.Background(), hc, nil, "cell", 2).(*discoveryGateway) + dg := New(context.Background(), hc, nil, "cell", 2) // no tablet hc.Reset() @@ -347,7 +346,7 @@ func testDiscoveryGatewayGeneric(t *testing.T, f func(dg Gateway, target *queryp } } -func testDiscoveryGatewayTransact(t *testing.T, f func(dg Gateway, target *querypb.Target) error) { +func testDiscoveryGatewayTransact(t *testing.T, f func(dg *DiscoveryGateway, target *querypb.Target) error) { keyspace := "ks" shard := "0" tabletType := topodatapb.TabletType_REPLICA @@ -357,7 +356,7 @@ func testDiscoveryGatewayTransact(t *testing.T, f func(dg Gateway, target *query TabletType: tabletType, } hc := discovery.NewFakeHealthCheck() - dg := createDiscoveryGateway(context.Background(), hc, nil, "cell", 2).(*discoveryGateway) + dg := New(context.Background(), hc, nil, "cell", 2) // retry error - no retry hc.Reset() diff --git a/go/vt/vtgate/gateway/gateway.go b/go/vt/vtgate/gateway/gateway.go deleted file mode 100644 index 725df19f493..00000000000 --- a/go/vt/vtgate/gateway/gateway.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package gateway contains the routing layer of vtgate. A Gateway can take -// a query targeted to a keyspace/shard/tablet_type and send it off. -package gateway - -import ( - "flag" - "time" - - "golang.org/x/net/context" - "vitess.io/vitess/go/flagutil" - "vitess.io/vitess/go/vt/log" - - "vitess.io/vitess/go/vt/discovery" - "vitess.io/vitess/go/vt/srvtopo" - "vitess.io/vitess/go/vt/vttablet/queryservice" - - topodatapb "vitess.io/vitess/go/vt/proto/topodata" -) - -// This file contains the Gateway interface definition, and the -// implementations registry. - -var ( - implementation = flag.String("gateway_implementation", "discoverygateway", "The implementation of gateway") - initialTabletTimeout = flag.Duration("gateway_initial_tablet_timeout", 30*time.Second, "At startup, the gateway will wait up to that duration to get one tablet per keyspace/shard/tablettype") - - // KeyspacesToWatch - if provided this specifies which keyspaces should be - // visible to a vtgate. By default the vtgate will allow access to any - // keyspace. - KeyspacesToWatch flagutil.StringListValue -) - -func init() { - flag.Var(&KeyspacesToWatch, "keyspaces_to_watch", "Specifies which keyspaces this vtgate should have access to while routing queries or accessing the vschema") -} - -// A Gateway is the query processing module for each shard, -// which is used by ScatterConn. -type Gateway interface { - // the query service that this Gateway wraps around - queryservice.QueryService - - // WaitForTablets asks the gateway to wait for the provided - // tablets types to be available. It the context is canceled - // before the end, it should return ctx.Err(). - // The error returned will have specific effects: - // - nil: keep going with startup. - // - context.DeadlineExceeded: log a warning that we didn't get - // all tablets, and keep going with startup. - // - any other error: log.Fatalf out. - WaitForTablets(ctx context.Context, tabletTypesToWait []topodatapb.TabletType) error - - // RegisterStats registers exported stats for the gateway - RegisterStats() - - // CacheStatus returns a list of TabletCacheStatus per shard / tablet type. - CacheStatus() TabletCacheStatusList -} - -// Creator is the factory method which can create the actual gateway object. -type Creator func(ctx context.Context, hc discovery.HealthCheck, serv srvtopo.Server, cell string, retryCount int) Gateway - -var creators = make(map[string]Creator) - -// RegisterCreator registers a Creator with given name. -func RegisterCreator(name string, gc Creator) { - if _, ok := creators[name]; ok { - log.Fatalf("Gateway %s already exists", name) - } - creators[name] = gc -} - -// GetCreator returns the Creator specified by the gateway_implementation flag. -func GetCreator() Creator { - gc, ok := creators[*implementation] - if !ok { - log.Exitf("No gateway registered as %s", *implementation) - } - return gc -} - -// WaitForTablets is a helper method to wait for the provided tablets, -// up until the *initialTabletTimeout. It will log what it is doing. -// Note it has the same name as the Gateway's interface method, as it -// just calls it. -func WaitForTablets(gw Gateway, tabletTypesToWait []topodatapb.TabletType) error { - log.Infof("Gateway waiting for serving tablets of types %v ...", tabletTypesToWait) - ctx, cancel := context.WithTimeout(context.Background(), *initialTabletTimeout) - defer cancel() - - err := gw.WaitForTablets(ctx, tabletTypesToWait) - switch err { - case nil: - // Log so we know everything is fine. - log.Infof("Waiting for tablets completed") - case context.DeadlineExceeded: - // In this scenario, we were able to reach the - // topology service, but some tablets may not be - // ready. We just warn and keep going. - log.Warningf("Timeout waiting for all keyspaces / shards to have healthy tablets of types %v, may be in degraded mode", tabletTypesToWait) - err = nil - default: - // Nothing to do here, the caller will log.Fatalf. - } - return err -} diff --git a/go/vt/vtgate/gateway/shard_error.go b/go/vt/vtgate/gateway/shard_error.go deleted file mode 100644 index 15057f4c955..00000000000 --- a/go/vt/vtgate/gateway/shard_error.go +++ /dev/null @@ -1,40 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gateway - -import ( - "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/topotools" - "vitess.io/vitess/go/vt/vterrors" - - querypb "vitess.io/vitess/go/vt/proto/query" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" -) - -// NewShardError returns a new error with the shard info amended. -func NewShardError(in error, target *querypb.Target, tablet *topodatapb.Tablet) error { - if in == nil { - return nil - } - if tablet != nil { - return vterrors.Wrapf(in, "target: %s.%s.%s, used tablet: %s", target.Keyspace, target.Shard, topoproto.TabletTypeLString(target.TabletType), topotools.TabletIdent(tablet)) - } - if target != nil { - return vterrors.Wrapf(in, "target: %s.%s.%s", target.Keyspace, target.Shard, topoproto.TabletTypeLString(target.TabletType)) - } - return in -} diff --git a/go/vt/vtgate/gatewaytest/grpc_discovery_test.go b/go/vt/vtgate/gatewaytest/grpc_discovery_test.go index c52b105f6e7..f9f6e4207ad 100644 --- a/go/vt/vtgate/gatewaytest/grpc_discovery_test.go +++ b/go/vt/vtgate/gatewaytest/grpc_discovery_test.go @@ -22,13 +22,14 @@ import ( "testing" "time" + "vitess.io/vitess/go/vt/vtgate" + "golang.org/x/net/context" "google.golang.org/grpc" "vitess.io/vitess/go/vt/discovery" "vitess.io/vitess/go/vt/srvtopo" - "vitess.io/vitess/go/vt/vtgate/gateway" "vitess.io/vitess/go/vt/vttablet/grpcqueryservice" "vitess.io/vitess/go/vt/vttablet/tabletconntest" @@ -42,7 +43,6 @@ import ( // connection from the gateway to the fake tablet. func TestGRPCDiscovery(t *testing.T) { flag.Set("tablet_protocol", "grpc") - flag.Set("gateway_implementation", "discoverygateway") // Fake services for the tablet, topo server. service, ts, cell := CreateFakeServers(t) @@ -66,7 +66,7 @@ func TestGRPCDiscovery(t *testing.T) { // Wait for the right tablets to be present. hc := discovery.NewHealthCheck(10*time.Second, 2*time.Minute) rs := srvtopo.NewResilientServer(ts, "TestGRPCDiscovery") - dg := gateway.GetCreator()(context.Background(), hc, rs, cell, 2) + dg := vtgate.New(context.Background(), hc, rs, cell, 2) hc.AddTablet(&topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: cell, @@ -80,7 +80,7 @@ func TestGRPCDiscovery(t *testing.T) { "grpc": int32(port), }, }, "test_tablet") - err = gateway.WaitForTablets(dg, []topodatapb.TabletType{tabletconntest.TestTarget.TabletType}) + err = vtgate.WaitForTablets(dg, []topodatapb.TabletType{tabletconntest.TestTarget.TabletType}) if err != nil { t.Fatalf("WaitForTablets failed: %v", err) } diff --git a/go/vt/vtgate/gatewaytest/suite.go b/go/vt/vtgate/gatewaytest/suite.go index 0f00fd53832..8a68a7c5da1 100644 --- a/go/vt/vtgate/gatewaytest/suite.go +++ b/go/vt/vtgate/gatewaytest/suite.go @@ -27,12 +27,13 @@ package gatewaytest import ( "testing" + "vitess.io/vitess/go/vt/vtgate" + "golang.org/x/net/context" "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/memorytopo" - "vitess.io/vitess/go/vt/vtgate/gateway" "vitess.io/vitess/go/vt/vttablet/queryservice" "vitess.io/vitess/go/vt/vttablet/tabletconn" "vitess.io/vitess/go/vt/vttablet/tabletconntest" @@ -80,11 +81,11 @@ func CreateFakeServers(t *testing.T) (*tabletconntest.FakeQueryService, *topo.Se // gatewayAdapter implements the TabletConn interface, but sends the // queries to the Gateway. type gatewayAdapter struct { - gateway.Gateway + *vtgate.DiscoveryGateway } // Close should be overridden to make sure we don't close the underlying Gateway. -func (ga gatewayAdapter) Close(ctx context.Context) error { +func (ga *gatewayAdapter) Close(ctx context.Context) error { return nil } @@ -92,12 +93,12 @@ func (ga gatewayAdapter) Close(ctx context.Context) error { // gateway needs to be configured with one established connection for // tabletconntest.TestTarget.{Keyspace, Shard, TabletType} to the // provided tabletconntest.FakeQueryService. -func TestSuite(t *testing.T, name string, g gateway.Gateway, f *tabletconntest.FakeQueryService) { +func TestSuite(t *testing.T, name string, g *vtgate.DiscoveryGateway, f *tabletconntest.FakeQueryService) { protocolName := "gateway-test-" + name tabletconn.RegisterDialer(protocolName, func(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (queryservice.QueryService, error) { - return &gatewayAdapter{Gateway: g}, nil + return &gatewayAdapter{DiscoveryGateway: g}, nil }) tabletconntest.TestSuite(t, protocolName, &topodatapb.Tablet{ diff --git a/go/vt/vtgate/resolver.go b/go/vt/vtgate/resolver.go index fa3a6aa6f26..23367fbab26 100644 --- a/go/vt/vtgate/resolver.go +++ b/go/vt/vtgate/resolver.go @@ -20,13 +20,11 @@ import ( "golang.org/x/net/context" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" - "vitess.io/vitess/go/vt/srvtopo" - "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/vt/vtgate/gateway" - querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/srvtopo" + "vitess.io/vitess/go/vt/vterrors" ) // Resolver is the layer to resolve KeyspaceIds and KeyRanges @@ -163,6 +161,6 @@ func (res *Resolver) MessageStream(ctx context.Context, keyspace string, shard s } // GetGatewayCacheStatus returns a displayable version of the Gateway cache. -func (res *Resolver) GetGatewayCacheStatus() gateway.TabletCacheStatusList { +func (res *Resolver) GetGatewayCacheStatus() TabletCacheStatusList { return res.scatterConn.GetGatewayCacheStatus() } diff --git a/go/vt/vtgate/scatter_conn.go b/go/vt/vtgate/scatter_conn.go index 5d175751add..624560a0d9e 100644 --- a/go/vt/vtgate/scatter_conn.go +++ b/go/vt/vtgate/scatter_conn.go @@ -28,15 +28,13 @@ import ( "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/concurrency" "vitess.io/vitess/go/vt/discovery" - "vitess.io/vitess/go/vt/srvtopo" - "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/vt/vtgate/gateway" - querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/srvtopo" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/vterrors" ) var ( @@ -49,7 +47,7 @@ type ScatterConn struct { timings *stats.MultiTimings tabletCallErrorCount *stats.CountersWithMultiLabels txConn *TxConn - gateway gateway.Gateway + gateway *DiscoveryGateway healthCheck discovery.HealthCheck } @@ -71,7 +69,7 @@ type shardActionFunc func(rs *srvtopo.ResolvedShard, i int) error type shardActionTransactionFunc func(rs *srvtopo.ResolvedShard, i int, shouldBegin bool, transactionID int64) (int64, error) // NewScatterConn creates a new ScatterConn. -func NewScatterConn(statsName string, txConn *TxConn, gw gateway.Gateway, hc discovery.HealthCheck) *ScatterConn { +func NewScatterConn(statsName string, txConn *TxConn, gw *DiscoveryGateway, hc discovery.HealthCheck) *ScatterConn { tabletCallErrorCountStatsName := "" if statsName != "" { tabletCallErrorCountStatsName = statsName + "ErrorCount" @@ -417,7 +415,7 @@ func (stc *ScatterConn) Close() error { } // GetGatewayCacheStatus returns a displayable version of the Gateway cache. -func (stc *ScatterConn) GetGatewayCacheStatus() gateway.TabletCacheStatusList { +func (stc *ScatterConn) GetGatewayCacheStatus() TabletCacheStatusList { return stc.gateway.CacheStatus() } diff --git a/go/vt/vtgate/scatter_conn_test.go b/go/vt/vtgate/scatter_conn_test.go index 7566a63b327..90dbd15f66e 100644 --- a/go/vt/vtgate/scatter_conn_test.go +++ b/go/vt/vtgate/scatter_conn_test.go @@ -29,14 +29,12 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/discovery" "vitess.io/vitess/go/vt/key" - "vitess.io/vitess/go/vt/srvtopo" - "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/vt/vtgate/gateway" - querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/srvtopo" + "vitess.io/vitess/go/vt/vterrors" ) // This file uses the sandbox_test framework. @@ -654,7 +652,7 @@ func newTestScatterConn(hc discovery.HealthCheck, serv srvtopo.Server, cell stri // The topo.Server is used to start watching the cells described // in '-cells_to_watch' command line parameter, which is // empty by default. So it's unused in this test, set to nil. - gw := gateway.GetCreator()(context.Background(), hc, serv, cell, 3) + gw := New(context.Background(), hc, serv, cell, 3) tc := NewTxConn(gw, vtgatepb.TransactionMode_TWOPC) return NewScatterConn("", tc, gw, hc) } diff --git a/go/vt/vtgate/gateway/status.go b/go/vt/vtgate/status.go similarity index 99% rename from go/vt/vtgate/gateway/status.go rename to go/vt/vtgate/status.go index c6d6eb30752..40c545b0107 100644 --- a/go/vt/vtgate/gateway/status.go +++ b/go/vt/vtgate/status.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package gateway +package vtgate import ( "fmt" diff --git a/go/vt/vtgate/gateway/status_test.go b/go/vt/vtgate/status_test.go similarity index 99% rename from go/vt/vtgate/gateway/status_test.go rename to go/vt/vtgate/status_test.go index ce61c2893f4..8472faae5d2 100644 --- a/go/vt/vtgate/gateway/status_test.go +++ b/go/vt/vtgate/status_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package gateway +package vtgate import ( "bytes" @@ -22,7 +22,6 @@ import ( "reflect" "testing" "time" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) diff --git a/go/vt/vtgate/tx_conn.go b/go/vt/vtgate/tx_conn.go index 2089500237f..6efd46460ea 100644 --- a/go/vt/vtgate/tx_conn.go +++ b/go/vt/vtgate/tx_conn.go @@ -25,22 +25,20 @@ import ( "vitess.io/vitess/go/vt/concurrency" "vitess.io/vitess/go/vt/dtids" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/vt/vtgate/gateway" - querypb "vitess.io/vitess/go/vt/proto/query" vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" ) // TxConn is used for executing transactional requests. type TxConn struct { - gateway gateway.Gateway + gateway *DiscoveryGateway mode vtgatepb.TransactionMode } // NewTxConn builds a new TxConn. -func NewTxConn(gw gateway.Gateway, txMode vtgatepb.TransactionMode) *TxConn { +func NewTxConn(gw *DiscoveryGateway, txMode vtgatepb.TransactionMode) *TxConn { return &TxConn{ gateway: gw, mode: txMode, diff --git a/go/vt/vtgate/vstream_manager_test.go b/go/vt/vtgate/vstream_manager_test.go index 673c00d6b82..2871eabd09b 100644 --- a/go/vt/vtgate/vstream_manager_test.go +++ b/go/vt/vtgate/vstream_manager_test.go @@ -32,7 +32,6 @@ import ( vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/vt/vtgate/gateway" ) func TestVStreamEvents(t *testing.T) { @@ -874,7 +873,7 @@ func TestResolveVStreamParams(t *testing.T) { } func newTestVStreamManager(hc discovery.HealthCheck, serv srvtopo.Server, cell string) *vstreamManager { - gw := gateway.GetCreator()(context.Background(), hc, serv, cell, 3) + gw := New(context.Background(), hc, serv, cell, 3) srvResolver := srvtopo.NewResolver(serv, gw, cell) return newVStreamManager(srvResolver, serv, cell) } diff --git a/go/vt/vtgate/vtgate.go b/go/vt/vtgate/vtgate.go index ddf6339343b..be76e04b2c3 100644 --- a/go/vt/vtgate/vtgate.go +++ b/go/vt/vtgate/vtgate.go @@ -42,7 +42,6 @@ import ( "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/vt/vtgate/gateway" "vitess.io/vitess/go/vt/vtgate/vtgateservice" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" @@ -102,7 +101,7 @@ type VTGate struct { resolver *Resolver vsm *vstreamManager txConn *TxConn - gw gateway.Gateway + gw *DiscoveryGateway // stats objects. // TODO(sougou): This needs to be cleaned up. There @@ -134,18 +133,18 @@ func Init(ctx context.Context, hc discovery.HealthCheck, serv srvtopo.Server, ce // Build objects from low to high level. // Start with the gateway. If we can't reach the topology service, // we can't go on much further, so we log.Fatal out. - gw := gateway.GetCreator()(ctx, hc, serv, cell, retryCount) + gw := New(ctx, hc, serv, cell, retryCount) gw.RegisterStats() - if err := gateway.WaitForTablets(gw, tabletTypesToWait); err != nil { + if err := WaitForTablets(gw, tabletTypesToWait); err != nil { log.Fatalf("gateway.WaitForTablets failed: %v", err) } // If we want to filter keyspaces replace the srvtopo.Server with a // filtering server - if len(gateway.KeyspacesToWatch) > 0 { - log.Infof("Keyspace filtering enabled, selecting %v", gateway.KeyspacesToWatch) + if len(KeyspacesToWatch) > 0 { + log.Infof("Keyspace filtering enabled, selecting %v", KeyspacesToWatch) var err error - serv, err = srvtopo.NewKeyspaceFilteringServer(serv, gateway.KeyspacesToWatch) + serv, err = srvtopo.NewKeyspaceFilteringServer(serv, KeyspacesToWatch) if err != nil { log.Fatalf("Unable to construct SrvTopo server: %v", err.Error()) } @@ -228,7 +227,7 @@ func (vtg *VTGate) IsHealthy() error { } // Gateway returns the current gateway implementation. Mostly used for tests. -func (vtg *VTGate) Gateway() gateway.Gateway { +func (vtg *VTGate) Gateway() *DiscoveryGateway { return vtg.gw } @@ -386,7 +385,7 @@ func (vtg *VTGate) VStream(ctx context.Context, tabletType topodatapb.TabletType } // GetGatewayCacheStatus returns a displayable version of the Gateway cache. -func (vtg *VTGate) GetGatewayCacheStatus() gateway.TabletCacheStatusList { +func (vtg *VTGate) GetGatewayCacheStatus() TabletCacheStatusList { return vtg.resolver.GetGatewayCacheStatus() } From 698519d7a8b2bf6bf0584bb883d36572364be374 Mon Sep 17 00:00:00 2001 From: deepthi Date: Thu, 26 Mar 2020 16:45:55 -0700 Subject: [PATCH 64/85] rename DiscoveryGateway -> tabletGateway, fix gofmt errors Signed-off-by: deepthi --- go/vt/vtexplain/vtexplain_vtgate.go | 2 +- .../suite.go => gateway_test_suite.go} | 10 ++--- .../{gatewaytest => }/grpc_discovery_test.go | 11 ++---- go/vt/vtgate/scatter_conn.go | 4 +- go/vt/vtgate/scatter_conn_test.go | 2 +- .../{discoverygateway.go => tabletgateway.go} | 37 ++++++++++--------- ...ygateway_test.go => tabletgateway_test.go} | 36 +++++++++--------- go/vt/vtgate/tx_conn.go | 4 +- go/vt/vtgate/vstream_manager_test.go | 2 +- go/vt/vtgate/vtgate.go | 6 +-- go/vt/vttablet/queryservice/wrapped.go | 2 +- 11 files changed, 56 insertions(+), 60 deletions(-) rename go/vt/vtgate/{gatewaytest/suite.go => gateway_test_suite.go} (93%) rename go/vt/vtgate/{gatewaytest => }/grpc_discovery_test.go (92%) rename go/vt/vtgate/{discoverygateway.go => tabletgateway.go} (90%) rename go/vt/vtgate/{discoverygateway_test.go => tabletgateway_test.go} (90%) diff --git a/go/vt/vtexplain/vtexplain_vtgate.go b/go/vt/vtexplain/vtexplain_vtgate.go index 14ed0d21b6c..9cedd20807a 100644 --- a/go/vt/vtexplain/vtexplain_vtgate.go +++ b/go/vt/vtexplain/vtexplain_vtgate.go @@ -72,7 +72,7 @@ func initVtgateExecutor(vSchemaStr string, opts *Options) error { func newFakeResolver(opts *Options, hc discovery.HealthCheck, serv srvtopo.Server, cell string) *vtgate.Resolver { ctx := context.Background() - gw := vtgate.New(ctx, hc, serv, cell, 3) + gw := vtgate.NewTabletGateway(ctx, hc, serv, cell, 3) gw.WaitForTablets(ctx, []topodatapb.TabletType{topodatapb.TabletType_REPLICA}) txMode := vtgatepb.TransactionMode_MULTI diff --git a/go/vt/vtgate/gatewaytest/suite.go b/go/vt/vtgate/gateway_test_suite.go similarity index 93% rename from go/vt/vtgate/gatewaytest/suite.go rename to go/vt/vtgate/gateway_test_suite.go index 8a68a7c5da1..db5c69bb62b 100644 --- a/go/vt/vtgate/gatewaytest/suite.go +++ b/go/vt/vtgate/gateway_test_suite.go @@ -22,13 +22,11 @@ limitations under the License. // - the error type returned: it's not a TabletError any more, but a ShardError. // We still check the error code is correct though which is really all we care // about. -package gatewaytest +package vtgate import ( "testing" - "vitess.io/vitess/go/vt/vtgate" - "golang.org/x/net/context" "vitess.io/vitess/go/vt/grpcclient" @@ -81,7 +79,7 @@ func CreateFakeServers(t *testing.T) (*tabletconntest.FakeQueryService, *topo.Se // gatewayAdapter implements the TabletConn interface, but sends the // queries to the Gateway. type gatewayAdapter struct { - *vtgate.DiscoveryGateway + *tabletGateway } // Close should be overridden to make sure we don't close the underlying Gateway. @@ -93,12 +91,12 @@ func (ga *gatewayAdapter) Close(ctx context.Context) error { // gateway needs to be configured with one established connection for // tabletconntest.TestTarget.{Keyspace, Shard, TabletType} to the // provided tabletconntest.FakeQueryService. -func TestSuite(t *testing.T, name string, g *vtgate.DiscoveryGateway, f *tabletconntest.FakeQueryService) { +func TestSuite(t *testing.T, name string, g *tabletGateway, f *tabletconntest.FakeQueryService) { protocolName := "gateway-test-" + name tabletconn.RegisterDialer(protocolName, func(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (queryservice.QueryService, error) { - return &gatewayAdapter{DiscoveryGateway: g}, nil + return &gatewayAdapter{tabletGateway: g}, nil }) tabletconntest.TestSuite(t, protocolName, &topodatapb.Tablet{ diff --git a/go/vt/vtgate/gatewaytest/grpc_discovery_test.go b/go/vt/vtgate/grpc_discovery_test.go similarity index 92% rename from go/vt/vtgate/gatewaytest/grpc_discovery_test.go rename to go/vt/vtgate/grpc_discovery_test.go index f9f6e4207ad..b6aa62633d4 100644 --- a/go/vt/vtgate/gatewaytest/grpc_discovery_test.go +++ b/go/vt/vtgate/grpc_discovery_test.go @@ -14,18 +14,15 @@ See the License for the specific language governing permissions and limitations under the License. */ -package gatewaytest +package vtgate import ( "flag" + "golang.org/x/net/context" "net" "testing" "time" - "vitess.io/vitess/go/vt/vtgate" - - "golang.org/x/net/context" - "google.golang.org/grpc" "vitess.io/vitess/go/vt/discovery" @@ -66,7 +63,7 @@ func TestGRPCDiscovery(t *testing.T) { // Wait for the right tablets to be present. hc := discovery.NewHealthCheck(10*time.Second, 2*time.Minute) rs := srvtopo.NewResilientServer(ts, "TestGRPCDiscovery") - dg := vtgate.New(context.Background(), hc, rs, cell, 2) + dg := NewTabletGateway(context.Background(), hc, rs, cell, 2) hc.AddTablet(&topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: cell, @@ -80,7 +77,7 @@ func TestGRPCDiscovery(t *testing.T) { "grpc": int32(port), }, }, "test_tablet") - err = vtgate.WaitForTablets(dg, []topodatapb.TabletType{tabletconntest.TestTarget.TabletType}) + err = WaitForTablets(dg, []topodatapb.TabletType{tabletconntest.TestTarget.TabletType}) if err != nil { t.Fatalf("WaitForTablets failed: %v", err) } diff --git a/go/vt/vtgate/scatter_conn.go b/go/vt/vtgate/scatter_conn.go index 624560a0d9e..bce0a26425a 100644 --- a/go/vt/vtgate/scatter_conn.go +++ b/go/vt/vtgate/scatter_conn.go @@ -47,7 +47,7 @@ type ScatterConn struct { timings *stats.MultiTimings tabletCallErrorCount *stats.CountersWithMultiLabels txConn *TxConn - gateway *DiscoveryGateway + gateway *tabletGateway healthCheck discovery.HealthCheck } @@ -69,7 +69,7 @@ type shardActionFunc func(rs *srvtopo.ResolvedShard, i int) error type shardActionTransactionFunc func(rs *srvtopo.ResolvedShard, i int, shouldBegin bool, transactionID int64) (int64, error) // NewScatterConn creates a new ScatterConn. -func NewScatterConn(statsName string, txConn *TxConn, gw *DiscoveryGateway, hc discovery.HealthCheck) *ScatterConn { +func NewScatterConn(statsName string, txConn *TxConn, gw *tabletGateway, hc discovery.HealthCheck) *ScatterConn { tabletCallErrorCountStatsName := "" if statsName != "" { tabletCallErrorCountStatsName = statsName + "ErrorCount" diff --git a/go/vt/vtgate/scatter_conn_test.go b/go/vt/vtgate/scatter_conn_test.go index 90dbd15f66e..6267272e2ef 100644 --- a/go/vt/vtgate/scatter_conn_test.go +++ b/go/vt/vtgate/scatter_conn_test.go @@ -652,7 +652,7 @@ func newTestScatterConn(hc discovery.HealthCheck, serv srvtopo.Server, cell stri // The topo.Server is used to start watching the cells described // in '-cells_to_watch' command line parameter, which is // empty by default. So it's unused in this test, set to nil. - gw := New(context.Background(), hc, serv, cell, 3) + gw := NewTabletGateway(context.Background(), hc, serv, cell, 3) tc := NewTxConn(gw, vtgatepb.TransactionMode_TWOPC) return NewScatterConn("", tc, gw, hc) } diff --git a/go/vt/vtgate/discoverygateway.go b/go/vt/vtgate/tabletgateway.go similarity index 90% rename from go/vt/vtgate/discoverygateway.go rename to go/vt/vtgate/tabletgateway.go index 2eded2f4f02..1e7db1a6f33 100644 --- a/go/vt/vtgate/discoverygateway.go +++ b/go/vt/vtgate/tabletgateway.go @@ -44,7 +44,7 @@ import ( ) var ( - _ = flag.String("gateway_implementation", "discoverygateway", "Deprecated") + _ = flag.String("gateway_implementation", "", "Deprecated") initialTabletTimeout = flag.Duration("gateway_initial_tablet_timeout", 30*time.Second, "At startup, the gateway will wait up to that duration to get one tablet per keyspace/shard/tablettype") // KeyspacesToWatch - if provided this specifies which keyspaces should be // visible to a vtgate. By default the vtgate will allow access to any @@ -54,18 +54,19 @@ var ( refreshKnownTablets = flag.Bool("tablet_refresh_known_tablets", true, "tablet refresh reloads the tablet address/port map from topo in case it changes") topoReadConcurrency = flag.Int("topo_read_concurrency", 32, "concurrent topo reads") - allowedTabletTypes []topodatapb.TabletType + allowedTabletTypes []topodatapb.TabletType - tabletFilters flagutil.StringListValue + tabletFilters flagutil.StringListValue KeyspacesToWatch flagutil.StringListValue ) + func init() { flag.Var(&KeyspacesToWatch, "keyspaces_to_watch", "Specifies which keyspaces this vtgate should have access to while routing queries or accessing the vschema") flag.Var(&tabletFilters, "tablet_filters", "Specifies a comma-separated list of 'keyspace|shard_name or keyrange' values to filter the tablets to watch") topoproto.TabletTypeListVar(&allowedTabletTypes, "allowed_tablet_types", "Specifies the tablet types this vtgate is allowed to route queries to") } -type DiscoveryGateway struct { +type tabletGateway struct { queryservice.QueryService hc discovery.HealthCheck tsc *discovery.TabletStatsCache @@ -87,7 +88,7 @@ type DiscoveryGateway struct { buffer *buffer.Buffer } -func New(ctx context.Context, hc discovery.HealthCheck, serv srvtopo.Server, cell string, retryCount int) *DiscoveryGateway { +func NewTabletGateway(ctx context.Context, hc discovery.HealthCheck, serv srvtopo.Server, cell string, retryCount int) *tabletGateway { var topoServer *topo.Server if serv != nil { var err error @@ -97,7 +98,7 @@ func New(ctx context.Context, hc discovery.HealthCheck, serv srvtopo.Server, cel } } - dg := &DiscoveryGateway{ + dg := &tabletGateway{ hc: hc, tsc: discovery.NewTabletStatsCacheDoNotSetListener(topoServer, cell), srvTopoServer: serv, @@ -141,7 +142,7 @@ func New(ctx context.Context, hc discovery.HealthCheck, serv srvtopo.Server, cel // RegisterStats registers the stats to export the lag since the last refresh // and the checksum of the topology -func (dg *DiscoveryGateway) RegisterStats() { +func (dg *tabletGateway) RegisterStats() { stats.NewGaugeDurationFunc( "TopologyWatcherMaxRefreshLag", "maximum time since the topology watcher refreshed a cell", @@ -157,7 +158,7 @@ func (dg *DiscoveryGateway) RegisterStats() { // topologyWatcherMaxRefreshLag returns the maximum lag since the watched // cells were refreshed from the topo server -func (dg *DiscoveryGateway) topologyWatcherMaxRefreshLag() time.Duration { +func (dg *tabletGateway) topologyWatcherMaxRefreshLag() time.Duration { var lag time.Duration for _, tw := range dg.tabletsWatchers { cellLag := tw.RefreshLag() @@ -169,7 +170,7 @@ func (dg *DiscoveryGateway) topologyWatcherMaxRefreshLag() time.Duration { } // topologyWatcherChecksum returns a checksum of the topology watcher state -func (dg *DiscoveryGateway) topologyWatcherChecksum() int64 { +func (dg *tabletGateway) topologyWatcherChecksum() int64 { var checksum int64 for _, tw := range dg.tabletsWatchers { checksum = checksum ^ int64(tw.TopoChecksum()) @@ -179,7 +180,7 @@ func (dg *DiscoveryGateway) topologyWatcherChecksum() int64 { // StatsUpdate forwards HealthCheck updates to TabletStatsCache and MasterBuffer. // It is part of the discovery.HealthCheckStatsListener interface. -func (dg *DiscoveryGateway) StatsUpdate(ts *discovery.TabletStats) { +func (dg *tabletGateway) StatsUpdate(ts *discovery.TabletStats) { dg.tsc.StatsUpdate(ts) if ts.Target.TabletType == topodatapb.TabletType_MASTER { @@ -188,7 +189,7 @@ func (dg *DiscoveryGateway) StatsUpdate(ts *discovery.TabletStats) { } // WaitForTablets is part of the gateway.Gateway interface. -func (dg *DiscoveryGateway) WaitForTablets(ctx context.Context, tabletTypesToWait []topodatapb.TabletType) error { +func (dg *tabletGateway) WaitForTablets(ctx context.Context, tabletTypesToWait []topodatapb.TabletType) error { // Skip waiting for tablets if we are not told to do so. if len(tabletTypesToWait) == 0 { return nil @@ -205,7 +206,7 @@ func (dg *DiscoveryGateway) WaitForTablets(ctx context.Context, tabletTypesToWai // Close shuts down underlying connections. // This function hides the inner implementation. -func (dg *DiscoveryGateway) Close(ctx context.Context) error { +func (dg *tabletGateway) Close(ctx context.Context) error { dg.buffer.Shutdown() for _, ctw := range dg.tabletsWatchers { ctw.Stop() @@ -215,7 +216,7 @@ func (dg *DiscoveryGateway) Close(ctx context.Context) error { // CacheStatus returns a list of TabletCacheStatus per // keyspace/shard/tablet_type. -func (dg *DiscoveryGateway) CacheStatus() TabletCacheStatusList { +func (dg *tabletGateway) CacheStatus() TabletCacheStatusList { dg.mu.RLock() res := make(TabletCacheStatusList, 0, len(dg.statusAggregators)) for _, aggr := range dg.statusAggregators { @@ -231,7 +232,7 @@ func (dg *DiscoveryGateway) CacheStatus() TabletCacheStatusList { // the middle of a transaction. While returning the error check if it maybe a result of // a resharding event, and set the re-resolve bit and let the upper layers // re-resolve and retry. -func (dg *DiscoveryGateway) withRetry(ctx context.Context, target *querypb.Target, unused queryservice.QueryService, name string, inTransaction bool, inner func(ctx context.Context, target *querypb.Target, conn queryservice.QueryService) (bool, error)) error { +func (dg *tabletGateway) withRetry(ctx context.Context, target *querypb.Target, unused queryservice.QueryService, name string, inTransaction bool, inner func(ctx context.Context, target *querypb.Target, conn queryservice.QueryService) (bool, error)) error { var tabletLastUsed *topodatapb.Tablet var err error invalidTablets := make(map[string]bool) @@ -370,13 +371,13 @@ func nextTablet(cell string, tablets []discovery.TabletStats, offset, length int return -1 } -func (dg *DiscoveryGateway) updateStats(target *querypb.Target, startTime time.Time, err error) { +func (dg *tabletGateway) updateStats(target *querypb.Target, startTime time.Time, err error) { elapsed := time.Since(startTime) aggr := dg.getStatsAggregator(target) aggr.UpdateQueryInfo("", target.TabletType, elapsed, err != nil) } -func (dg *DiscoveryGateway) getStatsAggregator(target *querypb.Target) *TabletStatusAggregator { +func (dg *tabletGateway) getStatsAggregator(target *querypb.Target) *TabletStatusAggregator { key := fmt.Sprintf("%v/%v/%v", target.Keyspace, target.Shard, target.TabletType.String()) // get existing aggregator @@ -400,9 +401,9 @@ func (dg *DiscoveryGateway) getStatsAggregator(target *querypb.Target) *TabletSt // WaitForTablets is a helper method to wait for the provided tablets, // up until the *initialTabletTimeout. It will log what it is doing. -// Note it has the same name as the DiscoveryGateway's method, as it +// Note it has the same name as the tabletGateway's method, as it // just calls it. -func WaitForTablets(gw *DiscoveryGateway, tabletTypesToWait []topodatapb.TabletType) error { +func WaitForTablets(gw *tabletGateway, tabletTypesToWait []topodatapb.TabletType) error { log.Infof("Gateway waiting for serving tablets of types %v ...", tabletTypesToWait) ctx, cancel := context.WithTimeout(context.Background(), *initialTabletTimeout) defer cancel() diff --git a/go/vt/vtgate/discoverygateway_test.go b/go/vt/vtgate/tabletgateway_test.go similarity index 90% rename from go/vt/vtgate/discoverygateway_test.go rename to go/vt/vtgate/tabletgateway_test.go index 7d2171055d6..ed231fbbee9 100644 --- a/go/vt/vtgate/discoverygateway_test.go +++ b/go/vt/vtgate/tabletgateway_test.go @@ -18,9 +18,9 @@ package vtgate import ( "fmt" + "golang.org/x/net/context" "strings" "testing" - "golang.org/x/net/context" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/discovery" @@ -36,23 +36,23 @@ import ( ) func TestDiscoveryGatewayExecute(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *tabletGateway, target *querypb.Target) error { _, err := dg.Execute(context.Background(), target, "query", nil, 0, nil) return err }) - testDiscoveryGatewayTransact(t, func(dg *DiscoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayTransact(t, func(dg *tabletGateway, target *querypb.Target) error { _, err := dg.Execute(context.Background(), target, "query", nil, 1, nil) return err }) } func TestDiscoveryGatewayExecuteBatch(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *tabletGateway, target *querypb.Target) error { queries := []*querypb.BoundQuery{{Sql: "query", BindVariables: nil}} _, err := dg.ExecuteBatch(context.Background(), target, queries, false, 0, nil) return err }) - testDiscoveryGatewayTransact(t, func(dg *DiscoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayTransact(t, func(dg *tabletGateway, target *querypb.Target) error { queries := []*querypb.BoundQuery{{Sql: "query", BindVariables: nil}} _, err := dg.ExecuteBatch(context.Background(), target, queries, false, 1, nil) return err @@ -60,7 +60,7 @@ func TestDiscoveryGatewayExecuteBatch(t *testing.T) { } func TestDiscoveryGatewayExecuteStream(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *tabletGateway, target *querypb.Target) error { err := dg.StreamExecute(context.Background(), target, "query", nil, 0, nil, func(qr *sqltypes.Result) error { return nil }) @@ -69,33 +69,33 @@ func TestDiscoveryGatewayExecuteStream(t *testing.T) { } func TestDiscoveryGatewayBegin(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *tabletGateway, target *querypb.Target) error { _, err := dg.Begin(context.Background(), target, nil) return err }) } func TestDiscoveryGatewayCommit(t *testing.T) { - testDiscoveryGatewayTransact(t, func(dg *DiscoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayTransact(t, func(dg *tabletGateway, target *querypb.Target) error { return dg.Commit(context.Background(), target, 1) }) } func TestDiscoveryGatewayRollback(t *testing.T) { - testDiscoveryGatewayTransact(t, func(dg *DiscoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayTransact(t, func(dg *tabletGateway, target *querypb.Target) error { return dg.Rollback(context.Background(), target, 1) }) } func TestDiscoveryGatewayBeginExecute(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *tabletGateway, target *querypb.Target) error { _, _, err := dg.BeginExecute(context.Background(), target, "query", nil, nil) return err }) } func TestDiscoveryGatewayBeginExecuteBatch(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *tabletGateway, target *querypb.Target) error { queries := []*querypb.BoundQuery{{Sql: "query", BindVariables: nil}} _, _, err := dg.BeginExecuteBatch(context.Background(), target, queries, false, nil) return err @@ -106,7 +106,7 @@ func TestDiscoveryGatewayGetTablets(t *testing.T) { keyspace := "ks" shard := "0" hc := discovery.NewFakeHealthCheck() - dg := New(context.Background(), hc, nil, "local", 2) + dg := NewTabletGateway(context.Background(), hc, nil, "local", 2) // replica should only use local ones hc.Reset() @@ -214,7 +214,7 @@ func TestDiscoveryGatewayGetTabletsInRegion(t *testing.T) { Cells: []string{"local-west", "local-east"}, } - dg := New(context.Background(), hc, srvTopo, "local-west", 2) + dg := NewTabletGateway(context.Background(), hc, srvTopo, "local-west", 2) ts.CreateCellsAlias(context.Background(), "local", cellsAlias) @@ -244,7 +244,7 @@ func TestDiscoveryGatewayGetTabletsWithRegion(t *testing.T) { Cells: []string{"local-west", "local-east"}, } - dg := New(context.Background(), hc, srvTopo, "local", 2) + dg := NewTabletGateway(context.Background(), hc, srvTopo, "local", 2) ts.CreateCellsAlias(context.Background(), "local", cellsAlias) @@ -263,7 +263,7 @@ func TestDiscoveryGatewayGetTabletsWithRegion(t *testing.T) { } } -func testDiscoveryGatewayGeneric(t *testing.T, f func(dg *DiscoveryGateway, target *querypb.Target) error) { +func testDiscoveryGatewayGeneric(t *testing.T, f func(dg *tabletGateway, target *querypb.Target) error) { keyspace := "ks" shard := "0" tabletType := topodatapb.TabletType_REPLICA @@ -273,7 +273,7 @@ func testDiscoveryGatewayGeneric(t *testing.T, f func(dg *DiscoveryGateway, targ TabletType: tabletType, } hc := discovery.NewFakeHealthCheck() - dg := New(context.Background(), hc, nil, "cell", 2) + dg := NewTabletGateway(context.Background(), hc, nil, "cell", 2) // no tablet hc.Reset() @@ -346,7 +346,7 @@ func testDiscoveryGatewayGeneric(t *testing.T, f func(dg *DiscoveryGateway, targ } } -func testDiscoveryGatewayTransact(t *testing.T, f func(dg *DiscoveryGateway, target *querypb.Target) error) { +func testDiscoveryGatewayTransact(t *testing.T, f func(dg *tabletGateway, target *querypb.Target) error) { keyspace := "ks" shard := "0" tabletType := topodatapb.TabletType_REPLICA @@ -356,7 +356,7 @@ func testDiscoveryGatewayTransact(t *testing.T, f func(dg *DiscoveryGateway, tar TabletType: tabletType, } hc := discovery.NewFakeHealthCheck() - dg := New(context.Background(), hc, nil, "cell", 2) + dg := NewTabletGateway(context.Background(), hc, nil, "cell", 2) // retry error - no retry hc.Reset() diff --git a/go/vt/vtgate/tx_conn.go b/go/vt/vtgate/tx_conn.go index 6efd46460ea..1cf7b4c0c93 100644 --- a/go/vt/vtgate/tx_conn.go +++ b/go/vt/vtgate/tx_conn.go @@ -33,12 +33,12 @@ import ( // TxConn is used for executing transactional requests. type TxConn struct { - gateway *DiscoveryGateway + gateway *tabletGateway mode vtgatepb.TransactionMode } // NewTxConn builds a new TxConn. -func NewTxConn(gw *DiscoveryGateway, txMode vtgatepb.TransactionMode) *TxConn { +func NewTxConn(gw *tabletGateway, txMode vtgatepb.TransactionMode) *TxConn { return &TxConn{ gateway: gw, mode: txMode, diff --git a/go/vt/vtgate/vstream_manager_test.go b/go/vt/vtgate/vstream_manager_test.go index 2871eabd09b..44eead7beac 100644 --- a/go/vt/vtgate/vstream_manager_test.go +++ b/go/vt/vtgate/vstream_manager_test.go @@ -873,7 +873,7 @@ func TestResolveVStreamParams(t *testing.T) { } func newTestVStreamManager(hc discovery.HealthCheck, serv srvtopo.Server, cell string) *vstreamManager { - gw := New(context.Background(), hc, serv, cell, 3) + gw := NewTabletGateway(context.Background(), hc, serv, cell, 3) srvResolver := srvtopo.NewResolver(serv, gw, cell) return newVStreamManager(srvResolver, serv, cell) } diff --git a/go/vt/vtgate/vtgate.go b/go/vt/vtgate/vtgate.go index be76e04b2c3..662dd64d6ac 100644 --- a/go/vt/vtgate/vtgate.go +++ b/go/vt/vtgate/vtgate.go @@ -101,7 +101,7 @@ type VTGate struct { resolver *Resolver vsm *vstreamManager txConn *TxConn - gw *DiscoveryGateway + gw *tabletGateway // stats objects. // TODO(sougou): This needs to be cleaned up. There @@ -133,7 +133,7 @@ func Init(ctx context.Context, hc discovery.HealthCheck, serv srvtopo.Server, ce // Build objects from low to high level. // Start with the gateway. If we can't reach the topology service, // we can't go on much further, so we log.Fatal out. - gw := New(ctx, hc, serv, cell, retryCount) + gw := NewTabletGateway(ctx, hc, serv, cell, retryCount) gw.RegisterStats() if err := WaitForTablets(gw, tabletTypesToWait); err != nil { log.Fatalf("gateway.WaitForTablets failed: %v", err) @@ -227,7 +227,7 @@ func (vtg *VTGate) IsHealthy() error { } // Gateway returns the current gateway implementation. Mostly used for tests. -func (vtg *VTGate) Gateway() *DiscoveryGateway { +func (vtg *VTGate) Gateway() *tabletGateway { return vtg.gw } diff --git a/go/vt/vttablet/queryservice/wrapped.go b/go/vt/vttablet/queryservice/wrapped.go index 185046dd9df..9443951d6fe 100644 --- a/go/vt/vttablet/queryservice/wrapped.go +++ b/go/vt/vttablet/queryservice/wrapped.go @@ -39,7 +39,7 @@ type WrapperFunc func(ctx context.Context, target *querypb.Target, conn QuerySer // Wrap returns a wrapped version of the original QueryService implementation. // This lets you avoid repeating boiler-plate code by consolidating it in the // wrapper function. -// A good example of this is go/vt/vtgate/gateway/discoverygateway.go. +// A good example of this is go/vt/vtgate/gateway/tabletgateway.go. // For every method invocation, the wrapper function is called, which can // in turn call the provided inner function that will use the input parameters // to call the implementation. In order to load balance across multiple From 66141183adc0af8330f99279cf6ae9f4fba9d238 Mon Sep 17 00:00:00 2001 From: Derek Perkins Date: Thu, 26 Mar 2020 16:28:45 -0600 Subject: [PATCH 65/85] Revert "Enable alternate registries to be able to be used" Signed-off-by: Derek Perkins --- helm/vitess/templates/_cron-jobs.tpl | 6 +--- helm/vitess/templates/_jobs.tpl | 8 ++--- helm/vitess/templates/_keyspace.tpl | 5 ++- helm/vitess/templates/_orchestrator.tpl | 5 ++- helm/vitess/templates/_pmm.tpl | 2 +- helm/vitess/templates/_shard.tpl | 7 ++-- helm/vitess/templates/_vtctld.tpl | 3 +- helm/vitess/templates/_vtgate.tpl | 10 +++--- helm/vitess/templates/_vttablet.tpl | 46 ++++++++----------------- helm/vitess/templates/vitess.yaml | 4 +-- helm/vitess/values.yaml | 20 +---------- 11 files changed, 32 insertions(+), 84 deletions(-) diff --git a/helm/vitess/templates/_cron-jobs.tpl b/helm/vitess/templates/_cron-jobs.tpl index 41a5986b7c8..3b4ffd13852 100644 --- a/helm/vitess/templates/_cron-jobs.tpl +++ b/helm/vitess/templates/_cron-jobs.tpl @@ -14,10 +14,6 @@ {{- $namespace := index . 8 -}} {{- $defaultVtctlclient := index . 9 }} - -{{- $vitessTag := $defaultVtctlclient.vitessTag -}} -{{- $vtctlclientImage := $defaultVtctlclient.vtctlclientImage -}} - {{ if $backup.enabled }} # create cron job for current shard --- @@ -59,7 +55,7 @@ spec: containers: - name: backup - image: "{{$vtctlclientImage}}:{{$vitessTag}}" + image: "vitess/vtctlclient:{{$vitessTag}}" volumeMounts: {{ include "user-secret-volumeMounts" $defaultVtctlclient.secrets | indent 14 }} diff --git a/helm/vitess/templates/_jobs.tpl b/helm/vitess/templates/_jobs.tpl index c27fe5105d6..45d389ff50e 100644 --- a/helm/vitess/templates/_jobs.tpl +++ b/helm/vitess/templates/_jobs.tpl @@ -8,7 +8,6 @@ {{- $namespace := index . 2 -}} {{- $vitessTag := $job.vitessTag | default $defaultVtctlclient.vitessTag -}} -{{- $vtctlclientImage := $defaultVtctlclient.vtctlclientImage -}} {{- $secrets := $job.secrets | default $defaultVtctlclient.secrets }} --- ################################### @@ -31,7 +30,7 @@ spec: restartPolicy: OnFailure containers: - name: vtjob - image: "{{$vtctlclientImage}}:{{$vitessTag}}" + image: "vitess/vtctlclient:{{$vitessTag}}" volumeMounts: {{ include "user-secret-volumeMounts" $defaultVtctlclient.secrets | indent 10 }} resources: @@ -57,9 +56,6 @@ spec: {{- $namespace := index . 2 -}} {{- $vitessTag := $job.vitessTag | default $defaultVtworker.vitessTag -}} -{{- $vtworkerImage := $job.vtworkerImage | default $defaultVtworker.vtworkerImage -}} - - {{- $secrets := $job.secrets | default $defaultVtworker.secrets }} --- ################################### @@ -83,7 +79,7 @@ spec: restartPolicy: OnFailure containers: - name: vtjob - image: "{{$vtworkerImage}}:{{$vitessTag}}" + image: "vitess/vtworker:{{$vitessTag}}" volumeMounts: {{ include "user-secret-volumeMounts" $defaultVtworker.secrets | indent 10 }} resources: diff --git a/helm/vitess/templates/_keyspace.tpl b/helm/vitess/templates/_keyspace.tpl index 8fc74efd9ae..2ba9f6872c3 100644 --- a/helm/vitess/templates/_keyspace.tpl +++ b/helm/vitess/templates/_keyspace.tpl @@ -15,7 +15,6 @@ # define image to use {{- $vitessTag := .vitessTag | default $defaultVtctlclient.vitessTag -}} -{{- $vtctlclientImage := .vtctlclientImage | default $defaultVtctlclient.vtctlclientImage -}} {{- $secrets := .secrets | default $defaultVtctlclient.secrets -}} {{- range $name, $schema := $keyspace.schema }} @@ -34,7 +33,7 @@ spec: restartPolicy: OnFailure containers: - name: apply-schema - image: "{{$vtctlclientImage}}:{{$vitessTag}}" + image: "vitess/vtctlclient:{{$vitessTag}}" volumeMounts: {{ include "user-secret-volumeMounts" $defaultVtctlclient.secrets | indent 10 }} @@ -101,7 +100,7 @@ spec: restartPolicy: OnFailure containers: - name: apply-vschema - image: "{{$vtctlclientImage}}:{{$vitessTag}}" + image: "vitess/vtctlclient:{{$vitessTag}}" volumeMounts: {{ include "user-secret-volumeMounts" $defaultVtctlclient.secrets | indent 10 }} diff --git a/helm/vitess/templates/_orchestrator.tpl b/helm/vitess/templates/_orchestrator.tpl index ba15ae4e4dc..13ad07831aa 100644 --- a/helm/vitess/templates/_orchestrator.tpl +++ b/helm/vitess/templates/_orchestrator.tpl @@ -5,7 +5,6 @@ # set tuple values to more recognizable variables {{- $orc := index . 0 -}} {{- $defaultVtctlclient := index . 1 }} -{{- $logtail := index . 2 }} apiVersion: v1 kind: Service @@ -124,7 +123,7 @@ spec: value: "15999" - name: recovery-log - image: "{{ $logtail.image }}:{{ $logtail.tag }}" + image: vitess/logtail:helm-1.0.7-5 imagePullPolicy: IfNotPresent env: - name: TAIL_FILEPATH @@ -134,7 +133,7 @@ spec: mountPath: /tmp - name: audit-log - image: "{{ $logtail.image }}:{{ $logtail.tag }}" + image: vitess/logtail:helm-1.0.7-5 imagePullPolicy: IfNotPresent env: - name: TAIL_FILEPATH diff --git a/helm/vitess/templates/_pmm.tpl b/helm/vitess/templates/_pmm.tpl index c5fec6a68e5..9669f58298d 100644 --- a/helm/vitess/templates/_pmm.tpl +++ b/helm/vitess/templates/_pmm.tpl @@ -50,7 +50,7 @@ spec: spec: containers: - name: pmm - image: "{{ $pmm.image}}:{{ $pmm.pmmTag }}" + image: "percona/pmm-server:{{ $pmm.pmmTag }}" ports: - name: web diff --git a/helm/vitess/templates/_shard.tpl b/helm/vitess/templates/_shard.tpl index 6f9d330306d..3d075c24f85 100644 --- a/helm/vitess/templates/_shard.tpl +++ b/helm/vitess/templates/_shard.tpl @@ -18,9 +18,6 @@ {{- with $cell.vtctld }} # define image to use {{- $vitessTag := .vitessTag | default $defaultVtctlclient.vitessTag }} -{{- $vtctlclientImage := .vtctlclientImage | default $defaultVtctlclient.vtctlclientImage }} - - --- ################################### # InitShardMaster Job @@ -45,7 +42,7 @@ spec: restartPolicy: OnFailure containers: - name: init-shard-master - image: "{{$vtctlclientImage}}:{{$vitessTag}}" + image: "vitess/vtctlclient:{{$vitessTag}}" volumeMounts: {{ include "user-secret-volumeMounts" $defaultVtctlclient.secrets | indent 10 }} @@ -148,7 +145,7 @@ spec: restartPolicy: OnFailure containers: - name: copy-schema - image: "{{$vtctlclientImage}}/:{{$vitessTag}}" + image: "vitess/vtctlclient:{{$vitessTag}}" volumeMounts: {{ include "user-secret-volumeMounts" $defaultVtctlclient.secrets | indent 10 }} diff --git a/helm/vitess/templates/_vtctld.tpl b/helm/vitess/templates/_vtctld.tpl index 98d8ba4291b..af178035e99 100644 --- a/helm/vitess/templates/_vtctld.tpl +++ b/helm/vitess/templates/_vtctld.tpl @@ -13,7 +13,6 @@ # define image to use {{- $vitessTag := .vitessTag | default $defaultVtctld.vitessTag -}} -{{- $vtcldImage := .vtcldImage | default $defaultVtctld.vtcldImage -}} {{- $cellClean := include "clean-label" $cell.name }} ################################### @@ -60,7 +59,7 @@ spec: {{ include "vtctld-affinity" (tuple $cellClean $cell.region) | indent 6 }} containers: - name: vtctld - image: {{$vtcldImage}}:{{$vitessTag}} + image: vitess/vtctld:{{$vitessTag}} imagePullPolicy: IfNotPresent readinessProbe: httpGet: diff --git a/helm/vitess/templates/_vtgate.tpl b/helm/vitess/templates/_vtgate.tpl index 62ec33ecd27..1455e9c47a2 100644 --- a/helm/vitess/templates/_vtgate.tpl +++ b/helm/vitess/templates/_vtgate.tpl @@ -12,7 +12,6 @@ # define image to use {{- $vitessTag := .vitessTag | default $defaultVtgate.vitessTag -}} -{{- $vtgateImage := .vtgateImage | default $defaultVtgate.vtgateImage -}} {{- $cellClean := include "clean-label" $cell.name }} ################################### @@ -69,13 +68,13 @@ spec: {{ if $cell.mysqlProtocol.enabled }} {{ if eq $cell.mysqlProtocol.authType "secret" }} initContainers: -{{ include "init-mysql-creds" (tuple $vitessTag $vtgateImage $cell) | indent 8 }} +{{ include "init-mysql-creds" (tuple $vitessTag $cell) | indent 8 }} {{ end }} {{ end }} containers: - name: vtgate - image: {{$vtgateImage}}:{{$vitessTag}} + image: vitess/vtgate:{{$vitessTag}} imagePullPolicy: IfNotPresent readinessProbe: httpGet: @@ -219,13 +218,12 @@ affinity: ################################### {{ define "init-mysql-creds" -}} {{- $vitessTag := index . 0 -}} -{{- $vtgateImage := index . 1 -}} -{{- $cell := index . 2 -}} +{{- $cell := index . 1 -}} {{- with $cell.mysqlProtocol }} - name: init-mysql-creds - image: "{{$vtgateImage}}:{{$vitessTag}}" + image: "vitess/vtgate:{{$vitessTag}}" imagePullPolicy: IfNotPresent volumeMounts: - name: creds diff --git a/helm/vitess/templates/_vttablet.tpl b/helm/vitess/templates/_vttablet.tpl index c27774ae936..3053407af4d 100644 --- a/helm/vitess/templates/_vttablet.tpl +++ b/helm/vitess/templates/_vttablet.tpl @@ -48,10 +48,6 @@ spec: {{- $config := index . 8 -}} {{- $pmm := index . 9 -}} {{- $orc := index . 10 -}} -{{- $mysqlctld := index . 11 -}} -{{- $logrotate := index . 12 -}} -{{- $logtail := index . 13 -}} -{{- $vtctl := index . 14 -}} # sanitize inputs for labels {{- $cellClean := include "clean-label" $cell.name -}} @@ -65,7 +61,6 @@ spec: # define images to use {{- $vitessTag := .vitessTag | default $defaultVttablet.vitessTag -}} -{{- $vtctlclientImage := .vtctlclientImage | default $defaultVttablet.vtctlclientImage -}} {{- $image := .image | default $defaultVttablet.image -}} {{- $mysqlImage := .mysqlImage | default $defaultVttablet.mysqlImage -}} {{- $mysqlImage := .mysqlImage | default $defaultVttablet.mysqlImage }} @@ -106,17 +101,17 @@ spec: {{ include "vttablet-affinity" (tuple $cellClean $keyspaceClean $shardClean $cell.region) | indent 6 }} initContainers: -{{ include "init-mysql" (tuple $vitessTag $cellClean $mysqlctld) | indent 8 }} -{{ include "init-vttablet" (tuple $vitessTag $cell $cellClean $namespace $mysqlctld $vtctl) | indent 8 }} +{{ include "init-mysql" (tuple $vitessTag $cellClean) | indent 8 }} +{{ include "init-vttablet" (tuple $vitessTag $cell $cellClean $namespace) | indent 8 }} containers: {{ include "cont-mysql" (tuple $topology $cell $keyspace $shard $tablet $defaultVttablet $uid) | indent 8 }} {{ include "cont-vttablet" (tuple $topology $cell $keyspace $shard $tablet $defaultVttablet $defaultVtctlclient $vitessTag $uid $namespace $config $orc) | indent 8 }} -{{ include "cont-logrotate" (tuple $logrotate) | indent 8 }} -{{ include "cont-mysql-generallog" (tuple $logrotate) | indent 8 }} -{{ include "cont-mysql-errorlog" (tuple $logrotate) | indent 8 }} -{{ include "cont-mysql-slowlog" (tuple $logrotate) | indent 8 }} -{{ if $pmm.enabled }}{{ include "cont-pmm-client" (tuple $pmm $namespace $keyspace $logtail) | indent 8 }}{{ end }} +{{ include "cont-logrotate" . | indent 8 }} +{{ include "cont-mysql-generallog" . | indent 8 }} +{{ include "cont-mysql-errorlog" . | indent 8 }} +{{ include "cont-mysql-slowlog" . | indent 8 }} +{{ if $pmm.enabled }}{{ include "cont-pmm-client" (tuple $pmm $namespace $keyspace) | indent 8 }}{{ end }} volumes: - name: vt @@ -169,13 +164,9 @@ spec: {{ define "init-mysql" -}} {{- $vitessTag := index . 0 -}} {{- $cellClean := index . 1 }} -{{- $mysqlctld := index . 2 }} - -{{- $vitessTag := $mysqlctld.vitessTag -}} -{{- $mysqlctldImage := $mysqlctld.mysqlctldImage -}} - name: "init-mysql" - image: "{{$mysqlctldImage}}:{{$vitessTag}}" + image: "vitess/mysqlctld:{{$vitessTag}}" imagePullPolicy: IfNotPresent volumeMounts: - name: vtdataroot @@ -218,11 +209,9 @@ spec: {{- $cell := index . 1 -}} {{- $cellClean := index . 2 -}} {{- $namespace := index . 3 }} -{{- $vtctl := index . 4 }} - - name: init-vttablet - image: "{{$vtctl.vtctlImage}}:{{$vitessTag}}" + image: "vitess/vtctl:{{$vitessTag}}" imagePullPolicy: IfNotPresent volumeMounts: - name: vtdataroot @@ -286,11 +275,8 @@ spec: {{- $cellClean := include "clean-label" $cell.name -}} {{- with $tablet.vttablet }} -{{- $vitessTag := $defaultVttablet.vitessTag -}} -{{- $vttabletImage := $defaultVttablet.vttabletImage -}} - - name: vttablet - image: "{{$vttabletImage}}:{{$vitessTag}}" + image: "vitess/vttablet:{{$vitessTag}}" imagePullPolicy: IfNotPresent readinessProbe: httpGet: @@ -546,10 +532,9 @@ spec: # run logrotate for all log files in /vtdataroot/tabletdata ########################## {{ define "cont-logrotate" }} -{{- $logrotate := index . 0 }} - name: logrotate - image: {{ $logrotate.image }}:{{ $logrotate.tag }} + image: vitess/logrotate:helm-1.0.7-5 imagePullPolicy: IfNotPresent volumeMounts: - name: vtdataroot @@ -561,10 +546,9 @@ spec: # redirect the error log file to stdout ########################## {{ define "cont-mysql-errorlog" }} -{{- $logtail := index . 0 -}} - name: error-log - image: {{ $logtail.image }}:{{ $logtail.tag }} + image: vitess/logtail:helm-1.0.7-5 imagePullPolicy: IfNotPresent env: @@ -580,10 +564,9 @@ spec: # redirect the slow log file to stdout ########################## {{ define "cont-mysql-slowlog" }} -{{- $logtail := index . 0 -}} - name: slow-log - image: {{ $logtail.image }}:{{ $logtail.tag }} + image: vitess/logtail:helm-1.0.7-5 imagePullPolicy: IfNotPresent env: @@ -599,10 +582,9 @@ spec: # redirect the general log file to stdout ########################## {{ define "cont-mysql-generallog" }} -{{- $logtail := index . 0 -}} - name: general-log - image: {{ $logtail.image }}:{{ $logtail.tag }} + image: vitess/logtail:helm-1.0.7-5 imagePullPolicy: IfNotPresent env: diff --git a/helm/vitess/templates/vitess.yaml b/helm/vitess/templates/vitess.yaml index 62dc8d50f42..7c3d36aac25 100644 --- a/helm/vitess/templates/vitess.yaml +++ b/helm/vitess/templates/vitess.yaml @@ -12,7 +12,7 @@ {{ if $.Values.orchestrator.enabled }} # create orchestrator global services and StatefulSet -{{ include "orchestrator" (tuple $.Values.orchestrator $.Values.vtctlclient $.Values.logtail) }} +{{ include "orchestrator" (tuple $.Values.orchestrator $.Values.vtctlclient) }} --- # create orchestrator config map {{ include "orchestrator-config" (tuple $.Values.orchestrator $.Release.Namespace $.Values.vttablet.enableHeartbeat $.Values.vtctlclient) }} @@ -66,7 +66,7 @@ # Tablet initializations {{ range $tablet := $shard.tablets }} - {{ include "vttablet" (tuple $.Values.topology $cell $keyspace $shard $tablet $.Values.vttablet $.Values.vtctlclient $.Release.Namespace $.Values.config $.Values.pmm $.Values.orchestrator $.Values.mysqlctld $.Values.logrotate $.Values.logtail $.Values.vtctl) }} + {{ include "vttablet" (tuple $.Values.topology $cell $keyspace $shard $tablet $.Values.vttablet $.Values.vtctlclient $.Release.Namespace $.Values.config $.Values.pmm $.Values.orchestrator) }} {{ end }} # range $tablet {{ end }} # range $shard {{ end }} # range $keyspace diff --git a/helm/vitess/values.yaml b/helm/vitess/values.yaml index 59af5639aee..96ef944051c 100644 --- a/helm/vitess/values.yaml +++ b/helm/vitess/values.yaml @@ -181,7 +181,6 @@ etcd: vtctld: serviceType: ClusterIP vitessTag: helm-1.0.7-5 - vtcldImage: vitess/vtctld resources: # requests: # cpu: 100m @@ -193,7 +192,6 @@ vtctld: vtgate: serviceType: ClusterIP vitessTag: helm-1.0.7-5 - vtgateImage: vitess/vtgate resources: # requests: # cpu: 500m @@ -213,14 +211,12 @@ vtgate: # Default values for vtctlclient resources defined in 'topology' vtctlclient: vitessTag: helm-1.0.7-5 - vtctlclientImage: vitess/vtctlclient extraFlags: {} secrets: [] # secrets are mounted under /vt/usersecrets/{secretname} # Default values for vtworker resources defined in 'jobs' vtworker: vitessTag: helm-1.0.7-5 - vtworkerImage: vitess/vtworker extraFlags: {} resources: # requests: @@ -232,7 +228,7 @@ vtworker: # Default values for vttablet resources defined in 'topology' vttablet: vitessTag: helm-1.0.7-5 - vttabletImage: vitess/vttablet + # valid values are # - mysql56 (for MySQL 8.0) # - mysql56 (for MySQL/Percona 5.6 or 5.7) @@ -325,20 +321,10 @@ vttablet: requests: storage: 10Gi -# Default values for mysqlctld resources defined in 'topology' -mysqlctld: - vitessTag: helm-1.0.7-5 - mysqlctldImage: vitess/mysqlctld - -vtctl: - vitessTag: helm-1.0.7-5 - vtctlImage: vitess/vtctl - # Default values for pmm pmm: enabled: false pmmTag: 1.17.0 - image : percona/pmm-server client: resources: requests: @@ -399,7 +385,3 @@ orchestrator: requests: cpu: 50m memory: 350Mi - -logrotate: - vitessTag: 3.1.1 - image: vitess/orchestrator From f72b878f675cf6195cf6d41546d12d8a0790f96e Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Thu, 30 Jan 2020 14:38:34 +0100 Subject: [PATCH 66/85] Remove ParenExp from code base Instead of keeping this information in the AST, we put in parens in the string representation of queries when needed when printing. This includes adding code that takes precendence into consideration - something that already we already need to do when parsing. Unfortunately I haven't found a good way to use the same source information for both sides of the precedence puzzle. Signed-off-by: Andres Taylor --- go/vt/sqlparser/analyzer.go | 2 - go/vt/sqlparser/analyzer_test.go | 14 --- go/vt/sqlparser/ast.go | 49 +++++----- go/vt/sqlparser/ast_funcs.go | 30 ++++--- go/vt/sqlparser/ast_test.go | 7 +- go/vt/sqlparser/parse_test.go | 24 +++-- go/vt/sqlparser/precedence.go | 90 +++++++++++++++++++ go/vt/sqlparser/precedence_test.go | 39 ++++++++ go/vt/sqlparser/rewriter.go | 17 ++-- go/vt/sqlparser/sql.go | 6 +- go/vt/sqlparser/sql.y | 6 +- go/vt/sqlparser/tracked_buffer.go | 6 ++ .../multi-output/selectsharded-output.txt | 20 ++--- go/vt/vtexplain/vtexplain_vttablet.go | 2 - go/vt/vtgate/planbuilder/dml.go | 7 -- go/vt/vtgate/planbuilder/expr.go | 47 +++------- go/vt/vtgate/planbuilder/expr_test.go | 12 --- go/vt/vtgate/planbuilder/plan_test.go | 2 +- go/vt/vtgate/planbuilder/route_option.go | 3 - .../vtgate/planbuilder/testdata/dml_cases.txt | 6 +- .../planbuilder/testdata/filter_cases.txt | 12 +-- .../planbuilder/testdata/from_cases.txt | 14 +-- .../testdata/postprocess_cases.txt | 10 +-- .../planbuilder/testdata/select_cases.txt | 10 +-- .../planbuilder/testdata/wireup_cases.txt | 2 +- go/vt/vttablet/endtoend/queries_test.go | 6 +- .../tabletserver/planbuilder/plan_test.go | 54 +++++------ .../tabletserver/vstreamer/planbuilder.go | 6 -- go/vt/wrangler/vdiff.go | 4 - go/vt/wrangler/vdiff_test.go | 16 ++-- 30 files changed, 298 insertions(+), 225 deletions(-) create mode 100644 go/vt/sqlparser/precedence.go diff --git a/go/vt/sqlparser/analyzer.go b/go/vt/sqlparser/analyzer.go index e9d573707d9..113a887fb3e 100644 --- a/go/vt/sqlparser/analyzer.go +++ b/go/vt/sqlparser/analyzer.go @@ -174,8 +174,6 @@ func SplitAndExpression(filters []Expr, node Expr) []Expr { case *AndExpr: filters = SplitAndExpression(filters, node.Left) return SplitAndExpression(filters, node.Right) - case *ParenExpr: - return SplitAndExpression(filters, node.Expr) } return append(filters, node) } diff --git a/go/vt/sqlparser/analyzer_test.go b/go/vt/sqlparser/analyzer_test.go index 42dd56989be..f27ce3d9504 100644 --- a/go/vt/sqlparser/analyzer_test.go +++ b/go/vt/sqlparser/analyzer_test.go @@ -402,14 +402,6 @@ func TestNewPlanValue(t *testing.T) { Value: sqltypes.NewVarBinary("strval"), }}, }, - }, { - in: ValTuple{ - &ParenExpr{Expr: &SQLVal{ - Type: ValArg, - Val: []byte(":valarg"), - }}, - }, - err: "expression is too complex", }, { in: ValTuple{ ListArg("::list"), @@ -418,12 +410,6 @@ func TestNewPlanValue(t *testing.T) { }, { in: &NullVal{}, out: sqltypes.PlanValue{}, - }, { - in: &ParenExpr{Expr: &SQLVal{ - Type: ValArg, - Val: []byte(":valarg"), - }}, - err: "expression is too complex", }} for _, tc := range tcases { got, err := NewPlanValue(tc.in) diff --git a/go/vt/sqlparser/ast.go b/go/vt/sqlparser/ast.go index 60de165ef1d..5a7e42dbeed 100644 --- a/go/vt/sqlparser/ast.go +++ b/go/vt/sqlparser/ast.go @@ -526,11 +526,6 @@ type ( Expr Expr } - // ParenExpr represents a parenthesized boolean expression. - ParenExpr struct { - Expr Expr - } - // ComparisonExpr represents a two-value comparison expression. ComparisonExpr struct { Operator string @@ -541,7 +536,7 @@ type ( // RangeCond represents a BETWEEN or a NOT BETWEEN expression. RangeCond struct { Operator string - Left Expr + Expr Expr From, To Expr } @@ -708,7 +703,6 @@ type ( func (*AndExpr) iExpr() {} func (*OrExpr) iExpr() {} func (*NotExpr) iExpr() {} -func (*ParenExpr) iExpr() {} func (*ComparisonExpr) iExpr() {} func (*RangeCond) iExpr() {} func (*IsExpr) iExpr() {} @@ -1406,27 +1400,30 @@ func (node Exprs) Format(buf *TrackedBuffer) { // Format formats the node. func (node *AndExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("%v and %v", node.Left, node.Right) + printWithParensIfNeeded(node, node.Left, buf) + buf.WriteString(" and ") + printWithParensIfNeeded(node, node.Right, buf) } // Format formats the node. func (node *OrExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("%v or %v", node.Left, node.Right) + printWithParensIfNeeded(node, node.Left, buf) + buf.WriteString(" or ") + printWithParensIfNeeded(node, node.Right, buf) } // Format formats the node. func (node *NotExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("not %v", node.Expr) -} - -// Format formats the node. -func (node *ParenExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("(%v)", node.Expr) + buf.WriteString("not ") + printWithParensIfNeeded(node, node.Expr, buf) } // Format formats the node. func (node *ComparisonExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("%v %s %v", node.Left, node.Operator, node.Right) + printWithParensIfNeeded(node, node.Left, buf) + buf.Myprintf(" %s ", node.Operator) + printWithParensIfNeeded(node, node.Right, buf) + if node.Escape != nil { buf.Myprintf(" escape %v", node.Escape) } @@ -1434,7 +1431,11 @@ func (node *ComparisonExpr) Format(buf *TrackedBuffer) { // Format formats the node. func (node *RangeCond) Format(buf *TrackedBuffer) { - buf.Myprintf("%v %s %v and %v", node.Left, node.Operator, node.From, node.To) + printWithParensIfNeeded(node, node.Expr, buf) + buf.Myprintf(" %s ", node.Operator) + printWithParensIfNeeded(node, node.From, buf) + buf.WriteString(" and ") + printWithParensIfNeeded(node, node.To, buf) } // Format formats the node. @@ -1504,21 +1505,27 @@ func (node ListArg) Format(buf *TrackedBuffer) { // Format formats the node. func (node *BinaryExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("%v %s %v", node.Left, node.Operator, node.Right) + printWithParensIfNeeded(node, node.Left, buf) + buf.Myprintf(" %s ", node.Operator) + printWithParensIfNeeded(node, node.Right, buf) } // Format formats the node. func (node *UnaryExpr) Format(buf *TrackedBuffer) { + buf.Myprintf("%s", node.Operator) if _, unary := node.Expr.(*UnaryExpr); unary { - buf.Myprintf("%s %v", node.Operator, node.Expr) + buf.Myprintf(" %v", node.Expr) return } - buf.Myprintf("%s%v", node.Operator, node.Expr) + + printWithParensIfNeeded(node, node.Expr, buf) } // Format formats the node. func (node *IntervalExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("interval %v %s", node.Expr, node.Unit) + buf.WriteString("interval ") + printWithParensIfNeeded(node, node.Expr, buf) + buf.Myprintf(" %s", node.Unit) } // Format formats the node. diff --git a/go/vt/sqlparser/ast_funcs.go b/go/vt/sqlparser/ast_funcs.go index 2403dc7c07f..97782cd6ce9 100644 --- a/go/vt/sqlparser/ast_funcs.go +++ b/go/vt/sqlparser/ast_funcs.go @@ -704,14 +704,8 @@ func (node *Select) SetLimit(limit *Limit) { } // AddWhere adds the boolean expression to the -// WHERE clause as an AND condition. If the expression -// is an OR clause, it parenthesizes it. Currently, -// the OR operator is the only one that's lower precedence -// than AND. +// WHERE clause as an AND condition. func (node *Select) AddWhere(expr Expr) { - if _, ok := expr.(*OrExpr); ok { - expr = &ParenExpr{Expr: expr} - } if node.Where == nil { node.Where = &Where{ Type: WhereStr, @@ -726,14 +720,8 @@ func (node *Select) AddWhere(expr Expr) { } // AddHaving adds the boolean expression to the -// HAVING clause as an AND condition. If the expression -// is an OR clause, it parenthesizes it. Currently, -// the OR operator is the only one that's lower precedence -// than AND. +// HAVING clause as an AND condition. func (node *Select) AddHaving(expr Expr) { - if _, ok := expr.(*OrExpr); ok { - expr = &ParenExpr{Expr: expr} - } if node.Having == nil { node.Having = &Where{ Type: HavingStr, @@ -767,6 +755,20 @@ func (node *Union) SetLimit(limit *Limit) { node.Limit = limit } +func needParens(op, val Expr) bool { + opBinding := precedenceFor(op) + valBinding := precedenceFor(val) + return !(opBinding == Syntactic || valBinding == Syntactic) && valBinding > opBinding +} + +func printWithParensIfNeeded(op, val Expr, buf *TrackedBuffer) { + if needParens(op, val) { + buf.WriteString("(") + defer buf.WriteString(")") + } + buf.Myprintf("%v", val) +} + type atCount int const ( diff --git a/go/vt/sqlparser/ast_test.go b/go/vt/sqlparser/ast_test.go index ae333445058..f229c0fa4ec 100644 --- a/go/vt/sqlparser/ast_test.go +++ b/go/vt/sqlparser/ast_test.go @@ -83,7 +83,6 @@ func TestSelect(t *testing.T) { t.Errorf("having: %q, want %s", buf.String(), want) } - // OR clauses must be parenthesized. tree, err = Parse("select * from t where a = 1 or b = 1") require.NoError(t, err) expr = tree.(*Select).Where.Expr @@ -91,7 +90,7 @@ func TestSelect(t *testing.T) { sel.AddWhere(expr) buf = NewTrackedBuffer(nil) sel.Where.Format(buf) - want = " where (a = 1 or b = 1)" + want = " where a = 1 or b = 1" if buf.String() != want { t.Errorf("where: %q, want %s", buf.String(), want) } @@ -99,7 +98,7 @@ func TestSelect(t *testing.T) { sel.AddHaving(expr) buf = NewTrackedBuffer(nil) sel.Having.Format(buf) - want = " having (a = 1 or b = 1)" + want = " having a = 1 or b = 1" if buf.String() != want { t.Errorf("having: %q, want %s", buf.String(), want) } @@ -439,7 +438,7 @@ func TestReplaceExpr(t *testing.T) { out: "not :a", }, { in: "select * from t where ((select a from b))", - out: "(:a)", + out: ":a", }, { in: "select * from t where (select a from b) = 1", out: ":a = 1", diff --git a/go/vt/sqlparser/parse_test.go b/go/vt/sqlparser/parse_test.go index a2946d804cc..37d927f8f17 100644 --- a/go/vt/sqlparser/parse_test.go +++ b/go/vt/sqlparser/parse_test.go @@ -26,6 +26,7 @@ import ( "sync" "testing" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" ) @@ -313,7 +314,8 @@ var ( }, { input: "select /* exists */ 1 from t where exists (select 1 from t)", }, { - input: "select /* (boolean) */ 1 from t where not (a = b)", + input: "select /* (boolean) */ 1 from t where not (a = b)", + output: "select /* (boolean) */ 1 from t where not a = b", }, { input: "select /* in value list */ 1 from t where a in (b, c)", }, { @@ -376,11 +378,14 @@ var ( }, { input: "select /* select as a value expression */ 1 from t where a = (select a from t)", }, { - input: "select /* parenthesised value */ 1 from t where a = (b)", + input: "select /* parenthesised value */ 1 from t where a = (b)", + output: "select /* parenthesised value */ 1 from t where a = b", }, { - input: "select /* over-parenthesize */ ((1)) from t where ((a)) in (((1))) and ((a, b)) in ((((1, 1))), ((2, 2)))", + input: "select /* over-parenthesize */ ((1)) from t where ((a)) in (((1))) and ((a, b)) in ((((1, 1))), ((2, 2)))", + output: "select /* over-parenthesize */ 1 from t where a in (1) and (a, b) in ((1, 1), (2, 2))", }, { - input: "select /* dot-parenthesize */ (a.b) from t where (b.c) = 2", + input: "select /* dot-parenthesize */ (a.b) from t where (b.c) = 2", + output: "select /* dot-parenthesize */ a.b from t where b.c = 2", }, { input: "select /* & */ 1 from t where a = b & c", }, { @@ -431,7 +436,7 @@ var ( input: "select /* function with distinct */ count(distinct a) from t", }, { input: "select count(distinctrow(1)) from (select (1) from dual union all select 1 from dual) a", - output: "select count(distinct (1)) from (select (1) from dual union all select 1 from dual) as a", + output: "select count(distinct 1) from (select 1 from dual union all select 1 from dual) as a", }, { input: "select /* if as func */ 1 from t where a = if(b)", }, { @@ -604,7 +609,8 @@ var ( }, { input: "select /* OR of mixed columns in where */ * from t where a = 5 or b and c is not null", }, { - input: "select /* OR in select columns */ (a or b) from t where c = 5", + input: "select /* OR in select columns */ (a or b) from t where c = 5", + output: "select /* OR in select columns */ a or b from t where c = 5", }, { input: "select /* bool as select value */ a, true from t", }, { @@ -1532,7 +1538,7 @@ var ( output: "delete a, b from tbl_a as a, tbl_b as b where a.id = b.id and b.name = 'test'", }, { input: "select distinctrow a.* from (select (1) from dual union all select 1 from dual) a", - output: "select distinct a.* from (select (1) from dual union all select 1 from dual) as a", + output: "select distinct a.* from (select 1 from dual union all select 1 from dual) as a", }, { input: "select `weird function name`() from t", }, { @@ -1572,8 +1578,8 @@ func TestValid(t *testing.T) { tree, err := Parse(tcase.input) require.NoError(t, err) out := String(tree) - if out != tcase.output { - t.Errorf("Parse(%q) = %q, want: %q", tcase.input, out, tcase.output) + if diff := cmp.Diff(tcase.output, out); diff != "" { + t.Errorf("Parse(%q):\n%s", tcase.input, diff) } // This test just exercises the tree walking functionality. // There's no way automated way to verify that a node calls diff --git a/go/vt/sqlparser/precedence.go b/go/vt/sqlparser/precedence.go new file mode 100644 index 00000000000..63a496b7ddf --- /dev/null +++ b/go/vt/sqlparser/precedence.go @@ -0,0 +1,90 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sqlparser + +// Precendence is used to know the precedence between operators, +// so we can introduce parens when needed in the String representation of the AST +type Precendence int + +const ( + Syntactic Precendence = iota + P1 + P2 + P3 + P4 + P5 + P6 + P7 + P8 + P9 + P10 + P11 + P12 + P13 + P14 + P15 + P16 + P17 +) + +func precedenceFor(in Expr) Precendence { + switch node := in.(type) { + case *OrExpr: + return P16 + //case *XorExpr: TODO add parser support for XOR + // return P15 + case *AndExpr: + return P14 + case *NotExpr: + return P13 + case *RangeCond: + return P12 + case *ComparisonExpr: + switch node.Operator { + case EqualStr, NotEqualStr, GreaterThanStr, GreaterEqualStr, LessThanStr, LessEqualStr, LikeStr, InStr, RegexpStr: + return P11 + } + case *BinaryExpr: + switch node.Operator { + case BitOrStr: + return P10 + case BitAndStr: + return P9 + case ShiftLeftStr, ShiftRightStr: + return P8 + case PlusStr, MinusStr: + return P7 + case DivStr, MultStr, ModStr, IntDivStr: + return P6 + case BitXorStr: + return P5 + } + case *UnaryExpr: + switch node.Operator { + case UPlusStr, UMinusStr: + return P4 + case BangStr: + return P3 + case BinaryStr: + return P2 + } + case *IntervalExpr: + return P1 + } + + return Syntactic +} diff --git a/go/vt/sqlparser/precedence_test.go b/go/vt/sqlparser/precedence_test.go index 1fdd74ada3f..38fcf1e5fa3 100644 --- a/go/vt/sqlparser/precedence_test.go +++ b/go/vt/sqlparser/precedence_test.go @@ -19,6 +19,8 @@ package sqlparser import ( "fmt" "testing" + + "github.com/stretchr/testify/require" ) func readable(node Expr) string { @@ -110,3 +112,40 @@ func TestIsPrecedence(t *testing.T) { } } } + +func TestParens(t *testing.T) { + tests := []struct { + in, expected string + }{ + {in: "12", expected: "12"}, + {in: "(12)", expected: "12"}, + {in: "((12))", expected: "12"}, + {in: "((true) and (false))", expected: "true and false"}, + {in: "((true) and (false)) and (true)", expected: "true and false and true"}, + {in: "((true) and (false))", expected: "true and false"}, + {in: "a=b and (c=d or e=f)", expected: "a = b and (c = d or e = f)"}, + {in: "(a=b and c=d) or e=f", expected: "a = b and c = d or e = f"}, + {in: "a & (b | c)", expected: "a & (b | c)"}, + {in: "(a & b) | c", expected: "a & b | c"}, + {in: "not (a=b and c=d)", expected: "not (a = b and c = d)"}, + {in: "not (a=b) and c=d", expected: "not a = b and c = d"}, + {in: "-(12)", expected: "-12"}, + {in: "-(12 + 12)", expected: "-(12 + 12)"}, + {in: "(1 > 2) and (1 = b)", expected: "1 > 2 and 1 = b"}, + {in: "(a / b) + c", expected: "a / b + c"}, + {in: "a / (b + c)", expected: "a / (b + c)"}, + {in: "(1,2,3)", expected: "(1, 2, 3)"}, + {in: "(a) between (5) and (7)", expected: "a between 5 and 7"}, + {in: "(a | b) between (5) and (7)", expected: "a | b between 5 and 7"}, + {in: "(a and b) between (5) and (7)", expected: "(a and b) between 5 and 7"}, + } + + for _, tc := range tests { + t.Run(tc.in, func(t *testing.T) { + stmt, err := Parse("select " + tc.in) + require.NoError(t, err) + out := String(stmt) + require.Equal(t, "select "+tc.expected+" from dual", out) + }) + } +} diff --git a/go/vt/sqlparser/rewriter.go b/go/vt/sqlparser/rewriter.go index 3b912a79bb8..490c3c876eb 100644 --- a/go/vt/sqlparser/rewriter.go +++ b/go/vt/sqlparser/rewriter.go @@ -459,10 +459,6 @@ func (r *replaceOrderByItems) inc() { *r++ } -func replaceParenExprExpr(newNode, parent SQLNode) { - parent.(*ParenExpr).Expr = newNode.(Expr) -} - func replaceParenSelectSelect(newNode, parent SQLNode) { parent.(*ParenSelect).Select = newNode.(SelectStatement) } @@ -503,12 +499,12 @@ func (r *replacePartitionsItems) inc() { *r++ } -func replaceRangeCondFrom(newNode, parent SQLNode) { - parent.(*RangeCond).From = newNode.(Expr) +func replaceRangeCondExpr(newNode, parent SQLNode) { + parent.(*RangeCond).Expr = newNode.(Expr) } -func replaceRangeCondLeft(newNode, parent SQLNode) { - parent.(*RangeCond).Left = newNode.(Expr) +func replaceRangeCondFrom(newNode, parent SQLNode) { + parent.(*RangeCond).From = newNode.(Expr) } func replaceRangeCondTo(newNode, parent SQLNode) { @@ -1090,9 +1086,6 @@ func (a *application) apply(parent, node SQLNode, replacer replacerFunc) { case *OtherRead: - case *ParenExpr: - a.apply(node, n.Expr, replaceParenExprExpr) - case *ParenSelect: a.apply(node, n.Select, replaceParenSelectSelect) @@ -1121,8 +1114,8 @@ func (a *application) apply(parent, node SQLNode, replacer replacerFunc) { } case *RangeCond: + a.apply(node, n.Expr, replaceRangeCondExpr) a.apply(node, n.From, replaceRangeCondFrom) - a.apply(node, n.Left, replaceRangeCondLeft) a.apply(node, n.To, replaceRangeCondTo) case ReferenceAction: diff --git a/go/vt/sqlparser/sql.go b/go/vt/sqlparser/sql.go index 897f02958af..452e79bf262 100644 --- a/go/vt/sqlparser/sql.go +++ b/go/vt/sqlparser/sql.go @@ -6037,13 +6037,13 @@ yydefault: yyDollar = yyS[yypt-5 : yypt+1] //line sql.y:2258 { - yyVAL.expr = &RangeCond{Left: yyDollar[1].expr, Operator: BetweenStr, From: yyDollar[3].expr, To: yyDollar[5].expr} + yyVAL.expr = &RangeCond{Expr: yyDollar[1].expr, Operator: BetweenStr, From: yyDollar[3].expr, To: yyDollar[5].expr} } case 426: yyDollar = yyS[yypt-6 : yypt+1] //line sql.y:2262 { - yyVAL.expr = &RangeCond{Left: yyDollar[1].expr, Operator: NotBetweenStr, From: yyDollar[4].expr, To: yyDollar[6].expr} + yyVAL.expr = &RangeCond{Expr: yyDollar[1].expr, Operator: NotBetweenStr, From: yyDollar[4].expr, To: yyDollar[6].expr} } case 427: yyDollar = yyS[yypt-2 : yypt+1] @@ -7097,7 +7097,7 @@ yydefault: //line sql.y:3079 { if len(yyDollar[1].valTuple) == 1 { - yyVAL.expr = &ParenExpr{yyDollar[1].valTuple[0]} + yyVAL.expr = yyDollar[1].valTuple[0] } else { yyVAL.expr = yyDollar[1].valTuple } diff --git a/go/vt/sqlparser/sql.y b/go/vt/sqlparser/sql.y index 1b5e2718bec..683dbf01cb9 100644 --- a/go/vt/sqlparser/sql.y +++ b/go/vt/sqlparser/sql.y @@ -2256,11 +2256,11 @@ condition: } | value_expression BETWEEN value_expression AND value_expression { - $$ = &RangeCond{Left: $1, Operator: BetweenStr, From: $3, To: $5} + $$ = &RangeCond{Expr: $1, Operator: BetweenStr, From: $3, To: $5} } | value_expression NOT BETWEEN value_expression AND value_expression { - $$ = &RangeCond{Left: $1, Operator: NotBetweenStr, From: $4, To: $6} + $$ = &RangeCond{Expr: $1, Operator: NotBetweenStr, From: $4, To: $6} } | EXISTS subquery { @@ -3078,7 +3078,7 @@ tuple_expression: row_tuple { if len($1) == 1 { - $$ = &ParenExpr{$1[0]} + $$ = $1[0] } else { $$ = $1 } diff --git a/go/vt/sqlparser/tracked_buffer.go b/go/vt/sqlparser/tracked_buffer.go index 5e5383ed0f1..07fb5e62df7 100644 --- a/go/vt/sqlparser/tracked_buffer.go +++ b/go/vt/sqlparser/tracked_buffer.go @@ -110,6 +110,12 @@ func (buf *TrackedBuffer) Myprintf(format string, values ...interface{}) { } } +// WriteString appends the contents of s to b's buffer. +// It returns the length of s and a nil error. +func (buf *TrackedBuffer) WriteString(s string) (int, error) { + return buf.Builder.WriteString(s) +} + // WriteArg writes a value argument into the buffer along with // tracking information for future substitutions. arg must contain // the ":" or "::" prefix. diff --git a/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt b/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt index 2b74d49911c..304294f340b 100644 --- a/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt +++ b/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt @@ -28,10 +28,10 @@ select * from user where name = 'bob' /* vindex lookup */ ---------------------------------------------------------------------- select * from user where name = 'bob' or nickname = 'bob' /* vindex lookup */ -1 ks_sharded/-40: select * from user where (name = 'bob' or nickname = 'bob') limit 10001 /* vindex lookup */ -1 ks_sharded/40-80: select * from user where (name = 'bob' or nickname = 'bob') limit 10001 /* vindex lookup */ -1 ks_sharded/80-c0: select * from user where (name = 'bob' or nickname = 'bob') limit 10001 /* vindex lookup */ -1 ks_sharded/c0-: select * from user where (name = 'bob' or nickname = 'bob') limit 10001 /* vindex lookup */ +1 ks_sharded/-40: select * from user where name = 'bob' or nickname = 'bob' limit 10001 /* vindex lookup */ +1 ks_sharded/40-80: select * from user where name = 'bob' or nickname = 'bob' limit 10001 /* vindex lookup */ +1 ks_sharded/80-c0: select * from user where name = 'bob' or nickname = 'bob' limit 10001 /* vindex lookup */ +1 ks_sharded/c0-: select * from user where name = 'bob' or nickname = 'bob' limit 10001 /* vindex lookup */ ---------------------------------------------------------------------- select u.id, u.name, u.nickname, n.info from user u join name_info n on u.name = n.name /* join on varchar */ @@ -97,16 +97,16 @@ select name from user where id = (select id from t1) /* non-correlated subquery select name from user where id in (select id from t1) /* non-correlated subquery in IN clause */ 1 ks_unsharded/-: select id from t1 limit 10001 /* non-correlated subquery in IN clause */ -2 ks_sharded/-40: select name from user where 1 = 1 and (id in (1)) limit 10001 /* non-correlated subquery in IN clause */ +2 ks_sharded/-40: select name from user where 1 = 1 and id in (1) limit 10001 /* non-correlated subquery in IN clause */ ---------------------------------------------------------------------- select name from user where id not in (select id from t1) /* non-correlated subquery in NOT IN clause */ 1 ks_unsharded/-: select id from t1 limit 10001 /* non-correlated subquery in NOT IN clause */ -2 ks_sharded/-40: select name from user where (1 = 0 or (id not in (1))) limit 10001 /* non-correlated subquery in NOT IN clause */ -2 ks_sharded/40-80: select name from user where (1 = 0 or (id not in (1))) limit 10001 /* non-correlated subquery in NOT IN clause */ -2 ks_sharded/80-c0: select name from user where (1 = 0 or (id not in (1))) limit 10001 /* non-correlated subquery in NOT IN clause */ -2 ks_sharded/c0-: select name from user where (1 = 0 or (id not in (1))) limit 10001 /* non-correlated subquery in NOT IN clause */ +2 ks_sharded/-40: select name from user where 1 = 0 or id not in (1) limit 10001 /* non-correlated subquery in NOT IN clause */ +2 ks_sharded/40-80: select name from user where 1 = 0 or id not in (1) limit 10001 /* non-correlated subquery in NOT IN clause */ +2 ks_sharded/80-c0: select name from user where 1 = 0 or id not in (1) limit 10001 /* non-correlated subquery in NOT IN clause */ +2 ks_sharded/c0-: select name from user where 1 = 0 or id not in (1) limit 10001 /* non-correlated subquery in NOT IN clause */ ---------------------------------------------------------------------- select name from user where exists (select id from t1) /* non-correlated subquery as EXISTS */ @@ -128,7 +128,7 @@ select * from name_info order by info /* select * and order by varchar column */ ---------------------------------------------------------------------- select distinct(name) from user where id = 1 /* select distinct */ -1 ks_sharded/-40: select distinct (name) from user where id = 1 limit 10001 /* select distinct */ +1 ks_sharded/-40: select distinct name from user where id = 1 limit 10001 /* select distinct */ ---------------------------------------------------------------------- select distinct name from user where id = 1 /* select distinct */ diff --git a/go/vt/vtexplain/vtexplain_vttablet.go b/go/vt/vtexplain/vtexplain_vttablet.go index 0e25ed72445..9fe4983dbd0 100644 --- a/go/vt/vtexplain/vtexplain_vttablet.go +++ b/go/vt/vtexplain/vtexplain_vttablet.go @@ -569,8 +569,6 @@ func inferColTypeFromExpr(node sqlparser.Expr, colTypeMap map[string]querypb.Typ default: log.Errorf("vtexplain: unsupported sql value %s", sqlparser.String(node)) } - case *sqlparser.ParenExpr: - colNames, colTypes = inferColTypeFromExpr(node.Expr, colTypeMap, colNames, colTypes) case *sqlparser.CaseExpr: colNames, colTypes = inferColTypeFromExpr(node.Whens[0].Val, colTypeMap, colNames, colTypes) case *sqlparser.NullVal: diff --git a/go/vt/vtgate/planbuilder/dml.go b/go/vt/vtgate/planbuilder/dml.go index 1d3f72dfb04..2c3b498fd5d 100644 --- a/go/vt/vtgate/planbuilder/dml.go +++ b/go/vt/vtgate/planbuilder/dml.go @@ -63,13 +63,6 @@ func getDMLRouting(where *sqlparser.Where, table *vindexes.Table) (engine.DMLOpc func getMatch(node sqlparser.Expr, col sqlparser.ColIdent) (pv sqltypes.PlanValue, ok bool) { filters := splitAndExpression(nil, node) for _, filter := range filters { - filter = skipParenthesis(filter) - if parenthesis, ok := node.(*sqlparser.ParenExpr); ok { - if pv, ok := getMatch(parenthesis.Expr, col); ok { - return pv, ok - } - continue - } comparison, ok := filter.(*sqlparser.ComparisonExpr) if !ok { continue diff --git a/go/vt/vtgate/planbuilder/expr.go b/go/vt/vtgate/planbuilder/expr.go index ad06a4f7219..e7a51664e6b 100644 --- a/go/vt/vtgate/planbuilder/expr.go +++ b/go/vt/vtgate/planbuilder/expr.go @@ -36,25 +36,10 @@ func splitAndExpression(filters []sqlparser.Expr, node sqlparser.Expr) []sqlpars case *sqlparser.AndExpr: filters = splitAndExpression(filters, node.Left) return splitAndExpression(filters, node.Right) - case *sqlparser.ParenExpr: - // If the inner expression is AndExpr, then we can remove - // the parenthesis because they are unnecessary. - if node, ok := node.Expr.(*sqlparser.AndExpr); ok { - return splitAndExpression(filters, node) - } } return append(filters, node) } -// skipParenthesis skips the parenthesis (if any) of an expression and -// returns the innermost unparenthesized expression. -func skipParenthesis(node sqlparser.Expr) sqlparser.Expr { - if node, ok := node.(*sqlparser.ParenExpr); ok { - return skipParenthesis(node.Expr) - } - return node -} - type subqueryInfo struct { ast *sqlparser.Subquery bldr builder @@ -179,33 +164,25 @@ func (pb *primitiveBuilder) findOrigin(expr sqlparser.Expr) (pullouts []*pullout case *sqlparser.ComparisonExpr: if construct.Operator == sqlparser.InStr { // a in (subquery) -> (:__sq_has_values = 1 and (a in ::__sq)) - newExpr := &sqlparser.ParenExpr{ - Expr: &sqlparser.AndExpr{ - Left: &sqlparser.ComparisonExpr{ - Left: sqlparser.NewValArg([]byte(":" + hasValues)), - Operator: sqlparser.EqualStr, - Right: sqlparser.NewIntVal([]byte("1")), - }, - Right: &sqlparser.ParenExpr{ - Expr: sqlparser.ReplaceExpr(construct, sqi.ast, sqlparser.ListArg([]byte("::"+sqName))), - }, + newExpr := &sqlparser.AndExpr{ + Left: &sqlparser.ComparisonExpr{ + Left: sqlparser.NewValArg([]byte(":" + hasValues)), + Operator: sqlparser.EqualStr, + Right: sqlparser.NewIntVal([]byte("1")), }, + Right: sqlparser.ReplaceExpr(construct, sqi.ast, sqlparser.ListArg([]byte("::"+sqName))), } expr = sqlparser.ReplaceExpr(expr, construct, newExpr) pullouts = append(pullouts, newPulloutSubquery(engine.PulloutIn, sqName, hasValues, sqi.bldr)) } else { // a not in (subquery) -> (:__sq_has_values = 0 or (a not in ::__sq)) - newExpr := &sqlparser.ParenExpr{ - Expr: &sqlparser.OrExpr{ - Left: &sqlparser.ComparisonExpr{ - Left: sqlparser.NewValArg([]byte(":" + hasValues)), - Operator: sqlparser.EqualStr, - Right: sqlparser.NewIntVal([]byte("0")), - }, - Right: &sqlparser.ParenExpr{ - Expr: sqlparser.ReplaceExpr(construct, sqi.ast, sqlparser.ListArg([]byte("::"+sqName))), - }, + newExpr := &sqlparser.OrExpr{ + Left: &sqlparser.ComparisonExpr{ + Left: sqlparser.NewValArg([]byte(":" + hasValues)), + Operator: sqlparser.EqualStr, + Right: sqlparser.NewIntVal([]byte("0")), }, + Right: sqlparser.ReplaceExpr(construct, sqi.ast, sqlparser.ListArg([]byte("::"+sqName))), } expr = sqlparser.ReplaceExpr(expr, construct, newExpr) pullouts = append(pullouts, newPulloutSubquery(engine.PulloutNotIn, sqName, hasValues, sqi.bldr)) diff --git a/go/vt/vtgate/planbuilder/expr_test.go b/go/vt/vtgate/planbuilder/expr_test.go index 9aca7b9596f..3046862a609 100644 --- a/go/vt/vtgate/planbuilder/expr_test.go +++ b/go/vt/vtgate/planbuilder/expr_test.go @@ -17,7 +17,6 @@ limitations under the License. package planbuilder import ( - "reflect" "testing" "vitess.io/vitess/go/vt/sqlparser" @@ -104,17 +103,6 @@ func TestValEqual(t *testing.T) { } } -func TestSkipParenthesis(t *testing.T) { - baseNode := newIntVal("1") - paren1 := &sqlparser.ParenExpr{Expr: baseNode} - paren2 := &sqlparser.ParenExpr{Expr: paren1} - for _, tcase := range []sqlparser.Expr{baseNode, paren1, paren2} { - if got, want := skipParenthesis(tcase), baseNode; !reflect.DeepEqual(got, want) { - t.Errorf("skipParenthesis(%v): %v, want %v", sqlparser.String(tcase), sqlparser.String(got), sqlparser.String(want)) - } - } -} - func newStrVal(in string) *sqlparser.SQLVal { return sqlparser.NewStrVal([]byte(in)) } diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 4e5c4760477..c4ac1d20067 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -243,7 +243,7 @@ func testFile(t *testing.T, filename, tempDir string, vschema *vindexes.VSchema) if out != tcase.output { fail = true - t.Errorf("File: %s, Line: %v\n %s", filename, tcase.lineno, cmp.Diff(out, tcase.output)) + t.Errorf("File: %s, Line: %v\n %s", filename, tcase.lineno, cmp.Diff(tcase.output, out)) } if err != nil { diff --git a/go/vt/vtgate/planbuilder/route_option.go b/go/vt/vtgate/planbuilder/route_option.go index fa3bab2c6a2..787254a813d 100644 --- a/go/vt/vtgate/planbuilder/route_option.go +++ b/go/vt/vtgate/planbuilder/route_option.go @@ -169,7 +169,6 @@ func (ro *routeOption) canMerge(rro *routeOption, customCheck func() bool) bool // mergeable by unique vindex. The constraint has to be an equality // like a.id = b.id where both columns have the same unique vindex. func (ro *routeOption) canMergeOnFilter(pb *primitiveBuilder, rro *routeOption, filter sqlparser.Expr) bool { - filter = skipParenthesis(filter) comparison, ok := filter.(*sqlparser.ComparisonExpr) if !ok { return false @@ -254,8 +253,6 @@ func (ro *routeOption) computePlan(pb *primitiveBuilder, filter sqlparser.Expr) case sqlparser.InStr: return ro.computeINPlan(pb, node) } - case *sqlparser.ParenExpr: - return ro.computePlan(pb, node.Expr) } return engine.SelectScatter, nil, nil } diff --git a/go/vt/vtgate/planbuilder/testdata/dml_cases.txt b/go/vt/vtgate/planbuilder/testdata/dml_cases.txt index 4c84422be4f..206c14014ff 100644 --- a/go/vt/vtgate/planbuilder/testdata/dml_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/dml_cases.txt @@ -185,7 +185,7 @@ "Name": "user", "Sharded": true }, - "Query": "update user set val = 1 where (id = 1)", + "Query": "update user set val = 1 where id = 1", "Vindex": "user_index", "Values": [ 1 @@ -204,7 +204,7 @@ "Name": "user", "Sharded": true }, - "Query": "update user set val = 1 where (name = 'foo' and id = 1)", + "Query": "update user set val = 1 where name = 'foo' and id = 1", "Vindex": "user_index", "Values": [ 1 @@ -1950,7 +1950,7 @@ "Name": "user", "Sharded": true }, - "Query": "update user_extra set val = 1 where (name = 'foo' or id = 1)", + "Query": "update user_extra set val = 1 where name = 'foo' or id = 1", "Table": "user_extra" } } diff --git a/go/vt/vtgate/planbuilder/testdata/filter_cases.txt b/go/vt/vtgate/planbuilder/testdata/filter_cases.txt index 6e059a0a7ac..9a580d21701 100644 --- a/go/vt/vtgate/planbuilder/testdata/filter_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/filter_cases.txt @@ -481,8 +481,8 @@ "Name": "user", "Sharded": true }, - "Query": "select (id or col) as val from user where user.col = 5 and user.id in (1, 2) and user.name = 'aa'", - "FieldQuery": "select (id or col) as val from user where 1 != 1", + "Query": "select id or col as val from user where user.col = 5 and user.id in (1, 2) and user.name = 'aa'", + "FieldQuery": "select id or col as val from user where 1 != 1", "Vindex": "name_user_map", "Values": [ "aa" @@ -561,7 +561,7 @@ "Name": "user", "Sharded": true }, - "Query": "select id from user where (user.id = 1 or user.name = 'aa' and user.id in (1, 2))", + "Query": "select id from user where user.id = 1 or user.name = 'aa' and user.id in (1, 2)", "FieldQuery": "select id from user where 1 != 1", "Table": "user" } @@ -898,7 +898,7 @@ "Name": "user", "Sharded": true }, - "Query": "select id from user where :__sq_has_values1 = 1 and (id in ::__vals)", + "Query": "select id from user where :__sq_has_values1 = 1 and id in ::__vals", "FieldQuery": "select id from user where 1 != 1", "Vindex": "user_index", "Values": [ @@ -933,7 +933,7 @@ "Name": "user", "Sharded": true }, - "Query": "select id from user where (:__sq_has_values1 = 0 or (id not in ::__sq1))", + "Query": "select id from user where :__sq_has_values1 = 0 or id not in ::__sq1", "FieldQuery": "select id from user where 1 != 1", "Table": "user" } @@ -1034,7 +1034,7 @@ "Name": "user", "Sharded": true }, - "Query": "select id2 from user where :__sq_has_values1 = 1 and (id2 in ::__sq1)", + "Query": "select id2 from user where :__sq_has_values1 = 1 and id2 in ::__sq1", "FieldQuery": "select id2 from user where 1 != 1", "Table": "user" } diff --git a/go/vt/vtgate/planbuilder/testdata/from_cases.txt b/go/vt/vtgate/planbuilder/testdata/from_cases.txt index 928ff6d0c22..d673886737c 100644 --- a/go/vt/vtgate/planbuilder/testdata/from_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/from_cases.txt @@ -664,8 +664,8 @@ "Name": "user", "Sharded": true }, - "Query": "select user.col from user join user_extra on (user.id = user_extra.user_id)", - "FieldQuery": "select user.col from user join user_extra on (user.id = user_extra.user_id) where 1 != 1", + "Query": "select user.col from user join user_extra on user.id = user_extra.user_id", + "FieldQuery": "select user.col from user join user_extra on user.id = user_extra.user_id where 1 != 1", "Table": "user" } } @@ -1530,8 +1530,8 @@ "Name": "main", "Sharded": false }, - "Query": "select unsharded_a.col from unsharded_a join unsharded_b on (:__sq_has_values1 = 1 and (unsharded_a.col in ::__sq1))", - "FieldQuery": "select unsharded_a.col from unsharded_a join unsharded_b on (:__sq_has_values1 = 1 and (unsharded_a.col in ::__sq1)) where 1 != 1", + "Query": "select unsharded_a.col from unsharded_a join unsharded_b on :__sq_has_values1 = 1 and unsharded_a.col in ::__sq1", + "FieldQuery": "select unsharded_a.col from unsharded_a join unsharded_b on :__sq_has_values1 = 1 and unsharded_a.col in ::__sq1 where 1 != 1", "Table": "unsharded_a" } } @@ -1573,7 +1573,7 @@ "Name": "user", "Sharded": true }, - "Query": "select 1 from user where :__sq_has_values1 = 1 and (user.col in ::__sq1)", + "Query": "select 1 from user where :__sq_has_values1 = 1 and user.col in ::__sq1", "FieldQuery": "select 1 from user where 1 != 1", "Table": "user" }, @@ -1621,7 +1621,7 @@ "Name": "user", "Sharded": true }, - "Query": "select 1 from user where :__sq_has_values1 = 1 and (user.col in ::__sq1)", + "Query": "select 1 from user where :__sq_has_values1 = 1 and user.col in ::__sq1", "FieldQuery": "select 1 from user where 1 != 1", "Table": "user" } @@ -1671,7 +1671,7 @@ "Name": "user", "Sharded": true }, - "Query": "select 1 from user where :__sq_has_values1 = 1 and (user.col in ::__sq1)", + "Query": "select 1 from user where :__sq_has_values1 = 1 and user.col in ::__sq1", "FieldQuery": "select 1 from user where 1 != 1", "Table": "user" }, diff --git a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt index 508e9ebee77..d1ab091100e 100644 --- a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt @@ -109,7 +109,7 @@ "Name": "user", "Sharded": true }, - "Query": "select id from user having :__sq_has_values1 = 1 and (id in ::__vals)", + "Query": "select id from user having :__sq_has_values1 = 1 and id in ::__vals", "FieldQuery": "select id from user where 1 != 1", "Vindex": "user_index", "Values": [ @@ -368,7 +368,7 @@ "Name": "user", "Sharded": true }, - "Query": "select col from user where :__sq_has_values1 = 1 and (col in ::__sq1) order by col asc", + "Query": "select col from user where :__sq_has_values1 = 1 and col in ::__sq1 order by col asc", "FieldQuery": "select col from user where 1 != 1", "OrderBy": [ { @@ -540,7 +540,7 @@ "Name": "user", "Sharded": true }, - "Query": "select col from user where :__sq_has_values1 = 1 and (col in ::__sq1) order by null", + "Query": "select col from user where :__sq_has_values1 = 1 and col in ::__sq1 order by null", "FieldQuery": "select col from user where 1 != 1", "Table": "user" } @@ -632,7 +632,7 @@ "Name": "user", "Sharded": true }, - "Query": "select col from user where :__sq_has_values1 = 1 and (col in ::__sq1) order by rand()", + "Query": "select col from user where :__sq_has_values1 = 1 and col in ::__sq1 order by rand()", "FieldQuery": "select col from user where 1 != 1", "Table": "user" } @@ -1040,7 +1040,7 @@ "Name": "user", "Sharded": true }, - "Query": "select col from user where :__sq_has_values1 = 1 and (col in ::__sq1) limit :__upper_limit", + "Query": "select col from user where :__sq_has_values1 = 1 and col in ::__sq1 limit :__upper_limit", "FieldQuery": "select col from user where 1 != 1", "Table": "user" } diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.txt b/go/vt/vtgate/planbuilder/testdata/select_cases.txt index c655e2f6d0e..8002a056fb4 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.txt @@ -781,7 +781,7 @@ "Name": "user", "Sharded": true }, - "Query": "select * from user where name = 'abc' and (id = 4) limit 5", + "Query": "select * from user where name = 'abc' and id = 4 limit 5", "FieldQuery": "select * from user where 1 != 1", "Vindex": "user_index", "Values": [ @@ -801,7 +801,7 @@ "Name": "user", "Sharded": true }, - "Query": "select * from user where (id = 4) and (name = 'abc') limit 5", + "Query": "select * from user where id = 4 and name = 'abc' limit 5", "FieldQuery": "select * from user where 1 != 1", "Vindex": "user_index", "Values": [ @@ -881,7 +881,7 @@ "Name": "user", "Sharded": true }, - "Query": "select * from user where (id = 1) and name = true limit 5", + "Query": "select * from user where id = 1 and name = true limit 5", "FieldQuery": "select * from user where 1 != 1", "Vindex": "user_index", "Values": [ @@ -901,7 +901,7 @@ "Name": "user", "Sharded": true }, - "Query": "select * from user where (id = 1) and name limit 5", + "Query": "select * from user where id = 1 and name limit 5", "FieldQuery": "select * from user where 1 != 1", "Vindex": "user_index", "Values": [ @@ -921,7 +921,7 @@ "Name": "user", "Sharded": true }, - "Query": "select * from user where (id = 5) and name = true limit 5", + "Query": "select * from user where id = 5 and name = true limit 5", "FieldQuery": "select * from user where 1 != 1", "Vindex": "user_index", "Values": [ diff --git a/go/vt/vtgate/planbuilder/testdata/wireup_cases.txt b/go/vt/vtgate/planbuilder/testdata/wireup_cases.txt index df5433d2349..2ce3e559caa 100644 --- a/go/vt/vtgate/planbuilder/testdata/wireup_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/wireup_cases.txt @@ -527,7 +527,7 @@ "Name": "user", "Sharded": true }, - "Query": "select 1 from user where :__sq_has_values1 = 1 and (id in ::__vals)", + "Query": "select 1 from user where :__sq_has_values1 = 1 and id in ::__vals", "FieldQuery": "select 1 from user where 1 != 1", "Vindex": "user_index", "Values": [ diff --git a/go/vt/vttablet/endtoend/queries_test.go b/go/vt/vttablet/endtoend/queries_test.go index 1cd00a795dc..992da441dcf 100644 --- a/go/vt/vttablet/endtoend/queries_test.go +++ b/go/vt/vttablet/endtoend/queries_test.go @@ -281,8 +281,8 @@ func TestQueries(t *testing.T) { {"1"}, }, Rewritten: []string{ - "select (eid) from vitess_a where 1 != 1", - "select /* parenthesised col */ (eid) from vitess_a where eid = 1 and id = 1 limit 10001", + "select eid from vitess_a where 1 != 1", + "select /* parenthesised col */ eid from vitess_a where eid = 1 and id = 1 limit 10001", }, RowsAffected: 1, }, @@ -355,7 +355,7 @@ func TestQueries(t *testing.T) { }, Rewritten: []string{ "select * from vitess_a where 1 != 1", - "select /* (condition) */ * from vitess_a where (eid = 1) limit 10001", + "select /* (condition) */ * from vitess_a where eid = 1 limit 10001", }, RowsAffected: 2, }, diff --git a/go/vt/vttablet/tabletserver/planbuilder/plan_test.go b/go/vt/vttablet/tabletserver/planbuilder/plan_test.go index 8cf06e6ff14..8f5f030c574 100644 --- a/go/vt/vttablet/tabletserver/planbuilder/plan_test.go +++ b/go/vt/vttablet/tabletserver/planbuilder/plan_test.go @@ -64,37 +64,39 @@ func (p *Plan) MarshalJSON() ([]byte, error) { func TestPlan(t *testing.T) { testSchema := loadSchema("schema_test.json") for tcase := range iterateExecFile("exec_cases.txt") { - if strings.Contains(tcase.options, "PassthroughDMLs") { - PassthroughDMLs = true - } - var plan *Plan - var err error - statement, err := sqlparser.Parse(tcase.input) - if err == nil { - plan, err = Build(statement, testSchema) - } - PassthroughDMLs = false - - var out string - if err != nil { - out = err.Error() - } else { - bout, err := json.Marshal(plan) - if err != nil { - t.Fatalf("Error marshalling %v: %v", plan, err) + t.Run(tcase.input, func(t *testing.T) { + if strings.Contains(tcase.options, "PassthroughDMLs") { + PassthroughDMLs = true } - out = string(bout) - } - if out != tcase.output { - t.Errorf("Line:%v\ngot = %s\nwant = %s", tcase.lineno, out, tcase.output) + var plan *Plan + var err error + statement, err := sqlparser.Parse(tcase.input) + if err == nil { + plan, err = Build(statement, testSchema) + } + PassthroughDMLs = false + + var out string if err != nil { - out = fmt.Sprintf("\"%s\"", out) + out = err.Error() } else { - bout, _ := json.MarshalIndent(plan, "", " ") + bout, err := json.Marshal(plan) + if err != nil { + t.Fatalf("Error marshalling %v: %v", plan, err) + } out = string(bout) } - fmt.Printf("\"%s\"\n%s\n\n", tcase.input, out) - } + if out != tcase.output { + t.Errorf("Line:%v\ngot = %s\nwant = %s", tcase.lineno, out, tcase.output) + if err != nil { + out = fmt.Sprintf("\"%s\"", out) + } else { + bout, _ := json.MarshalIndent(plan, "", " ") + out = string(bout) + } + fmt.Printf("\"%s\"\n%s\n\n", tcase.input, out) + } + }) } } diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index 901d8064cf5..c87c5cb462d 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -413,12 +413,6 @@ func splitAndExpression(filters []sqlparser.Expr, node sqlparser.Expr) []sqlpars case *sqlparser.AndExpr: filters = splitAndExpression(filters, node.Left) return splitAndExpression(filters, node.Right) - case *sqlparser.ParenExpr: - // If the inner expression is AndExpr, then we can remove - // the parenthesis because they are unnecessary. - if node, ok := node.Expr.(*sqlparser.AndExpr); ok { - return splitAndExpression(filters, node) - } } return append(filters, node) } diff --git a/go/vt/wrangler/vdiff.go b/go/vt/wrangler/vdiff.go index 02afd5e2cce..b45374242c7 100644 --- a/go/vt/wrangler/vdiff.go +++ b/go/vt/wrangler/vdiff.go @@ -882,10 +882,6 @@ func removeExprKeyrange(node sqlparser.Expr) sqlparser.Expr { Left: removeExprKeyrange(node.Left), Right: removeExprKeyrange(node.Right), } - case *sqlparser.ParenExpr: - return &sqlparser.ParenExpr{ - Expr: removeExprKeyrange(node.Expr), - } } return node } diff --git a/go/vt/wrangler/vdiff_test.go b/go/vt/wrangler/vdiff_test.go index 768d879b306..c81d3e6a742 100644 --- a/go/vt/wrangler/vdiff_test.go +++ b/go/vt/wrangler/vdiff_test.go @@ -308,7 +308,7 @@ func TestVDiffPlanSuccess(t *testing.T) { table: "t1", td: &tableDiffer{ targetTable: "t1", - sourceExpression: "select c1, c2 from t1 where (c2 = 2) order by c1 asc", + sourceExpression: "select c1, c2 from t1 where c2 = 2 order by c1 asc", targetExpression: "select c1, c2 from t1 order by c1 asc", compareCols: []int{-1, 1}, comparePKs: []int{0}, @@ -359,12 +359,14 @@ func TestVDiffPlanSuccess(t *testing.T) { }, }} for _, tcase := range testcases { - filter := &binlogdatapb.Filter{Rules: []*binlogdatapb.Rule{tcase.input}} - df := &vdiff{} - err := df.buildVDiffPlan(context.Background(), filter, schm) - require.NoError(t, err, tcase.input) - require.Equal(t, 1, len(df.differs), tcase.input) - assert.Equal(t, tcase.td, df.differs[tcase.table], tcase.input) + t.Run(tcase.input.Filter, func(t *testing.T) { + filter := &binlogdatapb.Filter{Rules: []*binlogdatapb.Rule{tcase.input}} + df := &vdiff{} + err := df.buildVDiffPlan(context.Background(), filter, schm) + require.NoError(t, err, tcase.input) + require.Equal(t, 1, len(df.differs), tcase.input) + assert.Equal(t, tcase.td, df.differs[tcase.table], tcase.input) + }) } } From 474bc95f7cbc1d720140ffca2557e988d5646038 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Tue, 17 Mar 2020 16:10:59 +0530 Subject: [PATCH 67/85] Refactoring of Myprintf Myprintf uses from AST now include the parent struct. In order to get better readability and guarantees that we don't introduce regressions in the future, this refactoring changes Myprintf uses from the AST to always provide the parent structure. Using this information, Myprintf can figure out when parens are needed. Signed-off-by: Harshit Gangal Signed-off-by: Andres Taylor --- go/vt/sqlparser/ast.go | 373 ++++++++++++++---------------- go/vt/sqlparser/ast_funcs.go | 14 -- go/vt/sqlparser/precedence.go | 4 + go/vt/sqlparser/rewriter.go | 10 +- go/vt/sqlparser/sql.go | 4 +- go/vt/sqlparser/sql.y | 5 +- go/vt/sqlparser/tracked_buffer.go | 59 ++++- 7 files changed, 243 insertions(+), 226 deletions(-) diff --git a/go/vt/sqlparser/ast.go b/go/vt/sqlparser/ast.go index 5a7e42dbeed..4f232fe4b02 100644 --- a/go/vt/sqlparser/ast.go +++ b/go/vt/sqlparser/ast.go @@ -536,7 +536,7 @@ type ( // RangeCond represents a BETWEEN or a NOT BETWEEN expression. RangeCond struct { Operator string - Expr Expr + Left Expr From, To Expr } @@ -809,7 +809,7 @@ type TableIdent struct { // Format formats the node. func (node *Select) Format(buf *TrackedBuffer) { - buf.Myprintf("select %v%s%s%s%v from %v%v%v%v%v%v%s", + buf.astPrintf(node, "select %v%s%s%s%v from %v%v%v%v%v%v%s", node.Comments, node.Cache, node.Distinct, node.Hints, node.SelectExprs, node.From, node.Where, node.GroupBy, node.Having, node.OrderBy, @@ -818,24 +818,24 @@ func (node *Select) Format(buf *TrackedBuffer) { // Format formats the node. func (node *ParenSelect) Format(buf *TrackedBuffer) { - buf.Myprintf("(%v)", node.Select) + buf.astPrintf(node, "(%v)", node.Select) } // Format formats the node. func (node *Union) Format(buf *TrackedBuffer) { - buf.Myprintf("%v %s %v%v%v%s", node.Left, node.Type, node.Right, + buf.astPrintf(node, "%v %s %v%v%v%s", node.Left, node.Type, node.Right, node.OrderBy, node.Limit, node.Lock) } // Format formats the node. func (node *Stream) Format(buf *TrackedBuffer) { - buf.Myprintf("stream %v%v from %v", + buf.astPrintf(node, "stream %v%v from %v", node.Comments, node.SelectExpr, node.Table) } // Format formats the node. func (node *Insert) Format(buf *TrackedBuffer) { - buf.Myprintf("%s %v%sinto %v%v%v %v%v", + buf.astPrintf(node, "%s %v%sinto %v%v%v %v%v", node.Action, node.Comments, node.Ignore, node.Table, node.Partitions, node.Columns, node.Rows, node.OnDup) @@ -843,26 +843,26 @@ func (node *Insert) Format(buf *TrackedBuffer) { // Format formats the node. func (node *Update) Format(buf *TrackedBuffer) { - buf.Myprintf("update %v%s%v set %v%v%v%v", + buf.astPrintf(node, "update %v%s%v set %v%v%v%v", node.Comments, node.Ignore, node.TableExprs, node.Exprs, node.Where, node.OrderBy, node.Limit) } // Format formats the node. func (node *Delete) Format(buf *TrackedBuffer) { - buf.Myprintf("delete %v", node.Comments) + buf.astPrintf(node, "delete %v", node.Comments) if node.Targets != nil { - buf.Myprintf("%v ", node.Targets) + buf.astPrintf(node, "%v ", node.Targets) } - buf.Myprintf("from %v%v%v%v%v", node.TableExprs, node.Partitions, node.Where, node.OrderBy, node.Limit) + buf.astPrintf(node, "from %v%v%v%v%v", node.TableExprs, node.Partitions, node.Where, node.OrderBy, node.Limit) } // Format formats the node. func (node *Set) Format(buf *TrackedBuffer) { if node.Scope == "" { - buf.Myprintf("set %v%v", node.Comments, node.Exprs) + buf.astPrintf(node, "set %v%v", node.Comments, node.Exprs) } else { - buf.Myprintf("set %v%s %v", node.Comments, node.Scope, node.Exprs) + buf.astPrintf(node, "set %v%s %v", node.Comments, node.Scope, node.Exprs) } } @@ -885,79 +885,79 @@ func (node *DDL) Format(buf *TrackedBuffer) { switch node.Action { case CreateStr: if node.OptLike != nil { - buf.Myprintf("%s table %v %v", node.Action, node.Table, node.OptLike) + buf.astPrintf(node, "%s table %v %v", node.Action, node.Table, node.OptLike) } else if node.TableSpec != nil { - buf.Myprintf("%s table %v %v", node.Action, node.Table, node.TableSpec) + buf.astPrintf(node, "%s table %v %v", node.Action, node.Table, node.TableSpec) } else { - buf.Myprintf("%s table %v", node.Action, node.Table) + buf.astPrintf(node, "%s table %v", node.Action, node.Table) } case DropStr: exists := "" if node.IfExists { exists = " if exists" } - buf.Myprintf("%s table%s %v", node.Action, exists, node.FromTables) + buf.astPrintf(node, "%s table%s %v", node.Action, exists, node.FromTables) case RenameStr: - buf.Myprintf("%s table %v to %v", node.Action, node.FromTables[0], node.ToTables[0]) + buf.astPrintf(node, "%s table %v to %v", node.Action, node.FromTables[0], node.ToTables[0]) for i := 1; i < len(node.FromTables); i++ { - buf.Myprintf(", %v to %v", node.FromTables[i], node.ToTables[i]) + buf.astPrintf(node, ", %v to %v", node.FromTables[i], node.ToTables[i]) } case AlterStr: if node.PartitionSpec != nil { - buf.Myprintf("%s table %v %v", node.Action, node.Table, node.PartitionSpec) + buf.astPrintf(node, "%s table %v %v", node.Action, node.Table, node.PartitionSpec) } else { - buf.Myprintf("%s table %v", node.Action, node.Table) + buf.astPrintf(node, "%s table %v", node.Action, node.Table) } case FlushStr: - buf.Myprintf("%s", node.Action) + buf.astPrintf(node, "%s", node.Action) case CreateVindexStr: - buf.Myprintf("alter vschema create vindex %v %v", node.Table, node.VindexSpec) + buf.astPrintf(node, "alter vschema create vindex %v %v", node.Table, node.VindexSpec) case DropVindexStr: - buf.Myprintf("alter vschema drop vindex %v", node.Table) + buf.astPrintf(node, "alter vschema drop vindex %v", node.Table) case AddVschemaTableStr: - buf.Myprintf("alter vschema add table %v", node.Table) + buf.astPrintf(node, "alter vschema add table %v", node.Table) case DropVschemaTableStr: - buf.Myprintf("alter vschema drop table %v", node.Table) + buf.astPrintf(node, "alter vschema drop table %v", node.Table) case AddColVindexStr: - buf.Myprintf("alter vschema on %v add vindex %v (", node.Table, node.VindexSpec.Name) + buf.astPrintf(node, "alter vschema on %v add vindex %v (", node.Table, node.VindexSpec.Name) for i, col := range node.VindexCols { if i != 0 { - buf.Myprintf(", %v", col) + buf.astPrintf(node, ", %v", col) } else { - buf.Myprintf("%v", col) + buf.astPrintf(node, "%v", col) } } - buf.Myprintf(")") + buf.astPrintf(node, ")") if node.VindexSpec.Type.String() != "" { - buf.Myprintf(" %v", node.VindexSpec) + buf.astPrintf(node, " %v", node.VindexSpec) } case DropColVindexStr: - buf.Myprintf("alter vschema on %v drop vindex %v", node.Table, node.VindexSpec.Name) + buf.astPrintf(node, "alter vschema on %v drop vindex %v", node.Table, node.VindexSpec.Name) case AddSequenceStr: - buf.Myprintf("alter vschema add sequence %v", node.Table) + buf.astPrintf(node, "alter vschema add sequence %v", node.Table) case AddAutoIncStr: - buf.Myprintf("alter vschema on %v add auto_increment %v", node.Table, node.AutoIncSpec) + buf.astPrintf(node, "alter vschema on %v add auto_increment %v", node.Table, node.AutoIncSpec) default: - buf.Myprintf("%s table %v", node.Action, node.Table) + buf.astPrintf(node, "%s table %v", node.Action, node.Table) } } // Format formats the node. func (node *OptLike) Format(buf *TrackedBuffer) { - buf.Myprintf("like %v", node.LikeTable) + buf.astPrintf(node, "like %v", node.LikeTable) } // Format formats the node. func (node *PartitionSpec) Format(buf *TrackedBuffer) { switch node.Action { case ReorganizeStr: - buf.Myprintf("%s %v into (", node.Action, node.Name) + buf.astPrintf(node, "%s %v into (", node.Action, node.Name) var prefix string for _, pd := range node.Definitions { - buf.Myprintf("%s%v", prefix, pd) + buf.astPrintf(node, "%s%v", prefix, pd) prefix = ", " } - buf.Myprintf(")") + buf.astPrintf(node, ")") default: panic("unimplemented") } @@ -966,50 +966,50 @@ func (node *PartitionSpec) Format(buf *TrackedBuffer) { // Format formats the node func (node *PartitionDefinition) Format(buf *TrackedBuffer) { if !node.Maxvalue { - buf.Myprintf("partition %v values less than (%v)", node.Name, node.Limit) + buf.astPrintf(node, "partition %v values less than (%v)", node.Name, node.Limit) } else { - buf.Myprintf("partition %v values less than (maxvalue)", node.Name) + buf.astPrintf(node, "partition %v values less than (maxvalue)", node.Name) } } // Format formats the node. func (ts *TableSpec) Format(buf *TrackedBuffer) { - buf.Myprintf("(\n") + buf.astPrintf(ts, "(\n") for i, col := range ts.Columns { if i == 0 { - buf.Myprintf("\t%v", col) + buf.astPrintf(ts, "\t%v", col) } else { - buf.Myprintf(",\n\t%v", col) + buf.astPrintf(ts, ",\n\t%v", col) } } for _, idx := range ts.Indexes { - buf.Myprintf(",\n\t%v", idx) + buf.astPrintf(ts, ",\n\t%v", idx) } for _, c := range ts.Constraints { - buf.Myprintf(",\n\t%v", c) + buf.astPrintf(ts, ",\n\t%v", c) } - buf.Myprintf("\n)%s", strings.Replace(ts.Options, ", ", ",\n ", -1)) + buf.astPrintf(ts, "\n)%s", strings.Replace(ts.Options, ", ", ",\n ", -1)) } // Format formats the node. func (col *ColumnDefinition) Format(buf *TrackedBuffer) { - buf.Myprintf("%v %v", col.Name, &col.Type) + buf.astPrintf(col, "%v %v", col.Name, &col.Type) } // Format returns a canonical string representation of the type and all relevant options func (ct *ColumnType) Format(buf *TrackedBuffer) { - buf.Myprintf("%s", ct.Type) + buf.astPrintf(ct, "%s", ct.Type) if ct.Length != nil && ct.Scale != nil { - buf.Myprintf("(%v,%v)", ct.Length, ct.Scale) + buf.astPrintf(ct, "(%v,%v)", ct.Length, ct.Scale) } else if ct.Length != nil { - buf.Myprintf("(%v)", ct.Length) + buf.astPrintf(ct, "(%v)", ct.Length) } if ct.EnumValues != nil { - buf.Myprintf("(%s)", strings.Join(ct.EnumValues, ", ")) + buf.astPrintf(ct, "(%s)", strings.Join(ct.EnumValues, ", ")) } opts := make([]string, 0, 16) @@ -1057,31 +1057,31 @@ func (ct *ColumnType) Format(buf *TrackedBuffer) { } if len(opts) != 0 { - buf.Myprintf(" %s", strings.Join(opts, " ")) + buf.astPrintf(ct, " %s", strings.Join(opts, " ")) } } // Format formats the node. func (idx *IndexDefinition) Format(buf *TrackedBuffer) { - buf.Myprintf("%v (", idx.Info) + buf.astPrintf(idx, "%v (", idx.Info) for i, col := range idx.Columns { if i != 0 { - buf.Myprintf(", %v", col.Column) + buf.astPrintf(idx, ", %v", col.Column) } else { - buf.Myprintf("%v", col.Column) + buf.astPrintf(idx, "%v", col.Column) } if col.Length != nil { - buf.Myprintf("(%v)", col.Length) + buf.astPrintf(idx, "(%v)", col.Length) } } - buf.Myprintf(")") + buf.astPrintf(idx, ")") for _, opt := range idx.Options { - buf.Myprintf(" %s", opt.Name) + buf.astPrintf(idx, " %s", opt.Name) if opt.Using != "" { - buf.Myprintf(" %s", opt.Using) + buf.astPrintf(idx, " %s", opt.Using) } else { - buf.Myprintf(" %v", opt.Value) + buf.astPrintf(idx, " %v", opt.Value) } } } @@ -1089,48 +1089,48 @@ func (idx *IndexDefinition) Format(buf *TrackedBuffer) { // Format formats the node. func (ii *IndexInfo) Format(buf *TrackedBuffer) { if ii.Primary { - buf.Myprintf("%s", ii.Type) + buf.astPrintf(ii, "%s", ii.Type) } else { - buf.Myprintf("%s", ii.Type) + buf.astPrintf(ii, "%s", ii.Type) if !ii.Name.IsEmpty() { - buf.Myprintf(" %v", ii.Name) + buf.astPrintf(ii, " %v", ii.Name) } } } // Format formats the node. func (node *AutoIncSpec) Format(buf *TrackedBuffer) { - buf.Myprintf("%v ", node.Column) - buf.Myprintf("using %v", node.Sequence) + buf.astPrintf(node, "%v ", node.Column) + buf.astPrintf(node, "using %v", node.Sequence) } // Format formats the node. The "CREATE VINDEX" preamble was formatted in // the containing DDL node Format, so this just prints the type, any // parameters, and optionally the owner func (node *VindexSpec) Format(buf *TrackedBuffer) { - buf.Myprintf("using %v", node.Type) + buf.astPrintf(node, "using %v", node.Type) numParams := len(node.Params) if numParams != 0 { - buf.Myprintf(" with ") + buf.astPrintf(node, " with ") for i, p := range node.Params { if i != 0 { - buf.Myprintf(", ") + buf.astPrintf(node, ", ") } - buf.Myprintf("%v", p) + buf.astPrintf(node, "%v", p) } } } // Format formats the node. func (node VindexParam) Format(buf *TrackedBuffer) { - buf.Myprintf("%s=%s", node.Key.String(), node.Val) + buf.astPrintf(node, "%s=%s", node.Key.String(), node.Val) } // Format formats the node. func (c *ConstraintDefinition) Format(buf *TrackedBuffer) { if c.Name != "" { - buf.Myprintf("constraint %s ", c.Name) + buf.astPrintf(c, "constraint %s ", c.Name) } c.Details.Format(buf) } @@ -1153,12 +1153,12 @@ func (a ReferenceAction) Format(buf *TrackedBuffer) { // Format formats the node. func (f *ForeignKeyDefinition) Format(buf *TrackedBuffer) { - buf.Myprintf("foreign key %v references %v %v", f.Source, f.ReferencedTable, f.ReferencedColumns) + buf.astPrintf(f, "foreign key %v references %v %v", f.Source, f.ReferencedTable, f.ReferencedColumns) if f.OnDelete != DefaultAction { - buf.Myprintf(" on delete %v", f.OnDelete) + buf.astPrintf(f, " on delete %v", f.OnDelete) } if f.OnUpdate != DefaultAction { - buf.Myprintf(" on update %v", f.OnUpdate) + buf.astPrintf(f, " on update %v", f.OnUpdate) } } @@ -1167,32 +1167,32 @@ func (node *Show) Format(buf *TrackedBuffer) { nodeType := strings.ToLower(node.Type) if (nodeType == "tables" || nodeType == "columns" || nodeType == "fields" || nodeType == "index" || nodeType == "keys") && node.ShowTablesOpt != nil { opt := node.ShowTablesOpt - buf.Myprintf("show %s%s", opt.Full, nodeType) + buf.astPrintf(node, "show %s%s", opt.Full, nodeType) if (nodeType == "columns" || nodeType == "fields" || nodeType == "index" || nodeType == "keys") && node.HasOnTable() { - buf.Myprintf(" from %v", node.OnTable) + buf.astPrintf(node, " from %v", node.OnTable) } if opt.DbName != "" { - buf.Myprintf(" from %s", opt.DbName) + buf.astPrintf(node, " from %s", opt.DbName) } - buf.Myprintf("%v", opt.Filter) + buf.astPrintf(node, "%v", opt.Filter) return } if node.Scope == "" { - buf.Myprintf("show %s", nodeType) + buf.astPrintf(node, "show %s", nodeType) } else { - buf.Myprintf("show %s %s", node.Scope, nodeType) + buf.astPrintf(node, "show %s %s", node.Scope, nodeType) } if node.HasOnTable() { - buf.Myprintf(" on %v", node.OnTable) + buf.astPrintf(node, " on %v", node.OnTable) } if nodeType == "collation" && node.ShowCollationFilterOpt != nil { - buf.Myprintf(" where %v", *node.ShowCollationFilterOpt) + buf.astPrintf(node, " where %v", *node.ShowCollationFilterOpt) } if nodeType == "charset" && node.ShowTablesOpt != nil { - buf.Myprintf("%v", node.ShowTablesOpt.Filter) + buf.astPrintf(node, "%v", node.ShowTablesOpt.Filter) } if node.HasTable() { - buf.Myprintf(" %v", node.Table) + buf.astPrintf(node, " %v", node.Table) } } @@ -1202,18 +1202,18 @@ func (node *ShowFilter) Format(buf *TrackedBuffer) { return } if node.Like != "" { - buf.Myprintf(" like '%s'", node.Like) + buf.astPrintf(node, " like '%s'", node.Like) } else { - buf.Myprintf(" where %v", node.Filter) + buf.astPrintf(node, " where %v", node.Filter) } } // Format formats the node. func (node *Use) Format(buf *TrackedBuffer) { if node.DBName.v != "" { - buf.Myprintf("use %v", node.DBName) + buf.astPrintf(node, "use %v", node.DBName) } else { - buf.Myprintf("use") + buf.astPrintf(node, "use") } } @@ -1245,7 +1245,7 @@ func (node *OtherAdmin) Format(buf *TrackedBuffer) { // Format formats the node. func (node Comments) Format(buf *TrackedBuffer) { for _, c := range node { - buf.Myprintf("%s ", c) + buf.astPrintf(node, "%s ", c) } } @@ -1253,7 +1253,7 @@ func (node Comments) Format(buf *TrackedBuffer) { func (node SelectExprs) Format(buf *TrackedBuffer) { var prefix string for _, n := range node { - buf.Myprintf("%s%v", prefix, n) + buf.astPrintf(node, "%s%v", prefix, n) prefix = ", " } } @@ -1261,22 +1261,22 @@ func (node SelectExprs) Format(buf *TrackedBuffer) { // Format formats the node. func (node *StarExpr) Format(buf *TrackedBuffer) { if !node.TableName.IsEmpty() { - buf.Myprintf("%v.", node.TableName) + buf.astPrintf(node, "%v.", node.TableName) } - buf.Myprintf("*") + buf.astPrintf(node, "*") } // Format formats the node. func (node *AliasedExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("%v", node.Expr) + buf.astPrintf(node, "%v", node.Expr) if !node.As.IsEmpty() { - buf.Myprintf(" as %v", node.As) + buf.astPrintf(node, " as %v", node.As) } } // Format formats the node. func (node Nextval) Format(buf *TrackedBuffer) { - buf.Myprintf("next %v values", node.Expr) + buf.astPrintf(node, "next %v values", node.Expr) } // Format formats the node. @@ -1286,7 +1286,7 @@ func (node Columns) Format(buf *TrackedBuffer) { } prefix := "(" for _, n := range node { - buf.Myprintf("%s%v", prefix, n) + buf.astPrintf(node, "%s%v", prefix, n) prefix = ", " } buf.WriteString(")") @@ -1299,7 +1299,7 @@ func (node Partitions) Format(buf *TrackedBuffer) { } prefix := " partition (" for _, n := range node { - buf.Myprintf("%s%v", prefix, n) + buf.astPrintf(node, "%s%v", prefix, n) prefix = ", " } buf.WriteString(")") @@ -1309,20 +1309,20 @@ func (node Partitions) Format(buf *TrackedBuffer) { func (node TableExprs) Format(buf *TrackedBuffer) { var prefix string for _, n := range node { - buf.Myprintf("%s%v", prefix, n) + buf.astPrintf(node, "%s%v", prefix, n) prefix = ", " } } // Format formats the node. func (node *AliasedTableExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("%v%v", node.Expr, node.Partitions) + buf.astPrintf(node, "%v%v", node.Expr, node.Partitions) if !node.As.IsEmpty() { - buf.Myprintf(" as %v", node.As) + buf.astPrintf(node, " as %v", node.As) } if node.Hints != nil { // Hint node provides the space padding. - buf.Myprintf("%v", node.Hints) + buf.astPrintf(node, "%v", node.Hints) } } @@ -1330,7 +1330,7 @@ func (node *AliasedTableExpr) Format(buf *TrackedBuffer) { func (node TableNames) Format(buf *TrackedBuffer) { var prefix string for _, n := range node { - buf.Myprintf("%s%v", prefix, n) + buf.astPrintf(node, "%s%v", prefix, n) prefix = ", " } } @@ -1341,43 +1341,43 @@ func (node TableName) Format(buf *TrackedBuffer) { return } if !node.Qualifier.IsEmpty() { - buf.Myprintf("%v.", node.Qualifier) + buf.astPrintf(node, "%v.", node.Qualifier) } - buf.Myprintf("%v", node.Name) + buf.astPrintf(node, "%v", node.Name) } // Format formats the node. func (node *ParenTableExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("(%v)", node.Exprs) + buf.astPrintf(node, "(%v)", node.Exprs) } // Format formats the node. func (node JoinCondition) Format(buf *TrackedBuffer) { if node.On != nil { - buf.Myprintf(" on %v", node.On) + buf.astPrintf(node, " on %v", node.On) } if node.Using != nil { - buf.Myprintf(" using %v", node.Using) + buf.astPrintf(node, " using %v", node.Using) } } // Format formats the node. func (node *JoinTableExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("%v %s %v%v", node.LeftExpr, node.Join, node.RightExpr, node.Condition) + buf.astPrintf(node, "%v %s %v%v", node.LeftExpr, node.Join, node.RightExpr, node.Condition) } // Format formats the node. func (node *IndexHints) Format(buf *TrackedBuffer) { - buf.Myprintf(" %sindex ", node.Type) + buf.astPrintf(node, " %sindex ", node.Type) if len(node.Indexes) == 0 { - buf.Myprintf("()") + buf.astPrintf(node, "()") } else { prefix := "(" for _, n := range node.Indexes { - buf.Myprintf("%s%v", prefix, n) + buf.astPrintf(node, "%s%v", prefix, n) prefix = ", " } - buf.Myprintf(")") + buf.astPrintf(node, ")") } } @@ -1386,66 +1386,54 @@ func (node *Where) Format(buf *TrackedBuffer) { if node == nil || node.Expr == nil { return } - buf.Myprintf(" %s %v", node.Type, node.Expr) + buf.astPrintf(node, " %s %v", node.Type, node.Expr) } // Format formats the node. func (node Exprs) Format(buf *TrackedBuffer) { var prefix string for _, n := range node { - buf.Myprintf("%s%v", prefix, n) + buf.astPrintf(node, "%s%v", prefix, n) prefix = ", " } } // Format formats the node. func (node *AndExpr) Format(buf *TrackedBuffer) { - printWithParensIfNeeded(node, node.Left, buf) - buf.WriteString(" and ") - printWithParensIfNeeded(node, node.Right, buf) + buf.astPrintf(node, "%v and %v", node.Left, node.Right) } // Format formats the node. func (node *OrExpr) Format(buf *TrackedBuffer) { - printWithParensIfNeeded(node, node.Left, buf) - buf.WriteString(" or ") - printWithParensIfNeeded(node, node.Right, buf) + buf.astPrintf(node, "%v or %v", node.Left, node.Right) } // Format formats the node. func (node *NotExpr) Format(buf *TrackedBuffer) { - buf.WriteString("not ") - printWithParensIfNeeded(node, node.Expr, buf) + buf.astPrintf(node, "not %v", node.Expr) } // Format formats the node. func (node *ComparisonExpr) Format(buf *TrackedBuffer) { - printWithParensIfNeeded(node, node.Left, buf) - buf.Myprintf(" %s ", node.Operator) - printWithParensIfNeeded(node, node.Right, buf) - + buf.astPrintf(node, "%v %s %v", node.Left, node.Operator, node.Right) if node.Escape != nil { - buf.Myprintf(" escape %v", node.Escape) + buf.astPrintf(node, " escape %v", node.Escape) } } // Format formats the node. func (node *RangeCond) Format(buf *TrackedBuffer) { - printWithParensIfNeeded(node, node.Expr, buf) - buf.Myprintf(" %s ", node.Operator) - printWithParensIfNeeded(node, node.From, buf) - buf.WriteString(" and ") - printWithParensIfNeeded(node, node.To, buf) + buf.astPrintf(node, "%v %s %v and %v", node.Left, node.Operator, node.From, node.To) } // Format formats the node. func (node *IsExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("%v %s", node.Expr, node.Operator) + buf.astPrintf(node, "%v %s", node.Expr, node.Operator) } // Format formats the node. func (node *ExistsExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("exists %v", node.Subquery) + buf.astPrintf(node, "exists %v", node.Subquery) } // Format formats the node. @@ -1454,11 +1442,11 @@ func (node *SQLVal) Format(buf *TrackedBuffer) { case StrVal: sqltypes.MakeTrusted(sqltypes.VarBinary, node.Val).EncodeSQL(buf) case IntVal, FloatVal, HexNum: - buf.Myprintf("%s", []byte(node.Val)) + buf.astPrintf(node, "%s", node.Val) case HexVal: - buf.Myprintf("X'%s'", []byte(node.Val)) + buf.astPrintf(node, "X'%s'", node.Val) case BitVal: - buf.Myprintf("B'%s'", []byte(node.Val)) + buf.astPrintf(node, "B'%s'", node.Val) case ValArg: buf.WriteArg(string(node.Val)) default: @@ -1468,34 +1456,34 @@ func (node *SQLVal) Format(buf *TrackedBuffer) { // Format formats the node. func (node *NullVal) Format(buf *TrackedBuffer) { - buf.Myprintf("null") + buf.astPrintf(node, "null") } // Format formats the node. func (node BoolVal) Format(buf *TrackedBuffer) { if node { - buf.Myprintf("true") + buf.astPrintf(node, "true") } else { - buf.Myprintf("false") + buf.astPrintf(node, "false") } } // Format formats the node. func (node *ColName) Format(buf *TrackedBuffer) { if !node.Qualifier.IsEmpty() { - buf.Myprintf("%v.", node.Qualifier) + buf.astPrintf(node, "%v.", node.Qualifier) } - buf.Myprintf("%v", node.Name) + buf.astPrintf(node, "%v", node.Name) } // Format formats the node. func (node ValTuple) Format(buf *TrackedBuffer) { - buf.Myprintf("(%v)", Exprs(node)) + buf.astPrintf(node, "(%v)", Exprs(node)) } // Format formats the node. func (node *Subquery) Format(buf *TrackedBuffer) { - buf.Myprintf("(%v)", node.Select) + buf.astPrintf(node, "(%v)", node.Select) } // Format formats the node. @@ -1505,42 +1493,37 @@ func (node ListArg) Format(buf *TrackedBuffer) { // Format formats the node. func (node *BinaryExpr) Format(buf *TrackedBuffer) { - printWithParensIfNeeded(node, node.Left, buf) - buf.Myprintf(" %s ", node.Operator) - printWithParensIfNeeded(node, node.Right, buf) + buf.astPrintf(node, "%v %s %v", node.Left, node.Operator, node.Right) } // Format formats the node. func (node *UnaryExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("%s", node.Operator) if _, unary := node.Expr.(*UnaryExpr); unary { - buf.Myprintf(" %v", node.Expr) + // They have same precedence so parenthesis is not required. + buf.astPrintf(node, "%s %v", node.Operator, node.Expr) return } - - printWithParensIfNeeded(node, node.Expr, buf) + buf.astPrintf(node, "%s%v", node.Operator, node.Expr) } // Format formats the node. func (node *IntervalExpr) Format(buf *TrackedBuffer) { - buf.WriteString("interval ") - printWithParensIfNeeded(node, node.Expr, buf) - buf.Myprintf(" %s", node.Unit) + buf.astPrintf(node, "interval %v %s", node.Expr, node.Unit) } // Format formats the node. func (node *TimestampFuncExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("%s(%s, %v, %v)", node.Name, node.Unit, node.Expr1, node.Expr2) + buf.astPrintf(node, "%s(%s, %v, %v)", node.Name, node.Unit, node.Expr1, node.Expr2) } // Format formats the node. func (node *CurTimeFuncExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("%s(%v)", node.Name.String(), node.Fsp) + buf.astPrintf(node, "%s(%v)", node.Name.String(), node.Fsp) } // Format formats the node. func (node *CollateExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("%v collate %s", node.Expr, node.Charset) + buf.astPrintf(node, "%v collate %s", node.Expr, node.Charset) } // Format formats the node. @@ -1550,7 +1533,7 @@ func (node *FuncExpr) Format(buf *TrackedBuffer) { distinct = "distinct " } if !node.Qualifier.IsEmpty() { - buf.Myprintf("%v.", node.Qualifier) + buf.astPrintf(node, "%v.", node.Qualifier) } // Function names should not be back-quoted even // if they match a reserved word, only if they contain illegal characters @@ -1561,17 +1544,17 @@ func (node *FuncExpr) Format(buf *TrackedBuffer) { } else { buf.WriteString(funcName) } - buf.Myprintf("(%s%v)", distinct, node.Exprs) + buf.astPrintf(node, "(%s%v)", distinct, node.Exprs) } // Format formats the node func (node *GroupConcatExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("group_concat(%s%v%v%s%v)", node.Distinct, node.Exprs, node.OrderBy, node.Separator, node.Limit) + buf.astPrintf(node, "group_concat(%s%v%v%s%v)", node.Distinct, node.Exprs, node.OrderBy, node.Separator, node.Limit) } // Format formats the node. func (node *ValuesFuncExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("values(%v)", node.Name) + buf.astPrintf(node, "values(%v)", node.Name) } // Format formats the node. @@ -1584,75 +1567,75 @@ func (node *SubstrExpr) Format(buf *TrackedBuffer) { } if node.To == nil { - buf.Myprintf("substr(%v, %v)", val, node.From) + buf.astPrintf(node, "substr(%v, %v)", val, node.From) } else { - buf.Myprintf("substr(%v, %v, %v)", val, node.From, node.To) + buf.astPrintf(node, "substr(%v, %v, %v)", val, node.From, node.To) } } // Format formats the node. func (node *ConvertExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("convert(%v, %v)", node.Expr, node.Type) + buf.astPrintf(node, "convert(%v, %v)", node.Expr, node.Type) } // Format formats the node. func (node *ConvertUsingExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("convert(%v using %s)", node.Expr, node.Type) + buf.astPrintf(node, "convert(%v using %s)", node.Expr, node.Type) } // Format formats the node. func (node *ConvertType) Format(buf *TrackedBuffer) { - buf.Myprintf("%s", node.Type) + buf.astPrintf(node, "%s", node.Type) if node.Length != nil { - buf.Myprintf("(%v", node.Length) + buf.astPrintf(node, "(%v", node.Length) if node.Scale != nil { - buf.Myprintf(", %v", node.Scale) + buf.astPrintf(node, ", %v", node.Scale) } - buf.Myprintf(")") + buf.astPrintf(node, ")") } if node.Charset != "" { - buf.Myprintf("%s %s", node.Operator, node.Charset) + buf.astPrintf(node, "%s %s", node.Operator, node.Charset) } } // Format formats the node func (node *MatchExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("match(%v) against (%v%s)", node.Columns, node.Expr, node.Option) + buf.astPrintf(node, "match(%v) against (%v%s)", node.Columns, node.Expr, node.Option) } // Format formats the node. func (node *CaseExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("case ") + buf.astPrintf(node, "case ") if node.Expr != nil { - buf.Myprintf("%v ", node.Expr) + buf.astPrintf(node, "%v ", node.Expr) } for _, when := range node.Whens { - buf.Myprintf("%v ", when) + buf.astPrintf(node, "%v ", when) } if node.Else != nil { - buf.Myprintf("else %v ", node.Else) + buf.astPrintf(node, "else %v ", node.Else) } - buf.Myprintf("end") + buf.astPrintf(node, "end") } // Format formats the node. func (node *Default) Format(buf *TrackedBuffer) { - buf.Myprintf("default") + buf.astPrintf(node, "default") if node.ColName != "" { - buf.Myprintf("(%s)", node.ColName) + buf.astPrintf(node, "(%s)", node.ColName) } } // Format formats the node. func (node *When) Format(buf *TrackedBuffer) { - buf.Myprintf("when %v then %v", node.Cond, node.Val) + buf.astPrintf(node, "when %v then %v", node.Cond, node.Val) } // Format formats the node. func (node GroupBy) Format(buf *TrackedBuffer) { prefix := " group by " for _, n := range node { - buf.Myprintf("%s%v", prefix, n) + buf.astPrintf(node, "%s%v", prefix, n) prefix = ", " } } @@ -1661,7 +1644,7 @@ func (node GroupBy) Format(buf *TrackedBuffer) { func (node OrderBy) Format(buf *TrackedBuffer) { prefix := " order by " for _, n := range node { - buf.Myprintf("%s%v", prefix, n) + buf.astPrintf(node, "%s%v", prefix, n) prefix = ", " } } @@ -1669,17 +1652,17 @@ func (node OrderBy) Format(buf *TrackedBuffer) { // Format formats the node. func (node *Order) Format(buf *TrackedBuffer) { if node, ok := node.Expr.(*NullVal); ok { - buf.Myprintf("%v", node) + buf.astPrintf(node, "%v", node) return } if node, ok := node.Expr.(*FuncExpr); ok { if node.Name.Lowered() == "rand" { - buf.Myprintf("%v", node) + buf.astPrintf(node, "%v", node) return } } - buf.Myprintf("%v %s", node.Expr, node.Direction) + buf.astPrintf(node, "%v %s", node.Expr, node.Direction) } // Format formats the node. @@ -1687,18 +1670,18 @@ func (node *Limit) Format(buf *TrackedBuffer) { if node == nil { return } - buf.Myprintf(" limit ") + buf.astPrintf(node, " limit ") if node.Offset != nil { - buf.Myprintf("%v, ", node.Offset) + buf.astPrintf(node, "%v, ", node.Offset) } - buf.Myprintf("%v", node.Rowcount) + buf.astPrintf(node, "%v", node.Rowcount) } // Format formats the node. func (node Values) Format(buf *TrackedBuffer) { prefix := "values " for _, n := range node { - buf.Myprintf("%s%v", prefix, n) + buf.astPrintf(node, "%s%v", prefix, n) prefix = ", " } } @@ -1707,21 +1690,21 @@ func (node Values) Format(buf *TrackedBuffer) { func (node UpdateExprs) Format(buf *TrackedBuffer) { var prefix string for _, n := range node { - buf.Myprintf("%s%v", prefix, n) + buf.astPrintf(node, "%s%v", prefix, n) prefix = ", " } } // Format formats the node. func (node *UpdateExpr) Format(buf *TrackedBuffer) { - buf.Myprintf("%v = %v", node.Name, node.Expr) + buf.astPrintf(node, "%v = %v", node.Name, node.Expr) } // Format formats the node. func (node SetExprs) Format(buf *TrackedBuffer) { var prefix string for _, n := range node { - buf.Myprintf("%s%v", prefix, n) + buf.astPrintf(node, "%s%v", prefix, n) prefix = ", " } } @@ -1730,12 +1713,12 @@ func (node SetExprs) Format(buf *TrackedBuffer) { func (node *SetExpr) Format(buf *TrackedBuffer) { // We don't have to backtick set variable names. if node.Name.EqualString("charset") || node.Name.EqualString("names") { - buf.Myprintf("%s %v", node.Name.String(), node.Expr) + buf.astPrintf(node, "%s %v", node.Name.String(), node.Expr) } else if node.Name.EqualString(TransactionStr) { sqlVal := node.Expr.(*SQLVal) - buf.Myprintf("%s %s", node.Name.String(), strings.ToLower(string(sqlVal.Val))) + buf.astPrintf(node, "%s %s", node.Name.String(), strings.ToLower(string(sqlVal.Val))) } else { - buf.Myprintf("%v = %v", node.Name, node.Expr) + buf.astPrintf(node, "%v = %v", node.Name, node.Expr) } } @@ -1744,7 +1727,7 @@ func (node OnDup) Format(buf *TrackedBuffer) { if node == nil { return } - buf.Myprintf(" on duplicate key update %v", UpdateExprs(node)) + buf.astPrintf(node, " on duplicate key update %v", UpdateExprs(node)) } // Format formats the node. diff --git a/go/vt/sqlparser/ast_funcs.go b/go/vt/sqlparser/ast_funcs.go index 97782cd6ce9..dada62f3125 100644 --- a/go/vt/sqlparser/ast_funcs.go +++ b/go/vt/sqlparser/ast_funcs.go @@ -755,20 +755,6 @@ func (node *Union) SetLimit(limit *Limit) { node.Limit = limit } -func needParens(op, val Expr) bool { - opBinding := precedenceFor(op) - valBinding := precedenceFor(val) - return !(opBinding == Syntactic || valBinding == Syntactic) && valBinding > opBinding -} - -func printWithParensIfNeeded(op, val Expr, buf *TrackedBuffer) { - if needParens(op, val) { - buf.WriteString("(") - defer buf.WriteString(")") - } - buf.Myprintf("%v", val) -} - type atCount int const ( diff --git a/go/vt/sqlparser/precedence.go b/go/vt/sqlparser/precedence.go index 63a496b7ddf..25f3838fc71 100644 --- a/go/vt/sqlparser/precedence.go +++ b/go/vt/sqlparser/precedence.go @@ -41,6 +41,10 @@ const ( P17 ) +// precedenceFor returns the precedence of an expression. +// +// * NOTE: If you change anything here, update sql.y to keep them consistent. +// Also make sure to add the new constructs to random_expr.go so we have test coverage for the new expressions * func precedenceFor(in Expr) Precendence { switch node := in.(type) { case *OrExpr: diff --git a/go/vt/sqlparser/rewriter.go b/go/vt/sqlparser/rewriter.go index 490c3c876eb..bb5e386b09c 100644 --- a/go/vt/sqlparser/rewriter.go +++ b/go/vt/sqlparser/rewriter.go @@ -499,14 +499,14 @@ func (r *replacePartitionsItems) inc() { *r++ } -func replaceRangeCondExpr(newNode, parent SQLNode) { - parent.(*RangeCond).Expr = newNode.(Expr) -} - func replaceRangeCondFrom(newNode, parent SQLNode) { parent.(*RangeCond).From = newNode.(Expr) } +func replaceRangeCondLeft(newNode, parent SQLNode) { + parent.(*RangeCond).Left = newNode.(Expr) +} + func replaceRangeCondTo(newNode, parent SQLNode) { parent.(*RangeCond).To = newNode.(Expr) } @@ -1114,8 +1114,8 @@ func (a *application) apply(parent, node SQLNode, replacer replacerFunc) { } case *RangeCond: - a.apply(node, n.Expr, replaceRangeCondExpr) a.apply(node, n.From, replaceRangeCondFrom) + a.apply(node, n.Left, replaceRangeCondLeft) a.apply(node, n.To, replaceRangeCondTo) case ReferenceAction: diff --git a/go/vt/sqlparser/sql.go b/go/vt/sqlparser/sql.go index 452e79bf262..9ce4bbe4f39 100644 --- a/go/vt/sqlparser/sql.go +++ b/go/vt/sqlparser/sql.go @@ -6037,13 +6037,13 @@ yydefault: yyDollar = yyS[yypt-5 : yypt+1] //line sql.y:2258 { - yyVAL.expr = &RangeCond{Expr: yyDollar[1].expr, Operator: BetweenStr, From: yyDollar[3].expr, To: yyDollar[5].expr} + yyVAL.expr = &RangeCond{Left: yyDollar[1].expr, Operator: BetweenStr, From: yyDollar[3].expr, To: yyDollar[5].expr} } case 426: yyDollar = yyS[yypt-6 : yypt+1] //line sql.y:2262 { - yyVAL.expr = &RangeCond{Expr: yyDollar[1].expr, Operator: NotBetweenStr, From: yyDollar[4].expr, To: yyDollar[6].expr} + yyVAL.expr = &RangeCond{Left: yyDollar[1].expr, Operator: NotBetweenStr, From: yyDollar[4].expr, To: yyDollar[6].expr} } case 427: yyDollar = yyS[yypt-2 : yypt+1] diff --git a/go/vt/sqlparser/sql.y b/go/vt/sqlparser/sql.y index 683dbf01cb9..e71f3742f7a 100644 --- a/go/vt/sqlparser/sql.y +++ b/go/vt/sqlparser/sql.y @@ -135,6 +135,7 @@ func skipToEnd(yylex interface{}) { // Some of these operators don't conflict in our situation. Nevertheless, // it's better to have these listed in the correct order. Also, we don't // support all operators yet. +// * NOTE: If you change anything here, update precedence.go as well * %left OR %left AND %right NOT '!' @@ -2256,11 +2257,11 @@ condition: } | value_expression BETWEEN value_expression AND value_expression { - $$ = &RangeCond{Expr: $1, Operator: BetweenStr, From: $3, To: $5} + $$ = &RangeCond{Left: $1, Operator: BetweenStr, From: $3, To: $5} } | value_expression NOT BETWEEN value_expression AND value_expression { - $$ = &RangeCond{Expr: $1, Operator: NotBetweenStr, From: $4, To: $6} + $$ = &RangeCond{Left: $1, Operator: NotBetweenStr, From: $4, To: $6} } | EXISTS subquery { diff --git a/go/vt/sqlparser/tracked_buffer.go b/go/vt/sqlparser/tracked_buffer.go index 07fb5e62df7..de366d47564 100644 --- a/go/vt/sqlparser/tracked_buffer.go +++ b/go/vt/sqlparser/tracked_buffer.go @@ -56,10 +56,23 @@ func (buf *TrackedBuffer) WriteNode(node SQLNode) *TrackedBuffer { // Myprintf mimics fmt.Fprintf(buf, ...), but limited to Node(%v), // Node.Value(%s) and string(%s). It also allows a %a for a value argument, in // which case it adds tracking info for future substitutions. +// It adds parens as needed to follow precedence rules when printing expressions // // The name must be something other than the usual Printf() to avoid "go vet" // warnings due to our custom format specifiers. +// *** THIS METHOD SHOULD NOT BE USED FROM ast.go. USE astPrintf INSTEAD *** func (buf *TrackedBuffer) Myprintf(format string, values ...interface{}) { + buf.astPrintf(nil, format, values...) +} + +// astPrintf is for internal use by the ast structs +func (buf *TrackedBuffer) astPrintf(currentNode SQLNode, format string, values ...interface{}) { + currentExpr, checkParens := currentNode.(Expr) + if checkParens { + // expressions that have Precedence Syntactic will never need parens + checkParens = precedenceFor(currentExpr) != Syntactic + } + end := len(format) fieldnum := 0 for i := 0; i < end; { @@ -94,12 +107,18 @@ func (buf *TrackedBuffer) Myprintf(format string, values ...interface{}) { panic(fmt.Sprintf("unexpected TrackedBuffer type %T", v)) } case 'v': - node := values[fieldnum].(SQLNode) - if buf.nodeFormatter == nil { - node.Format(buf) + value := values[fieldnum] + expr := getExpressionForParensEval(checkParens, value) + + if expr != nil { // + needParens := needParens(currentExpr, expr) + buf.printIf(needParens, "(") + buf.formatter(expr) + buf.printIf(needParens, ")") } else { - buf.nodeFormatter(buf, node) + buf.formatter(value.(SQLNode)) } + case 'a': buf.WriteArg(values[fieldnum].(string)) default: @@ -110,10 +129,34 @@ func (buf *TrackedBuffer) Myprintf(format string, values ...interface{}) { } } -// WriteString appends the contents of s to b's buffer. -// It returns the length of s and a nil error. -func (buf *TrackedBuffer) WriteString(s string) (int, error) { - return buf.Builder.WriteString(s) +func getExpressionForParensEval(checkParens bool, value interface{}) Expr { + if checkParens { + expr, isExpr := value.(Expr) + if isExpr { + return expr + } + } + return nil +} + +func (buf *TrackedBuffer) printIf(condition bool, text string) { + if condition { + buf.WriteString(text) + } +} + +func (buf *TrackedBuffer) formatter(node SQLNode) { + if buf.nodeFormatter == nil { + node.Format(buf) + } else { + buf.nodeFormatter(buf, node) + } +} + +func needParens(op, val Expr) bool { + opBinding := precedenceFor(op) + valBinding := precedenceFor(val) + return !(opBinding == Syntactic || valBinding == Syntactic) && valBinding > opBinding } // WriteArg writes a value argument into the buffer along with From 77cbf276592024506b4963eb0ba54e6929374009 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Wed, 18 Mar 2020 13:37:09 +0100 Subject: [PATCH 68/85] Added randomized testing to test parsing/unparsing Signed-off-by: Andres Taylor --- go/vt/sqlparser/precedence_test.go | 31 ++++ go/vt/sqlparser/random_expr.go | 241 +++++++++++++++++++++++++++++ 2 files changed, 272 insertions(+) create mode 100644 go/vt/sqlparser/random_expr.go diff --git a/go/vt/sqlparser/precedence_test.go b/go/vt/sqlparser/precedence_test.go index 38fcf1e5fa3..18a100f4568 100644 --- a/go/vt/sqlparser/precedence_test.go +++ b/go/vt/sqlparser/precedence_test.go @@ -18,7 +18,9 @@ package sqlparser import ( "fmt" + "math/rand" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -149,3 +151,32 @@ func TestParens(t *testing.T) { }) } } + +func TestRandom(t *testing.T) { + // The purpose of this test is to find discrepancies between Format and parsing. If for example our precedence rules are not consistent between the two, this test should find it. + // The idea is to generate random queries, and pass them through the parser and then the unparser, and one more time. The result of the first unparse should be the same as the second result. + seed := time.Now().UnixNano() + fmt.Println(fmt.Sprintf("seed is %d", seed)) + g := generator{ + seed: seed, + r: rand.New(rand.NewSource(seed)), + } + endBy := time.Now().Add(5 * time.Second) + + for { + if time.Now().After(endBy) { + break + } + // Given a random expression + randomExpr := g.expression() + inputQ := "select " + String(randomExpr) + " from t" + + // When it's parsed and unparsed + parsedInput, err := Parse(inputQ) + require.NoError(t, err) + + // Then the unparsing should be the same as the input query + outputOfParseResult := String(parsedInput) + require.Equal(t, outputOfParseResult, inputQ) + } +} diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go new file mode 100644 index 00000000000..169bac00c2e --- /dev/null +++ b/go/vt/sqlparser/random_expr.go @@ -0,0 +1,241 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sqlparser + +import ( + "fmt" + "math/rand" +) + +// This file is used to generate random expressions to be used for testing + +type generator struct { + seed int64 + r *rand.Rand + depth int +} + +// enter should be called whenever we are producing an intermediate node. it should be followed by a `defer g.exit()` +func (g *generator) enter() { + g.depth++ +} + +// exit should be called when exiting an intermediate node +func (g *generator) exit() { + g.depth-- +} + +// atMaxDepth returns true if we have reached the maximum allowed depth or the expression tree +func (g *generator) atMaxDepth() bool { + return g.depth >= 3 +} + +func (g *generator) expression() Expr { + if g.randomBool() { + return g.booleanExpr() + } + + return g.intExpr() +} + +func (g *generator) booleanExpr() Expr { + if g.atMaxDepth() { + return g.booleanLiteral() + } + + options := []exprF{ + func() Expr { return g.andExpr() }, + func() Expr { return g.orExpr() }, + func() Expr { return g.booleanLiteral() }, + func() Expr { return g.comparison() }, + func() Expr { return g.inExpr() }, + func() Expr { return g.between() }, + func() Expr { return g.isExpr() }, + } + + return g.randomOf(options) +} + +func (g *generator) intExpr() Expr { + if g.atMaxDepth() { + return g.intLiteral() + } + + options := []exprF{ + func() Expr { return g.arithmetic() }, + func() Expr { return g.intLiteral() }, + func() Expr { return g.caseExpr() }, + } + + return g.randomOf(options) +} + +func (g *generator) booleanLiteral() Expr { + return BoolVal(g.randomBool()) +} + +func (g *generator) randomBool() bool { + return g.r.Float32() < 0.5 +} + +func (g *generator) intLiteral() Expr { + t := fmt.Sprintf("%d", g.r.Intn(1000)-g.r.Intn((1000))) + + return NewIntVal([]byte(t)) +} + +var comparisonOps = []string{"=", ">", "<", ">=", "<=", "<=>", "!="} + +func (g *generator) comparison() Expr { + g.enter() + defer g.exit() + + v := g.r.Intn(len(comparisonOps)) + + cmp := &ComparisonExpr{ + Operator: comparisonOps[v], + Left: g.intExpr(), + Right: g.intExpr(), + } + + return cmp +} + +func (g *generator) caseExpr() Expr { + g.enter() + defer g.exit() + + var exp Expr + var elseExpr Expr + if g.randomBool() { + exp = g.intExpr() + } + if g.randomBool() { + elseExpr = g.intExpr() + } + + size := g.r.Intn(5) + 2 + var whens []*When + for i := 0; i < size; i++ { + var cond Expr + if exp == nil { + cond = g.booleanExpr() + } else { + cond = g.expression() + } + + whens = append(whens, &When{ + Cond: cond, + Val: g.expression(), + }) + } + + return &CaseExpr{ + Expr: exp, + Whens: whens, + Else: elseExpr, + } +} + +var arithmeticOps = []string{BitAndStr, BitOrStr, BitXorStr, PlusStr, MinusStr, MultStr, DivStr, IntDivStr, ModStr, ShiftRightStr, ShiftLeftStr} + +func (g *generator) arithmetic() Expr { + g.enter() + defer g.exit() + + op := arithmeticOps[g.r.Intn(len(arithmeticOps))] + + return &BinaryExpr{ + Operator: op, + Left: g.intExpr(), + Right: g.intExpr(), + } +} + +type exprF func() Expr + +func (g *generator) randomOf(options []exprF) Expr { + return options[g.r.Intn(len(options))]() +} + +func (g *generator) randomOfS(options []string) string { + return options[g.r.Intn(len(options))] +} + +func (g *generator) andExpr() Expr { + g.enter() + defer g.exit() + return &AndExpr{ + Left: g.booleanExpr(), + Right: g.booleanExpr(), + } +} + +func (g *generator) orExpr() Expr { + g.enter() + defer g.exit() + return &OrExpr{ + Left: g.booleanExpr(), + Right: g.booleanExpr(), + } +} + +func (g *generator) inExpr() Expr { + g.enter() + defer g.exit() + + expr := g.intExpr() + size := g.r.Intn(5) + 2 + tuples := ValTuple{} + for i := 0; i < size; i++ { + tuples = append(tuples, g.intExpr()) + } + op := InStr + if g.randomBool() { + op = NotInStr + } + + return &ComparisonExpr{ + Operator: op, + Left: expr, + Right: tuples, + } +} + +func (g *generator) between() Expr { + g.enter() + defer g.exit() + + return &RangeCond{ + Operator: BetweenStr, + Left: g.intExpr(), + From: g.intExpr(), + To: g.intExpr(), + } +} + +func (g *generator) isExpr() Expr { + g.enter() + defer g.exit() + + ops := []string{IsNullStr, IsNotNullStr, IsTrueStr, IsNotTrueStr, IsFalseStr, IsNotFalseStr} + + return &IsExpr{ + Operator: g.randomOfS(ops), + Expr: g.booleanExpr(), + } +} From 5b7b806aafbec334320e725b92fe314dc38ac951 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Wed, 25 Mar 2020 15:55:54 +0100 Subject: [PATCH 69/85] Handle IS precendence correctly Signed-off-by: Andres Taylor --- go/vt/sqlparser/precedence.go | 2 + go/vt/sqlparser/precedence_test.go | 11 +- go/vt/sqlparser/random_expr.go | 107 ++- go/vt/sqlparser/sql.go | 1218 ++++++++++++++-------------- go/vt/sqlparser/tracked_buffer.go | 17 + 5 files changed, 720 insertions(+), 635 deletions(-) diff --git a/go/vt/sqlparser/precedence.go b/go/vt/sqlparser/precedence.go index 25f3838fc71..dbb8343baa2 100644 --- a/go/vt/sqlparser/precedence.go +++ b/go/vt/sqlparser/precedence.go @@ -62,6 +62,8 @@ func precedenceFor(in Expr) Precendence { case EqualStr, NotEqualStr, GreaterThanStr, GreaterEqualStr, LessThanStr, LessEqualStr, LikeStr, InStr, RegexpStr: return P11 } + case *IsExpr: + return P11 case *BinaryExpr: switch node.Operator { case BitOrStr: diff --git a/go/vt/sqlparser/precedence_test.go b/go/vt/sqlparser/precedence_test.go index 18a100f4568..f159695c130 100644 --- a/go/vt/sqlparser/precedence_test.go +++ b/go/vt/sqlparser/precedence_test.go @@ -18,7 +18,6 @@ package sqlparser import ( "fmt" - "math/rand" "testing" "time" @@ -140,6 +139,7 @@ func TestParens(t *testing.T) { {in: "(a) between (5) and (7)", expected: "a between 5 and 7"}, {in: "(a | b) between (5) and (7)", expected: "a | b between 5 and 7"}, {in: "(a and b) between (5) and (7)", expected: "(a and b) between 5 and 7"}, + {in: "(true is true) is null", expected: "(true is true) is null"}, } for _, tc := range tests { @@ -157,11 +157,8 @@ func TestRandom(t *testing.T) { // The idea is to generate random queries, and pass them through the parser and then the unparser, and one more time. The result of the first unparse should be the same as the second result. seed := time.Now().UnixNano() fmt.Println(fmt.Sprintf("seed is %d", seed)) - g := generator{ - seed: seed, - r: rand.New(rand.NewSource(seed)), - } - endBy := time.Now().Add(5 * time.Second) + g := newGenerator(seed, 5) + endBy := time.Now().Add(1 * time.Second) for { if time.Now().After(endBy) { @@ -173,7 +170,7 @@ func TestRandom(t *testing.T) { // When it's parsed and unparsed parsedInput, err := Parse(inputQ) - require.NoError(t, err) + require.NoError(t, err, inputQ) // Then the unparsing should be the same as the input query outputOfParseResult := String(parsedInput) diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go index 169bac00c2e..510a187902e 100644 --- a/go/vt/sqlparser/random_expr.go +++ b/go/vt/sqlparser/random_expr.go @@ -23,10 +23,20 @@ import ( // This file is used to generate random expressions to be used for testing +func newGenerator(seed int64, maxDepth int) *generator { + g := generator{ + seed: seed, + r: rand.New(rand.NewSource(seed)), + maxDepth: maxDepth, + } + return &g +} + type generator struct { - seed int64 - r *rand.Rand - depth int + seed int64 + r *rand.Rand + depth int + maxDepth int } // enter should be called whenever we are producing an intermediate node. it should be followed by a `defer g.exit()` @@ -41,15 +51,32 @@ func (g *generator) exit() { // atMaxDepth returns true if we have reached the maximum allowed depth or the expression tree func (g *generator) atMaxDepth() bool { - return g.depth >= 3 + return g.depth >= g.maxDepth } +/* Creates a random expression. It builds an expression tree using the following constructs: + - true/false + - AND/OR/NOT + - string literalrs, numeric literals (-/+ 1000) + - =, >, <, >=, <=, <=>, != + - &, |, ^, +, -, *, /, div, %, <<, >> + - IN, BETWEEN and CASE + - IS NULL, IS NOT NULL, IS TRUE, IS NOT TRUE, IS FALSE, IS NOT FALSE + +Note: It's important to update this method so that it produces all expressions that need precedence checking. +It's currently missing function calls and string operators +*/ func (g *generator) expression() Expr { if g.randomBool() { return g.booleanExpr() } + options := []exprF{ + func() Expr { return g.intExpr() }, + func() Expr { return g.stringExpr() }, + func() Expr { return g.booleanExpr() }, + } - return g.intExpr() + return g.randomOf(options) } func (g *generator) booleanExpr() Expr { @@ -60,11 +87,14 @@ func (g *generator) booleanExpr() Expr { options := []exprF{ func() Expr { return g.andExpr() }, func() Expr { return g.orExpr() }, - func() Expr { return g.booleanLiteral() }, - func() Expr { return g.comparison() }, + func() Expr { return g.comparison(g.intExpr) }, + func() Expr { return g.comparison(g.stringExpr) }, + //func() Expr { return g.comparison(g.booleanExpr) }, // this is not accepted by the parser func() Expr { return g.inExpr() }, func() Expr { return g.between() }, func() Expr { return g.isExpr() }, + func() Expr { return g.notExpr() }, + func() Expr { return g.likeExpr() }, } return g.randomOf(options) @@ -78,7 +108,7 @@ func (g *generator) intExpr() Expr { options := []exprF{ func() Expr { return g.arithmetic() }, func() Expr { return g.intLiteral() }, - func() Expr { return g.caseExpr() }, + func() Expr { return g.caseExpr(g.intExpr) }, } return g.randomOf(options) @@ -98,34 +128,60 @@ func (g *generator) intLiteral() Expr { return NewIntVal([]byte(t)) } -var comparisonOps = []string{"=", ">", "<", ">=", "<=", "<=>", "!="} +var words = []string{"ox", "ant", "ape", "asp", "bat", "bee", "boa", "bug", "cat", "cod", "cow", "cub", "doe", "dog", "eel", "eft", "elf", "elk", "emu", "ewe", "fly", "fox", "gar", "gnu", "hen", "hog", "imp", "jay", "kid", "kit", "koi", "lab", "man", "owl", "pig", "pug", "pup", "ram", "rat", "ray", "yak", "bass", "bear", "bird", "boar", "buck", "bull", "calf", "chow", "clam", "colt", "crab", "crow", "dane", "deer", "dodo", "dory", "dove", "drum", "duck", "fawn", "fish", "flea", "foal", "fowl", "frog", "gnat", "goat", "grub", "gull", "hare", "hawk", "ibex", "joey", "kite", "kiwi", "lamb", "lark", "lion", "loon", "lynx", "mako", "mink", "mite", "mole", "moth", "mule", "mutt", "newt", "orca", "oryx", "pika", "pony", "puma", "seal", "shad", "slug", "sole", "stag", "stud", "swan", "tahr", "teal", "tick", "toad", "tuna", "wasp", "wolf", "worm", "wren", "yeti", "adder", "akita", "alien", "aphid", "bison", "boxer", "bream", "bunny", "burro", "camel", "chimp", "civet", "cobra", "coral", "corgi", "crane", "dingo", "drake", "eagle", "egret", "filly", "finch", "gator", "gecko", "ghost", "ghoul", "goose", "guppy", "heron", "hippo", "horse", "hound", "husky", "hyena", "koala", "krill", "leech", "lemur", "liger", "llama", "louse", "macaw", "midge", "molly", "moose", "moray", "mouse", "panda", "perch", "prawn", "quail", "racer", "raven", "rhino", "robin", "satyr", "shark", "sheep", "shrew", "skink", "skunk", "sloth", "snail", "snake", "snipe", "squid", "stork", "swift", "swine", "tapir", "tetra", "tiger", "troll", "trout", "viper", "wahoo", "whale", "zebra", "alpaca", "amoeba", "baboon", "badger", "beagle", "bedbug", "beetle", "bengal", "bobcat", "caiman", "cattle", "cicada", "collie", "condor", "cougar", "coyote", "dassie", "donkey", "dragon", "earwig", "falcon", "feline", "ferret", "gannet", "gibbon", "glider", "goblin", "gopher", "grouse", "guinea", "hermit", "hornet", "iguana", "impala", "insect", "jackal", "jaguar", "jennet", "kitten", "kodiak", "lizard", "locust", "maggot", "magpie", "mammal", "mantis", "marlin", "marmot", "marten", "martin", "mayfly", "minnow", "monkey", "mullet", "muskox", "ocelot", "oriole", "osprey", "oyster", "parrot", "pigeon", "piglet", "poodle", "possum", "python", "quagga", "rabbit", "raptor", "rodent", "roughy", "salmon", "sawfly", "serval", "shiner", "shrimp", "spider", "sponge", "tarpon", "thrush", "tomcat", "toucan", "turkey", "turtle", "urchin", "vervet", "walrus", "weasel", "weevil", "wombat", "anchovy", "anemone", "bluejay", "buffalo", "bulldog", "buzzard", "caribou", "catfish", "chamois", "cheetah", "chicken", "chigger", "cowbird", "crappie", "crawdad", "cricket", "dogfish", "dolphin", "firefly", "garfish", "gazelle", "gelding", "giraffe", "gobbler", "gorilla", "goshawk", "grackle", "griffon", "grizzly", "grouper", "haddock", "hagfish", "halibut", "hamster", "herring", "jackass", "javelin", "jawfish", "jaybird", "katydid", "ladybug", "lamprey", "lemming", "leopard", "lioness", "lobster", "macaque", "mallard", "mammoth", "manatee", "mastiff", "meerkat", "mollusk", "monarch", "mongrel", "monitor", "monster", "mudfish", "muskrat", "mustang", "narwhal", "oarfish", "octopus", "opossum", "ostrich", "panther", "peacock", "pegasus", "pelican", "penguin", "phoenix", "piranha", "polecat", "primate", "quetzal", "raccoon", "rattler", "redbird", "redfish", "reptile", "rooster", "sawfish", "sculpin", "seagull", "skylark", "snapper", "spaniel", "sparrow", "sunbeam", "sunbird", "sunfish", "tadpole", "termite", "terrier", "unicorn", "vulture", "wallaby", "walleye", "warthog", "whippet", "wildcat", "aardvark", "airedale", "albacore", "anteater", "antelope", "arachnid", "barnacle", "basilisk", "blowfish", "bluebird", "bluegill", "bonefish", "bullfrog", "cardinal", "chipmunk", "cockatoo", "crayfish", "dinosaur", "doberman", "duckling", "elephant", "escargot", "flamingo", "flounder", "foxhound", "glowworm", "goldfish", "grubworm", "hedgehog", "honeybee", "hookworm", "humpback", "kangaroo", "killdeer", "kingfish", "labrador", "lacewing", "ladybird", "lionfish", "longhorn", "mackerel", "malamute", "marmoset", "mastodon", "moccasin", "mongoose", "monkfish", "mosquito", "pangolin", "parakeet", "pheasant", "pipefish", "platypus", "polliwog", "porpoise", "reindeer", "ringtail", "sailfish", "scorpion", "seahorse", "seasnail", "sheepdog", "shepherd", "silkworm", "squirrel", "stallion", "starfish", "starling", "stingray", "stinkbug", "sturgeon", "terrapin", "titmouse", "tortoise", "treefrog", "werewolf", "woodcock"} + +func (g *generator) stringLiteral() Expr { + return NewStrVal([]byte(g.randomOfS(words))) +} + +func (g *generator) stringExpr() Expr { + if g.atMaxDepth() { + return g.stringLiteral() + } + + options := []exprF{ + func() Expr { return g.stringLiteral() }, + func() Expr { return g.caseExpr(g.stringExpr) }, + } + + return g.randomOf(options) +} -func (g *generator) comparison() Expr { +func (g *generator) likeExpr() Expr { g.enter() defer g.exit() + return &ComparisonExpr{ + Operator: LikeStr, + Left: g.stringExpr(), + Right: g.stringExpr(), + } +} - v := g.r.Intn(len(comparisonOps)) +var comparisonOps = []string{EqualStr, LessThanStr, GreaterThanStr, LessEqualStr, GreaterEqualStr, NotEqualStr, NullSafeEqualStr} + +func (g *generator) comparison(f func() Expr) Expr { + g.enter() + defer g.exit() cmp := &ComparisonExpr{ - Operator: comparisonOps[v], - Left: g.intExpr(), - Right: g.intExpr(), + Operator: g.randomOfS(comparisonOps), + Left: f(), + Right: f(), } - return cmp } -func (g *generator) caseExpr() Expr { +func (g *generator) caseExpr(valueF func() Expr) Expr { g.enter() defer g.exit() var exp Expr var elseExpr Expr if g.randomBool() { - exp = g.intExpr() + exp = valueF() } if g.randomBool() { - elseExpr = g.intExpr() + elseExpr = valueF() } size := g.r.Intn(5) + 2 @@ -194,6 +250,12 @@ func (g *generator) orExpr() Expr { } } +func (g *generator) notExpr() Expr { + g.enter() + defer g.exit() + return &NotExpr{g.booleanExpr()} +} + func (g *generator) inExpr() Expr { g.enter() defer g.exit() @@ -220,8 +282,15 @@ func (g *generator) between() Expr { g.enter() defer g.exit() + var op string + if g.randomBool() { + op = BetweenStr + } else { + op = NotBetweenStr + } + return &RangeCond{ - Operator: BetweenStr, + Operator: op, Left: g.intExpr(), From: g.intExpr(), To: g.intExpr(), diff --git a/go/vt/sqlparser/sql.go b/go/vt/sqlparser/sql.go index 9ce4bbe4f39..bb743ccfc7a 100644 --- a/go/vt/sqlparser/sql.go +++ b/go/vt/sqlparser/sql.go @@ -3603,53 +3603,53 @@ yydefault: case 1: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:324 +//line sql.y:325 { setParseTree(yylex, yyDollar[1].statement) } case 2: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:329 +//line sql.y:330 { } case 3: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:330 +//line sql.y:331 { } case 4: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:334 +//line sql.y:335 { yyVAL.statement = yyDollar[1].selStmt } case 23: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:356 +//line sql.y:357 { setParseTree(yylex, nil) } case 24: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:362 +//line sql.y:363 { yyVAL.colIdent = NewColIdentWithAt(string(yyDollar[1].bytes), NoAt) } case 25: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:366 +//line sql.y:367 { yyVAL.colIdent = NewColIdentWithAt(string(yyDollar[1].bytes), SingleAt) } case 26: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:370 +//line sql.y:371 { yyVAL.colIdent = NewColIdentWithAt(string(yyDollar[1].bytes), DoubleAt) } case 27: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:376 +//line sql.y:377 { sel := yyDollar[1].selStmt.(*Select) sel.OrderBy = yyDollar[2].orderBy @@ -3659,55 +3659,55 @@ yydefault: } case 28: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:384 +//line sql.y:385 { yyVAL.selStmt = &Union{Type: yyDollar[2].str, Left: yyDollar[1].selStmt, Right: yyDollar[3].selStmt, OrderBy: yyDollar[4].orderBy, Limit: yyDollar[5].limit, Lock: yyDollar[6].str} } case 29: yyDollar = yyS[yypt-7 : yypt+1] -//line sql.y:388 +//line sql.y:389 { yyVAL.selStmt = &Select{Comments: Comments(yyDollar[2].bytes2), Cache: yyDollar[3].str, SelectExprs: SelectExprs{Nextval{Expr: yyDollar[5].expr}}, From: TableExprs{&AliasedTableExpr{Expr: yyDollar[7].tableName}}} } case 30: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:394 +//line sql.y:395 { yyVAL.statement = &Stream{Comments: Comments(yyDollar[2].bytes2), SelectExpr: yyDollar[3].selectExpr, Table: yyDollar[5].tableName} } case 31: yyDollar = yyS[yypt-10 : yypt+1] -//line sql.y:401 +//line sql.y:402 { yyVAL.selStmt = &Select{Comments: Comments(yyDollar[2].bytes2), Cache: yyDollar[3].str, Distinct: yyDollar[4].str, Hints: yyDollar[5].str, SelectExprs: yyDollar[6].selectExprs, From: yyDollar[7].tableExprs, Where: NewWhere(WhereStr, yyDollar[8].expr), GroupBy: GroupBy(yyDollar[9].exprs), Having: NewWhere(HavingStr, yyDollar[10].expr)} } case 32: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:407 +//line sql.y:408 { yyVAL.selStmt = yyDollar[1].selStmt } case 33: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:411 +//line sql.y:412 { yyVAL.selStmt = &ParenSelect{Select: yyDollar[2].selStmt} } case 34: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:417 +//line sql.y:418 { yyVAL.selStmt = yyDollar[1].selStmt } case 35: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:421 +//line sql.y:422 { yyVAL.selStmt = &ParenSelect{Select: yyDollar[2].selStmt} } case 36: yyDollar = yyS[yypt-7 : yypt+1] -//line sql.y:428 +//line sql.y:429 { // insert_data returns a *Insert pre-filled with Columns & Values ins := yyDollar[6].ins @@ -3721,7 +3721,7 @@ yydefault: } case 37: yyDollar = yyS[yypt-8 : yypt+1] -//line sql.y:440 +//line sql.y:441 { cols := make(Columns, 0, len(yyDollar[7].updateExprs)) vals := make(ValTuple, 0, len(yyDollar[8].updateExprs)) @@ -3733,192 +3733,192 @@ yydefault: } case 38: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:452 +//line sql.y:453 { yyVAL.str = InsertStr } case 39: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:456 +//line sql.y:457 { yyVAL.str = ReplaceStr } case 40: yyDollar = yyS[yypt-9 : yypt+1] -//line sql.y:462 +//line sql.y:463 { yyVAL.statement = &Update{Comments: Comments(yyDollar[2].bytes2), Ignore: yyDollar[3].str, TableExprs: yyDollar[4].tableExprs, Exprs: yyDollar[6].updateExprs, Where: NewWhere(WhereStr, yyDollar[7].expr), OrderBy: yyDollar[8].orderBy, Limit: yyDollar[9].limit} } case 41: yyDollar = yyS[yypt-8 : yypt+1] -//line sql.y:468 +//line sql.y:469 { yyVAL.statement = &Delete{Comments: Comments(yyDollar[2].bytes2), TableExprs: TableExprs{&AliasedTableExpr{Expr: yyDollar[4].tableName}}, Partitions: yyDollar[5].partitions, Where: NewWhere(WhereStr, yyDollar[6].expr), OrderBy: yyDollar[7].orderBy, Limit: yyDollar[8].limit} } case 42: yyDollar = yyS[yypt-7 : yypt+1] -//line sql.y:472 +//line sql.y:473 { yyVAL.statement = &Delete{Comments: Comments(yyDollar[2].bytes2), Targets: yyDollar[4].tableNames, TableExprs: yyDollar[6].tableExprs, Where: NewWhere(WhereStr, yyDollar[7].expr)} } case 43: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:476 +//line sql.y:477 { yyVAL.statement = &Delete{Comments: Comments(yyDollar[2].bytes2), Targets: yyDollar[3].tableNames, TableExprs: yyDollar[5].tableExprs, Where: NewWhere(WhereStr, yyDollar[6].expr)} } case 44: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:480 +//line sql.y:481 { yyVAL.statement = &Delete{Comments: Comments(yyDollar[2].bytes2), Targets: yyDollar[3].tableNames, TableExprs: yyDollar[5].tableExprs, Where: NewWhere(WhereStr, yyDollar[6].expr)} } case 45: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:485 +//line sql.y:486 { } case 46: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:486 +//line sql.y:487 { } case 47: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:490 +//line sql.y:491 { yyVAL.tableNames = TableNames{yyDollar[1].tableName} } case 48: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:494 +//line sql.y:495 { yyVAL.tableNames = append(yyVAL.tableNames, yyDollar[3].tableName) } case 49: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:500 +//line sql.y:501 { yyVAL.tableNames = TableNames{yyDollar[1].tableName} } case 50: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:504 +//line sql.y:505 { yyVAL.tableNames = append(yyVAL.tableNames, yyDollar[3].tableName) } case 51: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:509 +//line sql.y:510 { yyVAL.partitions = nil } case 52: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:513 +//line sql.y:514 { yyVAL.partitions = yyDollar[3].partitions } case 53: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:519 +//line sql.y:520 { yyVAL.statement = &Set{Comments: Comments(yyDollar[2].bytes2), Exprs: yyDollar[3].setExprs} } case 54: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:523 +//line sql.y:524 { yyVAL.statement = &Set{Comments: Comments(yyDollar[2].bytes2), Scope: yyDollar[3].str, Exprs: yyDollar[4].setExprs} } case 55: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:527 +//line sql.y:528 { yyVAL.statement = &Set{Comments: Comments(yyDollar[2].bytes2), Scope: yyDollar[3].str, Exprs: yyDollar[5].setExprs} } case 56: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:531 +//line sql.y:532 { yyVAL.statement = &Set{Comments: Comments(yyDollar[2].bytes2), Exprs: yyDollar[4].setExprs} } case 57: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:537 +//line sql.y:538 { yyVAL.setExprs = SetExprs{yyDollar[1].setExpr} } case 58: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:541 +//line sql.y:542 { yyVAL.setExprs = append(yyVAL.setExprs, yyDollar[3].setExpr) } case 59: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:547 +//line sql.y:548 { yyVAL.setExpr = &SetExpr{Name: NewColIdent(TransactionStr), Expr: NewStrVal([]byte(yyDollar[3].str))} } case 60: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:551 +//line sql.y:552 { yyVAL.setExpr = &SetExpr{Name: NewColIdent(TransactionStr), Expr: NewStrVal([]byte(TxReadWrite))} } case 61: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:555 +//line sql.y:556 { yyVAL.setExpr = &SetExpr{Name: NewColIdent(TransactionStr), Expr: NewStrVal([]byte(TxReadOnly))} } case 62: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:561 +//line sql.y:562 { yyVAL.str = IsolationLevelRepeatableRead } case 63: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:565 +//line sql.y:566 { yyVAL.str = IsolationLevelReadCommitted } case 64: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:569 +//line sql.y:570 { yyVAL.str = IsolationLevelReadUncommitted } case 65: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:573 +//line sql.y:574 { yyVAL.str = IsolationLevelSerializable } case 66: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:579 +//line sql.y:580 { yyVAL.str = SessionStr } case 67: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:583 +//line sql.y:584 { yyVAL.str = GlobalStr } case 68: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:589 +//line sql.y:590 { yyDollar[1].ddl.TableSpec = yyDollar[2].TableSpec yyVAL.statement = yyDollar[1].ddl } case 69: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:594 +//line sql.y:595 { // Create table [name] like [name] yyDollar[1].ddl.OptLike = yyDollar[2].optLike @@ -3926,139 +3926,139 @@ yydefault: } case 70: yyDollar = yyS[yypt-8 : yypt+1] -//line sql.y:600 +//line sql.y:601 { // Change this to an alter statement yyVAL.statement = &DDL{Action: AlterStr, Table: yyDollar[7].tableName} } case 71: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:605 +//line sql.y:606 { yyVAL.statement = &DDL{Action: CreateStr, Table: yyDollar[3].tableName.ToViewName()} } case 72: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:609 +//line sql.y:610 { yyVAL.statement = &DDL{Action: CreateStr, Table: yyDollar[5].tableName.ToViewName()} } case 73: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:613 +//line sql.y:614 { yyVAL.statement = &DBDDL{Action: CreateStr, DBName: string(yyDollar[4].colIdent.String())} } case 74: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:617 +//line sql.y:618 { yyVAL.statement = &DBDDL{Action: CreateStr, DBName: string(yyDollar[4].colIdent.String())} } case 75: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:622 +//line sql.y:623 { yyVAL.colIdent = NewColIdent("") } case 76: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:626 +//line sql.y:627 { yyVAL.colIdent = yyDollar[2].colIdent } case 77: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:632 +//line sql.y:633 { yyVAL.colIdent = yyDollar[1].colIdent } case 78: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:637 +//line sql.y:638 { var v []VindexParam yyVAL.vindexParams = v } case 79: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:642 +//line sql.y:643 { yyVAL.vindexParams = yyDollar[2].vindexParams } case 80: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:648 +//line sql.y:649 { yyVAL.vindexParams = make([]VindexParam, 0, 4) yyVAL.vindexParams = append(yyVAL.vindexParams, yyDollar[1].vindexParam) } case 81: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:653 +//line sql.y:654 { yyVAL.vindexParams = append(yyVAL.vindexParams, yyDollar[3].vindexParam) } case 82: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:659 +//line sql.y:660 { yyVAL.vindexParam = VindexParam{Key: yyDollar[1].colIdent, Val: yyDollar[3].str} } case 83: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:665 +//line sql.y:666 { yyVAL.ddl = &DDL{Action: CreateStr, Table: yyDollar[4].tableName} setDDL(yylex, yyVAL.ddl) } case 84: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:672 +//line sql.y:673 { yyVAL.TableSpec = yyDollar[2].TableSpec yyVAL.TableSpec.Options = yyDollar[4].str } case 85: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:679 +//line sql.y:680 { yyVAL.optLike = &OptLike{LikeTable: yyDollar[2].tableName} } case 86: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:683 +//line sql.y:684 { yyVAL.optLike = &OptLike{LikeTable: yyDollar[3].tableName} } case 87: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:689 +//line sql.y:690 { yyVAL.TableSpec = &TableSpec{} yyVAL.TableSpec.AddColumn(yyDollar[1].columnDefinition) } case 88: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:694 +//line sql.y:695 { yyVAL.TableSpec.AddColumn(yyDollar[3].columnDefinition) } case 89: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:698 +//line sql.y:699 { yyVAL.TableSpec.AddIndex(yyDollar[3].indexDefinition) } case 90: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:702 +//line sql.y:703 { yyVAL.TableSpec.AddConstraint(yyDollar[3].constraintDefinition) } case 91: yyDollar = yyS[yypt-8 : yypt+1] -//line sql.y:708 +//line sql.y:709 { yyDollar[2].columnType.NotNull = yyDollar[3].boolVal yyDollar[2].columnType.Default = yyDollar[4].optVal @@ -4070,7 +4070,7 @@ yydefault: } case 92: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:719 +//line sql.y:720 { yyVAL.columnType = yyDollar[1].columnType yyVAL.columnType.Unsigned = yyDollar[2].boolVal @@ -4078,74 +4078,74 @@ yydefault: } case 96: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:730 +//line sql.y:731 { yyVAL.columnType = yyDollar[1].columnType yyVAL.columnType.Length = yyDollar[2].sqlVal } case 97: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:735 +//line sql.y:736 { yyVAL.columnType = yyDollar[1].columnType } case 98: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:741 +//line sql.y:742 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 99: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:745 +//line sql.y:746 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 100: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:749 +//line sql.y:750 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 101: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:753 +//line sql.y:754 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 102: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:757 +//line sql.y:758 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 103: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:761 +//line sql.y:762 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 104: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:765 +//line sql.y:766 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 105: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:769 +//line sql.y:770 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 106: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:773 +//line sql.y:774 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 107: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:779 +//line sql.y:780 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} yyVAL.columnType.Length = yyDollar[2].LengthScaleOption.Length @@ -4153,7 +4153,7 @@ yydefault: } case 108: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:785 +//line sql.y:786 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} yyVAL.columnType.Length = yyDollar[2].LengthScaleOption.Length @@ -4161,7 +4161,7 @@ yydefault: } case 109: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:791 +//line sql.y:792 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} yyVAL.columnType.Length = yyDollar[2].LengthScaleOption.Length @@ -4169,7 +4169,7 @@ yydefault: } case 110: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:797 +//line sql.y:798 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} yyVAL.columnType.Length = yyDollar[2].LengthScaleOption.Length @@ -4177,7 +4177,7 @@ yydefault: } case 111: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:803 +//line sql.y:804 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} yyVAL.columnType.Length = yyDollar[2].LengthScaleOption.Length @@ -4185,206 +4185,206 @@ yydefault: } case 112: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:811 +//line sql.y:812 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 113: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:815 +//line sql.y:816 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes), Length: yyDollar[2].sqlVal} } case 114: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:819 +//line sql.y:820 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes), Length: yyDollar[2].sqlVal} } case 115: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:823 +//line sql.y:824 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes), Length: yyDollar[2].sqlVal} } case 116: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:827 +//line sql.y:828 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 117: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:833 +//line sql.y:834 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes), Length: yyDollar[2].sqlVal, Charset: yyDollar[3].str, Collate: yyDollar[4].str} } case 118: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:837 +//line sql.y:838 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes), Length: yyDollar[2].sqlVal, Charset: yyDollar[3].str, Collate: yyDollar[4].str} } case 119: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:841 +//line sql.y:842 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes), Length: yyDollar[2].sqlVal} } case 120: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:845 +//line sql.y:846 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes), Length: yyDollar[2].sqlVal} } case 121: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:849 +//line sql.y:850 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes), Charset: yyDollar[2].str, Collate: yyDollar[3].str} } case 122: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:853 +//line sql.y:854 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes), Charset: yyDollar[2].str, Collate: yyDollar[3].str} } case 123: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:857 +//line sql.y:858 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes), Charset: yyDollar[2].str, Collate: yyDollar[3].str} } case 124: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:861 +//line sql.y:862 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes), Charset: yyDollar[2].str, Collate: yyDollar[3].str} } case 125: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:865 +//line sql.y:866 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 126: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:869 +//line sql.y:870 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 127: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:873 +//line sql.y:874 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 128: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:877 +//line sql.y:878 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 129: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:881 +//line sql.y:882 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 130: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:885 +//line sql.y:886 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes), EnumValues: yyDollar[3].strs, Charset: yyDollar[5].str, Collate: yyDollar[6].str} } case 131: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:890 +//line sql.y:891 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes), EnumValues: yyDollar[3].strs, Charset: yyDollar[5].str, Collate: yyDollar[6].str} } case 132: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:896 +//line sql.y:897 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 133: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:900 +//line sql.y:901 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 134: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:904 +//line sql.y:905 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 135: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:908 +//line sql.y:909 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 136: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:912 +//line sql.y:913 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 137: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:916 +//line sql.y:917 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 138: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:920 +//line sql.y:921 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 139: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:924 +//line sql.y:925 { yyVAL.columnType = ColumnType{Type: string(yyDollar[1].bytes)} } case 140: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:930 +//line sql.y:931 { yyVAL.strs = make([]string, 0, 4) yyVAL.strs = append(yyVAL.strs, "'"+string(yyDollar[1].bytes)+"'") } case 141: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:935 +//line sql.y:936 { yyVAL.strs = append(yyDollar[1].strs, "'"+string(yyDollar[3].bytes)+"'") } case 142: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:940 +//line sql.y:941 { yyVAL.sqlVal = nil } case 143: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:944 +//line sql.y:945 { yyVAL.sqlVal = NewIntVal(yyDollar[2].bytes) } case 144: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:949 +//line sql.y:950 { yyVAL.LengthScaleOption = LengthScaleOption{} } case 145: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:953 +//line sql.y:954 { yyVAL.LengthScaleOption = LengthScaleOption{ Length: NewIntVal(yyDollar[2].bytes), @@ -4393,13 +4393,13 @@ yydefault: } case 146: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:961 +//line sql.y:962 { yyVAL.LengthScaleOption = LengthScaleOption{} } case 147: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:965 +//line sql.y:966 { yyVAL.LengthScaleOption = LengthScaleOption{ Length: NewIntVal(yyDollar[2].bytes), @@ -4407,7 +4407,7 @@ yydefault: } case 148: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:971 +//line sql.y:972 { yyVAL.LengthScaleOption = LengthScaleOption{ Length: NewIntVal(yyDollar[2].bytes), @@ -4416,478 +4416,478 @@ yydefault: } case 149: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:979 +//line sql.y:980 { yyVAL.boolVal = BoolVal(false) } case 150: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:983 +//line sql.y:984 { yyVAL.boolVal = BoolVal(true) } case 151: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:988 +//line sql.y:989 { yyVAL.boolVal = BoolVal(false) } case 152: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:992 +//line sql.y:993 { yyVAL.boolVal = BoolVal(true) } case 153: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:998 +//line sql.y:999 { yyVAL.boolVal = BoolVal(false) } case 154: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1002 +//line sql.y:1003 { yyVAL.boolVal = BoolVal(false) } case 155: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1006 +//line sql.y:1007 { yyVAL.boolVal = BoolVal(true) } case 156: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1011 +//line sql.y:1012 { yyVAL.optVal = nil } case 157: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1015 +//line sql.y:1016 { yyVAL.optVal = yyDollar[2].expr } case 158: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1020 +//line sql.y:1021 { yyVAL.optVal = nil } case 159: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1024 +//line sql.y:1025 { yyVAL.optVal = yyDollar[3].expr } case 160: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1029 +//line sql.y:1030 { yyVAL.boolVal = BoolVal(false) } case 161: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1033 +//line sql.y:1034 { yyVAL.boolVal = BoolVal(true) } case 162: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1038 +//line sql.y:1039 { yyVAL.str = "" } case 163: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1042 +//line sql.y:1043 { yyVAL.str = string(yyDollar[3].colIdent.String()) } case 164: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1046 +//line sql.y:1047 { yyVAL.str = string(yyDollar[3].bytes) } case 165: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1051 +//line sql.y:1052 { yyVAL.str = "" } case 166: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1055 +//line sql.y:1056 { yyVAL.str = string(yyDollar[2].colIdent.String()) } case 167: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1059 +//line sql.y:1060 { yyVAL.str = string(yyDollar[2].bytes) } case 168: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1064 +//line sql.y:1065 { yyVAL.colKeyOpt = colKeyNone } case 169: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1068 +//line sql.y:1069 { yyVAL.colKeyOpt = colKeyPrimary } case 170: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1072 +//line sql.y:1073 { yyVAL.colKeyOpt = colKey } case 171: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1076 +//line sql.y:1077 { yyVAL.colKeyOpt = colKeyUniqueKey } case 172: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1080 +//line sql.y:1081 { yyVAL.colKeyOpt = colKeyUnique } case 173: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1085 +//line sql.y:1086 { yyVAL.sqlVal = nil } case 174: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1089 +//line sql.y:1090 { yyVAL.sqlVal = NewStrVal(yyDollar[2].bytes) } case 175: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:1095 +//line sql.y:1096 { yyVAL.indexDefinition = &IndexDefinition{Info: yyDollar[1].indexInfo, Columns: yyDollar[3].indexColumns, Options: yyDollar[5].indexOptions} } case 176: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1099 +//line sql.y:1100 { yyVAL.indexDefinition = &IndexDefinition{Info: yyDollar[1].indexInfo, Columns: yyDollar[3].indexColumns} } case 177: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1105 +//line sql.y:1106 { yyVAL.indexOptions = []*IndexOption{yyDollar[1].indexOption} } case 178: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1109 +//line sql.y:1110 { yyVAL.indexOptions = append(yyVAL.indexOptions, yyDollar[2].indexOption) } case 179: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1115 +//line sql.y:1116 { yyVAL.indexOption = &IndexOption{Name: string(yyDollar[1].bytes), Using: string(yyDollar[2].colIdent.String())} } case 180: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1119 +//line sql.y:1120 { // should not be string yyVAL.indexOption = &IndexOption{Name: string(yyDollar[1].bytes), Value: NewIntVal(yyDollar[3].bytes)} } case 181: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1124 +//line sql.y:1125 { yyVAL.indexOption = &IndexOption{Name: string(yyDollar[1].bytes), Value: NewStrVal(yyDollar[2].bytes)} } case 182: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1130 +//line sql.y:1131 { yyVAL.str = "" } case 183: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1134 +//line sql.y:1135 { yyVAL.str = string(yyDollar[1].bytes) } case 184: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1140 +//line sql.y:1141 { yyVAL.indexInfo = &IndexInfo{Type: string(yyDollar[1].bytes) + " " + string(yyDollar[2].bytes), Name: NewColIdent("PRIMARY"), Primary: true, Unique: true} } case 185: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1144 +//line sql.y:1145 { yyVAL.indexInfo = &IndexInfo{Type: string(yyDollar[1].bytes) + " " + string(yyDollar[2].str), Name: NewColIdent(yyDollar[3].str), Spatial: true, Unique: false} } case 186: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1148 +//line sql.y:1149 { yyVAL.indexInfo = &IndexInfo{Type: string(yyDollar[1].bytes) + " " + string(yyDollar[2].str), Name: NewColIdent(yyDollar[3].str), Unique: true} } case 187: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1152 +//line sql.y:1153 { yyVAL.indexInfo = &IndexInfo{Type: string(yyDollar[1].bytes), Name: NewColIdent(yyDollar[2].str), Unique: true} } case 188: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1156 +//line sql.y:1157 { yyVAL.indexInfo = &IndexInfo{Type: string(yyDollar[1].str), Name: NewColIdent(yyDollar[2].str), Unique: false} } case 189: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1162 +//line sql.y:1163 { yyVAL.str = string(yyDollar[1].bytes) } case 190: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1166 +//line sql.y:1167 { yyVAL.str = string(yyDollar[1].bytes) } case 191: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1171 +//line sql.y:1172 { yyVAL.str = "" } case 192: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1175 +//line sql.y:1176 { yyVAL.str = string(yyDollar[1].colIdent.String()) } case 193: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1181 +//line sql.y:1182 { yyVAL.indexColumns = []*IndexColumn{yyDollar[1].indexColumn} } case 194: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1185 +//line sql.y:1186 { yyVAL.indexColumns = append(yyVAL.indexColumns, yyDollar[3].indexColumn) } case 195: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1191 +//line sql.y:1192 { yyVAL.indexColumn = &IndexColumn{Column: yyDollar[1].colIdent, Length: yyDollar[2].sqlVal} } case 196: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1197 +//line sql.y:1198 { yyVAL.constraintDefinition = &ConstraintDefinition{Name: string(yyDollar[2].colIdent.String()), Details: yyDollar[3].constraintInfo} } case 197: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1201 +//line sql.y:1202 { yyVAL.constraintDefinition = &ConstraintDefinition{Details: yyDollar[1].constraintInfo} } case 198: yyDollar = yyS[yypt-10 : yypt+1] -//line sql.y:1208 +//line sql.y:1209 { yyVAL.constraintInfo = &ForeignKeyDefinition{Source: yyDollar[4].columns, ReferencedTable: yyDollar[7].tableName, ReferencedColumns: yyDollar[9].columns} } case 199: yyDollar = yyS[yypt-11 : yypt+1] -//line sql.y:1212 +//line sql.y:1213 { yyVAL.constraintInfo = &ForeignKeyDefinition{Source: yyDollar[4].columns, ReferencedTable: yyDollar[7].tableName, ReferencedColumns: yyDollar[9].columns, OnDelete: yyDollar[11].ReferenceAction} } case 200: yyDollar = yyS[yypt-11 : yypt+1] -//line sql.y:1216 +//line sql.y:1217 { yyVAL.constraintInfo = &ForeignKeyDefinition{Source: yyDollar[4].columns, ReferencedTable: yyDollar[7].tableName, ReferencedColumns: yyDollar[9].columns, OnUpdate: yyDollar[11].ReferenceAction} } case 201: yyDollar = yyS[yypt-12 : yypt+1] -//line sql.y:1220 +//line sql.y:1221 { yyVAL.constraintInfo = &ForeignKeyDefinition{Source: yyDollar[4].columns, ReferencedTable: yyDollar[7].tableName, ReferencedColumns: yyDollar[9].columns, OnDelete: yyDollar[11].ReferenceAction, OnUpdate: yyDollar[12].ReferenceAction} } case 202: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1226 +//line sql.y:1227 { yyVAL.ReferenceAction = yyDollar[3].ReferenceAction } case 203: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1232 +//line sql.y:1233 { yyVAL.ReferenceAction = yyDollar[3].ReferenceAction } case 204: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1238 +//line sql.y:1239 { yyVAL.ReferenceAction = Restrict } case 205: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1242 +//line sql.y:1243 { yyVAL.ReferenceAction = Cascade } case 206: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1246 +//line sql.y:1247 { yyVAL.ReferenceAction = NoAction } case 207: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1250 +//line sql.y:1251 { yyVAL.ReferenceAction = SetDefault } case 208: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1254 +//line sql.y:1255 { yyVAL.ReferenceAction = SetNull } case 209: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1259 +//line sql.y:1260 { yyVAL.str = "" } case 210: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1263 +//line sql.y:1264 { yyVAL.str = " " + string(yyDollar[1].str) } case 211: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1267 +//line sql.y:1268 { yyVAL.str = string(yyDollar[1].str) + ", " + string(yyDollar[3].str) } case 212: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1275 +//line sql.y:1276 { yyVAL.str = yyDollar[1].str } case 213: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1279 +//line sql.y:1280 { yyVAL.str = yyDollar[1].str + " " + yyDollar[2].str } case 214: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1283 +//line sql.y:1284 { yyVAL.str = yyDollar[1].str + "=" + yyDollar[3].str } case 215: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1289 +//line sql.y:1290 { yyVAL.str = yyDollar[1].colIdent.String() } case 216: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1293 +//line sql.y:1294 { yyVAL.str = "'" + string(yyDollar[1].bytes) + "'" } case 217: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1297 +//line sql.y:1298 { yyVAL.str = string(yyDollar[1].bytes) } case 218: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:1303 +//line sql.y:1304 { yyVAL.statement = &DDL{Action: AlterStr, Table: yyDollar[4].tableName} } case 219: yyDollar = yyS[yypt-7 : yypt+1] -//line sql.y:1307 +//line sql.y:1308 { yyVAL.statement = &DDL{Action: AlterStr, Table: yyDollar[4].tableName} } case 220: yyDollar = yyS[yypt-7 : yypt+1] -//line sql.y:1311 +//line sql.y:1312 { yyVAL.statement = &DDL{Action: AlterStr, Table: yyDollar[4].tableName} } case 221: yyDollar = yyS[yypt-7 : yypt+1] -//line sql.y:1315 +//line sql.y:1316 { // Change this to a rename statement yyVAL.statement = &DDL{Action: RenameStr, FromTables: TableNames{yyDollar[4].tableName}, ToTables: TableNames{yyDollar[7].tableName}} } case 222: yyDollar = yyS[yypt-7 : yypt+1] -//line sql.y:1320 +//line sql.y:1321 { // Rename an index can just be an alter yyVAL.statement = &DDL{Action: AlterStr, Table: yyDollar[4].tableName} } case 223: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1325 +//line sql.y:1326 { yyVAL.statement = &DDL{Action: AlterStr, Table: yyDollar[3].tableName.ToViewName()} } case 224: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:1329 +//line sql.y:1330 { yyVAL.statement = &DDL{Action: AlterStr, Table: yyDollar[4].tableName, PartitionSpec: yyDollar[5].partSpec} } case 225: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1333 +//line sql.y:1334 { yyVAL.statement = &DBDDL{Action: AlterStr, DBName: string(yyDollar[3].colIdent.String())} } case 226: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1337 +//line sql.y:1338 { yyVAL.statement = &DBDDL{Action: AlterStr, DBName: string(yyDollar[3].colIdent.String())} } case 227: yyDollar = yyS[yypt-7 : yypt+1] -//line sql.y:1341 +//line sql.y:1342 { yyVAL.statement = &DDL{ Action: CreateVindexStr, @@ -4901,7 +4901,7 @@ yydefault: } case 228: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:1353 +//line sql.y:1354 { yyVAL.statement = &DDL{ Action: DropVindexStr, @@ -4913,19 +4913,19 @@ yydefault: } case 229: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:1363 +//line sql.y:1364 { yyVAL.statement = &DDL{Action: AddVschemaTableStr, Table: yyDollar[5].tableName} } case 230: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:1367 +//line sql.y:1368 { yyVAL.statement = &DDL{Action: DropVschemaTableStr, Table: yyDollar[5].tableName} } case 231: yyDollar = yyS[yypt-12 : yypt+1] -//line sql.y:1371 +//line sql.y:1372 { yyVAL.statement = &DDL{ Action: AddColVindexStr, @@ -4940,7 +4940,7 @@ yydefault: } case 232: yyDollar = yyS[yypt-7 : yypt+1] -//line sql.y:1384 +//line sql.y:1385 { yyVAL.statement = &DDL{ Action: DropColVindexStr, @@ -4952,13 +4952,13 @@ yydefault: } case 233: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:1394 +//line sql.y:1395 { yyVAL.statement = &DDL{Action: AddSequenceStr, Table: yyDollar[5].tableName} } case 234: yyDollar = yyS[yypt-9 : yypt+1] -//line sql.y:1398 +//line sql.y:1399 { yyVAL.statement = &DDL{ Action: AddAutoIncStr, @@ -4971,49 +4971,49 @@ yydefault: } case 249: yyDollar = yyS[yypt-7 : yypt+1] -//line sql.y:1427 +//line sql.y:1428 { yyVAL.partSpec = &PartitionSpec{Action: ReorganizeStr, Name: yyDollar[3].colIdent, Definitions: yyDollar[6].partDefs} } case 250: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1433 +//line sql.y:1434 { yyVAL.partDefs = []*PartitionDefinition{yyDollar[1].partDef} } case 251: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1437 +//line sql.y:1438 { yyVAL.partDefs = append(yyDollar[1].partDefs, yyDollar[3].partDef) } case 252: yyDollar = yyS[yypt-8 : yypt+1] -//line sql.y:1443 +//line sql.y:1444 { yyVAL.partDef = &PartitionDefinition{Name: yyDollar[2].colIdent, Limit: yyDollar[7].expr} } case 253: yyDollar = yyS[yypt-8 : yypt+1] -//line sql.y:1447 +//line sql.y:1448 { yyVAL.partDef = &PartitionDefinition{Name: yyDollar[2].colIdent, Maxvalue: true} } case 254: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1453 +//line sql.y:1454 { yyVAL.statement = yyDollar[3].ddl } case 255: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1459 +//line sql.y:1460 { yyVAL.ddl = &DDL{Action: RenameStr, FromTables: TableNames{yyDollar[1].tableName}, ToTables: TableNames{yyDollar[3].tableName}} } case 256: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:1463 +//line sql.y:1464 { yyVAL.ddl = yyDollar[1].ddl yyVAL.ddl.FromTables = append(yyVAL.ddl.FromTables, yyDollar[3].tableName) @@ -5021,7 +5021,7 @@ yydefault: } case 257: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1471 +//line sql.y:1472 { var exists bool if yyDollar[3].byt != 0 { @@ -5031,14 +5031,14 @@ yydefault: } case 258: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:1479 +//line sql.y:1480 { // Change this to an alter statement yyVAL.statement = &DDL{Action: AlterStr, Table: yyDollar[5].tableName} } case 259: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:1484 +//line sql.y:1485 { var exists bool if yyDollar[3].byt != 0 { @@ -5048,150 +5048,150 @@ yydefault: } case 260: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1492 +//line sql.y:1493 { yyVAL.statement = &DBDDL{Action: DropStr, DBName: string(yyDollar[4].colIdent.String())} } case 261: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1496 +//line sql.y:1497 { yyVAL.statement = &DBDDL{Action: DropStr, DBName: string(yyDollar[4].colIdent.String())} } case 262: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1502 +//line sql.y:1503 { yyVAL.statement = &DDL{Action: TruncateStr, Table: yyDollar[3].tableName} } case 263: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1506 +//line sql.y:1507 { yyVAL.statement = &DDL{Action: TruncateStr, Table: yyDollar[2].tableName} } case 264: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1511 +//line sql.y:1512 { yyVAL.statement = &OtherRead{} } case 265: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1517 +//line sql.y:1518 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes) + " " + string(yyDollar[3].colIdent.String())} } case 266: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1522 +//line sql.y:1523 { showTablesOpt := &ShowTablesOpt{Filter: yyDollar[4].showFilter} yyVAL.statement = &Show{Type: CharsetStr, ShowTablesOpt: showTablesOpt} } case 267: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1527 +//line sql.y:1528 { showTablesOpt := &ShowTablesOpt{Filter: yyDollar[3].showFilter} yyVAL.statement = &Show{Type: string(yyDollar[2].bytes), ShowTablesOpt: showTablesOpt} } case 268: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1532 +//line sql.y:1533 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes) + " " + string(yyDollar[3].bytes)} } case 269: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1537 +//line sql.y:1538 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes) + " " + string(yyDollar[3].colIdent.String())} } case 270: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1541 +//line sql.y:1542 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes) + " " + string(yyDollar[3].bytes)} } case 271: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1545 +//line sql.y:1546 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes) + " " + string(yyDollar[3].bytes), Table: yyDollar[4].tableName} } case 272: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1549 +//line sql.y:1550 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes) + " " + string(yyDollar[3].bytes)} } case 273: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1553 +//line sql.y:1554 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes) + " " + string(yyDollar[3].bytes)} } case 274: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1557 +//line sql.y:1558 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes)} } case 275: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1561 +//line sql.y:1562 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes)} } case 276: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:1565 +//line sql.y:1566 { showTablesOpt := &ShowTablesOpt{DbName: yyDollar[5].str, Filter: yyDollar[6].showFilter} yyVAL.statement = &Show{Type: string(yyDollar[2].bytes), ShowTablesOpt: showTablesOpt, OnTable: yyDollar[4].tableName} } case 277: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:1570 +//line sql.y:1571 { showTablesOpt := &ShowTablesOpt{DbName: yyDollar[5].str, Filter: yyDollar[6].showFilter} yyVAL.statement = &Show{Type: string(yyDollar[2].bytes), ShowTablesOpt: showTablesOpt, OnTable: yyDollar[4].tableName} } case 278: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1575 +//line sql.y:1576 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes)} } case 279: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1579 +//line sql.y:1580 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes)} } case 280: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1583 +//line sql.y:1584 { yyVAL.statement = &Show{Scope: yyDollar[2].str, Type: string(yyDollar[3].bytes)} } case 281: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1587 +//line sql.y:1588 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes)} } case 282: yyDollar = yyS[yypt-7 : yypt+1] -//line sql.y:1591 +//line sql.y:1592 { showTablesOpt := &ShowTablesOpt{Full: yyDollar[2].str, DbName: yyDollar[6].str, Filter: yyDollar[7].showFilter} yyVAL.statement = &Show{Type: string(yyDollar[3].str), ShowTablesOpt: showTablesOpt, OnTable: yyDollar[5].tableName} } case 283: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:1596 +//line sql.y:1597 { // this is ugly, but I couldn't find a better way for now if yyDollar[3].str == "processlist" { @@ -5203,19 +5203,19 @@ yydefault: } case 284: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1606 +//line sql.y:1607 { yyVAL.statement = &Show{Scope: yyDollar[2].str, Type: string(yyDollar[3].bytes)} } case 285: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1610 +//line sql.y:1611 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes)} } case 286: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1614 +//line sql.y:1615 { // Cannot dereference $4 directly, or else the parser stackcannot be pooled. See yyParsePooled showCollationFilterOpt := yyDollar[4].expr @@ -5223,429 +5223,429 @@ yydefault: } case 287: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:1620 +//line sql.y:1621 { showTablesOpt := &ShowTablesOpt{Filter: yyDollar[4].showFilter} yyVAL.statement = &Show{Scope: string(yyDollar[2].bytes), Type: string(yyDollar[3].bytes), ShowTablesOpt: showTablesOpt} } case 288: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1625 +//line sql.y:1626 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes) + " " + string(yyDollar[3].bytes)} } case 289: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1629 +//line sql.y:1630 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes) + " " + string(yyDollar[3].bytes)} } case 290: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:1633 +//line sql.y:1634 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes) + " " + string(yyDollar[3].bytes), OnTable: yyDollar[5].tableName} } case 291: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1637 +//line sql.y:1638 { yyVAL.statement = &Show{Type: string(yyDollar[2].bytes)} } case 292: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1651 +//line sql.y:1652 { yyVAL.statement = &Show{Type: string(yyDollar[2].colIdent.String())} } case 293: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1657 +//line sql.y:1658 { yyVAL.str = string(yyDollar[1].bytes) } case 294: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1661 +//line sql.y:1662 { yyVAL.str = string(yyDollar[1].bytes) } case 295: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1667 +//line sql.y:1668 { yyVAL.str = "" } case 296: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1671 +//line sql.y:1672 { yyVAL.str = "full " } case 297: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1677 +//line sql.y:1678 { yyVAL.str = string(yyDollar[1].bytes) } case 298: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1681 +//line sql.y:1682 { yyVAL.str = string(yyDollar[1].bytes) } case 299: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1687 +//line sql.y:1688 { yyVAL.str = "" } case 300: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1691 +//line sql.y:1692 { yyVAL.str = yyDollar[2].tableIdent.v } case 301: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1695 +//line sql.y:1696 { yyVAL.str = yyDollar[2].tableIdent.v } case 302: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1701 +//line sql.y:1702 { yyVAL.showFilter = nil } case 303: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1705 +//line sql.y:1706 { yyVAL.showFilter = &ShowFilter{Like: string(yyDollar[2].bytes)} } case 304: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1709 +//line sql.y:1710 { yyVAL.showFilter = &ShowFilter{Filter: yyDollar[2].expr} } case 305: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1715 +//line sql.y:1716 { yyVAL.showFilter = nil } case 306: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1719 +//line sql.y:1720 { yyVAL.showFilter = &ShowFilter{Like: string(yyDollar[2].bytes)} } case 307: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1725 +//line sql.y:1726 { yyVAL.str = "" } case 308: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1729 +//line sql.y:1730 { yyVAL.str = SessionStr } case 309: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1733 +//line sql.y:1734 { yyVAL.str = GlobalStr } case 310: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1739 +//line sql.y:1740 { yyVAL.statement = &Use{DBName: yyDollar[2].tableIdent} } case 311: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1743 +//line sql.y:1744 { yyVAL.statement = &Use{DBName: TableIdent{v: ""}} } case 312: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1749 +//line sql.y:1750 { yyVAL.statement = &Begin{} } case 313: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1753 +//line sql.y:1754 { yyVAL.statement = &Begin{} } case 314: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1759 +//line sql.y:1760 { yyVAL.statement = &Commit{} } case 315: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1765 +//line sql.y:1766 { yyVAL.statement = &Rollback{} } case 316: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1771 +//line sql.y:1772 { yyVAL.statement = &OtherRead{} } case 317: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1775 +//line sql.y:1776 { yyVAL.statement = &OtherRead{} } case 318: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1779 +//line sql.y:1780 { yyVAL.statement = &OtherRead{} } case 319: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1783 +//line sql.y:1784 { yyVAL.statement = &OtherAdmin{} } case 320: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1787 +//line sql.y:1788 { yyVAL.statement = &OtherAdmin{} } case 321: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1791 +//line sql.y:1792 { yyVAL.statement = &OtherAdmin{} } case 322: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1795 +//line sql.y:1796 { yyVAL.statement = &OtherAdmin{} } case 323: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1801 +//line sql.y:1802 { yyVAL.statement = &DDL{Action: FlushStr} } case 324: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1805 +//line sql.y:1806 { setAllowComments(yylex, true) } case 325: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1809 +//line sql.y:1810 { yyVAL.bytes2 = yyDollar[2].bytes2 setAllowComments(yylex, false) } case 326: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1815 +//line sql.y:1816 { yyVAL.bytes2 = nil } case 327: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1819 +//line sql.y:1820 { yyVAL.bytes2 = append(yyDollar[1].bytes2, yyDollar[2].bytes) } case 328: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1825 +//line sql.y:1826 { yyVAL.str = UnionStr } case 329: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1829 +//line sql.y:1830 { yyVAL.str = UnionAllStr } case 330: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1833 +//line sql.y:1834 { yyVAL.str = UnionDistinctStr } case 331: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1838 +//line sql.y:1839 { yyVAL.str = "" } case 332: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1842 +//line sql.y:1843 { yyVAL.str = SQLNoCacheStr } case 333: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1846 +//line sql.y:1847 { yyVAL.str = SQLCacheStr } case 334: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1851 +//line sql.y:1852 { yyVAL.str = "" } case 335: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1855 +//line sql.y:1856 { yyVAL.str = DistinctStr } case 336: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1859 +//line sql.y:1860 { yyVAL.str = DistinctStr } case 337: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1864 +//line sql.y:1865 { yyVAL.str = "" } case 338: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1868 +//line sql.y:1869 { yyVAL.str = StraightJoinHint } case 339: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1873 +//line sql.y:1874 { yyVAL.selectExprs = nil } case 340: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1877 +//line sql.y:1878 { yyVAL.selectExprs = yyDollar[1].selectExprs } case 341: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1883 +//line sql.y:1884 { yyVAL.selectExprs = SelectExprs{yyDollar[1].selectExpr} } case 342: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1887 +//line sql.y:1888 { yyVAL.selectExprs = append(yyVAL.selectExprs, yyDollar[3].selectExpr) } case 343: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1893 +//line sql.y:1894 { yyVAL.selectExpr = &StarExpr{} } case 344: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1897 +//line sql.y:1898 { yyVAL.selectExpr = &AliasedExpr{Expr: yyDollar[1].expr, As: yyDollar[2].colIdent} } case 345: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1901 +//line sql.y:1902 { yyVAL.selectExpr = &StarExpr{TableName: TableName{Name: yyDollar[1].tableIdent}} } case 346: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:1905 +//line sql.y:1906 { yyVAL.selectExpr = &StarExpr{TableName: TableName{Qualifier: yyDollar[1].tableIdent, Name: yyDollar[3].tableIdent}} } case 347: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1910 +//line sql.y:1911 { yyVAL.colIdent = ColIdent{} } case 348: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1914 +//line sql.y:1915 { yyVAL.colIdent = yyDollar[1].colIdent } case 349: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1918 +//line sql.y:1919 { yyVAL.colIdent = yyDollar[2].colIdent } case 351: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1925 +//line sql.y:1926 { yyVAL.colIdent = NewColIdent(string(yyDollar[1].bytes)) } case 352: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:1930 +//line sql.y:1931 { yyVAL.tableExprs = TableExprs{&AliasedTableExpr{Expr: TableName{Name: NewTableIdent("dual")}}} } case 353: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:1934 +//line sql.y:1935 { yyVAL.tableExprs = yyDollar[2].tableExprs } case 354: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1940 +//line sql.y:1941 { yyVAL.tableExprs = TableExprs{yyDollar[1].tableExpr} } case 355: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1944 +//line sql.y:1945 { yyVAL.tableExprs = append(yyVAL.tableExprs, yyDollar[3].tableExpr) } case 358: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1954 +//line sql.y:1955 { yyVAL.tableExpr = yyDollar[1].aliasedTableName } case 359: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1958 +//line sql.y:1959 { yyVAL.tableExpr = &AliasedTableExpr{Expr: yyDollar[1].subquery, As: yyDollar[3].tableIdent} } case 360: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1962 +//line sql.y:1963 { // missed alias for subquery yylex.Error("Every derived table must have its own alias") @@ -5653,199 +5653,199 @@ yydefault: } case 361: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1968 +//line sql.y:1969 { yyVAL.tableExpr = &ParenTableExpr{Exprs: yyDollar[2].tableExprs} } case 362: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1974 +//line sql.y:1975 { yyVAL.aliasedTableName = &AliasedTableExpr{Expr: yyDollar[1].tableName, As: yyDollar[2].tableIdent, Hints: yyDollar[3].indexHints} } case 363: yyDollar = yyS[yypt-7 : yypt+1] -//line sql.y:1978 +//line sql.y:1979 { yyVAL.aliasedTableName = &AliasedTableExpr{Expr: yyDollar[1].tableName, Partitions: yyDollar[4].partitions, As: yyDollar[6].tableIdent, Hints: yyDollar[7].indexHints} } case 364: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1984 +//line sql.y:1985 { yyVAL.columns = Columns{yyDollar[1].colIdent} } case 365: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1988 +//line sql.y:1989 { yyVAL.columns = append(yyVAL.columns, yyDollar[3].colIdent) } case 366: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:1994 +//line sql.y:1995 { yyVAL.partitions = Partitions{yyDollar[1].colIdent} } case 367: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:1998 +//line sql.y:1999 { yyVAL.partitions = append(yyVAL.partitions, yyDollar[3].colIdent) } case 368: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2011 +//line sql.y:2012 { yyVAL.tableExpr = &JoinTableExpr{LeftExpr: yyDollar[1].tableExpr, Join: yyDollar[2].str, RightExpr: yyDollar[3].tableExpr, Condition: yyDollar[4].joinCondition} } case 369: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2015 +//line sql.y:2016 { yyVAL.tableExpr = &JoinTableExpr{LeftExpr: yyDollar[1].tableExpr, Join: yyDollar[2].str, RightExpr: yyDollar[3].tableExpr, Condition: yyDollar[4].joinCondition} } case 370: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2019 +//line sql.y:2020 { yyVAL.tableExpr = &JoinTableExpr{LeftExpr: yyDollar[1].tableExpr, Join: yyDollar[2].str, RightExpr: yyDollar[3].tableExpr, Condition: yyDollar[4].joinCondition} } case 371: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2023 +//line sql.y:2024 { yyVAL.tableExpr = &JoinTableExpr{LeftExpr: yyDollar[1].tableExpr, Join: yyDollar[2].str, RightExpr: yyDollar[3].tableExpr} } case 372: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2029 +//line sql.y:2030 { yyVAL.joinCondition = JoinCondition{On: yyDollar[2].expr} } case 373: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2031 +//line sql.y:2032 { yyVAL.joinCondition = JoinCondition{Using: yyDollar[3].columns} } case 374: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2035 +//line sql.y:2036 { yyVAL.joinCondition = JoinCondition{} } case 375: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2037 +//line sql.y:2038 { yyVAL.joinCondition = yyDollar[1].joinCondition } case 376: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2041 +//line sql.y:2042 { yyVAL.joinCondition = JoinCondition{} } case 377: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2043 +//line sql.y:2044 { yyVAL.joinCondition = JoinCondition{On: yyDollar[2].expr} } case 378: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2046 +//line sql.y:2047 { yyVAL.empty = struct{}{} } case 379: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2048 +//line sql.y:2049 { yyVAL.empty = struct{}{} } case 380: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2051 +//line sql.y:2052 { yyVAL.tableIdent = NewTableIdent("") } case 381: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2055 +//line sql.y:2056 { yyVAL.tableIdent = yyDollar[1].tableIdent } case 382: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2059 +//line sql.y:2060 { yyVAL.tableIdent = yyDollar[2].tableIdent } case 384: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2066 +//line sql.y:2067 { yyVAL.tableIdent = NewTableIdent(string(yyDollar[1].bytes)) } case 385: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2072 +//line sql.y:2073 { yyVAL.str = JoinStr } case 386: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2076 +//line sql.y:2077 { yyVAL.str = JoinStr } case 387: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2080 +//line sql.y:2081 { yyVAL.str = JoinStr } case 388: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2086 +//line sql.y:2087 { yyVAL.str = StraightJoinStr } case 389: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2092 +//line sql.y:2093 { yyVAL.str = LeftJoinStr } case 390: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2096 +//line sql.y:2097 { yyVAL.str = LeftJoinStr } case 391: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2100 +//line sql.y:2101 { yyVAL.str = RightJoinStr } case 392: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2104 +//line sql.y:2105 { yyVAL.str = RightJoinStr } case 393: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2110 +//line sql.y:2111 { yyVAL.str = NaturalJoinStr } case 394: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2114 +//line sql.y:2115 { if yyDollar[2].str == LeftJoinStr { yyVAL.str = NaturalLeftJoinStr @@ -5855,469 +5855,469 @@ yydefault: } case 395: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2124 +//line sql.y:2125 { yyVAL.tableName = yyDollar[2].tableName } case 396: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2128 +//line sql.y:2129 { yyVAL.tableName = yyDollar[1].tableName } case 397: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2134 +//line sql.y:2135 { yyVAL.tableName = TableName{Name: yyDollar[1].tableIdent} } case 398: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2138 +//line sql.y:2139 { yyVAL.tableName = TableName{Qualifier: yyDollar[1].tableIdent, Name: yyDollar[3].tableIdent} } case 399: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2144 +//line sql.y:2145 { yyVAL.tableName = TableName{Name: yyDollar[1].tableIdent} } case 400: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2149 +//line sql.y:2150 { yyVAL.indexHints = nil } case 401: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:2153 +//line sql.y:2154 { yyVAL.indexHints = &IndexHints{Type: UseStr, Indexes: yyDollar[4].columns} } case 402: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2157 +//line sql.y:2158 { yyVAL.indexHints = &IndexHints{Type: UseStr} } case 403: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:2161 +//line sql.y:2162 { yyVAL.indexHints = &IndexHints{Type: IgnoreStr, Indexes: yyDollar[4].columns} } case 404: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:2165 +//line sql.y:2166 { yyVAL.indexHints = &IndexHints{Type: ForceStr, Indexes: yyDollar[4].columns} } case 405: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2170 +//line sql.y:2171 { yyVAL.expr = nil } case 406: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2174 +//line sql.y:2175 { yyVAL.expr = yyDollar[2].expr } case 407: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2180 +//line sql.y:2181 { yyVAL.expr = yyDollar[1].expr } case 408: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2184 +//line sql.y:2185 { yyVAL.expr = &AndExpr{Left: yyDollar[1].expr, Right: yyDollar[3].expr} } case 409: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2188 +//line sql.y:2189 { yyVAL.expr = &OrExpr{Left: yyDollar[1].expr, Right: yyDollar[3].expr} } case 410: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2192 +//line sql.y:2193 { yyVAL.expr = &NotExpr{Expr: yyDollar[2].expr} } case 411: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2196 +//line sql.y:2197 { yyVAL.expr = &IsExpr{Operator: yyDollar[3].str, Expr: yyDollar[1].expr} } case 412: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2200 +//line sql.y:2201 { yyVAL.expr = yyDollar[1].expr } case 413: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2204 +//line sql.y:2205 { yyVAL.expr = &Default{ColName: yyDollar[2].str} } case 414: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2210 +//line sql.y:2211 { yyVAL.str = "" } case 415: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2214 +//line sql.y:2215 { yyVAL.str = string(yyDollar[2].colIdent.String()) } case 416: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2220 +//line sql.y:2221 { yyVAL.boolVal = BoolVal(true) } case 417: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2224 +//line sql.y:2225 { yyVAL.boolVal = BoolVal(false) } case 418: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2230 +//line sql.y:2231 { yyVAL.expr = &ComparisonExpr{Left: yyDollar[1].expr, Operator: yyDollar[2].str, Right: yyDollar[3].expr} } case 419: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2234 +//line sql.y:2235 { yyVAL.expr = &ComparisonExpr{Left: yyDollar[1].expr, Operator: InStr, Right: yyDollar[3].colTuple} } case 420: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2238 +//line sql.y:2239 { yyVAL.expr = &ComparisonExpr{Left: yyDollar[1].expr, Operator: NotInStr, Right: yyDollar[4].colTuple} } case 421: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2242 +//line sql.y:2243 { yyVAL.expr = &ComparisonExpr{Left: yyDollar[1].expr, Operator: LikeStr, Right: yyDollar[3].expr, Escape: yyDollar[4].expr} } case 422: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:2246 +//line sql.y:2247 { yyVAL.expr = &ComparisonExpr{Left: yyDollar[1].expr, Operator: NotLikeStr, Right: yyDollar[4].expr, Escape: yyDollar[5].expr} } case 423: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2250 +//line sql.y:2251 { yyVAL.expr = &ComparisonExpr{Left: yyDollar[1].expr, Operator: RegexpStr, Right: yyDollar[3].expr} } case 424: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2254 +//line sql.y:2255 { yyVAL.expr = &ComparisonExpr{Left: yyDollar[1].expr, Operator: NotRegexpStr, Right: yyDollar[4].expr} } case 425: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:2258 +//line sql.y:2259 { yyVAL.expr = &RangeCond{Left: yyDollar[1].expr, Operator: BetweenStr, From: yyDollar[3].expr, To: yyDollar[5].expr} } case 426: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:2262 +//line sql.y:2263 { yyVAL.expr = &RangeCond{Left: yyDollar[1].expr, Operator: NotBetweenStr, From: yyDollar[4].expr, To: yyDollar[6].expr} } case 427: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2266 +//line sql.y:2267 { yyVAL.expr = &ExistsExpr{Subquery: yyDollar[2].subquery} } case 428: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2272 +//line sql.y:2273 { yyVAL.str = IsNullStr } case 429: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2276 +//line sql.y:2277 { yyVAL.str = IsNotNullStr } case 430: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2280 +//line sql.y:2281 { yyVAL.str = IsTrueStr } case 431: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2284 +//line sql.y:2285 { yyVAL.str = IsNotTrueStr } case 432: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2288 +//line sql.y:2289 { yyVAL.str = IsFalseStr } case 433: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2292 +//line sql.y:2293 { yyVAL.str = IsNotFalseStr } case 434: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2298 +//line sql.y:2299 { yyVAL.str = EqualStr } case 435: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2302 +//line sql.y:2303 { yyVAL.str = LessThanStr } case 436: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2306 +//line sql.y:2307 { yyVAL.str = GreaterThanStr } case 437: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2310 +//line sql.y:2311 { yyVAL.str = LessEqualStr } case 438: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2314 +//line sql.y:2315 { yyVAL.str = GreaterEqualStr } case 439: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2318 +//line sql.y:2319 { yyVAL.str = NotEqualStr } case 440: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2322 +//line sql.y:2323 { yyVAL.str = NullSafeEqualStr } case 441: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2327 +//line sql.y:2328 { yyVAL.expr = nil } case 442: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2331 +//line sql.y:2332 { yyVAL.expr = yyDollar[2].expr } case 443: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2337 +//line sql.y:2338 { yyVAL.colTuple = yyDollar[1].valTuple } case 444: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2341 +//line sql.y:2342 { yyVAL.colTuple = yyDollar[1].subquery } case 445: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2345 +//line sql.y:2346 { yyVAL.colTuple = ListArg(yyDollar[1].bytes) } case 446: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2351 +//line sql.y:2352 { yyVAL.subquery = &Subquery{yyDollar[2].selStmt} } case 447: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2357 +//line sql.y:2358 { yyVAL.exprs = Exprs{yyDollar[1].expr} } case 448: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2361 +//line sql.y:2362 { yyVAL.exprs = append(yyDollar[1].exprs, yyDollar[3].expr) } case 449: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2367 +//line sql.y:2368 { yyVAL.expr = yyDollar[1].expr } case 450: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2371 +//line sql.y:2372 { yyVAL.expr = yyDollar[1].boolVal } case 451: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2375 +//line sql.y:2376 { yyVAL.expr = yyDollar[1].colName } case 452: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2379 +//line sql.y:2380 { yyVAL.expr = yyDollar[1].expr } case 453: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2383 +//line sql.y:2384 { yyVAL.expr = yyDollar[1].subquery } case 454: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2387 +//line sql.y:2388 { yyVAL.expr = &BinaryExpr{Left: yyDollar[1].expr, Operator: BitAndStr, Right: yyDollar[3].expr} } case 455: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2391 +//line sql.y:2392 { yyVAL.expr = &BinaryExpr{Left: yyDollar[1].expr, Operator: BitOrStr, Right: yyDollar[3].expr} } case 456: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2395 +//line sql.y:2396 { yyVAL.expr = &BinaryExpr{Left: yyDollar[1].expr, Operator: BitXorStr, Right: yyDollar[3].expr} } case 457: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2399 +//line sql.y:2400 { yyVAL.expr = &BinaryExpr{Left: yyDollar[1].expr, Operator: PlusStr, Right: yyDollar[3].expr} } case 458: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2403 +//line sql.y:2404 { yyVAL.expr = &BinaryExpr{Left: yyDollar[1].expr, Operator: MinusStr, Right: yyDollar[3].expr} } case 459: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2407 +//line sql.y:2408 { yyVAL.expr = &BinaryExpr{Left: yyDollar[1].expr, Operator: MultStr, Right: yyDollar[3].expr} } case 460: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2411 +//line sql.y:2412 { yyVAL.expr = &BinaryExpr{Left: yyDollar[1].expr, Operator: DivStr, Right: yyDollar[3].expr} } case 461: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2415 +//line sql.y:2416 { yyVAL.expr = &BinaryExpr{Left: yyDollar[1].expr, Operator: IntDivStr, Right: yyDollar[3].expr} } case 462: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2419 +//line sql.y:2420 { yyVAL.expr = &BinaryExpr{Left: yyDollar[1].expr, Operator: ModStr, Right: yyDollar[3].expr} } case 463: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2423 +//line sql.y:2424 { yyVAL.expr = &BinaryExpr{Left: yyDollar[1].expr, Operator: ModStr, Right: yyDollar[3].expr} } case 464: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2427 +//line sql.y:2428 { yyVAL.expr = &BinaryExpr{Left: yyDollar[1].expr, Operator: ShiftLeftStr, Right: yyDollar[3].expr} } case 465: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2431 +//line sql.y:2432 { yyVAL.expr = &BinaryExpr{Left: yyDollar[1].expr, Operator: ShiftRightStr, Right: yyDollar[3].expr} } case 466: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2435 +//line sql.y:2436 { yyVAL.expr = &BinaryExpr{Left: yyDollar[1].colName, Operator: JSONExtractOp, Right: yyDollar[3].expr} } case 467: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2439 +//line sql.y:2440 { yyVAL.expr = &BinaryExpr{Left: yyDollar[1].colName, Operator: JSONUnquoteExtractOp, Right: yyDollar[3].expr} } case 468: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2443 +//line sql.y:2444 { yyVAL.expr = &CollateExpr{Expr: yyDollar[1].expr, Charset: yyDollar[3].str} } case 469: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2447 +//line sql.y:2448 { yyVAL.expr = &UnaryExpr{Operator: BinaryStr, Expr: yyDollar[2].expr} } case 470: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2451 +//line sql.y:2452 { yyVAL.expr = &UnaryExpr{Operator: UBinaryStr, Expr: yyDollar[2].expr} } case 471: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2455 +//line sql.y:2456 { yyVAL.expr = &UnaryExpr{Operator: Utf8mb4Str, Expr: yyDollar[2].expr} } case 472: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2459 +//line sql.y:2460 { if num, ok := yyDollar[2].expr.(*SQLVal); ok && num.Type == IntVal { yyVAL.expr = num @@ -6327,7 +6327,7 @@ yydefault: } case 473: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2467 +//line sql.y:2468 { if num, ok := yyDollar[2].expr.(*SQLVal); ok && num.Type == IntVal { // Handle double negative @@ -6343,19 +6343,19 @@ yydefault: } case 474: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2481 +//line sql.y:2482 { yyVAL.expr = &UnaryExpr{Operator: TildaStr, Expr: yyDollar[2].expr} } case 475: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2485 +//line sql.y:2486 { yyVAL.expr = &UnaryExpr{Operator: BangStr, Expr: yyDollar[2].expr} } case 476: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2489 +//line sql.y:2490 { // This rule prevents the usage of INTERVAL // as a function. If support is needed for that, @@ -6365,325 +6365,325 @@ yydefault: } case 481: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2507 +//line sql.y:2508 { yyVAL.expr = &FuncExpr{Name: yyDollar[1].colIdent, Exprs: yyDollar[3].selectExprs} } case 482: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:2511 +//line sql.y:2512 { yyVAL.expr = &FuncExpr{Name: yyDollar[1].colIdent, Distinct: true, Exprs: yyDollar[4].selectExprs} } case 483: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:2515 +//line sql.y:2516 { yyVAL.expr = &FuncExpr{Name: yyDollar[1].colIdent, Distinct: true, Exprs: yyDollar[4].selectExprs} } case 484: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:2519 +//line sql.y:2520 { yyVAL.expr = &FuncExpr{Qualifier: yyDollar[1].tableIdent, Name: yyDollar[3].colIdent, Exprs: yyDollar[5].selectExprs} } case 485: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2529 +//line sql.y:2530 { yyVAL.expr = &FuncExpr{Name: NewColIdent("left"), Exprs: yyDollar[3].selectExprs} } case 486: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2533 +//line sql.y:2534 { yyVAL.expr = &FuncExpr{Name: NewColIdent("right"), Exprs: yyDollar[3].selectExprs} } case 487: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:2537 +//line sql.y:2538 { yyVAL.expr = &ConvertExpr{Expr: yyDollar[3].expr, Type: yyDollar[5].convertType} } case 488: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:2541 +//line sql.y:2542 { yyVAL.expr = &ConvertExpr{Expr: yyDollar[3].expr, Type: yyDollar[5].convertType} } case 489: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:2545 +//line sql.y:2546 { yyVAL.expr = &ConvertUsingExpr{Expr: yyDollar[3].expr, Type: yyDollar[5].str} } case 490: yyDollar = yyS[yypt-8 : yypt+1] -//line sql.y:2549 +//line sql.y:2550 { yyVAL.expr = &SubstrExpr{Name: yyDollar[3].colName, From: yyDollar[5].expr, To: yyDollar[7].expr} } case 491: yyDollar = yyS[yypt-8 : yypt+1] -//line sql.y:2553 +//line sql.y:2554 { yyVAL.expr = &SubstrExpr{Name: yyDollar[3].colName, From: yyDollar[5].expr, To: yyDollar[7].expr} } case 492: yyDollar = yyS[yypt-8 : yypt+1] -//line sql.y:2557 +//line sql.y:2558 { yyVAL.expr = &SubstrExpr{StrVal: NewStrVal(yyDollar[3].bytes), From: yyDollar[5].expr, To: yyDollar[7].expr} } case 493: yyDollar = yyS[yypt-8 : yypt+1] -//line sql.y:2561 +//line sql.y:2562 { yyVAL.expr = &SubstrExpr{StrVal: NewStrVal(yyDollar[3].bytes), From: yyDollar[5].expr, To: yyDollar[7].expr} } case 494: yyDollar = yyS[yypt-9 : yypt+1] -//line sql.y:2565 +//line sql.y:2566 { yyVAL.expr = &MatchExpr{Columns: yyDollar[3].selectExprs, Expr: yyDollar[7].expr, Option: yyDollar[8].str} } case 495: yyDollar = yyS[yypt-8 : yypt+1] -//line sql.y:2569 +//line sql.y:2570 { yyVAL.expr = &GroupConcatExpr{Distinct: yyDollar[3].str, Exprs: yyDollar[4].selectExprs, OrderBy: yyDollar[5].orderBy, Separator: yyDollar[6].str, Limit: yyDollar[7].limit} } case 496: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:2573 +//line sql.y:2574 { yyVAL.expr = &CaseExpr{Expr: yyDollar[2].expr, Whens: yyDollar[3].whens, Else: yyDollar[4].expr} } case 497: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2577 +//line sql.y:2578 { yyVAL.expr = &ValuesFuncExpr{Name: yyDollar[3].colName} } case 498: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2587 +//line sql.y:2588 { yyVAL.expr = &FuncExpr{Name: NewColIdent("current_timestamp")} } case 499: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2591 +//line sql.y:2592 { yyVAL.expr = &FuncExpr{Name: NewColIdent("utc_timestamp")} } case 500: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2595 +//line sql.y:2596 { yyVAL.expr = &FuncExpr{Name: NewColIdent("utc_time")} } case 501: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2600 +//line sql.y:2601 { yyVAL.expr = &FuncExpr{Name: NewColIdent("utc_date")} } case 502: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2605 +//line sql.y:2606 { yyVAL.expr = &FuncExpr{Name: NewColIdent("localtime")} } case 503: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2610 +//line sql.y:2611 { yyVAL.expr = &FuncExpr{Name: NewColIdent("localtimestamp")} } case 504: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2616 +//line sql.y:2617 { yyVAL.expr = &FuncExpr{Name: NewColIdent("current_date")} } case 505: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2621 +//line sql.y:2622 { yyVAL.expr = &FuncExpr{Name: NewColIdent("current_time")} } case 506: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2626 +//line sql.y:2627 { yyVAL.expr = &CurTimeFuncExpr{Name: NewColIdent("current_timestamp"), Fsp: yyDollar[2].expr} } case 507: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2630 +//line sql.y:2631 { yyVAL.expr = &CurTimeFuncExpr{Name: NewColIdent("utc_timestamp"), Fsp: yyDollar[2].expr} } case 508: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2634 +//line sql.y:2635 { yyVAL.expr = &CurTimeFuncExpr{Name: NewColIdent("utc_time"), Fsp: yyDollar[2].expr} } case 509: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2639 +//line sql.y:2640 { yyVAL.expr = &CurTimeFuncExpr{Name: NewColIdent("localtime"), Fsp: yyDollar[2].expr} } case 510: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2644 +//line sql.y:2645 { yyVAL.expr = &CurTimeFuncExpr{Name: NewColIdent("localtimestamp"), Fsp: yyDollar[2].expr} } case 511: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2649 +//line sql.y:2650 { yyVAL.expr = &CurTimeFuncExpr{Name: NewColIdent("current_time"), Fsp: yyDollar[2].expr} } case 512: yyDollar = yyS[yypt-8 : yypt+1] -//line sql.y:2653 +//line sql.y:2654 { yyVAL.expr = &TimestampFuncExpr{Name: string("timestampadd"), Unit: yyDollar[3].colIdent.String(), Expr1: yyDollar[5].expr, Expr2: yyDollar[7].expr} } case 513: yyDollar = yyS[yypt-8 : yypt+1] -//line sql.y:2657 +//line sql.y:2658 { yyVAL.expr = &TimestampFuncExpr{Name: string("timestampdiff"), Unit: yyDollar[3].colIdent.String(), Expr1: yyDollar[5].expr, Expr2: yyDollar[7].expr} } case 516: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2667 +//line sql.y:2668 { yyVAL.expr = yyDollar[2].expr } case 517: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2677 +//line sql.y:2678 { yyVAL.expr = &FuncExpr{Name: NewColIdent("if"), Exprs: yyDollar[3].selectExprs} } case 518: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2681 +//line sql.y:2682 { yyVAL.expr = &FuncExpr{Name: NewColIdent("database"), Exprs: yyDollar[3].selectExprs} } case 519: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2685 +//line sql.y:2686 { yyVAL.expr = &FuncExpr{Name: NewColIdent("schema"), Exprs: yyDollar[3].selectExprs} } case 520: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2689 +//line sql.y:2690 { yyVAL.expr = &FuncExpr{Name: NewColIdent("mod"), Exprs: yyDollar[3].selectExprs} } case 521: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2693 +//line sql.y:2694 { yyVAL.expr = &FuncExpr{Name: NewColIdent("replace"), Exprs: yyDollar[3].selectExprs} } case 522: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2697 +//line sql.y:2698 { yyVAL.expr = &FuncExpr{Name: NewColIdent("substr"), Exprs: yyDollar[3].selectExprs} } case 523: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2701 +//line sql.y:2702 { yyVAL.expr = &FuncExpr{Name: NewColIdent("substr"), Exprs: yyDollar[3].selectExprs} } case 524: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2707 +//line sql.y:2708 { yyVAL.str = "" } case 525: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2711 +//line sql.y:2712 { yyVAL.str = BooleanModeStr } case 526: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2715 +//line sql.y:2716 { yyVAL.str = NaturalLanguageModeStr } case 527: yyDollar = yyS[yypt-7 : yypt+1] -//line sql.y:2719 +//line sql.y:2720 { yyVAL.str = NaturalLanguageModeWithQueryExpansionStr } case 528: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2723 +//line sql.y:2724 { yyVAL.str = QueryExpansionStr } case 529: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2729 +//line sql.y:2730 { yyVAL.str = string(yyDollar[1].colIdent.String()) } case 530: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2733 +//line sql.y:2734 { yyVAL.str = string(yyDollar[1].bytes) } case 531: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2739 +//line sql.y:2740 { yyVAL.convertType = &ConvertType{Type: string(yyDollar[1].bytes), Length: yyDollar[2].sqlVal} } case 532: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2743 +//line sql.y:2744 { yyVAL.convertType = &ConvertType{Type: string(yyDollar[1].bytes), Length: yyDollar[2].sqlVal, Charset: yyDollar[3].str, Operator: CharacterSetStr} } case 533: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2747 +//line sql.y:2748 { yyVAL.convertType = &ConvertType{Type: string(yyDollar[1].bytes), Length: yyDollar[2].sqlVal, Charset: string(yyDollar[3].colIdent.String())} } case 534: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2751 +//line sql.y:2752 { yyVAL.convertType = &ConvertType{Type: string(yyDollar[1].bytes)} } case 535: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2755 +//line sql.y:2756 { yyVAL.convertType = &ConvertType{Type: string(yyDollar[1].bytes), Length: yyDollar[2].sqlVal} } case 536: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2759 +//line sql.y:2760 { yyVAL.convertType = &ConvertType{Type: string(yyDollar[1].bytes)} yyVAL.convertType.Length = yyDollar[2].LengthScaleOption.Length @@ -6691,169 +6691,169 @@ yydefault: } case 537: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2765 +//line sql.y:2766 { yyVAL.convertType = &ConvertType{Type: string(yyDollar[1].bytes)} } case 538: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2769 +//line sql.y:2770 { yyVAL.convertType = &ConvertType{Type: string(yyDollar[1].bytes), Length: yyDollar[2].sqlVal} } case 539: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2773 +//line sql.y:2774 { yyVAL.convertType = &ConvertType{Type: string(yyDollar[1].bytes)} } case 540: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2777 +//line sql.y:2778 { yyVAL.convertType = &ConvertType{Type: string(yyDollar[1].bytes)} } case 541: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2781 +//line sql.y:2782 { yyVAL.convertType = &ConvertType{Type: string(yyDollar[1].bytes), Length: yyDollar[2].sqlVal} } case 542: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2785 +//line sql.y:2786 { yyVAL.convertType = &ConvertType{Type: string(yyDollar[1].bytes)} } case 543: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2789 +//line sql.y:2790 { yyVAL.convertType = &ConvertType{Type: string(yyDollar[1].bytes)} } case 544: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2794 +//line sql.y:2795 { yyVAL.expr = nil } case 545: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2798 +//line sql.y:2799 { yyVAL.expr = yyDollar[1].expr } case 546: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2803 +//line sql.y:2804 { yyVAL.str = string("") } case 547: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2807 +//line sql.y:2808 { yyVAL.str = " separator '" + string(yyDollar[2].bytes) + "'" } case 548: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2813 +//line sql.y:2814 { yyVAL.whens = []*When{yyDollar[1].when} } case 549: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2817 +//line sql.y:2818 { yyVAL.whens = append(yyDollar[1].whens, yyDollar[2].when) } case 550: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2823 +//line sql.y:2824 { yyVAL.when = &When{Cond: yyDollar[2].expr, Val: yyDollar[4].expr} } case 551: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2828 +//line sql.y:2829 { yyVAL.expr = nil } case 552: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2832 +//line sql.y:2833 { yyVAL.expr = yyDollar[2].expr } case 553: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2838 +//line sql.y:2839 { yyVAL.colName = &ColName{Name: yyDollar[1].colIdent} } case 554: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2842 +//line sql.y:2843 { yyVAL.colName = &ColName{Qualifier: TableName{Name: yyDollar[1].tableIdent}, Name: yyDollar[3].colIdent} } case 555: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:2846 +//line sql.y:2847 { yyVAL.colName = &ColName{Qualifier: TableName{Qualifier: yyDollar[1].tableIdent, Name: yyDollar[3].tableIdent}, Name: yyDollar[5].colIdent} } case 556: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2852 +//line sql.y:2853 { yyVAL.expr = NewStrVal(yyDollar[1].bytes) } case 557: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2856 +//line sql.y:2857 { yyVAL.expr = NewHexVal(yyDollar[1].bytes) } case 558: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2860 +//line sql.y:2861 { yyVAL.expr = NewBitVal(yyDollar[1].bytes) } case 559: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2864 +//line sql.y:2865 { yyVAL.expr = NewIntVal(yyDollar[1].bytes) } case 560: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2868 +//line sql.y:2869 { yyVAL.expr = NewFloatVal(yyDollar[1].bytes) } case 561: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2872 +//line sql.y:2873 { yyVAL.expr = NewHexNum(yyDollar[1].bytes) } case 562: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2876 +//line sql.y:2877 { yyVAL.expr = NewValArg(yyDollar[1].bytes) } case 563: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2880 +//line sql.y:2881 { yyVAL.expr = &NullVal{} } case 564: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2886 +//line sql.y:2887 { // TODO(sougou): Deprecate this construct. if yyDollar[1].colIdent.Lowered() != "value" { @@ -6864,237 +6864,237 @@ yydefault: } case 565: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2895 +//line sql.y:2896 { yyVAL.expr = NewIntVal(yyDollar[1].bytes) } case 566: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2899 +//line sql.y:2900 { yyVAL.expr = NewValArg(yyDollar[1].bytes) } case 567: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2904 +//line sql.y:2905 { yyVAL.exprs = nil } case 568: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2908 +//line sql.y:2909 { yyVAL.exprs = yyDollar[3].exprs } case 569: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2913 +//line sql.y:2914 { yyVAL.expr = nil } case 570: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2917 +//line sql.y:2918 { yyVAL.expr = yyDollar[2].expr } case 571: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2922 +//line sql.y:2923 { yyVAL.orderBy = nil } case 572: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2926 +//line sql.y:2927 { yyVAL.orderBy = yyDollar[3].orderBy } case 573: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2932 +//line sql.y:2933 { yyVAL.orderBy = OrderBy{yyDollar[1].order} } case 574: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:2936 +//line sql.y:2937 { yyVAL.orderBy = append(yyDollar[1].orderBy, yyDollar[3].order) } case 575: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2942 +//line sql.y:2943 { yyVAL.order = &Order{Expr: yyDollar[1].expr, Direction: yyDollar[2].str} } case 576: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2947 +//line sql.y:2948 { yyVAL.str = AscScr } case 577: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2951 +//line sql.y:2952 { yyVAL.str = AscScr } case 578: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:2955 +//line sql.y:2956 { yyVAL.str = DescScr } case 579: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2960 +//line sql.y:2961 { yyVAL.limit = nil } case 580: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2964 +//line sql.y:2965 { yyVAL.limit = &Limit{Rowcount: yyDollar[2].expr} } case 581: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2968 +//line sql.y:2969 { yyVAL.limit = &Limit{Offset: yyDollar[2].expr, Rowcount: yyDollar[4].expr} } case 582: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2972 +//line sql.y:2973 { yyVAL.limit = &Limit{Offset: yyDollar[4].expr, Rowcount: yyDollar[2].expr} } case 583: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:2977 +//line sql.y:2978 { yyVAL.str = "" } case 584: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2981 +//line sql.y:2982 { yyVAL.str = ForUpdateStr } case 585: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:2985 +//line sql.y:2986 { yyVAL.str = ShareModeStr } case 586: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:2998 +//line sql.y:2999 { yyVAL.ins = &Insert{Rows: yyDollar[2].values} } case 587: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3002 +//line sql.y:3003 { yyVAL.ins = &Insert{Rows: yyDollar[1].selStmt} } case 588: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:3006 +//line sql.y:3007 { // Drop the redundant parenthesis. yyVAL.ins = &Insert{Rows: yyDollar[2].selStmt} } case 589: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:3011 +//line sql.y:3012 { yyVAL.ins = &Insert{Columns: yyDollar[2].columns, Rows: yyDollar[5].values} } case 590: yyDollar = yyS[yypt-4 : yypt+1] -//line sql.y:3015 +//line sql.y:3016 { yyVAL.ins = &Insert{Columns: yyDollar[2].columns, Rows: yyDollar[4].selStmt} } case 591: yyDollar = yyS[yypt-6 : yypt+1] -//line sql.y:3019 +//line sql.y:3020 { // Drop the redundant parenthesis. yyVAL.ins = &Insert{Columns: yyDollar[2].columns, Rows: yyDollar[5].selStmt} } case 592: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3026 +//line sql.y:3027 { yyVAL.columns = Columns{yyDollar[1].colIdent} } case 593: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:3030 +//line sql.y:3031 { yyVAL.columns = Columns{yyDollar[3].colIdent} } case 594: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:3034 +//line sql.y:3035 { yyVAL.columns = append(yyVAL.columns, yyDollar[3].colIdent) } case 595: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:3038 +//line sql.y:3039 { yyVAL.columns = append(yyVAL.columns, yyDollar[5].colIdent) } case 596: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:3043 +//line sql.y:3044 { yyVAL.updateExprs = nil } case 597: yyDollar = yyS[yypt-5 : yypt+1] -//line sql.y:3047 +//line sql.y:3048 { yyVAL.updateExprs = yyDollar[5].updateExprs } case 598: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3053 +//line sql.y:3054 { yyVAL.values = Values{yyDollar[1].valTuple} } case 599: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:3057 +//line sql.y:3058 { yyVAL.values = append(yyDollar[1].values, yyDollar[3].valTuple) } case 600: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3063 +//line sql.y:3064 { yyVAL.valTuple = yyDollar[1].valTuple } case 601: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:3067 +//line sql.y:3068 { yyVAL.valTuple = ValTuple{} } case 602: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:3073 +//line sql.y:3074 { yyVAL.valTuple = ValTuple(yyDollar[2].exprs) } case 603: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3079 +//line sql.y:3080 { if len(yyDollar[1].valTuple) == 1 { yyVAL.expr = yyDollar[1].valTuple[0] @@ -7104,277 +7104,277 @@ yydefault: } case 604: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3089 +//line sql.y:3090 { yyVAL.updateExprs = UpdateExprs{yyDollar[1].updateExpr} } case 605: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:3093 +//line sql.y:3094 { yyVAL.updateExprs = append(yyDollar[1].updateExprs, yyDollar[3].updateExpr) } case 606: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:3099 +//line sql.y:3100 { yyVAL.updateExpr = &UpdateExpr{Name: yyDollar[1].colName, Expr: yyDollar[3].expr} } case 607: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3105 +//line sql.y:3106 { yyVAL.setExprs = SetExprs{yyDollar[1].setExpr} } case 608: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:3109 +//line sql.y:3110 { yyVAL.setExprs = append(yyDollar[1].setExprs, yyDollar[3].setExpr) } case 609: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:3115 +//line sql.y:3116 { yyVAL.setExpr = &SetExpr{Name: yyDollar[1].colIdent, Expr: NewStrVal([]byte("on"))} } case 610: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:3119 +//line sql.y:3120 { yyVAL.setExpr = &SetExpr{Name: yyDollar[1].colIdent, Expr: NewStrVal([]byte("off"))} } case 611: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:3123 +//line sql.y:3124 { yyVAL.setExpr = &SetExpr{Name: yyDollar[1].colIdent, Expr: yyDollar[3].expr} } case 612: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:3127 +//line sql.y:3128 { yyVAL.setExpr = &SetExpr{Name: NewColIdent(string(yyDollar[1].bytes)), Expr: yyDollar[2].expr} } case 614: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:3134 +//line sql.y:3135 { yyVAL.bytes = []byte("charset") } case 616: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3141 +//line sql.y:3142 { yyVAL.expr = NewStrVal([]byte(yyDollar[1].colIdent.String())) } case 617: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3145 +//line sql.y:3146 { yyVAL.expr = NewStrVal(yyDollar[1].bytes) } case 618: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3149 +//line sql.y:3150 { yyVAL.expr = &Default{} } case 621: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:3158 +//line sql.y:3159 { yyVAL.byt = 0 } case 622: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:3160 +//line sql.y:3161 { yyVAL.byt = 1 } case 623: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:3163 +//line sql.y:3164 { yyVAL.empty = struct{}{} } case 624: yyDollar = yyS[yypt-3 : yypt+1] -//line sql.y:3165 +//line sql.y:3166 { yyVAL.empty = struct{}{} } case 625: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:3168 +//line sql.y:3169 { yyVAL.str = "" } case 626: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3170 +//line sql.y:3171 { yyVAL.str = IgnoreStr } case 627: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3174 +//line sql.y:3175 { yyVAL.empty = struct{}{} } case 628: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3176 +//line sql.y:3177 { yyVAL.empty = struct{}{} } case 629: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3178 +//line sql.y:3179 { yyVAL.empty = struct{}{} } case 630: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3180 +//line sql.y:3181 { yyVAL.empty = struct{}{} } case 631: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3182 +//line sql.y:3183 { yyVAL.empty = struct{}{} } case 632: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3184 +//line sql.y:3185 { yyVAL.empty = struct{}{} } case 633: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3186 +//line sql.y:3187 { yyVAL.empty = struct{}{} } case 634: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3188 +//line sql.y:3189 { yyVAL.empty = struct{}{} } case 635: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3190 +//line sql.y:3191 { yyVAL.empty = struct{}{} } case 636: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3192 +//line sql.y:3193 { yyVAL.empty = struct{}{} } case 637: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:3195 +//line sql.y:3196 { yyVAL.empty = struct{}{} } case 638: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3197 +//line sql.y:3198 { yyVAL.empty = struct{}{} } case 639: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3199 +//line sql.y:3200 { yyVAL.empty = struct{}{} } case 640: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3203 +//line sql.y:3204 { yyVAL.empty = struct{}{} } case 641: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3205 +//line sql.y:3206 { yyVAL.empty = struct{}{} } case 642: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:3208 +//line sql.y:3209 { yyVAL.empty = struct{}{} } case 643: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3210 +//line sql.y:3211 { yyVAL.empty = struct{}{} } case 644: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3212 +//line sql.y:3213 { yyVAL.empty = struct{}{} } case 645: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:3215 +//line sql.y:3216 { yyVAL.colIdent = ColIdent{} } case 646: yyDollar = yyS[yypt-2 : yypt+1] -//line sql.y:3217 +//line sql.y:3218 { yyVAL.colIdent = yyDollar[2].colIdent } case 647: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3221 +//line sql.y:3222 { yyVAL.colIdent = yyDollar[1].colIdent } case 648: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3225 +//line sql.y:3226 { yyVAL.colIdent = NewColIdent(string(yyDollar[1].bytes)) } case 650: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3232 +//line sql.y:3233 { yyVAL.colIdent = NewColIdent(string(yyDollar[1].bytes)) } case 651: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3238 +//line sql.y:3239 { yyVAL.tableIdent = NewTableIdent(string(yyDollar[1].colIdent.String())) } case 652: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3242 +//line sql.y:3243 { yyVAL.tableIdent = NewTableIdent(string(yyDollar[1].bytes)) } case 654: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3249 +//line sql.y:3250 { yyVAL.tableIdent = NewTableIdent(string(yyDollar[1].bytes)) } case 942: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3562 +//line sql.y:3563 { if incNesting(yylex) { yylex.Error("max nesting level reached") @@ -7383,31 +7383,31 @@ yydefault: } case 943: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3571 +//line sql.y:3572 { decNesting(yylex) } case 944: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:3576 +//line sql.y:3577 { skipToEnd(yylex) } case 945: yyDollar = yyS[yypt-0 : yypt+1] -//line sql.y:3581 +//line sql.y:3582 { skipToEnd(yylex) } case 946: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3585 +//line sql.y:3586 { skipToEnd(yylex) } case 947: yyDollar = yyS[yypt-1 : yypt+1] -//line sql.y:3589 +//line sql.y:3590 { skipToEnd(yylex) } diff --git a/go/vt/sqlparser/tracked_buffer.go b/go/vt/sqlparser/tracked_buffer.go index de366d47564..8f5fef1f689 100644 --- a/go/vt/sqlparser/tracked_buffer.go +++ b/go/vt/sqlparser/tracked_buffer.go @@ -154,11 +154,28 @@ func (buf *TrackedBuffer) formatter(node SQLNode) { } func needParens(op, val Expr) bool { + if areBothISExpr(op, val) { + return true + } + opBinding := precedenceFor(op) valBinding := precedenceFor(val) + return !(opBinding == Syntactic || valBinding == Syntactic) && valBinding > opBinding } +func areBothISExpr(op Expr, val Expr) bool { + _, isOpIS := op.(*IsExpr) + if isOpIS { + _, isValIS := val.(*IsExpr) + if isValIS { + // when using IS on an IS op, we need special handling + return true + } + } + return false +} + // WriteArg writes a value argument into the buffer along with // tracking information for future substitutions. arg must contain // the ":" or "::" prefix. From 1f82d5f3aa69e97a04891f1c5a7c7dc63e493e12 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Fri, 27 Mar 2020 19:30:31 +0100 Subject: [PATCH 70/85] Update migrater type and related objects and comments to reflect the renaming Signed-off-by: Rohit Nayak --- go/vt/wrangler/stream_migrater.go | 80 ++-- go/vt/wrangler/stream_migrater_test.go | 4 +- go/vt/wrangler/traffic_switcher.go | 548 ++++++++++++------------ go/vt/wrangler/traffic_switcher_test.go | 26 +- go/vt/wrangler/vdiff.go | 52 +-- 5 files changed, 355 insertions(+), 355 deletions(-) diff --git a/go/vt/wrangler/stream_migrater.go b/go/vt/wrangler/stream_migrater.go index 39ad5923713..e5edad90878 100644 --- a/go/vt/wrangler/stream_migrater.go +++ b/go/vt/wrangler/stream_migrater.go @@ -41,7 +41,7 @@ type streamMigrater struct { streams map[string][]*vrStream workflows []string templates []*vrStream - mi *migrater + ts *trafficSwitcher } type vrStream struct { @@ -51,9 +51,9 @@ type vrStream struct { pos mysql.Position } -func buildStreamMigrater(ctx context.Context, mi *migrater, cancelMigrate bool) (*streamMigrater, error) { - sm := &streamMigrater{mi: mi} - if sm.mi.migrationType == binlogdatapb.MigrationType_TABLES { +func buildStreamMigrater(ctx context.Context, ts *trafficSwitcher, cancelMigrate bool) (*streamMigrater, error) { + sm := &streamMigrater{ts: ts} + if sm.ts.migrationType == binlogdatapb.MigrationType_TABLES { // Source streams should be stopped only for shard migrations. return sm, nil } @@ -78,7 +78,7 @@ func buildStreamMigrater(ctx context.Context, mi *migrater, cancelMigrate bool) func (sm *streamMigrater) readSourceStreams(ctx context.Context, cancelMigrate bool) (map[string][]*vrStream, error) { streams := make(map[string][]*vrStream) var mu sync.Mutex - err := sm.mi.forAllSources(func(source *miSource) error { + err := sm.ts.forAllSources(func(source *tsSource) error { if !cancelMigrate { // This flow protects us from the following scenario: When we create streams, // we always do it in two phases. We start them off as Stopped, and then @@ -107,7 +107,7 @@ func (sm *streamMigrater) readSourceStreams(ctx context.Context, cancelMigrate b // No VReplication is running. So, we have no work to do. return nil } - p3qr, err := sm.mi.wr.tmc.VReplicationExec(ctx, source.master.Tablet, fmt.Sprintf("select vrepl_id from _vt.copy_state where vrepl_id in %s", tabletStreamValues(tabletStreams))) + p3qr, err := sm.ts.wr.tmc.VReplicationExec(ctx, source.master.Tablet, fmt.Sprintf("select vrepl_id from _vt.copy_state where vrepl_id in %s", tabletStreamValues(tabletStreams))) if err != nil { return err } @@ -197,7 +197,7 @@ func (sm *streamMigrater) blsIsReference(bls *binlogdatapb.BinlogSource) (bool, } func (sm *streamMigrater) identifyRuleType(rule *binlogdatapb.Rule) (int, error) { - vtable, ok := sm.mi.sourceKSSchema.Tables[rule.Match] + vtable, ok := sm.ts.sourceKSSchema.Tables[rule.Match] if !ok { return 0, fmt.Errorf("table %v not found in vschema", rule.Match) } @@ -212,11 +212,11 @@ func (sm *streamMigrater) identifyRuleType(rule *binlogdatapb.Rule) (int, error) func (sm *streamMigrater) readTabletStreams(ctx context.Context, ti *topo.TabletInfo, constraint string) ([]*vrStream, error) { var query string if constraint == "" { - query = fmt.Sprintf("select id, workflow, source, pos from _vt.vreplication where db_name=%s and workflow != %s", encodeString(ti.DbName()), encodeString(sm.mi.reverseWorkflow)) + query = fmt.Sprintf("select id, workflow, source, pos from _vt.vreplication where db_name=%s and workflow != %s", encodeString(ti.DbName()), encodeString(sm.ts.reverseWorkflow)) } else { - query = fmt.Sprintf("select id, workflow, source, pos from _vt.vreplication where db_name=%s and workflow != %s and %s", encodeString(ti.DbName()), encodeString(sm.mi.reverseWorkflow), constraint) + query = fmt.Sprintf("select id, workflow, source, pos from _vt.vreplication where db_name=%s and workflow != %s and %s", encodeString(ti.DbName()), encodeString(sm.ts.reverseWorkflow), constraint) } - p3qr, err := sm.mi.wr.tmc.VReplicationExec(ctx, ti.Tablet, query) + p3qr, err := sm.ts.wr.tmc.VReplicationExec(ctx, ti.Tablet, query) if err != nil { return nil, err } @@ -232,7 +232,7 @@ func (sm *streamMigrater) readTabletStreams(ctx context.Context, ti *topo.Tablet if workflow == "" { return nil, fmt.Errorf("VReplication streams must have named workflows for migration: shard: %s:%s, stream: %d", ti.Keyspace, ti.Shard, id) } - if workflow == sm.mi.workflow { + if workflow == sm.ts.workflow { return nil, fmt.Errorf("VReplication stream has the same workflow name as the resharding workflow: shard: %s:%s, stream: %d", ti.Keyspace, ti.Shard, id) } var bls binlogdatapb.BinlogSource @@ -244,7 +244,7 @@ func (sm *streamMigrater) readTabletStreams(ctx context.Context, ti *topo.Tablet return nil, vterrors.Wrap(err, "blsIsReference") } if isReference { - sm.mi.wr.Logger().Infof("readTabletStreams: ignoring reference table %+v", bls) + sm.ts.wr.Logger().Infof("readTabletStreams: ignoring reference table %+v", bls) continue } pos, err := mysql.DecodePosition(row[3].ToString()) @@ -264,13 +264,13 @@ func (sm *streamMigrater) readTabletStreams(ctx context.Context, ti *topo.Tablet func (sm *streamMigrater) stopSourceStreams(ctx context.Context) error { stoppedStreams := make(map[string][]*vrStream) var mu sync.Mutex - err := sm.mi.forAllSources(func(source *miSource) error { + err := sm.ts.forAllSources(func(source *tsSource) error { tabletStreams := sm.streams[source.si.ShardName()] if len(tabletStreams) == 0 { return nil } query := fmt.Sprintf("update _vt.vreplication set state='Stopped', message='for cutover' where id in %s", tabletStreamValues(tabletStreams)) - _, err := sm.mi.wr.tmc.VReplicationExec(ctx, source.master.Tablet, query) + _, err := sm.ts.wr.tmc.VReplicationExec(ctx, source.master.Tablet, query) if err != nil { return err } @@ -297,7 +297,7 @@ func (sm *streamMigrater) syncSourceStreams(ctx context.Context) (map[string]mys key := fmt.Sprintf("%s:%s", vrs.bls.Keyspace, vrs.bls.Shard) pos, ok := stopPositions[key] if !ok || vrs.pos.AtLeast(pos) { - sm.mi.wr.Logger().Infof("syncSourceStreams setting stopPositions +%s %+v %d", key, vrs.pos, vrs.id) + sm.ts.wr.Logger().Infof("syncSourceStreams setting stopPositions +%s %+v %d", key, vrs.pos, vrs.id) stopPositions[key] = vrs.pos } } @@ -308,36 +308,36 @@ func (sm *streamMigrater) syncSourceStreams(ctx context.Context) (map[string]mys for _, vrs := range tabletStreams { key := fmt.Sprintf("%s:%s", vrs.bls.Keyspace, vrs.bls.Shard) pos := stopPositions[key] - sm.mi.wr.Logger().Infof("syncSourceStreams before go func +%s %+v %d", key, pos, vrs.id) + sm.ts.wr.Logger().Infof("syncSourceStreams before go func +%s %+v %d", key, pos, vrs.id) if vrs.pos.Equal(pos) { continue } wg.Add(1) go func(vrs *vrStream, shard string, pos mysql.Position) { defer wg.Done() - sm.mi.wr.Logger().Infof("syncSourceStreams beginning of go func %s %s %+v %d", shard, vrs.bls.Shard, pos, vrs.id) + sm.ts.wr.Logger().Infof("syncSourceStreams beginning of go func %s %s %+v %d", shard, vrs.bls.Shard, pos, vrs.id) - si, err := sm.mi.wr.ts.GetShard(ctx, sm.mi.sourceKeyspace, shard) + si, err := sm.ts.wr.ts.GetShard(ctx, sm.ts.sourceKeyspace, shard) if err != nil { allErrors.RecordError(err) return } - master, err := sm.mi.wr.ts.GetTablet(ctx, si.MasterAlias) + master, err := sm.ts.wr.ts.GetTablet(ctx, si.MasterAlias) if err != nil { allErrors.RecordError(err) return } query := fmt.Sprintf("update _vt.vreplication set state='Running', stop_pos='%s', message='synchronizing for cutover' where id=%d", mysql.EncodePosition(pos), vrs.id) - if _, err := sm.mi.wr.tmc.VReplicationExec(ctx, master.Tablet, query); err != nil { + if _, err := sm.ts.wr.tmc.VReplicationExec(ctx, master.Tablet, query); err != nil { allErrors.RecordError(err) return } - sm.mi.wr.Logger().Infof("Waiting for keyspace:shard: %v:%v, position %v", sm.mi.sourceKeyspace, shard, pos) - if err := sm.mi.wr.tmc.VReplicationWaitForPos(ctx, master.Tablet, int(vrs.id), mysql.EncodePosition(pos)); err != nil { + sm.ts.wr.Logger().Infof("Waiting for keyspace:shard: %v:%v, position %v", sm.ts.sourceKeyspace, shard, pos) + if err := sm.ts.wr.tmc.VReplicationWaitForPos(ctx, master.Tablet, int(vrs.id), mysql.EncodePosition(pos)); err != nil { allErrors.RecordError(err) return } - sm.mi.wr.Logger().Infof("Position for keyspace:shard: %v:%v reached", sm.mi.sourceKeyspace, shard) + sm.ts.wr.Logger().Infof("Position for keyspace:shard: %v:%v reached", sm.ts.sourceKeyspace, shard) }(vrs, shard, pos) } } @@ -348,7 +348,7 @@ func (sm *streamMigrater) syncSourceStreams(ctx context.Context) (map[string]mys func (sm *streamMigrater) verifyStreamPositions(ctx context.Context, stopPositions map[string]mysql.Position) ([]string, error) { stoppedStreams := make(map[string][]*vrStream) var mu sync.Mutex - err := sm.mi.forAllSources(func(source *miSource) error { + err := sm.ts.forAllSources(func(source *tsSource) error { tabletStreams := sm.streams[source.si.ShardName()] if len(tabletStreams) == 0 { return nil @@ -448,7 +448,7 @@ func (sm *streamMigrater) templatize(ctx context.Context, tabletStreams []*vrStr } func (sm *streamMigrater) templatizeRule(ctx context.Context, rule *binlogdatapb.Rule) (int, error) { - vtable, ok := sm.mi.sourceKSSchema.Tables[rule.Match] + vtable, ok := sm.ts.sourceKSSchema.Tables[rule.Match] if !ok { return 0, fmt.Errorf("table %v not found in vschema", rule.Match) } @@ -516,7 +516,7 @@ func (sm *streamMigrater) templatizeKeyRange(ctx context.Context, rule *binlogda return nil } // There was no in_keyrange expression. Create a new one. - vtable := sm.mi.sourceKSSchema.Tables[rule.Match] + vtable := sm.ts.sourceKSSchema.Tables[rule.Match] inkr := &sqlparser.FuncExpr{ Name: sqlparser.NewColIdent("in_keyrange"), Exprs: sqlparser.SelectExprs{ @@ -535,7 +535,7 @@ func (sm *streamMigrater) createTargetStreams(ctx context.Context, tmpl []*vrStr if len(tmpl) == 0 { return nil } - return sm.mi.forAllTargets(func(target *miTarget) error { + return sm.ts.forAllTargets(func(target *tsTarget) error { tabletStreams := copyTabletStreams(tmpl) for _, vrs := range tabletStreams { for _, rule := range vrs.bls.Filter.Rules { @@ -552,7 +552,7 @@ func (sm *streamMigrater) createTargetStreams(ctx context.Context, tmpl []*vrStr for _, vrs := range tabletStreams { ig.AddRow(vrs.workflow, vrs.bls, mysql.EncodePosition(vrs.pos), "", "") } - _, err := sm.mi.wr.VReplicationExec(ctx, target.master.Alias, ig.String()) + _, err := sm.ts.wr.VReplicationExec(ctx, target.master.Alias, ig.String()) return err }) } @@ -565,13 +565,13 @@ func (sm *streamMigrater) cancelMigration(ctx context.Context) { // Ignore error. We still want to restart the source streams if deleteTargetStreams fails. _ = sm.deleteTargetStreams(ctx) - err := sm.mi.forAllSources(func(source *miSource) error { - query := fmt.Sprintf("update _vt.vreplication set state='Running', stop_pos=null, message='' where db_name=%s and workflow != %s", encodeString(source.master.DbName()), encodeString(sm.mi.reverseWorkflow)) - _, err := sm.mi.wr.VReplicationExec(ctx, source.master.Alias, query) + err := sm.ts.forAllSources(func(source *tsSource) error { + query := fmt.Sprintf("update _vt.vreplication set state='Running', stop_pos=null, message='' where db_name=%s and workflow != %s", encodeString(source.master.DbName()), encodeString(sm.ts.reverseWorkflow)) + _, err := sm.ts.wr.VReplicationExec(ctx, source.master.Alias, query) return err }) if err != nil { - sm.mi.wr.Logger().Errorf("Cancel migration failed: could not restart source streams: %v", err) + sm.ts.wr.Logger().Errorf("Cancel migration failed: could not restart source streams: %v", err) } } @@ -580,35 +580,35 @@ func (sm *streamMigrater) deleteTargetStreams(ctx context.Context) error { return nil } workflowList := stringListify(sm.workflows) - err := sm.mi.forAllTargets(func(target *miTarget) error { + err := sm.ts.forAllTargets(func(target *tsTarget) error { query := fmt.Sprintf("delete from _vt.vreplication where db_name=%s and workflow in (%s)", encodeString(target.master.DbName()), workflowList) - _, err := sm.mi.wr.VReplicationExec(ctx, target.master.Alias, query) + _, err := sm.ts.wr.VReplicationExec(ctx, target.master.Alias, query) return err }) if err != nil { - sm.mi.wr.Logger().Warningf("Could not delete migrated streams: %v", err) + sm.ts.wr.Logger().Warningf("Could not delete migrated streams: %v", err) } return err } // streamMigraterFinalize finalizes the stream migration. // It's a standalone function because it does not use the streamMigrater state. -func streamMigraterfinalize(ctx context.Context, mi *migrater, workflows []string) error { +func streamMigraterfinalize(ctx context.Context, ts *trafficSwitcher, workflows []string) error { if len(workflows) == 0 { return nil } workflowList := stringListify(workflows) - err := mi.forAllSources(func(source *miSource) error { + err := ts.forAllSources(func(source *tsSource) error { query := fmt.Sprintf("delete from _vt.vreplication where db_name=%s and workflow in (%s)", encodeString(source.master.DbName()), workflowList) - _, err := mi.wr.VReplicationExec(ctx, source.master.Alias, query) + _, err := ts.wr.VReplicationExec(ctx, source.master.Alias, query) return err }) if err != nil { return err } - err = mi.forAllTargets(func(target *miTarget) error { + err = ts.forAllTargets(func(target *tsTarget) error { query := fmt.Sprintf("update _vt.vreplication set state='Running' where db_name=%s and workflow in (%s)", encodeString(target.master.DbName()), workflowList) - _, err := mi.wr.VReplicationExec(ctx, target.master.Alias, query) + _, err := ts.wr.VReplicationExec(ctx, target.master.Alias, query) return err }) return err diff --git a/go/vt/wrangler/stream_migrater_test.go b/go/vt/wrangler/stream_migrater_test.go index 2d6a729d251..6aa1b52b212 100644 --- a/go/vt/wrangler/stream_migrater_test.go +++ b/go/vt/wrangler/stream_migrater_test.go @@ -1675,11 +1675,11 @@ func TestTemplatize(t *testing.T) { if err != nil { t.Fatal(err) } - mi := &migrater{ + ts := &trafficSwitcher{ sourceKSSchema: ksschema, } for _, tt := range tests { - sm := &streamMigrater{mi: mi} + sm := &streamMigrater{ts: ts} out, err := sm.templatize(context.Background(), tt.in) var gotErr string if err != nil { diff --git a/go/vt/wrangler/traffic_switcher.go b/go/vt/wrangler/traffic_switcher.go index 90089b3287a..4cef3aa9c9b 100644 --- a/go/vt/wrangler/traffic_switcher.go +++ b/go/vt/wrangler/traffic_switcher.go @@ -48,12 +48,12 @@ const ( frozenStr = "FROZEN" ) -// MigrateDirection specifies the migration direction. -type MigrateDirection int +// TrafficSwitchDirection specifies the switching direction. +type TrafficSwitchDirection int -// The following constants define the migration direction. +// The following constants define the switching direction. const ( - DirectionForward = MigrateDirection(iota) + DirectionForward = TrafficSwitchDirection(iota) DirectionBackward ) @@ -65,9 +65,9 @@ const ( disallowWrites ) -// migrater contains the metadata for migrating read and write traffic +// trafficSwitcher contains the metadata for switching read and write traffic // for vreplication streams. -type migrater struct { +type trafficSwitcher struct { migrationType binlogdatapb.MigrationType wr *Wrangler workflow string @@ -76,65 +76,65 @@ type migrater struct { frozen bool reverseWorkflow string id int64 - sources map[string]*miSource - targets map[string]*miTarget + sources map[string]*tsSource + targets map[string]*tsTarget sourceKeyspace string targetKeyspace string tables []string sourceKSSchema *vindexes.KeyspaceSchema } -// miTarget contains the metadata for each migration target. -type miTarget struct { +// tsTarget contains the metadata for each migration target. +type tsTarget struct { si *topo.ShardInfo master *topo.TabletInfo sources map[uint32]*binlogdatapb.BinlogSource position string } -// miSource contains the metadata for each migration source. -type miSource struct { +// tsSource contains the metadata for each migration source. +type tsSource struct { si *topo.ShardInfo master *topo.TabletInfo position string journaled bool } -// SwitchReads is a generic way of migrating read traffic for a resharding workflow. -func (wr *Wrangler) SwitchReads(ctx context.Context, targetKeyspace, workflow string, servedType topodatapb.TabletType, cells []string, direction MigrateDirection) error { +// SwitchReads is a generic way of switching read traffic for a resharding workflow. +func (wr *Wrangler) SwitchReads(ctx context.Context, targetKeyspace, workflow string, servedType topodatapb.TabletType, cells []string, direction TrafficSwitchDirection) error { if servedType != topodatapb.TabletType_REPLICA && servedType != topodatapb.TabletType_RDONLY { return fmt.Errorf("tablet type must be REPLICA or RDONLY: %v", servedType) } - mi, err := wr.buildMigrater(ctx, targetKeyspace, workflow) + ts, err := wr.buildTrafficSwitcher(ctx, targetKeyspace, workflow) if err != nil { - wr.Logger().Errorf("buildMigrater failed: %v", err) + wr.Logger().Errorf("buildTrafficSwitcher failed: %v", err) return err } - if mi.frozen { - return fmt.Errorf("cannot migrate reads while SwitchWrites is in progress") + if ts.frozen { + return fmt.Errorf("cannot switch reads while SwitchWrites is in progress") } - if err := mi.validate(ctx, false /* isWrite */); err != nil { - mi.wr.Logger().Errorf("validate failed: %v", err) + if err := ts.validate(ctx, false /* isWrite */); err != nil { + ts.wr.Logger().Errorf("validate failed: %v", err) return err } // For reads, locking the source keyspace is sufficient. - ctx, unlock, lockErr := wr.ts.LockKeyspace(ctx, mi.sourceKeyspace, "SwitchReads") + ctx, unlock, lockErr := wr.ts.LockKeyspace(ctx, ts.sourceKeyspace, "SwitchReads") if lockErr != nil { - mi.wr.Logger().Errorf("LockKeyspace failed: %v", lockErr) + ts.wr.Logger().Errorf("LockKeyspace failed: %v", lockErr) return lockErr } defer unlock(&err) - if mi.migrationType == binlogdatapb.MigrationType_TABLES { - if err := mi.migrateTableReads(ctx, cells, servedType, direction); err != nil { - mi.wr.Logger().Errorf("migrateTableReads failed: %v", err) + if ts.migrationType == binlogdatapb.MigrationType_TABLES { + if err := ts.switchTableReads(ctx, cells, servedType, direction); err != nil { + ts.wr.Logger().Errorf("switchTableReads failed: %v", err) return err } return nil } - if err := mi.migrateShardReads(ctx, cells, servedType, direction); err != nil { - mi.wr.Logger().Errorf("migrateShardReads failed: %v", err) + if err := ts.switchShardReads(ctx, cells, servedType, direction); err != nil { + ts.wr.Logger().Errorf("switchShardReads failed: %v", err) return err } return nil @@ -142,37 +142,37 @@ func (wr *Wrangler) SwitchReads(ctx context.Context, targetKeyspace, workflow st // SwitchWrites is a generic way of migrating write traffic for a resharding workflow. func (wr *Wrangler) SwitchWrites(ctx context.Context, targetKeyspace, workflow string, filteredReplicationWaitTime time.Duration, cancelMigrate, reverseReplication bool) (journalID int64, err error) { - mi, err := wr.buildMigrater(ctx, targetKeyspace, workflow) + ts, err := wr.buildTrafficSwitcher(ctx, targetKeyspace, workflow) if err != nil { - wr.Logger().Errorf("buildMigrater failed: %v", err) + wr.Logger().Errorf("buildTrafficSwitcher failed: %v", err) return 0, err } - if mi.frozen { - mi.wr.Logger().Infof("Replication has been frozen already. Deleting left-over streams") - if err := mi.deleteTargetVReplication(ctx); err != nil { - mi.wr.Logger().Errorf("deleteTargetVReplication failed: %v", err) + if ts.frozen { + ts.wr.Logger().Infof("Replication has been frozen already. Deleting left-over streams") + if err := ts.deleteTargetVReplication(ctx); err != nil { + ts.wr.Logger().Errorf("deleteTargetVReplication failed: %v", err) return 0, err } return 0, nil } - mi.wr.Logger().Infof("Built migration metadata: %+v", mi) - if err := mi.validate(ctx, true /* isWrite */); err != nil { - mi.wr.Logger().Errorf("validate failed: %v", err) + ts.wr.Logger().Infof("Built switching metadata: %+v", ts) + if err := ts.validate(ctx, true /* isWrite */); err != nil { + ts.wr.Logger().Errorf("validate failed: %v", err) return 0, err } // Need to lock both source and target keyspaces. - ctx, sourceUnlock, lockErr := wr.ts.LockKeyspace(ctx, mi.sourceKeyspace, "SwitchWrites") + ctx, sourceUnlock, lockErr := wr.ts.LockKeyspace(ctx, ts.sourceKeyspace, "SwitchWrites") if lockErr != nil { - mi.wr.Logger().Errorf("LockKeyspace failed: %v", lockErr) + ts.wr.Logger().Errorf("LockKeyspace failed: %v", lockErr) return 0, lockErr } defer sourceUnlock(&err) - if mi.targetKeyspace != mi.sourceKeyspace { - tctx, targetUnlock, lockErr := wr.ts.LockKeyspace(ctx, mi.targetKeyspace, "SwitchWrites") + if ts.targetKeyspace != ts.sourceKeyspace { + tctx, targetUnlock, lockErr := wr.ts.LockKeyspace(ctx, ts.targetKeyspace, "SwitchWrites") if lockErr != nil { - mi.wr.Logger().Errorf("LockKeyspace failed: %v", lockErr) + ts.wr.Logger().Errorf("LockKeyspace failed: %v", lockErr) return 0, lockErr } ctx = tctx @@ -180,105 +180,105 @@ func (wr *Wrangler) SwitchWrites(ctx context.Context, targetKeyspace, workflow s } // If no journals exist, sourceWorkflows will be initialized by sm.MigrateStreams. - journalsExist, sourceWorkflows, err := mi.checkJournals(ctx) + journalsExist, sourceWorkflows, err := ts.checkJournals(ctx) if err != nil { - mi.wr.Logger().Errorf("checkJournals failed: %v", err) + ts.wr.Logger().Errorf("checkJournals failed: %v", err) return 0, err } if !journalsExist { - mi.wr.Logger().Infof("No previous journals were found. Proceeding normally.") - sm, err := buildStreamMigrater(ctx, mi, cancelMigrate) + ts.wr.Logger().Infof("No previous journals were found. Proceeding normally.") + sm, err := buildStreamMigrater(ctx, ts, cancelMigrate) if err != nil { - mi.wr.Logger().Errorf("buildStreamMigrater failed: %v", err) + ts.wr.Logger().Errorf("buildStreamMigrater failed: %v", err) return 0, err } if cancelMigrate { - mi.wr.Logger().Infof("Cancel was requested.") - mi.cancelMigration(ctx, sm) + ts.wr.Logger().Infof("Cancel was requested.") + ts.cancelMigration(ctx, sm) return 0, nil } sourceWorkflows, err = sm.stopStreams(ctx) if err != nil { - mi.wr.Logger().Errorf("stopStreams failed: %v", err) + ts.wr.Logger().Errorf("stopStreams failed: %v", err) for key, streams := range sm.streams { for _, stream := range streams { - mi.wr.Logger().Errorf("stream in stopStreams: key %s shard %s stream %+v", key, stream.bls.Shard, stream.bls) + ts.wr.Logger().Errorf("stream in stopStreams: key %s shard %s stream %+v", key, stream.bls.Shard, stream.bls) } } - mi.cancelMigration(ctx, sm) + ts.cancelMigration(ctx, sm) return 0, err } - if err := mi.stopSourceWrites(ctx); err != nil { - mi.wr.Logger().Errorf("stopSourceWrites failed: %v", err) - mi.cancelMigration(ctx, sm) + if err := ts.stopSourceWrites(ctx); err != nil { + ts.wr.Logger().Errorf("stopSourceWrites failed: %v", err) + ts.cancelMigration(ctx, sm) return 0, err } - if err := mi.waitForCatchup(ctx, filteredReplicationWaitTime); err != nil { - mi.wr.Logger().Errorf("waitForCatchup failed: %v", err) - mi.cancelMigration(ctx, sm) + if err := ts.waitForCatchup(ctx, filteredReplicationWaitTime); err != nil { + ts.wr.Logger().Errorf("waitForCatchup failed: %v", err) + ts.cancelMigration(ctx, sm) return 0, err } if err := sm.migrateStreams(ctx); err != nil { - mi.wr.Logger().Errorf("migrateStreams failed: %v", err) - mi.cancelMigration(ctx, sm) + ts.wr.Logger().Errorf("migrateStreams failed: %v", err) + ts.cancelMigration(ctx, sm) return 0, err } - if err := mi.createReverseVReplication(ctx); err != nil { - mi.wr.Logger().Errorf("createReverseVReplication failed: %v", err) - mi.cancelMigration(ctx, sm) + if err := ts.createReverseVReplication(ctx); err != nil { + ts.wr.Logger().Errorf("createReverseVReplication failed: %v", err) + ts.cancelMigration(ctx, sm) return 0, err } } else { if cancelMigrate { - err := fmt.Errorf("migration has reached the point of no return, cannot cancel") - mi.wr.Logger().Errorf("%v", err) + err := fmt.Errorf("traffic switching has reached the point of no return, cannot cancel") + ts.wr.Logger().Errorf("%v", err) return 0, err } - mi.wr.Logger().Infof("Journals were found. Completing the left over steps.") + ts.wr.Logger().Infof("Journals were found. Completing the left over steps.") // Need to gather positions in case all journals were not created. - if err := mi.gatherPositions(ctx); err != nil { - mi.wr.Logger().Errorf("gatherPositions failed: %v", err) + if err := ts.gatherPositions(ctx); err != nil { + ts.wr.Logger().Errorf("gatherPositions failed: %v", err) return 0, err } } // This is the point of no return. Once a journal is created, // traffic can be redirected to target shards. - if err := mi.createJournals(ctx, sourceWorkflows); err != nil { - mi.wr.Logger().Errorf("createJournals failed: %v", err) + if err := ts.createJournals(ctx, sourceWorkflows); err != nil { + ts.wr.Logger().Errorf("createJournals failed: %v", err) return 0, err } - if err := mi.allowTargetWrites(ctx); err != nil { - mi.wr.Logger().Errorf("allowTargetWrites failed: %v", err) + if err := ts.allowTargetWrites(ctx); err != nil { + ts.wr.Logger().Errorf("allowTargetWrites failed: %v", err) return 0, err } - if err := mi.changeRouting(ctx); err != nil { - mi.wr.Logger().Errorf("changeRouting failed: %v", err) + if err := ts.changeRouting(ctx); err != nil { + ts.wr.Logger().Errorf("changeRouting failed: %v", err) return 0, err } - if err := streamMigraterfinalize(ctx, mi, sourceWorkflows); err != nil { - mi.wr.Logger().Errorf("finalize failed: %v", err) + if err := streamMigraterfinalize(ctx, ts, sourceWorkflows); err != nil { + ts.wr.Logger().Errorf("finalize failed: %v", err) return 0, err } if reverseReplication { - if err := mi.startReverseVReplication(ctx); err != nil { - mi.wr.Logger().Errorf("startReverseVReplication failed: %v", err) + if err := ts.startReverseVReplication(ctx); err != nil { + ts.wr.Logger().Errorf("startReverseVReplication failed: %v", err) return 0, err } } - if err := mi.deleteTargetVReplication(ctx); err != nil { - mi.wr.Logger().Errorf("deleteTargetVReplication failed: %v", err) + if err := ts.deleteTargetVReplication(ctx); err != nil { + ts.wr.Logger().Errorf("deleteTargetVReplication failed: %v", err) return 0, err } - return mi.id, nil + return ts.id, nil } -func (wr *Wrangler) buildMigrater(ctx context.Context, targetKeyspace, workflow string) (*migrater, error) { - targets, frozen, err := wr.buildMigrationTargets(ctx, targetKeyspace, workflow) +func (wr *Wrangler) buildTrafficSwitcher(ctx context.Context, targetKeyspace, workflow string) (*trafficSwitcher, error) { + targets, frozen, err := wr.buildTargets(ctx, targetKeyspace, workflow) if err != nil { return nil, err } if frozen { - return &migrater{ + return &trafficSwitcher{ wr: wr, workflow: workflow, targets: targets, @@ -286,86 +286,86 @@ func (wr *Wrangler) buildMigrater(ctx context.Context, targetKeyspace, workflow }, nil } - mi := &migrater{ + ts := &trafficSwitcher{ wr: wr, workflow: workflow, reverseWorkflow: reverseName(workflow), id: hashStreams(targetKeyspace, targets), targets: targets, - sources: make(map[string]*miSource), + sources: make(map[string]*tsSource), targetKeyspace: targetKeyspace, } - mi.wr.Logger().Infof("Migration ID for workflow %s: %d", workflow, mi.id) + ts.wr.Logger().Infof("Migration ID for workflow %s: %d", workflow, ts.id) // Build the sources for _, target := range targets { for _, bls := range target.sources { - if mi.sourceKeyspace == "" { - mi.sourceKeyspace = bls.Keyspace - } else if mi.sourceKeyspace != bls.Keyspace { - return nil, fmt.Errorf("source keyspaces are mismatched across streams: %v vs %v", mi.sourceKeyspace, bls.Keyspace) + if ts.sourceKeyspace == "" { + ts.sourceKeyspace = bls.Keyspace + } else if ts.sourceKeyspace != bls.Keyspace { + return nil, fmt.Errorf("source keyspaces are mismatched across streams: %v vs %v", ts.sourceKeyspace, bls.Keyspace) } - if mi.tables == nil { + if ts.tables == nil { for _, rule := range bls.Filter.Rules { - mi.tables = append(mi.tables, rule.Match) + ts.tables = append(ts.tables, rule.Match) } - sort.Strings(mi.tables) + sort.Strings(ts.tables) } else { var tables []string for _, rule := range bls.Filter.Rules { tables = append(tables, rule.Match) } sort.Strings(tables) - if !reflect.DeepEqual(mi.tables, tables) { - return nil, fmt.Errorf("table lists are mismatched across streams: %v vs %v", mi.tables, tables) + if !reflect.DeepEqual(ts.tables, tables) { + return nil, fmt.Errorf("table lists are mismatched across streams: %v vs %v", ts.tables, tables) } } - if _, ok := mi.sources[bls.Shard]; ok { + if _, ok := ts.sources[bls.Shard]; ok { continue } - sourcesi, err := mi.wr.ts.GetShard(ctx, bls.Keyspace, bls.Shard) + sourcesi, err := ts.wr.ts.GetShard(ctx, bls.Keyspace, bls.Shard) if err != nil { return nil, err } - sourceMaster, err := mi.wr.ts.GetTablet(ctx, sourcesi.MasterAlias) + sourceMaster, err := ts.wr.ts.GetTablet(ctx, sourcesi.MasterAlias) if err != nil { return nil, err } - mi.sources[bls.Shard] = &miSource{ + ts.sources[bls.Shard] = &tsSource{ si: sourcesi, master: sourceMaster, } } } - if mi.sourceKeyspace != mi.targetKeyspace { - mi.migrationType = binlogdatapb.MigrationType_TABLES + if ts.sourceKeyspace != ts.targetKeyspace { + ts.migrationType = binlogdatapb.MigrationType_TABLES } else { // TODO(sougou): for shard migration, validate that source and target combined // keyranges match. - mi.migrationType = binlogdatapb.MigrationType_SHARDS - for sourceShard := range mi.sources { - if _, ok := mi.targets[sourceShard]; ok { + ts.migrationType = binlogdatapb.MigrationType_SHARDS + for sourceShard := range ts.sources { + if _, ok := ts.targets[sourceShard]; ok { // If shards are overlapping, then this is a table migration. - mi.migrationType = binlogdatapb.MigrationType_TABLES + ts.migrationType = binlogdatapb.MigrationType_TABLES break } } } - vs, err := mi.wr.ts.GetVSchema(ctx, mi.sourceKeyspace) + vs, err := ts.wr.ts.GetVSchema(ctx, ts.sourceKeyspace) if err != nil { return nil, err } - mi.sourceKSSchema, err = vindexes.BuildKeyspaceSchema(vs, mi.sourceKeyspace) + ts.sourceKSSchema, err = vindexes.BuildKeyspaceSchema(vs, ts.sourceKeyspace) if err != nil { return nil, err } - return mi, nil + return ts, nil } -func (wr *Wrangler) buildMigrationTargets(ctx context.Context, targetKeyspace, workflow string) (targets map[string]*miTarget, frozen bool, err error) { - targets = make(map[string]*miTarget) +func (wr *Wrangler) buildTargets(ctx context.Context, targetKeyspace, workflow string) (targets map[string]*tsTarget, frozen bool, err error) { + targets = make(map[string]*tsTarget) targetShards, err := wr.ts.GetShardNames(ctx, targetKeyspace) if err != nil { return nil, false, err @@ -395,7 +395,7 @@ func (wr *Wrangler) buildMigrationTargets(ctx context.Context, targetKeyspace, w continue } - targets[targetShard] = &miTarget{ + targets[targetShard] = &tsTarget{ si: targetsi, master: targetMaster, sources: make(map[uint32]*binlogdatapb.BinlogSource), @@ -425,7 +425,7 @@ func (wr *Wrangler) buildMigrationTargets(ctx context.Context, targetKeyspace, w } // hashStreams produces a reproducible hash based on the input parameters. -func hashStreams(targetKeyspace string, targets map[string]*miTarget) int64 { +func hashStreams(targetKeyspace string, targets map[string]*tsTarget) int64 { var expanded []string for shard, target := range targets { for uid := range target.sources { @@ -442,57 +442,57 @@ func hashStreams(targetKeyspace string, targets map[string]*miTarget) int64 { return int64(hasher.Sum64() & math.MaxInt64) } -func (mi *migrater) validate(ctx context.Context, isWrite bool) error { - if mi.migrationType == binlogdatapb.MigrationType_TABLES { +func (ts *trafficSwitcher) validate(ctx context.Context, isWrite bool) error { + if ts.migrationType == binlogdatapb.MigrationType_TABLES { // All shards must be present. - if err := mi.compareShards(ctx, mi.sourceKeyspace, mi.sourceShards()); err != nil { + if err := ts.compareShards(ctx, ts.sourceKeyspace, ts.sourceShards()); err != nil { return err } - if err := mi.compareShards(ctx, mi.targetKeyspace, mi.targetShards()); err != nil { + if err := ts.compareShards(ctx, ts.targetKeyspace, ts.targetShards()); err != nil { return err } // Wildcard table names not allowed. - for _, table := range mi.tables { + for _, table := range ts.tables { if strings.HasPrefix(table, "/") { return fmt.Errorf("cannot migrate streams with wild card table names: %v", table) } } if isWrite { - return mi.validateTableForWrite(ctx) + return ts.validateTableForWrite(ctx) } } else { // binlogdatapb.MigrationType_SHARDS if isWrite { - return mi.validateShardForWrite(ctx) + return ts.validateShardForWrite(ctx) } } return nil } -func (mi *migrater) validateTableForWrite(ctx context.Context) error { - rules, err := mi.wr.getRoutingRules(ctx) +func (ts *trafficSwitcher) validateTableForWrite(ctx context.Context) error { + rules, err := ts.wr.getRoutingRules(ctx) if err != nil { return err } - for _, table := range mi.tables { + for _, table := range ts.tables { for _, tabletType := range []topodatapb.TabletType{topodatapb.TabletType_REPLICA, topodatapb.TabletType_RDONLY} { tt := strings.ToLower(tabletType.String()) - if rules[table+"@"+tt] == nil || rules[mi.targetKeyspace+"."+table+"@"+tt] == nil { - return fmt.Errorf("missing tablet type specific routing, read-only traffic must be migrated before migrating writes: %v", table) + if rules[table+"@"+tt] == nil || rules[ts.targetKeyspace+"."+table+"@"+tt] == nil { + return fmt.Errorf("missing tablet type specific routing, read-only traffic must be switched before switching writes: %v", table) } } } return nil } -func (mi *migrater) validateShardForWrite(ctx context.Context) error { - srvKeyspaces, err := mi.wr.ts.GetSrvKeyspaceAllCells(ctx, mi.sourceKeyspace) +func (ts *trafficSwitcher) validateShardForWrite(ctx context.Context) error { + srvKeyspaces, err := ts.wr.ts.GetSrvKeyspaceAllCells(ctx, ts.sourceKeyspace) if err != nil { return err } // Checking one shard is enough. var si *topo.ShardInfo - for _, source := range mi.sources { + for _, source := range ts.sources { si = source.si break } @@ -510,31 +510,31 @@ func (mi *migrater) validateShardForWrite(ctx context.Context) error { } } if len(shardServedTypes) > 0 { - return fmt.Errorf("cannot migrate MASTER away from %v/%v until everything else is migrated. Make sure that the following types are migrated first: %v", si.Keyspace(), si.ShardName(), strings.Join(shardServedTypes, ", ")) + return fmt.Errorf("cannot switch MASTER away from %v/%v until everything else is switched. Make sure that the following types are switched first: %v", si.Keyspace(), si.ShardName(), strings.Join(shardServedTypes, ", ")) } } return nil } -func (mi *migrater) compareShards(ctx context.Context, keyspace string, sis []*topo.ShardInfo) error { +func (ts *trafficSwitcher) compareShards(ctx context.Context, keyspace string, sis []*topo.ShardInfo) error { var shards []string for _, si := range sis { shards = append(shards, si.ShardName()) } - topoShards, err := mi.wr.ts.GetShardNames(ctx, keyspace) + topoShards, err := ts.wr.ts.GetShardNames(ctx, keyspace) if err != nil { return err } sort.Strings(topoShards) sort.Strings(shards) if !reflect.DeepEqual(topoShards, shards) { - return fmt.Errorf("mismatched shards for keyspace %s: topo: %v vs migrate command: %v", keyspace, topoShards, shards) + return fmt.Errorf("mismatched shards for keyspace %s: topo: %v vs switch command: %v", keyspace, topoShards, shards) } return nil } -func (mi *migrater) migrateTableReads(ctx context.Context, cells []string, servedType topodatapb.TabletType, direction MigrateDirection) error { - rules, err := mi.wr.getRoutingRules(ctx) +func (ts *trafficSwitcher) switchTableReads(ctx context.Context, cells []string, servedType topodatapb.TabletType, direction TrafficSwitchDirection) error { + rules, err := ts.wr.getRoutingRules(ctx) if err != nil { return err } @@ -544,49 +544,49 @@ func (mi *migrater) migrateTableReads(ctx context.Context, cells []string, serve // For forward migration, we add tablet type specific rules to redirect traffic to the target. // For backward, we delete them. tt := strings.ToLower(servedType.String()) - for _, table := range mi.tables { + for _, table := range ts.tables { if direction == DirectionForward { - rules[table+"@"+tt] = []string{mi.targetKeyspace + "." + table} - rules[mi.targetKeyspace+"."+table+"@"+tt] = []string{mi.targetKeyspace + "." + table} - rules[mi.sourceKeyspace+"."+table+"@"+tt] = []string{mi.targetKeyspace + "." + table} + rules[table+"@"+tt] = []string{ts.targetKeyspace + "." + table} + rules[ts.targetKeyspace+"."+table+"@"+tt] = []string{ts.targetKeyspace + "." + table} + rules[ts.sourceKeyspace+"."+table+"@"+tt] = []string{ts.targetKeyspace + "." + table} } else { delete(rules, table+"@"+tt) - delete(rules, mi.targetKeyspace+"."+table+"@"+tt) - delete(rules, mi.sourceKeyspace+"."+table+"@"+tt) + delete(rules, ts.targetKeyspace+"."+table+"@"+tt) + delete(rules, ts.sourceKeyspace+"."+table+"@"+tt) } } - if err := mi.wr.saveRoutingRules(ctx, rules); err != nil { + if err := ts.wr.saveRoutingRules(ctx, rules); err != nil { return err } - return mi.wr.ts.RebuildSrvVSchema(ctx, cells) + return ts.wr.ts.RebuildSrvVSchema(ctx, cells) } -func (mi *migrater) migrateShardReads(ctx context.Context, cells []string, servedType topodatapb.TabletType, direction MigrateDirection) error { +func (ts *trafficSwitcher) switchShardReads(ctx context.Context, cells []string, servedType topodatapb.TabletType, direction TrafficSwitchDirection) error { var fromShards, toShards []*topo.ShardInfo if direction == DirectionForward { - fromShards, toShards = mi.sourceShards(), mi.targetShards() + fromShards, toShards = ts.sourceShards(), ts.targetShards() } else { - fromShards, toShards = mi.targetShards(), mi.sourceShards() + fromShards, toShards = ts.targetShards(), ts.sourceShards() } - if err := mi.wr.updateShardRecords(ctx, mi.sourceKeyspace, fromShards, cells, servedType, true /* isFrom */, false /* clearSourceShards */); err != nil { + if err := ts.wr.updateShardRecords(ctx, ts.sourceKeyspace, fromShards, cells, servedType, true /* isFrom */, false /* clearSourceShards */); err != nil { return err } - if err := mi.wr.updateShardRecords(ctx, mi.sourceKeyspace, toShards, cells, servedType, false, false); err != nil { + if err := ts.wr.updateShardRecords(ctx, ts.sourceKeyspace, toShards, cells, servedType, false, false); err != nil { return err } - return mi.wr.ts.MigrateServedType(ctx, mi.sourceKeyspace, toShards, fromShards, servedType, cells) + return ts.wr.ts.MigrateServedType(ctx, ts.sourceKeyspace, toShards, fromShards, servedType, cells) } // checkJournals returns true if at least one journal has been created. -// If so, it also returns the list of sourceWorkflows that need to be migrated. -func (mi *migrater) checkJournals(ctx context.Context) (journalsExist bool, sourceWorkflows []string, err error) { +// If so, it also returns the list of sourceWorkflows that need to be switched. +func (ts *trafficSwitcher) checkJournals(ctx context.Context) (journalsExist bool, sourceWorkflows []string, err error) { var mu sync.Mutex journal := &binlogdatapb.Journal{} var exists bool - err = mi.forAllSources(func(source *miSource) error { - statement := fmt.Sprintf("select val from _vt.resharding_journal where id=%v", mi.id) - p3qr, err := mi.wr.tmc.VReplicationExec(ctx, source.master.Tablet, statement) + err = ts.forAllSources(func(source *tsSource) error { + statement := fmt.Sprintf("select val from _vt.resharding_journal where id=%v", ts.id) + p3qr, err := ts.wr.tmc.VReplicationExec(ctx, source.master.Tablet, statement) if err != nil { return err } @@ -608,49 +608,49 @@ func (mi *migrater) checkJournals(ctx context.Context) (journalsExist bool, sour return exists, journal.SourceWorkflows, err } -func (mi *migrater) stopSourceWrites(ctx context.Context) error { +func (ts *trafficSwitcher) stopSourceWrites(ctx context.Context) error { var err error - if mi.migrationType == binlogdatapb.MigrationType_TABLES { - err = mi.changeTableSourceWrites(ctx, disallowWrites) + if ts.migrationType == binlogdatapb.MigrationType_TABLES { + err = ts.changeTableSourceWrites(ctx, disallowWrites) } else { - err = mi.changeShardsAccess(ctx, mi.sourceKeyspace, mi.sourceShards(), disallowWrites) + err = ts.changeShardsAccess(ctx, ts.sourceKeyspace, ts.sourceShards(), disallowWrites) } if err != nil { return err } - return mi.forAllSources(func(source *miSource) error { + return ts.forAllSources(func(source *tsSource) error { var err error - source.position, err = mi.wr.tmc.MasterPosition(ctx, source.master.Tablet) - mi.wr.Logger().Infof("Position for source %v:%v: %v", mi.sourceKeyspace, source.si.ShardName(), source.position) + source.position, err = ts.wr.tmc.MasterPosition(ctx, source.master.Tablet) + ts.wr.Logger().Infof("Position for source %v:%v: %v", ts.sourceKeyspace, source.si.ShardName(), source.position) return err }) } -func (mi *migrater) changeTableSourceWrites(ctx context.Context, access accessType) error { - return mi.forAllSources(func(source *miSource) error { - if _, err := mi.wr.ts.UpdateShardFields(ctx, mi.sourceKeyspace, source.si.ShardName(), func(si *topo.ShardInfo) error { - return si.UpdateSourceBlacklistedTables(ctx, topodatapb.TabletType_MASTER, nil, access == allowWrites /* remove */, mi.tables) +func (ts *trafficSwitcher) changeTableSourceWrites(ctx context.Context, access accessType) error { + return ts.forAllSources(func(source *tsSource) error { + if _, err := ts.wr.ts.UpdateShardFields(ctx, ts.sourceKeyspace, source.si.ShardName(), func(si *topo.ShardInfo) error { + return si.UpdateSourceBlacklistedTables(ctx, topodatapb.TabletType_MASTER, nil, access == allowWrites /* remove */, ts.tables) }); err != nil { return err } - return mi.wr.tmc.RefreshState(ctx, source.master.Tablet) + return ts.wr.tmc.RefreshState(ctx, source.master.Tablet) }) } -func (mi *migrater) waitForCatchup(ctx context.Context, filteredReplicationWaitTime time.Duration) error { +func (ts *trafficSwitcher) waitForCatchup(ctx context.Context, filteredReplicationWaitTime time.Duration) error { ctx, cancel := context.WithTimeout(ctx, filteredReplicationWaitTime) defer cancel() var mu sync.Mutex - return mi.forAllUids(func(target *miTarget, uid uint32) error { + return ts.forAllUids(func(target *tsTarget, uid uint32) error { bls := target.sources[uid] - source := mi.sources[bls.Shard] - mi.wr.Logger().Infof("waiting for keyspace:shard: %v:%v, position %v", mi.targetKeyspace, target.si.ShardName(), source.position) - if err := mi.wr.tmc.VReplicationWaitForPos(ctx, target.master.Tablet, int(uid), source.position); err != nil { + source := ts.sources[bls.Shard] + ts.wr.Logger().Infof("waiting for keyspace:shard: %v:%v, position %v", ts.targetKeyspace, target.si.ShardName(), source.position) + if err := ts.wr.tmc.VReplicationWaitForPos(ctx, target.master.Tablet, int(uid), source.position); err != nil { return err } - mi.wr.Logger().Infof("position for keyspace:shard: %v:%v reached", mi.targetKeyspace, target.si.ShardName()) - if _, err := mi.wr.tmc.VReplicationExec(ctx, target.master.Tablet, binlogplayer.StopVReplication(uid, "stopped for cutover")); err != nil { + ts.wr.Logger().Infof("position for keyspace:shard: %v:%v reached", ts.targetKeyspace, target.si.ShardName()) + if _, err := ts.wr.tmc.VReplicationExec(ctx, target.master.Tablet, binlogplayer.StopVReplication(uid, "stopped for cutover")); err != nil { return err } @@ -661,67 +661,67 @@ func (mi *migrater) waitForCatchup(ctx context.Context, filteredReplicationWaitT return nil } var err error - target.position, err = mi.wr.tmc.MasterPosition(ctx, target.master.Tablet) - mi.wr.Logger().Infof("Position for uid %v: %v", uid, target.position) + target.position, err = ts.wr.tmc.MasterPosition(ctx, target.master.Tablet) + ts.wr.Logger().Infof("Position for uid %v: %v", uid, target.position) return err }) } -func (mi *migrater) cancelMigration(ctx context.Context, sm *streamMigrater) { +func (ts *trafficSwitcher) cancelMigration(ctx context.Context, sm *streamMigrater) { var err error - if mi.migrationType == binlogdatapb.MigrationType_TABLES { - err = mi.changeTableSourceWrites(ctx, allowWrites) + if ts.migrationType == binlogdatapb.MigrationType_TABLES { + err = ts.changeTableSourceWrites(ctx, allowWrites) } else { - err = mi.changeShardsAccess(ctx, mi.sourceKeyspace, mi.sourceShards(), allowWrites) + err = ts.changeShardsAccess(ctx, ts.sourceKeyspace, ts.sourceShards(), allowWrites) } if err != nil { - mi.wr.Logger().Errorf("Cancel migration failed:", err) + ts.wr.Logger().Errorf("Cancel migration failed:", err) } sm.cancelMigration(ctx) - err = mi.forAllTargets(func(target *miTarget) error { - query := fmt.Sprintf("update _vt.vreplication set state='Running', message='' where db_name=%s and workflow=%s", encodeString(target.master.DbName()), encodeString(mi.workflow)) - _, err := mi.wr.tmc.VReplicationExec(ctx, target.master.Tablet, query) + err = ts.forAllTargets(func(target *tsTarget) error { + query := fmt.Sprintf("update _vt.vreplication set state='Running', message='' where db_name=%s and workflow=%s", encodeString(target.master.DbName()), encodeString(ts.workflow)) + _, err := ts.wr.tmc.VReplicationExec(ctx, target.master.Tablet, query) return err }) if err != nil { - mi.wr.Logger().Errorf("Cancel migration failed: could not restart vreplication: %v", err) + ts.wr.Logger().Errorf("Cancel migration failed: could not restart vreplication: %v", err) } - err = mi.deleteReverseVReplication(ctx) + err = ts.deleteReverseVReplication(ctx) if err != nil { - mi.wr.Logger().Errorf("Cancel migration failed: could not delete revers vreplication entries: %v", err) + ts.wr.Logger().Errorf("Cancel migration failed: could not delete revers vreplication entries: %v", err) } } -func (mi *migrater) gatherPositions(ctx context.Context) error { - err := mi.forAllSources(func(source *miSource) error { +func (ts *trafficSwitcher) gatherPositions(ctx context.Context) error { + err := ts.forAllSources(func(source *tsSource) error { var err error - source.position, err = mi.wr.tmc.MasterPosition(ctx, source.master.Tablet) - mi.wr.Logger().Infof("Position for source %v:%v: %v", mi.sourceKeyspace, source.si.ShardName(), source.position) + source.position, err = ts.wr.tmc.MasterPosition(ctx, source.master.Tablet) + ts.wr.Logger().Infof("Position for source %v:%v: %v", ts.sourceKeyspace, source.si.ShardName(), source.position) return err }) if err != nil { return err } - return mi.forAllTargets(func(target *miTarget) error { + return ts.forAllTargets(func(target *tsTarget) error { var err error - target.position, err = mi.wr.tmc.MasterPosition(ctx, target.master.Tablet) - mi.wr.Logger().Infof("Position for target %v:%v: %v", mi.targetKeyspace, target.si.ShardName(), target.position) + target.position, err = ts.wr.tmc.MasterPosition(ctx, target.master.Tablet) + ts.wr.Logger().Infof("Position for target %v:%v: %v", ts.targetKeyspace, target.si.ShardName(), target.position) return err }) } -func (mi *migrater) createReverseVReplication(ctx context.Context) error { - if err := mi.deleteReverseVReplication(ctx); err != nil { +func (ts *trafficSwitcher) createReverseVReplication(ctx context.Context) error { + if err := ts.deleteReverseVReplication(ctx); err != nil { return err } - err := mi.forAllUids(func(target *miTarget, uid uint32) error { + err := ts.forAllUids(func(target *tsTarget, uid uint32) error { bls := target.sources[uid] - source := mi.sources[bls.Shard] + source := ts.sources[bls.Shard] reverseBls := &binlogdatapb.BinlogSource{ - Keyspace: mi.targetKeyspace, + Keyspace: ts.targetKeyspace, Shard: target.si.ShardName(), TabletType: bls.TabletType, Filter: &binlogdatapb.Filter{}, @@ -734,13 +734,13 @@ func (mi *migrater) createReverseVReplication(ctx context.Context) error { } var filter string if strings.HasPrefix(rule.Match, "/") { - if mi.sourceKSSchema.Keyspace.Sharded { + if ts.sourceKSSchema.Keyspace.Sharded { filter = key.KeyRangeString(source.si.KeyRange) } } else { var inKeyrange string - if mi.sourceKSSchema.Keyspace.Sharded { - vtable, ok := mi.sourceKSSchema.Tables[rule.Match] + if ts.sourceKSSchema.Keyspace.Sharded { + vtable, ok := ts.sourceKSSchema.Tables[rule.Match] if !ok { return fmt.Errorf("table %s not found in vschema", rule.Match) } @@ -756,42 +756,42 @@ func (mi *migrater) createReverseVReplication(ctx context.Context) error { }) } - _, err := mi.wr.VReplicationExec(ctx, source.master.Alias, binlogplayer.CreateVReplicationState(mi.reverseWorkflow, reverseBls, target.position, binlogplayer.BlpStopped, source.master.DbName())) + _, err := ts.wr.VReplicationExec(ctx, source.master.Alias, binlogplayer.CreateVReplicationState(ts.reverseWorkflow, reverseBls, target.position, binlogplayer.BlpStopped, source.master.DbName())) return err }) return err } -func (mi *migrater) deleteReverseVReplication(ctx context.Context) error { - return mi.forAllSources(func(source *miSource) error { - query := fmt.Sprintf("delete from _vt.vreplication where db_name=%s and workflow=%s", encodeString(source.master.DbName()), encodeString(mi.reverseWorkflow)) - _, err := mi.wr.tmc.VReplicationExec(ctx, source.master.Tablet, query) +func (ts *trafficSwitcher) deleteReverseVReplication(ctx context.Context) error { + return ts.forAllSources(func(source *tsSource) error { + query := fmt.Sprintf("delete from _vt.vreplication where db_name=%s and workflow=%s", encodeString(source.master.DbName()), encodeString(ts.reverseWorkflow)) + _, err := ts.wr.tmc.VReplicationExec(ctx, source.master.Tablet, query) return err }) } -func (mi *migrater) createJournals(ctx context.Context, sourceWorkflows []string) error { +func (ts *trafficSwitcher) createJournals(ctx context.Context, sourceWorkflows []string) error { log.Infof("In createJournals for source workflows %+v", sourceWorkflows) - return mi.forAllSources(func(source *miSource) error { + return ts.forAllSources(func(source *tsSource) error { if source.journaled { return nil } participants := make([]*binlogdatapb.KeyspaceShard, 0) participantMap := make(map[string]bool) journal := &binlogdatapb.Journal{ - Id: mi.id, - MigrationType: mi.migrationType, - Tables: mi.tables, + Id: ts.id, + MigrationType: ts.migrationType, + Tables: ts.tables, LocalPosition: source.position, Participants: participants, SourceWorkflows: sourceWorkflows, } - for targetShard, target := range mi.targets { + for targetShard, target := range ts.targets { for _, tsource := range target.sources { participantMap[tsource.Shard] = true } journal.ShardGtids = append(journal.ShardGtids, &binlogdatapb.ShardGtid{ - Keyspace: mi.targetKeyspace, + Keyspace: ts.targetKeyspace, Shard: targetShard, Gtid: target.position, }) @@ -809,45 +809,45 @@ func (mi *migrater) createJournals(ctx context.Context, sourceWorkflows []string } log.Infof("Creating journal %v", journal) - mi.wr.Logger().Infof("Creating journal: %v", journal) + ts.wr.Logger().Infof("Creating journal: %v", journal) statement := fmt.Sprintf("insert into _vt.resharding_journal "+ "(id, db_name, val) "+ "values (%v, %v, %v)", - mi.id, encodeString(source.master.DbName()), encodeString(journal.String())) - if _, err := mi.wr.tmc.VReplicationExec(ctx, source.master.Tablet, statement); err != nil { + ts.id, encodeString(source.master.DbName()), encodeString(journal.String())) + if _, err := ts.wr.tmc.VReplicationExec(ctx, source.master.Tablet, statement); err != nil { return err } return nil }) } -func (mi *migrater) allowTargetWrites(ctx context.Context) error { - if mi.migrationType == binlogdatapb.MigrationType_TABLES { - return mi.allowTableTargetWrites(ctx) +func (ts *trafficSwitcher) allowTargetWrites(ctx context.Context) error { + if ts.migrationType == binlogdatapb.MigrationType_TABLES { + return ts.allowTableTargetWrites(ctx) } - return mi.changeShardsAccess(ctx, mi.targetKeyspace, mi.targetShards(), allowWrites) + return ts.changeShardsAccess(ctx, ts.targetKeyspace, ts.targetShards(), allowWrites) } -func (mi *migrater) allowTableTargetWrites(ctx context.Context) error { - return mi.forAllTargets(func(target *miTarget) error { - if _, err := mi.wr.ts.UpdateShardFields(ctx, mi.targetKeyspace, target.si.ShardName(), func(si *topo.ShardInfo) error { - return si.UpdateSourceBlacklistedTables(ctx, topodatapb.TabletType_MASTER, nil, true, mi.tables) +func (ts *trafficSwitcher) allowTableTargetWrites(ctx context.Context) error { + return ts.forAllTargets(func(target *tsTarget) error { + if _, err := ts.wr.ts.UpdateShardFields(ctx, ts.targetKeyspace, target.si.ShardName(), func(si *topo.ShardInfo) error { + return si.UpdateSourceBlacklistedTables(ctx, topodatapb.TabletType_MASTER, nil, true, ts.tables) }); err != nil { return err } - return mi.wr.tmc.RefreshState(ctx, target.master.Tablet) + return ts.wr.tmc.RefreshState(ctx, target.master.Tablet) }) } -func (mi *migrater) changeRouting(ctx context.Context) error { - if mi.migrationType == binlogdatapb.MigrationType_TABLES { - return mi.changeTableRouting(ctx) +func (ts *trafficSwitcher) changeRouting(ctx context.Context) error { + if ts.migrationType == binlogdatapb.MigrationType_TABLES { + return ts.changeTableRouting(ctx) } - return mi.changeShardRouting(ctx) + return ts.changeShardRouting(ctx) } -func (mi *migrater) changeTableRouting(ctx context.Context) error { - rules, err := mi.wr.getRoutingRules(ctx) +func (ts *trafficSwitcher) changeTableRouting(ctx context.Context) error { + rules, err := ts.wr.getRoutingRules(ctx) if err != nil { return err } @@ -860,29 +860,29 @@ func (mi *migrater) changeTableRouting(ctx context.Context) error { // After this step, only the following rules will be left: // table -> targetKeyspace.table // sourceKeyspace.table -> targetKeyspace.table - for _, table := range mi.tables { + for _, table := range ts.tables { for _, tabletType := range []topodatapb.TabletType{topodatapb.TabletType_REPLICA, topodatapb.TabletType_RDONLY} { tt := strings.ToLower(tabletType.String()) delete(rules, table+"@"+tt) - delete(rules, mi.targetKeyspace+"."+table+"@"+tt) - delete(rules, mi.sourceKeyspace+"."+table+"@"+tt) - mi.wr.Logger().Infof("Delete routing: %v %v %v", table+"@"+tt, mi.targetKeyspace+"."+table+"@"+tt, mi.sourceKeyspace+"."+table+"@"+tt) + delete(rules, ts.targetKeyspace+"."+table+"@"+tt) + delete(rules, ts.sourceKeyspace+"."+table+"@"+tt) + ts.wr.Logger().Infof("Delete routing: %v %v %v", table+"@"+tt, ts.targetKeyspace+"."+table+"@"+tt, ts.sourceKeyspace+"."+table+"@"+tt) } - delete(rules, mi.targetKeyspace+"."+table) - mi.wr.Logger().Infof("Delete routing: %v", mi.targetKeyspace+"."+table) - rules[table] = []string{mi.targetKeyspace + "." + table} - rules[mi.sourceKeyspace+"."+table] = []string{mi.targetKeyspace + "." + table} - mi.wr.Logger().Infof("Add routing: %v %v", table, mi.sourceKeyspace+"."+table) + delete(rules, ts.targetKeyspace+"."+table) + ts.wr.Logger().Infof("Delete routing: %v", ts.targetKeyspace+"."+table) + rules[table] = []string{ts.targetKeyspace + "." + table} + rules[ts.sourceKeyspace+"."+table] = []string{ts.targetKeyspace + "." + table} + ts.wr.Logger().Infof("Add routing: %v %v", table, ts.sourceKeyspace+"."+table) } - if err := mi.wr.saveRoutingRules(ctx, rules); err != nil { + if err := ts.wr.saveRoutingRules(ctx, rules); err != nil { return err } - return mi.wr.ts.RebuildSrvVSchema(ctx, nil) + return ts.wr.ts.RebuildSrvVSchema(ctx, nil) } -func (mi *migrater) changeShardRouting(ctx context.Context) error { - err := mi.forAllSources(func(source *miSource) error { - _, err := mi.wr.ts.UpdateShardFields(ctx, mi.sourceKeyspace, source.si.ShardName(), func(si *topo.ShardInfo) error { +func (ts *trafficSwitcher) changeShardRouting(ctx context.Context) error { + err := ts.forAllSources(func(source *tsSource) error { + _, err := ts.wr.ts.UpdateShardFields(ctx, ts.sourceKeyspace, source.si.ShardName(), func(si *topo.ShardInfo) error { si.IsMasterServing = false return nil }) @@ -891,8 +891,8 @@ func (mi *migrater) changeShardRouting(ctx context.Context) error { if err != nil { return err } - err = mi.forAllTargets(func(target *miTarget) error { - _, err := mi.wr.ts.UpdateShardFields(ctx, mi.targetKeyspace, target.si.ShardName(), func(si *topo.ShardInfo) error { + err = ts.forAllTargets(func(target *tsTarget) error { + _, err := ts.wr.ts.UpdateShardFields(ctx, ts.targetKeyspace, target.si.ShardName(), func(si *topo.ShardInfo) error { si.IsMasterServing = true return nil }) @@ -901,49 +901,49 @@ func (mi *migrater) changeShardRouting(ctx context.Context) error { if err != nil { return err } - return mi.wr.ts.MigrateServedType(ctx, mi.targetKeyspace, mi.targetShards(), mi.sourceShards(), topodatapb.TabletType_MASTER, nil) + return ts.wr.ts.MigrateServedType(ctx, ts.targetKeyspace, ts.targetShards(), ts.sourceShards(), topodatapb.TabletType_MASTER, nil) } -func (mi *migrater) startReverseVReplication(ctx context.Context) error { - return mi.forAllSources(func(source *miSource) error { +func (ts *trafficSwitcher) startReverseVReplication(ctx context.Context) error { + return ts.forAllSources(func(source *tsSource) error { query := fmt.Sprintf("update _vt.vreplication set state='Running', message='' where db_name=%s", encodeString(source.master.DbName())) - _, err := mi.wr.VReplicationExec(ctx, source.master.Alias, query) + _, err := ts.wr.VReplicationExec(ctx, source.master.Alias, query) return err }) } -func (mi *migrater) deleteTargetVReplication(ctx context.Context) error { +func (ts *trafficSwitcher) deleteTargetVReplication(ctx context.Context) error { // Mark target streams as frozen before deleting. If SwitchWrites gets // re-invoked after a freeze, it will skip all the previous steps and // jump directly here for the final cleanup. - err := mi.forAllTargets(func(target *miTarget) error { - query := fmt.Sprintf("update _vt.vreplication set message = '%s' where db_name=%s and workflow=%s", frozenStr, encodeString(target.master.DbName()), encodeString(mi.workflow)) - _, err := mi.wr.tmc.VReplicationExec(ctx, target.master.Tablet, query) + err := ts.forAllTargets(func(target *tsTarget) error { + query := fmt.Sprintf("update _vt.vreplication set message = '%s' where db_name=%s and workflow=%s", frozenStr, encodeString(target.master.DbName()), encodeString(ts.workflow)) + _, err := ts.wr.tmc.VReplicationExec(ctx, target.master.Tablet, query) return err }) if err != nil { return err } - return mi.forAllTargets(func(target *miTarget) error { - query := fmt.Sprintf("delete from _vt.vreplication where db_name=%s and workflow=%s", encodeString(target.master.DbName()), encodeString(mi.workflow)) - _, err := mi.wr.tmc.VReplicationExec(ctx, target.master.Tablet, query) + return ts.forAllTargets(func(target *tsTarget) error { + query := fmt.Sprintf("delete from _vt.vreplication where db_name=%s and workflow=%s", encodeString(target.master.DbName()), encodeString(ts.workflow)) + _, err := ts.wr.tmc.VReplicationExec(ctx, target.master.Tablet, query) return err }) } -func (mi *migrater) changeShardsAccess(ctx context.Context, keyspace string, shards []*topo.ShardInfo, access accessType) error { - if err := mi.wr.ts.UpdateDisableQueryService(ctx, mi.sourceKeyspace, shards, topodatapb.TabletType_MASTER, nil, access == disallowWrites /* disable */); err != nil { +func (ts *trafficSwitcher) changeShardsAccess(ctx context.Context, keyspace string, shards []*topo.ShardInfo, access accessType) error { + if err := ts.wr.ts.UpdateDisableQueryService(ctx, ts.sourceKeyspace, shards, topodatapb.TabletType_MASTER, nil, access == disallowWrites /* disable */); err != nil { return err } - return mi.wr.refreshMasters(ctx, shards) + return ts.wr.refreshMasters(ctx, shards) } -func (mi *migrater) forAllSources(f func(*miSource) error) error { +func (ts *trafficSwitcher) forAllSources(f func(*tsSource) error) error { var wg sync.WaitGroup allErrors := &concurrency.AllErrorRecorder{} - for _, source := range mi.sources { + for _, source := range ts.sources { wg.Add(1) - go func(source *miSource) { + go func(source *tsSource) { defer wg.Done() if err := f(source); err != nil { @@ -955,12 +955,12 @@ func (mi *migrater) forAllSources(f func(*miSource) error) error { return allErrors.AggrError(vterrors.Aggregate) } -func (mi *migrater) forAllTargets(f func(*miTarget) error) error { +func (ts *trafficSwitcher) forAllTargets(f func(*tsTarget) error) error { var wg sync.WaitGroup allErrors := &concurrency.AllErrorRecorder{} - for _, target := range mi.targets { + for _, target := range ts.targets { wg.Add(1) - go func(target *miTarget) { + go func(target *tsTarget) { defer wg.Done() if err := f(target); err != nil { @@ -972,13 +972,13 @@ func (mi *migrater) forAllTargets(f func(*miTarget) error) error { return allErrors.AggrError(vterrors.Aggregate) } -func (mi *migrater) forAllUids(f func(target *miTarget, uid uint32) error) error { +func (ts *trafficSwitcher) forAllUids(f func(target *tsTarget, uid uint32) error) error { var wg sync.WaitGroup allErrors := &concurrency.AllErrorRecorder{} - for _, target := range mi.targets { + for _, target := range ts.targets { for uid := range target.sources { wg.Add(1) - go func(target *miTarget, uid uint32) { + go func(target *tsTarget, uid uint32) { defer wg.Done() if err := f(target, uid); err != nil { @@ -991,17 +991,17 @@ func (mi *migrater) forAllUids(f func(target *miTarget, uid uint32) error) error return allErrors.AggrError(vterrors.Aggregate) } -func (mi *migrater) sourceShards() []*topo.ShardInfo { - shards := make([]*topo.ShardInfo, 0, len(mi.sources)) - for _, source := range mi.sources { +func (ts *trafficSwitcher) sourceShards() []*topo.ShardInfo { + shards := make([]*topo.ShardInfo, 0, len(ts.sources)) + for _, source := range ts.sources { shards = append(shards, source.si) } return shards } -func (mi *migrater) targetShards() []*topo.ShardInfo { - shards := make([]*topo.ShardInfo, 0, len(mi.targets)) - for _, target := range mi.targets { +func (ts *trafficSwitcher) targetShards() []*topo.ShardInfo { + shards := make([]*topo.ShardInfo, 0, len(ts.targets)) + for _, target := range ts.targets { shards = append(shards, target.si) } return shards diff --git a/go/vt/wrangler/traffic_switcher_test.go b/go/vt/wrangler/traffic_switcher_test.go index dbe56392b0e..7d7d07be44d 100644 --- a/go/vt/wrangler/traffic_switcher_test.go +++ b/go/vt/wrangler/traffic_switcher_test.go @@ -105,7 +105,7 @@ func TestTableMigrateMainflow(t *testing.T) { // Other cell REPLICA migration. // The global routing already contains redirections for rdonly. // So, adding routes for replica and deploying to cell2 will also cause - // cell2 to migrate rdonly. This is a quirk that can be fixed later if necessary. + // cell2 to switch rdonly. This is a quirk that can be fixed later if necessary. // TODO(sougou): check if it's worth fixing, or clearly document the quirk. err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, []string{"cell2"}, DirectionForward) if err != nil { @@ -164,7 +164,7 @@ func TestTableMigrateMainflow(t *testing.T) { verifyQueries(t, tme.allDBClients) //------------------------------------------------------------------------------------------------------------------- - // Migrate all REPLICA. + // Switch all REPLICA. err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) @@ -224,7 +224,7 @@ func TestTableMigrateMainflow(t *testing.T) { verifyQueries(t, tme.allDBClients) //------------------------------------------------------------------------------------------------------------------- - // Can't migrate master with SwitchReads. + // Can't switch master with SwitchReads. err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_MASTER, nil, DirectionForward) want := "tablet type must be REPLICA or RDONLY: MASTER" if err == nil || err.Error() != want { @@ -233,9 +233,9 @@ func TestTableMigrateMainflow(t *testing.T) { verifyQueries(t, tme.allDBClients) //------------------------------------------------------------------------------------------------------------------- - // Can't migrate writes if REPLICA and RDONLY have not fully migrated yet. + // Can't switch writes if REPLICA and RDONLY have not fully switched yet. _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) - want = "missing tablet type specific routing, read-only traffic must be migrated before migrating writes" + want = "missing tablet type specific routing, read-only traffic must be switched before switching writes" if err == nil || !strings.Contains(err.Error(), want) { t.Errorf("SwitchWrites err: %v, want %v", err, want) } @@ -244,7 +244,7 @@ func TestTableMigrateMainflow(t *testing.T) { //------------------------------------------------------------------------------------------------------------------- // Test SwitchWrites cancelation on failure. - // Migrate all the reads first. + // Switch all the reads first. err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) @@ -499,7 +499,7 @@ func TestShardMigrateMainflow(t *testing.T) { verifyQueries(t, tme.allDBClients) //------------------------------------------------------------------------------------------------------------------- - // Migrate all RDONLY. + // Switch all RDONLY. // This is an extra step that does not exist in the tables test. // The per-cell migration mechanism is different for tables. So, this // extra step is needed to bring things in sync. @@ -514,7 +514,7 @@ func TestShardMigrateMainflow(t *testing.T) { verifyQueries(t, tme.allDBClients) //------------------------------------------------------------------------------------------------------------------- - // Migrate all REPLICA. + // Switch all REPLICA. err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) if err != nil { t.Fatal(err) @@ -538,7 +538,7 @@ func TestShardMigrateMainflow(t *testing.T) { verifyQueries(t, tme.allDBClients) //------------------------------------------------------------------------------------------------------------------- - // Can't migrate master with SwitchReads. + // Can't switch master with SwitchReads. err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_MASTER, nil, DirectionForward) want := "tablet type must be REPLICA or RDONLY: MASTER" if err == nil || err.Error() != want { @@ -547,9 +547,9 @@ func TestShardMigrateMainflow(t *testing.T) { verifyQueries(t, tme.allDBClients) //------------------------------------------------------------------------------------------------------------------- - // Can't migrate writes if REPLICA and RDONLY have not fully migrated yet. + // Can't switch writes if REPLICA and RDONLY have not fully switched yet. _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, true) - want = "cannot migrate MASTER away" + want = "cannot switch MASTER away" if err == nil || !strings.Contains(err.Error(), want) { t.Errorf("SwitchWrites err: %v, want %v", err, want) } @@ -558,7 +558,7 @@ func TestShardMigrateMainflow(t *testing.T) { //------------------------------------------------------------------------------------------------------------------- // Test SwitchWrites cancelation on failure. - // Migrate all the reads first. + // Switch all the reads first. err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_RDONLY, nil, DirectionForward) if err != nil { t.Fatal(err) @@ -1318,7 +1318,7 @@ func TestMigrateFrozen(t *testing.T) { tme.dbTargetClients[1].addQuery(vreplQueryks2, &sqltypes.Result{}, nil) err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", topodatapb.TabletType_REPLICA, nil, DirectionForward) - want := "cannot migrate reads while SwitchWrites is in progress" + want := "cannot switch reads while SwitchWrites is in progress" if err == nil || err.Error() != want { t.Errorf("SwitchReads(frozen) err: %v, want %v", err, want) } diff --git a/go/vt/wrangler/vdiff.go b/go/vt/wrangler/vdiff.go index 02afd5e2cce..da9b35d88c0 100644 --- a/go/vt/wrangler/vdiff.go +++ b/go/vt/wrangler/vdiff.go @@ -55,7 +55,7 @@ type DiffReport struct { // vdiff contains the metadata for performing vdiff for one workflow. type vdiff struct { - mi *migrater + ts *trafficSwitcher sourceCell string targetCell string tabletTypesStr string @@ -64,7 +64,7 @@ type vdiff struct { differs map[string]*tableDiffer // The key for sources and targets is the shard name. - // The source and target keyspaces are pulled from mi. + // The source and target keyspaces are pulled from ts. sources map[string]*shardStreamer targets map[string]*shardStreamer } @@ -131,32 +131,32 @@ func (wr *Wrangler) VDiff(ctx context.Context, targetKeyspace, workflow, sourceC } // Reuse migrater code to fetch and validate initial metadata about the workflow. - mi, err := wr.buildMigrater(ctx, targetKeyspace, workflow) + ts, err := wr.buildTrafficSwitcher(ctx, targetKeyspace, workflow) if err != nil { - wr.Logger().Errorf("buildMigrater: %v", err) + wr.Logger().Errorf("buildTrafficSwitcher: %v", err) return nil, err } - if err := mi.validate(ctx, false /* isWrite */); err != nil { - mi.wr.Logger().Errorf("validate: %v", err) + if err := ts.validate(ctx, false /* isWrite */); err != nil { + ts.wr.Logger().Errorf("validate: %v", err) return nil, err } // Initialize vdiff. df := &vdiff{ - mi: mi, + ts: ts, sourceCell: sourceCell, targetCell: targetCell, tabletTypesStr: tabletTypesStr, sources: make(map[string]*shardStreamer), targets: make(map[string]*shardStreamer), } - for shard, source := range mi.sources { + for shard, source := range ts.sources { df.sources[shard] = &shardStreamer{ master: source.master, } } - var oneTarget *miTarget - for shard, target := range mi.targets { + var oneTarget *tsTarget + for shard, target := range ts.targets { df.targets[shard] = &shardStreamer{ master: target.master, } @@ -198,7 +198,7 @@ func (wr *Wrangler) VDiff(ctx context.Context, targetKeyspace, workflow, sourceC return nil, vterrors.Wrap(err, "stopTargets") } // Make sure all sources are past the target's positions and start a query stream that records the current source positions. - if err := df.startQueryStreams(ctx, df.mi.sourceKeyspace, df.sources, td.sourceExpression, filteredReplicationWaitTime); err != nil { + if err := df.startQueryStreams(ctx, df.ts.sourceKeyspace, df.sources, td.sourceExpression, filteredReplicationWaitTime); err != nil { return nil, vterrors.Wrap(err, "startQueryStreams(sources)") } // Fast forward the targets to the newly recorded source positions. @@ -206,7 +206,7 @@ func (wr *Wrangler) VDiff(ctx context.Context, targetKeyspace, workflow, sourceC return nil, vterrors.Wrap(err, "syncTargets") } // Sources and targets are in sync. Start query streams on the targets. - if err := df.startQueryStreams(ctx, df.mi.targetKeyspace, df.targets, td.targetExpression, filteredReplicationWaitTime); err != nil { + if err := df.startQueryStreams(ctx, df.ts.targetKeyspace, df.targets, td.targetExpression, filteredReplicationWaitTime); err != nil { return nil, vterrors.Wrap(err, "startQueryStreams(targets)") } // Now that queries are running, target vreplication streams can be restarted. @@ -214,7 +214,7 @@ func (wr *Wrangler) VDiff(ctx context.Context, targetKeyspace, workflow, sourceC return nil, vterrors.Wrap(err, "restartTargets") } // Perform the diff of source and target streams. - dr, err := td.diff(ctx, df.mi.wr) + dr, err := td.diff(ctx, df.ts.wr) if err != nil { return nil, vterrors.Wrap(err, "diff") } @@ -427,7 +427,7 @@ func (df *vdiff) selectTablets(ctx context.Context, healthcheckTopologyRefresh, go func() { defer wg.Done() err1 = df.forAll(df.sources, func(shard string, source *shardStreamer) error { - tp, err := discovery.NewTabletPicker(ctx, df.mi.wr.ts, df.sourceCell, df.mi.sourceKeyspace, shard, df.tabletTypesStr, healthcheckTopologyRefresh, healthcheckRetryDelay, healthcheckTimeout) + tp, err := discovery.NewTabletPicker(ctx, df.ts.wr.ts, df.sourceCell, df.ts.sourceKeyspace, shard, df.tabletTypesStr, healthcheckTopologyRefresh, healthcheckRetryDelay, healthcheckTimeout) if err != nil { return err } @@ -446,7 +446,7 @@ func (df *vdiff) selectTablets(ctx context.Context, healthcheckTopologyRefresh, go func() { defer wg.Done() err2 = df.forAll(df.targets, func(shard string, target *shardStreamer) error { - tp, err := discovery.NewTabletPicker(ctx, df.mi.wr.ts, df.targetCell, df.mi.targetKeyspace, shard, df.tabletTypesStr, healthcheckTopologyRefresh, healthcheckRetryDelay, healthcheckTimeout) + tp, err := discovery.NewTabletPicker(ctx, df.ts.wr.ts, df.targetCell, df.ts.targetKeyspace, shard, df.tabletTypesStr, healthcheckTopologyRefresh, healthcheckRetryDelay, healthcheckTimeout) if err != nil { return err } @@ -473,13 +473,13 @@ func (df *vdiff) stopTargets(ctx context.Context) error { var mu sync.Mutex err := df.forAll(df.targets, func(shard string, target *shardStreamer) error { - query := fmt.Sprintf("update _vt.vreplication set state='Stopped', message='for vdiff' where db_name=%s and workflow=%s", encodeString(target.master.DbName()), encodeString(df.mi.workflow)) - _, err := df.mi.wr.tmc.VReplicationExec(ctx, target.master.Tablet, query) + query := fmt.Sprintf("update _vt.vreplication set state='Stopped', message='for vdiff' where db_name=%s and workflow=%s", encodeString(target.master.DbName()), encodeString(df.ts.workflow)) + _, err := df.ts.wr.tmc.VReplicationExec(ctx, target.master.Tablet, query) if err != nil { return err } - query = fmt.Sprintf("select source, pos from _vt.vreplication where db_name=%s and workflow=%s", encodeString(target.master.DbName()), encodeString(df.mi.workflow)) - p3qr, err := df.mi.wr.tmc.VReplicationExec(ctx, target.master.Tablet, query) + query = fmt.Sprintf("select source, pos from _vt.vreplication where db_name=%s and workflow=%s", encodeString(target.master.DbName()), encodeString(df.ts.workflow)) + p3qr, err := df.ts.wr.tmc.VReplicationExec(ctx, target.master.Tablet, query) if err != nil { return err } @@ -525,7 +525,7 @@ func (df *vdiff) startQueryStreams(ctx context.Context, keyspace string, partici defer cancel() return df.forAll(participants, func(shard string, participant *shardStreamer) error { // Iteration for each participant. - if err := df.mi.wr.tmc.WaitForPosition(waitCtx, participant.tablet, mysql.EncodePosition(participant.position)); err != nil { + if err := df.ts.wr.tmc.WaitForPosition(waitCtx, participant.tablet, mysql.EncodePosition(participant.position)); err != nil { return vterrors.Wrapf(err, "WaitForPosition for tablet %v", topoproto.TabletAliasString(participant.tablet.Alias)) } participant.result = make(chan *sqltypes.Result, 1) @@ -600,14 +600,14 @@ func (df *vdiff) streamOne(ctx context.Context, keyspace, shard string, particip func (df *vdiff) syncTargets(ctx context.Context, filteredReplicationWaitTime time.Duration) error { waitCtx, cancel := context.WithTimeout(ctx, filteredReplicationWaitTime) defer cancel() - err := df.mi.forAllUids(func(target *miTarget, uid uint32) error { + err := df.ts.forAllUids(func(target *tsTarget, uid uint32) error { bls := target.sources[uid] pos := df.sources[bls.Shard].snapshotPosition query := fmt.Sprintf("update _vt.vreplication set state='Running', stop_pos='%s', message='synchronizing for vdiff' where id=%d", pos, uid) - if _, err := df.mi.wr.tmc.VReplicationExec(ctx, target.master.Tablet, query); err != nil { + if _, err := df.ts.wr.tmc.VReplicationExec(ctx, target.master.Tablet, query); err != nil { return err } - if err := df.mi.wr.tmc.VReplicationWaitForPos(waitCtx, target.master.Tablet, int(uid), pos); err != nil { + if err := df.ts.wr.tmc.VReplicationWaitForPos(waitCtx, target.master.Tablet, int(uid), pos); err != nil { return vterrors.Wrapf(err, "VReplicationWaitForPos for tablet %v", topoproto.TabletAliasString(target.master.Tablet.Alias)) } return nil @@ -617,7 +617,7 @@ func (df *vdiff) syncTargets(ctx context.Context, filteredReplicationWaitTime ti } err = df.forAll(df.targets, func(shard string, target *shardStreamer) error { - pos, err := df.mi.wr.tmc.MasterPosition(ctx, target.master.Tablet) + pos, err := df.ts.wr.tmc.MasterPosition(ctx, target.master.Tablet) if err != nil { return err } @@ -634,8 +634,8 @@ func (df *vdiff) syncTargets(ctx context.Context, filteredReplicationWaitTime ti // restartTargets restarts the stopped target vreplication streams. func (df *vdiff) restartTargets(ctx context.Context) error { return df.forAll(df.targets, func(shard string, target *shardStreamer) error { - query := fmt.Sprintf("update _vt.vreplication set state='Running', message='', stop_pos='' where db_name=%s and workflow=%s", encodeString(target.master.DbName()), encodeString(df.mi.workflow)) - _, err := df.mi.wr.tmc.VReplicationExec(ctx, target.master.Tablet, query) + query := fmt.Sprintf("update _vt.vreplication set state='Running', message='', stop_pos='' where db_name=%s and workflow=%s", encodeString(target.master.DbName()), encodeString(df.ts.workflow)) + _, err := df.ts.wr.tmc.VReplicationExec(ctx, target.master.Tablet, query) return err }) } From 35f49a180cb57e7a6512f24a7f81ab3c897d8665 Mon Sep 17 00:00:00 2001 From: Mohammad Aziz Date: Sat, 28 Mar 2020 09:29:11 +0530 Subject: [PATCH 71/85] Fix typo functionnality -> functionality Signed-off-by: Mohammad Aziz --- go/vt/vtctld/vtctld.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtctld/vtctld.go b/go/vt/vtctld/vtctld.go index f29967cfdbe..96c4d18925e 100644 --- a/go/vt/vtctld/vtctld.go +++ b/go/vt/vtctld/vtctld.go @@ -47,7 +47,7 @@ const ( appPrefix = "/app/" ) -// InitVtctld initializes all the vtctld functionnality. +// InitVtctld initializes all the vtctld functionality. func InitVtctld(ts *topo.Server) { actionRepo := NewActionRepository(ts) From 220a9315336988c2c10c4f8909c927be30ae711f Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 26 Mar 2020 14:03:34 +0530 Subject: [PATCH 72/85] added parsedestination method Signed-off-by: Harshit Gangal --- go/vt/vtgate/executor.go | 6 +- go/vt/vtgate/executor_test.go | 23 +++--- go/vt/vtgate/vcursor_impl.go | 39 ++++++++- go/vt/vtgate/vcursor_impl_test.go | 133 ++++++++++++++++++++++++++++++ 4 files changed, 186 insertions(+), 15 deletions(-) create mode 100644 go/vt/vtgate/vcursor_impl_test.go diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 49f4efea8e8..a6d9f1d3faa 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -312,7 +312,7 @@ func (e *Executor) handleExec(ctx context.Context, safeSession *SafeSession, sql // V3 mode. query, comments := sqlparser.SplitMarginComments(sql) - vcursor := newVCursorImpl(ctx, safeSession, destKeyspace, destTabletType, comments, e, logStats, e.VSchema(), e.resolver.resolver) + vcursor, _ := newVCursorImpl(ctx, safeSession, comments, e, logStats, e.VSchema(), e.resolver.resolver) plan, err := e.getPlan( vcursor, query, @@ -1217,7 +1217,7 @@ func (e *Executor) StreamExecute(ctx context.Context, method string, safeSession bindVars = make(map[string]*querypb.BindVariable) } query, comments := sqlparser.SplitMarginComments(sql) - vcursor := newVCursorImpl(ctx, safeSession, target.Keyspace, target.TabletType, comments, e, logStats, e.VSchema(), e.resolver.resolver) + vcursor, _ := newVCursorImpl(ctx, safeSession, comments, e, logStats, e.VSchema(), e.resolver.resolver) // check if this is a stream statement for messaging // TODO: support keyRange syntax @@ -1667,7 +1667,7 @@ func (e *Executor) prepare(ctx context.Context, safeSession *SafeSession, sql st func (e *Executor) handlePrepare(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, destKeyspace string, destTabletType topodatapb.TabletType, logStats *LogStats) ([]*querypb.Field, error) { // V3 mode. query, comments := sqlparser.SplitMarginComments(sql) - vcursor := newVCursorImpl(ctx, safeSession, destKeyspace, destTabletType, comments, e, logStats, e.VSchema(), e.resolver.resolver) + vcursor, _ := newVCursorImpl(ctx, safeSession, comments, e, logStats, e.VSchema(), e.resolver.resolver) plan, err := e.getPlan( vcursor, query, diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index e9b8a67ac75..c97854e27da 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -1959,8 +1959,8 @@ func TestVSchemaStats(t *testing.T) { func TestGetPlanUnnormalized(t *testing.T) { r, _, _, _ := createExecutorEnv() - emptyvc := newVCursorImpl(context.Background(), nil, "", 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) - unshardedvc := newVCursorImpl(context.Background(), nil, KsTestUnsharded, 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) + emptyvc, _ := newVCursorImpl(context.Background(), NewSafeSession(&vtgatepb.Session{TargetString: "@unknown"}), makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) + unshardedvc, _ := newVCursorImpl(context.Background(), NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "@unknown"}), makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) logStats1 := NewLogStats(context.Background(), "Test", "", nil) query1 := "select * from music_user_map where id = 1" @@ -2005,9 +2005,12 @@ func TestGetPlanUnnormalized(t *testing.T) { KsTestUnsharded + "@unknown:" + query1, "@unknown:" + query1, } - if keys := r.plans.Keys(); !reflect.DeepEqual(keys, want) { - t.Errorf("Plan keys: %s, want %s", keys, want) + if diff := cmp.Diff(want, r.plans.Keys()); diff != "" { + t.Errorf("\n-want,+got:\n%s", diff) } + //if keys := r.plans.Keys(); !reflect.DeepEqual(keys, want) { + // t.Errorf("Plan keys: %s, want %s", keys, want) + //} if logStats4.SQL != wantSQL { t.Errorf("logstats sql want \"%s\" got \"%s\"", wantSQL, logStats4.SQL) } @@ -2015,7 +2018,7 @@ func TestGetPlanUnnormalized(t *testing.T) { func TestGetPlanCacheUnnormalized(t *testing.T) { r, _, _, _ := createExecutorEnv() - emptyvc := newVCursorImpl(context.Background(), nil, "", 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) + emptyvc, _ := newVCursorImpl(context.Background(), NewSafeSession(&vtgatepb.Session{TargetString: "@unknown"}), makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) query1 := "select * from music_user_map where id = 1" logStats1 := NewLogStats(context.Background(), "Test", "", nil) _, err := r.getPlan(emptyvc, query1, makeComments(" /* comment */"), map[string]*querypb.BindVariable{}, true /* skipQueryPlanCache */, logStats1) @@ -2040,7 +2043,7 @@ func TestGetPlanCacheUnnormalized(t *testing.T) { // Skip cache using directive r, _, _, _ = createExecutorEnv() - unshardedvc := newVCursorImpl(context.Background(), nil, KsTestUnsharded, 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) + unshardedvc, _ := newVCursorImpl(context.Background(), NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "@unknown"}), makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) query1 = "insert /*vt+ SKIP_QUERY_PLAN_CACHE=1 */ into user(id) values (1), (2)" logStats1 = NewLogStats(context.Background(), "Test", "", nil) @@ -2061,7 +2064,7 @@ func TestGetPlanCacheUnnormalized(t *testing.T) { func TestGetPlanCacheNormalized(t *testing.T) { r, _, _, _ := createExecutorEnv() r.normalize = true - emptyvc := newVCursorImpl(context.Background(), nil, "", 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) + emptyvc, _ := newVCursorImpl(context.Background(), NewSafeSession(&vtgatepb.Session{TargetString: "@unknown"}), makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) query1 := "select * from music_user_map where id = 1" logStats1 := NewLogStats(context.Background(), "Test", "", nil) _, err := r.getPlan(emptyvc, query1, makeComments(" /* comment */"), map[string]*querypb.BindVariable{}, true /* skipQueryPlanCache */, logStats1) @@ -2086,7 +2089,7 @@ func TestGetPlanCacheNormalized(t *testing.T) { // Skip cache using directive r, _, _, _ = createExecutorEnv() r.normalize = true - unshardedvc := newVCursorImpl(context.Background(), nil, KsTestUnsharded, 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) + unshardedvc, _ := newVCursorImpl(context.Background(), NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "@unknown"}), makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) query1 = "insert /*vt+ SKIP_QUERY_PLAN_CACHE=1 */ into user(id) values (1), (2)" logStats1 = NewLogStats(context.Background(), "Test", "", nil) @@ -2107,8 +2110,8 @@ func TestGetPlanCacheNormalized(t *testing.T) { func TestGetPlanNormalized(t *testing.T) { r, _, _, _ := createExecutorEnv() r.normalize = true - emptyvc := newVCursorImpl(context.Background(), nil, "", 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) - unshardedvc := newVCursorImpl(context.Background(), nil, KsTestUnsharded, 0, makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) + emptyvc, _ := newVCursorImpl(context.Background(), NewSafeSession(&vtgatepb.Session{TargetString: "@unknown"}), makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) + unshardedvc, _ := newVCursorImpl(context.Background(), NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "@unknown"}), makeComments(""), r, nil, r.VSchema(), r.resolver.resolver) query1 := "select * from music_user_map where id = 1" query2 := "select * from music_user_map where id = 2" diff --git a/go/vt/vtgate/vcursor_impl.go b/go/vt/vtgate/vcursor_impl.go index bba6596c4b3..7bf6c8dbf30 100644 --- a/go/vt/vtgate/vcursor_impl.go +++ b/go/vt/vtgate/vcursor_impl.go @@ -34,6 +34,7 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + topoprotopb "vitess.io/vitess/go/vt/topo/topoproto" ) var _ engine.VCursor = (*vcursorImpl)(nil) @@ -56,6 +57,7 @@ type vcursorImpl struct { safeSession *SafeSession keyspace string tabletType topodatapb.TabletType + destination key.Destination marginComments sqlparser.MarginComments executor iExecute resolver *srvtopo.Resolver @@ -71,18 +73,23 @@ type vcursorImpl struct { // the query and supply it here. Trailing comments are typically sent by the application for various reasons, // including as identifying markers. So, they have to be added back to all queries that are executed // on behalf of the original query. -func newVCursorImpl(ctx context.Context, safeSession *SafeSession, keyspace string, tabletType topodatapb.TabletType, marginComments sqlparser.MarginComments, executor *Executor, logStats *LogStats, vschema *vindexes.VSchema, resolver *srvtopo.Resolver) *vcursorImpl { +func newVCursorImpl(ctx context.Context, safeSession *SafeSession, marginComments sqlparser.MarginComments, executor *Executor, logStats *LogStats, vschema *vindexes.VSchema, resolver *srvtopo.Resolver) (*vcursorImpl, error) { + keyspace, tabletType, destination, err := parseDestinationTarget(safeSession.TargetString, vschema) + if err != nil { + return nil, err + } return &vcursorImpl{ ctx: ctx, safeSession: safeSession, keyspace: keyspace, tabletType: tabletType, + destination: destination, marginComments: marginComments, executor: executor, logStats: logStats, vschema: vschema, resolver: resolver, - } + }, nil } // Context returns the current Context. @@ -253,3 +260,31 @@ func commentedShardQueries(shardQueries []*querypb.BoundQuery, marginComments sq } return newQueries } + +// TargetDestination implements the ContextVSchema interface +func (vc *vcursorImpl) TargetDestination(qualifier string) (key.Destination, *vindexes.Keyspace, topodatapb.TabletType, error) { + keyspaceName := vc.keyspace + if vc.destination == nil && qualifier != "" { + keyspaceName = qualifier + } + if keyspaceName == "" { + return nil, nil, 0, vterrors.New(vtrpcpb.Code_INVALID_ARGUMENT, "keyspace not specified") + } + keyspace := vc.vschema.Keyspaces[keyspaceName] + if keyspace == nil { + return nil, nil, 0, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "no keyspace with name [%s] found", keyspaceName) + } + return vc.destination, keyspace.Keyspace, vc.tabletType, nil +} + +// ParseDestinationTarget parses destination target string and sets default keyspace if possible. +func parseDestinationTarget(targetString string, vschema *vindexes.VSchema) (string, topodatapb.TabletType, key.Destination, error) { + destKeyspace, destTabletType, dest, err := topoprotopb.ParseDestination(targetString, defaultTabletType) + // Set default keyspace + if destKeyspace == "" && len(vschema.Keyspaces) == 1 { + for k := range vschema.Keyspaces { + destKeyspace = k + } + } + return destKeyspace, destTabletType, dest, err +} diff --git a/go/vt/vtgate/vcursor_impl_test.go b/go/vt/vtgate/vcursor_impl_test.go new file mode 100644 index 00000000000..61b70505f72 --- /dev/null +++ b/go/vt/vtgate/vcursor_impl_test.go @@ -0,0 +1,133 @@ +package vtgate + +import ( + "context" + "testing" + + "vitess.io/vitess/go/vt/key" + "vitess.io/vitess/go/vt/vtgate/vindexes" + + "github.com/stretchr/testify/require" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" + "vitess.io/vitess/go/vt/sqlparser" +) + +func TestDestinationKeyspace(t *testing.T) { + + ks1 := &vindexes.Keyspace{ + Name: "ks1", + Sharded: false, + } + ks1Schema := &vindexes.KeyspaceSchema{ + Keyspace: ks1, + Tables: nil, + Vindexes: nil, + Error: nil, + } + ks2 := &vindexes.Keyspace{ + Name: "ks2", + Sharded: false, + } + ks2Schema := &vindexes.KeyspaceSchema{ + Keyspace: ks2, + Tables: nil, + Vindexes: nil, + Error: nil, + } + vschemaWith2KS := &vindexes.VSchema{ + Keyspaces: map[string]*vindexes.KeyspaceSchema{ + ks1.Name: ks1Schema, + ks2.Name: ks2Schema, + }} + + vschemaWith1KS := &vindexes.VSchema{ + Keyspaces: map[string]*vindexes.KeyspaceSchema{ + ks1.Name: ks1Schema, + }} + + type testCase struct { + vschema *vindexes.VSchema + targetString, qualifier string + expectedError string + expectedKeyspace string + expectedDest key.Destination + expectedTabletType topodatapb.TabletType + } + + tests := []testCase{{ + vschema: vschemaWith1KS, + targetString: "", + qualifier: "", + expectedKeyspace: ks1.Name, + expectedDest: nil, + expectedTabletType: topodatapb.TabletType_MASTER, + }, { + vschema: vschemaWith1KS, + targetString: "ks1", + qualifier: "", + expectedKeyspace: ks1.Name, + expectedDest: nil, + expectedTabletType: topodatapb.TabletType_MASTER, + }, { + vschema: vschemaWith1KS, + targetString: "ks1:-80", + qualifier: "", + expectedKeyspace: ks1.Name, + expectedDest: key.DestinationShard("-80"), + expectedTabletType: topodatapb.TabletType_MASTER, + }, { + vschema: vschemaWith1KS, + targetString: "ks1@replica", + qualifier: "", + expectedKeyspace: ks1.Name, + expectedDest: nil, + expectedTabletType: topodatapb.TabletType_REPLICA, + }, { + vschema: vschemaWith1KS, + targetString: "ks1:-80@replica", + qualifier: "", + expectedKeyspace: ks1.Name, + expectedDest: key.DestinationShard("-80"), + expectedTabletType: topodatapb.TabletType_REPLICA, + }, { + vschema: vschemaWith1KS, + targetString: "", + qualifier: "ks1", + expectedKeyspace: ks1.Name, + expectedDest: nil, + expectedTabletType: topodatapb.TabletType_MASTER, + }, { + vschema: vschemaWith1KS, + targetString: "ks2", + qualifier: "", + expectedError: "no keyspace with name [ks2] found", + }, { + vschema: vschemaWith1KS, + targetString: "ks2:-80", + qualifier: "", + expectedError: "no keyspace with name [ks2] found", + }, { + vschema: vschemaWith1KS, + targetString: "", + qualifier: "ks2", + expectedError: "no keyspace with name [ks2] found", + }, { + vschema: vschemaWith2KS, + targetString: "", + expectedError: "keyspace not specified", + }} + + for _, tc := range tests { + impl, _ := newVCursorImpl(context.Background(), NewSafeSession(&vtgatepb.Session{TargetString: tc.targetString}), sqlparser.MarginComments{}, nil, nil, tc.vschema, nil) + dest, keyspace, tabletType, err := impl.TargetDestination(tc.qualifier) + if tc.expectedError == "" { + require.NoError(t, err) + require.Equal(t, tc.expectedDest, dest) + require.Equal(t, tc.expectedKeyspace, keyspace.Name) + require.Equal(t, tc.expectedTabletType, tabletType) + } else { + require.EqualError(t, err, tc.expectedError) + } + } +} From 8c343a07f625c3b339b844cdb1353cb7f14d4a41 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Thu, 26 Mar 2020 11:54:52 +0100 Subject: [PATCH 73/85] Added Destination() to the ContextVSchema interface Signed-off-by: Andres Taylor --- go/vt/vtgate/planbuilder/builder.go | 1 + go/vt/vtgate/planbuilder/plan_test.go | 6 ++++++ go/vt/vtgate/vcursor_impl.go | 8 ++++++++ 3 files changed, 15 insertions(+) diff --git a/go/vt/vtgate/planbuilder/builder.go b/go/vt/vtgate/planbuilder/builder.go index bd016333523..5409838bb55 100644 --- a/go/vt/vtgate/planbuilder/builder.go +++ b/go/vt/vtgate/planbuilder/builder.go @@ -122,6 +122,7 @@ type ContextVSchema interface { FindTablesOrVindex(tablename sqlparser.TableName) ([]*vindexes.Table, vindexes.Vindex, string, topodatapb.TabletType, key.Destination, error) DefaultKeyspace() (*vindexes.Keyspace, error) TargetString() string + Destination() key.Destination } //------------------------------------------------------------------------- diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index c4ac1d20067..8828e609d3e 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -186,10 +186,16 @@ func loadSchema(t *testing.T, filename string) *vindexes.VSchema { return vschema } +var _ ContextVSchema = (*vschemaWrapper)(nil) + type vschemaWrapper struct { v *vindexes.VSchema } +func (vw *vschemaWrapper) Destination() key.Destination { + panic("implement me") +} + func (vw *vschemaWrapper) FindTable(tab sqlparser.TableName) (*vindexes.Table, string, topodatapb.TabletType, key.Destination, error) { destKeyspace, destTabletType, destTarget, err := topoproto.ParseDestination(tab.Qualifier.String(), topodatapb.TabletType_MASTER) if err != nil { diff --git a/go/vt/vtgate/vcursor_impl.go b/go/vt/vtgate/vcursor_impl.go index 7bf6c8dbf30..f8863037b97 100644 --- a/go/vt/vtgate/vcursor_impl.go +++ b/go/vt/vtgate/vcursor_impl.go @@ -20,6 +20,8 @@ import ( "sync/atomic" "time" + "vitess.io/vitess/go/vt/vtgate/planbuilder" + "golang.org/x/net/context" "vitess.io/vitess/go/sqltypes" @@ -38,6 +40,7 @@ import ( ) var _ engine.VCursor = (*vcursorImpl)(nil) +var _ planbuilder.ContextVSchema = (*vcursorImpl)(nil) var _ iExecute = (*Executor)(nil) // vcursor_impl needs these facilities to be able to be able to execute queries for vindexes @@ -247,6 +250,11 @@ func (vc *vcursorImpl) ResolveDestinations(keyspace string, ids []*querypb.Value return vc.resolver.ResolveDestinations(vc.ctx, keyspace, vc.tabletType, ids, destinations) } +// Destination implements the ContextVSchema interface +func (vc *vcursorImpl) Destination() key.Destination { + return vc.destination +} + func commentedShardQueries(shardQueries []*querypb.BoundQuery, marginComments sqlparser.MarginComments) []*querypb.BoundQuery { if marginComments.Leading == "" && marginComments.Trailing == "" { return shardQueries From fe9975aaaad725286fecc076d7a348e6208bda4c Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 26 Mar 2020 19:31:04 +0530 Subject: [PATCH 74/85] added bypass planbuilder Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/send.go | 63 +++++++++++++++++++++++++ go/vt/vtgate/executor.go | 2 +- go/vt/vtgate/planbuilder/builder.go | 68 +++++++++++++++------------ go/vt/vtgate/planbuilder/bypass.go | 35 ++++++++++++++ go/vt/vtgate/planbuilder/plan_test.go | 39 ++++++++++++++- go/vt/vtgate/vcursor_impl.go | 8 ++++ 6 files changed, 181 insertions(+), 34 deletions(-) create mode 100644 go/vt/vtgate/engine/send.go create mode 100644 go/vt/vtgate/planbuilder/bypass.go diff --git a/go/vt/vtgate/engine/send.go b/go/vt/vtgate/engine/send.go new file mode 100644 index 00000000000..e5f3f10b000 --- /dev/null +++ b/go/vt/vtgate/engine/send.go @@ -0,0 +1,63 @@ +package engine + +import ( + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/key" + "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/vtgate/vindexes" +) + +var _ Primitive = (*Send)(nil) + +// Send is an operator to send query to the specific keyspace, tabletType and destination +type Send struct { + // Keyspace specifies the keyspace to send the query to. + Keyspace *vindexes.Keyspace + + // TargetDestination specifies an explicit target destination to send the query to. + // This bypases the core of the v3 engine. + TargetDestination key.Destination + + // TargetTabletType specifies an explicit target destination tablet type + // this is only used in conjunction with TargetDestination + TargetTabletType topodatapb.TabletType + + // Query specifies the query to be executed. + Query string +} + +// RouteType implements Primitive interface +func (s Send) RouteType() string { + panic("implement me") +} + +// GetKeyspaceName implements Primitive interface +func (s Send) GetKeyspaceName() string { + panic("implement me") +} + +// GetTableName implements Primitive interface +func (s Send) GetTableName() string { + panic("implement me") +} + +// Execute implements Primitive interface +func (s Send) Execute(vcursor VCursor, bindVars map[string]*query.BindVariable, wantfields bool) (*sqltypes.Result, error) { + panic("implement me") +} + +// StreamExecute implements Primitive interface +func (s Send) StreamExecute(vcursor VCursor, bindVars map[string]*query.BindVariable, wantields bool, callback func(*sqltypes.Result) error) error { + panic("implement me") +} + +// GetFields implements Primitive interface +func (s Send) GetFields(vcursor VCursor, bindVars map[string]*query.BindVariable) (*sqltypes.Result, error) { + panic("implement me") +} + +// Inputs implements Primitive interface +func (s Send) Inputs() []Primitive { + panic("implement me") +} diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index a6d9f1d3faa..1d7e2138fb7 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -173,7 +173,7 @@ func (e *Executor) execute(ctx context.Context, safeSession *SafeSession, sql st if err != nil { return nil, err } - + // ks@replica , :-80-> ks:-80, -80@replica, ks:-80@replica if safeSession.InTransaction() && destTabletType != topodatapb.TabletType_MASTER { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "transactions are supported only for master tablet types, current type: %v", destTabletType) } diff --git a/go/vt/vtgate/planbuilder/builder.go b/go/vt/vtgate/planbuilder/builder.go index 5409838bb55..ed4df32b86d 100644 --- a/go/vt/vtgate/planbuilder/builder.go +++ b/go/vt/vtgate/planbuilder/builder.go @@ -123,6 +123,7 @@ type ContextVSchema interface { DefaultKeyspace() (*vindexes.Keyspace, error) TargetString() string Destination() key.Destination + TabletType() topodatapb.TabletType } //------------------------------------------------------------------------- @@ -274,37 +275,42 @@ func Build(query string, vschema ContextVSchema) (*engine.Plan, error) { func BuildFromStmt(query string, stmt sqlparser.Statement, vschema ContextVSchema, bindVarNeeds sqlparser.BindVarNeeds) (*engine.Plan, error) { var err error var instruction engine.Primitive - switch stmt := stmt.(type) { - case *sqlparser.Select: - instruction, err = buildSelectPlan(stmt, vschema) - case *sqlparser.Insert: - instruction, err = buildInsertPlan(stmt, vschema) - case *sqlparser.Update: - instruction, err = buildUpdatePlan(stmt, vschema) - case *sqlparser.Delete: - instruction, err = buildDeletePlan(stmt, vschema) - case *sqlparser.Union: - instruction, err = buildUnionPlan(stmt, vschema) - case *sqlparser.Set: - return nil, errors.New("unsupported construct: set") - case *sqlparser.Show: - return nil, errors.New("unsupported construct: show") - case *sqlparser.DDL: - return nil, errors.New("unsupported construct: ddl") - case *sqlparser.DBDDL: - return nil, errors.New("unsupported construct: ddl on database") - case *sqlparser.OtherRead: - return nil, errors.New("unsupported construct: other read") - case *sqlparser.OtherAdmin: - return nil, errors.New("unsupported construct: other admin") - case *sqlparser.Begin: - return nil, errors.New("unsupported construct: begin") - case *sqlparser.Commit: - return nil, errors.New("unsupported construct: commit") - case *sqlparser.Rollback: - return nil, errors.New("unsupported construct: rollback") - default: - return nil, fmt.Errorf("BUG: unexpected statement type: %T", stmt) + + if vschema.Destination() != nil { + instruction, err = buildPlanForBypass(stmt, vschema) + } else { + switch stmt := stmt.(type) { + case *sqlparser.Select: + instruction, err = buildSelectPlan(stmt, vschema) + case *sqlparser.Insert: + instruction, err = buildInsertPlan(stmt, vschema) + case *sqlparser.Update: + instruction, err = buildUpdatePlan(stmt, vschema) + case *sqlparser.Delete: + instruction, err = buildDeletePlan(stmt, vschema) + case *sqlparser.Union: + instruction, err = buildUnionPlan(stmt, vschema) + case *sqlparser.Set: + return nil, errors.New("unsupported construct: set") + case *sqlparser.Show: + return nil, errors.New("unsupported construct: show") + case *sqlparser.DDL: + return nil, errors.New("unsupported construct: ddl") + case *sqlparser.DBDDL: + return nil, errors.New("unsupported construct: ddl on database") + case *sqlparser.OtherRead: + return nil, errors.New("unsupported construct: other read") + case *sqlparser.OtherAdmin: + return nil, errors.New("unsupported construct: other admin") + case *sqlparser.Begin: + return nil, errors.New("unsupported construct: begin") + case *sqlparser.Commit: + return nil, errors.New("unsupported construct: commit") + case *sqlparser.Rollback: + return nil, errors.New("unsupported construct: rollback") + default: + return nil, fmt.Errorf("BUG: unexpected statement type: %T", stmt) + } } if err != nil { return nil, err diff --git a/go/vt/vtgate/planbuilder/bypass.go b/go/vt/vtgate/planbuilder/bypass.go new file mode 100644 index 00000000000..945f9b0db7c --- /dev/null +++ b/go/vt/vtgate/planbuilder/bypass.go @@ -0,0 +1,35 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package planbuilder + +import ( + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vtgate/engine" +) + +func buildPlanForBypass(stmt sqlparser.Statement, vschema ContextVSchema) (engine.Primitive, error) { + keyspace, err := vschema.DefaultKeyspace() + if err != nil { + return nil, err + } + return &engine.Send{ + Keyspace: keyspace, + TargetDestination: vschema.Destination(), + TargetTabletType: vschema.TabletType(), + Query: sqlparser.String(stmt), + }, nil +} diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 8828e609d3e..90c96bbf4ca 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -169,6 +169,34 @@ func TestOne(t *testing.T) { testFile(t, "onecase.txt", "", vschema) } +func TestBypassPlanning(t *testing.T) { + query := "select * from ks.t1" + stmt, err := sqlparser.Parse(query) + require.NoError(t, err) + vschema := &vschemaWrapper{ + v: loadSchema(t, "schema_test.json"), + keyspace: &vindexes.Keyspace{ + Name: "main", + Sharded: false, + }, + tabletType: topodatapb.TabletType_MASTER, + dest: key.DestinationShard("-80"), + } + plan, err := BuildFromStmt(query, stmt, vschema, sqlparser.BindVarNeeds{}) + require.NoError(t, err) + expected := &engine.Send{Query: query, + Keyspace: &vindexes.Keyspace{ + Name: "main", + Sharded: false, + }, + TargetTabletType: topodatapb.TabletType_MASTER, + TargetDestination: key.DestinationShard("-80"), + } + if diff := cmp.Diff(expected, plan.Instructions); diff != "" { + t.Errorf("-want,+actual\n%s", diff) + } +} + func loadSchema(t *testing.T, filename string) *vindexes.VSchema { formal, err := vindexes.LoadFormal(locateFile(filename)) if err != nil { @@ -189,11 +217,18 @@ func loadSchema(t *testing.T, filename string) *vindexes.VSchema { var _ ContextVSchema = (*vschemaWrapper)(nil) type vschemaWrapper struct { - v *vindexes.VSchema + v *vindexes.VSchema + keyspace *vindexes.Keyspace + tabletType topodatapb.TabletType + dest key.Destination +} + +func (vw *vschemaWrapper) TabletType() topodatapb.TabletType { + return vw.tabletType } func (vw *vschemaWrapper) Destination() key.Destination { - panic("implement me") + return vw.dest } func (vw *vschemaWrapper) FindTable(tab sqlparser.TableName) (*vindexes.Table, string, topodatapb.TabletType, key.Destination, error) { diff --git a/go/vt/vtgate/vcursor_impl.go b/go/vt/vtgate/vcursor_impl.go index f8863037b97..745630e60ee 100644 --- a/go/vt/vtgate/vcursor_impl.go +++ b/go/vt/vtgate/vcursor_impl.go @@ -78,6 +78,9 @@ type vcursorImpl struct { // on behalf of the original query. func newVCursorImpl(ctx context.Context, safeSession *SafeSession, marginComments sqlparser.MarginComments, executor *Executor, logStats *LogStats, vschema *vindexes.VSchema, resolver *srvtopo.Resolver) (*vcursorImpl, error) { keyspace, tabletType, destination, err := parseDestinationTarget(safeSession.TargetString, vschema) + // ks + // select b from ts.a + // select b from ps.a if err != nil { return nil, err } @@ -255,6 +258,11 @@ func (vc *vcursorImpl) Destination() key.Destination { return vc.destination } +// TabletType implements the ContextVSchema interface +func (vc *vcursorImpl) TabletType() topodatapb.TabletType { + return vc.tabletType +} + func commentedShardQueries(shardQueries []*querypb.BoundQuery, marginComments sqlparser.MarginComments) []*querypb.BoundQuery { if marginComments.Leading == "" && marginComments.Trailing == "" { return shardQueries From 3add0bdc8c943529a9626393510ad8b9d96d4221 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Fri, 27 Mar 2020 12:09:54 +0530 Subject: [PATCH 75/85] added test for send in engine Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/fake_vcursor_test.go | 2 + go/vt/vtgate/engine/send.go | 33 +++++++++--- go/vt/vtgate/engine/send_test.go | 66 ++++++++++++++++++++++++ go/vt/vtgate/planbuilder/bypass.go | 11 +++- go/vt/vtgate/planbuilder/plan_test.go | 1 - 5 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 go/vt/vtgate/engine/send_test.go diff --git a/go/vt/vtgate/engine/fake_vcursor_test.go b/go/vt/vtgate/engine/fake_vcursor_test.go index 7ff09281eb8..5e392c0d534 100644 --- a/go/vt/vtgate/engine/fake_vcursor_test.go +++ b/go/vt/vtgate/engine/fake_vcursor_test.go @@ -202,6 +202,8 @@ func (f *loggingVCursor) ResolveDestinations(keyspace string, ids []*querypb.Val shards = f.shards[:1] case key.DestinationNone: // Nothing to do here. + case key.DestinationShard: + shards = []string{destination.String()} default: return nil, nil, fmt.Errorf("unsupported destination: %v", destination) } diff --git a/go/vt/vtgate/engine/send.go b/go/vt/vtgate/engine/send.go index e5f3f10b000..134f937b2ef 100644 --- a/go/vt/vtgate/engine/send.go +++ b/go/vt/vtgate/engine/send.go @@ -4,8 +4,11 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/proto/query" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/vindexes" + + querypb "vitess.io/vitess/go/vt/proto/query" ) var _ Primitive = (*Send)(nil) @@ -19,10 +22,6 @@ type Send struct { // This bypases the core of the v3 engine. TargetDestination key.Destination - // TargetTabletType specifies an explicit target destination tablet type - // this is only used in conjunction with TargetDestination - TargetTabletType topodatapb.TabletType - // Query specifies the query to be executed. Query string } @@ -44,7 +43,29 @@ func (s Send) GetTableName() string { // Execute implements Primitive interface func (s Send) Execute(vcursor VCursor, bindVars map[string]*query.BindVariable, wantfields bool) (*sqltypes.Result, error) { - panic("implement me") + rss, _, err := vcursor.ResolveDestinations(s.Keyspace.Name, nil, []key.Destination{s.TargetDestination}) + if err != nil { + return nil, vterrors.Wrap(err, "sendExecute") + } + + if !s.Keyspace.Sharded && len(rss) != 1 { + return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "Keyspace does not have exactly one shard: %v", rss) + } + + queries := make([]*querypb.BoundQuery, len(rss)) + for i := range rss { + queries[i] = &querypb.BoundQuery{ + Sql: s.Query, + BindVariables: bindVars, + } + } + + result, errs := vcursor.ExecuteMultiShard(rss, queries, false, true) + err = vterrors.Aggregate(errs) + if err != nil { + return nil, err + } + return result, nil } // StreamExecute implements Primitive interface diff --git a/go/vt/vtgate/engine/send_test.go b/go/vt/vtgate/engine/send_test.go new file mode 100644 index 00000000000..1e9f824e6aa --- /dev/null +++ b/go/vt/vtgate/engine/send_test.go @@ -0,0 +1,66 @@ +package engine + +import ( + "errors" + "testing" + + "vitess.io/vitess/go/vt/key" + querypb "vitess.io/vitess/go/vt/proto/query" + "vitess.io/vitess/go/vt/vtgate/vindexes" +) + +func TestSendUnsharded(t *testing.T) { + send := &Send{ + Keyspace: &vindexes.Keyspace{ + Name: "ks", + Sharded: false, + }, + Query: "dummy_query", + TargetDestination: key.DestinationAllShards{}, + } + + vc := &loggingVCursor{shards: []string{"0"}} + _, err := send.Execute(vc, map[string]*querypb.BindVariable{}, false) + if err != nil { + t.Fatal(err) + } + vc.ExpectLog(t, []string{ + `ResolveDestinations ks [] Destinations:DestinationAllShards()`, + `ExecuteMultiShard ks.0: dummy_query {} false true`, + }) + + // Failure cases + vc = &loggingVCursor{shardErr: errors.New("shard_error")} + _, err = send.Execute(vc, map[string]*querypb.BindVariable{}, false) + expectError(t, "Execute", err, "sendExecute: shard_error") + + vc = &loggingVCursor{} + _, err = send.Execute(vc, map[string]*querypb.BindVariable{}, false) + expectError(t, "Execute", err, "Keyspace does not have exactly one shard: []") +} + +func TestSendSharded(t *testing.T) { + send := &Send{ + Keyspace: &vindexes.Keyspace{ + Name: "ks", + Sharded: true, + }, + Query: "dummy_query", + TargetDestination: key.DestinationShard("20-"), + } + + vc := &loggingVCursor{shards: []string{"-20", "20-"}} + _, err := send.Execute(vc, map[string]*querypb.BindVariable{}, false) + if err != nil { + t.Fatal(err) + } + vc.ExpectLog(t, []string{ + `ResolveDestinations ks [] Destinations:DestinationShard(20-)`, + `ExecuteMultiShard ks.DestinationShard(20-): dummy_query {} false true`, + }) + + // Failure cases + vc = &loggingVCursor{shardErr: errors.New("shard_error")} + _, err = send.Execute(vc, map[string]*querypb.BindVariable{}, false) + expectError(t, "Execute", err, "sendExecute: shard_error") +} diff --git a/go/vt/vtgate/planbuilder/bypass.go b/go/vt/vtgate/planbuilder/bypass.go index 945f9b0db7c..142c04ffc34 100644 --- a/go/vt/vtgate/planbuilder/bypass.go +++ b/go/vt/vtgate/planbuilder/bypass.go @@ -17,11 +17,21 @@ limitations under the License. package planbuilder import ( + "vitess.io/vitess/go/vt/key" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/engine" ) func buildPlanForBypass(stmt sqlparser.Statement, vschema ContextVSchema) (engine.Primitive, error) { + switch vschema.Destination().(type) { + case key.DestinationExactKeyRange: + if _, ok := stmt.(*sqlparser.Insert); ok { + return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "range queries not supported for inserts: %s", vschema.TargetString()) + } + } + keyspace, err := vschema.DefaultKeyspace() if err != nil { return nil, err @@ -29,7 +39,6 @@ func buildPlanForBypass(stmt sqlparser.Statement, vschema ContextVSchema) (engin return &engine.Send{ Keyspace: keyspace, TargetDestination: vschema.Destination(), - TargetTabletType: vschema.TabletType(), Query: sqlparser.String(stmt), }, nil } diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 90c96bbf4ca..aab3853de72 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -189,7 +189,6 @@ func TestBypassPlanning(t *testing.T) { Name: "main", Sharded: false, }, - TargetTabletType: topodatapb.TabletType_MASTER, TargetDestination: key.DestinationShard("-80"), } if diff := cmp.Diff(expected, plan.Instructions); diff != "" { From 59173b4236e80a7601614a38db365880cd16fac2 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 27 Mar 2020 08:43:08 +0100 Subject: [PATCH 76/85] Added route type and testing Signed-off-by: Andres Taylor --- go/test/utils/diff.go | 83 +++++++++++++++++++++++++++ go/vt/vtgate/engine/send.go | 32 +++++------ go/vt/vtgate/engine/send_test.go | 10 ++-- go/vt/vtgate/planbuilder/bypass.go | 4 ++ go/vt/vtgate/planbuilder/plan_test.go | 15 ++++- 5 files changed, 119 insertions(+), 25 deletions(-) create mode 100644 go/test/utils/diff.go diff --git a/go/test/utils/diff.go b/go/test/utils/diff.go new file mode 100644 index 00000000000..05a1f9eb66d --- /dev/null +++ b/go/test/utils/diff.go @@ -0,0 +1,83 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +// MustMatchFn is used to create a common diff function for a test file +// Usage in *_test.go file: +// +// Top declaration: +// +// var mustMatch = testutils.MustMatchFn( +// []interface{}{ // types with unexported fields +// type1{}, +// type2{}, +// ... +// typeN{}, +// }, +// []string{ // ignored fields +// ".id", // id numbers are unstable +// ".createAt", // created dates might not be interesting to compare +// }, +// ) +// +// In Test*() function: +// +// mustMatch(t, want, got, "something doesn't match") +func MustMatchFn(allowUnexportedTypes []interface{}, ignoredFields []string, extraOpts ...cmp.Option) func(t *testing.T, want, got interface{}, errMsg string) { + diffOpts := append([]cmp.Option{ + cmp.AllowUnexported(allowUnexportedTypes...), + cmpIgnoreFields(ignoredFields...), + }, extraOpts...) + // Diffs want/got and fails with errMsg on any failure. + return func(t *testing.T, want, got interface{}, errMsg string) { + t.Helper() + diff := cmp.Diff(want, got, diffOpts...) + if diff != "" { + t.Fatalf("%s: (-want +got)\n%v", errMsg, diff) + } + } +} + +// MustMatch is a convenience version of MustMatchFn with no overrides. +// Usage in Test*() function: +// +// testutils.MustMatch(t, want, got, "something doesn't match") +var MustMatch = MustMatchFn(nil, nil) + +// Skips fields of pathNames for cmp.Diff. +// Similar to standard cmpopts.IgnoreFields, but allows unexported fields. +func cmpIgnoreFields(pathNames ...string) cmp.Option { + skipFields := make(map[string]bool, len(pathNames)) + for _, name := range pathNames { + skipFields[name] = true + } + + return cmp.FilterPath(func(path cmp.Path) bool { + for _, ps := range path { + if skipFields[ps.String()] { + return true + } + } + return false + }, cmp.Ignore()) +} diff --git a/go/vt/vtgate/engine/send.go b/go/vt/vtgate/engine/send.go index 134f937b2ef..76fefdf7beb 100644 --- a/go/vt/vtgate/engine/send.go +++ b/go/vt/vtgate/engine/send.go @@ -24,25 +24,30 @@ type Send struct { // Query specifies the query to be executed. Query string + + // OpCode specifies the route type + OpCode string + + noInputs } // RouteType implements Primitive interface -func (s Send) RouteType() string { - panic("implement me") +func (s *Send) RouteType() string { + return s.OpCode } // GetKeyspaceName implements Primitive interface -func (s Send) GetKeyspaceName() string { - panic("implement me") +func (s *Send) GetKeyspaceName() string { + return s.Keyspace.Name } // GetTableName implements Primitive interface -func (s Send) GetTableName() string { - panic("implement me") +func (s *Send) GetTableName() string { + return "" } // Execute implements Primitive interface -func (s Send) Execute(vcursor VCursor, bindVars map[string]*query.BindVariable, wantfields bool) (*sqltypes.Result, error) { +func (s *Send) Execute(vcursor VCursor, bindVars map[string]*query.BindVariable, _ bool) (*sqltypes.Result, error) { rss, _, err := vcursor.ResolveDestinations(s.Keyspace.Name, nil, []key.Destination{s.TargetDestination}) if err != nil { return nil, vterrors.Wrap(err, "sendExecute") @@ -69,16 +74,11 @@ func (s Send) Execute(vcursor VCursor, bindVars map[string]*query.BindVariable, } // StreamExecute implements Primitive interface -func (s Send) StreamExecute(vcursor VCursor, bindVars map[string]*query.BindVariable, wantields bool, callback func(*sqltypes.Result) error) error { - panic("implement me") +func (s *Send) StreamExecute(vcursor VCursor, bindVars map[string]*query.BindVariable, wantields bool, callback func(*sqltypes.Result) error) error { + return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "not reachable") // TODO: systay - this should work } // GetFields implements Primitive interface -func (s Send) GetFields(vcursor VCursor, bindVars map[string]*query.BindVariable) (*sqltypes.Result, error) { - panic("implement me") -} - -// Inputs implements Primitive interface -func (s Send) Inputs() []Primitive { - panic("implement me") +func (s *Send) GetFields(vcursor VCursor, bindVars map[string]*query.BindVariable) (*sqltypes.Result, error) { + return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "not reachable") // TODO: systay - @sugu, is this correct? } diff --git a/go/vt/vtgate/engine/send_test.go b/go/vt/vtgate/engine/send_test.go index 1e9f824e6aa..c8883661794 100644 --- a/go/vt/vtgate/engine/send_test.go +++ b/go/vt/vtgate/engine/send_test.go @@ -4,6 +4,8 @@ import ( "errors" "testing" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/vt/key" querypb "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/vtgate/vindexes" @@ -21,9 +23,7 @@ func TestSendUnsharded(t *testing.T) { vc := &loggingVCursor{shards: []string{"0"}} _, err := send.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `ExecuteMultiShard ks.0: dummy_query {} false true`, @@ -51,9 +51,7 @@ func TestSendSharded(t *testing.T) { vc := &loggingVCursor{shards: []string{"-20", "20-"}} _, err := send.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationShard(20-)`, `ExecuteMultiShard ks.DestinationShard(20-): dummy_query {} false true`, diff --git a/go/vt/vtgate/planbuilder/bypass.go b/go/vt/vtgate/planbuilder/bypass.go index 142c04ffc34..88583d2104f 100644 --- a/go/vt/vtgate/planbuilder/bypass.go +++ b/go/vt/vtgate/planbuilder/bypass.go @@ -40,5 +40,9 @@ func buildPlanForBypass(stmt sqlparser.Statement, vschema ContextVSchema) (engin Keyspace: keyspace, TargetDestination: vschema.Destination(), Query: sqlparser.String(stmt), + OpCode: ByPassOpCode, }, nil } + +//ByPassOpCode is the opcode +const ByPassOpCode = "ByPass" diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index aab3853de72..93a9504bb95 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -26,6 +26,8 @@ import ( "strings" "testing" + "vitess.io/vitess/go/test/utils" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" @@ -169,6 +171,14 @@ func TestOne(t *testing.T) { testFile(t, "onecase.txt", "", vschema) } +var mustMatch = utils.MustMatchFn( + []interface{}{ // types with unexported fields + engine.Send{}, + }, + []string{ // ignored fields + }, +) + func TestBypassPlanning(t *testing.T) { query := "select * from ks.t1" stmt, err := sqlparser.Parse(query) @@ -190,10 +200,9 @@ func TestBypassPlanning(t *testing.T) { Sharded: false, }, TargetDestination: key.DestinationShard("-80"), + OpCode: ByPassOpCode, } - if diff := cmp.Diff(expected, plan.Instructions); diff != "" { - t.Errorf("-want,+actual\n%s", diff) - } + mustMatch(t, expected, plan.Instructions, "plan output not what we expected") } func loadSchema(t *testing.T, filename string) *vindexes.VSchema { From cc6c455a61b9e448afe8d8d57d38364da3fe9956 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Fri, 27 Mar 2020 17:02:40 +0530 Subject: [PATCH 77/85] Removed dest logic from handleExec and added destination in plan key Signed-off-by: Harshit Gangal --- go/vt/vtgate/autocommit_test.go | 10 +- go/vt/vtgate/executor.go | 48 +---- go/vt/vtgate/executor_dml_test.go | 255 +++++++++------------------ go/vt/vtgate/executor_select_test.go | 10 +- go/vt/vtgate/executor_test.go | 10 +- go/vt/vtgate/vcursor_impl.go | 8 + 6 files changed, 109 insertions(+), 232 deletions(-) diff --git a/go/vt/vtgate/autocommit_test.go b/go/vt/vtgate/autocommit_test.go index 1188cab4802..a3c689f1652 100644 --- a/go/vt/vtgate/autocommit_test.go +++ b/go/vt/vtgate/autocommit_test.go @@ -424,17 +424,17 @@ func TestAutocommitDirectRangeTarget(t *testing.T) { Autocommit: true, TransactionMode: vtgatepb.TransactionMode_MULTI, } - sql := "DELETE FROM sharded_user_msgs LIMIT 1000" + sql := "delete from sharded_user_msgs limit 1000" _, err := executor.Execute(context.Background(), "TestExecute", NewSafeSession(session), sql, map[string]*querypb.BindVariable{}) require.NoError(t, err) - testQueries(t, "sbc1", sbc1, []*querypb.BoundQuery{{ + testBatchQuery(t, "sbc1", sbc1, &querypb.BoundQuery{ Sql: sql, BindVariables: map[string]*querypb.BindVariable{}, - }}) - testAsTransactionCount(t, "sbc1", sbc1, 0) - testCommitCount(t, "sbc1", sbc1, 1) + }) + testAsTransactionCount(t, "sbc1", sbc1, 1) + testCommitCount(t, "sbc1", sbc1, 0) } func autocommitExec(executor *Executor, sql string) (*sqltypes.Result, error) { diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 1d7e2138fb7..3436297cce1 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -266,49 +266,6 @@ func (e *Executor) execute(ctx context.Context, safeSession *SafeSession, sql st } func (e *Executor) handleExec(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, destKeyspace string, destTabletType topodatapb.TabletType, dest key.Destination, logStats *LogStats, stmtType sqlparser.StatementType) (*sqltypes.Result, error) { - if dest != nil { - if destKeyspace == "" { - return nil, errNoKeyspace - } - - switch dest.(type) { - case key.DestinationExactKeyRange: - stmtType := sqlparser.Preview(sql) - if stmtType == sqlparser.StmtInsert { - return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "range queries not supported for inserts: %s", safeSession.TargetString) - } - - } - - execStart := time.Now() - if e.normalize { - query, comments := sqlparser.SplitMarginComments(sql) - stmt, err := sqlparser.Parse(query) - if err != nil { - return nil, err - } - rewriteResult, err := sqlparser.PrepareAST(stmt, bindVars, "vtg") - if err != nil { - return nil, err - } - normalized := sqlparser.String(rewriteResult.AST) - sql = comments.Leading + normalized + comments.Trailing - neededBindVariables, err := e.createNeededBindVariables(rewriteResult.BindVarNeeds, safeSession) - if err != nil { - return nil, err - } - for k, v := range neededBindVariables { - bindVars[k] = v - } - } - logStats.PlanTime = execStart.Sub(logStats.StartTime) - logStats.SQL = sql - logStats.BindVariables = bindVars - result, err := e.resolver.Execute(ctx, sql, bindVars, destKeyspace, destTabletType, dest, safeSession, false /* notInTransaction */, safeSession.Options, logStats, true /* canAutocommit */) - logStats.ExecuteTime = time.Since(execStart) - e.updateQueryCounts("ShardDirect", "", "", int64(logStats.ShardQueries)) - return result, err - } // V3 mode. query, comments := sqlparser.SplitMarginComments(sql) @@ -1376,8 +1333,7 @@ func (e *Executor) getPlan(vcursor *vcursorImpl, sql string, comments sqlparser. if e.VSchema() == nil { return nil, errors.New("vschema not initialized") } - keyspace := vcursor.keyspace - planKey := keyspace + vindexes.TabletTypeSuffix[vcursor.tabletType] + ":" + sql + planKey := vcursor.planPrefixKey() + ":" + sql if plan, ok := e.plans.Get(planKey); ok { return plan.(*engine.Plan), nil } @@ -1409,7 +1365,7 @@ func (e *Executor) getPlan(vcursor *vcursorImpl, sql string, comments sqlparser. logStats.BindVariables = bindVars } - planKey = keyspace + vindexes.TabletTypeSuffix[vcursor.tabletType] + ":" + normalized + planKey = vcursor.planPrefixKey() + ":" + normalized if plan, ok := e.plans.Get(planKey); ok { return plan.(*engine.Plan), nil } diff --git a/go/vt/vtgate/executor_dml_test.go b/go/vt/vtgate/executor_dml_test.go index d8f2a3cc62b..6116d8d5d43 100644 --- a/go/vt/vtgate/executor_dml_test.go +++ b/go/vt/vtgate/executor_dml_test.go @@ -21,6 +21,9 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/test/utils" + "github.com/stretchr/testify/require" "golang.org/x/net/context" @@ -314,12 +317,8 @@ func TestUpdateNormalize(t *testing.T) { "vtg2": sqltypes.TestBindVariable(int64(1)), }, }} - if sbc1.Queries != nil { - t.Errorf("sbc1.Queries: %+v, want nil\n", sbc1.Queries) - } - if !reflect.DeepEqual(sbc2.Queries, wantQueries) { - t.Errorf("sbc2.Queries: %+v, want %+v\n", sbc2.Queries, wantQueries) - } + assert.Empty(t, sbc1.BatchQueries) + utils.MustMatch(t, sbc2.BatchQueries[0], wantQueries, "didn't get expected queries") sbc2.Queries = nil masterSession.TargetString = "" } @@ -1524,186 +1523,102 @@ func TestInsertBadAutoInc(t *testing.T) { } func TestKeyDestRangeQuery(t *testing.T) { - executor, sbc1, sbc2, _ := createExecutorEnv() - // it works in a single shard key range - masterSession.TargetString = "TestExecutor[40-60]" - _, err := executorExec(executor, "DELETE FROM sharded_user_msgs LIMIT 1000", nil) - require.NoError(t, err) - sql := "DELETE FROM sharded_user_msgs LIMIT 1000" - wantQueries := []*querypb.BoundQuery{{ - Sql: sql, - BindVariables: map[string]*querypb.BindVariable{}, - }} - - if len(sbc1.Queries) != 0 { - t.Errorf("sbc1.Queries: %+v, want %+v\n", sbc1.Queries, []*querypb.BoundQuery{}) + type testCase struct { + inputQuery, targetString string + expectedSbc1Query string + expectedSbc2Query string + } + deleteInput := "DELETE FROM sharded_user_msgs LIMIT 1000" + deleteOutput := "delete from sharded_user_msgs limit 1000" + + selectInput := "SELECT * FROM sharded_user_msgs LIMIT 1" + selectOutput := "select * from sharded_user_msgs limit 1" + updateInput := "UPDATE sharded_user_msgs set message='test' LIMIT 1" + updateOutput := "update sharded_user_msgs set message = 'test' limit 1" + insertInput := "INSERT INTO sharded_user_msgs(message) VALUES('test')" + insertOutput := "insert into sharded_user_msgs(message) values ('test')" + tests := []testCase{ + { + inputQuery: deleteInput, + targetString: "TestExecutor[-60]", + expectedSbc1Query: deleteOutput, + expectedSbc2Query: deleteOutput, + }, + { + inputQuery: deleteInput, + targetString: "TestExecutor[40-60]", + expectedSbc2Query: deleteOutput, + }, + { + inputQuery: deleteInput, + targetString: "TestExecutor[-]", + expectedSbc1Query: deleteOutput, + expectedSbc2Query: deleteOutput, + }, + { + inputQuery: selectInput, + targetString: "TestExecutor[-]", + expectedSbc1Query: selectOutput, + expectedSbc2Query: selectOutput, + }, + { + inputQuery: updateInput, + targetString: "TestExecutor[-]", + expectedSbc1Query: updateOutput, + expectedSbc2Query: updateOutput, + }, + { + inputQuery: insertInput, + targetString: "TestExecutor:40-60", + expectedSbc2Query: insertOutput, + }, + { + inputQuery: insertInput, + targetString: "TestExecutor:-20", + expectedSbc1Query: insertOutput, + }, } - testQueries(t, "sbc2", sbc2, wantQueries) - - sbc1.Queries = nil - sbc2.Queries = nil - - // it works with keyrange spanning two shards - masterSession.TargetString = "TestExecutor[-60]" - - _, err = executorExec(executor, sql, nil) - require.NoError(t, err) - testQueries(t, "sbc1", sbc1, wantQueries) - testQueries(t, "sbc1", sbc2, wantQueries) - sbc1.Queries = nil - sbc2.Queries = nil - - // it works with open ended key range - masterSession.TargetString = "TestExecutor[-]" + for _, tc := range tests { + t.Run(tc.targetString+" - "+tc.inputQuery, func(t *testing.T) { + executor, sbc1, sbc2, _ := createExecutorEnv() - _, err = executorExec(executor, sql, nil) - require.NoError(t, err) - - testQueries(t, "sbc1", sbc1, wantQueries) - testQueries(t, "sbc2", sbc2, wantQueries) - - sbc1.Queries = nil - sbc2.Queries = nil - - // it works for select - sql = "SELECT * FROM sharded_user_msgs LIMIT 1" - wantQueries = []*querypb.BoundQuery{{ - Sql: sql, - BindVariables: map[string]*querypb.BindVariable{}, - }} - - _, err = executorExec(executor, sql, nil) - require.NoError(t, err) + masterSession.TargetString = tc.targetString + _, err := executorExec(executor, tc.inputQuery, nil) + require.NoError(t, err) - testQueries(t, "sbc1", sbc1, wantQueries) - testQueries(t, "sbc2", sbc2, wantQueries) - - sbc1.Queries = nil - sbc2.Queries = nil - - // it works for updates - sql = "UPDATE sharded_user_msgs set message='test' LIMIT 1" - - wantQueries = []*querypb.BoundQuery{{ - Sql: sql, - BindVariables: map[string]*querypb.BindVariable{}, - }} - - _, err = executorExec(executor, sql, nil) - require.NoError(t, err) - - testQueries(t, "sbc1", sbc1, wantQueries) - testQueries(t, "sbc2", sbc2, wantQueries) + if tc.expectedSbc1Query == "" { + require.Empty(t, sbc1.BatchQueries, "sbc1") + } else { + assertBatchQueriesContain(t, tc.expectedSbc1Query, "sbc1", sbc1) + } - sbc1.Queries = nil - sbc2.Queries = nil + if tc.expectedSbc2Query == "" { + require.Empty(t, sbc2.BatchQueries) + } else { + assertBatchQueriesContain(t, tc.expectedSbc2Query, "sbc2", sbc2) + } + }) + } // it does not work for inserts - _, err = executorExec(executor, "INSERT INTO sharded_user_msgs(message) VALUES('test')", nil) + executor, _, _, _ := createExecutorEnv() + masterSession.TargetString = "TestExecutor[-]" + _, err := executorExec(executor, insertInput, nil) - want := "range queries not supported for inserts: TestExecutor[-]" - if err == nil || err.Error() != want { - t.Errorf("got: %v, want %s", err, want) - } + require.EqualError(t, err, "range queries not supported for inserts: TestExecutor[-]") - sbc1.Queries = nil - sbc2.Queries = nil masterSession.TargetString = "" } -func TestKeyShardDestQuery(t *testing.T) { - executor, sbc1, sbc2, _ := createExecutorEnv() - // it works in a single shard key range - masterSession.TargetString = "TestExecutor:40-60" - - _, err := executorExec(executor, "DELETE FROM sharded_user_msgs LIMIT 1000", nil) - require.NoError(t, err) - sql := "DELETE FROM sharded_user_msgs LIMIT 1000" - wantQueries := []*querypb.BoundQuery{{ - Sql: sql, - BindVariables: map[string]*querypb.BindVariable{}, - }} - - if len(sbc1.Queries) != 0 { - t.Errorf("sbc1.Queries: %+v, want %+v\n", sbc1.Queries, []*querypb.BoundQuery{}) - } - testQueries(t, "sbc2", sbc2, wantQueries) - - sbc1.Queries = nil - sbc2.Queries = nil - - masterSession.TargetString = "TestExecutor:40-60" - - _, err = executorExec(executor, sql, nil) - require.NoError(t, err) - - testQueries(t, "sbc2", sbc2, wantQueries) - - sbc1.Queries = nil - sbc2.Queries = nil - - // it works for select - sql = "SELECT * FROM sharded_user_msgs LIMIT 1" - wantQueries = []*querypb.BoundQuery{{ - Sql: sql, - BindVariables: map[string]*querypb.BindVariable{}, - }} - - _, err = executorExec(executor, sql, nil) - require.NoError(t, err) - - if len(sbc1.Queries) != 0 { - t.Errorf("sbc1.Queries: %+v, want %+v\n", sbc1.Queries, []*querypb.BoundQuery{}) - } - - testQueries(t, "sbc2", sbc2, wantQueries) - - sbc1.Queries = nil - sbc2.Queries = nil - - // it works for updates - sql = "UPDATE sharded_user_msgs set message='test' LIMIT 1" - - wantQueries = []*querypb.BoundQuery{{ +func assertBatchQueriesContain(t *testing.T, sql, sbcName string, sbc *sandboxconn.SandboxConn) { + t.Helper() + expectedQuery := &querypb.BoundQuery{ Sql: sql, BindVariables: map[string]*querypb.BindVariable{}, - }} - - _, err = executorExec(executor, sql, nil) - - require.NoError(t, err) - - if len(sbc1.Queries) != 0 { - t.Errorf("sbc1.Queries: %+v, want %+v\n", sbc1.Queries, []*querypb.BoundQuery{}) } - - testQueries(t, "sbc2", sbc2, wantQueries) - - sbc1.Queries = nil - sbc2.Queries = nil - - // it works for inserts - - sql = "INSERT INTO sharded_user_msgs(message) VALUES('test')" - _, err = executorExec(executor, sql, nil) - - wantQueries = []*querypb.BoundQuery{{ - Sql: sql, - BindVariables: map[string]*querypb.BindVariable{}, - }} - require.NoError(t, err) - - if len(sbc1.Queries) != 0 { - t.Errorf("sbc1.Queries: %+v, want %+v\n", sbc1.Queries, []*querypb.BoundQuery{}) - } - - testQueries(t, "sbc2", sbc2, wantQueries) - - sbc1.Queries = nil - sbc2.Queries = nil - masterSession.TargetString = "" + testBatchQuery(t, sbcName, sbc, expectedQuery) } // Prepared statement tests diff --git a/go/vt/vtgate/executor_select_test.go b/go/vt/vtgate/executor_select_test.go index 502680c4945..f70ce6da7de 100644 --- a/go/vt/vtgate/executor_select_test.go +++ b/go/vt/vtgate/executor_select_test.go @@ -22,6 +22,8 @@ import ( "strings" "testing" + "vitess.io/vitess/go/test/utils" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -632,12 +634,8 @@ func TestSelectNormalize(t *testing.T) { "vtg1": sqltypes.TestBindVariable(int64(1)), }, }} - if sbc1.Queries != nil { - t.Errorf("sbc1.Queries: %+v, want nil\n", sbc1.Queries) - } - if !reflect.DeepEqual(sbc2.Queries, wantQueries) { - t.Errorf("sbc2.Queries: %+v, want %+v\n", sbc2.Queries, wantQueries) - } + require.Empty(t, sbc1.BatchQueries) + utils.MustMatch(t, sbc2.BatchQueries[0], wantQueries, "sbc2.Queries") sbc2.Queries = nil masterSession.TargetString = "" } diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index c97854e27da..543cd155806 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -194,6 +194,7 @@ func TestExecutorTransactionsNoAutoCommit(t *testing.T) { } } +//TODO - what about these? func TestDirectTargetRewrites(t *testing.T) { executor, _, _, sbclookup := createExecutorEnv() executor.normalize = true @@ -205,13 +206,12 @@ func TestDirectTargetRewrites(t *testing.T) { } sql := "select database()" - if _, err := executor.Execute(context.Background(), "TestExecute", NewSafeSession(session), sql, map[string]*querypb.BindVariable{}); err != nil { - t.Error(err) - } - testQueries(t, "sbclookup", sbclookup, []*querypb.BoundQuery{{ + _, err := executor.Execute(context.Background(), "TestExecute", NewSafeSession(session), sql, map[string]*querypb.BindVariable{}) + require.NoError(t, err) + testBatchQuery(t, "sbclookup", sbclookup, &querypb.BoundQuery{ Sql: "select :__vtdbname as `database()` from dual", BindVariables: map[string]*querypb.BindVariable{"__vtdbname": sqltypes.StringBindVariable("TestUnsharded")}, - }}) + }) } func TestExecutorTransactionsAutoCommit(t *testing.T) { diff --git a/go/vt/vtgate/vcursor_impl.go b/go/vt/vtgate/vcursor_impl.go index 745630e60ee..11bee348d9d 100644 --- a/go/vt/vtgate/vcursor_impl.go +++ b/go/vt/vtgate/vcursor_impl.go @@ -17,6 +17,7 @@ limitations under the License. package vtgate import ( + "fmt" "sync/atomic" "time" @@ -304,3 +305,10 @@ func parseDestinationTarget(targetString string, vschema *vindexes.VSchema) (str } return destKeyspace, destTabletType, dest, err } + +func (vc *vcursorImpl) planPrefixKey() string { + if vc.destination != nil { + return fmt.Sprintf("%s%s%s", vc.keyspace, vindexes.TabletTypeSuffix[vc.tabletType], vc.destination.String()) + } + return fmt.Sprintf("%s%s", vc.keyspace, vindexes.TabletTypeSuffix[vc.tabletType]) +} From 289ef8108c8b8c79d58b5b1cffb3babaf86bc90f Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 27 Mar 2020 12:58:07 +0100 Subject: [PATCH 78/85] Moved to plan test using text file Signed-off-by: Andres Taylor --- go/vt/vtgate/planbuilder/plan_test.go | 68 +++++++------------ .../planbuilder/testdata/bypass_cases.txt | 59 ++++++++++++++++ 2 files changed, 85 insertions(+), 42 deletions(-) create mode 100644 go/vt/vtgate/planbuilder/testdata/bypass_cases.txt diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 93a9504bb95..a73ed6c8c88 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -26,8 +26,6 @@ import ( "strings" "testing" - "vitess.io/vitess/go/test/utils" - "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" @@ -144,7 +142,10 @@ func init() { } func TestPlan(t *testing.T) { - vschema := loadSchema(t, "schema_test.json") + vschemaWrapper := &vschemaWrapper{ + v: loadSchema(t, "schema_test.json"), + } + testOutputTempDir, err := ioutil.TempDir("", "plan_test") require.NoError(t, err) // You will notice that some tests expect user.Id instead of user.id. @@ -153,35 +154,29 @@ func TestPlan(t *testing.T) { // the column is named as Id. This is to make sure that // column names are case-preserved, but treated as // case-insensitive even if they come from the vschema. - testFile(t, "aggr_cases.txt", testOutputTempDir, vschema) - testFile(t, "dml_cases.txt", testOutputTempDir, vschema) - testFile(t, "from_cases.txt", testOutputTempDir, vschema) - testFile(t, "filter_cases.txt", testOutputTempDir, vschema) - testFile(t, "postprocess_cases.txt", testOutputTempDir, vschema) - testFile(t, "select_cases.txt", testOutputTempDir, vschema) - testFile(t, "symtab_cases.txt", testOutputTempDir, vschema) - testFile(t, "unsupported_cases.txt", testOutputTempDir, vschema) - testFile(t, "vindex_func_cases.txt", testOutputTempDir, vschema) - testFile(t, "wireup_cases.txt", testOutputTempDir, vschema) - testFile(t, "memory_sort_cases.txt", testOutputTempDir, vschema) + testFile(t, "aggr_cases.txt", testOutputTempDir, vschemaWrapper) + testFile(t, "dml_cases.txt", testOutputTempDir, vschemaWrapper) + testFile(t, "from_cases.txt", testOutputTempDir, vschemaWrapper) + testFile(t, "filter_cases.txt", testOutputTempDir, vschemaWrapper) + testFile(t, "postprocess_cases.txt", testOutputTempDir, vschemaWrapper) + testFile(t, "select_cases.txt", testOutputTempDir, vschemaWrapper) + testFile(t, "symtab_cases.txt", testOutputTempDir, vschemaWrapper) + testFile(t, "unsupported_cases.txt", testOutputTempDir, vschemaWrapper) + testFile(t, "vindex_func_cases.txt", testOutputTempDir, vschemaWrapper) + testFile(t, "wireup_cases.txt", testOutputTempDir, vschemaWrapper) + testFile(t, "memory_sort_cases.txt", testOutputTempDir, vschemaWrapper) } func TestOne(t *testing.T) { - vschema := loadSchema(t, "schema_test.json") + vschema := &vschemaWrapper{ + v: loadSchema(t, "schema_test.json"), + } + testFile(t, "onecase.txt", "", vschema) } -var mustMatch = utils.MustMatchFn( - []interface{}{ // types with unexported fields - engine.Send{}, - }, - []string{ // ignored fields - }, -) - -func TestBypassPlanning(t *testing.T) { - query := "select * from ks.t1" - stmt, err := sqlparser.Parse(query) +func TestBypassPlanningFromFile(t *testing.T) { + testOutputTempDir, err := ioutil.TempDir("", "plan_test") require.NoError(t, err) vschema := &vschemaWrapper{ v: loadSchema(t, "schema_test.json"), @@ -192,17 +187,8 @@ func TestBypassPlanning(t *testing.T) { tabletType: topodatapb.TabletType_MASTER, dest: key.DestinationShard("-80"), } - plan, err := BuildFromStmt(query, stmt, vschema, sqlparser.BindVarNeeds{}) - require.NoError(t, err) - expected := &engine.Send{Query: query, - Keyspace: &vindexes.Keyspace{ - Name: "main", - Sharded: false, - }, - TargetDestination: key.DestinationShard("-80"), - OpCode: ByPassOpCode, - } - mustMatch(t, expected, plan.Instructions, "plan output not what we expected") + + testFile(t, "bypass_cases.txt", testOutputTempDir, vschema) } func loadSchema(t *testing.T, filename string) *vindexes.VSchema { @@ -278,21 +264,19 @@ type testPlan struct { Instructions engine.Primitive `json:",omitempty"` } -func testFile(t *testing.T, filename, tempDir string, vschema *vindexes.VSchema) { +func testFile(t *testing.T, filename, tempDir string, vschema *vschemaWrapper) { t.Run(filename, func(t *testing.T) { expected := &strings.Builder{} fail := false for tcase := range iterateExecFile(filename) { t.Run(tcase.comments, func(t *testing.T) { - plan, err := Build(tcase.input, &vschemaWrapper{ - v: vschema, - }) + plan, err := Build(tcase.input, vschema) out := getPlanOrErrorOutput(err, plan) if out != tcase.output { fail = true - t.Errorf("File: %s, Line: %v\n %s", filename, tcase.lineno, cmp.Diff(tcase.output, out)) + t.Errorf("File: %s, Line: %v\n %s\n%s", filename, tcase.lineno, cmp.Diff(tcase.output, out), out) } if err != nil { diff --git a/go/vt/vtgate/planbuilder/testdata/bypass_cases.txt b/go/vt/vtgate/planbuilder/testdata/bypass_cases.txt new file mode 100644 index 00000000000..30022a7cb59 --- /dev/null +++ b/go/vt/vtgate/planbuilder/testdata/bypass_cases.txt @@ -0,0 +1,59 @@ +# select bypass +"select count(*), col from unsharded" +{ + "Original": "select count(*), col from unsharded", + "Instructions": { + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetDestination": "-80", + "Query": "select count(*), col from unsharded", + "OpCode": "ByPass" + } +} + +# update bypass +"update user set val = 1 where id = 18446744073709551616 and id = 1" +{ + "Original": "update user set val = 1 where id = 18446744073709551616 and id = 1", + "Instructions": { + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetDestination": "-80", + "Query": "update user set val = 1 where id = 18446744073709551616 and id = 1", + "OpCode": "ByPass" + } +} + +# delete bypass +"DELETE FROM USER WHERE ID = 42" +{ + "Original": "DELETE FROM USER WHERE ID = 42", + "Instructions": { + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetDestination": "-80", + "Query": "delete from USER where ID = 42", + "OpCode": "ByPass" + } +} + +# insert bypass +"INSERT INTO USER (ID, NAME) VALUES (42, 'ms X')" +{ + "Original": "INSERT INTO USER (ID, NAME) VALUES (42, 'ms X')", + "Instructions": { + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetDestination": "-80", + "Query": "insert into USER(ID, NAME) values (42, 'ms X')", + "OpCode": "ByPass" + } +} From f1010d55acbd20a90821979b994ca040c0cb7b1c Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Fri, 27 Mar 2020 19:19:58 +0530 Subject: [PATCH 79/85] fix vtexplain test Signed-off-by: Harshit Gangal --- go/vt/vtexplain/testdata/multi-output/target-output.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/go/vt/vtexplain/testdata/multi-output/target-output.txt b/go/vt/vtexplain/testdata/multi-output/target-output.txt index abf70f5b57d..a4a11506c3e 100644 --- a/go/vt/vtexplain/testdata/multi-output/target-output.txt +++ b/go/vt/vtexplain/testdata/multi-output/target-output.txt @@ -1,12 +1,16 @@ ---------------------------------------------------------------------- select * from user where email='null@void.com' +1 ks_sharded/40-80: begin 1 ks_sharded/40-80: select * from user where email = 'null@void.com' limit 10001 +1 ks_sharded/40-80: commit ---------------------------------------------------------------------- select * from user where id in (1,2,3,4,5,6,7,8) +1 ks_sharded/40-80: begin 1 ks_sharded/40-80: select * from user where id in (1, 2, 3, 4, 5, 6, 7, 8) limit 10001 +1 ks_sharded/40-80: commit ---------------------------------------------------------------------- insert into user (id, name) values (2, 'bob') From 8219745666085e3b74320c5b5fb15de7e3816934 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 30 Mar 2020 11:32:23 +0530 Subject: [PATCH 80/85] Renamed isDML to rollbackOnError Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/delete.go | 4 +-- go/vt/vtgate/engine/fake_vcursor_test.go | 14 +++++----- go/vt/vtgate/engine/insert.go | 2 +- go/vt/vtgate/engine/primitive.go | 6 ++--- go/vt/vtgate/engine/route.go | 8 +++--- go/vt/vtgate/engine/update.go | 4 +-- go/vt/vtgate/executor.go | 2 +- go/vt/vtgate/vcursor_impl.go | 26 +++++++++---------- go/vt/vtgate/vcursor_impl_test.go | 1 - go/vt/vtgate/vindexes/consistent_lookup.go | 8 +++--- .../vtgate/vindexes/consistent_lookup_test.go | 12 ++++----- go/vt/vtgate/vindexes/lookup_internal.go | 8 +++--- go/vt/vtgate/vindexes/lookup_test.go | 10 +++---- go/vt/vtgate/vindexes/vindex.go | 4 +-- 14 files changed, 54 insertions(+), 55 deletions(-) diff --git a/go/vt/vtgate/engine/delete.go b/go/vt/vtgate/engine/delete.go index c1bf66db083..409ce0d5d97 100644 --- a/go/vt/vtgate/engine/delete.go +++ b/go/vt/vtgate/engine/delete.go @@ -166,7 +166,7 @@ func (del *Delete) execDeleteEqual(vcursor VCursor, bindVars map[string]*querypb return nil, vterrors.Wrap(err, "execDeleteEqual") } } - return execShard(vcursor, del.Query, bindVars, rs, true /* isDML */, true /* canAutocommit */) + return execShard(vcursor, del.Query, bindVars, rs, true /* rollbackOnError */, true /* canAutocommit */) } // deleteVindexEntries performs an delete if table owns vindex. @@ -231,6 +231,6 @@ func (del *Delete) execDeleteByDestination(vcursor VCursor, bindVars map[string] } } autocommit := (len(rss) == 1 || del.MultiShardAutocommit) && vcursor.AutocommitApproval() - res, errs := vcursor.ExecuteMultiShard(rss, queries, true /* isDML */, autocommit) + res, errs := vcursor.ExecuteMultiShard(rss, queries, true /* rollbackOnError */, autocommit) return res, vterrors.Aggregate(errs) } diff --git a/go/vt/vtgate/engine/fake_vcursor_test.go b/go/vt/vtgate/engine/fake_vcursor_test.go index 5e392c0d534..c0d93370f4b 100644 --- a/go/vt/vtgate/engine/fake_vcursor_test.go +++ b/go/vt/vtgate/engine/fake_vcursor_test.go @@ -58,11 +58,11 @@ func (t noopVCursor) SetContextTimeout(timeout time.Duration) context.CancelFunc func (t noopVCursor) RecordWarning(warning *querypb.QueryWarning) { } -func (t noopVCursor) Execute(method string, query string, bindvars map[string]*querypb.BindVariable, isDML bool, co vtgatepb.CommitOrder) (*sqltypes.Result, error) { +func (t noopVCursor) Execute(method string, query string, bindvars map[string]*querypb.BindVariable, rollbackOnError bool, co vtgatepb.CommitOrder) (*sqltypes.Result, error) { panic("unimplemented") } -func (t noopVCursor) ExecuteMultiShard(rss []*srvtopo.ResolvedShard, queries []*querypb.BoundQuery, isDML, autocommit bool) (*sqltypes.Result, []error) { +func (t noopVCursor) ExecuteMultiShard(rss []*srvtopo.ResolvedShard, queries []*querypb.BoundQuery, rollbackOnError, canAutocommit bool) (*sqltypes.Result, []error) { panic("unimplemented") } @@ -78,7 +78,7 @@ func (t noopVCursor) StreamExecuteMulti(query string, rss []*srvtopo.ResolvedSha panic("unimplemented") } -func (t noopVCursor) ExecuteKeyspaceID(keyspace string, ksid []byte, query string, bindVars map[string]*querypb.BindVariable, isDML, autocommit bool) (*sqltypes.Result, error) { +func (t noopVCursor) ExecuteKeyspaceID(keyspace string, ksid []byte, query string, bindVars map[string]*querypb.BindVariable, rollbackOnError, autocommit bool) (*sqltypes.Result, error) { panic("unimplemented") } @@ -121,7 +121,7 @@ func (f *loggingVCursor) RecordWarning(warning *querypb.QueryWarning) { f.warnings = append(f.warnings, warning) } -func (f *loggingVCursor) Execute(method string, query string, bindvars map[string]*querypb.BindVariable, isDML bool, co vtgatepb.CommitOrder) (*sqltypes.Result, error) { +func (f *loggingVCursor) Execute(method string, query string, bindvars map[string]*querypb.BindVariable, rollbackOnError bool, co vtgatepb.CommitOrder) (*sqltypes.Result, error) { name := "Unknown" switch co { case vtgatepb.CommitOrder_NORMAL: @@ -133,12 +133,12 @@ func (f *loggingVCursor) Execute(method string, query string, bindvars map[strin case vtgatepb.CommitOrder_AUTOCOMMIT: name = "ExecuteAutocommit" } - f.log = append(f.log, fmt.Sprintf("%s %s %v %v", name, query, printBindVars(bindvars), isDML)) + f.log = append(f.log, fmt.Sprintf("%s %s %v %v", name, query, printBindVars(bindvars), rollbackOnError)) return f.nextResult() } -func (f *loggingVCursor) ExecuteMultiShard(rss []*srvtopo.ResolvedShard, queries []*querypb.BoundQuery, isDML, canAutocommit bool) (*sqltypes.Result, []error) { - f.log = append(f.log, fmt.Sprintf("ExecuteMultiShard %v%v %v", printResolvedShardQueries(rss, queries), isDML, canAutocommit)) +func (f *loggingVCursor) ExecuteMultiShard(rss []*srvtopo.ResolvedShard, queries []*querypb.BoundQuery, rollbackOnError, canAutocommit bool) (*sqltypes.Result, []error) { + f.log = append(f.log, fmt.Sprintf("ExecuteMultiShard %v%v %v", printResolvedShardQueries(rss, queries), rollbackOnError, canAutocommit)) res, err := f.nextResult() if err != nil { return nil, []error{err} diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index 625a4656231..e08d5398ea2 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -277,7 +277,7 @@ func (ins *Insert) execInsertSharded(vcursor VCursor, bindVars map[string]*query } autocommit := (len(rss) == 1 || ins.MultiShardAutocommit) && vcursor.AutocommitApproval() - result, errs := vcursor.ExecuteMultiShard(rss, queries, true /* isDML */, autocommit) + result, errs := vcursor.ExecuteMultiShard(rss, queries, true /* rollbackOnError */, autocommit) if errs != nil { return nil, vterrors.Wrap(vterrors.Aggregate(errs), "execInsertSharded") } diff --git a/go/vt/vtgate/engine/primitive.go b/go/vt/vtgate/engine/primitive.go index 34a8d96a226..1ee2389b048 100644 --- a/go/vt/vtgate/engine/primitive.go +++ b/go/vt/vtgate/engine/primitive.go @@ -57,16 +57,16 @@ type VCursor interface { RecordWarning(warning *querypb.QueryWarning) // V3 functions. - Execute(method string, query string, bindvars map[string]*querypb.BindVariable, isDML bool, co vtgatepb.CommitOrder) (*sqltypes.Result, error) + Execute(method string, query string, bindvars map[string]*querypb.BindVariable, rollbackOnError bool, co vtgatepb.CommitOrder) (*sqltypes.Result, error) AutocommitApproval() bool // Shard-level functions. - ExecuteMultiShard(rss []*srvtopo.ResolvedShard, queries []*querypb.BoundQuery, isDML, canAutocommit bool) (*sqltypes.Result, []error) + ExecuteMultiShard(rss []*srvtopo.ResolvedShard, queries []*querypb.BoundQuery, rollbackOnError, canAutocommit bool) (*sqltypes.Result, []error) ExecuteStandalone(query string, bindvars map[string]*querypb.BindVariable, rs *srvtopo.ResolvedShard) (*sqltypes.Result, error) StreamExecuteMulti(query string, rss []*srvtopo.ResolvedShard, bindVars []map[string]*querypb.BindVariable, callback func(reply *sqltypes.Result) error) error // Keyspace ID level functions. - ExecuteKeyspaceID(keyspace string, ksid []byte, query string, bindVars map[string]*querypb.BindVariable, isDML, autocommit bool) (*sqltypes.Result, error) + ExecuteKeyspaceID(keyspace string, ksid []byte, query string, bindVars map[string]*querypb.BindVariable, rollbackOnError, autocommit bool) (*sqltypes.Result, error) // Resolver methods, from key.Destination to srvtopo.ResolvedShard. // Will replace all of the Topo functions. diff --git a/go/vt/vtgate/engine/route.go b/go/vt/vtgate/engine/route.go index 8a0f2f326a4..d1d78a9bc13 100644 --- a/go/vt/vtgate/engine/route.go +++ b/go/vt/vtgate/engine/route.go @@ -267,7 +267,7 @@ func (route *Route) execute(vcursor VCursor, bindVars map[string]*querypb.BindVa } queries := getQueries(route.Query, bvs) - result, errs := vcursor.ExecuteMultiShard(rss, queries, false /* isDML */, false /* autocommit */) + result, errs := vcursor.ExecuteMultiShard(rss, queries, false /* rollbackOnError */, false /* autocommit */) if errs != nil { if route.ScatterErrorsAsWarnings { @@ -362,7 +362,7 @@ func (route *Route) GetFields(vcursor VCursor, bindVars map[string]*querypb.Bind // This code is unreachable. It's just a sanity check. return nil, fmt.Errorf("no shards for keyspace: %s", route.Keyspace.Name) } - qr, err := execShard(vcursor, route.FieldQuery, bindVars, rss[0], false /* isDML */, false /* canAutocommit */) + qr, err := execShard(vcursor, route.FieldQuery, bindVars, rss[0], false /* rollbackOnError */, false /* canAutocommit */) if err != nil { return nil, err } @@ -518,14 +518,14 @@ func resolveKeyspaceID(vcursor VCursor, vindex vindexes.SingleColumn, vindexKey } } -func execShard(vcursor VCursor, query string, bindVars map[string]*querypb.BindVariable, rs *srvtopo.ResolvedShard, isDML, canAutocommit bool) (*sqltypes.Result, error) { +func execShard(vcursor VCursor, query string, bindVars map[string]*querypb.BindVariable, rs *srvtopo.ResolvedShard, rollbackOnError, canAutocommit bool) (*sqltypes.Result, error) { autocommit := canAutocommit && vcursor.AutocommitApproval() result, errs := vcursor.ExecuteMultiShard([]*srvtopo.ResolvedShard{rs}, []*querypb.BoundQuery{ { Sql: query, BindVariables: bindVars, }, - }, isDML, autocommit) + }, rollbackOnError, autocommit) return result, vterrors.Aggregate(errs) } diff --git a/go/vt/vtgate/engine/update.go b/go/vt/vtgate/engine/update.go index 565f86e1e14..684fe65d7d3 100644 --- a/go/vt/vtgate/engine/update.go +++ b/go/vt/vtgate/engine/update.go @@ -173,7 +173,7 @@ func (upd *Update) execUpdateEqual(vcursor VCursor, bindVars map[string]*querypb return nil, vterrors.Wrap(err, "execUpdateEqual") } } - return execShard(vcursor, upd.Query, bindVars, rs, true /* isDML */, true /* canAutocommit */) + return execShard(vcursor, upd.Query, bindVars, rs, true /* rollbackOnError */, true /* canAutocommit */) } // updateVindexEntries performs an update when a vindex is being modified @@ -260,6 +260,6 @@ func (upd *Update) execUpdateByDestination(vcursor VCursor, bindVars map[string] } autocommit := (len(rss) == 1 || upd.MultiShardAutocommit) && vcursor.AutocommitApproval() - result, errs := vcursor.ExecuteMultiShard(rss, queries, true /* isDML */, autocommit) + result, errs := vcursor.ExecuteMultiShard(rss, queries, true /* rollbackOnError */, autocommit) return result, vterrors.Aggregate(errs) } diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 3436297cce1..0debe8a97e5 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -312,7 +312,7 @@ func (e *Executor) handleExec(ctx context.Context, safeSession *SafeSession, sql } // Check if there was partial DML execution. If so, rollback the transaction. - if err != nil && safeSession.InTransaction() && vcursor.hasPartialDML { + if err != nil && safeSession.InTransaction() && vcursor.rollbackOnPartialExec { _ = e.txConn.Rollback(ctx, safeSession) err = vterrors.Errorf(vtrpcpb.Code_ABORTED, "transaction rolled back due to partial DML execution: %v", err) } diff --git a/go/vt/vtgate/vcursor_impl.go b/go/vt/vtgate/vcursor_impl.go index 11bee348d9d..3235086c4db 100644 --- a/go/vt/vtgate/vcursor_impl.go +++ b/go/vt/vtgate/vcursor_impl.go @@ -66,11 +66,11 @@ type vcursorImpl struct { executor iExecute resolver *srvtopo.Resolver logStats *LogStats - // hasPartialDML is set to true if any DML was successfully + // rollbackOnPartialExec is set to true if any DML was successfully // executed. If there was a subsequent failure, the transaction // must be forced to rollback. - hasPartialDML bool - vschema *vindexes.VSchema + rollbackOnPartialExec bool + vschema *vindexes.VSchema } // newVcursorImpl creates a vcursorImpl. Before creating this object, you have to separate out any marginComments that came with @@ -174,7 +174,7 @@ func (vc *vcursorImpl) TargetString() string { } // Execute is part of the engine.VCursor interface. -func (vc *vcursorImpl) Execute(method string, query string, bindVars map[string]*querypb.BindVariable, isDML bool, co vtgatepb.CommitOrder) (*sqltypes.Result, error) { +func (vc *vcursorImpl) Execute(method string, query string, bindVars map[string]*querypb.BindVariable, rollbackOnError bool, co vtgatepb.CommitOrder) (*sqltypes.Result, error) { session := vc.safeSession if co == vtgatepb.CommitOrder_AUTOCOMMIT { // For autocommit, we have to create an independent session. @@ -185,19 +185,19 @@ func (vc *vcursorImpl) Execute(method string, query string, bindVars map[string] } qr, err := vc.executor.Execute(vc.ctx, method, session, vc.marginComments.Leading+query+vc.marginComments.Trailing, bindVars) - if err == nil && isDML { - vc.hasPartialDML = true + if err == nil && rollbackOnError { + vc.rollbackOnPartialExec = true } return qr, err } // ExecuteMultiShard is part of the engine.VCursor interface. -func (vc *vcursorImpl) ExecuteMultiShard(rss []*srvtopo.ResolvedShard, queries []*querypb.BoundQuery, isDML, autocommit bool) (*sqltypes.Result, []error) { +func (vc *vcursorImpl) ExecuteMultiShard(rss []*srvtopo.ResolvedShard, queries []*querypb.BoundQuery, rollbackOnError, autocommit bool) (*sqltypes.Result, []error) { atomic.AddUint32(&vc.logStats.ShardQueries, uint32(len(queries))) qr, errs := vc.executor.ExecuteMultiShard(vc.ctx, rss, commentedShardQueries(queries, vc.marginComments), vc.tabletType, vc.safeSession, false, autocommit) - if errs == nil && isDML { - vc.hasPartialDML = true + if errs == nil && rollbackOnError { + vc.rollbackOnPartialExec = true } return qr, errs } @@ -229,7 +229,7 @@ func (vc *vcursorImpl) StreamExecuteMulti(query string, rss []*srvtopo.ResolvedS } // ExecuteKeyspaceID is part of the engine.VCursor interface. -func (vc *vcursorImpl) ExecuteKeyspaceID(keyspace string, ksid []byte, query string, bindVars map[string]*querypb.BindVariable, isDML, autocommit bool) (*sqltypes.Result, error) { +func (vc *vcursorImpl) ExecuteKeyspaceID(keyspace string, ksid []byte, query string, bindVars map[string]*querypb.BindVariable, rollbackOnError, autocommit bool) (*sqltypes.Result, error) { atomic.AddUint32(&vc.logStats.ShardQueries, 1) rss, _, err := vc.ResolveDestinations(keyspace, nil, []key.Destination{key.DestinationKeyspaceID(ksid)}) if err != nil { @@ -239,11 +239,11 @@ func (vc *vcursorImpl) ExecuteKeyspaceID(keyspace string, ksid []byte, query str Sql: query, BindVariables: bindVars, }} - qr, errs := vc.ExecuteMultiShard(rss, queries, isDML, autocommit) + qr, errs := vc.ExecuteMultiShard(rss, queries, rollbackOnError, autocommit) if len(errs) == 0 { - if isDML { - vc.hasPartialDML = true + if rollbackOnError { + vc.rollbackOnPartialExec = true } return qr, nil } diff --git a/go/vt/vtgate/vcursor_impl_test.go b/go/vt/vtgate/vcursor_impl_test.go index 61b70505f72..d6062dca443 100644 --- a/go/vt/vtgate/vcursor_impl_test.go +++ b/go/vt/vtgate/vcursor_impl_test.go @@ -14,7 +14,6 @@ import ( ) func TestDestinationKeyspace(t *testing.T) { - ks1 := &vindexes.Keyspace{ Name: "ks1", Sharded: false, diff --git a/go/vt/vtgate/vindexes/consistent_lookup.go b/go/vt/vtgate/vindexes/consistent_lookup.go index 1c64d218cc7..84072e75beb 100644 --- a/go/vt/vtgate/vindexes/consistent_lookup.go +++ b/go/vt/vtgate/vindexes/consistent_lookup.go @@ -266,19 +266,19 @@ func (lu *clCommon) handleDup(vcursor VCursor, values []sqltypes.Value, ksid []b bindVars[lu.lkp.To] = sqltypes.BytesBindVariable(ksid) // Lock the lookup row using pre priority. - qr, err := vcursor.Execute("VindexCreate", lu.lockLookupQuery, bindVars, false /* isDML */, vtgatepb.CommitOrder_PRE) + qr, err := vcursor.Execute("VindexCreate", lu.lockLookupQuery, bindVars, false /* rollbackOnError */, vtgatepb.CommitOrder_PRE) if err != nil { return err } switch len(qr.Rows) { case 0: - if _, err := vcursor.Execute("VindexCreate", lu.insertLookupQuery, bindVars, true /* isDML */, vtgatepb.CommitOrder_PRE); err != nil { + if _, err := vcursor.Execute("VindexCreate", lu.insertLookupQuery, bindVars, true /* rollbackOnError */, vtgatepb.CommitOrder_PRE); err != nil { return err } case 1: existingksid := qr.Rows[0][0].ToBytes() // Lock the target row using normal transaction priority. - qr, err = vcursor.ExecuteKeyspaceID(lu.keyspace, existingksid, lu.lockOwnerQuery, bindVars, false /* isDML */, false /* autocommit */) + qr, err = vcursor.ExecuteKeyspaceID(lu.keyspace, existingksid, lu.lockOwnerQuery, bindVars, false /* rollbackOnError */, false /* autocommit */) if err != nil { return err } @@ -288,7 +288,7 @@ func (lu *clCommon) handleDup(vcursor VCursor, values []sqltypes.Value, ksid []b if bytes.Equal(existingksid, ksid) { return nil } - if _, err := vcursor.Execute("VindexCreate", lu.updateLookupQuery, bindVars, true /* isDML */, vtgatepb.CommitOrder_PRE); err != nil { + if _, err := vcursor.Execute("VindexCreate", lu.updateLookupQuery, bindVars, true /* rollbackOnError */, vtgatepb.CommitOrder_PRE); err != nil { return err } default: diff --git a/go/vt/vtgate/vindexes/consistent_lookup_test.go b/go/vt/vtgate/vindexes/consistent_lookup_test.go index dbb95dbc4d6..346063def6d 100644 --- a/go/vt/vtgate/vindexes/consistent_lookup_test.go +++ b/go/vt/vtgate/vindexes/consistent_lookup_test.go @@ -522,7 +522,7 @@ func (vc *loggingVCursor) AddResult(qr *sqltypes.Result, err error) { vc.errors = append(vc.errors, err) } -func (vc *loggingVCursor) Execute(method string, query string, bindvars map[string]*querypb.BindVariable, isDML bool, co vtgatepb.CommitOrder) (*sqltypes.Result, error) { +func (vc *loggingVCursor) Execute(method string, query string, bindvars map[string]*querypb.BindVariable, rollbackOnError bool, co vtgatepb.CommitOrder) (*sqltypes.Result, error) { name := "Unknown" switch co { case vtgatepb.CommitOrder_NORMAL: @@ -534,14 +534,14 @@ func (vc *loggingVCursor) Execute(method string, query string, bindvars map[stri case vtgatepb.CommitOrder_AUTOCOMMIT: name = "ExecuteAutocommit" } - return vc.execute(name, query, bindvars, isDML) + return vc.execute(name, query, bindvars, rollbackOnError) } -func (vc *loggingVCursor) ExecuteKeyspaceID(keyspace string, ksid []byte, query string, bindVars map[string]*querypb.BindVariable, isDML, autocommit bool) (*sqltypes.Result, error) { - return vc.execute("ExecuteKeyspaceID", query, bindVars, isDML) +func (vc *loggingVCursor) ExecuteKeyspaceID(keyspace string, ksid []byte, query string, bindVars map[string]*querypb.BindVariable, rollbackOnError, autocommit bool) (*sqltypes.Result, error) { + return vc.execute("ExecuteKeyspaceID", query, bindVars, rollbackOnError) } -func (vc *loggingVCursor) execute(method string, query string, bindvars map[string]*querypb.BindVariable, isDML bool) (*sqltypes.Result, error) { +func (vc *loggingVCursor) execute(method string, query string, bindvars map[string]*querypb.BindVariable, rollbackOnError bool) (*sqltypes.Result, error) { if vc.index >= len(vc.results) { return nil, fmt.Errorf("ran out of results to return: %s", query) } @@ -550,7 +550,7 @@ func (vc *loggingVCursor) execute(method string, query string, bindvars map[stri bvl = append(bvl, bv{Name: k, Bv: string(v.Value)}) } sort.Slice(bvl, func(i, j int) bool { return bvl[i].Name < bvl[j].Name }) - vc.log = append(vc.log, fmt.Sprintf("%s %s %v %v", method, query, bvl, isDML)) + vc.log = append(vc.log, fmt.Sprintf("%s %s %v %v", method, query, bvl, rollbackOnError)) idx := vc.index vc.index++ if vc.errors[idx] != nil { diff --git a/go/vt/vtgate/vindexes/lookup_internal.go b/go/vt/vtgate/vindexes/lookup_internal.go index bfcc38a2c17..2ea8a77e455 100644 --- a/go/vt/vtgate/vindexes/lookup_internal.go +++ b/go/vt/vtgate/vindexes/lookup_internal.go @@ -76,7 +76,7 @@ func (lkp *lookupInternal) Lookup(vcursor VCursor, ids []sqltypes.Value) ([]*sql if lkp.Autocommit { co = vtgatepb.CommitOrder_AUTOCOMMIT } - result, err = vcursor.Execute("VindexLookup", lkp.sel, bindVars, false /* isDML */, co) + result, err = vcursor.Execute("VindexLookup", lkp.sel, bindVars, false /* rollbackOnError */, co) if err != nil { return nil, fmt.Errorf("lookup.Map: %v", err) } @@ -101,7 +101,7 @@ func (lkp *lookupInternal) VerifyCustom(vcursor VCursor, ids, values []sqltypes. lkp.FromColumns[0]: sqltypes.ValueBindVariable(id), lkp.To: sqltypes.ValueBindVariable(values[i]), } - result, err := vcursor.Execute("VindexVerify", lkp.ver, bindVars, false /* isDML */, co) + result, err := vcursor.Execute("VindexVerify", lkp.ver, bindVars, false /* rollbackOnError */, co) if err != nil { return nil, fmt.Errorf("lookup.Verify: %v", err) } @@ -206,7 +206,7 @@ func (lkp *lookupInternal) createCustom(vcursor VCursor, rowsColValues [][]sqlty fmt.Fprintf(buf, "%s=values(%s)", lkp.To, lkp.To) } - if _, err := vcursor.Execute("VindexCreate", buf.String(), bindVars, true /* isDML */, co); err != nil { + if _, err := vcursor.Execute("VindexCreate", buf.String(), bindVars, true /* rollbackOnError */, co); err != nil { return fmt.Errorf("lookup.Create: %v", err) } return nil @@ -247,7 +247,7 @@ func (lkp *lookupInternal) Delete(vcursor VCursor, rowsColValues [][]sqltypes.Va bindVars[lkp.FromColumns[colIdx]] = sqltypes.ValueBindVariable(columnValue) } bindVars[lkp.To] = sqltypes.ValueBindVariable(value) - _, err := vcursor.Execute("VindexDelete", lkp.del, bindVars, true /* isDML */, co) + _, err := vcursor.Execute("VindexDelete", lkp.del, bindVars, true /* rollbackOnError */, co) if err != nil { return fmt.Errorf("lookup.Delete: %v", err) } diff --git a/go/vt/vtgate/vindexes/lookup_test.go b/go/vt/vtgate/vindexes/lookup_test.go index 961a63725a9..827e0708dda 100644 --- a/go/vt/vtgate/vindexes/lookup_test.go +++ b/go/vt/vtgate/vindexes/lookup_test.go @@ -45,7 +45,7 @@ type vcursor struct { pre, post int } -func (vc *vcursor) Execute(method string, query string, bindvars map[string]*querypb.BindVariable, isDML bool, co vtgatepb.CommitOrder) (*sqltypes.Result, error) { +func (vc *vcursor) Execute(method string, query string, bindvars map[string]*querypb.BindVariable, rollbackOnError bool, co vtgatepb.CommitOrder) (*sqltypes.Result, error) { switch co { case vtgatepb.CommitOrder_PRE: vc.pre++ @@ -54,14 +54,14 @@ func (vc *vcursor) Execute(method string, query string, bindvars map[string]*que case vtgatepb.CommitOrder_AUTOCOMMIT: vc.autocommits++ } - return vc.execute(method, query, bindvars, isDML) + return vc.execute(method, query, bindvars, rollbackOnError) } -func (vc *vcursor) ExecuteKeyspaceID(keyspace string, ksid []byte, query string, bindVars map[string]*querypb.BindVariable, isDML, autocommit bool) (*sqltypes.Result, error) { - return vc.execute("ExecuteKeyspaceID", query, bindVars, isDML) +func (vc *vcursor) ExecuteKeyspaceID(keyspace string, ksid []byte, query string, bindVars map[string]*querypb.BindVariable, rollbackOnError, autocommit bool) (*sqltypes.Result, error) { + return vc.execute("ExecuteKeyspaceID", query, bindVars, rollbackOnError) } -func (vc *vcursor) execute(method string, query string, bindvars map[string]*querypb.BindVariable, isDML bool) (*sqltypes.Result, error) { +func (vc *vcursor) execute(method string, query string, bindvars map[string]*querypb.BindVariable, rollbackOnError bool) (*sqltypes.Result, error) { vc.queries = append(vc.queries, &querypb.BoundQuery{ Sql: query, BindVariables: bindvars, diff --git a/go/vt/vtgate/vindexes/vindex.go b/go/vt/vtgate/vindexes/vindex.go index ce7951aec65..0fea9233bb7 100644 --- a/go/vt/vtgate/vindexes/vindex.go +++ b/go/vt/vtgate/vindexes/vindex.go @@ -35,8 +35,8 @@ import ( // in the current context and session of a VTGate request. Vindexes // can use this interface to execute lookup queries. type VCursor interface { - Execute(method string, query string, bindvars map[string]*querypb.BindVariable, isDML bool, co vtgatepb.CommitOrder) (*sqltypes.Result, error) - ExecuteKeyspaceID(keyspace string, ksid []byte, query string, bindVars map[string]*querypb.BindVariable, isDML, autocommit bool) (*sqltypes.Result, error) + Execute(method string, query string, bindvars map[string]*querypb.BindVariable, rollbackOnError bool, co vtgatepb.CommitOrder) (*sqltypes.Result, error) + ExecuteKeyspaceID(keyspace string, ksid []byte, query string, bindVars map[string]*querypb.BindVariable, rollbackOnError, autocommit bool) (*sqltypes.Result, error) } // Vindex defines the interface required to register a vindex. From d93107be6874cda599e977b9399f3ee3810ac942 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 30 Mar 2020 13:59:50 +0530 Subject: [PATCH 81/85] Added autocommit logic to send primitive Signed-off-by: Harshit Gangal --- .../testdata/multi-output/target-output.txt | 4 ---- go/vt/vtgate/autocommit_test.go | 8 ++++---- go/vt/vtgate/engine/send.go | 3 ++- go/vt/vtgate/engine/send_test.go | 4 ++-- go/vt/vtgate/executor_dml_test.go | 16 ++++++++-------- go/vt/vtgate/executor_select_test.go | 4 ++-- go/vt/vtgate/executor_test.go | 4 ++-- 7 files changed, 20 insertions(+), 23 deletions(-) diff --git a/go/vt/vtexplain/testdata/multi-output/target-output.txt b/go/vt/vtexplain/testdata/multi-output/target-output.txt index a4a11506c3e..abf70f5b57d 100644 --- a/go/vt/vtexplain/testdata/multi-output/target-output.txt +++ b/go/vt/vtexplain/testdata/multi-output/target-output.txt @@ -1,16 +1,12 @@ ---------------------------------------------------------------------- select * from user where email='null@void.com' -1 ks_sharded/40-80: begin 1 ks_sharded/40-80: select * from user where email = 'null@void.com' limit 10001 -1 ks_sharded/40-80: commit ---------------------------------------------------------------------- select * from user where id in (1,2,3,4,5,6,7,8) -1 ks_sharded/40-80: begin 1 ks_sharded/40-80: select * from user where id in (1, 2, 3, 4, 5, 6, 7, 8) limit 10001 -1 ks_sharded/40-80: commit ---------------------------------------------------------------------- insert into user (id, name) values (2, 'bob') diff --git a/go/vt/vtgate/autocommit_test.go b/go/vt/vtgate/autocommit_test.go index a3c689f1652..e0ebb8c6705 100644 --- a/go/vt/vtgate/autocommit_test.go +++ b/go/vt/vtgate/autocommit_test.go @@ -429,12 +429,12 @@ func TestAutocommitDirectRangeTarget(t *testing.T) { _, err := executor.Execute(context.Background(), "TestExecute", NewSafeSession(session), sql, map[string]*querypb.BindVariable{}) require.NoError(t, err) - testBatchQuery(t, "sbc1", sbc1, &querypb.BoundQuery{ + testQueries(t, "sbc1", sbc1, []*querypb.BoundQuery{{ Sql: sql, BindVariables: map[string]*querypb.BindVariable{}, - }) - testAsTransactionCount(t, "sbc1", sbc1, 1) - testCommitCount(t, "sbc1", sbc1, 0) + }}) + testAsTransactionCount(t, "sbc1", sbc1, 0) + testCommitCount(t, "sbc1", sbc1, 1) } func autocommitExec(executor *Executor, sql string) (*sqltypes.Result, error) { diff --git a/go/vt/vtgate/engine/send.go b/go/vt/vtgate/engine/send.go index 76fefdf7beb..e9097a2ecd0 100644 --- a/go/vt/vtgate/engine/send.go +++ b/go/vt/vtgate/engine/send.go @@ -65,7 +65,8 @@ func (s *Send) Execute(vcursor VCursor, bindVars map[string]*query.BindVariable, } } - result, errs := vcursor.ExecuteMultiShard(rss, queries, false, true) + canAutocommit := len(rss) == 1 && vcursor.AutocommitApproval() + result, errs := vcursor.ExecuteMultiShard(rss, queries, true, canAutocommit) err = vterrors.Aggregate(errs) if err != nil { return nil, err diff --git a/go/vt/vtgate/engine/send_test.go b/go/vt/vtgate/engine/send_test.go index c8883661794..c3e03ec755c 100644 --- a/go/vt/vtgate/engine/send_test.go +++ b/go/vt/vtgate/engine/send_test.go @@ -26,7 +26,7 @@ func TestSendUnsharded(t *testing.T) { require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, - `ExecuteMultiShard ks.0: dummy_query {} false true`, + `ExecuteMultiShard ks.0: dummy_query {} true true`, }) // Failure cases @@ -54,7 +54,7 @@ func TestSendSharded(t *testing.T) { require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationShard(20-)`, - `ExecuteMultiShard ks.DestinationShard(20-): dummy_query {} false true`, + `ExecuteMultiShard ks.DestinationShard(20-): dummy_query {} true true`, }) // Failure cases diff --git a/go/vt/vtgate/executor_dml_test.go b/go/vt/vtgate/executor_dml_test.go index 6116d8d5d43..b0a47608df5 100644 --- a/go/vt/vtgate/executor_dml_test.go +++ b/go/vt/vtgate/executor_dml_test.go @@ -317,8 +317,8 @@ func TestUpdateNormalize(t *testing.T) { "vtg2": sqltypes.TestBindVariable(int64(1)), }, }} - assert.Empty(t, sbc1.BatchQueries) - utils.MustMatch(t, sbc2.BatchQueries[0], wantQueries, "didn't get expected queries") + assert.Empty(t, sbc1.Queries) + utils.MustMatch(t, sbc2.Queries, wantQueries, "didn't get expected queries") sbc2.Queries = nil masterSession.TargetString = "" } @@ -1591,13 +1591,13 @@ func TestKeyDestRangeQuery(t *testing.T) { if tc.expectedSbc1Query == "" { require.Empty(t, sbc1.BatchQueries, "sbc1") } else { - assertBatchQueriesContain(t, tc.expectedSbc1Query, "sbc1", sbc1) + assertQueriesContain(t, tc.expectedSbc1Query, "sbc1", sbc1) } if tc.expectedSbc2Query == "" { require.Empty(t, sbc2.BatchQueries) } else { - assertBatchQueriesContain(t, tc.expectedSbc2Query, "sbc2", sbc2) + assertQueriesContain(t, tc.expectedSbc2Query, "sbc2", sbc2) } }) } @@ -1612,13 +1612,13 @@ func TestKeyDestRangeQuery(t *testing.T) { masterSession.TargetString = "" } -func assertBatchQueriesContain(t *testing.T, sql, sbcName string, sbc *sandboxconn.SandboxConn) { +func assertQueriesContain(t *testing.T, sql, sbcName string, sbc *sandboxconn.SandboxConn) { t.Helper() - expectedQuery := &querypb.BoundQuery{ + expectedQuery := []*querypb.BoundQuery{{ Sql: sql, BindVariables: map[string]*querypb.BindVariable{}, - } - testBatchQuery(t, sbcName, sbc, expectedQuery) + }} + testQueries(t, sbcName, sbc, expectedQuery) } // Prepared statement tests diff --git a/go/vt/vtgate/executor_select_test.go b/go/vt/vtgate/executor_select_test.go index f70ce6da7de..6e53ec590d3 100644 --- a/go/vt/vtgate/executor_select_test.go +++ b/go/vt/vtgate/executor_select_test.go @@ -634,8 +634,8 @@ func TestSelectNormalize(t *testing.T) { "vtg1": sqltypes.TestBindVariable(int64(1)), }, }} - require.Empty(t, sbc1.BatchQueries) - utils.MustMatch(t, sbc2.BatchQueries[0], wantQueries, "sbc2.Queries") + require.Empty(t, sbc1.Queries) + utils.MustMatch(t, sbc2.Queries, wantQueries, "sbc2.Queries") sbc2.Queries = nil masterSession.TargetString = "" } diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index 543cd155806..6c1a690d143 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -208,10 +208,10 @@ func TestDirectTargetRewrites(t *testing.T) { _, err := executor.Execute(context.Background(), "TestExecute", NewSafeSession(session), sql, map[string]*querypb.BindVariable{}) require.NoError(t, err) - testBatchQuery(t, "sbclookup", sbclookup, &querypb.BoundQuery{ + testQueries(t, "sbclookup", sbclookup, []*querypb.BoundQuery{{ Sql: "select :__vtdbname as `database()` from dual", BindVariables: map[string]*querypb.BindVariable{"__vtdbname": sqltypes.StringBindVariable("TestUnsharded")}, - }) + }}) } func TestExecutorTransactionsAutoCommit(t *testing.T) { From c4658bf2010e8fdfbc922a2fc536a994d52cab53 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Mon, 30 Mar 2020 14:35:17 +0200 Subject: [PATCH 82/85] Handle DML/non-DML separately for the Send primitive Signed-off-by: Andres Taylor --- go/vt/sqlparser/analyzer.go | 10 ++ go/vt/vtgate/engine/send.go | 16 ++- go/vt/vtgate/engine/send_test.go | 136 ++++++++++++------ go/vt/vtgate/planbuilder/bypass.go | 5 +- .../planbuilder/testdata/bypass_cases.txt | 8 +- 5 files changed, 119 insertions(+), 56 deletions(-) diff --git a/go/vt/sqlparser/analyzer.go b/go/vt/sqlparser/analyzer.go index 113a887fb3e..c221787fa4f 100644 --- a/go/vt/sqlparser/analyzer.go +++ b/go/vt/sqlparser/analyzer.go @@ -163,6 +163,16 @@ func IsDML(sql string) bool { return false } +//IsDMLStatement returns true if the query is an INSERT, UPDATE or DELETE statement. +func IsDMLStatement(stmt Statement) bool { + switch stmt.(type) { + case *Insert, *Update, *Delete: + return true + } + + return false +} + // SplitAndExpression breaks up the Expr into AND-separated conditions // and appends them to filters. Outer parenthesis are removed. Precedence // should be taken into account if expressions are recombined. diff --git a/go/vt/vtgate/engine/send.go b/go/vt/vtgate/engine/send.go index e9097a2ecd0..dd8314e0f42 100644 --- a/go/vt/vtgate/engine/send.go +++ b/go/vt/vtgate/engine/send.go @@ -25,15 +25,19 @@ type Send struct { // Query specifies the query to be executed. Query string - // OpCode specifies the route type - OpCode string + // NoAutoCommit specifies if we need to check autocommit behaviour + NoAutoCommit bool noInputs } // RouteType implements Primitive interface func (s *Send) RouteType() string { - return s.OpCode + if s.NoAutoCommit { + return "SendNoAutoCommit" + } + + return "Send" } // GetKeyspaceName implements Primitive interface @@ -65,7 +69,11 @@ func (s *Send) Execute(vcursor VCursor, bindVars map[string]*query.BindVariable, } } - canAutocommit := len(rss) == 1 && vcursor.AutocommitApproval() + canAutocommit := false + if !s.NoAutoCommit { + canAutocommit = len(rss) == 1 && vcursor.AutocommitApproval() + } + result, errs := vcursor.ExecuteMultiShard(rss, queries, true, canAutocommit) err = vterrors.Aggregate(errs) if err != nil { diff --git a/go/vt/vtgate/engine/send_test.go b/go/vt/vtgate/engine/send_test.go index c3e03ec755c..2096d22be1c 100644 --- a/go/vt/vtgate/engine/send_test.go +++ b/go/vt/vtgate/engine/send_test.go @@ -11,54 +11,102 @@ import ( "vitess.io/vitess/go/vt/vtgate/vindexes" ) -func TestSendUnsharded(t *testing.T) { - send := &Send{ - Keyspace: &vindexes.Keyspace{ - Name: "ks", - Sharded: false, - }, - Query: "dummy_query", - TargetDestination: key.DestinationAllShards{}, +func TestSendTable(t *testing.T) { + type testCase struct { + testName string + sharded bool + shards []string + destination key.Destination + expectedQueryLog []string + noAutoCommit bool } - vc := &loggingVCursor{shards: []string{"0"}} - _, err := send.Execute(vc, map[string]*querypb.BindVariable{}, false) - require.NoError(t, err) - vc.ExpectLog(t, []string{ - `ResolveDestinations ks [] Destinations:DestinationAllShards()`, - `ExecuteMultiShard ks.0: dummy_query {} true true`, - }) - - // Failure cases - vc = &loggingVCursor{shardErr: errors.New("shard_error")} - _, err = send.Execute(vc, map[string]*querypb.BindVariable{}, false) - expectError(t, "Execute", err, "sendExecute: shard_error") - - vc = &loggingVCursor{} - _, err = send.Execute(vc, map[string]*querypb.BindVariable{}, false) - expectError(t, "Execute", err, "Keyspace does not have exactly one shard: []") -} - -func TestSendSharded(t *testing.T) { - send := &Send{ - Keyspace: &vindexes.Keyspace{ - Name: "ks", - Sharded: true, + singleShard := []string{"0"} + twoShards := []string{"-20", "20-"} + tests := []testCase{ + { + testName: "unsharded with no autocommit", + sharded: false, + shards: singleShard, + destination: key.DestinationAllShards{}, + expectedQueryLog: []string{ + `ResolveDestinations ks [] Destinations:DestinationAllShards()`, + `ExecuteMultiShard ks.0: dummy_query {} true true`, + }, + noAutoCommit: true, + }, + { + testName: "sharded with no autocommit", + sharded: true, + shards: twoShards, + destination: key.DestinationShard("20-"), + expectedQueryLog: []string{ + `ResolveDestinations ks [] Destinations:DestinationShard(20-)`, + `ExecuteMultiShard ks.DestinationShard(20-): dummy_query {} true true`, + }, + noAutoCommit: true, + }, + { + testName: "unsharded", + sharded: false, + shards: singleShard, + destination: key.DestinationAllShards{}, + expectedQueryLog: []string{ + `ResolveDestinations ks [] Destinations:DestinationAllShards()`, + `ExecuteMultiShard ks.0: dummy_query {} true true`, + }, + noAutoCommit: false, + }, + { + testName: "sharded with single shard destination", + sharded: true, + shards: twoShards, + destination: key.DestinationShard("20-"), + expectedQueryLog: []string{ + `ResolveDestinations ks [] Destinations:DestinationShard(20-)`, + `ExecuteMultiShard ks.DestinationShard(20-): dummy_query {} true true`, + }, + noAutoCommit: false, + }, + { + testName: "sharded with multi shard destination", + sharded: true, + shards: twoShards, + destination: key.DestinationAllShards{}, + expectedQueryLog: []string{ + `ResolveDestinations ks [] Destinations:DestinationAllShards()`, + `ExecuteMultiShard ks.-20: dummy_query {} ks.20-: dummy_query {} true false`, + }, + noAutoCommit: false, }, - Query: "dummy_query", - TargetDestination: key.DestinationShard("20-"), } - vc := &loggingVCursor{shards: []string{"-20", "20-"}} - _, err := send.Execute(vc, map[string]*querypb.BindVariable{}, false) - require.NoError(t, err) - vc.ExpectLog(t, []string{ - `ResolveDestinations ks [] Destinations:DestinationShard(20-)`, - `ExecuteMultiShard ks.DestinationShard(20-): dummy_query {} true true`, - }) + for _, tc := range tests { + t.Run(tc.testName, func(t *testing.T) { + send := &Send{ + Keyspace: &vindexes.Keyspace{ + Name: "ks", + Sharded: tc.sharded, + }, + Query: "dummy_query", + TargetDestination: tc.destination, + NoAutoCommit: false, + } + vc := &loggingVCursor{shards: tc.shards} + _, err := send.Execute(vc, map[string]*querypb.BindVariable{}, false) + require.NoError(t, err) + vc.ExpectLog(t, tc.expectedQueryLog) - // Failure cases - vc = &loggingVCursor{shardErr: errors.New("shard_error")} - _, err = send.Execute(vc, map[string]*querypb.BindVariable{}, false) - expectError(t, "Execute", err, "sendExecute: shard_error") + // Failure cases + vc = &loggingVCursor{shardErr: errors.New("shard_error")} + _, err = send.Execute(vc, map[string]*querypb.BindVariable{}, false) + require.EqualError(t, err, "sendExecute: shard_error") + + if !tc.sharded { + vc = &loggingVCursor{} + _, err = send.Execute(vc, map[string]*querypb.BindVariable{}, false) + require.EqualError(t, err, "Keyspace does not have exactly one shard: []") + } + }) + } } diff --git a/go/vt/vtgate/planbuilder/bypass.go b/go/vt/vtgate/planbuilder/bypass.go index 88583d2104f..2ffb3f415ac 100644 --- a/go/vt/vtgate/planbuilder/bypass.go +++ b/go/vt/vtgate/planbuilder/bypass.go @@ -40,9 +40,6 @@ func buildPlanForBypass(stmt sqlparser.Statement, vschema ContextVSchema) (engin Keyspace: keyspace, TargetDestination: vschema.Destination(), Query: sqlparser.String(stmt), - OpCode: ByPassOpCode, + NoAutoCommit: !sqlparser.IsDMLStatement(stmt), }, nil } - -//ByPassOpCode is the opcode -const ByPassOpCode = "ByPass" diff --git a/go/vt/vtgate/planbuilder/testdata/bypass_cases.txt b/go/vt/vtgate/planbuilder/testdata/bypass_cases.txt index 30022a7cb59..672bbc87f1d 100644 --- a/go/vt/vtgate/planbuilder/testdata/bypass_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/bypass_cases.txt @@ -9,7 +9,7 @@ }, "TargetDestination": "-80", "Query": "select count(*), col from unsharded", - "OpCode": "ByPass" + "NoAutoCommit": true } } @@ -24,7 +24,7 @@ }, "TargetDestination": "-80", "Query": "update user set val = 1 where id = 18446744073709551616 and id = 1", - "OpCode": "ByPass" + "NoAutoCommit": false } } @@ -39,7 +39,7 @@ }, "TargetDestination": "-80", "Query": "delete from USER where ID = 42", - "OpCode": "ByPass" + "NoAutoCommit": false } } @@ -54,6 +54,6 @@ }, "TargetDestination": "-80", "Query": "insert into USER(ID, NAME) values (42, 'ms X')", - "OpCode": "ByPass" + "NoAutoCommit": false } } From 5a9f4da8cee33780c8f4106450fe903bd25724b8 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Mon, 30 Mar 2020 15:05:13 +0200 Subject: [PATCH 83/85] Don't rollback on error when not needed Signed-off-by: Andres Taylor --- go/vt/vtgate/engine/send.go | 3 ++- go/vt/vtgate/engine/send_test.go | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/go/vt/vtgate/engine/send.go b/go/vt/vtgate/engine/send.go index dd8314e0f42..7bd8bc9c77d 100644 --- a/go/vt/vtgate/engine/send.go +++ b/go/vt/vtgate/engine/send.go @@ -74,7 +74,8 @@ func (s *Send) Execute(vcursor VCursor, bindVars map[string]*query.BindVariable, canAutocommit = len(rss) == 1 && vcursor.AutocommitApproval() } - result, errs := vcursor.ExecuteMultiShard(rss, queries, true, canAutocommit) + rollbackOnError := !s.NoAutoCommit // for non-dml queries, there's no need to do a rollback + result, errs := vcursor.ExecuteMultiShard(rss, queries, rollbackOnError, canAutocommit) err = vterrors.Aggregate(errs) if err != nil { return nil, err diff --git a/go/vt/vtgate/engine/send_test.go b/go/vt/vtgate/engine/send_test.go index 2096d22be1c..38d8ad510bb 100644 --- a/go/vt/vtgate/engine/send_test.go +++ b/go/vt/vtgate/engine/send_test.go @@ -31,7 +31,7 @@ func TestSendTable(t *testing.T) { destination: key.DestinationAllShards{}, expectedQueryLog: []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, - `ExecuteMultiShard ks.0: dummy_query {} true true`, + `ExecuteMultiShard ks.0: dummy_query {} false false`, }, noAutoCommit: true, }, @@ -42,7 +42,7 @@ func TestSendTable(t *testing.T) { destination: key.DestinationShard("20-"), expectedQueryLog: []string{ `ResolveDestinations ks [] Destinations:DestinationShard(20-)`, - `ExecuteMultiShard ks.DestinationShard(20-): dummy_query {} true true`, + `ExecuteMultiShard ks.DestinationShard(20-): dummy_query {} false false`, }, noAutoCommit: true, }, @@ -90,7 +90,7 @@ func TestSendTable(t *testing.T) { }, Query: "dummy_query", TargetDestination: tc.destination, - NoAutoCommit: false, + NoAutoCommit: tc.noAutoCommit, } vc := &loggingVCursor{shards: tc.shards} _, err := send.Execute(vc, map[string]*querypb.BindVariable{}, false) From c6e02cb572471952b79a2ab40692ab58f227576a Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 30 Mar 2020 19:14:24 +0530 Subject: [PATCH 84/85] removed destination from handleExec signature Signed-off-by: Harshit Gangal --- go/vt/vtgate/executor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 0debe8a97e5..9da0a08c418 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -207,7 +207,7 @@ func (e *Executor) execute(ctx context.Context, safeSession *SafeSession, sql st switch specStmt := stmt.(type) { case *sqlparser.Select, *sqlparser.Union: - return e.handleExec(ctx, safeSession, sql, bindVars, destKeyspace, destTabletType, dest, logStats, stmtType) + return e.handleExec(ctx, safeSession, sql, bindVars, logStats, stmtType) case *sqlparser.Insert, *sqlparser.Update, *sqlparser.Delete: safeSession := safeSession @@ -232,7 +232,7 @@ func (e *Executor) execute(ctx context.Context, safeSession *SafeSession, sql st // at the beginning, but never after. safeSession.SetAutocommittable(mustCommit) - qr, err := e.handleExec(ctx, safeSession, sql, bindVars, destKeyspace, destTabletType, dest, logStats, stmtType) + qr, err := e.handleExec(ctx, safeSession, sql, bindVars, logStats, stmtType) if err != nil { return nil, err } @@ -265,7 +265,7 @@ func (e *Executor) execute(ctx context.Context, safeSession *SafeSession, sql st return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "unrecognized statement: %s", sql) } -func (e *Executor) handleExec(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, destKeyspace string, destTabletType topodatapb.TabletType, dest key.Destination, logStats *LogStats, stmtType sqlparser.StatementType) (*sqltypes.Result, error) { +func (e *Executor) handleExec(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, logStats *LogStats, stmtType sqlparser.StatementType) (*sqltypes.Result, error) { // V3 mode. query, comments := sqlparser.SplitMarginComments(sql) From 87d1fe894d3b834f3ce73068155af5d868c125b2 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 31 Mar 2020 07:46:19 +0200 Subject: [PATCH 85/85] Minor fixes Signed-off-by: Andres Taylor --- go/vt/vtgate/engine/send.go | 24 ++++++++++++++++++- go/vt/vtgate/executor.go | 2 -- go/vt/vtgate/executor_test.go | 1 - .../planbuilder/testdata/bypass_cases.txt | 4 ++++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/go/vt/vtgate/engine/send.go b/go/vt/vtgate/engine/send.go index 7bd8bc9c77d..8a8d0a880fb 100644 --- a/go/vt/vtgate/engine/send.go +++ b/go/vt/vtgate/engine/send.go @@ -1,6 +1,8 @@ package engine import ( + "encoding/json" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/proto/query" @@ -31,6 +33,26 @@ type Send struct { noInputs } +// MarshalJSON serializes the Send into a JSON representation. +// It's used for testing and diagnostics. +func (s *Send) MarshalJSON() ([]byte, error) { + marshalSend := struct { + Opcode string + Keyspace *vindexes.Keyspace + TargetDestination key.Destination + Query string + NoAutoCommit bool + }{ + Opcode: "Send", + Keyspace: s.Keyspace, + TargetDestination: s.TargetDestination, + NoAutoCommit: s.NoAutoCommit, + Query: s.Query, + } + + return json.Marshal(marshalSend) +} + // RouteType implements Primitive interface func (s *Send) RouteType() string { if s.NoAutoCommit { @@ -90,5 +112,5 @@ func (s *Send) StreamExecute(vcursor VCursor, bindVars map[string]*query.BindVar // GetFields implements Primitive interface func (s *Send) GetFields(vcursor VCursor, bindVars map[string]*query.BindVariable) (*sqltypes.Result, error) { - return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "not reachable") // TODO: systay - @sugu, is this correct? + return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "not reachable") } diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 9da0a08c418..3535022cd9f 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -266,8 +266,6 @@ func (e *Executor) execute(ctx context.Context, safeSession *SafeSession, sql st } func (e *Executor) handleExec(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, logStats *LogStats, stmtType sqlparser.StatementType) (*sqltypes.Result, error) { - - // V3 mode. query, comments := sqlparser.SplitMarginComments(sql) vcursor, _ := newVCursorImpl(ctx, safeSession, comments, e, logStats, e.VSchema(), e.resolver.resolver) plan, err := e.getPlan( diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index 6c1a690d143..938f4340843 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -194,7 +194,6 @@ func TestExecutorTransactionsNoAutoCommit(t *testing.T) { } } -//TODO - what about these? func TestDirectTargetRewrites(t *testing.T) { executor, _, _, sbclookup := createExecutorEnv() executor.normalize = true diff --git a/go/vt/vtgate/planbuilder/testdata/bypass_cases.txt b/go/vt/vtgate/planbuilder/testdata/bypass_cases.txt index 672bbc87f1d..e68e934d78a 100644 --- a/go/vt/vtgate/planbuilder/testdata/bypass_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/bypass_cases.txt @@ -3,6 +3,7 @@ { "Original": "select count(*), col from unsharded", "Instructions": { + "Opcode": "Send", "Keyspace": { "Name": "main", "Sharded": false @@ -18,6 +19,7 @@ { "Original": "update user set val = 1 where id = 18446744073709551616 and id = 1", "Instructions": { + "Opcode": "Send", "Keyspace": { "Name": "main", "Sharded": false @@ -33,6 +35,7 @@ { "Original": "DELETE FROM USER WHERE ID = 42", "Instructions": { + "Opcode": "Send", "Keyspace": { "Name": "main", "Sharded": false @@ -48,6 +51,7 @@ { "Original": "INSERT INTO USER (ID, NAME) VALUES (42, 'ms X')", "Instructions": { + "Opcode": "Send", "Keyspace": { "Name": "main", "Sharded": false