query routing: multi-table plan options

In this change the query routing takes the possibility that
there could be multiple target options for a given table. The
design for this is explained in vitessio#4790.

At a high level:
* VSchema.FindTableOrVindex function can return a list of
  tables instead of a single one.
* The route planbuilder creates multiple routeOptions, one
  for each table returned.
* All actions that affected the plan of a route are changed
  to update all routeOptions.
* If a particular routeOption cannot accommodate a pushed
  down construct, it's removed from the list. Previously,
  this was an error case. But if no options are left, then
  we return an error.
* If two routeOptions qualify for a merge of routes, then
  all other combinations that don't qualify are discarded.
  This is the case for joins, subqueries and unions.

More details:
vindexTable was renamed to the more appropriate vschemaTable.

In order to achieve this, a new routeOption data type was
introduced, and route was changed to contain a list of
routeOptions.

In symtab, tables used to point at the vschema table that
was used to build them. Since a table can now represent
multiple target tables, this field has been moved into
routeOption.

In symtab, columns used to contain a vindex member. Since
this can change depending on the target table, the routeOption
now contains a map of column to vindexes instead.

The routeOption also contains the vschemaTable. DMLs use
this information. Since DMLs have to be more deterministic
about the table they write to, they always choose the
first option.

At the beginning of the Wireup phase, we evaluate all existing
options and decide on the best available.

To be done:
When a table has multiple targets, the targets can have different
names than the original table. If so, the queries have to be
rewritten to address the new target tables. In order to do this,
each routeOption will contain a list of substitutions that will
be made during the Wireup phase.

Tests have to be written for the new flows.

Signed-off-by: Sugu Sougoumarane <ssougou@gmail.com>

go/vt/sqlparser/ast.go

@@ -532,7 +532,7 @@ func (node *Stream) walkSubtree(visit Visit) error {
 // the row and re-inserts with new values. For that reason we keep it as an Insert struct.
 // Replaces are currently disallowed in sharded schemas because
 // of the implications the deletion part may have on vindexes.
-// If you add fields here, consider adding them to calls to validateSubquerySamePlan.
+// If you add fields here, consider adding them to calls to validateUnshardedRoute.
 type Insert struct {
 	Action     string
 	Comments   Comments
@@ -584,7 +584,7 @@ func (Values) iInsertRows()       {}
 func (*ParenSelect) iInsertRows() {}
 
 // Update represents an UPDATE statement.
-// If you add fields here, consider adding them to calls to validateSubquerySamePlan.
+// If you add fields here, consider adding them to calls to validateUnshardedRoute.
 type Update struct {
 	Comments   Comments
 	Ignore     string
@@ -618,7 +618,7 @@ func (node *Update) walkSubtree(visit Visit) error {
 }
 
 // Delete represents a DELETE statement.
-// If you add fields here, consider adding them to calls to validateSubquerySamePlan.
+// If you add fields here, consider adding them to calls to validateUnshardedRoute.
 type Delete struct {
 	Comments   Comments
 	Targets    TableNames

go/vt/vtgate/engine/insert.go

@@ -456,9 +456,7 @@ func (ins *Insert) processPrimary(vcursor VCursor, vindexKeys [][]sqltypes.Value
 	var flattenedVindexKeys []sqltypes.Value
 	// TODO: @rafael - this will change once vindex Primary keys also support multicolumns
 	for _, val := range vindexKeys {
-		for _, internalVal := range val {
-			flattenedVindexKeys = append(flattenedVindexKeys, internalVal)
-		}
+		flattenedVindexKeys = append(flattenedVindexKeys, val...)
 	}
 
 	destinations, err := colVindex.Vindex.Map(vcursor, flattenedVindexKeys)

go/vt/vtgate/engine/limit_test.go

@@ -90,12 +90,6 @@ func TestLimitExecute(t *testing.T) {
 	}
 
 	// Test with limit higher than input.
-	wantResult = sqltypes.MakeTestResult(
-		fields,
-		"a|1",
-		"b|2",
-		"c|3",
-	)
 	inputResult = sqltypes.MakeTestResult(
 		fields,
 		"a|1",

go/vt/vtgate/engine/route.go

@@ -143,7 +143,8 @@ func (route *Route) MarshalJSON() ([]byte, error) {
 
 // RouteOpcode is a number representing the opcode
 // for the Route primitve. Adding new opcodes here
-// will require review of the join code in planbuilder.
+// will require review of the join code and
+// the finalizeOptions code in planbuilder.
 type RouteOpcode int
 
 // This is the list of RouteOpcode values.
@@ -321,7 +322,7 @@ func (route *Route) GetFields(vcursor VCursor, bindVars map[string]*querypb.Bind
 	}
 	if len(rss) != 1 {
 		// This code is unreachable. It's just a sanity check.
-		return nil, fmt.Errorf("No shards for keyspace: %s", route.Keyspace.Name)
+		return nil, fmt.Errorf("no shards for keyspace: %s", route.Keyspace.Name)
 	}
 	qr, err := execShard(vcursor, route.FieldQuery, bindVars, rss[0], false /* isDML */, false /* canAutocommit */)
 	if err != nil {
@@ -461,7 +462,7 @@ func execAnyShard(vcursor VCursor, query string, bindVars map[string]*querypb.Bi
 	}
 	if len(rss) != 1 {
 		// This code is unreachable. It's just a sanity check.
-		return nil, fmt.Errorf("No shards for keyspace: %s", keyspace.Name)
+		return nil, fmt.Errorf("no shards for keyspace: %s", keyspace.Name)
 	}
 	return vcursor.ExecuteStandalone(query, bindVars, rss[0])
 }

go/vt/vtgate/planbuilder/builder.go

@@ -109,7 +109,7 @@ type builder interface {
 // info about tables.
 type ContextVSchema interface {
 	FindTable(tablename sqlparser.TableName) (*vindexes.Table, string, topodatapb.TabletType, key.Destination, error)
-	FindTableOrVindex(tablename sqlparser.TableName) (*vindexes.Table, vindexes.Vindex, string, topodatapb.TabletType, key.Destination, error)
+	FindTablesOrVindex(tablename sqlparser.TableName) ([]*vindexes.Table, vindexes.Vindex, string, topodatapb.TabletType, key.Destination, error)
 	DefaultKeyspace() (*vindexes.Keyspace, error)
 	TargetString() string
 }

go/vt/vtgate/planbuilder/delete.go

@@ -41,46 +41,39 @@ func buildDeletePlan(del *sqlparser.Delete, vschema ContextVSchema) (*engine.Del
 	if !ok {
 		return nil, errors.New("unsupported: multi-table/vindex delete statement in sharded keyspace")
 	}
-	edel.Keyspace = rb.ERoute.Keyspace
+	ro := rb.routeOptions[0]
+	edel.Keyspace = ro.ERoute.Keyspace
 	if !edel.Keyspace.Sharded {
 		// We only validate non-table subexpressions because the previous analysis has already validated them.
-		if !pb.validateSubquerySamePlan(del.Targets, del.Where, del.OrderBy, del.Limit) {
+		if !pb.validateUnshardedRoute(del.Targets, del.Where, del.OrderBy, del.Limit) {
 			return nil, errors.New("unsupported: sharded subqueries in DML")
 		}
 		edel.Opcode = engine.DeleteUnsharded
 		return edel, nil
 	}
-	if del.Targets != nil || len(pb.st.tables) != 1 {
+	if del.Targets != nil || ro.vschemaTable == nil {
 		return nil, errors.New("unsupported: multi-table delete statement in sharded keyspace")
 	}
 	if hasSubquery(del) {
 		return nil, errors.New("unsupported: subqueries in sharded DML")
 	}
-	var vindexTable *vindexes.Table
-	for _, tval := range pb.st.tables {
-		vindexTable = tval.vindexTable
-	}
-	edel.Table = vindexTable
-	if edel.Table == nil {
-		return nil, errors.New("internal error: table.vindexTable is mysteriously nil")
-	}
-	var err error
+	edel.Table = ro.vschemaTable
 
 	directives := sqlparser.ExtractCommentDirectives(del.Comments)
 	if directives.IsSet(sqlparser.DirectiveMultiShardAutocommit) {
 		edel.MultiShardAutocommit = true
 	}
 
 	edel.QueryTimeout = queryTimeout(directives)
-
-	if rb.ERoute.TargetDestination != nil {
-		if rb.ERoute.TargetTabletType != topodatapb.TabletType_MASTER {
+	if rb.routeOptions[0].ERoute.TargetDestination != nil {
+		if rb.routeOptions[0].ERoute.TargetTabletType != topodatapb.TabletType_MASTER {
 			return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "unsupported: DELETE statement with a replica target")
 		}
 		edel.Opcode = engine.DeleteByDestination
-		edel.TargetDestination = rb.ERoute.TargetDestination
+		edel.TargetDestination = rb.routeOptions[0].ERoute.TargetDestination
 		return edel, nil
 	}
+	var err error
 	edel.Vindex, edel.Values, err = getDMLRouting(del.Where, edel.Table)
 	// We couldn't generate a route for a single shard
 	// Execute a delete sharded

go/vt/vtgate/planbuilder/expr.go

@@ -154,7 +154,7 @@ func (pb *primitiveBuilder) findOrigin(expr sqlparser.Expr) (pullouts []*pullout
 			switch {
 			// If it's last_insert_id, ensure it's a single unsharded route.
 			case node.Name.EqualString("last_insert_id"):
-				if rb, isRoute := pb.bldr.(*route); !isRoute || rb.ERoute.Keyspace.Sharded {
+				if rb, isRoute := pb.bldr.(*route); !isRoute || !rb.removeShardedOptions() {
 					return false, errors.New("unsupported: LAST_INSERT_ID is only allowed for unsharded keyspaces")
 				}
 			}
@@ -169,8 +169,7 @@ func (pb *primitiveBuilder) findOrigin(expr sqlparser.Expr) (pullouts []*pullout
 	highestRoute, _ := highestOrigin.(*route)
 	for _, sqi := range subqueries {
 		subroute, _ := sqi.bldr.(*route)
-		if highestRoute != nil && subroute != nil && highestRoute.SubqueryCanMerge(pb, subroute) {
-			subroute.Redirect = highestRoute
+		if highestRoute != nil && subroute != nil && highestRoute.MergeSubquery(pb, subroute) {
 			continue
 		}
 		if sqi.origin != nil {
@@ -241,14 +240,17 @@ func hasSubquery(node sqlparser.SQLNode) bool {
 	return has
 }
 
-func (pb *primitiveBuilder) validateSubquerySamePlan(nodes ...sqlparser.SQLNode) bool {
+func (pb *primitiveBuilder) validateUnshardedRoute(nodes ...sqlparser.SQLNode) bool {
 	var keyspace string
 	if rb, ok := pb.bldr.(*route); ok {
-		keyspace = rb.ERoute.Keyspace.Name
+		keyspace = rb.routeOptions[0].ERoute.Keyspace.Name
+	} else {
+		// This code is unreachable because the caller checks.
+		return false
 	}
-	samePlan := true
 
 	for _, node := range nodes {
+		samePlan := true
 		inSubQuery := false
 		_ = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) {
 			switch nodeType := node.(type) {
@@ -269,7 +271,7 @@ func (pb *primitiveBuilder) validateSubquerySamePlan(nodes ...sqlparser.SQLNode)
 					samePlan = false
 					return false, errors.New("dummy")
 				}
-				if innerRoute.ERoute.Keyspace.Name != keyspace {
+				if !innerRoute.removeOptionsWithUnmatchedKeyspace(keyspace) {
 					samePlan = false
 					return false, errors.New("dummy")
 				}
@@ -287,7 +289,7 @@ func (pb *primitiveBuilder) validateSubquerySamePlan(nodes ...sqlparser.SQLNode)
 					samePlan = false
 					return false, errors.New("dummy")
 				}
-				if innerRoute.ERoute.Keyspace.Name != keyspace {
+				if !innerRoute.removeOptionsWithUnmatchedKeyspace(keyspace) {
 					samePlan = false
 					return false, errors.New("dummy")
 				}