Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use evalengine for Vindex lookups in route primitive #9291

Merged
merged 15 commits into from
Dec 5, 2021
Merged

Use evalengine for Vindex lookups in route primitive #9291

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
d2fabf4
refactor: split route into route and routeLegacy
systay Nov 29, 2021
e0add06
feat: use evalengine.Expr instead of PlanValue on route
systay Dec 1, 2021
e81ae1e
feat: update plan description to handle evalengine better
systay Dec 1, 2021
ed4cc4d
refactor: remove unused field
systay Dec 1, 2021
1926c6d
feat: support ListArg in evalengine
systay Dec 1, 2021
ce33106
feat: handle cases where columns are not allowed
systay Dec 2, 2021
e1598ef
feat: evalEngineExpr function to not return an error and plan operato…
harshit-gangal Dec 2, 2021
af0e7a0
refactor: addressed comments and change route to routeGen4 and routeL…
harshit-gangal Dec 2, 2021
4f40d2a
refactor: remove routeLegacy and use engine.route for both v3 and gen4
systay Dec 2, 2021
957f7b3
refactor: rename function
systay Dec 2, 2021
9a1a56a
test: re-insert accidentally removed test case
systay Dec 2, 2021
9373533
chore: year was wrong
systay Dec 2, 2021
11878b4
feat: support tuple evaluation in evalEngine
harshit-gangal Dec 2, 2021
5821afb
feat: error out for unsupported multi-column value vindex
harshit-gangal Dec 2, 2021
754a568
test: add e2e test in gen4 for select with non-correlated subquery
harshit-gangal Dec 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions go/vt/vtgate/engine/cached_size.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 0 additions & 4 deletions go/vt/vtgate/engine/plan_description_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@ import (

"vitess.io/vitess/go/test/utils"

"vitess.io/vitess/go/sqltypes"

"vitess.io/vitess/go/vt/key"
"vitess.io/vitess/go/vt/vtgate/vindexes"
)
Expand Down Expand Up @@ -59,8 +57,6 @@ func createRoute() *Route {
TableName: "tableName",
FieldQuery: "more query",
Vindex: hash.(*vindexes.Hash),
Values: []sqltypes.PlanValue{},
OrderBy: []OrderByParams{},
}
}

Expand Down
31 changes: 21 additions & 10 deletions go/vt/vtgate/engine/route.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,9 @@ type Route struct {

// Vindex specifies the vindex to be used.
Vindex vindexes.SingleColumn
// Values specifies the vindex values to use for routing.
Values []sqltypes.PlanValue

// Value specifies the vindex value to use for routing.
Value RouteValue

// OrderBy specifies the key order for merge sorting. This will be
// set only for scatter queries that need the results to be
Expand Down Expand Up @@ -106,6 +107,16 @@ type Route struct {
noTxNeeded
}

type RouteValue interface {
// ResolveValue allows for retrieval of the value we expose for public consumption
ResolveValue(bindVars map[string]*querypb.BindVariable) (sqltypes.Value, error)

// ResolveList allows for retrieval of the value we expose for public consumption
ResolveList(bindVars map[string]*querypb.BindVariable) ([]sqltypes.Value, error)

MarshalJSON() ([]byte, error)
}

// NewSimpleRoute creates a Route with the bare minimum of parameters.
func NewSimpleRoute(opcode RouteOpcode, keyspace *vindexes.Keyspace) *Route {
return &Route{
Expand Down Expand Up @@ -588,11 +599,11 @@ func (route *Route) paramsAnyShard(vcursor VCursor, bindVars map[string]*querypb
}

func (route *Route) paramsSelectEqual(vcursor VCursor, bindVars map[string]*querypb.BindVariable) ([]*srvtopo.ResolvedShard, []map[string]*querypb.BindVariable, error) {
key, err := route.Values[0].ResolveValue(bindVars)
value, err := route.Value.ResolveValue(bindVars)
if err != nil {
return nil, nil, err
}
rss, _, err := resolveShards(vcursor, route.Vindex, route.Keyspace, []sqltypes.Value{key})
rss, _, err := resolveShards(vcursor, route.Vindex, route.Keyspace, []sqltypes.Value{value})
frouioui marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return nil, nil, err
}
Expand All @@ -604,23 +615,23 @@ func (route *Route) paramsSelectEqual(vcursor VCursor, bindVars map[string]*quer
}

func (route *Route) paramsSelectIn(vcursor VCursor, bindVars map[string]*querypb.BindVariable) ([]*srvtopo.ResolvedShard, []map[string]*querypb.BindVariable, error) {
keys, err := route.Values[0].ResolveList(bindVars)
value, err := route.Value.ResolveList(bindVars)
if err != nil {
return nil, nil, err
}
rss, values, err := resolveShards(vcursor, route.Vindex, route.Keyspace, keys)
rss, values, err := resolveShards(vcursor, route.Vindex, route.Keyspace, value)
if err != nil {
return nil, nil, err
}
return rss, shardVars(bindVars, values), nil
}

func (route *Route) paramsSelectMultiEqual(vcursor VCursor, bindVars map[string]*querypb.BindVariable) ([]*srvtopo.ResolvedShard, []map[string]*querypb.BindVariable, error) {
keys, err := route.Values[0].ResolveList(bindVars)
value, err := route.Value.ResolveList(bindVars)
if err != nil {
return nil, nil, err
}
rss, _, err := resolveShards(vcursor, route.Vindex, route.Keyspace, keys)
rss, _, err := resolveShards(vcursor, route.Vindex, route.Keyspace, value)
if err != nil {
return nil, nil, err
}
Expand Down Expand Up @@ -795,8 +806,8 @@ func (route *Route) description() PrimitiveDescription {
if route.Vindex != nil {
other["Vindex"] = route.Vindex.String()
}
if len(route.Values) > 0 {
other["Values"] = route.Values
if route.Value != nil {
other["Values"] = route.Value
}
if len(route.SysTableTableSchema) != 0 {
sysTabSchema := "["
Expand Down
83 changes: 42 additions & 41 deletions go/vt/vtgate/engine/route_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,8 +234,10 @@ func TestSelectEqualUnique(t *testing.T) {
"dummy_select_field",
)
sel.Vindex = vindex.(vindexes.SingleColumn)
sel.Values = []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}}

sel.Value = &evalengine.RouteValue{
Expr: evalengine.NewLiteralInt(1),
}
vc := &loggingVCursor{
shards: []string{"-20", "20-"},
results: []*sqltypes.Result{defaultSelectResult},
Expand Down Expand Up @@ -270,7 +272,7 @@ func TestSelectNone(t *testing.T) {
"dummy_select_field",
)
sel.Vindex = vindex.(vindexes.SingleColumn)
sel.Values = nil
sel.Value = nil

vc := &loggingVCursor{
shards: []string{"-20", "20-"},
Expand Down Expand Up @@ -305,8 +307,9 @@ func TestSelectEqualUniqueScatter(t *testing.T) {
"dummy_select_field",
)
sel.Vindex = vindex.(vindexes.SingleColumn)
sel.Values = []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}}

sel.Value = &evalengine.RouteValue{
Expr: evalengine.NewLiteralInt(1),
}
vc := &loggingVCursor{
shards: []string{"-20", "20-"},
shardForKsid: []string{"-20", "20-"},
Expand Down Expand Up @@ -346,8 +349,9 @@ func TestSelectEqual(t *testing.T) {
"dummy_select_field",
)
sel.Vindex = vindex.(vindexes.SingleColumn)
sel.Values = []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}}

sel.Value = &evalengine.RouteValue{
Expr: evalengine.NewLiteralInt(1),
}
vc := &loggingVCursor{
shards: []string{"-20", "20-"},
results: []*sqltypes.Result{
Expand Down Expand Up @@ -398,7 +402,9 @@ func TestSelectEqualNoRoute(t *testing.T) {
"dummy_select_field",
)
sel.Vindex = vindex.(vindexes.SingleColumn)
sel.Values = []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}}
sel.Value = &evalengine.RouteValue{
Expr: evalengine.NewLiteralInt(1),
}

vc := &loggingVCursor{shards: []string{"-20", "20-"}}
result, err := sel.TryExecute(vc, map[string]*querypb.BindVariable{}, false)
Expand Down Expand Up @@ -431,16 +437,13 @@ func TestSelectINUnique(t *testing.T) {
"dummy_select_field",
)
sel.Vindex = vindex.(vindexes.SingleColumn)
sel.Values = []sqltypes.PlanValue{{
Values: []sqltypes.PlanValue{{
Value: sqltypes.NewInt64(1),
}, {
Value: sqltypes.NewInt64(2),
}, {
Value: sqltypes.NewInt64(4),
}},
}}

sel.Value = &evalengine.RouteValue{
Expr: evalengine.TupleExpr{
evalengine.NewLiteralInt(1),
evalengine.NewLiteralInt(2),
evalengine.NewLiteralInt(4),
},
}
vc := &loggingVCursor{
shards: []string{"-20", "20-"},
shardForKsid: []string{"-20", "-20", "20-"},
Expand Down Expand Up @@ -483,15 +486,13 @@ func TestSelectINNonUnique(t *testing.T) {
"dummy_select_field",
)
sel.Vindex = vindex.(vindexes.SingleColumn)
sel.Values = []sqltypes.PlanValue{{
Values: []sqltypes.PlanValue{{
Value: sqltypes.NewInt64(1),
}, {
Value: sqltypes.NewInt64(2),
}, {
Value: sqltypes.NewInt64(4),
}},
}}
sel.Value = &evalengine.RouteValue{
Expr: evalengine.TupleExpr{
evalengine.NewLiteralInt(1),
evalengine.NewLiteralInt(2),
evalengine.NewLiteralInt(4),
},
}

fields := sqltypes.MakeTestFields(
"fromc|toc",
Expand Down Expand Up @@ -548,15 +549,13 @@ func TestSelectMultiEqual(t *testing.T) {
"dummy_select_field",
)
sel.Vindex = vindex.(vindexes.SingleColumn)
sel.Values = []sqltypes.PlanValue{{
Values: []sqltypes.PlanValue{{
Value: sqltypes.NewInt64(1),
}, {
Value: sqltypes.NewInt64(2),
}, {
Value: sqltypes.NewInt64(4),
}},
}}
sel.Value = &evalengine.RouteValue{
Expr: evalengine.TupleExpr{
evalengine.NewLiteralInt(1),
evalengine.NewLiteralInt(2),
evalengine.NewLiteralInt(4),
},
}

vc := &loggingVCursor{
shards: []string{"-20", "20-"},
Expand Down Expand Up @@ -601,8 +600,8 @@ func TestSelectLike(t *testing.T) {
)

sel.Vindex = vindex
sel.Values = []sqltypes.PlanValue{
{Value: sqltypes.NewVarBinary("a%")},
sel.Value = &evalengine.RouteValue{
Expr: evalengine.NewLiteralString([]byte("a%"), collations.TypedCollation{}),
}
// md5("a") = 0cc175b9c0f1b6a831c399e269772661
// keyspace id prefix for "a" is 0x0c
Expand Down Expand Up @@ -631,8 +630,8 @@ func TestSelectLike(t *testing.T) {

vc.Rewind()

sel.Values = []sqltypes.PlanValue{
{Value: sqltypes.NewVarBinary("ab%")},
sel.Value = &evalengine.RouteValue{
Expr: evalengine.NewLiteralString([]byte("ab%"), collations.TypedCollation{}),
}
// md5("b") = 92eb5ffee6ae2fec3ad71c777531578f
// keyspace id prefix for "ab" is 0x0c92
Expand Down Expand Up @@ -773,7 +772,9 @@ func TestRouteGetFields(t *testing.T) {
"dummy_select_field",
)
sel.Vindex = vindex.(vindexes.SingleColumn)
sel.Values = []sqltypes.PlanValue{{Value: sqltypes.NewInt64(1)}}
sel.Value = &evalengine.RouteValue{
Expr: evalengine.NewLiteralInt(1),
}

vc := &loggingVCursor{shards: []string{"-20", "20-"}}
result, err := sel.TryExecute(vc, map[string]*querypb.BindVariable{}, true)
Expand Down Expand Up @@ -1204,7 +1205,7 @@ func TestRouteStreamTruncate(t *testing.T) {
expectResult(t, "sel.Execute", result, wantResult)
}

func XTestRouteStreamSortTruncate(t *testing.T) {
func TestRouteStreamSortTruncate(t *testing.T) {
sel := NewRoute(
SelectUnsharded,
&vindexes.Keyspace{
Expand Down
3 changes: 3 additions & 0 deletions go/vt/vtgate/evalengine/convert.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,9 @@ func convertExpr(e sqlparser.Expr, lookup ConverterLookup) (Expr, error) {
case sqlparser.Argument:
collation := getCollation(e, lookup)
return NewBindVar(string(node), collation), nil
case sqlparser.ListArg:
collation := getCollation(e, lookup)
return NewBindVar(string(node), collation), nil
case *sqlparser.Literal:
switch node.Type {
case sqlparser.IntVal:
Expand Down
Loading