Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Gen4: groupby support #8453

Merged
merged 22 commits into from
Jul 15, 2021
Merged

Gen4: groupby support #8453

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
bcb288a
gen4:group by planning - initial changes
harshit-gangal Jul 9, 2021
120458f
Moved queryprojection into the abstract package
systay Jul 9, 2021
422d76e
start writing unit test for query projection
systay Jul 12, 2021
f8b00fa
simplified qp logic
systay Jul 12, 2021
31f1be2
added additional qp tests
harshit-gangal Jul 12, 2021
89faf69
plan order by for group by expressions
harshit-gangal Jul 12, 2021
6a1b3bd
added resultcolumns to ordered aggregated and memory sort primitive
harshit-gangal Jul 12, 2021
10870f9
added support for group by with weight string needed. Add truncation …
harshit-gangal Jul 12, 2021
7809688
store column and weight string column information in group by params
harshit-gangal Jul 13, 2021
20a93a0
removed col from groupbyparams as redundant, added logic to updated k…
harshit-gangal Jul 13, 2021
819a83d
Merge remote-tracking branch 'upstream/main' into gen4-groupby
harshit-gangal Jul 13, 2021
72d6a00
gen4: plan group by and order by together
systay Jul 13, 2021
e85e70d
Removed weight string needed enumeration and replaced it with boolean
frouioui Jul 14, 2021
5045179
Added weight string columns for Memory Sort Plan On Aggregation
frouioui Jul 14, 2021
82ec9cf
Plan group by for unique vindex
frouioui Jul 14, 2021
847b0ec
Changed order and group by plan output to include weightstring offsets
frouioui Jul 14, 2021
bea8ef6
Fixed up tests
systay Jul 14, 2021
bf13372
Handle grouping without aggregate functions, and fail when trying to …
systay Jul 14, 2021
95cee45
Minor updates
systay Jul 14, 2021
03aa03a
Simpler return statement for planOrderByForJoin
frouioui Jul 15, 2021
42bae55
Changed createMemorySortPlanOnAggregation error message
frouioui Jul 15, 2021
041bb58
Merge branch main into gen4-groupby
systay Jul 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions go/mysql/sql_error.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ var stateToMysqlCode = map[vterrors.State]struct {
vterrors.NoSuchTable: {num: ERNoSuchTable, state: SSUnknownTable},
vterrors.NotSupportedYet: {num: ERNotSupportedYet, state: SSClientError},
vterrors.ForbidSchemaChange: {num: ERForbidSchemaChange, state: SSUnknownSQLState},
vterrors.MixOfGroupFuncAndFields: {num: ERMixOfGroupFuncAndFields, state: SSClientError},
vterrors.NetPacketTooLarge: {num: ERNetPacketTooLarge, state: SSNetError},
vterrors.NonUniqError: {num: ERNonUniq, state: SSConstraintViolation},
vterrors.NonUniqTable: {num: ERNonUniqTable, state: SSClientError},
Expand Down
38 changes: 38 additions & 0 deletions go/test/endtoend/vtgate/gen4/gen4_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ func TestOrderBy(t *testing.T) {
require.NoError(t, err)
defer conn.Close()

defer func() {
_, _ = exec(t, conn, `delete from t1`)
}()

// insert some data.
checkedExec(t, conn, `insert into t1(id, col) values (100, 123),(10, 12),(1, 13),(1000, 1234)`)

Expand All @@ -48,6 +52,40 @@ func TestOrderBy(t *testing.T) {
require.Error(t, err)
}

func TestGroupBy(t *testing.T) {
ctx := context.Background()
conn, err := mysql.Connect(ctx, &vtParams)
require.NoError(t, err)
defer conn.Close()

defer func() {
_, _ = exec(t, conn, `delete from t1`)
_, _ = exec(t, conn, `delete from t2`)
}()

// insert some data.
checkedExec(t, conn, `insert into t1(id, col) values (1, 123),(2, 12),(3, 13),(4, 1234)`)
checkedExec(t, conn, `insert into t2(id, tcol1, tcol2) values (1, 'A', 'A'),(2, 'B', 'C'),(3, 'A', 'C'),(4, 'C', 'A'),(5, 'A', 'A'),(6, 'B', 'C'),(7, 'B', 'A'),(8, 'C', 'B')`)

// Gen4 only supported query.
assertMatches(t, conn, `select tcol2, tcol1, count(id) from t2 group by tcol2, tcol1`,
`[[VARCHAR("A") VARCHAR("A") INT64(2)] [VARCHAR("A") VARCHAR("B") INT64(1)] [VARCHAR("A") VARCHAR("C") INT64(1)] [VARCHAR("B") VARCHAR("C") INT64(1)] [VARCHAR("C") VARCHAR("A") INT64(1)] [VARCHAR("C") VARCHAR("B") INT64(2)]]`)

assertMatches(t, conn, `select tcol1, tcol1 from t2 order by tcol1`,
`[[VARCHAR("A") VARCHAR("A")] [VARCHAR("A") VARCHAR("A")] [VARCHAR("A") VARCHAR("A")] [VARCHAR("B") VARCHAR("B")] [VARCHAR("B") VARCHAR("B")] [VARCHAR("B") VARCHAR("B")] [VARCHAR("C") VARCHAR("C")] [VARCHAR("C") VARCHAR("C")]]`)

assertMatches(t, conn, `select tcol1, tcol1 from t1 join t2 on t1.id = t2.id order by tcol1`,
`[[VARCHAR("A") VARCHAR("A")] [VARCHAR("A") VARCHAR("A")] [VARCHAR("B") VARCHAR("B")] [VARCHAR("C") VARCHAR("C")]]`)

assertMatches(t, conn, `select count(*) k, tcol1, tcol2, "abc" b from t2 group by tcol1, tcol2, b order by k, tcol2, tcol1`,
`[[INT64(1) VARCHAR("B") VARCHAR("A") VARCHAR("abc")] `+
`[INT64(1) VARCHAR("C") VARCHAR("A") VARCHAR("abc")] `+
`[INT64(1) VARCHAR("C") VARCHAR("B") VARCHAR("abc")] `+
`[INT64(1) VARCHAR("A") VARCHAR("C") VARCHAR("abc")] `+
`[INT64(2) VARCHAR("A") VARCHAR("A") VARCHAR("abc")] `+
`[INT64(2) VARCHAR("B") VARCHAR("C") VARCHAR("abc")]]`)
}

func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) {
t.Helper()
qr := checkedExec(t, conn, query)
Expand Down
23 changes: 22 additions & 1 deletion go/test/endtoend/vtgate/gen4/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,13 @@ var (
col bigint,
primary key(id)
) Engine=InnoDB;

create table t2(
id bigint,
tcol1 varchar(50),
tcol2 varchar(50),
primary key(id)
) Engine=InnoDB;
`

VSchema = `
Expand All @@ -53,6 +60,20 @@ var (
"name": "xxhash"
}
]
},
"t2": {
"column_vindexes": [
{
"column": "id",
"name": "xxhash"
}
],
"columns": [
{
"name": "tcol1",
"type": "VARCHAR"
}
]
}
}
}`
Expand Down Expand Up @@ -84,7 +105,7 @@ func TestMain(m *testing.M) {
}

// Start vtgate
clusterInstance.VtGateExtraArgs = []string{"-planner_version", "Gen4Fallback"} // enable Gen4 planner.
clusterInstance.VtGateExtraArgs = []string{"-planner_version", "Gen4"} // enable Gen4 planner.
err = clusterInstance.StartVtgate()
if err != nil {
return 1
Expand Down
19 changes: 19 additions & 0 deletions go/vt/sqlparser/ast_funcs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1366,3 +1366,22 @@ func ToString(exprs []TableExpr) string {
}
return buf.String()
}

// ContainsAggregation returns true if the expression contains aggregation
func ContainsAggregation(e Expr) bool {
hasAggregates := false
_ = Walk(func(node SQLNode) (kontinue bool, err error) {
switch node := node.(type) {
case *FuncExpr:
if node.IsAggregate() {
hasAggregates = true
return false, nil
}
case *GroupConcatExpr:
hasAggregates = true
return false, nil
}
return true, nil
}, e)
return hasAggregates
}
Comment on lines +1369 to +1387
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similar method exists in ordered_aggregates

func nodeHasAggregates(node sqlparser.SQLNode) bool {
	hasAggregates := false
	_ = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) {
		switch node := node.(type) {
		case *sqlparser.FuncExpr:
			if node.IsAggregate() {
				hasAggregates = true
				return false, errors.New("unused error")
			}
		case *sqlparser.GroupConcatExpr:
			hasAggregates = true
			return false, errors.New("unused error")
		case *sqlparser.Subquery:
			// Subqueries are analyzed by themselves.
			return false, nil
		}
		return true, nil
	}, node)
	return hasAggregates
}

1 change: 1 addition & 0 deletions go/vt/vterrors/state.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ const (
WrongTypeForVar
WrongValueForVar
LockOrActiveTransaction
MixOfGroupFuncAndFields

// failed precondition
NoDB
Expand Down
35 changes: 28 additions & 7 deletions go/vt/vtgate/engine/cached_size.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions go/vt/vtgate/engine/comparer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ func (c *comparer) compare(r1, r2 []sqltypes.Value) (int, error) {
return cmp, nil
}

// extractSlices extracts the three fields of OrderbyParams into a slice of comparers
func extractSlices(input []OrderbyParams) []*comparer {
// extractSlices extracts the three fields of OrderByParams into a slice of comparers
func extractSlices(input []OrderByParams) []*comparer {
var result []*comparer
for _, order := range input {
result = append(result, &comparer{
Expand Down
7 changes: 5 additions & 2 deletions go/vt/vtgate/engine/memory_sort.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ var _ Primitive = (*MemorySort)(nil)
// MemorySort is a primitive that performs in-memory sorting.
type MemorySort struct {
UpperLimit sqltypes.PlanValue
OrderBy []OrderbyParams
OrderBy []OrderByParams
Input Primitive

// TruncateColumnCount specifies the number of columns to return
Expand Down Expand Up @@ -183,6 +183,9 @@ func (ms *MemorySort) description() PrimitiveDescription {
if !value.IsNull() {
other["UpperLimit"] = value.String()
}
if ms.TruncateColumnCount > 0 {
other["ResultColumns"] = ms.TruncateColumnCount
}
return PrimitiveDescription{
OperatorType: "Sort",
Variant: "Memory",
Expand All @@ -191,7 +194,7 @@ func (ms *MemorySort) description() PrimitiveDescription {
}

func orderByParamsToString(i interface{}) string {
return i.(OrderbyParams).String()
return i.(OrderByParams).String()
}

//GenericJoin will iterate over arrays, slices or maps, and executes the f function to get a
Expand Down
18 changes: 9 additions & 9 deletions go/vt/vtgate/engine/memory_sort_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func TestMemorySortExecute(t *testing.T) {
}

ms := &MemorySort{
OrderBy: []OrderbyParams{{
OrderBy: []OrderByParams{{
WeightStringCol: -1,
Col: 1,
}},
Expand Down Expand Up @@ -100,7 +100,7 @@ func TestMemorySortStreamExecuteWeightString(t *testing.T) {
}

ms := &MemorySort{
OrderBy: []OrderbyParams{{
OrderBy: []OrderByParams{{
WeightStringCol: 0,
Col: 1,
}},
Expand Down Expand Up @@ -168,7 +168,7 @@ func TestMemorySortExecuteWeightString(t *testing.T) {
}

ms := &MemorySort{
OrderBy: []OrderbyParams{{
OrderBy: []OrderByParams{{
WeightStringCol: 1,
Col: 0,
}},
Expand Down Expand Up @@ -223,7 +223,7 @@ func TestMemorySortStreamExecute(t *testing.T) {
}

ms := &MemorySort{
OrderBy: []OrderbyParams{{
OrderBy: []OrderByParams{{
WeightStringCol: -1,
Col: 1,
}},
Expand Down Expand Up @@ -302,7 +302,7 @@ func TestMemorySortExecuteTruncate(t *testing.T) {
}

ms := &MemorySort{
OrderBy: []OrderbyParams{{
OrderBy: []OrderByParams{{
WeightStringCol: -1,
Col: 1,
}},
Expand Down Expand Up @@ -341,7 +341,7 @@ func TestMemorySortStreamExecuteTruncate(t *testing.T) {
}

ms := &MemorySort{
OrderBy: []OrderbyParams{{
OrderBy: []OrderByParams{{
WeightStringCol: -1,
Col: 1,
}},
Expand Down Expand Up @@ -384,7 +384,7 @@ func TestMemorySortMultiColumn(t *testing.T) {
}

ms := &MemorySort{
OrderBy: []OrderbyParams{{
OrderBy: []OrderByParams{{
Col: 1,
WeightStringCol: -1,
}, {
Expand Down Expand Up @@ -459,7 +459,7 @@ func TestMemorySortMaxMemoryRows(t *testing.T) {
}

ms := &MemorySort{
OrderBy: []OrderbyParams{{
OrderBy: []OrderByParams{{
WeightStringCol: -1,
Col: 1,
}},
Expand Down Expand Up @@ -495,7 +495,7 @@ func TestMemorySortExecuteNoVarChar(t *testing.T) {
}

ms := &MemorySort{
OrderBy: []OrderbyParams{{
OrderBy: []OrderByParams{{
WeightStringCol: -1,
Col: 0,
}},
Expand Down
2 changes: 1 addition & 1 deletion go/vt/vtgate/engine/merge_sort.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ var _ Primitive = (*MergeSort)(nil)
// so that vdiff can use it. In that situation, only StreamExecute is used.
type MergeSort struct {
Primitives []StreamExecutor
OrderBy []OrderbyParams
OrderBy []OrderByParams
ScatterErrorsAsWarnings bool
noInputs
noTxNeeded
Expand Down
Loading