Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

planner, executor: cherry-pick naaj to v6.2 #37833

Closed
wants to merge 5 commits into from
Closed

planner, executor: cherry-pick naaj to v6.2 #37833

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7a02201
implement the null aware anti semi join and null aware anti left out…
AilinKid Aug 31, 2022
ac89caf
move naaj to v6.2
AilinKid Sep 15, 2022
c15b711
remove some useless code
AilinKid Sep 15, 2022
ed8bd7f
fix test
AilinKid Sep 15, 2022
98aabd8
make fmt
AilinKid Sep 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/explaintest/r/index_merge.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ id estRows task access object operator info
Sort_8 4433.77 root test.t1.c1
└─Projection_10 4433.77 root test.t1.c1, test.t1.c2, test.t1.c3
└─Selection_11 4433.77 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), Column#9))
└─HashJoin_12 5542.21 root CARTESIAN anti left outer semi join, other cond:eq(test.t1.c3, test.t1.c3)
└─HashJoin_12 5542.21 root Null-aware anti left outer semi join, equal:[eq(test.t1.c3, test.t1.c3)]
├─TableReader_18(Build) 10000.00 root data:TableFullScan_17
│ └─TableFullScan_17 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
└─IndexMerge_16(Probe) 5542.21 root
Expand Down
410 changes: 410 additions & 0 deletions cmd/explaintest/r/naaj.result
View file
Open in desktop

Large diffs are not rendered by default.

211 changes: 211 additions & 0 deletions cmd/explaintest/t/naaj.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
# naaj.test file is for null-aware anti join
use test;
# assert the cases for the left side without null.
select "***************************************************** PART 1 *****************************************************************" as name;
drop table if exists naaj_A, naaj_B;
create table naaj_A(a int, b int, c int);
create table naaj_B(a int, b int, c int);
insert into naaj_A values (1,1,1);
insert into naaj_B values (1,2,2);

# assert 1: both side don't have null values.
# AntiLeftOuterSemiJoin
explain format = 'brief' select (a, b) not in (select a, b from naaj_B) from naaj_A;
select (a, b) not in (select a, b from naaj_B) from naaj_A;

# AntiSemiJoin
explain format = 'brief' select * from naaj_A where (a, b) not in (select a, b from naaj_B);
select * from naaj_A where (a, b) not in (select a, b from naaj_B);

# assert 2: right side has same key bucket.
insert into naaj_B values(1,1,1);
select (a, b) not in (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) not in (select a, b from naaj_B);

# assert 3: right side has null values.
insert into naaj_B values(1, null, 2);
select (a, b) not in (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) not in (select a, b from naaj_B);

# assert 4: right side have null values, but it can't pass the inner(join key related or not) filter.
explain format = 'brief' select (a, b) not in (select a, b from naaj_B where naaj_A.c > naaj_B.c) from naaj_A;
select (a, b) not in (select a, b from naaj_B where naaj_A.c > naaj_B.c) from naaj_A;

explain format = 'brief' select * from naaj_A where (a, b) not in (select a, b from naaj_B where naaj_A.c > naaj_B.c);
select * from naaj_A where (a, b) not in (select a, b from naaj_B where naaj_A.c > naaj_B.c);

explain format = 'brief' select (a, b) not in (select a, b from naaj_B where naaj_A.a != naaj_B.a) from naaj_A;
select (a, b) not in (select a, b from naaj_B where naaj_A.a != naaj_B.a) from naaj_A;

explain format = 'brief' select * from naaj_A where (a, b) not in (select a, b from naaj_B where naaj_A.a != naaj_B.a);
select * from naaj_A where (a, b) not in (select a, b from naaj_B where naaj_A.a != naaj_B.a);

# assert 5: right side is empty.
select * from naaj_A where (a, b) not in (select a, b from naaj_B where false);
select (a, b) not in (select a, b from naaj_B where false) from naaj_A;

# assert 6: right side null bucket filter (not-null join key should match with each other).
insert into naaj_B values(2, null, 2);
select (a, b) not in (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) not in (select a, b from naaj_B);

delete from naaj_B where a=1 and b=1 and c=1;
select (a, b) not in (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) not in (select a, b from naaj_B);

# case 2: assert the cases for the left side has null.
select "***************************************************** PART 2 *****************************************************************" as name;
delete from naaj_A;
delete from naaj_B;
insert into naaj_A values(1,null,1);

# assert 1: left side has null, while the right is empty.
select (a, b) not in (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) not in (select a, b from naaj_B);

# assert 2: left side has null, while the right has a invalid null row (can't pass the nullBit filter).
insert into naaj_B values(2, null, 2);
select (a, b) not in (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) not in (select a, b from naaj_B);

# left side has null, while the right has a valid null row. (passed the nullBit filter).
insert into naaj_B values(null, null, 2);
select (a, b) not in (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) not in (select a, b from naaj_B);

# assert 3: left side has null, while the right has a valid non-null row.
delete from naaj_B;
insert into naaj_B values(2, 2, 2);
select (a, b) not in (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) not in (select a, b from naaj_B);

# assert 4: left side has null, while the right has no valid rows (equivalent to ).
insert into naaj_B values(2, null, 2);
insert into naaj_B values(null, null, 2);
explain format = 'brief' select (a, b) not in (select a, b from naaj_B where naaj_A.c > naaj_B.c) from naaj_A;
select (a, b) not in (select a, b from naaj_B where naaj_A.c > naaj_B.c) from naaj_A;
explain format = 'brief' select * from naaj_A where (a, b) not in (select a, b from naaj_B where naaj_A.c > naaj_B.c);
select * from naaj_A where (a, b) not in (select a, b from naaj_B where naaj_A.c > naaj_B.c);

# assert 5: When the inner subq has a correlated EQ condition, we won't built the NA-EQ connecting condition here.
explain format = 'brief' select (a, b) not in (select a, b from naaj_B where naaj_A.c = naaj_B.c) from naaj_A;
select (a, b) not in (select a, b from naaj_B where naaj_A.c = naaj_B.c) from naaj_A;
explain format = 'brief' select * from naaj_A where (a, b) not in (select a, b from naaj_B where naaj_A.c = naaj_B.c);
select * from naaj_A where (a, b) not in (select a, b from naaj_B where naaj_A.c = naaj_B.c);

# case 3: assert the cases for the equivalent semantic predicate of != ALL
select "***************************************************** PART 3 *****************************************************************" as name;
drop table if exists naaj_A, naaj_B;
create table naaj_A(a int, b int, c int);
create table naaj_B(a int, b int, c int);
insert into naaj_A values (1,1,1);
insert into naaj_B values (1,2,2);

# assert 1: both side don't have null values.
# AntiLeftOuterSemiJoin
explain format = 'brief' select (a, b) != all (select a, b from naaj_B) from naaj_A;
select (a, b) != all (select a, b from naaj_B) from naaj_A;

# AntiSemiJoin
explain format = 'brief' select * from naaj_A where (a, b) != all (select a, b from naaj_B);
select * from naaj_A where (a, b) != all (select a, b from naaj_B);

# assert 2: right side has same key bucket.
insert into naaj_B values(1,1,1);
select (a, b) != all (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) != all (select a, b from naaj_B);

# assert 3: right side has null values.
insert into naaj_B values(1, null, 2);
select (a, b) != all (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) != all (select a, b from naaj_B);

# assert 4: right side have null values, but it can't pass the inner(join key related or not) filter.
explain format = 'brief' select (a, b) != all (select a, b from naaj_B where naaj_A.c > naaj_B.c) from naaj_A;
select (a, b) != all (select a, b from naaj_B where naaj_A.c > naaj_B.c) from naaj_A;

explain format = 'brief' select * from naaj_A where (a, b) != all (select a, b from naaj_B where naaj_A.c > naaj_B.c);
select * from naaj_A where (a, b) != all (select a, b from naaj_B where naaj_A.c > naaj_B.c);

explain format = 'brief' select (a, b) != all (select a, b from naaj_B where naaj_A.a != naaj_B.a) from naaj_A;
select (a, b) != all (select a, b from naaj_B where naaj_A.a != naaj_B.a) from naaj_A;

explain format = 'brief' select * from naaj_A where (a, b) != all (select a, b from naaj_B where naaj_A.a != naaj_B.a);
select * from naaj_A where (a, b) != all (select a, b from naaj_B where naaj_A.a != naaj_B.a);

# assert 5: right side is empty.
select * from naaj_A where (a, b) != all (select a, b from naaj_B where false);
select (a, b) != all (select a, b from naaj_B where false) from naaj_A;

# assert 6: right side null bucket filter (not-null join key should match with each other).
insert into naaj_B values(2, null, 2);
select (a, b) != all (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) != all (select a, b from naaj_B);

delete from naaj_B where a=1 and b=1 and c=1;
select (a, b) != all (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) != all (select a, b from naaj_B);

# case 4: assert the cases for the equivalent semantic predicate of != ALL
select "***************************************************** PART 4 *****************************************************************" as name;
delete from naaj_A;
delete from naaj_B;
insert into naaj_A values(1,null,1);

# assert 1: left side has null, while the right is empty.
select (a, b) != all (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) != all (select a, b from naaj_B);

# assert 2: left side has null, while the right has a invalid null row (can't pass the nullBit filter).
insert into naaj_B values(2, null, 2);
select (a, b) != all (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) != all (select a, b from naaj_B);

# left side has null, while the right has a valid null row. (passed the nullBit filter).
insert into naaj_B values(null, null, 2);
select (a, b) != all (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) != all (select a, b from naaj_B);

# assert 3: left side has null, while the right has a valid non-null row.
delete from naaj_B;
insert into naaj_B values(2, 2, 2);
select (a, b) != all (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a, b) != all (select a, b from naaj_B);

# assert 4: left side has null, while the right has no valid rows (equivalent to ).
insert into naaj_B values(2, null, 2);
insert into naaj_B values(null, null, 2);
explain format = 'brief' select (a, b) != all (select a, b from naaj_B where naaj_A.c > naaj_B.c) from naaj_A;
select (a, b) != all (select a, b from naaj_B where naaj_A.c > naaj_B.c) from naaj_A;
explain format = 'brief' select * from naaj_A where (a, b) != all (select a, b from naaj_B where naaj_A.c > naaj_B.c);
select * from naaj_A where (a, b) != all (select a, b from naaj_B where naaj_A.c > naaj_B.c);

# assert 5: When the inner subq has a correlated EQ condition, we won't built the NA-EQ connecting condition here.
explain format = 'brief' select (a, b) != all (select a, b from naaj_B where naaj_A.c = naaj_B.c) from naaj_A;
select (a, b) != all (select a, b from naaj_B where naaj_A.c = naaj_B.c) from naaj_A;
explain format = 'brief' select * from naaj_A where (a, b) != all (select a, b from naaj_B where naaj_A.c = naaj_B.c);
select * from naaj_A where (a, b) != all (select a, b from naaj_B where naaj_A.c = naaj_B.c);

# case 5: assert some bugs.
select "***************************************************** PART 5 *****************************************************************" as name;
delete from naaj_A;
delete from naaj_B;
insert into naaj_A values(1,1,1);
insert into naaj_B values(2,null,2);

# assert 1: although the probe key doesn't have null values, we still need to use buildNullBits to guarantee the non-null position has the exactly the same value.
select (a,b) not in (select a, b from naaj_B) from naaj_A;
select * from naaj_A where (a,b) not in (select a, b from naaj_B);

# assert 2: should inject the projection under join.
explain select (a+1,b*2) not in (select a, b from naaj_B) from naaj_A;
select (a+1,b*2) not in (select a, b from naaj_B) from naaj_A;
insert into naaj_B values(2,2,2);
select (a+1,b*2) not in (select a, b from naaj_B) from naaj_A;

explain select * from naaj_A where (a+1,b*2) not in (select a+1, b-1 from naaj_B);
select * from naaj_A where (a+1,b*2) not in (select a, b from naaj_B);

# assert 3: NA-EQ and EQ can't co-exist at the same time.
explain select (a+1,b*2) not in (select a, b=1 from naaj_B where naaj_A.a = naaj_B.a) from naaj_A;
explain select * from naaj_A where (a+1,b*2) not in (select a, b=1 from naaj_B where naaj_A.a = naaj_B.a);
7 changes: 4 additions & 3 deletions executor/benchmark_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -929,7 +929,7 @@ func prepare4HashJoin(testCase *hashJoinTestCase, innerExec, outerExec Executor)
e.joiners = make([]joiner, e.concurrency)
for i := uint(0); i < e.concurrency; i++ {
e.joiners[i] = newJoiner(testCase.ctx, e.joinType, true, defaultValues,
nil, lhsTypes, rhsTypes, childrenUsedSchema)
nil, lhsTypes, rhsTypes, childrenUsedSchema, false)
}
memLimit := int64(-1)
if testCase.disk {
Expand Down Expand Up @@ -1336,7 +1336,7 @@ func prepare4IndexInnerHashJoin(tc *indexJoinTestCase, outerDS *mockDataSource,
hashCols: tc.innerHashKeyIdx,
},
workerWg: new(sync.WaitGroup),
joiner: newJoiner(tc.ctx, 0, false, defaultValues, nil, leftTypes, rightTypes, nil),
joiner: newJoiner(tc.ctx, 0, false, defaultValues, nil, leftTypes, rightTypes, nil, false),
isOuterJoin: false,
keyOff2IdxOff: keyOff2IdxOff,
lastColHelper: nil,
Expand Down Expand Up @@ -1420,7 +1420,7 @@ func prepare4IndexMergeJoin(tc *indexJoinTestCase, outerDS *mockDataSource, inne
concurrency := e.ctx.GetSessionVars().IndexLookupJoinConcurrency()
joiners := make([]joiner, concurrency)
for i := 0; i < concurrency; i++ {
joiners[i] = newJoiner(tc.ctx, 0, false, defaultValues, nil, leftTypes, rightTypes, nil)
joiners[i] = newJoiner(tc.ctx, 0, false, defaultValues, nil, leftTypes, rightTypes, nil, false)
}
e.joiners = joiners
return e, nil
Expand Down Expand Up @@ -1539,6 +1539,7 @@ func prepareMergeJoinExec(tc *mergeJoinTestCase, joinSchema *expression.Schema,
retTypes(leftExec),
retTypes(rightExec),
tc.childrenUsedSchema,
false,
)

mergeJoinExec.innerTable = &mergeJoinTable{
Expand Down
Loading