From 2d909b969675a2f626d45d64f512d884b9decd51 Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 2 Feb 2023 15:56:11 +0800 Subject: [PATCH 1/5] fix --- executor/index_join_inner_pattern_test.go | 28 +++++ planner/core/exhaust_physical_plans.go | 146 +++++++++++++++++----- sessionctx/variable/session.go | 3 + sessionctx/variable/sysvar.go | 9 ++ sessionctx/variable/tidb_vars.go | 4 + 5 files changed, 159 insertions(+), 31 deletions(-) create mode 100644 executor/index_join_inner_pattern_test.go diff --git a/executor/index_join_inner_pattern_test.go b/executor/index_join_inner_pattern_test.go new file mode 100644 index 0000000000000..21bdcad0f2812 --- /dev/null +++ b/executor/index_join_inner_pattern_test.go @@ -0,0 +1,28 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor + +import ( + "testing" + + "github.com/pingcap/tidb/testkit" +) + +func TestIndexJoinMultiPattern(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + +} diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 61575810da1fb..90681ac99094b 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -717,33 +717,72 @@ func (p *LogicalJoin) getIndexJoinByOuterIdx(prop *property.PhysicalProperty, ou } else { innerJoinKeys, outerJoinKeys, _, _ = p.GetJoinKeys() } - ds, isDataSource := innerChild.(*DataSource) - us, isUnionScan := innerChild.(*LogicalUnionScan) - if (!isDataSource && !isUnionScan) || (isDataSource && ds.preferStoreType&preferTiFlash != 0) { + innerChildWrapper := p.extractIndexJoinInnerChildPattern(innerChild) + if innerChildWrapper == nil { return nil } - if isUnionScan { - // The child of union scan may be union all for partition table. - ds, isDataSource = us.Children()[0].(*DataSource) + + var avgInnerRowCnt float64 + if outerChild.statsInfo().RowCount > 0 { + avgInnerRowCnt = p.equalCondOutCnt / outerChild.statsInfo().RowCount + } + joins = p.buildIndexJoinInner2TableScan(prop, innerChildWrapper, innerJoinKeys, outerJoinKeys, outerIdx, avgInnerRowCnt) + if joins != nil { + return + } + return p.buildIndexJoinInner2IndexScan(prop, innerChildWrapper, innerJoinKeys, outerJoinKeys, outerIdx, avgInnerRowCnt) +} + +type indexJoinInnerChildWrapper struct { + ds *DataSource + us *LogicalUnionScan + proj *LogicalProjection + sel *LogicalSelection +} + +func (p *LogicalJoin) extractIndexJoinInnerChildPattern(innerChild LogicalPlan) *indexJoinInnerChildWrapper { + wrapper := &indexJoinInnerChildWrapper{} + switch child := innerChild.(type) { + case *DataSource: + wrapper.ds = child + case *LogicalUnionScan: + wrapper.us = child + ds, isDataSource := wrapper.us.Children()[0].(*DataSource) if !isDataSource { return nil } + wrapper.ds = ds // If one of the union scan children is a TiFlash table, then we can't choose index join. - for _, child := range us.Children() { + for _, child := range wrapper.us.Children() { if ds, ok := child.(*DataSource); ok && ds.preferStoreType&preferTiFlash != 0 { return nil } } + case *LogicalProjection: + if !p.ctx.GetSessionVars().EnableIndexJoinInnerSideMultiPattern { + return nil + } + wrapper.proj = child + ds, isDataSource := wrapper.proj.Children()[0].(*DataSource) + if !isDataSource { + return nil + } + wrapper.ds = ds + case *LogicalSelection: + if !p.ctx.GetSessionVars().EnableIndexJoinInnerSideMultiPattern { + return nil + } + wrapper.sel = child + ds, isDataSource := wrapper.sel.Children()[0].(*DataSource) + if !isDataSource { + return nil + } + wrapper.ds = ds } - var avgInnerRowCnt float64 - if outerChild.statsInfo().RowCount > 0 { - avgInnerRowCnt = p.equalCondOutCnt / outerChild.statsInfo().RowCount - } - joins = p.buildIndexJoinInner2TableScan(prop, ds, innerJoinKeys, outerJoinKeys, outerIdx, us, avgInnerRowCnt) - if joins != nil { - return + if wrapper.ds == nil || wrapper.ds.preferStoreType&preferTiFlash != 0 { + return nil } - return p.buildIndexJoinInner2IndexScan(prop, ds, innerJoinKeys, outerJoinKeys, outerIdx, us, avgInnerRowCnt) + return wrapper } func (p *LogicalJoin) getIndexJoinBuildHelper(ds *DataSource, innerJoinKeys []*expression.Column, checkPathValid func(path *util.AccessPath) bool, outerJoinKeys []*expression.Column) (*indexJoinBuildHelper, []int) { @@ -783,8 +822,10 @@ func (p *LogicalJoin) getIndexJoinBuildHelper(ds *DataSource, innerJoinKeys []*e // fetched from the inner side for every tuple from the outer side. This will be // promised to be no worse than building IndexScan as the inner child. func (p *LogicalJoin) buildIndexJoinInner2TableScan( - prop *property.PhysicalProperty, ds *DataSource, innerJoinKeys, outerJoinKeys []*expression.Column, - outerIdx int, us *LogicalUnionScan, avgInnerRowCnt float64) (joins []PhysicalPlan) { + prop *property.PhysicalProperty, wrapper *indexJoinInnerChildWrapper, innerJoinKeys, outerJoinKeys []*expression.Column, + outerIdx int, avgInnerRowCnt float64) (joins []PhysicalPlan) { + ds := wrapper.ds + us := wrapper.us var tblPath *util.AccessPath for _, path := range ds.possibleAccessPaths { if path.IsTablePath() && path.StoreType == kv.TiKV { @@ -806,13 +847,13 @@ func (p *LogicalJoin) buildIndexJoinInner2TableScan( return nil } rangeInfo := helper.buildRangeDecidedByInformation(helper.chosenPath.IdxCols, outerJoinKeys) - innerTask = p.constructInnerTableScanTask(ds, helper.chosenRanges.Range(), outerJoinKeys, us, rangeInfo, false, false, avgInnerRowCnt) + innerTask = p.constructInnerTableScanTask(wrapper, helper.chosenRanges.Range(), outerJoinKeys, rangeInfo, false, false, avgInnerRowCnt) // The index merge join's inner plan is different from index join, so we // should construct another inner plan for it. // Because we can't keep order for union scan, if there is a union scan in inner task, // we can't construct index merge join. if us == nil { - innerTask2 = p.constructInnerTableScanTask(ds, helper.chosenRanges.Range(), outerJoinKeys, us, rangeInfo, true, !prop.IsSortItemEmpty() && prop.SortItems[0].Desc, avgInnerRowCnt) + innerTask2 = p.constructInnerTableScanTask(wrapper, helper.chosenRanges.Range(), outerJoinKeys, rangeInfo, true, !prop.IsSortItemEmpty() && prop.SortItems[0].Desc, avgInnerRowCnt) } ranges = helper.chosenRanges } else { @@ -846,13 +887,13 @@ func (p *LogicalJoin) buildIndexJoinInner2TableScan( } buffer.WriteString("]") rangeInfo := buffer.String() - innerTask = p.constructInnerTableScanTask(ds, ranges, outerJoinKeys, us, rangeInfo, false, false, avgInnerRowCnt) + innerTask = p.constructInnerTableScanTask(wrapper, ranges, outerJoinKeys, rangeInfo, false, false, avgInnerRowCnt) // The index merge join's inner plan is different from index join, so we // should construct another inner plan for it. // Because we can't keep order for union scan, if there is a union scan in inner task, // we can't construct index merge join. if us == nil { - innerTask2 = p.constructInnerTableScanTask(ds, ranges, outerJoinKeys, us, rangeInfo, true, !prop.IsSortItemEmpty() && prop.SortItems[0].Desc, avgInnerRowCnt) + innerTask2 = p.constructInnerTableScanTask(wrapper, ranges, outerJoinKeys, rangeInfo, true, !prop.IsSortItemEmpty() && prop.SortItems[0].Desc, avgInnerRowCnt) } } var ( @@ -880,8 +921,10 @@ func (p *LogicalJoin) buildIndexJoinInner2TableScan( } func (p *LogicalJoin) buildIndexJoinInner2IndexScan( - prop *property.PhysicalProperty, ds *DataSource, innerJoinKeys, outerJoinKeys []*expression.Column, - outerIdx int, us *LogicalUnionScan, avgInnerRowCnt float64) (joins []PhysicalPlan) { + prop *property.PhysicalProperty, wrapper *indexJoinInnerChildWrapper, innerJoinKeys, outerJoinKeys []*expression.Column, + outerIdx int, avgInnerRowCnt float64) (joins []PhysicalPlan) { + ds := wrapper.ds + us := wrapper.us helper, keyOff2IdxOff := p.getIndexJoinBuildHelper(ds, innerJoinKeys, func(path *util.AccessPath) bool { return !path.IsTablePath() }, outerJoinKeys) if helper == nil { return nil @@ -898,7 +941,7 @@ func (p *LogicalJoin) buildIndexJoinInner2IndexScan( maxOneRow = ok && (sf.FuncName.L == ast.EQ) } } - innerTask := p.constructInnerIndexScanTask(ds, helper.chosenPath, helper.chosenRanges.Range(), helper.chosenRemained, outerJoinKeys, us, rangeInfo, false, false, avgInnerRowCnt, maxOneRow) + innerTask := p.constructInnerIndexScanTask(wrapper, helper.chosenPath, helper.chosenRanges.Range(), helper.chosenRemained, outerJoinKeys, rangeInfo, false, false, avgInnerRowCnt, maxOneRow) failpoint.Inject("MockOnlyEnableIndexHashJoin", func(val failpoint.Value) { if val.(bool) && !p.ctx.GetSessionVars().InRestrictedSQL { failpoint.Return(p.constructIndexHashJoin(prop, outerIdx, innerTask, helper.chosenRanges, keyOff2IdxOff, helper.chosenPath, helper.lastColManager)) @@ -913,7 +956,7 @@ func (p *LogicalJoin) buildIndexJoinInner2IndexScan( // Because we can't keep order for union scan, if there is a union scan in inner task, // we can't construct index merge join. if us == nil { - innerTask2 := p.constructInnerIndexScanTask(ds, helper.chosenPath, helper.chosenRanges.Range(), helper.chosenRemained, outerJoinKeys, us, rangeInfo, true, !prop.IsSortItemEmpty() && prop.SortItems[0].Desc, avgInnerRowCnt, maxOneRow) + innerTask2 := p.constructInnerIndexScanTask(wrapper, helper.chosenPath, helper.chosenRanges.Range(), helper.chosenRemained, outerJoinKeys, rangeInfo, true, !prop.IsSortItemEmpty() && prop.SortItems[0].Desc, avgInnerRowCnt, maxOneRow) if innerTask2 != nil { joins = append(joins, p.constructIndexMergeJoin(prop, outerIdx, innerTask2, helper.chosenRanges, keyOff2IdxOff, helper.chosenPath, helper.lastColManager)...) } @@ -968,15 +1011,15 @@ func (ijHelper *indexJoinBuildHelper) buildRangeDecidedByInformation(idxCols []* // constructInnerTableScanTask is specially used to construct the inner plan for PhysicalIndexJoin. func (p *LogicalJoin) constructInnerTableScanTask( - ds *DataSource, + wrapper *indexJoinInnerChildWrapper, ranges ranger.Ranges, outerJoinKeys []*expression.Column, - us *LogicalUnionScan, rangeInfo string, keepOrder bool, desc bool, rowCount float64, ) task { + ds := wrapper.ds // If `ds.tableInfo.GetPartitionInfo() != nil`, // it means the data source is a partition table reader. // If the inner task need to keep order, the partition table reader can't satisfy it. @@ -1038,10 +1081,51 @@ func (p *LogicalJoin) constructInnerTableScanTask( ts.addPushedDownSelection(copTask, selStats) t := copTask.convertToRootTask(ds.ctx) reader := t.p - t.p = p.constructInnerUnionScan(us, reader) + t.p = p.constructInnerByWrapper(wrapper, reader) return t } +func (p *LogicalJoin) constructInnerByWrapper(wrapper *indexJoinInnerChildWrapper, child PhysicalPlan) PhysicalPlan { + if p.ctx.GetSessionVars().EnableIndexJoinInnerSideMultiPattern { + if wrapper.us != nil { + return p.constructInnerUnionScan(wrapper.us, child) + } else if wrapper.proj != nil { + return p.constructInnerProj(wrapper.proj, child) + } else if wrapper.sel != nil { + return p.constructInnerSel(wrapper.sel, child) + } + } else { + if wrapper.us != nil { + return p.constructInnerUnionScan(wrapper.us, child) + } + } + return child +} + +func (p *LogicalJoin) constructInnerSel(sel *LogicalSelection, child PhysicalPlan) PhysicalPlan { + if sel == nil { + return child + } + physicalSel := PhysicalSelection{ + Conditions: sel.Conditions, + }.Init(sel.ctx, sel.stats, sel.blockOffset, nil) + physicalSel.SetChildren(child) + return physicalSel +} + +func (p *LogicalJoin) constructInnerProj(proj *LogicalProjection, child PhysicalPlan) PhysicalPlan { + if proj == nil { + return child + } + physicalProj := PhysicalProjection{ + Exprs: proj.Exprs, + CalculateNoDelay: proj.CalculateNoDelay, + AvoidColumnEvaluator: proj.AvoidColumnEvaluator, + }.Init(proj.ctx, proj.stats, proj.blockOffset, nil) + physicalProj.SetChildren(child) + return physicalProj +} + func (p *LogicalJoin) constructInnerUnionScan(us *LogicalUnionScan, reader PhysicalPlan) PhysicalPlan { if us == nil { return reader @@ -1058,18 +1142,18 @@ func (p *LogicalJoin) constructInnerUnionScan(us *LogicalUnionScan, reader Physi // constructInnerIndexScanTask is specially used to construct the inner plan for PhysicalIndexJoin. func (p *LogicalJoin) constructInnerIndexScanTask( - ds *DataSource, + wrapper *indexJoinInnerChildWrapper, path *util.AccessPath, ranges ranger.Ranges, filterConds []expression.Expression, _ []*expression.Column, - us *LogicalUnionScan, rangeInfo string, keepOrder bool, desc bool, rowCount float64, maxOneRow bool, ) task { + ds := wrapper.ds // If `ds.tableInfo.GetPartitionInfo() != nil`, // it means the data source is a partition table reader. // If the inner task need to keep order, the partition table reader can't satisfy it. @@ -1194,7 +1278,7 @@ func (p *LogicalJoin) constructInnerIndexScanTask( is.addPushedDownSelection(cop, ds, tmpPath, finalStats) t := cop.convertToRootTask(ds.ctx) reader := t.p - t.p = p.constructInnerUnionScan(us, reader) + t.p = p.constructInnerByWrapper(wrapper, reader) return t } diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index eb6eeadc72d79..a218332cd23c4 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1337,6 +1337,9 @@ type SessionVars struct { // PessimisticTransactionAggressiveLocking controls whether aggressive locking for pessimistic transaction // is enabled. PessimisticTransactionAggressiveLocking bool + + // EnableIndexJoinInnerSideMultiPattern indicates whether enable multi pattern for index join inner side + EnableIndexJoinInnerSideMultiPattern bool } // planReplayerSessionFinishedTaskKeyLen is used to control the max size for the finished plan replayer task key in session diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index de0e5ffc90bd3..2f6912d7280e8 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -2288,6 +2288,15 @@ var defaultSysVars = []*SysVar{ s.EnablePlanCacheForParamLimit = TiDBOptOn(val) return nil }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableIndexJoinInnerMultiPattern, Value: BoolToOnOff(DefTiDBEnableIndexJoinInnerMultiPattern), Type: TypeBool, + SetSession: func(s *SessionVars, val string) error { + s.EnableIndexJoinInnerSideMultiPattern = TiDBOptOn(val) + return nil + }, + GetSession: func(s *SessionVars) (string, error) { + return BoolToOnOff(s.EnableIndexJoinInnerSideMultiPattern), nil + }, + }, } // FeedbackProbability points to the FeedbackProbability in statistics package. diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index c86795937d544..5f778686f25d5 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -799,6 +799,9 @@ const ( // TiDBEnablePlanCacheForParamLimit controls whether prepare statement with parameterized limit can be cached TiDBEnablePlanCacheForParamLimit = "tidb_enable_plan_cache_for_param_limit" + + // TiDBEnableIndexJoinInnerMultiPattern indicates whether support multi pattern on index join's inner side + TiDBEnableIndexJoinInnerMultiPattern = "tidb_enable_index_join_inner_multi_pattern" ) // TiDB vars that have only global scope @@ -1171,6 +1174,7 @@ const ( DefTiDBEnableResourceControl = false DefTiDBPessimisticTransactionAggressiveLocking = false DefTiDBEnablePlanCacheForParamLimit = true + DefTiDBEnableIndexJoinInnerMultiPattern = false ) // Process global variables. From eca0e9b6020d992fdf0b1b0fd38e247c0daba3d5 Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 2 Feb 2023 17:30:17 +0800 Subject: [PATCH 2/5] fix --- executor/index_advise_test.go | 134 ++++++++++++++++++++++ executor/index_join_inner_pattern_test.go | 28 ----- 2 files changed, 134 insertions(+), 28 deletions(-) delete mode 100644 executor/index_join_inner_pattern_test.go diff --git a/executor/index_advise_test.go b/executor/index_advise_test.go index 3415ffe83537b..de1cfd460c700 100644 --- a/executor/index_advise_test.go +++ b/executor/index_advise_test.go @@ -65,3 +65,137 @@ func TestIndexAdvise(t *testing.T) { require.Equal(t, uint64(4), ia.MaxIndexNum.PerTable) require.Equal(t, uint64(5), ia.MaxIndexNum.PerDB) } + +func TestIndexJoinProjPattern(t *testing.T) { + store := testkit.CreateMockStore(t) + + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec(`create table t1( +pnbrn_cnaps varchar(5) not null, +new_accno varchar(18) not null, +primary key(pnbrn_cnaps,new_accno) nonclustered +);`) + tk.MustExec(`create table t2( +pnbrn_cnaps varchar(5) not null, +txn_accno varchar(18) not null, +txn_dt date not null, +yn_frz varchar(1) default null, +txn_curr_tp varchar(3) default null +);`) + sql := `explain update +/*+ inl_join(a) */ +t2 b, +( +select t1.pnbrn_cnaps, +t1.new_accno +from t1 +where t1.pnbrn_cnaps = '40001' +) a +set b.yn_frz = '1' +where b._tidb_rowid between 1 and 10000 +and b.txn_dt = str_to_date('20221201', '%Y%m%d') +and b.pnbrn_cnaps = a.pnbrn_cnaps +and b.txn_accno = a.new_accno;` + rows := [][]interface{}{ + {"Update_8"}, + {"└─IndexJoin_14"}, + {" ├─TableReader_25(Build)"}, + {" │ └─Selection_24"}, + {" │ └─TableRangeScan_23"}, + {" └─IndexReader_12(Probe)"}, + {" └─Selection_11"}, + {" └─IndexRangeScan_10"}, + } + tk.MustExec("set @@session.tidb_enable_index_join_inner_multi_pattern='ON';") + tk.MustQuery(sql).CheckAt([]int{0}, rows) + rows = [][]interface{}{ + {"Update_8"}, + {"└─HashJoin_11"}, + {" ├─TableReader_14(Build)"}, + {" │ └─Selection_13"}, + {" │ └─TableRangeScan_12"}, + {" └─IndexReader_17(Probe)"}, + {" └─IndexRangeScan_16"}, + } + tk.MustExec("set @@session.tidb_enable_index_join_inner_multi_pattern='OFF';") + tk.MustQuery(sql).CheckAt([]int{0}, rows) +} + +func TestIndexJoinSelPattern(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec(`create table tbl_miss( +id bigint(20) unsigned not null auto_random(5) +,txn_dt date default null +,perip_sys_uuid varchar(32) not null +,rvrs_idr varchar(1) not null +,file_pnbrn_cnaps varchar(5) default null +,glbl_sn varchar(34) default null +,glbl_bsn_trck_no varchar(200) +,sbmsn_scn_refno varchar(8) default null +,primary key(id) clustered +,key idx1 (txn_dt, perip_sys_uuid, rvrs_idr) +,key idx2 (txn_dt, glbl_sn,glbl_bsn_trck_no,sbmsn_scn_refno,rvrs_idr) +);`) + tk.MustExec(`create table tbl_src( +txn_dt date default null +,uuid varchar(32) not null +,ctiq_trty char(4) +,txn_tlr_refno char(7) +,txn_org_refno char(5) +,atomt_cd varchar(10) +,orgnt_cd varchar(10) +,txn_chnl char(6) +,rvrs_idr char(1) +,expd_inf varchar(5000) +,last_mnplt_idr char(2) +,glbl_sn varchar(34) +,primary key(uuid,rvrs_idr) nonclustered +);`) + sql := `explain select /*+ use_index(mis,) inl_join(src) */ + * + from tbl_miss mis + ,tbl_src src + where src.txn_dt >= str_to_date('20221201', '%Y%m%d') + and mis.id between 1 and 10000 + and mis.perip_sys_uuid = src.uuid + and mis.rvrs_idr = src.rvrs_idr + and mis.txn_dt = src.txn_dt + and ( + case when isnull(src.expd_inf) = 1 then '' + when instr(concat_ws('',src.expd_inf,'~~'), '~~a4' ) = 0 then '' + else + substr(concat_ws('',src.expd_inf,'~~'), + instr(concat_ws('',src.expd_inf,'~~'),'~~a4') + 4, + instr(substr(concat_ws('',src.expd_inf,'~~'), + instr(concat_ws('',src.expd_inf,'~~'),'~~a4') + 4, length(concat_ws('',src.expd_inf,'~~'))),'~~') -1) + end + ) != '01';` + rows := [][]interface{}{ + {"HashJoin_9"}, + {"├─TableReader_12(Build)"}, + {"│ └─Selection_11"}, + {"│ └─TableRangeScan_10"}, + {"└─Selection_13(Probe)"}, + {" └─TableReader_16"}, + {" └─Selection_15"}, + {" └─TableFullScan_14"}, + } + tk.MustExec("set @@session.tidb_enable_index_join_inner_multi_pattern='OFF';") + tk.MustQuery(sql).CheckAt([]int{0}, rows) + rows = [][]interface{}{ + {"IndexJoin_13"}, + {"├─TableReader_25(Build)"}, + {"│ └─Selection_24"}, + {"│ └─TableRangeScan_23"}, + {"└─Selection_12(Probe)"}, + {" └─IndexLookUp_11"}, + {" ├─IndexRangeScan_8(Build)"}, + {" └─Selection_10(Probe)"}, + {" └─TableRowIDScan_9"}, + } + tk.MustExec("set @@session.tidb_enable_index_join_inner_multi_pattern='ON';") + tk.MustQuery(sql).CheckAt([]int{0}, rows) +} diff --git a/executor/index_join_inner_pattern_test.go b/executor/index_join_inner_pattern_test.go deleted file mode 100644 index 21bdcad0f2812..0000000000000 --- a/executor/index_join_inner_pattern_test.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package executor - -import ( - "testing" - - "github.com/pingcap/tidb/testkit" -) - -func TestIndexJoinMultiPattern(t *testing.T) { - store := testkit.CreateMockStore(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("use test") - -} From 7bd7f97e426aebe9afe05bd8e1e3e0dbfbc4c980 Mon Sep 17 00:00:00 2001 From: yisaer Date: Fri, 3 Feb 2023 12:14:43 +0800 Subject: [PATCH 3/5] fix --- executor/index_advise_test.go | 8 ++++---- sessionctx/variable/session.go | 1 + sessionctx/variable/sysvar.go | 9 --------- sessionctx/variable/tidb_vars.go | 4 ---- 4 files changed, 5 insertions(+), 17 deletions(-) diff --git a/executor/index_advise_test.go b/executor/index_advise_test.go index de1cfd460c700..29165bcbaac3a 100644 --- a/executor/index_advise_test.go +++ b/executor/index_advise_test.go @@ -107,7 +107,7 @@ and b.txn_accno = a.new_accno;` {" └─Selection_11"}, {" └─IndexRangeScan_10"}, } - tk.MustExec("set @@session.tidb_enable_index_join_inner_multi_pattern='ON';") + tk.Session().GetSessionVars().EnableIndexJoinInnerSideMultiPattern = true tk.MustQuery(sql).CheckAt([]int{0}, rows) rows = [][]interface{}{ {"Update_8"}, @@ -118,7 +118,7 @@ and b.txn_accno = a.new_accno;` {" └─IndexReader_17(Probe)"}, {" └─IndexRangeScan_16"}, } - tk.MustExec("set @@session.tidb_enable_index_join_inner_multi_pattern='OFF';") + tk.Session().GetSessionVars().EnableIndexJoinInnerSideMultiPattern = false tk.MustQuery(sql).CheckAt([]int{0}, rows) } @@ -183,7 +183,7 @@ txn_dt date default null {" └─Selection_15"}, {" └─TableFullScan_14"}, } - tk.MustExec("set @@session.tidb_enable_index_join_inner_multi_pattern='OFF';") + tk.Session().GetSessionVars().EnableIndexJoinInnerSideMultiPattern = false tk.MustQuery(sql).CheckAt([]int{0}, rows) rows = [][]interface{}{ {"IndexJoin_13"}, @@ -196,6 +196,6 @@ txn_dt date default null {" └─Selection_10(Probe)"}, {" └─TableRowIDScan_9"}, } - tk.MustExec("set @@session.tidb_enable_index_join_inner_multi_pattern='ON';") + tk.Session().GetSessionVars().EnableIndexJoinInnerSideMultiPattern = true tk.MustQuery(sql).CheckAt([]int{0}, rows) } diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index a218332cd23c4..f3d60f644d70c 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1339,6 +1339,7 @@ type SessionVars struct { PessimisticTransactionAggressiveLocking bool // EnableIndexJoinInnerSideMultiPattern indicates whether enable multi pattern for index join inner side + // For now it is not public to user EnableIndexJoinInnerSideMultiPattern bool } diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 2f6912d7280e8..de0e5ffc90bd3 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -2288,15 +2288,6 @@ var defaultSysVars = []*SysVar{ s.EnablePlanCacheForParamLimit = TiDBOptOn(val) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableIndexJoinInnerMultiPattern, Value: BoolToOnOff(DefTiDBEnableIndexJoinInnerMultiPattern), Type: TypeBool, - SetSession: func(s *SessionVars, val string) error { - s.EnableIndexJoinInnerSideMultiPattern = TiDBOptOn(val) - return nil - }, - GetSession: func(s *SessionVars) (string, error) { - return BoolToOnOff(s.EnableIndexJoinInnerSideMultiPattern), nil - }, - }, } // FeedbackProbability points to the FeedbackProbability in statistics package. diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 5f778686f25d5..c86795937d544 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -799,9 +799,6 @@ const ( // TiDBEnablePlanCacheForParamLimit controls whether prepare statement with parameterized limit can be cached TiDBEnablePlanCacheForParamLimit = "tidb_enable_plan_cache_for_param_limit" - - // TiDBEnableIndexJoinInnerMultiPattern indicates whether support multi pattern on index join's inner side - TiDBEnableIndexJoinInnerMultiPattern = "tidb_enable_index_join_inner_multi_pattern" ) // TiDB vars that have only global scope @@ -1174,7 +1171,6 @@ const ( DefTiDBEnableResourceControl = false DefTiDBPessimisticTransactionAggressiveLocking = false DefTiDBEnablePlanCacheForParamLimit = true - DefTiDBEnableIndexJoinInnerMultiPattern = false ) // Process global variables. From 623f2065f757af629e345bf400887582c2d34da5 Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 6 Feb 2023 13:41:08 +0800 Subject: [PATCH 4/5] fix --- executor/index_advise_test.go | 70 +++++++++++++------------- planner/core/exhaust_physical_plans.go | 24 +++++---- 2 files changed, 50 insertions(+), 44 deletions(-) diff --git a/executor/index_advise_test.go b/executor/index_advise_test.go index 29165bcbaac3a..62379d47bb13a 100644 --- a/executor/index_advise_test.go +++ b/executor/index_advise_test.go @@ -80,10 +80,12 @@ primary key(pnbrn_cnaps,new_accno) nonclustered pnbrn_cnaps varchar(5) not null, txn_accno varchar(18) not null, txn_dt date not null, -yn_frz varchar(1) default null, -txn_curr_tp varchar(3) default null +yn_frz varchar(1) default null );`) - sql := `explain update + tk.MustExec(`insert into t1(pnbrn_cnaps,new_accno) values ("40001","123")`) + tk.MustExec(`insert into t2(pnbrn_cnaps, txn_accno, txn_dt, yn_frz) values ("40001","123","20221201","0");`) + + sql := `update /*+ inl_join(a) */ t2 b, ( @@ -93,8 +95,7 @@ from t1 where t1.pnbrn_cnaps = '40001' ) a set b.yn_frz = '1' -where b._tidb_rowid between 1 and 10000 -and b.txn_dt = str_to_date('20221201', '%Y%m%d') +where b.txn_dt = str_to_date('20221201', '%Y%m%d') and b.pnbrn_cnaps = a.pnbrn_cnaps and b.txn_accno = a.new_accno;` rows := [][]interface{}{ @@ -102,59 +103,55 @@ and b.txn_accno = a.new_accno;` {"└─IndexJoin_14"}, {" ├─TableReader_25(Build)"}, {" │ └─Selection_24"}, - {" │ └─TableRangeScan_23"}, + {" │ └─TableFullScan_23"}, {" └─IndexReader_12(Probe)"}, {" └─Selection_11"}, {" └─IndexRangeScan_10"}, } tk.Session().GetSessionVars().EnableIndexJoinInnerSideMultiPattern = true - tk.MustQuery(sql).CheckAt([]int{0}, rows) + tk.MustQuery("explain "+sql).CheckAt([]int{0}, rows) rows = [][]interface{}{ {"Update_8"}, - {"└─HashJoin_11"}, - {" ├─TableReader_14(Build)"}, - {" │ └─Selection_13"}, - {" │ └─TableRangeScan_12"}, - {" └─IndexReader_17(Probe)"}, - {" └─IndexRangeScan_16"}, + {"└─HashJoin_10"}, + {" ├─IndexReader_17(Build)"}, + {" │ └─IndexRangeScan_16"}, + {" └─TableReader_14(Probe)"}, + {" └─Selection_13"}, + {" └─TableFullScan_12"}, } tk.Session().GetSessionVars().EnableIndexJoinInnerSideMultiPattern = false - tk.MustQuery(sql).CheckAt([]int{0}, rows) + tk.MustQuery("explain "+sql).CheckAt([]int{0}, rows) + + tk.Session().GetSessionVars().EnableIndexJoinInnerSideMultiPattern = true + tk.MustExec(sql) + tk.MustQuery("select yn_frz from t2").Check(testkit.Rows("1")) + } func TestIndexJoinSelPattern(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") - tk.MustExec(`create table tbl_miss( -id bigint(20) unsigned not null auto_random(5) + tk.MustExec(` create table tbl_miss( +id bigint(20) unsigned not null ,txn_dt date default null ,perip_sys_uuid varchar(32) not null ,rvrs_idr varchar(1) not null -,file_pnbrn_cnaps varchar(5) default null -,glbl_sn varchar(34) default null -,glbl_bsn_trck_no varchar(200) -,sbmsn_scn_refno varchar(8) default null ,primary key(id) clustered ,key idx1 (txn_dt, perip_sys_uuid, rvrs_idr) -,key idx2 (txn_dt, glbl_sn,glbl_bsn_trck_no,sbmsn_scn_refno,rvrs_idr) -);`) +); +`) + tk.MustExec(`insert into tbl_miss (id,txn_dt,perip_sys_uuid,rvrs_idr) values (1,"20221201","123","1");`) tk.MustExec(`create table tbl_src( txn_dt date default null ,uuid varchar(32) not null -,ctiq_trty char(4) -,txn_tlr_refno char(7) -,txn_org_refno char(5) -,atomt_cd varchar(10) -,orgnt_cd varchar(10) -,txn_chnl char(6) ,rvrs_idr char(1) ,expd_inf varchar(5000) -,last_mnplt_idr char(2) -,glbl_sn varchar(34) ,primary key(uuid,rvrs_idr) nonclustered -);`) - sql := `explain select /*+ use_index(mis,) inl_join(src) */ +); +`) + tk.MustExec(`insert into tbl_src (txn_dt,uuid,rvrs_idr) values ("20221201","123","1");`) + sql := `select /*+ use_index(mis,) inl_join(src) */ * from tbl_miss mis ,tbl_src src @@ -165,7 +162,6 @@ txn_dt date default null and mis.txn_dt = src.txn_dt and ( case when isnull(src.expd_inf) = 1 then '' - when instr(concat_ws('',src.expd_inf,'~~'), '~~a4' ) = 0 then '' else substr(concat_ws('',src.expd_inf,'~~'), instr(concat_ws('',src.expd_inf,'~~'),'~~a4') + 4, @@ -184,7 +180,7 @@ txn_dt date default null {" └─TableFullScan_14"}, } tk.Session().GetSessionVars().EnableIndexJoinInnerSideMultiPattern = false - tk.MustQuery(sql).CheckAt([]int{0}, rows) + tk.MustQuery("explain "+sql).CheckAt([]int{0}, rows) rows = [][]interface{}{ {"IndexJoin_13"}, {"├─TableReader_25(Build)"}, @@ -197,5 +193,9 @@ txn_dt date default null {" └─TableRowIDScan_9"}, } tk.Session().GetSessionVars().EnableIndexJoinInnerSideMultiPattern = true - tk.MustQuery(sql).CheckAt([]int{0}, rows) + tk.MustQuery("explain "+sql).CheckAt([]int{0}, rows) + tk.Session().GetSessionVars().EnableIndexJoinInnerSideMultiPattern = true + tk.MustQuery(sql).Check(testkit.Rows("1 2022-12-01 123 1 2022-12-01 123 1 ")) + tk.Session().GetSessionVars().EnableIndexJoinInnerSideMultiPattern = false + tk.MustQuery(sql).Check(testkit.Rows("1 2022-12-01 123 1 2022-12-01 123 1 ")) } diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 90681ac99094b..a39a14874ce7c 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -762,6 +762,12 @@ func (p *LogicalJoin) extractIndexJoinInnerChildPattern(innerChild LogicalPlan) if !p.ctx.GetSessionVars().EnableIndexJoinInnerSideMultiPattern { return nil } + // For now, we only allow proj with all Column expression can be the inner side of index join + for _, expr := range child.Exprs { + if _, ok := expr.(*expression.Column); !ok { + return nil + } + } wrapper.proj = child ds, isDataSource := wrapper.proj.Children()[0].(*DataSource) if !isDataSource { @@ -1086,18 +1092,18 @@ func (p *LogicalJoin) constructInnerTableScanTask( } func (p *LogicalJoin) constructInnerByWrapper(wrapper *indexJoinInnerChildWrapper, child PhysicalPlan) PhysicalPlan { - if p.ctx.GetSessionVars().EnableIndexJoinInnerSideMultiPattern { - if wrapper.us != nil { - return p.constructInnerUnionScan(wrapper.us, child) - } else if wrapper.proj != nil { - return p.constructInnerProj(wrapper.proj, child) - } else if wrapper.sel != nil { - return p.constructInnerSel(wrapper.sel, child) - } - } else { + if !p.ctx.GetSessionVars().EnableIndexJoinInnerSideMultiPattern { if wrapper.us != nil { return p.constructInnerUnionScan(wrapper.us, child) } + return child + } + if wrapper.us != nil { + return p.constructInnerUnionScan(wrapper.us, child) + } else if wrapper.proj != nil { + return p.constructInnerProj(wrapper.proj, child) + } else if wrapper.sel != nil { + return p.constructInnerSel(wrapper.sel, child) } return child } From cb8a294308f04d8f23e5f368b149ba1b833f8b2c Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 6 Feb 2023 15:12:58 +0800 Subject: [PATCH 5/5] fix --- executor/index_advise_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/executor/index_advise_test.go b/executor/index_advise_test.go index 62379d47bb13a..a50c294294cea 100644 --- a/executor/index_advise_test.go +++ b/executor/index_advise_test.go @@ -68,7 +68,6 @@ func TestIndexAdvise(t *testing.T) { func TestIndexJoinProjPattern(t *testing.T) { store := testkit.CreateMockStore(t) - tk := testkit.NewTestKit(t, store) tk.MustExec("use test") tk.MustExec(`create table t1( @@ -125,7 +124,6 @@ and b.txn_accno = a.new_accno;` tk.Session().GetSessionVars().EnableIndexJoinInnerSideMultiPattern = true tk.MustExec(sql) tk.MustQuery("select yn_frz from t2").Check(testkit.Rows("1")) - } func TestIndexJoinSelPattern(t *testing.T) {