Skip to content

Commit

Permalink
tests: disable later materialize in some tiflash tests (#58189)
Browse files Browse the repository at this point in the history 
close #58188
  • Loading branch information
@windtalker
windtalker authored Dec 12, 2024
1 parent e54cc5b commit 96103da
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 1 deletion.
56 changes: 55 additions & 1 deletion pkg/executor/test/tiflashtest/tiflash_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ func TestReadPartitionTable(t *testing.T) {
tk.MustExec("insert into t values(2,0)")
tk.MustExec("insert into t values(3,0)")
tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
// mock executor does not support use outer table as build side for outer join, so need to
// force the inner table as build side
tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1")
Expand Down Expand Up @@ -131,6 +133,8 @@ func TestAggPushDownApplyAll(t *testing.T) {

tk.MustExec("set @@session.tidb_allow_mpp=1")
tk.MustExec("set @@session.tidb_enforce_mpp=1")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")

tk.MustQuery("select * from foo where a=all(select a from bar where bar.b=foo.b)").Check(testkit.Rows("0 <nil>"))
}
Expand Down Expand Up @@ -165,6 +169,8 @@ func TestReadUnsigedPK(t *testing.T) {

tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
tk.MustExec("set @@session.tidb_allow_mpp=ON")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
// mock executor does not support use outer table as build side for outer join, so need to
// force the inner table as build side
tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1")
Expand Down Expand Up @@ -199,6 +205,8 @@ func TestJoinRace(t *testing.T) {
tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
tk.MustExec("set @@session.tidb_enforce_mpp=ON")
tk.MustExec("set @@tidb_opt_broadcast_cartesian_join=0")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
tk.MustQuery("select count(*) from (select count(a) x from t group by b) t1 join (select count(a) x from t group by b) t2 on t1.x > t2.x").Check(testkit.Rows("6"))
}

Expand Down Expand Up @@ -358,6 +366,8 @@ func TestTiFlashPartitionTableShuffledHashJoin(t *testing.T) {
// mock executor does not support use outer table as build side for outer join, so need to
// force the inner table as build side
tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")

lr := func() (int, int) {
l, r := rand.Intn(400), rand.Intn(400)
Expand Down Expand Up @@ -418,6 +428,8 @@ func TestTiFlashPartitionTableReader(t *testing.T) {
// mock executor does not support use outer table as build side for outer join, so need to
// force the inner table as build side
tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")

vals := make([]string, 0, 500)
for i := 0; i < 500; i++ {
Expand Down Expand Up @@ -564,6 +576,8 @@ func TestMppEnum(t *testing.T) {
tk.MustExec("insert into t values(3,'zca')")
tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
tk.MustExec("set @@session.tidb_allow_mpp=ON")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
// mock executor does not support use outer table as build side for outer join, so need to
// force the inner table as build side
tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1")
Expand All @@ -583,6 +597,8 @@ func TestTiFlashPlanCacheable(t *testing.T) {
err := domain.GetDomain(tk.Session()).DDLExecutor().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
require.NoError(t, err)
tk.MustExec("set @@session.tidb_isolation_read_engines = 'tikv, tiflash'")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
tk.MustExec("insert into t values(1);")
tk.MustExec("prepare stmt from 'select /*+ read_from_storage(tiflash[t]) */ * from t;';")
tk.MustQuery("execute stmt;").Check(testkit.Rows("1"))
Expand Down Expand Up @@ -679,6 +695,8 @@ func TestCancelMppTasks(t *testing.T) {
require.NoError(t, err)
tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
tk.MustExec("set @@session.tidb_allow_mpp=ON")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
// mock executor does not support use outer table as build side for outer join, so need to
// force the inner table as build side
tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1")
Expand Down Expand Up @@ -727,6 +745,8 @@ func TestMppGoroutinesExitFromErrors(t *testing.T) {
tk.MustExec("insert into t1 values(3,0)")
tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
tk.MustExec("set @@session.tidb_allow_mpp=ON")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
// mock executor does not support use outer table as build side for outer join, so need to
// force the inner table as build side
tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1")
Expand Down Expand Up @@ -815,6 +835,8 @@ func TestUnionWithEmptyDualTable(t *testing.T) {
tk.MustExec("insert into t values(1,2,3)")
tk.MustExec("insert into t1 values(1,2,3)")
tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
tk.MustExec("set @@session.tidb_enforce_mpp=ON")
tk.MustQuery("select count(*) from (select a , b from t union all select a , c from t1 where false) tt").Check(testkit.Rows("1"))
}
Expand All @@ -836,6 +858,8 @@ func TestAvgOverflow(t *testing.T) {
}
tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
tk.MustExec("set @@session.tidb_enforce_mpp=ON")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
tk.MustQuery("select avg(a) from t group by a").Check(testkit.Rows("9.0000"))
tk.MustExec("drop table if exists t")

Expand Down Expand Up @@ -885,6 +909,8 @@ func TestMppApply(t *testing.T) {
tk.MustExec("analyze table x1, x2;")

tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
tk.MustExec("set @@session.tidb_allow_mpp=ON")
// mock executor does not support use outer table as build side for outer join, so need to
// force the inner table as build side
Expand Down Expand Up @@ -922,6 +948,8 @@ func TestTiFlashVirtualColumn(t *testing.T) {
tk.MustExec("insert into t3(a,b) values(b'01',b'01'),(b'10',b'10'),(b'11',b'11')")

tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
tk.MustExec("set @@session.tidb_allow_mpp=ON")
// mock executor does not support use outer table as build side for outer join, so need to
// force the inner table as build side
Expand Down Expand Up @@ -970,6 +998,8 @@ func TestTiFlashPartitionTableShuffledHashAggregation(t *testing.T) {
tk.MustExec(fmt.Sprintf("analyze table %v", tbl))
}
tk.MustExec("set @@session.tidb_isolation_read_engines='tiflash'")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
tk.MustExec("set @@session.tidb_enforce_mpp=1")
// mock executor does not support use outer table as build side for outer join, so need to
// force the inner table as build side
Expand Down Expand Up @@ -1014,6 +1044,8 @@ func TestIssue57149(t *testing.T) {
tk.MustExec("INSERT INTO t1(i, j) VALUES (1, '{\"a\": 2}')")
tk.MustExec("analyze table test.t1")
tk.MustExec("set @@tidb_isolation_read_engines = 'tiflash'")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
tk.MustExec("set @@tidb_enforce_mpp = 1")
result := tk.MustQuery("SELECT i, (j = (SELECT j FROM t1 WHERE j = CAST('null' AS JSON))) AS c11, (j = (SELECT j FROM t1 WHERE 1<>1)) AS c13 from t1 ORDER BY i limit 1")
require.Len(t, result.Rows(), 1)
Expand All @@ -1039,7 +1071,7 @@ func TestTiFlashPartitionTableBroadcastJoin(t *testing.T) {
tk.MustExec(`create table tlist (a int, b int) partition by list(a) (
partition p0 values in (` + listPartitions[0] + `), partition p1 values in (` + listPartitions[1] + `),
partition p2 values in (` + listPartitions[2] + `), partition p3 values in (` + listPartitions[3] + `))`)
tk.MustExec(`create table tnormal (a int, b int) partition by hash(a) partitions 4`)
tk.MustExec(`create table tnormal (a int, b int)`)

for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} {
tk.MustExec("alter table " + tbl + " set tiflash replica 1")
Expand All @@ -1061,6 +1093,8 @@ func TestTiFlashPartitionTableBroadcastJoin(t *testing.T) {
// mock executor does not support use outer table as build side for outer join, so need to
// force the inner table as build side
tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")

lr := func() (int, int) {
l, r := rand.Intn(400), rand.Intn(400)
Expand Down Expand Up @@ -1190,6 +1224,8 @@ func TestTiflashPartitionTableScan(t *testing.T) {
tk.MustExec("insert into t values(1),(11),(21),(31),(41);")
tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic';")
tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\";")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
tk.MustExec("set @@session.tidb_allow_tiflash_cop=ON")
// MPP
tk.MustExec("set @@session.tidb_allow_mpp=ON;")
Expand Down Expand Up @@ -1355,6 +1391,8 @@ func TestTiflashEmptyDynamicPruneResult(t *testing.T) {
tk.MustExec("insert into IDT_RP24833 values(-8448770111093677011, \"郇鋺篤堯擈斥鍮啸赠璭饱磟朅闑傒聎疫ᛄ怖霃\", \"8781-05-02 04:23:03\", -27252736532807028, -1.34554e38);")
tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic';")
tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\";")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
tk.MustExec("set @@session.tidb_allow_mpp=ON;")
tk.MustExec("set @@session.tidb_enforce_mpp = on;")
tk.MustQuery("select /*+ read_from_storage(tiflash[t1]) */ * from IDT_RP24833 partition(p3, p4) t1 where t1. col1 between -8448770111093677011 and -8448770111093677011;").Check(testkit.Rows())
Expand Down Expand Up @@ -1384,6 +1422,8 @@ func TestDisaggregatedTiFlash(t *testing.T) {
err = domain.GetDomain(tk.Session()).DDLExecutor().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
require.NoError(t, err)
tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")

err = tk.ExecToErr("select * from t;")
// Expect error, because TestAutoScaler return empty topo.
Expand Down Expand Up @@ -1421,6 +1461,8 @@ func TestDisaggregatedTiFlashNonAutoScaler(t *testing.T) {
err = domain.GetDomain(tk.Session()).DDLExecutor().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
require.NoError(t, err)
tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")

err = tk.ExecToErr("select * from t;")
// This error message means we use PD instead of AutoScaler.
Expand Down Expand Up @@ -1544,6 +1586,8 @@ func TestTiFlashComputeDispatchPolicy(t *testing.T) {
err = domain.GetDomain(tk.Session()).DDLExecutor().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
require.NoError(t, err)
tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")

err = tiflashcompute.InitGlobalTopoFetcher(tiflashcompute.TestASStr, "tmpAddr", "tmpClusterID", false)
require.NoError(t, err)
Expand Down Expand Up @@ -1596,6 +1640,8 @@ func TestDisaggregatedTiFlashGeneratedColumn(t *testing.T) {
err = domain.GetDomain(tk.Session()).DDLExecutor().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
require.NoError(t, err)
tk.MustExec("alter table t2 add index idx2((lower(c2)));")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")

nthPlan := 100
test1 := func(forceTiFlash bool) {
Expand Down Expand Up @@ -1910,6 +1956,8 @@ func TestMPPRecovery(t *testing.T) {
}
tk.MustExec(insertStr)
tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
sql := "select * from t order by 1, 2"
const packagePath = "github.com/pingcap/tidb/pkg/executor/internal/mpp/"

Expand Down Expand Up @@ -2001,6 +2049,8 @@ func TestIssue50358(t *testing.T) {
tk.MustExec("insert into t1 values(3)")

tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
tk.MustExec("set @@session.tidb_allow_mpp=ON")
for i := 0; i < 20; i++ {
// test if it is stable.
Expand Down Expand Up @@ -2028,6 +2078,8 @@ func TestMppAggShouldAlignFinalMode(t *testing.T) {
err = failpoint.Enable("github.com/pingcap/tidb/pkg/expression/aggregation/show-agg-mode", "return(true)")
require.Nil(t, err)

// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
tk.MustQuery("explain format='brief' select 1 from (" +
" select /*+ read_from_storage(tiflash[t]) */ sum(1)" +
Expand Down Expand Up @@ -2079,6 +2131,8 @@ func TestMppTableReaderCacheForSingleSQL(t *testing.T) {
tk.MustExec("insert into t2 values(4, 4)")
tk.MustExec("insert into t2 values(5, 5)")

// unistore does not support later materialization
tk.MustExec("set tidb_opt_enable_late_materialization=0")
tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
tk.MustExec("set @@session.tidb_allow_mpp=ON")
tk.MustExec("set @@session.tidb_enforce_mpp=ON")
Expand Down
1 change: 1 addition & 0 deletions pkg/store/mockstore/unistore/cophandler/mpp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,7 @@ func (b *mppExecBuilder) buildMPPJoin(pb *tipb.Join, children []*tipb.Executor)
if err != nil {
return nil, errors.Trace(err)
}
e.baseMPPExec.children = []mppExec{leftCh, rightCh}
if pb.JoinType == tipb.JoinType_TypeLeftOuterJoin {
for _, tp := range rightCh.getFieldTypes() {
tp.DelFlag(mysql.NotNullFlag)
Expand Down
5 changes: 5 additions & 0 deletions pkg/store/mockstore/unistore/cophandler/mpp_exec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -666,6 +666,7 @@ func (e *exchSenderExec) next() (*chunk.Chunk, error) {
} else if !(chk != nil && chk.NumRows() != 0) {
return nil, nil
}
e.execSummary.updateOnlyRows(chk.NumRows())
if e.exchangeTp == tipb.ExchangeType_Hash {
rows := chk.NumRows()
targetChunks := make([]*chunk.Chunk, 0, len(e.tunnels))
Expand Down Expand Up @@ -767,6 +768,9 @@ func (e *exchRecvExec) next() (*chunk.Chunk, error) {
e.chk = nil
}()
}
if e.chk != nil {
e.execSummary.updateOnlyRows(e.chk.NumRows())
}
return e.chk, nil
}

Expand Down Expand Up @@ -984,6 +988,7 @@ func (e *joinExec) next() (*chunk.Chunk, error) {
if e.idx < len(e.reservedRows) {
idx := e.idx
e.idx++
e.execSummary.updateOnlyRows(e.reservedRows[idx].Chunk().NumRows())
return e.reservedRows[idx].Chunk(), nil
}
eof, err := e.fetchRows()
Expand Down

0 comments on commit 96103da

Please sign in to comment.