Merge remote-tracking branch 'upstream/master' into add-innodb-defaul…

…t-row-format

cmd/ddltest/ddl_test.go

@@ -41,6 +41,7 @@ import (
  "github.com/pingcap/tidb/kv"
  "github.com/pingcap/tidb/session"
  "github.com/pingcap/tidb/sessionctx"
+ "github.com/pingcap/tidb/sessionctx/variable"
  "github.com/pingcap/tidb/store"
  tidbdriver "github.com/pingcap/tidb/store/driver"
  "github.com/pingcap/tidb/table"
@@ -559,7 +560,7 @@ func (s *TestDDLSuite) Bootstrap(c *C) {
  tk.MustExec("create table test_mixed (c1 int, c2 int, primary key(c1))")
  tk.MustExec("create table test_inc (c1 int, c2 int, primary key(c1))")
 
- tk.Se.GetSessionVars().EnableClusteredIndex = true
+ tk.Se.GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeOn
  tk.MustExec("drop table if exists test_insert_common, test_conflict_insert_common, " +
  "test_update_common, test_conflict_update_common, test_delete_common, test_conflict_delete_common, " +
  "test_mixed_common, test_inc_common")
@@ -571,7 +572,7 @@ func (s *TestDDLSuite) Bootstrap(c *C) {
  tk.MustExec("create table test_conflict_delete_common (c1 int, c2 int, primary key(c1, c2))")
  tk.MustExec("create table test_mixed_common (c1 int, c2 int, primary key(c1, c2))")
  tk.MustExec("create table test_inc_common (c1 int, c2 int, primary key(c1, c2))")
- tk.Se.GetSessionVars().EnableClusteredIndex = false
+ tk.Se.GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeIntOnly
 }
 
 func (s *TestDDLSuite) TestSimple(c *C) {

cmd/explaintest/main.go

@@ -94,7 +94,6 @@ func newTester(name string) *tester {
  t.enableQueryLog = true
  t.ctx = mock.NewContext()
  t.ctx.GetSessionVars().EnableWindowFunction = true
- t.ctx.GetSessionVars().IntPrimaryKeyDefaultAsClustered = true
  return t
 }
 
@@ -658,7 +657,6 @@ func main() {
  "set @@tidb_projection_concurrency=4",
  "set @@tidb_distsql_scan_concurrency=15",
  "set @@global.tidb_enable_clustered_index=0;",
- "set @@tidb_int_primary_key_default_as_clustered=1",
  }
  for _, sql := range resets {
  if _, err = mdb.Exec(sql); err != nil {

config/config.go

@@ -126,6 +126,8 @@ type Config struct {
  IndexLimit int `toml:"index-limit" json:"index-limit"`
  TableColumnCountLimit uint32 `toml:"table-column-count-limit" json:"table-column-count-limit"`
  GracefulWaitBeforeShutdown int `toml:"graceful-wait-before-shutdown" json:"graceful-wait-before-shutdown"`
+ // AlterPrimaryKey is used to control alter primary key feature.
+ AlterPrimaryKey bool `toml:"alter-primary-key" json:"alter-primary-key"`
  // TreatOldVersionUTF8AsUTF8MB4 is use to treat old version table/column UTF8 charset as UTF8MB4. This is for compatibility.
  // Currently not support dynamic modify, because this need to reload all old version schema.
  TreatOldVersionUTF8AsUTF8MB4 bool `toml:"treat-old-version-utf8-as-utf8mb4" json:"treat-old-version-utf8-as-utf8mb4"`
@@ -560,6 +562,7 @@ var defaultConf = Config{
  MaxIndexLength: 3072,
  IndexLimit: 64,
  TableColumnCountLimit: 1017,
+ AlterPrimaryKey: false,
  TreatOldVersionUTF8AsUTF8MB4: true,
  EnableTableLock: false,
  DelayCleanTableLock: 0,

config/config.toml.example

@@ -86,6 +86,12 @@ delay-clean-table-lock = 0
 # Maximum number of the splitting region, which is used by the split region statement.
 split-region-max-num = 1000
 
+# alter-primary-key is used to control whether the primary keys are clustered. 
+# Note that this config is deprecated. Only valid when @@global.tidb_enable_clustered_index = 'int_only'.
+# Default is false, only the integer primary keys are clustered.
+# If it is true, all types of primary keys are nonclustered.
+alter-primary-key = false
+
 # server-version is used to change the version string of TiDB in the following scenarios:
 # 1. the server version returned by builtin-function `VERSION()`.
 # 2. the server version filled in handshake packets of MySQL Connection Protocol, see https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake for more details.

config/config_test.go

@@ -184,6 +184,7 @@ unrecognized-option-test = true
  _, err = f.WriteString(`
 token-limit = 0
 enable-table-lock = true
+alter-primary-key = true
 delay-clean-table-lock = 5
 split-region-max-num=10000
 enable-batch-dml = true
@@ -243,6 +244,7 @@ spilled-file-encryption-method = "plaintext"
 
  // Test that the value will be overwritten by the config file.
  c.Assert(conf.Performance.TxnTotalSizeLimit, Equals, uint64(2000))
+ c.Assert(conf.AlterPrimaryKey, Equals, true)
  c.Assert(conf.Performance.TCPNoDelay, Equals, false)
 
  c.Assert(conf.TiKVClient.CommitTimeout, Equals, "41s")

ddl/db_change_test.go

@@ -793,9 +793,10 @@ func (s *testStateChangeSuite) TestWriteOnlyForDropColumn(c *C) {
  c.Assert(err, IsNil)
  }()
 
- sqls := make([]sqlWithErr, 2)
+ sqls := make([]sqlWithErr, 3)
  sqls[0] = sqlWithErr{"update t set c1='5', c3='2020-03-01';", errors.New("[planner:1054]Unknown column 'c3' in 'field list'")}
- sqls[1] = sqlWithErr{"update t t1, tt t2 set t1.c1='5', t1.c3='2020-03-01', t2.c1='10' where t1.c4=t2.c4",
+ sqls[1] = sqlWithErr{"update t set c1='5', c3='2020-03-01' where c4 = 8;", errors.New("[planner:1054]Unknown column 'c3' in 'field list'")}
+ sqls[2] = sqlWithErr{"update t t1, tt t2 set t1.c1='5', t1.c3='2020-03-01', t2.c1='10' where t1.c4=t2.c4",
  errors.New("[planner:1054]Unknown column 'c3' in 'field list'")}
  // TODO: Fix the case of sqls[2].
  // sqls[2] = sqlWithErr{"update t set c1='5' where c3='2017-07-01';", errors.New("[planner:1054]Unknown column 'c3' in 'field list'")}

ddl/db_integration_test.go

@@ -38,6 +38,7 @@ import (
  "github.com/pingcap/tidb/session"
  "github.com/pingcap/tidb/sessionctx"
  "github.com/pingcap/tidb/sessionctx/stmtctx"
+ "github.com/pingcap/tidb/sessionctx/variable"
  "github.com/pingcap/tidb/store/mockstore"
  "github.com/pingcap/tidb/store/tikv/mockstore/cluster"
  "github.com/pingcap/tidb/store/tikv/oracle"
@@ -2638,7 +2639,7 @@ func (s *testIntegrationSuite7) TestDuplicateErrorMessage(c *C) {
  config.UpdateGlobal(func(conf *config.Config) {
  conf.EnableGlobalIndex = globalIndex
  })
- for _, clusteredIndex := range []bool{false, true} {
+ for _, clusteredIndex := range []variable.ClusteredIndexDefMode{variable.ClusteredIndexDefModeOn, variable.ClusteredIndexDefModeOff, variable.ClusteredIndexDefModeIntOnly} {
  tk.Se.GetSessionVars().EnableClusteredIndex = clusteredIndex
  for _, t := range tests {
  tk.MustExec("drop table if exists t;")

ddl/db_test.go

@@ -178,7 +178,7 @@ func (s *testDBSuite7) TestAddIndexWithPK(c *C) {
  tk.MustExec("use " + s.schemaName)
 
  testAddIndexWithPK(tk)
- tk.Se.GetSessionVars().EnableClusteredIndex = true
+ tk.Se.GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeOn
  testAddIndexWithPK(tk)
 }
 
@@ -1056,7 +1056,7 @@ func (s *testDBSuite6) TestAddMultiColumnsIndexClusterIndex(c *C) {
  tk.MustExec("create database test_add_multi_col_index_clustered;")
  tk.MustExec("use test_add_multi_col_index_clustered;")
 
- tk.Se.GetSessionVars().EnableClusteredIndex = true
+ tk.Se.GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeOn
  tk.MustExec("create table t (a int, b varchar(10), c int, primary key (a, b));")
  tk.MustExec("insert into t values (1, '1', 1), (2, '2', NULL), (3, '3', 3);")
  tk.MustExec("create index idx on t (a, c);")
@@ -1156,7 +1156,7 @@ func testAddIndex(c *C, store kv.Storage, lease time.Duration, tp testAddIndexTy
  case testPartition:
  tk.MustExec("set @@session.tidb_enable_table_partition = '1';")
  case testClusteredIndex:
- tk.Se.GetSessionVars().EnableClusteredIndex = true
+ tk.Se.GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeOn
  }
  tk.MustExec("drop table if exists test_add_index")
  tk.MustExec(createTableSQL)

ddl/ddl_api.go

@@ -1523,8 +1523,14 @@ func isSingleIntPK(constr *ast.Constraint, lastCol *model.ColumnInfo) bool {
 // ShouldBuildClusteredIndex is used to determine whether the CREATE TABLE statement should build a clustered index table.
 func ShouldBuildClusteredIndex(ctx sessionctx.Context, opt *ast.IndexOption, isSingleIntPK bool) bool {
  if opt == nil || opt.PrimaryKeyTp == model.PrimaryKeyTypeDefault {
- return ctx.GetSessionVars().EnableClusteredIndex ||
- (isSingleIntPK && ctx.GetSessionVars().IntPrimaryKeyDefaultAsClustered)
+ switch ctx.GetSessionVars().EnableClusteredIndex {
+ case variable.ClusteredIndexDefModeOn:
+ return true
+ case variable.ClusteredIndexDefModeIntOnly:
+ return !config.GetGlobalConfig().AlterPrimaryKey && isSingleIntPK
+ default:
+ return false
+ }
  }
  return opt.PrimaryKeyTp == model.PrimaryKeyTypeClustered
 }

ddl/failtest/fail_db_test.go

@@ -354,7 +354,7 @@ func (s *testFailDBSuite) TestAddIndexWorkerNum(c *C) {
  tk.MustExec("use test_db")
  tk.MustExec("drop table if exists test_add_index")
  if s.IsCommonHandle {
- tk.Se.GetSessionVars().EnableClusteredIndex = true
+ tk.Se.GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeOn
  tk.MustExec("create table test_add_index (c1 bigint, c2 bigint, c3 bigint, primary key(c1, c3))")
  } else {
  tk.MustExec("create table test_add_index (c1 bigint, c2 bigint, c3 bigint, primary key(c1))")

ddl/partition.go

@@ -129,16 +129,17 @@ func (w *worker) onAddTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (v
  if tblInfo.TiFlashReplica != nil && tblInfo.TiFlashReplica.Available {
  // For available state, the new added partition should wait it's replica to
  // be finished. Otherwise the query to this partition will be blocked.
- needWait, err := checkPartitionReplica(addingDefinitions, d)
+ needRetry, err := checkPartitionReplica(tblInfo.TiFlashReplica.Count, addingDefinitions, d)
  if err != nil {
  ver, err = convertAddTablePartitionJob2RollbackJob(t, job, err, tblInfo)
  return ver, err
  }
- if needWait {
+ if needRetry {
  // The new added partition hasn't been replicated.
  // Do nothing to the job this time, wait next worker round.
  time.Sleep(tiflashCheckTiDBHTTPAPIHalfInterval)
- return ver, nil
+ // Set the error here which will lead this job exit when it's retry times beyond the limitation.
+ return ver, errors.Errorf("[ddl] add partition wait for tiflash replica to complete")
  }
  }
 
@@ -222,13 +223,29 @@ func checkAddPartitionValue(meta *model.TableInfo, part *model.PartitionInfo) er
  return nil
 }
 
-func checkPartitionReplica(addingDefinitions []model.PartitionDefinition, d *ddlCtx) (needWait bool, err error) {
+func checkPartitionReplica(replicaCount uint64, addingDefinitions []model.PartitionDefinition, d *ddlCtx) (needWait bool, err error) {
+ failpoint.Inject("mockWaitTiFlashReplica", func(val failpoint.Value) {
+ if val.(bool) {
+ failpoint.Return(true, nil)
+ }
+ })
+
  ctx := context.Background()
  pdCli := d.store.(tikv.Storage).GetRegionCache().PDClient()
  stores, err := pdCli.GetAllStores(ctx)
  if err != nil {
  return needWait, errors.Trace(err)
  }
+ // Check whether stores have `count` tiflash engines.
+ tiFlashStoreCount := uint64(0)
+ for _, store := range stores {
+ if storeHasEngineTiFlashLabel(store) {
+ tiFlashStoreCount++
+ }
+ }
+ if replicaCount > tiFlashStoreCount {
+ return false, errors.Errorf("[ddl] the tiflash replica count: %d should be less than the total tiflash server count: %d", replicaCount, tiFlashStoreCount)
+ }
  for _, pd := range addingDefinitions {
  startKey, endKey := tablecodec.GetTableHandleKeyRange(pd.ID)
  regions, err := pdCli.ScanRegions(ctx, startKey, endKey, -1)

ddl/partition_test.go

@@ -17,14 +17,16 @@ import (
  "context"
 
  . "github.com/pingcap/check"
+ "github.com/pingcap/failpoint"
  "github.com/pingcap/parser/model"
  "github.com/pingcap/parser/mysql"
  "github.com/pingcap/tidb/kv"
  "github.com/pingcap/tidb/sessionctx"
+ "github.com/pingcap/tidb/sessionctx/variable"
  "github.com/pingcap/tidb/types"
 )
 
-var _ = Suite(&testPartitionSuite{})
+var _ = SerialSuites(&testPartitionSuite{})
 
 type testPartitionSuite struct {
  store kv.Storage
@@ -155,3 +157,111 @@ func testTruncatePartition(c *C, ctx sessionctx.Context, d *ddl, dbInfo *model.D
  checkHistoryJobArgs(c, ctx, job.ID, &historyJobArgs{ver: v, tbl: tblInfo})
  return job
 }
+
+func testAddPartition(c *C, ctx sessionctx.Context, d *ddl, dbInfo *model.DBInfo, tblInfo *model.TableInfo) error {
+ ids, err := d.genGlobalIDs(1)
+ c.Assert(err, IsNil)
+ partitionInfo := &model.PartitionInfo{
+ Type: model.PartitionTypeRange,
+ Expr: tblInfo.Columns[0].Name.L,
+ Enable: true,
+ Definitions: []model.PartitionDefinition{
+ {
+ ID: ids[0],
+ Name: model.NewCIStr("p2"),
+ LessThan: []string{"300"},
+ },
+ },
+ }
+ addPartitionJob := &model.Job{
+ SchemaID: dbInfo.ID,
+ TableID: tblInfo.ID,
+ Type: model.ActionAddTablePartition,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []interface{}{partitionInfo},
+ }
+ return d.doDDLJob(ctx, addPartitionJob)
+}
+
+func (s *testPartitionSuite) TestAddPartitionReplicaBiggerThanTiFlashStores(c *C) {
+ d := testNewDDLAndStart(
+ context.Background(),
+ c,
+ WithStore(s.store),
+ WithLease(testLease),
+ )
+ defer func() {
+ err := d.Stop()
+ c.Assert(err, IsNil)
+ }()
+ dbInfo := testSchemaInfo(c, d, "test_partition2")
+ testCreateSchema(c, testNewContext(d), d, dbInfo)
+ // Build a tableInfo with replica count = 1 while there is no real tiFlash store.
+ tblInfo := buildTableInfoWithReplicaInfo(c, d)
+ ctx := testNewContext(d)
+ testCreateTable(c, ctx, d, dbInfo, tblInfo)
+
+ err := testAddPartition(c, ctx, d, dbInfo, tblInfo)
+ // Since there is no real TiFlash store (less than replica count), adding a partition will error here.
+ c.Assert(err, NotNil)
+ c.Assert(err.Error(), Equals, "[ddl:-1][ddl] the tiflash replica count: 1 should be less than the total tiflash server count: 0")
+
+ // Test `add partition` waiting TiFlash replica can exit when its retry count is beyond the limitation.
+ originErrCountLimit := variable.GetDDLErrorCountLimit()
+ variable.SetDDLErrorCountLimit(3)
+ defer func() {
+ variable.SetDDLErrorCountLimit(originErrCountLimit)
+ }()
+ c.Assert(failpoint.Enable("github.com/pingcap/tidb/ddl/mockWaitTiFlashReplica", `return(true)`), IsNil)
+ defer func() {
+ c.Assert(failpoint.Disable("github.com/pingcap/tidb/ddl/mockWaitTiFlashReplica"), IsNil)
+ }()
+ err = testAddPartition(c, ctx, d, dbInfo, tblInfo)
+ c.Assert(err, NotNil)
+ c.Assert(err.Error(), Equals, "[ddl:-1]DDL job rollback, error msg: [ddl] add partition wait for tiflash replica to complete")
+}
+
+func buildTableInfoWithReplicaInfo(c *C, d *ddl) *model.TableInfo {
+ tbl := &model.TableInfo{
+ Name: model.NewCIStr("t1"),
+ }
+ col := &model.ColumnInfo{
+ Name: model.NewCIStr("c"),
+ Offset: 0,
+ State: model.StatePublic,
+ FieldType: *types.NewFieldType(mysql.TypeLong),
+ ID: allocateColumnID(tbl),
+ }
+ genIDs, err := d.genGlobalIDs(1)
+ c.Assert(err, IsNil)
+ tbl.ID = genIDs[0]
+ tbl.Columns = []*model.ColumnInfo{col}
+ tbl.Charset = "utf8"
+ tbl.Collate = "utf8_bin"
+ tbl.TiFlashReplica = &model.TiFlashReplicaInfo{
+ Count: 1,
+ Available: true,
+ }
+
+ partIDs, err := d.genGlobalIDs(2)
+ c.Assert(err, IsNil)
+ partInfo := &model.PartitionInfo{
+ Type: model.PartitionTypeRange,
+ Expr: tbl.Columns[0].Name.L,
+ Enable: true,
+ Definitions: []model.PartitionDefinition{
+ {
+ ID: partIDs[0],
+ Name: model.NewCIStr("p0"),
+ LessThan: []string{"100"},
+ },
+ {
+ ID: partIDs[1],
+ Name: model.NewCIStr("p1"),
+ LessThan: []string{"200"},
+ },
+ },
+ }
+ tbl.Partition = partInfo
+ return tbl
+}

ddl/serial_test.go

@@ -117,7 +117,7 @@ func (s *testIntegrationSuite9) TestPrimaryKey(c *C) {
  tk.MustExec("drop database if exists test_primary_key;")
  tk.MustExec("create database test_primary_key;")
  tk.MustExec("use test_primary_key;")
- tk.Se.GetSessionVars().EnableClusteredIndex = false
+ tk.Se.GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeIntOnly
 
  // Test add/drop primary key on a plain table.
  tk.MustExec("drop table if exists t;")
@@ -325,7 +325,7 @@ func (s *testIntegrationSuite9) TestMultiRegionGetTableEndCommonHandle(c *C) {
  tk.MustExec("drop database if exists test_get_endhandle")
  tk.MustExec("create database test_get_endhandle")
  tk.MustExec("use test_get_endhandle")
- tk.Se.GetSessionVars().EnableClusteredIndex = true
+ tk.Se.GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeOn
 
  tk.MustExec("create table t(a varchar(20), b int, c float, d bigint, primary key (a, b, c))")
  var builder strings.Builder
@@ -369,7 +369,7 @@ func (s *testIntegrationSuite9) TestGetTableEndCommonHandle(c *C) {
  tk.MustExec("drop database if exists test_get_endhandle")
  tk.MustExec("create database test_get_endhandle")
  tk.MustExec("use test_get_endhandle")
- tk.Se.GetSessionVars().EnableClusteredIndex = true
+ tk.Se.GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeOn
 
  tk.MustExec("create table t(a varchar(15), b bigint, c int, primary key (a, b))")
  tk.MustExec("create table t1(a varchar(15), b bigint, c int, primary key (a(2), b))")
@@ -1406,7 +1406,7 @@ func (s *testIntegrationSuite9) TestInvisibleIndex(c *C) {
 
 func (s *testIntegrationSuite9) TestCreateClusteredIndex(c *C) {
  tk := testkit.NewTestKitWithInit(c, s.store)
- tk.Se.GetSessionVars().EnableClusteredIndex = true
+ tk.Se.GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeOn
  tk.MustExec("CREATE TABLE t1 (a int primary key, b int)")
  tk.MustExec("CREATE TABLE t2 (a varchar(255) primary key, b int)")
  tk.MustExec("CREATE TABLE t3 (a int, b int, c int, primary key (a, b))")
@@ -1447,7 +1447,7 @@ func (s *testIntegrationSuite9) TestCreateClusteredIndex(c *C) {
  c.Assert(err, IsNil)
  c.Assert(tbl.Meta().IsCommonHandle, IsTrue)
 
- tk.Se.GetSessionVars().EnableClusteredIndex = false
+ tk.Se.GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeIntOnly
  tk.MustExec("CREATE TABLE t7 (a varchar(255) primary key, b int)")
  is = domain.GetDomain(ctx).InfoSchema()
  tbl, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t7"))