Merge remote-tracking branch 'upstream/master' into support_set_trans…

…action_read_only_as_of

README.md

@@ -55,7 +55,6 @@ In addition, you may enjoy following:
 - [@PingCAP](https://twitter.com/PingCAP) on Twitter
 - Question tagged [#tidb on StackOverflow](https://stackoverflow.com/questions/tagged/tidb)
 - The PingCAP Team [English Blog](https://en.pingcap.com/blog) and [Chinese Blog](https://pingcap.com/blog-cn/)
-- [TiDB Monthly](https://pingcap.com/weekly/)
 
 For support, please contact [PingCAP](http://bit.ly/contact_us_via_github).
 

ddl/db_integration_test.go

@@ -41,7 +41,6 @@ import (
 	"github.com/pingcap/tidb/sessionctx/variable"
 	"github.com/pingcap/tidb/store/mockstore"
 	"github.com/pingcap/tidb/store/tikv/mockstore/cluster"
-	"github.com/pingcap/tidb/store/tikv/oracle"
 	"github.com/pingcap/tidb/table"
 	"github.com/pingcap/tidb/tablecodec"
 	"github.com/pingcap/tidb/types"
@@ -1234,7 +1233,7 @@ func (s *testIntegrationSuite5) TestBackwardCompatibility(c *C) {
 
 	// Split the table.
 	tableStart := tablecodec.GenTableRecordPrefix(tbl.Meta().ID)
-	s.cluster.SplitKeys(tableStart, tableStart.PrefixNext(), 100)
+	s.cluster.SplitKeys(tableStart, tableStart.PrefixNext(), 10)
 
 	unique := false
 	indexName := model.NewCIStr("idx_b")
@@ -1276,7 +1275,6 @@ func (s *testIntegrationSuite5) TestBackwardCompatibility(c *C) {
 		historyJob, err := getHistoryDDLJob(s.store, job.ID)
 		c.Assert(err, IsNil)
 		if historyJob == nil {
-
 			continue
 		}
 		c.Assert(historyJob.Error, IsNil)
@@ -1343,7 +1341,7 @@ func getMaxTableHandle(ctx *testMaxTableRowIDContext, store kv.Storage) (kv.Hand
 	c := ctx.c
 	d := ctx.d
 	tbl := ctx.tbl
-	curVer, err := store.CurrentVersion(oracle.GlobalTxnScope)
+	curVer, err := store.CurrentVersion(kv.GlobalTxnScope)
 	c.Assert(err, IsNil)
 	maxHandle, emptyTable, err := d.GetTableMaxHandle(curVer.Ver, tbl.(table.PhysicalTable))
 	c.Assert(err, IsNil)
@@ -2616,24 +2614,24 @@ func (s *testIntegrationSuite5) TestDropColumnsWithMultiIndex(c *C) {
 	tk.MustQuery(query).Check(testkit.Rows())
 }
 
-func (s *testIntegrationSuite5) TestDropLastVisibleColumn(c *C) {
+func (s *testSerialDBSuite) TestDropLastVisibleColumnOrColumns(c *C) {
+	defer config.RestoreFunc()
+	config.UpdateGlobal(func(conf *config.Config) {
+		conf.Experimental.AllowsExpressionIndex = true
+	})
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test_db")
 	tk.MustExec("create table t_drop_last_column(x int, key((1+1)))")
-	defer tk.MustExec("drop table if exists t_drop_last_column")
 	_, err := tk.Exec("alter table t_drop_last_column drop column x")
 	c.Assert(err, NotNil)
 	c.Assert(err.Error(), Equals, "[ddl:1113]A table must have at least 1 column")
-}
-
-func (s *testIntegrationSuite5) TestDropLastVisibleColumns(c *C) {
-	tk := testkit.NewTestKit(c, s.store)
-	tk.MustExec("use test_db")
+	// for visible columns
 	tk.MustExec("create table t_drop_last_columns(x int, y int, key((1+1)))")
-	defer tk.MustExec("drop table if exists t_drop_last_columns")
-	_, err := tk.Exec("alter table t_drop_last_columns drop column x, drop column y")
+	_, err = tk.Exec("alter table t_drop_last_columns drop column x, drop column y")
 	c.Assert(err, NotNil)
 	c.Assert(err.Error(), Equals, "[ddl:1113]A table must have at least 1 column")
+
+	tk.MustExec("drop table if exists t_drop_last_column, t_drop_last_columns")
 }
 
 func (s *testIntegrationSuite7) TestAutoIncrementTableOption(c *C) {
@@ -2803,3 +2801,27 @@ func (s *testIntegrationSuite3) TestIssue21835(c *C) {
 	_, err := tk.Exec("create table t( col decimal(1,2) not null default 0);")
 	c.Assert(err.Error(), Equals, "[types:1427]For float(M,D), double(M,D) or decimal(M,D), M must be >= D (column 'col').")
 }
+
+func (s *testIntegrationSuite3) TestCreateTemporaryTable(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t;")
+
+	// Grammar error.
+	tk.MustGetErrCode("create global temporary table t(a double(0, 0))", errno.ErrParse)
+	tk.MustGetErrCode("create temporary table t(id int) on commit delete rows", errno.ErrParse)
+	tk.MustGetErrCode("create temporary table t(id int) on commit preserve rows", errno.ErrParse)
+	tk.MustGetErrCode("create table t(id int) on commit delete rows", errno.ErrParse)
+	tk.MustGetErrCode("create table t(id int) on commit preserve rows", errno.ErrParse)
+
+	// Not support yet.
+	tk.MustGetErrCode("create global temporary table t (id int) on commit preserve rows", errno.ErrUnsupportedDDLOperation)
+	// Engine type can only be 'memory' or empty for now.
+	tk.MustGetErrCode("create global temporary table t (id int) engine = 'innodb' on commit delete rows", errno.ErrUnsupportedDDLOperation)
+	// Follow the behaviour of the old version TiDB: parse and ignore the 'temporary' keyword.
+	tk.MustGetErrCode("create temporary table t(id int)", errno.ErrNotSupportedYet)
+
+	tk.MustExec("set @@tidb_enable_noop_functions = 1")
+	tk.MustExec("create temporary table t (id int)")
+	tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning 1105 local TEMPORARY TABLE is not supported yet, TEMPORARY will be parsed but ignored"))
+}

ddl/ddl_api.go

@@ -1736,8 +1736,14 @@ func buildTableInfoWithStmt(ctx sessionctx.Context, s *ast.CreateTableStmt, dbCh
 	switch s.TemporaryKeyword {
 	case ast.TemporaryGlobal:
 		tbInfo.TempTableType = model.TempTableGlobal
+		// "create global temporary table ... on commit preserve rows"
+		if !s.OnCommitDelete {
+			return nil, errors.Trace(errUnsupportedOnCommitPreserve)
+		}
 	case ast.TemporaryLocal:
-		tbInfo.TempTableType = model.TempTableLocal
+		// TODO: set "tbInfo.TempTableType = model.TempTableLocal" after local temporary table is supported.
+		tbInfo.TempTableType = model.TempTableNone
+		ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("local TEMPORARY TABLE is not supported yet, TEMPORARY will be parsed but ignored"))
 	case ast.TemporaryNone:
 		tbInfo.TempTableType = model.TempTableNone
 	}
@@ -2217,6 +2223,12 @@ func handleTableOptions(options []*ast.TableOption, tbInfo *model.TableInfo) err
 			tbInfo.PreSplitRegions = op.UintValue
 		case ast.TableOptionCharset, ast.TableOptionCollate:
 			// We don't handle charset and collate here since they're handled in `getCharsetAndCollateInTableOption`.
+		case ast.TableOptionEngine:
+			if tbInfo.TempTableType != model.TempTableNone {
+				if op.StrValue != "" && !strings.EqualFold(op.StrValue, "memory") {
+					return errors.Trace(errUnsupportedEngineTemporary)
+				}
+			}
 		}
 	}
 	shardingBits := shardingBits(tbInfo)

ddl/delete_range.go

@@ -27,7 +27,6 @@ import (
 	"github.com/pingcap/tidb/ddl/util"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/sessionctx"
-	"github.com/pingcap/tidb/store/tikv/oracle"
 	"github.com/pingcap/tidb/tablecodec"
 	"github.com/pingcap/tidb/util/logutil"
 	"github.com/pingcap/tidb/util/sqlexec"
@@ -451,7 +450,7 @@ func doBatchInsert(s sqlexec.SQLExecutor, jobID int64, tableIDs []int64, ts uint
 
 // getNowTS gets the current timestamp, in TSO.
 func getNowTSO(ctx sessionctx.Context) (uint64, error) {
-	currVer, err := ctx.GetStore().CurrentVersion(oracle.GlobalTxnScope)
+	currVer, err := ctx.GetStore().CurrentVersion(kv.GlobalTxnScope)
 	if err != nil {
 		return 0, errors.Trace(err)
 	}

ddl/error.go

@@ -277,4 +277,7 @@ var (
 
 	// ErrPartitionNoTemporary returns when partition at temporary mode
 	ErrPartitionNoTemporary = dbterror.ClassDDL.NewStd(mysql.ErrPartitionNoTemporary)
+
+	errUnsupportedOnCommitPreserve = dbterror.ClassDDL.NewStdErr(mysql.ErrUnsupportedDDLOperation, parser_mysql.Message("TiDB doesn't support ON COMMIT PRESERVE ROWS for now", nil))
+	errUnsupportedEngineTemporary  = dbterror.ClassDDL.NewStdErr(mysql.ErrUnsupportedDDLOperation, parser_mysql.Message("TiDB doesn't support this kind of engine for temporary table", nil))
 )

ddl/index.go

@@ -1217,10 +1217,9 @@ func (w *worker) updateReorgInfo(t table.PartitionedTable, reorg *reorgInfo) (bo
 
 	failpoint.Inject("mockUpdateCachedSafePoint", func(val failpoint.Value) {
 		if val.(bool) {
-			// 18 is for the logical time.
-			ts := oracle.GetPhysical(time.Now()) << 18
+			ts := oracle.GoTimeToTS(time.Now())
 			s := reorg.d.store.(tikv.Storage)
-			s.UpdateSPCache(uint64(ts), time.Now())
+			s.UpdateSPCache(ts, time.Now())
 			time.Sleep(time.Millisecond * 3)
 		}
 	})

ddl/reorg.go

@@ -33,7 +33,6 @@ import (
 	"github.com/pingcap/tidb/sessionctx"
 	"github.com/pingcap/tidb/sessionctx/stmtctx"
 	"github.com/pingcap/tidb/statistics"
-	"github.com/pingcap/tidb/store/tikv/oracle"
 	"github.com/pingcap/tidb/table"
 	"github.com/pingcap/tidb/table/tables"
 	"github.com/pingcap/tidb/tablecodec"
@@ -534,7 +533,7 @@ func getTableRange(d *ddlCtx, tbl table.PhysicalTable, snapshotVer uint64, prior
 }
 
 func getValidCurrentVersion(store kv.Storage) (ver kv.Version, err error) {
-	ver, err = store.CurrentVersion(oracle.GlobalTxnScope)
+	ver, err = store.CurrentVersion(kv.GlobalTxnScope)
 	if err != nil {
 		return ver, errors.Trace(err)
 	} else if ver.Ver <= 0 {

distsql/request_builder.go

@@ -27,7 +27,6 @@ import (
 	"github.com/pingcap/tidb/sessionctx/stmtctx"
 	"github.com/pingcap/tidb/sessionctx/variable"
 	"github.com/pingcap/tidb/statistics"
-	"github.com/pingcap/tidb/store/tikv/oracle"
 	"github.com/pingcap/tidb/tablecodec"
 	"github.com/pingcap/tidb/types"
 	"github.com/pingcap/tidb/util/codec"
@@ -236,7 +235,7 @@ func (builder *RequestBuilder) SetFromSessionVars(sv *variable.SessionVars) *Req
 	}
 	builder.txnScope = sv.TxnCtx.TxnScope
 	builder.IsStaleness = sv.TxnCtx.IsStaleness
-	if builder.IsStaleness && builder.txnScope != oracle.GlobalTxnScope {
+	if builder.IsStaleness && builder.txnScope != kv.GlobalTxnScope {
 		builder.MatchStoreLabels = []*metapb.StoreLabel{
 			{
 				Key:   placement.DCLabelKey,
@@ -279,9 +278,9 @@ func (builder *RequestBuilder) SetFromInfoSchema(is infoschema.InfoSchema) *Requ
 
 func (builder *RequestBuilder) verifyTxnScope() error {
 	if builder.txnScope == "" {
-		builder.txnScope = oracle.GlobalTxnScope
+		builder.txnScope = kv.GlobalTxnScope
 	}
-	if builder.txnScope == oracle.GlobalTxnScope || builder.is == nil {
+	if builder.txnScope == kv.GlobalTxnScope || builder.is == nil {
 		return nil
 	}
 	visitPhysicalTableID := make(map[int64]struct{})
@@ -600,7 +599,7 @@ func CommonHandleRangesToKVRanges(sc *stmtctx.StatementContext, tids []int64, ra
 
 // VerifyTxnScope verify whether the txnScope and visited physical table break the leader rule's dcLocation.
 func VerifyTxnScope(txnScope string, physicalTableID int64, is infoschema.InfoSchema) bool {
-	if txnScope == "" || txnScope == oracle.GlobalTxnScope {
+	if txnScope == "" || txnScope == kv.GlobalTxnScope {
 		return true
 	}
 	bundle, ok := is.BundleByName(placement.GroupID(physicalTableID))

docs/design/2021-04-26-lock-view.md

@@ -1,7 +1,7 @@
 # TiDB Design Documents
 
 - Author(s): [longfangsong](https://github.com/longfangsong), [MyonKeminta](http://github.com/MyonKeminta)
-- Last updated: May 6, 2021
+- Last updated: May 18, 2021
 - Discussion PR: N/A
 - Tracking Issue: https://github.com/pingcap/tidb/issues/24199
 
@@ -35,14 +35,14 @@ Several tables will be provided in `information_schema`. Some tables has both lo
 
 | Field      | Type       | Comment |
 |------------|------------|---------|
-|`TRX_ID`    | `unsigned bigint`   | The transaction ID (aka. start ts) |
-|`TRX_STARTED`|`time`| Human readable start time of the transaction |
-|`DIGEST`|`text`| The digest of the current executing SQL statement |
-|`SQLS` | `text` | A list of all executed SQL statements' digests |
-|`STATE`| `enum('Running', 'Lock waiting', 'Committing', 'RollingBack')`| The state of the transaction |
+| `TRX_ID`    | `unsigned bigint`   | The transaction ID (aka. start ts) |
+| `TRX_STARTED`|`time`| Human readable start time of the transaction |
+| `DIGEST`|`text`| The digest of the current executing SQL statement |
+| `ALL_SQLS` | `text` | A list of all executed SQL statements' digests |
+| `STATE`| `enum('Running', 'Lock waiting', 'Committing', 'RollingBack')`| The state of the transaction |
 | `WAITING_START_TIME` | `time` | The elapsed time since the start of the current lock waiting (if any) |
 | `SCOPE` | `enum('Global', 'Local')` | The scope of the transaction |
-| `ISOLATION_LEVEL` | `enum('RR', 'RC')` | |
+| `ISOLATION_LEVEL` | `enum('REPEATABLE-READ', 'READ-COMMITTED')` | |
 | `AUTOCOMMIT` | `bool` |  |
 | `SESSION_ID` | `unsigned bigint` | | 
 | `USER` | `varchar` | |
@@ -79,24 +79,28 @@ Several tables will be provided in `information_schema`. Some tables has both lo
 * Permission:
     * `PROCESS` privilege is needed to access this table.
 
-### Table `(CLUSTER_)DEAD_LOCK`
+### Table `(CLUSTER_)DEADLOCKS`
 
 | Field      | Type       | Comment |
 |------------|------------|---------|
 | `DEADLOCK_ID` | `int` | There needs multiple rows to represent information of a single deadlock event. This field is used to distinguish different events. |
 | `OCCUR_TIME`  | `time` | The physical time when the deadlock occurs |
+| `RETRYABLE` | `bool` | Is the deadlock retryable. TiDB tries to determine if the current statement is (indirectly) waiting for a lock locked by the current statement. |
 | `TRY_LOCK_TRX_ID` | `unsigned bigint` | The transaction ID (start ts) of the transaction that's trying to acquire the lock |
 | `CURRENT_SQL_DIGEST` | `text` | The SQL that's being blocked |
 | `KEY` | `varchar` | The key that's being locked, but locked by another transaction in the deadlock event |
-| `SQLS` | `text` | A list of the digest of SQL statements that the transaction has executed |
+| `ALL_SQLS` | `text` | A list of the digest of SQL statements that the transaction has executed |
 | `TRX_HOLDING_LOCK` | `unsigned bigint` | The transaction that's currently holding the lock. There will be another record in the table with the same `DEADLOCK_ID` for that transaction. |
 
 * Life span of rows:
     * Create after TiDB receive a deadlock error
     * FIFO，clean the oldest after buffer is full
 * Collecting, storing and querying:
-    * All of these information can be collected on TiDB side. It just need to add the information to the table when receives deadlock error from TiKV. The information of other transactions involved in the deadlock circle needed to be fetched from elsewhere (the `TIDB_TRX` table) when handling the deadlock error.
-    * Currently there are no much information in the deadlock error (it doesn't has the SQLs and keys' information),  which needs to be improved.
+    * All of these information can be collected on TiDB side. It just need to add the information to the table when receives deadlock error from TiKV. The information of other transactions involved in the deadlock circle needed to be fetched from elsewhere (the `CLUSTER_TIDB_TRX` table) when handling the deadlock error.
+    * TiKV needs to report more rich information in the deadlock error for collecting.
+    * There are two types of deadlock errors internally: retryable or non-retryable. The transaction will internally retry on retryable deadlocks and won't report error to the client. Therefore, the user are typically more interested in the non-retryable deadlocks. 
+        * Retryable deadlock errors are by default not collected, and can be enabled with configuration.
+        * Collecting `CLUSTER_TIDB_TRX` for more rich information for retryable deadlock is possible to make the performance worse. Whether it will be collected for retryable deadlock will be decided after some tests.
 * Permission:
   * `PROCESS` privilege is needed to access this table.
 
@@ -151,9 +155,25 @@ The locking key and `resource_group_tag` that comes from the `Context` of the pe
 
 The wait chain will be added to the `Deadlock` error which is returned by the `PessimisticLock` request, so that when deadlock happens, the full wait chain information can be passed to TiDB.
 
+### Configurations
+
+#### TiDB Config File `pessimistic-txn.tidb_deadlock_history_capacity`
+
+Specifies how many recent deadlock events each TiDB node should keep.
+Dynamically changeable via HTTP API.
+Value: 0 to 10000
+Default: 10
+
+#### TiDB Config File `pessimistic-txn.tidb_deadlock_history_collect_retryable`
+
+Specifies whether to collect retryable deadlock errors to the `(CLUSTER_)DEADLOCKS` table.
+Dynamically changeable via HTTP API.
+Value: 0 (do not collect) or 1 (collect)
+Default: 0
+
 ## Compatibility
 
-This feature is not expected to be incompatible with other features. During upgrading, when there are different versions of TiDB nodes exists at the same time, it's possible that the `CLUSTER_` prefixed tables may encounter errors. But since this feature is typically used by user manually, this shouldn't be a severe problem. So we don't need to care much about that.
+This feature is not expected to be incompatible with other features. During upgrading, when there are different versions of TiDB nodes exists at the same time, it's possible that the `CLUSTER_` prefixed tables may encounter errors. However, since this feature is typically used by user manually, this shouldn't be a severe problem. So we don't need to care much about that.
 
 ## Test Design
 
@@ -190,7 +210,7 @@ This feature is not expected to be incompatible with other features. During upgr
 
 * Since lock waiting on TiKV may timeout and retry, it's possible that in a single query to `DATA_LOCK_WAIT` table doesn't shows all (logical) lock waiting.
 * Information about internal transactions may not be collected in our first version of implementation.
-* Since TiDB need to query transaction information after it receives the deadlock error, the transactions' status may be changed during that time. As a result the information in `(CLUSTER_)DEAD_LOCK` table can't be promised to be accurate and complete.
+* Since TiDB need to query transaction information after it receives the deadlock error, the transactions' status may be changed during that time. As a result the information in `(CLUSTER_)DEADLOCKS` table can't be promised to be accurate and complete.
 * Statistics about transaction conflicts is still not enough.
 * Historical information of `TIDB_TRX` and `DATA_LOCK_WAITS` is not kept, which possibly makes it still difficult to investigate some kind of problems.
 * The SQL digest that's holding lock and blocking the current transaction is hard to retrieve and is not included in the current design.

domain/domain.go

@@ -48,7 +48,6 @@ import (
 	"github.com/pingcap/tidb/sessionctx/variable"
 	"github.com/pingcap/tidb/statistics/handle"
 	"github.com/pingcap/tidb/store/tikv"
-	"github.com/pingcap/tidb/store/tikv/oracle"
 	"github.com/pingcap/tidb/telemetry"
 	"github.com/pingcap/tidb/util"
 	"github.com/pingcap/tidb/util/dbterror"
@@ -336,7 +335,7 @@ func (do *Domain) Reload() error {
 	defer do.m.Unlock()
 
 	startTime := time.Now()
-	ver, err := do.store.CurrentVersion(oracle.GlobalTxnScope)
+	ver, err := do.store.CurrentVersion(kv.GlobalTxnScope)
 	if err != nil {
 		return err
 	}

domain/domain_test.go

@@ -287,7 +287,7 @@ func (*testSuite) TestT(c *C) {
 	c.Assert(dd, NotNil)
 	c.Assert(dd.GetLease(), Equals, 80*time.Millisecond)
 
-	snapTS := oracle.EncodeTSO(oracle.GetPhysical(time.Now()))
+	snapTS := oracle.GoTimeToTS(time.Now())
 	cs := &ast.CharsetOpt{
 		Chs: "utf8",
 		Col: "utf8_bin",
@@ -317,7 +317,7 @@ func (*testSuite) TestT(c *C) {
 	c.Assert(err, IsNil)
 
 	// for GetSnapshotInfoSchema
-	currSnapTS := oracle.EncodeTSO(oracle.GetPhysical(time.Now()))
+	currSnapTS := oracle.GoTimeToTS(time.Now())
 	currSnapIs, err := dom.GetSnapshotInfoSchema(currSnapTS)
 	c.Assert(err, IsNil)
 	c.Assert(currSnapIs, NotNil)
@@ -347,7 +347,7 @@ func (*testSuite) TestT(c *C) {
 
 	// for schemaValidator
 	schemaVer := dom.SchemaValidator.(*schemaValidator).LatestSchemaVersion()
-	ver, err := store.CurrentVersion(oracle.GlobalTxnScope)
+	ver, err := store.CurrentVersion(kv.GlobalTxnScope)
 	c.Assert(err, IsNil)
 	ts := ver.Ver
 
@@ -360,7 +360,7 @@ func (*testSuite) TestT(c *C) {
 	c.Assert(succ, Equals, ResultSucc)
 	time.Sleep(ddlLease)
 
-	ver, err = store.CurrentVersion(oracle.GlobalTxnScope)
+	ver, err = store.CurrentVersion(kv.GlobalTxnScope)
 	c.Assert(err, IsNil)
 	ts = ver.Ver
 	_, succ = dom.SchemaValidator.Check(ts, schemaVer, nil)