meta: make auto increment id can be adjust. (pingcap#10978) (pingcap#…

…10997)

ddl/db_test.go

@@ -25,6 +25,7 @@ import (
 
 	. "github.com/pingcap/check"
 	"github.com/pingcap/errors"
+	"github.com/pingcap/failpoint"
 	"github.com/pingcap/parser/ast"
 	"github.com/pingcap/parser/charset"
 	"github.com/pingcap/parser/model"
@@ -2850,6 +2851,10 @@ func (s *testDBSuite) TestComment(c *C) {
 }
 
 func (s *testDBSuite) TestRebaseAutoID(c *C) {
+	c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
+	defer func() {
+		c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
+	}()
 	s.tk = testkit.NewTestKit(c, s.store)
 	s.tk.MustExec("use " + s.schemaName)
 
@@ -4657,6 +4662,10 @@ func (s *testDBSuite) TestCanceledJobTakeTime(c *C) {
 }
 
 func (s *testDBSuite) TestAlterShardRowIDBits(c *C) {
+	c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
+	defer func() {
+		c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
+	}()
 	s.tk = testkit.NewTestKit(c, s.store)
 	tk := s.tk
 

executor/admin_test.go

@@ -17,6 +17,7 @@ import (
 	"fmt"
 
 	. "github.com/pingcap/check"
+	"github.com/pingcap/failpoint"
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/tidb/executor"
 	"github.com/pingcap/tidb/meta/autoid"
@@ -536,6 +537,10 @@ func (s *testSuite) TestAdminCheckPrimaryIndex(c *C) {
 }
 
 func (s *testSuite) TestAdminShowNextID(c *C) {
+	c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
+	defer func() {
+		c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
+	}()
 	step := int64(10)
 	autoIDStep := autoid.GetStep()
 	autoid.SetStep(step)

executor/ddl_test.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	. "github.com/pingcap/check"
+	"github.com/pingcap/failpoint"
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/parser/mysql"
 	"github.com/pingcap/parser/terror"
@@ -260,6 +261,10 @@ func (s *testSuite) TestDefaultDBAfterDropCurDB(c *C) {
 }
 
 func (s *testSuite) TestRenameTable(c *C) {
+	c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
+	defer func() {
+		c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
+	}()
 	tk := testkit.NewTestKit(c, s.store)
 
 	tk.MustExec("create database rename1")

meta/autoid/autoid.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/cznic/mathutil"
 	"github.com/pingcap/errors"
+	"github.com/pingcap/failpoint"
 	"github.com/pingcap/parser/terror"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/meta"
@@ -30,6 +31,12 @@ import (
 	"go.uber.org/zap"
 )
 
+const (
+	minStep            = 1000
+	maxStep            = 2000000
+	defaultConsumeTime = 10 * time.Second
+)
+
 // Test needs to change it, so it's a variable.
 var step = int64(30000)
 
@@ -59,8 +66,10 @@ type allocator struct {
 	end   int64
 	store kv.Storage
 	// dbID is current database's ID.
-	dbID       int64
-	isUnsigned bool
+	dbID          int64
+	isUnsigned    bool
+	lastAllocTime time.Time
+	step          int64
 }
 
 // GetStep is only used by tests
@@ -121,7 +130,7 @@ func (alloc *allocator) rebase4Unsigned(tableID int64, requiredBase uint64, allo
 		uCurrentEnd := uint64(currentEnd)
 		if allocIDs {
 			newBase = mathutil.MaxUint64(uCurrentEnd, requiredBase)
-			newEnd = mathutil.MinUint64(math.MaxUint64-uint64(step), newBase) + uint64(step)
+			newEnd = mathutil.MinUint64(math.MaxUint64-uint64(alloc.step), newBase) + uint64(alloc.step)
 		} else {
 			if uCurrentEnd >= requiredBase {
 				newBase = uCurrentEnd
@@ -166,7 +175,7 @@ func (alloc *allocator) rebase4Signed(tableID, requiredBase int64, allocIDs bool
 		}
 		if allocIDs {
 			newBase = mathutil.MaxInt64(currentEnd, requiredBase)
-			newEnd = mathutil.MinInt64(math.MaxInt64-step, newBase) + step
+			newEnd = mathutil.MinInt64(math.MaxInt64-alloc.step, newBase) + alloc.step
 		} else {
 			if currentEnd >= requiredBase {
 				newBase = currentEnd
@@ -212,21 +221,24 @@ func (alloc *allocator) alloc4Unsigned(tableID int64) (int64, error) {
 	if alloc.base == alloc.end { // step
 		var newBase, newEnd int64
 		startTime := time.Now()
+		consumeDur := startTime.Sub(alloc.lastAllocTime)
+		alloc.step = NextStep(alloc.step, consumeDur)
 		err := kv.RunInNewTxn(alloc.store, true, func(txn kv.Transaction) error {
 			m := meta.NewMeta(txn)
 			var err1 error
 			newBase, err1 = m.GetAutoTableID(alloc.dbID, tableID)
 			if err1 != nil {
 				return errors.Trace(err1)
 			}
-			tmpStep := int64(mathutil.MinUint64(math.MaxUint64-uint64(newBase), uint64(step)))
+			tmpStep := int64(mathutil.MinUint64(math.MaxUint64-uint64(newBase), uint64(alloc.step)))
 			newEnd, err1 = m.GenAutoTableID(alloc.dbID, tableID, tmpStep)
 			return err1
 		})
 		metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDAlloc, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
 		if err != nil {
 			return 0, err
 		}
+		alloc.lastAllocTime = time.Now()
 		if uint64(newBase) == math.MaxUint64 {
 			return 0, ErrAutoincReadFailed
 		}
@@ -245,21 +257,24 @@ func (alloc *allocator) alloc4Signed(tableID int64) (int64, error) {
 	if alloc.base == alloc.end { // step
 		var newBase, newEnd int64
 		startTime := time.Now()
+		consumeDur := startTime.Sub(alloc.lastAllocTime)
+		alloc.step = NextStep(alloc.step, consumeDur)
 		err := kv.RunInNewTxn(alloc.store, true, func(txn kv.Transaction) error {
 			m := meta.NewMeta(txn)
 			var err1 error
 			newBase, err1 = m.GetAutoTableID(alloc.dbID, tableID)
 			if err1 != nil {
 				return errors.Trace(err1)
 			}
-			tmpStep := mathutil.MinInt64(math.MaxInt64-newBase, step)
+			tmpStep := mathutil.MinInt64(math.MaxInt64-newBase, alloc.step)
 			newEnd, err1 = m.GenAutoTableID(alloc.dbID, tableID, tmpStep)
 			return err1
 		})
 		metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDAlloc, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
 		if err != nil {
 			return 0, err
 		}
+		alloc.lastAllocTime = time.Now()
 		if newBase == math.MaxInt64 {
 			return 0, ErrAutoincReadFailed
 		}
@@ -287,6 +302,35 @@ func (alloc *allocator) Alloc(tableID int64) (int64, error) {
 	return alloc.alloc4Signed(tableID)
 }
 
+// NextStep return new auto id step according to previous step and consuming time.
+func NextStep(curStep int64, consumeDur time.Duration) int64 {
+	failpoint.Inject("mockAutoIDChange", func(val failpoint.Value) {
+		if val.(bool) {
+			failpoint.Return(step)
+		}
+	})
+
+	consumeRate := defaultConsumeTime.Seconds() / consumeDur.Seconds()
+	res := int64(float64(curStep) * consumeRate)
+	if res < minStep {
+		return minStep
+	} else if res > maxStep {
+		return maxStep
+	}
+	return res
+}
+
+// NewAllocator returns a new auto increment id generator on the store.
+func NewAllocator(store kv.Storage, dbID int64, isUnsigned bool) Allocator {
+	return &allocator{
+		store:         store,
+		dbID:          dbID,
+		isUnsigned:    isUnsigned,
+		step:          step,
+		lastAllocTime: time.Now(),
+	}
+}
+
 var (
 	memID     int64
 	memIDLock sync.Mutex
@@ -340,15 +384,6 @@ func (alloc *memoryAllocator) Alloc(tableID int64) (int64, error) {
 	return alloc.base, nil
 }
 
-// NewAllocator returns a new auto increment id generator on the store.
-func NewAllocator(store kv.Storage, dbID int64, isUnsigned bool) Allocator {
-	return &allocator{
-		store:      store,
-		dbID:       dbID,
-		isUnsigned: isUnsigned,
-	}
-}
-
 // NewMemoryAllocator returns a new auto increment id generator in memory.
 func NewMemoryAllocator(dbID int64) Allocator {
 	return &memoryAllocator{

meta/autoid/autoid_test.go

@@ -21,6 +21,7 @@ import (
 
 	. "github.com/pingcap/check"
 	"github.com/pingcap/errors"
+	"github.com/pingcap/failpoint"
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/meta"
@@ -39,6 +40,11 @@ type testSuite struct {
 }
 
 func (*testSuite) TestT(c *C) {
+	c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
+	defer func() {
+		c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
+	}()
+
 	store, err := mockstore.NewMockTikvStore()
 	c.Assert(err, IsNil)
 	defer store.Close()
@@ -130,6 +136,11 @@ func (*testSuite) TestT(c *C) {
 }
 
 func (*testSuite) TestUnsignedAutoid(c *C) {
+	c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
+	defer func() {
+		c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
+	}()
+
 	store, err := mockstore.NewMockTikvStore()
 	c.Assert(err, IsNil)
 	defer store.Close()
@@ -315,3 +326,13 @@ func (*testSuite) TestRollbackAlloc(c *C) {
 	c.Assert(alloc.Base(), Equals, int64(0))
 	c.Assert(alloc.End(), Equals, int64(0))
 }
+
+// TestNextStep tests generate next auto id step.
+func (*testSuite) TestNextStep(c *C) {
+	nextStep := autoid.NextStep(2000000, 1*time.Nanosecond)
+	c.Assert(nextStep, Equals, int64(2000000))
+	nextStep = autoid.NextStep(678910, 10*time.Second)
+	c.Assert(nextStep, Equals, int64(678910))
+	nextStep = autoid.NextStep(50000, 10*time.Minute)
+	c.Assert(nextStep, Equals, int64(1000))
+}