executor, session: refine insert unsigned bigint autoIncreID (#8181) (#…

…9339)

ddl/column_change_test.go

@@ -356,7 +356,7 @@ func getCurrentTable(d *ddl, schemaID, tableID int64) (table.Table, error) {
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
-	alloc := autoid.NewAllocator(d.store, schemaID)
+	alloc := autoid.NewAllocator(d.store, schemaID, false)
 	tbl, err := table.TableFromMeta(alloc, tblInfo)
 	if err != nil {
 		return nil, errors.Trace(err)

ddl/ddl_api.go

@@ -1069,7 +1069,7 @@ func checkCharsetAndCollation(cs string, co string) error {
 // handleAutoIncID handles auto_increment option in DDL. It creates a ID counter for the table and initiates the counter to a proper value.
 // For example if the option sets auto_increment to 10. The counter will be set to 9. So the next allocated ID will be 10.
 func (d *ddl) handleAutoIncID(tbInfo *model.TableInfo, schemaID int64) error {
-	alloc := autoid.NewAllocator(d.store, tbInfo.GetDBID(schemaID))
+	alloc := autoid.NewAllocator(d.store, tbInfo.GetDBID(schemaID), tbInfo.IsAutoIncColUnsigned())
 	tbInfo.State = model.StatePublic
 	tb, err := table.TableFromMeta(alloc, tbInfo)
 	if err != nil {

ddl/table.go

@@ -154,7 +154,7 @@ func splitTableRegion(store kv.Storage, tableID int64) {
 }
 
 func getTable(store kv.Storage, schemaID int64, tblInfo *model.TableInfo) (table.Table, error) {
-	alloc := autoid.NewAllocator(store, tblInfo.GetDBID(schemaID))
+	alloc := autoid.NewAllocator(store, tblInfo.GetDBID(schemaID), tblInfo.IsAutoIncColUnsigned())
 	tbl, err := table.TableFromMeta(alloc, tblInfo)
 	return tbl, errors.Trace(err)
 }

ddl/table_test.go

@@ -177,7 +177,7 @@ func testGetTableWithError(d *ddl, schemaID, tableID int64) (table.Table, error)
 	if tblInfo == nil {
 		return nil, errors.New("table not found")
 	}
-	alloc := autoid.NewAllocator(d.store, schemaID)
+	alloc := autoid.NewAllocator(d.store, schemaID, false)
 	tbl, err := table.TableFromMeta(alloc, tblInfo)
 	if err != nil {
 		return nil, errors.Trace(err)

executor/executor_test.go

@@ -2848,7 +2848,7 @@ func (s *testSuite) TestCheckIndex(c *C) {
 	c.Assert(err, IsNil)
 	tbInfo := tbl.Meta()
 
-	alloc := autoid.NewAllocator(s.store, dbInfo.ID)
+	alloc := autoid.NewAllocator(s.store, dbInfo.ID, false)
 	tb, err := tables.TableFromMeta(alloc, tbInfo)
 	c.Assert(err, IsNil)
 

executor/insert_common.go

@@ -458,10 +458,12 @@ func (e *InsertValues) adjustAutoIncrementDatum(d types.Datum, hasValue bool, c
 		d.SetNull()
 	}
 	if !d.IsNull() {
-		recordID, err = d.ToInt64(e.ctx.GetSessionVars().StmtCtx)
-		if e.filterErr(err) != nil {
-			return types.Datum{}, errors.Trace(err)
+		sc := e.ctx.GetSessionVars().StmtCtx
+		datum, err1 := d.ConvertTo(sc, &c.FieldType)
+		if e.filterErr(err1) != nil {
+			return types.Datum{}, err1
 		}
+		recordID = datum.GetInt64()
 	}
 	// Use the value if it's not null and not 0.
 	if recordID != 0 {

go.mod

@@ -48,7 +48,7 @@ require (
 	github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3
 	github.com/pingcap/goleveldb v0.0.0-20171020084629-8d44bfdf1030
 	github.com/pingcap/kvproto v0.0.0-20181109035735-8e3f33ac4929
-	github.com/pingcap/parser v0.0.0-20190118033454-a52e5bde3bd2
+	github.com/pingcap/parser v0.0.0-20190218023123-90a796aef0c5
 	github.com/pingcap/pd v2.1.0-rc.4+incompatible
 	github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible
 	github.com/pingcap/tipb v0.0.0-20180910045846-371b48b15d93

go.sum

@@ -94,6 +94,10 @@ github.com/pingcap/kvproto v0.0.0-20181109035735-8e3f33ac4929 h1:NAq95+VGsS2G7Sj
 github.com/pingcap/kvproto v0.0.0-20181109035735-8e3f33ac4929/go.mod h1:0gwbe1F2iBIjuQ9AH0DbQhL+Dpr5GofU8fgYyXk+ykk=
 github.com/pingcap/parser v0.0.0-20190118033454-a52e5bde3bd2 h1:7YGx4hF6M0nlFJVZrLF3EbMRI+XOizL+9aB8Txe745U=
 github.com/pingcap/parser v0.0.0-20190118033454-a52e5bde3bd2/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
+github.com/pingcap/parser v0.0.0-20190214121452-6d10a0b75f3e h1:tfl2np1PRmAQQoHjeyzy1qYY62RD6camcHf6wVjQ19M=
+github.com/pingcap/parser v0.0.0-20190214121452-6d10a0b75f3e/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
+github.com/pingcap/parser v0.0.0-20190218023123-90a796aef0c5 h1:0sHmsTSdOkatWgUbz1bWDz57z4c1drsR2aAXpEgK6Co=
+github.com/pingcap/parser v0.0.0-20190218023123-90a796aef0c5/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
 github.com/pingcap/pd v2.1.0-rc.4+incompatible h1:/buwGk04aHO5odk/+O8ZOXGs4qkUjYTJ2UpCJXna8NE=
 github.com/pingcap/pd v2.1.0-rc.4+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E=
 github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible h1:e9Gi/LP9181HT3gBfSOeSBA+5JfemuE4aEAhqNgoE4k=

infoschema/builder.go

@@ -173,7 +173,7 @@ func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID i
 	}
 	if alloc == nil {
 		schemaID := dbInfo.ID
-		alloc = autoid.NewAllocator(b.handle.store, tblInfo.GetDBID(schemaID))
+		alloc = autoid.NewAllocator(b.handle.store, tblInfo.GetDBID(schemaID), tblInfo.IsAutoIncColUnsigned())
 	}
 	tbl, err := tables.TableFromMeta(alloc, tblInfo)
 	if err != nil {
@@ -276,7 +276,7 @@ func (b *Builder) createSchemaTablesForDB(di *model.DBInfo) error {
 	b.is.schemaMap[di.Name.L] = schTbls
 	for _, t := range di.Tables {
 		schemaID := di.ID
-		alloc := autoid.NewAllocator(b.handle.store, t.GetDBID(schemaID))
+		alloc := autoid.NewAllocator(b.handle.store, t.GetDBID(schemaID), t.IsAutoIncColUnsigned())
 		var tbl table.Table
 		tbl, err := tables.TableFromMeta(alloc, t)
 		if err != nil {

meta/autoid/autoid.go

@@ -57,7 +57,8 @@ type allocator struct {
 	end   int64
 	store kv.Storage
 	// dbID is current database's ID.
-	dbID int64
+	dbID       int64
+	isUnsigned bool
 }
 
 // GetStep is only used by tests
@@ -91,25 +92,65 @@ func (alloc *allocator) NextGlobalAutoID(tableID int64) (int64, error) {
 		return errors.Trace(err1)
 	})
 	metrics.AutoIDHistogram.WithLabelValues(metrics.GlobalAutoID, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
-	return autoID + 1, errors.Trace(err)
+	if alloc.isUnsigned {
+		return int64(uint64(autoID) + 1), err
+	}
+	return autoID + 1, err
 }
 
-// Rebase implements autoid.Allocator Rebase interface.
-// The requiredBase is the minimum base value after Rebase.
-// The real base may be greater than the required base.
-func (alloc *allocator) Rebase(tableID, requiredBase int64, allocIDs bool) error {
-	if tableID == 0 {
-		return errInvalidTableID.GenWithStack("Invalid tableID")
+func (alloc *allocator) rebase4Unsigned(tableID int64, requiredBase uint64, allocIDs bool) error {
+	// Satisfied by alloc.base, nothing to do.
+	if requiredBase <= uint64(alloc.base) {
+		return nil
+	}
+	// Satisfied by alloc.end, need to update alloc.base.
+	if requiredBase <= uint64(alloc.end) {
+		alloc.base = int64(requiredBase)
+		return nil
+	}
+	var newBase, newEnd uint64
+	startTime := time.Now()
+	err := kv.RunInNewTxn(alloc.store, true, func(txn kv.Transaction) error {
+		m := meta.NewMeta(txn)
+		currentEnd, err1 := m.GetAutoTableID(alloc.dbID, tableID)
+		if err1 != nil {
+			return errors.Trace(err1)
+		}
+		uCurrentEnd := uint64(currentEnd)
+		if allocIDs {
+			newBase = mathutil.MaxUint64(uCurrentEnd, requiredBase)
+			newEnd = mathutil.MinUint64(math.MaxUint64-uint64(step), newBase) + uint64(step)
+		} else {
+			if uCurrentEnd >= requiredBase {
+				newBase = uCurrentEnd
+				newEnd = uCurrentEnd
+				// Required base satisfied, we don't need to update KV.
+				return nil
+			}
+			// If we don't want to allocate IDs, for example when creating a table with a given base value,
+			// We need to make sure when other TiDB server allocates ID for the first time, requiredBase + 1
+			// will be allocated, so we need to increase the end to exactly the requiredBase.
+			newBase = requiredBase
+			newEnd = requiredBase
+		}
+		_, err1 = m.GenAutoTableID(alloc.dbID, tableID, int64(newEnd-uCurrentEnd))
+		return err1
+	})
+	metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDRebase, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
+	if err != nil {
+		return err
 	}
+	alloc.base, alloc.end = int64(newBase), int64(newEnd)
+	return nil
+}
 
-	alloc.mu.Lock()
-	defer alloc.mu.Unlock()
+func (alloc *allocator) rebase4Signed(tableID, requiredBase int64, allocIDs bool) error {
+	// Satisfied by alloc.base, nothing to do.
 	if requiredBase <= alloc.base {
-		// Satisfied by alloc.base, nothing to do.
 		return nil
 	}
+	// Satisfied by alloc.end, need to update alloc.base.
 	if requiredBase <= alloc.end {
-		// Satisfied by alloc.end, need to updata alloc.base.
 		alloc.base = requiredBase
 		return nil
 	}
@@ -123,7 +164,7 @@ func (alloc *allocator) Rebase(tableID, requiredBase int64, allocIDs bool) error
 		}
 		if allocIDs {
 			newBase = mathutil.MaxInt64(currentEnd, requiredBase)
-			newEnd = newBase + step
+			newEnd = mathutil.MinInt64(math.MaxInt64-step, newBase) + step
 		} else {
 			if currentEnd >= requiredBase {
 				newBase = currentEnd
@@ -138,23 +179,34 @@ func (alloc *allocator) Rebase(tableID, requiredBase int64, allocIDs bool) error
 			newEnd = requiredBase
 		}
 		_, err1 = m.GenAutoTableID(alloc.dbID, tableID, newEnd-currentEnd)
-		return errors.Trace(err1)
+		return err1
 	})
 	metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDRebase, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
 	if err != nil {
-		return errors.Trace(err)
+		return err
 	}
 	alloc.base, alloc.end = newBase, newEnd
 	return nil
 }
 
-// Alloc implements autoid.Allocator Alloc interface.
-func (alloc *allocator) Alloc(tableID int64) (int64, error) {
+// Rebase implements autoid.Allocator Rebase interface.
+// The requiredBase is the minimum base value after Rebase.
+// The real base may be greater than the required base.
+func (alloc *allocator) Rebase(tableID, requiredBase int64, allocIDs bool) error {
 	if tableID == 0 {
-		return 0, errInvalidTableID.GenWithStack("Invalid tableID")
+		return errInvalidTableID.GenWithStack("Invalid tableID")
 	}
+
 	alloc.mu.Lock()
 	defer alloc.mu.Unlock()
+
+	if alloc.isUnsigned {
+		return alloc.rebase4Unsigned(tableID, uint64(requiredBase), allocIDs)
+	}
+	return alloc.rebase4Signed(tableID, requiredBase, allocIDs)
+}
+
+func (alloc *allocator) alloc4Unsigned(tableID int64) (int64, error) {
 	if alloc.base == alloc.end { // step
 		var newBase, newEnd int64
 		startTime := time.Now()
@@ -165,15 +217,46 @@ func (alloc *allocator) Alloc(tableID int64) (int64, error) {
 			if err1 != nil {
 				return errors.Trace(err1)
 			}
-			newEnd, err1 = m.GenAutoTableID(alloc.dbID, tableID, step)
+			tmpStep := int64(mathutil.MinUint64(math.MaxUint64-uint64(newBase), uint64(step)))
+			newEnd, err1 = m.GenAutoTableID(alloc.dbID, tableID, tmpStep)
+			return err1
+		})
+		metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDAlloc, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
+		if err != nil {
+			return 0, err
+		}
+		if uint64(newBase) == math.MaxUint64 {
+			return 0, ErrAutoincReadFailed
+		}
+		alloc.base, alloc.end = newBase, newEnd
+	}
+
+	alloc.base = int64(uint64(alloc.base) + 1)
+	log.Debugf("[kv] Alloc id %d, table ID:%d, from %p, database ID:%d", uint64(alloc.base), tableID, alloc, alloc.dbID)
+	return alloc.base, nil
+}
+
+func (alloc *allocator) alloc4Signed(tableID int64) (int64, error) {
+	if alloc.base == alloc.end { // step
+		var newBase, newEnd int64
+		startTime := time.Now()
+		err := kv.RunInNewTxn(alloc.store, true, func(txn kv.Transaction) error {
+			m := meta.NewMeta(txn)
+			var err1 error
+			newBase, err1 = m.GetAutoTableID(alloc.dbID, tableID)
 			if err1 != nil {
 				return errors.Trace(err1)
 			}
-			return nil
+			tmpStep := mathutil.MinInt64(math.MaxInt64-newBase, step)
+			newEnd, err1 = m.GenAutoTableID(alloc.dbID, tableID, tmpStep)
+			return err1
 		})
 		metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDAlloc, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
 		if err != nil {
-			return 0, errors.Trace(err)
+			return 0, err
+		}
+		if newBase == math.MaxInt64 {
+			return 0, ErrAutoincReadFailed
 		}
 		alloc.base, alloc.end = newBase, newEnd
 	}
@@ -183,6 +266,19 @@ func (alloc *allocator) Alloc(tableID int64) (int64, error) {
 	return alloc.base, nil
 }
 
+// Alloc implements autoid.Allocator Alloc interface.
+func (alloc *allocator) Alloc(tableID int64) (int64, error) {
+	if tableID == 0 {
+		return 0, errInvalidTableID.GenWithStack("Invalid tableID")
+	}
+	alloc.mu.Lock()
+	defer alloc.mu.Unlock()
+	if alloc.isUnsigned {
+		return alloc.alloc4Unsigned(tableID)
+	}
+	return alloc.alloc4Signed(tableID)
+}
+
 var (
 	memID     int64
 	memIDLock sync.Mutex
@@ -237,10 +333,11 @@ func (alloc *memoryAllocator) Alloc(tableID int64) (int64, error) {
 }
 
 // NewAllocator returns a new auto increment id generator on the store.
-func NewAllocator(store kv.Storage, dbID int64) Allocator {
+func NewAllocator(store kv.Storage, dbID int64, isUnsigned bool) Allocator {
 	return &allocator{
-		store: store,
-		dbID:  dbID,
+		store:      store,
+		dbID:       dbID,
+		isUnsigned: isUnsigned,
 	}
 }