[2.1] Cherry-pick #170 and #172 (#174)

* kv: fix handling of column default values (#170)

* kv: fix handling of column default values

* if the column is AUTO_INCREMENT, fill in with row_id (assume it is
  missing for the entire table instead of just a few values)
* if the column has DEFAULT, fill in that value

* tests: ensure DEFAULT CURRENT_TIMESTAMP works

* tests,restore: re-enable the exotic_filenames test (#172)

lightning/kv/sql2kv.go

@@ -69,13 +69,16 @@ func (kvcodec *TableKVEncoder) Encode(
 	}
 
 	for i, col := range cols {
-		if j := columnPermutation[i]; j >= 0 {
+		if j := columnPermutation[i]; j >= 0 && j < len(row) {
 			value, err = table.CastValue(kvcodec.se, row[j], col.ToInfo())
 			if err == nil {
 				value, err = col.HandleBadNull(value, kvcodec.se.vars.StmtCtx)
 			}
+		} else if mysql.HasAutoIncrementFlag(col.Flag) {
+			// we still need a conversion, e.g. to catch overflow with a TINYINT column.
+			value, err = table.CastValue(kvcodec.se, types.NewIntDatum(rowID), col.ToInfo())
 		} else {
-			value, err = table.GetColOriginDefaultValue(kvcodec.se, col.ToInfo())
+			value, err = table.GetColDefaultValue(kvcodec.se, col.ToInfo())
 		}
 		if err != nil {
 			return nil, errors.Trace(err)

lightning/restore/tidb.go

@@ -20,6 +20,7 @@ import (
 	"net/http"
 	"net/url"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/pingcap/errors"
@@ -73,12 +74,18 @@ func (timgr *TiDBManager) Close() {
 }
 
 func (timgr *TiDBManager) InitSchema(ctx context.Context, database string, tablesSchema map[string]string) error {
-	createDatabase := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", database)
+	var createDatabaseBuilder strings.Builder
+	createDatabaseBuilder.WriteString("CREATE DATABASE IF NOT EXISTS ")
+	common.WriteMySQLIdentifier(&createDatabaseBuilder, database)
+	createDatabase := createDatabaseBuilder.String()
 	err := common.ExecWithRetry(ctx, timgr.db, createDatabase, createDatabase)
 	if err != nil {
 		return errors.Trace(err)
 	}
-	useDB := fmt.Sprintf("USE `%s`", database)
+	var useDBBuilder strings.Builder
+	useDBBuilder.WriteString("USE ")
+	common.WriteMySQLIdentifier(&useDBBuilder, database)
+	useDB := useDBBuilder.String()
 	err = common.ExecWithRetry(ctx, timgr.db, useDB, useDB)
 	if err != nil {
 		return errors.Trace(err)

tests/default-columns/config.toml

@@ -0,0 +1,18 @@
+[lightning]
+check-requirements = false
+file = "/tmp/lightning_test_result/lightning.log"
+level = "debug"
+
+[tikv-importer]
+addr = "127.0.0.1:8808"
+
+[mydumper]
+data-source-dir = "tests/default-columns/data"
+
+[tidb]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+status-port = 10080
+pd-addr = "127.0.0.1:2379"
+log-level = "error"

tests/default-columns/data/defcol-schema-create.sql

@@ -0,0 +1 @@
+CREATE DATABASE defcol;

tests/default-columns/data/defcol.t-schema.sql

@@ -0,0 +1,6 @@
+CREATE TABLE t (
+    pk INT PRIMARY KEY AUTO_INCREMENT,
+    x INT NULL,
+    y INT NOT NULL DEFAULT 123,
+    z DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+);

tests/default-columns/data/defcol.t.1.sql

@@ -0,0 +1 @@
+INSERT INTO t () VALUES (), (), (), (), (), ();

tests/default-columns/data/defcol.t.2.sql

@@ -0,0 +1 @@
+INSERT INTO t VALUES (), (), ();

tests/default-columns/data/defcol.u-schema.sql

@@ -0,0 +1,4 @@
+CREATE TABLE u (
+    xx INT UNIQUE AUTO_INCREMENT,
+    yy INT PRIMARY KEY
+);

tests/default-columns/data/defcol.u.1.sql

@@ -0,0 +1 @@
+INSERT INTO u (yy) VALUES (40), (60);

tests/default-columns/run.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+# Copyright 2019 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+run_sql 'DROP DATABASE IF EXISTS defcol'
+
+run_lightning
+
+run_sql 'SELECT min(pk), count(pk) FROM defcol.t'
+check_contains 'min(pk): 1'
+check_contains 'count(pk): 9'
+
+run_sql 'SELECT pk FROM defcol.t WHERE x IS NOT NULL OR y <> 123 OR z IS NULL OR z NOT BETWEEN now() - INTERVAL 5 MINUTE AND now()'
+check_not_contains 'pk:'
+
+run_sql 'SELECT xx FROM defcol.u WHERE yy = 40'
+check_contains 'xx: 1'
+
+run_sql 'SELECT xx FROM defcol.u WHERE yy = 60'
+check_contains 'xx: 2'

tests/exotic_filenames/data/中文庫🥳-schema-create.sql

@@ -0,0 +1 @@
+create database `中文庫🥳`;

tests/exotic_filenames/data/中文庫🥳.中文表-schema.sql

@@ -0,0 +1 @@
+create table 中文表(a int primary key);

tests/exotic_filenames/data/中文庫🥳.中文表.sql

@@ -0,0 +1 @@
+insert into 中文表 values (2345);

tests/exotic_filenames/run.sh

@@ -16,11 +16,10 @@
 # Confirm the behavior for some exotic filenames
 # Do not enable until https://github.com/pingcap/tidb/pull/8302 is merged.
 
-exit 0
-
 set -eu
 
 run_sql 'DROP DATABASE IF EXISTS `x``f"n`;'
+run_sql 'DROP DATABASE IF EXISTS `中文庫🥳`;'
 run_lightning
 echo 'Import finished'
 
@@ -33,3 +32,6 @@ check_contains 'b > 80000: 1'
 run_sql 'SELECT _tidb_rowid > 80000, b > 80000 FROM `x``f"n`.`exotic``table````name` WHERE a = "gggggg"'
 check_contains '_tidb_rowid > 80000: 1'
 check_contains 'b > 80000: 1'
+
+run_sql 'SELECT * FROM `中文庫🥳`.中文表'
+check_contains 'a: 2345'