From 4575c8f5d073c8b1c01b3cb71cde884c7b8aacea Mon Sep 17 00:00:00 2001 From: kennytm Date: Wed, 24 Apr 2019 06:34:37 +0800 Subject: [PATCH 1/2] kv: fix handling of column default values * if the column is AUTO_INCREMENT, fill in with row_id (assume it is missing for the entire table instead of just a few values) * if the column has DEFAULT, fill in that value --- lightning/kv/sql2kv.go | 7 ++-- tests/default-columns/config.toml | 18 ++++++++++ .../data/defcol-schema-create.sql | 1 + .../default-columns/data/defcol.t-schema.sql | 5 +++ tests/default-columns/data/defcol.t.1.sql | 1 + tests/default-columns/data/defcol.t.2.sql | 1 + .../default-columns/data/defcol.u-schema.sql | 4 +++ tests/default-columns/data/defcol.u.1.sql | 1 + tests/default-columns/run.sh | 33 +++++++++++++++++++ 9 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 tests/default-columns/config.toml create mode 100644 tests/default-columns/data/defcol-schema-create.sql create mode 100644 tests/default-columns/data/defcol.t-schema.sql create mode 100644 tests/default-columns/data/defcol.t.1.sql create mode 100644 tests/default-columns/data/defcol.t.2.sql create mode 100644 tests/default-columns/data/defcol.u-schema.sql create mode 100644 tests/default-columns/data/defcol.u.1.sql create mode 100755 tests/default-columns/run.sh diff --git a/lightning/kv/sql2kv.go b/lightning/kv/sql2kv.go index b73aa0674..58517f26c 100644 --- a/lightning/kv/sql2kv.go +++ b/lightning/kv/sql2kv.go @@ -69,13 +69,16 @@ func (kvcodec *TableKVEncoder) Encode( } for i, col := range cols { - if j := columnPermutation[i]; j >= 0 { + if j := columnPermutation[i]; j >= 0 && j < len(row) { value, err = table.CastValue(kvcodec.se, row[j], col.ToInfo()) if err == nil { value, err = col.HandleBadNull(value, kvcodec.se.vars.StmtCtx) } + } else if mysql.HasAutoIncrementFlag(col.Flag) { + // we still need a conversion, e.g. to catch overflow with a TINYINT column. + value, err = table.CastValue(kvcodec.se, types.NewIntDatum(rowID), col.ToInfo()) } else { - value, err = table.GetColOriginDefaultValue(kvcodec.se, col.ToInfo()) + value, err = table.GetColDefaultValue(kvcodec.se, col.ToInfo()) } if err != nil { return nil, errors.Trace(err) diff --git a/tests/default-columns/config.toml b/tests/default-columns/config.toml new file mode 100644 index 000000000..bb9f5b054 --- /dev/null +++ b/tests/default-columns/config.toml @@ -0,0 +1,18 @@ +[lightning] +check-requirements = false +file = "/tmp/lightning_test_result/lightning.log" +level = "debug" + +[tikv-importer] +addr = "127.0.0.1:8808" + +[mydumper] +data-source-dir = "tests/default-columns/data" + +[tidb] +host = "127.0.0.1" +port = 4000 +user = "root" +status-port = 10080 +pd-addr = "127.0.0.1:2379" +log-level = "error" diff --git a/tests/default-columns/data/defcol-schema-create.sql b/tests/default-columns/data/defcol-schema-create.sql new file mode 100644 index 000000000..02bc0983c --- /dev/null +++ b/tests/default-columns/data/defcol-schema-create.sql @@ -0,0 +1 @@ +CREATE DATABASE defcol; diff --git a/tests/default-columns/data/defcol.t-schema.sql b/tests/default-columns/data/defcol.t-schema.sql new file mode 100644 index 000000000..ba6fa6911 --- /dev/null +++ b/tests/default-columns/data/defcol.t-schema.sql @@ -0,0 +1,5 @@ +CREATE TABLE t ( + pk INT PRIMARY KEY AUTO_INCREMENT, + x INT NULL, + y INT NOT NULL DEFAULT 123 +); diff --git a/tests/default-columns/data/defcol.t.1.sql b/tests/default-columns/data/defcol.t.1.sql new file mode 100644 index 000000000..aef45b86c --- /dev/null +++ b/tests/default-columns/data/defcol.t.1.sql @@ -0,0 +1 @@ +INSERT INTO t () VALUES (), (), (), (), (), (); diff --git a/tests/default-columns/data/defcol.t.2.sql b/tests/default-columns/data/defcol.t.2.sql new file mode 100644 index 000000000..253dfdd8d --- /dev/null +++ b/tests/default-columns/data/defcol.t.2.sql @@ -0,0 +1 @@ +INSERT INTO t VALUES (), (), (); diff --git a/tests/default-columns/data/defcol.u-schema.sql b/tests/default-columns/data/defcol.u-schema.sql new file mode 100644 index 000000000..c85dc9826 --- /dev/null +++ b/tests/default-columns/data/defcol.u-schema.sql @@ -0,0 +1,4 @@ +CREATE TABLE u ( + xx INT UNIQUE AUTO_INCREMENT, + yy INT PRIMARY KEY +); diff --git a/tests/default-columns/data/defcol.u.1.sql b/tests/default-columns/data/defcol.u.1.sql new file mode 100644 index 000000000..19c7c31cb --- /dev/null +++ b/tests/default-columns/data/defcol.u.1.sql @@ -0,0 +1 @@ +INSERT INTO u (yy) VALUES (40), (60); diff --git a/tests/default-columns/run.sh b/tests/default-columns/run.sh new file mode 100755 index 000000000..8b98f9c5f --- /dev/null +++ b/tests/default-columns/run.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +run_sql 'DROP DATABASE IF EXISTS defcol' + +run_lightning + +run_sql 'SELECT min(pk), count(pk) FROM defcol.t' +check_contains 'min(pk): 1' +check_contains 'count(pk): 9' + +run_sql 'SELECT pk FROM defcol.t WHERE x IS NOT NULL OR y <> 123' +check_not_contains 'pk:' + +run_sql 'SELECT xx FROM defcol.u WHERE yy = 40' +check_contains 'xx: 1' + +run_sql 'SELECT xx FROM defcol.u WHERE yy = 60' +check_contains 'xx: 2' From c04b0d07c907969b874d6be37f99b8cbeb56fcf7 Mon Sep 17 00:00:00 2001 From: kennytm Date: Thu, 25 Apr 2019 07:58:01 +0800 Subject: [PATCH 2/2] tests: ensure DEFAULT CURRENT_TIMESTAMP works --- tests/default-columns/data/defcol.t-schema.sql | 3 ++- tests/default-columns/run.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/default-columns/data/defcol.t-schema.sql b/tests/default-columns/data/defcol.t-schema.sql index ba6fa6911..05f2ae676 100644 --- a/tests/default-columns/data/defcol.t-schema.sql +++ b/tests/default-columns/data/defcol.t-schema.sql @@ -1,5 +1,6 @@ CREATE TABLE t ( pk INT PRIMARY KEY AUTO_INCREMENT, x INT NULL, - y INT NOT NULL DEFAULT 123 + y INT NOT NULL DEFAULT 123, + z DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ); diff --git a/tests/default-columns/run.sh b/tests/default-columns/run.sh index 8b98f9c5f..7a7130bbb 100755 --- a/tests/default-columns/run.sh +++ b/tests/default-columns/run.sh @@ -23,7 +23,7 @@ run_sql 'SELECT min(pk), count(pk) FROM defcol.t' check_contains 'min(pk): 1' check_contains 'count(pk): 9' -run_sql 'SELECT pk FROM defcol.t WHERE x IS NOT NULL OR y <> 123' +run_sql 'SELECT pk FROM defcol.t WHERE x IS NOT NULL OR y <> 123 OR z IS NULL OR z NOT BETWEEN now() - INTERVAL 5 MINUTE AND now()' check_not_contains 'pk:' run_sql 'SELECT xx FROM defcol.u WHERE yy = 40'