lightning: evaluate all generated columns even if they are virtual (#…

…1407) (#1417)

pkg/lightning/backend/kv/sql2kv.go

@@ -121,17 +121,17 @@ func autoRandomIncrementBits(col *table.Column, randomBits int) int {
 }
 
 // collectGeneratedColumns collects all expressions required to evaluate the
-// results of all stored generated columns. The returning slice is in evaluation
-// order.
+// results of all generated columns. The returning slice is in evaluation order.
 func collectGeneratedColumns(se *session, meta *model.TableInfo, cols []*table.Column) ([]genCol, error) {
-	maxGenColOffset := -1
+	hasGenCol := false
 	for _, col := range cols {
-		if col.GeneratedStored && col.Offset > maxGenColOffset {
-			maxGenColOffset = col.Offset
+		if col.GeneratedExpr != nil {
+			hasGenCol = true
+			break
 		}
 	}
 
-	if maxGenColOffset < 0 {
+	if !hasGenCol {
 		return nil, nil
 	}
 
@@ -166,7 +166,7 @@ func collectGeneratedColumns(se *session, meta *model.TableInfo, cols []*table.C
 	// for simplicity we just evaluate all generated columns (virtual or not) before the last stored one.
 	var genCols []genCol
 	for i, col := range cols {
-		if col.GeneratedExpr != nil && col.Offset <= maxGenColOffset {
+		if col.GeneratedExpr != nil {
 			expr, err := expression.RewriteAstExpr(se, col.GeneratedExpr, schema, names)
 			if err != nil {
 				return nil, err

tests/config/tidb.toml

@@ -11,3 +11,6 @@ ssl-key = "/tmp/backup_restore_test/certs/tidb.key"
 cluster-ssl-ca = "/tmp/backup_restore_test/certs/ca.pem"
 cluster-ssl-cert = "/tmp/backup_restore_test/certs/tidb.pem"
 cluster-ssl-key = "/tmp/backup_restore_test/certs/tidb.key"
+
+[experimental]
+allow-expression-index = true

tests/lightning_generated_columns/data/gencol.expr_index-schema.sql

@@ -0,0 +1,11 @@
+-- https://github.com/pingcap/br/issues/1404
+-- expression indices just use a hidden virtual generated column behind the scene.
+
+CREATE TABLE `expr_index` (
+    `id` int(11) NOT NULL AUTO_INCREMENT,
+    `a` varchar(20) DEFAULT NULL,
+    `b` varchar(20) DEFAULT NULL,
+    PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,
+    KEY `idx_a` (`a`),
+    KEY `idx_lower_b` ((lower(`b`)))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=90003;

tests/lightning_generated_columns/data/gencol.expr_index.0.sql

@@ -0,0 +1 @@
+insert into expr_index (id, a, b) values (1, 'aaa', 'bbb'), (2, 'ABC', 'CDSFDS');

tests/lightning_generated_columns/data/gencol.nested-schema.sql

@@ -3,5 +3,6 @@ create table nested (
     b int as (a + 1) virtual unique,
     c int as (b + 1) stored unique,
     d int as (c + 1) virtual unique,
-    e int as (d + 1) stored unique
+    e int as (d + 1) stored unique,
+    f int as (e + 1) virtual unique
 );

tests/lightning_generated_columns/data/gencol.virtual_only-schema.sql

@@ -0,0 +1,5 @@
+create table virtual_only (
+    id int primary key,
+    id_plus_1 int as (id + 1) virtual,
+    id_plus_2 int as (id + 2) virtual
+);

tests/lightning_generated_columns/data/gencol.virtual_only.0.sql

@@ -0,0 +1 @@
+insert into virtual_only (id) values (30), (40);

tests/lightning_generated_columns/run.sh

@@ -43,12 +43,21 @@ for BACKEND in 'local' 'tidb' 'importer'; do
 
     run_lightning --backend $BACKEND
 
+    run_sql 'ADMIN CHECK TABLE gencol.nested'
     run_sql 'SELECT * FROM gencol.nested WHERE a = 100'
     check_contains 'a: 100'
     check_contains 'b: 101'
     check_contains 'c: 102'
     check_contains 'd: 103'
     check_contains 'e: 104'
+    check_contains 'f: 105'
+    run_sql 'SELECT * FROM gencol.nested WHERE f = 1005'
+    check_contains 'a: 1000'
+    check_contains 'b: 1001'
+    check_contains 'c: 1002'
+    check_contains 'd: 1003'
+    check_contains 'e: 1004'
+    check_contains 'f: 1005'
 
     run_sql 'SELECT * FROM gencol.various_types' --binary-as-hex
     check_contains 'int64: 3'
@@ -68,4 +77,18 @@ for BACKEND in 'local' 'tidb' 'importer'; do
     # FIXME: test below disabled due to pingcap/tidb#21510
     # check_contains 'week: 6'
     check_contains 'tz: 1969-12-31 16:00:01'
+
+    run_sql 'ADMIN CHECK TABLE gencol.virtual_only'
+    run_sql 'SELECT * FROM gencol.virtual_only WHERE id = 30'
+    check_contains 'id_plus_1: 31'
+    check_contains 'id_plus_2: 32'
+    run_sql 'SELECT * FROM gencol.virtual_only WHERE id_plus_2 = 42'
+    check_contains 'id: 40'
+    check_contains 'id_plus_1: 41'
+
+    run_sql 'ADMIN CHECK TABLE gencol.expr_index'
+    run_sql 'SELECT /*+ use_index(gencol.expr_index, idx_lower_b) */ * FROM gencol.expr_index WHERE lower(b) = "cdsfds"'
+    check_contains 'id: 2'
+    check_contains 'a: ABC'
+    check_contains 'b: CDSFDS'
 done