Skip to content

Commit

Permalink
Merge #62050
Browse files Browse the repository at this point in the history 
62050: sql: allow VIEW and SEQUENCEs for multi-region databases r=ajstorm a=otan

Refs #61382 -- missing materialized views.

Release note (bug fix): Allow VIEWs and SEQUENCEs in multi-region
databases. They will have REGIONAL BY TABLE set.



Co-authored-by: Oliver Tan <otan@cockroachlabs.com>
  • Loading branch information
@otan
craig[bot] and otan committed Mar 17, 2021
2 parents 5766c37 + f40aa61 commit 054169b
Show file tree
Hide file tree
Showing 13 changed files with 378 additions and 83 deletions.
193 changes: 125 additions & 68 deletions pkg/ccl/importccl/import_stmt_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6477,7 +6477,7 @@ func TestImportMultiRegion(t *testing.T) {
defer leaktest.AfterTest(t)()
defer log.Scope(t).Close(t)

baseDir := filepath.Join("testdata", "avro")
baseDir := filepath.Join("testdata")
_, sqlDB, cleanup := multiregionccltestutils.TestingCreateMultiRegionCluster(
t, 1 /* numServers */, base.TestingKnobs{}, &baseDir,
)
Expand All @@ -6493,95 +6493,152 @@ func TestImportMultiRegion(t *testing.T) {
_, err = sqlDB.Exec(`CREATE DATABASE multi_region PRIMARY REGION "us-east1"`)
require.NoError(t, err)

simpleOcf := fmt.Sprintf("nodelocal://0/%s", "simple.ocf")
simpleOcf := fmt.Sprintf("nodelocal://0/avro/%s", "simple.ocf")

// Table schemas for USING
tableSchemaMR := fmt.Sprintf("nodelocal://0/%s", "simple-schema-multi-region.sql")
tableSchemaMRRegionalByRow := fmt.Sprintf("nodelocal://0/%s",
tableSchemaMR := fmt.Sprintf("nodelocal://0/avro/%s", "simple-schema-multi-region.sql")
tableSchemaMRRegionalByRow := fmt.Sprintf("nodelocal://0/avro/%s",
"simple-schema-multi-region-regional-by-row.sql")

tests := []struct {
name string
db string
table string
sql string
create string
args []interface{}
errString string
viewsAndSequencesTestCases := []struct {
desc string
importSQL string
expected map[string]string
}{
{
name: "import-create-using-multi-region-to-non-multi-region-database",
db: "foo",
table: "simple",
sql: "IMPORT TABLE simple CREATE USING $1 AVRO DATA ($2)",
args: []interface{}{tableSchemaMR, simpleOcf},
errString: "cannot write descriptor for multi-region table",
},
{
name: "import-create-using-multi-region-regional-by-table-to-multi-region-database",
db: "multi_region",
table: "simple",
sql: "IMPORT TABLE simple CREATE USING $1 AVRO DATA ($2)",
args: []interface{}{tableSchemaMR, simpleOcf},
},
{
name: "import-create-using-multi-region-regional-by-row-to-multi-region-database",
db: "multi_region",
table: "simple",
sql: "IMPORT TABLE simple CREATE USING $1 AVRO DATA ($2)",
args: []interface{}{tableSchemaMRRegionalByRow, simpleOcf},
errString: "IMPORT to REGIONAL BY ROW table not supported",
},
{
name: "import-into-multi-region-regional-by-row-to-multi-region-database",
db: "multi_region",
table: "mr_regional_by_row",
create: "CREATE TABLE mr_regional_by_row (i INT8 PRIMARY KEY, s text, b bytea) LOCALITY REGIONAL BY ROW",
sql: "IMPORT INTO mr_regional_by_row AVRO DATA ($1)",
args: []interface{}{simpleOcf},
errString: "IMPORT into REGIONAL BY ROW table not supported",
desc: "pgdump",
importSQL: `IMPORT PGDUMP 'nodelocal://0/pgdump/views_and_sequences.sql' WITH ignore_unsupported_statements`,
expected: map[string]string{
"tbl": "REGIONAL BY TABLE IN PRIMARY REGION",
"s": "REGIONAL BY TABLE IN PRIMARY REGION",
// views are ignored.
},
},
{
name: "import-into-using-multi-region-global-to-multi-region-database",
db: "multi_region",
table: "mr_global",
create: "CREATE TABLE mr_global (i INT8 PRIMARY KEY, s text, b bytea) LOCALITY GLOBAL",
sql: "IMPORT INTO mr_global AVRO DATA ($1)",
args: []interface{}{simpleOcf},
desc: "mysqldump",
importSQL: `IMPORT MYSQLDUMP 'nodelocal://0/mysqldump/views_and_sequences.sql'`,
expected: map[string]string{
"tbl": "REGIONAL BY TABLE IN PRIMARY REGION",
"tbl_auto_inc": "REGIONAL BY TABLE IN PRIMARY REGION",
// views are ignored.
},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, err = sqlDB.Exec(fmt.Sprintf(`SET DATABASE = %q`, test.db))
for _, tc := range viewsAndSequencesTestCases {
t.Run(tc.desc, func(t *testing.T) {
_, err = sqlDB.Exec(`USE multi_region`)
require.NoError(t, err)
defer func() {
_, err := sqlDB.Exec(`
DROP TABLE IF EXISTS tbl;
DROP SEQUENCE IF EXISTS s;
DROP SEQUENCE IF EXISTS table_auto_inc;
DROP VIEW IF EXISTS v`,
)
require.NoError(t, err)
}()

_, err = sqlDB.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %q CASCADE", test.table))
_, err = sqlDB.Exec(tc.importSQL)
require.NoError(t, err)
rows, err := sqlDB.Query("SELECT table_name, locality FROM [SHOW TABLES] ORDER BY table_name")
require.NoError(t, err)

if test.create != "" {
_, err = sqlDB.Exec(test.create)
require.NoError(t, err)
results := make(map[string]string)
for rows.Next() {
require.NoError(t, rows.Err())
var tableName, locality string
require.NoError(t, rows.Scan(&tableName, &locality))
results[tableName] = locality
}
require.Equal(t, tc.expected, results)
})
}

_, err = sqlDB.ExecContext(context.Background(), test.sql, test.args...)
if test.errString != "" {
testutils.IsError(err, test.errString)
} else {
t.Run("avro", func(t *testing.T) {
tests := []struct {
name string
db string
table string
sql string
create string
args []interface{}
errString string
}{
{
name: "import-create-using-multi-region-to-non-multi-region-database",
db: "foo",
table: "simple",
sql: "IMPORT TABLE simple CREATE USING $1 AVRO DATA ($2)",
args: []interface{}{tableSchemaMR, simpleOcf},
errString: "cannot write descriptor for multi-region table",
},
{
name: "import-create-using-multi-region-regional-by-table-to-multi-region-database",
db: "multi_region",
table: "simple",
sql: "IMPORT TABLE simple CREATE USING $1 AVRO DATA ($2)",
args: []interface{}{tableSchemaMR, simpleOcf},
},
{
name: "import-create-using-multi-region-regional-by-row-to-multi-region-database",
db: "multi_region",
table: "simple",
sql: "IMPORT TABLE simple CREATE USING $1 AVRO DATA ($2)",
args: []interface{}{tableSchemaMRRegionalByRow, simpleOcf},
errString: "IMPORT to REGIONAL BY ROW table not supported",
},
{
name: "import-into-multi-region-regional-by-row-to-multi-region-database",
db: "multi_region",
table: "mr_regional_by_row",
create: "CREATE TABLE mr_regional_by_row (i INT8 PRIMARY KEY, s text, b bytea) LOCALITY REGIONAL BY ROW",
sql: "IMPORT INTO mr_regional_by_row AVRO DATA ($1)",
args: []interface{}{simpleOcf},
errString: "IMPORT into REGIONAL BY ROW table not supported",
},
{
name: "import-into-using-multi-region-global-to-multi-region-database",
db: "multi_region",
table: "mr_global",
create: "CREATE TABLE mr_global (i INT8 PRIMARY KEY, s text, b bytea) LOCALITY GLOBAL",
sql: "IMPORT INTO mr_global AVRO DATA ($1)",
args: []interface{}{simpleOcf},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, err = sqlDB.Exec(fmt.Sprintf(`SET DATABASE = %q`, test.db))
require.NoError(t, err)
res := sqlDB.QueryRow(fmt.Sprintf("SELECT count(*) FROM %q", test.table))
require.NoError(t, res.Err())

var numRows int
err = res.Scan(&numRows)
_, err = sqlDB.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %q CASCADE", test.table))
require.NoError(t, err)

if numRows == 0 {
t.Error("expected some rows after import")
if test.create != "" {
_, err = sqlDB.Exec(test.create)
require.NoError(t, err)
}
}
})
}

_, err = sqlDB.ExecContext(context.Background(), test.sql, test.args...)
if test.errString != "" {
testutils.IsError(err, test.errString)
} else {
require.NoError(t, err)
res := sqlDB.QueryRow(fmt.Sprintf("SELECT count(*) FROM %q", test.table))
require.NoError(t, res.Err())

var numRows int
err = res.Scan(&numRows)
require.NoError(t, err)

if numRows == 0 {
t.Error("expected some rows after import")
}
}
})
}
})
}

// TestImportClientDisconnect ensures that an import job can complete even if
Expand Down
4 changes: 4 additions & 0 deletions pkg/ccl/importccl/read_import_mysql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -432,6 +432,8 @@ func mysqlTableToCockroach(
priv,
tree.PersistencePermanent,
nil, /* params */
// If this is multi-region, this will get added by WriteDescriptors.
false, /* isMultiRegion */
)
} else {
priv := descpb.NewDefaultPrivilegeDescriptor(owner)
Expand All @@ -446,6 +448,8 @@ func mysqlTableToCockroach(
priv,
tree.PersistencePermanent,
nil, /* params */
// If this is multi-region, this will get added by WriteDescriptors.
false, /* isMultiRegion */
)
}
if err != nil {
Expand Down
13 changes: 11 additions & 2 deletions pkg/ccl/importccl/read_import_pgdump.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,8 @@ func createPostgresSequences(
descpb.NewDefaultPrivilegeDescriptor(owner),
tree.PersistencePermanent,
nil, /* params */
// If this is multi-region, this will get added by WriteDescriptors.
false, /* isMultiRegion */
)
if err != nil {
return nil, err
Expand Down Expand Up @@ -456,8 +458,15 @@ func readPostgresCreateTable(
}

// Construct sequence descriptors.
seqs, err := createPostgresSequences(ctx, parentID, schemaObjects.createSeq, fks,
walltime, owner, schemaNameToDesc)
seqs, err := createPostgresSequences(
ctx,
parentID,
schemaObjects.createSeq,
fks,
walltime,
owner,
schemaNameToDesc,
)
if err != nil {
return nil, nil, err
}
Expand Down
81 changes: 81 additions & 0 deletions pkg/ccl/importccl/testdata/mysqldump/views_and_sequences.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
-- MySQL dump 10.13 Distrib 8.0.23, for osx10.15 (x86_64)
--
-- Host: localhost Database: bob
-- ------------------------------------------------------
-- Server version 8.0.23

/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!50503 SET NAMES utf8mb4 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;

--
-- Table structure for table `tbl`
--

DROP TABLE IF EXISTS `tbl`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `tbl` (
`id` int NOT NULL AUTO_INCREMENT,
`a` int DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Dumping data for table `tbl`
--

LOCK TABLES `tbl` WRITE;
/*!40000 ALTER TABLE `tbl` DISABLE KEYS */;
/*!40000 ALTER TABLE `tbl` ENABLE KEYS */;
UNLOCK TABLES;

--
-- Temporary view structure for view `v`
--

DROP TABLE IF EXISTS `v`;
/*!50001 DROP VIEW IF EXISTS `v`*/;
SET @saved_cs_client = @@character_set_client;
/*!50503 SET character_set_client = utf8mb4 */;
/*!50001 CREATE VIEW `v` AS SELECT
1 AS `id`,
1 AS `a`*/;
SET character_set_client = @saved_cs_client;

--
-- Final view structure for view `v`
--

/*!50001 DROP VIEW IF EXISTS `v`*/;
/*!50001 SET @saved_cs_client = @@character_set_client */;
/*!50001 SET @saved_cs_results = @@character_set_results */;
/*!50001 SET @saved_col_connection = @@collation_connection */;
/*!50001 SET character_set_client = utf8mb4 */;
/*!50001 SET character_set_results = utf8mb4 */;
/*!50001 SET collation_connection = utf8mb4_0900_ai_ci */;
/*!50001 CREATE ALGORITHM=UNDEFINED */
/*!50013 DEFINER=`root`@`localhost` SQL SECURITY DEFINER */
/*!50001 VIEW `v` AS select `tbl`.`id` AS `id`,`tbl`.`a` AS `a` from `tbl` */;
/*!50001 SET character_set_client = @saved_cs_client */;
/*!50001 SET character_set_results = @saved_cs_results */;
/*!50001 SET collation_connection = @saved_col_connection */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;

/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;

-- Dump completed on 2021-03-17 8:23:08
Loading

0 comments on commit 054169b

Please sign in to comment.