Merge #54896 #55827

54896: cli: add import pgdump/mysqldump CLI command r=miretskiy,knz a=adityamaru

This change introduces a new CLI command to import
locally saved PGDUMP or MYSQLDUMP files into a running
cockroach cluster.

The underlying logic relies on user scoped userfile
storage to upload the local dump file to, and subsequently
import data from. Most of the heavy lifting was already
completed during the development of the userfile storage.

We can only support bundle formats that have the table
schemas to be imported, baked into the dump files. The newly
added CLI commands support the same options as IMPORT PGDUMP
and IMPORT MYSQLDUMP from the SQL shell do. These options
are to be passed in as CLI flags.

Following are the CLI examples:
`./cockroach import db <format> <source>`
`./cockroach import table <tablename> <format> <source>`

Release note (cli change): Adds an import CLI command which
allows users to upload and import local dump files into a
running cockroach cluster. We currently support PGDUMP and
MYSQLDUMP formats.

55827: sql: stub ALTER TABLE ... SET LOCALITY/AFFINITY related commands r=ajstorm a=otan

Following the doc on proposed ideas and fleshed them all out. They
default to displaying REGIONAL AFFINITY for now.

Release note (sql change): Implemented ALTER TABLE ... SET
LOCALITY/REGIONAL AFFINITY commands that allow users to configure
multiregion properties of given tables. These may change later.

Co-authored-by: Aditya Maru <adityamaru@gmail.com>
Co-authored-by: Oliver Tan <otan@cockroachlabs.com>

docs/generated/sql/bnf/show_var.bnf

@@ -11,6 +11,7 @@ show_stmt ::=
 	| show_indexes_stmt
 	| show_partitions_stmt
 	| show_jobs_stmt
+	| show_locality_stmt
 	| show_schedules_stmt
 	| show_queries_stmt
 	| show_ranges_stmt

docs/generated/sql/bnf/stmt_block.bnf

@@ -211,6 +211,7 @@ show_stmt ::=
 	| show_indexes_stmt
 	| show_partitions_stmt
 	| show_jobs_stmt
+	| show_locality_stmt
 	| show_schedules_stmt
 	| show_queries_stmt
 	| show_ranges_stmt
@@ -638,6 +639,9 @@ show_jobs_stmt ::=
 	| 'SHOW' 'JOB' a_expr
 	| 'SHOW' 'JOB' 'WHEN' 'COMPLETE' a_expr
 
+show_locality_stmt ::=
+	'SHOW' 'LOCALITY'
+
 show_schedules_stmt ::=
 	'SHOW' 'SCHEDULES' opt_schedule_executor_type
 	| 'SHOW' schedule_state 'SCHEDULES' opt_schedule_executor_type
@@ -759,6 +763,7 @@ unreserved_keyword ::=
 	| 'ACCESS'
 	| 'ADD'
 	| 'ADMIN'
+	| 'AFFINITY'
 	| 'AFTER'
 	| 'AGGREGATE'
 	| 'ALTER'
@@ -897,6 +902,7 @@ unreserved_keyword ::=
 	| 'LOCAL'
 	| 'LOCKED'
 	| 'LOGIN'
+	| 'LOCALITY'
 	| 'LOOKUP'
 	| 'LOW'
 	| 'MATCH'
@@ -985,6 +991,7 @@ unreserved_keyword ::=
 	| 'REF'
 	| 'REFRESH'
 	| 'REGION'
+	| 'REGIONAL'
 	| 'REGIONS'
 	| 'REINDEX'
 	| 'RELEASE'
@@ -1202,6 +1209,7 @@ alter_table_stmt ::=
 	| alter_zone_table_stmt
 	| alter_rename_table_stmt
 	| alter_table_set_schema_stmt
+	| alter_table_regional_affinity_stmt
 
 alter_index_stmt ::=
 	alter_oneindex_stmt
@@ -1586,6 +1594,10 @@ alter_table_set_schema_stmt ::=
 	'ALTER' 'TABLE' relation_expr 'SET' 'SCHEMA' schema_name
 	| 'ALTER' 'TABLE' 'IF' 'EXISTS' relation_expr 'SET' 'SCHEMA' schema_name
 
+alter_table_regional_affinity_stmt ::=
+	'ALTER' 'TABLE' relation_expr 'SET' regional_affinity
+	| 'ALTER' 'TABLE' 'IF' 'NOT' 'EXISTS' relation_expr 'SET' regional_affinity
+
 alter_oneindex_stmt ::=
 	'ALTER' 'INDEX' table_index_name alter_index_cmds
 	| 'ALTER' 'INDEX' 'IF' 'EXISTS' table_index_name alter_index_cmds
@@ -1742,7 +1754,7 @@ opt_connection_limit ::=
 	| 
 
 opt_regions_list ::=
-	region_or_regions opt_equal name_list
+	region_or_regions opt_equal region_name_list
 	| 
 
 opt_survive_clause ::=
@@ -2090,6 +2102,20 @@ set_zone_config ::=
 	'CONFIGURE' 'ZONE' 'USING' var_set_list
 	| 'CONFIGURE' 'ZONE' 'DISCARD'
 
+regional_affinity ::=
+	'LOCALITY' 'GLOBAL'
+	| 'LOCALITY' 'REGIONAL' 'BY' 'TABLE' 'IN' region_name
+	| 'LOCALITY' 'REGIONAL' 'BY' 'ROW'
+	| 'REGIONAL' 'AFFINITY' 'TO' 'NONE'
+	| 'REGIONAL' 'AFFINITY' 'TO' region_name
+	| 'REGIONAL' 'AFFINITY' 'AT' 'ROW' 'LEVEL'
+	| 'REGIONAL' 'AFFINITY' 'NONE'
+	| 'REGIONAL' 'AFFINITY' region_name
+	| 'REGIONAL' 'AFFINITY' 'ROW' 'LEVEL'
+	| 'NO' 'REGIONAL' 'AFFINITY'
+	| 'TABLE' 'LEVEL' 'REGIONAL' 'AFFINITY' 'TO' region_name
+	| 'ROW' 'LEVEL' 'REGIONAL' 'AFFINITY'
+
 alter_index_cmds ::=
 	( alter_index_cmd ) ( ( ',' alter_index_cmd ) )*
 
@@ -2191,6 +2217,9 @@ signed_iconst ::=
 	'ICONST'
 	| only_signed_iconst
 
+region_name_list ::=
+	name_list
+
 opt_name ::=
 	name
 	| 
@@ -2421,6 +2450,9 @@ alter_table_cmd ::=
 var_set_list ::=
 	( var_name '=' 'COPY' 'FROM' 'PARENT' | var_name '=' var_value ) ( ( ',' var_name '=' var_value | ',' var_name '=' 'COPY' 'FROM' 'PARENT' ) )*
 
+region_name ::=
+	name
+
 alter_index_cmd ::=
 	partition_by
 

pkg/cli/cli.go

@@ -197,6 +197,7 @@ func init() {
 		dumpCmd,
 		nodeLocalCmd,
 		userFileCmd,
+		importCmd,
 
 		// Miscellaneous commands.
 		// TODO(pmattis): stats

pkg/cli/cli_test.go

@@ -1417,6 +1417,7 @@ Available Commands:
 
   nodelocal         upload and delete nodelocal files
   userfile          upload, list and delete user scoped files
+  import            import a db or table from a local PGDUMP or MYSQLDUMP file
   demo              open a demo sql shell
   gen               generate auxiliary files
   version           output version information

pkg/cli/cliflags/flags.go

@@ -1264,4 +1264,21 @@ The zip command will block for the duration specified. Zero disables this featur
 		Name:        "all",
 		Description: `Cancel all outstanding requests.`,
 	}
+
+	ImportSkipForeignKeys = FlagInfo{
+		Name: "skip-foreign-keys",
+		Description: `
+Speed up data import by ignoring foreign key constraints in the dump file's DDL.
+Also enables importing individual tables that would otherwise fail due to
+dependencies on other tables.
+`,
+	}
+
+	ImportMaxRowSize = FlagInfo{
+		Name: "max-row-size",
+		Description: `
+Override limits on line size when importing Postgres dump files. This setting 
+may need to be tweaked if the Postgres dump file has extremely long lines.
+`,
+	}
 )

pkg/cli/context.go

@@ -51,6 +51,7 @@ func initCLIDefaults() {
 	setDemoContextDefaults()
 	setStmtDiagContextDefaults()
 	setAuthContextDefaults()
+	setImportContextDefaults()
 
 	initPreFlagsDefaults()
 
@@ -565,6 +566,17 @@ func setStmtDiagContextDefaults() {
 	stmtDiagCtx.all = false
 }
 
+// importCtx captures the command-line parameters of the 'import' command.
+var importCtx struct {
+	maxRowSize      int
+	skipForeignKeys bool
+}
+
+func setImportContextDefaults() {
+	importCtx.maxRowSize = 512 * (1 << 10) // 512 KiB
+	importCtx.skipForeignKeys = false
+}
+
 // GetServerCfgStores provides direct public access to the StoreSpecList inside
 // serverCfg. This is used by CCL code to populate some fields.
 //

pkg/cli/flags.go

@@ -553,6 +553,7 @@ func init() {
 	clientCmds = append(clientCmds, nodeCmds...)
 	clientCmds = append(clientCmds, systemBenchCmds...)
 	clientCmds = append(clientCmds, nodeLocalCmds...)
+	clientCmds = append(clientCmds, importCmds...)
 	clientCmds = append(clientCmds, userFileCmds...)
 	clientCmds = append(clientCmds, stmtDiagCmds...)
 	for _, cmd := range clientCmds {
@@ -678,6 +679,7 @@ func init() {
 	sqlCmds = append(sqlCmds, demoCmd.Commands()...)
 	sqlCmds = append(sqlCmds, stmtDiagCmds...)
 	sqlCmds = append(sqlCmds, nodeLocalCmds...)
+	sqlCmds = append(sqlCmds, importCmds...)
 	sqlCmds = append(sqlCmds, userFileCmds...)
 	for _, cmd := range sqlCmds {
 		f := cmd.Flags()
@@ -778,6 +780,17 @@ func init() {
 		boolFlag(stmtDiagCancelCmd.Flags(), &stmtDiagCtx.all, cliflags.StmtDiagCancelAll)
 	}
 
+	// import dump command.
+	{
+		d := importDumpFileCmd.Flags()
+		boolFlag(d, &importCtx.skipForeignKeys, cliflags.ImportSkipForeignKeys)
+		intFlag(d, &importCtx.maxRowSize, cliflags.ImportMaxRowSize)
+
+		t := importDumpTableCmd.Flags()
+		boolFlag(t, &importCtx.skipForeignKeys, cliflags.ImportSkipForeignKeys)
+		intFlag(t, &importCtx.maxRowSize, cliflags.ImportMaxRowSize)
+	}
+
 	// sqlfmt command.
 	{
 		f := sqlfmtCmd.Flags()