Skip to content

Commit

Permalink
Add DDL support null-shard-key tables(#6778/#6784/#6787/#6859)
Browse files Browse the repository at this point in the history
Add tests for ddl coverage:
* indexes
* partitioned tables + indexes with long names
* triggers
* foreign keys
* statistics
* grant & revoke statements
* truncate & vacuum
* create/test/drop view that depends on a dist table with no shard key
* policy & rls test

* alter table add/drop/alter_type column (using sequences/different data
  types/identity columns)
* alter table add constraint (not null, check, exclusion constraint)
* alter table add column with a default value / set default / drop
  default
* alter table set option (autovacuum)

* indexes / constraints without names
* multiple subcommands

Adds support for
* Creating new partitions after distributing (with null key) the parent
table
* Attaching partitions to a distributed table with null distribution key
(and automatically distribute the new partition with null key as well)
* Detaching partitions from it
  • Loading branch information
agedemenli authored and onurctirtir committed May 3, 2023
1 parent fa467e0 commit cdf54ff
Show file tree
Hide file tree
Showing 9 changed files with 956 additions and 29 deletions.
3 changes: 1 addition & 2 deletions src/backend/distributed/commands/create_distributed_table.c
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,6 @@ static List * HashSplitPointsForShardList(List *shardList);
static List * HashSplitPointsForShardCount(int shardCount);
static List * WorkerNodesForShardList(List *shardList);
static List * RoundRobinWorkerNodeList(List *workerNodeList, int listLength);
static void CreateNullShardKeyDistTable(Oid relationId, char *colocateWithTableName);
static CitusTableParams DecideCitusTableParams(CitusTableType tableType,
DistributedTableParams *
distributedTableParams);
Expand Down Expand Up @@ -1031,7 +1030,7 @@ CreateReferenceTable(Oid relationId)
* CreateNullShardKeyDistTable is a wrapper around CreateCitusTable that creates a
* single shard distributed table that doesn't have a shard key.
*/
static void
void
CreateNullShardKeyDistTable(Oid relationId, char *colocateWithTableName)
{
DistributedTableParams distributedTableParams = {
Expand Down
35 changes: 30 additions & 5 deletions src/backend/distributed/commands/table.c
Original file line number Diff line number Diff line change
Expand Up @@ -384,18 +384,30 @@ PostprocessCreateTableStmtPartitionOf(CreateStmt *createStatement, const
*/
if (IsCitusTable(parentRelationId))
{
/*
* We can create Citus local tables and distributed tables with null shard keys
* right away, without switching to sequential mode, because they are going to
* have only one shard.
*/
if (IsCitusTableType(parentRelationId, CITUS_LOCAL_TABLE))
{
CreateCitusLocalTablePartitionOf(createStatement, relationId,
parentRelationId);
return;
}

char *parentRelationName = generate_qualified_relation_name(parentRelationId);

if (IsCitusTableType(parentRelationId, NULL_KEY_DISTRIBUTED_TABLE))
{
CreateNullShardKeyDistTable(relationId, parentRelationName);
return;
}

Var *parentDistributionColumn = DistPartitionKeyOrError(parentRelationId);
char *distributionColumnName =
ColumnToColumnName(parentRelationId, (Node *) parentDistributionColumn);
char parentDistributionMethod = DISTRIBUTE_BY_HASH;
char *parentRelationName = generate_qualified_relation_name(parentRelationId);

SwitchToSequentialAndLocalExecutionIfPartitionNameTooLong(parentRelationId,
relationId);
Expand Down Expand Up @@ -589,19 +601,32 @@ PreprocessAttachCitusPartitionToCitusTable(Oid parentCitusRelationId, Oid

/*
* DistributePartitionUsingParent takes a parent and a partition relation and
* distributes the partition, using the same distribution column as the parent.
* It creates a *hash* distributed table by default, as partitioned tables can only be
* distributed by hash.
* distributes the partition, using the same distribution column as the parent, if the
* parent has a distribution column. It creates a *hash* distributed table by default, as
* partitioned tables can only be distributed by hash, unless it's null key distributed.
*
* If the parent has no distribution key, we distribute the partition with null key too.
*/
static void
DistributePartitionUsingParent(Oid parentCitusRelationId, Oid partitionRelationId)
{
char *parentRelationName = generate_qualified_relation_name(parentCitusRelationId);

if (!HasDistributionKey(parentCitusRelationId))
{
/*
* If the parent is null key distributed, we should distribute the partition
* with null distribution key as well.
*/
CreateNullShardKeyDistTable(partitionRelationId, parentRelationName);
return;
}

Var *distributionColumn = DistPartitionKeyOrError(parentCitusRelationId);
char *distributionColumnName = ColumnToColumnName(parentCitusRelationId,
(Node *) distributionColumn);

char distributionMethod = DISTRIBUTE_BY_HASH;
char *parentRelationName = generate_qualified_relation_name(parentCitusRelationId);

SwitchToSequentialAndLocalExecutionIfPartitionNameTooLong(
parentCitusRelationId, partitionRelationId);
Expand Down
2 changes: 1 addition & 1 deletion src/backend/distributed/planner/multi_join_order.c
Original file line number Diff line number Diff line change
Expand Up @@ -1404,7 +1404,7 @@ DistPartitionKeyOrError(Oid relationId)
if (partitionKey == NULL)
{
ereport(ERROR, (errmsg(
"no distribution column found for relation %d, because it is a reference table",
"no distribution column found for relation %d",
relationId)));
}

Expand Down
1 change: 1 addition & 0 deletions src/include/distributed/metadata_utility.h
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,7 @@ extern void DeletePartitionRow(Oid distributedRelationId);
extern void DeleteShardRow(uint64 shardId);
extern void UpdatePlacementGroupId(uint64 placementId, int groupId);
extern void DeleteShardPlacementRow(uint64 placementId);
extern void CreateNullShardKeyDistTable(Oid relationId, char *colocateWithTableName);
extern void CreateDistributedTable(Oid relationId, char *distributionColumnName,
char distributionMethod, int shardCount,
bool shardCountIsStrict, char *colocateWithTableName);
Expand Down
154 changes: 154 additions & 0 deletions src/test/regress/expected/alter_table_null_dist_key.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
CREATE SCHEMA alter_null_dist_key;
SET search_path TO alter_null_dist_key;
SET citus.next_shard_id TO 1720000;
SET citus.shard_count TO 32;
SET citus.shard_replication_factor TO 1;
CREATE SEQUENCE dist_seq;
CREATE TABLE null_dist_table(a bigint DEFAULT nextval('dist_seq') UNIQUE, "b" text, c bigint GENERATED BY DEFAULT AS IDENTITY);
INSERT INTO null_dist_table("b") VALUES ('test');
SELECT create_distributed_table('null_dist_table', null, colocate_with=>'none', distribution_type=>null);
NOTICE: Copying data from local table...
NOTICE: copying the data has completed
DETAIL: The local data in the table is no longer visible, but is still on disk.
HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$alter_null_dist_key.null_dist_table$$)
create_distributed_table
---------------------------------------------------------------------

(1 row)

-- add column
ALTER TABLE null_dist_table ADD COLUMN d bigint DEFAULT 2;
SELECT * FROM null_dist_table ORDER BY c;
a | b | c | d
---------------------------------------------------------------------
1 | test | 1 | 2
(1 row)

-- alter default, set to 3
ALTER TABLE null_dist_table ALTER COLUMN d SET DEFAULT 3;
INSERT INTO null_dist_table("b") VALUES ('test');
SELECT * FROM null_dist_table ORDER BY c;
a | b | c | d
---------------------------------------------------------------------
1 | test | 1 | 2
2 | test | 2 | 3
(2 rows)

-- drop default, see null
ALTER TABLE null_dist_table ALTER COLUMN d DROP DEFAULT;
INSERT INTO null_dist_table("b") VALUES ('test');
SELECT * FROM null_dist_table ORDER BY c;
a | b | c | d
---------------------------------------------------------------------
1 | test | 1 | 2
2 | test | 2 | 3
3 | test | 3 |
(3 rows)

-- cleanup the rows that were added to test the default behavior
DELETE FROM null_dist_table WHERE "b" = 'test' AND a > 1;
-- alter column type
ALTER TABLE null_dist_table ALTER COLUMN d TYPE text;
UPDATE null_dist_table SET d = 'this is a text' WHERE d = '2';
SELECT * FROM null_dist_table ORDER BY c;
a | b | c | d
---------------------------------------------------------------------
1 | test | 1 | this is a text
(1 row)

-- drop seq column
ALTER TABLE null_dist_table DROP COLUMN a;
SELECT * FROM null_dist_table ORDER BY c;
b | c | d
---------------------------------------------------------------------
test | 1 | this is a text
(1 row)

-- add not null constraint
ALTER TABLE null_dist_table ALTER COLUMN b SET NOT NULL;
-- not null constraint violation, error out
INSERT INTO null_dist_table VALUES (NULL, 2, 'test');
ERROR: null value in column "b" violates not-null constraint
DETAIL: Failing row contains (null, 2, test).
CONTEXT: while executing command on localhost:xxxxx
-- drop not null constraint and try again
ALTER TABLE null_dist_table ALTER COLUMN b DROP NOT NULL;
INSERT INTO null_dist_table VALUES (NULL, 3, 'test');
SELECT * FROM null_dist_table ORDER BY c;
b | c | d
---------------------------------------------------------------------
test | 1 | this is a text
| 3 | test
(2 rows)

-- add exclusion constraint
ALTER TABLE null_dist_table ADD CONSTRAINT exc_b EXCLUDE USING btree (b with =);
-- rename the exclusion constraint, errors out
ALTER TABLE null_dist_table RENAME CONSTRAINT exc_b TO exc_b_1;
ERROR: renaming constraints belonging to distributed tables is currently unsupported
-- create exclusion constraint without a name
ALTER TABLE null_dist_table ADD EXCLUDE USING btree (b with =);
-- test setting autovacuum option
ALTER TABLE null_dist_table SET (autovacuum_enabled = false);
-- test multiple subcommands
ALTER TABLE null_dist_table ADD COLUMN int_column1 INTEGER,
DROP COLUMN d;
SELECT * FROM null_dist_table ORDER BY c;
b | c | int_column1
---------------------------------------------------------------------
test | 1 |
| 3 |
(2 rows)

-- test policy and row level security
CREATE TABLE null_dist_key_with_policy (table_user text);
INSERT INTO null_dist_key_with_policy VALUES ('user_1');
SELECT create_distributed_table('null_dist_key_with_policy', null);
NOTICE: Copying data from local table...
NOTICE: copying the data has completed
DETAIL: The local data in the table is no longer visible, but is still on disk.
HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$alter_null_dist_key.null_dist_key_with_policy$$)
create_distributed_table
---------------------------------------------------------------------

(1 row)

-- enable rls
ALTER TABLE null_dist_key_with_policy ENABLE ROW LEVEL SECURITY;
-- user_1 will be allowed to see the inserted row
CREATE ROLE user_1 WITH LOGIN;
GRANT ALL ON SCHEMA alter_null_dist_key TO user_1;
GRANT ALL ON TABLE alter_null_dist_key.null_dist_key_with_policy TO user_1;
CREATE POLICY table_policy ON null_dist_key_with_policy TO user_1
USING (table_user = current_user);
-- user_2 will not be allowed to see the inserted row
CREATE ROLE user_2 WITH LOGIN;
GRANT ALL ON SCHEMA alter_null_dist_key TO user_2;
GRANT ALL ON TABLE alter_null_dist_key.null_dist_key_with_policy TO user_2;
CREATE POLICY table_policy_1 ON null_dist_key_with_policy TO user_2
USING (table_user = current_user);
\c - user_1 -
SELECT * FROM alter_null_dist_key.null_dist_key_with_policy;
table_user
---------------------------------------------------------------------
user_1
(1 row)

\c - user_2 -
SELECT * FROM alter_null_dist_key.null_dist_key_with_policy;
table_user
---------------------------------------------------------------------
(0 rows)

-- postgres will always be allowed to see the row as a superuser
\c - postgres -
SELECT * FROM alter_null_dist_key.null_dist_key_with_policy;
table_user
---------------------------------------------------------------------
user_1
(1 row)

-- cleanup
SET client_min_messages TO ERROR;
DROP SCHEMA alter_null_dist_key CASCADE;
DROP ROLE user_1, user_2;
Loading

0 comments on commit cdf54ff

Please sign in to comment.