Merge branch 'main' into circleci-gha-migration

.circleci/config.yml

@@ -6,7 +6,7 @@ orbs:
 parameters:
   image_suffix:
     type: string
-    default: '-v641cdcd'
+    default: '-v87fd773'
   pg14_version:
     type: string
     default: '14.9'
@@ -15,10 +15,10 @@ parameters:
     default: '15.4'
   pg16_version:
     type: string
-    default: '16rc1'
+    default: '16.0'
   upgrade_pg_versions:
     type: string
-    default: '14.9-15.4-16rc1'
+    default: '14.9-15.4-16.0'
   style_checker_tools_version:
     type: string
     default: '0.8.18'

.github/workflows/packaging-test-pipelines.yml

@@ -44,12 +44,22 @@ jobs:
         # For this reason, we need to use a "matrix" to generate names of
         # rpm images, e.g. citus/packaging:centos-7-pg12
         packaging_docker_image:
-          - oraclelinux-7
           - oraclelinux-8
-          - centos-7
           - almalinux-8
           - almalinux-9
         POSTGRES_VERSION: ${{ fromJson(needs.get_postgres_versions_from_file.outputs.pg_versions) }}
+        # Postgres removed support for CentOS 7 in PG 16. Below block is needed to
+        # keep the build for CentOS 7 working for PG 14 and PG 15.
+        # Once dependent systems drop support for Centos 7, we can remove this block.
+        include:
+          - packaging_docker_image: centos-7
+            POSTGRES_VERSION: 14
+          - packaging_docker_image: centos-7
+            POSTGRES_VERSION: 15
+          - packaging_docker_image: oraclelinux-7
+            POSTGRES_VERSION: 14
+          - packaging_docker_image: oraclelinux-7
+            POSTGRES_VERSION: 15
 
     container:
       image: citus/packaging:${{ matrix.packaging_docker_image }}-pg${{ matrix.POSTGRES_VERSION }}
@@ -125,7 +135,6 @@ jobs:
           - debian-bullseye-all
           - ubuntu-focal-all
           - ubuntu-jammy-all
-          - ubuntu-kinetic-all
 
         POSTGRES_VERSION: ${{ fromJson(needs.get_postgres_versions_from_file.outputs.pg_versions) }}
 

CHANGELOG.md

@@ -1,3 +1,55 @@
+### citus v12.1.0 (September 12, 2023) ###
+
+* Adds support for PostgreSQL 16.0 (#7173)
+
+* Add `citus_schema_move()` function which moves tables within a
+  distributed schema to another node (#7180)
+
+* Adds `citus_pause_node()` UDF that allows pausing the node with given id
+  (#7089)
+
+* Makes sure to enforce shard level colocation with the GUC
+  `citus.enable_non_colocated_router_query_pushdown` (#7076)
+
+* Allows creating reference / distributed-schema tables from local tables added
+  to metadata and that use identity columns (#7131)
+
+* Propagates `BUFFER_USAGE_LIMIT` option in `VACUUM` and `ANALYZE` (#7114)
+
+* Propagates `PROCESS_MAIN`, `SKIP_DATABASE_STATS`, `ONLY_DATABASE_STATS`
+  options in `VACUUM` (#7114)
+
+* Propagates `GENERIC_PLAN` option in `EXPLAIN` (#7141)
+
+* Propagates "rules" option in `CREATE COLLATION` (#7185)
+
+* Propagates `GRANT`/ `REVOKE` for database privileges (#7109)
+
+* Adds TRUNCATE trigger support on Citus foreign tables (#7170)
+
+* Removes `pg_send_cancellation` (#7135)
+
+* Prevents unnecessarily pulling the data into coordinator for some
+  `INSERT .. SELECT` queries that target a single-shard group (#7077)
+
+* Makes sure that rebalancer throws an error if replication factor is greater
+  than the shard allowed node count. Also makes sure to avoid moving a shard
+  to a node that it already exists on. (#7074)
+
+* Fixes a bug that may appear during 2PC recovery when there are multiple
+  databases (#7174)
+
+* Fixes a bug that could cause `COPY` logic to skip data in case of
+  out-of-memory (#7152)
+
+* Fixes a bug that causes an unexpected error when adding a column with
+  a `NULL` constraint (#7093)
+
+* Fixes `PROCESS_TOAST` default value to `true` (#7122)
+
+* Improves the error thrown when there is datatype mismatch in `MERGE ON`
+  (#7081)
+
 ### citus v12.0.0 (July 11, 2023) ###
 
 * Adds support for schema-based sharding.

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for Citus 12.1devel.
+# Generated by GNU Autoconf 2.69 for Citus 12.2devel.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -579,8 +579,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='Citus'
 PACKAGE_TARNAME='citus'
-PACKAGE_VERSION='12.1devel'
-PACKAGE_STRING='Citus 12.1devel'
+PACKAGE_VERSION='12.2devel'
+PACKAGE_STRING='Citus 12.2devel'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1262,7 +1262,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures Citus 12.1devel to adapt to many kinds of systems.
+\`configure' configures Citus 12.2devel to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1324,7 +1324,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of Citus 12.1devel:";;
+     short | recursive ) echo "Configuration of Citus 12.2devel:";;
    esac
   cat <<\_ACEOF
 
@@ -1429,7 +1429,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-Citus configure 12.1devel
+Citus configure 12.2devel
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1912,7 +1912,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by Citus $as_me 12.1devel, which was
+It was created by Citus $as_me 12.2devel, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -5393,7 +5393,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by Citus $as_me 12.1devel, which was
+This file was extended by Citus $as_me 12.2devel, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -5455,7 +5455,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-Citus config.status 12.1devel
+Citus config.status 12.2devel
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -5,7 +5,7 @@
 # everyone needing autoconf installed, the resulting files are checked
 # into the SCM.
 
-AC_INIT([Citus], [12.1devel])
+AC_INIT([Citus], [12.2devel])
 AC_COPYRIGHT([Copyright (c) Citus Data, Inc.])
 
 # we'll need sed and awk for some of the version commands

src/backend/columnar/citus_columnar.control

@@ -1,6 +1,6 @@
 # Columnar extension
 comment = 'Citus Columnar extension'
-default_version = '11.3-1'
+default_version = '12.2-1'
 module_pathname = '$libdir/citus_columnar'
 relocatable = false
 schema = pg_catalog

src/backend/columnar/sql/citus_columnar--11.3-1--12.2-1.sql

@@ -0,0 +1 @@
+-- citus_columnar--11.3-1--12.2-1

src/backend/columnar/sql/downgrades/citus_columnar--12.2-1--11.3-1.sql

@@ -0,0 +1 @@
+-- citus_columnar--12.2-1--11.3-1

src/backend/distributed/citus.control

@@ -1,6 +1,6 @@
 # Citus extension
 comment = 'Citus distributed database'
-default_version = '12.1-1'
+default_version = '12.2-1'
 module_pathname = '$libdir/citus'
 relocatable = false
 schema = pg_catalog

src/backend/distributed/commands/database.c

@@ -147,3 +147,98 @@ PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString,
 
 	return NodeDDLTaskList(NON_COORDINATOR_NODES, commands);
 }
+
+
+/*
+ * PreprocessAlterDatabaseStmt is executed before the statement is applied to the local
+ * postgres instance.
+ *
+ * In this stage we can prepare the commands that need to be run on all workers to grant
+ * on databases.
+ */
+List *
+PreprocessAlterDatabaseStmt(Node *node, const char *queryString,
+							ProcessUtilityContext processUtilityContext)
+{
+	if (!ShouldPropagate())
+	{
+		return NIL;
+	}
+
+	AlterDatabaseStmt *stmt = castNode(AlterDatabaseStmt, node);
+
+	EnsureCoordinator();
+
+	char *sql = DeparseTreeNode((Node *) stmt);
+
+	List *commands = list_make3(DISABLE_DDL_PROPAGATION,
+								(void *) sql,
+								ENABLE_DDL_PROPAGATION);
+
+	return NodeDDLTaskList(NON_COORDINATOR_NODES, commands);
+}
+
+
+#if PG_VERSION_NUM >= PG_VERSION_15
+
+/*
+ * PreprocessAlterDatabaseSetStmt is executed before the statement is applied to the local
+ * postgres instance.
+ *
+ * In this stage we can prepare the commands that need to be run on all workers to grant
+ * on databases.
+ */
+List *
+PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString,
+									   ProcessUtilityContext processUtilityContext)
+{
+	if (!ShouldPropagate())
+	{
+		return NIL;
+	}
+
+	AlterDatabaseRefreshCollStmt *stmt = castNode(AlterDatabaseRefreshCollStmt, node);
+
+	EnsureCoordinator();
+
+	char *sql = DeparseTreeNode((Node *) stmt);
+
+	List *commands = list_make3(DISABLE_DDL_PROPAGATION,
+								(void *) sql,
+								ENABLE_DDL_PROPAGATION);
+
+	return NodeDDLTaskList(NON_COORDINATOR_NODES, commands);
+}
+
+
+#endif
+
+
+/*
+ * PreprocessAlterDatabaseSetStmt is executed before the statement is applied to the local
+ * postgres instance.
+ *
+ * In this stage we can prepare the commands that need to be run on all workers to grant
+ * on databases.
+ */
+List *
+PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString,
+							   ProcessUtilityContext processUtilityContext)
+{
+	if (!ShouldPropagate())
+	{
+		return NIL;
+	}
+
+	AlterDatabaseSetStmt *stmt = castNode(AlterDatabaseSetStmt, node);
+
+	EnsureCoordinator();
+
+	char *sql = DeparseTreeNode((Node *) stmt);
+
+	List *commands = list_make3(DISABLE_DDL_PROPAGATION,
+								(void *) sql,
+								ENABLE_DDL_PROPAGATION);
+
+	return NodeDDLTaskList(NON_COORDINATOR_NODES, commands);
+}

src/backend/distributed/commands/distribute_object_ops.c

@@ -444,6 +444,42 @@ static DistributeObjectOps Database_Grant = {
 	.markDistributed = false,
 };
 
+static DistributeObjectOps Database_Alter = {
+	.deparse = DeparseAlterDatabaseStmt,
+	.qualify = NULL,
+	.preprocess = PreprocessAlterDatabaseStmt,
+	.postprocess = NULL,
+	.objectType = OBJECT_DATABASE,
+	.operationType = DIST_OPS_ALTER,
+	.address = NULL,
+	.markDistributed = false,
+};
+
+#if PG_VERSION_NUM >= PG_VERSION_15
+static DistributeObjectOps Database_RefreshColl = {
+	.deparse = DeparseAlterDatabaseRefreshCollStmt,
+	.qualify = NULL,
+	.preprocess = PreprocessAlterDatabaseRefreshCollStmt,
+	.postprocess = NULL,
+	.objectType = OBJECT_DATABASE,
+	.operationType = DIST_OPS_ALTER,
+	.address = NULL,
+	.markDistributed = false,
+};
+#endif
+
+static DistributeObjectOps Database_Set = {
+	.deparse = DeparseAlterDatabaseSetStmt,
+	.qualify = NULL,
+	.preprocess = PreprocessAlterDatabaseSetStmt,
+	.postprocess = NULL,
+	.objectType = OBJECT_DATABASE,
+	.operationType = DIST_OPS_ALTER,
+	.address = NULL,
+	.markDistributed = false,
+};
+
+
 static DistributeObjectOps Domain_Alter = {
 	.deparse = DeparseAlterDomainStmt,
 	.qualify = QualifyAlterDomainStmt,
@@ -1272,7 +1308,6 @@ static DistributeObjectOps Trigger_Rename = {
 	.markDistributed = false,
 };
 
-
 /*
  * GetDistributeObjectOps looks up the DistributeObjectOps which handles the node.
  *
@@ -1283,6 +1318,25 @@ GetDistributeObjectOps(Node *node)
 {
 	switch (nodeTag(node))
 	{
+		case T_AlterDatabaseStmt:
+		{
+			return &Database_Alter;
+		}
+
+#if PG_VERSION_NUM >= PG_VERSION_15
+		case T_AlterDatabaseRefreshCollStmt:
+		{
+			return &Database_RefreshColl;
+		}
+
+#endif
+
+		case T_AlterDatabaseSetStmt:
+		{
+			return &Database_Set;
+		}
+
+
 		case T_AlterDomainStmt:
 		{
 			return &Domain_Alter;