From 9e18547688114fae58c052ab7a76278661c42d9b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 2 May 2025 16:27:24 -0400 Subject: [PATCH 01/32] feat: if upgrade 17 -> 17 or 17-orioledb -> 17-orioledb do not run these queries --- .../admin_api_scripts/pg_upgrade_scripts/initiate.sh | 8 ++++++-- ansible/vars.yml | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 92c1516a5..2b54557e8 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -43,8 +43,12 @@ POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" POST_UPGRADE_POSTGRES_PERMS_SCRIPT="/tmp/pg_upgrade/pg_upgrade_postgres_perms.sql" OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;") -SERVER_LC_COLLATE=$(run_sql -A -t -c "SHOW lc_collate;") -SERVER_LC_CTYPE=$(run_sql -A -t -c "SHOW lc_ctype;") +# Skip locale settings if both versions are PostgreSQL 17+ or 17-orioledb +if ! [[ (("$OLD_PGVERSION" =~ ^17.* || "$OLD_PGVERSION" == "17-orioledb") && ("$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb")) ]]; then + SERVER_LC_COLLATE=$(run_sql -A -t -c "SHOW lc_collate;") + SERVER_LC_CTYPE=$(run_sql -A -t -c "SHOW lc_ctype;") +fi + SERVER_ENCODING=$(run_sql -A -t -c "SHOW server_encoding;") POSTGRES_CONFIG_PATH="/etc/postgresql/postgresql.conf" diff --git a/ansible/vars.yml b/ansible/vars.yml index 0e658202d..4dc843c9e 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.078-orioledb" - postgres17: "17.4.1.028" - postgres15: "15.8.1.085" + postgresorioledb-17: "17.0.1.078-orioledb-upgrade-1" + postgres17: "17.4.1.028-upgrade-1" + postgres15: "15.8.1.085-upgrade-1" # Non Postgres Extensions pgbouncer_release: "1.19.0" From a1a0b6e642949570e2512b1fe06f172f0d9a0a94 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 5 May 2025 14:44:35 -0400 Subject: [PATCH 02/32] feat: handle all cases of SERVER_LC_COLLATE and SERVER_LC_CTYPE --- .../pg_upgrade_scripts/initiate.sh | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 2b54557e8..58b78ea37 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -41,7 +41,7 @@ LOG_FILE="/var/log/pg-upgrade-initiate.log" POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" POST_UPGRADE_POSTGRES_PERMS_SCRIPT="/tmp/pg_upgrade/pg_upgrade_postgres_perms.sql" -OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;") +OLD_PGVERSION=$(pg_config --version | sed 's/PostgreSQL \([0-9]*\.[0-9]*\).*/\1/') # Skip locale settings if both versions are PostgreSQL 17+ or 17-orioledb if ! [[ (("$OLD_PGVERSION" =~ ^17.* || "$OLD_PGVERSION" == "17-orioledb") && ("$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb")) ]]; then @@ -398,7 +398,11 @@ function initiate_upgrade { rm -rf "${PGDATANEW:?}/" if [ "$IS_NIX_UPGRADE" = "true" ]; then - LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --lc-collate=$SERVER_LC_COLLATE --lc-ctype=$SERVER_LC_CTYPE -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres + if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then + LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres + else + LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --lc-collate=$SERVER_LC_COLLATE --lc-ctype=$SERVER_LC_CTYPE -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres + fi else su -c "$PGBINNEW/initdb -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres fi @@ -431,7 +435,12 @@ EOF if [ "$IS_NIX_BASED_SYSTEM" = "true" ]; then UPGRADE_COMMAND=". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $UPGRADE_COMMAND" fi - GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres + + if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then + GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres + else + GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres + fi echo "10. Stopping postgres; running pg_upgrade" # Extra work to ensure postgres is actually stopped @@ -447,9 +456,12 @@ EOF CI_stop_postgres fi - GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres + if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then + GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres + else + GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres + fi - # copying custom configurations echo "11. Copying custom configurations" mkdir -p "$MOUNT_POINT/conf" cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/" From 1dbf0fecf1a898eb125359932f1e2c492c8f9ddb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 5 May 2025 14:46:30 -0400 Subject: [PATCH 03/32] chore: bump version for testing --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 4dc843c9e..a23ea3742 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.078-orioledb-upgrade-1" - postgres17: "17.4.1.028-upgrade-1" - postgres15: "15.8.1.085-upgrade-1" + postgresorioledb-17: "17.0.1.078-orioledb-upgrade-2" + postgres17: "17.4.1.028-upgrade-2" + postgres15: "15.8.1.085-upgrade-2" # Non Postgres Extensions pgbouncer_release: "1.19.0" From fdcd1c425055a1b975bd68e3b949793c9c15e597 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 6 May 2025 10:28:17 -0400 Subject: [PATCH 04/32] fix: explixit set on 17/oriole --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 2 +- ansible/vars.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 58b78ea37..98a2b24f6 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -399,7 +399,7 @@ function initiate_upgrade { if [ "$IS_NIX_UPGRADE" = "true" ]; then if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then - LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres + LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres else LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --lc-collate=$SERVER_LC_COLLATE --lc-ctype=$SERVER_LC_CTYPE -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres fi diff --git a/ansible/vars.yml b/ansible/vars.yml index a23ea3742..002886a7a 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.078-orioledb-upgrade-2" - postgres17: "17.4.1.028-upgrade-2" - postgres15: "15.8.1.085-upgrade-2" + postgresorioledb-17: "17.0.1.078-orioledb-upgrade-3" + postgres17: "17.4.1.028-upgrade-3" + postgres15: "15.8.1.085-upgrade-3" # Non Postgres Extensions pgbouncer_release: "1.19.0" From cf59660a7813d13d5dc894820e64f9228c626d0e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 6 May 2025 14:49:10 -0400 Subject: [PATCH 05/32] feat: handling max_slot_wal_keep_size for pg 17 was needed as well --- .../admin_api_scripts/pg_upgrade_scripts/complete.sh | 12 ++++++++++++ .../admin_api_scripts/pg_upgrade_scripts/initiate.sh | 5 +++++ ansible/vars.yml | 6 +++--- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh index 108409463..3527c61fa 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh @@ -22,6 +22,18 @@ function cleanup { ship_logs "$LOG_FILE" || true + # Restore max_slot_wal_keep_size to 4096 only if both versions were PostgreSQL 17+ + if [[ ("$OLD_PGVERSION" =~ ^17.* || "$OLD_PGVERSION" == "17-orioledb") && ("$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb") ]]; then + sed -i 's/max_slot_wal_keep_size = -1/max_slot_wal_keep_size = 4096/' /etc/postgresql/postgresql.conf + fi + + # Restart postgres to apply any configuration changes + if [ -z "$IS_CI" ]; then + systemctl restart postgresql + else + CI_start_postgres + fi + exit "$EXIT_CODE" } diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 98a2b24f6..8e63eb8a0 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -397,6 +397,11 @@ function initiate_upgrade { chown -R postgres:postgres "$MOUNT_POINT/" rm -rf "${PGDATANEW:?}/" + # Change max_slot_wal_keep_size to -1 for upgrade only if both versions are PostgreSQL 17+ + if [[ ("$OLD_PGVERSION" =~ ^17.* || "$OLD_PGVERSION" == "17-orioledb") && ("$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb") ]]; then + sed -i 's/max_slot_wal_keep_size = [0-9]*/max_slot_wal_keep_size = -1/' /etc/postgresql/postgresql.conf + fi + if [ "$IS_NIX_UPGRADE" = "true" ]; then if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres diff --git a/ansible/vars.yml b/ansible/vars.yml index 002886a7a..faa307141 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.078-orioledb-upgrade-3" - postgres17: "17.4.1.028-upgrade-3" - postgres15: "15.8.1.085-upgrade-3" + postgresorioledb-17: "17.0.1.078-orioledb-upgrade-4" + postgres17: "17.4.1.028-upgrade-4" + postgres15: "15.8.1.085-upgrade-4" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 11dab86773c04e4a190e086a90afc5f23c6525ed Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 6 May 2025 14:57:20 -0400 Subject: [PATCH 06/32] feat: binary upgrades require max_slot_wal_keep_size to be -1 during binary upgrade --- .../files/admin_api_scripts/pg_upgrade_scripts/complete.sh | 6 ++---- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh index 3527c61fa..b08b57dac 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh @@ -22,10 +22,8 @@ function cleanup { ship_logs "$LOG_FILE" || true - # Restore max_slot_wal_keep_size to 4096 only if both versions were PostgreSQL 17+ - if [[ ("$OLD_PGVERSION" =~ ^17.* || "$OLD_PGVERSION" == "17-orioledb") && ("$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb") ]]; then - sed -i 's/max_slot_wal_keep_size = -1/max_slot_wal_keep_size = 4096/' /etc/postgresql/postgresql.conf - fi + # Restore max_slot_wal_keep_size to 4096 after binary upgrade + sed -i 's/max_slot_wal_keep_size = -1/max_slot_wal_keep_size = 4096/' /etc/postgresql/postgresql.conf # Restart postgres to apply any configuration changes if [ -z "$IS_CI" ]; then diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 8e63eb8a0..81abe5fb0 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -397,10 +397,8 @@ function initiate_upgrade { chown -R postgres:postgres "$MOUNT_POINT/" rm -rf "${PGDATANEW:?}/" - # Change max_slot_wal_keep_size to -1 for upgrade only if both versions are PostgreSQL 17+ - if [[ ("$OLD_PGVERSION" =~ ^17.* || "$OLD_PGVERSION" == "17-orioledb") && ("$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb") ]]; then - sed -i 's/max_slot_wal_keep_size = [0-9]*/max_slot_wal_keep_size = -1/' /etc/postgresql/postgresql.conf - fi + # Change max_slot_wal_keep_size to -1 for binary upgrade + sed -i 's/max_slot_wal_keep_size = [0-9]*/max_slot_wal_keep_size = -1/' /etc/postgresql/postgresql.conf if [ "$IS_NIX_UPGRADE" = "true" ]; then if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then From 3ec98188eeb623b8eca46df6653ad7149849112b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 6 May 2025 16:07:56 -0400 Subject: [PATCH 07/32] fix: Better to override that during the upgrade process by specifying the option directly when pg_upgrade is executed (-c) --- .../files/admin_api_scripts/pg_upgrade_scripts/complete.sh | 3 --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 7 +++---- ansible/vars.yml | 6 +++--- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh index b08b57dac..735b727ed 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh @@ -22,9 +22,6 @@ function cleanup { ship_logs "$LOG_FILE" || true - # Restore max_slot_wal_keep_size to 4096 after binary upgrade - sed -i 's/max_slot_wal_keep_size = -1/max_slot_wal_keep_size = 4096/' /etc/postgresql/postgresql.conf - # Restart postgres to apply any configuration changes if [ -z "$IS_CI" ]; then systemctl restart postgresql diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 81abe5fb0..8b6910b30 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -397,9 +397,6 @@ function initiate_upgrade { chown -R postgres:postgres "$MOUNT_POINT/" rm -rf "${PGDATANEW:?}/" - # Change max_slot_wal_keep_size to -1 for binary upgrade - sed -i 's/max_slot_wal_keep_size = [0-9]*/max_slot_wal_keep_size = -1/' /etc/postgresql/postgresql.conf - if [ "$IS_NIX_UPGRADE" = "true" ]; then if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres @@ -430,8 +427,10 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf --jobs="${WORKERS}" -r \ --old-options='-c config_file=${POSTGRES_CONFIG_PATH}' \ --old-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ + --old-options="-c max_slot_wal_keep_size=-1" \ --new-options="-c data_directory=${PGDATANEW}" \ - --new-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" + --new-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ + --new-options="-c max_slot_wal_keep_size=-1" EOF ) diff --git a/ansible/vars.yml b/ansible/vars.yml index faa307141..ae3725f9a 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.078-orioledb-upgrade-4" - postgres17: "17.4.1.028-upgrade-4" - postgres15: "15.8.1.085-upgrade-4" + postgresorioledb-17: "17.0.1.078-orioledb-upgrade-5" + postgres17: "17.4.1.028-upgrade-5" + postgres15: "15.8.1.085-upgrade-5" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 715afe025bba061aedd9053dc64720fb0c01c1ef Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 07:59:14 -0400 Subject: [PATCH 08/32] fix: cover only pg 17 --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 8b6910b30..522f8d71c 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -438,7 +438,7 @@ EOF UPGRADE_COMMAND=". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $UPGRADE_COMMAND" fi - if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then + if [[ "$PGVERSION" =~ ^17.* ]]; then GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres else GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres @@ -458,7 +458,7 @@ EOF CI_stop_postgres fi - if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then + if [[ "$PGVERSION" =~ ^17.* ]]; then GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres else GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres From b079397d6f7e50540a4444af1b4169ae0f1c6078 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 08:15:22 -0400 Subject: [PATCH 09/32] chore: bump version --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index ae3725f9a..2d8565918 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.078-orioledb-upgrade-5" - postgres17: "17.4.1.028-upgrade-5" - postgres15: "15.8.1.085-upgrade-5" + postgresorioledb-17: "17.0.1.078-orioledb-upgrade-6" + postgres17: "17.4.1.028-upgrade-6" + postgres15: "15.8.1.085-upgrade-6" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 1a70efecbf9a9e18d0f6ecd12b59a4e5637f2525 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 10:01:05 -0400 Subject: [PATCH 10/32] fix: rm oriole handling --- ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 522f8d71c..566470fc2 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -398,7 +398,7 @@ function initiate_upgrade { rm -rf "${PGDATANEW:?}/" if [ "$IS_NIX_UPGRADE" = "true" ]; then - if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then + if [[ "$PGVERSION" =~ ^17.* ]]; then LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres else LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --lc-collate=$SERVER_LC_COLLATE --lc-ctype=$SERVER_LC_CTYPE -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres From 0a9fb889ea2ac282bf5cf53df549934c12ef1cf3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 11:18:37 -0400 Subject: [PATCH 11/32] fix: do not need max_slot_wal_keep_size on old version --- ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 566470fc2..ba0c3c162 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -427,7 +427,6 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf --jobs="${WORKERS}" -r \ --old-options='-c config_file=${POSTGRES_CONFIG_PATH}' \ --old-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ - --old-options="-c max_slot_wal_keep_size=-1" \ --new-options="-c data_directory=${PGDATANEW}" \ --new-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ --new-options="-c max_slot_wal_keep_size=-1" From 86c93bd3ca252c48ecf20913cb5875e0359de53b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 13:19:07 -0400 Subject: [PATCH 12/32] fix: temp config on new-options too --- ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index ba0c3c162..da303ad1e 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -429,6 +429,7 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf --old-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ --new-options="-c data_directory=${PGDATANEW}" \ --new-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ + --new-options='-c config_file=${TEMP_POSTGRES_CONFIG_PATH}' \ --new-options="-c max_slot_wal_keep_size=-1" EOF ) From cf74e758886d374685a84f7e2dbdd35d5f561cbb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 13:20:33 -0400 Subject: [PATCH 13/32] test: bump test version --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 2d8565918..f88336ac3 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.078-orioledb-upgrade-6" - postgres17: "17.4.1.028-upgrade-6" - postgres15: "15.8.1.085-upgrade-6" + postgresorioledb-17: "17.0.1.078-orioledb-upgrade-7" + postgres17: "17.4.1.028-upgrade-7" + postgres15: "15.8.1.085-upgrade-7" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 1ca847303698c023ddea3e88ff5625da1be43225 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 15:17:25 -0400 Subject: [PATCH 14/32] fix: remove unbound var --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 1 - ansible/vars.yml | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index da303ad1e..ba0c3c162 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -429,7 +429,6 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf --old-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ --new-options="-c data_directory=${PGDATANEW}" \ --new-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ - --new-options='-c config_file=${TEMP_POSTGRES_CONFIG_PATH}' \ --new-options="-c max_slot_wal_keep_size=-1" EOF ) diff --git a/ansible/vars.yml b/ansible/vars.yml index f88336ac3..31cf89b7b 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.078-orioledb-upgrade-7" - postgres17: "17.4.1.028-upgrade-7" - postgres15: "15.8.1.085-upgrade-7" + postgresorioledb-17: "17.0.1.078-orioledb-upgrade-8" + postgres17: "17.4.1.028-upgrade-8" + postgres15: "15.8.1.085-upgrade-8" # Non Postgres Extensions pgbouncer_release: "1.19.0" From cfa3f19bfb4788f6c49e96f3ac17a31f873307ff Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 17:10:12 -0400 Subject: [PATCH 15/32] chore: remove complete.sh change that should not have been committed --- .../files/admin_api_scripts/pg_upgrade_scripts/complete.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh index 735b727ed..108409463 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh @@ -22,13 +22,6 @@ function cleanup { ship_logs "$LOG_FILE" || true - # Restart postgres to apply any configuration changes - if [ -z "$IS_CI" ]; then - systemctl restart postgresql - else - CI_start_postgres - fi - exit "$EXIT_CODE" } From ecff04786ef79c74c812e21a75447b56a6790ff5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 17:14:25 -0400 Subject: [PATCH 16/32] chore: bump for testing --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 31cf89b7b..858ab7d14 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.078-orioledb-upgrade-8" - postgres17: "17.4.1.028-upgrade-8" - postgres15: "15.8.1.085-upgrade-8" + postgresorioledb-17: "17.0.1.078-orioledb-upgrade-9" + postgres17: "17.4.1.028-upgrade-9" + postgres15: "15.8.1.085-upgrade-9" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 8d2ccb351d8304f57c49e28cc1df9db4675480d8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 9 May 2025 12:57:42 -0400 Subject: [PATCH 17/32] chore: stash code --- .../pg_upgrade_scripts/initiate.sh | 68 +++++++++-- nix/docs/testing-pg-upgrade-scripts.md | 115 ++++++++++++++++++ 2 files changed, 171 insertions(+), 12 deletions(-) create mode 100644 nix/docs/testing-pg-upgrade-scripts.md diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index ba0c3c162..82353e7ea 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -52,7 +52,7 @@ fi SERVER_ENCODING=$(run_sql -A -t -c "SHOW server_encoding;") POSTGRES_CONFIG_PATH="/etc/postgresql/postgresql.conf" -PGBINOLD="/usr/lib/postgresql/bin" +PGBINOLD="$(pg_config --bindir)" PG_UPGRADE_BIN_DIR="/tmp/pg_upgrade_bin/$PGVERSION" NIX_INSTALLER_PATH="/tmp/persistent/nix-installer" @@ -242,6 +242,10 @@ function initiate_upgrade { fi PGDATAOLD=$(cat "$POSTGRES_CONFIG_PATH" | grep data_directory | sed "s/data_directory = '\(.*\)'.*/\1/") + # Resolve symlink if it exists + if [ -L "$PGDATAOLD" ]; then + PGDATAOLD=$(readlink -f "$PGDATAOLD") + fi PGDATANEW="$MOUNT_POINT/pgdata" @@ -255,7 +259,12 @@ function initiate_upgrade { if [ -n "$IS_LOCAL_UPGRADE" ]; then mkdir -p "$PG_UPGRADE_BIN_DIR" mkdir -p /tmp/persistent/ - echo "a7189a68ed4ea78c1e73991b5f271043636cf074" > "$PG_UPGRADE_BIN_DIR/nix_flake_version" + # Use NIX_FLAKE_VERSION if set, otherwise use default + if [ -n "$NIX_FLAKE_VERSION" ]; then + echo "$NIX_FLAKE_VERSION" > "$PG_UPGRADE_BIN_DIR/nix_flake_version" + else + echo "a7189a68ed4ea78c1e73991b5f271043636cf074" > "$PG_UPGRADE_BIN_DIR/nix_flake_version" + fi tar -czf "/tmp/persistent/pg_upgrade_bin.tar.gz" -C "/tmp/pg_upgrade_bin" . rm -rf /tmp/pg_upgrade_bin/ fi @@ -320,6 +329,7 @@ function initiate_upgrade { # upgrade job outputs a log in the cwd; needs write permissions mkdir -p /tmp/pg_upgrade/ chown -R postgres:postgres /tmp/pg_upgrade/ + chmod 0700 /tmp/pg_upgrade/ cd /tmp/pg_upgrade/ # Fixing erros generated by previous dpkg executions (package upgrades et co) @@ -417,25 +427,51 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf run_sql -c "select pg_reload_conf();" fi + # Build the base upgrade command UPGRADE_COMMAND=$(cat <> "$TMP_CONFIG" + chown postgres:postgres "$TMP_CONFIG" + + UPGRADE_COMMAND="$UPGRADE_COMMAND \ + --old-options='-c config_file=$TMP_CONFIG' \ + --new-options='-c max_slot_wal_keep_size=-1' \ + --new-options='-c unix_socket_directories=/tmp/pg_upgrade'" + else + UPGRADE_COMMAND="$UPGRADE_COMMAND \ + --old-options='-c config_file=${POSTGRES_CONFIG_PATH}'" + fi + if [ "$IS_NIX_BASED_SYSTEM" = "true" ]; then - UPGRADE_COMMAND=". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $UPGRADE_COMMAND" - fi + # Ensure nix environment is loaded for both old and new instances + UPGRADE_COMMAND=". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && PGBINOLD=${PGBINOLD} $UPGRADE_COMMAND" + fi + + # Set max_slot_wal_keep_size in environment for PG17 upgrades + if [[ "$PGVERSION" =~ ^17.* ]]; then + export PGDATA_UPGRADE_MAX_SLOT_WAL_KEEP_SIZE=-1 + fi if [[ "$PGVERSION" =~ ^17.* ]]; then GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres @@ -457,6 +493,14 @@ EOF CI_stop_postgres fi + # Create and set permissions on socket directory for PG17 upgrades + if [[ "$PGVERSION" =~ ^17.* ]]; then + mkdir -p /tmp/pg_upgrade + chown postgres:postgres /tmp/pg_upgrade + chmod 0700 /tmp/pg_upgrade + fi + + # Start the old PostgreSQL instance with version-specific options if [[ "$PGVERSION" =~ ^17.* ]]; then GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres else diff --git a/nix/docs/testing-pg-upgrade-scripts.md b/nix/docs/testing-pg-upgrade-scripts.md new file mode 100644 index 000000000..49c966b5f --- /dev/null +++ b/nix/docs/testing-pg-upgrade-scripts.md @@ -0,0 +1,115 @@ +# Testing PostgreSQL Upgrade Scripts + +This document describes how to test changes to the PostgreSQL upgrade scripts on a running machine. + +## Prerequisites + +- A running PostgreSQL instance +- Access to the Supabase Postgres repository +- Permissions to run GitHub Actions workflows +- ssh access to the ec2 instance + +## Development Workflow + +1. **Make Changes to Upgrade Scripts** + - Make your changes to the scripts in `ansible/files/admin_api_scripts/pg_upgrade_scripts/` + - Commit and push your changes to your feature branch + - For quick testing, you can also edit the script directly on the server at `/etc/adminapi/pg_upgrade_scripts/initiate.sh` + +2. **Publish Script Changes** (Only needed for deploying to new instances) + - Go to [publish-nix-pgupgrade-scripts.yml](https://github.com/supabase/postgres/actions/workflows/publish-nix-pgupgrade-scripts.yml) + - Click "Run workflow" + - Select your branch + - Run the workflow + +3. **Publish Binary Flake Version** (Only needed for deploying to new instances) + - Go to [publish-nix-pgupgrade-bin-flake-version.yml](https://github.com/supabase/postgres/actions/workflows/publish-nix-pgupgrade-bin-flake-version.yml) + - Click "Run workflow" + - Select your branch + - Run the workflow + - Note: Make sure the flake version includes the PostgreSQL version you're testing (e.g., 17) + +4. **Test on Running Machine** + ssh into the machine + ```bash + # Stop PostgreSQL + sudo systemctl stop postgresql + + # Run the upgrade script in local mode with your desired flake version + sudo NIX_FLAKE_VERSION="your-flake-version-here" IS_LOCAL_UPGRADE=true /etc/adminapi/pg_upgrade_scripts/initiate.sh 17 + ``` + Note: This will use the version of the script that exists at `/etc/adminapi/pg_upgrade_scripts/initiate.sh` on the server. + The script should be run as the ubuntu user with sudo privileges. The script will handle switching to the postgres user when needed. + + In local mode: + - The script at `/etc/adminapi/pg_upgrade_scripts/initiate.sh` will be used (your edited version) + - Only the PostgreSQL binaries will be downloaded from the specified flake version + - No new upgrade scripts will be downloaded + - You can override the flake version by setting the NIX_FLAKE_VERSION environment variable + - If NIX_FLAKE_VERSION is not set, it will use the default flake version + +5. **Monitor Progress** + ```bash + # Watch the upgrade log + tail -f /var/log/pg-upgrade-initiate.log + ``` + +6. **Check Results** + In local mode, the script will: + - Create a new data directory at `/data_migration/pgdata` + - Run pg_upgrade to test the upgrade process + - Generate SQL files in `/data_migration/sql/` for any needed post-upgrade steps + - Log the results in `/var/log/pg-upgrade-initiate.log` + + To verify success: + ```bash + # Check the upgrade log for completion + grep "Upgrade complete" /var/log/pg-upgrade-initiate.log + + # Check for any generated SQL files + ls -l /data_migration/sql/ + + # Check the new data directory + ls -l /data_migration/pgdata/ + ``` + + Note: The instance will not be upgraded to the new version in local mode. This is just a test run to verify the upgrade process works correctly. + +## Important Notes + +- The `IS_LOCAL_UPGRADE=true` flag makes the script run in the foreground and skip disk mounting steps +- The script will use the existing data directory +- All output is logged to `/var/log/pg-upgrade-initiate.log` +- The script will automatically restart PostgreSQL after completion or failure +- For testing, you can edit the script directly on the server - the GitHub Actions workflows are only needed for deploying to new instances +- Run the script as the ubuntu user with sudo privileges - the script will handle user switching internally +- Local mode is for testing only - it will not actually upgrade the instance +- The Nix flake version must include the PostgreSQL version you're testing (e.g., 17) +- In local mode, only the PostgreSQL binaries are downloaded from the flake - the upgrade scripts are used from the local filesystem +- You can override the flake version by setting the NIX_FLAKE_VERSION environment variable when running the script + +## Troubleshooting + +If the upgrade fails: +1. Check the logs at `/var/log/pg-upgrade-initiate.log` +2. Look for any error messages in the PostgreSQL logs +3. The script will attempt to clean up and restore the original state +4. If you see an error about missing Nix flake attributes, make sure the flake version includes the PostgreSQL version you're testing + +Common Errors: +- `error: flake 'github:supabase/postgres/...' does not provide attribute 'packages.aarch64-linux.psql_17/bin'` + - This means the Nix flake version doesn't include PostgreSQL 17 binaries + - You need to specify a flake version that includes your target version + - You can find valid flake versions by looking at the commit history of the publish-nix-pgupgrade-bin-flake-version.yml workflow + +## Cleanup + +After testing: +1. The script will automatically clean up temporary files +2. PostgreSQL will be restarted +3. The original configuration will be restored + +## References + +- [publish-nix-pgupgrade-scripts.yml](https://github.com/supabase/postgres/actions/workflows/publish-nix-pgupgrade-scripts.yml) +- [publish-nix-pgupgrade-bin-flake-version.yml](https://github.com/supabase/postgres/actions/workflows/publish-nix-pgupgrade-bin-flake-version.yml) \ No newline at end of file From 05d8a6fa228899c5349d9ad5f8c05197610375c4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 9 May 2025 14:14:05 -0400 Subject: [PATCH 18/32] feat: working pg 17 upgrade --- .../pg_upgrade_scripts/initiate.sh | 80 +++++++------------ 1 file changed, 29 insertions(+), 51 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 82353e7ea..b3b232f85 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -52,7 +52,7 @@ fi SERVER_ENCODING=$(run_sql -A -t -c "SHOW server_encoding;") POSTGRES_CONFIG_PATH="/etc/postgresql/postgresql.conf" -PGBINOLD="$(pg_config --bindir)" +PGBINOLD="/usr/lib/postgresql/bin" PG_UPGRADE_BIN_DIR="/tmp/pg_upgrade_bin/$PGVERSION" NIX_INSTALLER_PATH="/tmp/persistent/nix-installer" @@ -242,10 +242,6 @@ function initiate_upgrade { fi PGDATAOLD=$(cat "$POSTGRES_CONFIG_PATH" | grep data_directory | sed "s/data_directory = '\(.*\)'.*/\1/") - # Resolve symlink if it exists - if [ -L "$PGDATAOLD" ]; then - PGDATAOLD=$(readlink -f "$PGDATAOLD") - fi PGDATANEW="$MOUNT_POINT/pgdata" @@ -259,12 +255,12 @@ function initiate_upgrade { if [ -n "$IS_LOCAL_UPGRADE" ]; then mkdir -p "$PG_UPGRADE_BIN_DIR" mkdir -p /tmp/persistent/ - # Use NIX_FLAKE_VERSION if set, otherwise use default - if [ -n "$NIX_FLAKE_VERSION" ]; then + if [ -n "$NIX_FLAKE_VERSION" ]; then echo "$NIX_FLAKE_VERSION" > "$PG_UPGRADE_BIN_DIR/nix_flake_version" else echo "a7189a68ed4ea78c1e73991b5f271043636cf074" > "$PG_UPGRADE_BIN_DIR/nix_flake_version" fi + tar -czf "/tmp/persistent/pg_upgrade_bin.tar.gz" -C "/tmp/pg_upgrade_bin" . rm -rf /tmp/pg_upgrade_bin/ fi @@ -329,7 +325,6 @@ function initiate_upgrade { # upgrade job outputs a log in the cwd; needs write permissions mkdir -p /tmp/pg_upgrade/ chown -R postgres:postgres /tmp/pg_upgrade/ - chmod 0700 /tmp/pg_upgrade/ cd /tmp/pg_upgrade/ # Fixing erros generated by previous dpkg executions (package upgrades et co) @@ -415,6 +410,7 @@ function initiate_upgrade { fi else su -c "$PGBINNEW/initdb -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres + fi # This line avoids the need to supply the supabase_admin password on the old @@ -427,51 +423,39 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf run_sql -c "select pg_reload_conf();" fi - # Build the base upgrade command + TMP_CONFIG="/tmp/pg_upgrade/postgresql.conf" + cp "$POSTGRES_CONFIG_PATH" "$TMP_CONFIG" + + # Check if max_slot_wal_keep_size exists in the config + if grep -q "max_slot_wal_keep_size" "$TMP_CONFIG"; then + # Find and replace the existing setting + sed -i 's/^\s*max_slot_wal_keep_size\s*=.*$/max_slot_wal_keep_size = -1/' "$TMP_CONFIG" + else + # Add the setting if not found + echo "max_slot_wal_keep_size = -1" >> "$TMP_CONFIG" + fi + + chown postgres:postgres "$TMP_CONFIG" + UPGRADE_COMMAND=$(cat <> "$TMP_CONFIG" - chown postgres:postgres "$TMP_CONFIG" - - UPGRADE_COMMAND="$UPGRADE_COMMAND \ - --old-options='-c config_file=$TMP_CONFIG' \ - --new-options='-c max_slot_wal_keep_size=-1' \ - --new-options='-c unix_socket_directories=/tmp/pg_upgrade'" - else - UPGRADE_COMMAND="$UPGRADE_COMMAND \ - --old-options='-c config_file=${POSTGRES_CONFIG_PATH}'" - fi - if [ "$IS_NIX_BASED_SYSTEM" = "true" ]; then - # Ensure nix environment is loaded for both old and new instances - UPGRADE_COMMAND=". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && PGBINOLD=${PGBINOLD} $UPGRADE_COMMAND" - fi - - # Set max_slot_wal_keep_size in environment for PG17 upgrades - if [[ "$PGVERSION" =~ ^17.* ]]; then - export PGDATA_UPGRADE_MAX_SLOT_WAL_KEEP_SIZE=-1 - fi + UPGRADE_COMMAND=". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $UPGRADE_COMMAND" + fi if [[ "$PGVERSION" =~ ^17.* ]]; then GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres @@ -493,13 +477,6 @@ EOF CI_stop_postgres fi - # Create and set permissions on socket directory for PG17 upgrades - if [[ "$PGVERSION" =~ ^17.* ]]; then - mkdir -p /tmp/pg_upgrade - chown postgres:postgres /tmp/pg_upgrade - chmod 0700 /tmp/pg_upgrade - fi - # Start the old PostgreSQL instance with version-specific options if [[ "$PGVERSION" =~ ^17.* ]]; then GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres @@ -507,6 +484,7 @@ EOF GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres fi + # copying custom configurations echo "11. Copying custom configurations" mkdir -p "$MOUNT_POINT/conf" cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/" From 7876ed8cf4f639bdb3bfe5c3d4e46c3636e82ba8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 9 May 2025 14:15:45 -0400 Subject: [PATCH 19/32] chore: bump version --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 858ab7d14..c03eaf10c 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.078-orioledb-upgrade-9" - postgres17: "17.4.1.028-upgrade-9" - postgres15: "15.8.1.085-upgrade-9" + postgresorioledb-17: "17.0.1.078-orioledb-upgrade-10" + postgres17: "17.4.1.028-upgrade-10" + postgres15: "15.8.1.085-upgrade-10" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 1352205b0b98f0425a17e95d17cb0a3d32002761 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 9 May 2025 19:37:06 -0400 Subject: [PATCH 20/32] feat: pg 15 handling --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index b3b232f85..3a1f770d2 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -435,6 +435,11 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf echo "max_slot_wal_keep_size = -1" >> "$TMP_CONFIG" fi + # Remove db_user_namespace if upgrading from PG15 + if [[ "$OLD_PGVERSION" =~ ^15.* ]]; then + sed -i '/^db_user_namespace/d' "$TMP_CONFIG" + fi + chown postgres:postgres "$TMP_CONFIG" UPGRADE_COMMAND=$(cat < Date: Fri, 9 May 2025 19:41:04 -0400 Subject: [PATCH 21/32] chore: bump versions --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index c03eaf10c..5b702bbad 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.078-orioledb-upgrade-10" - postgres17: "17.4.1.028-upgrade-10" - postgres15: "15.8.1.085-upgrade-10" + postgresorioledb-17: "17.0.1.078-orioledb-upgrade-11" + postgres17: "17.4.1.028-upgrade-11" + postgres15: "15.8.1.085-upgrade-11" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 7a38abbfe795156e5ec2b179aa530d9b7f056c09 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Sat, 10 May 2025 09:36:09 -0400 Subject: [PATCH 22/32] chore: versions for release --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 5b702bbad..4821ee48b 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.078-orioledb-upgrade-11" - postgres17: "17.4.1.028-upgrade-11" - postgres15: "15.8.1.085-upgrade-11" + postgresorioledb-17: "17.0.1.079-orioledb" + postgres17: "17.4.1.029" + postgres15: "15.8.1.086" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 6754a60563eec2622d12485380e988772c6f5214 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Sat, 10 May 2025 13:00:33 -0400 Subject: [PATCH 23/32] feat: rm oriole handling, refine 15 -> 17 config --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 3a1f770d2..1c5612e34 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -43,8 +43,8 @@ POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" POST_UPGRADE_POSTGRES_PERMS_SCRIPT="/tmp/pg_upgrade/pg_upgrade_postgres_perms.sql" OLD_PGVERSION=$(pg_config --version | sed 's/PostgreSQL \([0-9]*\.[0-9]*\).*/\1/') -# Skip locale settings if both versions are PostgreSQL 17+ or 17-orioledb -if ! [[ (("$OLD_PGVERSION" =~ ^17.* || "$OLD_PGVERSION" == "17-orioledb") && ("$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb")) ]]; then +# Skip locale settings if both versions are PostgreSQL 17+ +if ! [[ "$OLD_PGVERSION" =~ ^17.* && "$PGVERSION" =~ ^17.* ]]; then SERVER_LC_COLLATE=$(run_sql -A -t -c "SHOW lc_collate;") SERVER_LC_CTYPE=$(run_sql -A -t -c "SHOW lc_ctype;") fi @@ -436,7 +436,7 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf fi # Remove db_user_namespace if upgrading from PG15 - if [[ "$OLD_PGVERSION" =~ ^15.* ]]; then + if [[ "$OLD_PGVERSION" =~ ^15.* && "$PGVERSION" =~ ^17.* ]]; then sed -i '/^db_user_namespace/d' "$TMP_CONFIG" fi From b81245b29269932a1d49c4260bd82530cbb66867 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Sat, 10 May 2025 13:01:25 -0400 Subject: [PATCH 24/32] chore: bump ver for testing --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 4821ee48b..7e0fb2bef 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.079-orioledb" - postgres17: "17.4.1.029" - postgres15: "15.8.1.086" + postgresorioledb-17: "17.0.1.079-orioledb-upgrade-12" + postgres17: "17.4.1.029-upgrade-12" + postgres15: "15.8.1.086-upgrade-12" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 888bd4736b489bd609e2c83d97df3618d6407385 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Sat, 10 May 2025 21:14:31 -0400 Subject: [PATCH 25/32] feat: make sure old pg stops if not force stop --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 1c5612e34..08ac37233 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -478,6 +478,13 @@ EOF sleep 3 systemctl stop postgresql + + # Additional check to ensure postgres is really stopped + if [ -f "${PGDATAOLD}/postmaster.pid" ]; then + echo "PostgreSQL still running, forcing stop..." + kill -9 $(head -n 1 "${PGDATAOLD}/postmaster.pid") || true + rm -f "${PGDATAOLD}/postmaster.pid" + fi else CI_stop_postgres fi From c27f98709938be6f3863bb0756ef183770fa1309 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Sat, 10 May 2025 21:15:32 -0400 Subject: [PATCH 26/32] chore: bump version --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 7e0fb2bef..715a4cc57 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.079-orioledb-upgrade-12" - postgres17: "17.4.1.029-upgrade-12" - postgres15: "15.8.1.086-upgrade-12" + postgresorioledb-17: "17.0.1.079-orioledb-upgrade-13" + postgres17: "17.4.1.029-upgrade-13" + postgres15: "15.8.1.086-upgrade-13" # Non Postgres Extensions pgbouncer_release: "1.19.0" From be98ca1c9c12d52cba507def991b8efd467cc0b2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 12 May 2025 09:29:00 -0400 Subject: [PATCH 27/32] chore: cleanup + bump version for test --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 3 ++- ansible/vars.yml | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 08ac37233..b576b6fd7 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -482,7 +482,8 @@ EOF # Additional check to ensure postgres is really stopped if [ -f "${PGDATAOLD}/postmaster.pid" ]; then echo "PostgreSQL still running, forcing stop..." - kill -9 $(head -n 1 "${PGDATAOLD}/postmaster.pid") || true + pid=$(head -n 1 "${PGDATAOLD}/postmaster.pid") + kill -9 "$pid" || true rm -f "${PGDATAOLD}/postmaster.pid" fi else diff --git a/ansible/vars.yml b/ansible/vars.yml index 715a4cc57..a6a6f623a 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.079-orioledb-upgrade-13" - postgres17: "17.4.1.029-upgrade-13" - postgres15: "15.8.1.086-upgrade-13" + postgresorioledb-17: "17.0.1.079-orioledb-upgrade-14" + postgres17: "17.4.1.029-upgrade-14" + postgres15: "15.8.1.086-upgrade-14" # Non Postgres Extensions pgbouncer_release: "1.19.0" From ed3ea7811df1ef23f18200afd9fe67c9e02b67eb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 12 May 2025 15:09:05 -0400 Subject: [PATCH 28/32] chore: release bump --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index a6a6f623a..4821ee48b 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.079-orioledb-upgrade-14" - postgres17: "17.4.1.029-upgrade-14" - postgres15: "15.8.1.086-upgrade-14" + postgresorioledb-17: "17.0.1.079-orioledb" + postgres17: "17.4.1.029" + postgres15: "15.8.1.086" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 600c1bce25e4a89d2db137d32c9259002b65e4a2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 14 May 2025 11:14:43 -0400 Subject: [PATCH 29/32] feat: no longer support pg12, try to stop gracefully no matter what --- .../pg_upgrade_scripts/initiate.sh | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index b576b6fd7..ed621685a 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -469,22 +469,26 @@ EOF fi echo "10. Stopping postgres; running pg_upgrade" - # Extra work to ensure postgres is actually stopped - # Mostly needed for PG12 projects with odd systemd unit behavior + # Stop PostgreSQL service if [ -z "$IS_CI" ]; then - retry 5 systemctl restart postgresql + echo "Stopping PostgreSQL service..." + + # Disable service first so it doesn't restart automatically systemctl disable postgresql - retry 5 systemctl stop postgresql - - sleep 3 - systemctl stop postgresql - # Additional check to ensure postgres is really stopped - if [ -f "${PGDATAOLD}/postmaster.pid" ]; then - echo "PostgreSQL still running, forcing stop..." - pid=$(head -n 1 "${PGDATAOLD}/postmaster.pid") - kill -9 "$pid" || true - rm -f "${PGDATAOLD}/postmaster.pid" + # Use systemctl stop with a longer timeout + if ! systemctl stop -t 120 postgresql; then + echo "Standard stop failed, trying one more time..." + sleep 5 + systemctl stop -t 120 postgresql || true + fi + + # Quick verification + if systemctl is-active --quiet postgresql; then + echo "WARNING: PostgreSQL service still active after stop commands" + echo "You may need to investigate what's preventing proper shutdown" + else + echo "PostgreSQL successfully stopped" fi else CI_stop_postgres From 0d531253a03cad0528f9bfacc4c1692b21cab63f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 14 May 2025 11:15:44 -0400 Subject: [PATCH 30/32] chore: bump version for testing --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 4821ee48b..06e02f009 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.079-orioledb" - postgres17: "17.4.1.029" - postgres15: "15.8.1.086" + postgresorioledb-17: "17.0.1.079-orioledb-pgupgrade-1" + postgres17: "17.4.1.029-pgupgrade-1" + postgres15: "15.8.1.086-pgupgrade-1" # Non Postgres Extensions pgbouncer_release: "1.19.0" From be24deab04735ff1d9a5f40394d84a8ecba08d6a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 14 May 2025 11:31:03 -0400 Subject: [PATCH 31/32] chore: always apply -1 setting and bump versions --- .../admin_api_scripts/pg_upgrade_scripts/initiate.sh | 9 +-------- ansible/vars.yml | 6 +++--- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index ed621685a..a5d23439c 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -426,14 +426,7 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf TMP_CONFIG="/tmp/pg_upgrade/postgresql.conf" cp "$POSTGRES_CONFIG_PATH" "$TMP_CONFIG" - # Check if max_slot_wal_keep_size exists in the config - if grep -q "max_slot_wal_keep_size" "$TMP_CONFIG"; then - # Find and replace the existing setting - sed -i 's/^\s*max_slot_wal_keep_size\s*=.*$/max_slot_wal_keep_size = -1/' "$TMP_CONFIG" - else - # Add the setting if not found - echo "max_slot_wal_keep_size = -1" >> "$TMP_CONFIG" - fi + echo "max_slot_wal_keep_size = -1" >> "$TMP_CONFIG" # Remove db_user_namespace if upgrading from PG15 if [[ "$OLD_PGVERSION" =~ ^15.* && "$PGVERSION" =~ ^17.* ]]; then diff --git a/ansible/vars.yml b/ansible/vars.yml index 06e02f009..497af503c 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.079-orioledb-pgupgrade-1" - postgres17: "17.4.1.029-pgupgrade-1" - postgres15: "15.8.1.086-pgupgrade-1" + postgresorioledb-17: "17.0.1.079-orioledb-pgupgrade-2" + postgres17: "17.4.1.029-pgupgrade-2" + postgres15: "15.8.1.086-pgupgrade-2" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 4c393d7f717c0f7c7d17e49557e04aac03c0c2d7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 14 May 2025 13:38:09 -0400 Subject: [PATCH 32/32] feat: remove timeout flag bump version for testing --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 4 ++-- ansible/vars.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index a5d23439c..db67418e5 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -470,10 +470,10 @@ EOF systemctl disable postgresql # Use systemctl stop with a longer timeout - if ! systemctl stop -t 120 postgresql; then + if ! systemctl stop postgresql; then echo "Standard stop failed, trying one more time..." sleep 5 - systemctl stop -t 120 postgresql || true + systemctl stop postgresql || true fi # Quick verification diff --git a/ansible/vars.yml b/ansible/vars.yml index 497af503c..391951612 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.079-orioledb-pgupgrade-2" - postgres17: "17.4.1.029-pgupgrade-2" - postgres15: "15.8.1.086-pgupgrade-2" + postgresorioledb-17: "17.0.1.079-orioledb-pgupgrade-3" + postgres17: "17.4.1.029-pgupgrade-3" + postgres15: "15.8.1.086-pgupgrade-3" # Non Postgres Extensions pgbouncer_release: "1.19.0"