diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-ami-build.yml similarity index 100% rename from .github/workflows/testinfra-nix.yml rename to .github/workflows/testinfra-ami-build.yml diff --git a/.gitignore b/.gitignore index 005d3ece6..d32cc8f60 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ result* .vscode/ db/schema.sql +common-nix.vars.pkr.hcl diff --git a/flake.nix b/flake.nix index 70810e194..d49c9e3fa 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ rust-overlay.url = "github:oxalica/rust-overlay"; }; - outputs = { self, nixpkgs, flake-utils, nix2container, nix-editor, rust-overlay, ...}: + outputs = { self, nixpkgs, flake-utils, nix-editor, rust-overlay, nix2container, ... }: let gitRev = "vcs=${self.shortRev or "dirty"}+${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}"; @@ -23,14 +23,13 @@ let pgsqlDefaultPort = "5435"; pgsqlSuperuser = "supabase_admin"; - nix2img = nix2container.packages.${system}.nix2container; pkgs = import nixpkgs { - config = { + config = { allowUnfree = true; permittedInsecurePackages = [ "v8-9.7.106.18" - ]; + ]; }; inherit system; overlays = [ @@ -84,6 +83,16 @@ }) ]; }; + # Define pythonEnv here + pythonEnv = pkgs.python3.withPackages (ps: with ps; [ + boto3 + docker + pytest + pytest-testinfra + requests + ec2instanceconnectcli + paramiko + ]); sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { }; supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { }; mecab-naist-jdic = pkgs.callPackage ./nix/ext/mecab-naist-jdic/default.nix { }; @@ -150,15 +159,16 @@ #Where we import and build the orioledb extension, we add on our custom extensions # plus the orioledb option #we're not using timescaledb or plv8 in the orioledb-17 version or pg 17 of supabase extensions - orioleFilteredExtensions = builtins.filter ( - x: + orioleFilteredExtensions = builtins.filter + ( + x: x != ./nix/ext/timescaledb.nix && x != ./nix/ext/timescaledb-2.9.1.nix && x != ./nix/ext/plv8.nix ) ourExtensions; orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; - dbExtensions17 = orioleFilteredExtensions; + dbExtensions17 = orioleFilteredExtensions; getPostgresqlPackage = version: pkgs.postgresql."postgresql_${version}"; # Create a 'receipt' file for a given postgresql package. This is a way @@ -196,14 +206,16 @@ }; makeOurPostgresPkgs = version: - let + let postgresql = getPostgresqlPackage version; - extensionsToUse = if (builtins.elem version ["orioledb-17"]) + extensionsToUse = + if (builtins.elem version [ "orioledb-17" ]) then orioledbExtensions - else if (builtins.elem version ["17"]) - then dbExtensions17 + else if (builtins.elem version [ "17" ]) + then dbExtensions17 else ourExtensions; - in map (path: pkgs.callPackage path { inherit postgresql; }) extensionsToUse; + in + map (path: pkgs.callPackage path { inherit postgresql; }) extensionsToUse; # Create an attrset that contains all the extensions included in a server. makeOurPostgresPkgsSet = version: @@ -252,372 +264,841 @@ recurseForDerivations = true; }; - makePostgresDevSetup = { pkgs, name, extraSubstitutions ? {} }: - let - paths = { - migrationsDir = builtins.path { - name = "migrations"; - path = ./migrations/db; - }; - postgresqlSchemaSql = builtins.path { - name = "postgresql-schema"; - path = ./nix/tools/postgresql_schema.sql; - }; - pgbouncerAuthSchemaSql = builtins.path { - name = "pgbouncer-auth-schema"; - path = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; - }; - statExtensionSql = builtins.path { - name = "stat-extension"; - path = ./ansible/files/stat_extension.sql; - }; - pgconfigFile = builtins.path { - name = "postgresql.conf"; - path = ./ansible/files/postgresql_config/postgresql.conf.j2; - }; - supautilsConfigFile = builtins.path { - name = "supautils.conf"; - path = ./ansible/files/postgresql_config/supautils.conf.j2; - }; - loggingConfigFile = builtins.path { - name = "logging.conf"; - path = ./ansible/files/postgresql_config/postgresql-csvlog.conf; - }; - readReplicaConfigFile = builtins.path { - name = "readreplica.conf"; - path = ./ansible/files/postgresql_config/custom_read_replica.conf.j2; - }; - pgHbaConfigFile = builtins.path { - name = "pg_hba.conf"; - path = ./ansible/files/postgresql_config/pg_hba.conf.j2; - }; - pgIdentConfigFile = builtins.path { - name = "pg_ident.conf"; - path = ./ansible/files/postgresql_config/pg_ident.conf.j2; - }; - postgresqlExtensionCustomScriptsPath = builtins.path { - name = "extension-custom-scripts"; - path = ./ansible/files/postgresql_extension_custom_scripts; - }; - getkeyScript = builtins.path { - name = "pgsodium_getkey.sh"; - path = ./nix/tests/util/pgsodium_getkey.sh; + makePostgresDevSetup = { pkgs, name, extraSubstitutions ? { } }: + let + paths = { + migrationsDir = builtins.path { + name = "migrations"; + path = ./migrations/db; + }; + postgresqlSchemaSql = builtins.path { + name = "postgresql-schema"; + path = ./nix/tools/postgresql_schema.sql; + }; + pgbouncerAuthSchemaSql = builtins.path { + name = "pgbouncer-auth-schema"; + path = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; + }; + statExtensionSql = builtins.path { + name = "stat-extension"; + path = ./ansible/files/stat_extension.sql; + }; + pgconfigFile = builtins.path { + name = "postgresql.conf"; + path = ./ansible/files/postgresql_config/postgresql.conf.j2; + }; + supautilsConfigFile = builtins.path { + name = "supautils.conf"; + path = ./ansible/files/postgresql_config/supautils.conf.j2; + }; + loggingConfigFile = builtins.path { + name = "logging.conf"; + path = ./ansible/files/postgresql_config/postgresql-csvlog.conf; + }; + readReplicaConfigFile = builtins.path { + name = "readreplica.conf"; + path = ./ansible/files/postgresql_config/custom_read_replica.conf.j2; + }; + pgHbaConfigFile = builtins.path { + name = "pg_hba.conf"; + path = ./ansible/files/postgresql_config/pg_hba.conf.j2; + }; + pgIdentConfigFile = builtins.path { + name = "pg_ident.conf"; + path = ./ansible/files/postgresql_config/pg_ident.conf.j2; + }; + postgresqlExtensionCustomScriptsPath = builtins.path { + name = "extension-custom-scripts"; + path = ./ansible/files/postgresql_extension_custom_scripts; + }; + getkeyScript = builtins.path { + name = "pgsodium_getkey.sh"; + path = ./nix/tests/util/pgsodium_getkey.sh; + }; }; - }; - - localeArchive = if pkgs.stdenv.isDarwin - then "${pkgs.darwin.locale}/share/locale" - else "${pkgs.glibcLocales}/lib/locale/locale-archive"; - - substitutions = { - SHELL_PATH = "${pkgs.bash}/bin/bash"; - PGSQL_DEFAULT_PORT = "${pgsqlDefaultPort}"; - PGSQL_SUPERUSER = "${pgsqlSuperuser}"; - PSQL15_BINDIR = "${basePackages.psql_15.bin}"; - PSQL17_BINDIR = "${basePackages.psql_17.bin}"; - PSQL_CONF_FILE = "${paths.pgconfigFile}"; - PSQLORIOLEDB17_BINDIR = "${basePackages.psql_orioledb-17.bin}"; - PGSODIUM_GETKEY = "${paths.getkeyScript}"; - READREPL_CONF_FILE = "${paths.readReplicaConfigFile}"; - LOGGING_CONF_FILE = "${paths.loggingConfigFile}"; - SUPAUTILS_CONF_FILE = "${paths.supautilsConfigFile}"; - PG_HBA = "${paths.pgHbaConfigFile}"; - PG_IDENT = "${paths.pgIdentConfigFile}"; - LOCALES = "${localeArchive}"; - EXTENSION_CUSTOM_SCRIPTS_DIR = "${paths.postgresqlExtensionCustomScriptsPath}"; - MECAB_LIB = "${basePackages.psql_15.exts.pgroonga}/lib/groonga/plugins/tokenizers/tokenizer_mecab.so"; - GROONGA_DIR = "${supabase-groonga}"; - MIGRATIONS_DIR = "${paths.migrationsDir}"; - POSTGRESQL_SCHEMA_SQL = "${paths.postgresqlSchemaSql}"; - PGBOUNCER_AUTH_SCHEMA_SQL = "${paths.pgbouncerAuthSchemaSql}"; - STAT_EXTENSION_SQL = "${paths.statExtensionSql}"; - CURRENT_SYSTEM = "${system}"; - } // extraSubstitutions; # Merge in any extra substitutions - in pkgs.runCommand name { - inherit (paths) migrationsDir postgresqlSchemaSql pgbouncerAuthSchemaSql statExtensionSql; - } '' - set -x - mkdir -p $out/bin $out/etc/postgresql-custom $out/etc/postgresql $out/extension-custom-scripts + + localeArchive = + if pkgs.stdenv.isDarwin + then "${pkgs.darwin.locale}/share/locale" + else "${pkgs.glibcLocales}/lib/locale/locale-archive"; + + substitutions = { + SHELL_PATH = "${pkgs.bash}/bin/bash"; + PGSQL_DEFAULT_PORT = "${pgsqlDefaultPort}"; + PGSQL_SUPERUSER = "${pgsqlSuperuser}"; + PSQL15_BINDIR = "${basePackages.psql_15.bin}"; + PSQL17_BINDIR = "${basePackages.psql_17.bin}"; + PSQL_CONF_FILE = "${paths.pgconfigFile}"; + PSQLORIOLEDB17_BINDIR = "${basePackages.psql_orioledb-17.bin}"; + PGSODIUM_GETKEY = "${paths.getkeyScript}"; + READREPL_CONF_FILE = "${paths.readReplicaConfigFile}"; + LOGGING_CONF_FILE = "${paths.loggingConfigFile}"; + SUPAUTILS_CONF_FILE = "${paths.supautilsConfigFile}"; + PG_HBA = "${paths.pgHbaConfigFile}"; + PG_IDENT = "${paths.pgIdentConfigFile}"; + LOCALES = "${localeArchive}"; + EXTENSION_CUSTOM_SCRIPTS_DIR = "${paths.postgresqlExtensionCustomScriptsPath}"; + MECAB_LIB = "${basePackages.psql_15.exts.pgroonga}/lib/groonga/plugins/tokenizers/tokenizer_mecab.so"; + GROONGA_DIR = "${supabase-groonga}"; + MIGRATIONS_DIR = "${paths.migrationsDir}"; + POSTGRESQL_SCHEMA_SQL = "${paths.postgresqlSchemaSql}"; + PGBOUNCER_AUTH_SCHEMA_SQL = "${paths.pgbouncerAuthSchemaSql}"; + STAT_EXTENSION_SQL = "${paths.statExtensionSql}"; + CURRENT_SYSTEM = "${system}"; + } // extraSubstitutions; # Merge in any extra substitutions + in + pkgs.runCommand name + { + inherit (paths) migrationsDir postgresqlSchemaSql pgbouncerAuthSchemaSql statExtensionSql; + } '' + set -x + mkdir -p $out/bin $out/etc/postgresql-custom $out/etc/postgresql $out/extension-custom-scripts - # Copy config files with error handling - cp ${paths.supautilsConfigFile} $out/etc/postgresql-custom/supautils.conf || { echo "Failed to copy supautils.conf"; exit 1; } - cp ${paths.pgconfigFile} $out/etc/postgresql/postgresql.conf || { echo "Failed to copy postgresql.conf"; exit 1; } - cp ${paths.loggingConfigFile} $out/etc/postgresql-custom/logging.conf || { echo "Failed to copy logging.conf"; exit 1; } - cp ${paths.readReplicaConfigFile} $out/etc/postgresql-custom/read-replica.conf || { echo "Failed to copy read-replica.conf"; exit 1; } - cp ${paths.pgHbaConfigFile} $out/etc/postgresql/pg_hba.conf || { echo "Failed to copy pg_hba.conf"; exit 1; } - cp ${paths.pgIdentConfigFile} $out/etc/postgresql/pg_ident.conf || { echo "Failed to copy pg_ident.conf"; exit 1; } - cp -r ${paths.postgresqlExtensionCustomScriptsPath}/* $out/extension-custom-scripts/ || { echo "Failed to copy custom scripts"; exit 1; } + # Copy config files with error handling + cp ${paths.supautilsConfigFile} $out/etc/postgresql-custom/supautils.conf || { echo "Failed to copy supautils.conf"; exit 1; } + cp ${paths.pgconfigFile} $out/etc/postgresql/postgresql.conf || { echo "Failed to copy postgresql.conf"; exit 1; } + cp ${paths.loggingConfigFile} $out/etc/postgresql-custom/logging.conf || { echo "Failed to copy logging.conf"; exit 1; } + cp ${paths.readReplicaConfigFile} $out/etc/postgresql-custom/read-replica.conf || { echo "Failed to copy read-replica.conf"; exit 1; } + cp ${paths.pgHbaConfigFile} $out/etc/postgresql/pg_hba.conf || { echo "Failed to copy pg_hba.conf"; exit 1; } + cp ${paths.pgIdentConfigFile} $out/etc/postgresql/pg_ident.conf || { echo "Failed to copy pg_ident.conf"; exit 1; } + cp -r ${paths.postgresqlExtensionCustomScriptsPath}/* $out/extension-custom-scripts/ || { echo "Failed to copy custom scripts"; exit 1; } - echo "Copy operation completed" - chmod 644 $out/etc/postgresql-custom/supautils.conf - chmod 644 $out/etc/postgresql/postgresql.conf - chmod 644 $out/etc/postgresql-custom/logging.conf - chmod 644 $out/etc/postgresql/pg_hba.conf - - substitute ${./nix/tools/run-server.sh.in} $out/bin/start-postgres-server \ - ${builtins.concatStringsSep " " (builtins.attrValues (builtins.mapAttrs - (name: value: "--subst-var-by '${name}' '${value}'") - substitutions - ))} - chmod +x $out/bin/start-postgres-server - ''; + echo "Copy operation completed" + chmod 644 $out/etc/postgresql-custom/supautils.conf + chmod 644 $out/etc/postgresql/postgresql.conf + chmod 644 $out/etc/postgresql-custom/logging.conf + chmod 644 $out/etc/postgresql/pg_hba.conf + + substitute ${./nix/tools/run-server.sh.in} $out/bin/start-postgres-server \ + ${builtins.concatStringsSep " " (builtins.attrValues (builtins.mapAttrs + (name: value: "--subst-var-by '${name}' '${value}'") + substitutions + ))} + chmod +x $out/bin/start-postgres-server + ''; # The base set of packages that we export from this Nix Flake, that can # be used with 'nix build'. Don't use the names listed below; check the # name in 'nix flake show' in order to make sure exactly what name you # want. - basePackages = let - # Function to get the PostgreSQL version from the attribute name - getVersion = name: - let - match = builtins.match "psql_([0-9]+)" name; - in - if match == null then null else builtins.head match; - - # Define the available PostgreSQL versions - postgresVersions = { - psql_15 = makePostgres "15"; - psql_17 = makePostgres "17"; - psql_orioledb-17 = makePostgres "orioledb-17" ; - }; + basePackages = + let + # Function to get the PostgreSQL version from the attribute name + getVersion = name: + let + match = builtins.match "psql_([0-9]+)" name; + in + if match == null then null else builtins.head match; + + # Define the available PostgreSQL versions + postgresVersions = { + psql_15 = makePostgres "15"; + psql_17 = makePostgres "17"; + psql_orioledb-17 = makePostgres "orioledb-17"; + }; - # Find the active PostgreSQL version - activeVersion = getVersion (builtins.head (builtins.attrNames postgresVersions)); + # Find the active PostgreSQL version + activeVersion = getVersion (builtins.head (builtins.attrNames postgresVersions)); - # Function to create the pg_regress package - makePgRegress = version: - let - postgresqlPackage = pkgs."postgresql_${version}"; - in - pkgs.callPackage ./nix/ext/pg_regress.nix { + # Function to create the pg_regress package + makePgRegress = version: + let + postgresqlPackage = pkgs."postgresql_${version}"; + in + pkgs.callPackage ./nix/ext/pg_regress.nix { postgresql = postgresqlPackage; }; - postgresql_15 = getPostgresqlPackage "15"; - postgresql_17 = getPostgresqlPackage "17"; - postgresql_orioledb-17 = getPostgresqlPackage "orioledb-17"; - in - postgresVersions // { - supabase-groonga = supabase-groonga; - cargo-pgrx_0_11_3 = pkgs.cargo-pgrx.cargo-pgrx_0_11_3; - cargo-pgrx_0_12_6 = pkgs.cargo-pgrx.cargo-pgrx_0_12_6; - cargo-pgrx_0_12_9 = pkgs.cargo-pgrx.cargo-pgrx_0_12_9; - # PostgreSQL versions. - psql_15 = postgresVersions.psql_15; - psql_17 = postgresVersions.psql_17; - psql_orioledb-17 = postgresVersions.psql_orioledb-17; - wal-g-2 = wal-g-2; - wal-g-3 = wal-g-3; - sfcgal = sfcgal; - pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; - inherit postgresql_15 postgresql_17 postgresql_orioledb-17; - postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; - postgresql_17_debug = if pkgs.stdenv.isLinux then postgresql_17.debug else null; - postgresql_orioledb-17_debug = if pkgs.stdenv.isLinux then postgresql_orioledb-17.debug else null; - postgresql_15_src = pkgs.stdenv.mkDerivation { - pname = "postgresql-15-src"; - version = postgresql_15.version; - - src = postgresql_15.src; - - nativeBuildInputs = [ pkgs.bzip2 ]; - - phases = [ "unpackPhase" "installPhase" ]; - - installPhase = '' - mkdir -p $out - cp -r . $out - ''; + postgresql_15 = getPostgresqlPackage "15"; + postgresql_17 = getPostgresqlPackage "17"; + postgresql_orioledb-17 = getPostgresqlPackage "orioledb-17"; + in + postgresVersions // { + supabase-groonga = supabase-groonga; + cargo-pgrx_0_11_3 = pkgs.cargo-pgrx.cargo-pgrx_0_11_3; + cargo-pgrx_0_12_6 = pkgs.cargo-pgrx.cargo-pgrx_0_12_6; + cargo-pgrx_0_12_9 = pkgs.cargo-pgrx.cargo-pgrx_0_12_9; + # PostgreSQL versions. + psql_15 = postgresVersions.psql_15; + psql_17 = postgresVersions.psql_17; + psql_orioledb-17 = postgresVersions.psql_orioledb-17; + wal-g-2 = wal-g-2; + wal-g-3 = wal-g-3; + sfcgal = sfcgal; + pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; + inherit postgresql_15 postgresql_17 postgresql_orioledb-17; + postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; + postgresql_17_debug = if pkgs.stdenv.isLinux then postgresql_17.debug else null; + postgresql_orioledb-17_debug = if pkgs.stdenv.isLinux then postgresql_orioledb-17.debug else null; + postgresql_15_src = pkgs.stdenv.mkDerivation { + pname = "postgresql-15-src"; + version = postgresql_15.version; + + src = postgresql_15.src; + + nativeBuildInputs = [ pkgs.bzip2 ]; + + phases = [ "unpackPhase" "installPhase" ]; - meta = with pkgs.lib; { - description = "PostgreSQL 15 source files"; - homepage = "https://www.postgresql.org/"; - license = licenses.postgresql; - platforms = platforms.all; + installPhase = '' + mkdir -p $out + cp -r . $out + ''; + + meta = with pkgs.lib; { + description = "PostgreSQL 15 source files"; + homepage = "https://www.postgresql.org/"; + license = licenses.postgresql; + platforms = platforms.all; + }; }; - }; - postgresql_17_src = pkgs.stdenv.mkDerivation { - pname = "postgresql-17-src"; - version = postgresql_17.version; - src = postgresql_17.src; + postgresql_17_src = pkgs.stdenv.mkDerivation { + pname = "postgresql-17-src"; + version = postgresql_17.version; + src = postgresql_17.src; - nativeBuildInputs = [ pkgs.bzip2 ]; + nativeBuildInputs = [ pkgs.bzip2 ]; - phases = [ "unpackPhase" "installPhase" ]; + phases = [ "unpackPhase" "installPhase" ]; - installPhase = '' - mkdir -p $out - cp -r . $out - ''; - meta = with pkgs.lib; { - description = "PostgreSQL 17 source files"; - homepage = "https://www.postgresql.org/"; - license = licenses.postgresql; - platforms = platforms.all; + installPhase = '' + mkdir -p $out + cp -r . $out + ''; + meta = with pkgs.lib; { + description = "PostgreSQL 17 source files"; + homepage = "https://www.postgresql.org/"; + license = licenses.postgresql; + platforms = platforms.all; + }; }; - }; - postgresql_orioledb-17_src = pkgs.stdenv.mkDerivation { - pname = "postgresql-17-src"; - version = postgresql_orioledb-17.version; + postgresql_orioledb-17_src = pkgs.stdenv.mkDerivation { + pname = "postgresql-17-src"; + version = postgresql_orioledb-17.version; - src = postgresql_orioledb-17.src; + src = postgresql_orioledb-17.src; - nativeBuildInputs = [ pkgs.bzip2 ]; + nativeBuildInputs = [ pkgs.bzip2 ]; - phases = [ "unpackPhase" "installPhase" ]; + phases = [ "unpackPhase" "installPhase" ]; - installPhase = '' - mkdir -p $out - cp -r . $out - ''; + installPhase = '' + mkdir -p $out + cp -r . $out + ''; - meta = with pkgs.lib; { - description = "PostgreSQL 15 source files"; - homepage = "https://www.postgresql.org/"; - license = licenses.postgresql; - platforms = platforms.all; + meta = with pkgs.lib; { + description = "PostgreSQL 15 source files"; + homepage = "https://www.postgresql.org/"; + license = licenses.postgresql; + platforms = platforms.all; + }; }; - }; - mecab_naist_jdic = mecab-naist-jdic; - supabase_groonga = supabase-groonga; - pg_regress = makePgRegress activeVersion; - # Start a version of the server. - start-server = makePostgresDevSetup { - inherit pkgs; - name = "start-postgres-server"; - }; + mecab_naist_jdic = mecab-naist-jdic; + supabase_groonga = supabase-groonga; + pg_regress = makePgRegress activeVersion; + # Start a version of the server. + start-server = makePostgresDevSetup { + inherit pkgs; + name = "start-postgres-server"; + }; + + # Start a version of the client and runs migrations script on server. + start-client = + let + migrationsDir = ./migrations/db; + postgresqlSchemaSql = ./nix/tools/postgresql_schema.sql; + pgbouncerAuthSchemaSql = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; + statExtensionSql = ./ansible/files/stat_extension.sql; + in + pkgs.runCommand "start-postgres-client" { } '' + mkdir -p $out/bin + substitute ${./nix/tools/run-client.sh.in} $out/bin/start-postgres-client \ + --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ + --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ + --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ + --subst-var-by 'PSQL17_BINDIR' '${basePackages.psql_17.bin}' \ + --subst-var-by 'PSQLORIOLEDB17_BINDIR' '${basePackages.psql_orioledb-17.bin}' \ + --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \ + --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \ + --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ + --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}' + chmod +x $out/bin/start-postgres-client + ''; + + # Migrate between two data directories. + migrate-tool = + let + configFile = ./nix/tests/postgresql.conf.in; + getkeyScript = ./nix/tests/util/pgsodium_getkey.sh; + primingScript = ./nix/tests/prime.sql; + migrationData = ./nix/tests/migrations/data.sql; + in + pkgs.runCommand "migrate-postgres" { } '' + mkdir -p $out/bin + substitute ${./nix/tools/migrate-tool.sh.in} $out/bin/migrate-postgres \ + --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ + --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \ + --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ + --subst-var-by 'PRIMING_SCRIPT' '${primingScript}' \ + --subst-var-by 'MIGRATION_DATA' '${migrationData}' + + chmod +x $out/bin/migrate-postgres + ''; - # Start a version of the client and runs migrations script on server. - start-client = - let - migrationsDir = ./migrations/db; - postgresqlSchemaSql = ./nix/tools/postgresql_schema.sql; - pgbouncerAuthSchemaSql = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; - statExtensionSql = ./ansible/files/stat_extension.sql; - in - pkgs.runCommand "start-postgres-client" { } '' + start-replica = pkgs.runCommand "start-postgres-replica" { } '' mkdir -p $out/bin - substitute ${./nix/tools/run-client.sh.in} $out/bin/start-postgres-client \ - --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ + substitute ${./nix/tools/run-replica.sh.in} $out/bin/start-postgres-replica \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ - --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ - --subst-var-by 'PSQL17_BINDIR' '${basePackages.psql_17.bin}' \ - --subst-var-by 'PSQLORIOLEDB17_BINDIR' '${basePackages.psql_orioledb-17.bin}' \ - --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \ - --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \ - --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ - --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}' - chmod +x $out/bin/start-postgres-client + --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' + chmod +x $out/bin/start-postgres-replica + ''; + pg-restore = + pkgs.runCommand "run-pg-restore" { } '' + mkdir -p $out/bin + substitute ${./nix/tools/run-restore.sh.in} $out/bin/pg-restore \ + --subst-var-by PSQL15_BINDIR '${basePackages.psql_15.bin}' + chmod +x $out/bin/pg-restore + ''; + sync-exts-versions = pkgs.runCommand "sync-exts-versions" { } '' + mkdir -p $out/bin + substitute ${./nix/tools/sync-exts-versions.sh.in} $out/bin/sync-exts-versions \ + --subst-var-by 'YQ' '${pkgs.yq}/bin/yq' \ + --subst-var-by 'JQ' '${pkgs.jq}/bin/jq' \ + --subst-var-by 'NIX_EDITOR' '${nix-editor.packages.${system}.nix-editor}/bin/nix-editor' \ + --subst-var-by 'NIXPREFETCHURL' '${pkgs.nixVersions.nix_2_20}/bin/nix-prefetch-url' \ + --subst-var-by 'NIX' '${pkgs.nixVersions.nix_2_20}/bin/nix' + chmod +x $out/bin/sync-exts-versions ''; - # Migrate between two data directories. - migrate-tool = - let - configFile = ./nix/tests/postgresql.conf.in; - getkeyScript = ./nix/tests/util/pgsodium_getkey.sh; - primingScript = ./nix/tests/prime.sql; - migrationData = ./nix/tests/migrations/data.sql; - in - pkgs.runCommand "migrate-postgres" { } '' + local-infra-bootstrap = pkgs.runCommand "local-infra-bootstrap" { } '' mkdir -p $out/bin - substitute ${./nix/tools/migrate-tool.sh.in} $out/bin/migrate-postgres \ - --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ - --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \ - --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ - --subst-var-by 'PRIMING_SCRIPT' '${primingScript}' \ - --subst-var-by 'MIGRATION_DATA' '${migrationData}' - - chmod +x $out/bin/migrate-postgres + substitute ${./nix/tools/local-infra-bootstrap.sh.in} $out/bin/local-infra-bootstrap + chmod +x $out/bin/local-infra-bootstrap ''; - - start-replica = pkgs.runCommand "start-postgres-replica" { } '' - mkdir -p $out/bin - substitute ${./nix/tools/run-replica.sh.in} $out/bin/start-postgres-replica \ - --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ - --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' - chmod +x $out/bin/start-postgres-replica - ''; - pg-restore = - pkgs.runCommand "run-pg-restore" { } '' + dbmate-tool = + let + migrationsDir = ./migrations/db; + ansibleVars = ./ansible/vars.yml; + pgbouncerAuthSchemaSql = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; + statExtensionSql = ./ansible/files/stat_extension.sql; + in + pkgs.runCommand "dbmate-tool" + { + buildInputs = with pkgs; [ + overmind + dbmate + nix + jq + yq + ]; + nativeBuildInputs = with pkgs; [ + makeWrapper + ]; + } '' + mkdir -p $out/bin $out/migrations + cp -r ${migrationsDir}/* $out + substitute ${./nix/tools/dbmate-tool.sh.in} $out/bin/dbmate-tool \ + --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ + --subst-var-by 'MIGRATIONS_DIR' $out \ + --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ + --subst-var-by 'ANSIBLE_VARS' ${ansibleVars} \ + --subst-var-by 'CURRENT_SYSTEM' '${system}' \ + --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ + --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}' + chmod +x $out/bin/dbmate-tool + wrapProgram $out/bin/dbmate-tool \ + --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.overmind pkgs.dbmate pkgs.nix pkgs.jq pkgs.yq ]} + ''; + show-commands = pkgs.runCommand "show-commands" + { + nativeBuildInputs = [ pkgs.makeWrapper ]; + buildInputs = [ pkgs.nushell ]; + } '' mkdir -p $out/bin - substitute ${./nix/tools/run-restore.sh.in} $out/bin/pg-restore \ - --subst-var-by PSQL15_BINDIR '${basePackages.psql_15.bin}' - chmod +x $out/bin/pg-restore + cat > $out/bin/show-commands << 'EOF' + #!${pkgs.nushell}/bin/nu + let json_output = (nix flake show --json --quiet --all-systems | from json) + let apps = ($json_output | get apps.${system}) + $apps | transpose name info | select name | each { |it| echo $"Run this app with: nix run .#($it.name)" } + EOF + chmod +x $out/bin/show-commands + wrapProgram $out/bin/show-commands \ + --prefix PATH : ${pkgs.nushell}/bin ''; - sync-exts-versions = pkgs.runCommand "sync-exts-versions" { } '' - mkdir -p $out/bin - substitute ${./nix/tools/sync-exts-versions.sh.in} $out/bin/sync-exts-versions \ - --subst-var-by 'YQ' '${pkgs.yq}/bin/yq' \ - --subst-var-by 'JQ' '${pkgs.jq}/bin/jq' \ - --subst-var-by 'NIX_EDITOR' '${nix-editor.packages.${system}.nix-editor}/bin/nix-editor' \ - --subst-var-by 'NIXPREFETCHURL' '${pkgs.nixVersions.nix_2_20}/bin/nix-prefetch-url' \ - --subst-var-by 'NIX' '${pkgs.nixVersions.nix_2_20}/bin/nix' - chmod +x $out/bin/sync-exts-versions - ''; - - local-infra-bootstrap = pkgs.runCommand "local-infra-bootstrap" { } '' - mkdir -p $out/bin - substitute ${./nix/tools/local-infra-bootstrap.sh.in} $out/bin/local-infra-bootstrap - chmod +x $out/bin/local-infra-bootstrap - ''; - dbmate-tool = - let - migrationsDir = ./migrations/db; - ansibleVars = ./ansible/vars.yml; - pgbouncerAuthSchemaSql = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; - statExtensionSql = ./ansible/files/stat_extension.sql; - in - pkgs.runCommand "dbmate-tool" { - buildInputs = with pkgs; [ - overmind - dbmate - nix - jq - yq - ]; - nativeBuildInputs = with pkgs; [ - makeWrapper - ]; - } '' - mkdir -p $out/bin $out/migrations - cp -r ${migrationsDir}/* $out - substitute ${./nix/tools/dbmate-tool.sh.in} $out/bin/dbmate-tool \ - --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ - --subst-var-by 'MIGRATIONS_DIR' $out \ - --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ - --subst-var-by 'ANSIBLE_VARS' ${ansibleVars} \ - --subst-var-by 'CURRENT_SYSTEM' '${system}' \ - --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ - --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}' - chmod +x $out/bin/dbmate-tool - wrapProgram $out/bin/dbmate-tool \ - --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.overmind pkgs.dbmate pkgs.nix pkgs.jq pkgs.yq ]} + update-readme = pkgs.runCommand "update-readme" + { + nativeBuildInputs = [ pkgs.makeWrapper ]; + buildInputs = [ pkgs.nushell ]; + } '' + mkdir -p $out/bin + cp ${./nix/tools/update_readme.nu} $out/bin/update-readme + chmod +x $out/bin/update-readme + wrapProgram $out/bin/update-readme \ + --prefix PATH : ${pkgs.nushell}/bin ''; - show-commands = pkgs.runCommand "show-commands" { - nativeBuildInputs = [ pkgs.makeWrapper ]; - buildInputs = [ pkgs.nushell ]; - } '' - mkdir -p $out/bin - cat > $out/bin/show-commands << 'EOF' - #!${pkgs.nushell}/bin/nu - let json_output = (nix flake show --json --quiet --all-systems | from json) - let apps = ($json_output | get apps.${system}) - $apps | transpose name info | select name | each { |it| echo $"Run this app with: nix run .#($it.name)" } - EOF - chmod +x $out/bin/show-commands - wrapProgram $out/bin/show-commands \ - --prefix PATH : ${pkgs.nushell}/bin - ''; - update-readme = pkgs.runCommand "update-readme" { - nativeBuildInputs = [ pkgs.makeWrapper ]; - buildInputs = [ pkgs.nushell ]; - } '' - mkdir -p $out/bin - cp ${./nix/tools/update_readme.nu} $out/bin/update-readme - chmod +x $out/bin/update-readme - wrapProgram $out/bin/update-readme \ - --prefix PATH : ${pkgs.nushell}/bin - ''; - }; + # Script to run the AMI build and tests locally + build-test-ami = pkgs.runCommand "build-test-ami" + { + buildInputs = with pkgs; [ + packer + awscli2 + yq + jq + openssl + git + coreutils + aws-vault + ]; + } '' + mkdir -p $out/bin + cat > $out/bin/build-test-ami << 'EOL' + #!/usr/bin/env bash + set -euo pipefail + + show_help() { + cat << EOF + Usage: build-test-ami [--help] + + Build AMI images for PostgreSQL testing. + + This script will: + 1. Check for required tools and AWS authentication + 2. Build two AMI stages using Packer + 3. Clean up any temporary instances + 4. Output the final AMI name for use with run-testinfra + + Arguments: + postgres-version PostgreSQL major version to build (required) + + Options: + --help Show this help message and exit + + Requirements: + - AWS Vault profile must be set in AWS_VAULT environment variable + - Packer, AWS CLI, yq, jq, and OpenSSL must be installed + - Must be run from a git repository + + Example: + aws-vault exec -- nix run .#build-test-ami 15 + EOF + } + + # Handle help flag + if [[ "$#" -gt 0 && "$1" == "--help" ]]; then + show_help + exit 0 + fi + + export PATH="${pkgs.lib.makeBinPath (with pkgs; [ + packer + awscli2 + yq + jq + openssl + git + coreutils + aws-vault + ])}:$PATH" + + # Check for required tools + for cmd in packer aws-vault yq jq openssl; do + if ! command -v $cmd &> /dev/null; then + echo "Error: $cmd is required but not found" + exit 1 + fi + done + + # Check AWS Vault profile + if [ -z "''${AWS_VAULT:-}" ]; then + echo "Error: AWS_VAULT environment variable must be set with the profile name" + echo "Usage: aws-vault exec -- nix run .#build-test-ami " + exit 1 + fi + + # Set values + REGION="ap-southeast-1" + POSTGRES_VERSION="$1" + RANDOM_STRING=$(openssl rand -hex 8) + GIT_SHA=$(git rev-parse HEAD) + RUN_ID=$(date +%s) + + # Generate common-nix.vars.pkr.hcl + PG_VERSION=$(yq -r ".postgres_release[\"postgres$POSTGRES_VERSION\"]" ansible/vars.yml) + echo "postgres-version = \"$PG_VERSION\"" > common-nix.vars.pkr.hcl + + # Build AMI Stage 1 + packer init amazon-arm64-nix.pkr.hcl + packer build \ + -var "git-head-version=$GIT_SHA" \ + -var "packer-execution-id=$RUN_ID" \ + -var-file="development-arm.vars.pkr.hcl" \ + -var-file="common-nix.vars.pkr.hcl" \ + -var "ansible_arguments=" \ + -var "postgres-version=$RANDOM_STRING" \ + -var "region=$REGION" \ + -var 'ami_regions=["'"$REGION"'"]' \ + -var "force-deregister=true" \ + -var "ansible_arguments=-e postgresql_major=$POSTGRES_VERSION" \ + amazon-arm64-nix.pkr.hcl + + # Build AMI Stage 2 + packer init stage2-nix-psql.pkr.hcl + packer build \ + -var "git-head-version=$GIT_SHA" \ + -var "packer-execution-id=$RUN_ID" \ + -var "postgres_major_version=$POSTGRES_VERSION" \ + -var-file="development-arm.vars.pkr.hcl" \ + -var-file="common-nix.vars.pkr.hcl" \ + -var "postgres-version=$RANDOM_STRING" \ + -var "region=$REGION" \ + -var 'ami_regions=["'"$REGION"'"]' \ + -var "force-deregister=true" \ + -var "git_sha=$GIT_SHA" \ + stage2-nix-psql.pkr.hcl + + # Cleanup instances from AMI builds + cleanup_instances() { + echo "Terminating EC2 instances with tag testinfra-run-id=$RUN_ID..." + aws ec2 --region $REGION describe-instances \ + --filters "Name=tag:testinfra-run-id,Values=$RUN_ID" \ + --query "Reservations[].Instances[].InstanceId" \ + --output text | xargs -r aws ec2 terminate-instances \ + --region $REGION --instance-ids || true + } + + # Set up traps for various signals to ensure cleanup + trap cleanup_instances EXIT HUP INT QUIT TERM + + # Create and activate virtual environment + VENV_DIR=$(mktemp -d) + trap 'rm -rf "$VENV_DIR"' EXIT HUP INT QUIT TERM + python3 -m venv "$VENV_DIR" + source "$VENV_DIR/bin/activate" + + # Install required Python packages + echo "Installing required Python packages..." + pip install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest paramiko requests + + # Run the tests with aws-vault + echo "Running tests for AMI: $RANDOM_STRING using AWS Vault profile: $AWS_VAULT_PROFILE" + aws-vault exec $AWS_VAULT_PROFILE -- pytest -vv -s testinfra/test_ami_nix.py + + # Deactivate virtual environment (cleanup is handled by trap) + deactivate + EOL + chmod +x $out/bin/build-test-ami + ''; + + run-testinfra = pkgs.runCommand "run-testinfra" + { + buildInputs = with pkgs; [ + aws-vault + python3 + python3Packages.pip + coreutils + ]; + } '' + mkdir -p $out/bin + cat > $out/bin/run-testinfra << 'EOL' + #!/usr/bin/env bash + set -euo pipefail + + show_help() { + cat << EOF + Usage: run-testinfra --ami-name NAME [--aws-vault-profile PROFILE] + + Run the testinfra tests locally against a specific AMI. + + This script will: + 1. Check if aws-vault is installed and configured + 2. Set up the required environment variables + 3. Create and activate a virtual environment + 4. Install required Python packages from pip + 5. Run the tests with aws-vault credentials + 6. Clean up the virtual environment + + Required flags: + --ami-name NAME The name of the AMI to test + + Optional flags: + --aws-vault-profile PROFILE AWS Vault profile to use (default: staging) + --help Show this help message and exit + + Requirements: + - aws-vault installed and configured + - Python 3 with pip + - Must be run from the repository root + + Examples: + run-testinfra --ami-name supabase-postgres-abc123 + run-testinfra --ami-name supabase-postgres-abc123 --aws-vault-profile production + EOF + } + + # Default values + AWS_VAULT_PROFILE="staging" + AMI_NAME="" + + # Parse arguments + while [[ $# -gt 0 ]]; do + case $1 in + --aws-vault-profile) + AWS_VAULT_PROFILE="$2" + shift 2 + ;; + --ami-name) + AMI_NAME="$2" + shift 2 + ;; + --help) + show_help + exit 0 + ;; + *) + echo "Error: Unexpected argument: $1" + show_help + exit 1 + ;; + esac + done + + # Check for required tools + if ! command -v aws-vault &> /dev/null; then + echo "Error: aws-vault is required but not found" + exit 1 + fi + + # Check for AMI name argument + if [ -z "$AMI_NAME" ]; then + echo "Error: --ami-name is required" + show_help + exit 1 + fi + + # Set environment variables + export AWS_REGION="ap-southeast-1" + export AWS_DEFAULT_REGION="ap-southeast-1" + export AMI_NAME="$AMI_NAME" # Export AMI_NAME for pytest + export RUN_ID="local-$(date +%s)" # Generate a unique RUN_ID + + # Function to terminate EC2 instances + terminate_instances() { + echo "Terminating EC2 instances with tag testinfra-run-id=$RUN_ID..." + aws-vault exec $AWS_VAULT_PROFILE -- aws ec2 --region ap-southeast-1 describe-instances \ + --filters "Name=tag:testinfra-run-id,Values=$RUN_ID" \ + --query "Reservations[].Instances[].InstanceId" \ + --output text | xargs -r aws-vault exec $AWS_VAULT_PROFILE -- aws ec2 terminate-instances \ + --region ap-southeast-1 --instance-ids || true + } + + # Set up traps for various signals to ensure cleanup + trap terminate_instances EXIT HUP INT QUIT TERM + + # Create and activate virtual environment + VENV_DIR=$(mktemp -d) + trap 'rm -rf "$VENV_DIR"' EXIT HUP INT QUIT TERM + python3 -m venv "$VENV_DIR" + source "$VENV_DIR/bin/activate" + + # Install required Python packages + echo "Installing required Python packages..." + pip install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest paramiko requests + + # Function to run tests and ensure cleanup + run_tests() { + local exit_code=0 + echo "Running tests for AMI: $AMI_NAME using AWS Vault profile: $AWS_VAULT_PROFILE" + aws-vault exec "$AWS_VAULT_PROFILE" -- pytest -vv -s testinfra/test_ami_nix.py || exit_code=$? + return $exit_code + } + + # Run tests and capture exit code + run_tests + test_exit_code=$? + + # Deactivate virtual environment + deactivate + + # Explicitly call cleanup + terminate_instances + + # Exit with the test exit code + exit $test_exit_code + EOL + chmod +x $out/bin/run-testinfra + ''; + + cleanup-ami = pkgs.runCommand "cleanup-ami" + { + buildInputs = with pkgs; [ + awscli2 + aws-vault + ]; + } '' + mkdir -p $out/bin + cat > $out/bin/cleanup-ami << 'EOL' + #!/usr/bin/env bash + set -euo pipefail + + export PATH="${pkgs.lib.makeBinPath (with pkgs; [ + awscli2 + aws-vault + ])}:$PATH" + + # Check for required tools + for cmd in aws-vault; do + if ! command -v $cmd &> /dev/null; then + echo "Error: $cmd is required but not found" + exit 1 + fi + done + + # Check AWS Vault profile + if [ -z "''${AWS_VAULT:-}" ]; then + echo "Error: AWS_VAULT environment variable must be set with the profile name" + echo "Usage: aws-vault exec -- nix run .#cleanup-ami " + exit 1 + fi + + # Check for AMI name argument + if [ -z "''${1:-}" ]; then + echo "Error: AMI name must be provided" + echo "Usage: aws-vault exec -- nix run .#cleanup-ami " + exit 1 + fi + + AMI_NAME="$1" + REGION="ap-southeast-1" + + # Deregister AMIs + for AMI_PATTERN in "supabase-postgres-ci-ami-test-stage-1" "$AMI_NAME"; do + aws ec2 describe-images --region $REGION --owners self \ + --filters "Name=name,Values=$AMI_PATTERN" \ + --query 'Images[*].ImageId' --output text | while read -r ami_id; do + echo "Deregistering AMI: $ami_id" + aws ec2 deregister-image --region $REGION --image-id "$ami_id" || true + done + done + EOL + chmod +x $out/bin/cleanup-ami + ''; + + trigger-nix-build = pkgs.runCommand "trigger-nix-build" + { + buildInputs = with pkgs; [ + gh + git + coreutils + ]; + } '' + mkdir -p $out/bin + cat > $out/bin/trigger-nix-build << 'EOL' + #!/usr/bin/env bash + set -euo pipefail + + show_help() { + cat << EOF + Usage: trigger-nix-build [--help] + + Trigger the nix-build workflow for the current branch and watch its progress. + + This script will: + 1. Check if you're authenticated with GitHub + 2. Get the current branch and commit + 3. Verify you're on a standard branch (develop or release/*) or prompt for confirmation + 4. Trigger the nix-build workflow + 5. Watch the workflow progress until completion + + Options: + --help Show this help message and exit + + Requirements: + - GitHub CLI (gh) installed and authenticated + - Git installed + - Must be run from a git repository + + Example: + trigger-nix-build + EOF + } + + # Handle help flag + if [[ "$#" -gt 0 && "$1" == "--help" ]]; then + show_help + exit 0 + fi + + export PATH="${pkgs.lib.makeBinPath (with pkgs; [ + gh + git + coreutils + ])}:$PATH" + + # Check for required tools + for cmd in gh git; do + if ! command -v $cmd &> /dev/null; then + echo "Error: $cmd is required but not found" + exit 1 + fi + done + + # Check if user is authenticated with GitHub + if ! gh auth status &>/dev/null; then + echo "Error: Not authenticated with GitHub. Please run 'gh auth login' first." + exit 1 + fi + + # Get current branch and commit + BRANCH=$(git rev-parse --abbrev-ref HEAD) + COMMIT=$(git rev-parse HEAD) + + # Check if we're on a standard branch + if [[ "$BRANCH" != "develop" && ! "$BRANCH" =~ ^release/ ]]; then + echo "Warning: Running workflow from non-standard branch: $BRANCH" + echo "This is supported for testing purposes." + read -p "Continue? [y/N] " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Aborted." + exit 1 + fi + fi + + # Trigger the workflow + echo "Triggering nix-build workflow for branch $BRANCH (commit: $COMMIT)" + gh workflow run nix-build.yml --ref "$BRANCH" + + # Wait for workflow to start and get the run ID + echo "Waiting for workflow to start..." + sleep 5 + + # Get the latest run ID for this workflow + RUN_ID=$(gh run list --workflow=nix-build.yml --limit 1 --json databaseId --jq '.[0].databaseId') + + if [ -z "$RUN_ID" ]; then + echo "Error: Could not find workflow run ID" + exit 1 + fi + + echo "Watching workflow run $RUN_ID..." + echo "The script will automatically exit when the workflow completes." + echo "Press Ctrl+C to stop watching (workflow will continue running)" + echo "----------------------------------------" + + # Try to watch the run, but handle network errors gracefully + while true; do + if gh run watch "$RUN_ID" --exit-status; then + break + else + echo "Network error while watching workflow. Retrying in 5 seconds..." + echo "You can also check the status manually with: gh run view $RUN_ID" + sleep 5 + fi + done + EOL + chmod +x $out/bin/trigger-nix-build + ''; + }; # Create a testing harness for a PostgreSQL package. This is used for @@ -685,10 +1166,10 @@ let name = pkg.version; in - if builtins.match "15.*" name != null then "15" - else if builtins.match "17.*" name != null then "17" - else if builtins.match "orioledb-17.*" name != null then "orioledb-17" - else throw "Unsupported PostgreSQL version: ${name}"; + if builtins.match "15.*" name != null then "15" + else if builtins.match "17.*" name != null then "17" + else if builtins.match "orioledb-17.*" name != null then "orioledb-17" + else throw "Unsupported PostgreSQL version: ${name}"; # Helper function to filter SQL files based on version filterTestFiles = version: dir: @@ -697,7 +1178,7 @@ isValidFile = name: let isVersionSpecific = builtins.match "z_.*" name != null; - matchesVersion = + matchesVersion = if isVersionSpecific then if version == "orioledb-17" @@ -712,163 +1193,174 @@ pkgs.lib.filterAttrs (name: _: isValidFile name) files; # Get the major version for filtering - majorVersion = - let - version = builtins.trace "pgpkg.version is: ${pgpkg.version}" pgpkg.version; - _ = builtins.trace "Entering majorVersion logic"; - isOrioledbMatch = builtins.match "^17_[0-9]+$" version != null; - isSeventeenMatch = builtins.match "^17[.][0-9]+$" version != null; - result = - if isOrioledbMatch - then "orioledb-17" - else if isSeventeenMatch - then "17" - else "15"; - in - builtins.trace "Major version result: ${result}" result; # Trace the result # For "15.8" + majorVersion = + let + version = builtins.trace "pgpkg.version is: ${pgpkg.version}" pgpkg.version; + _ = builtins.trace "Entering majorVersion logic"; + isOrioledbMatch = builtins.match "^17_[0-9]+$" version != null; + isSeventeenMatch = builtins.match "^17[.][0-9]+$" version != null; + result = + if isOrioledbMatch + then "orioledb-17" + else if isSeventeenMatch + then "17" + else "15"; + in + builtins.trace "Major version result: ${result}" result; # Trace the result # For "15.8" # Filter SQL test files filteredSqlTests = filterTestFiles majorVersion ./nix/tests/sql; - + # Convert filtered tests to a sorted list of basenames (without extension) - testList = pkgs.lib.mapAttrsToList (name: _: - builtins.substring 0 (pkgs.lib.stringLength name - 4) name - ) filteredSqlTests; + testList = pkgs.lib.mapAttrsToList + (name: _: + builtins.substring 0 (pkgs.lib.stringLength name - 4) name + ) + filteredSqlTests; sortedTestList = builtins.sort (a: b: a < b) testList; in pkgs.runCommand "postgres-${pgpkg.version}-check-harness" { - nativeBuildInputs = with pkgs; [ - coreutils bash perl pgpkg pg_prove pg_regress procps - start-postgres-server-bin which getkey-script supabase-groonga + nativeBuildInputs = with pkgs; [ + coreutils + bash + perl + pgpkg + pg_prove + pg_regress + procps + start-postgres-server-bin + which + getkey-script + supabase-groonga ]; } '' - set -e - - #First we need to create a generic pg cluster for pgtap tests and run those - export GRN_PLUGINS_DIR=${supabase-groonga}/lib/groonga/plugins - PGTAP_CLUSTER=$(mktemp -d) - initdb --locale=C --username=supabase_admin -D "$PGTAP_CLUSTER" - substitute ${./nix/tests/postgresql.conf.in} "$PGTAP_CLUSTER"/postgresql.conf \ - --subst-var-by PGSODIUM_GETKEY_SCRIPT "${getkey-script}/bin/pgsodium-getkey" - echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf - echo "port = 5435" >> "$PGTAP_CLUSTER"/postgresql.conf - echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf - echo "Checking shared_preload_libraries setting:" - grep -rn "shared_preload_libraries" "$PGTAP_CLUSTER"/postgresql.conf - # Remove timescaledb if running orioledb-17 check - echo "I AM ${pgpkg.version}====================================================" - if [[ "${pgpkg.version}" == *"17"* ]]; then - perl -pi -e 's/ timescaledb,//g' "$PGTAP_CLUSTER/postgresql.conf" + set -e + + #First we need to create a generic pg cluster for pgtap tests and run those + export GRN_PLUGINS_DIR=${supabase-groonga}/lib/groonga/plugins + PGTAP_CLUSTER=$(mktemp -d) + initdb --locale=C --username=supabase_admin -D "$PGTAP_CLUSTER" + substitute ${./nix/tests/postgresql.conf.in} "$PGTAP_CLUSTER"/postgresql.conf \ + --subst-var-by PGSODIUM_GETKEY_SCRIPT "${getkey-script}/bin/pgsodium-getkey" + echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf + echo "port = 5435" >> "$PGTAP_CLUSTER"/postgresql.conf + echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf + echo "Checking shared_preload_libraries setting:" + grep -rn "shared_preload_libraries" "$PGTAP_CLUSTER"/postgresql.conf + # Remove timescaledb if running orioledb-17 check + echo "I AM ${pgpkg.version}====================================================" + if [[ "${pgpkg.version}" == *"17"* ]]; then + perl -pi -e 's/ timescaledb,//g' "$PGTAP_CLUSTER/postgresql.conf" + fi + #NOTE in the future we may also need to add the orioledb extension to the cluster when cluster is oriole + echo "PGTAP_CLUSTER directory contents:" + ls -la "$PGTAP_CLUSTER" + + # Check if postgresql.conf exists + if [ ! -f "$PGTAP_CLUSTER/postgresql.conf" ]; then + echo "postgresql.conf is missing!" + exit 1 + fi + + # PostgreSQL startup + if [[ "$(uname)" == "Darwin" ]]; then + pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k "$PGTAP_CLUSTER" -p 5435 -d 5" start 2>&1 + else + mkdir -p "$PGTAP_CLUSTER/sockets" + pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k $PGTAP_CLUSTER/sockets -p 5435 -d 5" start 2>&1 + fi || { + echo "pg_ctl failed to start PostgreSQL" + echo "Contents of postgresql.log:" + cat "$PGTAP_CLUSTER"/postgresql.log + exit 1 + } + for i in {1..60}; do + if pg_isready -h localhost -p 5435; then + echo "PostgreSQL is ready" + break fi - #NOTE in the future we may also need to add the orioledb extension to the cluster when cluster is oriole - echo "PGTAP_CLUSTER directory contents:" - ls -la "$PGTAP_CLUSTER" - - # Check if postgresql.conf exists - if [ ! -f "$PGTAP_CLUSTER/postgresql.conf" ]; then - echo "postgresql.conf is missing!" - exit 1 + sleep 1 + if [ $i -eq 60 ]; then + echo "PostgreSQL is not ready after 60 seconds" + echo "PostgreSQL status:" + pg_ctl -D "$PGTAP_CLUSTER" status + echo "PostgreSQL log content:" + cat "$PGTAP_CLUSTER"/postgresql.log + exit 1 fi - - # PostgreSQL startup - if [[ "$(uname)" == "Darwin" ]]; then - pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k "$PGTAP_CLUSTER" -p 5435 -d 5" start 2>&1 - else - mkdir -p "$PGTAP_CLUSTER/sockets" - pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k $PGTAP_CLUSTER/sockets -p 5435 -d 5" start 2>&1 - fi || { - echo "pg_ctl failed to start PostgreSQL" - echo "Contents of postgresql.log:" + done + createdb -p 5435 -h localhost --username=supabase_admin testing + if ! psql -p 5435 -h localhost --username=supabase_admin -d testing -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then + echo "Error executing SQL file. PostgreSQL log content:" cat "$PGTAP_CLUSTER"/postgresql.log + pg_ctl -D "$PGTAP_CLUSTER" stop exit 1 - } - for i in {1..60}; do - if pg_isready -h localhost -p 5435; then - echo "PostgreSQL is ready" - break + fi + SORTED_DIR=$(mktemp -d) + for t in $(printf "%s\n" ${builtins.concatStringsSep " " sortedTestList}); do + psql -p 5435 -h localhost --username=supabase_admin -d testing -f "${./nix/tests/sql}/$t.sql" || true + done + rm -rf "$SORTED_DIR" + pg_ctl -D "$PGTAP_CLUSTER" stop + rm -rf $PGTAP_CLUSTER + + # End of pgtap tests + # from here on out we are running pg_regress tests, we use a different cluster for this + # which is start by the start-postgres-server-bin script + # start-postgres-server-bin script closely matches our AMI setup, configurations and migrations + + # Ensure pgsodium key directory exists with proper permissions + if [[ "$(uname)" == "Darwin" ]]; then + mkdir -p /private/tmp/pgsodium + chmod 1777 /private/tmp/pgsodium + fi + unset GRN_PLUGINS_DIR + ${start-postgres-server-bin}/bin/start-postgres-server ${getVersionArg pgpkg} --daemonize + + for i in {1..60}; do + if pg_isready -h localhost -p 5435 -U supabase_admin -q; then + echo "PostgreSQL is ready" + break fi sleep 1 if [ $i -eq 60 ]; then - echo "PostgreSQL is not ready after 60 seconds" - echo "PostgreSQL status:" - pg_ctl -D "$PGTAP_CLUSTER" status - echo "PostgreSQL log content:" - cat "$PGTAP_CLUSTER"/postgresql.log - exit 1 + echo "PostgreSQL failed to start" + exit 1 fi - done - createdb -p 5435 -h localhost --username=supabase_admin testing - if ! psql -p 5435 -h localhost --username=supabase_admin -d testing -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then - echo "Error executing SQL file. PostgreSQL log content:" - cat "$PGTAP_CLUSTER"/postgresql.log - pg_ctl -D "$PGTAP_CLUSTER" stop - exit 1 - fi - SORTED_DIR=$(mktemp -d) - for t in $(printf "%s\n" ${builtins.concatStringsSep " " sortedTestList}); do - psql -p 5435 -h localhost --username=supabase_admin -d testing -f "${./nix/tests/sql}/$t.sql" || true - done - rm -rf "$SORTED_DIR" - pg_ctl -D "$PGTAP_CLUSTER" stop - rm -rf $PGTAP_CLUSTER - - # End of pgtap tests - # from here on out we are running pg_regress tests, we use a different cluster for this - # which is start by the start-postgres-server-bin script - # start-postgres-server-bin script closely matches our AMI setup, configurations and migrations - - # Ensure pgsodium key directory exists with proper permissions - if [[ "$(uname)" == "Darwin" ]]; then - mkdir -p /private/tmp/pgsodium - chmod 1777 /private/tmp/pgsodium - fi - unset GRN_PLUGINS_DIR - ${start-postgres-server-bin}/bin/start-postgres-server ${getVersionArg pgpkg} --daemonize - - for i in {1..60}; do - if pg_isready -h localhost -p 5435 -U supabase_admin -q; then - echo "PostgreSQL is ready" - break - fi - sleep 1 - if [ $i -eq 60 ]; then - echo "PostgreSQL failed to start" - exit 1 - fi - done - - if ! psql -p 5435 -h localhost --no-password --username=supabase_admin -d postgres -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then - echo "Error executing SQL file" - exit 1 - fi + done - mkdir -p $out/regression_output - if ! pg_regress \ - --use-existing \ - --dbname=postgres \ - --inputdir=${./nix/tests} \ - --outputdir=$out/regression_output \ - --host=localhost \ - --port=5435 \ - --user=supabase_admin \ - ${builtins.concatStringsSep " " sortedTestList}; then - echo "pg_regress tests failed" - cat $out/regression_output/regression.diffs - exit 1 - fi + if ! psql -p 5435 -h localhost --no-password --username=supabase_admin -d postgres -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then + echo "Error executing SQL file" + exit 1 + fi + + mkdir -p $out/regression_output + if ! pg_regress \ + --use-existing \ + --dbname=postgres \ + --inputdir=${./nix/tests} \ + --outputdir=$out/regression_output \ + --host=localhost \ + --port=5435 \ + --user=supabase_admin \ + ${builtins.concatStringsSep " " sortedTestList}; then + echo "pg_regress tests failed" + cat $out/regression_output/regression.diffs + exit 1 + fi - echo "Running migrations tests" - pg_prove -p 5435 -U supabase_admin -h localhost -d postgres -v ${./migrations/tests}/test.sql + echo "Running migrations tests" + pg_prove -p 5435 -U supabase_admin -h localhost -d postgres -v ${./migrations/tests}/test.sql - # Copy logs to output - for logfile in $(find /tmp -name postgresql.log -type f); do - cp "$logfile" $out/postgresql.log - done - exit 0 - ''; - in + # Copy logs to output + for logfile in $(find /tmp -name postgresql.log -type f); do + cp "$logfile" $out/postgresql.log + done + exit 0 + ''; + in rec { # The list of all packages that can be built with 'nix build'. The list # of names that can be used can be shown with 'nix flake show' @@ -907,6 +1399,10 @@ dbmate-tool = mkApp "dbmate-tool" "dbmate-tool"; update-readme = mkApp "update-readme" "update-readme"; show-commands = mkApp "show-commands" "show-commands"; + build-test-ami = mkApp "build-test-ami" "build-test-ami"; + run-testinfra = mkApp "run-testinfra" "run-testinfra"; + cleanup-ami = mkApp "cleanup-ami" "cleanup-ami"; + trigger-nix-build = mkApp "trigger-nix-build" "trigger-nix-build"; }; # 'devShells.default' lists the set of packages that are included in the @@ -914,54 +1410,59 @@ # for development and puts many convenient devtools instantly within # reach. - devShells = let - mkCargoPgrxDevShell = { pgrxVersion, rustVersion }: pkgs.mkShell { - packages = with pkgs; [ - basePackages."cargo-pgrx_${pgrxVersion}" - (rust-bin.stable.${rustVersion}.default.override { - extensions = [ "rust-src" ]; - }) - ]; - shellHook = '' - export HISTFILE=.history - ''; - }; - in { - default = pkgs.mkShell { - packages = with pkgs; [ - coreutils - just - nix-update - #pg_prove - shellcheck - ansible - ansible-lint - (packer.overrideAttrs (oldAttrs: { - version = "1.7.8"; - })) - - basePackages.start-server - basePackages.start-client - basePackages.start-replica - basePackages.migrate-tool - basePackages.sync-exts-versions - dbmate - nushell - ]; - shellHook = '' - export HISTFILE=.history - export DATABASE_URL="postgres://supabase_admin@localhost:5435/postgres?sslmode=disable" - ''; - }; - cargo-pgrx_0_11_3 = mkCargoPgrxDevShell { - pgrxVersion = "0_11_3"; - rustVersion = "1.80.0"; - }; - cargo-pgrx_0_12_6 = mkCargoPgrxDevShell { - pgrxVersion = "0_12_6"; - rustVersion = "1.80.0"; - }; - }; - } - ); + devShells = + let + mkCargoPgrxDevShell = { pgrxVersion, rustVersion }: pkgs.mkShell { + packages = with pkgs; [ + basePackages."cargo-pgrx_${pgrxVersion}" + (rust-bin.stable.${rustVersion}.default.override { + extensions = [ "rust-src" ]; + }) + ]; + shellHook = '' + export HISTFILE=.history + ''; + }; + in + { + default = pkgs.mkShell { + packages = with pkgs; [ + coreutils + just + nix-update + #pg_prove + shellcheck + ansible + ansible-lint + (packer.overrideAttrs (oldAttrs: { + version = "1.7.8"; + })) + + basePackages.start-server + basePackages.start-client + basePackages.start-replica + basePackages.migrate-tool + basePackages.sync-exts-versions + basePackages.build-test-ami + basePackages.run-testinfra + basePackages.cleanup-ami + dbmate + nushell + pythonEnv + ]; + shellHook = '' + export HISTFILE=.history + ''; + }; + cargo-pgrx_0_11_3 = mkCargoPgrxDevShell { + pgrxVersion = "0_11_3"; + rustVersion = "1.80.0"; + }; + cargo-pgrx_0_12_6 = mkCargoPgrxDevShell { + pgrxVersion = "0_12_6"; + rustVersion = "1.80.0"; + }; + }; + } + ); } diff --git a/nix/docs/development-workflow.md b/nix/docs/development-workflow.md new file mode 100644 index 000000000..695427abc --- /dev/null +++ b/nix/docs/development-workflow.md @@ -0,0 +1,141 @@ +# PostgreSQL Development Workflow + +This document outlines the workflow for developing and testing PostgreSQL in an ec2 instance using the tools provided in this repo. + +## Prerequisites + +- Nix installed and configured +- AWS credentials configured with aws-vault (you must set up aws-vault beforehand) +- GitHub access to the repository + +## Workflow Steps + +### 1. Trigger Remote Build and Cache + +To build, test, and cache your changes in the Supabase Nix binary cache: + +```bash +# From your branch +nix run .#trigger-nix-build +``` + +This will: +- Trigger a GitHub Actions workflow +- Build PostgreSQL and extensions +- Run nix flake check tests (evaluation of nix code, pg_regress and migrations tests) +- Cache the results in the Supabase Nix binary cache +- Watch the workflow progress until completion + +The workflow will run on the branch you're currently on. + +If you're on a feature different branch, you'll be prompted to confirm before proceeding. + +### 2. Build AMI + +After the build is complete and cached, build the AMI: + +```bash +# Build AMI for PostgreSQL 15 +aws-vault exec -- nix run .#build-test-ami 15 + +# Or for PostgreSQL 17 +aws-vault exec -- nix run .#build-test-ami 17 + +# Or for PostgreSQL orioledb-17 +aws-vault exec -- nix run .#build-test-ami orioledb-17 +``` + +This will: +- Build two AMI stages using Packer +- Clean up temporary instances after AMI builds +- Output the final AMI name (e.g., `supabase-postgres-abc123`) + +**Important**: Take note of the AMI name output at the end, as you'll need it for the next step. + +### 3. Run Testinfra + +Run the testinfra tests against the AMI: + +```bash +# Run tests against the AMI +nix run .#run-testinfra -- --aws-vault-profile --ami-name supabase-postgres-abc123 +``` + +This will: +- Create a Python virtual environment +- Install required Python packages +- Create an EC2 instance from the AMI +- Run the test suite +- Automatically terminate the EC2 instance when done + +The script handles: +- Setting up AWS credentials via aws-vault +- Creating and managing the Python virtual environment +- Running the tests +- Cleaning up EC2 instances +- Proper error handling and cleanup on interruption + +### 4. Optional: Cleanup AMI + +If you want to clean up the AMI after testing: + +```bash +# Clean up the AMI +aws-vault exec -- nix run .#cleanup-ami supabase-postgres-abc123 +``` + +This will: +- Deregister the AMI +- Clean up any associated resources + +## Troubleshooting + +### Common Issues + +1. **AWS Credentials** + - Ensure aws-vault is properly configured + - Use the `--aws-vault-profile` argument to specify your AWS profile + - Default profile is "staging" if not specified + +2. **EC2 Instance Not Terminating** + - The script includes multiple safeguards for cleanup + - If instances aren't terminated, check AWS console and terminate manually + +3. **Test Failures** + - Check the test output for specific failures + - Ensure you're using the correct AMI name + - Verify AWS region and permissions + +### Environment Variables + +The following environment variables are used: +- `AWS_VAULT`: AWS Vault profile name (default: staging) +- `AWS_REGION`: AWS region (default: ap-southeast-1) +- `AMI_NAME`: Name of the AMI to test + +## Best Practices + +1. **Branch Management** + - Use feature branches for development + - Merge to develop for testing + - Use release branches for version-specific changes + +2. **Resource Cleanup** + - Always run the cleanup step after testing + - Monitor AWS console for any lingering resources + - Use the cleanup-ami command when done with an AMI + +3. **Testing** + - Run tests locally before pushing changes + - Verify AMI builds before running testinfra + - Check test output for any warnings or errors + +## Additional Commands + +```bash +# Show available commands +nix run .#show-commands + +# Update README with latest command information +nix run .#update-readme +``` \ No newline at end of file diff --git a/nix/ext/pg_jsonschema.nix b/nix/ext/pg_jsonschema.nix index 642519f08..654bb93f5 100644 --- a/nix/ext/pg_jsonschema.nix +++ b/nix/ext/pg_jsonschema.nix @@ -26,7 +26,10 @@ buildPgrxExtension_0_12_6 rec { env = lib.optionalAttrs stdenv.isDarwin { POSTGRES_LIB = "${postgresql}/lib"; RUSTFLAGS = "-C link-arg=-undefined -C link-arg=dynamic_lookup"; - PGPORT = "5433"; + PGPORT = toString (5441 + + (if builtins.match ".*_.*" postgresql.version != null then 1 else 0) + # +1 for OrioleDB + ((builtins.fromJSON (builtins.substring 0 2 postgresql.version)) - 15) * 2); # +2 for each major version + }; cargoLock = { diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py index 4d354fac3..1975818d6 100644 --- a/testinfra/test_ami_nix.py +++ b/testinfra/test_ami_nix.py @@ -6,10 +6,11 @@ import pytest import requests import socket -import testinfra from ec2instanceconnectcli.EC2InstanceConnectLogger import EC2InstanceConnectLogger from ec2instanceconnectcli.EC2InstanceConnectKey import EC2InstanceConnectKey from time import sleep +import subprocess +import paramiko # if GITHUB_RUN_ID is not set, use a default value that includes the user and hostname RUN_ID = os.environ.get( @@ -170,6 +171,51 @@ logger.setLevel(logging.DEBUG) +def get_ssh_connection(instance_ip, ssh_identity_file, max_retries=10): + """Create and return a single SSH connection that can be reused.""" + for attempt in range(max_retries): + try: + # Create SSH client + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + # Connect with our working parameters + ssh.connect( + hostname=instance_ip, + username='ubuntu', + key_filename=ssh_identity_file, + timeout=10, + banner_timeout=10 + ) + + # Test the connection + stdin, stdout, stderr = ssh.exec_command('echo "SSH test"') + if stdout.channel.recv_exit_status() == 0 and "SSH test" in stdout.read().decode(): + logger.info("SSH connection established successfully") + return ssh + else: + raise Exception("SSH test command failed") + + except Exception as e: + if attempt == max_retries - 1: + raise + logger.warning( + f"Ssh connection failed, retrying: {attempt + 1}/{max_retries} failed, retrying ..." + ) + sleep(5) + + +def run_ssh_command(ssh, command): + """Run a command over the established SSH connection.""" + stdin, stdout, stderr = ssh.exec_command(command) + exit_code = stdout.channel.recv_exit_status() + return { + 'succeeded': exit_code == 0, + 'stdout': stdout.read().decode(), + 'stderr': stderr.read().decode() + } + + # scope='session' uses the same container for all the tests; # scope='function' uses a new container per test function. @pytest.fixture(scope="session") @@ -230,6 +276,7 @@ def gzip_then_base64_encode(s: str) -> str: - 'sudo echo \"pgbouncer\" \"postgres\" >> /etc/pgbouncer/userlist.txt' - 'cd /tmp && aws s3 cp --region ap-southeast-1 s3://init-scripts-staging/project/init.sh .' - 'bash init.sh "staging"' + - 'touch /var/lib/init-complete' - 'rm -rf /tmp/*' """, TagSpecifications=[ @@ -256,108 +303,95 @@ def gzip_then_base64_encode(s: str) -> str: ) assert response["Success"] - # instance doesn't have public ip yet + # Wait for instance to have public IP while not instance.public_ip_address: logger.warning("waiting for ip to be available") sleep(5) instance.reload() - while True: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if sock.connect_ex((instance.public_ip_address, 22)) == 0: - break - else: - logger.warning("waiting for ssh to be available") - sleep(10) - - def get_ssh_connection(instance_ip, ssh_identity_file, max_retries=10): - for attempt in range(max_retries): - try: - return testinfra.get_host( - f"paramiko://ubuntu@{instance_ip}?timeout=60", - ssh_identity_file=ssh_identity_file, - ) - except Exception as e: - if attempt == max_retries - 1: - raise - logger.warning( - f"Ssh connection failed, retrying: {attempt + 1}/{max_retries} failed, retrying ..." - ) - sleep(5) - - host = get_ssh_connection( - # paramiko is an ssh backend + # Create single SSH connection + ssh = get_ssh_connection( instance.public_ip_address, temp_key.get_priv_key_file(), ) - def is_healthy(host, instance_ip, ssh_identity_file) -> bool: + # Check PostgreSQL data directory + logger.info("Checking PostgreSQL data directory...") + result = run_ssh_command(ssh, "ls -la /var/lib/postgresql") + if result['succeeded']: + logger.info("PostgreSQL data directory contents:\n" + result['stdout']) + else: + logger.warning("Failed to list PostgreSQL data directory: " + result['stderr']) + + # Wait for init.sh to complete + logger.info("Waiting for init.sh to complete...") + max_attempts = 60 # 5 minutes + attempt = 0 + while attempt < max_attempts: + try: + result = run_ssh_command(ssh, "test -f /var/lib/init-complete") + if result['succeeded']: + logger.info("init.sh has completed") + break + except Exception as e: + logger.warning(f"Error checking init.sh status: {str(e)}") + + attempt += 1 + logger.warning(f"Waiting for init.sh to complete (attempt {attempt}/{max_attempts})") + sleep(5) + + if attempt >= max_attempts: + logger.error("init.sh failed to complete within the timeout period") + instance.terminate() + raise TimeoutError("init.sh failed to complete within the timeout period") + + def is_healthy(ssh) -> bool: health_checks = [ - ( - "postgres", - lambda h: h.run("sudo -u postgres /usr/bin/pg_isready -U postgres"), - ), - ( - "adminapi", - lambda h: h.run( - f"curl -sf -k --connect-timeout 30 --max-time 60 https://localhost:8085/health -H 'apikey: {supabase_admin_key}'" - ), - ), - ( - "postgrest", - lambda h: h.run( - "curl -sf --connect-timeout 30 --max-time 60 http://localhost:3001/ready" - ), - ), - ( - "gotrue", - lambda h: h.run( - "curl -sf --connect-timeout 30 --max-time 60 http://localhost:8081/health" - ), - ), - ("kong", lambda h: h.run("sudo kong health")), - ("fail2ban", lambda h: h.run("sudo fail2ban-client status")), + ("postgres", "sudo -u postgres /usr/bin/pg_isready -U postgres"), + ("adminapi", f"curl -sf -k --connect-timeout 30 --max-time 60 https://localhost:8085/health -H 'apikey: {supabase_admin_key}'"), + ("postgrest", "curl -sf --connect-timeout 30 --max-time 60 http://localhost:3001/ready"), + ("gotrue", "curl -sf --connect-timeout 30 --max-time 60 http://localhost:8081/health"), + ("kong", "sudo kong health"), + ("fail2ban", "sudo fail2ban-client status"), ] - for service, check in health_checks: + for service, command in health_checks: try: - cmd = check(host) - if cmd.failed is True: + result = run_ssh_command(ssh, command) + if not result['succeeded']: logger.warning(f"{service} not ready") return False except Exception: - logger.warning( - f"Connection failed during {service} check, attempting reconnect..." - ) - host = get_ssh_connection(instance_ip, ssh_identity_file) + logger.warning(f"Connection failed during {service} check") return False return True while True: - if is_healthy( - host=host, - instance_ip=instance.public_ip_address, - ssh_identity_file=temp_key.get_priv_key_file(), - ): + if is_healthy(ssh): break sleep(1) - # return a testinfra connection to the instance - yield host + # Return both the SSH connection and instance IP for use in tests + yield { + 'ssh': ssh, + 'ip': instance.public_ip_address + } # at the end of the test suite, destroy the instance instance.terminate() def test_postgrest_is_running(host): - postgrest = host.service("postgrest") - assert postgrest.is_running + """Check if postgrest service is running using our SSH connection.""" + result = run_ssh_command(host['ssh'], "systemctl is-active postgrest") + assert result['succeeded'] and result['stdout'].strip() == 'active', "PostgREST service is not running" def test_postgrest_responds_to_requests(host): + """Test if PostgREST responds to requests.""" res = requests.get( - f"http://{host.backend.get_hostname()}/rest/v1/", + f"http://{host['ip']}/rest/v1/", headers={ "apikey": anon_key, "authorization": f"Bearer {anon_key}", @@ -367,8 +401,9 @@ def test_postgrest_responds_to_requests(host): def test_postgrest_can_connect_to_db(host): + """Test if PostgREST can connect to the database.""" res = requests.get( - f"http://{host.backend.get_hostname()}/rest/v1/buckets", + f"http://{host['ip']}/rest/v1/buckets", headers={ "apikey": service_role_key, "authorization": f"Bearer {service_role_key}", @@ -378,14 +413,10 @@ def test_postgrest_can_connect_to_db(host): assert res.ok -# There would be an error if the `apikey` query parameter isn't removed, -# since PostgREST treats query parameters as conditions. -# -# Worth testing since remove_apikey_query_parameters uses regexp instead -# of parsed query parameters. def test_postgrest_starting_apikey_query_parameter_is_removed(host): + """Test if PostgREST removes apikey query parameter at start.""" res = requests.get( - f"http://{host.backend.get_hostname()}/rest/v1/buckets", + f"http://{host['ip']}/rest/v1/buckets", headers={ "accept-profile": "storage", }, @@ -399,8 +430,9 @@ def test_postgrest_starting_apikey_query_parameter_is_removed(host): def test_postgrest_middle_apikey_query_parameter_is_removed(host): + """Test if PostgREST removes apikey query parameter in middle.""" res = requests.get( - f"http://{host.backend.get_hostname()}/rest/v1/buckets", + f"http://{host['ip']}/rest/v1/buckets", headers={ "accept-profile": "storage", }, @@ -414,8 +446,9 @@ def test_postgrest_middle_apikey_query_parameter_is_removed(host): def test_postgrest_ending_apikey_query_parameter_is_removed(host): + """Test if PostgREST removes apikey query parameter at end.""" res = requests.get( - f"http://{host.backend.get_hostname()}/rest/v1/buckets", + f"http://{host['ip']}/rest/v1/buckets", headers={ "accept-profile": "storage", }, @@ -428,14 +461,10 @@ def test_postgrest_ending_apikey_query_parameter_is_removed(host): assert res.ok -# There would be an error if the empty key query parameter isn't removed, -# since PostgREST treats empty key query parameters as malformed input. -# -# Worth testing since remove_apikey_and_empty_key_query_parameters uses regexp instead -# of parsed query parameters. def test_postgrest_starting_empty_key_query_parameter_is_removed(host): + """Test if PostgREST removes empty key query parameter at start.""" res = requests.get( - f"http://{host.backend.get_hostname()}/rest/v1/buckets", + f"http://{host['ip']}/rest/v1/buckets", headers={ "accept-profile": "storage", }, @@ -449,8 +478,9 @@ def test_postgrest_starting_empty_key_query_parameter_is_removed(host): def test_postgrest_middle_empty_key_query_parameter_is_removed(host): + """Test if PostgREST removes empty key query parameter in middle.""" res = requests.get( - f"http://{host.backend.get_hostname()}/rest/v1/buckets", + f"http://{host['ip']}/rest/v1/buckets", headers={ "accept-profile": "storage", }, @@ -464,8 +494,9 @@ def test_postgrest_middle_empty_key_query_parameter_is_removed(host): def test_postgrest_ending_empty_key_query_parameter_is_removed(host): + """Test if PostgREST removes empty key query parameter at end.""" res = requests.get( - f"http://{host.backend.get_hostname()}/rest/v1/buckets", + f"http://{host['ip']}/rest/v1/buckets", headers={ "accept-profile": "storage", },