diff --git a/nix/checks.nix b/nix/checks.nix index 13be10f23..016dd6d59 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -307,12 +307,11 @@ postgresql_17_src ; } - // pkgs.lib.optionalAttrs (system == "x86_64-linux") { - wrappers = import ./ext/tests/wrappers.nix { + // pkgs.lib.optionalAttrs (system == "x86_64-linux") ( + import ./ext/tests { inherit self; inherit pkgs; - }; - devShell = self'.devShells.default; - }; + } + ); }; } diff --git a/nix/ext/tests/default.nix b/nix/ext/tests/default.nix new file mode 100644 index 000000000..1097765ca --- /dev/null +++ b/nix/ext/tests/default.nix @@ -0,0 +1,173 @@ +{ self, pkgs }: +let + testsDir = ./.; + testFiles = builtins.attrNames (builtins.readDir testsDir); + nixFiles = builtins.filter ( + name: builtins.match ".*\\.nix$" name != null && name != "default.nix" + ) testFiles; + extTest = + extension_name: + let + pname = extension_name; + inherit (pkgs) lib; + installedExtension = + postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; + postgresqlWithExtension = + postgresql: + let + majorVersion = lib.versions.major postgresql.version; + pkg = pkgs.buildEnv { + name = "postgresql-${majorVersion}-${pname}"; + paths = [ + postgresql + postgresql.lib + (installedExtension majorVersion) + ]; + passthru = { + inherit (postgresql) version psqlSchema; + lib = pkg; + withPackages = _: pkg; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + pathsToLink = [ + "/" + "/bin" + "/lib" + ]; + postBuild = '' + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib + ''; + }; + in + pkg; + in + self.inputs.nixpkgs.lib.nixos.runTest { + name = pname; + hostPkgs = pkgs; + nodes.server = + { config, ... }: + { + virtualisation = { + forwardPorts = [ + { + from = "host"; + host.port = 13022; + guest.port = 22; + } + ]; + }; + services.openssh = { + enable = true; + }; + + services.postgresql = { + enable = true; + package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + enableTCPIP = true; + initialScript = pkgs.writeText "init-postgres-with-password" '' + CREATE USER test WITH PASSWORD 'secret'; + ''; + authentication = '' + host test postgres samenet scram-sha-256 + ''; + }; + + networking.firewall.allowedTCPPorts = [ config.services.postgresql.settings.port ]; + + specialisation.postgresql17.configuration = { + services.postgresql = { + package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); + }; + + systemd.services.postgresql-migrate = { + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = "postgres"; + Group = "postgres"; + StateDirectory = "postgresql"; + WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; + }; + script = + let + oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; + oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; + newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; + in + '' + if [[ ! -d ${newDataDir} ]]; then + install -d -m 0700 -o postgres -g postgres "${newDataDir}" + ${newPostgresql}/bin/initdb -D "${newDataDir}" + ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ + --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" + else + echo "${newDataDir} already exists" + fi + ''; + }; + + systemd.services.postgresql = { + after = [ "postgresql-migrate.service" ]; + requires = [ "postgresql-migrate.service" ]; + }; + }; + }; + testScript = + { nodes, ... }: + let + pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; + in + '' + versions = { + "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], + "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], + } + extension_name = "${pname}" + support_upgrade = True + pg17_configuration = "${pg17-configuration}" + + ${builtins.readFile ./lib.py} + + start_all() + + server.wait_for_unit("multi-user.target") + server.wait_for_unit("postgresql.service") + + test = PostgresExtensionTest(server, extension_name, versions, support_upgrade) + + with subtest("Check upgrade path with postgresql 15"): + test.check_upgrade_path("15") + + last_version = None + with subtest("Check the install of the last version of the extension"): + last_version = test.check_install_last_version("15") + + with subtest("switch to postgresql 17"): + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + + with subtest("Check last version of the extension after upgrade"): + test.assert_version_matches(last_version) + + with subtest("Check upgrade path with postgresql 17"): + test.check_upgrade_path("17") + ''; + }; +in +builtins.listToAttrs ( + map (file: { + name = "ext-" + builtins.replaceStrings [ ".nix" ] [ "" ] file; + value = import (testsDir + "/${file}") { inherit self pkgs; }; + }) nixFiles +) +// builtins.listToAttrs ( + map (extName: { + name = "ext-${extName}"; + value = extTest extName; + }) [ "wrappers" ] +) diff --git a/nix/ext/tests/lib.py b/nix/ext/tests/lib.py new file mode 100644 index 000000000..16e43cbfb --- /dev/null +++ b/nix/ext/tests/lib.py @@ -0,0 +1,109 @@ +"""PostgreSQL extension testing framework for multi-version compatibility. + +This module provides a test framework for PostgreSQL extensions that need to be +tested across multiple PostgreSQL versions and extension versions. It handles +installation, upgrades, and version verification of PostgreSQL extensions. +""" + +from typing import Sequence, Mapping +from test_driver.machine import Machine + +Versions = Mapping[str, Sequence[str]] + +class PostgresExtensionTest(object): + def __init__(self, vm: Machine, extension_name: str, versions: Versions, support_upgrade: bool = True): + """Initialize the PostgreSQL extension test framework. + + Args: + vm: Test machine instance for executing commands + extension_name: Name of the PostgreSQL extension to test + versions: Mapping of PostgreSQL versions to available extension versions + support_upgrade: Whether the extension supports in-place upgrades + """ + self.vm = vm + self.extension_name = extension_name + self.versions = versions + self.support_upgrade = support_upgrade + + def run_sql(self, query: str) -> str: + return self.vm.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() + + def drop_extension(self): + self.run_sql(f"DROP EXTENSION IF EXISTS {self.extension_name};") + + def install_extension(self, version: str): + self.run_sql(f"""CREATE EXTENSION {self.extension_name} WITH VERSION '{version}' CASCADE;""") + # Verify version was installed correctly + self.assert_version_matches(version) + + def update_extension(self, version: str): + self.run_sql(f"""ALTER EXTENSION {self.extension_name} UPDATE TO '{version}';""") + # Verify version was installed correctly + self.assert_version_matches(version) + + def get_installed_version(self) -> str: + """Get the currently installed version of the extension. + + Returns: + Version string of the currently installed extension, + or empty string if extension is not installed + """ + return self.run_sql(f"""SELECT extversion FROM pg_extension WHERE extname = '{self.extension_name}';""") + + def assert_version_matches(self, expected_version: str): + """Check if the installed version matches the expected version. + + Args: + expected_version: Expected version string to verify against + + Raises: + AssertionError: If the installed version does not match the expected version + """ + installed_version = self.get_installed_version() + assert installed_version == expected_version, f"Expected version {expected_version}, but found {installed_version}" + + def check_upgrade_path(self, pg_version): + """Test the complete upgrade path for a PostgreSQL version. + + This method tests all available extension versions for a given PostgreSQL + version, either through in-place upgrades or reinstallation depending on + the support_upgrade setting. + + Args: + pg_version: PostgreSQL version to test (e.g., "14", "15") + + Raises: + ValueError: If no versions are available for the specified PostgreSQL version + AssertionError: If version installation or upgrade fails + """ + available_versions = self.versions.get(pg_version, []) + if not available_versions: + raise ValueError(f"No versions available for PostgreSQL version {pg_version}") + + # Install and verify first version + firstVersion = available_versions[0] + self.drop_extension() + self.install_extension(firstVersion) + + # Test remaining versions + for version in available_versions[1:]: + if self.support_upgrade: + self.update_extension(version) + else: + self.drop_extension() + self.install_extension(version) + + + def check_install_last_version(self, pg_version: str) -> str: + """Test if the install of the last version of the extension works for a given PostgreSQL version. + + Args: + pg_version: PostgreSQL version to check (e.g., "14", "15") + """ + available_versions = self.versions.get(pg_version, []) + if not available_versions: + raise ValueError(f"No versions available for PostgreSQL version {pg_version}") + last_version = available_versions[-1] + self.drop_extension() + self.install_extension(last_version) + return last_version diff --git a/nix/ext/tests/wrappers.nix b/nix/ext/tests/wrappers.nix deleted file mode 100644 index 54cc63944..000000000 --- a/nix/ext/tests/wrappers.nix +++ /dev/null @@ -1,157 +0,0 @@ -{ self, pkgs }: -let - pname = "wrappers"; - inherit (pkgs) lib; - installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; - versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; - postgresqlWithExtension = - postgresql: - let - majorVersion = lib.versions.major postgresql.version; - pkg = pkgs.buildEnv { - name = "postgresql-${majorVersion}-${pname}"; - paths = [ - postgresql - postgresql.lib - (installedExtension majorVersion) - ]; - passthru = { - inherit (postgresql) version psqlSchema; - lib = pkg; - withPackages = _: pkg; - }; - nativeBuildInputs = [ pkgs.makeWrapper ]; - pathsToLink = [ - "/" - "/bin" - "/lib" - ]; - postBuild = '' - wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib - wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib - wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib - ''; - }; - in - pkg; -in -self.inputs.nixpkgs.lib.nixos.runTest { - name = pname; - hostPkgs = pkgs; - nodes.server = - { config, ... }: - { - virtualisation = { - forwardPorts = [ - { - from = "host"; - host.port = 13022; - guest.port = 22; - } - ]; - }; - services.openssh = { - enable = true; - }; - users.users.root.openssh.authorizedKeys.keys = [ - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIo+ulCUfJjnCVgfM4946Ih5Nm8DeZZiayYeABHGPEl7 jfroche" - ]; - - services.postgresql = { - enable = true; - package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; - }; - - specialisation.postgresql17.configuration = { - services.postgresql = { - package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); - }; - - systemd.services.postgresql-migrate = { - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - User = "postgres"; - Group = "postgres"; - StateDirectory = "postgresql"; - WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; - }; - script = - let - oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; - newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; - oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; - newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; - in - '' - if [[ ! -d ${newDataDir} ]]; then - install -d -m 0700 -o postgres -g postgres "${newDataDir}" - ${newPostgresql}/bin/initdb -D "${newDataDir}" - ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ - --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" - else - echo "${newDataDir} already exists" - fi - ''; - }; - - systemd.services.postgresql = { - after = [ "postgresql-migrate.service" ]; - requires = [ "postgresql-migrate.service" ]; - }; - }; - }; - testScript = - { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in - '' - versions = { - "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], - "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], - } - - def run_sql(query): - return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() - - def check_upgrade_path(pg_version): - with subtest("Check ${pname} upgrade path"): - firstVersion = versions[pg_version][0] - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'") - run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}';""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}" - for version in versions[pg_version][1:]: - run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}" - - start_all() - - server.wait_for_unit("multi-user.target") - server.wait_for_unit("postgresql.service") - - check_upgrade_path("15") - - with subtest("Check ${pname} latest extension version"): - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'") - server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname};'") - installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") - latestVersion = versions["15"][-1] - assert f"${pname},{latestVersion}" in installed_extensions - - with subtest("switch to postgresql 17"): - server.succeed( - "${pg17-configuration}/bin/switch-to-configuration test >&2" - ) - - with subtest("Check ${pname} latest extension version"): - installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") - latestVersion = versions["17"][-1] - assert f"${pname},{latestVersion}" in installed_extensions - - check_upgrade_path("17") - ''; -} diff --git a/nix/ext/update_versions_json.py b/nix/ext/update_versions_json.py new file mode 100755 index 000000000..85d25d6b1 --- /dev/null +++ b/nix/ext/update_versions_json.py @@ -0,0 +1,90 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i python3 -p python3 git nix-prefetch-git python3Packages.packaging + +import subprocess +import json +import argparse +from pathlib import Path +from typing import Dict, List, Union +from packaging.version import parse as parse_version, InvalidVersion + +Schema = Dict[str, Dict[str, Dict[str, Union[List[str], str, bool]]]] + +POSTGRES_VERSIONS: List[str] = ["15", "17"] +VERSIONS_JSON_PATH = "versions.json" + + +def run(cmd: List[str]) -> str: + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + return result.stdout.strip() + + +def get_tags(url: str) -> Dict[str, str]: + output = run(["git", "ls-remote", "--tags", url]) + tags: Dict[str, str] = {} + for line in output.splitlines(): + if "^{}" not in line: + parts = line.split("\t") + if len(parts) == 2: + commit_hash, ref = parts + if ref.startswith("refs/tags/"): + tag = ref.removeprefix("refs/tags/") + try: + parse_version(tag) + except InvalidVersion: + continue + tags[tag] = commit_hash + return tags + + +def get_sri_hash(url: str, commit_hash: str) -> str: + output = run(["nix-prefetch-git", "--quiet", "--url", url, "--rev", commit_hash]) + nix_hash = json.loads(output)["sha256"] + return "sha256-" + run(["nix", "hash", "to-base64", "--type", "sha256", nix_hash]) + + +def load() -> Schema: + if not Path(VERSIONS_JSON_PATH).exists(): + return {} + with open(VERSIONS_JSON_PATH, "r", encoding="utf-8") as f: + return json.load(f) + + +def build(name: str, url: str, data: Schema, ignore: bool = False) -> Schema: + tags = get_tags(url) + versions = data.get(name, {}) + for tag, commit_hash in tags.items(): + if tag in versions: + continue + if ignore: + versions[tag] = {"ignore": True} + else: + sri_hash = get_sri_hash(url, commit_hash) + versions[tag] = {"postgresql": POSTGRES_VERSIONS, "hash": sri_hash} + data[name] = versions + return data + + +def save(data: Schema) -> None: + sorted_data = {} + for name, versions in data.items(): + sorted_data[name] = dict( + sorted(versions.items(), key=lambda item: parse_version(item[0])) + ) + with open(VERSIONS_JSON_PATH, "w", encoding="utf-8") as f: + json.dump(sorted_data, f, indent=2) + f.write("\n") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("extension_name") + parser.add_argument("git_repo_url") + parser.add_argument("--ignore", action="store_true") + args = parser.parse_args() + + save(build(args.extension_name, args.git_repo_url, load(), ignore=args.ignore)) + + +if __name__ == "__main__": + main() diff --git a/nix/ext/versions.json b/nix/ext/versions.json index e5c4e90b1..b4bebf39c 100644 --- a/nix/ext/versions.json +++ b/nix/ext/versions.json @@ -1,5 +1,91 @@ { "wrappers": { + "0.3.0": { + "postgresql": [ + "15" + ], + "hash": "sha256-ogpF8NJ7kW3Ut8jaKMDiKYIXnI38nfRq2mMK4rqFAIA=", + "pgrx": "0.11.3", + "rust": "1.76.0" + }, + "0.4.1": { + "postgresql": [ + "15" + ], + "hash": "sha256-AU9Y43qEMcIBVBThu+Aor1HCtfFIg+CdkzK9IxVdkzM=", + "pgrx": "0.11.3", + "rust": "1.76.0" + }, + "0.4.2": { + "postgresql": [ + "15" + ], + "hash": "sha256-ut3IQED6ANXgabiHoEUdfSrwkuuYYSpRoeWdtBvSe64=", + "pgrx": "0.11.3", + "rust": "1.76.0" + }, + "0.4.3": { + "postgresql": [ + "15" + ], + "hash": "sha256-CkoNMoh40zbQL4V49ZNYgv3JjoNWjODtTpHn+L8DdZA=", + "pgrx": "0.12.6", + "rust": "1.80.0" + }, + "0.4.4": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-QoGFJpq8PuvMM8SS+VZd7MlNl56uFivRjs1tCtwX+oE=", + "pgrx": "0.12.6", + "rust": "1.80.0" + }, + "0.4.5": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-IgDfVFROMCHYLZ/Iqj12MsQjPPCdRoH+3oi3Ki/iaRI=", + "pgrx": "0.12.9", + "rust": "1.81.0" + }, + "0.4.6": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-hthb3qEXT1Kf4yPoq0udEbQzlyLtI5tug6sK4YAPFjU=", + "pgrx": "0.12.9", + "rust": "1.84.0" + }, + "0.5.0": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-FbRTUcpEHBa5DI6dutvBeahYM0RZVAXIzIAZWIaxvn0=", + "pgrx": "0.12.9", + "rust": "1.84.0" + }, + "0.5.1": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-3GfN3vZMFWf4FV/fSOe9ZN6KETmjoNw3Paz+JRzaH3c=", + "pgrx": "0.12.9", + "rust": "1.87.0" + }, + "0.5.2": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-9VqQHduoAWnY8gtfRZLDOKiibfwuSTzyVFbH0uhsfCU=", + "pgrx": "0.14.3", + "rust": "1.87.0" + }, "0.5.3": { "postgresql": [ "15", diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 218c65e77..8bdbce163 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -155,6 +155,8 @@ let }; } // lib.optionalAttrs (version == "0.3.0") { + # TODO: there is an inference error on crate `time` caused by an API change in Rust 1.80.0; + # so we should patch `Cargo.toml` with `time >= 0.3.35`, to use a more recent Rust version! patches = [ ./0001-bump-pgrx-to-0.11.3.patch ]; cargoLock = { @@ -167,7 +169,9 @@ let ); allVersions = (builtins.fromJSON (builtins.readFile ../versions.json)).wrappers; supportedVersions = lib.filterAttrs ( - _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + _: value: + (!value ? ignore || value.ignore != true) + && builtins.elem (lib.versions.major postgresql.version) value.postgresql ) allVersions; versions = lib.naturalSort (lib.attrNames supportedVersions); latestVersion = lib.last versions;