From dea5cbc19cdc055fd118ac6498232fc11ac488fe Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Fri, 29 Oct 2021 18:09:47 -0400 Subject: [PATCH 01/19] Initial changes to postgresql extension handling --- plugins/modules/postgresql_ext.py | 139 ++++++++++++++---------------- 1 file changed, 63 insertions(+), 76 deletions(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index 579c59b7..ce83434c 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -171,8 +171,6 @@ import traceback -from distutils.version import LooseVersion - try: from psycopg2.extras import DictCursor except ImportError: @@ -198,14 +196,9 @@ # PostgreSQL module specific support methods. # -def ext_exists(cursor, ext): - query = "SELECT * FROM pg_extension WHERE extname=%(ext)s" - cursor.execute(query, {'ext': ext}) - return cursor.rowcount == 1 - -def ext_delete(cursor, ext, cascade): - if ext_exists(cursor, ext): +def ext_delete(cursor, ext, curr_version, cascade): + if curr_version: query = "DROP EXTENSION \"%s\"" % ext if cascade: query += " CASCADE" @@ -245,7 +238,7 @@ def ext_create(cursor, ext, schema, cascade, version): if schema: query += " WITH SCHEMA \"%s\"" % schema - if version: + if version != 'latest': query += " VERSION %(ver)s" params['ver'] = version if cascade: @@ -286,46 +279,38 @@ def ext_get_versions(cursor, ext): query = ("SELECT version FROM pg_available_extension_versions " "WHERE name = %(ext)s") cursor.execute(query, {'ext': ext}) - res = cursor.fetchall() - - available_versions = parse_ext_versions(current_version, res) + available_versions = cursor.fetchall() if current_version is None: current_version = False return (current_version, available_versions) - -def parse_ext_versions(current_version, ext_ver_list): - """Parse ext versions. - - Args: - current_version (str) -- version to compare elements of ext_ver_list with - ext_ver_list (list) -- list containing dicts with versions - - Return a sorted list with versions that are higher than current_version. - - Note: Incomparable versions (e.g., postgis version "unpackaged") are skipped. +def ext_valid_update_path(cursor, ext, current_version, version): + """ + Value of 'latest' is always a valid path. """ - available_versions = [] - - for line in ext_ver_list: - if line['version'] == 'unpackaged': - continue - try: - if current_version is None: - if LooseVersion(line['version']) >= LooseVersion('0'): - available_versions.append(line['version']) - else: - if LooseVersion(line['version']) > LooseVersion(current_version): - available_versions.append(line['version']) - except Exception: - # When a version cannot be compared, skip it - # (there's a note in the documentation) - continue + valid_path = False + if version != 'latest': + query = ("SELECT path FROM pg_extension_update_paths(%(ext)s)" + "WHERE source = %(current_version)s" + "AND target = %(version)s") + params = {} + + params['ext'] = ext + params['current_version'] = current_version + params['version'] = version + + cursor.execute(query, params) + res = cursor.fetchone() + if res != None: + valid_path = True + else: + valid_path = True + + return (valid_path) - return sorted(available_versions, key=LooseVersion) # =========================================== # Module execution. @@ -374,65 +359,67 @@ def main(): curr_version, available_versions = ext_get_versions(cursor, ext) if state == "present": - if version == 'latest': - if available_versions: - version = available_versions[-1] - else: - version = '' + # If version passed if version: - # If the specific version is passed and it is not available for update: - if version not in available_versions: - if not curr_version: - module.fail_json(msg="Passed version '%s' is not available" % version) - - elif LooseVersion(curr_version) == LooseVersion(version): + # If extension is installed, update to passed version if a valid path exists + if curr_version: + if curr_version == version: changed = False - else: - module.fail_json(msg="Passed version '%s' is lower than " - "the current created version '%s' or " - "the passed version is not available" % (version, curr_version)) - - # If the specific version is passed and it is higher that the current version: - if curr_version: - if LooseVersion(curr_version) < LooseVersion(version): + valid_update_path = ext_valid_update_path(cursor, ext, curr_version, version) + if valid_update_path: + if module.check_mode: + changed = True + else: + changed = ext_update_version(cursor, ext, version) + else: + module.fail_json(msg="Passed version '%s' has no valid update path from " + "the current created version '%s' or " + "the passed version is not available" % (version, curr_version)) + else: + # If not requesting latest version and passed version not available + if version != 'latest' and version not in available_versions: + module.fail_json(msg="Passed version '%s' is not available" % version) + # Else install the passed version when available + else: if module.check_mode: changed = True else: - changed = ext_update_version(cursor, ext, version) - - # If the specific version is passed and it is created now: - if curr_version == version: - changed = False + changed = ext_create(cursor, ext, schema, cascade, version) - # If the ext doesn't exist and installed: - elif not curr_version and available_versions: + # If version is not passed: + else: + # Extension exists, attempt to update to latest version defined in extension control file + # ALTER EXTENSION is actually run, so 'changed' is technically true even if nothing updated + if curr_version and version == 'latest': if module.check_mode: changed = True else: - changed = ext_create(cursor, ext, schema, cascade, version) - - # If version is not passed: - else: - if not curr_version: - # If the ext doesn't exist and it's installed: + changed = ext_update_version(cursor, ext, version) + # Extension exists, no request to update so no change + elif curr_version: + if module.check_mode: + changed = False + else: + # If the ext doesn't exist and is available: if available_versions: if module.check_mode: changed = True else: changed = ext_create(cursor, ext, schema, cascade, version) - # If the ext doesn't exist and not installed: + # If the ext doesn't exist and is not available: else: - module.fail_json(msg="Extension %s is not installed" % ext) + module.fail_json(msg="Extension %s is not available" % ext) + elif state == "absent": if curr_version: if module.check_mode: changed = True else: - changed = ext_delete(cursor, ext, cascade) + changed = ext_delete(cursor, ext, curr_version, cascade) else: changed = False From db473abb44aeda6984a81392d0b927d409c1048b Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Mon, 1 Nov 2021 18:57:55 -0400 Subject: [PATCH 02/19] Properly handle latest version flag. Formatting fixes. --- plugins/modules/postgresql_ext.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index ce83434c..ac2abd2c 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -267,10 +267,10 @@ def ext_get_versions(cursor, ext): # 1. Get the current extension version: query = ("SELECT extversion FROM pg_catalog.pg_extension " - "WHERE extname = %(ext)s") + "WHERE extname = '%s'" % ext) current_version = None - cursor.execute(query, {'ext': ext}) + cursor.execute(query) res = cursor.fetchone() if res: current_version = res[0] @@ -279,28 +279,30 @@ def ext_get_versions(cursor, ext): query = ("SELECT version FROM pg_available_extension_versions " "WHERE name = %(ext)s") cursor.execute(query, {'ext': ext}) - available_versions = cursor.fetchall() + + available_versions = [r[0] for r in cursor.fetchall()] if current_version is None: current_version = False return (current_version, available_versions) + def ext_valid_update_path(cursor, ext, current_version, version): """ Value of 'latest' is always a valid path. """ valid_path = False + params = {} if version != 'latest': query = ("SELECT path FROM pg_extension_update_paths(%(ext)s)" "WHERE source = %(current_version)s" "AND target = %(version)s") - params = {} params['ext'] = ext - params['current_version'] = current_version - params['version'] = version + params['cv'] = current_version + params['ver'] = version cursor.execute(query, params) res = cursor.fetchone() @@ -364,6 +366,7 @@ def main(): if version: # If extension is installed, update to passed version if a valid path exists if curr_version: + # Given version already installed if curr_version == version: changed = False else: @@ -386,28 +389,28 @@ def main(): if module.check_mode: changed = True else: - changed = ext_create(cursor, ext, schema, cascade, version) + changed = ext_create(cursor, ext, schema, cascade, 'latest') # If version is not passed: else: # Extension exists, attempt to update to latest version defined in extension control file - # ALTER EXTENSION is actually run, so 'changed' is technically true even if nothing updated + # ALTER EXTENSION is actually run, so 'changed' is technically true even if nothing updated if curr_version and version == 'latest': if module.check_mode: changed = True else: changed = ext_update_version(cursor, ext, version) - # Extension exists, no request to update so no change - elif curr_version: - if module.check_mode: - changed = False + # Extension exists, no request to update so no change + elif curr_version: + if module.check_mode: + changed = False else: # If the ext doesn't exist and is available: if available_versions: if module.check_mode: changed = True else: - changed = ext_create(cursor, ext, schema, cascade, version) + changed = ext_create(cursor, ext, schema, cascade, 'latest') # If the ext doesn't exist and is not available: else: From 0c4f4be7a51a5a3c3f23d5d0626ed4abeec26b75 Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Mon, 15 Nov 2021 17:11:49 -0500 Subject: [PATCH 03/19] Fix variable usage. Also make param usage consistent. Update docs. --- plugins/modules/postgresql_ext.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index ac2abd2c..0191f914 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -72,9 +72,9 @@ description: - Extension version to add or update to. Has effect with I(state=present) only. - If not specified, the latest extension version will be created. - - It can't downgrade an extension version. - When version downgrade is needed, remove the extension and create new one with appropriate version. - - Set I(version=latest) to update the extension to the latest available version. + - Version downgrade is only supported if extension provides a downgrade path. + Otherwise extension must be removed and lower version of extension must be made available. + - Set I(version=latest) to always update the extension to the latest available version. type: str trust_input: description: @@ -114,6 +114,7 @@ - Thomas O'Donnell (@andytom) - Sandro Santilli (@strk) - Andrew Klychkov (@Andersson007) +- Keith Fiske (@keithf4) extends_documentation_fragment: - community.postgresql.postgres @@ -147,13 +148,13 @@ cascade: yes state: absent -- name: Create extension foo of version 1.2 or update it if it's already created +- name: Create extension foo of version 1.2 or update it to that version if it's already created and a valid update path exists community.postgresql.postgresql_ext: db: acme name: foo version: 1.2 -- name: Assuming extension foo is created, update it to the latest version +- name: Create latest available version of extension foo. If already installed, update it to the latest version community.postgresql.postgresql_ext: db: acme name: foo @@ -265,12 +266,16 @@ def ext_get_versions(cursor, ext): ext (str) -- extension name """ + current_version = None + params = {} + params['ext'] = ext + # 1. Get the current extension version: query = ("SELECT extversion FROM pg_catalog.pg_extension " - "WHERE extname = '%s'" % ext) + "WHERE extname = %(ext)s") + + cursor.execute(query, params) - current_version = None - cursor.execute(query) res = cursor.fetchone() if res: current_version = res[0] @@ -278,7 +283,8 @@ def ext_get_versions(cursor, ext): # 2. Get available versions: query = ("SELECT version FROM pg_available_extension_versions " "WHERE name = %(ext)s") - cursor.execute(query, {'ext': ext}) + + cursor.execute(query, params) available_versions = [r[0] for r in cursor.fetchall()] @@ -297,8 +303,8 @@ def ext_valid_update_path(cursor, ext, current_version, version): params = {} if version != 'latest': query = ("SELECT path FROM pg_extension_update_paths(%(ext)s)" - "WHERE source = %(current_version)s" - "AND target = %(version)s") + "WHERE source = %(cv)s" + "AND target = %(ver)s") params['ext'] = ext params['cv'] = current_version @@ -378,7 +384,7 @@ def main(): changed = ext_update_version(cursor, ext, version) else: module.fail_json(msg="Passed version '%s' has no valid update path from " - "the current created version '%s' or " + "the currently installed version '%s' or " "the passed version is not available" % (version, curr_version)) else: # If not requesting latest version and passed version not available @@ -402,7 +408,6 @@ def main(): changed = ext_update_version(cursor, ext, version) # Extension exists, no request to update so no change elif curr_version: - if module.check_mode: changed = False else: # If the ext doesn't exist and is available: From 2ec1b965f2cbb1b5afb442997a83ca6af4964c83 Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Tue, 16 Nov 2021 09:50:41 -0500 Subject: [PATCH 04/19] Update plugins/modules/postgresql_ext.py Co-authored-by: Douglas J Hunley --- plugins/modules/postgresql_ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index 0191f914..f8b460bc 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -154,7 +154,7 @@ name: foo version: 1.2 -- name: Create latest available version of extension foo. If already installed, update it to the latest version +- name: Create the latest available version of extension foo. If already installed, update it to the latest version community.postgresql.postgresql_ext: db: acme name: foo From 709febe2b298669895da65dcd67fca47b1ae2fa4 Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Tue, 16 Nov 2021 09:51:20 -0500 Subject: [PATCH 05/19] Update plugins/modules/postgresql_ext.py Co-authored-by: Douglas J Hunley --- plugins/modules/postgresql_ext.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index f8b460bc..8b36eda8 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -72,8 +72,7 @@ description: - Extension version to add or update to. Has effect with I(state=present) only. - If not specified, the latest extension version will be created. - - Version downgrade is only supported if extension provides a downgrade path. - Otherwise extension must be removed and lower version of extension must be made available. + - Downgrading is only supported if the extension provides a downgrade path otherwise the extension must be removed and a lower version of the extension must be made available. - Set I(version=latest) to always update the extension to the latest available version. type: str trust_input: From 7bbaaf2f38eb3fa68cea2a74cb71460fcddce075 Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Tue, 16 Nov 2021 14:32:26 -0500 Subject: [PATCH 06/19] Fix CI test failures --- plugins/modules/postgresql_ext.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index 8b36eda8..54961553 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -147,13 +147,14 @@ cascade: yes state: absent -- name: Create extension foo of version 1.2 or update it to that version if it's already created and a valid update path exists - community.postgresql.postgresql_ext: +- name: Create extension foo of version 1.2 or update it to that version if it's already + created and a valid update path exists community.postgresql.postgresql_ext: db: acme name: foo version: 1.2 -- name: Create the latest available version of extension foo. If already installed, update it to the latest version +- name: Create the latest available version of extension foo. If already installed, + update it to the latest version community.postgresql.postgresql_ext: db: acme name: foo @@ -302,8 +303,8 @@ def ext_valid_update_path(cursor, ext, current_version, version): params = {} if version != 'latest': query = ("SELECT path FROM pg_extension_update_paths(%(ext)s)" - "WHERE source = %(cv)s" - "AND target = %(ver)s") + "WHERE source = %(cv)s" + "AND target = %(ver)s") params['ext'] = ext params['cv'] = current_version @@ -311,11 +312,11 @@ def ext_valid_update_path(cursor, ext, current_version, version): cursor.execute(query, params) res = cursor.fetchone() - if res != None: + if res is not None: valid_path = True else: valid_path = True - + return (valid_path) @@ -407,7 +408,7 @@ def main(): changed = ext_update_version(cursor, ext, version) # Extension exists, no request to update so no change elif curr_version: - changed = False + changed = False else: # If the ext doesn't exist and is available: if available_versions: @@ -420,7 +421,6 @@ def main(): else: module.fail_json(msg="Extension %s is not available" % ext) - elif state == "absent": if curr_version: if module.check_mode: From 8ae685302d810a0c69d22e0a8ec46e545335d6fe Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Tue, 16 Nov 2021 14:37:06 -0500 Subject: [PATCH 07/19] Wrap long doc line --- plugins/modules/postgresql_ext.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index 54961553..8b086940 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -72,7 +72,8 @@ description: - Extension version to add or update to. Has effect with I(state=present) only. - If not specified, the latest extension version will be created. - - Downgrading is only supported if the extension provides a downgrade path otherwise the extension must be removed and a lower version of the extension must be made available. + - Downgrading is only supported if the extension provides a downgrade path otherwise + the extension must be removed and a lower version of the extension must be made available. - Set I(version=latest) to always update the extension to the latest available version. type: str trust_input: From 360ec492ec64374afdc2fd95d70b1a549343f17d Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Tue, 16 Nov 2021 15:16:54 -0500 Subject: [PATCH 08/19] Fix invalid yaml in examples --- plugins/modules/postgresql_ext.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index 8b086940..ddd806ae 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -148,14 +148,13 @@ cascade: yes state: absent -- name: Create extension foo of version 1.2 or update it to that version if it's already - created and a valid update path exists community.postgresql.postgresql_ext: +- name: Create extension foo of version 1.2 or update it to that version if it's already created and a valid update path exists + community.postgresql.postgresql_ext: db: acme name: foo version: 1.2 -- name: Create the latest available version of extension foo. If already installed, - update it to the latest version +- name: Create the latest available version of extension foo. If already installed, update it to the latest version community.postgresql.postgresql_ext: db: acme name: foo From e4748cba806d80aeedd5de9b20fa4fb3f3e8da30 Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Tue, 16 Nov 2021 17:14:39 -0500 Subject: [PATCH 09/19] Fix handling given version on extension creation. Use safe sql string generation with identifiers. --- plugins/modules/postgresql_ext.py | 32 ++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index ddd806ae..bbd22513 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -174,6 +174,7 @@ try: from psycopg2.extras import DictCursor + from psycopg2 import sql except ImportError: # psycopg2 is checked by connect_to_db() # from ansible.module_utils.postgres @@ -200,11 +201,12 @@ def ext_delete(cursor, ext, curr_version, cascade): if curr_version: - query = "DROP EXTENSION \"%s\"" % ext + query = sql.SQL("DROP EXTENSION {extension}").format( + extension=sql.Identifier(ext)) if cascade: - query += " CASCADE" + query += sql.SQL(" CASCADE") cursor.execute(query) - executed_queries.append(query) + executed_queries.append(cursor.mogrify(query)) return True else: return False @@ -220,11 +222,12 @@ def ext_update_version(cursor, ext, version): ext (str) -- extension name version (str) -- extension version """ - query = "ALTER EXTENSION \"%s\" UPDATE" % ext + query = sql.SQL("ALTER EXTENSION {extension} UPDATE").format( + extension=sql.Identifier(ext)) params = {} if version != 'latest': - query += " TO %(ver)s" + query += sql.SQL( " TO %(ver)s") params['ver'] = version cursor.execute(query, params) @@ -234,16 +237,18 @@ def ext_update_version(cursor, ext, version): def ext_create(cursor, ext, schema, cascade, version): - query = "CREATE EXTENSION \"%s\"" % ext + query = sql.SQL("CREATE EXTENSION {extension}").format( + extension=sql.Identifier(ext)) params = {} if schema: - query += " WITH SCHEMA \"%s\"" % schema + query += sql.SQL(" WITH SCHEMA {ext_schema}").format( + ext_schema=sql.Identifier(schema)) if version != 'latest': - query += " VERSION %(ver)s" + query += sql.SQL(" VERSION %(ver)s") params['ver'] = version if cascade: - query += " CASCADE" + query += sql.SQL(" CASCADE") cursor.execute(query, params) executed_queries.append(cursor.mogrify(query, params)) @@ -302,8 +307,8 @@ def ext_valid_update_path(cursor, ext, current_version, version): valid_path = False params = {} if version != 'latest': - query = ("SELECT path FROM pg_extension_update_paths(%(ext)s)" - "WHERE source = %(cv)s" + query = ("SELECT path FROM pg_extension_update_paths(%(ext)s) " + "WHERE source = %(cv)s " "AND target = %(ver)s") params['ext'] = ext @@ -311,6 +316,7 @@ def ext_valid_update_path(cursor, ext, current_version, version): params['ver'] = version cursor.execute(query, params) + executed_queries.append(cursor.mogrify(query, params)) res = cursor.fetchone() if res is not None: valid_path = True @@ -395,7 +401,7 @@ def main(): if module.check_mode: changed = True else: - changed = ext_create(cursor, ext, schema, cascade, 'latest') + changed = ext_create(cursor, ext, schema, cascade, version) # If version is not passed: else: @@ -432,7 +438,7 @@ def main(): except Exception as e: db_connection.close() - module.fail_json(msg="Database query failed: %s" % to_native(e), exception=traceback.format_exc()) + module.fail_json(msg="Management of PostgreSQL extension failed: %s" % to_native(e), exception=traceback.format_exc()) db_connection.close() module.exit_json(changed=changed, db=module.params["db"], ext=ext, queries=executed_queries) From 2fae4ea28290a58c046e7f7d84709e9d395ee2f1 Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Tue, 16 Nov 2021 17:28:20 -0500 Subject: [PATCH 10/19] Fix CI test failures --- plugins/modules/postgresql_ext.py | 14 +++++--------- .../tasks/postgresql_ext_version_opt.yml | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index bbd22513..d165c8ca 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -201,8 +201,7 @@ def ext_delete(cursor, ext, curr_version, cascade): if curr_version: - query = sql.SQL("DROP EXTENSION {extension}").format( - extension=sql.Identifier(ext)) + query = sql.SQL("DROP EXTENSION {extension}").format(extension=sql.Identifier(ext)) if cascade: query += sql.SQL(" CASCADE") cursor.execute(query) @@ -222,12 +221,11 @@ def ext_update_version(cursor, ext, version): ext (str) -- extension name version (str) -- extension version """ - query = sql.SQL("ALTER EXTENSION {extension} UPDATE").format( - extension=sql.Identifier(ext)) + query = sql.SQL("ALTER EXTENSION {extension} UPDATE").format(extension=sql.Identifier(ext)) params = {} if version != 'latest': - query += sql.SQL( " TO %(ver)s") + query += sql.SQL(" TO %(ver)s") params['ver'] = version cursor.execute(query, params) @@ -237,13 +235,11 @@ def ext_update_version(cursor, ext, version): def ext_create(cursor, ext, schema, cascade, version): - query = sql.SQL("CREATE EXTENSION {extension}").format( - extension=sql.Identifier(ext)) + query = sql.SQL("CREATE EXTENSION {extension}").format(extension=sql.Identifier(ext)) params = {} if schema: - query += sql.SQL(" WITH SCHEMA {ext_schema}").format( - ext_schema=sql.Identifier(schema)) + query += sql.SQL(" WITH SCHEMA {ext_schema}").format(ext_schema=sql.Identifier(schema)) if version != 'latest': query += sql.SQL(" VERSION %(ver)s") params['ver'] = version diff --git a/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml b/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml index 1c47ef4e..0380df22 100644 --- a/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml +++ b/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml @@ -153,7 +153,7 @@ - assert: that: - result is changed - - result.queries == ["ALTER EXTENSION \"{{ test_ext }}\" UPDATE TO '2.0'"] + - result.queries == ["SELECT path FROM pg_extension_update_paths('dummy') WHERE source = '1.0' AND target = '2.0'", "ALTER EXTENSION \"{{ test_ext }}\" UPDATE TO '2.0'"] - name: postgresql_ext_version - check, the version must be 2.0 <<: *task_parameters From f85df4601b9669cd5aaf4608156c6aa26c0a603b Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Tue, 16 Nov 2021 19:45:51 -0500 Subject: [PATCH 11/19] Revert safe sql string building. Not supported in CentOS7/U20 --- plugins/modules/postgresql_ext.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index d165c8ca..155d3d75 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -174,7 +174,6 @@ try: from psycopg2.extras import DictCursor - from psycopg2 import sql except ImportError: # psycopg2 is checked by connect_to_db() # from ansible.module_utils.postgres @@ -201,9 +200,9 @@ def ext_delete(cursor, ext, curr_version, cascade): if curr_version: - query = sql.SQL("DROP EXTENSION {extension}").format(extension=sql.Identifier(ext)) + query = "DROP EXTENSION \"%s\"" % ext if cascade: - query += sql.SQL(" CASCADE") + query += " CASCADE" cursor.execute(query) executed_queries.append(cursor.mogrify(query)) return True @@ -221,11 +220,11 @@ def ext_update_version(cursor, ext, version): ext (str) -- extension name version (str) -- extension version """ - query = sql.SQL("ALTER EXTENSION {extension} UPDATE").format(extension=sql.Identifier(ext)) + query = "ALTER EXTENSION \"%s\" UPDATE" % ext params = {} if version != 'latest': - query += sql.SQL(" TO %(ver)s") + query += " TO %(ver)s" params['ver'] = version cursor.execute(query, params) @@ -235,16 +234,16 @@ def ext_update_version(cursor, ext, version): def ext_create(cursor, ext, schema, cascade, version): - query = sql.SQL("CREATE EXTENSION {extension}").format(extension=sql.Identifier(ext)) + query = "CREATE EXTENSION \"%s\"" % ext params = {} if schema: - query += sql.SQL(" WITH SCHEMA {ext_schema}").format(ext_schema=sql.Identifier(schema)) + query += " WITH SCHEMA \"%s\"" % schema if version != 'latest': - query += sql.SQL(" VERSION %(ver)s") + query += " VERSION %(ver)s" params['ver'] = version if cascade: - query += sql.SQL(" CASCADE") + query += " CASCADE" cursor.execute(query, params) executed_queries.append(cursor.mogrify(query, params)) From 206f1fc0e43237c4f3e3d124260f9d7dd5ed7655 Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Tue, 16 Nov 2021 19:55:55 -0500 Subject: [PATCH 12/19] Fix CI Test on Ubuntu to use new extension update syntax for latest version --- .../targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml b/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml index 0380df22..6f8c8c49 100644 --- a/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml +++ b/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml @@ -199,7 +199,7 @@ - assert: that: - result is changed - - result.queries == ["ALTER EXTENSION \"{{ test_ext }}\" UPDATE TO '3.0'"] + - result.queries == ["ALTER EXTENSION \"{{ test_ext }}\" UPDATE"] - name: postgresql_ext_version - check <<: *task_parameters From 81cdbdb90c3fe3526addb5b67ef735be6b5fc961 Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Tue, 16 Nov 2021 20:07:19 -0500 Subject: [PATCH 13/19] Fix CI test for U20 when updating to latest version --- .../tasks/postgresql_ext_version_opt.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml b/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml index 6f8c8c49..a917ad51 100644 --- a/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml +++ b/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml @@ -211,7 +211,7 @@ that: - result.rowcount == 1 - - name: postgresql_ext_version - try to update the extension to the latest version again + - name: postgresql_ext_version - try to update the extension to the latest version again which always runs an update. <<: *task_parameters postgresql_ext: <<: *pg_parameters @@ -222,7 +222,17 @@ - assert: that: - - result is not changed + - result is changed + + - name: postgresql_ext_version - check that version number did not change even though update ran + <<: *task_parameters + postgresql_query: + <<: *pg_parameters + query: "SELECT 1 FROM pg_extension WHERE extname = '{{ test_ext }}' AND extversion = '3.0'" + + - assert: + that: + - result.rowcount == 1 - name: postgresql_ext_version - try to downgrade the extension version, must fail <<: *task_parameters From 5fc4758d1adc0044647dd81a50e512e81766e0f7 Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Wed, 17 Nov 2021 11:43:42 -0500 Subject: [PATCH 14/19] Fix CI test for non-existent extension. Update python doc. --- plugins/modules/postgresql_ext.py | 46 +++++++++++++++++-- .../tasks/postgresql_ext_version_opt.yml | 4 +- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index 155d3d75..ef8c115c 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -198,8 +198,20 @@ # -def ext_delete(cursor, ext, curr_version, cascade): - if curr_version: +def ext_delete(cursor, ext, current_version, cascade): + """Remove the extension from the database + + Return True if success. + + Args: + cursor (cursor) -- cursor object of psycopg2 library + ext (str) -- extension name + current_version (str) -- installed version of the extension. + Value obtained from ext_get_versions and used to + determine if the extension was installed. + cascade (boolean) -- Pass the CASCADE flag to the DROP commmand + """ + if current_version: query = "DROP EXTENSION \"%s\"" % ext if cascade: query += " CASCADE" @@ -234,6 +246,17 @@ def ext_update_version(cursor, ext, version): def ext_create(cursor, ext, schema, cascade, version): + """ + Create the extension objects inside the database + + Return True if success. + + Args: + cursor (cursor) -- cursor object of psycopg2 library + ext (str) -- extension name + schema (str) -- target schema for extension objects + version (str) -- extension version + """ query = "CREATE EXTENSION \"%s\"" % ext params = {} @@ -252,7 +275,9 @@ def ext_create(cursor, ext, schema, cascade, version): def ext_get_versions(cursor, ext): """ - Get the current created extension version and available versions. + Get the currently created extension version if it is installed + in the database and versions that are available if it is + installed on the system. Return tuple (current_version, [list of available versions]). @@ -296,7 +321,18 @@ def ext_get_versions(cursor, ext): def ext_valid_update_path(cursor, ext, current_version, version): """ - Value of 'latest' is always a valid path. + Check to see if the installed extension version has a valid update + path to the given version. A version of 'latest' is always a valid path. + + Return True if a valid path exists. Otherwise return False. + + Args: + cursor (cursor) -- cursor object of psycopg2 library + ext (str) -- extension name + current_version (str) -- installed version of the extension. + version (str) -- target extension version to update to. + A value of 'latest' is always a valid path and will result + in the extension update command always being run. """ valid_path = False @@ -401,7 +437,7 @@ def main(): # If version is not passed: else: # Extension exists, attempt to update to latest version defined in extension control file - # ALTER EXTENSION is actually run, so 'changed' is technically true even if nothing updated + # ALTER EXTENSION is actually run, so 'changed' will be true even if nothing updated if curr_version and version == 'latest': if module.check_mode: changed = True diff --git a/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml b/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml index a917ad51..5a30e883 100644 --- a/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml +++ b/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml @@ -327,7 +327,7 @@ that: - result.rowcount == 1 - - name: postgresql_ext_version - try to install non-existent version + - name: postgresql_ext_version - try to install non-existent extension <<: *task_parameters postgresql_ext: <<: *pg_parameters @@ -338,7 +338,7 @@ - assert: that: - result.failed == true - - result.msg == "Extension non_existent is not installed" + - result.msg == "Extension non_existent is not available" ###################################################################### # https://github.com/ansible-collections/community.general/issues/1095 From 14f6c3be0fa49f136ee6a921c024e8cf1b396848 Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Fri, 19 Nov 2021 15:45:44 -0500 Subject: [PATCH 15/19] Remove unused codepath --- plugins/modules/postgresql_ext.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index ef8c115c..5963ffd8 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -412,6 +412,8 @@ def main(): # Given version already installed if curr_version == version: changed = False + # Attempt to update to given version or latest version defined in extension control file + # ALTER EXTENSION is actually run if valid, so 'changed' will be true even if nothing updated else: valid_update_path = ext_valid_update_path(cursor, ext, curr_version, version) if valid_update_path: @@ -436,15 +438,8 @@ def main(): # If version is not passed: else: - # Extension exists, attempt to update to latest version defined in extension control file - # ALTER EXTENSION is actually run, so 'changed' will be true even if nothing updated - if curr_version and version == 'latest': - if module.check_mode: - changed = True - else: - changed = ext_update_version(cursor, ext, version) - # Extension exists, no request to update so no change - elif curr_version: + ## Extension exists, no request to update so no change + if curr_version: changed = False else: # If the ext doesn't exist and is available: From d669797dcc9f3d8254f4ad03f8ff809704e031ba Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Fri, 19 Nov 2021 15:53:59 -0500 Subject: [PATCH 16/19] Remove extra comment marker --- plugins/modules/postgresql_ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index 5963ffd8..f2a5a3c0 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -438,7 +438,7 @@ def main(): # If version is not passed: else: - ## Extension exists, no request to update so no change + # Extension exists, no request to update so no change if curr_version: changed = False else: From 427e76721d1726056d1c81e7799bb23f27a2b85f Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Fri, 19 Nov 2021 16:16:43 -0500 Subject: [PATCH 17/19] Add changlog fragment --- .../fragments/163-better_handling_of_postgresql_extensions.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelogs/fragments/163-better_handling_of_postgresql_extensions.yml diff --git a/changelogs/fragments/163-better_handling_of_postgresql_extensions.yml b/changelogs/fragments/163-better_handling_of_postgresql_extensions.yml new file mode 100644 index 00000000..f3b980c1 --- /dev/null +++ b/changelogs/fragments/163-better_handling_of_postgresql_extensions.yml @@ -0,0 +1,2 @@ +bugfixes: +- postgresql_ext - Handle postgresql extension updates through path validation instead of version comparison (https://github.com/ansible-collections/community.postgresql/issues/129). From 5a2d9938a0816f54601df30913d3a42dc1a45225 Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Mon, 22 Nov 2021 10:08:06 -0500 Subject: [PATCH 18/19] Apply suggestions from code review Co-authored-by: Andrew Klychkov --- plugins/modules/postgresql_ext.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index f2a5a3c0..6498b353 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -199,7 +199,7 @@ def ext_delete(cursor, ext, current_version, cascade): - """Remove the extension from the database + """Remove the extension from the database. Return True if success. @@ -247,7 +247,7 @@ def ext_update_version(cursor, ext, version): def ext_create(cursor, ext, schema, cascade, version): """ - Create the extension objects inside the database + Create the extension objects inside the database. Return True if success. From 317a8fd6b4e25926e5136b77cc7e6194bfe0958b Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Tue, 23 Nov 2021 16:59:48 -0500 Subject: [PATCH 19/19] Make available_extensions a set. Remove SELECT from executed queries. Update doc. --- plugins/modules/postgresql_ext.py | 8 +++++--- .../postgresql_ext/tasks/postgresql_ext_version_opt.yml | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/modules/postgresql_ext.py b/plugins/modules/postgresql_ext.py index f2a5a3c0..10cdd828 100644 --- a/plugins/modules/postgresql_ext.py +++ b/plugins/modules/postgresql_ext.py @@ -71,7 +71,10 @@ version: description: - Extension version to add or update to. Has effect with I(state=present) only. - - If not specified, the latest extension version will be created. + - If not specified and extension is not installed in the database, + the latest version available will be created. + - If extension is already installed, will update to the given version if a valid update + path exists. - Downgrading is only supported if the extension provides a downgrade path otherwise the extension must be removed and a lower version of the extension must be made available. - Set I(version=latest) to always update the extension to the latest available version. @@ -311,7 +314,7 @@ def ext_get_versions(cursor, ext): cursor.execute(query, params) - available_versions = [r[0] for r in cursor.fetchall()] + available_versions = set(r[0] for r in cursor.fetchall()) if current_version is None: current_version = False @@ -347,7 +350,6 @@ def ext_valid_update_path(cursor, ext, current_version, version): params['ver'] = version cursor.execute(query, params) - executed_queries.append(cursor.mogrify(query, params)) res = cursor.fetchone() if res is not None: valid_path = True diff --git a/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml b/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml index 5a30e883..36249627 100644 --- a/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml +++ b/tests/integration/targets/postgresql_ext/tasks/postgresql_ext_version_opt.yml @@ -153,7 +153,7 @@ - assert: that: - result is changed - - result.queries == ["SELECT path FROM pg_extension_update_paths('dummy') WHERE source = '1.0' AND target = '2.0'", "ALTER EXTENSION \"{{ test_ext }}\" UPDATE TO '2.0'"] + - result.queries == ["ALTER EXTENSION \"{{ test_ext }}\" UPDATE TO '2.0'"] - name: postgresql_ext_version - check, the version must be 2.0 <<: *task_parameters