From 2f04aff99ffc12c7c2abe16fdc5d9dec89c92f3e Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 21 Nov 2024 08:35:48 -0800 Subject: [PATCH 01/13] Add admin-only view to display applied migrations This view allows Aerie admin users to read migration information about their venue (information was formerly locked to Hasura/Postgres Admins) --- .../tables/migrations/applied_migrations_view.yaml | 11 +++++++++++ .../hasura/metadata/databases/tables/tables.yaml | 5 +++++ .../Aerie/12_applied_migrations_view/down.sql | 2 ++ .../Aerie/12_applied_migrations_view/up.sql | 4 ++++ .../postgres-init-db/sql/applied_migrations.sql | 1 + deployment/postgres-init-db/sql/init.sql | 1 + .../sql/views/migrations/applied_migrations_view.sql | 3 +++ 7 files changed, 27 insertions(+) create mode 100644 deployment/hasura/metadata/databases/tables/migrations/applied_migrations_view.yaml create mode 100644 deployment/hasura/migrations/Aerie/12_applied_migrations_view/down.sql create mode 100644 deployment/hasura/migrations/Aerie/12_applied_migrations_view/up.sql create mode 100644 deployment/postgres-init-db/sql/views/migrations/applied_migrations_view.sql diff --git a/deployment/hasura/metadata/databases/tables/migrations/applied_migrations_view.yaml b/deployment/hasura/metadata/databases/tables/migrations/applied_migrations_view.yaml new file mode 100644 index 0000000000..2ede6993f7 --- /dev/null +++ b/deployment/hasura/metadata/databases/tables/migrations/applied_migrations_view.yaml @@ -0,0 +1,11 @@ +table: + name: applied_migrations + schema: migrations +configuration: + custom_name: "applied_migrations" +select_permissions: + - role: aerie_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true diff --git a/deployment/hasura/metadata/databases/tables/tables.yaml b/deployment/hasura/metadata/databases/tables/tables.yaml index 40b07b3cf3..0930fc8d5a 100644 --- a/deployment/hasura/metadata/databases/tables/tables.yaml +++ b/deployment/hasura/metadata/databases/tables/tables.yaml @@ -1,6 +1,11 @@ # Would prefer to do this as one file that delegates to others, as was done with init.sql # but doing so currently throws an error: "parse-failed: parsing Object failed, expected Object, but encountered Array" +#################### +#### MIGRATIONS #### +#################### +- "!include migrations/applied_migrations_view.yaml" + ##################### #### PERMISSIONS #### ##################### diff --git a/deployment/hasura/migrations/Aerie/12_applied_migrations_view/down.sql b/deployment/hasura/migrations/Aerie/12_applied_migrations_view/down.sql new file mode 100644 index 0000000000..37141aedc5 --- /dev/null +++ b/deployment/hasura/migrations/Aerie/12_applied_migrations_view/down.sql @@ -0,0 +1,2 @@ +drop view migrations.applied_migrations; +call migrations.mark_migration_rolled_back('12'); diff --git a/deployment/hasura/migrations/Aerie/12_applied_migrations_view/up.sql b/deployment/hasura/migrations/Aerie/12_applied_migrations_view/up.sql new file mode 100644 index 0000000000..1147a09abe --- /dev/null +++ b/deployment/hasura/migrations/Aerie/12_applied_migrations_view/up.sql @@ -0,0 +1,4 @@ +create view migrations.applied_migrations as + select migration_id::int + from migrations.schema_migrations; +call migrations.mark_migration_applied('12'); diff --git a/deployment/postgres-init-db/sql/applied_migrations.sql b/deployment/postgres-init-db/sql/applied_migrations.sql index 9f1d007a71..347ff9e5ab 100644 --- a/deployment/postgres-init-db/sql/applied_migrations.sql +++ b/deployment/postgres-init-db/sql/applied_migrations.sql @@ -14,3 +14,4 @@ call migrations.mark_migration_applied('8'); call migrations.mark_migration_applied('9'); call migrations.mark_migration_applied('10'); call migrations.mark_migration_applied('11'); +call migrations.mark_migration_applied('12'); diff --git a/deployment/postgres-init-db/sql/init.sql b/deployment/postgres-init-db/sql/init.sql index a42a30c46b..4169ca177b 100644 --- a/deployment/postgres-init-db/sql/init.sql +++ b/deployment/postgres-init-db/sql/init.sql @@ -12,6 +12,7 @@ begin; -- Migrations \ir tables/migrations/schema_migrations.sql \ir applied_migrations.sql + \ir views/migrations/applied_migrations_view.sql -- Util Functions \ir functions/util_functions/shared_update_functions.sql diff --git a/deployment/postgres-init-db/sql/views/migrations/applied_migrations_view.sql b/deployment/postgres-init-db/sql/views/migrations/applied_migrations_view.sql new file mode 100644 index 0000000000..0cb94574fa --- /dev/null +++ b/deployment/postgres-init-db/sql/views/migrations/applied_migrations_view.sql @@ -0,0 +1,3 @@ +create view migrations.applied_migrations as + select migration_id::int + from migrations.schema_migrations; From 5f71792197c5fbd87a8523c5a1f1ecaf83314ee4 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Tue, 3 Dec 2024 14:48:14 -0800 Subject: [PATCH 02/13] Extract repeated exit code into `exit_with_error` --- deployment/aerie_db_migration.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/deployment/aerie_db_migration.py b/deployment/aerie_db_migration.py index 3dbe829ba5..674c0d9ec6 100755 --- a/deployment/aerie_db_migration.py +++ b/deployment/aerie_db_migration.py @@ -11,6 +11,17 @@ def clear_screen(): os.system('cls' if os.name == 'nt' else 'clear') + +def exit_with_error(message: str, exit_code=1): + """ + Exit the program with the specified error message and exit code. + + :param message: Error message to display before exiting. + :param exit_code: Error code to exit with. Defaults to 1. + """ + print("\033[91mError\033[0m: "+message) + sys.exit(exit_code) + # internal class class DB_Migration: steps = [] From 050a6e7f275a8c4ee3de2a1a4fef13b34e6d70af Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Tue, 3 Dec 2024 15:00:04 -0800 Subject: [PATCH 03/13] Extract migration steps inside DB_Migration initializer Additionally, record where the migration steps are located and update step-by-step mode to reference this location --- deployment/aerie_db_migration.py | 47 +++++++++++++++++++------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/deployment/aerie_db_migration.py b/deployment/aerie_db_migration.py index 674c0d9ec6..51ddfc773f 100755 --- a/deployment/aerie_db_migration.py +++ b/deployment/aerie_db_migration.py @@ -24,13 +24,30 @@ def exit_with_error(message: str, exit_code=1): # internal class class DB_Migration: + """ + Container class for Migration steps to be applied/reverted. + """ steps = [] - db_name = '' - def __init__(self, db_name): - self.db_name = db_name + migrations_folder = '' + def __init__(self, migrations_folder: str, reverse: bool): + """ + :param migrations_folder: Folder where the migrations are stored. + :param reverse: If true, reverses the list of migration steps. + """ + self.migrations_folder = migrations_folder + try: + for root, dirs, files in os.walk(migrations_folder): + if dirs: + self.add_migration_step(dirs) + except FileNotFoundError as fne: + exit_with_error(str(fne).split("]")[1]) + if len(self.steps) <= 0: + exit_with_error("No database migrations found.") + if reverse: + self.steps.reverse() def add_migration_step(self, _migration_step): - self.steps = sorted(_migration_step, key=lambda x:int(x.split('_')[0])) + self.steps = sorted(_migration_step, key=lambda x: int(x.split('_')[0])) def step_by_step_migration(db_migration, apply): display_string = "\n\033[4mMIGRATION STEPS AVAILABLE:\033[0m\n" @@ -49,14 +66,16 @@ def step_by_step_migration(db_migration, apply): input("Press Enter to continue...") return + folder = os.path.join(db_migration.migrations_folder, f'{split[0]}_{split[1]}') if apply: - if (len(split) == 4) or (not os.path.isfile(f'migrations/{db_migration.db_name}/{split[0]}_{split[1]}/up.sql')): + # If there are four words, they must be " Present Present" + if (len(split) == 4 and "Present" == split[-1]) or (not os.path.isfile(os.path.join(folder, 'up.sql'))): available_steps.remove(f'{split[0]}_{split[1]}') else: display_string += _output[i] + "\n" else: - if (len(split) == 5 and "Not Present" == (split[3] + " " + split[4])) \ - or (not os.path.isfile(f'migrations/{db_migration.db_name}/{split[0]}_{split[1]}/down.sql')): + # If there are only five words, they must be " Present Not Present" + if (len(split) == 5 and "Not Present" == (split[-2] + " " + split[-1])) or (not os.path.isfile(os.path.join(folder, 'down.sql'))): available_steps.remove(f'{split[0]}_{split[1]}') else: display_string += _output[i] + "\n" @@ -189,20 +208,10 @@ def main(): HASURA_PATH = "./hasura" if args.hasura_path: HASURA_PATH = args.hasura_path - MIGRATION_PATH = HASURA_PATH+"/migrations/Aerie" + MIGRATION_PATH = os.path.abspath(HASURA_PATH+"/migrations/Aerie") # Find all migration folders for the database - migration = DB_Migration("Aerie") - try: - for root,dirs,files in os.walk(MIGRATION_PATH): - if dirs: - migration.add_migration_step(dirs) - except FileNotFoundError as fne: - print("\033[91mError\033[0m:"+ str(fne).split("]")[1]) - sys.exit(1) - if len(migration.steps) <= 0: - print("\033[91mError\033[0m: No database migrations found.") - sys.exit(1) + migration = DB_Migration("Aerie", MIGRATION_PATH) # If reverting, reverse the list if args.revert: From 56f5e00eae7be9bc084d9ccf010499cac71bd2bd Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Tue, 3 Dec 2024 15:01:43 -0800 Subject: [PATCH 04/13] Extract creating args parser to a method --- deployment/aerie_db_migration.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/deployment/aerie_db_migration.py b/deployment/aerie_db_migration.py index 51ddfc773f..4055d45229 100755 --- a/deployment/aerie_db_migration.py +++ b/deployment/aerie_db_migration.py @@ -167,11 +167,12 @@ def mark_current_version(username, password, netloc): return current_schema -def main(): +def createArgsParser() -> argparse.ArgumentParser: # Create a cli parser parser = argparse.ArgumentParser(description=__doc__) + # Applying and Reverting are exclusive arguments - exclusive_args = parser.add_mutually_exclusive_group(required='true') + exclusive_args = parser.add_mutually_exclusive_group(required=True) # Add arguments exclusive_args.add_argument( @@ -202,8 +203,10 @@ def main(): help="the network location of the database. defaults to localhost", default='localhost') + return parser +def main(): # Generate arguments - args = parser.parse_args() + args = migrateArgsParser().parse_args() HASURA_PATH = "./hasura" if args.hasura_path: From 92d9f2fcdd0e0b0ea911130b84506b58e0e8544a Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Wed, 4 Dec 2024 05:53:01 -0800 Subject: [PATCH 05/13] Update Configuration Loading Hasura CLI loads first from arguments passed to the command, then from environment variables, then from a passed .env, then finally from a config file. This change updates how the script determines its connection credentials to match how HasuraCLI works. --- deployment/aerie_db_migration.py | 116 ++++++++++++++++++++++--------- deployment/requirements.txt | 5 +- 2 files changed, 86 insertions(+), 35 deletions(-) diff --git a/deployment/aerie_db_migration.py b/deployment/aerie_db_migration.py index 4055d45229..24d58177b6 100755 --- a/deployment/aerie_db_migration.py +++ b/deployment/aerie_db_migration.py @@ -7,6 +7,7 @@ import shutil import subprocess import psycopg +from dotenv import load_dotenv def clear_screen(): os.system('cls' if os.name == 'nt' else 'clear') @@ -167,7 +168,58 @@ def mark_current_version(username, password, netloc): return current_schema +def loadConfigFile(endpoint: str, secret: str, config_folder: str) -> (str, str): + """ + Extract the endpoint and admin secret from a Hasura config file. + Values passed as arguments take priority over the contents of the config file. + + :param endpoint: Initial value of the endpoint for Hasura. Will be extracted if empty. + :param secret: Initial value of the admin secret for Hasura. Will be extracted if empty. + :param config_folder: Folder to look for the config file in. + :return: A tuple containing the Hasura endpoint and the Hasura admin secret. + """ + hasura_endpoint = endpoint + hasura_admin_secret = secret + + # Check if config.YAML exists + configPath = os.path.join(config_folder, 'config.yaml') + if not os.path.isfile(configPath): + # Check for .YML + configPath = os.path.join(config_folder, 'config.yml') + if not os.path.isfile(configPath): + errorMsg = "HASURA_GRAPHQL_ENDPOINT and HASURA_GRAPHQL_ADMIN_SECRET" if not endpoint and not secret \ + else "HASURA_GRAPHQL_ENDPOINT" if not endpoint \ + else "HASURA_GRAPHQL_ADMIN_SECRET" + errorMsg += " must be defined by either environment variables or in a config.yaml located in " + config_folder + "." + exit_with_error(errorMsg) + + # Extract admin secret and/or endpoint from the config.yaml, if they were not already set + with open(configPath) as configFile: + for line in configFile: + if hasura_endpoint and hasura_admin_secret: + break + line = line.strip() + if line.startswith("endpoint") and not hasura_endpoint: + hasura_endpoint = line.removeprefix("endpoint:").strip() + continue + if line.startswith("admin_secret") and not hasura_admin_secret: + hasura_admin_secret = line.removeprefix("admin_secret:").strip() + continue + + if not hasura_endpoint or not hasura_admin_secret: + errorMsg = "HASURA_GRAPHQL_ENDPOINT and HASURA_GRAPHQL_ADMIN_SECRET" if not hasura_endpoint and not hasura_admin_secret \ + else "HASURA_GRAPHQL_ENDPOINT" if not hasura_endpoint \ + else "HASURA_GRAPHQL_ADMIN_SECRET" + errorMsg += " must be defined by either environment variables or in a config.yaml located in " + config_folder + "." + exit_with_error(errorMsg) + + return hasura_endpoint, hasura_admin_secret + + def createArgsParser() -> argparse.ArgumentParser: + """ + Create an ArgumentParser for this script. + """ # Create a cli parser parser = argparse.ArgumentParser(description=__doc__) @@ -177,26 +229,37 @@ def createArgsParser() -> argparse.ArgumentParser: # Add arguments exclusive_args.add_argument( '-a', '--apply', - help="apply migration steps to the database", + help='apply migration steps to the database', action='store_true') exclusive_args.add_argument( '-r', '--revert', - help="revert migration steps to the databases", + help='revert migration steps to the databases', action='store_true') parser.add_argument( '--all', - help="apply[revert] ALL unapplied[applied] migration steps to the database", + help='apply[revert] ALL unapplied[applied] migration steps to the database', action='store_true') parser.add_argument( '-p', '--hasura-path', - help="the path to the directory containing the config.yaml for Aerie. defaults to ./hasura") + help='directory containing the config.yaml and migrations folder for the venue. defaults to ./hasura', + default='./hasura') parser.add_argument( '-e', '--env-path', - help="the path to the .env file used to deploy aerie. must define AERIE_USERNAME and AERIE_PASSWORD") + help='envfile to load envvars from.') + + parser.add_argument( + '--endpoint', + help="http(s) endpoint for the venue's Hasura instance", + required=False) + + parser.add_argument( + '--admin_secret', + help="admin secret for the venue's Hasura instance", + required=False) parser.add_argument( '-n', '--network-location', @@ -204,6 +267,8 @@ def createArgsParser() -> argparse.ArgumentParser: default='localhost') return parser + + def main(): # Generate arguments args = migrateArgsParser().parse_args() @@ -213,6 +278,20 @@ def main(): HASURA_PATH = args.hasura_path MIGRATION_PATH = os.path.abspath(HASURA_PATH+"/migrations/Aerie") + if args.env_path: + if not os.path.isfile(args.env_path): + exit_with_error(f'Specified envfile does not exist: {args.env_path}') + load_dotenv(args.env_path) + + # Grab the credentials from the environment if needed + hasura_endpoint = args.endpoint if args.endpoint else os.environ.get('HASURA_GRAPHQL_ENDPOINT', "") + hasura_admin_secret = args.admin_secret if args.admin_secret else os.environ.get('HASURA_GRAPHQL_ADMIN_SECRET', "") + + if not (hasura_endpoint and hasura_admin_secret): + (e, s) = loadConfigFile(hasura_endpoint, hasura_admin_secret, HASURA_PATH) + hasura_endpoint = e + hasura_admin_secret = s + # Find all migration folders for the database migration = DB_Migration("Aerie", MIGRATION_PATH) @@ -226,33 +305,6 @@ def main(): else: os.system('hasura version') - # Get the Username/Password - username = os.environ.get('AERIE_USERNAME', "") - password = os.environ.get('AERIE_PASSWORD', "") - - if args.env_path: - usernameFound = False - passwordFound = False - with open(args.env_path) as envFile: - for line in envFile: - if usernameFound and passwordFound: - break - line = line.strip() - if line.startswith("AERIE_USERNAME"): - username = line.removeprefix("AERIE_USERNAME=") - usernameFound = True - continue - if line.startswith("AERIE_PASSWORD"): - password = line.removeprefix("AERIE_PASSWORD=") - passwordFound = True - continue - if not usernameFound: - print("\033[91mError\033[0m: AERIE_USERNAME environment variable is not defined in "+args.env_path+".") - sys.exit(1) - if not passwordFound: - print("\033[91mError\033[0m: AERIE_PASSWORD environment variable is not defined in "+args.env_path+".") - sys.exit(1) - # Navigate to the hasura directory os.chdir(HASURA_PATH) diff --git a/deployment/requirements.txt b/deployment/requirements.txt index d55b17b8fc..589448fbb9 100644 --- a/deployment/requirements.txt +++ b/deployment/requirements.txt @@ -1,3 +1,2 @@ -psycopg>=3.1.18 -psycopg-binary>=3.1.18 -typing_extensions>=4.11.0 +requests~=2.31.0 +python-dotenv~=1.0.1 From 2d2221d7ad714f54394ff69fc7c65a37a743555b Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Mon, 9 Dec 2024 13:18:45 -0800 Subject: [PATCH 06/13] Use Hasura endpoint to fetch current migration information Changes the script to use a `run_sql` request to get the current migration, rather than a direct connection to a database. This removes the ability for the user to specify one database in the 'netloc' argument and another in the Hasura configuration --- deployment/aerie_db_migration.py | 72 ++++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/deployment/aerie_db_migration.py b/deployment/aerie_db_migration.py index 24d58177b6..a4debd269f 100755 --- a/deployment/aerie_db_migration.py +++ b/deployment/aerie_db_migration.py @@ -6,8 +6,8 @@ import sys import shutil import subprocess -import psycopg from dotenv import load_dotenv +import requests def clear_screen(): os.system('cls' if os.name == 'nt' else 'clear') @@ -152,21 +152,56 @@ def bulk_migration(db_migration, apply, current_version): print("\n".join(_output)) exit(exit_with) -def mark_current_version(username, password, netloc): - # Connect to DB - connectionString = "postgres://"+username+":"+password+"@"+netloc+":5432/aerie" - with psycopg.connect(connectionString) as connection: - # Open a cursor to perform database operations - with connection.cursor() as cursor: - # Get the current schema version - cursor.execute("SELECT migration_id FROM migrations.schema_migrations ORDER BY migration_id::int DESC LIMIT 1") - current_schema = int(cursor.fetchone()[0]) - - # Mark everything up to that as applied - for i in range(0, current_schema+1): - os.system('hasura migrate apply --skip-execution --version '+str(i)+' --database-name Aerie >/dev/null 2>&1') +def mark_current_version(admin_secret: str, endpoint: str) -> int: + """ + Queries the database behind the Hasura instance for its current schema information. + Ensures that all applied migrations are marked as such in Hasura's migration tracker. - return current_schema + :param admin_secret: The Admin Secret for the Hasura instance + :param endpoint: The connection URL for the Hasura instance, in the format "https://URL:PORT" + :return: The migration the database is currently on + """ + # Remove potential trailing "/" + endpoint = endpoint.strip() + endpoint = endpoint.rstrip('/') + + # Query the database + run_sql_url = f'{endpoint}/v2/query' + headers = { + "content-type": "application/json", + "x-hasura-admin-secret": admin_secret, + "x-hasura-role": "admin" + } + body = { + "type": "run_sql", + "args": { + "source": "Aerie", + "sql": "SELECT migration_id FROM migrations.schema_migrations;", + "read_only": True + } + } + session = requests.Session() + resp = session.post(url=run_sql_url, headers=headers, json=body) + if not resp.ok: + exit_with_error("Error while fetching current schema information.") + + migration_ids = resp.json()['result'] + if migration_ids.pop(0)[0] != 'migration_id': + exit_with_error("Error while fetching current schema information.") + + # migration_ids currently looks like [['0'], ['1'], ... ['n']] + prev_id = -1 + cur_id = 0 + for i in migration_ids: + cur_id = int(i[0]) + if cur_id != prev_id + 1: + exit_with_error(f'Gap detected in applied migrations. \n\tLast migration: {prev_id} \tNext migration: {cur_id}' + f'\n\tTo resolve, manually revert all migrations following {prev_id}, then run this script again.') + # Ensure migration is marked as applied + os.system(f'hasura migrate apply --skip-execution --version {cur_id} --database-name Aerie >/dev/null 2>&1') + prev_id = cur_id + + return cur_id def loadConfigFile(endpoint: str, secret: str, config_folder: str) -> (str, str): """ @@ -261,11 +296,6 @@ def createArgsParser() -> argparse.ArgumentParser: help="admin secret for the venue's Hasura instance", required=False) - parser.add_argument( - '-n', '--network-location', - help="the network location of the database. defaults to localhost", - default='localhost') - return parser @@ -309,7 +339,7 @@ def main(): os.chdir(HASURA_PATH) # Mark all migrations previously applied to the databases to be updated as such - current_version = mark_current_version(username, password, args.network_location) + current_version = mark_current_version(endpoint=hasura_endpoint, admin_secret=hasura_admin_secret) clear_screen() print(f'\n###############################' From aef4d2a03d4851006eabfeac472adffca0df6c47 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Mon, 9 Dec 2024 13:31:06 -0800 Subject: [PATCH 07/13] Create CLI Wrapper Class Hasura replaces the os.system and subprocess.getoutput calls. This was done as there are three flags that need to be provided to every call to the hasura cli: - skip-update-check: cleans up the output by skipping the automatic update check - project: directory where the cli command is executed - envfile: env file to load envvars from. Set to '.env' in the directory the CLI is run from to avoid the CLI and the Migration script loading separate envfiles --- deployment/aerie_db_migration.py | 252 ++++++++++++++++++++----------- 1 file changed, 160 insertions(+), 92 deletions(-) diff --git a/deployment/aerie_db_migration.py b/deployment/aerie_db_migration.py index a4debd269f..842b7180a8 100755 --- a/deployment/aerie_db_migration.py +++ b/deployment/aerie_db_migration.py @@ -23,7 +23,138 @@ def exit_with_error(message: str, exit_code=1): print("\033[91mError\033[0m: "+message) sys.exit(exit_code) -# internal class + +class Hasura: + """ + Class for communicating with Hasura via the CLI and API. + """ + command_suffix = '' + migrate_suffix = '' + endpoint = '' + admin_secret = '' + db_name = 'Aerie' + current_version = 0 + + def __init__(self, endpoint: str, admin_secret: str, hasura_path: str, env_path: str, db_name='Aerie'): + """ + Initialize a Hasura object. + + :param endpoint: The http(s) endpoint for the Hasura instance. + :param admin_secret: The admin secret for the Hasura instance. + :param hasura_path: The directory containing the config.yaml and migrations folder for the Hasura instance. + :param env_path: The path to the envfile, if provided. + :param db_name: The name that the Hasura instance calls the database. Defaults to 'Aerie'. + """ + self.admin_secret = admin_secret + self.db_name = db_name + + # Sanitize endpoint + self.endpoint = endpoint + self.endpoint = self.endpoint.strip() + self.endpoint = self.endpoint.rstrip('/') + + # Set up the suffix flags to use when calling the Hasura CLI + self.command_suffix = f'--skip-update-check --project {hasura_path}' + if env_path: + self.command_suffix += f' --envfile {env_path}' + + # Set up the suffix flags to use when calling the 'migrate' subcommand on the CLI + self.migrate_suffix = f"--database-name {self.db_name} --endpoint {self.endpoint} --admin-secret '{self.admin_secret}'" + + # Check that Hasura CLI is installed + if not shutil.which('hasura'): + sys.exit(f'Hasura CLI is not installed. Exiting...') + else: + self.execute('version') + + # Mark the current schema version in Hasura + self.current_version = self.mark_current_version() + + def execute(self, subcommand: str, flags='', no_output=False) -> int: + """ + Execute an arbitrary "hasura" command. + + :param subcommand: The subcommand to execute. + :param flags: The flags to be passed to the subcommand. + :param no_output: If true, swallows both STDERR and STDOUT output from the command. + :return: The exit code of the command. + """ + command = f'hasura {subcommand} {flags} {self.command_suffix}' + if no_output: + command += ' > /dev/null 2>&1' + return os.system(command) + + def migrate(self, subcommand: str, flags='', no_output=False) -> int: + """ + Execute a "hasura migrate" subcommand. + + :param subcommand: A subcommand of "hasura migrate" + :param flags: Flags specific to the subcommand call to be passed. + :param no_output: If true, swallows both STDERR and STDOUT output from the command. + :return: The exit code of the command. + """ + command = f'hasura migrate {subcommand} {flags} {self.migrate_suffix} {self.command_suffix}' + if no_output: + command += ' > /dev/null 2>&1' + return os.system(command) + + def get_migrate_output(self, subcommand: str, flags='') -> [str]: + """ + Get the output of a "hasura migrate" subcommand. + + :param subcommand: A subcommand of "hasura migrate" + :param flags: Flags specific to the subcommand call to be passed. + :return: The STDOUT response of the subcommand, split on newlines. + """ + command = f'hasura migrate {subcommand} {flags} {self.migrate_suffix} {self.command_suffix}' + return subprocess.getoutput(command).split("\n") + + def mark_current_version(self) -> int: + """ + Queries the database behind the Hasura instance for its current schema information. + Ensures that all applied migrations are marked as "applied" in Hasura's internal migration tracker. + + :return: The migration the underlying database is currently on + """ + # Query the database + run_sql_url = f'{self.endpoint}/v2/query' + headers = { + "content-type": "application/json", + "x-hasura-admin-secret": self.admin_secret, + "x-hasura-role": "admin" + } + body = { + "type": "run_sql", + "args": { + "source": self.db_name, + "sql": "SELECT migration_id FROM migrations.schema_migrations;", + "read_only": True + } + } + session = requests.Session() + resp = session.post(url=run_sql_url, headers=headers, json=body) + if not resp.ok: + exit_with_error("Error while fetching current schema information.") + + migration_ids = resp.json()['result'] + if migration_ids.pop(0)[0] != 'migration_id': + exit_with_error("Error while fetching current schema information.") + + # migration_ids now looks like [['0'], ['1'], ... ['n']] + prev_id = -1 + cur_id = 0 + for i in migration_ids: + cur_id = int(i[0]) + if cur_id != prev_id + 1: + exit_with_error(f'Gap detected in applied migrations. \n\tLast migration: {prev_id} \tNext migration: {cur_id}' + f'\n\tTo resolve, manually revert all migrations following {prev_id}, then run this script again.') + # Ensure migration is marked as applied + self.migrate('apply', f'--skip-execution --version {cur_id}', no_output=True) + prev_id = cur_id + + return cur_id + + class DB_Migration: """ Container class for Migration steps to be applied/reverted. @@ -50,9 +181,10 @@ def __init__(self, migrations_folder: str, reverse: bool): def add_migration_step(self, _migration_step): self.steps = sorted(_migration_step, key=lambda x: int(x.split('_')[0])) -def step_by_step_migration(db_migration, apply): + +def step_by_step_migration(hasura: Hasura, db_migration: DB_Migration, apply: bool): display_string = "\n\033[4mMIGRATION STEPS AVAILABLE:\033[0m\n" - _output = subprocess.getoutput(f'hasura migrate status --database-name {db_migration.db_name}').split("\n") + _output = hasura.get_migrate_output('status') del _output[0:3] display_string += _output[0] + "\n" @@ -91,9 +223,9 @@ def step_by_step_migration(db_migration, apply): timestamp = step.split("_")[0] if apply: - os.system(f'hasura migrate apply --version {timestamp} --database-name {db_migration.db_name} --dry-run --log-level WARN') + hasura.migrate('apply', f'--version {timestamp} --dry-run --log-level WARN') else: - os.system(f'hasura migrate apply --version {timestamp} --type down --database-name {db_migration.db_name} --dry-run --log-level WARN') + hasura.migrate('apply', f'--version {timestamp} --type down --dry-run --log-level WARN') print() _value = '' @@ -108,11 +240,11 @@ def step_by_step_migration(db_migration, apply): if _value == "y": if apply: print('Applying...') - exit_code = os.system(f'hasura migrate apply --version {timestamp} --type up --database-name {db_migration.db_name}') + exit_code = hasura.migrate('apply', f'--version {timestamp} --type up') else: print('Reverting...') - exit_code = os.system(f'hasura migrate apply --version {timestamp} --type down --database-name {db_migration.db_name}') - os.system('hasura metadata reload') + exit_code = hasura.migrate('apply', f'--version {timestamp} --type down') + hasura.execute('metadata reload') print() if exit_code != 0: return @@ -120,88 +252,32 @@ def step_by_step_migration(db_migration, apply): return input("Press Enter to continue...") -def bulk_migration(db_migration, apply, current_version): + +def bulk_migration(hasura: Hasura, apply: bool): # Migrate the database exit_with = 0 if apply: - os.system(f'hasura migrate apply --database-name {db_migration.db_name} --dry-run --log-level WARN') - exit_code = os.system(f'hasura migrate apply --database-name {db_migration.db_name}') + hasura.migrate('apply', f'--dry-run --log-level WARN') + exit_code = hasura.migrate('apply') if exit_code != 0: exit_with = 1 else: - # Performing GOTO 1 when the database is at migration 1 will cause Hasura to attempt to reapply migration 1 - if current_version == 1: - os.system(f'hasura migrate apply --down 1 --database-name {db_migration.db_name} --dry-run --log-level WARN') - exit_code = os.system(f'hasura migrate apply --down 1 --database-name {db_migration.db_name}') - else: - os.system(f'hasura migrate apply --goto 1 --database-name {db_migration.db_name} --dry-run --log-level WARN &&' - f'hasura migrate apply --down 1 --database-name {db_migration.db_name} --dry-run --log-level WARN') - exit_code = os.system(f'hasura migrate apply --goto 1 --database-name {db_migration.db_name} &&' - f'hasura migrate apply --down 1 --database-name {db_migration.db_name}') + hasura.migrate('apply', f'--down {hasura.current_version} --dry-run --log-level WARN') + exit_code = hasura.migrate('apply', f'--down {hasura.current_version}') if exit_code != 0: exit_with = 1 - os.system('hasura metadata reload') + hasura.execute('metadata reload') # Show the result after the migration print(f'\n###############' f'\nDatabase Status' f'\n###############') - _output = subprocess.getoutput(f'hasura migrate status --database-name {db_migration.db_name}').split("\n") + _output = hasura.get_migrate_output('status') del _output[0:3] print("\n".join(_output)) exit(exit_with) -def mark_current_version(admin_secret: str, endpoint: str) -> int: - """ - Queries the database behind the Hasura instance for its current schema information. - Ensures that all applied migrations are marked as such in Hasura's migration tracker. - - :param admin_secret: The Admin Secret for the Hasura instance - :param endpoint: The connection URL for the Hasura instance, in the format "https://URL:PORT" - :return: The migration the database is currently on - """ - # Remove potential trailing "/" - endpoint = endpoint.strip() - endpoint = endpoint.rstrip('/') - - # Query the database - run_sql_url = f'{endpoint}/v2/query' - headers = { - "content-type": "application/json", - "x-hasura-admin-secret": admin_secret, - "x-hasura-role": "admin" - } - body = { - "type": "run_sql", - "args": { - "source": "Aerie", - "sql": "SELECT migration_id FROM migrations.schema_migrations;", - "read_only": True - } - } - session = requests.Session() - resp = session.post(url=run_sql_url, headers=headers, json=body) - if not resp.ok: - exit_with_error("Error while fetching current schema information.") - - migration_ids = resp.json()['result'] - if migration_ids.pop(0)[0] != 'migration_id': - exit_with_error("Error while fetching current schema information.") - - # migration_ids currently looks like [['0'], ['1'], ... ['n']] - prev_id = -1 - cur_id = 0 - for i in migration_ids: - cur_id = int(i[0]) - if cur_id != prev_id + 1: - exit_with_error(f'Gap detected in applied migrations. \n\tLast migration: {prev_id} \tNext migration: {cur_id}' - f'\n\tTo resolve, manually revert all migrations following {prev_id}, then run this script again.') - # Ensure migration is marked as applied - os.system(f'hasura migrate apply --skip-execution --version {cur_id} --database-name Aerie >/dev/null 2>&1') - prev_id = cur_id - - return cur_id def loadConfigFile(endpoint: str, secret: str, config_folder: str) -> (str, str): """ @@ -303,9 +379,7 @@ def main(): # Generate arguments args = migrateArgsParser().parse_args() - HASURA_PATH = "./hasura" - if args.hasura_path: - HASURA_PATH = args.hasura_path + HASURA_PATH = args.hasura_path MIGRATION_PATH = os.path.abspath(HASURA_PATH+"/migrations/Aerie") if args.env_path: @@ -322,35 +396,29 @@ def main(): hasura_endpoint = e hasura_admin_secret = s - # Find all migration folders for the database - migration = DB_Migration("Aerie", MIGRATION_PATH) - - # If reverting, reverse the list - if args.revert: - migration.steps.reverse() - - # Check that hasura cli is installed - if not shutil.which('hasura'): - sys.exit(f'Hasura CLI is not installed. Exiting...') - else: - os.system('hasura version') + hasura = Hasura(endpoint=hasura_endpoint, + admin_secret=hasura_admin_secret, + db_name="Aerie", + hasura_path=os.path.abspath(HASURA_PATH), + env_path=os.path.abspath(args.env_path) if args.env_path else None) # Navigate to the hasura directory os.chdir(HASURA_PATH) - # Mark all migrations previously applied to the databases to be updated as such - current_version = mark_current_version(endpoint=hasura_endpoint, admin_secret=hasura_admin_secret) - clear_screen() print(f'\n###############################' f'\nAERIE DATABASE MIGRATION HELPER' f'\n###############################') # Enter step-by-step mode if not otherwise specified if not args.all: + # Find all migration folders for the database + migration = DB_Migration(MIGRATION_PATH, args.revert) + # Go step-by-step through the migrations available for the selected database - step_by_step_migration(migration, args.apply) + step_by_step_migration(hasura, migration, args.apply) else: - bulk_migration(migration, args.apply, current_version) + bulk_migration(hasura, args.apply) + if __name__ == "__main__": main() From 229f1c3cb8316f6ea85fd0685a9dfff0ecb8ae45 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Wed, 4 Dec 2024 13:40:14 -0800 Subject: [PATCH 08/13] Print the Hasura Endpoint that the migration script is migrating --- deployment/aerie_db_migration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployment/aerie_db_migration.py b/deployment/aerie_db_migration.py index 842b7180a8..55748a8272 100755 --- a/deployment/aerie_db_migration.py +++ b/deployment/aerie_db_migration.py @@ -408,7 +408,8 @@ def main(): clear_screen() print(f'\n###############################' f'\nAERIE DATABASE MIGRATION HELPER' - f'\n###############################') + f'\n###############################' + f'\n\nMigrating database at {hasura.endpoint}') # Enter step-by-step mode if not otherwise specified if not args.all: # Find all migration folders for the database From bdeaac4656e8a58bb8691ce855b9b3b55d14da5a Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Mon, 9 Dec 2024 09:41:37 -0800 Subject: [PATCH 09/13] Create 'status' subfunction - Extracts initializing a Hasura object from a Namespace into a function - moves migration logic into 'migrate' subfunction --- deployment/aerie_db_migration.py | 143 ++++++++++++++++++++----------- 1 file changed, 95 insertions(+), 48 deletions(-) diff --git a/deployment/aerie_db_migration.py b/deployment/aerie_db_migration.py index 55748a8272..83030976b3 100755 --- a/deployment/aerie_db_migration.py +++ b/deployment/aerie_db_migration.py @@ -333,9 +333,51 @@ def createArgsParser() -> argparse.ArgumentParser: """ # Create a cli parser parser = argparse.ArgumentParser(description=__doc__) + parent_parser = argparse.ArgumentParser(add_help=False) + subparser = parser.add_subparsers(title='commands', metavar="") + + # Add global arguments to Parent parser + parent_parser.add_argument( + '-p', '--hasura-path', + dest='hasura_path', + help='directory containing the config.yaml and migrations folder for the venue. defaults to ./hasura', + default='./hasura') + + parent_parser.add_argument( + '-e', '--env-path', + dest='env_path', + help='envfile to load envvars from.') + + parent_parser.add_argument( + '--endpoint', + help="http(s) endpoint for the venue's Hasura instance", + required=False) + + parent_parser.add_argument( + '--admin-secret', + dest='admin_secret', + help="admin secret for the venue's Hasura instance", + required=False) + + # Add 'status' subcommand + status_parser = subparser.add_parser( + 'status', + help='Get the current migration status of the database', + description='Get the current migration status of the database.', + parents=[parent_parser]) + + status_parser.set_defaults(func=status) + + # Add 'migrate' subcommand + migrate_parser = subparser.add_parser( + 'migrate', + help='Migrate the database', + description='Migrate the database.', + parents=[parent_parser]) + migrate_parser.set_defaults(func=migrate) # Applying and Reverting are exclusive arguments - exclusive_args = parser.add_mutually_exclusive_group(required=True) + exclusive_args = migrate_parser.add_mutually_exclusive_group(required=True) # Add arguments exclusive_args.add_argument( @@ -348,40 +390,61 @@ def createArgsParser() -> argparse.ArgumentParser: help='revert migration steps to the databases', action='store_true') - parser.add_argument( + migrate_parser.add_argument( '--all', help='apply[revert] ALL unapplied[applied] migration steps to the database', action='store_true') - parser.add_argument( - '-p', '--hasura-path', - help='directory containing the config.yaml and migrations folder for the venue. defaults to ./hasura', - default='./hasura') + return parser - parser.add_argument( - '-e', '--env-path', - help='envfile to load envvars from.') - parser.add_argument( - '--endpoint', - help="http(s) endpoint for the venue's Hasura instance", - required=False) +def migrate(args: argparse.Namespace): + hasura = create_hasura(arguments) - parser.add_argument( - '--admin_secret', - help="admin secret for the venue's Hasura instance", - required=False) + clear_screen() + print(f'\n###############################' + f'\nAERIE DATABASE MIGRATION HELPER' + f'\n###############################' + f'\n\nMigrating database at {hasura.endpoint}') + # Enter step-by-step mode if not otherwise specified + if not args.all: + # Find all migration folders for the database + migration_path = os.path.abspath(args.hasura_path+"/migrations/Aerie") + migration = DB_Migration(migration_path, args.revert) + + # Go step-by-step through the migrations available for the selected database + step_by_step_migration(hasura, migration, args.apply) + else: + bulk_migration(hasura, args.apply) - return parser +def status(args: argparse.Namespace): + hasura = create_hasura(args) + + clear_screen() + print(f'\n###############################' + f'\nAERIE DATABASE MIGRATION STATUS' + f'\n###############################' + f'\n\nDisplaying status of database at {hasura.endpoint}') -def main(): - # Generate arguments - args = migrateArgsParser().parse_args() + display_string = f"\n\033[4mMIGRATION STATUS:\033[0m\n" + output = hasura.get_migrate_output('status') + del output[0:3] + display_string += "\n".join(output) + print(display_string) - HASURA_PATH = args.hasura_path - MIGRATION_PATH = os.path.abspath(HASURA_PATH+"/migrations/Aerie") +def create_hasura(args: argparse.Namespace) -> Hasura: + """ + Create a Hasura object from the CLI arguments + + :param args: Namespace containing the CLI arguments passed to the script. Relevant fields in Namespace: + - hasura_path (mandatory): Directory containing the config.yaml and migrations folder for the venue + - env_path (optional): Envfile to load envvars from + - endpoint (optional): Http(s) endpoint for the venue's Hasura instance + - admin_secret (optional): Admin secret for the venue's Hasura instance + :return: A Hasura object connected to the specified instance + """ if args.env_path: if not os.path.isfile(args.env_path): exit_with_error(f'Specified envfile does not exist: {args.env_path}') @@ -392,34 +455,18 @@ def main(): hasura_admin_secret = args.admin_secret if args.admin_secret else os.environ.get('HASURA_GRAPHQL_ADMIN_SECRET', "") if not (hasura_endpoint and hasura_admin_secret): - (e, s) = loadConfigFile(hasura_endpoint, hasura_admin_secret, HASURA_PATH) + (e, s) = loadConfigFile(hasura_endpoint, hasura_admin_secret, args.hasura_path) hasura_endpoint = e hasura_admin_secret = s - hasura = Hasura(endpoint=hasura_endpoint, - admin_secret=hasura_admin_secret, - db_name="Aerie", - hasura_path=os.path.abspath(HASURA_PATH), - env_path=os.path.abspath(args.env_path) if args.env_path else None) - - # Navigate to the hasura directory - os.chdir(HASURA_PATH) - - clear_screen() - print(f'\n###############################' - f'\nAERIE DATABASE MIGRATION HELPER' - f'\n###############################' - f'\n\nMigrating database at {hasura.endpoint}') - # Enter step-by-step mode if not otherwise specified - if not args.all: - # Find all migration folders for the database - migration = DB_Migration(MIGRATION_PATH, args.revert) - - # Go step-by-step through the migrations available for the selected database - step_by_step_migration(hasura, migration, args.apply) - else: - bulk_migration(hasura, args.apply) + return Hasura(endpoint=hasura_endpoint, + admin_secret=hasura_admin_secret, + db_name="Aerie", + hasura_path=os.path.abspath(args.hasura_path), + env_path=os.path.abspath(args.env_path) if args.env_path else None) if __name__ == "__main__": - main() + # Generate arguments and kick off correct subfunction + arguments = createArgsParser().parse_args() + arguments.func(arguments) From eafb1c2bced5f3d6bedc8aec48094e0a5d176d01 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Mon, 9 Dec 2024 13:56:27 -0800 Subject: [PATCH 10/13] Reorder functions --- deployment/aerie_db_migration.py | 137 +++++++++++++++---------------- 1 file changed, 68 insertions(+), 69 deletions(-) diff --git a/deployment/aerie_db_migration.py b/deployment/aerie_db_migration.py index 83030976b3..59ea3ce3e0 100755 --- a/deployment/aerie_db_migration.py +++ b/deployment/aerie_db_migration.py @@ -279,6 +279,74 @@ def bulk_migration(hasura: Hasura, apply: bool): exit(exit_with) +def migrate(args: argparse.Namespace): + hasura = create_hasura(arguments) + + clear_screen() + print(f'\n###############################' + f'\nAERIE DATABASE MIGRATION HELPER' + f'\n###############################' + f'\n\nMigrating database at {hasura.endpoint}') + # Enter step-by-step mode if not otherwise specified + if not args.all: + # Find all migration folders for the database + migration_path = os.path.abspath(args.hasura_path+"/migrations/Aerie") + migration = DB_Migration(migration_path, args.revert) + + # Go step-by-step through the migrations available for the selected database + step_by_step_migration(hasura, migration, args.apply) + else: + bulk_migration(hasura, args.apply) + + +def status(args: argparse.Namespace): + hasura = create_hasura(args) + + clear_screen() + print(f'\n###############################' + f'\nAERIE DATABASE MIGRATION STATUS' + f'\n###############################' + f'\n\nDisplaying status of database at {hasura.endpoint}') + + display_string = f"\n\033[4mMIGRATION STATUS:\033[0m\n" + output = hasura.get_migrate_output('status') + del output[0:3] + display_string += "\n".join(output) + print(display_string) + + +def create_hasura(args: argparse.Namespace) -> Hasura: + """ + Create a Hasura object from the CLI arguments + + :param args: Namespace containing the CLI arguments passed to the script. Relevant fields in Namespace: + - hasura_path (mandatory): Directory containing the config.yaml and migrations folder for the venue + - env_path (optional): Envfile to load envvars from + - endpoint (optional): Http(s) endpoint for the venue's Hasura instance + - admin_secret (optional): Admin secret for the venue's Hasura instance + :return: A Hasura object connected to the specified instance + """ + if args.env_path: + if not os.path.isfile(args.env_path): + exit_with_error(f'Specified envfile does not exist: {args.env_path}') + load_dotenv(args.env_path) + + # Grab the credentials from the environment if needed + hasura_endpoint = args.endpoint if args.endpoint else os.environ.get('HASURA_GRAPHQL_ENDPOINT', "") + hasura_admin_secret = args.admin_secret if args.admin_secret else os.environ.get('HASURA_GRAPHQL_ADMIN_SECRET', "") + + if not (hasura_endpoint and hasura_admin_secret): + (e, s) = loadConfigFile(hasura_endpoint, hasura_admin_secret, args.hasura_path) + hasura_endpoint = e + hasura_admin_secret = s + + return Hasura(endpoint=hasura_endpoint, + admin_secret=hasura_admin_secret, + db_name="Aerie", + hasura_path=os.path.abspath(args.hasura_path), + env_path=os.path.abspath(args.env_path) if args.env_path else None) + + def loadConfigFile(endpoint: str, secret: str, config_folder: str) -> (str, str): """ Extract the endpoint and admin secret from a Hasura config file. @@ -397,75 +465,6 @@ def createArgsParser() -> argparse.ArgumentParser: return parser - -def migrate(args: argparse.Namespace): - hasura = create_hasura(arguments) - - clear_screen() - print(f'\n###############################' - f'\nAERIE DATABASE MIGRATION HELPER' - f'\n###############################' - f'\n\nMigrating database at {hasura.endpoint}') - # Enter step-by-step mode if not otherwise specified - if not args.all: - # Find all migration folders for the database - migration_path = os.path.abspath(args.hasura_path+"/migrations/Aerie") - migration = DB_Migration(migration_path, args.revert) - - # Go step-by-step through the migrations available for the selected database - step_by_step_migration(hasura, migration, args.apply) - else: - bulk_migration(hasura, args.apply) - - -def status(args: argparse.Namespace): - hasura = create_hasura(args) - - clear_screen() - print(f'\n###############################' - f'\nAERIE DATABASE MIGRATION STATUS' - f'\n###############################' - f'\n\nDisplaying status of database at {hasura.endpoint}') - - display_string = f"\n\033[4mMIGRATION STATUS:\033[0m\n" - output = hasura.get_migrate_output('status') - del output[0:3] - display_string += "\n".join(output) - print(display_string) - - -def create_hasura(args: argparse.Namespace) -> Hasura: - """ - Create a Hasura object from the CLI arguments - - :param args: Namespace containing the CLI arguments passed to the script. Relevant fields in Namespace: - - hasura_path (mandatory): Directory containing the config.yaml and migrations folder for the venue - - env_path (optional): Envfile to load envvars from - - endpoint (optional): Http(s) endpoint for the venue's Hasura instance - - admin_secret (optional): Admin secret for the venue's Hasura instance - :return: A Hasura object connected to the specified instance - """ - if args.env_path: - if not os.path.isfile(args.env_path): - exit_with_error(f'Specified envfile does not exist: {args.env_path}') - load_dotenv(args.env_path) - - # Grab the credentials from the environment if needed - hasura_endpoint = args.endpoint if args.endpoint else os.environ.get('HASURA_GRAPHQL_ENDPOINT', "") - hasura_admin_secret = args.admin_secret if args.admin_secret else os.environ.get('HASURA_GRAPHQL_ADMIN_SECRET', "") - - if not (hasura_endpoint and hasura_admin_secret): - (e, s) = loadConfigFile(hasura_endpoint, hasura_admin_secret, args.hasura_path) - hasura_endpoint = e - hasura_admin_secret = s - - return Hasura(endpoint=hasura_endpoint, - admin_secret=hasura_admin_secret, - db_name="Aerie", - hasura_path=os.path.abspath(args.hasura_path), - env_path=os.path.abspath(args.env_path) if args.env_path else None) - - if __name__ == "__main__": # Generate arguments and kick off correct subfunction arguments = createArgsParser().parse_args() From e912936a319bd6eeb6f7036c7880630c9278d56d Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 5 Dec 2024 09:22:23 -0800 Subject: [PATCH 11/13] Update doc comments --- deployment/aerie_db_migration.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/deployment/aerie_db_migration.py b/deployment/aerie_db_migration.py index 59ea3ce3e0..1b905b8c8e 100755 --- a/deployment/aerie_db_migration.py +++ b/deployment/aerie_db_migration.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Migrate the Aerie Database""" +"""Migrate the database of an Aerie venue.""" import os import argparse @@ -183,6 +183,13 @@ def add_migration_step(self, _migration_step): def step_by_step_migration(hasura: Hasura, db_migration: DB_Migration, apply: bool): + """ + Migrate the database one migration at a time until there are no more migrations left or the user decides to quit. + + :param hasura: Hasura object connected to the venue to be migrated + :param db_migration: DB_Migration containing the complete list of migrations available + :param apply: Whether to apply or revert migrations + """ display_string = "\n\033[4mMIGRATION STEPS AVAILABLE:\033[0m\n" _output = hasura.get_migrate_output('status') del _output[0:3] @@ -254,6 +261,12 @@ def step_by_step_migration(hasura: Hasura, db_migration: DB_Migration, apply: bo def bulk_migration(hasura: Hasura, apply: bool): + """ + Migrate the database until there are no migrations left to be applied[reverted]. + + :param hasura: Hasura object connected to the venue to be migrated + :param apply: Whether to apply or revert migrations. + """ # Migrate the database exit_with = 0 if apply: @@ -280,6 +293,11 @@ def bulk_migration(hasura: Hasura, apply: bool): def migrate(args: argparse.Namespace): + """ + Handle the 'migrate' subcommand. + + :param args: The arguments passed to the script. + """ hasura = create_hasura(arguments) clear_screen() @@ -300,6 +318,11 @@ def migrate(args: argparse.Namespace): def status(args: argparse.Namespace): + """ + Handle the 'status' subcommand. + + :param args: The arguments passed to the script. + """ hasura = create_hasura(args) clear_screen() From 174f6b3eea23ff36449d50eab8944b7ea1dc9f6a Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Tue, 10 Dec 2024 07:17:33 -0800 Subject: [PATCH 12/13] Update pgcmp workflow --- .github/workflows/pgcmp.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pgcmp.yml b/.github/workflows/pgcmp.yml index d55accf29b..d03c017690 100644 --- a/.github/workflows/pgcmp.yml +++ b/.github/workflows/pgcmp.yml @@ -111,7 +111,7 @@ jobs: AERIE_PASSWORD=${AERIE_PASSWORD} EOF python -m pip install -r requirements.txt - python aerie_db_migration.py --apply --all + python aerie_db_migration.py migrate --apply --all cd .. - name: Clone PGCMP uses: actions/checkout@v4 @@ -204,7 +204,7 @@ jobs: AERIE_PASSWORD=${AERIE_PASSWORD} EOF python -m pip install -r requirements.txt - python aerie_db_migration.py --revert --all + python aerie_db_migration.py migrate --revert --all cd .. - name: Dump Migrated Database run: | From 0fa71a3f7b4a29ebc68e16f8fef7db94a5536b7e Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Tue, 10 Dec 2024 14:06:43 -0800 Subject: [PATCH 13/13] Fix Hasura failing to reload metadata The migration script now runs "hasura metadata apply" before "hasura metadata reload". - As metadata reloading takes noticeably longer, in step-by-step mode it now only runs when exiting, rather than between each step. --- deployment/aerie_db_migration.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/deployment/aerie_db_migration.py b/deployment/aerie_db_migration.py index 1b905b8c8e..5e9018ad1e 100755 --- a/deployment/aerie_db_migration.py +++ b/deployment/aerie_db_migration.py @@ -109,6 +109,17 @@ def get_migrate_output(self, subcommand: str, flags='') -> [str]: command = f'hasura migrate {subcommand} {flags} {self.migrate_suffix} {self.command_suffix}' return subprocess.getoutput(command).split("\n") + def get_migrate_status(self, flags='') -> str: + """ + Execute 'hasura migrate status' and format the output. + + :param flags: Any additional flags to be passed to 'hasura migrate status' + :return: The output of the CLI command with the first three lines removed + """ + output = self.get_migrate_output('status', flags) + del output[0:3] + return output + def mark_current_version(self) -> int: """ Queries the database behind the Hasura instance for its current schema information. @@ -154,6 +165,13 @@ def mark_current_version(self) -> int: return cur_id + def reload_metadata(self): + """ + Apply and reload the metadata. + """ + self.execute('metadata apply') + self.execute('metadata reload') + class DB_Migration: """ @@ -191,8 +209,7 @@ def step_by_step_migration(hasura: Hasura, db_migration: DB_Migration, apply: bo :param apply: Whether to apply or revert migrations """ display_string = "\n\033[4mMIGRATION STEPS AVAILABLE:\033[0m\n" - _output = hasura.get_migrate_output('status') - del _output[0:3] + _output = hasura.get_migrate_status() display_string += _output[0] + "\n" # Filter out the steps that can't be applied given the current mode and currently applied steps @@ -243,6 +260,7 @@ def step_by_step_migration(hasura: Hasura, db_migration: DB_Migration, apply: bo _value = input(f'Revert {step}? (y/n/\033[4mq\033[0muit): ').lower() if _value == "q" or _value == "quit": + hasura.reload_metadata() sys.exit() if _value == "y": if apply: @@ -251,12 +269,14 @@ def step_by_step_migration(hasura: Hasura, db_migration: DB_Migration, apply: bo else: print('Reverting...') exit_code = hasura.migrate('apply', f'--version {timestamp} --type down') - hasura.execute('metadata reload') print() if exit_code != 0: + hasura.reload_metadata() return elif _value == "n": + hasura.reload_metadata() return + hasura.reload_metadata() input("Press Enter to continue...") @@ -280,7 +300,7 @@ def bulk_migration(hasura: Hasura, apply: bool): if exit_code != 0: exit_with = 1 - hasura.execute('metadata reload') + hasura.reload_metadata() # Show the result after the migration print(f'\n###############' @@ -332,9 +352,7 @@ def status(args: argparse.Namespace): f'\n\nDisplaying status of database at {hasura.endpoint}') display_string = f"\n\033[4mMIGRATION STATUS:\033[0m\n" - output = hasura.get_migrate_output('status') - del output[0:3] - display_string += "\n".join(output) + display_string += "\n".join(hasura.get_migrate_status()) print(display_string)