diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/autodetect-sqlite-archive.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/autodetect-sqlite-archive.yaml index 83f2d6d..a3aebbe 100644 --- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/autodetect-sqlite-archive.yaml +++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/autodetect-sqlite-archive.yaml @@ -11,4 +11,4 @@ - name: Load sqlite image if sqlite3.tar exists shell: cmd: podman image import --change 'ENV PATH=/opt/app-root/src/bin:/opt/app-root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' --change 'ENV container=oci' --change 'ENTRYPOINT=["/usr/bin/sqlite3"]' - {{ sqlite_image }} < {{ quay_root }}/sqlite3.tar - when: s.stat.exists and local_install == "false" \ No newline at end of file + when: s.stat.exists and local_install == "false" diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml index f8136a1..6961d77 100644 --- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml +++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml @@ -32,16 +32,6 @@ msg: "sqlite cli is not available via the container, cannot proceed to migrate" when: sqlite_version.rc != 0 or sqlite_version.stdout.split(' ')[0] | regex_search('(\\d+\\.\\d+\\.\\d+)') is not defined -- name: Check for Python 3 command - command: which python3 - register: python3_result - ignore_errors: yes - -- name: Fail if Python 3 is not installed - fail: - msg: "Python 3 is not installed, cannot proceed with upgrade" - when: python3_result.rc != 0 - - name: Take a pg_dump of the data from running quay-postgres container command: > podman exec -it quay-postgres @@ -148,13 +138,14 @@ state: absent path: "{{ systemd_unit_dir }}/quay-migrate.service" -- name: Copy postgres-to-sqlite python script to host machine +- name: Copy postgres-to-sqlite bash script to host machine template: - src: /runner/project/roles/mirror_appliance/templates/pg_to_sqlite.py - dest: "{{ expanded_quay_root }}/quay-postgres-backup/" + src: ../templates/pg_to_sqlite.sh + dest: "{{ expanded_quay_root }}/quay-postgres-backup/pg_to_sqlite.sh" + mode: '0755' -- name: Transform PostgreSQL data only dump to SQLite-compatible .sql file - command: python3 "{{ expanded_quay_root }}/quay-postgres-backup/pg_to_sqlite.py" "{{ expanded_quay_root }}/quay-postgres-backup/pg_data_dump.sql" "{{ expanded_quay_root }}/quay-postgres-backup/transformed_pgdata.sql" +- name: Convert PostgreSQL data-only dump to SQLite-compatible SQL file + command: /bin/bash "{{ expanded_quay_root }}/quay-postgres-backup/pg_to_sqlite.sh" "{{ expanded_quay_root }}/quay-postgres-backup/pg_data_dump.sql" "{{ expanded_quay_root }}/quay-postgres-backup/transformed_pgdata.sql" - name: Concatenate sqlite schema .sql and transformed postgres data into single merged_sqlite.sql shell: cat "{{ expanded_quay_root }}/quay-postgres-backup/sqlite_schema_dump.sql" "{{ expanded_quay_root }}/quay-postgres-backup/transformed_pgdata.sql" > "{{ expanded_quay_root }}/quay-postgres-backup/merged_sqlite.sql" diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/templates/pg_to_sqlite.py b/ansible-runner/context/app/project/roles/mirror_appliance/templates/pg_to_sqlite.py deleted file mode 100644 index ed963a2..0000000 --- a/ansible-runner/context/app/project/roles/mirror_appliance/templates/pg_to_sqlite.py +++ /dev/null @@ -1,47 +0,0 @@ -import re - -# Function that takes postgres data-only dump as "input_file" arg and converts it into a sqlite compatible .sql file -def pg_to_sqlite(input_file, output_file): - with open(input_file, 'r') as file: - sql = file.read() - - # Replace PostgreSQL-specific data types with SQLite equivalents - sql = re.sub(r'\'true\'', '1', sql) - sql = re.sub(r'\'false\'', '0', sql) - - # Remove PostgreSQL-specific commands not supported by SQLite - sql = re.sub(r'SET\s+\w+\s*=\s*[^;]+;', '', sql) # Remove all SET statements - sql = re.sub(r'^\s*--.*$', '', sql, flags=re.MULTILINE) # Remove comments - sql = re.sub(r'ALTER TABLE .*? DISABLE TRIGGER ALL;', '', sql) # Remove disable triggers - sql = re.sub(r'ALTER TABLE .*? ENABLE TRIGGER ALL;', '', sql) # Remove enable triggers - sql = re.sub(r"SELECT pg_catalog\.set_config\('search_path', '', false\);", '', sql) # Remove search path config - sql = re.sub(r'SET SESSION AUTHORIZATION DEFAULT;', '', sql) # Remove session authorization - sql = re.sub(r'SET client_encoding = \'UTF8\';', '', sql) # Remove client_encoding statement - - # Remove lines starting with "pg_dump:" - sql = re.sub(r'^pg_dump:.*$', '', sql, flags=re.MULTILINE) - - # Remove original PostgreSQL sequence set statements - sql = re.sub(r"SELECT pg_catalog\.setval\('public\..*?\', \d+, (true|false)\);", '', sql) - - # Remove the `public.` schema prefix from table names - sql = re.sub(r'INSERT INTO public\.', 'INSERT INTO ', sql) - - # Remove blank lines - sql = re.sub(r'\n\s*\n', '\n', sql) - sql = sql.strip() # Remove leading and trailing whitespace, including newlines - - with open(output_file, 'w') as file: - file.write(sql) - -if __name__ == "__main__": - import sys - if len(sys.argv) != 3: - print("Usage: python pg_to_sqlite.py ") - sys.exit(1) - - input_file = sys.argv[1] - output_file = sys.argv[2] - - pg_to_sqlite(input_file, output_file) - diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/templates/pg_to_sqlite.sh b/ansible-runner/context/app/project/roles/mirror_appliance/templates/pg_to_sqlite.sh new file mode 100755 index 0000000..d6f01cc --- /dev/null +++ b/ansible-runner/context/app/project/roles/mirror_appliance/templates/pg_to_sqlite.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Function that takes postgres data-only dump as "input_file" arg and converts it into a sqlite compatible .sql file +pg_to_sqlite() { + local input_file="$1" + local output_file="$2" + + # Read the input file + sql=$(<"$input_file") + + # Replace PostgreSQL-specific data types with SQLite equivalents + sql=$(echo "$sql" | sed "s/'true'/1/g") + sql=$(echo "$sql" | sed "s/'false'/0/g") + + # Remove PostgreSQL-specific commands not supported by SQLite + sql=$(echo "$sql" | sed -E '/SET\s+\w+\s*=\s*[^;]+;/d') + sql=$(echo "$sql" | sed -E '/^\s*--.*$/d') + sql=$(echo "$sql" | sed -E '/ALTER TABLE .*? DISABLE TRIGGER ALL;/d') + sql=$(echo "$sql" | sed -E '/ALTER TABLE .*? ENABLE TRIGGER ALL;/d') + sql=$(echo "$sql" | sed -E "s/SELECT pg_catalog\.set_config\('search_path', '', false\);//g") + sql=$(echo "$sql" | sed -E '/SET SESSION AUTHORIZATION DEFAULT;/d') + sql=$(echo "$sql" | sed -E '/SET client_encoding = '\''UTF8'\'';/d') + + # Remove lines starting with "pg_dump:" + sql=$(echo "$sql" | sed -E '/^pg_dump:.*$/d') + + # Remove original PostgreSQL sequence set statements + sql=$(echo "$sql" | sed -E "s/SELECT pg_catalog\.setval\('public\..*?', [0-9]+, (true|false)\);//g") + + # Remove the `public.` schema prefix from table names + sql=$(echo "$sql" | sed "s/INSERT INTO public\./INSERT INTO /g") + + # Remove empty lines + sql=$(echo "$sql" | sed "/^\s*$/d") + + # Trim whitespace from the beginning and end + sql=$(echo "$sql" | sed 's/^[ \t]*//;s/[ \t]*$//') + + # Write the output to the specified file + echo "$sql" > "$output_file" +} + +# Check if the script receives two arguments +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +input_file="$1" +output_file="$2" + +# Call the function +pg_to_sqlite "$input_file" "$output_file"