Replace python with a bash script for postgres to sqlite conversion (…

…PROJQUAY-7613) (#158)

* Write bash script for postgres to sqlite sql conversion (PROJQUAY-7613)

Removes the python dependency since some of the host machine
don't provide python3 out of the box

* Remove python sql version script

* Fix regex

---------

Signed-off-by: harishsurf <hgovinda@redhat.com>

ansible-runner/context/app/project/roles/mirror_appliance/tasks/autodetect-sqlite-archive.yaml

@@ -11,4 +11,4 @@
 - name: Load sqlite image if sqlite3.tar exists
   shell: 
     cmd: podman image import --change 'ENV PATH=/opt/app-root/src/bin:/opt/app-root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' --change 'ENV container=oci' --change 'ENTRYPOINT=["/usr/bin/sqlite3"]' - {{ sqlite_image }} < {{ quay_root }}/sqlite3.tar
-  when: s.stat.exists and local_install == "false"
+  when: s.stat.exists and local_install == "false"

ansible-runner/context/app/project/roles/mirror_appliance/tasks/migrate.yaml

@@ -32,16 +32,6 @@
     msg: "sqlite cli is not available via the container, cannot proceed to migrate"
   when: sqlite_version.rc != 0 or sqlite_version.stdout.split(' ')[0] | regex_search('(\\d+\\.\\d+\\.\\d+)') is not defined
 
-- name: Check for Python 3 command
-  command: which python3
-  register: python3_result
-  ignore_errors: yes
-
-- name: Fail if Python 3 is not installed
-  fail:
-    msg: "Python 3 is not installed, cannot proceed with upgrade"
-  when: python3_result.rc != 0
-
 - name: Take a pg_dump of the data from running quay-postgres container
   command: >
     podman exec -it quay-postgres
@@ -148,13 +138,14 @@
     state: absent
     path: "{{ systemd_unit_dir }}/quay-migrate.service"
 
-- name: Copy postgres-to-sqlite python script to host machine
+- name: Copy postgres-to-sqlite bash script to host machine
   template:
-    src: /runner/project/roles/mirror_appliance/templates/pg_to_sqlite.py
-    dest: "{{ expanded_quay_root }}/quay-postgres-backup/"
+    src: ../templates/pg_to_sqlite.sh
+    dest: "{{ expanded_quay_root }}/quay-postgres-backup/pg_to_sqlite.sh"
+    mode: '0755'
 
-- name: Transform PostgreSQL data only dump to SQLite-compatible .sql file
-  command: python3 "{{ expanded_quay_root }}/quay-postgres-backup/pg_to_sqlite.py" "{{ expanded_quay_root }}/quay-postgres-backup/pg_data_dump.sql" "{{ expanded_quay_root }}/quay-postgres-backup/transformed_pgdata.sql"
+- name: Convert PostgreSQL data-only dump to SQLite-compatible SQL file
+  command: /bin/bash "{{ expanded_quay_root }}/quay-postgres-backup/pg_to_sqlite.sh" "{{ expanded_quay_root }}/quay-postgres-backup/pg_data_dump.sql" "{{ expanded_quay_root }}/quay-postgres-backup/transformed_pgdata.sql"
 
 - name: Concatenate sqlite schema .sql and transformed postgres data into single merged_sqlite.sql
   shell: cat "{{ expanded_quay_root }}/quay-postgres-backup/sqlite_schema_dump.sql" "{{ expanded_quay_root }}/quay-postgres-backup/transformed_pgdata.sql" > "{{ expanded_quay_root }}/quay-postgres-backup/merged_sqlite.sql"

ansible-runner/context/app/project/roles/mirror_appliance/templates/pg_to_sqlite.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Function that takes postgres data-only dump as "input_file" arg and converts it into a sqlite compatible .sql file
+pg_to_sqlite() {
+    local input_file="$1"
+    local output_file="$2"
+
+    # Read the input file
+    sql=$(<"$input_file")
+
+    # Replace PostgreSQL-specific data types with SQLite equivalents
+    sql=$(echo "$sql" | sed "s/'true'/1/g")
+    sql=$(echo "$sql" | sed "s/'false'/0/g")
+
+    # Remove PostgreSQL-specific commands not supported by SQLite
+    sql=$(echo "$sql" | sed -E '/SET\s+\w+\s*=\s*[^;]+;/d')
+    sql=$(echo "$sql" | sed -E '/^\s*--.*$/d')
+    sql=$(echo "$sql" | sed -E '/ALTER TABLE .*? DISABLE TRIGGER ALL;/d')
+    sql=$(echo "$sql" | sed -E '/ALTER TABLE .*? ENABLE TRIGGER ALL;/d')
+    sql=$(echo "$sql" | sed -E "s/SELECT pg_catalog\.set_config\('search_path', '', false\);//g")
+    sql=$(echo "$sql" | sed -E '/SET SESSION AUTHORIZATION DEFAULT;/d')
+    sql=$(echo "$sql" | sed -E '/SET client_encoding = '\''UTF8'\'';/d')
+
+    # Remove lines starting with "pg_dump:"
+    sql=$(echo "$sql" | sed -E '/^pg_dump:.*$/d')
+
+    # Remove original PostgreSQL sequence set statements
+    sql=$(echo "$sql" | sed -E "s/SELECT pg_catalog\.setval\('public\..*?', [0-9]+, (true|false)\);//g")
+
+    # Remove the `public.` schema prefix from table names
+    sql=$(echo "$sql" | sed "s/INSERT INTO public\./INSERT INTO /g")
+
+    # Remove empty lines
+    sql=$(echo "$sql" | sed "/^\s*$/d")
+
+    # Trim whitespace from the beginning and end
+    sql=$(echo "$sql" | sed 's/^[ \t]*//;s/[ \t]*$//')
+
+    # Write the output to the specified file
+    echo "$sql" > "$output_file"
+}
+
+# Check if the script receives two arguments
+if [ "$#" -ne 2 ]; then
+    echo "Usage: $0 <input_file> <output_file>"
+    exit 1
+fi
+
+input_file="$1"
+output_file="$2"
+
+# Call the function
+pg_to_sqlite "$input_file" "$output_file"