From 9429a8573ccc802a8a559688090aa7d92b0f853e Mon Sep 17 00:00:00 2001 From: Jason Montleon Date: Tue, 23 Jul 2024 09:32:28 -0400 Subject: [PATCH] :bug: Use postgresql modules for migration (#364) When using psql, errors in applying the sql generated from the dump do not reach bash without adding additional options. psql only generates a non-zero exit code when something fatal happens, like being unable to connect to the database host. So, all ansible sees is exit code 0 from psql and continues on, as if everything is great. I did some reading up on postgres modules for ansible and we do not need to shell out to do this, so I have implemented it with modules. Since handling the actual migration with the ansible modules I have not encountered a failure. Continuing to run upgrades to test ... --------- Signed-off-by: Jason Montleon --- Dockerfile | 2 +- roles/tackle/tasks/main.yml | 56 +++++++++++++++---- .../deployment-keycloak-postgresql.yml.j2 | 6 +- ...rvice-keycloak-postgresql-migration.yml.j2 | 2 +- .../service-keycloak-postgresql.yml.j2 | 2 +- 5 files changed, 50 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index a0f9118d340..ed1242ae088 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN echo -e "[almalinux8-appstream]" \ "\nbaseurl = https://repo.almalinux.org/almalinux/8/AppStream/\$basearch/os/" \ "\nenabled = 1" \ "\ngpgcheck = 0" > /etc/yum.repos.d/almalinux.repo -RUN dnf -y module enable postgresql:15 && dnf -y install postgresql && dnf clean all +RUN dnf -y module enable postgresql:15 && dnf -y install postgresql python38-psycopg2 && dnf clean all USER 1001 COPY requirements.yml ${HOME}/requirements.yml diff --git a/roles/tackle/tasks/main.yml b/roles/tackle/tasks/main.yml index 7fd6082c676..beba23d90dc 100644 --- a/roles/tackle/tasks/main.yml +++ b/roles/tackle/tasks/main.yml @@ -241,18 +241,50 @@ dbm_user: "{{ pgsql_secret.resources[0].data['database-user'] | b64decode }}" dbm_pass: "{{ pgsql_secret.resources[0].data['database-password'] | b64decode }}" - - name: "Generate DB URLs" - set_fact: - dbm_src: postgresql://{{ dbm_user }}:{{ dbm_pass }}@{{ keycloak_database_service_k8s_resource_name }}/{{ keycloak_database_db_name }} - dbm_dst: postgresql://{{ dbm_user }}:{{ dbm_pass }}@{{ keycloak_database_service_k8s_resource_name }}-migration/{{ keycloak_database_db_name }} - - - name: "Perform the DB upgrade" - shell: | - set -euo pipefail - until pg_isready -U {{ dbm_user }} -h {{ keycloak_database_service_k8s_resource_name }} -d {{ keycloak_database_db_name }}; do sleep 10; done - until pg_isready -U {{ dbm_user }} -h {{ keycloak_database_service_k8s_resource_name }}-migration -d {{ keycloak_database_db_name }}; do sleep 10; done - pg_dump {{ dbm_src }} | psql {{ dbm_dst }} - changed_when: false + - name: Ping source database + postgresql_ping: + db: "{{ keycloak_database_db_name }}" + login_host: "{{ keycloak_database_service_k8s_resource_name }}" + login_user: "{{ dbm_user }}" + login_password: "{{ dbm_pass }}" + register: src_ping + retries: 12 + until: src_ping.is_available and src_ping.server_version.major == 12 + delay: 10 + + - name: Ping destination database + postgresql_ping: + db: "{{ keycloak_database_db_name }}" + login_host: "{{ keycloak_database_service_k8s_resource_name }}-migration" + login_user: "{{ dbm_user }}" + login_password: "{{ dbm_pass }}" + register: dst_ping + retries: 12 + until: dst_ping.is_available and dst_ping.server_version.major == 15 + delay: 10 + + - name: Dump database + postgresql_db: + state: dump + name: "{{ keycloak_database_db_name }}" + target: /tmp/keycloak.sql + login_host: "{{ keycloak_database_service_k8s_resource_name }}" + login_user: "{{ dbm_user }}" + login_password: "{{ dbm_pass }}" + + - name: Restore database + postgresql_db: + state: restore + name: "{{ keycloak_database_db_name }}" + target: /tmp/keycloak.sql + login_host: "{{ keycloak_database_service_k8s_resource_name }}-migration" + login_user: "{{ dbm_user }}" + login_password: "{{ dbm_pass }}" + + - name: Clean up database dump file + file: + state: absent + path: /tmp/keycloak.sql - name: "Remove the temporary migration service" k8s: diff --git a/roles/tackle/templates/deployment-keycloak-postgresql.yml.j2 b/roles/tackle/templates/deployment-keycloak-postgresql.yml.j2 index 42526f768fd..768b509d3b0 100644 --- a/roles/tackle/templates/deployment-keycloak-postgresql.yml.j2 +++ b/roles/tackle/templates/deployment-keycloak-postgresql.yml.j2 @@ -5,7 +5,7 @@ metadata: name: {{ keycloak_database_deployment_name }}-{{ keycloak_database_db_version }} namespace: {{ app_namespace }} labels: - app.kubernetes.io/name: {{ keycloak_database_service_name }} + app.kubernetes.io/name: {{ keycloak_database_service_name }}-{{ keycloak_database_db_version }} app.kubernetes.io/component: {{ keycloak_database_component_name }} app.kubernetes.io/part-of: {{ app_name }} version: "{{ keycloak_database_db_version }}" @@ -13,7 +13,7 @@ spec: replicas: {{ keycloak_database_deployment_replicas }} selector: matchLabels: - app.kubernetes.io/name: {{ keycloak_database_service_name }} + app.kubernetes.io/name: {{ keycloak_database_service_name }}-{{ keycloak_database_db_version }} app.kubernetes.io/component: {{ keycloak_database_component_name }} app.kubernetes.io/part-of: {{ app_name }} version: "{{ keycloak_database_db_version }}" @@ -24,7 +24,7 @@ spec: template: metadata: labels: - app.kubernetes.io/name: {{ keycloak_database_service_name }} + app.kubernetes.io/name: {{ keycloak_database_service_name }}-{{ keycloak_database_db_version }} app.kubernetes.io/component: {{ keycloak_database_component_name }} app.kubernetes.io/part-of: {{ app_name }} app: {{ app_name }} diff --git a/roles/tackle/templates/service-keycloak-postgresql-migration.yml.j2 b/roles/tackle/templates/service-keycloak-postgresql-migration.yml.j2 index 25a184ca78c..c3aa624f78b 100644 --- a/roles/tackle/templates/service-keycloak-postgresql-migration.yml.j2 +++ b/roles/tackle/templates/service-keycloak-postgresql-migration.yml.j2 @@ -15,7 +15,7 @@ spec: targetPort: 5432 protocol: TCP selector: - app.kubernetes.io/name: {{ keycloak_database_service_name }} + app.kubernetes.io/name: {{ keycloak_database_service_name }}-{{ keycloak_database_db_version }} app.kubernetes.io/component: {{ keycloak_database_component_name }} app.kubernetes.io/part-of: {{ app_name }} version: "{{ keycloak_database_db_version }}" diff --git a/roles/tackle/templates/service-keycloak-postgresql.yml.j2 b/roles/tackle/templates/service-keycloak-postgresql.yml.j2 index 081e1e2e981..acd0805e40d 100644 --- a/roles/tackle/templates/service-keycloak-postgresql.yml.j2 +++ b/roles/tackle/templates/service-keycloak-postgresql.yml.j2 @@ -15,7 +15,7 @@ spec: targetPort: 5432 protocol: TCP selector: - app.kubernetes.io/name: {{ keycloak_database_service_name }} + app.kubernetes.io/name: {{ keycloak_database_service_name }}-{{ keycloak_database_db_version }} app.kubernetes.io/component: {{ keycloak_database_component_name }} app.kubernetes.io/part-of: {{ app_name }} version: "{{ keycloak_database_db_version }}"