Add CI test for replace_failed_postgresql plan

Author: petergmurphy

.github/workflows/test-replace-failed-postgresql.yaml

@@ -0,0 +1,120 @@
+---
+name: Test replace failed PostgreSQL
+on:
+  pull_request:
+    paths:
+      - .github/workflows/**/*
+      - spec/**/*
+      - lib/**/*
+      - tasks/**/*
+      - functions/**/*
+      - types/**/*
+      - plans/**/*
+      - hiera/**/*
+      - manifests/**/*
+      - templates/**/*
+      - files/**/*
+      - metadata.json
+      - Rakefile
+      - Gemfile
+      - provision.yaml
+      - .rspec
+      - .rubocop.yml
+      - .puppet-lint.rc
+      - .fixtures.yml
+    branches: [main]
+  workflow_dispatch: {}
+jobs:
+  test-replace-failed-postgresql:
+    name: PE ${{ matrix.version }} ${{ matrix.architecture }} on ${{ matrix.image }}
+    runs-on: ubuntu-20.04
+    env:
+      BOLT_GEM: true
+      BOLT_DISABLE_ANALYTICS: true
+      LANG: en_US.UTF-8
+    strategy:
+      fail-fast: false
+      matrix:
+        architecture: [extra-large-with-dr-and-spare-replica]
+        install_architecture: [extra-large-with-dr]
+        version: [2023.8.0]  # can change to test multiple versions once this one works
+        image: [almalinux-cloud/almalinux-8]
+    steps:
+      - name: Checkout Source
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+      - name: Activate Ruby 2.7
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '2.7'
+          bundler-cache: true
+      - name: Print bundle environment
+        if: ${{ github.repository_owner == 'puppetlabs' }}
+        run: |
+          echo ::group::info:bundler
+            bundle env
+          echo ::endgroup::
+      - name: Provision test cluster
+        timeout-minutes: 15
+        run: |
+          echo ::group::prepare
+            mkdir -p $HOME/.ssh
+            echo 'Host *'                      >  $HOME/.ssh/config
+            echo '    ServerAliveInterval 150' >> $HOME/.ssh/config
+            echo '    ServerAliveCountMax 2'   >> $HOME/.ssh/config
+            bundle exec rake spec_prep
+          echo ::endgroup::
+          echo ::group::provision
+            bundle exec bolt plan run peadm_spec::provision_test_cluster \
+              --modulepath spec/fixtures/modules \
+              provider=provision_service \
+              image=${{ matrix.image }} \
+              architecture=${{ matrix.architecture }} \
+              --log-level trace
+          echo ::endgroup::
+          echo ::group::info:request
+            cat request.json || true; echo
+          echo ::endgroup::
+          echo ::group::info:inventory
+            sed -e 's/password: .*/password: "[redacted]"/' < spec/fixtures/litmus_inventory.yaml || true
+          echo ::endgroup::
+      - name: Install PE on test cluster
+        timeout-minutes: 120
+        run: |
+          bundle exec bolt plan run peadm_spec::install_test_cluster \
+            --inventoryfile spec/fixtures/litmus_inventory.yaml \
+            --modulepath spec/fixtures/modules \
+            architecture=${{ matrix.install_architecture }} \
+            version=${{ matrix.version }} \
+            console_password=${{ secrets.CONSOLE_PASSWORD }}
+      - name: Replace failed PostgreSQL
+        run: |
+          echo ::group::prepare
+            mkdir -p $HOME/.ssh
+            echo 'Host *'                      >  $HOME/.ssh/config
+            echo '    ServerAliveInterval 150' >> $HOME/.ssh/config
+            echo '    ServerAliveCountMax 2'   >> $HOME/.ssh/config
+            bundle exec rake spec_prep
+          echo ::endgroup::
+          bundle exec bolt plan run peadm::replace_failed_postgresql \
+            --inventoryfile=spec/fixtures/litmus_inventory.yaml \
+            --modulepath=spec/fixtures/modules \
+            primary_host=$(yq '.groups[].targets[] | select(.vars.role == "primary") | .uri' spec/fixtures/litmus_inventory.yaml) \
+            replica_host=$(yq '.groups[].targets[] | select(.vars.role == "replica") | .uri' spec/fixtures/litmus_inventory.yaml) \
+            working_postgresql_host=$(yq '.groups[].targets[] | select(.vars.role == "replica-pdb-postgresql") | .uri' spec/fixtures/litmus_inventory.yaml) \
+            failed_postgresql_host=$(yq '.groups[].targets[] | select(.vars.role == "primary-pdb-postgresql") | .uri' spec/fixtures/litmus_inventory.yaml) \
+            replacement_postgresql_host=$(yq '.groups[].targets[] | select(.vars.role == "spare-replica") | .uri' spec/fixtures/litmus_inventory.yaml) \
+          --no-host-key-check
+      - name: Tear down PE XL DR test cluster
+        if: ${{ always() }}
+        continue-on-error: true
+        run: |-
+          if [ -f spec/fixtures/litmus_inventory.yaml ]; then
+            echo ::group::tear_down
+              bundle exec rake 'litmus:tear_down'
+            echo ::endgroup::
+            echo ::group::info:request
+              cat request.json || true; echo
+            echo ::endgroup::
+          fi