18263 - New queues use PUB/SUB (#1438)

* Update requirements.txt

* Fix CI

* Changes to convert project to new queues

* Remove nats from workflows

* remove nats from docker compose

* remove q_cli

* Remove more NATS

* Remove more nats

* Gun events-listener, combining the queues for less tech debt

* More nats removal

* Move payment-reconciliations into pay-queue

* Change project structure of queues

* Add in tests from events listener

* Getting rid of some infrastructure and tech debt we don't need

* Enhance gcp_queue_publisher

* Wire up ftp poller to new queues

* Add in enum, change util func

* Small tweaks for gcp_queue_publisher

* Put in adhoc test for publishing to queue

* Add in PaymentToken dataclass shared across apps

* shorten enums

* Replace queue messages in mailer in payment-jobs

* Add more MessageTypes to enum

* More enum

* Convert to PUB

* missing comma

* Add in pub

* Rework entire PUB

* Clean up clean up everybody clean up!

* Small fix for ftp-poller

* Fix ref

* Holy moly third try it must be friday

* fix lint

* remove unused receipt method

* payment jobs updates, makefile for pay-queue

* linting fixes, requirements fixes

* Dependencies are being funny with this one - will need to relook at it. This should work for now.

* Update old references

* Add in pay jobs CI

* directory change

* Use pip 24.0.0

* Flask-Migrate upgrade + disable compare_type

* Small tweaks for jobs, still some failing tests

* Use scalars to fix where conditions when there are tuples, as those cause exceptions

* Fix CI

* Fix unit test for pay-api

* Lint fixes and others

* Update CI

* fix deps

* update reqs

* Take out tracing for BCOL-API

* more tracing removal

* 1 more spot for tracing

* Fix linting, move pubsub to local, had issues referencing it as a library

* Update requirements

* More test / lint fixes

* Cleanup conftest

* more cleanup

* Some unit tests working, more to go

* More cleanup

* 4 unit tests to go

* Fix eft reconciliation tests

* Lint fixes more unit test fixes

* Unit tests should be all passing

* Remove redundant comment

* Cleanup

* Lint fix

* Add in code coverage for pay-queue

* Put in event listener topic

* Some cleanup

* Update requirements

.github/workflows/bcol-api-ci.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     branches:
       - main
-      - queue_python_upgrade
+      - feature-queue-python-upgrade
     paths:
       - "bcol-api/**"
 

.github/workflows/ftp-poller-ci.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     branches:
       - main
-      - queue_python_upgrade
+      - feature-queue-python-upgrade
     paths:
       - "jobs/ftp-poller/**"
 
@@ -53,10 +53,6 @@ jobs:
     needs: setup-job
     env:
       DATABASE_TEST_URL: "postgresql://postgres:postgres@localhost:5432/pay-test"
-      NATS_QUEUE: "account-worker"
-      NATS_CLUSTER_ID: "test-cluster"
-      NATS_CLIENT_NAME: "account.events.worker"
-      NATS_SUBJECT: "account.events"
       USE_DOCKER_MOCK: "YES"
       JWT_OIDC_ISSUER: "http://localhost:8081/auth/realms/demo"
       SBC_AUTH_ADMIN_CLIENT_ID: "sbc-auth-admin"

.github/workflows/pay-admin-ci.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     branches:
       - main
-      - queue_python_upgrade
+      - feature-queue-python-upgrade
     paths:
       - "pay-admin/**"
       - "pay-api/src/pay_api/models/**"

.github/workflows/pay-api-ci.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     branches:
       - main
-      - queue_python_upgrade
+      - feature-queue-python-upgrade
     paths:
       - "pay-api/**"
 
@@ -55,7 +55,6 @@ jobs:
       FLASK_ENV: "testing"
       # Needs different database than POSTGRES otherwise dropping database doesn't work
       DATABASE_TEST_URL: "postgresql://postgres:postgres@localhost:5432/pay-test"
-      TEST_NATS_DOCKER: "YES"
       USE_TEST_KEYCLOAK_DOCKER: "YES"
       USE_DOCKER_MOCK: "YES"
 
@@ -66,9 +65,6 @@ jobs:
       JWT_OIDC_TEST_CLIENT_SECRET: "1111111111"
       JWT_OIDC_TEST_JWKS_CACHE_TIMEOUT: "6000"
 
-      NATS_QUEUE: "test-worker"
-      NATS_SUBJECT: "entity.payment.test"
-
       SBC_AUTH_ADMIN_CLIENT_ID: "sbc-auth-admin"
       SBC_AUTH_ADMIN_CLIENT_SECRET: "2222222222"
 

.github/workflows/events-listener-cd.yml → .github/workflows/pay-queue-cd.yml

@@ -1,12 +1,13 @@
-name: Events Listener Queue CD
+name: Pay Queue CD
 
 on:
   push:
     branches:
       - main
     paths:
-      - "queue_services/events-listener/**"
+      - "pay-queue/**"
       - "pay-api/src/pay_api/models/**"
+      - "pay-api/src/pay_api/services/cfs_service.py"
   workflow_dispatch:
     inputs:
       environment:
@@ -17,14 +18,14 @@ on:
 defaults:
   run:
     shell: bash
-    working-directory: ./queue_services/events-listener
+    working-directory: ./pay-queue
 
 env:
-  APP_NAME: "events-listener"
+  APP_NAME: "pay-queue"
   TAG_NAME: "dev"
 
 jobs:
-  events-listener-cd-by-push:
+  pay-queue-cd-by-push:
     runs-on: ubuntu-20.04
 
     if: github.event_name == 'push' && github.repository == 'bcgov/sbc-pay'
@@ -61,13 +62,13 @@ jobs:
         if: failure()
         with:
           type: ${{ job.status }}
-          job_name: "*Events Listener Queue Built and Deployed to ${{env.TAG_NAME}}*"
+          job_name: "*Pay Queue Built and Deployed to ${{env.TAG_NAME}}*"
           channel: "#registries-bot"
           url: ${{ secrets.ROCKETCHAT_WEBHOOK }}
           commit: true
           token: ${{ secrets.GITHUB_TOKEN }}
 
-  events-listener-cd-by-dispatch:
+  pay-queue-cd-by-dispatch:
     runs-on: ubuntu-20.04
 
     if: github.event_name == 'workflow_dispatch' && github.repository == 'bcgov/sbc-pay'
@@ -107,7 +108,7 @@ jobs:
         if: failure()
         with:
           type: ${{ job.status }}
-          job_name: "*Events Listener Queue Built and Deployed to ${{env.TAG_NAME}}*"
+          job_name: "*Pay Queue Built and Deployed to ${{env.TAG_NAME}}*"
           channel: "#registries-bot"
           url: ${{ secrets.ROCKETCHAT_WEBHOOK }}
           commit: true

.github/workflows/payment-reconciliations-ci.yml → .github/workflows/pay-queue-ci.yml

@@ -1,18 +1,19 @@
-name: Payment Reconciliations Queue CI
+name: Payment Queue CI
 
 on:
   pull_request:
     branches:
       - main
+      - feature-queue-python-upgrade
     paths:
-      - "queue_services/payment-reconciliations/**"
+      - "pay-queue/**"
       - "pay-api/src/pay_api/models/**"
       - "pay-api/src/pay_api/services/cfs_service.py"
 
 defaults:
   run:
     shell: bash
-    working-directory: ./queue_services/payment-reconciliations
+    working-directory: ./pay-queue
 
 jobs:
   setup-job:
@@ -53,10 +54,8 @@ jobs:
   testing:
     needs: setup-job
     env:
-      DATABASE_TEST_URL: "postgresql://postgres:postgres@localhost:5432/postgres"
-      TEST_NATS_DOCKER: "YES"
+      DATABASE_TEST_URL: "postgresql://postgres:postgres@localhost:5432/pay-test"
       USE_DOCKER_MOCK: "YES"
-      STAN_CLUSTER_NAME: "test-cluster"
       MINIO_ENDPOINT: "localhost:9000"
       MINIO_ACCESS_KEY: "minio"
       MINIO_ACCESS_SECRET: "minio123"
@@ -80,7 +79,7 @@ jobs:
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: postgres
+          POSTGRES_DB: pay-test
         ports:
           - 5432:5432
         # needed because the postgres container does not provide a healthcheck
@@ -102,7 +101,7 @@ jobs:
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
-          file: ./queue_services/payment-reconciliations/coverage.xml
+          file: ./pay-queue/coverage.xml
           flags: paymentreconciliationsqueue
           name: codecov-payment-reconciliations
           fail_ci_if_error: true

.github/workflows/payment-jobs-ci.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - main
+      - feature-queue-python-upgrade
     paths:
       - "jobs/payment-jobs/**"
       - "pay-api/src/pay_api/models/**"
@@ -53,7 +54,7 @@ jobs:
   testing:
     needs: setup-job
     env:
-      DATABASE_TEST_URL: "postgresql://postgres:postgres@localhost:5432/postgres"
+      DATABASE_TEST_URL: "postgresql://postgres:postgres@localhost:5432/pay-test"
       USE_DOCKER_MOCK: "YES"
       JWT_OIDC_ISSUER: "http://localhost:8081/auth/realms/demo"
       SBC_AUTH_ADMIN_CLIENT_ID: "sbc-auth-admin"
@@ -71,7 +72,7 @@ jobs:
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: postgres
+          POSTGRES_DB: pay-test
         ports:
           - 5432:5432
         # needed because the postgres container does not provide a healthcheck