21464 - Resync EFT branch with main (#1547)

* Merge branch 'main' of https://github.com/bcgov/sbc-pay into feature-queue-python-upgrade * Fix linting issues. Make units tests pass for pay-api. * Fix lint and test for ftp-poller * pay-queue lint, ci passing * update sbc-common-components, also remove because pay-api already includes it. * remove requirements.txt for pay-admin * poetry update pay-api * Fix lint plus unit tests for payment-jobs
bcgov · May 31, 2024 · 23fed0f · 23fed0f
1 parent cd3901d
commit 23fed0f
Show file tree

Hide file tree

Showing 282 changed files with 1,908 additions and 1,950 deletions.
diff --git a/.github/workflows/pay-queue-cd.yml b/.github/workflows/pay-queue-cd.yml
@@ -4,30 +4,112 @@ on:
   push:
     branches:
       - main
-      - feature*
     paths:
       - "pay-queue/**"
       - "pay-api/src/pay_api/models/**"
       - "pay-api/src/pay_api/services/cfs_service.py"
   workflow_dispatch:
     inputs:
-      target:
-        description: "Deploy To"
+      environment:
+        description: "Environment (dev/test/prod)"
         required: true
-        type: choice
-        options:
-        - dev
-        - test
-        - sandbox
-        - prod
+        default: "dev"
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./pay-queue
+
+env:
+  APP_NAME: "pay-queue"
+  TAG_NAME: "dev"
 
 jobs:
-  pay-queue-cd:
-    uses: bcgov/bcregistry-sre/.github/workflows/cloud-run-service-cd.yaml@main
-    with:
-      target: ${{ inputs.target }}
-      app_name: "pay-queue"
-      working_directory: "./pay-queue"
-    secrets:
-      WORKLOAD_IDENTIFY_POOLS_PROVIDER: ${{ secrets.WORKLOAD_IDENTIFY_POOLS_PROVIDER }}
-      GCP_SERVICE_ACCOUNT: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+  pay-queue-cd-by-push:
+    runs-on: ubuntu-20.04
+
+    if: github.event_name == 'push' && github.repository == 'bcgov/sbc-pay'
+    environment:
+      name: "dev"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Login Openshift
+        shell: bash
+        run: |
+          oc login --server=${{secrets.OPENSHIFT4_LOGIN_REGISTRY}} --token=${{secrets.OPENSHIFT4_SA_TOKEN}}
+
+      - name: CD Flow
+        shell: bash
+        env:
+          OPS_REPOSITORY: ${{ secrets.OPS_REPOSITORY }}
+          OPENSHIFT_DOCKER_REGISTRY: ${{ secrets.OPENSHIFT4_DOCKER_REGISTRY }}
+          OPENSHIFT_SA_NAME: ${{ secrets.OPENSHIFT4_SA_NAME }}
+          OPENSHIFT_SA_TOKEN: ${{ secrets.OPENSHIFT4_SA_TOKEN }}
+          OPENSHIFT_REPOSITORY: ${{ secrets.OPENSHIFT4_REPOSITORY }}
+          TAG_NAME: ${{ env.TAG_NAME }}
+        run: |
+          make cd
+
+      - name: Watch new rollout (trigger by image change in Openshift)
+        shell: bash
+        run: |
+          oc rollout status dc/${{ env.APP_NAME }}-${{ env.TAG_NAME }} -n ${{ secrets.OPENSHIFT4_REPOSITORY }}-${{ env.TAG_NAME }} -w
+
+      - name: Rocket.Chat Notification
+        uses: RocketChat/Rocket.Chat.GitHub.Action.Notification@master
+        if: failure()
+        with:
+          type: ${{ job.status }}
+          job_name: "*Payment Reconciliations Queue Built and Deployed to ${{env.TAG_NAME}}*"
+          channel: "#registries-bot"
+          url: ${{ secrets.ROCKETCHAT_WEBHOOK }}
+          commit: true
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+  pay-queue-cd-by-dispatch:
+    runs-on: ubuntu-20.04
+
+    if: github.event_name == 'workflow_dispatch' && github.repository == 'bcgov/sbc-pay'
+    environment:
+      name: "${{ github.event.inputs.environment }}"
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set env by input
+        run: |
+          echo "TAG_NAME=${{ github.event.inputs.environment }}" >> $GITHUB_ENV
+
+      - name: Login Openshift
+        shell: bash
+        run: |
+          oc login --server=${{secrets.OPENSHIFT4_LOGIN_REGISTRY}} --token=${{secrets.OPENSHIFT4_SA_TOKEN}}
+
+      - name: CD Flow
+        shell: bash
+        env:
+          OPS_REPOSITORY: ${{ secrets.OPS_REPOSITORY }}
+          OPENSHIFT_DOCKER_REGISTRY: ${{ secrets.OPENSHIFT4_DOCKER_REGISTRY }}
+          OPENSHIFT_SA_NAME: ${{ secrets.OPENSHIFT4_SA_NAME }}
+          OPENSHIFT_SA_TOKEN: ${{ secrets.OPENSHIFT4_SA_TOKEN }}
+          OPENSHIFT_REPOSITORY: ${{ secrets.OPENSHIFT4_REPOSITORY }}
+          TAG_NAME: ${{ env.TAG_NAME }}
+        run: |
+          make cd
+
+      - name: Watch new rollout (trigger by image change in Openshift)
+        shell: bash
+        run: |
+          oc rollout status dc/${{ env.APP_NAME }}-${{ env.TAG_NAME }} -n ${{ secrets.OPENSHIFT4_REPOSITORY }}-${{ env.TAG_NAME }} -w
+
+      - name: Rocket.Chat Notification
+        uses: RocketChat/Rocket.Chat.GitHub.Action.Notification@master
+        if: failure()
+        with:
+          type: ${{ job.status }}
+          job_name: "*Payment Reconciliations Queue Built and Deployed to ${{env.TAG_NAME}}*"
+          channel: "#registries-bot"
+          url: ${{ secrets.ROCKETCHAT_WEBHOOK }}
+          commit: true
+          token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/pay-queue-ci.yml b/.github/workflows/pay-queue-ci.yml
@@ -14,7 +14,6 @@ defaults:
   run:
     shell: bash
     working-directory: ./pay-queue
-
 jobs:
   setup-job:
     runs-on: ubuntu-20.04

diff --git a/.github/workflows/pay-queue-gcp-cd.yml b/.github/workflows/pay-queue-gcp-cd.yml
@@ -0,0 +1,33 @@
+name: Pay Queue CD
+
+on:
+  push:
+    branches:
+      - main
+      - feature*
+    paths:
+      - "pay-queue/**"
+      - "pay-api/src/pay_api/models/**"
+      - "pay-api/src/pay_api/services/cfs_service.py"
+  workflow_dispatch:
+    inputs:
+      target:
+        description: "Deploy To"
+        required: true
+        type: choice
+        options:
+        - dev
+        - test
+        - sandbox
+        - prod
+
+jobs:
+  pay-queue-cd:
+    uses: bcgov/bcregistry-sre/.github/workflows/cloud-run-service-cd.yaml@main
+    with:
+      target: ${{ inputs.target }}
+      app_name: "pay-queue"
+      working_directory: "./pay-queue"
+    secrets:
+      WORKLOAD_IDENTIFY_POOLS_PROVIDER: ${{ secrets.WORKLOAD_IDENTIFY_POOLS_PROVIDER }}
+      GCP_SERVICE_ACCOUNT: ${{ secrets.GCP_SERVICE_ACCOUNT }}
diff --git a/.gitignore b/.gitignore
@@ -129,3 +129,6 @@ ACK.INBOX.F12022020202
 pay-queue/ACK.*
 pay-queue/FEEDBACK.*
 jobs/notebook-report/data/
+test_eft_tdi17.txt
+ACK.INBOX*
+FEEDBACK.INBOX*
diff --git a/bcol-api/requirements.txt b/bcol-api/requirements.txt
@@ -36,7 +36,7 @@ python-ldap==3.4.4
 pytz==2024.1
 requests-file==2.0.0
 requests-toolbelt==1.0.0
-requests==2.31.0
+requests==2.32.2
 rsa==4.9
 sentry-sdk==1.41.0
 six==1.16.0

diff --git a/jobs/ftp-poller/Makefile b/jobs/ftp-poller/Makefile
@@ -115,7 +115,7 @@ tag: push ## tag image
 #################################################################################
 
 run: ## Run the project in local
-	. venv/bin/activate && python app.py
+	echo "unimplememted use docker"
 
 #################################################################################
 # Self Documenting Commands                                                     #

diff --git a/jobs/ftp-poller/config.py b/jobs/ftp-poller/config.py
@@ -136,11 +136,9 @@ class _Config(object):  # pylint: disable=too-few-public-methods
     SENTRY_ENABLE = os.getenv('SENTRY_ENABLE', 'False')
     SENTRY_DSN = os.getenv('SENTRY_DSN', None)
 
-    # GCP PubSub
-    AUDIENCE = os.getenv('AUDIENCE', None)
-    GCP_AUTH_KEY = os.getenv('GCP_AUTH_KEY', None)
-    PUBLISHER_AUDIENCE = os.getenv('PUBLISHER_AUDIENCE', None)
-    FTP_POLLER_TOPIC = os.getenv('FTP_POLLER_TOPIC', None)
+    # PUB/SUB - PUB: ftp-poller-payment-reconciliation-dev
+    FTP_POLLER_TOPIC = os.getenv('FTP_POLLER_TOPIC', 'ftp-poller-payment-reconciliation-dev')
+    GCP_AUTH_KEY = os.getenv('AUTHPAY_GCP_AUTH_KEY', None)
 
     TESTING = False
     DEBUG = True

diff --git a/jobs/ftp-poller/devops/vaults.json b/jobs/ftp-poller/devops/vaults.json
@@ -19,5 +19,12 @@
         "application": [
             "relationship-api"
         ]
+    },
+    {
+        "vault": "gcp-queue",
+        "application": [
+            "gtksf3",
+            "topics"
+        ]
     }
 ]
diff --git a/jobs/ftp-poller/invoke_jobs.py b/jobs/ftp-poller/invoke_jobs.py
@@ -21,6 +21,7 @@
 import sentry_sdk
 from flask import Flask
 from sentry_sdk.integrations.flask import FlaskIntegration
+from pay_api.services.gcp_queue import queue
 
 import config
 from utils.logger import setup_logging
@@ -43,7 +44,8 @@ def create_app(run_mode=os.getenv('FLASK_ENV', 'production')):
                 dsn=app.config.get('SENTRY_DSN'),
                 integrations=[FlaskIntegration()]
             )
-    app.logger.info(f'<<<< Starting Ftp Poller Job >>>>')
+    app.logger.info('<<<< Starting Ftp Poller Job >>>>')
+    queue.init_app(app)
     ma.init_app(app)
 
     register_shellcontext(app)

diff --git a/jobs/ftp-poller/openshift/ftp-poller-build.json b/jobs/ftp-poller/openshift/ftp-poller-build.json