From de52841f06c5fb69a6afa9813d8c17a7c845b5f0 Mon Sep 17 00:00:00 2001
From: katie-gardner-AND <katharine.gardner@and.digital>
Date: Tue, 26 Nov 2024 12:01:22 +0000
Subject: [PATCH] feat: Add qa-visualiser to github workflows

---
 .../azure-get-db-connectionstring/action.yml  | 26 ------
 .../azure-get-keyvault-secret/action.yml      | 26 ++++++
 .../action.yml                                | 20 ++--
 .github/scripts/qa-data.py                    | 65 -------------
 .github/scripts/qa-visualisations.py          | 67 --------------
 .github/workflows/clear-user-data-from-db.yml |  6 +-
 .github/workflows/deploy-image.yml            |  8 +-
 .github/workflows/e2e-tests.yml               |  6 +-
 .github/workflows/qa-viz.yml                  | 91 +++++++++++--------
 9 files changed, 98 insertions(+), 217 deletions(-)
 delete mode 100644 .github/actions/azure-get-db-connectionstring/action.yml
 create mode 100644 .github/actions/azure-get-keyvault-secret/action.yml
 delete mode 100644 .github/scripts/qa-data.py
 delete mode 100644 .github/scripts/qa-visualisations.py

diff --git a/.github/actions/azure-get-db-connectionstring/action.yml b/.github/actions/azure-get-db-connectionstring/action.yml
deleted file mode 100644
index b5321e40d..000000000
--- a/.github/actions/azure-get-db-connectionstring/action.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-name: Get DB connection string
-description: Gets DB connection string from Azure Keyvault
-
-inputs:
-  az_keyvault_database_connectionstring_name:
-    required: true
-    type: string
-  az_keyvault_name:
-    required: true
-    type: string
-    
-outputs:
-  connection_string:
-    description: DB connection string
-    value: ${{ steps.get-connection-string.outputs.connection_string }}
-
-runs:
-  using: composite
-  steps:
-    - name: Get connection string
-      id: get-connection-string
-      shell: bash
-      run: |
-        connection_string=$(az keyvault secret show --name ${{ inputs.az_keyvault_database_connectionstring_name }} --vault-name ${{ inputs.az_keyvault_name }} --query value -o tsv)
-        echo "::add-mask::$connection_string"
-        echo "connection_string=$connection_string" >> $GITHUB_OUTPUT
\ No newline at end of file
diff --git a/.github/actions/azure-get-keyvault-secret/action.yml b/.github/actions/azure-get-keyvault-secret/action.yml
new file mode 100644
index 000000000..505c643c2
--- /dev/null
+++ b/.github/actions/azure-get-keyvault-secret/action.yml
@@ -0,0 +1,26 @@
+name: Get Keyvault secret
+description: Gets a secret from the Azure Keyvault
+
+inputs:
+  az_keyvault_secret_name:
+    required: true
+    type: string
+  az_keyvault_name:
+    required: true
+    type: string
+
+outputs:
+  secret_value:
+    description: Value of secret from the keyvault
+    value: ${{ steps.get-secret.outputs.secret_value }}
+
+runs:
+  using: composite
+  steps:
+    - name: Get secret
+      id: get-secret
+      shell: bash
+      run: |
+        secret_value=$(az keyvault secret show --name ${{ inputs.az_keyvault_secret_name }} --vault-name ${{ inputs.az_keyvault_name }} --query value -o tsv)
+        echo "::add-mask::$secret_value"
+        echo "secret_value=$secret_value" >> $GITHUB_OUTPUT
diff --git a/.github/actions/build-and-run-database-upgrader/action.yml b/.github/actions/build-and-run-database-upgrader/action.yml
index c5077789f..4f375d6c4 100644
--- a/.github/actions/build-and-run-database-upgrader/action.yml
+++ b/.github/actions/build-and-run-database-upgrader/action.yml
@@ -7,16 +7,16 @@ inputs:
     type: string
   az_keyvault_name:
     required: true
-    type: string  
+    type: string
   az_keyvault_database_connectionstring_name:
     required: true
-    type: string  
+    type: string
   az_sql_database_server_name:
     required: true
-    type: string  
+    type: string
   az_resource_group_name:
     required: true
-    type: string 
+    type: string
   environment:
     required: true
     type: string
@@ -30,16 +30,16 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: Build database upgrader  
+    - name: Build database upgrader
       uses: ./.github/actions/build-dotnet-app
       with:
-        dotnet_version: ${{ inputs.dotnet_version }}  
+        dotnet_version: ${{ inputs.dotnet_version }}
         solution_filename: Dfe.PlanTech.DatabaseUpgrader.sln
 
     - name: Get workflow IP address
       id: whats-my-ip
       uses: ./.github/actions/whats-my-ip-address
-      
+
     - name: Add Azure firewall rules
       shell: bash
       run: |
@@ -48,14 +48,14 @@ runs:
 
     - name: Get connection string
       id: get-db-connection-string
-      uses: ./.github/actions/azure-get-db-connectionstring
+      uses: ./.github/actions/azure-get-keyvault-secret
       with:
         az_keyvault_name: ${{ inputs.az_keyvault_name }}
-        az_keyvault_database_connectionstring_name: ${{ inputs.az_keyvault_database_connectionstring_name }}
+        az_keyvault_secret_name: ${{ inputs.az_keyvault_database_connectionstring_name }}
 
     - name: Run database upgrader
       shell: bash
-      run: dotnet ./build/Dfe.PlanTech.DatabaseUpgrader.dll -c "${{ steps.get-db-connection-string.outputs.connection_string }}" --env ${{ inputs.environment }} -p Analytics_Username=${{ inputs.sql_analytics_username }} Analytics_Password=${{ inputs.sql_analytics_password }}
+      run: dotnet ./build/Dfe.PlanTech.DatabaseUpgrader.dll -c "${{ steps.get-db-connection-string.outputs.secret_value }}" --env ${{ inputs.environment }} -p Analytics_Username=${{ inputs.sql_analytics_username }} Analytics_Password=${{ inputs.sql_analytics_password }}
 
     - name: Remove Azure firewall rules
       shell: bash
diff --git a/.github/scripts/qa-data.py b/.github/scripts/qa-data.py
deleted file mode 100644
index 1bf9e6c57..000000000
--- a/.github/scripts/qa-data.py
+++ /dev/null
@@ -1,65 +0,0 @@
-import os
-import pyodbc
-import json
-import struct
-from azure import identity
-
-def query_to_json(connection_string, sql_query):
-
-    if os.getenv("ENV") != "dev":
-        credential = identity.DefaultAzureCredential(exclude_interactive_browser_credential=True)
-        token_bytes = credential.get_token("https://database.windows.net/.default").token.encode("UTF-16-LE")
-        token_struct = struct.pack(f'<I{len(token_bytes)}s', len(token_bytes), token_bytes)
-        SQL_COPT_SS_ACCESS_TOKEN = 1256
-        connection = pyodbc.connect(connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct})
-    else:
-        connection = pyodbc.connect(connection_string)
-    cursor = connection.cursor()
-    cursor.execute(sql_query)
-    result = cursor.fetchall()
-    connection.close()
-    return cursor, result
-
-def main():
-
-    connection_string = os.getenv("SQL_CONNECTION_STRING")
-    
-    data_folder = 'data'
-    if not os.path.exists(data_folder):
-        os.makedirs(data_folder)
-
-    section_query = "SELECT id, name FROM [Contentful].[Sections]"
-    cursor, sections = query_to_json(connection_string, section_query)
-
-    for section in sections:
-        section_id, section_name = section
-
-        sql_query = f'''
-            SELECT
-                Q.Id AS QuestionId,
-                Q.Text AS QuestionText,
-                O.[Order] AS QuestionOrder,
-                A.Text,
-                A.NextQuestionId
-            FROM
-                [Contentful].[Questions] Q
-            JOIN
-                [Contentful].[ContentComponents] O ON Q.Id = O.Id
-            LEFT JOIN
-                [Contentful].[Answers] A ON Q.Id = A.ParentQuestionId
-            WHERE
-                Q.SectionId = '{section_id}'
-            ORDER BY
-                O.[Order] ASC;
-        '''
-
-        cursor, result = query_to_json(connection_string, sql_query)
-
-        json_result = json.dumps([dict(zip([column[0] for column in cursor.description], row)) for row in result], indent=2)
-
-        output_file = os.path.join(data_folder, f'{section_name}.json')
-        with open(output_file, 'w') as file:
-            file.write(json_result)
-
-if __name__ == "__main__":
-    main()
diff --git a/.github/scripts/qa-visualisations.py b/.github/scripts/qa-visualisations.py
deleted file mode 100644
index 7da9b8898..000000000
--- a/.github/scripts/qa-visualisations.py
+++ /dev/null
@@ -1,67 +0,0 @@
-import os
-import json
-from graphviz import Digraph
-
-def split_text(text, max_length):
-    if len(text) <= max_length:
-        return [text]
-    else:
-        split_index = text.rfind(' ', 0, max_length)
-        if split_index == -1:
-            return [text[:max_length]] + split_text(text[max_length:], max_length)
-        else:
-            return [text[:split_index]] + split_text(text[split_index+1:], max_length)
-
-def create_questionnaire_flowchart(question_data):
-    tree = Digraph(format='png')
-
-    end_nodes = set()
-
-    for item in question_data:
-        current_question = item["QuestionId"]
-        next_question_id = item["NextQuestionId"]
-        answer_text = item["Text"]
-
-        question_lines = split_text(item["QuestionText"], 20)
-        question_text = "\n".join(question_lines)
-
-        tree.node(current_question, question_text, shape='box', style='filled', fillcolor='grey:white', width='2', gradient='200')
-
-        answer_lines = split_text(answer_text, 20)
-        answer_text = "\n".join(answer_lines)
-
-        if next_question_id:
-            tree.node(next_question_id, "Missing Content", shape='diamond', style='filled', fillcolor='red:white', width='2')
-            tree.edge(current_question, next_question_id, label=answer_text, color='black')
-        else:
-            end_nodes.add(current_question)
-
-    for end_node in end_nodes:
-        tree.node("end", "Check Answers", shape='box', style='filled', fillcolor='lightblue:white', width='2', gradient='300')
-        tree.edge(end_node, "end", label="No More Questions", color='black')
-
-    return tree
-
-def process_data_files():
-    data_folder = 'data'
-    png_folder = 'visualisations'
-    if not os.path.exists(png_folder):
-        os.makedirs(png_folder)
-
-    for filename in os.listdir(data_folder):
-        if filename.endswith('.json'):
-            file_path = os.path.join(data_folder, filename)
-            with open(file_path, 'r') as file:
-                json_data = json.load(file)
-
-            flowchart = create_questionnaire_flowchart(json_data)
-            output_file = os.path.join(png_folder, f'{os.path.splitext(filename)[0]}')
-            flowchart.graph_attr['rankdir'] = 'LR'
-            flowchart.graph_attr['beautify'] = 'true'
-            flowchart.edge_attr['arrowhead'] = 'vee'
-            flowchart.edge_attr['arrowsize'] = '0.5'
-
-            flowchart.render(output_file, cleanup=True)
-
-if __name__ == '__main__':
-    process_data_files()
diff --git a/.github/workflows/clear-user-data-from-db.yml b/.github/workflows/clear-user-data-from-db.yml
index 795745dbd..b6f5b1b57 100644
--- a/.github/workflows/clear-user-data-from-db.yml
+++ b/.github/workflows/clear-user-data-from-db.yml
@@ -57,15 +57,15 @@ jobs:
 
       - name: Get connection string
         id: get-connection-string
-        uses: ./.github/actions/azure-get-db-connectionstring
+        uses: ./.github/actions/azure-get-keyvault-secret
         with:
           az_keyvault_name: ${{ env.az_keyvault_name }}
-          az_keyvault_database_connectionstring_name: ${{ env.az_keyvault_database_connectionstring_name }}
+          az_keyvault_secret_name: ${{ env.az_keyvault_database_connectionstring_name }}
 
       - name: Run clear DB SQL script
         uses: azure/sql-action@v2.2
         with:
-          connection-string: ${{ steps.get-connection-string.outputs.connection_string }}
+          connection-string: ${{ steps.get-connection-string.outputs.secret_value }}
           path: "./.github/scripts/clear-user-data-from-db.sql"
 
       - name: Remove Azure firewall rules
diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml
index f88d17f97..84b32cd64 100644
--- a/.github/workflows/deploy-image.yml
+++ b/.github/workflows/deploy-image.yml
@@ -157,17 +157,17 @@ jobs:
           az_resource_group: ${{ env.az_resource_group_name}}
           az_sql_database_server_name: ${{ env.az_sql_database_server_name }}
 
-      - name: Get Connection String
+      - name: Get Connection string
         id: get-connection-string
-        uses: ./.github/actions/azure-get-db-connectionstring
+        uses: ./.github/actions/azure-get-keyvault-secret
         with:
           az_keyvault_name: ${{ env.az_keyvault_name }}
-          az_keyvault_database_connectionstring_name: ${{ env.az_keyvault_database_connectionstring_name }}
+          az_keyvault_secret_name: ${{ env.az_keyvault_database_connectionstring_name }}
 
       - name: Execute Clear Database Stored Procedure
         uses: azure/sql-action@v2.2
         with:
-          connection-string: ${{ steps.get-connection-string.outputs.connection_string }}
+          connection-string: ${{ steps.get-connection-string.outputs.secret_value }}
           path: "./.github/scripts/drop-establishment-data.sql"
 
       - name: Cypress Testing
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
index d9cb45190..efb526242 100644
--- a/.github/workflows/e2e-tests.yml
+++ b/.github/workflows/e2e-tests.yml
@@ -55,15 +55,15 @@ jobs:
 
       - name: Get connection string
         id: get-connection-string
-        uses: ./.github/actions/azure-get-db-connectionstring
+        uses: ./.github/actions/azure-get-keyvault-secret
         with:
           az_keyvault_name: ${{ env.az_keyvault_name }}
-          az_keyvault_database_connectionstring_name: ${{ env.az_keyvault_database_connectionstring_name }}
+          az_keyvault_secret_name: ${{ env.az_keyvault_database_connectionstring_name }}
 
       - name: Execute stored procedure
         uses: azure/sql-action@v2.2
         with:
-          connection-string: ${{ steps.get-connection-string.outputs.connection_string }}
+          connection-string: ${{ steps.get-connection-string.outputs.secret_value }}
           path: "./.github/scripts/drop-establishment-data.sql"
 
       - name: Remove Azure firewall rules
diff --git a/.github/workflows/qa-viz.yml b/.github/workflows/qa-viz.yml
index 5250bcf84..a6bd97afc 100644
--- a/.github/workflows/qa-viz.yml
+++ b/.github/workflows/qa-viz.yml
@@ -3,18 +3,21 @@ name: QA Visualisation Workflow
 
 on:
   workflow_dispatch:
-    inputs: 
+    inputs:
       environment:
         description: Environment to run the visualisation for
         required: true
         type: choice
-        options: ['Dev', 'Tst', 'Staging', 'Production']
+        options: [ 'Dev', 'Tst', 'Staging', 'Production' ]
   workflow_call:
     inputs:
       environment:
         type: string
         required: true
 
+env:
+  DOTNET_VERSION: ${{ vars.DOTNET_VERSION }}
+
 jobs:
   generate_images:
     runs-on: ubuntu-latest
@@ -22,12 +25,13 @@ jobs:
       az_keyvault_name: ${{ secrets.AZ_ENVIRONMENT }}${{ secrets.DFE_PROJECT_NAME }}-kv
       az_resource_group_name: ${{ secrets.AZ_ENVIRONMENT }}${{ secrets.DFE_PROJECT_NAME }}
       az_sql_database_server_name: ${{ secrets.AZ_ENVIRONMENT }}${{ secrets.DFE_PROJECT_NAME }}
+      az_keyvault_plantech_api_key_name: api--authentication--keyvalue
       SQL_IP_NAME: qa-viz
     environment: ${{ inputs.environment || 'Staging' }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
-        
+
       - name: Login with AZ
         uses: ./.github/actions/azure-login
         with:
@@ -36,6 +40,12 @@ jobs:
           az_client_id: ${{ secrets.AZ_CLIENT_ID }}
           az_client_secret: ${{ secrets.AZ_CLIENT_SECRET }}
 
+      - name: Build web app
+        uses: ./.github/actions/build-dotnet-app
+        with:
+          dotnet_version: ${{ env.DOTNET_VERSION }}
+          solution_filename: plan-technology-for-your-school.sln
+
       - name: Get workflow IP address
         id: whats-my-ip
         uses: ./.github/actions/whats-my-ip-address
@@ -49,37 +59,48 @@ jobs:
           az_ip_name: ${{ env.SQL_IP_NAME }}
           az_resource_group: ${{ env.az_resource_group_name}}
           az_sql_database_server_name: ${{ env.az_sql_database_server_name }}
-        
+
+      - name: Run project
+        shell: bash
+        working-directory: ./build
+        run: |
+          export ASPNETCORE_ENVIRONMENT=E2E
+          export KeyVaultName=${{ env.az_keyvault_name }}
+          export Kestrel__Endpoints__Https__Url=https://*:8081
+          export DfeSignIn__FrontDoorUrl=https://localhost:8081
+          export GTM__Id=gtm_id
+          export GTM__SiteVerificationId=testing_site_verification_id
+          dotnet ./Dfe.PlanTech.Web.dll --DfeSignIn:FrontDoorUrl=https://localhost:8081 &
+
       - name: Install Graphviz
         run: sudo apt-get install graphviz -y
-          
-      - name: Install Microsoft ODBC
-        run: sudo ACCEPT_EULA=Y apt-get install msodbcsql17 -y
-      
-      - name: Set up Python
-        uses: actions/setup-python@v5
+
+      - name: Get api key
+        id: get-plantech-api-key
+        uses: ./.github/actions/azure-get-keyvault-secret
         with:
-          python-version: 3.9
-          
-      - name: Install pyodbc
-        run: pip install pyodbc
-    
-      - name: Install graphviz
-        run: pip install graphviz
-          
-      - name: Install azure identity
-        run: pip install azure-identity
-          
-      - name: Run data script
-        run: python ./.github/scripts/qa-data.py
+          az_keyvault_name: ${{ env.az_keyvault_name }}
+          az_keyvault_secret_name: ${{ env.az_keyvault_plantech_api_key_name }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
+      - name: Set up Python
+        working-directory: ./qa-visualiser
+        run: uv python install
+
+      - name: Install requirements
+        working-directory: ./qa-visualiser
+        run: uv sync
+
+      - name: Run visualisation script
+        working-directory: ./qa-visualiser
+        run: uv run main.py
         env:
-          SQL_CONNECTION_STRING: "Driver={ODBC Driver 17 for SQL Server};Server=tcp:${{ env.az_sql_database_server_name }}.database.windows.net,1433;Database=${{ env.az_sql_database_server_name }}-sqldb;Encrypt=yes;TrustServerCertificate=no;Connection Timeout=30"
-          
-      - name: List data files
-        run: ls -l ./data
-          
+          PLANTECH_API_URL: https://localhost:8081/api/cms
+          PLANTECH_API_KEY: ${{ steps.get-plantech-api-key.outputs.secret_value }}
+
       - name: Remove Azure firewall rules
-        if: always()
         uses: ./.github/actions/azure-ip-whitelist
         with:
           ip_address: ${{ steps.whats-my-ip.outputs.ip }}
@@ -88,18 +109,10 @@ jobs:
           az_ip_name: ${{ env.SQL_IP_NAME }}
           az_resource_group: ${{ env.az_resource_group_name}}
           az_sql_database_server_name: ${{ env.az_sql_database_server_name }}
-            
-      - name: Run Visaulisation Script
-        run: python ./.github/scripts/qa-visualisations.py 
-        
+
       - name: Upload image
         id: artifact-upload-step
         uses: actions/upload-artifact@v4
         with:
           name: qa-visualisations
-          path: ./visualisations
-        
-      - name: Call Teams Webhook
-        run: |
-            curl -X POST -H 'Content-Type: application/json' -d '{"text": "New Visualisations Available for ${{ inputs.environment }} : https://github.com/DFE-Digital/plan-technology-for-your-school/actions/runs/${{ github.run_id }}/artifacts/${{ steps.artifact-upload-step.outputs.artifact-id }}"}' ${{ secrets.PLANTECH_TEAMS_WEBHOOK }}
-          
+          path: ./qa-visualiser/visualisations