Merge branch 'main' into tamirkamara/3100-firewall-policy

.github/actions/devcontainer_run_command/action.yml

@@ -109,6 +109,10 @@ inputs:
   RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE:
     description: "The number of resource processor processes to create for parallel operations"
     required: false
+  E2E_TESTS_NUMBER_PROCESSES:
+    description: "The number of e2e tests running in parallel"
+    required: false
+    default: ""
 
 runs:
   using: composite
@@ -183,5 +187,6 @@ runs:
             && inputs.WORKSPACE_APP_SERVICE_PLAN_SKU) || 'P1v2' }}" \
           -e TF_VAR_resource_processor_number_processes_per_instance="${{ (inputs.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE != ''
             && inputs.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE) || 5  }}" \
+          -e E2E_TESTS_NUMBER_PROCESSES="${{ inputs.E2E_TESTS_NUMBER_PROCESSES }}" \
           '${{ inputs.CI_CACHE_ACR_NAME }}.azurecr.io/tredev:${{ inputs.DEVCONTAINER_TAG }}' \
         bash -c "${{ inputs.COMMAND }}"

.github/workflows/deploy_tre.yml

@@ -60,3 +60,4 @@ jobs:
       CORE_APP_SERVICE_PLAN_SKU: ${{ secrets.CORE_APP_SERVICE_PLAN_SKU }}
       WORKSPACE_APP_SERVICE_PLAN_SKU: ${{ secrets.WORKSPACE_APP_SERVICE_PLAN_SKU }}
       RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE: ${{ secrets.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE }}
+      E2E_TESTS_NUMBER_PROCESSES: "1"

.github/workflows/deploy_tre_branch.yml

@@ -87,3 +87,4 @@ jobs:
       CORE_APP_SERVICE_PLAN_SKU: ${{ secrets.CORE_APP_SERVICE_PLAN_SKU }}
       WORKSPACE_APP_SERVICE_PLAN_SKU: ${{ secrets.WORKSPACE_APP_SERVICE_PLAN_SKU }}
       RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE: ${{ secrets.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE }}
+      E2E_TESTS_NUMBER_PROCESSES: "1"

.github/workflows/deploy_tre_reusable.yml

@@ -110,7 +110,10 @@ on:  # yamllint disable-line rule:truthy
         description: ""
         required: false
       RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE:
-        description: "Inputs"
+        description: ""
+        required: false
+      E2E_TESTS_NUMBER_PROCESSES:
+        description: ""
         required: false
 
 # This will prevent multiple runs of this entire workflow.
@@ -784,6 +787,7 @@ jobs:
           TRE_ID: "${{ secrets.TRE_ID }}"
           IS_API_SECURED: false
           WORKSPACE_APP_SERVICE_PLAN_SKU: ${{ secrets.WORKSPACE_APP_SERVICE_PLAN_SKU }}
+          E2E_TESTS_NUMBER_PROCESSES: ${{ secrets.E2E_TESTS_NUMBER_PROCESSES }}
 
       - name: Upload Test Results
         if: always()

.github/workflows/pr_comment_bot.yml

@@ -176,3 +176,4 @@ jobs:
       TRE_ID: ${{ format('tre{0}', needs.pr_comment.outputs.prRefId) }}
       CI_CACHE_ACR_NAME: ${{ secrets.ACR_NAME }}
       TF_LOG: ${{ secrets.TF_LOG }}
+      E2E_TESTS_NUMBER_PROCESSES: "1"

e2e_tests/resources/deployment.py

@@ -10,31 +10,41 @@
 
 async def delete_done(client, operation_endpoint, headers):
     delete_terminal_states = [strings.RESOURCE_STATUS_DELETED, strings.RESOURCE_STATUS_DELETING_FAILED]
-    deployment_status, message = await check_deployment(client, operation_endpoint, headers)
-    return (True, deployment_status, message) if deployment_status in delete_terminal_states else (False, deployment_status, message)
+    deployment_status, message, operation_steps = await check_deployment(client, operation_endpoint, headers)
+    return (True, deployment_status, message, operation_steps) if deployment_status in delete_terminal_states else (False, deployment_status, message, operation_steps)
 
 
 async def install_done(client, operation_endpoint, headers):
     install_terminal_states = [strings.RESOURCE_STATUS_DEPLOYED, strings.RESOURCE_STATUS_DEPLOYMENT_FAILED]
-    deployment_status, message = await check_deployment(client, operation_endpoint, headers)
-    return (True, deployment_status, message) if deployment_status in install_terminal_states else (False, deployment_status, message)
+    deployment_status, message, operation_steps = await check_deployment(client, operation_endpoint, headers)
+    return (True, deployment_status, message, operation_steps) if deployment_status in install_terminal_states else (False, deployment_status, message, operation_steps)
 
 
 async def patch_done(client, operation_endpoint, headers):
     install_terminal_states = [strings.RESOURCE_STATUS_UPDATED, strings.RESOURCE_STATUS_UPDATING_FAILED]
-    deployment_status, message = await check_deployment(client, operation_endpoint, headers)
-    return (True, deployment_status, message) if deployment_status in install_terminal_states else (False, deployment_status, message)
+    deployment_status, message, operation_steps = await check_deployment(client, operation_endpoint, headers)
+    return (True, deployment_status, message, operation_steps) if deployment_status in install_terminal_states else (False, deployment_status, message, operation_steps)
 
 
 async def check_deployment(client, operation_endpoint, headers):
     full_endpoint = get_full_endpoint(operation_endpoint)
 
     response = await client.get(full_endpoint, headers=headers, timeout=TIMEOUT)
     if response.status_code == 200:
-        deployment_status = response.json()["operation"]["status"]
-        message = response.json()["operation"]["message"]
-        return deployment_status, message
+        response_json = response.json()
+        deployment_status = response_json["operation"]["status"]
+        message = response_json["operation"]["message"]
+        operation_steps = stringify_operation_steps(response_json["operation"]["steps"])
+        return deployment_status, message, operation_steps
     else:
         LOGGER.error(f"Non 200 response in check_deployment: {response.status_code}")
         LOGGER.error(f"Full response: {response}")
         raise Exception("Non 200 response in check_deployment")
+
+
+def stringify_operation_steps(steps):
+    string = ''
+    for i, step in enumerate(steps, 1):
+        string += f'Step {i}: {step["stepTitle"]}\n'
+        string += f'{step["message"]}\n\n'
+    return string

e2e_tests/resources/resource.py

@@ -80,7 +80,7 @@ async def disable_and_delete_resource(endpoint, access_token, verify):
 
 
 async def wait_for(func, client, operation_endpoint, headers, failure_states: list):
-    done, done_state, message = await func(client, operation_endpoint, headers)
+    done, done_state, message, operation_steps = await func(client, operation_endpoint, headers)
     LOGGER.info(f'WAITING FOR OP: {operation_endpoint}')
     while not done:
         await asyncio.sleep(30)
@@ -90,5 +90,5 @@ async def wait_for(func, client, operation_endpoint, headers, failure_states: li
     try:
         assert done_state not in failure_states
     except Exception:
-        LOGGER.exception(f"Failed to deploy status message: {message}")
+        LOGGER.exception(f"Failed to deploy. Status message: {message}.\n{operation_steps}")
         raise