diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 8eab5df6aa..268a4fe34d 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -55,7 +55,6 @@ ARG PORTER_MIRROR=https://cdn.porter.sh ARG PORTER_VERSION=v0.38.13 ARG PORTER_TERRAFORM_MIXIN_VERSION=v1.0.0-rc.1 ARG PORTER_AZ_MIXIN_VERSION=v0.7.3 -ARG PORTER_DOCKER_MIXIN_VERSION=v0.3.0 ARG PORTER_AZURE_PLUGIN_VERSION=v0.11.2 ARG PORTER_HOME=/home/$USERNAME/.porter/ COPY .devcontainer/scripts/porter.sh /tmp/ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 573a093045..c3ac9972b6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -266,6 +266,6 @@ "forwardPorts": [ 8000 ], - // Give permission to access docker socket + // Run commands after the container is created. "postCreateCommand": "./.devcontainer/scripts/post-create.sh" } diff --git a/.devcontainer/scripts/porter.sh b/.devcontainer/scripts/porter.sh index a3ef83e46d..e99baae056 100755 --- a/.devcontainer/scripts/porter.sh +++ b/.devcontainer/scripts/porter.sh @@ -13,7 +13,6 @@ ln -s "${PORTER_HOME}/porter" "${PORTER_HOME}/runtimes/porter-runtime" "${PORTER_HOME}/porter" mixin install exec --version "${PORTER_VERSION}" "${PORTER_HOME}/porter" mixin install terraform --version "${PORTER_TERRAFORM_MIXIN_VERSION}" "${PORTER_HOME}/porter" mixin install az --version "${PORTER_AZ_MIXIN_VERSION}" -"${PORTER_HOME}/porter" mixin install docker --version "${PORTER_DOCKER_MIXIN_VERSION}" "${PORTER_HOME}/porter" plugin install azure --version "${PORTER_AZURE_PLUGIN_VERSION}" chown -R "${USERNAME}" "${PORTER_HOME}" diff --git a/.devcontainer/scripts/post-create.sh b/.devcontainer/scripts/post-create.sh index fc0f8cc417..6f04ffa5e2 100755 --- a/.devcontainer/scripts/post-create.sh +++ b/.devcontainer/scripts/post-create.sh @@ -1,5 +1,9 @@ #!/bin/bash -set -e +set -o errexit +set -o pipefail +set -o nounset +# Uncomment this line to see each command for debugging (careful: this will show secrets!) +# set -o xtrace # docker socket fixup sudo bash ./devops/scripts/set_docker_sock_permission.sh diff --git a/.github/actions/devcontainer_run_command/action.yml b/.github/actions/devcontainer_run_command/action.yml index ca76f3e1a9..e72c285d19 100644 --- a/.github/actions/devcontainer_run_command/action.yml +++ b/.github/actions/devcontainer_run_command/action.yml @@ -106,8 +106,6 @@ inputs: RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE: description: "The number of resource processor processes to create for parallel operations" required: false - E2E_TESTS_NUMBER_PROCESSES: - description: "Number of processes to run e2e tests" runs: using: composite diff --git a/.github/workflows/deploy_tre.yml b/.github/workflows/deploy_tre.yml index fa4b0375b9..9e93d6955d 100644 --- a/.github/workflows/deploy_tre.yml +++ b/.github/workflows/deploy_tre.yml @@ -59,4 +59,3 @@ jobs: CORE_APP_SERVICE_PLAN_SKU: ${{ secrets.CORE_APP_SERVICE_PLAN_SKU }} WORKSPACE_APP_SERVICE_PLAN_SKU: ${{ secrets.WORKSPACE_APP_SERVICE_PLAN_SKU }} RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE: ${{ secrets.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE }} - E2E_TESTS_NUMBER_PROCESSES: ${{ secrets.E2E_TESTS_NUMBER_PROCESSES }} diff --git a/.github/workflows/deploy_tre_branch.yml b/.github/workflows/deploy_tre_branch.yml index f4506ed0a9..4108b9979a 100644 --- a/.github/workflows/deploy_tre_branch.yml +++ b/.github/workflows/deploy_tre_branch.yml @@ -86,4 +86,3 @@ jobs: CORE_APP_SERVICE_PLAN_SKU: ${{ secrets.CORE_APP_SERVICE_PLAN_SKU }} WORKSPACE_APP_SERVICE_PLAN_SKU: ${{ secrets.WORKSPACE_APP_SERVICE_PLAN_SKU }} RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE: ${{ secrets.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE }} - E2E_TESTS_NUMBER_PROCESSES: ${{ secrets.E2E_TESTS_NUMBER_PROCESSES }} diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml index fa19819a4d..48a397e589 100644 --- a/.github/workflows/deploy_tre_reusable.yml +++ b/.github/workflows/deploy_tre_reusable.yml @@ -109,9 +109,6 @@ on: # yamllint disable-line rule:truthy RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE: description: "Inputs" required: false - E2E_TESTS_NUMBER_PROCESSES: - description: "" - required: false # This will prevent multiple runs of this entire workflow. # We should NOT cancel in progress runs as that can destabilize the environment. @@ -625,7 +622,6 @@ jobs: TRE_ID: "${{ secrets.TRE_ID }}" IS_API_SECURED: false WORKSPACE_APP_SERVICE_PLAN_SKU: ${{ secrets.WORKSPACE_APP_SERVICE_PLAN_SKU }} - E2E_TESTS_NUMBER_PROCESSES: ${{ secrets.E2E_TESTS_NUMBER_PROCESSES }} - name: Upload Test Results if: always() diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c31b338f0..4282c01819 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,17 @@ - -## 0.6.1 (Unreleased) +## 0.8.0 (Unreleased) + +**BREAKING CHANGES & MIGRATIONS**: + +ENHANCEMENTS: +* Remove Porter's Docker mixin as it's not in use ([#2889](https://github.com/microsoft/AzureTRE/pull/2889)) + +BUG FIXES: +* Private endpoints for AppInsights are now provisioning successfully and consistently ([#2841](https://github.com/microsoft/AzureTRE/pull/2841)) + +COMPONENTS: + +## 0.7.0 (November 17, 2022) **BREAKING CHANGES & MIGRATIONS**: * The airlock request object has changed. Make sure you have ran the db migration step after deploying the new API image and UI (which runs automatically in `make all`/`make tre-deploy` but can be manually invoked with `make db-migrate`) so that existing requests in your DB are migrated to the new model. @@ -15,16 +26,16 @@ * Fields in AirlockNotification event have changed without backward compatibility. If Airlock Notifier shared service is deployed, it needs to be re-deployed. Any other consumers of AirlockNotification event need to be updated. For more details, see [#2798](https://github.com/microsoft/AzureTRE/pull/2798) FEATURES: -* Display workspace and shared services total costs for admin role in UI [#2738](https://github.com/microsoft/AzureTRE/pull/2772) -* Automatically validate all resources have tre_id tag via TFLint [#2774](https://github.com/microsoft/AzureTRE/pull/2774) -* Add metadata endpoint and simplify `tre` CLI login (also adds API version to UI) (#2794) -* Updated resource card in UI with visual improvements, disabled state badge and resource ID in info popout [#2846](https://github.com/microsoft/AzureTRE/pull/2846) -* Add health information for backend services to UI info popout in footer [#2846](https://github.com/microsoft/AzureTRE/pull/2846) +* Display workspace and shared services total costs for admin role in UI ([#2772](https://github.com/microsoft/AzureTRE/pull/2772)) +* Automatically validate all resources have tre_id tag via TFLint ([#2774](https://github.com/microsoft/AzureTRE/pull/2774)) +* Add metadata endpoint and simplify `tre` CLI login (also adds API version to UI) ([#2794](https://github.com/microsoft/AzureTRE/pull/2794)) +* Updated resource card in UI with visual improvements, disabled state badge and resource ID in info popout ([#2846](https://github.com/microsoft/AzureTRE/pull/2846)) +* Add health information for backend services to UI info popout in footer ([#2846](https://github.com/microsoft/AzureTRE/pull/2846)) ENHANCEMENTS: -* Renamed several airlock fields to make them more descriptive and added a createdBy field. Included migration for backwards compatibility ([#2779](https://github.com/microsoft/AzureTRE/pull/2779)) +* Renamed several airlock fields to make them more descriptive and added a createdBy field. Included migration for backwards compatibility [#2779](https://github.com/microsoft/AzureTRE/pull/2779) * Show error message when Review VMs are not configured in the current workspace -* CLI: Add missing endpoints and minor bug fixes (#2784) +* CLI: Add missing endpoints and minor bug fixes ([#2784](https://github.com/microsoft/AzureTRE/pull/2784)) * Airlock Notifier: Provide a link to request in the UI in the email ([#2754](https://github.com/microsoft/AzureTRE/pull/2754)) * Add additional fields for Airlock Notification event ([#2798](https://github.com/microsoft/AzureTRE/pull/2798)) * Fail firewall database migration if there's no firewall deployed ([#2792](https://github.com/microsoft/AzureTRE/pull/2792)) @@ -34,7 +45,7 @@ ENHANCEMENTS: * Adds extra dns zones and links into core network ([#2828](https://github.com/microsoft/AzureTRE/pull/2828)). * Add UI version to its footer card ([#2849](https://github.com/microsoft/AzureTRE/pull/2849)). * Use `log_category_types` in `azurerm_monitor_diagnostic_categories` to remove deprecation warning ([#2855](https://github.com/microsoft/AzureTRE/pull/2855)). -* Gitea workspace bundle has a number of updates as detailed in PR ([#2862](https://github.com/microsoft/AzureTRE/pull/2862). +* Gitea workspace bundle has a number of updates as detailed in PR ([#2862](https://github.com/microsoft/AzureTRE/pull/2862)). BUG FIXES: * Show the correct createdBy value for airlock requests in UI and in API queries ([#2779](https://github.com/microsoft/AzureTRE/pull/2779)) @@ -46,6 +57,31 @@ BUG FIXES: * Fix ML Flow deployment issues and update version ([#2865](https://github.com/microsoft/AzureTRE/pull/2865)) COMPONENTS: +| name | version | +| ----- | ----- | +| devops | 0.4.2 | +| core | 0.4.43 | +| tre-workspace-base | 0.5.1 | +| tre-workspace-unrestricted | 0.5.0 | +| tre-workspace-airlock-import-review | 0.5.0 | +| tre-service-mlflow | 0.4.0 | +| tre-service-innereye | 0.4.0 | +| tre-workspace-service-gitea | 0.6.0 | +| tre-workspace-service-mysql | 0.2.0 | +| tre-service-guacamole-linuxvm | 0.5.2 | +| tre-service-guacamole-export-reviewvm | 0.0.6 | +| tre-service-guacamole-windowsvm | 0.5.2 | +| tre-service-guacamole-import-reviewvm | 0.1.3 | +| tre-service-guacamole | 0.5.0 | +| tre-user-resource-aml-compute-instance | 0.4.1 | +| tre-service-azureml | 0.5.6 | +| tre-shared-service-cyclecloud | 0.3.0 | +| tre-shared-service-gitea | 0.4.0 | +| tre-shared-service-airlock-notifier | 0.2.3 | +| tre-shared-service-admin-vm | 0.2.0 | +| tre-shared-service-certs | 0.2.2 | +| tre-shared-service-sonatype-nexus | 2.2.3 | +| tre-shared-service-firewall | 0.6.2 | ## 0.6.0 (October 24, 2022) @@ -159,7 +195,7 @@ ENHANCEMENTS: * Airlock requests with status "blocked_by_scan" have the reason for being blocked by the malware scanner in the status_message field ([#2666](https://github.com/microsoft/AzureTRE/pull/2666)) * Move admin-vm from core to a shared service ([#2624](https://github.com/microsoft/AzureTRE/pull/2624)) * Remove obsolete docker environment variables ([#2675](https://github.com/microsoft/AzureTRE/pull/2675)) -* Using Porter's Terrform mixin 1.0.0-rc.1 where mirror in done internally ([#2677](https://github.com/microsoft/AzureTRE/pull/2677)) +* Using Porter's Terraform mixin 1.0.0-rc.1 where mirror in done internally ([#2677](https://github.com/microsoft/AzureTRE/pull/2677)) * Airlock function internal storage is accessed with private endpoints ([#2679](https://github.com/microsoft/AzureTRE/pull/2679)) BUG FIXES: @@ -219,9 +255,9 @@ ENHANCEMENTS: * Keyvault diagnostic settings in base workspace ([#2521](https://github.com/microsoft/AzureTRE/pull/2521)) * Airlock requests contain a field with information about the files that were submitted ([#2504](https://github.com/microsoft/AzureTRE/pull/2504)) * UI - Operations and notifications stability improvements ([[#2530](https://github.com/microsoft/AzureTRE/pull/2530)) -* UI - Initial implemetation of Workspace Airlock Request View ([#2512](https://github.com/microsoft/AzureTRE/pull/2512)) +* UI - Initial implementation of Workspace Airlock Request View ([#2512](https://github.com/microsoft/AzureTRE/pull/2512)) * Add ability to automatically create Azure AD groups for each application role. Requires API version 0.4.30 or later ([#2532](https://github.com/microsoft/AzureTRE/pull/2532)) -* Add `is_expsed_externally` option to Azure ML Workspace Service ([#2548](https://github.com/microsoft/AzureTRE/pull2548)) +* Add `is_exposed_externally` option to Azure ML Workspace Service ([#2548](https://github.com/microsoft/AzureTRE/pull2548)) * Azure ML workspace service assigns Azure ML Data Scientist role to Workspace Researchers ([#2539](https://github.com/microsoft/AzureTRE/pull/2539)) * UI is deployed by default ([#2554](https://github.com/microsoft/AzureTRE/pull/2554)) * Remove manual/makefile option to install Gitea/Nexus ([#2573](https://github.com/microsoft/AzureTRE/pull/2573)) @@ -234,7 +270,7 @@ BUG FIXES: * Temporary disable AppInsight's private endpoint in base workspace ([#2543](https://github.com/microsoft/AzureTRE/pull/2543)) * Resource Processor execution optimization (`porter show`) for long-standing services ([#2542](https://github.com/microsoft/AzureTRE/pull/2542)) * Move AML Compute deployment to use AzApi Terraform Provider {[#2555]((https://github.com/microsoft/AzureTRE/pull/2555)) -* Invalid token exceptions in the API app are catched, throwing 401 instead of 500 Internal server error ([#2572](https://github.com/microsoft/AzureTRE/pull/2572)) +* Invalid token exceptions in the API app are caught, throwing 401 instead of 500 Internal server error ([#2572](https://github.com/microsoft/AzureTRE/pull/2572)) COMPONENTS: @@ -276,7 +312,7 @@ ENHANCEMENTS: * 'CreationTime' field was added to Airlock requests ([#2432](https://github.com/microsoft/AzureTRE/pull/2432)) * Bundles mirror Terraform plugins when built ([#2446](https://github.com/microsoft/AzureTRE/pull/2446)) * 'Get all Airlock requests' endpoint supports filtering ([#2433](https://github.com/microsoft/AzureTRE/pull/2433)) -* API uses user delagation key when generating SAS token for airlock requests ([#2460](https://github.com/microsoft/AzureTRE/pull/2460)) +* API uses user delegation key when generating SAS token for airlock requests ([#2460](https://github.com/microsoft/AzureTRE/pull/2460)) * Longer docker caching in Resource Processor ([#2486](https://github.com/microsoft/AzureTRE/pull/2486)) * Remove AppInsights Profiler support in base workspace bundle and deploy with native Terraform resources ([#2478](https://github.com/microsoft/AzureTRE/pull/2478)) diff --git a/Makefile b/Makefile index ffe3915575..872f2db717 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ IMAGE_NAME_PREFIX?="microsoft/azuretre" FULL_CONTAINER_REGISTRY_NAME?="$${ACR_NAME}.azurecr.io" FULL_IMAGE_NAME_PREFIX:=`echo "${FULL_CONTAINER_REGISTRY_NAME}/${IMAGE_NAME_PREFIX}" | tr A-Z a-z` LINTER_REGEX_INCLUDE?=all # regular expression used to specify which files to include in local linting (defaults to "all") +E2E_TESTS_NUMBER_PROCESSES_DEFAULT=4 # can be overridden in e2e_tests/.env target_title = @echo -e "\n\e[34m»»» 🧩 \e[96m$(1)\e[0m..." @@ -313,9 +314,13 @@ test-e2e-custom: $(call target_title, "Running E2E tests with custom selector ${SELECTOR}") \ && . ${MAKEFILE_DIR}/devops/scripts/load_env.sh ${MAKEFILE_DIR}/e2e_tests/.env \ && cd e2e_tests \ - && if [[ -n "$${E2E_TESTS_NUMBER_PROCESSES}" && "$${E2E_TESTS_NUMBER_PROCESSES}" -ne 1 ]]; then \ - python -m pytest -n "$${E2E_TESTS_NUMBER_PROCESSES}" -m "${SELECTOR}" --verify $${IS_API_SECURED:-true} --junit-xml "pytest_e2e_$${SELECTOR// /_}.xml"; else \ - python -m pytest -m "${SELECTOR}" --verify $${IS_API_SECURED:-true} --junit-xml "pytest_e2e_$${SELECTOR// /_}.xml"; fi + && \ + if [[ -n "$${E2E_TESTS_NUMBER_PROCESSES}" && "$${E2E_TESTS_NUMBER_PROCESSES}" -ne 1 ]]; then \ + python -m pytest -n "$${E2E_TESTS_NUMBER_PROCESSES}" -m "${SELECTOR}" --verify $${IS_API_SECURED:-true} --junit-xml "pytest_e2e_$${SELECTOR// /_}.xml"; \ + elif [[ "$${E2E_TESTS_NUMBER_PROCESSES}" -eq 1 ]]; then \ + python -m pytest -m "${SELECTOR}" --verify $${IS_API_SECURED:-true} --junit-xml "pytest_e2e_$${SELECTOR// /_}.xml"; \ + else \ + python -m pytest -n "${E2E_TESTS_NUMBER_PROCESSES_DEFAULT}" -m "${SELECTOR}" --verify $${IS_API_SECURED:-true} --junit-xml "pytest_e2e_$${SELECTOR// /_}.xml"; fi setup-local-debugging: $(call target_title,"Setting up the ability to debug the API and Resource Processor") \ diff --git a/docs/azure-tre-overview/airlock.md b/docs/azure-tre-overview/airlock.md index 0a7f8c0e0d..a73b20d209 100644 --- a/docs/azure-tre-overview/airlock.md +++ b/docs/azure-tre-overview/airlock.md @@ -1,8 +1,8 @@ # Airlock -In a Trusted Research Environment (TRE) the workspaces represent a security boundary that enables researchers to access data, execute analysis, apply algorithms and collect reports. The airlock capability brings the only mechanism that allows users to `import` or `export` data, tools or other file based artefacts in a secure fashion with a human approval. -This constitutes the mechanism focused in preventing data exfiltration and securing TRE and its workspaces from inappropriate data, while allowing researchers to work on their projects and execute their tasks. -The airlock feature brings several actions: ingress/egress Mechanism; Data movement; Security gates; Approval mechanism and Notifications. As part of TRE's Safe settings all activity must be tracked for auditiing purpose. +In a Trusted Research Environment (TRE) the workspaces represent a security boundary that enables researchers to access data, execute analysis, apply algorithms and collect reports. The airlock capability is the only mechanism that allows users to `import` or `export` data, tools or other file based artefacts in a secure fashion with a human approval. +This constitutes the mechanism focused on preventing data exfiltration and securing TRE and its workspaces from inappropriate data, while allowing researchers to work on their projects and execute their tasks. +The airlock feature brings several actions: ingress/egress Mechanism; Data movement; Security gates; Approval mechanism and Notifications. As part of TRE's Safe settings all activity must be tracked for auditing purposes. The Airlock feature aims to address these goals: @@ -10,7 +10,7 @@ The Airlock feature aims to address these goals: * Provide a process to allow approved data to be imported through the security boundary of a TRE Workspace. -* TRE provides functionality to track requests and decisions, supporting cycle of revision, approval or rejection. +* TRE provides functionality to track requests and decisions, supporting cycles of revision, approval or rejection. * Data being imported with an airlock import process can be automatically scanned for security issues. @@ -20,24 +20,24 @@ The Airlock feature aims to address these goals: * All steps within the airlock process are audited. -Typically in a TRE, the Airlock feature would be used to allow a researcher to export outputs of a research project such as summary results. With the airlock, data to export must go though a human review, typically done by a data governance team. +Typically in a TRE, the Airlock feature would be used to allow a researcher to export the outputs of a research project such as summary results. With the airlock, data to be exported must go through a human review, typically undertaken by a data governance team. -The Airlock feature will create events on every meaningful step of the processes. This will enable increased flexibility by allowing an organization to extend the notification mechanism. +The Airlock feature will create events on every meaningful step of the process. This will enable increased flexibility by allowing an organization to extend the notification mechanism. ## Ingress/Egress Mechanism -The Airlock allows a TRE user to start `import` or `export` process to a given workspace. A number of milestones must be reached in order to complete a successful import or export. These milestones are defined using the following states: +The Airlock allows a TRE user to start the `import` or `export` process to a given workspace. A number of milestones must be reached in order to complete a successful import or export. These milestones are defined using the following states: -1. **Draft**: An Airlock request has been created but has not yet started. The TRE User/Researcher has now access to a storage location and he must identify the data to be processed. At this point the airlock import/export processes allow a single file to be processed. However a compressed file may be used (zip). -2. **Submitted**: The request was submitted by the ressearcher (not yet processed). -3. **In-Review**: The request is ready to be reviewed. This state can be reached directly from Submitted state or after going through a successful security scan (found clean). +1. **Draft**: An Airlock request has been created but has not yet started. The TRE User/Researcher now has access to a storage location and he must identify the data to be processed. At this point the airlock import/export processes allow a single file to be processed. However a compressed file may be used (zip). +2. **Submitted**: The request was submitted by the researcher (not yet processed). +3. **In-Review**: The request is ready to be reviewed. This state can be reached directly from the Submitted state or after going through a successful security scan (found clean). 4. **Approval In-progress**: The Airlock request has been approved, however data movement is still ongoing. -5. **Approved**: The Airlock request has been approved. At this state, data has been securely verified and manually reviewed. The data is now in its final location. For an import process the data is now available in the TRE workspace. It can be accessed by the requestor from within the workspace. +5. **Approved**: The Airlock request has been approved. At this state, data has been securely verified and manually reviewed. The data is now in its final location. For an import process the data is now available in the TRE workspace, it can be accessed by the requestor from within the workspace. 6. **Rejection In-progress**: The Airlock request has been rejected, however data movement is still ongoing. 7. **Rejected**: The Airlock request has been rejected. The data in the process was rejected manually by the Airlock Manager. 8. **Cancelled**: The Airlock request was manually cancelled by the requestor TRE user, a Workspace owner or a TRE administrator. The cancelation is only allowed when the request is not actively changing (i.e. **Draft** or **In-Review** state). 9. **Blocking In-progress**: The Airlock request has been blocked, however data movement is still ongoing. -10. **Blocked By Scan**: The Airlock request has been blocked. The security analysis found issues in the submitted data, and consequently quarantined the data. +10. **Blocked By Scan**: The Airlock request has been blocked. The security analysis found issues in the submitted data and consequently quarantined the data. ```mermaid graph TD @@ -60,28 +60,28 @@ graph TD ``` > Airlock state flow diagram for an Airlock export request -When an airlock process is created the initial state is **Draft** and the required infrastructure will get created providing a single container to isolate the data in the request. Once completed, the user user will be able to get a link for this container inside the storage account (URL + SAS token) that he can use to upload the desired data to be processed (import or export). +When an airlock process is created the initial state is **Draft** and the required infrastructure will get created providing a single container to isolate the data in the request. Once completed, the user will be able to get a link for this container inside the storage account (URL + SAS token) that he can use to upload the desired data to be processed (import or export). This storage location is external for import (`stalimex`) or internal for export (`stalexint`), however only accessible to the requestor (ex: a TRE user/researcher). -The user will be able to upload a file to the provided storage location, using any tool of their preference: [Azure Storage Explorer](https://azure.microsoft.com/en-us/features/storage-explorer/) or [AzCopy](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-v10) which is a command line. +The user will be able to upload a file to the provided storage location, using any tool of their preference: [Azure Storage Explorer](https://azure.microsoft.com/en-us/features/storage-explorer/) or [AzCopy](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-v10) which is a command line tool. The user Submits the request (TRE API call) starting the data movement (to the `stalimip` - import in-progress or `stalexip` - export in-progress). The airlock request is now in state **Submitted**. -If enabled, the Security Scanning is started. In case security flaws are found, the request state becomes **Blocking In-progress** while the data is moved to blocked storage (either import blocked `stalimblocked` or export blocked`stalexblocked`). In this case, the request is finalized with the state **Blocked By Scan**. -If the Security Scanning does not identify security flaws, the request state becomes **In-Review**. Simultaneously a notification is sent to the Airlock Manager user. The user needs to ask for the container url using the TRE API (SAS token + URL with READ permission). +If enabled, the Security Scanning is started. In the case that security flaws are found, the request state becomes **Blocking In-progress** while the data is moved to blocked storage (either import blocked `stalimblocked` or export blocked `stalexblocked`). In this case, the request is finalized with the state **Blocked By Scan**. +If the Security Scanning does not identify any security flaws, the request state becomes **In-Review**. Simultaneously, a notification is sent to the Airlock Manager user. The user needs to ask for the container URL using the TRE API (SAS token + URL with READ permission). -> The Security Scanning can be disabled, chaging the request state from **Submitted** straight to **In-Review**. +> The Security Scanning can be disabled, changing the request state from **Submitted** straight to **In-Review**. The Airlock Manager will manually review the data using the tools of their choice available in the TRE workspace. Once review is completed, the Airlock Manager will have to *Approve* or *Reject* the airlock proces, though a TRE API call. At this point, the request will change state to either **Approval In-progress** or **Rejection In-progress**, while the data movement occurs moving afterwards to **Approved** or **Rejected** accordingly. The data will now be in the final storage destination: `stalexapp` - export approved or `stalimapp` - import approved. -With this state change a notification will be triggered to the requestor including the location for the processed data in the form of an URL + SAS token. +With this state change, a notification will be triggered to the requestor including the location of the processed data in the form of an URL + SAS token. ## Data movement -For any airlock process there is data movement either **into** a TRE workspace (in import process) or **from** a TRE workspace (in export process). Being the TRE a Workspace boundary, there are networking configurations designed to achieve this goal. The data movement will guarantee that the data is automatically verified for securty flaws and manually reviewed, before placing data inside the TRE Workspace. -Also, the process guarantees that data is not tampered with through out the process. +For any airlock process, there is data movement either **into** a TRE workspace (in import process) or **from** a TRE workspace (in export process). Being a TRE Workspace boundary, there are networking configurations designed to achieve this goal. The data movement will guarantee that the data is automatically verified for security flaws and manually reviewed, before placing data inside the TRE Workspace. +Also, the process guarantees that data is not tampered with throughout the process. -In an import process, data will transition from more public locations (yet confined to the requestor) to TRE workspace storage, after guaranteeding security automatically and by manual review. +In an import process, data will transition from more public locations (yet confined to the requestor) to TRE workspace storage, after guaranteeing security automatically and by manual review. -In an export process data will transition from internal locations (available to the requestor) to public locations in the TRE, after going through a manual review. +In an export process, data will transition from internal locations (available to the requestor) to public locations in the TRE, after going through a manual review. Considering that the Airlock requests may require large data movements, the operations can have longer durations, hence becoming the operations asynchronous. This is why states like **Approval In-progress**, **Rejection In-progress** or **Blocking In-progress** will be set while there are data movement operations. @@ -89,37 +89,37 @@ Considering that the Airlock requests may require large data movements, the oper ## Security Scan -The identified data in a airlock proces, will be submited to a security scan. If the security scan identifies issues the data is quarantined, and a report is added to the process metadata. Both the requestor and Workspace Owner are notified. For successful security scan, the data will remain in state **In-progress**, and accessible to the Workspace Owner. +The identified data in a airlock proces, will be submited to a security scan. If the security scan identifies issues the data is quarantined and a report is added to the process metadata. Both the requestor and Workspace Owner are notified. For a successful security scan, the data will remain in state **In-progress**, and accessible to the Workspace Owner. -> * The Security scan will be optional, behind a feature flag, enabled by script -> * The outcome of security scan will be either the in-progress (`stalexip`) storage or blocked (`stalexblocked`) -> * An airlock process will guarantee that the content being imported/exported is secure. It is envisioned that a set of **security gates** are identified to be executed successfully for a process to be approve. +> * The Security scan will be optional, behind a feature flag enabled by a script +> * The outcome of the security scan will be either the in-progress (`stalexip`) storage or blocked (`stalexblocked`) +> * An airlock process will guarantee that the content being imported/exported is secure. It is envisioned that a set of **security gates** are identified to be executed successfully for a process to be approved. ## Approval mechanism -The approval mechanism, is bundled with any airlock process, providing a specific way to `approve` or `reject` the data. This mechanism will allow the Airlock Managers to explicitly approve/reject the process, after having acess to the data. The Airlock Manager users will be able to execute a manual review on the data using the tools available to him in a reviewal TRE Workspace. -Once this manual review is executed, Airlock Managers can proactivelly approve or reject the airlock request. +The approval mechanism, is bundled with any airlock process, providing a specific way to `approve` or `reject` the data. This mechanism will allow the Airlock Managers to explicitly approve/reject the process, after having access to the data. The Airlock Manager users will be able to execute a manual review of the data using the tools available to him in a reviewal TRE Workspace. +Once this manual review is executed, the Airlock Managers can proactively approve or reject the airlock request. -The only goal of the Approval mechanism is to provide a cycle of revision, approval or rejection, while tracking the decision. +The only goal of the Approval mechanism is to provide a cycle of revision, approval or rejection while tracking the decision. This mechanism will provide access to the data in the airlock process, and will be able to use a VM in TRE workspace. The data review will be the Airlock Manager responsibility > * It is envisioned that this mechanism to be more flexible and extensible. -> * The `Airlock Manager` is a role defined at workspace instance level and assigned to identities. Initially the `Owner` role will be used. +> * The `Airlock Manager` is a role defined at the workspace instance level and assigned to identities. Initially, the `Owner` role will be used. ## Notifications -Throughout the airlock process, the notification mechanism will notify the relevant people to the process. Both the requestor (TRE User/Researcher) and the Workspace Owner will be notified by email, of the relevant process events. +Throughout the airlock process, the notification mechanism will notify the relevant people of the process. Both the requestor (TRE User/Researcher) and the Workspace Owner will be notified by email of the relevant process events. -Whenever the airlock process changes to a state of **Draft**, **Submitted**, **Approved**, **Rejected**, **Approval In-progress**,**Rejection In-progress**, **Blocked By Scan** or **Cancelled**, the process requestor gets notified. -When the state changes to `In-progress` the Workspace Onwer (Airlock Manager) gets notified. +Whenever the airlock process changes to a state of **Draft**, **Submitted**, **Approved**, **Rejected**, **Approval In-progress**, **Rejection In-progress**, **Blocked By Scan** or **Cancelled**, the process requestor gets notified. +When the state changes to `In-progress` the Workspace Owner (Airlock Manager) gets notified. > * The Notification mechanism is also data-driven, allowing an organization to extend the notifications behavior. The mechanism is exemplified with a Logic App determining the notifications logic. > * Notifications will work with All TRE users being AAD users (guests or not), with email defined – if not, notifications will not be sent. ## Architecture -The Airlock feature is supported by infrastructure at the TRE and workspace level, containing a set of storage accounts. Each Airlock request, will provision and use unique storage containers with the request id in it's name. +The Airlock feature is supported by infrastructure at the TRE and workspace level, containing a set of storage accounts. Each Airlock request will provision and use unique storage containers with the request id in its name. ```mermaid graph LR @@ -156,7 +156,7 @@ Workspace: * `stalexrej` - workspace storage (st) airlock (al) export (ex) rejected (rej) * `stalexblocked` - workspace storage (st) airlock (al) export (ex) blocked -> * The external storage accounts (`stalimex`, `stalexapp`), are not bound to any vnet and accessible (with SAS token) via internet. +> * The external storage accounts (`stalimex`, `stalexapp`), are not bound to any vnet and are accessible (with SAS token) via the internet > * The internal storage account (`stalexint`) is bound to the workspace vnet, so ONLY TRE Users/Researchers on that workspace can access it > * The (export) in-progress storage account (`stalexip`) is bound to the workspace vnet > * The (export) blocked storage account (`stalexblocked`) is bound to the workspace vnet @@ -173,12 +173,12 @@ In the TRE Core, the TRE API will provide the airlock API endpoints allowing to | Method | Endpoint | Description | |---|---|---| | `POST` | `/api/workspaces/{workspace_id}/requests` | Create an Airlock request (in **Draft**) | -| `POST` | `/api/workspaces/{workspace_id}/requests/{airlock_request_id}/link` | Get the url and token to acccess Airlock Request | `POST` | `/api/workspaces/{workspace_id}/requests/{airlock_request_id}/submit` | Submits an Airlock request | +| `POST` | `/api/workspaces/{workspace_id}/requests/{airlock_request_id}/link` | Get the url and token to access an Airlock Request | `POST` | `/api/workspaces/{workspace_id}/requests/{airlock_request_id}/submit` | Submits an Airlock request | | `POST` | `/api/workspaces/{workspace_id}/requests/{airlock_request_id}/review` | Reviews an Airlock request | | `POST` | `/api/workspaces/{workspace_id}/requests/{airlock_request_id}/cancel` | Cancels an Airlock request | -container | -Also in the airlock feature we have the **Airlock Processor** which will handle the events that are created throughout the process, signalling state changes from blobs created, status changed or security scans finalized. + +Also in the airlock feature there is the **Airlock Processor** which handles the events that are created throughout the process, signalling state changes from blobs created, status changed or security scans finalized. ## Airlock flow diff --git a/docs/tre-admins/environment-variables.md b/docs/tre-admins/environment-variables.md index b0208a0b35..ea3392c046 100644 --- a/docs/tre-admins/environment-variables.md +++ b/docs/tre-admins/environment-variables.md @@ -57,4 +57,3 @@ | -------- | ----------- | | `AZURE_CREDENTIALS`| Credentials used to authorize CI/CD workflows to provision resources for the TRE workspaces and workspace services. This is basically your ARM client credentials in json format. Read more about how to create it and its format [here](./setup-instructions/workflows.md##create-a-service principal-for-provisioning-resources)| | `MS_TEAMS_WEBHOOK_URI` | URI for the Teams channel webhook | - | `E2E_TESTS_NUMBER_PROCESSES` | (Optional) Number of processes to run End-to-end tests | diff --git a/docs/using-tre/templates/index.md b/docs/using-tre/templates/index.md index 6f5dd19531..2e8c636122 100644 --- a/docs/using-tre/templates/index.md +++ b/docs/using-tre/templates/index.md @@ -1,6 +1,6 @@ # Creating Custom templates -This document will show how to create custom templates, integrate them into your CI/CD pipelines. +This document will show how to create custom templates and integrate them into your CI/CD pipelines. ## Templates types @@ -14,18 +14,18 @@ Read more about them [here](../../index.md#workspace) ## How to add custom templates -AzureTRE deployment repository has directories setup for: workspace, workspace service and user resource template definitions. +AzureTRE deployment repository has directories set up for workspace, workspace service and user resource template definitions. -See [template authoring guide](../../tre-workspace-authors/authoring-workspace-templates.md) to learn more about how to author templates. +See the [template authoring guide](../../tre-workspace-authors/authoring-workspace-templates.md) to learn more about how to author templates. **To add your custom templates follow the next steps:** - Deployment requirements: - 1. Add your template under relevant folder (For example: if you are adding a new workspace template then place it under `/templates/workspaces` folder). + 1. Add your template under the relevant folder (For example: if you are adding a new workspace template then place it under `/templates/workspaces` folder). 2. Use existing templates in AzureTRE as a reference. 3. Add porter configuration - AzureTRE uses [Porter](https://porter.sh/) as a solution for implementing and deploying workspaces and workspace, learn more about how it is used in AzureTRE [here](https://microsoft.github.io/AzureTRE/tre-developers/resource-processor/#porter). - 4. Add terraform scripts to setup your deployment plan. + 4. Add terraform scripts to set up your deployment plan. - Define resource template in the API - follow [this readme](https://microsoft.github.io/AzureTRE/tre-admins/registering-templates/) to register your template. - Use the [AzureTRE UI](https://microsoft.github.io/AzureTRE/tre-developers/ui/) to deploy your resources - Add your custom templates to CI/CD workflows - in Deploy Azure TRE Reusable workflow make sure to add your bundles under register_bundles and publish_bundles steps. @@ -36,7 +36,7 @@ See the [pipelines documentation](../../tre-admins/setup-instructions/cicd-deply ## How to Contribute to our Documentation -If you have any comments or suggestions about our documentation then you can visit our GitHub project and either raise a new issue, or comment on one of the existing ones. +If you have any comments or suggestions about our documentation then you can visit our GitHub project and either raise a new issue or comment on one of the existing ones. You can find our existing documentation issues on GitHub by clicking on the link below: diff --git a/docs/using-tre/tre-for-research/importing-exporting-data-airlock.md b/docs/using-tre/tre-for-research/importing-exporting-data-airlock.md index ad33d6d001..8444db2679 100644 --- a/docs/using-tre/tre-for-research/importing-exporting-data-airlock.md +++ b/docs/using-tre/tre-for-research/importing-exporting-data-airlock.md @@ -3,9 +3,9 @@ This guide will take you through the process of importing data into a TRE workspace, and exporting data from a workspace to the outside world, using the Airlock feature. -The Airlock feature is intended for ad-hoc use when you need to bring in and export out files that you need for your research, and it ensures that when you import or export this data, all the appropriate approvals and procedures configured by your organisation take place. +The Airlock feature is intended for ad-hoc use when you need to bring in and export out files that you need for your research. It ensures that when you import or export this data all the appropriate approvals and procedures configured by your organisation take place. -You can read more about the Airock feature in the [Airlock documentation](../../azure-tre-overview/airlock.md). +You can read more about the Airlock feature in the [Airlock documentation](../../azure-tre-overview/airlock.md). ## Importing data to a workspace @@ -21,7 +21,7 @@ To bring in external data to a secure TRE workspace so you can use it for your r 1. Fill in a suitable **Title** for your request (make this short but descriptive to help you and others identify it in a list of many other requests) -1. Provide a **Business Jutification** for bringing the data into the workspace (this will be used to help your organisation's data stewards decide whether to approve or reject the request) +1. Provide a **Business Justification** for bringing the data into the workspace (this will be used to help your organisation's data stewards decide whether to approve or reject the request) 1. Click *Create* when ready. This will create your draft request and allow you to proceed with adding the data you'd like to import @@ -31,7 +31,7 @@ To bring in external data to a secure TRE workspace so you can use it for your r ### Step 2: Add data to your import request -1. The request you've just created should pop up automatically; however you can return to it at any time within the Airlock page by finding it in the list of requests. (Use the *My requests* quick filter to find it more easily) +1. The request you've just created should pop up automatically; however, you can return to it at any time within the Airlock page by finding it in the list of requests. (Use the *My requests* quick filter to find it more easily) 2. Click *Generate* in the **Files** section to generate a Storage SAS URL to use for uploading your data. @@ -50,7 +50,7 @@ To bring in external data to a secure TRE workspace so you can use it for your r ### Step 3: Get your approved data -The request will be in an *In Review* state until it's either approved or rejected by your Airlock Manager(s) manually or by an automated workflow, depending on your organisation's specific configuration. +The request will be in an *In Review* state until it is either approved or rejected by your Airlock Manager(s) manually or by an automated workflow (depending on your organisation's specific configuration). !!! note Your organisation may have the Airlock Notifier service configured which will send email notifications when your request has been approved/rejected, or you may have another mechanism in place. Please check with your TRE administrator. @@ -70,7 +70,7 @@ If your request is approved, you can follow the below steps to get your data fro - With the Azure CLI, you can use `az storage blob download --file /path/to/write/to --blob-url SAS_URL`. [More info](https://docs.microsoft.com/en-us/cli/azure/storage/blob?view=azure-cli-latest#az-storage-blob-download) !!! tip - If you're using a Workspace VM that uses one of the standard TRE Data Science VM images, you'll likely have both Storage Explorer and the Azure CLI pre-installed. + If you are using a Workspace VM that uses one of the standard TRE Data Science VM images, you will likely have both Storage Explorer and the Azure CLI pre-installed. ## Exporting data from a workspace diff --git a/e2e_tests/.env.sample b/e2e_tests/.env.sample index 0201fe40e7..8d9199e916 100644 --- a/e2e_tests/.env.sample +++ b/e2e_tests/.env.sample @@ -20,5 +20,5 @@ WORKSPACE_APP_SERVICE_PLAN_SKU="P1v2" TEST_WORKSPACE_ID= TEST_AAD_WORKSPACE_ID=ID of pre-created AAD workspace> -# Uncomment this if you wish to run tests in parallel -# E2E_TESTS_NUMBER_PROCESSES=4 +# Run tests sequentially. Change this value if you want to run tests in parallel locally +E2E_TESTS_NUMBER_PROCESSES=1 diff --git a/e2e_tests/helpers.py b/e2e_tests/helpers.py index ddb237a68c..99c6dd33e0 100644 --- a/e2e_tests/helpers.py +++ b/e2e_tests/helpers.py @@ -85,7 +85,7 @@ async def check_aad_auth_redirect(endpoint, verify) -> None: while (True): try: response = await client.get(url=endpoint, timeout=TIMEOUT) - LOGGER.info(f"Endpoint Response: {response}") + LOGGER.info(f"Endpoint Response: {endpoint} {response}") if response.status_code in terminal_http_status: break diff --git a/resource_processor/_version.py b/resource_processor/_version.py index 58ce5cd174..9b084a6099 100644 --- a/resource_processor/_version.py +++ b/resource_processor/_version.py @@ -1 +1 @@ -__version__ = "0.4.11" +__version__ = "0.4.12" diff --git a/resource_processor/scripts/porter.sh b/resource_processor/scripts/porter.sh index 10d5701cbd..0a6482bed1 100755 --- a/resource_processor/scripts/porter.sh +++ b/resource_processor/scripts/porter.sh @@ -13,5 +13,4 @@ ln -s "${PORTER_HOME}/porter" "${PORTER_HOME}/runtimes/porter-runtime" "${PORTER_HOME}/porter" mixin install exec --version "${PORTER_VERSION}" "${PORTER_HOME}/porter" mixin install terraform --version "${PORTER_TERRAFORM_MIXIN_VERSION}" "${PORTER_HOME}/porter" mixin install az --version "${PORTER_AZ_MIXIN_VERSION}" -"${PORTER_HOME}/porter" mixin install docker --version "${PORTER_DOCKER_MIXIN_VERSION}" "${PORTER_HOME}/porter" plugin install azure --version "${PORTER_AZURE_PLUGIN_VERSION}" diff --git a/resource_processor/vmss_porter/Dockerfile b/resource_processor/vmss_porter/Dockerfile index e3837b373e..bcda871a5b 100644 --- a/resource_processor/vmss_porter/Dockerfile +++ b/resource_processor/vmss_porter/Dockerfile @@ -13,7 +13,6 @@ ARG PORTER_MIRROR=https://cdn.porter.sh ARG PORTER_VERSION=v0.38.13 ARG PORTER_TERRAFORM_MIXIN_VERSION=v1.0.0-rc.1 ARG PORTER_AZ_MIXIN_VERSION=v0.7.3 -ARG PORTER_DOCKER_MIXIN_VERSION=v0.3.0 ARG PORTER_AZURE_PLUGIN_VERSION=v0.11.2 ARG PORTER_HOME=/root/.porter/ COPY scripts/porter.sh /tmp/ diff --git a/templates/workspace_services/innereye/porter.yaml b/templates/workspace_services/innereye/porter.yaml index 2c027ae625..b8d7b46b99 100644 --- a/templates/workspace_services/innereye/porter.yaml +++ b/templates/workspace_services/innereye/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-innereye -version: 0.4.0 +version: 0.4.1 description: "An Azure TRE service for InnerEye Deep Learning" registry: azuretre dockerfile: Dockerfile.tmpl @@ -50,7 +50,6 @@ parameters: mixins: - exec - az - - docker - terraform: clientVersion: 1.2.6 diff --git a/templates/workspaces/base/porter.yaml b/templates/workspaces/base/porter.yaml index b9d8933691..81cded6dde 100644 --- a/templates/workspaces/base/porter.yaml +++ b/templates/workspaces/base/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-workspace-base -version: 0.5.1 +version: 0.5.2 description: "A base Azure TRE workspace" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf b/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf index ba61a4d15f..4f32c77fe4 100644 --- a/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf +++ b/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf @@ -89,7 +89,6 @@ resource "azurerm_monitor_private_link_scoped_service" "ampls_app_insights" { } resource "azurerm_private_endpoint" "azure_monitor_private_endpoint" { - count = 0 # Remove with https://github.com/microsoft/AzureTRE/issues/2357 name = "pe-ampls-${var.tre_id}-ws-${local.short_workspace_id}" resource_group_name = var.resource_group_name location = var.location diff --git a/templates/workspaces/base/terraform/workspace.tf b/templates/workspaces/base/terraform/workspace.tf index 042e6c4766..494de2e869 100644 --- a/templates/workspaces/base/terraform/workspace.tf +++ b/templates/workspaces/base/terraform/workspace.tf @@ -76,6 +76,5 @@ module "azure_monitor" { enable_local_debugging = var.enable_local_debugging depends_on = [ module.network, - module.airlock, # shouldn't be required, related to: https://github.com/microsoft/AzureTRE/issues/2357 ] }