Merge branch 'main' into 1505-ohdsi-workspace

.devcontainer/Dockerfile

@@ -13,15 +13,15 @@ ARG USER_GID=$USER_UID
 
 # Set up non-root user
 COPY .devcontainer/scripts/non-root-user.sh /tmp/
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN bash /tmp/non-root-user.sh "${USERNAME}" "${USER_UID}" "${USER_GID}"
 
 # Set env for tracking that we're running in a devcontainer
 ENV DEVCONTAINER=true
 
-# [Option] Install Node.js
-ARG INSTALL_NODE="false"
+# Install Node.js for GH actions tests and UI
 ARG NODE_VERSION="lts/*"
-RUN if [ "${INSTALL_NODE}" = "true" ]; then su $USERNAME -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+RUN su $USERNAME -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"
 
 # Install terraform
 ARG TERRAFORM_VERSION="1.1.5"
@@ -33,6 +33,7 @@ COPY .devcontainer/scripts/docker-client.sh /tmp/
 RUN /tmp/docker-client.sh $USERNAME
 
 # Install Docker
+# hadolint ignore=DL3008,DL3004
 RUN apt-get update && sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release \
     && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \
     && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" \
@@ -41,9 +42,11 @@ RUN apt-get update && sudo apt-get install -y apt-transport-https ca-certificate
     && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
 # Install nekos act - run GitHub workflows locally https://github.com/nektos/act
+# hadolint ignore=DL3004
 RUN if [ "${INTERACTIVE}" = "true" ]; then curl https://raw.githubusercontent.com/nektos/act/master/install.sh | sudo bash; fi
 
 # Install Certbot
+# hadolint ignore=DL3008
 RUN apt-get update && apt-get install -y python3 python3-venv libaugeas0 \
     && python3 -m venv /opt/certbot/ \
     && /opt/certbot/bin/pip install --no-cache-dir --upgrade pip \
@@ -72,6 +75,7 @@ COPY ["api_app/requirements.txt", "api_app/requirements-dev.txt", "/tmp/pip-tmp/
 COPY ["resource_processor/vmss_porter/requirements.txt", "/tmp/pip-tmp/resource_processor/vmss_porter/" ]
 COPY ["docs/requirements.txt", "/tmp/pip-tmp/docs/"]
 COPY ["e2e_tests/requirements.txt", "/tmp/pip-tmp/e2e_tests/"]
+COPY ["airlock_processor/requirements.txt", "/tmp/pip-tmp/airlock_processor/"]
 RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt && rm -rf /tmp/pip-tmp
 
 # Install azure-cli
@@ -81,7 +85,7 @@ RUN export AZURE_CLI_VERSION=${AZURE_CLI_VERSION} \
     && /tmp/azure-cli.sh
 
 ARG YQ_VERSION="v4.13.0"
-RUN wget -O /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \
+RUN curl -L --fail -o /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \
     && chmod +x /usr/local/bin/yq
 
 USER $USERNAME

.devcontainer/devcontainer.json

@@ -161,6 +161,20 @@
           "purpose": [
             "debug-test"
           ]
+        },
+        {
+          "name": "Launch Edge (localhost)",
+          "type": "pwa-msedge",
+          "request": "launch",
+          "url": "http://localhost:3000",
+          "webRoot": "${workspaceFolder}/ui/app"
+        },
+        {
+          "name": "Launch Chrome (localhost)",
+          "type": "pwa-chrome",
+          "request": "launch",
+          "url": "http://localhost:3000",
+          "webRoot": "${workspaceFolder}/ui/app"
         }
       ],
       "compounds": []
@@ -223,7 +237,8 @@
     "mikestead.dotenv",
     "humao.rest-client",
     "timonwong.shellcheck",
-    "ms-azuretools.vscode-bicep"
+    "ms-azuretools.vscode-bicep",
+    "ms-azuretools.vscode-azurefunctions"
   ],
   "forwardPorts": [
     8000

.github/actions/devcontainer_run_command/action.yml

@@ -89,6 +89,12 @@ inputs:
   TF_VAR_api_client_secret:
     description: "The API Client Secret."
     required: false
+  TF_VAR_application_admin_client_id:
+    description: "The Client ID of an identity that can manage the AAD Applications."
+    required: false
+  TF_VAR_application_admin_client_secret:
+    description: "The Client secret of an identity that can manage the AAD Applications."
+    required: false
   ACR_NAME:
     description: "The Container Registry that holds our Research images."
     required: false
@@ -98,18 +104,21 @@ inputs:
   BUNDLE_TYPE:
     description: "The Bundle type (workspace / Workspace-service / User Resource)."
     required: false
+  WORKSPACE_SERVICE_NAME:
+    description: "The workspace service name for the bundle you are registering."
+    required: false
   IS_API_SECURED:
     description: "Indicates if the API endpoint has valid TLS certificate and if we validate it during E2E."
     required: false
     default: "true"
-  TF_VAR_keyvault_purge_protection_enabled:
-    description: "A value indicating if keyvaults will have purge protection."
-    required: false
-    default: "true"
   TF_VAR_stateful_resources_locked:
     description: "A value indicating if resources with state will be protected with locks."
     required: false
     default: "true"
+  TF_VAR_enable_airlock_malware_scanning:
+    description: "If False, Airlock requests will skip the malware scanning stage."
+    required: false
+    default: "false"
   CI_CACHE_ACR_NAME:
     description: "A secondary ACR used for caching in CI environments"
     required: false
@@ -156,6 +165,8 @@ runs:
         TF_VAR_aad_tenant_id: "${{ inputs.AAD_TENANT_ID }}"
         TF_VAR_api_client_id: "${{ inputs.TF_VAR_api_client_id }}"
         TF_VAR_api_client_secret: "${{ inputs.TF_VAR_api_client_secret }}"
+        TF_VAR_application_admin_client_id: "${{ inputs.TF_VAR_application_admin_client_id }}"
+        TF_VAR_application_admin_client_secret: "${{ inputs.TF_VAR_application_admin_client_secret }}"
         TF_VAR_acr_name: ${{ inputs.ACR_NAME }}
         IS_API_SECURED: ${{ inputs.IS_API_SECURED }}
       run: |
@@ -168,6 +179,7 @@ runs:
           -e TF_IN_AUTOMATION="${{ inputs.TF_IN_AUTOMATION }}" \
           -e USE_ENV_VARS_NOT_FILES="${{ inputs.USE_ENV_VARS_NOT_FILES }}" \
           -e BUNDLE_TYPE="${{ inputs.BUNDLE_TYPE }}" \
+          -e WORKSPACE_SERVICE_NAME="${{ inputs.WORKSPACE_SERVICE_NAME }}" \
           -e LOCATION="${{ inputs.LOCATION }}" \
           -e TF_VAR_location="${{ inputs.LOCATION }}" \
           -e RESOURCE_LOCATION="${{ inputs.LOCATION }}" \
@@ -183,6 +195,8 @@ runs:
           -e TF_VAR_aad_tenant_id \
           -e TF_VAR_api_client_id \
           -e TF_VAR_api_client_secret \
+          -e TF_VAR_application_admin_client_id \
+          -e TF_VAR_application_admin_client_secret \
           -e TF_VAR_arm_subscription_id="${{ inputs.ARM_SUBSCRIPTION_ID }}" \
           -e TF_VAR_swagger_ui_client_id \
           -e TF_VAR_core_address_space \
@@ -199,8 +213,8 @@ runs:
           -e TEST_ACCOUNT_CLIENT_SECRET \
           -e IS_API_SECURED \
           -e DOCKER_BUILDKIT=1 \
-          -e TF_VAR_keyvault_purge_protection_enabled=${{ inputs.TF_VAR_keyvault_purge_protection_enabled }} \
           -e TF_VAR_stateful_resources_locked=${{ inputs.TF_VAR_stateful_resources_locked }} \
+          -e TF_VAR_enable_airlock_malware_scanning=${{ inputs.TF_VAR_enable_airlock_malware_scanning }} \
           -e CI_CACHE_ACR_NAME="${{ inputs.CI_CACHE_ACR_NAME }}" \
           '${{ inputs.ACTIONS_ACR_URI }}tredev:${{ inputs.ACTIONS_DEVCONTAINER_TAG }}' \
         bash -c "${{ inputs.COMMAND }}"

.github/linters/.yaml-lint.yml

@@ -5,3 +5,5 @@ rules:
   line-length:
     max: 120  # Keep this updated with the editorconfig file
     level: warning
+  comments:
+    min-spaces-from-content: 1 # Used to follow prettier standard: https://github.com/prettier/prettier/pull/10926

.github/scripts/build.js

@@ -95,6 +95,15 @@ async function getCommandFromComment({ core, context, github }) {
           break;
         }
 
+      case "/test-extended-aad":
+        {
+          const runTests = await handleTestCommand({ core, github }, parts, "extended AAD tests", runId, { number: prNumber, authorUsername: prAuthorUsername, repoOwner, repoName, headSha: prHeadSha, refId: prRefId, details: pr }, { username: commentUsername, link: commentLink });
+          if (runTests) {
+            command = "run-tests-extended-aad";
+          }
+          break;
+        }
+
       case "/test-shared-services":
         {
           const runTests = await handleTestCommand({ core, github }, parts, "shared service tests", runId, { number: prNumber, authorUsername: prAuthorUsername, repoOwner, repoName, headSha: prHeadSha, refId: prRefId, details: pr }, { username: commentUsername, link: commentLink });
@@ -238,6 +247,7 @@ async function showHelp({ github }, repoOwner, repoName, prNumber, commentUser,
 You can use the following commands:
     /test - build, deploy and run smoke tests on a PR
     /test-extended - build, deploy and run smoke & extended tests on a PR
+    /test-extended-aad - build, deploy and run smoke & extended AAD tests on a PR
     /test-shared-services - test the deployment of shared services on a PR build
     /test-force-approve - force approval of the PR tests (i.e. skip the deployment checks)
     /test-destroy-env - delete the validation environment for a PR (e.g. to enable testing a deployment from a clean start after previous tests)

.github/scripts/build.test.js

@@ -386,6 +386,32 @@ describe('getCommandFromComment', () => {
         });
       });
 
+      describe(`for '/test-extended-aad'`, () => {
+        test(`should set command to 'run-tests-extended-aad'`, async () => {
+          const context = createCommentContext({
+            username: 'admin',
+            body: '/test-extended-aad',
+          });
+          await getCommandFromComment({ core, context, github });
+          expect(outputFor(mockCoreSetOutput, 'command')).toBe('run-tests-extended-aad');
+        });
+
+        test(`should add comment with run link`, async () => {
+          const context = createCommentContext({
+            username: 'admin',
+            body: '/test-extended-aad',
+            pullRequestNumber: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES,
+          });
+          await getCommandFromComment({ core, context, github });
+          expect(mockGithubRestIssuesCreateComment).toHaveComment({
+            owner: 'someOwner',
+            repo: 'someRepo',
+            issue_number: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES,
+            bodyMatcher: /Running extended AAD tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `cbce50da`\)/,
+          });
+        });
+      });
+
       describe(`for '/test-shared-services'`, () => {
         test(`should set command to 'run-tests-shared-services'`, async () => {
           const context = createCommentContext({

.github/workflows/build_docker_images.yml

@@ -17,6 +17,13 @@ jobs:
     name: Build images
     runs-on: ubuntu-latest
     steps:
+      - name: Upload Event File
+        # this step is required to publish test results from forks
+        uses: actions/upload-artifact@v2
+        with:
+          name: Event File
+          path: ${{ github.event_path }}
+
       - name: Checkout
         uses: actions/checkout@v2
         with:
@@ -43,6 +50,10 @@ jobs:
               - 'templates/shared_services/gitea/**/*'
             gitea_version:
               - 'templates/shared_services/gitea/version.txt'
+            airlock_processor:
+              - 'airlock_processor/**/*'
+            airlock_processor_version:
+              - 'airlock_processor/_version.py'
 
       - name: "Stale version: api"
         if: ${{ steps.filter.outputs.api == 'true' &&
@@ -64,6 +75,11 @@ jobs:
           steps.filter.outputs.gitea_version == 'false' }}
         run: echo "::error::Code update without version change" && exit 1
 
+      - name: "Stale version: airlock_processor"
+        if: ${{ steps.filter.outputs.airlock_processor == 'true' &&
+          steps.filter.outputs.airlock_processor_version == 'false' }}
+        run: echo "::error::Code update without version change" && exit 1
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v1
@@ -158,15 +174,46 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+      # Unit Tests are executed by calling the 'test-results' target in the
+      # Dockerfile's. Test runner exit codes must be swallowed (and kept) so we
+      # can output the test results. This means we have to check for failure
+      # trigger files in later steps.
+      - name: "Test image: airlock_processor"
+        # test should be before build since its docker target
+        # is prior to runtime
+        if: |
+          (steps.filter.outputs.airlock_processor == 'true'
+          || github.event_name == 'workflow_dispatch')
+        uses: docker/build-push-action@v2
+        with:
+          context: ./airlock_processor/
+          file: ./airlock_processor/Dockerfile
+          outputs: type=local,dest=test-results
+          target: test-results
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: "Check pytest failure file existence"
+        id: check_airlock_processor_test_result
+        uses: andstor/file-existence-action@v1
+        with:
+          files: "test-results/pytest_airlock_processor_unit_failed"
+
+      - name: "Build image: airlock_processor"
+        if: |
+          (steps.filter.outputs.airlock_processor == 'true'
+          || github.event_name == 'workflow_dispatch')
+          && steps.check_airlock_processor_test_result.outputs.files_exists == 'false'
+        uses: docker/build-push-action@v2
+        with:
+          context: ./airlock_processor/
+          file: ./airlock_processor/Dockerfile
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       - name: Upload Unit Test Results
         if: always()
         uses: actions/upload-artifact@v2
         with:
           name: test-results
           path: test-results
-
-      - name: Publish Unit Test Results
-        uses: EnricoMi/publish-unit-test-result-action@v1
-        if: always()
-        with:
-          files: test-results/**/*.xml

.github/workflows/deploy_tre.yml

@@ -22,7 +22,8 @@ jobs:
     uses: ./.github/workflows/deploy_tre_reusable.yml
     with:
       ciGitRef: ${{ github.ref }}
-      e2eTestsCustomSelector: "extended or shared_services"
+      e2eTestsCustomSelector: "extended or shared_services or extended_aad"
+      environmentName: CICD
     secrets:
       AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }}
       ACR_NAME: ${{ secrets.ACR_NAME }}
@@ -32,6 +33,8 @@ jobs:
       ACTIONS_DEVCONTAINER_TAG: 'latest'
       API_CLIENT_ID: ${{ secrets.API_CLIENT_ID }}
       API_CLIENT_SECRET: ${{ secrets.API_CLIENT_SECRET }}
+      APPLICATION_ADMIN_CLIENT_ID: ${{ secrets.APPLICATION_ADMIN_CLIENT_ID }}
+      APPLICATION_ADMIN_CLIENT_SECRET: ${{ secrets.APPLICATION_ADMIN_CLIENT_SECRET }}
       ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
       ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
       ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}

.github/workflows/deploy_tre_branch.yml

@@ -14,7 +14,11 @@ on:  # yamllint disable-line rule:truthy
         type: string
         default: ""
         required: false
-
+      environment:
+        description: The environment to run this workflow in
+        type: environment
+        default: CICD
+        required: true
 
 # This will prevent multiple runs of this entire workflow.
 # We should NOT cancel in progress runs as that can destabilize the environment.
@@ -51,6 +55,7 @@ jobs:
     with:
       ciGitRef: ${{ github.ref }}
       e2eTestsCustomSelector: ${{ github.event.inputs.e2eTestsCustomSelector }}
+      environmentName: ${{ github.event.inputs.environment }}
     secrets:
       AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }}
       ACR_NAME: ${{ format('tre{0}', needs.prepare-not-main.outputs.refid) }}
@@ -60,6 +65,8 @@ jobs:
       ACTIONS_DEVCONTAINER_TAG: ${{ needs.prepare-not-main.outputs.refid }}
       API_CLIENT_ID: ${{ secrets.API_CLIENT_ID }}
       API_CLIENT_SECRET: ${{ secrets.API_CLIENT_SECRET }}
+      APPLICATION_ADMIN_CLIENT_ID: ${{ secrets.APPLICATION_ADMIN_CLIENT_ID }}
+      APPLICATION_ADMIN_CLIENT_SECRET: ${{ secrets.APPLICATION_ADMIN_CLIENT_SECRET }}
       ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
       ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
       ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}