Merge branch 'microsoft:main' into main

CHANGELOG.md

@@ -8,8 +8,12 @@ FEATURES:
 ENHANCEMENTS:
 
 BUG FIXES:
+* Update Guacomole Linux VM Images to Ubuntu 22.04 LTS. Part of ([#3523](https://github.com/microsoft/AzureTRE/issues/3523))
+* Update Nexus Shared Service with new proxies. Part of ([#3523](https://github.com/microsoft/AzureTRE/issues/3523))
 * Update to Resource Processor Image, now using Ubuntu 22.04 (jammy). Part of ([#3523](https://github.com/microsoft/AzureTRE/issues/3523))
-* Remove TLS1.0/1.1 support from Application Gateway
+* Remove TLS1.0/1.1 support from Application Gateway ([#3914](https://github.com/microsoft/AzureTRE/issues/3914))
+* GitHub Actions version updates. ([#3847](https://github.com/microsoft/AzureTRE/issues/3847))
+* Add workaround to avoid name clashes for storage accounts([#3863](https://github.com/microsoft/AzureTRE/pull/3858))
 
 COMPONENTS:
 

api_app/_version.py

@@ -1 +1 @@
-__version__ = "0.18.6"
+__version__ = "0.18.7"

api_app/db/repositories/workspaces.py

@@ -1,12 +1,12 @@
 import uuid
 from typing import List, Tuple
-
+from azure.mgmt.storage import StorageManagementClient
 from pydantic import parse_obj_as
 from db.repositories.resources_history import ResourceHistoryRepository
 from models.domain.resource_template import ResourceTemplate
 from models.domain.authentication import User
 
-from core import config
+from core import config, credentials
 from db.errors import EntityDoesNotExist, InvalidInput, ResourceIsNotDeployed
 from db.repositories.resource_templates import ResourceTemplateRepository
 from db.repositories.resources import ResourceRepository, IS_NOT_DELETED_CLAUSE
@@ -66,17 +66,22 @@ async def get_workspace_by_id(self, workspace_id: str) -> Workspace:
         return parse_obj_as(Workspace, workspaces[0])
 
     # Remove this method once not using last 4 digits for naming - https://github.com/microsoft/AzureTRE/issues/3666
-    async def is_workspace_with_last_4_id(self, workspace_id: str) -> bool:
-        query = self.workspaces_query_string() + f' AND ENDSWITH(c.id, "{workspace_id[-4:]}")'
-        workspaces = await self.query(query=query)
-        return len(workspaces) > 0
+    async def is_worksapce_storage_account_available(self, workspace_id: str) -> bool:
+        storage_client = StorageManagementClient(credentials.get_credential(), config.SUBSCRIPTION_ID)
+        # check for storage account with last 4 digits of workspace_id
+        availability_result = storage_client.storage_accounts.check_name_availability(
+            {
+                "name": f"stgws{workspace_id[-4:]}"
+            }
+        )
+        return availability_result.name_available
 
     async def create_workspace_item(self, workspace_input: WorkspaceInCreate, auth_info: dict, workspace_owner_object_id: str, user_roles: List[str]) -> Tuple[Workspace, ResourceTemplate]:
 
         full_workspace_id = str(uuid.uuid4())
 
         # Ensure workspace with last four digits of ID does not already exist - remove when https://github.com/microsoft/AzureTRE/issues/3666 is resolved
-        while await self.is_workspace_with_last_4_id(full_workspace_id):
+        while not await self.is_worksapce_storage_account_available(full_workspace_id):
             full_workspace_id = str(uuid.uuid4())
 
         template = await self.validate_input_against_template(workspace_input.templateName, workspace_input, ResourceType.Workspace, user_roles)

api_app/requirements.txt

@@ -7,6 +7,7 @@ azure-mgmt-compute==30.3.0
 azure-mgmt-cosmosdb==9.3.0
 azure-mgmt-costmanagement==4.0.1
 azure-mgmt-resource==23.0.1
+azure-mgmt-storage==21.1.0
 azure-monitor-opentelemetry==1.2.0
 azure-servicebus==7.11.3
 azure-storage-blob==12.19.0

api_app/tests_ma/test_db/test_repositories/test_workpaces_repository.py

@@ -98,15 +98,18 @@ async def test_get_workspace_by_id_queries_db(workspace_repo, workspace):
 @pytest.mark.asyncio
 @patch('db.repositories.workspaces.generate_new_cidr')
 @patch('db.repositories.workspaces.WorkspaceRepository.validate_input_against_template')
+@patch('db.repositories.workspaces.WorkspaceRepository.is_worksapce_storage_account_available')
 @patch('core.config.RESOURCE_LOCATION', "useast2")
 @patch('core.config.TRE_ID', "9876")
-async def test_create_workspace_item_creates_a_workspace_with_the_right_values(validate_input_mock, new_cidr_mock, workspace_repo, basic_workspace_request, basic_resource_template):
+async def test_create_workspace_item_creates_a_workspace_with_the_right_values(mock_is_workspace_storage_account_available, validate_input_mock, new_cidr_mock, workspace_repo, basic_workspace_request, basic_resource_template):
     workspace_to_create = basic_workspace_request
     # make sure the input has 'None' for values that we expect to be set
     workspace_to_create.properties.pop("address_space", None)
     workspace_to_create.properties.pop("address_spaces", None)
     workspace_to_create.properties.pop("workspace_owner_object_id", None)
 
+    mock_is_workspace_storage_account_available.return_value = AsyncMock().return_value
+    mock_is_workspace_storage_account_available.return_value.return_value = False
     validate_input_mock.return_value = basic_resource_template
     new_cidr_mock.return_value = "1.2.3.4/24"
 
@@ -165,14 +168,18 @@ async def test_get_address_space_based_on_size_with_large_address_space(workspac
 
 @pytest.mark.asyncio
 @patch('db.repositories.workspaces.WorkspaceRepository.validate_input_against_template')
+@patch('db.repositories.workspaces.WorkspaceRepository.is_worksapce_storage_account_available')
 @patch('core.config.RESOURCE_LOCATION', "useast2")
 @patch('core.config.TRE_ID', "9876")
 @patch('core.config.CORE_ADDRESS_SPACE', "10.1.0.0/22")
 @patch('core.config.TRE_ADDRESS_SPACE', "10.0.0.0/12")
-async def test_create_workspace_item_creates_a_workspace_with_custom_address_space(validate_input_mock, workspace_repo, basic_workspace_request, basic_resource_template):
+async def test_create_workspace_item_creates_a_workspace_with_custom_address_space(mock_is_workspace_storage_account_available, validate_input_mock, workspace_repo, basic_workspace_request, basic_resource_template):
     workspace_to_create = basic_workspace_request
     workspace_to_create.properties["address_space_size"] = "custom"
     workspace_to_create.properties["address_space"] = "10.2.4.0/24"
+
+    mock_is_workspace_storage_account_available.return_value = AsyncMock().return_value
+    mock_is_workspace_storage_account_available.return_value.return_value = False
     validate_input_mock.return_value = basic_resource_template
 
     workspace, _ = await workspace_repo.create_workspace_item(workspace_to_create, {}, "test_object_id", ["test_role"])
@@ -182,14 +189,18 @@ async def test_create_workspace_item_creates_a_workspace_with_custom_address_spa
 
 @pytest.mark.asyncio
 @patch('db.repositories.workspaces.WorkspaceRepository.validate_input_against_template')
+@patch('db.repositories.workspaces.WorkspaceRepository.is_worksapce_storage_account_available')
 @patch('core.config.RESOURCE_LOCATION', "useast2")
 @patch('core.config.TRE_ID', "9876")
 @patch('core.config.CORE_ADDRESS_SPACE', "10.1.0.0/22")
 @patch('core.config.TRE_ADDRESS_SPACE', "10.0.0.0/12")
-async def test_create_workspace_item_throws_exception_with_bad_custom_address_space(validate_input_mock, workspace_repo, basic_workspace_request, basic_resource_template):
+async def test_create_workspace_item_throws_exception_with_bad_custom_address_space(mock_is_workspace_storage_account_available, validate_input_mock, workspace_repo, basic_workspace_request, basic_resource_template):
     workspace_to_create = basic_workspace_request
     workspace_to_create.properties["address_space_size"] = "custom"
     workspace_to_create.properties["address_space"] = "192.168.0.0/24"
+
+    mock_is_workspace_storage_account_available.return_value = AsyncMock().return_value
+    mock_is_workspace_storage_account_available.return_value.return_value = False
     validate_input_mock.return_value = basic_resource_template
 
     with pytest.raises(InvalidInput):
@@ -249,8 +260,12 @@ async def test_get_address_space_based_on_size_with_address_space_and_address_sp
 
 @pytest.mark.asyncio
 @patch('db.repositories.workspaces.WorkspaceRepository.validate_input_against_template')
-async def test_create_workspace_item_raises_value_error_if_template_is_invalid(validate_input_mock, workspace_repo, basic_workspace_request):
+@patch('db.repositories.workspaces.WorkspaceRepository.is_worksapce_storage_account_available')
+async def test_create_workspace_item_raises_value_error_if_template_is_invalid(mock_is_workspace_storage_account_available, validate_input_mock, workspace_repo, basic_workspace_request):
     workspace_input = basic_workspace_request
+
+    mock_is_workspace_storage_account_available.return_value = AsyncMock().return_value
+    mock_is_workspace_storage_account_available.return_value.return_value = False
     validate_input_mock.side_effect = ValueError
 
     with pytest.raises(ValueError):
@@ -281,3 +296,29 @@ def test_workspace_owner_is_not_overwritten_if_present_in_workspace_properties(w
     not_expected_object_id = "Not Expected"
 
     assert workspace_repo.get_workspace_owner(dictToTest, not_expected_object_id) == "Expected"
+
+
+@patch('azure.mgmt.storage.StorageManagementClient')
+async def test_is_worksapce_storage_account_available_when_name_available(mock_storage_client):
+    workspace_id = "workspace1234"
+    mock_storage_client.return_value.storage_accounts.check_name_availability.return_value = AsyncMock()
+    mock_storage_client.return_value.storage_accounts.check_name_availability.return_value.name_available = True
+    workspace_repo = WorkspaceRepository()
+
+    result = await workspace_repo.is_workspace_with_last_4_id(workspace_id)
+
+    mock_storage_client.return_value.storage_accounts.check_name_availability.assert_called_once_with({"name": f"stgws{workspace_id[-4:]}"})
+    assert result is False
+
+
+@patch('azure.mgmt.storage.StorageManagementClient')
+async def test_is_worksapce_storage_account_available_when_name_not_available(mock_storage_client):
+    workspace_id = "workspace1234"
+    mock_storage_client.return_value.storage_accounts.check_name_availability.return_value = AsyncMock()
+    mock_storage_client.return_value.storage_accounts.check_name_availability.return_value.name_available = False
+    workspace_repo = WorkspaceRepository()
+
+    result = await workspace_repo.is_workspace_with_last_4_id(workspace_id)
+
+    mock_storage_client.return_value.storage_accounts.check_name_availability.assert_called_once_with({"name": f"stgws{workspace_id[-4:]}"})
+    assert result is True

devops/scripts/load_and_validate_env.sh

@@ -85,4 +85,13 @@ else
     export TRE_URL
 fi
 
+# if local debugging is configured, then set vars required by ~/.porter/config.yaml
+if [ -f "$DIR/../../core/private.env" ]; then
+  # shellcheck disable=SC1091
+  source "$DIR/../../core/private.env"
+  # shellcheck disable=SC2153
+  KEY_VAULT_URL=$KEYVAULT_URI
+  export KEY_VAULT_URL
+fi
+
 set +o nounset

docs/tre-templates/shared-services/nexus.md

@@ -87,10 +87,12 @@ Nexus Shared Service requires access to resources outside of the Azure TRE VNET.
 | Ubuntu Security Packages | apt | [http://security.ubuntu.com/ubuntu/] | `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/repository/ubuntu-security/` | Provide access to Ubuntu Security apt packages on Ubuntu systems. |
 | Almalinux | yum | [https://repo.almalinux.org] | `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/repository/almalinux` | Install Almalinux packages |
 | R-Proxy | r | [https://cran.r-project.org/] | `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/repository/r-proxy` | Provide access to CRAN packages for R |
+| R-Studio Download | raw | [https://download1.rstudio.org] | `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/repository/r-studio-download` | Provide access to download R Studio |
 | Fedora Project | yum | [https://download-ib01.fedoraproject.org] | `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/repository/fedoraproject` | Install Fedora Project Linux packages |
 | Microsoft Apt | apt | [https://packages.microsoft.com] | `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/repository/microsoft-apt` | Provide access to Microsoft Apt packages |
 | Microsoft Keys | raw | [https://packages.microsoft.com/keys/] | `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/repository/microsoft-keys` | Provide access to Microsoft keys |
 | Microsoft Yum | yum | [https://packages.microsoft.com/yumrepos] | `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/repository/microsoft-yum` | Provide access to Microsoft Yum packages |
+| Microsoft Download | raw | [https://download.microsoft.com/download] | `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/repository/microsoft-download` | Provide access to Microsoft Downloads |
 
 ### Migrate from an existing V1 Nexus service (hosted on App Service)
 
@@ -111,3 +113,20 @@ If you still have an existing Nexus installation based on App Service (from the
 The Nexus service checks Key Vault regularly for the latest certificate matching the name you passed on deploy (`nexus-ssl` by default).
 
 When approaching expiry, you can either provide an updated certificate into the TRE core KeyVault (with the name you specified when installing Nexus) if you brought your own, or if you used the certs shared service to generate one, just call the `renew` custom action on that service. This will generate a new certificate and persist it to the Key Vault, replacing the expired one.
+
+## Updating to v3.0.0
+The newest version of Nexus is a significant update for the service.
+As a result, a new installation of Nexus will be necessary.
+
+We are currently in the process of developing an upgrade path for upcoming releases.
+
+## Using Docker Hub
+When using Docker with a VM, the image URL should be constructed as follows: {NEXUS_URL}:{port}/docker-image
+
+```bash
+sudo docker pull {NEXUS_URL}:8083/hello-world
+```
+
+the default port out of the box is 8083
+
+Nexus will also need "Anonymous Access" set to "Enable". This can be done by logging into the Nexus Portal with the Admin user and following the prompts.

docs/tre-templates/user-resources/custom.md

@@ -0,0 +1,62 @@
+# Guacamole User Resources
+
+- linuxvm - a Linux-based virtual machine
+- windowsvm - A Windows-based virtual machine
+
+## Customising the user resources
+
+The `guacamole-azure-linuxvm` and `guacamole-azure-windowsvm` folders follow a consistent layout.
+To update one of these templates (or to create a new template based on these folders) to use different image details or VM sizes, there are a few files that need to be updated:
+
+| File                   | Description                                                                                                                                                                                                                                                                        |
+| ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `porter.yaml`          | This file describes the template and the name should be updated when creating a template based on the folder.<br> This file also contains a `custom` data section that describes the VM properties.<br> Additionally, the version needs to be updated to deploy an updated version |
+| `template_schema.json` | This file controls the validation applied to the template, for example specifying the valid options for fields such as size and image                                                                                                                                              |
+
+### Configuration
+
+In `porter.yaml`, the `custom` section contains a couple of sub-sections (shown below)
+
+```yaml
+custom:
+  vm_sizes:
+    "2 CPU | 8GB RAM": Standard_D2s_v5
+    "4 CPU | 16GB RAM": Standard_D4s_v5
+    "8 CPU | 32GB RAM": Standard_D8s_v5
+    "16 CPU | 64GB RAM": Standard_D16s_v5
+  image_options:
+    "Ubuntu 22.04 LTS":
+      source_image_reference:
+        publisher: canonical
+        offer: 0001-com-ubuntu-server-jammy
+        sku: 22_04-lts-gen2
+        version: latest
+        apt_sku: 22.04
+      install_ui: true
+      conda_config: false
+    # "Custom Image From Gallery":
+    #   source_image_name: your-image
+    #   install_ui: true
+    #   conda_config: true
+```
+
+The `vm_sizes` section is a map of a custom SKU description to the SKU identifier.
+
+The `image_options` section defined the possible image choices for the template (note that the name of the image used here needs to be included in the corresponding enum in `template_schema.json`).
+
+Within the image definition in `image_options` there are a few properties that can be specified:
+
+| Name                     | Description                                                                                              |
+| ------------------------ | -------------------------------------------------------------------------------------------------------- |
+| `source_image_name`      | Specify VM image to use by name (see notes below for identifying the image gallery containing the image) |
+| `source_image_reference` | Specify VM image to use by `publisher`, `offer`, `sku` & `version` (e.g. for Azure Marketplace images)   |
+| `install_ui`             | (Linux only) Set `true` to install desktop environment                                                   |
+| `conda_config`           | Set true to configure conda                                                                              |
+
+When specifying images using `source_image_name`, the image must be stored in an [image gallery](https://learn.microsoft.com/en-us/azure/virtual-machines/azure-compute-gallery).
+To enable re-using built user resource templates across environments where the image may vary, the image gallery is configured via the `RP_BUNDLE_VALUES` environment variable when deploying the TRE.
+The `RP_BUNDLE_VALUES` variable is a JSON object, and the `image_gallery_id` property within it identifies the image gallery that contains the images specified by `source_image_name`:
+
+```bash
+RP_BUNDLE_VALUES='{"image_gallery_id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/<your-rg>/providers/Microsoft.Compute/galleries/<your-gallery-name>"}
+```

docs/tre-templates/user-resources/guacamole-linux-vm.md

@@ -7,3 +7,13 @@ It blocks all inbound and outbound traffic to the internet and allows only RDP c
 
 - [A base workspace bundle installed](../workspaces/base.md)
 - [A guacamole workspace service bundle installed](../workspace-services/guacamole.md)
+- [A Nexus shared service has been deployed](../shared-services/nexus.md)
+
+## Notes
+
+- Nexus is a prerequisite of installing the Linux VMs given the additional commands in the bootstrap scripts.
+- In production we recommend using VM images to avoid transient issues downloading and installing packages. The included user resource templates for VMs with bootstrap scripts should only be used for trial/demonstration purposes. More info can be found [here](./custom.md).
+- Snap (app store for linux via [snapcraft.io](https://snapcraft.io)) hasn't been configured to work via the nexus proxy
+
+## Using Custom Images
+For custom image usage, visit this [page](./custom.md).

e2e_tests/test_performance.py

@@ -106,7 +106,7 @@ async def test_bulk_updates_to_ensure_each_resource_updated_in_series(verify) ->
         "properties": {
             "display_name": "Perf test VM",
             "description": "",
-            "os_image": "Ubuntu 18.04"
+            "os_image": "Ubuntu 22.04 LTS"
         }
     }
 

templates/shared_services/sonatype-nexus-vm/porter.yaml

@@ -1,7 +1,7 @@
 ---
 schemaVersion: 1.0.0
 name: tre-shared-service-sonatype-nexus
-version: 2.8.13
+version: 3.0.0
 description: "A Sonatype Nexus shared service"
 dockerfile: Dockerfile.tmpl
 registry: azuretre

templates/shared_services/sonatype-nexus-vm/scripts/nexus_realms_config.json

@@ -1,5 +1,4 @@
 [
-  "NexusAuthenticatingRealm",
-  "NexusAuthorizingRealm",
-  "DockerToken"
+  "DockerToken",
+  "NexusAuthenticatingRealm"
 ]