Add firewall to Pulumi

data_safe_haven/administration/users/azure_ad_users.py

@@ -159,4 +159,4 @@ def set_user_attributes(self):
         #         self.error(
         #             f"Failed to set Linux attributes for user {user.preferred_username}.\n{str(exc)}"
         #         )
-        pass
+        pass

data_safe_haven/backend/backend.py

@@ -16,8 +16,8 @@ def __init__(
         self, settings: DotFileSettings, *args: Optional[Any], **kwargs: Optional[Any]
     ):
         super().__init__(*args, **kwargs)
-        self.azure_api_ = None
-        self.config = Config(
+        self.azure_api_: Optional[AzureApi] = None
+        self.config: Config = Config(
             name=settings.name,
             subscription_name=settings.subscription_name,
         )

data_safe_haven/commands/deploy_shm_command.py

@@ -2,6 +2,7 @@
 # Standard library imports
 import ipaddress
 import re
+from typing import cast, List
 
 # Third party imports
 import pytz
@@ -18,6 +19,7 @@
 from data_safe_haven.helpers import password
 from data_safe_haven.mixins import LoggingMixin
 from data_safe_haven.pulumi import PulumiStack
+from data_safe_haven.provisioning import SHMProvisioningManager
 
 
 class DeploySHMCommand(LoggingMixin, Command):
@@ -98,13 +100,17 @@ def handle(self) -> None:
                 stack.secret("password-domain-ldap-searcher"),
             )
             config.add_secret(
-                config.shm.domain_controllers["password_domain_admin"],
+                config.shm.domain_controllers["domain_admin_password_secret"],
                 stack.secret("password-domain-admin"),
             )
 
             # Upload config to blob storage
             config.upload()
 
+            # Provision SHM with anything that could not be done in Pulumi
+            manager = SHMProvisioningManager(config)
+            manager.run()
+
         except DataSafeHavenException as exc:
             error_msg = (
                 f"Could not deploy Data Safe Haven Management environment.\n{str(exc)}"
@@ -115,7 +121,7 @@ def handle(self) -> None:
     def add_missing_values(self, config: Config) -> None:
         """Request any missing config values and add them to the config"""
         # Request FQDN if not provided
-        fqdn = self.option("fqdn")
+        fqdn = cast(str, self.option("fqdn"))
         while not config.shm.fqdn:
             if fqdn:
                 config.shm.fqdn = fqdn
@@ -125,7 +131,7 @@ def add_missing_values(self, config: Config) -> None:
                 )
 
         # Request admin IP addresses if not provided
-        aad_tenant_id = self.option("aad-tenant-id")
+        aad_tenant_id = cast(str, self.option("aad-tenant-id"))
         while not config.shm.aad_tenant_id:
             if aad_tenant_id and re.match(
                 r"^[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}$",
@@ -139,7 +145,7 @@ def add_missing_values(self, config: Config) -> None:
                 aad_tenant_id = self.log_ask("AzureAD tenant ID:", None)
 
         # Request admin email address if not provided
-        admin_email_address = self.option("email")
+        admin_email_address = cast(str, self.option("email"))
         while not config.shm.admin_email_address:
             if not admin_email_address:
                 self.info(
@@ -154,7 +160,7 @@ def add_missing_values(self, config: Config) -> None:
             admin_email_address = None
 
         # Request admin IP addresses if not provided
-        admin_ip_addresses = " ".join(self.option("ip-address"))
+        admin_ip_addresses = " ".join(cast(List[str], self.option("ip-address")))
         while not config.shm.admin_ip_addresses:
             if not admin_ip_addresses:
                 self.info(
@@ -174,7 +180,7 @@ def add_missing_values(self, config: Config) -> None:
             admin_ip_addresses = None
 
         # Request timezone if not provided
-        timezone = self.option("timezone")
+        timezone = cast(str, self.option("timezone"))
         while not config.shm.timezone:
             if timezone in pytz.all_timezones:
                 config.shm.timezone = timezone

data_safe_haven/commands/deploy_sre_command.py

@@ -2,6 +2,7 @@
 # Third party imports
 import yaml
 from cleo import Command
+from typing import cast, List
 
 # Local imports
 from data_safe_haven.config import Config, DotFileSettings
@@ -47,7 +48,7 @@ def handle(self) -> None:
             config = Config(settings.name, settings.subscription_name)
 
             # Set a JSON-safe name for this SRE and add any missing values to the config
-            self.sre_name = alphanumeric(self.argument("name"))
+            self.sre_name = alphanumeric(cast(str, self.argument("name")))
             self.add_missing_values(config)
 
             # Load GraphAPI as this may require user-interaction that is not possible as part of a Pulumi declarative command
@@ -88,7 +89,9 @@ def handle(self) -> None:
             config.sre[self.sre_name].remote_desktop[
                 "connection_db_server_admin_password_secret"
             ] = f"password-user-database-admin-sre-{self.sre_name}"
-            for (vm_name, _, vm_ipaddress) in stack.output("srd"):
+            for (vm_name, vm_ipaddress) in zip(
+                stack.output("srd")["vm_names"], stack.output("srd")["vm_ip_addresses"]
+            ):
                 config.sre[self.sre_name].research_desktops[
                     vm_name
                 ].ip_address = vm_ipaddress
@@ -141,7 +144,7 @@ def add_missing_values(self, config: Config) -> None:
         available_vm_skus = azure_api.list_available_vm_skus(config.azure.location)
         vm_skus = [
             sku
-            for sku in self.option("research-desktop")
+            for sku in cast(List[str], self.option("research-desktop"))
             if sku in available_vm_skus
         ]
         while not vm_skus:
@@ -154,8 +157,8 @@ def add_missing_values(self, config: Config) -> None:
             vm_skus = [sku for sku in answer if sku in available_vm_skus]
         if hasattr(config.sre[self.sre_name], "research_desktops"):
             del config.sre[self.sre_name].research_desktops
+        idx_cpu, idx_gpu = 0, 0
         for vm_sku in vm_skus:
-            idx_cpu, idx_gpu = 0, 0
             if int(available_vm_skus[vm_sku]["GPUs"]) > 0:
                 vm_cfg = config.sre[self.sre_name].research_desktops[
                     f"srd-gpu-{idx_gpu:02d}"

data_safe_haven/commands/teardown_backend_command.py

@@ -17,6 +17,7 @@ class TeardownBackendCommand(LoggingMixin, Command):
     Teardown a deployed Data Safe Haven backend using local configuration files
 
     backend
+        {--o|output= : Path to an output log file}
     """
 
     def handle(self):

data_safe_haven/commands/teardown_sre_command.py

@@ -1,6 +1,7 @@
 """Command-line application for tearing down a Secure Research Environment"""
 # Third party imports
 from cleo import Command
+from typing import cast
 
 # Local imports
 from data_safe_haven.config import Config, DotFileSettings
@@ -23,6 +24,7 @@ class TeardownSRECommand(LoggingMixin, Command):
 
     def handle(self):
         try:
+            environment_name = "UNKNOWN"
             # Set up logging for anything called by this command
             self.initialise_logging(self.io.verbosity, self.option("output"))
 
@@ -34,14 +36,18 @@ def handle(self):
                     f"Unable to load project settings. Please run this command from inside the project directory.\n{str(exc)}"
                 ) from exc
             config = Config(settings.name, settings.subscription_name)
+            environment_name = config.name
+
             # Remove infrastructure deployed with Pulumi
             try:
-                stack = PulumiStack(config, "SRE", sre_name=self.argument("name"))
+                stack = PulumiStack(
+                    config, "SRE", sre_name=cast(str, self.argument("name"))
+                )
                 if stack.work_dir.exists():
                     stack.teardown()
                 else:
                     raise DataSafeHavenInputException(
-                        f"SRE {self.argument('name')} not found - check the name is spelt correctly." 
+                        f"SRE {self.argument('name')} not found - check the name is spelt correctly."
                     )
             except Exception as exc:
                 raise DataSafeHavenInputException(
@@ -59,7 +65,7 @@ def handle(self):
         except DataSafeHavenException as exc:
             for (
                 line
-            ) in f"Could not teardown Data Safe Haven '{config.environment_name}'.\n{str(exc)}".split(
+            ) in f"Could not teardown Data Safe Haven '{environment_name}'.\n{str(exc)}".split(
                 "\n"
             ):
                 self.error(line)

data_safe_haven/commands/users_add.py

@@ -1,6 +1,7 @@
 """Command-line application for initialising a Data Safe Haven deployment"""
 # Third party imports
 from cleo import Command
+from typing import cast
 
 # Local imports
 from data_safe_haven.administration.users import UserHandler
@@ -24,6 +25,7 @@ class UsersAddCommand(LoggingMixin, Command):
 
     def handle(self):
         try:
+            environment_name = "UNKNOWN"
             # Set up logging for anything called by this command
             self.initialise_logging(self.io.verbosity, self.option("output"))
 
@@ -35,6 +37,7 @@ def handle(self):
                     f"Unable to load project settings. Please run this command from inside the project directory.\n{str(exc)}"
                 ) from exc
             config = Config(settings.name, settings.subscription_name)
+            environment_name = config.name
 
             # Load GraphAPI as this may require user-interaction that is not possible as part of a Pulumi declarative command
             graph_api = GraphApi(
@@ -44,11 +47,11 @@ def handle(self):
 
             # Add users to SHM
             users = UserHandler(config, graph_api)
-            users.add(self.argument("csv"))
+            users.add(cast(str, self.argument("csv")))
         except DataSafeHavenException as exc:
             for (
                 line
-            ) in f"Could not add users to Data Safe Haven '{config.name}'.\n{str(exc)}".split(
+            ) in f"Could not add users to Data Safe Haven '{environment_name}'.\n{str(exc)}".split(
                 "\n"
             ):
                 self.error(line)

data_safe_haven/config/config.py

@@ -36,7 +36,7 @@ def __init__(
         self.shm_name = alphanumeric(self.name).lower()
 
         # Construct backend storage variables
-        backend_resource_group_name = f"rg-shm-{self.shm_name}-backend"
+        backend_resource_group_name = f"shm-{self.shm_name}-rg-backend"
         backend_storage_account_name = (
             f"shm{self.shm_name[:12]}backend"  # maximum of 24 characters allowed
         )
@@ -67,15 +67,17 @@ def __init__(
         if isinstance(self.tags.version, dotmap.DotMap):
             self.tags.version = __version__
         if isinstance(self.backend.key_vault_name, dotmap.DotMap):
-            self.backend.key_vault_name = f"kv-{self.shm_name[:13]}-backend"
+            self.backend.key_vault_name = f"shm-{self.shm_name[:9]}-kv-backend"
         if isinstance(self.backend.resource_group_name, dotmap.DotMap):
             self.backend.resource_group_name = backend_resource_group_name
         if isinstance(self.backend.storage_account_name, dotmap.DotMap):
             self.backend.storage_account_name = backend_storage_account_name
         if isinstance(self.backend.storage_container_name, dotmap.DotMap):
             self.backend.storage_container_name = backend_storage_container_name
         if isinstance(self.backend.managed_identity_name, dotmap.DotMap):
-            self.backend.managed_identity_name = "KeyVaultReaderIdentity"
+            self.backend.managed_identity_name = (
+                f"shm-{self.shm_name}-identity-reader-backend"
+            )
         if isinstance(self.backend.pulumi_encryption_key_name, dotmap.DotMap):
             self.backend.pulumi_encryption_key_name = "pulumi-encryption-key"
         if isinstance(self.pulumi.storage_container_name, dotmap.DotMap):

data_safe_haven/config/dotfilesettings.py

@@ -1,6 +1,7 @@
 """Load global and local settings from dotfiles"""
 # Standard library imports
 import pathlib
+from typing import cast, Dict, Optional
 
 # Third party imports
 import yaml
@@ -21,39 +22,28 @@ class DotFileSettings(LoggingMixin):
       name: Turing Development
     """
 
-    admin_group_id: str = None
-    location: str = None
-    name: str = None
-    subscription_name: str = None
-    config_file_name = ".dshconfig"
+    admin_group_id: str
+    location: str
+    name: str
+    subscription_name: str
+    config_file_name: str = ".dshconfig"
 
     def __init__(
         self,
-        admin_group_id: str = None,
-        location: str = None,
-        name: str = None,
-        subscription_name: str = None,
+        admin_group_id: Optional[str] = None,
+        location: Optional[str] = None,
+        name: Optional[str] = None,
+        subscription_name: Optional[str] = None,
     ):
         super().__init__()
+        # Load local dotfile settings (if any)
+        local_dotfile = pathlib.Path.cwd() / self.config_file_name
         try:
-            # Load local dotfile settings (if any)
-            local_dotfile = pathlib.Path.cwd() / self.config_file_name
             if local_dotfile.exists():
-                with open(pathlib.Path(local_dotfile), "r", encoding="utf-8") as f_yaml:
-                    settings = yaml.safe_load(f_yaml)
-                    self.admin_group_id = settings.get("azure", {}).get(
-                        "admin_group_id", self.admin_group_id
-                    )
-                    self.location = settings.get("azure", {}).get(
-                        "location", self.location
-                    )
-                    self.name = settings.get("shm", {}).get("name", self.name)
-                    self.subscription_name = settings.get("azure", {}).get(
-                        "subscription_name", self.subscription_name
-                    )
+                self.read(local_dotfile)
         except Exception as exc:
             raise DataSafeHavenInputException(
-                f"Could not load settings from YAML file '{local_dotfile}'"
+                f"Could not load settings from YAML file '{local_dotfile}'.\n{str(exc)}"
             ) from exc
 
         # Override with command-line settings (if any)
@@ -86,7 +76,23 @@ def __init__(
                 None,
             )
 
-    def write(self, directory: pathlib.Path) -> None:
+    def read(self, yaml_file: pathlib.Path) -> None:
+        """Read settings from YAML file"""
+        with open(pathlib.Path(yaml_file), "r", encoding="utf-8") as f_yaml:
+            settings = cast(Dict[str, Dict[str, str]], yaml.safe_load(f_yaml))
+        if admin_group_id := settings.get("azure", {}).get("admin_group_id", None):
+            self.admin_group_id = admin_group_id
+        if location := settings.get("azure", {}).get("location", None):
+            self.location = location
+        if name := settings.get("shm", {}).get("name", None):
+            self.name = name
+        if subscription_name := settings.get("azure", {}).get(
+            "subscription_name", None
+        ):
+            self.subscription_name = subscription_name
+
+    def write(self, directory: pathlib.Path) -> pathlib.Path:
+        """Write settings to YAML file"""
         settings = {
             "shm": {
                 "name": self.name,
@@ -98,6 +104,6 @@ def write(self, directory: pathlib.Path) -> None:
             },
         }
         filepath = (directory / self.config_file_name).resolve()
-        with open(filepath, "w", encoding="utf-8") as f_settings:
-            yaml.dump(settings, f_settings, indent=2)
+        with open(filepath, "w", encoding="utf-8") as f_yaml:
+            yaml.dump(settings, f_yaml, indent=2)
         return filepath

data_safe_haven/exceptions/__init__.py

@@ -14,6 +14,10 @@ class DataSafeHavenInternalException(DataSafeHavenException):
     pass
 
 
+class DataSafeHavenIPRangeException(DataSafeHavenException):
+    pass
+
+
 class DataSafeHavenNotImplementedException(DataSafeHavenInternalException):
     pass