From a279d7b6110930966d17c28522196d1e3fed1670 Mon Sep 17 00:00:00 2001 From: Eu Jing Chua Date: Wed, 9 Aug 2023 16:14:11 -0700 Subject: [PATCH 01/11] Initial scripts for automating rg setup --- .../rg-template.example.parameters.json | 30 ++ scripts/setup-rg/rg-template.json | 376 ++++++++++++++++++ scripts/setup-rg/setup-rg.ps1 | 89 +++++ 3 files changed, 495 insertions(+) create mode 100644 scripts/setup-rg/rg-template.example.parameters.json create mode 100644 scripts/setup-rg/rg-template.json create mode 100644 scripts/setup-rg/setup-rg.ps1 diff --git a/scripts/setup-rg/rg-template.example.parameters.json b/scripts/setup-rg/rg-template.example.parameters.json new file mode 100644 index 00000000000..b2059c701d5 --- /dev/null +++ b/scripts/setup-rg/rg-template.example.parameters.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "projectPrefix": { + "value": "mlos-mysql-autotune" + }, + "vmSKU": { + "value": "Standard_D2s_v3" + }, + "vmAdminUsername": { + "value": "PLACEHOLDER" + }, + "sshPublicKey": { + "value": "PLACEHOLDER" + }, + "fileShareName": { + "value": "mlos-file-share" + }, + "resultsDbName": { + "value": "mlos-mysql-autotune" + }, + "resultsDbAdminUsername": { + "value": "mlos" + }, + "resultsDbAdminPassword": { + "value": "PLACEHOLDER" + } + } +} diff --git a/scripts/setup-rg/rg-template.json b/scripts/setup-rg/rg-template.json new file mode 100644 index 00000000000..e984cd86849 --- /dev/null +++ b/scripts/setup-rg/rg-template.json @@ -0,0 +1,376 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "projectPrefix": { + "type": "string", + "metadata": { + "description": "Prefix used for naming resources." + }, + "defaultValue": "mlos" + }, + "vmSKU": { + "type": "string", + "defaultValue": "Standard_D2s_v3", + "metadata": { + "description": "VM SKU" + } + }, + "vmAdminUsername": { + "type": "string" + }, + "sshPublicKey": { + "type": "string" + }, + "fileShareName": { + "type": "string", + "defaultValue": "mlos-file-share" + }, + "resultsDbName": { + "type": "string" + }, + "resultsDbAdminUsername": { + "type": "string", + "defaultValue": "mlos" + }, + "resultsDbAdminPassword": { + "type": "securestring" + } + }, + "functions": [], + "variables": { + "storageAccountName": "[toLower(concat(replace(parameters('projectPrefix'), '-', ''), 'storage'))]", + "vmName": "[concat(parameters('projectPrefix'), '-vm')]", + "networkInterfaceName": "[concat(variables('vmName'), '-ni-', uniqueString(resourceGroup().id))]", + "vnetName": "[concat(variables('vmName'), '-vnet')]", + "publicIpName": "[concat(variables('vmName'), '-ip')]", + "nsgName": "[concat(variables('vmName'), '-nsg')]", + "kvName": "[concat(parameters('projectPrefix'), '-kv')]" + }, + "resources": [ + { + "name": "[variables('publicIpName')]", + "type": "Microsoft.Network/publicIPAddresses", + "apiVersion": "2020-11-01", + "location": "[resourceGroup().location]", + "tags": { + "displayName": "PublicIPAddress" + }, + "properties": { + "publicIPAllocationMethod": "Static", + "dnsSettings": { + "domainNameLabel": "[variables('vmName')]" + } + } + }, + { + "name": "[variables('nsgName')]", + "type": "Microsoft.Network/networkSecurityGroups", + "apiVersion": "2020-11-01", + "location": "[resourceGroup().location]", + "properties": { + "securityRules": [ + { + "name": "CorpSSH", + "properties": { + "description": "Corp. IP SSH access", + "protocol": "Tcp", + "sourcePortRange": "*", + "destinationPortRange": "22", + "sourceAddressPrefix": "131.107.0.0/16", + "destinationAddressPrefix": "*", + "access": "Allow", + "priority": 100, + "direction": "Inbound" + } + } + ] + } + }, + { + "name": "[variables('vnetName')]", + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2020-11-01", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', variables('nsgName'))]" + ], + "tags": { + "displayName": "[variables('vnetName')]" + }, + "properties": { + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/16" + ] + }, + "subnets": [ + { + "name": "default", + "properties": { + "addressPrefix": "10.0.0.0/24", + "serviceEndpoints": [ + { + "service": "Microsoft.Storage", + "locations": ["[resourceGroup().location]"] + }, + { + "service": "Microsoft.KeyVault", + "locations": ["[resourceGroup().location]"] + } + ], + "networkSecurityGroup": { + "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('nsgName'))]" + } + } + } + ] + } + }, + { + "name": "[variables('networkInterfaceName')]", + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2020-11-01", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIpName'))]", + "[resourceId('Microsoft.Network/virtualNetworks', variables('vnetName'))]", + "[resourceId('Microsoft.Network/networkSecurityGroups', variables('nsgName'))]" + ], + "tags": { + "displayName": "ubuntuVM1-NetworkInterface" + }, + "properties": { + "ipConfigurations": [ + { + "name": "ipConfig1", + "properties": { + "privateIPAllocationMethod": "Dynamic", + "publicIPAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIpName'))]" + }, + "subnet": { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('vnetName'), 'default')]" + } + } + } + ], + "networkSecurityGroup": { + "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('nsgName'))]" + } + } + }, + { + "name": "[variables('kvName')]", + "type": "Microsoft.KeyVault/vaults", + "apiVersion": "2019-09-01", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Network/virtualNetworks', variables('vnetName'))]" + ], + "tags": { + "displayName": "[variables('kvName')]" + }, + "properties": { + "enabledForDeployment": true, + "enabledForTemplateDeployment": true, + "enabledForDiskEncryption": true, + "tenantId": "72f988bf-86f1-41af-91ab-2d7cd011db47", + "accessPolicies": [], + "enableRbacAuthorization": true, + "networkAcls": { + "virtualNetworkRules": [ + { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('vnetName'), 'default')]", + "ignoreMissingVnetServiceEndpoint": false + } + ] + }, + "sku": { + "name": "standard", + "family": "A" + } + } + }, + { + "name": "[variables('vmName')]", + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2021-03-01", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]" + ], + "tags": { + "displayName": "[variables('vmName')]" + }, + "properties": { + "hardwareProfile": { + "vmSize": "[parameters('vmSKU')]" + }, + "osProfile": { + "computerName": "[variables('vmName')]", + "adminUsername": "[parameters('vmAdminUsername')]", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "ssh": { + "publicKeys": [ + { + "path": "[concat('/home/', parameters('vmAdminUsername'), '/.ssh/authorized_keys')]", + "keyData": "[parameters('sshPublicKey')]" + } + ] + } + } + }, + "storageProfile": { + "imageReference": { + "publisher": "Canonical", + "offer": "0001-com-ubuntu-server-jammy", + "sku": "22_04-lts-gen2", + "version": "latest" + }, + "osDisk": { + "name": "[concat(variables('vmName'), '-OsDisk')]", + "caching": "ReadWrite", + "createOption": "FromImage" + } + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]" + } + ] + }, + "diagnosticsProfile": { + "bootDiagnostics": { + "enabled": false + } + } + } + }, + { + "name": "[variables('storageAccountName')]", + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2021-04-01", + "tags": { + "displayName": "[variables('storageAccountName')]" + }, + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Network/virtualNetworks', variables('vnetName'))]" + ], + "kind": "StorageV2", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "virtualNetworkRules": [ + { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('vnetName'), 'default')]", + "action": "Allow" + } + ], + "ipRules": [], + "defaultAction": "Deny" + } + } + }, + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2022-09-01", + "name": "[concat(variables('storageAccountName'), '/default/', parameters('fileShareName'))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]" + ], + "properties": { + "accessTier": "TransactionOptimized", + "shareQuota": 5120, + "enabledProtocols": "SMB" + } + }, + { + "type": "Microsoft.DBforMySQL/flexibleServers", + "apiVersion": "2021-12-01-preview", + "name": "[parameters('resultsDbName')]", + "location": "[resourceGroup().location]", + "sku": { + "name": "Standard_B1s", + "tier": "Burstable" + }, + "properties": { + "administratorLogin": "[parameters('resultsDbAdminUsername')]", + "administratorLoginPassword": "[parameters('resultsDbAdminPassword')]", + "storage": { + "storageSizeGB": 20, + "iops": 360, + "autoGrow": "Enabled", + "autoIoScaling": "Disabled", + "logOnDisk": "Disabled" + }, + "version": "8.0.21", + "network": { + "publicNetworkAccess": "Enabled" + }, + "backup": { + "backupRetentionDays": 7, + "geoRedundantBackup": "Disabled" + }, + "highAvailability": { + "mode": "Disabled" + } + } + }, + { + "type": "Microsoft.DBforMySQL/flexibleServers/databases", + "apiVersion": "2021-12-01-preview", + "name": "[concat(parameters('resultsDbName'), '/mlos')]", + "dependsOn": [ + "[resourceId('Microsoft.DBforMySQL/flexibleServers', parameters('resultsDbName'))]" + ], + "properties": { + "charset": "utf8", + "collation": "utf8_general_ci" + } + }, + { + "type": "Microsoft.DBforMySQL/flexibleServers/firewallRules", + "apiVersion": "2021-12-01-preview", + "name": "[concat(parameters('resultsDbName'), '/CorpIps')]", + "dependsOn": [ + "[resourceId('Microsoft.DBforMySQL/flexibleServers', parameters('resultsDbName'))]" + ], + "properties": { + "startIpAddress": "131.107.0.0", + "endIpAddress": "131.107.255.255" + } + }, + { + "type": "Microsoft.DBforMySQL/flexibleServers/firewallRules", + "apiVersion": "2021-12-01-preview", + "name": "[concat(parameters('resultsDbName'), '/AllowVM-', variables('vmName'))]", + "dependsOn": [ + "[resourceId('Microsoft.DBforMySQL/flexibleServers', parameters('resultsDbName'))]", + "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIpName'))]", + "[resourceId('Microsoft.Compute/virtualMachines', variables('vmName'))]" + ], + "properties": { + "startIpAddress": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('publicIpName'))).ipAddress]", + "endIpAddress": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('publicIpName'))).ipAddress]" + } + } + ], + "outputs": { + "vmHostName": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('publicIpName'))).dnsSettings.fqdn]" + }, + "kvName": { + "type": "string", + "value": "[variables('kvName')]" + } + } +} diff --git a/scripts/setup-rg/setup-rg.ps1 b/scripts/setup-rg/setup-rg.ps1 new file mode 100644 index 00000000000..d0a6e0c3d26 --- /dev/null +++ b/scripts/setup-rg/setup-rg.ps1 @@ -0,0 +1,89 @@ + +param( + # ARM template params + [Parameter(Mandatory=$True)] + [string] $armParameters, + # Other params + [Parameter(Mandatory=$True)] + [string] $servicePrincipalName, + [Parameter(Mandatory=$True)] + [string] $resourceGroupName, + [Parameter(Mandatory=$True)] + [string] $certName +) + +$currentUserAlias = az account show --query user.name --output tsv + +$servicePrincipalId = az ad sp list ` + --display-name $servicePrincipalName ` + --query '[].id' ` + --output tsv + +# Assign 'Contributor' access to Service Principal +az role assignment create ` + --assignee $servicePrincipalId ` + --role "Contributor" ` + --resource-group $resourceGroupName + +# Provision resources into the resource group with ARM template +$deploymentResults = az deployment group create ` + --resource-group $resourceGroupName ` + --template-file .\rg-template.json ` + --parameters $armParameters + | ConvertFrom-JSON + +if (!$?) { + Write-Error "Error in provisioning resources!" + return +} + +# Assign 'Key Vault Administrator' access to current user +$kvName = $deploymentResults.properties.outputs.kvName.value +$kvId = az keyvault show --name $kvName --query "id" --output tsv +az role assignment create ` + --assignee $currentUserAlias ` + --role "Key Vault Administrator" ` + --scope $kvId + +# Generate certificate if doesnt exist +$certThumprint = az keyvault certificate show ` + --name $certName ` + --vault-name $kvName ` + --query "x509ThumbprintHex" --output tsv +if (!$?) { + Write-Output "Generating certificate in the key vault..." + az keyvault certificate get-default-policy | Out-File -Encoding utf8 defaultCertPolicy.json + az keyvault certificate create ` + --vault-name $kvName ` + --name $certName ` + --policy `@defaultCertPolicy.json + $certThumprint = az keyvault certificate show ` + --name $certName ` + --vault-name $kvName ` + --query "x509ThumbprintHex" --output tsv +} else { + Write-Output "Certificate exists in the key vault already." +} + +# Upload certificate to service principal if doesnt exist +$spCertThumbprints = az ad sp credential list ` + --id $servicePrincipalId ` + --cert ` + --query "[].customKeyIdentifier" --output tsv +if (!$spCertThumbprints.Contains($certThumprint)) { + # Download the public portion of the certificate from key vault + Write-Output "Downloading certificate from key vault..." + az keyvault certificate download ` + --vault-name $kvName ` + --name $certName ` + --file "$certName.pem" + + # Upload the certificate to the service principal + Write-Output "Uploading certificate to service principal..." + az ad sp credential reset ` + --id $servicePrincipalId ` + --cert "$certName.pem" ` + --append +} else { + Write-Output "Certificate uploaded to the Service Principal already." +} From 8d255ceac0c2b88ce3c216f8fb852312a59f03eb Mon Sep 17 00:00:00 2001 From: Eu Jing Chua Date: Wed, 9 Aug 2023 16:40:06 -0700 Subject: [PATCH 02/11] Add README --- scripts/setup-rg/README.md | 66 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 scripts/setup-rg/README.md diff --git a/scripts/setup-rg/README.md b/scripts/setup-rg/README.md new file mode 100644 index 00000000000..f7c14c768b8 --- /dev/null +++ b/scripts/setup-rg/README.md @@ -0,0 +1,66 @@ +# Resource Group Setup + +These scripts are helpers for setting up a new resource group (RG) as the control plane for MLOS. + +## Quickstart + +1. Starting in this current directory, ensure that we are logged in to Azure CLI. + + ```sh + az login + ``` + +2. Make a copy of the ARM parameters file. + + ```sh + cp rg-template.example.parameters.json rg-template..parameters.json + ``` + +3. Modify the ARM parameters in the newly created file as needed, especially the `PLACEHOLDER` values. + +4. Execute the main script and follow prompts: + + ```sh + ./setup-rg.ps1 + Supply values for the following parameters: + armParameters: rg-template..parameters.json + servicePrincipalName: + resourceGroupName: + certName: + ``` + +## Manual + +Parameters for script can also passed in manually (without prompt) as follows: + +```sh +./setup-rg.ps1 ` + -armParameters $armParams ` + -servicePrincipalName $servicePrincipalName ` + -resourceGroupName $resourceGroupName ` + -certName $certName +``` + +where `$armParams` follows the same usage as `--parameters` in [az deployment group create](https://learn.microsoft.com/en-us/cli/azure/deployment/group?view=azure-cli-latest#az-deployment-group-create-examples). + +## Workflow + +The high-level flow for what this script automates is as follows: + +1. Assign `Contributor` access to the Service Principal (SP) for write access over resources. + For now we assume the experiment's resources are provisioned in the same RG as the control plane, so the access is granted to the same RG. + +2. Provision control plane resources into the RG. + This includes: + - Networking (public IP, security group, vnet, subnet, network interface) + - Key Vault + - Storage (storage account, file share, MySQL Flex server) + - VM + +3. Assign `Key Vault Administrator` access to the current user. + This allows the current user to retrieve secrets / certificates from the VM once it is set up. + Ensure to log in as the same user in the VM. + +4. If not existing yet, generate a certificate with `certName` in the key vault. + +5. If not associated yet, upload the certificate from before to the SP. From 202645abf5d1a25f9215f9ddcb780d739dbde952 Mon Sep 17 00:00:00 2001 From: Eu Jing Chua Date: Fri, 25 Aug 2023 14:49:21 -0700 Subject: [PATCH 03/11] Initial address of PR comments --- scripts/setup-rg/README.md | 43 ++--- .../setup-rg/results-db/mysql-template.json | 74 ++++++++ .../mysql-template.parameters.example.json | 15 ++ .../rg-template.example.parameters.json | 18 +- scripts/setup-rg/rg-template.json | 174 ++++++++---------- scripts/setup-rg/setup-rg.ps1 | 134 +++++++++----- 6 files changed, 267 insertions(+), 191 deletions(-) create mode 100644 scripts/setup-rg/results-db/mysql-template.json create mode 100644 scripts/setup-rg/results-db/mysql-template.parameters.example.json diff --git a/scripts/setup-rg/README.md b/scripts/setup-rg/README.md index f7c14c768b8..ead8714bbf3 100644 --- a/scripts/setup-rg/README.md +++ b/scripts/setup-rg/README.md @@ -1,6 +1,7 @@ # Resource Group Setup -These scripts are helpers for setting up a new resource group (RG) as the control plane for MLOS. +These scripts are helpers for setting up a new resource group (RG) in Azure as the *control plane* for MLOS. +The *control plane RG* is a container for the *persistent* resources of MLOS (results/metrics storage, scheduler VM, notebook interface, etc.). ## Quickstart @@ -13,49 +14,37 @@ These scripts are helpers for setting up a new resource group (RG) as the contro 2. Make a copy of the ARM parameters file. ```sh - cp rg-template.example.parameters.json rg-template..parameters.json + cp rg-template.example.parameters.json rg-template.parameters.json ``` 3. Modify the ARM parameters in the newly created file as needed, especially the `PLACEHOLDER` values. -4. Execute the main script and follow prompts: +4. Execute the main script with CLI args as follows: - ```sh - ./setup-rg.ps1 - Supply values for the following parameters: - armParameters: rg-template..parameters.json - servicePrincipalName: - resourceGroupName: - certName: + ```shell + ./setup-rg.ps1 ` + -armParameters $armParams ` + -servicePrincipalName $servicePrincipalName ` + -resourceGroupName $resourceGroupName ` + -certName $certName ``` -## Manual - -Parameters for script can also passed in manually (without prompt) as follows: - -```sh -./setup-rg.ps1 ` - -armParameters $armParams ` - -servicePrincipalName $servicePrincipalName ` - -resourceGroupName $resourceGroupName ` - -certName $certName -``` - -where `$armParams` follows the same usage as `--parameters` in [az deployment group create](https://learn.microsoft.com/en-us/cli/azure/deployment/group?view=azure-cli-latest#az-deployment-group-create-examples). + where `$armParams` can be `rg-template.parameters.json` from before. However, it also follows the same usage as `--parameters` in [az deployment group create](https://learn.microsoft.com/en-us/cli/azure/deployment/group?view=azure-cli-latest#az-deployment-group-create-examples). ## Workflow The high-level flow for what this script automates is as follows: 1. Assign `Contributor` access to the Service Principal (SP) for write access over resources. - For now we assume the experiment's resources are provisioned in the same RG as the control plane, so the access is granted to the same RG. + Ideally, experiment resources are placed in their own RG. + When that isn't possible, they can also be placed in the control plane RG, in which case the SP can optionally be given access to the control plane RG as well. 2. Provision control plane resources into the RG. This includes: - Networking (public IP, security group, vnet, subnet, network interface) - - Key Vault - - Storage (storage account, file share, MySQL Flex server) - - VM + - Key Vault for storing the SP credentials. + - Storage (storage account, file share, DB) + - VM for running the `mlos_bench` scheduler. 3. Assign `Key Vault Administrator` access to the current user. This allows the current user to retrieve secrets / certificates from the VM once it is set up. diff --git a/scripts/setup-rg/results-db/mysql-template.json b/scripts/setup-rg/results-db/mysql-template.json new file mode 100644 index 00000000000..9c8f2962bff --- /dev/null +++ b/scripts/setup-rg/results-db/mysql-template.json @@ -0,0 +1,74 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "dbName": { + "type": "string" + }, + "dbLocation": { + "type": "string", + "defaultValue": "[resourceGroup().location]" + }, + "dbAdminUsername": { + "type": "string", + "defaultValue": "mlos" + }, + "dbAdminPassword": { + "type": "securestring" + } + }, + "functions": [], + "variables": {}, + "resources": [ + { + "type": "Microsoft.DBforMySQL/flexibleServers", + "apiVersion": "2021-12-01-preview", + "name": "[parameters('dbName')]", + "location": "[parameters('dbLocation')]", + "sku": { + "name": "Standard_B1s", + "tier": "Burstable" + }, + "properties": { + "administratorLogin": "[parameters('dbAdminUsername')]", + "administratorLoginPassword": "[parameters('dbAdminPassword')]", + "storage": { + "storageSizeGB": 20, + "iops": 360, + "autoGrow": "Enabled", + "autoIoScaling": "Disabled", + "logOnDisk": "Disabled" + }, + "version": "8.0.21", + "network": { + "publicNetworkAccess": "Enabled" + }, + "backup": { + "backupRetentionDays": 7, + "geoRedundantBackup": "Disabled" + }, + "highAvailability": { + "mode": "Disabled" + } + } + }, + { + "type": "Microsoft.DBforMySQL/flexibleServers/databases", + "apiVersion": "2021-12-01-preview", + "name": "[concat(parameters('dbName'), '/mlos')]", + "dependsOn": [ + "[resourceId('Microsoft.DBforMySQL/flexibleServers', parameters('dbName'))]" + ], + "properties": { + "charset": "utf8", + "collation": "utf8_general_ci" + } + } + ], + "outputs": { + "dbName": { + "type": "string", + "value": "[parameters('dbName')]" + } + } +} diff --git a/scripts/setup-rg/results-db/mysql-template.parameters.example.json b/scripts/setup-rg/results-db/mysql-template.parameters.example.json new file mode 100644 index 00000000000..e271c9d7774 --- /dev/null +++ b/scripts/setup-rg/results-db/mysql-template.parameters.example.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "dbName": { + "value": "mlos-autotune-db" + }, + "dbAdminUsername": { + "value": "mlos" + }, + "dbAdminPassword": { + "value": "PLACEHOLDER" + } + } +} diff --git a/scripts/setup-rg/rg-template.example.parameters.json b/scripts/setup-rg/rg-template.example.parameters.json index b2059c701d5..fc8c643b57c 100644 --- a/scripts/setup-rg/rg-template.example.parameters.json +++ b/scripts/setup-rg/rg-template.example.parameters.json @@ -3,7 +3,7 @@ "contentVersion": "1.0.0.0", "parameters": { "projectPrefix": { - "value": "mlos-mysql-autotune" + "value": "mlos-autotune" }, "vmSKU": { "value": "Standard_D2s_v3" @@ -11,20 +11,16 @@ "vmAdminUsername": { "value": "PLACEHOLDER" }, - "sshPublicKey": { + "sshPublicKeys": { + "value": [ + "PLACEHOLDER" + ] + }, + "vmSshSourceAddressPrefix": { "value": "PLACEHOLDER" }, "fileShareName": { "value": "mlos-file-share" - }, - "resultsDbName": { - "value": "mlos-mysql-autotune" - }, - "resultsDbAdminUsername": { - "value": "mlos" - }, - "resultsDbAdminPassword": { - "value": "PLACEHOLDER" } } } diff --git a/scripts/setup-rg/rg-template.json b/scripts/setup-rg/rg-template.json index e984cd86849..8cc69cbb376 100644 --- a/scripts/setup-rg/rg-template.json +++ b/scripts/setup-rg/rg-template.json @@ -19,27 +19,48 @@ "vmAdminUsername": { "type": "string" }, - "sshPublicKey": { - "type": "string" + "sshPublicKeys": { + "type": "array" + }, + "storageAccountLocations": { + "type": "array", + "defaultValue": [ + "[resourceGroup().location]" + ] }, "fileShareName": { "type": "string", "defaultValue": "mlos-file-share" }, - "resultsDbName": { - "type": "string" + "vmLocation": { + "type": "string", + "defaultValue": "[resourceGroup().location]" }, - "resultsDbAdminUsername": { + "vmSshPort": { "type": "string", - "defaultValue": "mlos" + "defaultValue": "22" }, - "resultsDbAdminPassword": { - "type": "securestring" + "vmSshSourceAddressPrefix": { + "type": "string" + }, + "storageSku": { + "type": "string", + "defaultValue": "Standard_LRS", + "allowedValues": [ + "Premium_LRS", + "Premium_ZRS", + "Standard_GRS", + "Standard_GZRS", + "Standard_LRS", + "Standard_RAGRS", + "Standard_RAGZRS", + "Standard_ZRS" + ] } }, "functions": [], "variables": { - "storageAccountName": "[toLower(concat(replace(parameters('projectPrefix'), '-', ''), 'storage'))]", + "storageAccountName": "[toLower(concat(replace(parameters('projectPrefix'), '-', '')))]", "vmName": "[concat(parameters('projectPrefix'), '-vm')]", "networkInterfaceName": "[concat(variables('vmName'), '-ni-', uniqueString(resourceGroup().id))]", "vnetName": "[concat(variables('vmName'), '-vnet')]", @@ -52,7 +73,7 @@ "name": "[variables('publicIpName')]", "type": "Microsoft.Network/publicIPAddresses", "apiVersion": "2020-11-01", - "location": "[resourceGroup().location]", + "location": "[parameters('vmLocation')]", "tags": { "displayName": "PublicIPAddress" }, @@ -67,17 +88,17 @@ "name": "[variables('nsgName')]", "type": "Microsoft.Network/networkSecurityGroups", "apiVersion": "2020-11-01", - "location": "[resourceGroup().location]", + "location": "[parameters('vmLocation')]", "properties": { "securityRules": [ { - "name": "CorpSSH", + "name": "UserSSH", "properties": { - "description": "Corp. IP SSH access", + "description": "For user SSH access", "protocol": "Tcp", "sourcePortRange": "*", - "destinationPortRange": "22", - "sourceAddressPrefix": "131.107.0.0/16", + "destinationPortRange": "[parameters('vmSshPort')]", + "sourceAddressPrefix": "[parameters('vmSshSourceAddressPrefix')]", "destinationAddressPrefix": "*", "access": "Allow", "priority": 100, @@ -91,7 +112,7 @@ "name": "[variables('vnetName')]", "type": "Microsoft.Network/virtualNetworks", "apiVersion": "2020-11-01", - "location": "[resourceGroup().location]", + "location": "[parameters('vmLocation')]", "dependsOn": [ "[resourceId('Microsoft.Network/networkSecurityGroups', variables('nsgName'))]" ], @@ -112,11 +133,11 @@ "serviceEndpoints": [ { "service": "Microsoft.Storage", - "locations": ["[resourceGroup().location]"] + "locations": "[parameters('storageAccountLocations')]" }, { "service": "Microsoft.KeyVault", - "locations": ["[resourceGroup().location]"] + "locations": ["[parameters('vmLocation')]"] } ], "networkSecurityGroup": { @@ -131,14 +152,14 @@ "name": "[variables('networkInterfaceName')]", "type": "Microsoft.Network/networkInterfaces", "apiVersion": "2020-11-01", - "location": "[resourceGroup().location]", + "location": "[parameters('vmLocation')]", "dependsOn": [ "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIpName'))]", "[resourceId('Microsoft.Network/virtualNetworks', variables('vnetName'))]", "[resourceId('Microsoft.Network/networkSecurityGroups', variables('nsgName'))]" ], "tags": { - "displayName": "ubuntuVM1-NetworkInterface" + "displayName": "[variables('networkInterfaceName')]" }, "properties": { "ipConfigurations": [ @@ -164,7 +185,7 @@ "name": "[variables('kvName')]", "type": "Microsoft.KeyVault/vaults", "apiVersion": "2019-09-01", - "location": "[resourceGroup().location]", + "location": "[parameters('vmLocation')]", "dependsOn": [ "[resourceId('Microsoft.Network/virtualNetworks', variables('vnetName'))]" ], @@ -196,7 +217,7 @@ "name": "[variables('vmName')]", "type": "Microsoft.Compute/virtualMachines", "apiVersion": "2021-03-01", - "location": "[resourceGroup().location]", + "location": "[parameters('vmLocation')]", "dependsOn": [ "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]" ], @@ -213,10 +234,14 @@ "linuxConfiguration": { "disablePasswordAuthentication": true, "ssh": { - "publicKeys": [ + "copy": [ { - "path": "[concat('/home/', parameters('vmAdminUsername'), '/.ssh/authorized_keys')]", - "keyData": "[parameters('sshPublicKey')]" + "name": "publicKeys", + "count": "[length(parameters('sshPublicKeys'))]", + "input": { + "path": "[concat('/home/', parameters('vmAdminUsername'), '/.ssh/authorized_keys')]", + "keyData": "[parameters('sshPublicKeys')[copyIndex('publicKeys')]]" + } } ] } @@ -250,20 +275,23 @@ } }, { - "name": "[variables('storageAccountName')]", + "copy": { + "name": "storageAccountCopy", + "count": "[length(parameters('storageAccountLocations'))]" + }, + "name": "[concat(variables('storageAccountName'), parameters('storageAccountLocations')[copyIndex('storageAccountCopy')])]", "type": "Microsoft.Storage/storageAccounts", "apiVersion": "2021-04-01", "tags": { - "displayName": "[variables('storageAccountName')]" + "displayName": "[concat(variables('storageAccountName'), parameters('storageAccountLocations')[copyIndex('storageAccountCopy')])]" }, - "location": "[resourceGroup().location]", + "location": "[parameters('storageAccountLocations')[copyIndex('storageAccountCopy')]]", "dependsOn": [ "[resourceId('Microsoft.Network/virtualNetworks', variables('vnetName'))]" ], "kind": "StorageV2", "sku": { - "name": "Standard_LRS", - "tier": "Standard" + "name": "[parameters('storageSku')]" }, "properties": { "networkAcls": { @@ -280,87 +308,21 @@ } }, { + "copy": { + "name": "fileShareCopy", + "count": "[length(parameters('storageAccountLocations'))]" + }, "type": "Microsoft.Storage/storageAccounts/fileServices/shares", "apiVersion": "2022-09-01", - "name": "[concat(variables('storageAccountName'), '/default/', parameters('fileShareName'))]", + "name": "[concat(variables('storageAccountName'), parameters('storageAccountLocations')[copyIndex('fileShareCopy')], '/default/', parameters('fileShareName'))]", "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]" + "[resourceId('Microsoft.Storage/storageAccounts', concat(variables('storageAccountName'), parameters('storageAccountLocations')[copyIndex('fileShareCopy')]))]" ], "properties": { "accessTier": "TransactionOptimized", "shareQuota": 5120, "enabledProtocols": "SMB" } - }, - { - "type": "Microsoft.DBforMySQL/flexibleServers", - "apiVersion": "2021-12-01-preview", - "name": "[parameters('resultsDbName')]", - "location": "[resourceGroup().location]", - "sku": { - "name": "Standard_B1s", - "tier": "Burstable" - }, - "properties": { - "administratorLogin": "[parameters('resultsDbAdminUsername')]", - "administratorLoginPassword": "[parameters('resultsDbAdminPassword')]", - "storage": { - "storageSizeGB": 20, - "iops": 360, - "autoGrow": "Enabled", - "autoIoScaling": "Disabled", - "logOnDisk": "Disabled" - }, - "version": "8.0.21", - "network": { - "publicNetworkAccess": "Enabled" - }, - "backup": { - "backupRetentionDays": 7, - "geoRedundantBackup": "Disabled" - }, - "highAvailability": { - "mode": "Disabled" - } - } - }, - { - "type": "Microsoft.DBforMySQL/flexibleServers/databases", - "apiVersion": "2021-12-01-preview", - "name": "[concat(parameters('resultsDbName'), '/mlos')]", - "dependsOn": [ - "[resourceId('Microsoft.DBforMySQL/flexibleServers', parameters('resultsDbName'))]" - ], - "properties": { - "charset": "utf8", - "collation": "utf8_general_ci" - } - }, - { - "type": "Microsoft.DBforMySQL/flexibleServers/firewallRules", - "apiVersion": "2021-12-01-preview", - "name": "[concat(parameters('resultsDbName'), '/CorpIps')]", - "dependsOn": [ - "[resourceId('Microsoft.DBforMySQL/flexibleServers', parameters('resultsDbName'))]" - ], - "properties": { - "startIpAddress": "131.107.0.0", - "endIpAddress": "131.107.255.255" - } - }, - { - "type": "Microsoft.DBforMySQL/flexibleServers/firewallRules", - "apiVersion": "2021-12-01-preview", - "name": "[concat(parameters('resultsDbName'), '/AllowVM-', variables('vmName'))]", - "dependsOn": [ - "[resourceId('Microsoft.DBforMySQL/flexibleServers', parameters('resultsDbName'))]", - "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIpName'))]", - "[resourceId('Microsoft.Compute/virtualMachines', variables('vmName'))]" - ], - "properties": { - "startIpAddress": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('publicIpName'))).ipAddress]", - "endIpAddress": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('publicIpName'))).ipAddress]" - } } ], "outputs": { @@ -368,6 +330,14 @@ "type": "string", "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('publicIpName'))).dnsSettings.fqdn]" }, + "vmIpAddress": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('publicIpName'))).ipAddress]" + }, + "vmName": { + "type": "string", + "value": "[variables('vmName')]" + }, "kvName": { "type": "string", "value": "[variables('kvName')]" diff --git a/scripts/setup-rg/setup-rg.ps1 b/scripts/setup-rg/setup-rg.ps1 index d0a6e0c3d26..5015af8d5de 100644 --- a/scripts/setup-rg/setup-rg.ps1 +++ b/scripts/setup-rg/setup-rg.ps1 @@ -1,42 +1,73 @@ param( - # ARM template params + # Main control plane ARM template params [Parameter(Mandatory=$True)] - [string] $armParameters, + [string] $controlPlaneArmParameters, + # Results DB ARM template params + [Parameter(Mandatory=$False)] + [string] $resultsDbArmParameters, # Other params [Parameter(Mandatory=$True)] [string] $servicePrincipalName, [Parameter(Mandatory=$True)] [string] $resourceGroupName, [Parameter(Mandatory=$True)] - [string] $certName + [string] $certName, + [Parameter(Mandatory=$False)] + [int] $certExpirationYears = 1 ) -$currentUserAlias = az account show --query user.name --output tsv - -$servicePrincipalId = az ad sp list ` - --display-name $servicePrincipalName ` - --query '[].id' ` - --output tsv - -# Assign 'Contributor' access to Service Principal -az role assignment create ` - --assignee $servicePrincipalId ` - --role "Contributor" ` - --resource-group $resourceGroupName - # Provision resources into the resource group with ARM template +Write-Output "Provisioning control plane resources..." $deploymentResults = az deployment group create ` --resource-group $resourceGroupName ` --template-file .\rg-template.json ` - --parameters $armParameters + --parameters $controlPlaneArmParameters ` + --parameters vmSshSourceAddressPrefix="131.107.0.0/16" | ConvertFrom-JSON if (!$?) { - Write-Error "Error in provisioning resources!" + Write-Error "Error in provisioning control plane resources!" return } +# Conditional provisioning of results DB +if ($resultsDbArmParameters) { + Write-Output "Provisioning results DB..." + $vmName = $deploymentResults.properties.outputs.vmName.value + $vmIpAddress = $deploymentResults.properties.outputs.vmIpAddress.value + $dbDeploymentResults = az deployment group create ` + --resource-group $resourceGroupName ` + --template-file ./results-db/mysql-template.json ` + --parameters $resultsDbArmParameters + + if (!$?) { + Write-Error "Error in provisioning results DB!" + } + else { + $dbName = $dbDeploymentResults.properties.outputs.dbName.value + + # VM IP access for results DB + az mysql flexible-server firewall-rule update ` + --resource-group $resourceGroupName ` + --name $dbName ` + --rule-name "AllowVM-$vmName" ` + --start-ip-address $vmIpAddress ` + --end-ip-address $vmIpAddress + + # Corp. IP access for results DB + az mysql flexible-server firewall-rule update ` + --resource-group $resourceGroupName ` + --name $dbName ` + --rule-name "CorpIps" ` + --start-ip-address "131.107.0.0" ` + --end-ip-address "131.107.255.255" + } +} + +$currentUserAlias = az account show --query "user.name" --output tsv +$resourceGroupId = az group show --name $resourceGroupName --query "id" --output tsv + # Assign 'Key Vault Administrator' access to current user $kvName = $deploymentResults.properties.outputs.kvName.value $kvId = az keyvault show --name $kvName --query "id" --output tsv @@ -45,45 +76,46 @@ az role assignment create ` --role "Key Vault Administrator" ` --scope $kvId -# Generate certificate if doesnt exist -$certThumprint = az keyvault certificate show ` +# Check if cert of same name exists in keyvault already +$certThumbprint = az keyvault certificate show ` --name $certName ` --vault-name $kvName ` --query "x509ThumbprintHex" --output tsv + if (!$?) { - Write-Output "Generating certificate in the key vault..." - az keyvault certificate get-default-policy | Out-File -Encoding utf8 defaultCertPolicy.json - az keyvault certificate create ` - --vault-name $kvName ` - --name $certName ` - --policy `@defaultCertPolicy.json - $certThumprint = az keyvault certificate show ` - --name $certName ` - --vault-name $kvName ` - --query "x509ThumbprintHex" --output tsv + # The cert does not exist yet. + # Create the service principal if doesn't exist, storing the cert in the keyvault + # If it does exist, this also patches the current service principal with the role + az ad sp create-for-rbac ` + --name $servicePrincipalName ` + --role "Contributor" ` + --scopes $resourceGroupId ` + --create-cert ` + --cert $certName ` + --keyvault $kvName ` + --years $certExpirationYears } else { - Write-Output "Certificate exists in the key vault already." -} + # The cert already exists in the keyvault. -# Upload certificate to service principal if doesnt exist -$spCertThumbprints = az ad sp credential list ` - --id $servicePrincipalId ` - --cert ` - --query "[].customKeyIdentifier" --output tsv -if (!$spCertThumbprints.Contains($certThumprint)) { - # Download the public portion of the certificate from key vault - Write-Output "Downloading certificate from key vault..." - az keyvault certificate download ` - --vault-name $kvName ` - --name $certName ` - --file "$certName.pem" + # Ensure the SP exists with correct roles, without creating a cert. + az ad sp create-for-rbac ` + --name $servicePrincipalName ` + --role "Contributor" ` + --scopes $resourceGroupId ` - # Upload the certificate to the service principal - Write-Output "Uploading certificate to service principal..." - az ad sp credential reset ` + # SP's certs + $servicePrincipalId = az ad sp list ` + --display-name $servicePrincipalName ` + --query "[?servicePrincipalType == 'Application'].objectId" ` + --output tsv + $spCertThumbprints = az ad sp credential list ` --id $servicePrincipalId ` - --cert "$certName.pem" ` - --append -} else { - Write-Output "Certificate uploaded to the Service Principal already." + --cert ` + --query "[].customKeyIdentifier" --output tsv + + if ($spCertThumbprints.Contains($certThumbprint)) { + Write-Output "Keyvault contains the certificate '$certName' that is linked to the service principal '$servicePrincipalName' already." + } else { + Write-Warning "Keyvault already contains a certificate called '$certname', but is not linked with the service principal '$servicePrincipalName'! Skipping cert handling" + } } From b9aae791b5ccda569c44a6b91997715d6dfb9aa2 Mon Sep 17 00:00:00 2001 From: Eu Jing Chua Date: Fri, 25 Aug 2023 19:11:00 -0700 Subject: [PATCH 04/11] Remove specific IP references --- scripts/setup-rg/README.md | 37 +++++++++++++------ .../setup-rg/results-db/mysql-template.json | 20 ++++++++++ .../mysql-template.parameters.example.json | 9 +++++ .../rg-template.example.parameters.json | 2 +- scripts/setup-rg/setup-rg.ps1 | 16 ++------ 5 files changed, 59 insertions(+), 25 deletions(-) diff --git a/scripts/setup-rg/README.md b/scripts/setup-rg/README.md index ead8714bbf3..51153780244 100644 --- a/scripts/setup-rg/README.md +++ b/scripts/setup-rg/README.md @@ -11,25 +11,32 @@ The *control plane RG* is a container for the *persistent* resources of MLOS (re az login ``` -2. Make a copy of the ARM parameters file. +1. Make a copy of the control plane ARM parameters file. ```sh cp rg-template.example.parameters.json rg-template.parameters.json ``` -3. Modify the ARM parameters in the newly created file as needed, especially the `PLACEHOLDER` values. +1. (Optional) Make a copy of the results DB parameters file, if planning to provision a results DB. -4. Execute the main script with CLI args as follows: + ```sh + cp results-db/mysql-template.parameters.example.json results-db/mysql-template.parameters.json + ``` + +1. Modify the ARM parameters in the newly created files as needed, especially the `PLACEHOLDER` values. + +1. Execute the main script with CLI args as follows: ```shell ./setup-rg.ps1 ` - -armParameters $armParams ` + -controlPlaneArmParameters $controlPlaneArmParams ` + -resultsDbArmParameters $resultsDbArmParams # If provisioning results DB, otherwise omit ` -servicePrincipalName $servicePrincipalName ` -resourceGroupName $resourceGroupName ` -certName $certName ``` - where `$armParams` can be `rg-template.parameters.json` from before. However, it also follows the same usage as `--parameters` in [az deployment group create](https://learn.microsoft.com/en-us/cli/azure/deployment/group?view=azure-cli-latest#az-deployment-group-create-examples). + where `$*ArmParams` can be the corresponding `*.parameters.json` and from before. However, it also follows the same usage as `--parameters` in [az deployment group create](https://learn.microsoft.com/en-us/cli/azure/deployment/group?view=azure-cli-latest#az-deployment-group-create-examples). ## Workflow @@ -39,17 +46,23 @@ The high-level flow for what this script automates is as follows: Ideally, experiment resources are placed in their own RG. When that isn't possible, they can also be placed in the control plane RG, in which case the SP can optionally be given access to the control plane RG as well. -2. Provision control plane resources into the RG. +1. Provision control plane resources into the RG. This includes: - - Networking (public IP, security group, vnet, subnet, network interface) + - Control VM for running the `mlos_bench` scheduler. + - Control VM's networking (public IP, security group, vnet, subnet, network interface) - Key Vault for storing the SP credentials. - - Storage (storage account, file share, DB) - - VM for running the `mlos_bench` scheduler. + - Storage (storage account, file share) -3. Assign `Key Vault Administrator` access to the current user. +1. The results DB is then optionally provisioned, adding appropriate firewall rules. + +1. Assign `Key Vault Administrator` access to the current user. This allows the current user to retrieve secrets / certificates from the VM once it is set up. Ensure to log in as the same user in the VM. -4. If not existing yet, generate a certificate with `certName` in the key vault. +1. Check if the desired certificate name already exists in the key vault. + +1. If certificate does not exist yet, create or update the Service Principal (SP) with `Contributor` access for write over resources. + Ideally, experiment resources are placed in their own RG. + When that isn't possible, they can also be placed in the control plane RG, in which case the SP can optionally be given access to the control plane RG as well. -5. If not associated yet, upload the certificate from before to the SP. +1. Otherwise, create or update the SP just with similar access as before. Now also verify that the existing certificate in the key vault matches one linked to the SP already, via thumbprint. diff --git a/scripts/setup-rg/results-db/mysql-template.json b/scripts/setup-rg/results-db/mysql-template.json index 9c8f2962bff..05ae9dfe7a8 100644 --- a/scripts/setup-rg/results-db/mysql-template.json +++ b/scripts/setup-rg/results-db/mysql-template.json @@ -15,6 +15,10 @@ }, "dbAdminPassword": { "type": "securestring" + }, + "dbFirewallRules": { + "type": "array", + "defaultValue": [] } }, "functions": [], @@ -63,6 +67,22 @@ "charset": "utf8", "collation": "utf8_general_ci" } + }, + { + "copy": { + "name": "firewallCopy", + "count": "[length(parameters('dbFirewallRules'))]" + }, + "type": "Microsoft.DBforMySQL/flexibleServers/firewallRules", + "apiVersion": "2021-12-01-preview", + "name": "[concat(parameters('dbName'), '/', parameters('dbFirewallRules')[copyIndex('firewallCopy')].name)]", + "dependsOn": [ + "[resourceId('Microsoft.DBforMySQL/flexibleServers', parameters('dbName'))]" + ], + "properties": { + "startIpAddress": "[parameters('dbFirewallRules')[copyIndex('firewallCopy')].startIpAddress]", + "endIpAddress": "[parameters('dbFirewallRules')[copyIndex('firewallCopy')].endIpAddress]" + } } ], "outputs": { diff --git a/scripts/setup-rg/results-db/mysql-template.parameters.example.json b/scripts/setup-rg/results-db/mysql-template.parameters.example.json index e271c9d7774..250aeb54129 100644 --- a/scripts/setup-rg/results-db/mysql-template.parameters.example.json +++ b/scripts/setup-rg/results-db/mysql-template.parameters.example.json @@ -10,6 +10,15 @@ }, "dbAdminPassword": { "value": "PLACEHOLDER" + }, + "dbFirewallRules": { + "value": [ + { + "name": "PLACEHOLDER", + "startIpAddress": "123.123.0.0", + "endIpAddress": "123.123.255.255" + } + ] } } } diff --git a/scripts/setup-rg/rg-template.example.parameters.json b/scripts/setup-rg/rg-template.example.parameters.json index fc8c643b57c..9c4ccc7a998 100644 --- a/scripts/setup-rg/rg-template.example.parameters.json +++ b/scripts/setup-rg/rg-template.example.parameters.json @@ -17,7 +17,7 @@ ] }, "vmSshSourceAddressPrefix": { - "value": "PLACEHOLDER" + "value": "PLACEHOLDER; e.g. 123.123.0.0/16" }, "fileShareName": { "value": "mlos-file-share" diff --git a/scripts/setup-rg/setup-rg.ps1 b/scripts/setup-rg/setup-rg.ps1 index 5015af8d5de..9b6121a7da5 100644 --- a/scripts/setup-rg/setup-rg.ps1 +++ b/scripts/setup-rg/setup-rg.ps1 @@ -23,7 +23,6 @@ $deploymentResults = az deployment group create ` --resource-group $resourceGroupName ` --template-file .\rg-template.json ` --parameters $controlPlaneArmParameters ` - --parameters vmSshSourceAddressPrefix="131.107.0.0/16" | ConvertFrom-JSON if (!$?) { @@ -34,18 +33,19 @@ if (!$?) { # Conditional provisioning of results DB if ($resultsDbArmParameters) { Write-Output "Provisioning results DB..." - $vmName = $deploymentResults.properties.outputs.vmName.value - $vmIpAddress = $deploymentResults.properties.outputs.vmIpAddress.value $dbDeploymentResults = az deployment group create ` --resource-group $resourceGroupName ` --template-file ./results-db/mysql-template.json ` - --parameters $resultsDbArmParameters + --parameters $resultsDbArmParameters ` + | ConvertFrom-JSON if (!$?) { Write-Error "Error in provisioning results DB!" } else { $dbName = $dbDeploymentResults.properties.outputs.dbName.value + $vmName = $deploymentResults.properties.outputs.vmName.value + $vmIpAddress = $deploymentResults.properties.outputs.vmIpAddress.value # VM IP access for results DB az mysql flexible-server firewall-rule update ` @@ -54,14 +54,6 @@ if ($resultsDbArmParameters) { --rule-name "AllowVM-$vmName" ` --start-ip-address $vmIpAddress ` --end-ip-address $vmIpAddress - - # Corp. IP access for results DB - az mysql flexible-server firewall-rule update ` - --resource-group $resourceGroupName ` - --name $dbName ` - --rule-name "CorpIps" ` - --start-ip-address "131.107.0.0" ` - --end-ip-address "131.107.255.255" } } From f9dad34db3823aab053ae81b5b5bc507b9abb1db Mon Sep 17 00:00:00 2001 From: Eu Jing Chua Date: Fri, 25 Aug 2023 19:36:37 -0700 Subject: [PATCH 05/11] Re-org scripts --- .../config/services/remote/azure/scripts}/setup-rg/README.md | 0 .../azure/scripts}/setup-rg/results-db/mysql-template.json | 0 .../setup-rg/results-db/mysql-template.parameters.example.json | 0 .../azure/scripts}/setup-rg/rg-template.example.parameters.json | 0 .../services/remote/azure/scripts}/setup-rg/rg-template.json | 0 .../config/services/remote/azure/scripts}/setup-rg/setup-rg.ps1 | 0 mlos_bench/mlos_bench/services/remote/azure/README.md | 2 +- 7 files changed, 1 insertion(+), 1 deletion(-) rename {scripts => mlos_bench/mlos_bench/config/services/remote/azure/scripts}/setup-rg/README.md (100%) rename {scripts => mlos_bench/mlos_bench/config/services/remote/azure/scripts}/setup-rg/results-db/mysql-template.json (100%) rename {scripts => mlos_bench/mlos_bench/config/services/remote/azure/scripts}/setup-rg/results-db/mysql-template.parameters.example.json (100%) rename {scripts => mlos_bench/mlos_bench/config/services/remote/azure/scripts}/setup-rg/rg-template.example.parameters.json (100%) rename {scripts => mlos_bench/mlos_bench/config/services/remote/azure/scripts}/setup-rg/rg-template.json (100%) rename {scripts => mlos_bench/mlos_bench/config/services/remote/azure/scripts}/setup-rg/setup-rg.ps1 (100%) diff --git a/scripts/setup-rg/README.md b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/README.md similarity index 100% rename from scripts/setup-rg/README.md rename to mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/README.md diff --git a/scripts/setup-rg/results-db/mysql-template.json b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/results-db/mysql-template.json similarity index 100% rename from scripts/setup-rg/results-db/mysql-template.json rename to mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/results-db/mysql-template.json diff --git a/scripts/setup-rg/results-db/mysql-template.parameters.example.json b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/results-db/mysql-template.parameters.example.json similarity index 100% rename from scripts/setup-rg/results-db/mysql-template.parameters.example.json rename to mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/results-db/mysql-template.parameters.example.json diff --git a/scripts/setup-rg/rg-template.example.parameters.json b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/rg-template.example.parameters.json similarity index 100% rename from scripts/setup-rg/rg-template.example.parameters.json rename to mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/rg-template.example.parameters.json diff --git a/scripts/setup-rg/rg-template.json b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/rg-template.json similarity index 100% rename from scripts/setup-rg/rg-template.json rename to mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/rg-template.json diff --git a/scripts/setup-rg/setup-rg.ps1 b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 similarity index 100% rename from scripts/setup-rg/setup-rg.ps1 rename to mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 diff --git a/mlos_bench/mlos_bench/services/remote/azure/README.md b/mlos_bench/mlos_bench/services/remote/azure/README.md index 4deb0df8503..30bebad4cfd 100644 --- a/mlos_bench/mlos_bench/services/remote/azure/README.md +++ b/mlos_bench/mlos_bench/services/remote/azure/README.md @@ -25,7 +25,7 @@ The requirements of this method are: - User logged in to Azure CLI - An existing SP with appropriate access to the target resource group (e.g., `Contributor` role). - > See scripts/setup-rg/README.md for details on setting up the SP. + > See [scripts/setup-rg/README.md](../../../config/services/remote/azure/scripts/setup-rg/README.md) for details on setting up the SP. - Key vault for storing certificate. - A certificate associated with the SP, which is stored in the key vault. - User has access to read the necessary secrets/certificates from the key vault (e.g., `Key Vault Administrator` role). From 01b16d6243a6c5eae9c9f94b1f079e9db4a533ca Mon Sep 17 00:00:00 2001 From: Eu Jing Chua Date: Mon, 28 Aug 2023 11:43:54 -0700 Subject: [PATCH 06/11] Symlink and README linking --- mlos_bench/mlos_bench/services/remote/azure/README.md | 2 +- scripts/azure/setup-rg | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 120000 scripts/azure/setup-rg diff --git a/mlos_bench/mlos_bench/services/remote/azure/README.md b/mlos_bench/mlos_bench/services/remote/azure/README.md index 30bebad4cfd..1644f3bd952 100644 --- a/mlos_bench/mlos_bench/services/remote/azure/README.md +++ b/mlos_bench/mlos_bench/services/remote/azure/README.md @@ -25,7 +25,7 @@ The requirements of this method are: - User logged in to Azure CLI - An existing SP with appropriate access to the target resource group (e.g., `Contributor` role). - > See [scripts/setup-rg/README.md](../../../config/services/remote/azure/scripts/setup-rg/README.md) for details on setting up the SP. + > See [scripts/azure/setup-rg/README.md](../../../../../scripts/azure/setup-rg/README.md) for details on setting up the SP. - Key vault for storing certificate. - A certificate associated with the SP, which is stored in the key vault. - User has access to read the necessary secrets/certificates from the key vault (e.g., `Key Vault Administrator` role). diff --git a/scripts/azure/setup-rg b/scripts/azure/setup-rg new file mode 120000 index 00000000000..bd8a238dcb3 --- /dev/null +++ b/scripts/azure/setup-rg @@ -0,0 +1 @@ +../../mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg \ No newline at end of file From 6ada9535b12b44054944a8a66d6d9a73d751af72 Mon Sep 17 00:00:00 2001 From: Eu Jing Chua Date: Tue, 29 Aug 2023 11:47:07 -0700 Subject: [PATCH 07/11] Initial sh script --- .../azure/scripts/setup-rg/setup-rg.ps1 | 15 +- .../remote/azure/scripts/setup-rg/setup-rg.sh | 150 ++++++++++++++++++ 2 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.sh diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 index 9b6121a7da5..d6090e1525b 100644 --- a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 +++ b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 @@ -62,7 +62,7 @@ $resourceGroupId = az group show --name $resourceGroupName --query "id" --output # Assign 'Key Vault Administrator' access to current user $kvName = $deploymentResults.properties.outputs.kvName.value -$kvId = az keyvault show --name $kvName --query "id" --output tsv +$kvId = az keyvault show --name $kvName --resource-group $resourceGroupName --query "id" --output tsv az role assignment create ` --assignee $currentUserAlias ` --role "Key Vault Administrator" ` @@ -95,19 +95,20 @@ if (!$?) { --role "Contributor" ` --scopes $resourceGroupId ` - # SP's certs - $servicePrincipalId = az ad sp list ` + # SP's certs, which are stored in the registered application instead + $servicePrincipalAppId = az ad sp list ` --display-name $servicePrincipalName ` - --query "[?servicePrincipalType == 'Application'].objectId" ` + --query "[?servicePrincipalType == 'Application'].appId" ` --output tsv - $spCertThumbprints = az ad sp credential list ` + $spCertThumbprints = az ad app credential list ` --id $servicePrincipalId ` --cert ` - --query "[].customKeyIdentifier" --output tsv + --query "[].customKeyIdentifier" ` + --output tsv if ($spCertThumbprints.Contains($certThumbprint)) { Write-Output "Keyvault contains the certificate '$certName' that is linked to the service principal '$servicePrincipalName' already." } else { - Write-Warning "Keyvault already contains a certificate called '$certname', but is not linked with the service principal '$servicePrincipalName'! Skipping cert handling" + Write-Warning "Keyvault already contains a certificate called '$certName', but is not linked with the service principal '$servicePrincipalName'! Skipping cert handling" } } diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.sh b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.sh new file mode 100644 index 00000000000..7529103dccf --- /dev/null +++ b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.sh @@ -0,0 +1,150 @@ +#!/bin/bash +## +## Copyright (c) Microsoft Corporation. +## Licensed under the MIT License. +## + +set -eu +set -x + +# Argument parsing +while [[ "$#" -gt 0 ]]; do +case $1 in + # Main control plane ARM template params + --controlPlaneArmParameters) + controlPlaneArmParameters="$2" + shift 2 + ;; + # Results DB ARM template params + --resultsDbArmParameters) + resultsDbArmParameters="$2" + shift 2 + ;; + # Other params + --servicePrincipalName) + servicePrincipalName="$2" + shift 2 + ;; + --resourceGroupName) + resourceGroupName="$2" + shift 2 + ;; + --certName) + certName="$2" + shift 2 + ;; + --certExpirationYears) + certExpirationYears="$2" + shift 2 + ;; + *) + echo "Unknown parameter passed $1" + shift 1 + ;; +esac +done + +# Default values +resultsDbArmParameters=${resultsDbArmParameters:-""} +certExpirationYears=${certExpirationYears:-1} + +# Provision resources into the resource group with ARM template +echo "Provisioning control plane resources..." +deploymentResults=$(az deployment group create \ + --resource-group "$resourceGroupName" \ + --template-file ./rg-template.json \ + --parameters "$controlPlaneArmParameters" \ + ) + +if [[ ! $? ]]; then + echo "Error in provisioning control plane resources!" + exit 1 +fi + +# Conditional provisioning of results DB +if [[ "$resultsDbArmParameters" ]]; then + echo "Provisioning results DB..." + dbDeploymentResults=$(az deployment group create \ + --resource-group "$resourceGroupName" \ + --template-file ./results-db/mysql-template.json \ + --parameters "$resultsDbArmParameters" \ + ) + + if [[ ! $? ]]; then + echo "Error in provisioning results DB!" + else + dbName=$(echo "$dbDeploymentResults" | jq -r ".properties.outputs.dbName.value") + vmName=$(echo "$deploymentResults" | jq -r ".properties.outputs.vmName.value") + vmIpAddress=$(echo "$deploymentResults" | jq -r ".properties.outputs.vmIpAddress.value") + + # VM IP access for results DB + az mysql flexible-server firewall-rule update \ + --resource-group "$resourceGroupName" \ + --name "$dbName" \ + --rule-name "AllowVM-$vmName" \ + --start-ip-address "$vmIpAddress" \ + --end-ip-address "$vmIpAddress" \ + + fi +fi + +currentUserAlias=$(az account show --query "user.name" --output tsv) +resourceGroupId=$(az group show --name "$resourceGroupName" --query "id" --output tsv) + +# Assign 'Key Vault Administrator' access to current user +kvName=$(echo "$deploymentResults" | jq -r ".properties.outputs.kvName.value") +kvId=$(az keyvault show --name "$kvName" --resource-group "$resourceGroupName" --query "id" --output tsv) +az role assignment create \ + --assignee "$currentUserAlias" \ + --role "Key Vault Administrator" \ + --scope "$kvId" \ + +# Check if cert of same name exists in keyvault already +certThumbprint=$(az keyvault certificate show \ + --name "$certName" \ + --vault-name "$kvName" \ + --query "x509ThumbprintHex" --output tsv \ + || echo "NOCERT") + +if [[ $certThumbprint == "NOCERT" ]]; then + # The cert does not exist yet. + # Create the service principal if doesn't exist, storing the cert in the keyvault + # If it does exist, this also patches the current service principal with the role + az ad sp create-for-rbac \ + --name "$servicePrincipalName" \ + --role "Contributor" \ + --scopes "$resourceGroupId" \ + --create-cert \ + --cert "$certName" \ + --keyvault "$kvName" \ + --years "$certExpirationYears" \ + +else + # The cert already exists in the keyvault. + + # Ensure the SP exists with correct roles, without creating a cert. + az ad sp create-for-rbac \ + --name "$servicePrincipalName" \ + --role "Contributor" \ + --scopes "$resourceGroupId" \ + + # SP's certs, which are stored in the registered application instead + servicePrincipalAppId=$(az ad sp list \ + --display-name "$servicePrincipalName" \ + --query "[?servicePrincipalType == 'Application'].appId" \ + --output tsv \ + ) + spCertThumbprints=$(az ad app credential list \ + --id "$servicePrincipalAppId" \ + --cert \ + --query "[].customKeyIdentifier" \ + --output tsv \ + ) + echo "KV cert thumprint is $certThumbprint" + echo "SP's certs are $spCertThumbprints" + if [[ $spCertThumbprints == *$certThumbprint* ]]; then + echo "Keyvault contains the certificate '$certName' that is linked to the service principal '$servicePrincipalName' already." + else + echo "Keyvault already contains a certificate called '$certName', but is not linked with the service principal '$servicePrincipalName'! Skipping cert handling" + fi +fi From 8095ae524ae93fde5a0e2868c00a3d638b566735 Mon Sep 17 00:00:00 2001 From: Eu Jing Chua Date: Tue, 29 Aug 2023 14:17:49 -0700 Subject: [PATCH 08/11] Script fixes --- .../services/remote/azure/scripts/setup-rg/setup-rg.ps1 | 5 +++-- .../services/remote/azure/scripts/setup-rg/setup-rg.sh | 4 +--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 index d6090e1525b..1b0ebae36c1 100644 --- a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 +++ b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 @@ -72,7 +72,8 @@ az role assignment create ` $certThumbprint = az keyvault certificate show ` --name $certName ` --vault-name $kvName ` - --query "x509ThumbprintHex" --output tsv + --query "x509ThumbprintHex" --output tsv ` + 2>$null if (!$?) { # The cert does not exist yet. @@ -101,7 +102,7 @@ if (!$?) { --query "[?servicePrincipalType == 'Application'].appId" ` --output tsv $spCertThumbprints = az ad app credential list ` - --id $servicePrincipalId ` + --id $servicePrincipalAppId ` --cert ` --query "[].customKeyIdentifier" ` --output tsv diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.sh b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.sh index 7529103dccf..049ccb81dae 100644 --- a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.sh +++ b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.sh @@ -5,7 +5,6 @@ ## set -eu -set -x # Argument parsing while [[ "$#" -gt 0 ]]; do @@ -104,6 +103,7 @@ certThumbprint=$(az keyvault certificate show \ --name "$certName" \ --vault-name "$kvName" \ --query "x509ThumbprintHex" --output tsv \ + 2> /dev/null \ || echo "NOCERT") if [[ $certThumbprint == "NOCERT" ]]; then @@ -140,8 +140,6 @@ else --query "[].customKeyIdentifier" \ --output tsv \ ) - echo "KV cert thumprint is $certThumbprint" - echo "SP's certs are $spCertThumbprints" if [[ $spCertThumbprints == *$certThumbprint* ]]; then echo "Keyvault contains the certificate '$certName' that is linked to the service principal '$servicePrincipalName' already." else From ed19c5f8abcac4a5260f51c1f3b151c0352c2d21 Mon Sep 17 00:00:00 2001 From: Eu Jing Chua Date: Thu, 31 Aug 2023 20:38:07 +0000 Subject: [PATCH 09/11] Script fixes and PR comments --- .../remote/azure/scripts/setup-rg/README.md | 38 ++- .../setup-rg/results-db/mysql-template.json | 4 +- .../mysql-template.parameters.example.json | 4 +- .../azure/scripts/setup-rg/rg-template.json | 19 +- .../azure/scripts/setup-rg/setup-rg.ps1 | 233 +++++++++--------- .../remote/azure/scripts/setup-rg/setup-rg.sh | 37 +-- 6 files changed, 179 insertions(+), 156 deletions(-) mode change 100644 => 100755 mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.sh diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/README.md b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/README.md index 51153780244..fb911620b8c 100644 --- a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/README.md +++ b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/README.md @@ -11,32 +11,44 @@ The *control plane RG* is a container for the *persistent* resources of MLOS (re az login ``` -1. Make a copy of the control plane ARM parameters file. +2. Make a copy of the control plane ARM parameters file. ```sh cp rg-template.example.parameters.json rg-template.parameters.json ``` -1. (Optional) Make a copy of the results DB parameters file, if planning to provision a results DB. +3. (Optional) Make a copy of the results DB parameters file, if planning to provision a results DB (suggested). ```sh cp results-db/mysql-template.parameters.example.json results-db/mysql-template.parameters.json ``` -1. Modify the ARM parameters in the newly created files as needed, especially the `PLACEHOLDER` values. +4. Modify the ARM parameters in the newly created files as needed, especially the `PLACEHOLDER` values. -1. Execute the main script with CLI args as follows: +5. Execute the main script with CLI args as follows: ```shell + # With Powershell ./setup-rg.ps1 ` - -controlPlaneArmParameters $controlPlaneArmParams ` - -resultsDbArmParameters $resultsDbArmParams # If provisioning results DB, otherwise omit ` + -controlPlaneArmParamsFile $controlPlaneArmParamsFile ` + -resultsDbArmParamsFile $resultsDbArmParamsFile # If provisioning results DB, otherwise omit ` -servicePrincipalName $servicePrincipalName ` -resourceGroupName $resourceGroupName ` -certName $certName ``` - where `$*ArmParams` can be the corresponding `*.parameters.json` and from before. However, it also follows the same usage as `--parameters` in [az deployment group create](https://learn.microsoft.com/en-us/cli/azure/deployment/group?view=azure-cli-latest#az-deployment-group-create-examples). + ```sh + # With bash + # If provisioning results DB include '--resultsDbArmsParamsFile', otherwise omit + ./setup-rg.sh \ + --controlPlaneArmParamsFile $controlPlaneArmParamsFile \ + --resultsDbArmParamsFile $resultsDbArmParamsFile \ + --servicePrincipalName $servicePrincipalName \ + --resourceGroupName $resourceGroupName \ + --certName $certName + ``` + + where `$*ArmParamsFile` can be the corresponding `*.parameters.json` and from before. However, it also follows the same usage as `--parameters` in [az deployment group create](https://learn.microsoft.com/en-us/cli/azure/deployment/group?view=azure-cli-latest#az-deployment-group-create-examples). ## Workflow @@ -46,23 +58,23 @@ The high-level flow for what this script automates is as follows: Ideally, experiment resources are placed in their own RG. When that isn't possible, they can also be placed in the control plane RG, in which case the SP can optionally be given access to the control plane RG as well. -1. Provision control plane resources into the RG. +2. Provision control plane resources into the RG. This includes: - Control VM for running the `mlos_bench` scheduler. - Control VM's networking (public IP, security group, vnet, subnet, network interface) - Key Vault for storing the SP credentials. - Storage (storage account, file share) -1. The results DB is then optionally provisioned, adding appropriate firewall rules. +3. The results DB is then optionally provisioned, adding appropriate firewall rules. -1. Assign `Key Vault Administrator` access to the current user. +4. Assign `Key Vault Administrator` access to the current user. This allows the current user to retrieve secrets / certificates from the VM once it is set up. Ensure to log in as the same user in the VM. -1. Check if the desired certificate name already exists in the key vault. +5. Check if the desired certificate name already exists in the key vault. -1. If certificate does not exist yet, create or update the Service Principal (SP) with `Contributor` access for write over resources. +6. If certificate does not exist yet, create or update the Service Principal (SP) with `Contributor` access for write over resources. Ideally, experiment resources are placed in their own RG. When that isn't possible, they can also be placed in the control plane RG, in which case the SP can optionally be given access to the control plane RG as well. -1. Otherwise, create or update the SP just with similar access as before. Now also verify that the existing certificate in the key vault matches one linked to the SP already, via thumbprint. +7. Otherwise, create or update the SP just with similar access as before. Now also verify that the existing certificate in the key vault matches one linked to the SP already, via thumbprint. diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/results-db/mysql-template.json b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/results-db/mysql-template.json index 05ae9dfe7a8..8b2784d792a 100644 --- a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/results-db/mysql-template.json +++ b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/results-db/mysql-template.json @@ -39,9 +39,7 @@ "storage": { "storageSizeGB": 20, "iops": 360, - "autoGrow": "Enabled", - "autoIoScaling": "Disabled", - "logOnDisk": "Disabled" + "autoGrow": "Enabled" }, "version": "8.0.21", "network": { diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/results-db/mysql-template.parameters.example.json b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/results-db/mysql-template.parameters.example.json index 250aeb54129..aa910fde5ca 100644 --- a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/results-db/mysql-template.parameters.example.json +++ b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/results-db/mysql-template.parameters.example.json @@ -15,8 +15,8 @@ "value": [ { "name": "PLACEHOLDER", - "startIpAddress": "123.123.0.0", - "endIpAddress": "123.123.255.255" + "startIpAddress": "192.168.0.0", + "endIpAddress": "192.168.255.255" } ] } diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/rg-template.json b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/rg-template.json index 8cc69cbb376..76f12d416e7 100644 --- a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/rg-template.json +++ b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/rg-template.json @@ -17,7 +17,8 @@ } }, "vmAdminUsername": { - "type": "string" + "type": "string", + "defaultValue": "mlos" }, "sshPublicKeys": { "type": "array" @@ -32,6 +33,13 @@ "type": "string", "defaultValue": "mlos-file-share" }, + "vmCustomData": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Custom data for the VM (e.g., for cloud-init)." + } + }, "vmLocation": { "type": "string", "defaultValue": "[resourceGroup().location]" @@ -60,12 +68,12 @@ }, "functions": [], "variables": { - "storageAccountName": "[toLower(concat(replace(parameters('projectPrefix'), '-', '')))]", + "storageAccountName": "[toLower(replace(parameters('projectPrefix'), '-', ''))]", "vmName": "[concat(parameters('projectPrefix'), '-vm')]", "networkInterfaceName": "[concat(variables('vmName'), '-ni-', uniqueString(resourceGroup().id))]", - "vnetName": "[concat(variables('vmName'), '-vnet')]", + "vnetName": "[concat(parameters('projectPrefix'), '-vnet')]", "publicIpName": "[concat(variables('vmName'), '-ip')]", - "nsgName": "[concat(variables('vmName'), '-nsg')]", + "nsgName": "[concat(parameters('projectPrefix'), '-nsg')]", "kvName": "[concat(parameters('projectPrefix'), '-kv')]" }, "resources": [ @@ -196,7 +204,7 @@ "enabledForDeployment": true, "enabledForTemplateDeployment": true, "enabledForDiskEncryption": true, - "tenantId": "72f988bf-86f1-41af-91ab-2d7cd011db47", + "tenantId": "[subscription().tenantId]", "accessPolicies": [], "enableRbacAuthorization": true, "networkAcls": { @@ -230,6 +238,7 @@ }, "osProfile": { "computerName": "[variables('vmName')]", + "customData": "[if(empty(parameters('vmCustomData')), null(), base64(parameters('vmCustomData')))]", "adminUsername": "[parameters('vmAdminUsername')]", "linuxConfiguration": { "disablePasswordAuthentication": true, diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 index 1b0ebae36c1..9472abdb877 100644 --- a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 +++ b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 @@ -1,115 +1,118 @@ - -param( - # Main control plane ARM template params - [Parameter(Mandatory=$True)] - [string] $controlPlaneArmParameters, - # Results DB ARM template params - [Parameter(Mandatory=$False)] - [string] $resultsDbArmParameters, - # Other params - [Parameter(Mandatory=$True)] - [string] $servicePrincipalName, - [Parameter(Mandatory=$True)] - [string] $resourceGroupName, - [Parameter(Mandatory=$True)] - [string] $certName, - [Parameter(Mandatory=$False)] - [int] $certExpirationYears = 1 -) - -# Provision resources into the resource group with ARM template -Write-Output "Provisioning control plane resources..." -$deploymentResults = az deployment group create ` - --resource-group $resourceGroupName ` - --template-file .\rg-template.json ` - --parameters $controlPlaneArmParameters ` - | ConvertFrom-JSON - -if (!$?) { - Write-Error "Error in provisioning control plane resources!" - return -} - -# Conditional provisioning of results DB -if ($resultsDbArmParameters) { - Write-Output "Provisioning results DB..." - $dbDeploymentResults = az deployment group create ` - --resource-group $resourceGroupName ` - --template-file ./results-db/mysql-template.json ` - --parameters $resultsDbArmParameters ` - | ConvertFrom-JSON - - if (!$?) { - Write-Error "Error in provisioning results DB!" - } - else { - $dbName = $dbDeploymentResults.properties.outputs.dbName.value - $vmName = $deploymentResults.properties.outputs.vmName.value - $vmIpAddress = $deploymentResults.properties.outputs.vmIpAddress.value - - # VM IP access for results DB - az mysql flexible-server firewall-rule update ` - --resource-group $resourceGroupName ` - --name $dbName ` - --rule-name "AllowVM-$vmName" ` - --start-ip-address $vmIpAddress ` - --end-ip-address $vmIpAddress - } -} - -$currentUserAlias = az account show --query "user.name" --output tsv -$resourceGroupId = az group show --name $resourceGroupName --query "id" --output tsv - -# Assign 'Key Vault Administrator' access to current user -$kvName = $deploymentResults.properties.outputs.kvName.value -$kvId = az keyvault show --name $kvName --resource-group $resourceGroupName --query "id" --output tsv -az role assignment create ` - --assignee $currentUserAlias ` - --role "Key Vault Administrator" ` - --scope $kvId - -# Check if cert of same name exists in keyvault already -$certThumbprint = az keyvault certificate show ` - --name $certName ` - --vault-name $kvName ` - --query "x509ThumbprintHex" --output tsv ` - 2>$null - -if (!$?) { - # The cert does not exist yet. - # Create the service principal if doesn't exist, storing the cert in the keyvault - # If it does exist, this also patches the current service principal with the role - az ad sp create-for-rbac ` - --name $servicePrincipalName ` - --role "Contributor" ` - --scopes $resourceGroupId ` - --create-cert ` - --cert $certName ` - --keyvault $kvName ` - --years $certExpirationYears -} else { - # The cert already exists in the keyvault. - - # Ensure the SP exists with correct roles, without creating a cert. - az ad sp create-for-rbac ` - --name $servicePrincipalName ` - --role "Contributor" ` - --scopes $resourceGroupId ` - - # SP's certs, which are stored in the registered application instead - $servicePrincipalAppId = az ad sp list ` - --display-name $servicePrincipalName ` - --query "[?servicePrincipalType == 'Application'].appId" ` - --output tsv - $spCertThumbprints = az ad app credential list ` - --id $servicePrincipalAppId ` - --cert ` - --query "[].customKeyIdentifier" ` - --output tsv - - if ($spCertThumbprints.Contains($certThumbprint)) { - Write-Output "Keyvault contains the certificate '$certName' that is linked to the service principal '$servicePrincipalName' already." - } else { - Write-Warning "Keyvault already contains a certificate called '$certName', but is not linked with the service principal '$servicePrincipalName'! Skipping cert handling" - } -} + +param( + # Main control plane ARM template params + [Parameter(Mandatory=$True)] + [string] $controlPlaneArmParamsFile, + # Results DB ARM template params + [Parameter(Mandatory=$False)] + [string] $resultsDbArmParamsFile, + # Other params + [Parameter(Mandatory=$True)] + [string] $servicePrincipalName, + [Parameter(Mandatory=$True)] + [string] $resourceGroupName, + [Parameter(Mandatory=$True)] + [string] $certName, + [Parameter(Mandatory=$False)] + [int] $certExpirationYears = 1 +) + +# Provision resources into the resource group with ARM template +Write-Output "Provisioning control plane resources..." +$deploymentResults = az deployment group create ` + --resource-group $resourceGroupName ` + --template-file .\rg-template.json ` + --parameters $controlPlaneArmParamsFile ` + --output json ` + | ConvertFrom-JSON + +if (!$?) { + Write-Error "Error in provisioning control plane resources!" + return +} + +# Conditional provisioning of results DB +if ($resultsDbArmParamsFile) { + Write-Output "Provisioning results DB..." + $dbDeploymentResults = az deployment group create ` + --resource-group $resourceGroupName ` + --template-file ./results-db/mysql-template.json ` + --parameters $resultsDbArmParamsFile ` + --output json ` + | ConvertFrom-JSON + + if (!$?) { + Write-Error "Error in provisioning results DB!" + } + else { + $dbName = $dbDeploymentResults.properties.outputs.dbName.value + $vmName = $deploymentResults.properties.outputs.vmName.value + $vmIpAddress = $deploymentResults.properties.outputs.vmIpAddress.value + + # VM IP access for results DB + az mysql flexible-server firewall-rule create ` + --resource-group $resourceGroupName ` + --name $dbName ` + --rule-name "AllowVM-$vmName" ` + --start-ip-address $vmIpAddress ` + --end-ip-address $vmIpAddress + } +} + +$currentUserAlias = az account show --query "user.name" --output tsv +$resourceGroupId = az group show --name $resourceGroupName --query "id" --output tsv + +# Assign 'Key Vault Administrator' access to current user +$kvName = $deploymentResults.properties.outputs.kvName.value +$kvId = az keyvault show --name $kvName --resource-group $resourceGroupName --query "id" --output tsv +az role assignment create ` + --assignee $currentUserAlias ` + --role "Key Vault Administrator" ` + --scope $kvId + +# Check if cert of same name exists in keyvault already +$certThumbprint = az keyvault certificate show ` + --name $certName ` + --vault-name $kvName ` + --query "x509ThumbprintHex" --output tsv ` + 2>$null ` + || "NOCERT" + +if ($certThumbprint == "NOCERT") { + # The cert does not exist yet. + # Create the service principal if doesn't exist, storing the cert in the keyvault + # If it does exist, this also patches the current service principal with the role + az ad sp create-for-rbac ` + --name $servicePrincipalName ` + --role "Contributor" ` + --scopes $resourceGroupId ` + --create-cert ` + --cert $certName ` + --keyvault $kvName ` + --years $certExpirationYears +} else { + # The cert already exists in the keyvault. + + # Ensure the SP exists with correct roles, without creating a cert. + az ad sp create-for-rbac ` + --name $servicePrincipalName ` + --role "Contributor" ` + --scopes $resourceGroupId ` + + # SP's certs, which are stored in the registered application instead + $servicePrincipalAppId = az ad sp list ` + --display-name $servicePrincipalName ` + --query "[?servicePrincipalType == 'Application'].appId" ` + --output tsv + $spCertThumbprints = az ad app credential list ` + --id $servicePrincipalAppId ` + --cert ` + --query "[].customKeyIdentifier" ` + --output tsv + + if ($spCertThumbprints.Contains($certThumbprint)) { + Write-Output "Keyvault contains the certificate '$certName' that is linked to the service principal '$servicePrincipalName' already." + } else { + Write-Warning "Keyvault already contains a certificate called '$certName', but is not linked with the service principal '$servicePrincipalName'! Skipping cert handling" + } +} diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.sh b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.sh old mode 100644 new mode 100755 index 049ccb81dae..be979ad00d6 --- a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.sh +++ b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.sh @@ -10,13 +10,13 @@ set -eu while [[ "$#" -gt 0 ]]; do case $1 in # Main control plane ARM template params - --controlPlaneArmParameters) - controlPlaneArmParameters="$2" + --controlPlaneArmParamsFile) + controlPlaneArmParamsFile="$2" shift 2 ;; # Results DB ARM template params - --resultsDbArmParameters) - resultsDbArmParameters="$2" + --resultsDbArmParamsFile) + resultsDbArmParamsFile="$2" shift 2 ;; # Other params @@ -44,7 +44,7 @@ esac done # Default values -resultsDbArmParameters=${resultsDbArmParameters:-""} +resultsDbArmParamsFile=${resultsDbArmParamsFile:-""} certExpirationYears=${certExpirationYears:-1} # Provision resources into the resource group with ARM template @@ -52,24 +52,26 @@ echo "Provisioning control plane resources..." deploymentResults=$(az deployment group create \ --resource-group "$resourceGroupName" \ --template-file ./rg-template.json \ - --parameters "$controlPlaneArmParameters" \ + --parameters "$controlPlaneArmParamsFile" \ + --output json \ ) -if [[ ! $? ]]; then +if [[ $? -ne 0 ]]; then echo "Error in provisioning control plane resources!" exit 1 fi # Conditional provisioning of results DB -if [[ "$resultsDbArmParameters" ]]; then +if [[ "$resultsDbArmParamsFile" ]]; then echo "Provisioning results DB..." dbDeploymentResults=$(az deployment group create \ --resource-group "$resourceGroupName" \ --template-file ./results-db/mysql-template.json \ - --parameters "$resultsDbArmParameters" \ + --parameters "$resultsDbArmParamsFile" \ + --output json \ ) - if [[ ! $? ]]; then + if [[ $? -ne 0 ]]; then echo "Error in provisioning results DB!" else dbName=$(echo "$dbDeploymentResults" | jq -r ".properties.outputs.dbName.value") @@ -77,13 +79,12 @@ if [[ "$resultsDbArmParameters" ]]; then vmIpAddress=$(echo "$deploymentResults" | jq -r ".properties.outputs.vmIpAddress.value") # VM IP access for results DB - az mysql flexible-server firewall-rule update \ + az mysql flexible-server firewall-rule create \ --resource-group "$resourceGroupName" \ --name "$dbName" \ --rule-name "AllowVM-$vmName" \ --start-ip-address "$vmIpAddress" \ - --end-ip-address "$vmIpAddress" \ - + --end-ip-address "$vmIpAddress" fi fi @@ -96,7 +97,7 @@ kvId=$(az keyvault show --name "$kvName" --resource-group "$resourceGroupName" - az role assignment create \ --assignee "$currentUserAlias" \ --role "Key Vault Administrator" \ - --scope "$kvId" \ + --scope "$kvId" # Check if cert of same name exists in keyvault already certThumbprint=$(az keyvault certificate show \ @@ -104,7 +105,8 @@ certThumbprint=$(az keyvault certificate show \ --vault-name "$kvName" \ --query "x509ThumbprintHex" --output tsv \ 2> /dev/null \ - || echo "NOCERT") + || echo "NOCERT" \ + ) if [[ $certThumbprint == "NOCERT" ]]; then # The cert does not exist yet. @@ -117,8 +119,7 @@ if [[ $certThumbprint == "NOCERT" ]]; then --create-cert \ --cert "$certName" \ --keyvault "$kvName" \ - --years "$certExpirationYears" \ - + --years "$certExpirationYears" else # The cert already exists in the keyvault. @@ -126,7 +127,7 @@ else az ad sp create-for-rbac \ --name "$servicePrincipalName" \ --role "Contributor" \ - --scopes "$resourceGroupId" \ + --scopes "$resourceGroupId" # SP's certs, which are stored in the registered application instead servicePrincipalAppId=$(az ad sp list \ From a4ad480abfe7ebf091d36c2144ab82a739258b1d Mon Sep 17 00:00:00 2001 From: Eu Jing Chua Date: Wed, 6 Sep 2023 18:31:28 +0000 Subject: [PATCH 10/11] Address CI issues --- .../services/remote/azure/scripts/setup-rg/README.md | 2 +- .../remote/azure/scripts/setup-rg/rg-template.json | 1 + .../services/remote/azure/scripts/setup-rg/setup-rg.ps1 | 4 ++++ .../config/services/test_load_service_config_examples.py | 9 +++++---- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/README.md b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/README.md index fb911620b8c..f3c0953d6f2 100644 --- a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/README.md +++ b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/README.md @@ -28,7 +28,7 @@ The *control plane RG* is a container for the *persistent* resources of MLOS (re 5. Execute the main script with CLI args as follows: ```shell - # With Powershell + # With Powershell, recommended to use Powershell 7 ./setup-rg.ps1 ` -controlPlaneArmParamsFile $controlPlaneArmParamsFile ` -resultsDbArmParamsFile $resultsDbArmParamsFile # If provisioning results DB, otherwise omit ` diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/rg-template.json b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/rg-template.json index 76f12d416e7..0a8998abb8c 100644 --- a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/rg-template.json +++ b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/rg-template.json @@ -257,6 +257,7 @@ } }, "storageProfile": { + // TODO: Look into automatically use the latest Ubuntu Server LTS "imageReference": { "publisher": "Canonical", "offer": "0001-com-ubuntu-server-jammy", diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 index 9472abdb877..f749bba52a6 100644 --- a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 +++ b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 @@ -1,3 +1,7 @@ +## +## Copyright (c) Microsoft Corporation. +## Licensed under the MIT License. +## param( # Main control plane ARM template params diff --git a/mlos_bench/mlos_bench/tests/config/services/test_load_service_config_examples.py b/mlos_bench/mlos_bench/tests/config/services/test_load_service_config_examples.py index 256b46c3211..d2849fb614a 100644 --- a/mlos_bench/mlos_bench/tests/config/services/test_load_service_config_examples.py +++ b/mlos_bench/mlos_bench/tests/config/services/test_load_service_config_examples.py @@ -27,10 +27,11 @@ def filter_configs(configs_to_filter: List[str]) -> List[str]: """If necessary, filter out json files that aren't for the module we're testing.""" - for config_path in configs_to_filter: - if config_path.endswith("arm-templates/azuredeploy-ubuntu-vm.jsonc"): - configs_to_filter.remove(config_path) - return configs_to_filter + def predicate(config_path: str) -> bool: + vm_template = config_path.endswith("arm-templates/azuredeploy-ubuntu-vm.jsonc") + setup_rg_scripts = config_path.find("azure/scripts/setup-rg") >= 0 + return not (vm_template or setup_rg_scripts) + return [config_path for config_path in configs_to_filter if predicate(config_path)] configs = locate_config_examples(ConfigPersistenceService.BUILTIN_CONFIG_PATH, CONFIG_TYPE, filter_configs) From bf370987d761f70642ab7e09cc9d47bf9767613e Mon Sep 17 00:00:00 2001 From: Eu Jing Chua Date: Wed, 6 Sep 2023 20:21:42 +0000 Subject: [PATCH 11/11] Update PS requirements --- .../config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 index f749bba52a6..c32e41279b0 100644 --- a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 +++ b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/setup-rg.ps1 @@ -1,3 +1,4 @@ +#Requires -Version 7 ## ## Copyright (c) Microsoft Corporation. ## Licensed under the MIT License.