feat: create kms key as part of cluster bootstrap

parts/k8s/cloud-init/artifacts/cse_config.sh

@@ -368,6 +368,13 @@ ensureDHCPv6() {
   fi
 }
 {{end}}
+{{- if EnableEncryptionWithExternalKms}}
+ensureKMSKeyvaultKey() {
+    wait_for_file 3600 1 {{GetKMSKeyvaultKeyServiceCSEScriptFilepath}} || exit {{GetCSEErrorCode "ERR_FILE_WATCH_TIMEOUT"}}
+    wait_for_file 3600 1 {{GetKMSKeyvaultKeyCSEScriptFilepath}} || exit {{GetCSEErrorCode "ERR_FILE_WATCH_TIMEOUT"}}
+    systemctlEnableAndStart kms-keyvault-key || exit {{GetCSEErrorCode "ERR_SYSTEMCTL_START_FAIL"}}
+}
+{{end}}
 ensureKubelet() {
   wait_for_file 1200 1 /etc/sysctl.d/11-aks-engine.conf || exit {{GetCSEErrorCode "ERR_FILE_WATCH_TIMEOUT"}}
   sysctl_reload 10 5 120 || exit {{GetCSEErrorCode "ERR_SYSCTL_RELOAD"}}
@@ -564,6 +571,7 @@ configAzurePolicyAddon() {
   sed -i "s|<resourceId>|/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP|g" $ADDONS_DIR/azure-policy-deployment.yaml
 }
 {{end}}
+
 configAddons() {
   {{if IsClusterAutoscalerAddonEnabled}}
   if [[ ${CLUSTER_AUTOSCALER_ADDON} == true ]]; then

parts/k8s/cloud-init/artifacts/cse_main.sh

@@ -219,6 +219,13 @@ time_metric "EnsureContainerd" ensureContainerd
 time_metric "EnsureDHCPv6" ensureDHCPv6
 {{end}}
 
+if [[ -n ${MASTER_NODE} ]]; then
+  {{/* configure and enable kms plugin */}}
+  {{- if EnableEncryptionWithExternalKms}}
+  time_metric "EnsureKMSKeyvaultKey" ensureKMSKeyvaultKey
+  {{end}}
+fi
+
 time_metric "EnsureKubelet" ensureKubelet
 {{if IsAzurePolicyAddonEnabled}}
 if [[ -n ${MASTER_NODE} ]]; then

parts/k8s/cloud-init/artifacts/kms-keyvault-key.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=setupkmskey
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart={{GetKMSKeyvaultKeyCSEScriptFilepath}}
+
+[Install]
+WantedBy=multi-user.target
+#EOF

parts/k8s/cloud-init/artifacts/kms-keyvault-key.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+set +x
+set -euo pipefail
+
+AZURE_JSON_PATH="/etc/kubernetes/azure.json"
+SERVICE_PRINCIPAL_CLIENT_ID=$(jq -r '.aadClientId' ${AZURE_JSON_PATH})
+SERVICE_PRINCIPAL_CLIENT_SECRET=$(jq -r '.aadClientSecret' ${AZURE_JSON_PATH})
+TENANT_ID=$(jq -r '.tenantId' ${AZURE_JSON_PATH})
+KMS_KEYVAULT_NAME=$(jq -r '.providerVaultName' ${AZURE_JSON_PATH})
+KMS_KEY_NAME=$(jq -r '.providerKeyName' ${AZURE_JSON_PATH})
+USER_ASSIGNED_IDENTITY_ID=$(jq -r '.userAssignedIdentityID' ${AZURE_JSON_PATH})
+PROVIDER_KEY_VERSION=$(jq -r '.providerKeyVersion' ${AZURE_JSON_PATH})
+
+TOKEN_URL="https://login.microsoftonline.com/$TENANT_ID/oauth2/token"
+SCOPE="https://vault.azure.net"
+KEYVAULT_URL="https://$KMS_KEYVAULT_NAME.vault.azure.net/keys/$KMS_KEY_NAME/versions?maxresults=1&api-version=7.1"
+KMS_KUBERNETES_FILE=/etc/kubernetes/manifests/kube-azure-kms.yaml
+
+# provider key version already exists
+# this will be the case for BYOK
+if [[ -n $PROVIDER_KEY_VERSION ]]; then
+    echo "KMS provider key version already exists"
+    exit 0
+fi
+
+echo "Generating token for Azure Key Vault"
+echo "------------------------------------------------------------------------"
+echo "Parameters"
+echo "------------------------------------------------------------------------"
+echo "SERVICE_PRINCIPAL_CLIENT_ID:     ..."
+echo "SERVICE_PRINCIPAL_CLIENT_SECRET: ..."
+echo "TENANT_ID:                       $TENANT_ID"
+echo "TOKEN_URL:                       $TOKEN_URL"
+echo "SCOPE:                           $SCOPE"
+echo "------------------------------------------------------------------------"
+
+if [[ $SERVICE_PRINCIPAL_CLIENT_ID == "msi" ]] && [[ $SERVICE_PRINCIPAL_CLIENT_SECRET == "msi" ]]; then
+    if [[ -z $USER_ASSIGNED_IDENTITY_ID ]]; then
+        # using system-assigned identity to access keyvault
+        TOKEN=$(curl -s --retry 5 --retry-delay 10 --max-time 60 \
+            -H Metadata:true \
+            "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=$SCOPE" | jq '.access_token' | xargs)
+    else
+        # using user-assigned managed identity to access keyvault
+        TOKEN=$(curl -s --retry 5 --retry-delay 10 --max-time 60 \
+            -H Metadata:true \
+            "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&client_id=$USER_ASSIGNED_IDENTITY_ID&resource=$SCOPE" | jq '.access_token' | xargs)
+    fi
+else
+    # use service principal token to access keyvault
+    TOKEN=$(curl -s --retry 5 --retry-delay 10 --max-time 60 -f -X POST \
+        -H "Content-Type: application/x-www-form-urlencoded" \
+        -d "grant_type=client_credentials" \
+        -d "client_id=$SERVICE_PRINCIPAL_CLIENT_ID" \
+        --data-urlencode "client_secret=$SERVICE_PRINCIPAL_CLIENT_SECRET" \
+        --data-urlencode "resource=$SCOPE" \
+        ${TOKEN_URL} | jq '.access_token' | xargs)
+fi
+
+
+if [[ -z $TOKEN ]]; then
+    echo "Error generating token for Azure Keyvault"
+    exit 120
+fi
+
+# Get the keyID for the kms key created as part of cluster bootstrap
+KEY_ID=$(curl -s --retry 5 --retry-delay 10 --max-time 60 -f \
+    ${KEYVAULT_URL} -H "Authorization: Bearer ${TOKEN}" | jq '.value[0].kid' | xargs)
+
+if [[ -z "$KEY_ID" || "$KEY_ID" == "null" ]]; then
+    echo "Error getting the kms key version"
+    exit 120
+fi
+
+# KID format: https://<keyvault name>.vault.azure.net/keys/<key name>/<key version>
+# Example KID: "https://akv0112master.vault.azure.net/keys/k8s/128a3d9956bc44feb6a0e2c2f35b732c"
+KEY_VERSION=${KEY_ID##*/}
+
+# Set the version in azure.json
+if [ -f $AZURE_JSON_PATH ]; then
+    # once the version is set in azure.json, kms plugin will just default to using the key
+    # this will be changed in upcoming kms release to set the version as container args
+    tmpDir=$(mktemp -d "$(pwd)/XXX")
+    jq --arg KEY_VERSION ${KEY_VERSION} '.providerKeyVersion=($KEY_VERSION)' "$AZURE_JSON_PATH" > $tmpDir/tmp
+    mv $tmpDir/tmp "$AZURE_JSON_PATH"
+    rm -Rf $tmpDir
+fi
+
+set -x
+#EOF

parts/k8s/cloud-init/masternodecustomdata.yml

@@ -380,6 +380,20 @@ write_files:
             endpoint: unix:///opt/azurekms.socket
             cachesize: 1000
         - identity: {}
+
+- path: {{GetKMSKeyvaultKeyServiceCSEScriptFilepath}}
+  permissions: "0644"
+  encoding: gzip
+  owner: root
+  content: !!binary |
+    {{CloudInitData "kmsKeyvaultKeySystemdService"}}
+
+- path: {{GetKMSKeyvaultKeyCSEScriptFilepath}}
+  permissions: "0544"
+  encoding: gzip
+  owner: root
+  content: !!binary |
+    {{CloudInitData "kmsKeyvaultKeyScript"}}
 {{end}}
 
 MASTER_MANIFESTS_CONFIG_PLACEHOLDER

pkg/api/const.go

@@ -229,7 +229,7 @@ const (
 	APIVersionAuthorizationSystem = "2018-09-01-preview"
 	APIVersionCompute             = "2019-07-01"
 	APIVersionDeployments         = "2018-06-01"
-	APIVersionKeyVault            = "2018-02-14"
+	APIVersionKeyVault            = "2019-09-01"
 	APIVersionManagedIdentity     = "2018-11-30"
 	APIVersionNetwork             = "2018-08-01"
 	APIVersionStorage             = "2018-07-01"

pkg/engine/armvariables.go

@@ -79,6 +79,7 @@ func getK8sMasterVars(cs *api.ContainerService) (map[string]interface{}, error)
 		excludeMasterFromStandardLB = to.Bool(kubernetesConfig.ExcludeMasterFromStandardLB)
 		maxLoadBalancerCount = kubernetesConfig.MaximumLoadBalancerRuleCount
 		provisionJumpbox = kubernetesConfig.PrivateJumpboxProvision()
+		enableEncryptionWithExternalKms = to.Bool(kubernetesConfig.EnableEncryptionWithExternalKms)
 
 		if kubernetesConfig.ShouldCreateNewUserAssignedIdentity() {
 			userAssignedIDReference = "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('userAssignedID'))]"
@@ -157,6 +158,12 @@ func getK8sMasterVars(cs *api.ContainerService) (map[string]interface{}, error)
 		"dockerMonitorSystemdService":  getBase64EncodedGzippedCustomScript(kubernetesDockerMonitorSystemdService, cs),
 	}
 
+	if enableEncryptionWithExternalKms {
+		fmt.Println("being processed")
+		cloudInitFiles["kmsKeyvaultKeySystemdService"] = getBase64EncodedGzippedCustomScript(kmsKeyvaultKeySystemdService, cs)
+		cloudInitFiles["kmsKeyvaultKeyScript"] = getBase64EncodedGzippedCustomScript(kmsKeyvaultKeyScript, cs)
+	}
+
 	if cs.Properties.OrchestratorProfile.KubernetesConfig.IsAddonEnabled(common.AADPodIdentityAddonName) {
 		cloudInitFiles["untaintNodesScript"] = getBase64EncodedGzippedCustomScript(untaintNodesScript, cs)
 		cloudInitFiles["untaintNodesSystemdService"] = getBase64EncodedGzippedCustomScript(untaintNodesSystemdService, cs)

pkg/engine/armvariables_test.go

@@ -90,7 +90,7 @@ func TestK8sVars(t *testing.T) {
 		"apiVersionAuthorizationUser":        "2018-09-01-preview",
 		"apiVersionCompute":                  "2019-07-01",
 		"apiVersionDeployments":              "2018-06-01",
-		"apiVersionKeyVault":                 "2018-02-14",
+		"apiVersionKeyVault":                 "2019-09-01",
 		"apiVersionManagedIdentity":          "2018-11-30",
 		"apiVersionNetwork":                  "2018-08-01",
 		"apiVersionStorage":                  "2018-07-01",
@@ -201,7 +201,6 @@ func TestK8sVars(t *testing.T) {
 	}
 
 	diff := cmp.Diff(varMap, expectedMap)
-
 	if diff != "" {
 		t.Errorf("unexpected diff while expecting equal structs: %s", diff)
 	}
@@ -810,7 +809,7 @@ func TestK8sVarsMastersOnly(t *testing.T) {
 		"apiVersionAuthorizationUser":        "2018-09-01-preview",
 		"apiVersionCompute":                  "2019-07-01",
 		"apiVersionDeployments":              "2018-06-01",
-		"apiVersionKeyVault":                 "2018-02-14",
+		"apiVersionKeyVault":                 "2019-09-01",
 		"apiVersionManagedIdentity":          "2018-11-30",
 		"apiVersionNetwork":                  "2018-08-01",
 		"apiVersionStorage":                  "2018-07-01",
@@ -929,6 +928,37 @@ func TestK8sVarsMastersOnly(t *testing.T) {
 	if diff != "" {
 		t.Errorf("unexpected diff while expecting equal structs: %s", diff)
 	}
+
+	// enable external kms encryption
+	cs.Properties.OrchestratorProfile.KubernetesConfig.EnableEncryptionWithExternalKms = to.BoolPtr(true)
+	expectedMap["clusterKeyVaultName"] = string("[take(concat('kv', tolower(uniqueString(concat(variables('masterFqdnPrefix'),variables('location'),parameters('nameSuffix'))))), 22)]")
+	expectedMap["cloudInitFiles"] = map[string]interface{}{
+		"provisionScript":              getBase64EncodedGzippedCustomScript(kubernetesCSEMainScript, cs),
+		"provisionSource":              getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScript, cs),
+		"provisionInstalls":            getBase64EncodedGzippedCustomScript(kubernetesCSEInstall, cs),
+		"provisionConfigs":             getBase64EncodedGzippedCustomScript(kubernetesCSEConfig, cs),
+		"customSearchDomainsScript":    getBase64EncodedGzippedCustomScript(kubernetesCustomSearchDomainsScript, cs),
+		"etcdSystemdService":           getBase64EncodedGzippedCustomScript(etcdSystemdService, cs),
+		"dhcpv6ConfigurationScript":    getBase64EncodedGzippedCustomScript(dhcpv6ConfigurationScript, cs),
+		"dhcpv6SystemdService":         getBase64EncodedGzippedCustomScript(dhcpv6SystemdService, cs),
+		"kubeletSystemdService":        getBase64EncodedGzippedCustomScript(kubeletSystemdService, cs),
+		"etcdMonitorSystemdService":    getBase64EncodedGzippedCustomScript(etcdMonitorSystemdService, cs),
+		"healthMonitorScript":          getBase64EncodedGzippedCustomScript(kubernetesHealthMonitorScript, cs),
+		"kubeletMonitorSystemdService": getBase64EncodedGzippedCustomScript(kubernetesKubeletMonitorSystemdService, cs),
+		"dockerMonitorSystemdService":  getBase64EncodedGzippedCustomScript(kubernetesDockerMonitorSystemdService, cs),
+		"kmsKeyvaultKeySystemdService": getBase64EncodedGzippedCustomScript(kmsKeyvaultKeySystemdService, cs),
+		"kmsKeyvaultKeyScript":         getBase64EncodedGzippedCustomScript(kmsKeyvaultKeyScript, cs),
+	}
+
+	varMap, err = GetKubernetesVariables(cs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	diff = cmp.Diff(varMap, expectedMap)
+
+	if diff != "" {
+		t.Errorf("unexpected diff while expecting equal structs: %s", diff)
+	}
 }
 
 func TestK8sVarsWindowsProfile(t *testing.T) {

pkg/engine/const.go

@@ -108,17 +108,22 @@ const (
 	// scripts and service for enabling ipv6 dual stack
 	dhcpv6SystemdService      = "k8s/cloud-init/artifacts/dhcpv6.service"
 	dhcpv6ConfigurationScript = "k8s/cloud-init/artifacts/enable-dhcpv6.sh"
+	// script for getting key version from keyvault for kms
+	kmsKeyvaultKeySystemdService = "k8s/cloud-init/artifacts/kms-keyvault-key.service"
+	kmsKeyvaultKeyScript         = "k8s/cloud-init/artifacts/kms-keyvault-key.sh"
 )
 
 // cloud-init destination file references
 const (
-	customCloudConfigCSEScriptFilepath   = "/opt/azure/containers/provision_configs_custom_cloud.sh"
-	cseHelpersScriptFilepath             = "/opt/azure/containers/provision_source.sh"
-	cseInstallScriptFilepath             = "/opt/azure/containers/provision_installs.sh"
-	cseConfigScriptFilepath              = "/opt/azure/containers/provision_configs.sh"
-	customSearchDomainsCSEScriptFilepath = "/opt/azure/containers/setup-custom-search-domains.sh"
-	dhcpV6ServiceCSEScriptFilepath       = "/etc/systemd/system/dhcpv6.service"
-	dhcpV6ConfigCSEScriptFilepath        = "/opt/azure/containers/enable-dhcpv6.sh"
+	customCloudConfigCSEScriptFilepath     = "/opt/azure/containers/provision_configs_custom_cloud.sh"
+	cseHelpersScriptFilepath               = "/opt/azure/containers/provision_source.sh"
+	cseInstallScriptFilepath               = "/opt/azure/containers/provision_installs.sh"
+	cseConfigScriptFilepath                = "/opt/azure/containers/provision_configs.sh"
+	customSearchDomainsCSEScriptFilepath   = "/opt/azure/containers/setup-custom-search-domains.sh"
+	dhcpV6ServiceCSEScriptFilepath         = "/etc/systemd/system/dhcpv6.service"
+	dhcpV6ConfigCSEScriptFilepath          = "/opt/azure/containers/enable-dhcpv6.sh"
+	kmsKeyvaultKeyServiceCSEScriptFilepath = "/etc/systemd/system/kms-keyvault-key.service"
+	kmsKeyvaultKeyCSEScriptFilepath        = "/opt/azure/containers/kms-keyvault-key.sh"
 )
 
 const (

pkg/engine/keyvaults.go

@@ -5,6 +5,7 @@ package engine
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/Azure/aks-engine/pkg/api"
 	"github.com/Azure/go-autorest/autorest/to"
@@ -146,3 +147,29 @@ func CreateKeyVaultVMSS(cs *api.ContainerService) map[string]interface{} {
 
 	return keyVaultMap
 }
+
+func CreateKeyVaultKey(cs *api.ContainerService) map[string]interface{} {
+	keyMap := map[string]interface{}{
+		"type":       "Microsoft.KeyVault/vaults/keys",
+		"name":       "[concat(variables('clusterKeyVaultName'), '/', 'k8s')]",
+		"apiVersion": "[variables('apiVersionKeyVault')]",
+		"location":   "[variables('location')]",
+		"dependsOn": []string{
+			"[resourceId('Microsoft.KeyVault/vaults', variables('clusterKeyVaultName'))]",
+		},
+	}
+	keyType := "RSA"
+	if strings.EqualFold(cs.Properties.OrchestratorProfile.KubernetesConfig.KeyVaultSku, "premium") {
+		keyType = "RSA-HSM"
+	}
+	keyProps := map[string]interface{}{
+		"kty": keyType,
+		"keyOps": []string{
+			"encrypt",
+			"decrypt",
+		},
+		"keySize": 2048,
+	}
+	keyMap["properties"] = keyProps
+	return keyMap
+}

pkg/engine/masterarmresources.go

@@ -92,7 +92,8 @@ func createKubernetesMasterResourcesVMAS(cs *api.ContainerService) []interface{}
 	if isKMSEnabled {
 		keyVaultStorageAccount := createKeyVaultStorageAccount()
 		keyVault := CreateKeyVaultVMAS(cs)
-		masterResources = append(masterResources, keyVaultStorageAccount, keyVault)
+		keyVaultKey := CreateKeyVaultKey(cs)
+		masterResources = append(masterResources, keyVaultStorageAccount, keyVault, keyVaultKey)
 	}
 
 	if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") {
@@ -194,7 +195,8 @@ func createKubernetesMasterResourcesVMSS(cs *api.ContainerService) []interface{}
 	if isKMSEnabled {
 		keyVaultStorageAccount := createKeyVaultStorageAccount()
 		keyVault := CreateKeyVaultVMSS(cs)
-		masterResources = append(masterResources, keyVaultStorageAccount, keyVault)
+		keyVaultKey := CreateKeyVaultKey(cs)
+		masterResources = append(masterResources, keyVaultStorageAccount, keyVault, keyVaultKey)
 	}
 
 	if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") {

pkg/engine/params_k8s.go

@@ -119,15 +119,18 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params
 			} else {
 				addValue(parametersMap, "servicePrincipalClientSecret", servicePrincipalProfile.Secret)
 			}
+		}
+	}
 
-			if kubernetesConfig != nil && to.Bool(kubernetesConfig.EnableEncryptionWithExternalKms) {
-				if kubernetesConfig.KeyVaultSku != "" {
-					addValue(parametersMap, "clusterKeyVaultSku", kubernetesConfig.KeyVaultSku)
-				}
-				if !to.Bool(kubernetesConfig.UseManagedIdentity) && servicePrincipalProfile.ObjectID != "" {
-					addValue(parametersMap, "servicePrincipalObjectId", servicePrincipalProfile.ObjectID)
-				}
-			}
+	// configure params required for external kms
+	if kubernetesConfig != nil && to.Bool(kubernetesConfig.EnableEncryptionWithExternalKms) {
+		servicePrincipalProfile := properties.ServicePrincipalProfile
+
+		if kubernetesConfig.KeyVaultSku != "" {
+			addValue(parametersMap, "clusterKeyVaultSku", kubernetesConfig.KeyVaultSku)
+		}
+		if !to.Bool(kubernetesConfig.UseManagedIdentity) && servicePrincipalProfile.ObjectID != "" {
+			addValue(parametersMap, "servicePrincipalObjectId", servicePrincipalProfile.ObjectID)
 		}
 	}
 

pkg/engine/template_generator.go

@@ -703,6 +703,12 @@ func getContainerServiceFuncMap(cs *api.ContainerService) template.FuncMap {
 		"GetDHCPv6ConfigCSEScriptFilepath": func() string {
 			return dhcpV6ConfigCSEScriptFilepath
 		},
+		"GetKMSKeyvaultKeyServiceCSEScriptFilepath": func() string {
+			return kmsKeyvaultKeyServiceCSEScriptFilepath
+		},
+		"GetKMSKeyvaultKeyCSEScriptFilepath": func() string {
+			return kmsKeyvaultKeyCSEScriptFilepath
+		},
 		"HasPrivateAzureRegistryServer": func() bool {
 			return cs.Properties.OrchestratorProfile.KubernetesConfig.PrivateAzureRegistryServer != ""
 		},