From 83bd5b42025c1ffe726cd8c8106568c19ff2a91f Mon Sep 17 00:00:00 2001 From: Anish Ramasekar Date: Thu, 14 Jan 2021 13:04:40 -0800 Subject: [PATCH] feat: create kms key as part of cluster bootstrap --- parts/k8s/cloud-init/artifacts/cse_config.sh | 8 + parts/k8s/cloud-init/artifacts/cse_main.sh | 7 + .../artifacts/kms-keyvault-key.service | 11 ++ .../cloud-init/artifacts/kms-keyvault-key.sh | 91 ++++++++++ parts/k8s/cloud-init/masternodecustomdata.yml | 14 ++ pkg/api/const.go | 2 +- pkg/engine/armvariables.go | 7 + pkg/engine/armvariables_test.go | 36 +++- pkg/engine/const.go | 19 +- pkg/engine/keyvaults.go | 27 +++ pkg/engine/masterarmresources.go | 6 +- pkg/engine/params_k8s.go | 19 +- pkg/engine/template_generator.go | 6 + pkg/engine/templates_generated.go | 171 ++++++++++++++++++ 14 files changed, 403 insertions(+), 21 deletions(-) create mode 100644 parts/k8s/cloud-init/artifacts/kms-keyvault-key.service create mode 100644 parts/k8s/cloud-init/artifacts/kms-keyvault-key.sh diff --git a/parts/k8s/cloud-init/artifacts/cse_config.sh b/parts/k8s/cloud-init/artifacts/cse_config.sh index 8ddb23ed1e5..d44af420cfe 100755 --- a/parts/k8s/cloud-init/artifacts/cse_config.sh +++ b/parts/k8s/cloud-init/artifacts/cse_config.sh @@ -368,6 +368,13 @@ ensureDHCPv6() { fi } {{end}} +{{- if EnableEncryptionWithExternalKms}} +ensureKMSKeyvaultKey() { + wait_for_file 3600 1 {{GetKMSKeyvaultKeyServiceCSEScriptFilepath}} || exit {{GetCSEErrorCode "ERR_FILE_WATCH_TIMEOUT"}} + wait_for_file 3600 1 {{GetKMSKeyvaultKeyCSEScriptFilepath}} || exit {{GetCSEErrorCode "ERR_FILE_WATCH_TIMEOUT"}} + systemctlEnableAndStart kms-keyvault-key || exit {{GetCSEErrorCode "ERR_SYSTEMCTL_START_FAIL"}} +} +{{end}} ensureKubelet() { wait_for_file 1200 1 /etc/sysctl.d/11-aks-engine.conf || exit {{GetCSEErrorCode "ERR_FILE_WATCH_TIMEOUT"}} sysctl_reload 10 5 120 || exit {{GetCSEErrorCode "ERR_SYSCTL_RELOAD"}} @@ -564,6 +571,7 @@ configAzurePolicyAddon() { sed -i "s||/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP|g" $ADDONS_DIR/azure-policy-deployment.yaml } {{end}} + configAddons() { {{if IsClusterAutoscalerAddonEnabled}} if [[ ${CLUSTER_AUTOSCALER_ADDON} == true ]]; then diff --git a/parts/k8s/cloud-init/artifacts/cse_main.sh b/parts/k8s/cloud-init/artifacts/cse_main.sh index 307f59b33ad..e88d548e08e 100755 --- a/parts/k8s/cloud-init/artifacts/cse_main.sh +++ b/parts/k8s/cloud-init/artifacts/cse_main.sh @@ -219,6 +219,13 @@ time_metric "EnsureContainerd" ensureContainerd time_metric "EnsureDHCPv6" ensureDHCPv6 {{end}} +if [[ -n ${MASTER_NODE} ]]; then + {{/* configure and enable kms plugin */}} + {{- if EnableEncryptionWithExternalKms}} + time_metric "EnsureKMSKeyvaultKey" ensureKMSKeyvaultKey + {{end}} +fi + time_metric "EnsureKubelet" ensureKubelet {{if IsAzurePolicyAddonEnabled}} if [[ -n ${MASTER_NODE} ]]; then diff --git a/parts/k8s/cloud-init/artifacts/kms-keyvault-key.service b/parts/k8s/cloud-init/artifacts/kms-keyvault-key.service new file mode 100644 index 00000000000..d8b63deacff --- /dev/null +++ b/parts/k8s/cloud-init/artifacts/kms-keyvault-key.service @@ -0,0 +1,11 @@ +[Unit] +Description=setupkmskey +After=network-online.target + +[Service] +Type=oneshot +ExecStart={{GetKMSKeyvaultKeyCSEScriptFilepath}} + +[Install] +WantedBy=multi-user.target +#EOF diff --git a/parts/k8s/cloud-init/artifacts/kms-keyvault-key.sh b/parts/k8s/cloud-init/artifacts/kms-keyvault-key.sh new file mode 100644 index 00000000000..d576035184e --- /dev/null +++ b/parts/k8s/cloud-init/artifacts/kms-keyvault-key.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +set +x +set -euo pipefail + +AZURE_JSON_PATH="/etc/kubernetes/azure.json" +SERVICE_PRINCIPAL_CLIENT_ID=$(jq -r '.aadClientId' ${AZURE_JSON_PATH}) +SERVICE_PRINCIPAL_CLIENT_SECRET=$(jq -r '.aadClientSecret' ${AZURE_JSON_PATH}) +TENANT_ID=$(jq -r '.tenantId' ${AZURE_JSON_PATH}) +KMS_KEYVAULT_NAME=$(jq -r '.providerVaultName' ${AZURE_JSON_PATH}) +KMS_KEY_NAME=$(jq -r '.providerKeyName' ${AZURE_JSON_PATH}) +USER_ASSIGNED_IDENTITY_ID=$(jq -r '.userAssignedIdentityID' ${AZURE_JSON_PATH}) +PROVIDER_KEY_VERSION=$(jq -r '.providerKeyVersion' ${AZURE_JSON_PATH}) + +TOKEN_URL="https://login.microsoftonline.com/$TENANT_ID/oauth2/token" +SCOPE="https://vault.azure.net" +KEYVAULT_URL="https://$KMS_KEYVAULT_NAME.vault.azure.net/keys/$KMS_KEY_NAME/versions?maxresults=1&api-version=7.1" +KMS_KUBERNETES_FILE=/etc/kubernetes/manifests/kube-azure-kms.yaml + +# provider key version already exists +# this will be the case for BYOK +if [[ -n $PROVIDER_KEY_VERSION ]]; then + echo "KMS provider key version already exists" + exit 0 +fi + +echo "Generating token for Azure Key Vault" +echo "------------------------------------------------------------------------" +echo "Parameters" +echo "------------------------------------------------------------------------" +echo "SERVICE_PRINCIPAL_CLIENT_ID: ..." +echo "SERVICE_PRINCIPAL_CLIENT_SECRET: ..." +echo "TENANT_ID: $TENANT_ID" +echo "TOKEN_URL: $TOKEN_URL" +echo "SCOPE: $SCOPE" +echo "------------------------------------------------------------------------" + +if [[ $SERVICE_PRINCIPAL_CLIENT_ID == "msi" ]] && [[ $SERVICE_PRINCIPAL_CLIENT_SECRET == "msi" ]]; then + if [[ -z $USER_ASSIGNED_IDENTITY_ID ]]; then + # using system-assigned identity to access keyvault + TOKEN=$(curl -s --retry 5 --retry-delay 10 --max-time 60 \ + -H Metadata:true \ + "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=$SCOPE" | jq '.access_token' | xargs) + else + # using user-assigned managed identity to access keyvault + TOKEN=$(curl -s --retry 5 --retry-delay 10 --max-time 60 \ + -H Metadata:true \ + "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&client_id=$USER_ASSIGNED_IDENTITY_ID&resource=$SCOPE" | jq '.access_token' | xargs) + fi +else + # use service principal token to access keyvault + TOKEN=$(curl -s --retry 5 --retry-delay 10 --max-time 60 -f -X POST \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=client_credentials" \ + -d "client_id=$SERVICE_PRINCIPAL_CLIENT_ID" \ + --data-urlencode "client_secret=$SERVICE_PRINCIPAL_CLIENT_SECRET" \ + --data-urlencode "resource=$SCOPE" \ + ${TOKEN_URL} | jq '.access_token' | xargs) +fi + + +if [[ -z $TOKEN ]]; then + echo "Error generating token for Azure Keyvault" + exit 120 +fi + +# Get the keyID for the kms key created as part of cluster bootstrap +KEY_ID=$(curl -s --retry 5 --retry-delay 10 --max-time 60 -f \ + ${KEYVAULT_URL} -H "Authorization: Bearer ${TOKEN}" | jq '.value[0].kid' | xargs) + +if [[ -z "$KEY_ID" || "$KEY_ID" == "null" ]]; then + echo "Error getting the kms key version" + exit 120 +fi + +# KID format: https://.vault.azure.net/keys// +# Example KID: "https://akv0112master.vault.azure.net/keys/k8s/128a3d9956bc44feb6a0e2c2f35b732c" +KEY_VERSION=${KEY_ID##*/} + +# Set the version in azure.json +if [ -f $AZURE_JSON_PATH ]; then + # once the version is set in azure.json, kms plugin will just default to using the key + # this will be changed in upcoming kms release to set the version as container args + tmpDir=$(mktemp -d "$(pwd)/XXX") + jq --arg KEY_VERSION ${KEY_VERSION} '.providerKeyVersion=($KEY_VERSION)' "$AZURE_JSON_PATH" > $tmpDir/tmp + mv $tmpDir/tmp "$AZURE_JSON_PATH" + rm -Rf $tmpDir +fi + +set -x +#EOF diff --git a/parts/k8s/cloud-init/masternodecustomdata.yml b/parts/k8s/cloud-init/masternodecustomdata.yml index a9942913666..78dcf94951d 100644 --- a/parts/k8s/cloud-init/masternodecustomdata.yml +++ b/parts/k8s/cloud-init/masternodecustomdata.yml @@ -380,6 +380,20 @@ write_files: endpoint: unix:///opt/azurekms.socket cachesize: 1000 - identity: {} + +- path: {{GetKMSKeyvaultKeyServiceCSEScriptFilepath}} + permissions: "0644" + encoding: gzip + owner: root + content: !!binary | + {{CloudInitData "kmsKeyvaultKeySystemdService"}} + +- path: {{GetKMSKeyvaultKeyCSEScriptFilepath}} + permissions: "0544" + encoding: gzip + owner: root + content: !!binary | + {{CloudInitData "kmsKeyvaultKeyScript"}} {{end}} MASTER_MANIFESTS_CONFIG_PLACEHOLDER diff --git a/pkg/api/const.go b/pkg/api/const.go index 119e36eeb61..2157e932510 100644 --- a/pkg/api/const.go +++ b/pkg/api/const.go @@ -229,7 +229,7 @@ const ( APIVersionAuthorizationSystem = "2018-09-01-preview" APIVersionCompute = "2019-07-01" APIVersionDeployments = "2018-06-01" - APIVersionKeyVault = "2018-02-14" + APIVersionKeyVault = "2019-09-01" APIVersionManagedIdentity = "2018-11-30" APIVersionNetwork = "2018-08-01" APIVersionStorage = "2018-07-01" diff --git a/pkg/engine/armvariables.go b/pkg/engine/armvariables.go index d85c15f4399..9a3035f5a2e 100644 --- a/pkg/engine/armvariables.go +++ b/pkg/engine/armvariables.go @@ -79,6 +79,7 @@ func getK8sMasterVars(cs *api.ContainerService) (map[string]interface{}, error) excludeMasterFromStandardLB = to.Bool(kubernetesConfig.ExcludeMasterFromStandardLB) maxLoadBalancerCount = kubernetesConfig.MaximumLoadBalancerRuleCount provisionJumpbox = kubernetesConfig.PrivateJumpboxProvision() + enableEncryptionWithExternalKms = to.Bool(kubernetesConfig.EnableEncryptionWithExternalKms) if kubernetesConfig.ShouldCreateNewUserAssignedIdentity() { userAssignedIDReference = "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('userAssignedID'))]" @@ -157,6 +158,12 @@ func getK8sMasterVars(cs *api.ContainerService) (map[string]interface{}, error) "dockerMonitorSystemdService": getBase64EncodedGzippedCustomScript(kubernetesDockerMonitorSystemdService, cs), } + if enableEncryptionWithExternalKms { + fmt.Println("being processed") + cloudInitFiles["kmsKeyvaultKeySystemdService"] = getBase64EncodedGzippedCustomScript(kmsKeyvaultKeySystemdService, cs) + cloudInitFiles["kmsKeyvaultKeyScript"] = getBase64EncodedGzippedCustomScript(kmsKeyvaultKeyScript, cs) + } + if cs.Properties.OrchestratorProfile.KubernetesConfig.IsAddonEnabled(common.AADPodIdentityAddonName) { cloudInitFiles["untaintNodesScript"] = getBase64EncodedGzippedCustomScript(untaintNodesScript, cs) cloudInitFiles["untaintNodesSystemdService"] = getBase64EncodedGzippedCustomScript(untaintNodesSystemdService, cs) diff --git a/pkg/engine/armvariables_test.go b/pkg/engine/armvariables_test.go index cf562b4b83c..e5b8f258e5a 100644 --- a/pkg/engine/armvariables_test.go +++ b/pkg/engine/armvariables_test.go @@ -90,7 +90,7 @@ func TestK8sVars(t *testing.T) { "apiVersionAuthorizationUser": "2018-09-01-preview", "apiVersionCompute": "2019-07-01", "apiVersionDeployments": "2018-06-01", - "apiVersionKeyVault": "2018-02-14", + "apiVersionKeyVault": "2019-09-01", "apiVersionManagedIdentity": "2018-11-30", "apiVersionNetwork": "2018-08-01", "apiVersionStorage": "2018-07-01", @@ -201,7 +201,6 @@ func TestK8sVars(t *testing.T) { } diff := cmp.Diff(varMap, expectedMap) - if diff != "" { t.Errorf("unexpected diff while expecting equal structs: %s", diff) } @@ -810,7 +809,7 @@ func TestK8sVarsMastersOnly(t *testing.T) { "apiVersionAuthorizationUser": "2018-09-01-preview", "apiVersionCompute": "2019-07-01", "apiVersionDeployments": "2018-06-01", - "apiVersionKeyVault": "2018-02-14", + "apiVersionKeyVault": "2019-09-01", "apiVersionManagedIdentity": "2018-11-30", "apiVersionNetwork": "2018-08-01", "apiVersionStorage": "2018-07-01", @@ -929,6 +928,37 @@ func TestK8sVarsMastersOnly(t *testing.T) { if diff != "" { t.Errorf("unexpected diff while expecting equal structs: %s", diff) } + + // enable external kms encryption + cs.Properties.OrchestratorProfile.KubernetesConfig.EnableEncryptionWithExternalKms = to.BoolPtr(true) + expectedMap["clusterKeyVaultName"] = string("[take(concat('kv', tolower(uniqueString(concat(variables('masterFqdnPrefix'),variables('location'),parameters('nameSuffix'))))), 22)]") + expectedMap["cloudInitFiles"] = map[string]interface{}{ + "provisionScript": getBase64EncodedGzippedCustomScript(kubernetesCSEMainScript, cs), + "provisionSource": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScript, cs), + "provisionInstalls": getBase64EncodedGzippedCustomScript(kubernetesCSEInstall, cs), + "provisionConfigs": getBase64EncodedGzippedCustomScript(kubernetesCSEConfig, cs), + "customSearchDomainsScript": getBase64EncodedGzippedCustomScript(kubernetesCustomSearchDomainsScript, cs), + "etcdSystemdService": getBase64EncodedGzippedCustomScript(etcdSystemdService, cs), + "dhcpv6ConfigurationScript": getBase64EncodedGzippedCustomScript(dhcpv6ConfigurationScript, cs), + "dhcpv6SystemdService": getBase64EncodedGzippedCustomScript(dhcpv6SystemdService, cs), + "kubeletSystemdService": getBase64EncodedGzippedCustomScript(kubeletSystemdService, cs), + "etcdMonitorSystemdService": getBase64EncodedGzippedCustomScript(etcdMonitorSystemdService, cs), + "healthMonitorScript": getBase64EncodedGzippedCustomScript(kubernetesHealthMonitorScript, cs), + "kubeletMonitorSystemdService": getBase64EncodedGzippedCustomScript(kubernetesKubeletMonitorSystemdService, cs), + "dockerMonitorSystemdService": getBase64EncodedGzippedCustomScript(kubernetesDockerMonitorSystemdService, cs), + "kmsKeyvaultKeySystemdService": getBase64EncodedGzippedCustomScript(kmsKeyvaultKeySystemdService, cs), + "kmsKeyvaultKeyScript": getBase64EncodedGzippedCustomScript(kmsKeyvaultKeyScript, cs), + } + + varMap, err = GetKubernetesVariables(cs) + if err != nil { + t.Fatal(err) + } + diff = cmp.Diff(varMap, expectedMap) + + if diff != "" { + t.Errorf("unexpected diff while expecting equal structs: %s", diff) + } } func TestK8sVarsWindowsProfile(t *testing.T) { diff --git a/pkg/engine/const.go b/pkg/engine/const.go index b5bacfbd331..f327b30b029 100644 --- a/pkg/engine/const.go +++ b/pkg/engine/const.go @@ -108,17 +108,22 @@ const ( // scripts and service for enabling ipv6 dual stack dhcpv6SystemdService = "k8s/cloud-init/artifacts/dhcpv6.service" dhcpv6ConfigurationScript = "k8s/cloud-init/artifacts/enable-dhcpv6.sh" + // script for getting key version from keyvault for kms + kmsKeyvaultKeySystemdService = "k8s/cloud-init/artifacts/kms-keyvault-key.service" + kmsKeyvaultKeyScript = "k8s/cloud-init/artifacts/kms-keyvault-key.sh" ) // cloud-init destination file references const ( - customCloudConfigCSEScriptFilepath = "/opt/azure/containers/provision_configs_custom_cloud.sh" - cseHelpersScriptFilepath = "/opt/azure/containers/provision_source.sh" - cseInstallScriptFilepath = "/opt/azure/containers/provision_installs.sh" - cseConfigScriptFilepath = "/opt/azure/containers/provision_configs.sh" - customSearchDomainsCSEScriptFilepath = "/opt/azure/containers/setup-custom-search-domains.sh" - dhcpV6ServiceCSEScriptFilepath = "/etc/systemd/system/dhcpv6.service" - dhcpV6ConfigCSEScriptFilepath = "/opt/azure/containers/enable-dhcpv6.sh" + customCloudConfigCSEScriptFilepath = "/opt/azure/containers/provision_configs_custom_cloud.sh" + cseHelpersScriptFilepath = "/opt/azure/containers/provision_source.sh" + cseInstallScriptFilepath = "/opt/azure/containers/provision_installs.sh" + cseConfigScriptFilepath = "/opt/azure/containers/provision_configs.sh" + customSearchDomainsCSEScriptFilepath = "/opt/azure/containers/setup-custom-search-domains.sh" + dhcpV6ServiceCSEScriptFilepath = "/etc/systemd/system/dhcpv6.service" + dhcpV6ConfigCSEScriptFilepath = "/opt/azure/containers/enable-dhcpv6.sh" + kmsKeyvaultKeyServiceCSEScriptFilepath = "/etc/systemd/system/kms-keyvault-key.service" + kmsKeyvaultKeyCSEScriptFilepath = "/opt/azure/containers/kms-keyvault-key.sh" ) const ( diff --git a/pkg/engine/keyvaults.go b/pkg/engine/keyvaults.go index f2e38716026..d161e6b3485 100644 --- a/pkg/engine/keyvaults.go +++ b/pkg/engine/keyvaults.go @@ -5,6 +5,7 @@ package engine import ( "fmt" + "strings" "github.com/Azure/aks-engine/pkg/api" "github.com/Azure/go-autorest/autorest/to" @@ -146,3 +147,29 @@ func CreateKeyVaultVMSS(cs *api.ContainerService) map[string]interface{} { return keyVaultMap } + +func CreateKeyVaultKey(cs *api.ContainerService) map[string]interface{} { + keyMap := map[string]interface{}{ + "type": "Microsoft.KeyVault/vaults/keys", + "name": "[concat(variables('clusterKeyVaultName'), '/', 'k8s')]", + "apiVersion": "[variables('apiVersionKeyVault')]", + "location": "[variables('location')]", + "dependsOn": []string{ + "[resourceId('Microsoft.KeyVault/vaults', variables('clusterKeyVaultName'))]", + }, + } + keyType := "RSA" + if strings.EqualFold(cs.Properties.OrchestratorProfile.KubernetesConfig.KeyVaultSku, "premium") { + keyType = "RSA-HSM" + } + keyProps := map[string]interface{}{ + "kty": keyType, + "keyOps": []string{ + "encrypt", + "decrypt", + }, + "keySize": 2048, + } + keyMap["properties"] = keyProps + return keyMap +} diff --git a/pkg/engine/masterarmresources.go b/pkg/engine/masterarmresources.go index 259f193a595..c6bc2a2247e 100644 --- a/pkg/engine/masterarmresources.go +++ b/pkg/engine/masterarmresources.go @@ -92,7 +92,8 @@ func createKubernetesMasterResourcesVMAS(cs *api.ContainerService) []interface{} if isKMSEnabled { keyVaultStorageAccount := createKeyVaultStorageAccount() keyVault := CreateKeyVaultVMAS(cs) - masterResources = append(masterResources, keyVaultStorageAccount, keyVault) + keyVaultKey := CreateKeyVaultKey(cs) + masterResources = append(masterResources, keyVaultStorageAccount, keyVault, keyVaultKey) } if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") { @@ -194,7 +195,8 @@ func createKubernetesMasterResourcesVMSS(cs *api.ContainerService) []interface{} if isKMSEnabled { keyVaultStorageAccount := createKeyVaultStorageAccount() keyVault := CreateKeyVaultVMSS(cs) - masterResources = append(masterResources, keyVaultStorageAccount, keyVault) + keyVaultKey := CreateKeyVaultKey(cs) + masterResources = append(masterResources, keyVaultStorageAccount, keyVault, keyVaultKey) } if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") { diff --git a/pkg/engine/params_k8s.go b/pkg/engine/params_k8s.go index 575dcba04ff..9d4f965f921 100644 --- a/pkg/engine/params_k8s.go +++ b/pkg/engine/params_k8s.go @@ -119,15 +119,18 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params } else { addValue(parametersMap, "servicePrincipalClientSecret", servicePrincipalProfile.Secret) } + } + } - if kubernetesConfig != nil && to.Bool(kubernetesConfig.EnableEncryptionWithExternalKms) { - if kubernetesConfig.KeyVaultSku != "" { - addValue(parametersMap, "clusterKeyVaultSku", kubernetesConfig.KeyVaultSku) - } - if !to.Bool(kubernetesConfig.UseManagedIdentity) && servicePrincipalProfile.ObjectID != "" { - addValue(parametersMap, "servicePrincipalObjectId", servicePrincipalProfile.ObjectID) - } - } + // configure params required for external kms + if kubernetesConfig != nil && to.Bool(kubernetesConfig.EnableEncryptionWithExternalKms) { + servicePrincipalProfile := properties.ServicePrincipalProfile + + if kubernetesConfig.KeyVaultSku != "" { + addValue(parametersMap, "clusterKeyVaultSku", kubernetesConfig.KeyVaultSku) + } + if !to.Bool(kubernetesConfig.UseManagedIdentity) && servicePrincipalProfile.ObjectID != "" { + addValue(parametersMap, "servicePrincipalObjectId", servicePrincipalProfile.ObjectID) } } diff --git a/pkg/engine/template_generator.go b/pkg/engine/template_generator.go index 59e475cf00b..cf4f64c73f2 100644 --- a/pkg/engine/template_generator.go +++ b/pkg/engine/template_generator.go @@ -703,6 +703,12 @@ func getContainerServiceFuncMap(cs *api.ContainerService) template.FuncMap { "GetDHCPv6ConfigCSEScriptFilepath": func() string { return dhcpV6ConfigCSEScriptFilepath }, + "GetKMSKeyvaultKeyServiceCSEScriptFilepath": func() string { + return kmsKeyvaultKeyServiceCSEScriptFilepath + }, + "GetKMSKeyvaultKeyCSEScriptFilepath": func() string { + return kmsKeyvaultKeyCSEScriptFilepath + }, "HasPrivateAzureRegistryServer": func() bool { return cs.Properties.OrchestratorProfile.KubernetesConfig.PrivateAzureRegistryServer != "" }, diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index 610e4bc40b0..8aab2fc099b 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -57,6 +57,8 @@ // ../../parts/k8s/cloud-init/artifacts/etcd.service // ../../parts/k8s/cloud-init/artifacts/generateproxycerts.sh // ../../parts/k8s/cloud-init/artifacts/health-monitor.sh +// ../../parts/k8s/cloud-init/artifacts/kms-keyvault-key.service +// ../../parts/k8s/cloud-init/artifacts/kms-keyvault-key.sh // ../../parts/k8s/cloud-init/artifacts/kubelet-monitor.service // ../../parts/k8s/cloud-init/artifacts/kubelet-monitor.timer // ../../parts/k8s/cloud-init/artifacts/kubelet.service @@ -12014,6 +12016,13 @@ ensureDHCPv6() { fi } {{end}} +{{- if EnableEncryptionWithExternalKms}} +ensureKMSKeyvaultKey() { + wait_for_file 3600 1 {{GetKMSKeyvaultKeyServiceCSEScriptFilepath}} || exit {{GetCSEErrorCode "ERR_FILE_WATCH_TIMEOUT"}} + wait_for_file 3600 1 {{GetKMSKeyvaultKeyCSEScriptFilepath}} || exit {{GetCSEErrorCode "ERR_FILE_WATCH_TIMEOUT"}} + systemctlEnableAndStart kms-keyvault-key || exit {{GetCSEErrorCode "ERR_SYSTEMCTL_START_FAIL"}} +} +{{end}} ensureKubelet() { wait_for_file 1200 1 /etc/sysctl.d/11-aks-engine.conf || exit {{GetCSEErrorCode "ERR_FILE_WATCH_TIMEOUT"}} sysctl_reload 10 5 120 || exit {{GetCSEErrorCode "ERR_SYSCTL_RELOAD"}} @@ -12210,6 +12219,7 @@ configAzurePolicyAddon() { sed -i "s||/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP|g" $ADDONS_DIR/azure-policy-deployment.yaml } {{end}} + configAddons() { {{if IsClusterAutoscalerAddonEnabled}} if [[ ${CLUSTER_AUTOSCALER_ADDON} == true ]]; then @@ -13341,6 +13351,13 @@ time_metric "EnsureContainerd" ensureContainerd time_metric "EnsureDHCPv6" ensureDHCPv6 {{end}} +if [[ -n ${MASTER_NODE} ]]; then + {{/* configure and enable kms plugin */}} + {{- if EnableEncryptionWithExternalKms}} + time_metric "EnsureKMSKeyvaultKey" ensureKMSKeyvaultKey + {{end}} +fi + time_metric "EnsureKubelet" ensureKubelet {{if IsAzurePolicyAddonEnabled}} if [[ -n ${MASTER_NODE} ]]; then @@ -13894,6 +13911,142 @@ func k8sCloudInitArtifactsHealthMonitorSh() (*asset, error) { return a, nil } +var _k8sCloudInitArtifactsKmsKeyvaultKeyService = []byte(`[Unit] +Description=setupkmskey +After=network-online.target + +[Service] +Type=oneshot +ExecStart={{GetKMSKeyvaultKeyCSEScriptFilepath}} + +[Install] +WantedBy=multi-user.target +#EOF +`) + +func k8sCloudInitArtifactsKmsKeyvaultKeyServiceBytes() ([]byte, error) { + return _k8sCloudInitArtifactsKmsKeyvaultKeyService, nil +} + +func k8sCloudInitArtifactsKmsKeyvaultKeyService() (*asset, error) { + bytes, err := k8sCloudInitArtifactsKmsKeyvaultKeyServiceBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "k8s/cloud-init/artifacts/kms-keyvault-key.service", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _k8sCloudInitArtifactsKmsKeyvaultKeySh = []byte(`#!/usr/bin/env bash + +set +x +set -euo pipefail + +AZURE_JSON_PATH="/etc/kubernetes/azure.json" +SERVICE_PRINCIPAL_CLIENT_ID=$(jq -r '.aadClientId' ${AZURE_JSON_PATH}) +SERVICE_PRINCIPAL_CLIENT_SECRET=$(jq -r '.aadClientSecret' ${AZURE_JSON_PATH}) +TENANT_ID=$(jq -r '.tenantId' ${AZURE_JSON_PATH}) +KMS_KEYVAULT_NAME=$(jq -r '.providerVaultName' ${AZURE_JSON_PATH}) +KMS_KEY_NAME=$(jq -r '.providerKeyName' ${AZURE_JSON_PATH}) +USER_ASSIGNED_IDENTITY_ID=$(jq -r '.userAssignedIdentityID' ${AZURE_JSON_PATH}) +PROVIDER_KEY_VERSION=$(jq -r '.providerKeyVersion' ${AZURE_JSON_PATH}) + +TOKEN_URL="https://login.microsoftonline.com/$TENANT_ID/oauth2/token" +SCOPE="https://vault.azure.net" +KEYVAULT_URL="https://$KMS_KEYVAULT_NAME.vault.azure.net/keys/$KMS_KEY_NAME/versions?maxresults=1&api-version=7.1" +KMS_KUBERNETES_FILE=/etc/kubernetes/manifests/kube-azure-kms.yaml + +# provider key version already exists +# this will be the case for BYOK +if [[ -n $PROVIDER_KEY_VERSION ]]; then + echo "KMS provider key version already exists" + exit 0 +fi + +echo "Generating token for Azure Key Vault" +echo "------------------------------------------------------------------------" +echo "Parameters" +echo "------------------------------------------------------------------------" +echo "SERVICE_PRINCIPAL_CLIENT_ID: ..." +echo "SERVICE_PRINCIPAL_CLIENT_SECRET: ..." +echo "TENANT_ID: $TENANT_ID" +echo "TOKEN_URL: $TOKEN_URL" +echo "SCOPE: $SCOPE" +echo "------------------------------------------------------------------------" + +if [[ $SERVICE_PRINCIPAL_CLIENT_ID == "msi" ]] && [[ $SERVICE_PRINCIPAL_CLIENT_SECRET == "msi" ]]; then + if [[ -z $USER_ASSIGNED_IDENTITY_ID ]]; then + # using system-assigned identity to access keyvault + TOKEN=$(curl -s --retry 5 --retry-delay 10 --max-time 60 \ + -H Metadata:true \ + "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=$SCOPE" | jq '.access_token' | xargs) + else + # using user-assigned managed identity to access keyvault + TOKEN=$(curl -s --retry 5 --retry-delay 10 --max-time 60 \ + -H Metadata:true \ + "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&client_id=$USER_ASSIGNED_IDENTITY_ID&resource=$SCOPE" | jq '.access_token' | xargs) + fi +else + # use service principal token to access keyvault + TOKEN=$(curl -s --retry 5 --retry-delay 10 --max-time 60 -f -X POST \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=client_credentials" \ + -d "client_id=$SERVICE_PRINCIPAL_CLIENT_ID" \ + --data-urlencode "client_secret=$SERVICE_PRINCIPAL_CLIENT_SECRET" \ + --data-urlencode "resource=$SCOPE" \ + ${TOKEN_URL} | jq '.access_token' | xargs) +fi + + +if [[ -z $TOKEN ]]; then + echo "Error generating token for Azure Keyvault" + exit 120 +fi + +# Get the keyID for the kms key created as part of cluster bootstrap +KEY_ID=$(curl -s --retry 5 --retry-delay 10 --max-time 60 -f \ + ${KEYVAULT_URL} -H "Authorization: Bearer ${TOKEN}" | jq '.value[0].kid' | xargs) + +if [[ -z "$KEY_ID" || "$KEY_ID" == "null" ]]; then + echo "Error getting the kms key version" + exit 120 +fi + +# KID format: https://.vault.azure.net/keys// +# Example KID: "https://akv0112master.vault.azure.net/keys/k8s/128a3d9956bc44feb6a0e2c2f35b732c" +KEY_VERSION=${KEY_ID##*/} + +# Set the version in azure.json +if [ -f $AZURE_JSON_PATH ]; then + # once the version is set in azure.json, kms plugin will just default to using the key + # this will be changed in upcoming kms release to set the version as container args + tmpDir=$(mktemp -d "$(pwd)/XXX") + jq --arg KEY_VERSION ${KEY_VERSION} '.providerKeyVersion=($KEY_VERSION)' "$AZURE_JSON_PATH" > $tmpDir/tmp + mv $tmpDir/tmp "$AZURE_JSON_PATH" + rm -Rf $tmpDir +fi + +set -x +#EOF +`) + +func k8sCloudInitArtifactsKmsKeyvaultKeyShBytes() ([]byte, error) { + return _k8sCloudInitArtifactsKmsKeyvaultKeySh, nil +} + +func k8sCloudInitArtifactsKmsKeyvaultKeySh() (*asset, error) { + bytes, err := k8sCloudInitArtifactsKmsKeyvaultKeyShBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "k8s/cloud-init/artifacts/kms-keyvault-key.sh", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _k8sCloudInitArtifactsKubeletMonitorService = []byte(`[Unit] Description=a script that checks kubelet health and restarts if needed After=kubelet.service @@ -15134,6 +15287,20 @@ write_files: endpoint: unix:///opt/azurekms.socket cachesize: 1000 - identity: {} + +- path: {{GetKMSKeyvaultKeyServiceCSEScriptFilepath}} + permissions: "0644" + encoding: gzip + owner: root + content: !!binary | + {{CloudInitData "kmsKeyvaultKeySystemdService"}} + +- path: {{GetKMSKeyvaultKeyCSEScriptFilepath}} + permissions: "0544" + encoding: gzip + owner: root + content: !!binary | + {{CloudInitData "kmsKeyvaultKeyScript"}} {{end}} MASTER_MANIFESTS_CONFIG_PLACEHOLDER @@ -19518,6 +19685,8 @@ var _bindata = map[string]func() (*asset, error){ "k8s/cloud-init/artifacts/etcd.service": k8sCloudInitArtifactsEtcdService, "k8s/cloud-init/artifacts/generateproxycerts.sh": k8sCloudInitArtifactsGenerateproxycertsSh, "k8s/cloud-init/artifacts/health-monitor.sh": k8sCloudInitArtifactsHealthMonitorSh, + "k8s/cloud-init/artifacts/kms-keyvault-key.service": k8sCloudInitArtifactsKmsKeyvaultKeyService, + "k8s/cloud-init/artifacts/kms-keyvault-key.sh": k8sCloudInitArtifactsKmsKeyvaultKeySh, "k8s/cloud-init/artifacts/kubelet-monitor.service": k8sCloudInitArtifactsKubeletMonitorService, "k8s/cloud-init/artifacts/kubelet-monitor.timer": k8sCloudInitArtifactsKubeletMonitorTimer, "k8s/cloud-init/artifacts/kubelet.service": k8sCloudInitArtifactsKubeletService, @@ -19668,6 +19837,8 @@ var _bintree = &bintree{nil, map[string]*bintree{ "etcd.service": {k8sCloudInitArtifactsEtcdService, map[string]*bintree{}}, "generateproxycerts.sh": {k8sCloudInitArtifactsGenerateproxycertsSh, map[string]*bintree{}}, "health-monitor.sh": {k8sCloudInitArtifactsHealthMonitorSh, map[string]*bintree{}}, + "kms-keyvault-key.service": {k8sCloudInitArtifactsKmsKeyvaultKeyService, map[string]*bintree{}}, + "kms-keyvault-key.sh": {k8sCloudInitArtifactsKmsKeyvaultKeySh, map[string]*bintree{}}, "kubelet-monitor.service": {k8sCloudInitArtifactsKubeletMonitorService, map[string]*bintree{}}, "kubelet-monitor.timer": {k8sCloudInitArtifactsKubeletMonitorTimer, map[string]*bintree{}}, "kubelet.service": {k8sCloudInitArtifactsKubeletService, map[string]*bintree{}},