feat: config application-level encryption for fl (#78)

platforms/gke/base/core/deploy.sh

@@ -17,8 +17,21 @@ set -o errexit
 
 start_timestamp=$(date +%s)
 
-# Disable gke_enterprise/servicemesh due to b/376312292
-declare -a terraservices=${CORE_TERRASERVICES_APPLY:-("networking" "container_cluster" "container_node_pool" "gke_enterprise/fleet_membership" "workloads/kueue")}
+declare -a terraservices
+if [[ -v CORE_TERRASERVICES_APPLY ]]; then
+    terraservices=("${CORE_TERRASERVICES_APPLY[@]}")
+else
+    terraservices=(
+        "networking"
+        "container_cluster"
+        "container_node_pool"
+        "gke_enterprise/fleet_membership"
+        # Disable gke_enterprise/servicemesh due to b/376312292
+        # "gke_enterprise/servicemesh"
+        "workloads/kueue"
+    )
+fi
+echo "Core platform terraservices to provision: ${terraservices[*]}"
 
 source ${ACP_PLATFORM_BASE_DIR}/_shared_config/scripts/set_environment_variables.sh ${ACP_PLATFORM_BASE_DIR}/_shared_config
 

platforms/gke/base/core/teardown.sh

@@ -17,7 +17,21 @@ set -o errexit
 
 start_timestamp=$(date +%s)
 
-declare -a terraservices=${CORE_TERRASERVICES_DESTROY:-("workloads/kueue" "gke_enterprise/fleet_membership" "container_node_pool" "container_cluster" "networking")}
+declare -a terraservices
+if [[ -v CORE_TERRASERVICES_DESTROY ]]; then
+    terraservices=("${CORE_TERRASERVICES_DESTROY[@]}")
+else
+    terraservices=(
+        "workloads/kueue"
+        # Disable gke_enterprise/servicemesh due to b/376312292
+        # "gke_enterprise/servicemesh"
+        "gke_enterprise/fleet_membership"
+        "container_node_pool"
+        "container_cluster"
+        "networking"
+    )
+fi
+echo "Core platform terraservices to destroy: ${terraservices[*]}"
 
 source ${ACP_PLATFORM_BASE_DIR}/_shared_config/scripts/set_environment_variables.sh ${ACP_PLATFORM_BASE_DIR}/_shared_config
 

platforms/gke/base/use-cases/federated-learning/common.sh

@@ -22,6 +22,8 @@ ACP_PLATFORM_SHARED_CONFIG_DIR="${ACP_PLATFORM_BASE_DIR}/_shared_config"
 
 # shellcheck disable=SC2034 # Variable is used in other scripts
 ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE="${ACP_PLATFORM_SHARED_CONFIG_DIR}/cluster.auto.tfvars"
+# shellcheck disable=SC2034 # Variable is used in other scripts
+ACP_PLATFORM_SHARED_CONFIG_INITIALIZE_AUTO_VARS_FILE="${ACP_PLATFORM_SHARED_CONFIG_DIR}/initialize.auto.tfvars"
 
 # shellcheck disable=SC1091
 source "${ACP_PLATFORM_SHARED_CONFIG_DIR}/scripts/set_environment_variables.sh" "${ACP_PLATFORM_SHARED_CONFIG_DIR}"
@@ -31,16 +33,29 @@ FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR="${FEDERATED_LEARNING_USE_CASE_DIR}/te
 # shellcheck disable=SC2034 # Variable is used in other scripts
 FEDERATED_LEARNING_SHARED_CONFIG_DIR="${FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR}/_shared_config"
 
+# shellcheck disable=SC2034 # Variable is used in other scripts
+# Terraservices that are necessary for the core platform
+federated_learning_core_platform_terraservices=(
+  "key_management_service"
+)
+
 # shellcheck disable=SC2034 # Variable is used in other scripts
 federated_learning_terraservices=(
   "container_image_repository"
   "private_google_access"
 )
 
+# shellcheck disable=SC2034 # Variable is used in other scripts
+TERRAFORM_CORE_INITIALIZE_CONFIGURATION=(
+  "initialize_backend_use_case_name = \"federated-learning\""
+)
+
 # shellcheck disable=SC2034 # Variable is used in other scripts
 TERRAFORM_CLUSTER_CONFIGURATION=(
   "cluster_binary_authorization_evaluation_mode = \"PROJECT_SINGLETON_POLICY_ENFORCE\""
   "cluster_confidential_nodes_enabled = false"
+  "cluster_database_encryption_state = \"ENCRYPTED\""
+  "cluster_database_encryption_key_name = \"cluster_database_encryption_key_name_placeholder\""
 )
 
 apply_or_destroy_terraservice() {
@@ -50,6 +65,7 @@ apply_or_destroy_terraservice() {
   local operation_mode
   operation_mode="${2:-"not set"}"
 
+  echo "Initializing ${terraservice} Terraform environment"
   cd "${FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR}/${terraservice}" &&
     terraform init
 
@@ -90,10 +106,58 @@ destroy_terraservice() {
 get_terraform_output() {
   terraservice="${1}"
   output_name="${2}"
+
+  if [[ ! -d "${terraservice}" ]]; then
+    echo "${terraservice} directory doesn't exist or is not readable"
+    return 1
+  fi
+
   local output
-  if ! output="$(cd "${FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR}/${terraservice}" && terraform output -raw "${output_name}")"; then
-    echo "Error while getting ${output_name} output"
+  if ! output="$(terraform -chdir="${terraservice}" init)"; then
+    echo "Error while initializing ${terraservice} to get ${output_name} output: ${output}"
+    return 1
+  fi
+
+  if ! output="$(
+    terraform -chdir="${terraservice}" output -raw "${output_name}"
+  )"; then
+    echo "Error while getting ${output_name} output: ${output}"
     return 1
   fi
   echo "${output}"
 }
+
+write_terraform_configuration_variable_to_file() {
+  local configuration_variable="${1}"
+  local destination_file_path="${2}"
+  local configuration_variable_name
+
+  configuration_variable_name="$(echo "${configuration_variable}" | awk '{ print $1 }')"
+  echo "Checking if ${configuration_variable_name} is in ${destination_file_path}"
+  grep -q "${configuration_variable_name}" "${destination_file_path}" || echo "${configuration_variable}" >>"${destination_file_path}"
+  terraform fmt "${destination_file_path}"
+}
+
+remove_terraform_configuration_variable_from_file() {
+  local configuration_variable="${1}"
+  local destination_file_path="${2}"
+  local configuration_variable_name
+
+  configuration_variable_name="$(echo "${configuration_variable}" | awk ' { print $1 }'))"
+  sed -i "/${configuration_variable_name}/d" "${destination_file_path}"
+  terraform fmt "${destination_file_path}"
+}
+
+edit_terraform_configuration_variable_value_in_file() {
+  local configuration_variable_placeholder_value="${1}"
+  local configuration_variable_value="${2}"
+  local destination_file_path="${3}"
+
+  echo "Changing the value of ${configuration_variable_placeholder_value} to ${configuration_variable_value} in ${destination_file_path}"
+
+  # Use | as a separator in the sed command because substitution values might contain slashes
+  local sed_command="s|${configuration_variable_placeholder_value}|${configuration_variable_value}|g"
+  echo "sed command: ${sed_command}"
+  sed -i "${sed_command}" "${destination_file_path}"
+  terraform fmt "${destination_file_path}"
+}

platforms/gke/base/use-cases/federated-learning/deploy.sh

@@ -23,16 +23,38 @@ source "${ACP_PLATFORM_BASE_DIR}/use-cases/federated-learning/common.sh"
 
 start_timestamp_federated_learning=$(date +%s)
 
+echo "Initializing the core platform"
+# Don't provision any core platform terraservice becuase we just need
+# to initialize the terraform environment and remote backend
+declare -a CORE_TERRASERVICES_APPLY
+CORE_TERRASERVICES_APPLY=("initialize")
+# shellcheck disable=SC1091
+source "${ACP_PLATFORM_CORE_DIR}/deploy.sh"
+
 echo "Preparing core platform configuration files"
 for configuration_variable in "${TERRAFORM_CLUSTER_CONFIGURATION[@]}"; do
-  configuration_variable_name="$(echo "${configuration_variable}" | awk '{ print $1 }')"
-  echo "Checking if ${configuration_variable_name} is in ${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
-  grep -q "${configuration_variable_name}" "${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}" || echo "${configuration_variable}" >>"${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
+  write_terraform_configuration_variable_to_file "${configuration_variable}" "${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
+done
+for configuration_variable in "${TERRAFORM_CORE_INITIALIZE_CONFIGURATION[@]}"; do
+  write_terraform_configuration_variable_to_file "${configuration_variable}" "${ACP_PLATFORM_SHARED_CONFIG_INITIALIZE_AUTO_VARS_FILE}"
 done
-terraform fmt "${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
+
+echo "Provision services that the core platform depends on"
+# shellcheck disable=SC2154 # variable defined in common.sh
+for terraservice in "${federated_learning_core_platform_terraservices[@]}"; do
+  provision_terraservice "${terraservice}"
+done
+
+if ! cluster_database_encryption_key_id="$(get_terraform_output "${FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR}/key_management_service" "cluster_database_encryption_key_id")"; then
+  exit 1
+fi
+edit_terraform_configuration_variable_value_in_file "cluster_database_encryption_key_name_placeholder" "${cluster_database_encryption_key_id}" "${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
 
 echo "Provisioning the core platform"
-"${ACP_PLATFORM_CORE_DIR}/deploy.sh"
+# shellcheck disable=SC2034 # Variable is used in other scripts
+CORE_TERRASERVICES_APPLY=("networking" "container_cluster" "gke_enterprise/fleet_membership")
+# shellcheck disable=SC1091
+source "${ACP_PLATFORM_CORE_DIR}/deploy.sh"
 
 echo "Provisioning the use case resources"
 # shellcheck disable=SC2154 # variable defined in common.sh

platforms/gke/base/use-cases/federated-learning/teardown.sh

@@ -35,10 +35,11 @@ echo "Destroying the core platform"
 "${ACP_PLATFORM_CORE_DIR}/teardown.sh"
 
 for configuration_variable in "${TERRAFORM_CLUSTER_CONFIGURATION[@]}"; do
-  configuration_variable_name="$(echo "${configuration_variable}" | awk ' { print $1 }'))"
-  sed -i "/${configuration_variable_name}/d" "${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
+  remove_terraform_configuration_variable_from_file "${configuration_variable}" "${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
+done
+for configuration_variable in "${TERRAFORM_CORE_INITIALIZE_CONFIGURATION[@]}"; do
+  remove_terraform_configuration_variable_from_file "${configuration_variable}" "${ACP_PLATFORM_SHARED_CONFIG_INITIALIZE_AUTO_VARS_FILE}"
 done
-terraform fmt "${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
 
 end_timestamp_federated_learning=$(date +%s)
 total_runtime_value_federated_learning=$((end_timestamp_federated_learning - start_timestamp_federated_learning))

platforms/gke/base/use-cases/federated-learning/terraform/_shared_config/uc_federated_learning_variables.tf

@@ -11,3 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+locals {
+  gke_robot_service_account           = "service-${data.google_project.default.number}@container-engine-robot.iam.gserviceaccount.com"
+  gke_robot_service_account_iam_email = "serviceAccount:${local.gke_robot_service_account}"
+}

platforms/gke/base/use-cases/federated-learning/terraform/key_management_service/_cluster.auto.tfvars

@@ -0,0 +1 @@
+../../../../_shared_config/cluster.auto.tfvars

platforms/gke/base/use-cases/federated-learning/terraform/key_management_service/_cluster_variables.tf

@@ -0,0 +1 @@
+../../../../_shared_config/cluster_variables.tf

platforms/gke/base/use-cases/federated-learning/terraform/key_management_service/_platform.auto.tfvars

@@ -0,0 +1 @@
+../../../../_shared_config/platform.auto.tfvars

platforms/gke/base/use-cases/federated-learning/terraform/key_management_service/_platform_variables.tf

@@ -0,0 +1 @@
+../../../../_shared_config/platform_variables.tf

platforms/gke/base/use-cases/federated-learning/terraform/key_management_service/_terraform.auto.tfvars

@@ -0,0 +1 @@
+../../../../_shared_config/terraform.auto.tfvars

platforms/gke/base/use-cases/federated-learning/terraform/key_management_service/_terraform_variables.tf

@@ -0,0 +1 @@
+../../../../_shared_config/terraform_variables.tf

platforms/gke/base/use-cases/federated-learning/terraform/key_management_service/_uc_federated_learning.auto.tfvars

@@ -0,0 +1 @@
+../_shared_config/uc_federated_learning.auto.tfvars

platforms/gke/base/use-cases/federated-learning/terraform/key_management_service/_uc_federated_learning_variables.tf

@@ -0,0 +1 @@
+../_shared_config/uc_federated_learning_variables.tf

platforms/gke/base/use-cases/federated-learning/terraform/key_management_service/main.tf

@@ -0,0 +1,57 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# KeyRings cannot be deleted; append a random suffix to the keyring name
+resource "random_id" "keyring_suffix" {
+  byte_length = 4
+}
+
+resource "google_kms_key_ring" "key_ring" {
+  name     = "${local.unique_identifier_prefix}-keyring-${random_id.keyring_suffix.hex}"
+  project  = google_project_service.cloudkms_googleapis_com.project
+  location = var.cluster_region
+}
+
+resource "google_kms_crypto_key" "cluster_secrects_key" {
+  name                          = "${local.unique_identifier_prefix}-clusterSecretsKey"
+  key_ring                      = google_kms_key_ring.key_ring.id
+  rotation_period               = "7776000s"
+  purpose                       = "ENCRYPT_DECRYPT"
+  import_only                   = false
+  skip_initial_version_creation = false
+
+  lifecycle {
+    prevent_destroy = false
+  }
+
+  version_template {
+    # Ref: https://cloud.google.com/kms/docs/reference/rest/v1/CryptoKeyVersionAlgorithm
+    algorithm = "GOOGLE_SYMMETRIC_ENCRYPTION"
+
+    # Ref: https://cloud.google.com/kms/docs/reference/rest/v1/ProtectionLevel
+    protection_level = "SOFTWARE"
+  }
+}
+
+resource "google_kms_crypto_key_iam_binding" "cluster_secrets_decrypters" {
+  role          = "roles/cloudkms.cryptoKeyDecrypter"
+  crypto_key_id = google_kms_crypto_key.cluster_secrects_key.id
+  members       = [local.gke_robot_service_account_iam_email]
+}
+
+resource "google_kms_crypto_key_iam_binding" "cluster_secrets_encrypters" {
+  role          = "roles/cloudkms.cryptoKeyEncrypter"
+  crypto_key_id = google_kms_crypto_key.cluster_secrects_key.id
+  members       = [local.gke_robot_service_account_iam_email]
+}

platforms/gke/base/use-cases/federated-learning/terraform/key_management_service/output.tf

@@ -0,0 +1,18 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+output "cluster_database_encryption_key_id" {
+  description = "Id of the cluster database encryption key"
+  value       = google_kms_crypto_key.cluster_secrects_key.id
+}