deploy: Modularize gcp terraform (#717)

refactor of gcp terraform into modules

* Add maintenance time variable with default
* Changes bastion image to centos
* Removes destroy trigger for patching reclaimpolicy PV
* Adds bash script to change pv reclaimpolicy from Retain to Delete

deploy/gcp/change-pv-reclaimpolicy.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# Set the ReclaimPolicy of persistent volumes bound to PVCs for a TiDB cluster in a given namespace
+# Inputs: Path to a valid kubeconfig file and the namespace in which the PVCs live.
+# Run before terraform destroy
+
+set -euo pipefail
+set -x
+
+KUBECONFIGFILE=$1
+NAMESPACE=$2
+
+if [[ ! -f ${KUBECONFIGFILE} ]]; then
+    echo "The given kubeconfig file does not exist"
+    exit 1
+fi
+
+if ! kubectl --kubeconfig ${KUBECONFIGFILE} get ns ${NAMESPACE}; then
+    echo "The given namespace was not found in the kubernetes cluster for the given kubeconfig file"
+    exit 1
+fi
+
+kubectl --kubeconfig ${KUBECONFIGFILE} get pvc -n ${NAMESPACE} -o jsonpath='{.items[*].spec.volumeName}'|fmt -1 | xargs -I {} kubectl --kubeconfig ${KUBECONFIGFILE} patch pv {} -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}'

deploy/gcp/create-service-account.sh

@@ -1,9 +1,15 @@
 #!/usr/bin/env bash
+# Create a service account with permissions needed for the terraform
+#
+# This script is currently designed to be idempotent and re-runnable, like terraform.
+#
+# We could write this in terraform, but there is a bootstrapping issue,
+# so it cannot just be added to the existing terraform.
 
 set -euo pipefail
 cd "$(dirname "$0")"
 PROJECT="${TF_VAR_GCP_PROJECT:-$(cat terraform.tfvars | awk -F '=' '/GCP_PROJECT/ {print $2}' | cut -d '"' -f 2)}"
-echo "$PROJECT"
+echo "using project: $PROJECT"
 
 cred_file=credentials.auto.tfvars
 if test -f "$cred_file" ; then
@@ -13,15 +19,33 @@ if test -f "$cred_file" ; then
   fi
 fi
 
-gcloud iam service-accounts create --display-name terraform terraform
-email="terraform@${PROJECT}.iam.gserviceaccount.com"
-gcloud projects add-iam-policy-binding "$PROJECT" --member "$email" --role roles/container.clusterAdmin
-gcloud projects add-iam-policy-binding "$PROJECT" --member "$email" --role roles/compute.networkAdmin
-gcloud projects add-iam-policy-binding "$PROJECT" --member "$email" --role roles/compute.viewer
-gcloud projects add-iam-policy-binding "$PROJECT" --member "$email" --role roles/compute.securityAdmin
-gcloud projects add-iam-policy-binding "$PROJECT" --member "$email" --role roles/iam.serviceAccountUser
-gcloud projects add-iam-policy-binding "$PROJECT" --member "$email" --role roles/compute.instanceAdmin.v1
+GCLOUD="gcloud --project $PROJECT"
 
 mkdir -p credentials
-gcloud iam service-accounts keys create credentials/terraform-key.json --iam-account "$email"
-echo GCP_CREDENTIALS_PATH="$(pwd)/credentials/terraform-key.json" > "$cred_file"
+key_file=credentials/terraform-key.json
+email="terraform@${PROJECT}.iam.gserviceaccount.com"
+
+sas=$($GCLOUD iam service-accounts list)
+if echo "$sas" | grep terraform >/dev/null ; then
+  if test -f $key_file && grep "$PROJECT" $key_file >/dev/null ; then
+    echo "service account terraform already exists along with the key file. Will set terraform variables"
+  else
+    echo "service account terraform already exists, will get a key for it"
+    $GCLOUD iam service-accounts keys create $key_file --iam-account "$email"
+  fi
+else
+  echo "creating a new service account terraform"
+  $GCLOUD iam service-accounts create --display-name terraform terraform
+  $GCLOUD iam service-accounts keys create $key_file --iam-account "$email"
+fi
+
+chmod 0600 $key_file
+
+$GCLOUD projects add-iam-policy-binding "$PROJECT" --member "serviceAccount:$email" --role roles/container.clusterAdmin
+$GCLOUD projects add-iam-policy-binding "$PROJECT" --member "serviceAccount:$email" --role roles/compute.networkAdmin
+$GCLOUD projects add-iam-policy-binding "$PROJECT" --member "serviceAccount:$email" --role roles/compute.viewer
+$GCLOUD projects add-iam-policy-binding "$PROJECT" --member "serviceAccount:$email" --role roles/compute.securityAdmin
+$GCLOUD projects add-iam-policy-binding "$PROJECT" --member "serviceAccount:$email" --role roles/iam.serviceAccountUser
+$GCLOUD projects add-iam-policy-binding "$PROJECT" --member "serviceAccount:$email" --role roles/compute.instanceAdmin.v1
+
+echo GCP_CREDENTIALS_PATH="\"$(pwd)/$key_file\"" > "$cred_file"