From 81fe8511eea277e10ce56ac4151c0aa94ece8c7d Mon Sep 17 00:00:00 2001 From: Jacob Lerche Date: Wed, 14 Aug 2019 21:56:37 -0700 Subject: [PATCH] deploy: Modularize gcp terraform (#717) refactor of gcp terraform into modules * Add maintenance time variable with default * Changes bastion image to centos * Removes destroy trigger for patching reclaimpolicy PV * Adds bash script to change pv reclaimpolicy from Retain to Delete --- deploy/gcp/change-pv-reclaimpolicy.sh | 22 + deploy/gcp/create-service-account.sh | 46 +- deploy/gcp/data.tf | 35 -- deploy/gcp/main.tf | 421 ++---------------- deploy/gcp/outputs.tf | 30 +- .../templates/tidb-cluster-values.yaml.tpl | 416 ----------------- deploy/gcp/tidbclusters.tf | 39 ++ deploy/gcp/variables.tf | 45 +- deploy/modules/gcp/bastion/bastion.tf | 64 +++ deploy/modules/gcp/bastion/data.tf | 5 + deploy/modules/gcp/bastion/outputs.tf | 3 + deploy/modules/gcp/bastion/variables.tf | 13 + .../modules/gcp/project-credentials/main.tf | 5 + .../gcp/project-credentials/outputs.tf | 3 + .../gcp/project-credentials/variables.tf | 4 + deploy/modules/gcp/tidb-cluster/data.tf | 21 + deploy/modules/gcp/tidb-cluster/main.tf | 161 +++++++ deploy/modules/gcp/tidb-cluster/outputs.tf | 33 ++ .../gcp/tidb-cluster/values/default.yaml | 21 + deploy/modules/gcp/tidb-cluster/variables.tf | 46 ++ deploy/modules/gcp/tidb-operator/main.tf | 140 ++++++ deploy/modules/gcp/tidb-operator/outputs.tf | 19 + deploy/modules/gcp/tidb-operator/variables.tf | 47 ++ deploy/modules/gcp/vpc/main.tf | 37 ++ deploy/modules/gcp/vpc/outputs.tf | 11 + deploy/modules/gcp/vpc/variables.tf | 40 ++ .../share/tidb-cluster-release/main.tf | 4 +- .../share/tidb-cluster-release/variables.tf | 5 + 28 files changed, 851 insertions(+), 885 deletions(-) create mode 100755 deploy/gcp/change-pv-reclaimpolicy.sh delete mode 100644 deploy/gcp/data.tf delete mode 100644 deploy/gcp/templates/tidb-cluster-values.yaml.tpl create mode 100644 deploy/gcp/tidbclusters.tf create mode 100644 deploy/modules/gcp/bastion/bastion.tf create mode 100644 deploy/modules/gcp/bastion/data.tf create mode 100644 deploy/modules/gcp/bastion/outputs.tf create mode 100644 deploy/modules/gcp/bastion/variables.tf create mode 100644 deploy/modules/gcp/project-credentials/main.tf create mode 100644 deploy/modules/gcp/project-credentials/outputs.tf create mode 100644 deploy/modules/gcp/project-credentials/variables.tf create mode 100644 deploy/modules/gcp/tidb-cluster/data.tf create mode 100644 deploy/modules/gcp/tidb-cluster/main.tf create mode 100644 deploy/modules/gcp/tidb-cluster/outputs.tf create mode 100644 deploy/modules/gcp/tidb-cluster/values/default.yaml create mode 100644 deploy/modules/gcp/tidb-cluster/variables.tf create mode 100644 deploy/modules/gcp/tidb-operator/main.tf create mode 100644 deploy/modules/gcp/tidb-operator/outputs.tf create mode 100644 deploy/modules/gcp/tidb-operator/variables.tf create mode 100644 deploy/modules/gcp/vpc/main.tf create mode 100644 deploy/modules/gcp/vpc/outputs.tf create mode 100644 deploy/modules/gcp/vpc/variables.tf diff --git a/deploy/gcp/change-pv-reclaimpolicy.sh b/deploy/gcp/change-pv-reclaimpolicy.sh new file mode 100755 index 0000000000..a780222961 --- /dev/null +++ b/deploy/gcp/change-pv-reclaimpolicy.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Set the ReclaimPolicy of persistent volumes bound to PVCs for a TiDB cluster in a given namespace +# Inputs: Path to a valid kubeconfig file and the namespace in which the PVCs live. +# Run before terraform destroy + +set -euo pipefail +set -x + +KUBECONFIGFILE=$1 +NAMESPACE=$2 + +if [[ ! -f ${KUBECONFIGFILE} ]]; then + echo "The given kubeconfig file does not exist" + exit 1 +fi + +if ! kubectl --kubeconfig ${KUBECONFIGFILE} get ns ${NAMESPACE}; then + echo "The given namespace was not found in the kubernetes cluster for the given kubeconfig file" + exit 1 +fi + +kubectl --kubeconfig ${KUBECONFIGFILE} get pvc -n ${NAMESPACE} -o jsonpath='{.items[*].spec.volumeName}'|fmt -1 | xargs -I {} kubectl --kubeconfig ${KUBECONFIGFILE} patch pv {} -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' diff --git a/deploy/gcp/create-service-account.sh b/deploy/gcp/create-service-account.sh index d0ca3223ee..805b45afce 100755 --- a/deploy/gcp/create-service-account.sh +++ b/deploy/gcp/create-service-account.sh @@ -1,9 +1,15 @@ #!/usr/bin/env bash +# Create a service account with permissions needed for the terraform +# +# This script is currently designed to be idempotent and re-runnable, like terraform. +# +# We could write this in terraform, but there is a bootstrapping issue, +# so it cannot just be added to the existing terraform. set -euo pipefail cd "$(dirname "$0")" PROJECT="${TF_VAR_GCP_PROJECT:-$(cat terraform.tfvars | awk -F '=' '/GCP_PROJECT/ {print $2}' | cut -d '"' -f 2)}" -echo "$PROJECT" +echo "using project: $PROJECT" cred_file=credentials.auto.tfvars if test -f "$cred_file" ; then @@ -13,15 +19,33 @@ if test -f "$cred_file" ; then fi fi -gcloud iam service-accounts create --display-name terraform terraform -email="terraform@${PROJECT}.iam.gserviceaccount.com" -gcloud projects add-iam-policy-binding "$PROJECT" --member "$email" --role roles/container.clusterAdmin -gcloud projects add-iam-policy-binding "$PROJECT" --member "$email" --role roles/compute.networkAdmin -gcloud projects add-iam-policy-binding "$PROJECT" --member "$email" --role roles/compute.viewer -gcloud projects add-iam-policy-binding "$PROJECT" --member "$email" --role roles/compute.securityAdmin -gcloud projects add-iam-policy-binding "$PROJECT" --member "$email" --role roles/iam.serviceAccountUser -gcloud projects add-iam-policy-binding "$PROJECT" --member "$email" --role roles/compute.instanceAdmin.v1 +GCLOUD="gcloud --project $PROJECT" mkdir -p credentials -gcloud iam service-accounts keys create credentials/terraform-key.json --iam-account "$email" -echo GCP_CREDENTIALS_PATH="$(pwd)/credentials/terraform-key.json" > "$cred_file" +key_file=credentials/terraform-key.json +email="terraform@${PROJECT}.iam.gserviceaccount.com" + +sas=$($GCLOUD iam service-accounts list) +if echo "$sas" | grep terraform >/dev/null ; then + if test -f $key_file && grep "$PROJECT" $key_file >/dev/null ; then + echo "service account terraform already exists along with the key file. Will set terraform variables" + else + echo "service account terraform already exists, will get a key for it" + $GCLOUD iam service-accounts keys create $key_file --iam-account "$email" + fi +else + echo "creating a new service account terraform" + $GCLOUD iam service-accounts create --display-name terraform terraform + $GCLOUD iam service-accounts keys create $key_file --iam-account "$email" +fi + +chmod 0600 $key_file + +$GCLOUD projects add-iam-policy-binding "$PROJECT" --member "serviceAccount:$email" --role roles/container.clusterAdmin +$GCLOUD projects add-iam-policy-binding "$PROJECT" --member "serviceAccount:$email" --role roles/compute.networkAdmin +$GCLOUD projects add-iam-policy-binding "$PROJECT" --member "serviceAccount:$email" --role roles/compute.viewer +$GCLOUD projects add-iam-policy-binding "$PROJECT" --member "serviceAccount:$email" --role roles/compute.securityAdmin +$GCLOUD projects add-iam-policy-binding "$PROJECT" --member "serviceAccount:$email" --role roles/iam.serviceAccountUser +$GCLOUD projects add-iam-policy-binding "$PROJECT" --member "serviceAccount:$email" --role roles/compute.instanceAdmin.v1 + +echo GCP_CREDENTIALS_PATH="\"$(pwd)/$key_file\"" > "$cred_file" diff --git a/deploy/gcp/data.tf b/deploy/gcp/data.tf deleted file mode 100644 index 0e760bfbd9..0000000000 --- a/deploy/gcp/data.tf +++ /dev/null @@ -1,35 +0,0 @@ -data "template_file" "tidb_cluster_values" { - template = file("${path.module}/templates/tidb-cluster-values.yaml.tpl") - - vars = { - cluster_version = var.tidb_version - pd_replicas = var.pd_replica_count - tikv_replicas = var.tikv_replica_count - tidb_replicas = var.tidb_replica_count - operator_version = var.tidb_operator_version - tidb_operator_registry = var.tidb_operator_registry - } -} - -data "google_compute_zones" "available" {} - -data "external" "tidb_ilb_ip" { - depends_on = [null_resource.deploy-tidb-cluster] - program = ["bash", "-c", "kubectl --kubeconfig ${local.kubeconfig} get svc -n tidb tidb-cluster-tidb -o json | jq '.status.loadBalancer.ingress[0]'"] -} - -data "external" "monitor_ilb_ip" { - depends_on = [null_resource.deploy-tidb-cluster] - program = ["bash", "-c", "kubectl --kubeconfig ${local.kubeconfig} get svc -n tidb tidb-cluster-grafana -o json | jq '.status.loadBalancer.ingress[0]'"] -} - -data "external" "tidb_port" { - depends_on = [null_resource.deploy-tidb-cluster] - program = ["bash", "-c", "kubectl --kubeconfig ${local.kubeconfig} get svc -n tidb tidb-cluster-tidb -o json | jq '.spec.ports | .[] | select( .name == \"mysql-client\") | {port: .port|tostring}'"] -} - -data "external" "monitor_port" { - depends_on = [null_resource.deploy-tidb-cluster] - program = ["bash", "-c", "kubectl --kubeconfig ${local.kubeconfig} get svc -n tidb tidb-cluster-grafana -o json | jq '.spec.ports | .[] | select( .name == \"grafana\") | {port: .port|tostring}'"] -} - diff --git a/deploy/gcp/main.tf b/deploy/gcp/main.tf index e510dcaf61..2935751a44 100644 --- a/deploy/gcp/main.tf +++ b/deploy/gcp/main.tf @@ -1,12 +1,3 @@ -variable "GCP_CREDENTIALS_PATH" { -} - -variable "GCP_REGION" { -} - -variable "GCP_PROJECT" { -} - provider "google" { credentials = file(var.GCP_CREDENTIALS_PATH) region = var.GCP_REGION @@ -21,397 +12,43 @@ provider "google-beta" { } locals { - credential_path = "${path.module}/credentials" - kubeconfig = "${local.credential_path}/kubeconfig_${var.cluster_name}" - tidb_cluster_values_path = "${path.module}/rendered/tidb-cluster-values.yaml" + credential_path = "${path.cwd}/credentials" + kubeconfig = "${local.credential_path}/kubeconfig_${var.gke_name}" } -resource "null_resource" "prepare-dir" { - provisioner "local-exec" { - command = "mkdir -p ${local.credential_path}" - } -} -resource "null_resource" "set-gcloud-project" { - provisioner "local-exec" { - command = "gcloud config set project ${var.GCP_PROJECT}" - } -} +module "project-credentials" { + source = "../modules/gcp/project-credentials" -resource "google_compute_network" "vpc_network" { - name = "${var.cluster_name}-vpc-network" - auto_create_subnetworks = false - project = var.GCP_PROJECT + path = local.credential_path } -resource "google_compute_subnetwork" "private_subnet" { - ip_cidr_range = "172.31.252.0/22" - name = "${var.cluster_name}-private-subnet" - network = google_compute_network.vpc_network.name - project = var.GCP_PROJECT - - secondary_ip_range { - ip_cidr_range = "172.30.0.0/16" - range_name = "pods-${var.GCP_REGION}" - } - - secondary_ip_range { - ip_cidr_range = "172.31.224.0/20" - range_name = "services-${var.GCP_REGION}" - } - - lifecycle { - ignore_changes = [secondary_ip_range] - } +module "vpc" { + source = "../modules/gcp/vpc" + create_vpc = var.create_vpc + gcp_project = var.GCP_PROJECT + gcp_region = var.GCP_REGION + vpc_name = var.vpc_name + private_subnet_name = "${var.gke_name}-private-subnet" + public_subnet_name = "${var.gke_name}-public-subnet" } -resource "google_compute_subnetwork" "public_subnet" { - ip_cidr_range = "172.29.252.0/22" - name = "${var.cluster_name}-public-subnet" - network = google_compute_network.vpc_network.name - project = var.GCP_PROJECT +module "tidb-operator" { + source = "../modules/gcp/tidb-operator" + gke_name = var.gke_name + vpc_name = var.vpc_name + subnetwork_name = module.vpc.private_subnetwork_name + gcp_project = var.GCP_PROJECT + gcp_region = var.GCP_REGION + kubeconfig_path = local.kubeconfig + tidb_operator_version = var.tidb_operator_version + maintenance_window_start_time = var.maintenance_window_start_time } -resource "google_container_cluster" "cluster" { - name = var.cluster_name - network = google_compute_network.vpc_network.name - subnetwork = google_compute_subnetwork.private_subnet.name - location = var.GCP_REGION - project = var.GCP_PROJECT - - master_auth { - username = "" - password = "" - - // due to https://github.com/terraform-providers/terraform-provider-google/issues/3369 - client_certificate_config { - issue_client_certificate = false - } - } - - master_authorized_networks_config { - cidr_blocks { - cidr_block = "0.0.0.0/0" - } - } - - ip_allocation_policy { - use_ip_aliases = true - } - - // See https://kubernetes.io/docs/setup/best-practices/cluster-large/#size-of-master-and-master-components for why initial_node_count is 5. The master node should accommodate up to 100 nodes like this - remove_default_node_pool = true - initial_node_count = 5 - - min_master_version = "latest" - - lifecycle { - ignore_changes = [master_auth] // see above linked issue - } - - maintenance_policy { - daily_maintenance_window { - start_time = "01:00" - } - } +module "bastion" { + source = "../modules/gcp/bastion" + vpc_name = module.vpc.vpc_name + public_subnet_name = module.vpc.public_subnetwork_name + gcp_project = var.GCP_PROJECT + bastion_name = "${var.gke_name}-tidb-bastion" } - -resource "google_container_node_pool" "pd_pool" { - // The monitor pool is where tiller must first be deployed to. - depends_on = [google_container_node_pool.monitor_pool] - provider = google-beta - project = var.GCP_PROJECT - cluster = google_container_cluster.cluster.name - location = google_container_cluster.cluster.location - name = "pd-pool" - node_count = var.pd_count - - management { - auto_repair = false - auto_upgrade = false - } - - node_config { - machine_type = var.pd_instance_type - local_ssd_count = 0 - - taint { - effect = "NO_SCHEDULE" - key = "dedicated" - value = "pd" - } - - labels = { - dedicated = "pd" - } - - tags = ["pd"] - oauth_scopes = ["storage-ro", "logging-write", "monitoring"] - } -} - -resource "google_container_node_pool" "tikv_pool" { - provider = google-beta - project = var.GCP_PROJECT - cluster = google_container_cluster.cluster.name - location = google_container_cluster.cluster.location - name = "tikv-pool" - node_count = var.tikv_count - - management { - auto_repair = false - auto_upgrade = false - } - - node_config { - machine_type = var.tikv_instance_type - image_type = "UBUNTU" - // This value cannot be changed (instead a new node pool is needed) - // 1 SSD is 375 GiB - local_ssd_count = 1 - - taint { - effect = "NO_SCHEDULE" - key = "dedicated" - value = "tikv" - } - - labels = { - dedicated = "tikv" - } - - tags = ["tikv"] - oauth_scopes = ["storage-ro", "logging-write", "monitoring"] - } -} - -resource "google_container_node_pool" "tidb_pool" { - // The pool order is tikv -> monitor -> pd -> tidb - depends_on = [google_container_node_pool.pd_pool] - provider = google-beta - project = var.GCP_PROJECT - cluster = google_container_cluster.cluster.name - location = google_container_cluster.cluster.location - name = "tidb-pool" - node_count = var.tidb_count - - management { - auto_repair = false - auto_upgrade = false - } - - node_config { - machine_type = var.tidb_instance_type - - taint { - effect = "NO_SCHEDULE" - key = "dedicated" - value = "tidb" - } - - labels = { - dedicated = "tidb" - } - - tags = ["tidb"] - oauth_scopes = ["storage-ro", "logging-write", "monitoring"] - } -} - -resource "google_container_node_pool" "monitor_pool" { - // Setup local SSD on TiKV nodes first (this can take some time) - // Create the monitor pool next because that is where tiller will be deployed to - depends_on = [google_container_node_pool.tikv_pool] - project = var.GCP_PROJECT - cluster = google_container_cluster.cluster.name - location = google_container_cluster.cluster.location - name = "monitor-pool" - node_count = var.monitor_count - - management { - auto_repair = false - auto_upgrade = false - } - - node_config { - machine_type = var.monitor_instance_type - tags = ["monitor"] - oauth_scopes = ["storage-ro", "logging-write", "monitoring"] - } -} - -resource "google_compute_firewall" "allow_ssh_bastion" { - name = "allow-ssh-bastion" - network = google_compute_network.vpc_network.self_link - project = var.GCP_PROJECT - - allow { - protocol = "tcp" - ports = ["22"] - } - - source_ranges = ["0.0.0.0/0"] - target_tags = ["bastion"] -} - -resource "google_compute_firewall" "allow_mysql_from_bastion" { - name = "allow-mysql-from-bastion" - network = google_compute_network.vpc_network.self_link - project = var.GCP_PROJECT - - allow { - protocol = "tcp" - ports = ["4000"] - } - - source_tags = ["bastion"] - target_tags = ["tidb"] -} - -resource "google_compute_firewall" "allow_ssh_from_bastion" { - name = "allow-ssh-from-bastion" - network = google_compute_network.vpc_network.self_link - project = var.GCP_PROJECT - - allow { - protocol = "tcp" - ports = ["22"] - } - - source_tags = ["bastion"] - target_tags = ["tidb", "tikv", "pd", "monitor"] -} - -resource "google_compute_instance" "bastion" { - project = var.GCP_PROJECT - zone = data.google_compute_zones.available.names[0] - machine_type = var.bastion_instance_type - name = "bastion" - - boot_disk { - initialize_params { - image = "ubuntu-os-cloud/ubuntu-1804-lts" - } - } - - network_interface { - subnetwork = google_compute_subnetwork.public_subnet.self_link - access_config { - } - } - - tags = ["bastion"] - - metadata_startup_script = "sudo apt-get install -y mysql-client && curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh | bash && sudo apt-get -y install sysbench" -} - -resource "null_resource" "get-credentials" { - provisioner "local-exec" { - command = "gcloud container clusters get-credentials ${google_container_cluster.cluster.name} --region ${var.GCP_REGION}" - - environment = { - KUBECONFIG = local.kubeconfig - } - } - - provisioner "local-exec" { - when = destroy - - command = </dev/null; then - kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $(gcloud config get-value account) -fi - -if ! kubectl get serviceaccount -n kube-system tiller 2>/dev/null ; then - kubectl create serviceaccount --namespace kube-system tiller -fi - -kubectl apply -f manifests/crd.yaml -kubectl apply -k manifests/local-ssd -kubectl apply -f manifests/gke/persistent-disk.yaml -kubectl apply -f manifests/tiller-rbac.yaml - -helm init --service-account tiller --upgrade -until helm ls; do - echo "Wait until tiller is ready" - sleep 1 -done -helm upgrade --install tidb-operator --namespace tidb-admin ${path.module}/charts/tidb-operator --set operatorImage=${var.tidb_operator_registry}/tidb-operator:${var.tidb_operator_version} -EOS - - - environment = { - KUBECONFIG = local.kubeconfig - } - } -} - -resource "null_resource" "deploy-tidb-cluster" { - depends_on = [ - null_resource.setup-env, - local_file.tidb-cluster-values, - google_container_node_pool.pd_pool, - google_container_node_pool.tikv_pool, - google_container_node_pool.tidb_pool, - google_container_node_pool.monitor_pool - ] - - triggers = { - values = data.template_file.tidb_cluster_values.rendered - } - - provisioner "local-exec" { - interpreter = ["bash", "-c"] -command = < 8, default thread pool size for coprocessors - # will be set to tikv.resources.limits.cpu * 0.8. - # readpoolCoprocessorConcurrency: 8 - - # scheduler's worker pool size, should increase it in heavy write cases, - # also should less than total cpu cores. - # storageSchedulerWorkerPoolSize: 4 - -tidb: - replicas: ${tidb_replicas} - # The secret name of root password, you can create secret with following command: - # kubectl create secret generic tidb-secret --from-literal=root= --namespace= - # If unset, the root password will be empty and you can set it after connecting - # passwordSecretName: tidb-secret - # initSql is the SQL statements executed after the TiDB cluster is bootstrapped. - # initSql: |- - # create database app; - image: "pingcap/tidb:${cluster_version}" - # Image pull policy. - imagePullPolicy: IfNotPresent - logLevel: info - preparedPlanCacheEnabled: false - preparedPlanCacheCapacity: 100 - # Enable local latches for transactions. Enable it when - # there are lots of conflicts between transactions. - txnLocalLatchesEnabled: false - txnLocalLatchesCapacity: "10240000" - # The limit of concurrent executed sessions. - tokenLimit: "1000" - # Set the memory quota for a query in bytes. Default: 32GB - memQuotaQuery: "34359738368" - # The limitation of the number for the entries in one transaction. - # If using TiKV as the storage, the entry represents a key/value pair. - # WARNING: Do not set the value too large, otherwise it will make a very large impact on the TiKV cluster. - # Please adjust this configuration carefully. - txnEntryCountLimit: "300000" - # The limitation of the size in byte for the entries in one transaction. - # If using TiKV as the storage, the entry represents a key/value pair. - # WARNING: Do not set the value too large, otherwise it will make a very large impact on the TiKV cluster. - # Please adjust this configuration carefully. - txnTotalSizeLimit: "104857600" - # enableBatchDml enables batch commit for the DMLs - enableBatchDml: false - # check mb4 value in utf8 is used to control whether to check the mb4 characters when the charset is utf8. - checkMb4ValueInUtf8: true - # treat-old-version-utf8-as-utf8mb4 use for upgrade compatibility. Set to true will treat old version table/column UTF8 charset as UTF8MB4. - treatOldVersionUtf8AsUtf8mb4: true - # lease is schema lease duration, very dangerous to change only if you know what you do. - lease: 45s - # Max CPUs to use, 0 use number of CPUs in the machine. - maxProcs: 0 - resources: - limits: {} - # cpu: 16000m - # memory: 16Gi - requests: {} - # cpu: 12000m - # memory: 12Gi - nodeSelector: - dedicated: tidb - # kind: tidb - # zone: cn-bj1-01,cn-bj1-02 - # region: cn-bj1 - tolerations: - - key: dedicated - operator: Equal - value: tidb - effect: "NoSchedule" - maxFailoverCount: 3 - service: - type: LoadBalancer - exposeStatus: true - annotations: - cloud.google.com/load-balancer-type: "Internal" - # separateSlowLog: true - slowLogTailer: - image: busybox:1.26.2 - resources: - limits: - cpu: 100m - memory: 50Mi - requests: - cpu: 20m - memory: 5Mi - - # tidb plugin configuration - plugin: - # enable plugin or not - enable: false - # the start argument to specify the folder containing - directory: /plugins - # the start argument to specify the plugin id (name "-" version) that needs to be loaded, e.g. 'conn_limit-1'. - list: ["whitelist-1"] - -# mysqlClient is used to set password for TiDB -# it must has Python MySQL client installed -mysqlClient: - image: tnir/mysqlclient - imagePullPolicy: IfNotPresent - -monitor: - create: true - # Also see rbac.create - # If you set rbac.create to false, you need to provide a value here. - # If you set rbac.create to true, you should leave this empty. - # serviceAccount: - persistent: true - storageClassName: pd-ssd - storage: 500Gi - grafana: - create: true - image: grafana/grafana:6.0.1 - imagePullPolicy: IfNotPresent - logLevel: info - resources: - limits: {} - # cpu: 8000m - # memory: 8Gi - requests: {} - # cpu: 4000m - # memory: 4Gi - username: admin - password: admin - config: - # Configure Grafana using environment variables except GF_PATHS_DATA, GF_SECURITY_ADMIN_USER and GF_SECURITY_ADMIN_PASSWORD - # Ref https://grafana.com/docs/installation/configuration/#using-environment-variables - GF_AUTH_ANONYMOUS_ENABLED: "true" - GF_AUTH_ANONYMOUS_ORG_NAME: "Main Org." - GF_AUTH_ANONYMOUS_ORG_ROLE: "Viewer" - # if grafana is running behind a reverse proxy with subpath http://foo.bar/grafana - # GF_SERVER_DOMAIN: foo.bar - # GF_SERVER_ROOT_URL: "%(protocol)s://%(domain)s/grafana/" - service: - type: LoadBalancer - prometheus: - image: prom/prometheus:v2.2.1 - imagePullPolicy: IfNotPresent - logLevel: info - resources: - limits: {} - # cpu: 8000m - # memory: 8Gi - requests: {} - # cpu: 4000m - # memory: 4Gi - service: - type: NodePort - reserveDays: 12 - # alertmanagerURL: "" - nodeSelector: {} - # kind: monitor - # zone: cn-bj1-01,cn-bj1-02 - # region: cn-bj1 - tolerations: [] - # - key: node-role - # operator: Equal - # value: tidb - # effect: "NoSchedule" - -binlog: - pump: - create: false - replicas: 1 - image: "pingcap/tidb-binlog:${cluster_version}" - imagePullPolicy: IfNotPresent - logLevel: info - # storageClassName is a StorageClass provides a way for administrators to describe the "classes" of storage they offer. - # different classes might map to quality-of-service levels, or to backup policies, - # or to arbitrary policies determined by the cluster administrators. - # refer to https://kubernetes.io/docs/concepts/storage/storage-classes - storageClassName: local-storage - storage: 10Gi - syncLog: true - # a integer value to control expiry date of the binlog data, indicates for how long (in days) the binlog data would be stored. - # must bigger than 0 - gc: 7 - # number of seconds between heartbeat ticks (in 2 seconds) - heartbeatInterval: 2 - - drainer: - create: false - image: "pingcap/tidb-binlog:${cluster_version}" - imagePullPolicy: IfNotPresent - logLevel: info - # storageClassName is a StorageClass provides a way for administrators to describe the "classes" of storage they offer. - # different classes might map to quality-of-service levels, or to backup policies, - # or to arbitrary policies determined by the cluster administrators. - # refer to https://kubernetes.io/docs/concepts/storage/storage-classes - storageClassName: local-storage - storage: 10Gi - # parallel worker count (default 16) - workerCount: 16 - # the interval time (in seconds) of detect pumps' status (default 10) - detectInterval: 10 - # disbale detect causality - disableDetect: false - # disable dispatching sqls that in one same binlog; if set true, work-count and txn-batch would be useless - disableDispatch: false - # # disable sync these schema - ignoreSchemas: "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql,test" - # if drainer donesn't have checkpoint, use initial commitTS to initial checkpoint - initialCommitTs: 0 - # enable safe mode to make syncer reentrant - safeMode: false - # number of binlog events in a transaction batch (default 20) - txnBatch: 20 - # downstream storage, equal to --dest-db-type - # valid values are "mysql", "pb", "kafka" - destDBType: pb - mysql: {} - # host: "127.0.0.1" - # user: "root" - # password: "" - # port: 3306 - # # Time and size limits for flash batch write - # timeLimit: "30s" - # sizeLimit: "100000" - kafka: {} - # only need config one of zookeeper-addrs and kafka-addrs, will get kafka address if zookeeper-addrs is configed. - # zookeeperAddrs: "127.0.0.1:2181" - # kafkaAddrs: "127.0.0.1:9092" - # kafkaVersion: "0.8.2.0" - -scheduledBackup: - create: false - binlogImage: "pingcap/tidb-binlog:${cluster_version}" - binlogImagePullPolicy: IfNotPresent - # https://github.com/tennix/tidb-cloud-backup - mydumperImage: pingcap/tidb-cloud-backup:20190610 - mydumperImagePullPolicy: IfNotPresent - # storageClassName is a StorageClass provides a way for administrators to describe the "classes" of storage they offer. - # different classes might map to quality-of-service levels, or to backup policies, - # or to arbitrary policies determined by the cluster administrators. - # refer to https://kubernetes.io/docs/concepts/storage/storage-classes - storageClassName: local-storage - storage: 100Gi - # https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule - schedule: "0 0 * * *" - # https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#suspend - suspend: false - # https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#jobs-history-limits - successfulJobsHistoryLimit: 3 - failedJobsHistoryLimit: 1 - # https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#starting-deadline - startingDeadlineSeconds: 3600 - # https://github.com/maxbube/mydumper/blob/master/docs/mydumper_usage.rst#options - options: "--chunk-filesize=100" - # secretName is the name of the secret which stores user and password used for backup - # Note: you must give the user enough privilege to do the backup - # you can create the secret by: - # kubectl create secret generic backup-secret --from-literal=user=root --from-literal=password= - secretName: backup-secret - # backup to gcp - gcp: {} - # bucket: "" - # secretName is the name of the secret which stores the gcp service account credentials json file - # The service account must have read/write permission to the above bucket. - # Read the following document to create the service account and download the credentials file as credentials.json: - # https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually - # And then create the secret by: kubectl create secret generic gcp-backup-secret --from-file=./credentials.json - # secretName: gcp-backup-secret - - # backup to ceph object storage - ceph: {} - # endpoint: "" - # bucket: "" - # secretName is the name of the secret which stores ceph object store access key and secret key - # You can create the secret by: - # kubectl create secret generic ceph-backup-secret --from-literal=access_key= --from-literal=secret_key= - # secretName: ceph-backup-secret - - # backup to s3 - s3: {} - # region: "" - # bucket: "" - # secretName is the name of the secret which stores s3 object store access key and secret key - # You can create the secret by: - # kubectl create secret generic s3-backup-secret --from-literal=access_key= --from-literal=secret_key= - # secretName: s3-backup-secret - -metaInstance: "{{ $labels.instance }}" -metaType: "{{ $labels.type }}" -metaValue: "{{ $value }}" diff --git a/deploy/gcp/tidbclusters.tf b/deploy/gcp/tidbclusters.tf new file mode 100644 index 0000000000..6588e6ab83 --- /dev/null +++ b/deploy/gcp/tidbclusters.tf @@ -0,0 +1,39 @@ +data "local_file" "kubeconfig" { + depends_on = [module.tidb-operator.cluster_id] + filename = module.tidb-operator.kubeconfig_path +} +resource "local_file" "kubeconfig" { + depends_on = [module.tidb-operator.cluster_id] + content = data.local_file.kubeconfig.content + filename = module.tidb-operator.kubeconfig_path +} + +provider "helm" { + alias = "gke" + insecure = true + install_tiller = false + kubernetes { + config_path = local_file.kubeconfig.filename + } +} +module "default-tidb-cluster" { + providers = { + helm = "helm.gke" + } + source = "../modules/gcp/tidb-cluster" + gcp_project = module.tidb-operator.gcp_project + gke_cluster_location = module.tidb-operator.gke_cluster_location + gke_cluster_name = module.tidb-operator.gke_cluster_name + cluster_name = var.default_tidb_cluster_name + cluster_version = var.tidb_version + kubeconfig_path = module.tidb-operator.kubeconfig_path + tidb_cluster_chart_version = coalesce(var.tidb_operator_chart_version, var.tidb_operator_version) + pd_instance_type = var.pd_instance_type + tikv_instance_type = var.tikv_instance_type + tidb_instance_type = var.tidb_instance_type + monitor_instance_type = var.monitor_instance_type + pd_node_count = var.pd_count + tikv_node_count = var.tikv_count + tidb_node_count = var.tidb_count + monitor_node_count = var.monitor_count +} diff --git a/deploy/gcp/variables.tf b/deploy/gcp/variables.tf index 14906d0c02..e0f25ccbba 100644 --- a/deploy/gcp/variables.tf +++ b/deploy/gcp/variables.tf @@ -1,6 +1,13 @@ -variable "cluster_name" { - description = "TiDB clustername" - default = "tidb" +variable "GCP_CREDENTIALS_PATH" { + description = "A path to to a service account key. See the docs for how to create one with the correct permissions" +} + +variable "GCP_REGION" { + description = "The GCP region in which to create the GKE cluster and associated resources" +} + +variable "GCP_PROJECT" { + description = "The GCP project in which to create the necessary resources" } variable "tidb_version" { @@ -9,13 +16,31 @@ variable "tidb_version" { } variable "tidb_operator_version" { - description = "TiDB operator version" - default = "v1.0.0" + default = "v1.0.0" } -variable "tidb_operator_registry" { - description = "TiDB operator registry" - default = "pingcap" +variable "tidb_operator_chart_version" { + description = "TiDB operator chart version, defaults to tidb_operator_version" + default = "" +} + +variable "create_vpc" { + default = true +} + +variable "gke_name" { + description = "Name of the GKE cluster. Also used as a prefix in names of related resources." + default = "tidb-cluster" +} + +variable "default_tidb_cluster_name" { + description = "The name that will be given to the default tidb cluster created." + default = "tidb-cluster" +} + +variable "vpc_name" { + description = "The name of the VPC network" + default = "tidb-cluster" } variable "pd_replica_count" { @@ -63,3 +88,7 @@ variable "bastion_instance_type" { default = "f1-micro" } +variable "maintenance_window_start_time" { + description = "The time in HH:MM GMT format to define the start of the daily maintenance window" + default = "01:00" +} diff --git a/deploy/modules/gcp/bastion/bastion.tf b/deploy/modules/gcp/bastion/bastion.tf new file mode 100644 index 0000000000..6a4b4e9266 --- /dev/null +++ b/deploy/modules/gcp/bastion/bastion.tf @@ -0,0 +1,64 @@ +resource "google_compute_firewall" "allow_ssh_bastion" { + name = "allow-ssh-bastion" + network = var.vpc_name + project = var.gcp_project + + allow { + protocol = "tcp" + ports = ["22"] + } + + source_ranges = ["0.0.0.0/0"] + target_tags = ["bastion"] +} + +resource "google_compute_firewall" "allow_mysql_from_bastion" { + name = "allow-mysql-from-bastion" + network = var.vpc_name + project = var.gcp_project + + allow { + protocol = "tcp" + ports = ["4000"] + } + + source_tags = ["bastion"] + target_tags = ["tidb"] +} + +resource "google_compute_firewall" "allow_ssh_from_bastion" { + name = "allow-ssh-from-bastion" + network = var.vpc_name + project = var.gcp_project + + allow { + protocol = "tcp" + ports = ["22"] + } + + source_tags = ["bastion"] + target_tags = ["tidb", "tikv", "pd", "monitor"] +} + +resource "google_compute_instance" "bastion" { + project = var.gcp_project + zone = data.google_compute_zones.available.names[0] + machine_type = var.bastion_instance_type + name = var.bastion_name + + boot_disk { + initialize_params { + image = data.google_compute_image.bastion_image.self_link + } + } + + network_interface { + subnetwork = var.public_subnet_name + // the empty access_config block will automatically generate an external IP for the instance + access_config {} + } + + tags = ["bastion"] + + metadata_startup_script = "sudo yum install -y mysql && curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.rpm.sh | sudo bash && sudo yum -y install sysbench" +} diff --git a/deploy/modules/gcp/bastion/data.tf b/deploy/modules/gcp/bastion/data.tf new file mode 100644 index 0000000000..f47d062c13 --- /dev/null +++ b/deploy/modules/gcp/bastion/data.tf @@ -0,0 +1,5 @@ +data "google_compute_zones" "available" {} +data "google_compute_image" "bastion_image" { + family = "centos-7" + project = "centos-cloud" +} diff --git a/deploy/modules/gcp/bastion/outputs.tf b/deploy/modules/gcp/bastion/outputs.tf new file mode 100644 index 0000000000..ffcc2f6cda --- /dev/null +++ b/deploy/modules/gcp/bastion/outputs.tf @@ -0,0 +1,3 @@ +output "how_to_ssh_to_bastion" { + value = "gcloud --project ${var.gcp_project} compute ssh ${var.bastion_name} --zone ${google_compute_instance.bastion.zone}" +} diff --git a/deploy/modules/gcp/bastion/variables.tf b/deploy/modules/gcp/bastion/variables.tf new file mode 100644 index 0000000000..7e99dda7ab --- /dev/null +++ b/deploy/modules/gcp/bastion/variables.tf @@ -0,0 +1,13 @@ +variable "vpc_name" { + description = "The VPC network name" +} + +variable "gcp_project" {} + +variable "bastion_instance_type" { + default = "f1-micro" +} + +variable "public_subnet_name" {} + +variable "bastion_name" {} diff --git a/deploy/modules/gcp/project-credentials/main.tf b/deploy/modules/gcp/project-credentials/main.tf new file mode 100644 index 0000000000..74d4189c8e --- /dev/null +++ b/deploy/modules/gcp/project-credentials/main.tf @@ -0,0 +1,5 @@ +resource "null_resource" "prepare-dir" { + provisioner "local-exec" { + command = "mkdir -p ${var.path}" + } +} diff --git a/deploy/modules/gcp/project-credentials/outputs.tf b/deploy/modules/gcp/project-credentials/outputs.tf new file mode 100644 index 0000000000..bef580d947 --- /dev/null +++ b/deploy/modules/gcp/project-credentials/outputs.tf @@ -0,0 +1,3 @@ +output "credentials_path" { + value = var.path +} diff --git a/deploy/modules/gcp/project-credentials/variables.tf b/deploy/modules/gcp/project-credentials/variables.tf new file mode 100644 index 0000000000..3d5676e29b --- /dev/null +++ b/deploy/modules/gcp/project-credentials/variables.tf @@ -0,0 +1,4 @@ +variable "path" { + description = "Path to a directory where the public and private key will be stored." + default = "" +} diff --git a/deploy/modules/gcp/tidb-cluster/data.tf b/deploy/modules/gcp/tidb-cluster/data.tf new file mode 100644 index 0000000000..c73e032c78 --- /dev/null +++ b/deploy/modules/gcp/tidb-cluster/data.tf @@ -0,0 +1,21 @@ +data "external" "tidb_ilb_ip" { + depends_on = [null_resource.wait-lb-ip] + program = ["bash", "-c", "kubectl --kubeconfig ${var.kubeconfig_path} get svc -n ${var.cluster_name} ${var.cluster_name}-tidb -o json | jq '.status.loadBalancer.ingress[0]'"] +} + +data "external" "monitor_lb_ip" { + depends_on = [null_resource.wait-lb-ip] + program = ["bash", "-c", "kubectl --kubeconfig ${var.kubeconfig_path} get svc -n ${var.cluster_name} ${var.cluster_name}-grafana -o json | jq '.status.loadBalancer.ingress[0]'"] +} + +data "external" "tidb_port" { + depends_on = [null_resource.wait-lb-ip] + program = ["bash", "-c", "kubectl --kubeconfig ${var.kubeconfig_path} get svc -n ${var.cluster_name} ${var.cluster_name}-tidb -o json | jq '.spec.ports | .[] | select( .name == \"mysql-client\") | {port: .port|tostring}'"] +} + +data "external" "monitor_port" { + depends_on = [null_resource.wait-lb-ip] + program = ["bash", "-c", "kubectl --kubeconfig ${var.kubeconfig_path} get svc -n ${var.cluster_name} ${var.cluster_name}-grafana -o json | jq '.spec.ports | .[] | select( .name == \"grafana\") | {port: .port|tostring}'"] +} + +data "google_compute_zones" "available" {} diff --git a/deploy/modules/gcp/tidb-cluster/main.tf b/deploy/modules/gcp/tidb-cluster/main.tf new file mode 100644 index 0000000000..3226371026 --- /dev/null +++ b/deploy/modules/gcp/tidb-cluster/main.tf @@ -0,0 +1,161 @@ +resource "google_container_node_pool" "pd_pool" { + // The monitor pool is where tiller must first be deployed to. + depends_on = [google_container_node_pool.monitor_pool] + provider = google-beta + project = var.gcp_project + cluster = var.gke_cluster_name + location = var.gke_cluster_location + name = "${var.cluster_name}-pd-pool" + node_count = var.pd_node_count + + management { + auto_repair = true + auto_upgrade = false + } + + node_config { + machine_type = var.pd_instance_type + local_ssd_count = 0 + + taint { + effect = "NO_SCHEDULE" + key = "dedicated" + value = "${var.cluster_name}-pd" + } + + labels = { + dedicated = "${var.cluster_name}-pd" + } + + tags = ["pd"] + oauth_scopes = ["storage-ro", "logging-write", "monitoring"] + } +} + +resource "google_container_node_pool" "tikv_pool" { + provider = google-beta + project = var.gcp_project + cluster = var.gke_cluster_name + location = var.gke_cluster_location + name = "${var.cluster_name}-tikv-pool" + node_count = var.tikv_node_count + + management { + auto_repair = false + auto_upgrade = false + } + + node_config { + machine_type = var.tikv_instance_type + // This value cannot be changed (instead a new node pool is needed) + // 1 SSD is 375 GiB + local_ssd_count = 1 + + taint { + effect = "NO_SCHEDULE" + key = "dedicated" + value = "${var.cluster_name}-tikv" + } + + labels = { + dedicated = "${var.cluster_name}-tikv" + } + + tags = ["tikv"] + oauth_scopes = ["storage-ro", "logging-write", "monitoring"] + } +} + +resource "google_container_node_pool" "tidb_pool" { + // The pool order is tikv -> monitor -> pd -> tidb + depends_on = [google_container_node_pool.pd_pool] + provider = google-beta + project = var.gcp_project + cluster = var.gke_cluster_name + location = var.gke_cluster_location + name = "${var.cluster_name}-tidb-pool" + node_count = var.tidb_node_count + + management { + auto_repair = true + auto_upgrade = false + } + + node_config { + machine_type = var.tidb_instance_type + + taint { + effect = "NO_SCHEDULE" + key = "dedicated" + value = "${var.cluster_name}-tidb" + } + + labels = { + dedicated = "${var.cluster_name}-tidb" + } + + tags = ["tidb"] + oauth_scopes = ["storage-ro", "logging-write", "monitoring"] + } +} + +resource "google_container_node_pool" "monitor_pool" { + // Setup local SSD on TiKV nodes first (this can take some time) + // Create the monitor pool next because that is where tiller will be deployed to + depends_on = [google_container_node_pool.tikv_pool] + project = var.gcp_project + cluster = var.gke_cluster_name + location = var.gke_cluster_location + name = "${var.cluster_name}-monitor-pool" + node_count = var.monitor_node_count + + management { + auto_repair = true + auto_upgrade = false + } + + node_config { + machine_type = var.monitor_instance_type + tags = ["monitor"] + oauth_scopes = ["storage-ro", "logging-write", "monitoring"] + } +} + +locals { + num_availability_zones = length(data.google_compute_zones.available) +} + +module "tidb-cluster" { + source = "../../share/tidb-cluster-release" + cluster_name = var.cluster_name + pd_count = var.pd_node_count * local.num_availability_zones + tikv_count = var.tikv_node_count * local.num_availability_zones + tidb_count = var.tidb_node_count * local.num_availability_zones + tidb_cluster_chart_version = var.tidb_cluster_chart_version + override_values = var.override_values + kubeconfig_filename = var.kubeconfig_path + base_values = file("${path.module}/values/default.yaml") + wait_on_resource = [google_container_node_pool.tidb_pool] +} + +resource "null_resource" "wait-lb-ip" { + depends_on = [ + google_container_node_pool.tidb_pool, + ] + provisioner "local-exec" { + interpreter = ["bash", "-c"] + working_dir = path.cwd + command = </dev/null; then + kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $(gcloud config get-value account) +fi + +if ! kubectl get serviceaccount -n kube-system tiller 2>/dev/null ; then + kubectl create serviceaccount --namespace kube-system tiller +fi + +kubectl apply -f https://raw.githubusercontent.com/pingcap/tidb-operator/${var.tidb_operator_version}/manifests/crd.yaml +kubectl apply -f https://raw.githubusercontent.com/pingcap/tidb-operator/${var.tidb_operator_version}/manifests/tiller-rbac.yaml +kubectl apply -k manifests/local-ssd +kubectl apply -f manifests/gke/persistent-disk.yaml + +helm init --service-account tiller --upgrade --wait +until helm ls; do + echo "Wait until tiller is ready" + sleep 5 +done +EOS + + + environment = { + KUBECONFIG = var.kubeconfig_path + } + } +} + +data "helm_repository" "pingcap" { + provider = "helm.initial" + depends_on = [null_resource.setup-env] + name = "pingcap" + url = "https://charts.pingcap.org/" +} + +resource "helm_release" "tidb-operator" { + provider = "helm.initial" + depends_on = [ + null_resource.setup-env, + null_resource.get-credentials, + data.helm_repository.pingcap, + ] + + repository = data.helm_repository.pingcap.name + chart = "tidb-operator" + version = var.tidb_operator_version + namespace = "tidb-admin" + name = "tidb-operator" + values = [var.operator_helm_values] + wait = false +} diff --git a/deploy/modules/gcp/tidb-operator/outputs.tf b/deploy/modules/gcp/tidb-operator/outputs.tf new file mode 100644 index 0000000000..84a9d10740 --- /dev/null +++ b/deploy/modules/gcp/tidb-operator/outputs.tf @@ -0,0 +1,19 @@ +output "cluster_id" { + value = google_container_cluster.cluster.id +} + +output "gke_cluster_name" { + value = google_container_cluster.cluster.name +} + +output "gcp_project" { + value = google_container_cluster.cluster.project +} + +output "gke_cluster_location" { + value = google_container_cluster.cluster.location +} + +output "kubeconfig_path" { + value = local_file.kubeconfig.filename +} diff --git a/deploy/modules/gcp/tidb-operator/variables.tf b/deploy/modules/gcp/tidb-operator/variables.tf new file mode 100644 index 0000000000..a94e2c3212 --- /dev/null +++ b/deploy/modules/gcp/tidb-operator/variables.tf @@ -0,0 +1,47 @@ +variable "gke_name" { + description = "Name of the GKE cluster. Also used as a prefix in names of related resources." + type = string +} + +variable "vpc_name" { + description = "The name of the VPC in which to place the cluster" +} + +variable "subnetwork_name" { + description = "The name of the subnetwork in which to place the cluster" +} + +variable "gcp_region" { + description = "The GCP region" +} + +variable "gcp_project" { + description = "The GCP project name" +} + +variable "gke_version" { + description = "Kubernetes version to use for the GKE cluster" + type = string + default = "latest" +} + +variable "tidb_operator_version" { + description = "TiDB Operator version" + type = string + default = "v1.0.0" +} + +variable "operator_helm_values" { + description = "Operator helm values" + type = string + default = "" +} + +variable "kubeconfig_path" { + description = "kubeconfig path" +} + +variable "maintenance_window_start_time" { + description = "The time in HH:MM GMT format to define the start of the daily maintenance window" + default = "01:00" +} diff --git a/deploy/modules/gcp/vpc/main.tf b/deploy/modules/gcp/vpc/main.tf new file mode 100644 index 0000000000..b4bd6e2018 --- /dev/null +++ b/deploy/modules/gcp/vpc/main.tf @@ -0,0 +1,37 @@ +resource "google_compute_network" "vpc_network" { + count = var.create_vpc ? 1 : 0 + name = var.vpc_name + auto_create_subnetworks = false + +} + +resource "google_compute_subnetwork" "private_subnet" { + depends_on = [google_compute_network.vpc_network] + name = var.private_subnet_name + ip_cidr_range = var.private_subnet_primary_cidr_range + network = var.vpc_name + project = var.gcp_project + + secondary_ip_range { + ip_cidr_range = var.private_subnet_secondary_cidr_ranges[0] + range_name = "pods-${var.gcp_region}" + } + + secondary_ip_range { + ip_cidr_range = var.private_subnet_secondary_cidr_ranges[1] + range_name = "services-${var.gcp_region}" + } + + + lifecycle { + ignore_changes = [secondary_ip_range] + } +} + +resource "google_compute_subnetwork" "public_subnet" { + depends_on = [google_compute_network.vpc_network] + ip_cidr_range = var.public_subnet_primary_cidr_range + name = var.public_subnet_name + network = var.vpc_name + project = var.gcp_project +} diff --git a/deploy/modules/gcp/vpc/outputs.tf b/deploy/modules/gcp/vpc/outputs.tf new file mode 100644 index 0000000000..8afa1e138d --- /dev/null +++ b/deploy/modules/gcp/vpc/outputs.tf @@ -0,0 +1,11 @@ +output "vpc_name" { + value = var.create_vpc ? google_compute_network.vpc_network[0].name : var.vpc_name +} + +output "private_subnetwork_name" { + value = google_compute_subnetwork.private_subnet.name +} + +output "public_subnetwork_name" { + value = google_compute_subnetwork.public_subnet.name +} diff --git a/deploy/modules/gcp/vpc/variables.tf b/deploy/modules/gcp/vpc/variables.tf new file mode 100644 index 0000000000..a99932a20b --- /dev/null +++ b/deploy/modules/gcp/vpc/variables.tf @@ -0,0 +1,40 @@ +variable "vpc_name" { + description = "Name of the VPC" +} + +variable "create_vpc" { + description = "Create a new VPC or not" +} + +variable "private_subnet_primary_cidr_range" { + description = "Primary CIDR range for the private subnetwork" + default = "172.31.252.0/22" +} + +variable "private_subnet_secondary_cidr_ranges" { + description = "Secondary ranges for private subnet" + default = ["172.30.0.0/16", "172.31.224.0/20"] +} + +variable "private_subnet_name" { + description = "Name for the private subnet" + default = "private-subnet" +} + +variable "public_subnet_name" { + description = "Name for the public subnet" + default = "public-subnet" +} + +variable "public_subnet_primary_cidr_range" { + description = "Primary CIDR range for the public subnetwork" + default = "172.29.252.0/22" +} + +variable "gcp_region" { + description = "The GCP region name" +} + +variable "gcp_project" { + description = "The GCP project name" +} diff --git a/deploy/modules/share/tidb-cluster-release/main.tf b/deploy/modules/share/tidb-cluster-release/main.tf index 4f89e914c8..de808a590d 100644 --- a/deploy/modules/share/tidb-cluster-release/main.tf +++ b/deploy/modules/share/tidb-cluster-release/main.tf @@ -1,5 +1,5 @@ resource "null_resource" "wait-tiller-ready" { - depends_on = [var.kubeconfig_filename] + depends_on = [var.kubeconfig_filename, var.wait_on_resource] provisioner "local-exec" { working_dir = path.cwd @@ -18,7 +18,7 @@ EOS data "helm_repository" "pingcap" { depends_on = [null_resource.wait-tiller-ready] name = "pingcap" - url = "http://charts.pingcap.org/" + url = "https://charts.pingcap.org/" } resource "helm_release" "tidb-cluster" { diff --git a/deploy/modules/share/tidb-cluster-release/variables.tf b/deploy/modules/share/tidb-cluster-release/variables.tf index 10d441ba3e..f0f3046cd7 100644 --- a/deploy/modules/share/tidb-cluster-release/variables.tf +++ b/deploy/modules/share/tidb-cluster-release/variables.tf @@ -53,3 +53,8 @@ variable "local_exec_interpreter" { type = list(string) default = ["/bin/sh", "-c"] } + +variable "wait_on_resource" { + description = "An optional resource to pass in to wait on before running" + default = [] +}