Streamlines startup daemonset and adds Linux Guest Environment instal…

…lation
pingcap · May 21, 2019 · 157f039 · 157f039
1 parent 25e0d2e
commit 157f039
Show file tree

Hide file tree

Showing 6 changed files with 94 additions and 40 deletions.
diff --git a/deploy/gcp/data.tf b/deploy/gcp/data.tf
@@ -2,10 +2,11 @@ data "template_file" "tidb_cluster_values" {
   template = "${file("${path.module}/templates/tidb-cluster-values.yaml.tpl")}"
 
   vars {
-    cluster_version = "${var.tidb_version}"
-    pd_replicas     = "${var.pd_replica_count}"
-    tikv_replicas   = "${var.tikv_replica_count}"
-    tidb_replicas   = "${var.tidb_replica_count}"
+    cluster_version  = "${var.tidb_version}"
+    pd_replicas      = "${var.pd_replica_count}"
+    tikv_replicas    = "${var.tikv_replica_count}"
+    tidb_replicas    = "${var.tidb_replica_count}"
+    operator_version = "${var.tidb_operator_version}"
   }
 }
 
@@ -18,3 +19,13 @@ data "external" "monitor_ilb_ip" {
   depends_on = ["null_resource.deploy-tidb-cluster"]
   program    = ["bash", "-c", "kubectl --kubeconfig ${local.kubeconfig} get svc -n tidb tidb-cluster-grafana -o json | jq '.status.loadBalancer.ingress[0]'"]
 }
+
+data "external" "tidb_port" {
+  depends_on = ["null_resource.deploy-tidb-cluster"]
+  program    = ["bash", "-c", "kubectl --kubeconfig ${local.kubeconfig} get svc -n tidb tidb-cluster-tidb -o json | jq '.spec.ports | .[] | select( .name == \"mysql-client\") | {port: .port|tostring}'"]
+}
+
+data "external" "monitor_port" {
+  depends_on = ["null_resource.deploy-tidb-cluster"]
+  program    = ["bash", "-c", "kubectl --kubeconfig ${local.kubeconfig} get svc -n tidb tidb-cluster-grafana -o json | jq '.spec.ports | .[] | select( .name == \"grafana\") | {port: .port|tostring}'"]
+}
diff --git a/deploy/gcp/main.tf b/deploy/gcp/main.tf
@@ -36,7 +36,7 @@ resource "google_compute_network" "vpc_network" {
 resource "google_compute_subnetwork" "private_subnet" {
   ip_cidr_range = "172.31.252.0/22"
   name          = "private-subnet"
-  network       = "${google_compute_network.vpc_network.self_link}"
+  network       = "${google_compute_network.vpc_network.name}"
   project       = "${var.GCP_PROJECT}"
 
   secondary_ip_range {
@@ -48,25 +48,34 @@ resource "google_compute_subnetwork" "private_subnet" {
     ip_cidr_range = "172.31.224.0/20"
     range_name    = "services-${var.GCP_REGION}"
   }
+
+  lifecycle {
+    ignore_changes = ["secondary_ip_range"]
+  }
 }
 
 resource "google_compute_subnetwork" "public_subnet" {
   ip_cidr_range = "172.29.252.0/22"
   name          = "public-subnet"
-  network       = "${google_compute_network.vpc_network.self_link}"
+  network       = "${google_compute_network.vpc_network.name}"
   project       = "${var.GCP_PROJECT}"
 }
 
 resource "google_container_cluster" "cluster" {
   name       = "${var.cluster_name}"
-  network    = "${google_compute_network.vpc_network.self_link}"
-  subnetwork = "${google_compute_subnetwork.private_subnet.self_link}"
+  network    = "${google_compute_network.vpc_network.name}"
+  subnetwork = "${google_compute_subnetwork.private_subnet.name}"
   location   = "${var.GCP_REGION}"
   project    = "${var.GCP_PROJECT}"
 
   master_auth {
     username = ""
     password = ""
+
+    // due to https://github.com/terraform-providers/terraform-provider-google/issues/3369
+    client_certificate_config {
+      issue_client_certificate = false
+    }
   }
 
   master_authorized_networks_config {
@@ -83,6 +92,10 @@ resource "google_container_cluster" "cluster" {
   initial_node_count       = 1
 
   min_master_version = "latest"
+
+  lifecycle {
+    ignore_changes = ["master_auth"] // see above linked issue
+  }
 }
 
 resource "google_container_node_pool" "pd_pool" {
@@ -95,7 +108,7 @@ resource "google_container_node_pool" "pd_pool" {
 
   node_config {
     machine_type    = "${var.pd_instance_type}"
-    image_type = "UBUNTU"
+    image_type      = "UBUNTU"
     local_ssd_count = 1
 
     taint {
@@ -123,7 +136,7 @@ resource "google_container_node_pool" "tikv_pool" {
 
   node_config {
     machine_type    = "${var.tikv_instance_type}"
-    image_type = "UBUNTU"
+    image_type      = "UBUNTU"
     local_ssd_count = 1
 
     taint {
@@ -259,7 +272,6 @@ resource "null_resource" "get-credentials" {
 
     command = <<EOS
 kubectl get pvc -n tidb -o jsonpath='{.items[*].spec.volumeName}'|fmt -1 | xargs -I {} kubectl patch pv {} -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}'
-kubectl delete namespace tidb
 EOS
 
     environment {
@@ -302,7 +314,7 @@ EOS
 }
 
 resource "null_resource" "deploy-tidb-cluster" {
-  depends_on = ["null_resource.setup-env", "local_file.tidb-cluster-values"]
+  depends_on = ["null_resource.setup-env", "local_file.tidb-cluster-values", "google_container_node_pool.pd_pool", "google_container_node_pool.tikv_pool", "google_container_node_pool.tidb_pool"]
 
   provisioner "local-exec" {
     command = <<EOS
@@ -311,6 +323,10 @@ until kubectl get po -n tidb -lapp.kubernetes.io/component=tidb | grep Running;
   echo "Wait for TiDB pod running"
   sleep 5
 done
+until kubectl get svc -n tidb tidb-cluster-tidb -o json | jq '.status.loadBalancer.ingress[0]' | grep ip; do
+  echo "Wait for TiDB internal loadbalancer IP"
+  sleep 5
+done
 EOS
 
     environment {

diff --git a/deploy/gcp/manifests/startup-script.yaml b/deploy/gcp/manifests/startup-script.yaml
@@ -1,41 +1,55 @@
-kind: DaemonSet
 apiVersion: extensions/v1beta1
+kind: DaemonSet
 metadata:
-  name: startup-script
+  name: local-ssd-startup
   namespace: kube-system
   labels:
-    app: startup-script
+    app: local-ssd-startup
 spec:
   template:
     metadata:
       labels:
-        app: startup-script
+        app: local-ssd-startup
     spec:
-      tolerations:
-        - key: dedicated
-          operator: Equal
-          value: pd
-          effect: "NoSchedule"
-        - key: dedicated
-          operator: Equal
-          value: tikv
-          effect: "NoSchedule"
       hostPID: true
+      nodeSelector:
+        cloud.google.com/gke-os-distribution: ubuntu
       containers:
-        - name: startup-script
+        - name: local-ssd-startup
           image: gcr.io/google-containers/startup-script:v1
-          imagePullPolicy: Always
           securityContext:
             privileged: true
           env:
             - name: STARTUP_SCRIPT
               value: |
-                #! /bin/bash
-                set -o errexit
-                set -o pipefail
-                set -o nounset
-                echo 'root        soft        nofile        1000000' >> /etc/security/limits.d/99-tidb.conf
-                echo 'root        hard        nofile        1000000' >> /etc/security/limits.d/99-tidb.conf
-                echo 'root        soft        core          unlimited' >> /etc/security/limits.d/99-tidb.conf
-                echo 'root        soft        stack         10240' >> /etc/security/limits.d/99-tidb.conf
-                echo done
+                #!/usr/bin/env bash
+                set -euo pipefail
+                apt-get update
+                apt-get install -y software-properties-common
+                apt-add-repository universe
+                apt-get update
+                declare -a PKG_LIST=(python-google-compute-engine \
+                python3-google-compute-engine \
+                google-compute-engine-oslogin \
+                gce-compute-image-packages)
+                for pkg in ${PKG_LIST[@]}; do
+                    apt-get install -y $pkg || echo "Not available: $pkg"
+                done
+                mount | grep -v nobarrier | awk '/ssd/{print $1}' | xargs -i mount {} -o remount,nobarrier
+                cat <<EOF > /etc/security/limits.d/99-tidb.conf
+                root        soft        nofile        1000000
+                root        hard        nofile        1000000
+                root        soft        core          unlimited
+                root        soft        stack         10240
+                EOF
+          volumeMounts:
+            - mountPath: /mnt/disks
+              name: local-ssd
+              mountPropagation: Bidirectional
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+      volumes:
+        - name: local-ssd
+          hostPath:
+            path: /mnt/disks
diff --git a/deploy/gcp/outputs.tf b/deploy/gcp/outputs.tf
@@ -25,3 +25,11 @@ output "tidb_ilb_ip" {
 output "monitor_ilb_ip" {
   value = "${data.external.monitor_ilb_ip.result["ip"]}"
 }
+
+output "how_to_ssh_to_bastion" {
+  value = "gcloud compute ssh bastion --zone ${var.GCP_REGION}-a"
+}
+
+output "how_to_connect_to_mysql_from_bastion" {
+  value = "mysql -h ${data.external.tidb_ilb_ip.result["ip"]} -P ${data.external.tidb_port.result["port"]} -u root"
+}
diff --git a/deploy/gcp/templates/tidb-cluster-values.yaml.tpl b/deploy/gcp/templates/tidb-cluster-values.yaml.tpl
@@ -30,7 +30,7 @@ services:
     type: ClusterIP
 
 discovery:
-  image: pingcap/tidb-operator:v1.0.0-beta.2
+  image: pingcap/tidb-operator:${operator_version}
   imagePullPolicy: IfNotPresent
   resources:
     limits:

diff --git a/deploy/gcp/variables.tf b/deploy/gcp/variables.tf
@@ -8,9 +8,9 @@ variable "tidb_version" {
   default     = "v2.1.8"
 }
 
-variable "pd_count" {
-  description = "Number of PD nodes per availability zone"
-  default     = 1
+variable "tidb_operator_version" {
+  description = "TiDB operator version"
+  default     = "v1.0.0-beta.2"
 }
 
 variable "pd_replica_count" {
@@ -25,6 +25,11 @@ variable "tidb_replica_count" {
   default = 3
 }
 
+variable "pd_count" {
+  description = "Number of PD nodes per availability zone"
+  default     = 1
+}
+
 variable "tikv_count" {
   description = "Number of TiKV nodes per availability zone"
   default     = 1