Merge pull request #830 from spiffxp/prow-build-clusters

add prow build clusters

infra/gcp/clusters/README.md

@@ -1,18 +1,35 @@
 # clusters
 
-This directory contains Terraform cluster configurations for the various GCP
+This directory contains Terraform modules and configurations for the various
+GCP projects and Kubernetes clusters that the Kubernetes project maintains.
 projects that the Kubernetes project maintains.
 
-Each directory represents a GCP project.  Each sub-directory of those represents
-a GKE cluster configuration.  We may template these into modules at some point,
-but for now they are designed to be straight forward and verbose.
+## Layout
+
+```
+.
+├── modules
+│   └── <module>
+└── projects
+    └── <project>
+        └── <cluster>
+```
+
+Each directory in `modules` represents a Terraform module intended for reuse
+inside of this repo. Not every configuration is able to use these modules yet
+due to differences in google provider version.
+
+Each directory in `projects` represents a GCP project. Each subdirectory of 
+those represents a GKE cluster configuration. 
+
+## Prerequsites
 
-Prerequisites:
 - Be a member of the k8s-infra-cluster-admins@kubernetes.io group.
 - Have Terraform installed
   (https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip)
 
-Instructions:
+## Instructions
+
 - Ensure you are logged into your GCP account with `gcloud auth application-default login`
 - From within a cluster directory:
   - `terraform init` will initialize your local state

infra/gcp/clusters/modules/gke-cluster/README.md

@@ -0,0 +1,16 @@
+# `gke-cluster` terraform module
+
+This terraform module defines a GKE cluster following wg-k8s-infra conventions:
+- GCP Service Account for nodes
+- BigQuery dataset for usage metering
+- GKE cluster with some useful defaults
+- No nodes are provided, they are expected to come from nodepools created via the [`gke-nodepool`] module
+
+It is assumed the GCP project for this cluster has been created via the [`gke-project`] module
+
+If this is a "prod" cluster:
+- the BigQuery dataset will NOT be deleted on `terraform destroy`
+- the GKE cluster will NOT be deleted on `terraform destroy`
+
+[`gke-project`]: /infra/gcp/clusters/modules/gke-project
+[`gke-nodepool`]: /infra/gcp/clusters/modules/gke-nodepool

infra/gcp/clusters/kubernetes-public/prow-build-test/k8s-infra-gke-cluster/main.tf → infra/gcp/clusters/modules/gke-cluster/main.tf

@@ -39,7 +39,14 @@ resource "google_project_iam_member" "cluster_node_sa_monitoring_metricwriter" {
 }
 
 // BigQuery dataset for usage data
-resource "google_bigquery_dataset" "usage_metering" {
+//
+// Uses a workaround from https://github.com/hashicorp/terraform/issues/22544#issuecomment-582974372
+// to set delete_contents_on_destroy to false if is_prod_cluster
+//
+// IMPORTANT: The prod_ and test_ forms of this resource MUST be kept in sync.
+//            Any changes in one MUST be reflected in the other.
+resource "google_bigquery_dataset" "prod_usage_metering" {
+  count       = var.is_prod_cluster == "true" ? 1 : 0
   dataset_id  = replace("usage_metering_${var.cluster_name}", "-", "_")
   project     = var.project_name
   description = "GKE Usage Metering for cluster '${var.cluster_name}'"
@@ -54,21 +61,144 @@ resource "google_bigquery_dataset" "usage_metering" {
     user_by_email = google_service_account.cluster_node_sa.email
   }
 
+  // NOTE: unique to prod_usage_metering
   // This restricts deletion of this dataset if there is data in it
-  // IMPORTANT: Should be true on test clusters
+  delete_contents_on_destroy = false
+}
+resource "google_bigquery_dataset" "test_usage_metering" {
+  count       = var.is_prod_cluster == "true" ? 0 : 1
+  dataset_id  = replace("usage_metering_${var.cluster_name}", "-", "_")
+  project     = var.project_name
+  description = "GKE Usage Metering for cluster '${var.cluster_name}'"
+  location    = var.bigquery_location
+
+  access {
+    role          = "OWNER"
+    special_group = "projectOwners"
+  }
+  access {
+    role          = "WRITER"
+    user_by_email = google_service_account.cluster_node_sa.email
+  }
+
+  // NOTE: unique to test_usage_metering
   delete_contents_on_destroy = true
 }
 
-// Create GKE cluster, but with no node pools. Node pools can be provisioned below
-resource "google_container_cluster" "cluster" {
+// Create GKE cluster, but with no node pools. Node pools are provisioned via another module.
+//
+// Uses a workaround from https://github.com/hashicorp/terraform/issues/22544#issuecomment-582974372
+// to set lifecycle.prevent_destroy to false if is_prod_cluster
+//
+// IMPORTANT: The prod_ and test_ forms of this resource MUST be kept in sync.
+//            Any changes in one MUST be reflected in the other.
+resource "google_container_cluster" "prod_cluster" {
+  count     = var.is_prod_cluster == "true" ? 1 : 0
+
   name     = var.cluster_name
   location = var.cluster_location
 
   provider = google-beta
   project  = var.project_name
 
+  // NOTE: unique to prod_cluster
   // GKE clusters are critical objects and should not be destroyed
-  // IMPORTANT: should be false on test clusters
+  lifecycle {
+    prevent_destroy = true
+  }
+
+  // Network config
+  network = "default"
+
+  // Start with a single node, because we're going to delete the default pool
+  initial_node_count = 1
+
+  // Removes the default node pool, so we can custom create them as separate
+  // objects
+  remove_default_node_pool = true
+
+  // Disable local and certificate auth
+  master_auth {
+    username = ""
+    password = ""
+
+    client_certificate_config {
+      issue_client_certificate = false
+    }
+  }
+
+  // Enable google-groups for RBAC
+  authenticator_groups_config {
+    security_group = "gke-security-groups@kubernetes.io"
+  }
+
+  // Enable workload identity for GCP IAM
+  workload_identity_config {
+    identity_namespace = "${var.project_name}.svc.id.goog"
+  }
+
+  // Enable Stackdriver Kubernetes Monitoring
+  logging_service    = "logging.googleapis.com/kubernetes"
+  monitoring_service = "monitoring.googleapis.com/kubernetes"
+
+  // Set maintenance time
+  maintenance_policy {
+    daily_maintenance_window {
+      start_time = "11:00" // (in UTC), 03:00 PST
+    }
+  }
+
+  // Restrict master to Google IP space; use Cloud Shell to access
+  master_authorized_networks_config {
+  }
+
+  // Enable GKE Usage Metering
+  resource_usage_export_config {
+    enable_network_egress_metering = true
+    bigquery_destination {
+      dataset_id = google_bigquery_dataset.prod_usage_metering[0].dataset_id
+    }
+  }
+
+  // Enable GKE Network Policy
+  network_policy {
+    enabled  = true
+    provider = "CALICO"
+  }
+
+  // Configure cluster addons
+  addons_config {
+    horizontal_pod_autoscaling {
+      disabled = false
+    }
+    http_load_balancing {
+      disabled = false
+    }
+    network_policy_config {
+      disabled = false
+    }
+  }
+
+  // Enable PodSecurityPolicy enforcement
+  pod_security_policy_config {
+    enabled = false // TODO: we should turn this on
+  }
+
+  // Enable VPA
+  vertical_pod_autoscaling {
+    enabled = true
+  }
+}
+resource "google_container_cluster" "test_cluster" {
+  count     = var.is_prod_cluster == "true" ? 0 : 1
+
+  name     = var.cluster_name
+  location = var.cluster_location
+
+  provider = google-beta
+  project  = var.project_name
+
+  // NOTE: unique to test_cluster
   lifecycle {
     prevent_destroy = false
   }
@@ -122,7 +252,7 @@ resource "google_container_cluster" "cluster" {
   resource_usage_export_config {
     enable_network_egress_metering = true
     bigquery_destination {
-      dataset_id = google_bigquery_dataset.usage_metering.dataset_id
+      dataset_id = google_bigquery_dataset.test_usage_metering[0].dataset_id
     }
   }
 

infra/gcp/clusters/kubernetes-public/prow-build-test/k8s-infra-gke-cluster/outputs.tf → infra/gcp/clusters/modules/gke-cluster/outputs.tf

@@ -16,7 +16,12 @@
 
 output "cluster" {
   description = "The cluster"
-  value       = google_container_cluster.cluster
+  // Workaround from https://github.com/hashicorp/terraform/issues/22544#issuecomment-582974372
+  // This should be either test_cluster or prod_cluster
+  value       = coalescelist(
+    google_container_cluster.test_cluster.*,
+    google_container_cluster.prod_cluster.*
+  )[0]
 }
 
 output "cluster_node_sa" {

infra/gcp/clusters/kubernetes-public/prow-build-test/k8s-infra-gke-cluster/variables.tf → infra/gcp/clusters/modules/gke-cluster/variables.tf

@@ -33,3 +33,9 @@ variable "bigquery_location" {
   description = "The bigquery specific location where the dataset should be created"
   type        = string
 }
+
+variable "is_prod_cluster" {
+  description = "If this is not a prod cluster it's safe to delete resources on destroy"
+  type        = string
+  default     = "false"
+}

infra/gcp/clusters/kubernetes-public/prow-build-test/k8s-infra-gke-nodepool/versions.tf → infra/gcp/clusters/modules/gke-cluster/versions.tf

@@ -15,7 +15,7 @@
  */
 
 terraform {
-  required_version = ">= 0.12.8"
+  required_version = "~> 0.12.20"
   required_providers {
     google      = "~> 3.19.0"
     google-beta = "~> 3.19.0"