Merge pull request kubernetes-sigs#46 from chizhg/gke-private-cluster

gke deployer supports creating private cluster

kubetest2-gke/deployer/deployer.go

@@ -40,6 +40,14 @@ const (
 	image         = "cos"
 )
 
+type privateClusterAccessLevel string
+
+const (
+	no           privateClusterAccessLevel = "no"
+	limited      privateClusterAccessLevel = "limited"
+	unrestricted privateClusterAccessLevel = "unrestricted"
+)
+
 var (
 	// poolRe matches instance group URLs of the form `https://www.googleapis.com/compute/v1/projects/some-project/zones/a-zone/instanceGroupManagers/gke-some-cluster-some-pool-90fcb815-grp`. Match meaning:
 	// m[0]: path starting with zones/
@@ -101,8 +109,14 @@ type deployer struct {
 	gcpSSHKeyIgnored bool
 
 	// Enable workload identity or not.
+	// See the details in https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity
 	workloadIdentityEnabled bool
 
+	// Private cluster access level, must be one of "no", "limited" and "unrestricted".
+	// See the details in https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters
+	privateClusterAccessLevel   string
+	privateClusterMasterIPRange string
+
 	boskosLocation              string
 	boskosResourceType          string
 	boskosAcquireTimeoutSeconds int
@@ -212,6 +226,8 @@ func bindFlags(d *deployer) *pflag.FlagSet {
 	flags.StringVar(&d.stageLocation, "stage", "", "Upload binaries to gs://bucket/ci/job-suffix if set")
 	flags.BoolVar(&d.gcpSSHKeyIgnored, "ignore-gcp-ssh-key", false, "Whether the GCP SSH key should be ignored or not for bringing up the cluster.")
 	flags.BoolVar(&d.workloadIdentityEnabled, "enable-workload-identity", false, "Whether enable workload identity for the cluster or not.")
+	flags.StringVar(&d.privateClusterAccessLevel, "private-cluster-access-level", "", "Private cluster access level, if not empty, must be one of 'no', 'limited' or 'unrestricted'")
+	flags.StringVar(&d.privateClusterMasterIPRange, "private-cluster-master-ip-range", "172.16.0.32/28", "Private cluster master IP range. It should be an IPv4 CIDR, and must not be empty if private cluster is requested.")
 	flags.StringVar(&d.boskosLocation, "boskos-location", defaultBoskosLocation, "If set, manually specifies the location of the Boskos server")
 	flags.StringVar(&d.boskosResourceType, "boskos-resource-type", defaultGKEProjectResourceType, "If set, manually specifies the resource type of GCP projects to acquire from Boskos")
 	flags.IntVar(&d.boskosAcquireTimeoutSeconds, "boskos-acquire-timeout-seconds", 300, "How long (in seconds) to hang on a request to Boskos to acquire a resource before erroring")

kubetest2-gke/deployer/network.go

@@ -64,6 +64,14 @@ func (d *deployer) verifyNetworkFlags() error {
 		}
 	}
 
+	if d.privateClusterAccessLevel != "" && d.privateClusterAccessLevel != string(no) &&
+		d.privateClusterAccessLevel != string(limited) && d.privateClusterAccessLevel != string(unrestricted) {
+		return fmt.Errorf("--private-cluster-access-level must be one of %v", []string{"", string(no), string(limited), string(unrestricted)})
+	}
+	if d.privateClusterAccessLevel != "" && d.privateClusterMasterIPRange == "" {
+		return fmt.Errorf("--private-cluster-master-ip-range must not be empty when requesting a private cluster")
+	}
+
 	return nil
 }
 
@@ -348,3 +356,33 @@ func removeHostServiceAgentUserRole(projects []string) error {
 	}
 	return nil
 }
+
+// This function returns the args required for creating a private cluster.
+// Reference: https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#top_of_page
+func privateClusterArgs(network, cluster, accessLevel, masterIPRange string) []string {
+	if accessLevel == "" {
+		return []string{}
+	}
+
+	subnetName := network + "-" + cluster
+	common := []string{
+		"--create-subnetwork name=" + subnetName,
+		"--enable-ip-alias",
+		"--enable-private-nodes",
+		"--no-enable-basic-auth",
+		"--master-ipv4-cidr=" + masterIPRange,
+		"--no-issue-client-certificate",
+	}
+
+	switch accessLevel {
+	case string(no):
+		common = append(common, "--enable-master-authorized-networks",
+			"--enable-private-endpoint")
+	case string(limited):
+		common = append(common, "--enable-master-authorized-networks")
+	case string(unrestricted):
+		common = append(common, "--no-enable-master-authorized-networks")
+	}
+
+	return common
+}

kubetest2-gke/deployer/up.go

@@ -62,6 +62,7 @@ func (d *deployer) Up() error {
 		subNetworkArgs := subNetworkArgs(d.projects, d.region, d.network, i)
 		for j := range d.projectClustersLayout[project] {
 			cluster := d.projectClustersLayout[project][j]
+			privateClusterArgs := privateClusterArgs(d.network, cluster, d.privateClusterAccessLevel, d.privateClusterMasterIPRange)
 			eg.Go(func() error {
 				// Create the cluster
 				args := make([]string, len(d.createCommand()))
@@ -78,6 +79,7 @@ func (d *deployer) Up() error {
 					args = append(args, fmt.Sprintf("--workload-pool=%s.svc.id.goog", project))
 				}
 				args = append(args, subNetworkArgs...)
+				args = append(args, privateClusterArgs...)
 				args = append(args, cluster)
 				klog.V(1).Infof("Gcloud command: gcloud %+v\n", args)
 				if err := runWithOutput(exec.CommandContext(ctx, "gcloud", args...)); err != nil {