diff --git a/kubetest2-gke/deployer/deployer.go b/kubetest2-gke/deployer/deployer.go index c12eb4b6..fb33879b 100644 --- a/kubetest2-gke/deployer/deployer.go +++ b/kubetest2-gke/deployer/deployer.go @@ -52,12 +52,18 @@ const ( ) var ( - // poolRe matches instance group URLs of the form `https://www.googleapis.com/compute/v1/projects/some-project/zones/a-zone/instanceGroupManagers/gke-some-cluster-some-pool-90fcb815-grp`. Match meaning: + // poolRe matches instance group URLs of the form `https://www.googleapis.com/compute/v1/projects/some-project/zones/a-zone/instanceGroupManagers/gke-some-cluster-some-pool-90fcb815-grp` + // or `https://www.googleapis.com/compute/v1/projects/some-project/zones/a-zone/instanceGroupManagers/gk3-some-cluster-some-pool-90fcb815-grp` for GKE in Autopilot mode. + // Match meaning: // m[0]: path starting with zones/ // m[1]: zone // m[2]: pool name (passed to e2es) // m[3]: unique hash (used as nonce for firewall rules) - poolRe = regexp.MustCompile(`zones/([^/]+)/instanceGroupManagers/(gke-.*-([0-9a-f]{8})-grp)$`) + poolRe = regexp.MustCompile(`zones/([^/]+)/instanceGroupManagers/(gk[e|3]-.*-([0-9a-f]{8})-grp)$`) + + // createAutopilotRe matches the cluster creation command to create GKE Autopilot clusters + // https://cloud.google.com/sdk/gcloud/reference/container/clusters/create-auto + createAutopilotRe = regexp.MustCompile(`container\s+clusters\s+create-auto`) urlRe = regexp.MustCompile(`https://.*/`) defaultNodePool = gkeNodePool{ @@ -106,6 +112,7 @@ type deployer struct { projectClustersLayout map[string][]cluster nodes int machineType string + imageType string network string subnetworkRanges []string environment string @@ -223,6 +230,7 @@ func bindFlags(d *deployer) *pflag.FlagSet { flags.StringVar(&d.zone, "zone", "", "For use with gcloud commands to specify the cluster zone.") flags.IntVar(&d.nodes, "num-nodes", defaultNodePool.Nodes, "For use with gcloud commands to specify the number of nodes for the cluster.") flags.StringVar(&d.machineType, "machine-type", defaultNodePool.MachineType, "For use with gcloud commands to specify the machine type for the cluster.") + flags.StringVar(&d.imageType, "image-type", image, "The image type to use for the cluster.") flags.BoolVar(&d.gcpSSHKeyIgnored, "ignore-gcp-ssh-key", true, "Whether the GCP SSH key should be ignored or not for bringing up the cluster.") flags.BoolVar(&d.workloadIdentityEnabled, "enable-workload-identity", false, "Whether enable workload identity for the cluster or not.") flags.StringVar(&d.privateClusterAccessLevel, "private-cluster-access-level", "", "Private cluster access level, if not empty, must be one of 'no', 'limited' or 'unrestricted'") diff --git a/kubetest2-gke/deployer/network.go b/kubetest2-gke/deployer/network.go index b6ea48f3..97ce09a9 100644 --- a/kubetest2-gke/deployer/network.go +++ b/kubetest2-gke/deployer/network.go @@ -169,7 +169,7 @@ func transformNetworkName(projects []string, network string) string { // Returns the sub network args needed for the cluster creation command. // Reference: https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-shared-vpc#creating_a_cluster_in_your_first_service_project -func subNetworkArgs(projects []string, region, network string, projectIndex int) []string { +func subNetworkArgs(createCommand string, projects []string, region, network string, projectIndex int) []string { // No sub network args need to be added for creating clusters in the host project. if projectIndex == 0 { return []string{} @@ -179,12 +179,16 @@ func subNetworkArgs(projects []string, region, network string, projectIndex int) hostProject := projects[0] curtProject := projects[projectIndex] subnetName := network + "-" + curtProject - return []string{ - "--enable-ip-alias", + args := []string{ fmt.Sprintf("--subnetwork=projects/%s/regions/%s/subnetworks/%s", hostProject, region, subnetName), fmt.Sprintf("--cluster-secondary-range-name=%s-pods", subnetName), fmt.Sprintf("--services-secondary-range-name=%s-services", subnetName), } + // GKE in Autopilot mode does not support --enable-ip-alias flag - https://cloud.google.com/sdk/gcloud/reference/container/clusters/create-auto + if !createAutopilotRe.MatchString(createCommand) { + args = append(args, "--enable-ip-alias") + } + return args } func (d *deployer) setupNetwork() error { diff --git a/kubetest2-gke/deployer/up.go b/kubetest2-gke/deployer/up.go index 61d97894..1f06ed1d 100644 --- a/kubetest2-gke/deployer/up.go +++ b/kubetest2-gke/deployer/up.go @@ -67,7 +67,7 @@ func (d *deployer) Up() error { loc := location(d.region, d.zone) for i := range d.projects { project := d.projects[i] - subNetworkArgs := subNetworkArgs(d.projects, d.region, d.network, i) + subNetworkArgs := subNetworkArgs(d.createCommandFlag, d.projects, d.region, d.network, i) for j := range d.projectClustersLayout[project] { cluster := d.projectClustersLayout[project][j] privateClusterArgs := privateClusterArgs(d.projects, d.network, d.privateClusterAccessLevel, d.privateClusterMasterIPRanges, cluster) @@ -78,11 +78,21 @@ func (d *deployer) Up() error { args = append(args, "--project="+project, loc, - "--machine-type="+d.machineType, - "--image-type="+image, - "--num-nodes="+strconv.Itoa(d.nodes), "--network="+transformNetworkName(d.projects, d.network), ) + // A few args are not supported in GKE Autopilot cluster creation, so they should be left unset + // when they are not provided via the flags. + // https://cloud.google.com/sdk/gcloud/reference/container/clusters/create-auto + if d.machineType != "" { + args = append(args, "--machine-type="+d.machineType) + } + if d.nodes != 0 { + args = append(args, "--num-nodes="+strconv.Itoa(d.nodes)) + } + if d.imageType != "" { + args = append(args, "--image-type="+image) + } + if d.workloadIdentityEnabled { args = append(args, fmt.Sprintf("--workload-pool=%s.svc.id.goog", project)) } @@ -234,8 +244,8 @@ func (d *deployer) verifyUpFlags() error { if err := d.verifyLocationFlags(); err != nil { return err } - if d.nodes <= 0 { - return fmt.Errorf("--num-nodes must be larger than 0") + if d.nodes < 0 { + return fmt.Errorf("--num-nodes must be no less than 0") } if err := validateVersion(d.Version); err != nil { return err