[GKE deployer] Enable building from source

go.mod

@@ -3,6 +3,7 @@ module sigs.k8s.io/kubetest2
 go 1.14
 
 require (
+	github.com/google/uuid v1.1.1
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/octago/sflags v0.2.0
 	github.com/pkg/errors v0.9.1

kubetest2-gke/deployer/build.go

@@ -18,19 +18,26 @@ package deployer
 
 import (
 	"fmt"
+	"strings"
 
-	"sigs.k8s.io/kubetest2/pkg/build"
+	"github.com/google/uuid"
 )
 
 func (d *deployer) Build() error {
-	if err := build.Build(); err != nil {
+	if err := d.BuildOptions.Validate(); err != nil {
 		return err
 	}
-
-	if d.stageLocation != "" {
-		if err := build.Stage(d.stageLocation); err != nil {
+	version, err := d.BuildOptions.Build()
+	if err != nil {
+		return err
+	}
+	version = strings.TrimPrefix(version, "v")
+	version += ".0+" + uuid.New().String()
+	if d.BuildOptions.StageLocation != "" {
+		if err := d.BuildOptions.Stage(version); err != nil {
 			return fmt.Errorf("error staging build: %v", err)
 		}
 	}
+	d.Version = version
 	return nil
 }

kubetest2-gke/deployer/deployer.go

@@ -24,9 +24,12 @@ import (
 	"regexp"
 	"sync"
 
+	"github.com/octago/sflags/gen/gpflag"
 	"github.com/spf13/pflag"
 	"k8s.io/klog"
 	"sigs.k8s.io/boskos/client"
+	"sigs.k8s.io/kubetest2/kubetest2-gke/deployer/options"
+	"sigs.k8s.io/kubetest2/pkg/build"
 
 	"sigs.k8s.io/kubetest2/pkg/types"
 )
@@ -78,6 +81,11 @@ type ig struct {
 type deployer struct {
 	// generic parts
 	commonOptions types.Options
+
+	BuildOptions *options.BuildOptions
+
+	Version string `desc:"Use a specific GKE version e.g. 1.16.13.gke-400 or 'latest'. If --build is specified it will default to building kubernetes from source."`
+
 	// doInit helps to make sure the initialization is performed only once
 	doInit sync.Once
 	// gke specific details
@@ -100,8 +108,6 @@ type deployer struct {
 	// project -> cluster -> instance groups
 	instanceGroups map[string]map[string][]*ig
 
-	stageLocation string
-
 	localLogsDir string
 	gcsLogsDir   string
 
@@ -147,7 +153,12 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) {
 	// create a deployer object and set fields that are not flag controlled
 	d := &deployer{
 		commonOptions: opts,
-		localLogsDir:  filepath.Join(opts.ArtifactsDir(), "logs"),
+		BuildOptions: &options.BuildOptions{
+			Builder:  &build.NoopBuilder{},
+			Stager:   &build.NoopStager{},
+			Strategy: "bazel",
+		},
+		localLogsDir: filepath.Join(opts.ArtifactsDir(), "logs"),
 	}
 
 	// register flags
@@ -176,6 +187,25 @@ func (d *deployer) verifyUpFlags() error {
 	if d.nodes <= 0 {
 		return fmt.Errorf("--num-nodes must be larger than 0")
 	}
+	if err := validateVersion(d.Version); err != nil {
+		return err
+	}
+	return nil
+}
+
+func validateVersion(version string) error {
+	switch version {
+	case "latest":
+		return nil
+	default:
+		re, err := regexp.Compile(`(\d)\.(\d)+\.(\d)*(-gke\.\d*\.\d*)(.*)`)
+		if err != nil {
+			return err
+		}
+		if !re.MatchString(version) {
+			return fmt.Errorf("unknown version %q", version)
+		}
+	}
 	return nil
 }
 
@@ -207,8 +237,11 @@ func (d *deployer) location() (string, error) {
 }
 
 func bindFlags(d *deployer) *pflag.FlagSet {
-	flags := pflag.NewFlagSet(Name, pflag.ContinueOnError)
-
+	flags, err := gpflag.Parse(d)
+	if err != nil {
+		klog.Fatalf("unable to generate flags from deployer")
+		return nil
+	}
 	flags.StringSliceVar(&d.clusters, "cluster-name", []string{}, "Cluster names separated by comma. Must be set. "+
 		"For multi-project profile, it should be in the format of clusterA:0,clusterB:1,clusterC:2, where the index means the index of the project.")
 	flags.StringVar(&d.createCommandFlag, "create-command", defaultCreate, "gcloud subcommand used to create a cluster. Modify if you need to pass arbitrary arguments to create.")
@@ -223,7 +256,6 @@ func bindFlags(d *deployer) *pflag.FlagSet {
 	flags.StringVar(&d.zone, "zone", "", "For use with gcloud commands to specify the cluster zone.")
 	flags.IntVar(&d.nodes, "num-nodes", defaultNodePool.Nodes, "For use with gcloud commands to specify the number of nodes for the cluster.")
 	flags.StringVar(&d.machineType, "machine-type", defaultNodePool.MachineType, "For use with gcloud commands to specify the machine type for the cluster.")
-	flags.StringVar(&d.stageLocation, "stage", "", "Upload binaries to gs://bucket/ci/job-suffix if set")
 	flags.BoolVar(&d.gcpSSHKeyIgnored, "ignore-gcp-ssh-key", false, "Whether the GCP SSH key should be ignored or not for bringing up the cluster.")
 	flags.BoolVar(&d.workloadIdentityEnabled, "enable-workload-identity", false, "Whether enable workload identity for the cluster or not.")
 	flags.StringVar(&d.privateClusterAccessLevel, "private-cluster-access-level", "", "Private cluster access level, if not empty, must be one of 'no', 'limited' or 'unrestricted'")

kubetest2-gke/deployer/options/build.go

@@ -0,0 +1,44 @@
+package options
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/kubetest2/pkg/build"
+)
+
+type BuildOptions struct {
+	Strategy      string `flag:"~strategy" desc:"Determines the build strategy to use either make or bazel."`
+	StageLocation string `flag:"~stage" desc:"Upload binaries to gs://bucket/ci/job-suffix if set"`
+	RepoRoot      string `flag:"~repo-root" desc:"Path to root of the kubernetes repo. Used with --build."`
+
+	build.Builder
+	build.Stager
+}
+
+func (bo *BuildOptions) Validate() error {
+	if bo.RepoRoot == "" {
+		return fmt.Errorf("required repo-root when building from source")
+	}
+	return bo.implementationFromStrategy()
+}
+
+func (bo *BuildOptions) implementationFromStrategy() error {
+	switch bo.Strategy {
+	case "bazel":
+		bazel := &build.Bazel{
+			RepoRoot:      bo.RepoRoot,
+			StageLocation: bo.StageLocation,
+			ImageLocation: "gke.gcr.io",
+		}
+		bo.Builder = bazel
+		bo.Stager = bazel
+	case "make":
+		bo.Builder = &build.MakeBuilder{}
+		bo.Stager = &build.ReleasePushBuild{
+			Location: bo.StageLocation,
+		}
+	default:
+		return fmt.Errorf("unknown build strategy: %v", bo.Strategy)
+	}
+	return nil
+}

kubetest2-gke/deployer/up.go

@@ -49,7 +49,7 @@ func (d *deployer) Up() error {
 		return err
 	}
 
-	klog.V(1).Infof("Environment: %v", os.Environ())
+	klog.V(2).Infof("Environment: %v", os.Environ())
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	eg, ctx := errgroup.WithContext(ctx)
@@ -74,6 +74,7 @@ func (d *deployer) Up() error {
 					"--image-type="+image,
 					"--num-nodes="+strconv.Itoa(d.nodes),
 					"--network="+transformNetworkName(d.projects, d.network),
+					"--cluster-version="+d.Version,
 				)
 				if d.workloadIdentityEnabled {
 					args = append(args, fmt.Sprintf("--workload-pool=%s.svc.id.goog", project))

pkg/build/bazel.go

@@ -0,0 +1,50 @@
+package build
+
+import (
+	"fmt"
+	"os"
+
+	"k8s.io/klog"
+
+	"sigs.k8s.io/kubetest2/pkg/exec"
+)
+
+type Bazel struct {
+	RepoRoot      string
+	StageLocation string
+	ImageLocation string
+}
+
+var _ Builder = &Bazel{}
+var _ Stager = &Bazel{}
+
+const (
+	defaultImageRegistry = "k8s.gcr.io"
+)
+
+func (b *Bazel) Stage(version string) error {
+	if b.ImageLocation == "" {
+		b.ImageLocation = defaultImageRegistry
+	}
+	location := b.StageLocation + "/v" + version
+	klog.V(0).Infof("Staging builds to %s ...", location)
+	cmd := exec.Command("bazel", "run", "//:push-build", "--", location)
+	env := os.Environ()
+	env = append(env, "KUBE_DOCKER_REGISTRY="+b.ImageLocation)
+	cmd.SetDir(b.RepoRoot)
+	cmd.SetEnv(env...)
+	exec.InheritOutput(cmd)
+	return cmd.Run()
+}
+
+func (b *Bazel) Build() (string, error) {
+	klog.V(0).Infof("Building kubernetes from %s ...", b.RepoRoot)
+	version, err := sourceVersion(b.RepoRoot)
+	if err != nil {
+		return "", fmt.Errorf("failed to get version: %v", err)
+	}
+	cmd := exec.Command("bazel", "build", "//build/release-tars")
+	cmd = cmd.SetDir(b.RepoRoot)
+	exec.InheritOutput(cmd)
+	return version, cmd.Run()
+}