kubetest2-gke: support boskos

kubetest2-gce/deployer/common.go

@@ -22,6 +22,11 @@ import (
 	"time"
 
 	"k8s.io/klog"
+	"sigs.k8s.io/kubetest2/pkg/boskos"
+)
+
+const (
+	gceProjectResourceType = "gce-project"
 )
 
 func (d *deployer) init() error {
@@ -46,22 +51,23 @@ func (d *deployer) initialize() error {
 		if d.GCPProject == "" {
 			klog.V(1).Info("No GCP project provided, acquiring from Boskos")
 
-			boskos, err := makeBoskosClient(d.BoskosLocation)
+			boskosClient, err := boskos.NewClient(d.BoskosLocation)
 			if err != nil {
 				return fmt.Errorf("failed to make boskos client: %s", err)
 			}
-			d.boskos = boskos
+			d.boskos = boskosClient
 
-			projectName, err := getProjectFromBoskos(
+			resource, err := boskos.Acquire(
 				d.boskos,
+				gceProjectResourceType,
 				time.Duration(d.BoskosAcquireTimeoutSeconds)*time.Second,
 				d.boskosHeartbeatClose,
 			)
 
 			if err != nil {
 				return fmt.Errorf("init failed to get project from boskos: %s", err)
 			}
-			d.GCPProject = projectName
+			d.GCPProject = resource.Name
 			klog.V(1).Infof("Got project %s from boskos", d.GCPProject)
 		}
 

kubetest2-gce/deployer/down.go

@@ -21,7 +21,7 @@ import (
 	"path/filepath"
 
 	"k8s.io/klog"
-
+	"sigs.k8s.io/kubetest2/pkg/boskos"
 	"sigs.k8s.io/kubetest2/pkg/exec"
 )
 
@@ -46,7 +46,7 @@ func (d *deployer) Down() error {
 
 	if d.boskos != nil {
 		klog.V(2).Info("releasing boskos project")
-		err := releaseBoskosProject(
+		err := boskos.Release(
 			d.boskos,
 			d.GCPProject,
 			d.boskosHeartbeatClose,

kubetest2-gke/deployer/common.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package deployer
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/klog"
+	"sigs.k8s.io/kubetest2/pkg/boskos"
+)
+
+const (
+	gkeProjectResourceType = "gke-project"
+)
+
+func (d *deployer) init() error {
+	var err error
+	d.doInit.Do(func() { err = d.initialize() })
+	return err
+}
+
+// initialize should only be called by init(), behind a sync.Once
+func (d *deployer) initialize() error {
+	if d.commonOptions.ShouldUp() {
+		if err := d.verifyUpFlags(); err != nil {
+			return fmt.Errorf("init failed to verify flags for up: %s", err)
+		}
+
+		if d.project == "" {
+			klog.V(1).Info("No GCP project provided, acquiring from Boskos")
+
+			boskosClient, err := boskos.NewClient(d.boskosLocation)
+			if err != nil {
+				return fmt.Errorf("failed to make boskos client: %s", err)
+			}
+			d.boskos = boskosClient
+
+			resource, err := boskos.Acquire(
+				d.boskos,
+				gkeProjectResourceType,
+				time.Duration(d.boskosAcquireTimeoutSeconds)*time.Second,
+				d.boskosHeartbeatClose,
+			)
+
+			if err != nil {
+				return fmt.Errorf("init failed to get project from boskos: %s", err)
+			}
+			d.project = resource.Name
+			klog.V(1).Infof("Got project %s from boskos", d.project)
+		}
+
+	}
+
+	if d.commonOptions.ShouldDown() {
+		if err := d.verifyDownFlags(); err != nil {
+			return fmt.Errorf("init failed to verify flags for down: %s", err)
+		}
+	}
+
+	return nil
+}

kubetest2-gke/deployer/deployer.go

@@ -18,6 +18,7 @@ limitations under the License.
 package deployer
 
 import (
+	"flag"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -28,13 +29,15 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/spf13/pflag"
-
 	"sigs.k8s.io/kubetest2/pkg/build"
 	"sigs.k8s.io/kubetest2/pkg/exec"
 	"sigs.k8s.io/kubetest2/pkg/metadata"
 	"sigs.k8s.io/kubetest2/pkg/types"
+	"k8s.io/klog"
+	"sigs.k8s.io/boskos/client"
 )
 
 // Name is the name of the deployer
@@ -77,6 +80,8 @@ type ig struct {
 type deployer struct {
 	// generic parts
 	commonOptions types.Options
+	// doInit helps to make sure the initialization is performed only once
+	doInit sync.Once
 	// gke specific details
 	project           string
 	zone              string
@@ -96,6 +101,17 @@ type deployer struct {
 
 	localLogsDir string
 	gcsLogsDir   string
+
+	boskosLocation              string
+	boskosAcquireTimeoutSeconds int
+
+	// boskos struct field will be non-nil when the deployer is
+	// using boskos to acquire a GCP project
+	boskos *client.Client
+
+	// this channel serves as a signal channel for the hearbeat goroutine
+	// so that it can be explicitly closed
+	boskosHeartbeatClose chan struct{}
 }
 
 // New implements deployer.New for gke
@@ -106,13 +122,28 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) {
 		localLogsDir:  filepath.Join(opts.ArtifactsDir(), "logs"),
 	}
 
-	// register flags and return
-	return d, bindFlags(d)
+	// register flags
+	fs := bindFlags(d)
+
+	// register flags for klog
+	klog.InitFlags(nil)
+	fs.AddGoFlagSet(flag.CommandLine)
+	return d, fs
+}
+
+// verifyCommonFlags validates flags for up phase.
+func (d *deployer) verifyUpFlags() error {
+	if d.cluster == "" {
+		return fmt.Errorf("--cluster-name must be set for GKE deployment")
+	}
+	if _, err := d.location(); err != nil {
+		return err
+	}
+	return nil
 }
 
-// verifyFlags validates that required flags are set, as well as
-// ensuring that location() will not return errors.
-func (d *deployer) verifyFlags() error {
+// verifyDownFlags validates flags for down phase.
+func (d *deployer) verifyDownFlags() error {
 	if d.cluster == "" {
 		return fmt.Errorf("--cluster-name must be set for GKE deployment")
 	}
@@ -157,6 +188,8 @@ func bindFlags(d *deployer) *pflag.FlagSet {
 	flags.StringVar(&d.region, "region", "", "For use with gcloud commands")
 	flags.StringVar(&d.zone, "zone", "", "For use with gcloud commands")
 	flags.StringVar(&d.stageLocation, "stage", "", "Upload binaries to gs://bucket/ci/job-suffix if set")
+	flags.StringVar(&d.boskosLocation, "boskos-location", "http://boskos.test-pods.svc.cluster.local.", "If set, manually specifies the location of the boskos server")
+	flags.IntVar(&d.boskosAcquireTimeoutSeconds, "boskos-acquire-timeout-seconds", 300, "How long (in seconds) to hang on a request to Boskos to acquire a resource before erroring")
 	return flags
 }
 
@@ -182,7 +215,7 @@ func (d *deployer) Build() error {
 
 // Deployer implementation methods below
 func (d *deployer) Up() error {
-	if err := d.verifyFlags(); err != nil {
+	if err := d.init(); err != nil {
 		return err
 	}
 	if err := d.prepareGcpIfNeeded(); err != nil {
@@ -453,7 +486,7 @@ func (d *deployer) cleanupNetworkFirewalls() (int, error) {
 }
 
 func (d *deployer) Down() error {
-	if err := d.verifyFlags(); err != nil {
+	if err := d.init(); err != nil {
 		return err
 	}
 	if err := d.prepareGcpIfNeeded(); err != nil {

kubetest2-gce/deployer/boskos.go → pkg/boskos/boskos.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package deployer
+package boskos
 
 import (
 	"context"
@@ -24,13 +24,14 @@ import (
 
 	"k8s.io/klog"
 	"sigs.k8s.io/boskos/client"
-	boskosCommon "sigs.k8s.io/boskos/common"
+	"sigs.k8s.io/boskos/common"
 )
 
 // const (for the run) owner string for consistency between up and down
 var boskosOwner = os.Getenv("JOB_NAME") + "-kubetest2"
 
-func makeBoskosClient(boskosLocation string) (*client.Client, error) {
+// NewClient creates a boskos client for kubetest2 deployers.
+func NewClient(boskosLocation string) (*client.Client, error) {
 	boskos, err := client.NewClient(
 		boskosOwner,
 		boskosLocation,
@@ -44,36 +45,34 @@ func makeBoskosClient(boskosLocation string) (*client.Client, error) {
 	return boskos, nil
 }
 
-// getProjectFromBoskos creates a boskos client, acquires a gcp project
-// and starts a heartbeat goroutine to keep the project reserved
-func getProjectFromBoskos(boskosClient *client.Client, timeout time.Duration, heartbeatClose chan struct{}) (string, error) {
-	resourceType := "gce-project"
+// Acquire acquires a resource for the given type and starts a heartbeat goroutine to keep the resource reserved.
+func Acquire(boskosClient *client.Client, resourceType string, timeout time.Duration, heartbeatClose chan struct{}) (*common.Resource, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
-	boskosProject, err := boskosClient.AcquireWait(ctx, resourceType, "free", "busy")
+	boskosResource, err := boskosClient.AcquireWait(ctx, resourceType, "free", "busy")
 	if err != nil {
-		return "", fmt.Errorf("failed to get a %q from boskos: %s", resourceType, err)
+		return nil, fmt.Errorf("failed to get a %q from boskos: %s", resourceType, err)
 	}
-	if boskosProject == nil {
-		return "", fmt.Errorf("boskos had no %s available", resourceType)
+	if boskosResource == nil {
+		return nil, fmt.Errorf("boskos had no %s available", resourceType)
 	}
 
 	startBoskosHeartbeat(
 		boskosClient,
-		boskosProject,
+		boskosResource,
 		5*time.Minute,
 		heartbeatClose,
 	)
 
-	return boskosProject.Name, nil
+	return boskosResource, nil
 }
 
 // startBoskosHeartbeat starts a goroutine that sends periodic updates to boskos
 // about the provided resource until the channel is closed. This prevents
 // reaper from taking the resource from the deployer while it is still in use.
-func startBoskosHeartbeat(boskosClient *client.Client, resource *boskosCommon.Resource, interval time.Duration, close chan struct{}) {
-	go func(c *client.Client, resource *boskosCommon.Resource) {
+func startBoskosHeartbeat(boskosClient *client.Client, resource *common.Resource, interval time.Duration, close chan struct{}) {
+	go func(c *client.Client, resource *common.Resource) {
 		klog.V(2).Info("boskos hearbeat starting")
 
 		for {
@@ -91,12 +90,11 @@ func startBoskosHeartbeat(boskosClient *client.Client, resource *boskosCommon.Re
 	}(boskosClient, resource)
 }
 
-func releaseBoskosProject(client *client.Client, projectName string, heartbeatClose chan struct{}) error {
-	if err := client.Release(projectName, "free"); err != nil {
-		return fmt.Errorf("failed to release %s: %s", projectName, err)
+// Release releases a resource.
+func Release(client *client.Client, resourceName string, heartbeatClose chan struct{}) error {
+	if err := client.Release(resourceName, "free"); err != nil {
+		return fmt.Errorf("failed to release %s: %s", resourceName, err)
 	}
-
 	close(heartbeatClose)
-
 	return nil
 }