From 7060dd647de33f68db7609fe25ea22b18e10aba1 Mon Sep 17 00:00:00 2001 From: Chi Zhang Date: Sun, 13 Jun 2021 18:50:13 -0700 Subject: [PATCH] Do not send heartbeat requests to Boskos if the interval is set to 0 --- kubetest2-gce/deployer/common.go | 1 + kubetest2-gce/deployer/deployer.go | 30 +++++++++++++++------------- kubetest2-gke/deployer/common.go | 8 +++++--- kubetest2-gke/deployer/deployer.go | 9 +++++---- kubetest2-gke/deployer/options/up.go | 9 +++++---- pkg/boskos/boskos.go | 16 ++++++++------- 6 files changed, 41 insertions(+), 32 deletions(-) diff --git a/kubetest2-gce/deployer/common.go b/kubetest2-gce/deployer/common.go index f09bf5e4..9c54d827 100644 --- a/kubetest2-gce/deployer/common.go +++ b/kubetest2-gce/deployer/common.go @@ -64,6 +64,7 @@ func (d *deployer) initialize() error { d.boskos, gceProjectResourceType, time.Duration(d.BoskosAcquireTimeoutSeconds)*time.Second, + time.Duration(d.BoskosHeartbeatIntervalSeconds)*time.Second, d.boskosHeartbeatClose, ) diff --git a/kubetest2-gce/deployer/deployer.go b/kubetest2-gce/deployer/deployer.go index e427afc3..229aae86 100644 --- a/kubetest2-gce/deployer/deployer.go +++ b/kubetest2-gce/deployer/deployer.go @@ -65,15 +65,16 @@ type deployer struct { // network is set for firewall rule creation, see buildEnv() and firewall.go network string - BoskosAcquireTimeoutSeconds int `desc:"How long (in seconds) to hang on a request to Boskos to acquire a resource before erroring."` - RepoRoot string `desc:"The path to the root of the local kubernetes/cloud-provider-gcp repo. Necessary to call certain scripts. Defaults to the current directory. If operating in legacy mode, this should be set to the local kubernetes/kubernetes repo."` - GCPProject string `desc:"GCP Project to create VMs in. If unset, the deployer will attempt to get a project from boskos."` - GCPZone string `desc:"GCP Zone to create VMs in. If unset, kube-up.sh and kube-down.sh defaults apply."` - EnableComputeAPI bool `desc:"If set, the deployer will enable the compute API for the project during the Up phase. This is necessary if the project has not been used before. WARNING: The currently configured GCP account must have permission to enable this API on the configured project."` - OverwriteLogsDir bool `desc:"If set, will overwrite an existing logs directory if one is encountered during dumping of logs. Useful when runnning tests locally."` - BoskosLocation string `desc:"If set, manually specifies the location of the boskos server. If unset and boskos is needed, defaults to http://boskos.test-pods.svc.cluster.local."` - LegacyMode bool `desc:"Set if the provided repo root is the kubernetes/kubernetes repo and not kubernetes/cloud-provider-gcp."` - NumNodes int `desc:"The number of nodes in the cluster."` + BoskosAcquireTimeoutSeconds int `desc:"How long (in seconds) to hang on a request to Boskos to acquire a resource before erroring."` + BoskosHeartbeatIntervalSeconds int `desc:"How often (in seconds) to send a heartbeat to Boskos to hold the acquired resource. 0 means no heartbeat."` + RepoRoot string `desc:"The path to the root of the local kubernetes/cloud-provider-gcp repo. Necessary to call certain scripts. Defaults to the current directory. If operating in legacy mode, this should be set to the local kubernetes/kubernetes repo."` + GCPProject string `desc:"GCP Project to create VMs in. If unset, the deployer will attempt to get a project from boskos."` + GCPZone string `desc:"GCP Zone to create VMs in. If unset, kube-up.sh and kube-down.sh defaults apply."` + EnableComputeAPI bool `desc:"If set, the deployer will enable the compute API for the project during the Up phase. This is necessary if the project has not been used before. WARNING: The currently configured GCP account must have permission to enable this API on the configured project."` + OverwriteLogsDir bool `desc:"If set, will overwrite an existing logs directory if one is encountered during dumping of logs. Useful when runnning tests locally."` + BoskosLocation string `desc:"If set, manually specifies the location of the boskos server. If unset and boskos is needed, defaults to http://boskos.test-pods.svc.cluster.local."` + LegacyMode bool `desc:"Set if the provided repo root is the kubernetes/kubernetes repo and not kubernetes/cloud-provider-gcp."` + NumNodes int `desc:"The number of nodes in the cluster."` EnableCacheMutationDetector bool `desc:"Sets the environment variable ENABLE_CACHE_MUTATION_DETECTOR=true during deployment. This should cause a panic if anything mutates a shared informer cache."` RuntimeConfig string `desc:"Sets the KUBE_RUNTIME_CONFIG environment variable during deployment."` @@ -117,11 +118,12 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { logsDir: filepath.Join(opts.RunDir(), "cluster-logs"), boskosHeartbeatClose: make(chan struct{}), // names need to start with an alphabet - instancePrefix: "kt2-" + pseudoUniqueSubstring(opts.RunID()), - network: "kt2-" + pseudoUniqueSubstring(opts.RunID()), - BoskosAcquireTimeoutSeconds: 5 * 60, - BoskosLocation: "http://boskos.test-pods.svc.cluster.local.", - NumNodes: 3, + instancePrefix: "kt2-" + pseudoUniqueSubstring(opts.RunID()), + network: "kt2-" + pseudoUniqueSubstring(opts.RunID()), + BoskosAcquireTimeoutSeconds: 5 * 60, + BoskosHeartbeatIntervalSeconds: 5 * 60, + BoskosLocation: "http://boskos.test-pods.svc.cluster.local.", + NumNodes: 3, } flagSet, err := gpflag.Parse(d) diff --git a/kubetest2-gke/deployer/common.go b/kubetest2-gke/deployer/common.go index 3d65476f..bc46a87d 100644 --- a/kubetest2-gke/deployer/common.go +++ b/kubetest2-gke/deployer/common.go @@ -29,9 +29,10 @@ import ( ) const ( - defaultBoskosLocation = "http://boskos.test-pods.svc.cluster.local." - defaultGKEProjectResourceType = "gke-project" - defaultBoskosAcquireTimeoutSeconds = 300 + defaultBoskosLocation = "http://boskos.test-pods.svc.cluster.local." + defaultGKEProjectResourceType = "gke-project" + defaultBoskosAcquireTimeoutSeconds = 300 + defaultBoskosHeartbeatIntervalSeconds = 300 ) func (d *Deployer) Init() error { @@ -71,6 +72,7 @@ func (d *Deployer) Initialize() error { d.boskos, d.BoskosResourceType, time.Duration(d.BoskosAcquireTimeoutSeconds)*time.Second, + time.Duration(d.BoskosHeartbeatIntervalSeconds)*time.Second, d.boskosHeartbeatClose, ) diff --git a/kubetest2-gke/deployer/deployer.go b/kubetest2-gke/deployer/deployer.go index ddad7eb1..66f67546 100644 --- a/kubetest2-gke/deployer/deployer.go +++ b/kubetest2-gke/deployer/deployer.go @@ -159,10 +159,11 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { Version: "", GCPSSHKeyIgnored: true, - BoskosLocation: defaultBoskosLocation, - BoskosResourceType: defaultGKEProjectResourceType, - BoskosAcquireTimeoutSeconds: defaultBoskosAcquireTimeoutSeconds, - BoskosProjectsRequested: 1, + BoskosLocation: defaultBoskosLocation, + BoskosResourceType: defaultGKEProjectResourceType, + BoskosAcquireTimeoutSeconds: defaultBoskosAcquireTimeoutSeconds, + BoskosHeartbeatIntervalSeconds: defaultBoskosHeartbeatIntervalSeconds, + BoskosProjectsRequested: 1, RetryableErrorPatterns: []string{gceStockoutErrorPattern}, }, diff --git a/kubetest2-gke/deployer/options/up.go b/kubetest2-gke/deployer/options/up.go index cbe78def..f5590a1a 100644 --- a/kubetest2-gke/deployer/options/up.go +++ b/kubetest2-gke/deployer/options/up.go @@ -33,10 +33,11 @@ type UpOptions struct { WorkloadIdentityEnabled bool `flag:"~enable-workload-identity" desc:"Whether enable workload identity for the cluster or not. See the details in https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity."` GCPSSHKeyIgnored bool `flag:"~ignore-gcp-ssh-key" desc:"Whether the GCP SSH key should be ignored or not for bringing up the cluster."` - BoskosLocation string `flag:"~boskos-location" desc:"If set, manually specifies the location of the Boskos server."` - BoskosResourceType string `flag:"~boskos-resource-type" desc:"If set, manually specifies the resource type of GCP projects to acquire from Boskos."` - BoskosAcquireTimeoutSeconds int `flag:"~boskos-acquire-timeout-seconds" desc:"How long (in seconds) to hang on a request to Boskos to acquire a resource before erroring."` - BoskosProjectsRequested int `flag:"~projects-requested" desc:"Number of projects to request from Boskos. It is only respected if projects is empty, and must be larger than zero."` + BoskosLocation string `flag:"~boskos-location" desc:"If set, manually specifies the location of the Boskos server."` + BoskosResourceType string `flag:"~boskos-resource-type" desc:"If set, manually specifies the resource type of GCP projects to acquire from Boskos."` + BoskosAcquireTimeoutSeconds int `flag:"~boskos-acquire-timeout-seconds" desc:"How long (in seconds) to hang on a request to Boskos to acquire a resource before erroring."` + BoskosHeartbeatIntervalSeconds int `flag:"~boskos-heartbeat-interval-seconds" desc:"How often (in seconds) to send a heartbeat to Boskos to hold the acquired resource. 0 means no heartbeat."` + BoskosProjectsRequested int `flag:"~projects-requested" desc:"Number of projects to request from Boskos. It is only respected if projects is empty, and must be larger than zero."` PrivateClusterAccessLevel string `flag:"~private-cluster-access-level" desc:"Private cluster access level, if not empty, must be one of 'no', 'limited' or 'unrestricted'. See the details in https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters."` PrivateClusterMasterIPRanges []string `flag:"~private-cluster-master-ip-range" desc:"Private cluster master IP ranges. It should be IPv4 CIDR(s), and its length must be the same as the number of clusters if private cluster is requested."` diff --git a/pkg/boskos/boskos.go b/pkg/boskos/boskos.go index 73ce0529..27a21c34 100644 --- a/pkg/boskos/boskos.go +++ b/pkg/boskos/boskos.go @@ -46,7 +46,7 @@ func NewClient(boskosLocation string) (*client.Client, error) { } // Acquire acquires a resource for the given type and starts a heartbeat goroutine to keep the resource reserved. -func Acquire(boskosClient *client.Client, resourceType string, timeout time.Duration, heartbeatClose chan struct{}) (*common.Resource, error) { +func Acquire(boskosClient *client.Client, resourceType string, timeout, heartbeatInterval time.Duration, heartbeatClose chan struct{}) (*common.Resource, error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -58,12 +58,14 @@ func Acquire(boskosClient *client.Client, resourceType string, timeout time.Dura return nil, fmt.Errorf("boskos had no %s available", resourceType) } - startBoskosHeartbeat( - boskosClient, - boskosResource, - 5*time.Minute, - heartbeatClose, - ) + if heartbeatInterval != 0 { + startBoskosHeartbeat( + boskosClient, + boskosResource, + heartbeatInterval, + heartbeatClose, + ) + } return boskosResource, nil }