use parameter in ContextTimeout

chatwork · Dec 1, 2023 · 81c4595 · 81c4595
1 parent cc4f589
commit 81c4595
Show file tree

Hide file tree

Showing 6 changed files with 55 additions and 19 deletions.
diff --git a/cmd/cert-manager/cert-manager.go b/cmd/cert-manager/cert-manager.go
@@ -87,6 +87,9 @@ func (c *CertManager) Check() error {
 	cert := c.createCertificateObject()
 
 	if err := c.createResources(cert); err != nil {
+		if err := c.cleanUpResources(cert); err != nil {
+			c.Chatwork.AddMessage(fmt.Sprintf("Error Delete Resources: %s", err))
+		}
 		return err
 	}
 	defer func() {
@@ -231,7 +234,7 @@ func (c *CertManager) createCert(cert certificates) error {
 
 	secretClient := c.Clientset.CoreV1().Secrets(c.Namespace)
 
-	err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
+	err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, c.Timeout, true, func(ctx context.Context) (bool, error) {
 		secret, err := secretClient.Get(ctx, cert.rootCA.Spec.SecretName, metav1.GetOptions{})
 		if err != nil {
 			c.Logger().WithError(err).Errorf("Waiting for secret %s to be ready", cert.rootCA.Spec.SecretName)
@@ -262,7 +265,7 @@ func (c *CertManager) createCert(cert certificates) error {
 		return err
 	}
 
-	err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
+	err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, c.Timeout, true, func(ctx context.Context) (bool, error) {
 		secret, err := secretClient.Get(ctx, cert.certificate.Spec.SecretName, metav1.GetOptions{})
 		if err != nil {
 			c.Logger().WithError(err).Errorf("Waiting for secret %s to be ready\n", cert.certificate.Spec.SecretName)

diff --git a/cmd/checker.go b/cmd/checker.go
@@ -18,6 +18,7 @@ type Checker struct {
 }
 
 func NewChecker(namespace string, clientset *kubernetes.Clientset, debug bool, logger func() *logrus.Entry, chatwork *notify.Chatwork, timeout time.Duration) *Checker {
+	logger().Info("Checker timeout: ", timeout)
 	return &Checker{
 		Namespace: namespace,
 		Clientset: clientset,

diff --git a/cmd/cluster-autoscaler/clusterAutoscalerCheck.go b/cmd/cluster-autoscaler/clusterAutoscalerCheck.go
@@ -16,6 +16,7 @@ import (
 	"github.com/chatwork/kibertas/util"
 	"github.com/chatwork/kibertas/util/k8s"
 	"github.com/chatwork/kibertas/util/notify"
+	"github.com/hashicorp/go-multierror"
 	"github.com/sirupsen/logrus"
 )
 
@@ -83,11 +84,36 @@ func (c *ClusterAutoscaler) Check() error {
 	c.Chatwork.AddMessage(fmt.Sprintf("spot nodes: %d\n", len(nodes.Items)))
 
 	if err := c.createResources(); err != nil {
+		if err := c.cleanUpResources(); err != nil {
+			c.Chatwork.AddMessage(fmt.Sprintf("Error Delete Resources: %s", err))
+		}
 		return err
 	}
+	defer func() {
+		if err := c.cleanUpResources(); err != nil {
+			c.Chatwork.AddMessage(fmt.Sprintf("Error Delete Resources: %s", err))
+		}
+	}()
+
 	return nil
 }
 
+func (c *ClusterAutoscaler) cleanUpResources() error {
+	k := k8s.NewK8s(c.Namespace, c.Clientset, c.Debug, c.Logger)
+	var result *multierror.Error
+	var err error
+	if err = k.DeleteDeployment(c.DeploymentName); err != nil {
+		c.Chatwork.AddMessage(fmt.Sprintf("Error Delete Deployment: %s", err))
+		result = multierror.Append(result, err)
+	}
+
+	if err = k.DeleteNamespace(); err != nil {
+		c.Chatwork.AddMessage(fmt.Sprintf("Error Delete Namespace: %s", err))
+		result = multierror.Append(result, err)
+	}
+	return result.ErrorOrNil()
+}
+
 func (c *ClusterAutoscaler) createResources() error {
 	k := k8s.NewK8s(c.Namespace, c.Clientset, c.Debug, c.Logger)
 
@@ -98,22 +124,13 @@ func (c *ClusterAutoscaler) createResources() error {
 		c.Chatwork.AddMessage(fmt.Sprint("Error Create Namespace:", err))
 		return err
 	}
-	defer func() {
-		if err := k.DeleteNamespace(); err != nil {
-			c.Chatwork.AddMessage(fmt.Sprint("Error Delete Namespace:", err))
-		}
-	}()
 
 	c.Chatwork.AddMessage(fmt.Sprintf("Create Deployment with desire replicas %d\n", c.ReplicaCount))
-	if err := k.CreateDeployment(c.createDeploymentObject()); err != nil {
+	if err := k.CreateDeployment(c.createDeploymentObject(), c.Timeout); err != nil {
 		c.Chatwork.AddMessage(fmt.Sprint("Error Create Deployment:", err))
 		return err
 	}
-	defer func() {
-		if err := k.DeleteDeployment(c.DeploymentName); err != nil {
-			c.Chatwork.AddMessage(fmt.Sprint("Error Delete Deployment:", err))
-		}
-	}()
+
 	c.Chatwork.AddMessage("cluster-autoscaler check finished\n")
 
 	return nil

diff --git a/cmd/datadog-agent/datadog-agent.go b/cmd/datadog-agent/datadog-agent.go
@@ -116,7 +116,7 @@ func (d *DatadogAgent) checkMetrics() error {
 	d.Chatwork.AddMessage(fmt.Sprintf("Querying metrics with query: %s", d.QueryMetrics))
 	now := time.Now().Unix()
 	from := now - 60*2
-	err := wait.PollUntilContextTimeout(context.Background(), 30*time.Second, 10*time.Minute, true, func(ctx context.Context) (bool, error) {
+	err := wait.PollUntilContextTimeout(context.Background(), 30*time.Second, d.Timeout, true, func(ctx context.Context) (bool, error) {
 		resp, r, err := api.QueryMetrics(ddctx, from, now, d.QueryMetrics)
 
 		if err != nil {

diff --git a/cmd/fluent/fluent.go b/cmd/fluent/fluent.go
@@ -102,6 +102,9 @@ func (f *Fluent) Check() error {
 	f.Chatwork.AddMessage(fmt.Sprintf("%s replica counts: %d", f.DeploymentName, f.ReplicaCount))
 
 	if err := f.createResources(); err != nil {
+		if err := f.cleanUpResources(); err != nil {
+			f.Chatwork.AddMessage(fmt.Sprintf("Error Delete Resources: %s", err))
+		}
 		return err
 	}
 	defer func() {
@@ -146,7 +149,7 @@ func (f *Fluent) createResources() error {
 		return err
 	}
 
-	if err := k.CreateDeployment(f.createDeploymentObject()); err != nil {
+	if err := k.CreateDeployment(f.createDeploymentObject(), f.Timeout); err != nil {
 		f.Chatwork.AddMessage(fmt.Sprint("Error Create Deployment:", err))
 		return err
 	}

diff --git a/cmd/ingress/ingressCheck.go b/cmd/ingress/ingressCheck.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strconv"
 	"time"
 
 	"github.com/chatwork/kibertas/cmd"
@@ -30,7 +31,6 @@ type Ingress struct {
 	IngressClassName string
 	ResourceName     string
 	ExternalHostname string
-	Timeout          time.Duration
 }
 
 func NewIngress(debug bool, logger func() *logrus.Entry, chatwork *notify.Chatwork, noDnsCheck bool, ingressClassName string) (*Ingress, error) {
@@ -51,6 +51,15 @@ func NewIngress(debug bool, logger func() *logrus.Entry, chatwork *notify.Chatwo
 		externalHostName = v
 	}
 
+	var err error
+	if v := os.Getenv("CHECK_TIMEOUT"); v != "" {
+		timeout, err = strconv.Atoi(v)
+		if err != nil {
+			logger().Errorf("strconv.Atoi: %s", err)
+			return nil, err
+		}
+	}
+
 	k8sclient, err := config.NewK8sClientset()
 	if err != nil {
 		logger().Errorf("NewK8sClientset: %s", err)
@@ -71,6 +80,9 @@ func (i *Ingress) Check() error {
 	defer i.Chatwork.Send()
 
 	if err := i.createResources(); err != nil {
+		if err := i.cleanUpResources(); err != nil {
+			i.Chatwork.AddMessage(fmt.Sprintf("Error Delete Resources: %s", err))
+		}
 		return err
 	}
 	defer func() {
@@ -128,15 +140,15 @@ func (i *Ingress) createResources() error {
 		i.Chatwork.AddMessage(fmt.Sprintf("Error Create Namespace: %s", err))
 		return err
 	}
-	if err := k.CreateDeployment(i.createDeploymentObject()); err != nil {
+	if err := k.CreateDeployment(i.createDeploymentObject(), i.Timeout); err != nil {
 		i.Chatwork.AddMessage(fmt.Sprintf("Error Create Deployment: %s", err))
 		return err
 	}
 	if err := k.CreateService(i.createServiceObject()); err != nil {
 		i.Chatwork.AddMessage(fmt.Sprintf("Error Create Service: %s", err))
 		return err
 	}
-	if err := k.CreateIngress(i.createIngressObject()); err != nil {
+	if err := k.CreateIngress(i.createIngressObject(), i.Timeout); err != nil {
 		i.Chatwork.AddMessage(fmt.Sprintf("Error Create Ingress: %s", err))
 		return err
 	}
@@ -260,7 +272,7 @@ func (i *Ingress) checkDNSRecord() error {
 
 	i.Chatwork.AddMessage("ingress create finished\n")
 	i.Logger().Println("Check DNS Record for: ", i.ExternalHostname)
-	err := wait.PollUntilContextTimeout(context.Background(), 30*time.Second, 10*time.Minute, false, func(ctx context.Context) (bool, error) {
+	err := wait.PollUntilContextTimeout(context.Background(), 30*time.Second, i.Timeout, false, func(ctx context.Context) (bool, error) {
 		m.SetQuestion(dns.Fqdn(i.ExternalHostname), dns.TypeA)
 		r, _, err := c.Exchange(m, "8.8.8.8:53")