wait for a given duration in case of imagePullBackOff

We have implemented imagePullBackOff as fail fast. The issue with this approach is, the node where the pod is scheduled often experiences registry rate limit. The image pull failure because of the rate limit returns the same warning (reason: Failed and message: ImagePullBackOff). The pod can potentially recover after waiting for enough time until the cap is expired. Kubernetes can then successfully pull the image and bring the pod up. Introducing a default configuration to specify cluster level timeout to allow the imagePullBackOff to retry for a given duration. Once that duration has passed, return a permanent failure. tektoncd#5987 tektoncd#7184 Signed-off-by: Priti Desai <pdesai@us.ibm.com> wait for a given duration in case of imagePullBackOff Signed-off-by: Priti Desai <pdesai@us.ibm.com>
pritidesai · Feb 13, 2024 · e77d6a2 · e77d6a2
1 parent 1568ed1
commit e77d6a2
Show file tree

Hide file tree

Showing 4 changed files with 86 additions and 3 deletions.
diff --git a/config/config-defaults.yaml b/config/config-defaults.yaml
@@ -87,6 +87,10 @@ data:
     # no default-resolver-type is specified by default
     default-resolver-type:
 
+    # default-imagepullbackoff-timeout contains the default number of minutes to wait
+    # before requeuing the TaskRun to retry
+    # default-imagepullbackoff-timeout: "5"
+
     # default-container-resource-requirements allow users to update default resource requirements
     # to a init-containers and containers of a pods create by the controller
     # Onet: All the resource requirements are applied to init-containers and containers

diff --git a/docs/additional-configs.md b/docs/additional-configs.md
@@ -31,6 +31,7 @@ installation.
     - [Verify the transparency logs using `rekor-cli`](#verify-the-transparency-logs-using-rekor-cli)
   - [Verify Tekton Resources](#verify-tekton-resources)
   - [Pipelinerun with Affinity Assistant](#pipelineruns-with-affinity-assistant)
+  - [TaskRuns with `imagePullBackOff` Timeout](#taskruns-with-imagepullbackoff-timeout)
   - [Next steps](#next-steps)
 
 
@@ -672,6 +673,28 @@ please take a look at [Trusted Resources](./trusted-resources.md).
 The cluster operators can review the [guidelines](developers/affinity-assistant.md) to `cordon` a node in the cluster
 with the tekton controller and the affinity assistant is enabled.
 
+## TaskRuns with `imagePullBackOff` Timeout
+
+Tekton pipelines has adopted a fail fast strategy with a taskRun failing with `TaskRunImagePullFailed` in case of an
+`imagePullBackOff`. This can be limited in some cases, and it generally depends on the infrastructure. To allow the
+cluster operators to decide whether to wait in case of an `imagePullBackOff`, a setting is available to configure
+the wait time in minutes such that the controller will wait for the specified duration before declaring a failure.
+For example, with the following `config-defaults`, the controller does not mark the taskRun as failure for 5 minutes since
+the pod is scheduled in case the image pull fails with `imagePullBackOff`.
+See issue https://github.com/tektoncd/pipeline/issues/5987 for more details.
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-defaults
+  namespace: tekton-pipelines
+data:
+  default-imagepullbackoff-timeout: "5"
+```
+
+
+
 ## Next steps
 
 To get started with Tekton check the [Introductory tutorials][quickstarts],

diff --git a/pkg/apis/config/default.go b/pkg/apis/config/default.go
@@ -49,6 +49,8 @@ const (
 	// default resource requirements, will be applied to all the containers, which has empty resource requirements
 	ResourceRequirementDefaultContainerKey = "default"
 
+	DefaultImagePullBackOffTimeout = 0
+
 	defaultTimeoutMinutesKey                = "default-timeout-minutes"
 	defaultServiceAccountKey                = "default-service-account"
 	defaultManagedByLabelValueKey           = "default-managed-by-label-value"
@@ -60,6 +62,7 @@ const (
 	defaultForbiddenEnv                     = "default-forbidden-env"
 	defaultResolverTypeKey                  = "default-resolver-type"
 	defaultContainerResourceRequirementsKey = "default-container-resource-requirements"
+	defaultImagePullBackOffTimeout          = "default-imagepullbackoff-timeout"
 )
 
 // DefaultConfig holds all the default configurations for the config.
@@ -79,6 +82,7 @@ type Defaults struct {
 	DefaultForbiddenEnv                  []string
 	DefaultResolverType                  string
 	DefaultContainerResourceRequirements map[string]corev1.ResourceRequirements
+	DefaultImagePullBackOffTimeout       int
 }
 
 // GetDefaultsConfigName returns the name of the configmap containing all
@@ -109,6 +113,7 @@ func (cfg *Defaults) Equals(other *Defaults) bool {
 		other.DefaultTaskRunWorkspaceBinding == cfg.DefaultTaskRunWorkspaceBinding &&
 		other.DefaultMaxMatrixCombinationsCount == cfg.DefaultMaxMatrixCombinationsCount &&
 		other.DefaultResolverType == cfg.DefaultResolverType &&
+		other.DefaultImagePullBackOffTimeout == cfg.DefaultImagePullBackOffTimeout &&
 		reflect.DeepEqual(other.DefaultForbiddenEnv, cfg.DefaultForbiddenEnv)
 }
 
@@ -121,6 +126,7 @@ func NewDefaultsFromMap(cfgMap map[string]string) (*Defaults, error) {
 		DefaultCloudEventsSink:            DefaultCloudEventSinkValue,
 		DefaultMaxMatrixCombinationsCount: DefaultMaxMatrixCombinationsCount,
 		DefaultResolverType:               DefaultResolverTypeValue,
+		DefaultImagePullBackOffTimeout:    DefaultImagePullBackOffTimeout,
 	}
 
 	if defaultTimeoutMin, ok := cfgMap[defaultTimeoutMinutesKey]; ok {
@@ -191,6 +197,14 @@ func NewDefaultsFromMap(cfgMap map[string]string) (*Defaults, error) {
 		tc.DefaultContainerResourceRequirements = resourceRequirementsValue
 	}
 
+	if defaultImagePullBackOff, ok := cfgMap[defaultImagePullBackOffTimeout]; ok {
+		timeout, err := strconv.ParseInt(defaultImagePullBackOff, 10, 0)
+		if err != nil {
+			return nil, fmt.Errorf("failed parsing tracing config %q", defaultImagePullBackOffTimeout)
+		}
+		tc.DefaultImagePullBackOffTimeout = int(timeout)
+	}
+
 	return &tc, nil
 }
 

diff --git a/pkg/reconciler/taskrun/taskrun.go b/pkg/reconciler/taskrun/taskrun.go
@@ -94,13 +94,15 @@ type Reconciler struct {
 	tracerProvider           trace.TracerProvider
 }
 
+const ImagePullBackOff = "ImagePullBackOff"
+
 var (
 	// Check that our Reconciler implements taskrunreconciler.Interface
 	_ taskrunreconciler.Interface = (*Reconciler)(nil)
 
 	// Pod failure reasons that trigger failure of the TaskRun
 	podFailureReasons = map[string]struct{}{
-		"ImagePullBackOff": {},
+		ImagePullBackOff:   {},
 		"InvalidImageName": {},
 	}
 )
@@ -171,7 +173,7 @@ func (c *Reconciler) ReconcileKind(ctx context.Context, tr *v1.TaskRun) pkgrecon
 	}
 
 	// Check for Pod Failures
-	if failed, reason, message := c.checkPodFailed(tr); failed {
+	if failed, reason, message := c.checkPodFailed(tr, ctx); failed {
 		err := c.failTaskRun(ctx, tr, reason, message)
 		return c.finishReconcileUpdateEmitEvents(ctx, tr, before, err)
 	}
@@ -222,10 +224,30 @@ func (c *Reconciler) ReconcileKind(ctx context.Context, tr *v1.TaskRun) pkgrecon
 	return nil
 }
 
-func (c *Reconciler) checkPodFailed(tr *v1.TaskRun) (bool, v1.TaskRunReason, string) {
+func (c *Reconciler) checkPodFailed(tr *v1.TaskRun, ctx context.Context) (bool, v1.TaskRunReason, string) {
 	for _, step := range tr.Status.Steps {
 		if step.Waiting != nil {
 			if _, found := podFailureReasons[step.Waiting.Reason]; found {
+				if step.Waiting.Reason == ImagePullBackOff {
+					imagePullBackOffTimeOut := config.FromContextOrDefaults(ctx).Defaults.DefaultImagePullBackOffTimeout
+					// only attempt to recover from the imagePullBackOff if specified
+					if imagePullBackOffTimeOut > 0 {
+						p, err := c.KubeClientSet.CoreV1().Pods(tr.Namespace).Get(ctx, tr.Status.PodName, metav1.GetOptions{})
+						if err != nil {
+							message := fmt.Sprintf(`The step %q in TaskRun %q failed to pull the image %q. The pod could not be retrieved with error: "%s."`, step.Name, tr.Name, step.ImageID, err)
+							return true, v1.TaskRunReasonImagePullFailed, message
+						}
+						for _, condition := range p.Status.Conditions {
+							// check the pod condition to get the time when the pod was scheduled
+							// keep trying until the pod schedule time has exceeded the specified imagePullBackOff timeout duration
+							if condition.Type == corev1.PodScheduled {
+								if c.Clock.Since(condition.LastTransitionTime.Time) < time.Duration(imagePullBackOffTimeOut)*time.Minute {
+									return false, "", ""
+								}
+							}
+						}
+					}
+				}
 				image := step.ImageID
 				message := fmt.Sprintf(`The step %q in TaskRun %q failed to pull the image %q. The pod errored with the message: "%s."`, step.Name, tr.Name, image, step.Waiting.Message)
 				return true, v1.TaskRunReasonImagePullFailed, message
@@ -235,6 +257,26 @@ func (c *Reconciler) checkPodFailed(tr *v1.TaskRun) (bool, v1.TaskRunReason, str
 	for _, sidecar := range tr.Status.Sidecars {
 		if sidecar.Waiting != nil {
 			if _, found := podFailureReasons[sidecar.Waiting.Reason]; found {
+				if sidecar.Waiting.Reason == ImagePullBackOff {
+					imagePullBackOffTimeOut := config.FromContextOrDefaults(ctx).Defaults.DefaultImagePullBackOffTimeout
+					// only attempt to recover from the imagePullBackOff if specified
+					if imagePullBackOffTimeOut > 0 {
+						p, err := c.KubeClientSet.CoreV1().Pods(tr.Namespace).Get(ctx, tr.Status.PodName, metav1.GetOptions{})
+						if err != nil {
+							message := fmt.Sprintf(`The step %q in TaskRun %q failed to pull the image %q. The pod could not be retrieved with error: "%s."`, sidecar.Name, tr.Name, sidecar.ImageID, err)
+							return true, v1.TaskRunReasonImagePullFailed, message
+						}
+						for _, condition := range p.Status.Conditions {
+							// check the pod condition to get the time when the pod was scheduled
+							// keep trying until the pod schedule time has exceeded the specified imagePullBackOff timeout duration
+							if condition.Type == corev1.PodScheduled {
+								if c.Clock.Since(condition.LastTransitionTime.Time) < time.Duration(imagePullBackOffTimeOut)*time.Minute {
+									return false, "", ""
+								}
+							}
+						}
+					}
+				}
 				image := sidecar.ImageID
 				message := fmt.Sprintf(`The sidecar %q in TaskRun %q failed to pull the image %q. The pod errored with the message: "%s."`, sidecar.Name, tr.Name, image, sidecar.Waiting.Message)
 				return true, v1.TaskRunReasonImagePullFailed, message