perf: create (pipeline|task)run timeout checks in background

Both the pipelinerun and taskrun controllers start timeout checks at
startup. They do this by iterating though each namespace and spawning a
go routine for each pipelinerun/taskrun to run the check in the
background. Although each timeout check is done in a go routine, the
iteration through namespaces and creation of the timeout checks is done
in a blocking manner. This adds significant latency at startup when the
number of namespaces is large. Ultimately, this causes a delay in how
fast each controllers can actually start reconciling resources. To
speed up the startup time, this changes the logic so that the iteration
through namespaces is done in the background. The timeout checks were
already carried out in separate go routines and were therefore safe to
use in a concurrent context, so no extra logic was needed to make this
change work in a concurrent context.

pkg/reconciler/pipelinerun/controller.go

@@ -87,7 +87,7 @@ func NewController(namespace string, images pipeline.Images) func(context.Contex
  })
 
  timeoutHandler.SetCallbackFunc(impl.EnqueueKey)
- timeoutHandler.CheckTimeouts(ctx, namespace, kubeclientset, pipelineclientset)
+ go timeoutHandler.CheckTimeouts(ctx, namespace, kubeclientset, pipelineclientset)
 
  logger.Info("Setting up event handlers")
  pipelineRunInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{

pkg/reconciler/taskrun/controller.go

@@ -88,7 +88,7 @@ func NewController(namespace string, images pipeline.Images) func(context.Contex
  })
 
  timeoutHandler.SetCallbackFunc(impl.EnqueueKey)
- timeoutHandler.CheckTimeouts(ctx, namespace, kubeclientset, pipelineclientset)
+ go timeoutHandler.CheckTimeouts(ctx, namespace, kubeclientset, pipelineclientset)
 
  logger.Info("Setting up event handlers")
  taskRunInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{

pkg/timeout/handler_test.go

@@ -111,7 +111,7 @@ func TestTaskRunCheckTimeouts(t *testing.T) {
  }
 
  th.SetCallbackFunc(f)
- th.CheckTimeouts(context.Background(), testNs, c.Kube, c.Pipeline)
+ go th.CheckTimeouts(context.Background(), testNs, c.Kube, c.Pipeline)
 
  for _, tc := range []struct {
  name string
@@ -203,7 +203,7 @@ func TestTaskRunSingleNamespaceCheckTimeouts(t *testing.T) {
  // Note that since f843899a11d5d09b29fac750f72f4a7e4882f615 CheckTimeouts is always called
  // with a namespace so there is no reason to maintain all namespaces functionality;
  // however in #2905 we should remove CheckTimeouts completely
- th.CheckTimeouts(context.Background(), "", c.Kube, c.Pipeline)
+ go th.CheckTimeouts(context.Background(), "", c.Kube, c.Pipeline)
 
  for _, tc := range []struct {
  name string
@@ -316,7 +316,7 @@ func TestPipelinRunCheckTimeouts(t *testing.T) {
  }
 
  th.SetCallbackFunc(f)
- th.CheckTimeouts(context.Background(), allNs, c.Kube, c.Pipeline)
+ go th.CheckTimeouts(context.Background(), allNs, c.Kube, c.Pipeline)
  for _, tc := range []struct {
  name string
  pr *v1beta1.PipelineRun
@@ -395,7 +395,7 @@ func TestWithNoFunc(t *testing.T) {
  t.Fatal("Expected CheckTimeouts function not to panic")
  }
  }()
- testHandler.CheckTimeouts(context.Background(), allNs, c.Kube, c.Pipeline)
+ go testHandler.CheckTimeouts(context.Background(), allNs, c.Kube, c.Pipeline)
 
 }