Add CRD waiter for migration job and webhook server

It may take some time before Catalog CRDs registration shows up in main API Server.
Because of that the CRD waiter is executed for each binary. Additionally, we added
retries when restoring Service Catalog resources during the migration process.

cmd/controller-manager/app/controller_manager.go

@@ -25,8 +25,8 @@ import (
 	"os"
 	goruntime "runtime"
 	"strconv"
-	"time"
 
+	"github.com/kubernetes-sigs/service-catalog/pkg/util"
 	"k8s.io/client-go/kubernetes"
 	v1coordination "k8s.io/client-go/kubernetes/typed/coordination/v1"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -43,7 +43,6 @@ import (
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/client-go/tools/leaderelection"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
@@ -234,7 +233,7 @@ func Run(controllerManagerOptions *options.ControllerManagerServer) error {
 		// 	k8sClientBuilder = rootClientBuilder
 		// }
 
-		err := StartControllers(controllerManagerOptions, k8sKubeconfig, serviceCatalogClientBuilder, recorder, readinessProbe, ctx.Done())
+		err := StartControllers(controllerManagerOptions, k8sKubeconfig, serviceCatalogClientBuilder, recorder, ctx.Done())
 		klog.Fatalf("error running controllers: %v", err)
 		panic("unreachable")
 	}
@@ -294,20 +293,12 @@ func StartControllers(s *options.ControllerManagerServer,
 	coreKubeconfig *rest.Config,
 	serviceCatalogClientBuilder controller.ClientBuilder,
 	recorder record.EventRecorder,
-	rProbe *probe.ReadinessCRD,
 	stop <-chan struct{}) error {
 
-	// When Catalog Controller and Catalog API Server are started at the
-	// same time with API Aggregation enabled, it may take some time before
-	// Catalog registration shows up in API Server.  Attempt to get resources
-	// every 10 seconds and quit after 3 minutes if unsuccessful.
-	err := wait.PollImmediate(10*time.Second, 3*time.Minute, rProbe.IsReady)
-
-	if err != nil {
-		if err == wait.ErrWaitTimeout {
-			return fmt.Errorf("unable to start service-catalog controller: CRDs are not available")
-		}
-		return err
+	// It may take some time before Catalog CRDs registration shows up in main API Server.
+	// We can start Service Catalog clients/informers only when CRDs are available.
+	if err := util.WaitForServiceCatalogCRDs(coreKubeconfig); err != nil {
+		return fmt.Errorf("unable to start service-catalog controller: %v", err)
 	}
 
 	// Launch service-catalog controller

cmd/migration/migration.go

@@ -23,6 +23,7 @@ import (
 	"github.com/kubernetes-sigs/service-catalog/pkg/cleaner"
 	sc "github.com/kubernetes-sigs/service-catalog/pkg/client/clientset_generated/clientset"
 	"github.com/kubernetes-sigs/service-catalog/pkg/migration"
+	"github.com/kubernetes-sigs/service-catalog/pkg/util"
 	k8sClientSet "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -43,6 +44,16 @@ func RunCommand(opt *Options) error {
 	if err != nil {
 		return fmt.Errorf("failed to get Kubernetes client config: %s", err)
 	}
+
+	// In case of restore action we are using CRDs.
+	// It may take some time before Catalog CRDs registration shows up in main API Server.
+	// We can start Service Catalog clients/informers only when CRDs are available.
+	if opt.Action == restoreActionName {
+		if err := util.WaitForServiceCatalogCRDs(restConfig); err != nil {
+			return fmt.Errorf("while waiting for ready Service Catalog CRDs: %v", err)
+		}
+	}
+
 	scClient, err := sc.NewForConfig(restConfig)
 	if err != nil {
 		return fmt.Errorf("failed to create Service Catalog client: %s", err)

cmd/webhook/server/webhook.go

@@ -21,6 +21,7 @@ import (
 	"net/http"
 
 	scTypes "github.com/kubernetes-sigs/service-catalog/pkg/apis/servicecatalog/v1beta1"
+	"github.com/kubernetes-sigs/service-catalog/pkg/util"
 	csbmutation "github.com/kubernetes-sigs/service-catalog/pkg/webhook/servicecatalog/clusterservicebroker/mutation"
 	cscmutation "github.com/kubernetes-sigs/service-catalog/pkg/webhook/servicecatalog/clusterserviceclass/mutation"
 	cspmutation "github.com/kubernetes-sigs/service-catalog/pkg/webhook/servicecatalog/clusterserviceplan/mutation"
@@ -66,16 +67,27 @@ func RunServer(opts *WebhookServerOptions, stopCh <-chan struct{}) error {
 }
 
 func run(opts *WebhookServerOptions, stopCh <-chan struct{}) error {
-	cfg := config.GetConfigOrDie()
-	mgr, err := manager.New(cfg, manager.Options{})
+	cfg, err := config.GetConfig()
 	if err != nil {
-		return errors.Wrap(err, "while set up overall controller manager for webhook server")
+		return errors.Wrap(err, "while getting Kubernetes client config")
 	}
+
 	apiextensionsClient, err := apiextensionsclientset.NewForConfig(cfg)
 	if err != nil {
 		return errors.Wrap(err, "while create apiextension clientset")
 	}
 
+	// It may take some time before Catalog CRDs registration shows up in main API Server.
+	// We can start Service Catalog clients/informers only when CRDs are available.
+	if err := util.WaitForServiceCatalogCRDs(cfg); err != nil {
+		return fmt.Errorf("while waiting for ready Service Catalog CRDs: %v", err)
+	}
+
+	mgr, err := manager.New(cfg, manager.Options{})
+	if err != nil {
+		return errors.Wrap(err, "while set up overall controller manager for webhook server")
+	}
+
 	err = scTypes.AddToScheme(mgr.GetScheme())
 	if err != nil {
 		return errors.Wrap(err, "while register Service Catalog scheme into manager")