Avoid setting owner refs if the service mesh/ingress is on a different cluster

cmd/flagger/main.go

@@ -168,15 +168,15 @@ func main() {
  if kubeconfigServiceMesh == "" {
  kubeconfigServiceMesh = kubeconfig
  }
- cfgHost, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigServiceMesh)
+ serviceMeshCfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigServiceMesh)
  if err != nil {
  logger.Fatalf("Error building host kubeconfig: %v", err)
  }
 
- cfgHost.QPS = float32(kubeconfigQPS)
- cfgHost.Burst = kubeconfigBurst
+ serviceMeshCfg.QPS = float32(kubeconfigQPS)
+ serviceMeshCfg.Burst = kubeconfigBurst
 
- meshClient, err := clientset.NewForConfig(cfgHost)
+ meshClient, err := clientset.NewForConfig(serviceMeshCfg)
  if err != nil {
  logger.Fatalf("Error building mesh clientset: %v", err)
  }
@@ -212,7 +212,14 @@ func main() {
  // start HTTP server
  go server.ListenAndServe(port, 3*time.Second, logger, stopCh)
 
- routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, ingressClass, logger, meshClient)
+ setOwnerRefs := true
+ // Router shouldn't set OwnerRefs on resources that they create since the
+ // service mesh/ingress controller is in a different cluster.
+ if cfg.Host != serviceMeshCfg.Host {
+ setOwnerRefs = false
+ }
+
+ routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, ingressClass, logger, meshClient, setOwnerRefs)
 
  var configTracker canary.Tracker
  if enableConfigTracking {

pkg/controller/scheduler_daemonset_fixture_test.go

@@ -92,7 +92,7 @@ func newDaemonSetFixture(c *flaggerv1.Canary) daemonSetFixture {
  }
 
  // init router
- rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient)
+ rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient, true)
 
  // init observer
  observerFactory, _ := observers.NewFactory(testMetricsServerURL)

pkg/controller/scheduler_deployment_fixture_test.go

@@ -120,7 +120,7 @@ func newDeploymentFixture(c *flaggerv1.Canary) fixture {
  }
 
  // init router
- rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient)
+ rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient, true)
 
  // init observer
  observerFactory, _ := observers.NewFactory(testMetricsServerURL)

pkg/router/appmesh.go

@@ -43,6 +43,7 @@ type AppMeshRouter struct {
  appmeshClient clientset.Interface
  flaggerClient clientset.Interface
  logger *zap.SugaredLogger
+ setOwnerRefs bool
 }
 
 // Reconcile creates or updates App Mesh virtual nodes and virtual services
@@ -161,16 +162,18 @@ func (ar *AppMeshRouter) reconcileVirtualNode(canary *flaggerv1.Canary, name str
  Namespace: canary.Namespace,
  Labels: metadata.Labels,
  Annotations: filterMetadata(metadata.Annotations),
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(canary, schema.GroupVersionKind{
- Group: flaggerv1.SchemeGroupVersion.Group,
- Version: flaggerv1.SchemeGroupVersion.Version,
- Kind: flaggerv1.CanaryKind,
- }),
- },
  },
  Spec: vnSpec,
  }
+ if ar.setOwnerRefs {
+ virtualnode.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(canary, schema.GroupVersionKind{
+ Group: flaggerv1.SchemeGroupVersion.Group,
+ Version: flaggerv1.SchemeGroupVersion.Version,
+ Kind: flaggerv1.CanaryKind,
+ }),
+ }
+ }
  _, err = ar.appmeshClient.AppmeshV1beta1().VirtualNodes(canary.Namespace).Create(context.TODO(), virtualnode, metav1.CreateOptions{})
  if err != nil {
  return fmt.Errorf("VirtualNode %s.%s create error %w", name, canary.Namespace, err)
@@ -314,16 +317,18 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name
  ObjectMeta: metav1.ObjectMeta{
  Name: name,
  Namespace: canary.Namespace,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(canary, schema.GroupVersionKind{
- Group: flaggerv1.SchemeGroupVersion.Group,
- Version: flaggerv1.SchemeGroupVersion.Version,
- Kind: flaggerv1.CanaryKind,
- }),
- },
  },
  Spec: vsSpec,
  }
+ if ar.setOwnerRefs {
+ virtualService.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(canary, schema.GroupVersionKind{
+ Group: flaggerv1.SchemeGroupVersion.Group,
+ Version: flaggerv1.SchemeGroupVersion.Version,
+ Kind: flaggerv1.CanaryKind,
+ }),
+ }
+ }
 
  // set App Mesh Gateway annotation on primary virtual service
  if canaryWeight == 0 {

pkg/router/appmesh_v1beta2.go

@@ -44,6 +44,7 @@ type AppMeshv1beta2Router struct {
  flaggerClient clientset.Interface
  logger *zap.SugaredLogger
  labelSelector string
+ setOwnerRefs bool
 }
 
 // Reconcile creates or updates App Mesh virtual nodes and virtual services
@@ -175,16 +176,18 @@ func (ar *AppMeshv1beta2Router) reconcileVirtualNode(canary *flaggerv1.Canary, n
  Namespace: canary.Namespace,
  Labels: metadata.Labels,
  Annotations: filterMetadata(metadata.Annotations),
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(canary, schema.GroupVersionKind{
- Group: flaggerv1.SchemeGroupVersion.Group,
- Version: flaggerv1.SchemeGroupVersion.Version,
- Kind: flaggerv1.CanaryKind,
- }),
- },
  },
  Spec: vnSpec,
  }
+ if ar.setOwnerRefs {
+ virtualnode.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(canary, schema.GroupVersionKind{
+ Group: flaggerv1.SchemeGroupVersion.Group,
+ Version: flaggerv1.SchemeGroupVersion.Version,
+ Kind: flaggerv1.CanaryKind,
+ }),
+ }
+ }
  _, err = ar.appmeshClient.AppmeshV1beta2().VirtualNodes(canary.Namespace).Create(context.TODO(), virtualnode, metav1.CreateOptions{})
  if err != nil {
  return fmt.Errorf("VirtualNode %s.%s create error %w", name, canary.Namespace, err)
@@ -341,16 +344,18 @@ func (ar *AppMeshv1beta2Router) reconcileVirtualRouter(canary *flaggerv1.Canary,
  ObjectMeta: metav1.ObjectMeta{
  Name: name,
  Namespace: canary.Namespace,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(canary, schema.GroupVersionKind{
- Group: flaggerv1.SchemeGroupVersion.Group,
- Version: flaggerv1.SchemeGroupVersion.Version,
- Kind: flaggerv1.CanaryKind,
- }),
- },
  },
  Spec: vrSpec,
  }
+ if ar.setOwnerRefs {
+ virtualRouter.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(canary, schema.GroupVersionKind{
+ Group: flaggerv1.SchemeGroupVersion.Group,
+ Version: flaggerv1.SchemeGroupVersion.Version,
+ Kind: flaggerv1.CanaryKind,
+ }),
+ }
+ }
 
  _, err = ar.appmeshClient.AppmeshV1beta2().VirtualRouters(canary.Namespace).Create(context.TODO(), virtualRouter, metav1.CreateOptions{})
  if err != nil {
@@ -363,13 +368,6 @@ func (ar *AppMeshv1beta2Router) reconcileVirtualRouter(canary *flaggerv1.Canary,
  ObjectMeta: metav1.ObjectMeta{
  Name: name,
  Namespace: canary.Namespace,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(canary, schema.GroupVersionKind{
- Group: flaggerv1.SchemeGroupVersion.Group,
- Version: flaggerv1.SchemeGroupVersion.Version,
- Kind: flaggerv1.CanaryKind,
- }),
- },
  },
  Spec: appmeshv1.VirtualServiceSpec{
  Provider: &appmeshv1.VirtualServiceProvider{
@@ -381,6 +379,15 @@ func (ar *AppMeshv1beta2Router) reconcileVirtualRouter(canary *flaggerv1.Canary,
  },
  },
  }
+ if ar.setOwnerRefs {
+ virtualService.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(canary, schema.GroupVersionKind{
+ Group: flaggerv1.SchemeGroupVersion.Group,
+ Version: flaggerv1.SchemeGroupVersion.Version,
+ Kind: flaggerv1.CanaryKind,
+ }),
+ }
+ }
 
  // set App Mesh Gateway annotation on primary virtual service
  if canaryWeight == 0 {

pkg/router/contour.go

@@ -41,6 +41,7 @@ type ContourRouter struct {
  flaggerClient clientset.Interface
  logger *zap.SugaredLogger
  ingressClass string
+ setOwnerRefs bool
 }
 
 // Reconcile creates or updates the HTTP proxy
@@ -169,20 +170,22 @@ func (cr *ContourRouter) Reconcile(canary *flaggerv1.Canary) error {
  Namespace: canary.Namespace,
  Labels: metadata.Labels,
  Annotations: filterMetadata(metadata.Annotations),
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(canary, schema.GroupVersionKind{
- Group: flaggerv1.SchemeGroupVersion.Group,
- Version: flaggerv1.SchemeGroupVersion.Version,
- Kind: flaggerv1.CanaryKind,
- }),
- },
  },
  Spec: newSpec,
  Status: contourv1.HTTPProxyStatus{
  CurrentStatus: "valid",
  Description: "valid HTTPProxy",
  },
  }
+ if cr.setOwnerRefs {
+ proxy.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(canary, schema.GroupVersionKind{
+ Group: flaggerv1.SchemeGroupVersion.Group,
+ Version: flaggerv1.SchemeGroupVersion.Version,
+ Kind: flaggerv1.CanaryKind,
+ }),
+ }
+ }
 
  if cr.ingressClass != "" {
  proxy.Annotations = map[string]string{

pkg/router/factory.go

@@ -35,14 +35,16 @@ type Factory struct {
  ingressAnnotationsPrefix string
  ingressClass string
  logger *zap.SugaredLogger
+ setOwnerRefs bool
 }
 
 func NewFactory(kubeConfig *restclient.Config, kubeClient kubernetes.Interface,
  flaggerClient clientset.Interface,
  ingressAnnotationsPrefix string,
  ingressClass string,
  logger *zap.SugaredLogger,
- meshClient clientset.Interface) *Factory {
+ meshClient clientset.Interface,
+ setOwnerRefs bool) *Factory {
  return &Factory{
  kubeConfig: kubeConfig,
  meshClient: meshClient,
@@ -51,6 +53,7 @@ func NewFactory(kubeConfig *restclient.Config, kubeClient kubernetes.Interface,
  ingressAnnotationsPrefix: ingressAnnotationsPrefix,
  ingressClass: ingressClass,
  logger: logger,
+ setOwnerRefs: setOwnerRefs,
  }
 }
 
@@ -81,13 +84,15 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
  kubeClient: factory.kubeClient,
  appmeshClient: factory.meshClient,
  labelSelector: labelSelector,
+ setOwnerRefs: factory.setOwnerRefs,
  }
  case provider == flaggerv1.AppMeshProvider:
  return &AppMeshRouter{
  logger: factory.logger,
  flaggerClient: factory.flaggerClient,
  kubeClient: factory.kubeClient,
  appmeshClient: factory.meshClient,
+ setOwnerRefs: factory.setOwnerRefs,
  }
  case provider == flaggerv1.LinkerdProvider:
  return &SmiRouter{
@@ -96,13 +101,15 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
  kubeClient: factory.kubeClient,
  smiClient: factory.meshClient,
  targetMesh: flaggerv1.LinkerdProvider,
+ setOwnerRefs: factory.setOwnerRefs,
  }
  case provider == flaggerv1.IstioProvider:
  return &IstioRouter{
  logger: factory.logger,
  flaggerClient: factory.flaggerClient,
  kubeClient: factory.kubeClient,
  istioClient: factory.meshClient,
+ setOwnerRefs: factory.setOwnerRefs,
  }
  case strings.HasPrefix(provider, flaggerv1.SMIProvider+":v1alpha1"):
  mesh := strings.TrimPrefix(provider, flaggerv1.SMIProvider+":v1alpha1:")
@@ -112,6 +119,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
  kubeClient: factory.kubeClient,
  smiClient: factory.meshClient,
  targetMesh: mesh,
+ setOwnerRefs: factory.setOwnerRefs,
  }
  case strings.HasPrefix(provider, flaggerv1.SMIProvider+":v1alpha2"):
  mesh := strings.TrimPrefix(provider, flaggerv1.SMIProvider+":v1alpha2:")
@@ -121,6 +129,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
  kubeClient: factory.kubeClient,
  smiClient: factory.meshClient,
  targetMesh: mesh,
+ setOwnerRefs: factory.setOwnerRefs,
  }
  case strings.HasPrefix(provider, flaggerv1.SMIProvider+":v1alpha3"):
  mesh := strings.TrimPrefix(provider, flaggerv1.SMIProvider+":v1alpha3:")
@@ -130,6 +139,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
  kubeClient: factory.kubeClient,
  smiClient: factory.meshClient,
  targetMesh: mesh,
+ setOwnerRefs: factory.setOwnerRefs,
  }
  case provider == flaggerv1.ContourProvider:
  return &ContourRouter{
@@ -138,29 +148,34 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
  kubeClient: factory.kubeClient,
  contourClient: factory.meshClient,
  ingressClass: factory.ingressClass,
+ setOwnerRefs: factory.setOwnerRefs,
  }
  case strings.HasPrefix(provider, flaggerv1.GlooProvider):
  return &GlooRouter{
  logger: factory.logger,
  flaggerClient: factory.flaggerClient,
  kubeClient: factory.kubeClient,
  glooClient: factory.meshClient,
+ setOwnerRefs: factory.setOwnerRefs,
  }
  case provider == flaggerv1.NGINXProvider:
  return &IngressRouter{
  logger: factory.logger,
  kubeClient: factory.kubeClient,
  annotationsPrefix: factory.ingressAnnotationsPrefix,
+ setOwnerRefs: factory.setOwnerRefs,
  }
  case provider == flaggerv1.SkipperProvider:
  return &SkipperRouter{
- logger: factory.logger,
- kubeClient: factory.kubeClient,
+ logger: factory.logger,
+ kubeClient: factory.kubeClient,
+ setOwnerRefs: factory.setOwnerRefs,
  }
  case provider == flaggerv1.TraefikProvider:
  return &TraefikRouter{
  logger: factory.logger,
  traefikClient: factory.meshClient,
+ setOwnerRefs: factory.setOwnerRefs,
  }
  case provider == flaggerv1.OsmProvider:
  return &Smiv1alpha2Router{
@@ -169,6 +184,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
  kubeClient: factory.kubeClient,
  smiClient: factory.meshClient,
  targetMesh: flaggerv1.OsmProvider,
+ setOwnerRefs: factory.setOwnerRefs,
  }
  case provider == flaggerv1.KumaProvider:
  return &KumaRouter{
@@ -182,6 +198,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
  logger: factory.logger,
  kubeClient: factory.kubeClient,
  gatewayAPIClient: factory.meshClient,
+ setOwnerRefs: factory.setOwnerRefs,
  }
  case provider == flaggerv1.KubernetesProvider:
  return &NopRouter{}
@@ -191,6 +208,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
  flaggerClient: factory.flaggerClient,
  kubeClient: factory.kubeClient,
  istioClient: factory.meshClient,
+ setOwnerRefs: factory.setOwnerRefs,
  }
  }
 }