Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request karmada-io#3475 from chaunceyjiang/graceful
Browse files Browse the repository at this point in the history
fix: graceful-eviction still works after an unexpected restart of karmada-controller-manager
karmada-bot authored and calvin committed May 4, 2023
2 parents 7843bc8 + 9592aad commit eedec14
Showing 9 changed files with 88 additions and 17 deletions.
5 changes: 5 additions & 0 deletions operator/pkg/controlplane/apiserver/apiserver.go
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@ import (
"github.com/karmada-io/karmada/operator/pkg/constants"
"github.com/karmada-io/karmada/operator/pkg/util"
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
"github.com/karmada-io/karmada/operator/pkg/util/patcher"
)

// EnsureKarmadaAPIServer creates karmada apiserver deployment and service resource
@@ -58,6 +59,8 @@ func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.K
return fmt.Errorf("error when decoding karmadaApiserver deployment: %w", err)
}

patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(apiserverDeployment)

if err := apiclient.CreateOrUpdateDeployment(client, apiserverDeployment); err != nil {
return fmt.Errorf("error when creating deployment for %s, err: %w", apiserverDeployment.Name, err)
}
@@ -113,6 +116,8 @@ func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operator
return fmt.Errorf("err when decoding karmadaApiserver deployment: %w", err)
}

patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(aggregatedAPIServerDeployment)

if err := apiclient.CreateOrUpdateDeployment(client, aggregatedAPIServerDeployment); err != nil {
return fmt.Errorf("error when creating deployment for %s, err: %w", aggregatedAPIServerDeployment.Name, err)
}
1 change: 1 addition & 0 deletions operator/pkg/controlplane/apiserver/mainfests.go
Original file line number Diff line number Diff line change
@@ -192,6 +192,7 @@ spec:
secret:
secretName: {{ .EtcdCertsSecret }}
`

// KarmadaAggregatedAPIServerService is karmada aggregated APIServer Service manifest
KarmadaAggregatedAPIServerService = `
apiVersion: v1
13 changes: 5 additions & 8 deletions operator/pkg/controlplane/controlplane.go
Original file line number Diff line number Diff line change
@@ -12,13 +12,11 @@ import (
"github.com/karmada-io/karmada/operator/pkg/constants"
"github.com/karmada-io/karmada/operator/pkg/util"
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
"github.com/karmada-io/karmada/operator/pkg/util/patcher"
)

// PatchManifest defines a function to patch deployment
type PatchManifest func(cfg *operatorv1alpha1.KarmadaComponents, deployment *appsv1.Deployment)

// EnsureControlPlaneComponent creates karmada controllerManager, kubeControllerManager, scheduler, webhook component
func EnsureControlPlaneComponent(component, name, namespace string, client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, patchManifestFunc PatchManifest) error {
func EnsureControlPlaneComponent(component, name, namespace string, client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents) error {
deployments, err := getComponentManifests(name, namespace, cfg)
if err != nil {
return err
@@ -29,10 +27,6 @@ func EnsureControlPlaneComponent(component, name, namespace string, client clien
return fmt.Errorf("no exist manifest for %s", component)
}

if patchManifestFunc != nil {
patchManifestFunc(cfg, deployment)
}

if err := apiclient.CreateOrUpdateDeployment(client, deployment); err != nil {
return fmt.Errorf("failed to create deployment resource for component %s, err: %w", component, err)
}
@@ -82,6 +76,7 @@ func getKubeControllerManagerManifest(name, namespace string, cfg *operatorv1alp
return nil, fmt.Errorf("err when decoding KubeControllerManager Deployment: %w", err)
}

patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(kcm)
return kcm, nil
}

@@ -106,6 +101,7 @@ func karmadaControllerManagerManifest(name, namespace string, cfg *operatorv1alp
return nil, fmt.Errorf("err when decoding KarmadaControllerManager Deployment: %w", err)
}

patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(kcm)
return kcm, nil
}

@@ -130,5 +126,6 @@ func karmadaSchedulerManifest(name, namespace string, cfg *operatorv1alpha1.Karm
return nil, fmt.Errorf("err when decoding KarmadaScheduler Deployment: %w", err)
}

patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(scheduler)
return scheduler, nil
}
3 changes: 3 additions & 0 deletions operator/pkg/controlplane/etcd/etcd.go
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@ import (
"github.com/karmada-io/karmada/operator/pkg/constants"
"github.com/karmada-io/karmada/operator/pkg/util"
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
"github.com/karmada-io/karmada/operator/pkg/util/patcher"
)

// EnsureKarmadaEtcd creates etcd StatefulSet and service resource.
@@ -69,6 +70,8 @@ func installKarmadaEtcd(client clientset.Interface, name, namespace string, cfg
return fmt.Errorf("error when decoding Etcd StatefulSet: %w", err)
}

patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForStatefulSet(etcdStatefulSet)

if err := apiclient.CreateOrUpdateStatefulSet(client, etcdStatefulSet); err != nil {
return fmt.Errorf("error when creating Etcd statefulset, err: %w", err)
}
3 changes: 3 additions & 0 deletions operator/pkg/controlplane/webhook/webhook.go
Original file line number Diff line number Diff line change
@@ -12,6 +12,7 @@ import (
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
"github.com/karmada-io/karmada/operator/pkg/util"
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
"github.com/karmada-io/karmada/operator/pkg/util/patcher"
)

// EnsureKarmadaWebhook creates karmada webhook deployment and service resource.
@@ -45,6 +46,8 @@ func installKarmadaWebhook(client clientset.Interface, cfg *operatorv1alpha1.Kar
return fmt.Errorf("err when decoding KarmadaWebhook Deployment: %w", err)
}

patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(webhookDeployment)

if err := apiclient.CreateOrUpdateDeployment(client, webhookDeployment); err != nil {
return fmt.Errorf("error when creating deployment for %s, err: %w", webhookDeployment.Name, err)
}
13 changes: 6 additions & 7 deletions operator/pkg/tasks/init/component.go
Original file line number Diff line number Diff line change
@@ -19,9 +19,9 @@ func NewComponentTask() workflow.Task {
Run: runComponents,
RunSubTasks: true,
Tasks: []workflow.Task{
newComponentSubTask(constants.KubeControllerManagerComponent, nil),
newComponentSubTask(constants.KarmadaControllerManagerComponent, nil),
newComponentSubTask(constants.KarmadaSchedulerComponent, nil),
newComponentSubTask(constants.KubeControllerManagerComponent),
newComponentSubTask(constants.KarmadaControllerManagerComponent),
newComponentSubTask(constants.KarmadaSchedulerComponent),
{
Name: "KarmadaWebhook",
Run: runKarmadaWebhook,
@@ -40,14 +40,14 @@ func runComponents(r workflow.RunData) error {
return nil
}

func newComponentSubTask(component string, patchManifestFunc controlplane.PatchManifest) workflow.Task {
func newComponentSubTask(component string) workflow.Task {
return workflow.Task{
Name: component,
Run: runComponentSubTask(component, patchManifestFunc),
Run: runComponentSubTask(component),
}
}

func runComponentSubTask(component string, patchManifestFunc controlplane.PatchManifest) func(r workflow.RunData) error {
func runComponentSubTask(component string) func(r workflow.RunData) error {
return func(r workflow.RunData) error {
data, ok := r.(InitData)
if !ok {
@@ -60,7 +60,6 @@ func runComponentSubTask(component string, patchManifestFunc controlplane.PatchM
data.GetNamespace(),
data.RemoteClient(),
data.Components(),
patchManifestFunc,
)
if err != nil {
return fmt.Errorf("failed to apply component %s, err: %w", component, err)
47 changes: 47 additions & 0 deletions operator/pkg/util/patcher/pather.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package patcher

import (
appsv1 "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/labels"
)

// Patcher defines various of var that need be pathed.
type Patcher struct {
labels map[string]string
annotations map[string]string
}

// NewPatcher returns a Patcher
func NewPatcher() *Patcher {
return &Patcher{}
}

// WithSettings set the labels to Patcher
func (p *Patcher) WithLabels(labels labels.Set) *Patcher {
p.labels = labels
return p
}

// WithAnnotations set annotations to Patcher
func (p *Patcher) WithAnnotations(annotations labels.Set) *Patcher {
p.annotations = annotations
return p
}

// ForDeployment patchs various of var with the manifest
func (p *Patcher) ForDeployment(deployment *appsv1.Deployment) {
deployment.Labels = labels.Merge(deployment.Labels, p.labels)
deployment.Spec.Template.Labels = labels.Merge(deployment.Spec.Template.Labels, p.labels)

deployment.Annotations = labels.Merge(deployment.Annotations, p.annotations)
deployment.Spec.Template.Annotations = labels.Merge(deployment.Spec.Template.Annotations, p.annotations)
}

// ForStatefulSet patchs various of var with the manifest
func (p *Patcher) ForStatefulSet(sts *appsv1.StatefulSet) {
sts.Labels = labels.Merge(sts.Labels, p.labels)
sts.Spec.Template.Labels = labels.Merge(sts.Spec.Template.Labels, p.labels)

sts.Annotations = labels.Merge(sts.Annotations, p.annotations)
sts.Spec.Template.Annotations = labels.Merge(sts.Spec.Template.Annotations, p.annotations)
}
Original file line number Diff line number Diff line change
@@ -84,7 +84,15 @@ func (c *CRBGracefulEvictionController) syncBinding(binding *workv1alpha2.Cluste
// SetupWithManager creates a controller and register to controller manager.
func (c *CRBGracefulEvictionController) SetupWithManager(mgr controllerruntime.Manager) error {
clusterResourceBindingPredicateFn := predicate.Funcs{
CreateFunc: func(createEvent event.CreateEvent) bool { return false },
CreateFunc: func(createEvent event.CreateEvent) bool {
newObj := createEvent.Object.(*workv1alpha2.ClusterResourceBinding)
if len(newObj.Spec.GracefulEvictionTasks) == 0 {
return false
}
// When the current component is restarted and there are still tasks in the
// GracefulEvictionTasks queue, we need to continue the procession.
return newObj.Status.SchedulerObservedGeneration == newObj.Generation
},
UpdateFunc: func(updateEvent event.UpdateEvent) bool {
newObj := updateEvent.ObjectNew.(*workv1alpha2.ClusterResourceBinding)

Original file line number Diff line number Diff line change
@@ -84,7 +84,15 @@ func (c *RBGracefulEvictionController) syncBinding(binding *workv1alpha2.Resourc
// SetupWithManager creates a controller and register to controller manager.
func (c *RBGracefulEvictionController) SetupWithManager(mgr controllerruntime.Manager) error {
resourceBindingPredicateFn := predicate.Funcs{
CreateFunc: func(createEvent event.CreateEvent) bool { return false },
CreateFunc: func(createEvent event.CreateEvent) bool {
newObj := createEvent.Object.(*workv1alpha2.ResourceBinding)
if len(newObj.Spec.GracefulEvictionTasks) == 0 {
return false
}
// When the current component is restarted and there are still tasks in the
// GracefulEvictionTasks queue, we need to continue the procession.
return newObj.Status.SchedulerObservedGeneration == newObj.Generation
},
UpdateFunc: func(updateEvent event.UpdateEvent) bool {
newObj := updateEvent.ObjectNew.(*workv1alpha2.ResourceBinding)

0 comments on commit eedec14

Please sign in to comment.