Skip to content

Commit

Permalink
Update metadata during subsequent promote
Browse files Browse the repository at this point in the history 
Signed-off-by: Karl Heins <karlheins@northwesternmutual.com>

Support updating primary Deployment/DaemonSet/HPA/Service labels and annotations after first-time rollout
  • Loading branch information
@kh34
kh34 committed Jan 18, 2022
1 parent 5776f0b commit 0aa9eee
Show file tree
Hide file tree
Showing 7 changed files with 235 additions and 111 deletions.
99 changes: 58 additions & 41 deletions pkg/canary/daemonset_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/util/retry"

flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
clientset "github.com/fluxcd/flagger/pkg/client/clientset/versioned"
Expand Down Expand Up @@ -119,59 +120,75 @@ func (c *DaemonSetController) Promote(cd *flaggerv1.Canary) error {
targetName := cd.Spec.TargetRef.Name
primaryName := fmt.Sprintf("%s-primary", targetName)

canary, err := c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("damonset %s.%s get query error: %v", targetName, cd.Namespace, err)
}
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
canary, err := c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("damonset %s.%s get query error: %v", targetName, cd.Namespace, err)
}

label, labelValue, err := c.getSelectorLabel(canary)
primaryLabelValue := fmt.Sprintf("%s-primary", labelValue)
if err != nil {
return fmt.Errorf("getSelectorLabel failed: %w", err)
}
label, labelValue, err := c.getSelectorLabel(canary)
primaryLabelValue := fmt.Sprintf("%s-primary", labelValue)
if err != nil {
return fmt.Errorf("getSelectorLabel failed: %w", err)
}

primary, err := c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("daemonset %s.%s get query error: %w", primaryName, cd.Namespace, err)
}
primary, err := c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("daemonset %s.%s get query error: %w", primaryName, cd.Namespace, err)
}

// promote secrets and config maps
configRefs, err := c.configTracker.GetTargetConfigs(cd)
if err != nil {
return fmt.Errorf("GetTargetConfigs failed: %w", err)
}
if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs, c.includeLabelPrefix); err != nil {
return fmt.Errorf("CreatePrimaryConfigs failed: %w", err)
}
// promote secrets and config maps
configRefs, err := c.configTracker.GetTargetConfigs(cd)
if err != nil {
return fmt.Errorf("GetTargetConfigs failed: %w", err)
}
if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs, c.includeLabelPrefix); err != nil {
return fmt.Errorf("CreatePrimaryConfigs failed: %w", err)
}

primaryCopy := primary.DeepCopy()
primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds
primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit
primaryCopy.Spec.UpdateStrategy = canary.Spec.UpdateStrategy
primaryCopy := primary.DeepCopy()
primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds
primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit
primaryCopy.Spec.UpdateStrategy = canary.Spec.UpdateStrategy

// update spec with primary secrets and config maps
primaryCopy.Spec.Template.Spec = c.configTracker.ApplyPrimaryConfigs(canary.Spec.Template.Spec, configRefs)
// update spec with primary secrets and config maps
primaryCopy.Spec.Template.Spec = c.configTracker.ApplyPrimaryConfigs(canary.Spec.Template.Spec, configRefs)

// ignore `daemonSetScaleDownNodeSelector` node selector
for key := range daemonSetScaleDownNodeSelector {
delete(primaryCopy.Spec.Template.Spec.NodeSelector, key)
}
// ignore `daemonSetScaleDownNodeSelector` node selector
for key := range daemonSetScaleDownNodeSelector {
delete(primaryCopy.Spec.Template.Spec.NodeSelector, key)
}

// update pod annotations to ensure a rolling update
annotations, err := makeAnnotations(canary.Spec.Template.Annotations)
if err != nil {
return fmt.Errorf("makeAnnotations failed: %w", err)
}
// update pod annotations to ensure a rolling update
annotations, err := makeAnnotations(canary.Spec.Template.Annotations)
if err != nil {
return fmt.Errorf("makeAnnotations failed: %w", err)
}

primaryCopy.Spec.Template.Annotations = annotations
primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryLabelValue, label)
primaryCopy.Spec.Template.Annotations = annotations
primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryLabelValue, label)

// apply update
_, err = c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{})
// update deploy annotations
primaryCopy.ObjectMeta.Annotations = make(map[string]string)
for k, v := range canary.ObjectMeta.Annotations {
primaryCopy.ObjectMeta.Annotations[k] = v
}
// update deploy labels
primaryCopy.ObjectMeta.Labels = make(map[string]string)
filteredLabels := includeLabelsByPrefix(canary.ObjectMeta.Labels, c.includeLabelPrefix)
for k, v := range filteredLabels {
primaryCopy.ObjectMeta.Labels[k] = v
}

// apply update
_, err = c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{})
return err
})
if err != nil {
return fmt.Errorf("updating daemonset %s.%s template spec failed: %w",
primaryCopy.GetName(), primaryCopy.Namespace, err)
primaryName, cd.Namespace, err)
}

return nil
}

Expand Down
8 changes: 8 additions & 0 deletions pkg/canary/daemonset_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,14 @@ func TestDaemonSetController_Promote(t *testing.T) {
sourceImage := dae2.Spec.Template.Spec.Containers[0].Image
assert.Equal(t, primaryImage, sourceImage)

daePrimaryLabels := daePrimary.ObjectMeta.Labels
daeSourceLabels := dae2.ObjectMeta.Labels
assert.Equal(t, daeSourceLabels["app.kubernetes.io/test-label-1"], daePrimaryLabels["app.kubernetes.io/test-label-1"])

daePrimaryAnnotations := daePrimary.ObjectMeta.Annotations
daeSourceAnnotations := dae2.ObjectMeta.Annotations
assert.Equal(t, daeSourceAnnotations["app.kubernetes.io/test-annotation-1"], daePrimaryAnnotations["app.kubernetes.io/test-annotation-1"])

configPrimary, err := mocks.kubeClient.CoreV1().ConfigMaps("default").Get(context.TODO(), "podinfo-config-env-primary", metav1.GetOptions{})
if assert.NoError(t, err) {
assert.Equal(t, configPrimary.Data["color"], config2.Data["color"])
Expand Down
12 changes: 11 additions & 1 deletion pkg/canary/daemonset_fixture_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,12 @@ func newDaemonSetControllerTestPodInfo(dc daemonsetConfigs) *appsv1.DaemonSet {
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: dc.name,
Annotations: map[string]string{
"test-annotation-1": "test-annotation-value-1",
},
Labels: map[string]string{
"test-label-1": "test-label-value-1",
},
},
Spec: appsv1.DaemonSetSpec{
Selector: &metav1.LabelSelector{
Expand All @@ -379,7 +385,11 @@ func newDaemonSetControllerTestPodInfo(dc daemonsetConfigs) *appsv1.DaemonSet {
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
dc.label: dc.labelValue,
dc.label: dc.labelValue,
"test-label-1": "test-label-value-1",
},
Annotations: map[string]string{
"test-annotation-1": "test-annotation-value-1",
},
},
Spec: corev1.PodSpec{
Expand Down
130 changes: 84 additions & 46 deletions pkg/canary/deployment_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/util/retry"

flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
clientset "github.com/fluxcd/flagger/pkg/client/clientset/versioned"
Expand Down Expand Up @@ -84,54 +85,69 @@ func (c *DeploymentController) Promote(cd *flaggerv1.Canary) error {
targetName := cd.Spec.TargetRef.Name
primaryName := fmt.Sprintf("%s-primary", targetName)

canary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("deployment %s.%s get query error: %w", targetName, cd.Namespace, err)
}
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
canary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("deployment %s.%s get query error: %w", targetName, cd.Namespace, err)
}

label, labelValue, err := c.getSelectorLabel(canary)
primaryLabelValue := fmt.Sprintf("%s-primary", labelValue)
if err != nil {
return fmt.Errorf("getSelectorLabel failed: %w", err)
}
label, labelValue, err := c.getSelectorLabel(canary)
primaryLabelValue := fmt.Sprintf("%s-primary", labelValue)
if err != nil {
return fmt.Errorf("getSelectorLabel failed: %w", err)
}

primary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("deployment %s.%s get query error: %w", primaryName, cd.Namespace, err)
}
primary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("deployment %s.%s get query error: %w", primaryName, cd.Namespace, err)
}

// promote secrets and config maps
configRefs, err := c.configTracker.GetTargetConfigs(cd)
if err != nil {
return fmt.Errorf("GetTargetConfigs failed: %w", err)
}
if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs, c.includeLabelPrefix); err != nil {
return fmt.Errorf("CreatePrimaryConfigs failed: %w", err)
}
// promote secrets and config maps
configRefs, err := c.configTracker.GetTargetConfigs(cd)
if err != nil {
return fmt.Errorf("GetTargetConfigs failed: %w", err)
}
if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs, c.includeLabelPrefix); err != nil {
return fmt.Errorf("CreatePrimaryConfigs failed: %w", err)
}

primaryCopy := primary.DeepCopy()
primaryCopy.Spec.ProgressDeadlineSeconds = canary.Spec.ProgressDeadlineSeconds
primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds
primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit
primaryCopy.Spec.Strategy = canary.Spec.Strategy
primaryCopy := primary.DeepCopy()
primaryCopy.Spec.ProgressDeadlineSeconds = canary.Spec.ProgressDeadlineSeconds
primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds
primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit
primaryCopy.Spec.Strategy = canary.Spec.Strategy

// update spec with primary secrets and config maps
primaryCopy.Spec.Template.Spec = c.getPrimaryDeploymentTemplateSpec(canary, configRefs)
// update spec with primary secrets and config maps
primaryCopy.Spec.Template.Spec = c.getPrimaryDeploymentTemplateSpec(canary, configRefs)

// update pod annotations to ensure a rolling update
annotations, err := makeAnnotations(canary.Spec.Template.Annotations)
if err != nil {
return fmt.Errorf("makeAnnotations failed: %w", err)
}
// update pod annotations to ensure a rolling update
podAnnotations, err := makeAnnotations(canary.Spec.Template.Annotations)
if err != nil {
return fmt.Errorf("makeAnnotations for podAnnotations failed: %w", err)
}

primaryCopy.Spec.Template.Annotations = podAnnotations
primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryLabelValue, label)

primaryCopy.Spec.Template.Annotations = annotations
primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryLabelValue, label)
// update deploy annotations
primaryCopy.ObjectMeta.Annotations = make(map[string]string)
for k, v := range canary.ObjectMeta.Annotations {
primaryCopy.ObjectMeta.Annotations[k] = v
}
// update deploy labels
primaryCopy.ObjectMeta.Labels = make(map[string]string)
filteredLabels := includeLabelsByPrefix(canary.ObjectMeta.Labels, c.includeLabelPrefix)
for k, v := range filteredLabels {
primaryCopy.ObjectMeta.Labels[k] = v
}

// apply update
_, err = c.kubeClient.AppsV1().Deployments(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{})
// apply update
_, err = c.kubeClient.AppsV1().Deployments(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{})
return err
})
if err != nil {
return fmt.Errorf("updating deployment %s.%s template spec failed: %w",
primaryCopy.GetName(), primaryCopy.Namespace, err)
primaryName, cd.Namespace, err)
}

// update HPA
Expand Down Expand Up @@ -364,18 +380,40 @@ func (c *DeploymentController) reconcilePrimaryHpa(cd *flaggerv1.Canary, init bo
if !init && primaryHpa != nil {
diffMetrics := cmp.Diff(hpaSpec.Metrics, primaryHpa.Spec.Metrics)
diffBehavior := cmp.Diff(hpaSpec.Behavior, primaryHpa.Spec.Behavior)
if diffMetrics != "" || diffBehavior != "" || int32Default(hpaSpec.MinReplicas) != int32Default(primaryHpa.Spec.MinReplicas) || hpaSpec.MaxReplicas != primaryHpa.Spec.MaxReplicas {
diffLabels := cmp.Diff(hpa.ObjectMeta.Labels, primaryHpa.ObjectMeta.Labels)
diffAnnotations := cmp.Diff(hpa.ObjectMeta.Annotations, primaryHpa.ObjectMeta.Annotations)
if diffMetrics != "" || diffBehavior != "" || diffLabels != "" || diffAnnotations != "" || int32Default(hpaSpec.MinReplicas) != int32Default(primaryHpa.Spec.MinReplicas) || hpaSpec.MaxReplicas != primaryHpa.Spec.MaxReplicas {
fmt.Println(diffMetrics, diffBehavior, hpaSpec.MinReplicas, primaryHpa.Spec.MinReplicas, hpaSpec.MaxReplicas, primaryHpa.Spec.MaxReplicas)
hpaClone := primaryHpa.DeepCopy()
hpaClone.Spec.MaxReplicas = hpaSpec.MaxReplicas
hpaClone.Spec.MinReplicas = hpaSpec.MinReplicas
hpaClone.Spec.Metrics = hpaSpec.Metrics
hpaClone.Spec.Behavior = hpaSpec.Behavior

_, err := c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Update(context.TODO(), hpaClone, metav1.UpdateOptions{})
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
primaryHpa, err := c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Get(context.TODO(), primaryHpaName, metav1.GetOptions{})
if err != nil {
return err
}
hpaClone := primaryHpa.DeepCopy()
hpaClone.Spec.MaxReplicas = hpaSpec.MaxReplicas
hpaClone.Spec.MinReplicas = hpaSpec.MinReplicas
hpaClone.Spec.Metrics = hpaSpec.Metrics
hpaClone.Spec.Behavior = hpaSpec.Behavior

// update hpa annotations
hpaClone.ObjectMeta.Annotations = make(map[string]string)
for k, v := range hpa.ObjectMeta.Annotations {
hpaClone.ObjectMeta.Annotations[k] = v
}
// update hpa labels
hpaClone.ObjectMeta.Labels = make(map[string]string)
filteredLabels := includeLabelsByPrefix(hpa.ObjectMeta.Labels, c.includeLabelPrefix)
for k, v := range filteredLabels {
hpaClone.ObjectMeta.Labels[k] = v
}

_, err = c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Update(context.TODO(), hpaClone, metav1.UpdateOptions{})
return err
})
if err != nil {
return fmt.Errorf("updating HorizontalPodAutoscaler %s.%s failed: %w",
hpaClone.Name, hpaClone.Namespace, err)
primaryHpa.Name, primaryHpa.Namespace, err)
}
c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).
Infof("HorizontalPodAutoscaler %s.%s updated", primaryHpa.GetName(), cd.Namespace)
Expand Down
16 changes: 16 additions & 0 deletions pkg/canary/deployment_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,14 @@ func TestDeploymentController_Promote(t *testing.T) {
sourceImage := dep2.Spec.Template.Spec.Containers[0].Image
assert.Equal(t, sourceImage, primaryImage)

depPrimaryLabels := depPrimary.ObjectMeta.Labels
depSourceLabels := dep2.ObjectMeta.Labels
assert.Equal(t, depSourceLabels["app.kubernetes.io/test-label-1"], depPrimaryLabels["app.kubernetes.io/test-label-1"])

depPrimaryAnnotations := depPrimary.ObjectMeta.Annotations
depSourceAnnotations := dep2.ObjectMeta.Annotations
assert.Equal(t, depSourceAnnotations["app.kubernetes.io/test-annotation-1"], depPrimaryAnnotations["app.kubernetes.io/test-annotation-1"])

configPrimary, err := mocks.kubeClient.CoreV1().ConfigMaps("default").Get(context.TODO(), "podinfo-config-env-primary", metav1.GetOptions{})
require.NoError(t, err)
assert.Equal(t, config2.Data["color"], configPrimary.Data["color"])
Expand All @@ -117,6 +125,14 @@ func TestDeploymentController_Promote(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, int32(2), hpaPrimary.Spec.MaxReplicas)

hpaPrimaryLabels := hpaPrimary.ObjectMeta.Labels
hpaSourceLabels := hpa.ObjectMeta.Labels
assert.Equal(t, hpaSourceLabels["app.kubernetes.io/test-label-1"], hpaPrimaryLabels["app.kubernetes.io/test-label-1"])

hpaPrimaryAnnotations := hpaPrimary.ObjectMeta.Annotations
hpaSourceAnnotations := hpa.ObjectMeta.Annotations
assert.Equal(t, hpaSourceAnnotations["app.kubernetes.io/test-annotation-1"], hpaPrimaryAnnotations["app.kubernetes.io/test-annotation-1"])

value := depPrimary.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm.LabelSelector.MatchExpressions[0].Values[0]
assert.Equal(t, "podinfo-primary", value)

Expand Down
Loading

0 comments on commit 0aa9eee

Please sign in to comment.