From 1e4643a708bc14a655ab297ac4594951fc0cfb59 Mon Sep 17 00:00:00 2001 From: Kai Kummerer Date: Tue, 13 Jun 2023 10:10:56 +0200 Subject: [PATCH 01/29] Add revision as annotation to LBM --- .../loadbalancerset/loadbalancerset_controller.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go index f1662dc5..7bfdb7fb 100644 --- a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go +++ b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go @@ -309,6 +309,7 @@ func isMachineReady(machine yawolv1beta1.LoadBalancerMachine) bool { func (r *LoadBalancerSetReconciler) createMachine(ctx context.Context, set *yawolv1beta1.LoadBalancerSet) error { machineLabels := r.getMachineLabelsFromSet(set) + revision := set.Annotations[helper.RevisionAnnotation] machine := yawolv1beta1.LoadBalancerMachine{ ObjectMeta: v1.ObjectMeta{ Name: set.Name + "-" + randomString(5), @@ -322,6 +323,9 @@ func (r *LoadBalancerSetReconciler) createMachine(ctx context.Context, set *yawo }, }, Labels: machineLabels, + Annotations: map[string]string{ + helper.RevisionAnnotation: revision, + }, }, Spec: set.Spec.Template.Spec, } From 83af865c13683057dae6d8ae33eb6e984178844d Mon Sep 17 00:00:00 2001 From: Maximilian Geberl <48486938+dergeberl@users.noreply.github.com> Date: Tue, 13 Jun 2023 11:36:26 +0200 Subject: [PATCH 02/29] add yawol keepalived file to change prio of keepalived --- .../yawollet/loadbalancer_controller.go | 4 +++ .../yawollet/loadbalancer_controller_test.go | 34 +++++++++++++++++++ internal/helper/const.go | 1 + internal/helper/loadbalancermachine.go | 9 +++++ internal/helper/yawollet.go | 21 ++++++++++++ 5 files changed, 69 insertions(+) diff --git a/controllers/yawollet/loadbalancer_controller.go b/controllers/yawollet/loadbalancer_controller.go index a030e17c..eee26295 100644 --- a/controllers/yawollet/loadbalancer_controller.go +++ b/controllers/yawollet/loadbalancer_controller.go @@ -93,6 +93,10 @@ func (r *LoadBalancerReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } + if err := helper.UpdateKeepalivedFile(lb, lbm); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: time.Duration(r.RequeueTime) * time.Second}, reconcileError } diff --git a/controllers/yawollet/loadbalancer_controller_test.go b/controllers/yawollet/loadbalancer_controller_test.go index ff008d3d..b6237feb 100644 --- a/controllers/yawollet/loadbalancer_controller_test.go +++ b/controllers/yawollet/loadbalancer_controller_test.go @@ -2,8 +2,11 @@ package controllers import ( "context" + "errors" "io" + "io/fs" "net/http" + "os" "os/exec" "strings" "time" @@ -46,6 +49,9 @@ var _ = Describe("check loadbalancer reconcile", Serial, Ordered, func() { ObjectMeta: metav1.ObjectMeta{ Name: nameLBM, Namespace: Namespace, + Annotations: map[string]string{ + helper.RevisionAnnotation: "1", + }, }, Spec: yawolv1beta1.LoadBalancerMachineSpec{ PortID: "", @@ -58,6 +64,9 @@ var _ = Describe("check loadbalancer reconcile", Serial, Ordered, func() { ObjectMeta: metav1.ObjectMeta{ Name: nameLB, Namespace: Namespace, + Annotations: map[string]string{ + helper.RevisionAnnotation: "1", + }, }, Spec: yawolv1beta1.LoadBalancerSpec{ Selector: metav1.LabelSelector{}, @@ -441,7 +450,32 @@ var _ = Describe("check loadbalancer reconcile", Serial, Ordered, func() { ) }) }) + When("lb and lbm revision annotation are the same", func() { + It("should create yawolKeepalivedFile", func() { + Eventually(func() error { + _, err := os.Stat(helper.YawolKeepalivedFile) + return err + }, TIMEOUT, INTERVAL).Should(Succeed()) + }) + }) + When("lb and lbm revision annotation are the same", func() { + BeforeEach(func() { + lb.Annotations = map[string]string{ + helper.RevisionAnnotation: "2", + } + }) + It("should not create or delete yawolKeepalivedFile", func() { + Eventually(func() error { + _, err := os.Stat(helper.YawolKeepalivedFile) + if err == nil || !errors.Is(err, fs.ErrNotExist) { + return errors.New("keepalived file still exists") + } + return nil + }, TIMEOUT, INTERVAL).Should(Succeed()) + + }) + }) When("envoy gets killed and restarted", func() { It("should set the correct conditions", func() { By("killing the envoy process") diff --git a/internal/helper/const.go b/internal/helper/const.go index 69d2cdff..da3d821b 100644 --- a/internal/helper/const.go +++ b/internal/helper/const.go @@ -6,6 +6,7 @@ const ( OpenstackReconcileTime = 5 * time.Minute DefaultRequeueTime = 10 * time.Millisecond RevisionAnnotation = "loadbalancer.yawol.stackit.cloud/revision" + YawolKeepalivedFile = "/tmp/yawolKeepalivedLastSet" HashLabel = "lbm-template-hash" LoadBalancerKind = "LoadBalancer" VRRPInstanceName = "ENVOY" diff --git a/internal/helper/loadbalancermachine.go b/internal/helper/loadbalancermachine.go index 5a346601..536a83f6 100644 --- a/internal/helper/loadbalancermachine.go +++ b/internal/helper/loadbalancermachine.go @@ -315,6 +315,11 @@ vrrp_track_process envoy { weight 100 } +vrrp_track_file yawolfile { + file "` + YawolKeepalivedFile + `" + weight 10 +} + vrrp_instance ` + VRRPInstanceName + ` { state BACKUP interface eth0 @@ -335,6 +340,10 @@ vrrp_instance ` + VRRPInstanceName + ` { track_process { envoy } + + track_file { + yawolfile + } }` } diff --git a/internal/helper/yawollet.go b/internal/helper/yawollet.go index b173b46b..30d69276 100644 --- a/internal/helper/yawollet.go +++ b/internal/helper/yawollet.go @@ -1,7 +1,9 @@ package helper import ( + "errors" "fmt" + "io/fs" "net" "os" "os/exec" @@ -888,3 +890,22 @@ func EnableAdHocDebugging( "Please make sure to disable debug access once you are finished and to roll all LoadBalancerMachines", lbmName) return nil } + +func UpdateKeepalivedFile(lb *yawolv1beta1.LoadBalancer, lbm *yawolv1beta1.LoadBalancerMachine) error { + if lbmIsLatestRevision(lb, lbm) { + //file should exist + f, err := os.Create(YawolKeepalivedFile) + defer f.Close() + return err + } + //file should not exist + err := os.Remove(YawolKeepalivedFile) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + return nil +} + +func lbmIsLatestRevision(lb *yawolv1beta1.LoadBalancer, lbm *yawolv1beta1.LoadBalancerMachine) bool { + return lb.Annotations[RevisionAnnotation] == lbm.Annotations[RevisionAnnotation] +} From 623f3fa59f8407720474339d4d3b53a32864a3e1 Mon Sep 17 00:00:00 2001 From: Kai Kummerer Date: Tue, 13 Jun 2023 11:51:57 +0200 Subject: [PATCH 03/29] WIP --- api/v1beta1/loadbalancerset_types.go | 3 + api/v1beta1/zz_generated.deepcopy.go | 7 ++ .../yawol.stackit.cloud_loadbalancersets.yaml | 69 +++++++++++++++++++ .../loadbalancerset_controller.go | 38 +++++++++- internal/helper/const.go | 1 + 5 files changed, 115 insertions(+), 3 deletions(-) diff --git a/api/v1beta1/loadbalancerset_types.go b/api/v1beta1/loadbalancerset_types.go index aabd3c32..25447f40 100644 --- a/api/v1beta1/loadbalancerset_types.go +++ b/api/v1beta1/loadbalancerset_types.go @@ -48,6 +48,9 @@ type LoadBalancerSetStatus struct { // AvailableReplicas are the current running replicas. // +optional AvailableReplicas *int `json:"availableReplicas,omitempty"` + // Conditions contains condition information for a LoadBalancerSet. + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` // ReadyReplicas are the current ready replicas. // +optional ReadyReplicas *int `json:"readyReplicas,omitempty"` diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 4530875a..ae9f73fa 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -529,6 +529,13 @@ func (in *LoadBalancerSetStatus) DeepCopyInto(out *LoadBalancerSetStatus) { *out = new(int) **out = **in } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.ReadyReplicas != nil { in, out := &in.ReadyReplicas, &out.ReadyReplicas *out = new(int) diff --git a/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml b/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml index 6dd2ac39..d82ec301 100644 --- a/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml +++ b/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml @@ -289,6 +289,75 @@ spec: availableReplicas: description: AvailableReplicas are the current running replicas. type: integer + conditions: + description: Conditions contains condition information for a LoadBalancerSet. + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array readyReplicas: description: ReadyReplicas are the current ready replicas. type: integer diff --git a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go index 7bfdb7fb..ab53a997 100644 --- a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go +++ b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go @@ -13,7 +13,7 @@ import ( helpermetrics "github.com/stackitcloud/yawol/internal/metrics" "github.com/go-logr/logr" - v12 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -75,6 +75,7 @@ func (r *LoadBalancerSetReconciler) Reconcile(ctx context.Context, req ctrl.Requ var ( readyMachineCount int deletedMachineCount int + setContainsMaster bool ) for i := range childMachines.Items { @@ -82,13 +83,17 @@ func (r *LoadBalancerSetReconciler) Reconcile(ctx context.Context, req ctrl.Requ deletedMachineCount++ continue } + if isMachineMaster(childMachines.Items[i]) { + setContainsMaster = true + } + if isMachineReady(childMachines.Items[i]) { readyMachineCount++ continue } } - if res, err := r.reconcileStatus(ctx, &set, readyMachineCount); err != nil || res.Requeue || res.RequeueAfter != 0 { + if res, err := r.reconcileStatus(ctx, &set, readyMachineCount, setContainsMaster); err != nil || res.Requeue || res.RequeueAfter != 0 { return res, err } @@ -146,6 +151,7 @@ func (r *LoadBalancerSetReconciler) reconcileStatus( ctx context.Context, set *yawolv1beta1.LoadBalancerSet, readyMachinesCount int, + setContainsMaster bool, ) (ctrl.Result, error) { // Write replicas into status if set.Status.Replicas == nil || *set.Status.Replicas != set.Spec.Replicas { @@ -169,6 +175,19 @@ func (r *LoadBalancerSetReconciler) reconcileStatus( return ctrl.Result{RequeueAfter: time.Second * 2}, nil } + // Write set contains master condition + if setContainsMaster { + + + r.patchLoadBalancerSetStatus(ctx, set, yawolv1beta1.LoadBalancerSetStatus{ + Conditions: []v1.Condition{ + v1.Condition{ + Type: helper.ContainsKeepAlivedMaster, + }, + }, + }) + } + return ctrl.Result{}, nil } @@ -307,6 +326,19 @@ func isMachineReady(machine yawolv1beta1.LoadBalancerMachine) bool { return true } +func isMachineMaster(machine yawolv1beta1.LoadBalancerMachine) bool { + if machine.Status.Conditions != nil { + for _, condition := range *machine.Status.Conditions { + if condition.Type == corev1.NodeConditionType(helper.KeepalivedMaster) { + if condition.Status == corev1.ConditionTrue { + return true + } + } + } + } + return false +} + func (r *LoadBalancerSetReconciler) createMachine(ctx context.Context, set *yawolv1beta1.LoadBalancerSet) error { machineLabels := r.getMachineLabelsFromSet(set) revision := set.Annotations[helper.RevisionAnnotation] @@ -351,7 +383,7 @@ func (r *LoadBalancerSetReconciler) deleteAllMachines(ctx context.Context, set * if err := r.Client.Delete(ctx, &childMachines.Items[i]); err != nil { return err } - r.Recorder.Event(set, v12.EventTypeNormal, "Deleted", "Deleted loadBalancerMachine "+childMachines.Items[i].Name) + r.Recorder.Event(set, corev1.EventTypeNormal, "Deleted", "Deleted loadBalancerMachine "+childMachines.Items[i].Name) } r.Log.Info("finished cleaning up old loadBalancerMachines") return nil diff --git a/internal/helper/const.go b/internal/helper/const.go index da3d821b..8e33b2f0 100644 --- a/internal/helper/const.go +++ b/internal/helper/const.go @@ -10,4 +10,5 @@ const ( HashLabel = "lbm-template-hash" LoadBalancerKind = "LoadBalancer" VRRPInstanceName = "ENVOY" + ContainsKeepAlivedMaster = "ContainsKeepAlivedMaster" ) From 98ebd341d2ecf4cdbede10b1b4791b57f54c2cbe Mon Sep 17 00:00:00 2001 From: Kai Kummerer Date: Tue, 13 Jun 2023 13:06:02 +0200 Subject: [PATCH 04/29] Write LBM ContainsKeepAlivedMaster condition --- .../loadbalancerset_controller.go | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go index ab53a997..2ea1ac30 100644 --- a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go +++ b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "encoding/json" "fmt" + "strconv" "time" yawolv1beta1 "github.com/stackitcloud/yawol/api/v1beta1" @@ -75,7 +76,7 @@ func (r *LoadBalancerSetReconciler) Reconcile(ctx context.Context, req ctrl.Requ var ( readyMachineCount int deletedMachineCount int - setContainsMaster bool + setContainsMaster bool ) for i := range childMachines.Items { @@ -176,16 +177,29 @@ func (r *LoadBalancerSetReconciler) reconcileStatus( } // Write set contains master condition - if setContainsMaster { - + var conditions []v1.Condition + if set.Status.Conditions != nil { + conditions = set.Status.Conditions + for _, condition := range conditions { + if condition.Type == helper.ContainsKeepAlivedMaster { + if condition.Status == v1.ConditionStatus(strconv.FormatBool(setContainsMaster)) { + return ctrl.Result{}, nil + } + } + } + } else { + conditions = []v1.Condition{} + } - r.patchLoadBalancerSetStatus(ctx, set, yawolv1beta1.LoadBalancerSetStatus{ - Conditions: []v1.Condition{ - v1.Condition{ - Type: helper.ContainsKeepAlivedMaster, - }, - }, - }) + status := strconv.FormatBool(setContainsMaster) + lastTransitionTime := v1.Time{Time: time.Now()} + conditions = append(conditions, v1.Condition{ + Type: helper.ContainsKeepAlivedMaster, + Status: v1.ConditionStatus(status), + LastTransitionTime: lastTransitionTime, + }) + if err := r.patchLoadBalancerSetStatus(ctx, set, yawolv1beta1.LoadBalancerSetStatus{Conditions: conditions}); err != nil { + return ctrl.Result{}, err } return ctrl.Result{}, nil From 263ccc5f9c51a7d82ef39266f327379a7c39dc82 Mon Sep 17 00:00:00 2001 From: Kai Kummerer Date: Tue, 13 Jun 2023 16:51:10 +0200 Subject: [PATCH 05/29] Add tests for set contains master condition & refactor --- .../loadbalancerset_controller.go | 49 ++++++++++--------- .../loadbalancerset_controller_test.go | 24 +++++++++ internal/helper/const.go | 16 +++--- 3 files changed, 59 insertions(+), 30 deletions(-) diff --git a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go index 2ea1ac30..f0ecea12 100644 --- a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go +++ b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go @@ -5,7 +5,7 @@ import ( "crypto/rand" "encoding/json" "fmt" - "strconv" + "reflect" "time" yawolv1beta1 "github.com/stackitcloud/yawol/api/v1beta1" @@ -177,27 +177,34 @@ func (r *LoadBalancerSetReconciler) reconcileStatus( } // Write set contains master condition - var conditions []v1.Condition - if set.Status.Conditions != nil { - conditions = set.Status.Conditions - for _, condition := range conditions { - if condition.Type == helper.ContainsKeepAlivedMaster { - if condition.Status == v1.ConditionStatus(strconv.FormatBool(setContainsMaster)) { - return ctrl.Result{}, nil - } - } + status := v1.ConditionFalse + reason := "NoKeepalivedMasterInSet" + + if setContainsMaster { + status = v1.ConditionTrue + reason = "KeepalivedMasterInSet" + } + + transitionTime := v1.Now() + for _, condition := range set.Status.Conditions { + if condition.Type == helper.ContainsKeepalivedMaster && condition.Status == status { + transitionTime = condition.LastTransitionTime } - } else { - conditions = []v1.Condition{} } - status := strconv.FormatBool(setContainsMaster) - lastTransitionTime := v1.Time{Time: time.Now()} - conditions = append(conditions, v1.Condition{ - Type: helper.ContainsKeepAlivedMaster, - Status: v1.ConditionStatus(status), - LastTransitionTime: lastTransitionTime, - }) + conditions := []v1.Condition{ + { + Type: helper.ContainsKeepalivedMaster, + Status: status, + LastTransitionTime: transitionTime, + Reason: reason, + }, + } + + if reflect.DeepEqual(set.Status.Conditions, conditions) { + return ctrl.Result{}, nil + } + if err := r.patchLoadBalancerSetStatus(ctx, set, yawolv1beta1.LoadBalancerSetStatus{Conditions: conditions}); err != nil { return ctrl.Result{}, err } @@ -344,9 +351,7 @@ func isMachineMaster(machine yawolv1beta1.LoadBalancerMachine) bool { if machine.Status.Conditions != nil { for _, condition := range *machine.Status.Conditions { if condition.Type == corev1.NodeConditionType(helper.KeepalivedMaster) { - if condition.Status == corev1.ConditionTrue { - return true - } + return condition.Status == corev1.ConditionTrue } } } diff --git a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go index bf598674..0186777d 100644 --- a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go +++ b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go @@ -106,6 +106,30 @@ var _ = Describe("LoadBalancerSet controller", Serial, Ordered, func() { machine := getChildMachines(ctx, &setStub)[0] Eventually(isValidMachine(&setStub, &machine)).Should(BeTrue()) }) + It("Should eventually have a master", func() { + By("setting LBM as master") + machine := getChildMachines(ctx, &setStub)[0] + machine.Status.Conditions = &[]v1.NodeCondition{ + { + Type: v1.NodeConditionType(helper.KeepalivedMaster), + Status: v1.ConditionTrue, + Reason: "KeepalivedStatus", + }, + } + Expect(k8sClient.Status().Update(ctx, &machine)).To(Succeed()) + + Eventually(func() metav1.ConditionStatus { + var set yawolv1beta1.LoadBalancerSet + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(&setStub), &set)).To(Succeed()) + Expect(set.Status.Conditions).To(Not(BeNil())) + for _, condition := range set.Status.Conditions { + if condition.Type == helper.ContainsKeepalivedMaster { + return condition.Status + } + } + return metav1.ConditionFalse + }, timeout, interval).Should(Equal(metav1.ConditionTrue)) + }) }) // patch diff --git a/internal/helper/const.go b/internal/helper/const.go index 8e33b2f0..3ab377ef 100644 --- a/internal/helper/const.go +++ b/internal/helper/const.go @@ -3,12 +3,12 @@ package helper import "time" const ( - OpenstackReconcileTime = 5 * time.Minute - DefaultRequeueTime = 10 * time.Millisecond - RevisionAnnotation = "loadbalancer.yawol.stackit.cloud/revision" - YawolKeepalivedFile = "/tmp/yawolKeepalivedLastSet" - HashLabel = "lbm-template-hash" - LoadBalancerKind = "LoadBalancer" - VRRPInstanceName = "ENVOY" - ContainsKeepAlivedMaster = "ContainsKeepAlivedMaster" + OpenstackReconcileTime = 5 * time.Minute + DefaultRequeueTime = 10 * time.Millisecond + RevisionAnnotation = "loadbalancer.yawol.stackit.cloud/revision" + YawolKeepalivedFile = "/tmp/yawolKeepalivedLastSet" + HashLabel = "lbm-template-hash" + LoadBalancerKind = "LoadBalancer" + VRRPInstanceName = "ENVOY" + ContainsKeepalivedMaster = "ContainsKeepalivedMaster" ) From 0ba69834756f1c220adce83c30d7fd8a36ef1687 Mon Sep 17 00:00:00 2001 From: Maximilian Geberl <48486938+dergeberl@users.noreply.github.com> Date: Wed, 14 Jun 2023 08:03:35 +0200 Subject: [PATCH 06/29] Remove nopreemt from keepalived to ensure keepalived will swap state when another keepalived with higher priority is available --- internal/helper/loadbalancermachine.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/helper/loadbalancermachine.go b/internal/helper/loadbalancermachine.go index 536a83f6..4272e87c 100644 --- a/internal/helper/loadbalancermachine.go +++ b/internal/helper/loadbalancermachine.go @@ -323,7 +323,6 @@ vrrp_track_file yawolfile { vrrp_instance ` + VRRPInstanceName + ` { state BACKUP interface eth0 - nopreempt virtual_router_id 100 priority 50 advert_int 4 From 254034f35d59e3112476df1e44d70e75d137cf01 Mon Sep 17 00:00:00 2001 From: Maximilian Geberl <48486938+dergeberl@users.noreply.github.com> Date: Wed, 14 Jun 2023 08:07:18 +0200 Subject: [PATCH 07/29] Modify scaledown of old loadbalancersets to ensure that in the new set is a keepalived master --- .../loadbalancer/loadbalancer_controller.go | 3 + .../loadbalancer_controller_test.go | 89 +++++++++++++++++++ internal/helper/loadbalancerset.go | 19 ++++ 3 files changed, 111 insertions(+) diff --git a/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go b/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go index becf3adf..5c857009 100644 --- a/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go +++ b/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go @@ -983,6 +983,9 @@ func (r *Reconciler) reconcileLoadBalancerSet( } if !downscaled { + if !helper.LBSetContainsKeepalivedMaster(loadBalancerSet) { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } r.Log.Info("scale down all lbsets except of", "lbs", loadBalancerSet.Name) return helper.ScaleDownAllLoadBalancerSetsForLBBut(ctx, r.Client, lb, loadBalancerSet.Name) } diff --git a/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go b/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go index 72db01cb..45b42669 100644 --- a/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go +++ b/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go @@ -3,6 +3,7 @@ package loadbalancer import ( "context" "encoding/json" + "errors" "fmt" "strings" "time" @@ -17,6 +18,7 @@ import ( "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" yawolv1beta1 "github.com/stackitcloud/yawol/api/v1beta1" + "github.com/stackitcloud/yawol/internal/helper" "github.com/stackitcloud/yawol/internal/openstack" "github.com/stackitcloud/yawol/internal/openstack/testing" v1 "k8s.io/api/core/v1" @@ -248,6 +250,93 @@ var _ = Describe("loadbalancer controller", Serial, Ordered, func() { }) }) + It("should scale down old lbset after new one has ready keepalived", func() { + var oldLbs runtimeClient.ObjectKey + By("waiting for lbset creation") + hopefully(lbNN, func(g Gomega, act LB) error { + var lbsetList yawolv1beta1.LoadBalancerSetList + g.Expect(k8sClient.List(ctx, &lbsetList, &runtimeClient.ListOptions{ + LabelSelector: labels.SelectorFromSet(lb.Spec.Selector.MatchLabels), + })).Should(Succeed()) + + g.Expect(len(lbsetList.Items)).Should(Equal(1)) + oldLbs = runtimeClient.ObjectKeyFromObject(&lbsetList.Items[0]) + return nil + }) + + By("changing flavorid") + updateLB(lbNN, func(act *LB) { + act.Spec.Infrastructure.Flavor = yawolv1beta1.OpenstackFlavorRef{ + FlavorID: pointer.String("somenewid"), + } + }) + + var newLbs runtimeClient.ObjectKey + By("checking for a new lbset") + hopefully(lbNN, func(g Gomega, act LB) error { + var lbsetList yawolv1beta1.LoadBalancerSetList + g.Expect(k8sClient.List(ctx, &lbsetList, &runtimeClient.ListOptions{ + LabelSelector: labels.SelectorFromSet(lb.Spec.Selector.MatchLabels), + })).Should(Succeed()) + + g.Expect(len(lbsetList.Items)).Should(Equal(2)) + for i := range lbsetList.Items { + if lbsetList.Items[i].Annotations[helper.RevisionAnnotation] == "2" { + newLbs = runtimeClient.ObjectKeyFromObject(&lbsetList.Items[i]) + } + } + return nil + }) + + By("Make both lbsets available by path status", func() { + var lbsetList yawolv1beta1.LoadBalancerSetList + Expect(k8sClient.List(ctx, &lbsetList, &runtimeClient.ListOptions{ + LabelSelector: labels.SelectorFromSet(lb.Spec.Selector.MatchLabels), + })).Should(Succeed()) + for _, lbs := range lbsetList.Items { + lbs.Status.ReadyReplicas = &lbs.Spec.Replicas + lbs.Status.Replicas = &lbs.Spec.Replicas + Expect(k8sClient.Status().Update(ctx, &lbs)).Should(Succeed()) + } + }) + + By("Old lbset should be scaled up because keepalived is not ready now") + Consistently(func() error { + var lbs yawolv1beta1.LoadBalancerSet + if err := k8sClient.Get(ctx, oldLbs, &lbs); err != nil { + return err + } + if lbs.Spec.Replicas == 0 { + return errors.New("already down scaled") + } + + return nil + }, time.Second*8).Should(Succeed()) + + By("Make latest lbsets keepalived condition true by path status", func() { + var lbs yawolv1beta1.LoadBalancerSet + Expect(k8sClient.Get(ctx, newLbs, &lbs)).Should(Succeed()) + lbs.Status.Conditions = []metav1.Condition{ + { + Type: helper.ContainsKeepalivedMaster, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.Time{Time: metav1.Now().Add(-120 * time.Second)}, + Reason: "ready", + Message: "ready", + }, + } + Expect(k8sClient.Status().Update(ctx, &lbs)).Should(Succeed()) + }) + + By("Old lbs should be downscaled") + hopefully(lbNN, func(g Gomega, act LB) error { + var lbs yawolv1beta1.LoadBalancerSet + g.Expect(k8sClient.Get(ctx, oldLbs, &lbs)).Should(Succeed()) + g.Expect(lbs.Spec.Replicas).To(Equal(0)) + return nil + }) + }) + It("should up- and downscale loadbalancer machines", func() { By("waiting for lb and lbset creation") hopefully(lbNN, func(g Gomega, act LB) error { diff --git a/internal/helper/loadbalancerset.go b/internal/helper/loadbalancerset.go index fcf1dc7e..4bb22b90 100644 --- a/internal/helper/loadbalancerset.go +++ b/internal/helper/loadbalancerset.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strconv" + "time" v1 "k8s.io/api/core/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -130,6 +131,24 @@ func LoadBalancerSetIsReady( return false, fmt.Errorf("active LoadBalancerSet not found") } +// LBSetContainsKeepalivedMaster returns true if the keepalived condition on set is ready for more than 2 min +// or keepalived is not ready for more than 10 min (to make sure this do not block updates) +func LBSetContainsKeepalivedMaster(set *yawolv1beta1.LoadBalancerSet) bool { + before2Minutes := metaV1.Time{Time: time.Now().Add(-2 * time.Minute)} + before10Minutes := metaV1.Time{Time: time.Now().Add(-10 * time.Minute)} + + for _, condition := range set.Status.Conditions { + if condition.Type != ContainsKeepalivedMaster { + continue + } + if condition.Status == metaV1.ConditionTrue { + return condition.LastTransitionTime.Before(&before2Minutes) + } + return condition.LastTransitionTime.Before(&before10Minutes) + } + return false +} + // Checks if LoadBalancerSets deriving from LoadBalancers are downscaled except for the LoadBalancerSet with the name of exceptionName // Returns true if all are downscaled; false if not // Follows error contract of getLoadBalancerSetsForLoadBalancer From 3aa993cb13d0e583f2269b5511f3250a592ff28d Mon Sep 17 00:00:00 2001 From: Maximilian Geberl <48486938+dergeberl@users.noreply.github.com> Date: Wed, 14 Jun 2023 08:08:38 +0200 Subject: [PATCH 08/29] remove redundant annotation --- .../yawol-controller/loadbalancer/loadbalancer_controller.go | 1 - .../loadbalancer/loadbalancerset_status_controller.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go b/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go index 5c857009..81ae2e4f 100644 --- a/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go +++ b/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go @@ -32,7 +32,6 @@ import ( const ( DefaultRequeueTime = 10 * time.Millisecond - RevisionAnnotation = "loadbalancer.yawol.stackit.cloud/revision" ServiceFinalizer = "yawol.stackit.cloud/controller2" ) diff --git a/controllers/yawol-controller/loadbalancer/loadbalancerset_status_controller.go b/controllers/yawol-controller/loadbalancer/loadbalancerset_status_controller.go index 316c2a2d..cc92da6b 100644 --- a/controllers/yawol-controller/loadbalancer/loadbalancerset_status_controller.go +++ b/controllers/yawol-controller/loadbalancer/loadbalancerset_status_controller.go @@ -57,7 +57,7 @@ func (r *LoadBalancerSetStatusReconciler) Reconcile(ctx context.Context, req ctr } // Not the current revision - if lb.Annotations[RevisionAnnotation] != loadBalancerSet.Annotations[RevisionAnnotation] { + if lb.Annotations[helper.RevisionAnnotation] != loadBalancerSet.Annotations[helper.RevisionAnnotation] { return ctrl.Result{}, nil } From ca8f9d4ab2bfcdbcf18afee1e092a1e7dd178fbb Mon Sep 17 00:00:00 2001 From: Maximilian Geberl <48486938+dergeberl@users.noreply.github.com> Date: Wed, 14 Jun 2023 08:58:16 +0200 Subject: [PATCH 09/29] make linter happy --- internal/helper/yawollet.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/helper/yawollet.go b/internal/helper/yawollet.go index 30d69276..6c5d41c5 100644 --- a/internal/helper/yawollet.go +++ b/internal/helper/yawollet.go @@ -893,12 +893,14 @@ func EnableAdHocDebugging( func UpdateKeepalivedFile(lb *yawolv1beta1.LoadBalancer, lbm *yawolv1beta1.LoadBalancerMachine) error { if lbmIsLatestRevision(lb, lbm) { - //file should exist + // file should exist f, err := os.Create(YawolKeepalivedFile) - defer f.Close() - return err + if err != nil { + return err + } + return f.Close() } - //file should not exist + // file should not exist err := os.Remove(YawolKeepalivedFile) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err From 935cc9c8df8a98611eb10b10e772055e3b196f9e Mon Sep 17 00:00:00 2001 From: Kai Kummerer Date: Wed, 14 Jun 2023 09:51:56 +0200 Subject: [PATCH 10/29] Rename v1 to metav1 --- .../loadbalancerset_controller.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go index f0ecea12..ba816037 100644 --- a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go +++ b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go @@ -15,7 +15,7 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -177,22 +177,22 @@ func (r *LoadBalancerSetReconciler) reconcileStatus( } // Write set contains master condition - status := v1.ConditionFalse + status := metav1.ConditionFalse reason := "NoKeepalivedMasterInSet" if setContainsMaster { - status = v1.ConditionTrue + status = metav1.ConditionTrue reason = "KeepalivedMasterInSet" } - transitionTime := v1.Now() + transitionTime := metav1.Now() for _, condition := range set.Status.Conditions { if condition.Type == helper.ContainsKeepalivedMaster && condition.Status == status { transitionTime = condition.LastTransitionTime } } - conditions := []v1.Condition{ + conditions := []metav1.Condition{ { Type: helper.ContainsKeepalivedMaster, Status: status, @@ -283,8 +283,8 @@ func (r *LoadBalancerSetReconciler) patchLoadBalancerSetStatus( // True if LastHeartbeatTime is > 5 minutes // True if a condition is not good for 5 minutes func shouldMachineBeDeleted(machine yawolv1beta1.LoadBalancerMachine) (bool, error) { - before5Minutes := v1.Time{Time: time.Now().Add(-5 * time.Minute)} - before10Minutes := v1.Time{Time: time.Now().Add(-10 * time.Minute)} + before5Minutes := metav1.Time{Time: time.Now().Add(-5 * time.Minute)} + before10Minutes := metav1.Time{Time: time.Now().Add(-10 * time.Minute)} // to handle the initial 10 minutes if machine.CreationTimestamp.Before(&before10Minutes) && @@ -325,7 +325,7 @@ func shouldMachineBeDeleted(machine yawolv1beta1.LoadBalancerMachine) (bool, err // False if LastHeartbeatTime is older than 180sec // False if ConfigReady, EnvoyReady or EnvoyUpToDate are false func isMachineReady(machine yawolv1beta1.LoadBalancerMachine) bool { - before180seconds := v1.Time{Time: time.Now().Add(-180 * time.Second)} + before180seconds := metav1.Time{Time: time.Now().Add(-180 * time.Second)} // not ready if no conditions are available if machine.Status.Conditions == nil || len(*machine.Status.Conditions) < 6 { @@ -362,10 +362,10 @@ func (r *LoadBalancerSetReconciler) createMachine(ctx context.Context, set *yawo machineLabels := r.getMachineLabelsFromSet(set) revision := set.Annotations[helper.RevisionAnnotation] machine := yawolv1beta1.LoadBalancerMachine{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: set.Name + "-" + randomString(5), Namespace: set.Namespace, - OwnerReferences: []v1.OwnerReference{ + OwnerReferences: []metav1.OwnerReference{ { APIVersion: set.APIVersion, Kind: set.Kind, From 453674a59ef2bc463009120f3cf46f138fb52e1a Mon Sep 17 00:00:00 2001 From: Kai Kummerer Date: Wed, 14 Jun 2023 10:22:40 +0200 Subject: [PATCH 11/29] Simplify patching `lbs` status --- .../loadbalancerset_controller.go | 63 +++++++------------ 1 file changed, 21 insertions(+), 42 deletions(-) diff --git a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go index ba816037..da47c829 100644 --- a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go +++ b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go @@ -5,7 +5,6 @@ import ( "crypto/rand" "encoding/json" "fmt" - "reflect" "time" yawolv1beta1 "github.com/stackitcloud/yawol/api/v1beta1" @@ -15,11 +14,14 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -94,7 +96,8 @@ func (r *LoadBalancerSetReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } - if res, err := r.reconcileStatus(ctx, &set, readyMachineCount, setContainsMaster); err != nil || res.Requeue || res.RequeueAfter != 0 { + // TODO: patchStatus *after* we've reconciled the replicas + if res, err := r.patchStatus(ctx, &set, readyMachineCount, setContainsMaster); err != nil || res.Requeue || res.RequeueAfter != 0 { return res, err } @@ -148,68 +151,44 @@ func (r *LoadBalancerSetReconciler) deletionRoutine( return ctrl.Result{}, nil } -func (r *LoadBalancerSetReconciler) reconcileStatus( +func (r *LoadBalancerSetReconciler) patchStatus( ctx context.Context, set *yawolv1beta1.LoadBalancerSet, readyMachinesCount int, setContainsMaster bool, ) (ctrl.Result, error) { + + setCopy := set.DeepCopy() + // Write replicas into status - if set.Status.Replicas == nil || *set.Status.Replicas != set.Spec.Replicas { - replicas := set.Spec.Replicas - if err := r.patchLoadBalancerSetStatus(ctx, set, yawolv1beta1.LoadBalancerSetStatus{ - Replicas: &replicas, - }); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{Requeue: true}, nil - } + set.Status.Replicas = pointer.Int(set.Spec.Replicas) // Write ready replicas into status - if set.Status.ReadyReplicas == nil || *set.Status.ReadyReplicas != readyMachinesCount { - replicas := readyMachinesCount - if err := r.patchLoadBalancerSetStatus(ctx, set, yawolv1beta1.LoadBalancerSetStatus{ - ReadyReplicas: &replicas, - }); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{RequeueAfter: time.Second * 2}, nil - } + set.Status.ReadyReplicas = pointer.Int(readyMachinesCount) // Write set contains master condition status := metav1.ConditionFalse reason := "NoKeepalivedMasterInSet" + message := "LoadBalancerSet doesn't contain a master machine" if setContainsMaster { status = metav1.ConditionTrue reason = "KeepalivedMasterInSet" + message = "LoadBalancerSet contains a master machine" } - transitionTime := metav1.Now() - for _, condition := range set.Status.Conditions { - if condition.Type == helper.ContainsKeepalivedMaster && condition.Status == status { - transitionTime = condition.LastTransitionTime - } - } - - conditions := []metav1.Condition{ - { - Type: helper.ContainsKeepalivedMaster, - Status: status, - LastTransitionTime: transitionTime, - Reason: reason, - }, - } + meta.SetStatusCondition(&set.Status.Conditions, metav1.Condition{ + Type: helper.ContainsKeepalivedMaster, + Status: status, + Reason: reason, + Message: message, + }) - if reflect.DeepEqual(set.Status.Conditions, conditions) { + if equality.Semantic.DeepEqual(set, setCopy) { return ctrl.Result{}, nil } - if err := r.patchLoadBalancerSetStatus(ctx, set, yawolv1beta1.LoadBalancerSetStatus{Conditions: conditions}); err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil + return ctrl.Result{}, r.Client.Status().Patch(ctx, set, client.MergeFrom(setCopy)) } func (r *LoadBalancerSetReconciler) reconcileReplicas( From 37051da7d7aa603747c47362edc83770023069e0 Mon Sep 17 00:00:00 2001 From: Kai Kummerer Date: Wed, 14 Jun 2023 10:28:20 +0200 Subject: [PATCH 12/29] =?UTF-8?q?Rename=20`ContainsKeepalivedMaster=C2=B4?= =?UTF-8?q?=20to=20`HasKeepalivedMaster`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../loadbalancer/loadbalancer_controller.go | 2 +- .../loadbalancer_controller_test.go | 2 +- .../loadbalancerset_controller.go | 18 +++++++++--------- .../loadbalancerset_controller_test.go | 2 +- internal/helper/const.go | 16 ++++++++-------- internal/helper/loadbalancerset.go | 6 +++--- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go b/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go index 81ae2e4f..9cbea31d 100644 --- a/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go +++ b/controllers/yawol-controller/loadbalancer/loadbalancer_controller.go @@ -982,7 +982,7 @@ func (r *Reconciler) reconcileLoadBalancerSet( } if !downscaled { - if !helper.LBSetContainsKeepalivedMaster(loadBalancerSet) { + if !helper.LBSetHasKeepalivedMaster(loadBalancerSet) { return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } r.Log.Info("scale down all lbsets except of", "lbs", loadBalancerSet.Name) diff --git a/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go b/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go index 45b42669..1d7ee96a 100644 --- a/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go +++ b/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go @@ -318,7 +318,7 @@ var _ = Describe("loadbalancer controller", Serial, Ordered, func() { Expect(k8sClient.Get(ctx, newLbs, &lbs)).Should(Succeed()) lbs.Status.Conditions = []metav1.Condition{ { - Type: helper.ContainsKeepalivedMaster, + Type: helper.HasKeepalivedMaster, Status: metav1.ConditionTrue, LastTransitionTime: metav1.Time{Time: metav1.Now().Add(-120 * time.Second)}, Reason: "ready", diff --git a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go index da47c829..79f58cb8 100644 --- a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go +++ b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go @@ -78,7 +78,7 @@ func (r *LoadBalancerSetReconciler) Reconcile(ctx context.Context, req ctrl.Requ var ( readyMachineCount int deletedMachineCount int - setContainsMaster bool + hasKeepalivedMaster bool ) for i := range childMachines.Items { @@ -87,7 +87,7 @@ func (r *LoadBalancerSetReconciler) Reconcile(ctx context.Context, req ctrl.Requ continue } if isMachineMaster(childMachines.Items[i]) { - setContainsMaster = true + hasKeepalivedMaster = true } if isMachineReady(childMachines.Items[i]) { @@ -97,7 +97,7 @@ func (r *LoadBalancerSetReconciler) Reconcile(ctx context.Context, req ctrl.Requ } // TODO: patchStatus *after* we've reconciled the replicas - if res, err := r.patchStatus(ctx, &set, readyMachineCount, setContainsMaster); err != nil || res.Requeue || res.RequeueAfter != 0 { + if res, err := r.patchStatus(ctx, &set, readyMachineCount, hasKeepalivedMaster); err != nil || res.Requeue || res.RequeueAfter != 0 { return res, err } @@ -155,7 +155,7 @@ func (r *LoadBalancerSetReconciler) patchStatus( ctx context.Context, set *yawolv1beta1.LoadBalancerSet, readyMachinesCount int, - setContainsMaster bool, + hasKeepalivedMaster bool, ) (ctrl.Result, error) { setCopy := set.DeepCopy() @@ -166,19 +166,19 @@ func (r *LoadBalancerSetReconciler) patchStatus( // Write ready replicas into status set.Status.ReadyReplicas = pointer.Int(readyMachinesCount) - // Write set contains master condition + // Write HasKeepalivedMaster condition status := metav1.ConditionFalse reason := "NoKeepalivedMasterInSet" - message := "LoadBalancerSet doesn't contain a master machine" + message := "LoadBalancerSet doesn't have a master machine" - if setContainsMaster { + if hasKeepalivedMaster { status = metav1.ConditionTrue reason = "KeepalivedMasterInSet" - message = "LoadBalancerSet contains a master machine" + message = "LoadBalancerSet has a master machine" } meta.SetStatusCondition(&set.Status.Conditions, metav1.Condition{ - Type: helper.ContainsKeepalivedMaster, + Type: helper.HasKeepalivedMaster, Status: status, Reason: reason, Message: message, diff --git a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go index 0186777d..28472e72 100644 --- a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go +++ b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go @@ -123,7 +123,7 @@ var _ = Describe("LoadBalancerSet controller", Serial, Ordered, func() { Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(&setStub), &set)).To(Succeed()) Expect(set.Status.Conditions).To(Not(BeNil())) for _, condition := range set.Status.Conditions { - if condition.Type == helper.ContainsKeepalivedMaster { + if condition.Type == helper.HasKeepalivedMaster { return condition.Status } } diff --git a/internal/helper/const.go b/internal/helper/const.go index 3ab377ef..f2aca47a 100644 --- a/internal/helper/const.go +++ b/internal/helper/const.go @@ -3,12 +3,12 @@ package helper import "time" const ( - OpenstackReconcileTime = 5 * time.Minute - DefaultRequeueTime = 10 * time.Millisecond - RevisionAnnotation = "loadbalancer.yawol.stackit.cloud/revision" - YawolKeepalivedFile = "/tmp/yawolKeepalivedLastSet" - HashLabel = "lbm-template-hash" - LoadBalancerKind = "LoadBalancer" - VRRPInstanceName = "ENVOY" - ContainsKeepalivedMaster = "ContainsKeepalivedMaster" + OpenstackReconcileTime = 5 * time.Minute + DefaultRequeueTime = 10 * time.Millisecond + RevisionAnnotation = "loadbalancer.yawol.stackit.cloud/revision" + YawolKeepalivedFile = "/tmp/yawolKeepalivedLastSet" + HashLabel = "lbm-template-hash" + LoadBalancerKind = "LoadBalancer" + VRRPInstanceName = "ENVOY" + HasKeepalivedMaster = "HasKeepalivedMaster" ) diff --git a/internal/helper/loadbalancerset.go b/internal/helper/loadbalancerset.go index 4bb22b90..77c1c8f7 100644 --- a/internal/helper/loadbalancerset.go +++ b/internal/helper/loadbalancerset.go @@ -131,14 +131,14 @@ func LoadBalancerSetIsReady( return false, fmt.Errorf("active LoadBalancerSet not found") } -// LBSetContainsKeepalivedMaster returns true if the keepalived condition on set is ready for more than 2 min +// LBSetHasKeepalivedMaster returns true if the keepalived condition on set is ready for more than 2 min // or keepalived is not ready for more than 10 min (to make sure this do not block updates) -func LBSetContainsKeepalivedMaster(set *yawolv1beta1.LoadBalancerSet) bool { +func LBSetHasKeepalivedMaster(set *yawolv1beta1.LoadBalancerSet) bool { before2Minutes := metaV1.Time{Time: time.Now().Add(-2 * time.Minute)} before10Minutes := metaV1.Time{Time: time.Now().Add(-10 * time.Minute)} for _, condition := range set.Status.Conditions { - if condition.Type != ContainsKeepalivedMaster { + if condition.Type != HasKeepalivedMaster { continue } if condition.Status == metaV1.ConditionTrue { From 4d51e84b7370d808e3e31badd67a8f12bcf7383d Mon Sep 17 00:00:00 2001 From: Kai Kummerer Date: Wed, 14 Jun 2023 10:48:36 +0200 Subject: [PATCH 13/29] Do patches instead of updates; Improve tests --- .../loadbalancer_controller_test.go | 73 +++++++++++++------ .../loadbalancerset_controller_test.go | 9 ++- 2 files changed, 56 insertions(+), 26 deletions(-) diff --git a/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go b/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go index 1d7ee96a..a4e9e31c 100644 --- a/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go +++ b/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go @@ -288,17 +288,17 @@ var _ = Describe("loadbalancer controller", Serial, Ordered, func() { return nil }) - By("Make both lbsets available by path status", func() { - var lbsetList yawolv1beta1.LoadBalancerSetList - Expect(k8sClient.List(ctx, &lbsetList, &runtimeClient.ListOptions{ - LabelSelector: labels.SelectorFromSet(lb.Spec.Selector.MatchLabels), - })).Should(Succeed()) - for _, lbs := range lbsetList.Items { - lbs.Status.ReadyReplicas = &lbs.Spec.Replicas - lbs.Status.Replicas = &lbs.Spec.Replicas - Expect(k8sClient.Status().Update(ctx, &lbs)).Should(Succeed()) - } - }) + By("Make both lbsets available by patching status") + var lbsetList yawolv1beta1.LoadBalancerSetList + Expect(k8sClient.List(ctx, &lbsetList, &runtimeClient.ListOptions{ + LabelSelector: labels.SelectorFromSet(lb.Spec.Selector.MatchLabels), + })).Should(Succeed()) + for _, lbs := range lbsetList.Items { + patch := runtimeClient.MergeFrom(lbs.DeepCopy()) + lbs.Status.ReadyReplicas = &lbs.Spec.Replicas + lbs.Status.Replicas = &lbs.Spec.Replicas + Expect(k8sClient.Status().Patch(ctx, &lbs, patch)).Should(Succeed()) + } By("Old lbset should be scaled up because keepalived is not ready now") Consistently(func() error { @@ -313,20 +313,47 @@ var _ = Describe("loadbalancer controller", Serial, Ordered, func() { return nil }, time.Second*8).Should(Succeed()) - By("Make latest lbsets keepalived condition true by path status", func() { + By("Make latest lbsets keepalived condition true by patching status") + var lbs yawolv1beta1.LoadBalancerSet + Expect(k8sClient.Get(ctx, newLbs, &lbs)).Should(Succeed()) + patch := runtimeClient.MergeFrom(lbs.DeepCopy()) + lbs.Status.Conditions = []metav1.Condition{ + { + Type: helper.HasKeepalivedMaster, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.Now(), + Reason: "ready", + Message: "ready", + }, + } + Expect(k8sClient.Status().Patch(ctx, &lbs, patch)).Should(Succeed()) + + By("Old lbset should be scaled up because keepalived is not ready long enough") + Consistently(func() error { var lbs yawolv1beta1.LoadBalancerSet - Expect(k8sClient.Get(ctx, newLbs, &lbs)).Should(Succeed()) - lbs.Status.Conditions = []metav1.Condition{ - { - Type: helper.HasKeepalivedMaster, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.Time{Time: metav1.Now().Add(-120 * time.Second)}, - Reason: "ready", - Message: "ready", - }, + if err := k8sClient.Get(ctx, oldLbs, &lbs); err != nil { + return err } - Expect(k8sClient.Status().Update(ctx, &lbs)).Should(Succeed()) - }) + if lbs.Spec.Replicas == 0 { + return errors.New("already down scaled") + } + + return nil + }, time.Second*8).Should(Succeed()) + + By("Make latest lbsets keepalived condition true since longer") + Expect(k8sClient.Get(ctx, newLbs, &lbs)).Should(Succeed()) + patch = runtimeClient.MergeFrom(lbs.DeepCopy()) + lbs.Status.Conditions = []metav1.Condition{ + { + Type: helper.HasKeepalivedMaster, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.Time{Time: metav1.Now().Add(-120 * time.Second)}, + Reason: "ready", + Message: "ready", + }, + } + Expect(k8sClient.Status().Patch(ctx, &lbs, patch)).Should(Succeed()) By("Old lbs should be downscaled") hopefully(lbNN, func(g Gomega, act LB) error { diff --git a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go index 28472e72..8a32239f 100644 --- a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go +++ b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go @@ -109,6 +109,7 @@ var _ = Describe("LoadBalancerSet controller", Serial, Ordered, func() { It("Should eventually have a master", func() { By("setting LBM as master") machine := getChildMachines(ctx, &setStub)[0] + patch := client.MergeFrom(machine.DeepCopy()) machine.Status.Conditions = &[]v1.NodeCondition{ { Type: v1.NodeConditionType(helper.KeepalivedMaster), @@ -116,7 +117,7 @@ var _ = Describe("LoadBalancerSet controller", Serial, Ordered, func() { Reason: "KeepalivedStatus", }, } - Expect(k8sClient.Status().Update(ctx, &machine)).To(Succeed()) + Expect(k8sClient.Status().Patch(ctx, &machine, patch)).To(Succeed()) Eventually(func() metav1.ConditionStatus { var set yawolv1beta1.LoadBalancerSet @@ -155,8 +156,9 @@ var _ = Describe("LoadBalancerSet controller", Serial, Ordered, func() { It("Should be successfully", func() { childMachines := getChildMachines(ctx, &setStub) for _, machine := range childMachines { + patch := client.MergeFrom(machine.DeepCopy()) machine.Status.Conditions = &cpy - err := k8sClient.Status().Update(ctx, &machine) + err := k8sClient.Status().Patch(ctx, &machine, patch) Expect(err).NotTo(HaveOccurred()) } }) @@ -185,8 +187,9 @@ var _ = Describe("LoadBalancerSet controller", Serial, Ordered, func() { childMachines := getChildMachines(ctx, &setStub) for _, machine := range childMachines { + patch := client.MergeFrom(machine.DeepCopy()) machine.Status.Conditions = &conditions - err := k8sClient.Status().Update(ctx, &machine) + err := k8sClient.Status().Patch(ctx, &machine, patch) Expect(err).NotTo(HaveOccurred()) } }) From ba4da05a4bf1da161739d4bb187efdb44889a324 Mon Sep 17 00:00:00 2001 From: Maximilian Geberl <48486938+dergeberl@users.noreply.github.com> Date: Wed, 14 Jun 2023 10:21:58 +0200 Subject: [PATCH 14/29] use aferoFs for revisionFile and rename function --- controllers/yawollet/loadbalancer_controller.go | 7 ++++++- controllers/yawollet/loadbalancer_controller_test.go | 5 ++--- controllers/yawollet/suite_test.go | 5 +++++ internal/helper/yawollet.go | 10 +++++++--- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/controllers/yawollet/loadbalancer_controller.go b/controllers/yawollet/loadbalancer_controller.go index eee26295..e7f0ab49 100644 --- a/controllers/yawollet/loadbalancer_controller.go +++ b/controllers/yawollet/loadbalancer_controller.go @@ -4,6 +4,7 @@ package controllers import ( "context" "fmt" + "github.com/spf13/afero" "sort" "time" @@ -36,6 +37,7 @@ type LoadBalancerReconciler struct { ListenAddress string RequeueTime int KeepalivedStatsFile string + AferoFs afero.Fs } // Reconcile handles reconciliation of loadbalancer object @@ -93,7 +95,7 @@ func (r *LoadBalancerReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - if err := helper.UpdateKeepalivedFile(lb, lbm); err != nil { + if err := helper.ReconcileLatestRevisionFile(r.AferoFs, lb, lbm); err != nil { return ctrl.Result{}, err } @@ -175,6 +177,9 @@ func (r *LoadBalancerReconciler) reconcile( // SetupWithManager is used by kubebuilder to init the controller loop func (r *LoadBalancerReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.AferoFs == nil { + r.AferoFs = afero.NewOsFs() + } return ctrl.NewControllerManagedBy(mgr). For(&yawolv1beta1.LoadBalancer{}). WithEventFilter(predicate.And( diff --git a/controllers/yawollet/loadbalancer_controller_test.go b/controllers/yawollet/loadbalancer_controller_test.go index b6237feb..69f27c97 100644 --- a/controllers/yawollet/loadbalancer_controller_test.go +++ b/controllers/yawollet/loadbalancer_controller_test.go @@ -6,7 +6,6 @@ import ( "io" "io/fs" "net/http" - "os" "os/exec" "strings" "time" @@ -453,7 +452,7 @@ var _ = Describe("check loadbalancer reconcile", Serial, Ordered, func() { When("lb and lbm revision annotation are the same", func() { It("should create yawolKeepalivedFile", func() { Eventually(func() error { - _, err := os.Stat(helper.YawolKeepalivedFile) + _, err := aferoFs.Stat(helper.YawolKeepalivedFile) return err }, TIMEOUT, INTERVAL).Should(Succeed()) @@ -467,7 +466,7 @@ var _ = Describe("check loadbalancer reconcile", Serial, Ordered, func() { }) It("should not create or delete yawolKeepalivedFile", func() { Eventually(func() error { - _, err := os.Stat(helper.YawolKeepalivedFile) + _, err := aferoFs.Stat(helper.YawolKeepalivedFile) if err == nil || !errors.Is(err, fs.ErrNotExist) { return errors.New("keepalived file still exists") } diff --git a/controllers/yawollet/suite_test.go b/controllers/yawollet/suite_test.go index 15a974df..be51d6aa 100644 --- a/controllers/yawollet/suite_test.go +++ b/controllers/yawollet/suite_test.go @@ -18,6 +18,7 @@ import ( testv3 "github.com/envoyproxy/go-control-plane/pkg/test/v3" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/spf13/afero" "google.golang.org/grpc" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -48,6 +49,7 @@ var ( envoyCmd *exec.Cmd ctx context.Context cancel context.CancelFunc + aferoFs afero.Fs ) func TestAPIs(t *testing.T) { @@ -59,6 +61,8 @@ func TestAPIs(t *testing.T) { var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + aferoFs = afero.NewMemMapFs() + ctx, cancel = context.WithCancel(context.Background()) By("bootstrapping test environment") @@ -138,6 +142,7 @@ var _ = BeforeSuite(func() { EnvoyCache: cache, ListenAddress: "127.0.0.1", RequeueTime: 1, + AferoFs: aferoFs, }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) diff --git a/internal/helper/yawollet.go b/internal/helper/yawollet.go index 6c5d41c5..3f2a2c65 100644 --- a/internal/helper/yawollet.go +++ b/internal/helper/yawollet.go @@ -23,6 +23,7 @@ import ( "k8s.io/client-go/tools/record" "github.com/shirou/gopsutil/v3/process" + "github.com/spf13/afero" yawolv1beta1 "github.com/stackitcloud/yawol/api/v1beta1" "github.com/stackitcloud/yawol/internal/envoystatus" @@ -891,17 +892,20 @@ func EnableAdHocDebugging( return nil } -func UpdateKeepalivedFile(lb *yawolv1beta1.LoadBalancer, lbm *yawolv1beta1.LoadBalancerMachine) error { +// ReconcileLatestRevisionFile makes sure that the YawolKeepalivedFile exists if the lbm is from the current revision. +// Otherwise, make sure that the file is deleted. +// Use aferoFs to use it in tests. +func ReconcileLatestRevisionFile(aferoFs afero.Fs, lb *yawolv1beta1.LoadBalancer, lbm *yawolv1beta1.LoadBalancerMachine) error { if lbmIsLatestRevision(lb, lbm) { // file should exist - f, err := os.Create(YawolKeepalivedFile) + f, err := aferoFs.Create(YawolKeepalivedFile) if err != nil { return err } return f.Close() } // file should not exist - err := os.Remove(YawolKeepalivedFile) + err := aferoFs.Remove(YawolKeepalivedFile) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } From d4543eda77f885883a857cc92c71c8059ae0c35a Mon Sep 17 00:00:00 2001 From: Maximilian Geberl <48486938+dergeberl@users.noreply.github.com> Date: Wed, 14 Jun 2023 10:42:07 +0200 Subject: [PATCH 15/29] add comment to yawolfile in keepalived --- internal/helper/loadbalancermachine.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/helper/loadbalancermachine.go b/internal/helper/loadbalancermachine.go index 4272e87c..2f753903 100644 --- a/internal/helper/loadbalancermachine.go +++ b/internal/helper/loadbalancermachine.go @@ -315,6 +315,7 @@ vrrp_track_process envoy { weight 100 } +# the yawolfile is used to change priority if loadbalancermachine is from current revision vrrp_track_file yawolfile { file "` + YawolKeepalivedFile + `" weight 10 From bfaac5bf1f98569ce480afc03b33409c5190185c Mon Sep 17 00:00:00 2001 From: Maximilian Geberl <48486938+dergeberl@users.noreply.github.com> Date: Wed, 14 Jun 2023 10:47:59 +0200 Subject: [PATCH 16/29] rename yawollet keepalived file and create folder of needed --- internal/helper/const.go | 7 ++++--- internal/helper/yawollet.go | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/helper/const.go b/internal/helper/const.go index f2aca47a..2609f155 100644 --- a/internal/helper/const.go +++ b/internal/helper/const.go @@ -6,9 +6,10 @@ const ( OpenstackReconcileTime = 5 * time.Minute DefaultRequeueTime = 10 * time.Millisecond RevisionAnnotation = "loadbalancer.yawol.stackit.cloud/revision" - YawolKeepalivedFile = "/tmp/yawolKeepalivedLastSet" + YawolKeepalivedDir = "/var/yawol/" + YawolKeepalivedFile = YawolKeepalivedDir + "set_is_latest_revision" HashLabel = "lbm-template-hash" - LoadBalancerKind = "LoadBalancer" + LoadBalancerKind = "LoadBalancer" VRRPInstanceName = "ENVOY" - HasKeepalivedMaster = "HasKeepalivedMaster" + HasKeepalivedMaster = "HasKeepalivedMaster" ) diff --git a/internal/helper/yawollet.go b/internal/helper/yawollet.go index 3f2a2c65..cd06d8fa 100644 --- a/internal/helper/yawollet.go +++ b/internal/helper/yawollet.go @@ -896,6 +896,9 @@ func EnableAdHocDebugging( // Otherwise, make sure that the file is deleted. // Use aferoFs to use it in tests. func ReconcileLatestRevisionFile(aferoFs afero.Fs, lb *yawolv1beta1.LoadBalancer, lbm *yawolv1beta1.LoadBalancerMachine) error { + if err := aferoFs.MkdirAll(YawolKeepalivedDir, 0755); err != nil { + return err + } if lbmIsLatestRevision(lb, lbm) { // file should exist f, err := aferoFs.Create(YawolKeepalivedFile) From c895dd135ec47550f22e23df9640699380d18f0a Mon Sep 17 00:00:00 2001 From: Maximilian Geberl <48486938+dergeberl@users.noreply.github.com> Date: Wed, 14 Jun 2023 10:48:50 +0200 Subject: [PATCH 17/29] LBSetContainsKeepalivedMaster is also true if there is no condition after 15 min to not block update --- internal/helper/loadbalancerset.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/helper/loadbalancerset.go b/internal/helper/loadbalancerset.go index 77c1c8f7..eee54da5 100644 --- a/internal/helper/loadbalancerset.go +++ b/internal/helper/loadbalancerset.go @@ -131,11 +131,14 @@ func LoadBalancerSetIsReady( return false, fmt.Errorf("active LoadBalancerSet not found") } -// LBSetHasKeepalivedMaster returns true if the keepalived condition on set is ready for more than 2 min -// or keepalived is not ready for more than 10 min (to make sure this do not block updates) +// LBSetHasKeepalivedMaster returns true one of the following conditions are met: +// - if the keepalived condition on set is ready for more than 2 min +// - keepalived condition is not ready for more than 10 min (to make sure this does not block updates) +// - no keepalived condition is in lbs but lbs is older than 15 min (to make sure this does not block updates) func LBSetHasKeepalivedMaster(set *yawolv1beta1.LoadBalancerSet) bool { before2Minutes := metaV1.Time{Time: time.Now().Add(-2 * time.Minute)} before10Minutes := metaV1.Time{Time: time.Now().Add(-10 * time.Minute)} + before15Minutes := metaV1.Time{Time: time.Now().Add(-15 * time.Minute)} for _, condition := range set.Status.Conditions { if condition.Type != HasKeepalivedMaster { @@ -146,7 +149,7 @@ func LBSetHasKeepalivedMaster(set *yawolv1beta1.LoadBalancerSet) bool { } return condition.LastTransitionTime.Before(&before10Minutes) } - return false + return set.CreationTimestamp.Before(&before15Minutes) } // Checks if LoadBalancerSets deriving from LoadBalancers are downscaled except for the LoadBalancerSet with the name of exceptionName From 5f875e5f26d82c72a56cafda3e2749e887519081 Mon Sep 17 00:00:00 2001 From: Kai Kummerer Date: Wed, 14 Jun 2023 10:57:27 +0200 Subject: [PATCH 18/29] Use MatchingLabels instead of ListOptions --- .../loadbalancer_controller_test.go | 51 +++++-------------- 1 file changed, 12 insertions(+), 39 deletions(-) diff --git a/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go b/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go index a4e9e31c..a30af7ae 100644 --- a/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go +++ b/controllers/yawol-controller/loadbalancer/loadbalancer_controller_test.go @@ -26,7 +26,6 @@ import ( k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" runtimeClient "sigs.k8s.io/controller-runtime/pkg/client" @@ -178,10 +177,7 @@ var _ = Describe("loadbalancer controller", Serial, Ordered, func() { It("should create a loadbalancerset", func() { hopefully(lbNN, func(g Gomega, act LB) error { var lbsetList yawolv1beta1.LoadBalancerSetList - g.Expect(k8sClient.List(ctx, &lbsetList, &runtimeClient.ListOptions{ - LabelSelector: labels.SelectorFromSet(lb.Spec.Selector.MatchLabels), - })).Should(Succeed()) - + g.Expect(k8sClient.List(ctx, &lbsetList, runtimeClient.MatchingLabels(lb.Spec.Selector.MatchLabels))).Should(Succeed()) g.Expect(len(lbsetList.Items)).Should(Equal(1)) // prevent later panic @@ -206,10 +202,7 @@ var _ = Describe("loadbalancer controller", Serial, Ordered, func() { By("waiting for lbset creation") hopefully(lbNN, func(g Gomega, act LB) error { var lbsetList yawolv1beta1.LoadBalancerSetList - g.Expect(k8sClient.List(ctx, &lbsetList, &runtimeClient.ListOptions{ - LabelSelector: labels.SelectorFromSet(lb.Spec.Selector.MatchLabels), - })).Should(Succeed()) - + g.Expect(k8sClient.List(ctx, &lbsetList, runtimeClient.MatchingLabels(lb.Spec.Selector.MatchLabels))).Should(Succeed()) g.Expect(len(lbsetList.Items)).Should(Equal(1)) lbset := lbsetList.Items[0] @@ -229,10 +222,7 @@ var _ = Describe("loadbalancer controller", Serial, Ordered, func() { By("checking for a new lbset") hopefully(lbNN, func(g Gomega, act LB) error { var lbsetList yawolv1beta1.LoadBalancerSetList - g.Expect(k8sClient.List(ctx, &lbsetList, &runtimeClient.ListOptions{ - LabelSelector: labels.SelectorFromSet(lb.Spec.Selector.MatchLabels), - })).Should(Succeed()) - + g.Expect(k8sClient.List(ctx, &lbsetList, runtimeClient.MatchingLabels(lb.Spec.Selector.MatchLabels))).Should(Succeed()) g.Expect(len(lbsetList.Items)).Should(Equal(2)) By("testing if the new set got a different hash") @@ -255,10 +245,7 @@ var _ = Describe("loadbalancer controller", Serial, Ordered, func() { By("waiting for lbset creation") hopefully(lbNN, func(g Gomega, act LB) error { var lbsetList yawolv1beta1.LoadBalancerSetList - g.Expect(k8sClient.List(ctx, &lbsetList, &runtimeClient.ListOptions{ - LabelSelector: labels.SelectorFromSet(lb.Spec.Selector.MatchLabels), - })).Should(Succeed()) - + g.Expect(k8sClient.List(ctx, &lbsetList, runtimeClient.MatchingLabels(lb.Spec.Selector.MatchLabels))).Should(Succeed()) g.Expect(len(lbsetList.Items)).Should(Equal(1)) oldLbs = runtimeClient.ObjectKeyFromObject(&lbsetList.Items[0]) return nil @@ -275,14 +262,11 @@ var _ = Describe("loadbalancer controller", Serial, Ordered, func() { By("checking for a new lbset") hopefully(lbNN, func(g Gomega, act LB) error { var lbsetList yawolv1beta1.LoadBalancerSetList - g.Expect(k8sClient.List(ctx, &lbsetList, &runtimeClient.ListOptions{ - LabelSelector: labels.SelectorFromSet(lb.Spec.Selector.MatchLabels), - })).Should(Succeed()) - + g.Expect(k8sClient.List(ctx, &lbsetList, runtimeClient.MatchingLabels(lb.Spec.Selector.MatchLabels))).Should(Succeed()) g.Expect(len(lbsetList.Items)).Should(Equal(2)) - for i := range lbsetList.Items { - if lbsetList.Items[i].Annotations[helper.RevisionAnnotation] == "2" { - newLbs = runtimeClient.ObjectKeyFromObject(&lbsetList.Items[i]) + for _, lbs := range lbsetList.Items { + if lbs.Annotations[helper.RevisionAnnotation] == "2" { + newLbs = runtimeClient.ObjectKeyFromObject(&lbs) } } return nil @@ -290,9 +274,7 @@ var _ = Describe("loadbalancer controller", Serial, Ordered, func() { By("Make both lbsets available by patching status") var lbsetList yawolv1beta1.LoadBalancerSetList - Expect(k8sClient.List(ctx, &lbsetList, &runtimeClient.ListOptions{ - LabelSelector: labels.SelectorFromSet(lb.Spec.Selector.MatchLabels), - })).Should(Succeed()) + Expect(k8sClient.List(ctx, &lbsetList, runtimeClient.MatchingLabels(lb.Spec.Selector.MatchLabels))).Should(Succeed()) for _, lbs := range lbsetList.Items { patch := runtimeClient.MergeFrom(lbs.DeepCopy()) lbs.Status.ReadyReplicas = &lbs.Spec.Replicas @@ -368,10 +350,7 @@ var _ = Describe("loadbalancer controller", Serial, Ordered, func() { By("waiting for lb and lbset creation") hopefully(lbNN, func(g Gomega, act LB) error { var lbsetList yawolv1beta1.LoadBalancerSetList - g.Expect(k8sClient.List(ctx, &lbsetList, &runtimeClient.ListOptions{ - LabelSelector: labels.SelectorFromSet(lb.Spec.Selector.MatchLabels), - })).Should(Succeed()) - + g.Expect(k8sClient.List(ctx, &lbsetList, runtimeClient.MatchingLabels(lb.Spec.Selector.MatchLabels))).Should(Succeed()) g.Expect(len(lbsetList.Items)).Should(Equal(1)) lbset := lbsetList.Items[0] @@ -386,10 +365,7 @@ var _ = Describe("loadbalancer controller", Serial, Ordered, func() { Eventually(func(g Gomega) { var lbsetList yawolv1beta1.LoadBalancerSetList - g.Expect(k8sClient.List(ctx, &lbsetList, &runtimeClient.ListOptions{ - LabelSelector: labels.SelectorFromSet(lb.Spec.Selector.MatchLabels), - })).Should(Succeed()) - + g.Expect(k8sClient.List(ctx, &lbsetList, runtimeClient.MatchingLabels(lb.Spec.Selector.MatchLabels))).Should(Succeed()) g.Expect(len(lbsetList.Items)).Should(Equal(1)) lbset := lbsetList.Items[0] @@ -403,10 +379,7 @@ var _ = Describe("loadbalancer controller", Serial, Ordered, func() { Eventually(func(g Gomega) { var lbsetList yawolv1beta1.LoadBalancerSetList - g.Expect(k8sClient.List(ctx, &lbsetList, &runtimeClient.ListOptions{ - LabelSelector: labels.SelectorFromSet(lb.Spec.Selector.MatchLabels), - })).Should(Succeed()) - + g.Expect(k8sClient.List(ctx, &lbsetList, runtimeClient.MatchingLabels(lb.Spec.Selector.MatchLabels))).Should(Succeed()) g.Expect(len(lbsetList.Items)).Should(Equal(1)) lbset := lbsetList.Items[0] From 46fc90921e26097768962fe72f1e52e11d1c7996 Mon Sep 17 00:00:00 2001 From: Kai Kummerer Date: Wed, 14 Jun 2023 11:06:19 +0200 Subject: [PATCH 19/29] Rename isMachineMaster to isMachineKeepalivedMaster --- .../loadbalancerset/loadbalancerset_controller.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go index 79f58cb8..d11eb2f1 100644 --- a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go +++ b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go @@ -86,7 +86,7 @@ func (r *LoadBalancerSetReconciler) Reconcile(ctx context.Context, req ctrl.Requ deletedMachineCount++ continue } - if isMachineMaster(childMachines.Items[i]) { + if isMachineKeepalivedMaster(childMachines.Items[i]) { hasKeepalivedMaster = true } @@ -326,7 +326,7 @@ func isMachineReady(machine yawolv1beta1.LoadBalancerMachine) bool { return true } -func isMachineMaster(machine yawolv1beta1.LoadBalancerMachine) bool { +func isMachineKeepalivedMaster(machine yawolv1beta1.LoadBalancerMachine) bool { if machine.Status.Conditions != nil { for _, condition := range *machine.Status.Conditions { if condition.Type == corev1.NodeConditionType(helper.KeepalivedMaster) { @@ -339,7 +339,6 @@ func isMachineMaster(machine yawolv1beta1.LoadBalancerMachine) bool { func (r *LoadBalancerSetReconciler) createMachine(ctx context.Context, set *yawolv1beta1.LoadBalancerSet) error { machineLabels := r.getMachineLabelsFromSet(set) - revision := set.Annotations[helper.RevisionAnnotation] machine := yawolv1beta1.LoadBalancerMachine{ ObjectMeta: metav1.ObjectMeta{ Name: set.Name + "-" + randomString(5), @@ -354,7 +353,7 @@ func (r *LoadBalancerSetReconciler) createMachine(ctx context.Context, set *yawo }, Labels: machineLabels, Annotations: map[string]string{ - helper.RevisionAnnotation: revision, + helper.RevisionAnnotation: set.Annotations[helper.RevisionAnnotation], }, }, Spec: set.Spec.Template.Spec, From 2a9be2526a31334b88aa2fc563fb5cb737eb4e93 Mon Sep 17 00:00:00 2001 From: Tim Ebert Date: Wed, 14 Jun 2023 11:14:05 +0200 Subject: [PATCH 20/29] Rename filesystem variables and members --- .../yawollet/loadbalancer_controller.go | 13 ++++++------- .../yawollet/loadbalancer_controller_test.go | 4 ++-- controllers/yawollet/suite_test.go | 18 +++++++++--------- internal/helper/yawollet.go | 8 ++++---- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/controllers/yawollet/loadbalancer_controller.go b/controllers/yawollet/loadbalancer_controller.go index e7f0ab49..d6575075 100644 --- a/controllers/yawollet/loadbalancer_controller.go +++ b/controllers/yawollet/loadbalancer_controller.go @@ -4,7 +4,6 @@ package controllers import ( "context" "fmt" - "github.com/spf13/afero" "sort" "time" @@ -12,16 +11,16 @@ import ( "github.com/stackitcloud/yawol/internal/helper" "github.com/stackitcloud/yawol/internal/helper/kubernetes" + envoycache "github.com/envoyproxy/go-control-plane/pkg/cache/v3" "github.com/envoyproxy/go-control-plane/pkg/resource/v3" "github.com/go-logr/logr" + "github.com/spf13/afero" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/predicate" - - envoycache "github.com/envoyproxy/go-control-plane/pkg/cache/v3" ) // LoadBalancerReconciler reconciles service Objects with type LoadBalancer @@ -37,7 +36,7 @@ type LoadBalancerReconciler struct { ListenAddress string RequeueTime int KeepalivedStatsFile string - AferoFs afero.Fs + Filesystem afero.Fs } // Reconcile handles reconciliation of loadbalancer object @@ -95,7 +94,7 @@ func (r *LoadBalancerReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - if err := helper.ReconcileLatestRevisionFile(r.AferoFs, lb, lbm); err != nil { + if err := helper.ReconcileLatestRevisionFile(r.Filesystem, lb, lbm); err != nil { return ctrl.Result{}, err } @@ -177,8 +176,8 @@ func (r *LoadBalancerReconciler) reconcile( // SetupWithManager is used by kubebuilder to init the controller loop func (r *LoadBalancerReconciler) SetupWithManager(mgr ctrl.Manager) error { - if r.AferoFs == nil { - r.AferoFs = afero.NewOsFs() + if r.Filesystem == nil { + r.Filesystem = afero.NewOsFs() } return ctrl.NewControllerManagedBy(mgr). For(&yawolv1beta1.LoadBalancer{}). diff --git a/controllers/yawollet/loadbalancer_controller_test.go b/controllers/yawollet/loadbalancer_controller_test.go index 69f27c97..60a252c6 100644 --- a/controllers/yawollet/loadbalancer_controller_test.go +++ b/controllers/yawollet/loadbalancer_controller_test.go @@ -452,7 +452,7 @@ var _ = Describe("check loadbalancer reconcile", Serial, Ordered, func() { When("lb and lbm revision annotation are the same", func() { It("should create yawolKeepalivedFile", func() { Eventually(func() error { - _, err := aferoFs.Stat(helper.YawolKeepalivedFile) + _, err := filesystem.Stat(helper.YawolKeepalivedFile) return err }, TIMEOUT, INTERVAL).Should(Succeed()) @@ -466,7 +466,7 @@ var _ = Describe("check loadbalancer reconcile", Serial, Ordered, func() { }) It("should not create or delete yawolKeepalivedFile", func() { Eventually(func() error { - _, err := aferoFs.Stat(helper.YawolKeepalivedFile) + _, err := filesystem.Stat(helper.YawolKeepalivedFile) if err == nil || !errors.Is(err, fs.ErrNotExist) { return errors.New("keepalived file still exists") } diff --git a/controllers/yawollet/suite_test.go b/controllers/yawollet/suite_test.go index be51d6aa..ef4da535 100644 --- a/controllers/yawollet/suite_test.go +++ b/controllers/yawollet/suite_test.go @@ -43,13 +43,13 @@ const ( ) var ( - cfg *rest.Config - k8sClient client.Client - testEnv *envtest.Environment - envoyCmd *exec.Cmd - ctx context.Context - cancel context.CancelFunc - aferoFs afero.Fs + cfg *rest.Config + k8sClient client.Client + testEnv *envtest.Environment + envoyCmd *exec.Cmd + ctx context.Context + cancel context.CancelFunc + filesystem afero.Fs ) func TestAPIs(t *testing.T) { @@ -61,7 +61,7 @@ func TestAPIs(t *testing.T) { var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - aferoFs = afero.NewMemMapFs() + filesystem = afero.NewMemMapFs() ctx, cancel = context.WithCancel(context.Background()) @@ -142,7 +142,7 @@ var _ = BeforeSuite(func() { EnvoyCache: cache, ListenAddress: "127.0.0.1", RequeueTime: 1, - AferoFs: aferoFs, + Filesystem: filesystem, }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) diff --git a/internal/helper/yawollet.go b/internal/helper/yawollet.go index cd06d8fa..f2bec03d 100644 --- a/internal/helper/yawollet.go +++ b/internal/helper/yawollet.go @@ -895,20 +895,20 @@ func EnableAdHocDebugging( // ReconcileLatestRevisionFile makes sure that the YawolKeepalivedFile exists if the lbm is from the current revision. // Otherwise, make sure that the file is deleted. // Use aferoFs to use it in tests. -func ReconcileLatestRevisionFile(aferoFs afero.Fs, lb *yawolv1beta1.LoadBalancer, lbm *yawolv1beta1.LoadBalancerMachine) error { - if err := aferoFs.MkdirAll(YawolKeepalivedDir, 0755); err != nil { +func ReconcileLatestRevisionFile(filesystem afero.Fs, lb *yawolv1beta1.LoadBalancer, lbm *yawolv1beta1.LoadBalancerMachine) error { + if err := filesystem.MkdirAll(YawolKeepalivedDir, 0755); err != nil { return err } if lbmIsLatestRevision(lb, lbm) { // file should exist - f, err := aferoFs.Create(YawolKeepalivedFile) + f, err := filesystem.Create(YawolKeepalivedFile) if err != nil { return err } return f.Close() } // file should not exist - err := aferoFs.Remove(YawolKeepalivedFile) + err := filesystem.Remove(YawolKeepalivedFile) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } From b8a4b0c022cfdedeb3d0fb250abfc6ed57568ec5 Mon Sep 17 00:00:00 2001 From: Tim Ebert Date: Wed, 14 Jun 2023 11:16:49 +0200 Subject: [PATCH 21/29] Move `/var/yawol` to `/var/lib/yawol` --- .../yawollet/loadbalancer_controller_test.go | 4 ++-- internal/helper/const.go | 18 +++++++++--------- internal/helper/loadbalancermachine.go | 2 +- internal/helper/yawollet.go | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/controllers/yawollet/loadbalancer_controller_test.go b/controllers/yawollet/loadbalancer_controller_test.go index 60a252c6..3dc70ee3 100644 --- a/controllers/yawollet/loadbalancer_controller_test.go +++ b/controllers/yawollet/loadbalancer_controller_test.go @@ -452,7 +452,7 @@ var _ = Describe("check loadbalancer reconcile", Serial, Ordered, func() { When("lb and lbm revision annotation are the same", func() { It("should create yawolKeepalivedFile", func() { Eventually(func() error { - _, err := filesystem.Stat(helper.YawolKeepalivedFile) + _, err := filesystem.Stat(helper.YawolSetIsLatestRevisionFile) return err }, TIMEOUT, INTERVAL).Should(Succeed()) @@ -466,7 +466,7 @@ var _ = Describe("check loadbalancer reconcile", Serial, Ordered, func() { }) It("should not create or delete yawolKeepalivedFile", func() { Eventually(func() error { - _, err := filesystem.Stat(helper.YawolKeepalivedFile) + _, err := filesystem.Stat(helper.YawolSetIsLatestRevisionFile) if err == nil || !errors.Is(err, fs.ErrNotExist) { return errors.New("keepalived file still exists") } diff --git a/internal/helper/const.go b/internal/helper/const.go index 2609f155..709d7179 100644 --- a/internal/helper/const.go +++ b/internal/helper/const.go @@ -3,13 +3,13 @@ package helper import "time" const ( - OpenstackReconcileTime = 5 * time.Minute - DefaultRequeueTime = 10 * time.Millisecond - RevisionAnnotation = "loadbalancer.yawol.stackit.cloud/revision" - YawolKeepalivedDir = "/var/yawol/" - YawolKeepalivedFile = YawolKeepalivedDir + "set_is_latest_revision" - HashLabel = "lbm-template-hash" - LoadBalancerKind = "LoadBalancer" - VRRPInstanceName = "ENVOY" - HasKeepalivedMaster = "HasKeepalivedMaster" + OpenstackReconcileTime = 5 * time.Minute + DefaultRequeueTime = 10 * time.Millisecond + RevisionAnnotation = "loadbalancer.yawol.stackit.cloud/revision" + YawolLibDir = "/var/lib/yawol/" + YawolSetIsLatestRevisionFile = YawolLibDir + "set_is_latest_revision" + HashLabel = "lbm-template-hash" + LoadBalancerKind = "LoadBalancer" + VRRPInstanceName = "ENVOY" + HasKeepalivedMaster = "HasKeepalivedMaster" ) diff --git a/internal/helper/loadbalancermachine.go b/internal/helper/loadbalancermachine.go index 2f753903..8572a5cd 100644 --- a/internal/helper/loadbalancermachine.go +++ b/internal/helper/loadbalancermachine.go @@ -317,7 +317,7 @@ vrrp_track_process envoy { # the yawolfile is used to change priority if loadbalancermachine is from current revision vrrp_track_file yawolfile { - file "` + YawolKeepalivedFile + `" + file "` + YawolSetIsLatestRevisionFile + `" weight 10 } diff --git a/internal/helper/yawollet.go b/internal/helper/yawollet.go index f2bec03d..45b22d9f 100644 --- a/internal/helper/yawollet.go +++ b/internal/helper/yawollet.go @@ -892,23 +892,23 @@ func EnableAdHocDebugging( return nil } -// ReconcileLatestRevisionFile makes sure that the YawolKeepalivedFile exists if the lbm is from the current revision. +// ReconcileLatestRevisionFile makes sure that the YawolSetIsLatestRevisionFile exists if the lbm is from the current revision. // Otherwise, make sure that the file is deleted. // Use aferoFs to use it in tests. func ReconcileLatestRevisionFile(filesystem afero.Fs, lb *yawolv1beta1.LoadBalancer, lbm *yawolv1beta1.LoadBalancerMachine) error { - if err := filesystem.MkdirAll(YawolKeepalivedDir, 0755); err != nil { + if err := filesystem.MkdirAll(YawolLibDir, 0755); err != nil { return err } if lbmIsLatestRevision(lb, lbm) { // file should exist - f, err := filesystem.Create(YawolKeepalivedFile) + f, err := filesystem.Create(YawolSetIsLatestRevisionFile) if err != nil { return err } return f.Close() } // file should not exist - err := filesystem.Remove(YawolKeepalivedFile) + err := filesystem.Remove(YawolSetIsLatestRevisionFile) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } From 34080fe7b13d94f759747f1c8747aaa32d9c4e34 Mon Sep 17 00:00:00 2001 From: Tim Ebert Date: Wed, 14 Jun 2023 11:22:43 +0200 Subject: [PATCH 22/29] Make linter happy --- .../loadbalancerset_controller.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go index d11eb2f1..56188ff4 100644 --- a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go +++ b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go @@ -3,7 +3,6 @@ package loadbalancerset import ( "context" "crypto/rand" - "encoding/json" "fmt" "time" @@ -19,7 +18,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" @@ -157,7 +155,6 @@ func (r *LoadBalancerSetReconciler) patchStatus( readyMachinesCount int, hasKeepalivedMaster bool, ) (ctrl.Result, error) { - setCopy := set.DeepCopy() // Write replicas into status @@ -244,19 +241,6 @@ func findFirstMachineForDeletion(machines []yawolv1beta1.LoadBalancerMachine) (y return machines[0], nil } -func (r *LoadBalancerSetReconciler) patchLoadBalancerSetStatus( - ctx context.Context, - lbs *yawolv1beta1.LoadBalancerSet, - status yawolv1beta1.LoadBalancerSetStatus, -) error { - statusJSON, err := json.Marshal(status) - if err != nil { - return err - } - patch := []byte(`{"status":` + string(statusJSON) + `}`) - return r.Client.Status().Patch(ctx, lbs, client.RawPatch(types.MergePatchType, patch)) -} - // Decides whether the machine should be deleted or not // True if created before 10 minutes and no condition added yet // True if LastHeartbeatTime is > 5 minutes From 3bbeefa434f63590148c02837362610589eebe4e Mon Sep 17 00:00:00 2001 From: Kai Kummerer Date: Wed, 14 Jun 2023 11:35:28 +0200 Subject: [PATCH 23/29] Improve `Should eventually have a master` test --- .../loadbalancerset_controller_test.go | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go index 8a32239f..dbef08ac 100644 --- a/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go +++ b/controllers/yawol-controller/loadbalancerset/loadbalancerset_controller_test.go @@ -9,6 +9,7 @@ import ( "time" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" @@ -16,6 +17,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" yawolv1beta1 "github.com/stackitcloud/yawol/api/v1beta1" "github.com/stackitcloud/yawol/internal/helper" @@ -107,6 +109,17 @@ var _ = Describe("LoadBalancerSet controller", Serial, Ordered, func() { Eventually(isValidMachine(&setStub, &machine)).Should(BeTrue()) }) It("Should eventually have a master", func() { + By("Check current set condition") + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(&setStub), &setStub)).To(Succeed()) + Expect(setStub.Status.Conditions).To(HaveLen(1)) + + Expect(setStub.Status.Conditions).To(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Type": Equal(helper.HasKeepalivedMaster), + "Status": Equal(metav1.ConditionFalse), + }))) + + oldLastTransitionTime := meta.FindStatusCondition(setStub.Status.Conditions, helper.HasKeepalivedMaster).LastTransitionTime + By("setting LBM as master") machine := getChildMachines(ctx, &setStub)[0] patch := client.MergeFrom(machine.DeepCopy()) @@ -119,17 +132,17 @@ var _ = Describe("LoadBalancerSet controller", Serial, Ordered, func() { } Expect(k8sClient.Status().Patch(ctx, &machine, patch)).To(Succeed()) - Eventually(func() metav1.ConditionStatus { + Eventually(func(g Gomega) { var set yawolv1beta1.LoadBalancerSet - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(&setStub), &set)).To(Succeed()) - Expect(set.Status.Conditions).To(Not(BeNil())) - for _, condition := range set.Status.Conditions { - if condition.Type == helper.HasKeepalivedMaster { - return condition.Status - } - } - return metav1.ConditionFalse - }, timeout, interval).Should(Equal(metav1.ConditionTrue)) + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(&setStub), &set)).To(Succeed()) + + g.Expect(set.Status.Conditions).To(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Type": Equal(helper.HasKeepalivedMaster), + "Status": Equal(metav1.ConditionTrue), + }))) + newLastTransitionTime := meta.FindStatusCondition(set.Status.Conditions, helper.HasKeepalivedMaster).LastTransitionTime + g.Expect(newLastTransitionTime).ToNot(Equal(oldLastTransitionTime)) + }, timeout, interval).Should(Succeed()) }) }) From 74ae42652d8f4c9fa76d731d0adad798f5d9f36e Mon Sep 17 00:00:00 2001 From: Maximilian Geberl <48486938+dergeberl@users.noreply.github.com> Date: Wed, 14 Jun 2023 12:27:29 +0200 Subject: [PATCH 24/29] create /var/lib/yawol folder in image --- image/install-alpine.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/image/install-alpine.yaml b/image/install-alpine.yaml index ac89e33a..cad084b7 100644 --- a/image/install-alpine.yaml +++ b/image/install-alpine.yaml @@ -79,6 +79,14 @@ state: directory mode: '0755' + - name: Create a /var/lib directory for yawol + file: + path: /var/lib/yawol + owner: yawol + group: yawol + state: directory + mode: '0755' + - name: add sudoers file for yawol copy: src: ./yawol-sudoers From 5d32018eb2fc5c5eb61147dcf9d53b0387347d05 Mon Sep 17 00:00:00 2001 From: Maximilian Geberl <48486938+dergeberl@users.noreply.github.com> Date: Wed, 14 Jun 2023 12:28:12 +0200 Subject: [PATCH 25/29] add printcolumn for lbs --- api/v1beta1/loadbalancerset_types.go | 1 + .../crds/yawol.stackit.cloud_loadbalancersets.yaml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/api/v1beta1/loadbalancerset_types.go b/api/v1beta1/loadbalancerset_types.go index 25447f40..52bf5776 100644 --- a/api/v1beta1/loadbalancerset_types.go +++ b/api/v1beta1/loadbalancerset_types.go @@ -11,6 +11,7 @@ import ( // +kubebuilder:printcolumn:name="DESIRED",type=string,JSONPath=`.spec.replicas` // +kubebuilder:printcolumn:name="CURRENT",type=string,JSONPath=`.status.replicas` // +kubebuilder:printcolumn:name="READY",type=string,JSONPath=`.status.readyReplicas` +// +kubebuilder:printcolumn:name="HasKeepalivedMaster",type=string,JSONPath=`.status.conditions[?(@.type=="HasKeepalivedMaster")].status` // +kubebuilder:printcolumn:name="AGE",type=date,JSONPath=`.metadata.creationTimestamp` // LoadBalancerSet is the Schema for the LoadBalancerSet's API. diff --git a/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml b/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml index d82ec301..a92daef7 100644 --- a/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml +++ b/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml @@ -27,6 +27,9 @@ spec: - jsonPath: .status.readyReplicas name: READY type: string + - jsonPath: .status.conditions[?(@.type=="HasKeepalivedMaster")].status + name: HasKeepalivedMaster + type: string - jsonPath: .metadata.creationTimestamp name: AGE type: date From bc9a2ac6a41fed9a0be1d209969f8924e9282e80 Mon Sep 17 00:00:00 2001 From: Tim Ebert Date: Wed, 14 Jun 2023 12:28:20 +0200 Subject: [PATCH 26/29] Add missing yawollet permissions for `PATCH events` --- internal/helper/yawollet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/helper/yawollet.go b/internal/helper/yawollet.go index 45b22d9f..7b2ccd24 100644 --- a/internal/helper/yawollet.go +++ b/internal/helper/yawollet.go @@ -446,7 +446,7 @@ func GetRoleRules( loadBalancerMachine *yawolv1beta1.LoadBalancerMachine, ) []rbac.PolicyRule { return []rbac.PolicyRule{{ - Verbs: []string{"create"}, + Verbs: []string{"create", "patch"}, APIGroups: []string{""}, Resources: []string{"events"}, }, { From d4637bbf884b6c2b52d18fe3c4660960bf10b5a6 Mon Sep 17 00:00:00 2001 From: Maximilian Geberl <48486938+dergeberl@users.noreply.github.com> Date: Wed, 14 Jun 2023 12:44:35 +0200 Subject: [PATCH 27/29] write on 1 in yawol RevisionFile --- internal/helper/yawollet.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/helper/yawollet.go b/internal/helper/yawollet.go index 7b2ccd24..8cab4186 100644 --- a/internal/helper/yawollet.go +++ b/internal/helper/yawollet.go @@ -905,6 +905,9 @@ func ReconcileLatestRevisionFile(filesystem afero.Fs, lb *yawolv1beta1.LoadBalan if err != nil { return err } + if _, err = f.WriteString("1"); err != nil { + return err + } return f.Close() } // file should not exist From 6e38e48b1d89914fa537b1c1aa6600c502a426fc Mon Sep 17 00:00:00 2001 From: Maximilian Geberl <48486938+dergeberl@users.noreply.github.com> Date: Wed, 14 Jun 2023 12:57:36 +0200 Subject: [PATCH 28/29] rename vrrp_track_file to track_file --- internal/helper/loadbalancermachine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/helper/loadbalancermachine.go b/internal/helper/loadbalancermachine.go index 8572a5cd..34bbed6e 100644 --- a/internal/helper/loadbalancermachine.go +++ b/internal/helper/loadbalancermachine.go @@ -316,7 +316,7 @@ vrrp_track_process envoy { } # the yawolfile is used to change priority if loadbalancermachine is from current revision -vrrp_track_file yawolfile { +track_file yawolfile { file "` + YawolSetIsLatestRevisionFile + `" weight 10 } From a46f4ff02048659622d960156e7865feae22c742 Mon Sep 17 00:00:00 2001 From: Maximilian Geberl <48486938+dergeberl@users.noreply.github.com> Date: Wed, 14 Jun 2023 13:03:37 +0200 Subject: [PATCH 29/29] write on 1 in yawol RevisionFile --- internal/helper/yawollet.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/helper/yawollet.go b/internal/helper/yawollet.go index 8cab4186..b226940a 100644 --- a/internal/helper/yawollet.go +++ b/internal/helper/yawollet.go @@ -905,10 +905,11 @@ func ReconcileLatestRevisionFile(filesystem afero.Fs, lb *yawolv1beta1.LoadBalan if err != nil { return err } + defer f.Close() if _, err = f.WriteString("1"); err != nil { return err } - return f.Close() + return nil } // file should not exist err := filesystem.Remove(YawolSetIsLatestRevisionFile)