Skip to content

Commit

Permalink
Make the kube proxy upgrader re-entrant
Browse files Browse the repository at this point in the history
  • Loading branch information
@g-gaston
g-gaston committed Mar 23, 2023
1 parent 6930780 commit 5ab642f
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 49 deletions.
109 changes: 60 additions & 49 deletions pkg/clustermanager/kube_proxy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,33 +100,31 @@ func PrepareKubeProxyForUpgrade(ctx context.Context, log logr.Logger, management
}

// Add label to nodes so we can use nodeAffinity to control the kube-proxy scheduling
if err := addIPTablesLEgacyLabelToAllNodes(ctx, log, workloadClusterClient); err != nil {
if err := addIPTablesLegacyLabelToAllNodes(ctx, log, workloadClusterClient); err != nil {
return err
}

kubeProxy, err := getKubeProxy(ctx, workloadClusterClient)
originalKubeProxy, err := getKubeProxy(ctx, workloadClusterClient)
if err != nil {
return err
}

// Generate the iptables legacy kube-proxy before we modify the kubeProxy object
iptablesLegacyKubeProxy := iptablesLegacyKubeProxyFromCurrentDaemonSet(kubeProxy)

// Make sure original kube-proxy DS is only scheduled in new nodes and it stops running in current nodes.
if err := restrictKubeProxyToNewNodes(ctx, workloadClusterClient, kubeProxy); err != nil {
updatedKubeProxy, err := restrictKubeProxyToNewNodes(ctx, workloadClusterClient, originalKubeProxy)
if err != nil {
return err
}

// Once old kube-proxy pods are deleted, create the new DS that will only be scheduled in the old nodes.
if err := workloadClusterClient.Create(ctx, iptablesLegacyKubeProxy); err != nil {
return errors.Wrap(err, "creating secondary kube-proxy DS with iptables-legacy for old nodes")
if err := createIPTablesLegacyKubeProxy(ctx, workloadClusterClient, kcp, originalKubeProxy); err != nil {
return err
}

// Finally update the main kube-proxy DS to reflect the new version so all the new nodes
// get that one scheduled from the beggining.
// get that one scheduled from the beginning.
log.V(4).Info("Updating kube-proxy DS version", "oldVersion", kcp.Spec.Version, "newVersion", spec.VersionsBundle.KubeDistro.KubeProxy.URI)
kubeProxy.Spec.Template.Spec.Containers[0].Image = spec.VersionsBundle.KubeDistro.KubeProxy.URI
if err := workloadClusterClient.Update(ctx, kubeProxy); err != nil {
updatedKubeProxy.Spec.Template.Spec.Containers[0].Image = spec.VersionsBundle.KubeDistro.KubeProxy.URI
if err := workloadClusterClient.Update(ctx, updatedKubeProxy); err != nil {
return errors.Wrap(err, "updating main kube-proxy version before upgrade")
}
return nil
Expand Down Expand Up @@ -171,7 +169,7 @@ func needsKubeProxyPreUpgrade(spec *cluster.Spec, currentKCP *controlplanev1.Kub
return specIncludesNewKubeProxy(spec) && !eksdIncludesNewKubeProxy(currentKubeVersion, currentEKSDNumber), nil
}

func addIPTablesLEgacyLabelToAllNodes(ctx context.Context, log logr.Logger, client client.Client) error {
func addIPTablesLegacyLabelToAllNodes(ctx context.Context, log logr.Logger, client client.Client) error {
nodeList := &corev1.NodeList{}
if err := client.List(ctx, nodeList); err != nil {
return errors.Wrap(err, "listing workload cluster nodes for kube-proxy upgrade")
Expand All @@ -194,32 +192,14 @@ func addIPTablesLEgacyLabelToAllNodes(ctx context.Context, log logr.Logger, clie
return nil
}

func iptablesLegacyKubeProxyFromCurrentDaemonSet(kubeProxy *appsv1.DaemonSet) *appsv1.DaemonSet {
iptablesLegacyKubeProxy := kubeProxy.DeepCopy()

// Generate a new DS with the old kube-proxy version with nodeAffinity so it only
// gets scheduled in the old (current) nodes.
iptablesLegacyKubeProxy.Name = "kube-proxy-iptables-legacy"
iptablesLegacyKubeProxy.ObjectMeta.ResourceVersion = ""
iptablesLegacyKubeProxy.ObjectMeta.UID = ""
iptablesLegacyKubeProxy.Spec.Template.Spec.Affinity = &corev1.Affinity{
NodeAffinity: &corev1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: iptablesLegacyLabel,
Operator: corev1.NodeSelectorOpExists,
},
},
},
},
},
},
func getKubeProxy(ctx context.Context, c client.Client) (*appsv1.DaemonSet, error) {
kubeProxy := &appsv1.DaemonSet{}
kubeProxyKey := client.ObjectKey{Name: kubeProxyDSName, Namespace: kubeProxyDSNamespace}
if err := c.Get(ctx, kubeProxyKey, kubeProxy); err != nil {
return nil, errors.Wrap(err, "reading kube-proxy for upgrade")
}

return iptablesLegacyKubeProxy
return kubeProxy, nil
}

func addAntiNodeAffinityToKubeProxy(ctx context.Context, client client.Client, kubeProxy *appsv1.DaemonSet) error {
Expand Down Expand Up @@ -259,26 +239,57 @@ func deleteAllOriginalKubeProxyPods(ctx context.Context, c client.Client) error
return nil
}

func getKubeProxy(ctx context.Context, c client.Client) (*appsv1.DaemonSet, error) {
kubeProxy := &appsv1.DaemonSet{}
kubeProxyKey := client.ObjectKey{Name: kubeProxyDSName, Namespace: kubeProxyDSNamespace}
if err := c.Get(ctx, kubeProxyKey, kubeProxy); err != nil {
return nil, errors.Wrap(err, "reading kube-proxy for upgrade")
}

return kubeProxy, nil
}

func restrictKubeProxyToNewNodes(ctx context.Context, client client.Client, kubeProxy *appsv1.DaemonSet) error {
func restrictKubeProxyToNewNodes(ctx context.Context, client client.Client, kubeProxy *appsv1.DaemonSet) (*appsv1.DaemonSet, error) {
kubeProxy = kubeProxy.DeepCopy()
// Add nodeAffinity to kube-proxy so it's not scheduled in new nodes without our label
if err := addAntiNodeAffinityToKubeProxy(ctx, client, kubeProxy); err != nil {
return err
return nil, err
}

// Delete original kube-proxy pods to ensure there is only one copy of kube-proxy running
// on each node.
if err := deleteAllOriginalKubeProxyPods(ctx, client); err != nil {
return err
return nil, err
}

return getKubeProxy(ctx, client)
}

func iptablesLegacyKubeProxyFromCurrentDaemonSet(kcp *controlplanev1.KubeadmControlPlane, kubeProxy *appsv1.DaemonSet) *appsv1.DaemonSet {
iptablesLegacyKubeProxy := kubeProxy.DeepCopy()

// Generate a new DS with the old kube-proxy version with nodeAffinity so it only
// gets scheduled in the old (current) nodes.
iptablesLegacyKubeProxy.Name = "kube-proxy-iptables-legacy"
iptablesLegacyKubeProxy.ObjectMeta.ResourceVersion = ""
iptablesLegacyKubeProxy.ObjectMeta.UID = ""
image := kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.ImageRepository + "/kube-proxy" +
":" + kcp.Spec.Version
iptablesLegacyKubeProxy.Spec.Template.Spec.Containers[0].Image = image
iptablesLegacyKubeProxy.Spec.Template.Spec.Affinity = &corev1.Affinity{
NodeAffinity: &corev1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: iptablesLegacyLabel,
Operator: corev1.NodeSelectorOpExists,
},
},
},
},
},
},
}

return iptablesLegacyKubeProxy
}

func createIPTablesLegacyKubeProxy(ctx context.Context, client client.Client, kcp *controlplanev1.KubeadmControlPlane, originalKubeProxy *appsv1.DaemonSet) error {
iptablesLegacyKubeProxy := iptablesLegacyKubeProxyFromCurrentDaemonSet(kcp, originalKubeProxy)
if err := client.Create(ctx, iptablesLegacyKubeProxy); err != nil && !apierrors.IsAlreadyExists(err) {
return errors.Wrap(err, "creating secondary kube-proxy DS with iptables-legacy for old nodes")
}

return nil
Expand Down
6 changes: 6 additions & 0 deletions pkg/clustermanager/kube_proxy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
Expand Down Expand Up @@ -58,6 +59,11 @@ func newPrepareKubeProxyTest() *prepareKubeProxyTest {
},
Spec: controlplanev1.KubeadmControlPlaneSpec{
Version: "v1.23.16-eks-1-23-16",
KubeadmConfigSpec: v1beta1.KubeadmConfigSpec{
ClusterConfiguration: &v1beta1.ClusterConfiguration{
ImageRepository: "public.ecr.aws/eks-distro/kubernetes",
},
},
},
}
tt.kubeProxy = &appsv1.DaemonSet{
Expand Down

0 comments on commit 5ab642f

Please sign in to comment.