From cb88bc67e476d1f48dedec4d9f9d9b04155cd036 Mon Sep 17 00:00:00 2001 From: Aaron U'Ren Date: Fri, 8 Dec 2023 14:48:53 -0600 Subject: [PATCH 1/3] k8s-1.12.yaml.template: update kube-router * Update version v1.6.0 -> v2.1.0 * expose container runtime socket to kube-router * kube-router loadbalancer controller * Add access / information that is needed by the load balancer controller in kube-router. * add access to endpoint slices for services controller * enable hairpin mode on kube-router * There are integration tests in kops that expect that hairpin functionality is always enabled, as such we now enable hairpin-mode in kube-router by default. * enable hostPID access * kube-router now requires access to the host's process namespace. See https://github.com/cloudnativelabs/kube-router/pull/1584 for more information. --- tests/e2e/pkg/tester/skip_regex.go | 2 +- .../k8s-1.12.yaml.template | 44 ++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/tests/e2e/pkg/tester/skip_regex.go b/tests/e2e/pkg/tester/skip_regex.go index 45d67d01908f2..eef09f5391dae 100644 --- a/tests/e2e/pkg/tester/skip_regex.go +++ b/tests/e2e/pkg/tester/skip_regex.go @@ -98,7 +98,7 @@ func (t *Tester) setSkipRegexFlag() error { skipRegex += "|should.create.a.Pod.with.SCTP.HostPort" } } else if networking.KubeRouter != nil { - skipRegex += "|load-balancer|hairpin|service\\.kubernetes\\.io|CLOSE_WAIT" + skipRegex += "|load-balancer|service\\.kubernetes\\.io|CLOSE_WAIT" skipRegex += "|EndpointSlice.should.support.a.Service.with.multiple" skipRegex += "|internalTrafficPolicy|externallTrafficPolicy|only.terminating.endpoints" } else if networking.Kubenet != nil { diff --git a/upup/models/cloudup/resources/addons/networking.kuberouter/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/networking.kuberouter/k8s-1.12.yaml.template index 4ff424422b45e..6001e98c35a86 100644 --- a/upup/models/cloudup/resources/addons/networking.kuberouter/k8s-1.12.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.kuberouter/k8s-1.12.yaml.template @@ -62,7 +62,7 @@ spec: serviceAccountName: kube-router containers: - name: kube-router - image: docker.io/cloudnativelabs/kube-router:v1.6.0 + image: docker.io/cloudnativelabs/kube-router:v2.1.0 args: - --run-router=true - --run-firewall=true @@ -70,11 +70,17 @@ spec: - --bgp-graceful-restart=true - --kubeconfig=/var/lib/kube-router/kubeconfig - --metrics-port=12013 + - --runtime-endpoint=unix:///run/containerd/containerd.sock + - --hairpin-mode=true env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name - name: KUBE_ROUTER_CNI_CONF_FILE value: /etc/cni/net.d/10-kuberouter.conflist livenessProbe: @@ -101,6 +107,12 @@ spec: - name: xtables-lock mountPath: /run/xtables.lock readOnly: false + - name: rt-tables + mountPath: /etc/iproute2/rt_tables + readOnly: false + - name: containerd-sock + mountPath: /run/containerd/containerd.sock + readOnly: true initContainers: - name: install-cni image: docker.io/cloudnativelabs/kube-router:v1.6.0 @@ -122,6 +134,7 @@ spec: - mountPath: /etc/kube-router name: kube-router-cfg hostNetwork: true + hostPID: true tolerations: - operator: Exists volumes: @@ -141,6 +154,13 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + - name: rt-tables + hostPath: + path: /etc/iproute2/rt_tables + type: FileOrCreate + - name: containerd-sock + hostPath: + path: /run/containerd/containerd.sock --- apiVersion: v1 kind: ServiceAccount @@ -181,6 +201,28 @@ rules: - get - list - watch + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - get + - create + - update + - apiGroups: + - "" + resources: + - services/status + verbs: + - update + - apiGroups: + - "discovery.k8s.io" + resources: + - endpointslices + verbs: + - get + - list + - watch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 From d3e4d74947981e4a18a11311cd8d68a29b29e9c2 Mon Sep 17 00:00:00 2001 From: Aaron U'Ren Date: Tue, 26 Dec 2023 16:40:48 -0600 Subject: [PATCH 2/3] skip_regex.go: update kube-router regex * kube-router enable service.kubernetes.io tests * kube-router enable load-balancer tests * kube-router enable endpointslice tests --- tests/e2e/pkg/tester/skip_regex.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/e2e/pkg/tester/skip_regex.go b/tests/e2e/pkg/tester/skip_regex.go index eef09f5391dae..61cc6f85aa46b 100644 --- a/tests/e2e/pkg/tester/skip_regex.go +++ b/tests/e2e/pkg/tester/skip_regex.go @@ -98,9 +98,7 @@ func (t *Tester) setSkipRegexFlag() error { skipRegex += "|should.create.a.Pod.with.SCTP.HostPort" } } else if networking.KubeRouter != nil { - skipRegex += "|load-balancer|service\\.kubernetes\\.io|CLOSE_WAIT" - skipRegex += "|EndpointSlice.should.support.a.Service.with.multiple" - skipRegex += "|internalTrafficPolicy|externallTrafficPolicy|only.terminating.endpoints" + skipRegex += "|should set TCP CLOSE_WAIT timeout|should check kube-proxy urls" } else if networking.Kubenet != nil { skipRegex += "|Services.*affinity" } From 821ab186499cbb0fd722b554405db1ba8b0e53c0 Mon Sep 17 00:00:00 2001 From: Aaron U'Ren Date: Sat, 30 Mar 2024 22:19:01 -0500 Subject: [PATCH 3/3] iam_builder.go: ensure kube-router src/dst permissions --- pkg/model/iam/iam_builder.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index 6fee94d5fdc79..49ef3ea99e839 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -468,6 +468,10 @@ func (r *NodeRoleNode) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) { addCalicoSrcDstCheckPermissions(p) } + if b.Cluster.Spec.Networking.KubeRouter != nil { + addKubeRouterSrcDstCheckPermissions(p) + } + return p, nil } @@ -769,6 +773,12 @@ func addCalicoSrcDstCheckPermissions(p *Policy) { ) } +func addKubeRouterSrcDstCheckPermissions(p *Policy) { + p.unconditionalAction.Insert( + "ec2:ModifyInstanceAttribute", + ) +} + func (b *PolicyBuilder) addNodeupPermissions(p *Policy, enableHookSupport bool) { addCertIAMPolicies(p) addKMSGenerateRandomPolicies(p)