Skip to content

Commit

Permalink
fix(kuma-cni): support port exclusion for UIDs (#8319)
Browse files Browse the repository at this point in the history
* fix(kuma-cni): support for 'traffic.kuma.io/exclude-outbound-tcp-ports-for-uids' and 'traffic.kuma.io/exclude-outbound-udp-ports-for-uids' annotations

Signed-off-by: Ilya Lobkov <ilya.lobkov@konghq.com>

* fix(kuma-cni): make check

Signed-off-by: Ilya Lobkov <ilya.lobkov@konghq.com>

* fix(kuma-cni): fix test

Signed-off-by: Ilya Lobkov <ilya.lobkov@konghq.com>

* fix(kuma-cni): remove node cleanup

Signed-off-by: Ilya Lobkov <ilya.lobkov@konghq.com>

* fix(kuma-cni): replace with a single annotation

Signed-off-by: Ilya Lobkov <ilya.lobkov@konghq.com>

* fix(kuma-cni): add comments, pass 'kuma.io/sidecar-uid' to CNI

Signed-off-by: Ilya Lobkov <ilya.lobkov@konghq.com>

---------

Signed-off-by: Ilya Lobkov <ilya.lobkov@konghq.com>
  • Loading branch information
lobkovilya authored Nov 13, 2023
1 parent 3550457 commit c06cc52
Show file tree
Hide file tree
Showing 7 changed files with 174 additions and 34 deletions.
63 changes: 35 additions & 28 deletions app/cni/pkg/cni/annotations.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,34 @@ const (
)

var annotationRegistry = map[string]*annotationParam{
"inject": {"kuma.io/sidecar-injection", "", alwaysValidFunc},
"ports": {"kuma.io/envoy-admin-port", "", validatePortList},
"excludeInboundPorts": {"traffic.kuma.io/exclude-inbound-ports", defaultRedirectExcludePort, validatePortList},
"excludeOutboundPorts": {"traffic.kuma.io/exclude-outbound-ports", defaultRedirectExcludePort, validatePortList},
"inboundPort": {"kuma.io/transparent-proxying-inbound-port", defaultInboundPort, validatePortList},
"inboundPortV6": {"kuma.io/transparent-proxying-inbound-v6-port", defaultInboundPortV6, validatePortList},
"outboundPort": {"kuma.io/transparent-proxying-outbound-port", defaultOutboundPort, validatePortList},
"isGateway": {"kuma.io/gateway", "false", alwaysValidFunc},
"builtinDNS": {"kuma.io/builtin-dns", "false", alwaysValidFunc},
"builtinDNSPort": {"kuma.io/builtin-dns-port", defaultBuiltinDNSPort, validatePortList},
"inject": {"kuma.io/sidecar-injection", "", alwaysValidFunc},
"ports": {"kuma.io/envoy-admin-port", "", validatePortList},
"excludeInboundPorts": {"traffic.kuma.io/exclude-inbound-ports", defaultRedirectExcludePort, validatePortList},
"excludeOutboundPorts": {"traffic.kuma.io/exclude-outbound-ports", defaultRedirectExcludePort, validatePortList},
"inboundPort": {"kuma.io/transparent-proxying-inbound-port", defaultInboundPort, validatePortList},
"inboundPortV6": {"kuma.io/transparent-proxying-inbound-v6-port", defaultInboundPortV6, validatePortList},
"outboundPort": {"kuma.io/transparent-proxying-outbound-port", defaultOutboundPort, validatePortList},
"isGateway": {"kuma.io/gateway", "false", alwaysValidFunc},
"builtinDNS": {"kuma.io/builtin-dns", "false", alwaysValidFunc},
"builtinDNSPort": {"kuma.io/builtin-dns-port", defaultBuiltinDNSPort, validatePortList},
"excludeOutboundPortsForUIDs": {"traffic.kuma.io/exclude-outbound-ports-for-uids", "", alwaysValidFunc},
"noRedirectUID": {"kuma.io/sidecar-uid", defaultNoRedirectUID, alwaysValidFunc},
}

type IntermediateConfig struct {
targetPort string
inboundPort string
inboundPortV6 string
noRedirectUID string
excludeInboundPorts string
excludeOutboundPorts string
isGateway string
builtinDNS string
builtinDNSPort string
// while https://github.com/kumahq/kuma/issues/8324 is not implemented, when changing the config,
// keep in mind to update all other places listed in the issue

targetPort string
inboundPort string
inboundPortV6 string
noRedirectUID string
excludeInboundPorts string
excludeOutboundPorts string
excludeOutboundPortsForUIDs string
isGateway string
builtinDNS string
builtinDNSPort string
}

type annotationValidationFunc func(value string) error
Expand Down Expand Up @@ -107,17 +113,18 @@ func getAnnotationOrDefault(name string, annotations map[string]string) (string,
// NewIntermediateConfig returns a new IntermediateConfig Object constructed from a list of ports and annotations
func NewIntermediateConfig(annotations map[string]string) (*IntermediateConfig, error) {
intermediateConfig := &IntermediateConfig{}
intermediateConfig.noRedirectUID = defaultNoRedirectUID

allFields := map[string]*string{
"outboundPort": &intermediateConfig.targetPort,
"inboundPort": &intermediateConfig.inboundPort,
"inboundPortV6": &intermediateConfig.inboundPortV6,
"excludeInboundPorts": &intermediateConfig.excludeInboundPorts,
"excludeOutboundPorts": &intermediateConfig.excludeOutboundPorts,
"isGateway": &intermediateConfig.isGateway,
"builtinDNS": &intermediateConfig.builtinDNS,
"builtinDNSPort": &intermediateConfig.builtinDNSPort,
"outboundPort": &intermediateConfig.targetPort,
"inboundPort": &intermediateConfig.inboundPort,
"inboundPortV6": &intermediateConfig.inboundPortV6,
"excludeInboundPorts": &intermediateConfig.excludeInboundPorts,
"excludeOutboundPorts": &intermediateConfig.excludeOutboundPorts,
"isGateway": &intermediateConfig.isGateway,
"builtinDNS": &intermediateConfig.builtinDNS,
"builtinDNSPort": &intermediateConfig.builtinDNSPort,
"excludeOutboundPortsForUIDs": &intermediateConfig.excludeOutboundPortsForUIDs,
"noRedirectUID": &intermediateConfig.noRedirectUID,
}

for fieldName, fieldPointer := range allFields {
Expand Down
24 changes: 24 additions & 0 deletions app/cni/pkg/cni/annotations_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package cni

import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var _ = Describe("NewIntermediateConfig", func() {
It("should set UID to default value if annotation is not specified", func() {
a := map[string]string{}
cfg, err := NewIntermediateConfig(a)
Expect(err).ToNot(HaveOccurred())
Expect(cfg.noRedirectUID).To(Equal(defaultNoRedirectUID))
})

It("should override UID when annotation is specified", func() {
a := map[string]string{
"kuma.io/sidecar-uid": "1234",
}
cfg, err := NewIntermediateConfig(a)
Expect(err).ToNot(HaveOccurred())
Expect(cfg.noRedirectUID).To(Equal("1234"))
})
})
18 changes: 15 additions & 3 deletions app/cni/pkg/cni/injector_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,16 +76,28 @@ func mapToConfig(intermediateConfig *IntermediateConfig, logWriter *bufio.Writer
if err != nil {
return nil, err
}

excludePortsForUIDs := []string{}
if intermediateConfig.excludeOutboundPortsForUIDs != "" {
excludePortsForUIDs = strings.Split(intermediateConfig.excludeOutboundPortsForUIDs, ";")
}

excludePortsForUIDsParsed, err := transparentproxy.ParseExcludePortsForUIDs(excludePortsForUIDs)
if err != nil {
return nil, err
}

cfg := config.Config{
RuntimeStdout: logWriter,
Owner: config.Owner{
UID: intermediateConfig.noRedirectUID,
},
Redirect: config.Redirect{
Outbound: config.TrafficFlow{
Enabled: true,
Port: port,
ExcludePorts: excludePorts,
Enabled: true,
Port: port,
ExcludePorts: excludePorts,
ExcludePortsForUIDs: excludePortsForUIDsParsed,
},
},
}
Expand Down
3 changes: 3 additions & 0 deletions pkg/transparentproxy/kubernetes/kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ import (
)

type PodRedirect struct {
// while https://github.com/kumahq/kuma/issues/8324 is not implemented, when changing the config,
// keep in mind to update all other places listed in the issue

BuiltinDNSEnabled bool
BuiltinDNSPort uint32
ExcludeOutboundPorts string
Expand Down
4 changes: 2 additions & 2 deletions pkg/transparentproxy/transparentproxy_v2.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ func (tp *TransparentProxyV2) Setup(tpConfig *config.TransparentProxyConfig) (st
}
var excludeOutboundPortsForUids []config.UIDsToPorts
if len(tpConfig.ExcludedOutboundsForUIDs) > 0 {
excludeOutboundPortsForUids, err = parseExcludePortsForUIDs(tpConfig.ExcludedOutboundsForUIDs)
excludeOutboundPortsForUids, err = ParseExcludePortsForUIDs(tpConfig.ExcludedOutboundsForUIDs)
if err != nil {
return "", errors.Wrap(err, "parsing excluded outbound ports for uids failed")
}
Expand Down Expand Up @@ -194,7 +194,7 @@ func (tp *TransparentProxyV2) Setup(tpConfig *config.TransparentProxyConfig) (st
return Setup(cfg)
}

func parseExcludePortsForUIDs(excludeOutboundPortsForUIDs []string) ([]config.UIDsToPorts, error) {
func ParseExcludePortsForUIDs(excludeOutboundPortsForUIDs []string) ([]config.UIDsToPorts, error) {
var uidsToPorts []config.UIDsToPorts
for _, excludePort := range excludeOutboundPortsForUIDs {
parts := strings.Split(excludePort, ":")
Expand Down
5 changes: 4 additions & 1 deletion test/e2e/cni/e2e_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,7 @@ func TestE2E(t *testing.T) {
test.RunE2ESpecs(t, "E2E CNI Suite")
}

var _ = Describe("Taint controller", Label("job-0"), Label("kind-not-supported"), Label("legacy-k3s-not-supported"), cni.AppDeploymentWithCniAndTaintController)
var (
_ = Describe("Taint controller", Label("job-0"), Label("kind-not-supported"), Label("legacy-k3s-not-supported"), cni.AppDeploymentWithCniAndTaintController)
_ = Describe("Connectivity - Exclude Outbound Port", Label("job-0"), Label("kind-not-supported"), Label("legacy-k3s-not-supported"), cni.ExcludeOutboundPort, Ordered)
)
91 changes: 91 additions & 0 deletions test/e2e/cni/exclude_outbound_port.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package cni

import (
"fmt"
"strings"
"time"

"github.com/gruntwork-io/terratest/modules/random"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"

"github.com/kumahq/kuma/pkg/config/core"
"github.com/kumahq/kuma/pkg/plugins/runtime/k8s/metadata"
"github.com/kumahq/kuma/pkg/util/pointer"
. "github.com/kumahq/kuma/test/framework"
"github.com/kumahq/kuma/test/framework/deployments/testserver"
)

func ExcludeOutboundPort() {
meshName := "exclude-outbound-port"

namespace := "exclude-outbound-port"
namespaceExternal := "exclude-outbound-port-external"

var cluster Cluster
var k8sCluster *K8sCluster

BeforeAll(func() {
k8sCluster = NewK8sCluster(NewTestingT(), Kuma1, Silent)
cluster = k8sCluster.
WithTimeout(6 * time.Second).
WithRetries(60)

releaseName := fmt.Sprintf("kuma-%s", strings.ToLower(random.UniqueId()))

Expect(NewClusterSetup().
Install(Kuma(core.Standalone,
WithInstallationMode(HelmInstallationMode),
WithHelmReleaseName(releaseName),
WithSkipDefaultMesh(true), // it's common case for HELM deployments that Mesh is also managed by HELM therefore it's not created by default
WithHelmOpt("cni.delayStartupSeconds", "40"),
WithHelmOpt("cni.logLevel", "debug"),
WithCNI(),
)).
Install(MTLSMeshKubernetes(meshName)).
Install(NamespaceWithSidecarInjection(namespace)).
Install(Namespace(namespaceExternal)).
Install(testserver.Install(
testserver.WithName("test-server"),
testserver.WithMesh(meshName),
testserver.WithNamespace(namespaceExternal),
)).
Setup(cluster)).To(Succeed())
})

E2EAfterAll(func() {
Expect(cluster.DeleteNamespace(namespace)).To(Succeed())
Expect(cluster.DeleteNamespace(namespaceExternal)).To(Succeed())
Expect(cluster.DeleteKuma()).To(Succeed())
Expect(cluster.DismissCluster()).To(Succeed())
})

It("should be able to use network from init container if we ignore ports for uid", func() {
Expect(cluster.Install(testserver.Install(
testserver.WithName("test-server"),
testserver.WithMesh(meshName),
testserver.WithNamespace(namespace),
testserver.WithPodAnnotations(map[string]string{
metadata.KumaTrafficExcludeOutboundPortsForUIDs: "tcp:80:1234;udp:53:1234",
}),
testserver.AddInitContainer(corev1.Container{
Name: "init-test-server",
Image: Config.GetUniversalImage(),
ImagePullPolicy: "IfNotPresent",
Command: []string{"curl"},
Args: []string{"-v", "-m", "3", "--fail", "test-server.exclude-outbound-port-external.svc.cluster.local:80"},
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
"cpu": resource.MustParse("50m"),
"memory": resource.MustParse("64Mi"),
},
},
SecurityContext: &corev1.SecurityContext{
RunAsUser: pointer.To(int64(1234)),
},
}),
))).To(Succeed())
})
}

0 comments on commit c06cc52

Please sign in to comment.