From ed1be22d2f46eae00a28d48fa434428034e215dc Mon Sep 17 00:00:00 2001 From: Jakub Dyszkiewicz Date: Fri, 27 Oct 2023 14:23:55 +0200 Subject: [PATCH] feat(xds): auto reachable services based on MeshTrafficPermission (#8125) Signed-off-by: Jakub Dyszkiewicz Signed-off-by: slonka Co-authored-by: slonka --- docs/generated/kuma-cp.md | 3 + docs/generated/raw/kuma-cp.yaml | 3 + pkg/api-server/api_server_suite_test.go | 1 + .../customization/customization_suite_test.go | 1 + pkg/api-server/inspect_endpoints_test.go | 11 +- pkg/api-server/server.go | 1 + .../inspect_dataplane_rules.golden.json | 2 +- ...inspect_dataplane_rules_subset.golden.json | 4 +- pkg/config/app/kuma-cp/config.go | 3 + pkg/config/app/kuma-cp/kuma-cp.defaults.yaml | 3 + pkg/config/loader_test.go | 3 + pkg/core/bootstrap/bootstrap.go | 6 + .../graph/graph_suite_test.go | 11 + .../graph/reachable_services_graph.go | 176 ++++++++ .../graph/reachable_services_graph_test.go | 378 ++++++++++++++++++ pkg/plugins/runtime/gateway/suite_test.go | 1 + .../builders/meshtrafficpermission_builder.go | 4 +- .../resources/builders/targetref_builder.go | 7 + .../resources/builders/zoneingress_builder.go | 17 + pkg/test/runtime/runtime.go | 1 + pkg/xds/cache/mesh/cache_test.go | 1 + pkg/xds/context/context.go | 21 +- pkg/xds/context/mesh_context_builder.go | 24 +- pkg/xds/context/reachable_services_graph.go | 31 ++ pkg/xds/server/v3/snapshot_generator_test.go | 1 + pkg/xds/sync/dataplane_proxy_builder.go | 14 +- pkg/xds/sync/dataplane_watchdog_test.go | 1 + pkg/xds/sync/proxy_builder_test.go | 1 + .../auto_reachable_services_k8s.go | 94 +++++ test/e2e/reachableservices/e2e_suite_test.go | 16 + 30 files changed, 808 insertions(+), 32 deletions(-) create mode 100644 pkg/plugins/policies/meshtrafficpermission/graph/graph_suite_test.go create mode 100644 pkg/plugins/policies/meshtrafficpermission/graph/reachable_services_graph.go create mode 100644 pkg/plugins/policies/meshtrafficpermission/graph/reachable_services_graph_test.go create mode 100644 pkg/xds/context/reachable_services_graph.go create mode 100644 test/e2e/reachableservices/auto_reachable_services_k8s.go create mode 100644 test/e2e/reachableservices/e2e_suite_test.go diff --git a/docs/generated/kuma-cp.md b/docs/generated/kuma-cp.md index 8ffcc24ecc51..e239b960dae5 100644 --- a/docs/generated/kuma-cp.md +++ b/docs/generated/kuma-cp.md @@ -743,6 +743,9 @@ experimental: fullResyncInterval: 60s # ENV: KUMA_EXPERIMENTAL_KDS_EVENT_BASED_WATCHDOG_FULL_RESYNC_INTERVAL # If true, then initial full resync is going to be delayed by 0 to FullResyncInterval. delayFullResync: false # ENV: KUMA_EXPERIMENTAL_KDS_EVENT_BASED_WATCHDOG_DELAY_FULL_RESYNC + # If true then control plane computes reachable services automatically based on MeshTrafficPermission. + # Lack of MeshTrafficPermission is treated as Deny the traffic. + autoReachableServices: false # ENV: KUMA_EXPERIMENTAL_AUTO_REACHABLE_SERVICES proxy: gateway: diff --git a/docs/generated/raw/kuma-cp.yaml b/docs/generated/raw/kuma-cp.yaml index e4441ddd83d7..d645e81e56f7 100644 --- a/docs/generated/raw/kuma-cp.yaml +++ b/docs/generated/raw/kuma-cp.yaml @@ -740,6 +740,9 @@ experimental: fullResyncInterval: 60s # ENV: KUMA_EXPERIMENTAL_KDS_EVENT_BASED_WATCHDOG_FULL_RESYNC_INTERVAL # If true, then initial full resync is going to be delayed by 0 to FullResyncInterval. delayFullResync: false # ENV: KUMA_EXPERIMENTAL_KDS_EVENT_BASED_WATCHDOG_DELAY_FULL_RESYNC + # If true then control plane computes reachable services automatically based on MeshTrafficPermission. + # Lack of MeshTrafficPermission is treated as Deny the traffic. + autoReachableServices: false # ENV: KUMA_EXPERIMENTAL_AUTO_REACHABLE_SERVICES proxy: gateway: diff --git a/pkg/api-server/api_server_suite_test.go b/pkg/api-server/api_server_suite_test.go index 4f24972e4507..94832f7a37f6 100644 --- a/pkg/api-server/api_server_suite_test.go +++ b/pkg/api-server/api_server_suite_test.go @@ -257,6 +257,7 @@ func tryStartApiServer(t *testApiServerConfigurer) (*api_server.ApiServer, kuma_ vips.NewPersistence(resManager, config_manager.NewConfigManager(t.store), false), cfg.DNSServer.Domain, 80, + xds_context.AnyToAnyReachableServicesGraphBuilder, ), customization.NewAPIList(), registry.Global().ObjectDescriptors(model.HasWsEnabled()), diff --git a/pkg/api-server/customization/customization_suite_test.go b/pkg/api-server/customization/customization_suite_test.go index 19afbc079a27..5baed08b4bea 100644 --- a/pkg/api-server/customization/customization_suite_test.go +++ b/pkg/api-server/customization/customization_suite_test.go @@ -67,6 +67,7 @@ func createTestApiServer(store store.ResourceStore, config *config_api_server.Ap vips.NewPersistence(resManager, config_manager.NewConfigManager(store), false), cfg.DNSServer.Domain, cfg.DNSServer.ServiceVipPort, + xds_context.AnyToAnyReachableServicesGraphBuilder, ), wsManager, registry.Global().ObjectDescriptors(core_model.HasWsEnabled()), diff --git a/pkg/api-server/inspect_endpoints_test.go b/pkg/api-server/inspect_endpoints_test.go index 4ac4f56090e4..fc27f22d8592 100644 --- a/pkg/api-server/inspect_endpoints_test.go +++ b/pkg/api-server/inspect_endpoints_test.go @@ -22,6 +22,7 @@ import ( core_model "github.com/kumahq/kuma/pkg/core/resources/model" "github.com/kumahq/kuma/pkg/core/resources/store" _ "github.com/kumahq/kuma/pkg/plugins/policies" + "github.com/kumahq/kuma/pkg/plugins/policies/meshtrafficpermission/api/v1alpha1" "github.com/kumahq/kuma/pkg/plugins/resources/memory" "github.com/kumahq/kuma/pkg/test/kds/samples" "github.com/kumahq/kuma/pkg/test/matchers" @@ -180,7 +181,7 @@ var _ = Describe("Inspect WS", func() { }, builders.MeshTrafficPermission(). WithTargetRef(builders.TargetRefMesh()). - AddFrom(builders.TargetRefMesh(), "ALLOW"). + AddFrom(builders.TargetRefMesh(), v1alpha1.Allow). Build(), }, contentType: restful.MIME_JSON, @@ -799,7 +800,7 @@ var _ = Describe("Inspect WS", func() { builders.MeshTrafficPermission(). WithMesh("mesh-1"). WithTargetRef(builders.TargetRefService("backend")). - AddFrom(builders.TargetRefMesh(), "ALLOW"). + AddFrom(builders.TargetRefMesh(), v1alpha1.Allow). Build(), }, contentType: restful.MIME_JSON, @@ -915,7 +916,7 @@ var _ = Describe("Inspect WS", func() { samples2.DataplaneWeb(), builders.MeshTrafficPermission(). WithTargetRef(builders.TargetRefService("web")). - AddFrom(builders.TargetRefServiceSubset("client", "kuma.io/zone", "east"), "DENY"). + AddFrom(builders.TargetRefServiceSubset("client", "kuma.io/zone", "east"), v1alpha1.Deny). Build(), builders.MeshAccessLog(). WithTargetRef(builders.TargetRefService("web")). @@ -943,12 +944,12 @@ var _ = Describe("Inspect WS", func() { Build(), builders.MeshTrafficPermission(). WithTargetRef(builders.TargetRefService("web")). - AddFrom(builders.TargetRefServiceSubset("client", "kuma.io/zone", "east"), "DENY"). + AddFrom(builders.TargetRefServiceSubset("client", "kuma.io/zone", "east"), v1alpha1.Deny). Build(), builders.MeshTrafficPermission(). WithName("mtp-2"). WithTargetRef(builders.TargetRefService("web")). - AddFrom(builders.TargetRefServiceSubset("client", "kuma.io/zone", "east", "version", "2"), "ALLOW"). + AddFrom(builders.TargetRefServiceSubset("client", "kuma.io/zone", "east", "version", "2"), v1alpha1.Allow). Build(), builders.MeshAccessLog(). WithTargetRef(builders.TargetRefService("web")). diff --git a/pkg/api-server/server.go b/pkg/api-server/server.go index b3d168b37e88..6c89280ecfcd 100644 --- a/pkg/api-server/server.go +++ b/pkg/api-server/server.go @@ -464,6 +464,7 @@ func SetupServer(rt runtime.Runtime) error { vips.NewPersistence(rt.ResourceManager(), rt.ConfigManager(), cfg.Experimental.UseTagFirstVirtualOutboundModel), cfg.DNSServer.Domain, cfg.DNSServer.ServiceVipPort, + xds_context.AnyToAnyReachableServicesGraphBuilder, ), rt.APIInstaller(), registry.Global().ObjectDescriptors(model.HasWsEnabled()), diff --git a/pkg/api-server/testdata/inspect_dataplane_rules.golden.json b/pkg/api-server/testdata/inspect_dataplane_rules.golden.json index 1200957fabeb..617e15702f7a 100644 --- a/pkg/api-server/testdata/inspect_dataplane_rules.golden.json +++ b/pkg/api-server/testdata/inspect_dataplane_rules.golden.json @@ -68,7 +68,7 @@ "kuma.io/zone": "east" }, "conf": { - "action": "DENY" + "action": "Deny" }, "origins": [ { diff --git a/pkg/api-server/testdata/inspect_dataplane_rules_subset.golden.json b/pkg/api-server/testdata/inspect_dataplane_rules_subset.golden.json index b20b37880821..0ec70e0cbdd1 100644 --- a/pkg/api-server/testdata/inspect_dataplane_rules_subset.golden.json +++ b/pkg/api-server/testdata/inspect_dataplane_rules_subset.golden.json @@ -130,7 +130,7 @@ "version": "2" }, "conf": { - "action": "ALLOW" + "action": "Allow" }, "origins": [ { @@ -161,7 +161,7 @@ "version": "!2" }, "conf": { - "action": "DENY" + "action": "Deny" }, "origins": [ { diff --git a/pkg/config/app/kuma-cp/config.go b/pkg/config/app/kuma-cp/config.go index 4b7d736138fc..8785614bd9fe 100644 --- a/pkg/config/app/kuma-cp/config.go +++ b/pkg/config/app/kuma-cp/config.go @@ -405,6 +405,9 @@ type ExperimentalConfig struct { IngressTagFilters []string `json:"ingressTagFilters" envconfig:"KUMA_EXPERIMENTAL_INGRESS_TAG_FILTERS"` // KDS event based watchdog settings. It is a more optimal way to generate KDS snapshot config. KDSEventBasedWatchdog ExperimentalKDSEventBasedWatchdog `json:"kdsEventBasedWatchdog"` + // If true then control plane computes reachable services automatically based on MeshTrafficPermission. + // Lack of MeshTrafficPermission is treated as Deny the traffic. + AutoReachableServices bool `json:"autoReachableServices" envconfig:"KUMA_EXPERIMENTAL_AUTO_REACHABLE_SERVICES"` } type ExperimentalKDSEventBasedWatchdog struct { diff --git a/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml b/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml index e4441ddd83d7..d645e81e56f7 100644 --- a/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml +++ b/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml @@ -740,6 +740,9 @@ experimental: fullResyncInterval: 60s # ENV: KUMA_EXPERIMENTAL_KDS_EVENT_BASED_WATCHDOG_FULL_RESYNC_INTERVAL # If true, then initial full resync is going to be delayed by 0 to FullResyncInterval. delayFullResync: false # ENV: KUMA_EXPERIMENTAL_KDS_EVENT_BASED_WATCHDOG_DELAY_FULL_RESYNC + # If true then control plane computes reachable services automatically based on MeshTrafficPermission. + # Lack of MeshTrafficPermission is treated as Deny the traffic. + autoReachableServices: false # ENV: KUMA_EXPERIMENTAL_AUTO_REACHABLE_SERVICES proxy: gateway: diff --git a/pkg/config/loader_test.go b/pkg/config/loader_test.go index 1f0e8b57678c..84e1a1eed7d4 100644 --- a/pkg/config/loader_test.go +++ b/pkg/config/loader_test.go @@ -354,6 +354,7 @@ var _ = Describe("Config loader", func() { Expect(cfg.Experimental.KDSEventBasedWatchdog.FlushInterval.Duration).To(Equal(10 * time.Second)) Expect(cfg.Experimental.KDSEventBasedWatchdog.FullResyncInterval.Duration).To(Equal(15 * time.Second)) Expect(cfg.Experimental.KDSEventBasedWatchdog.DelayFullResync).To(BeTrue()) + Expect(cfg.Experimental.AutoReachableServices).To(BeTrue()) Expect(cfg.Proxy.Gateway.GlobalDownstreamMaxConnections).To(BeNumerically("==", 1)) Expect(cfg.EventBus.BufferSize).To(Equal(uint(30))) @@ -699,6 +700,7 @@ experimental: flushInterval: 10s fullResyncInterval: 15s delayFullResync: true + autoReachableServices: true proxy: gateway: globalDownstreamMaxConnections: 1 @@ -962,6 +964,7 @@ tracing: "KUMA_EXPERIMENTAL_KDS_EVENT_BASED_WATCHDOG_FLUSH_INTERVAL": "10s", "KUMA_EXPERIMENTAL_KDS_EVENT_BASED_WATCHDOG_FULL_RESYNC_INTERVAL": "15s", "KUMA_EXPERIMENTAL_KDS_EVENT_BASED_WATCHDOG_DELAY_FULL_RESYNC": "true", + "KUMA_EXPERIMENTAL_AUTO_REACHABLE_SERVICES": "true", "KUMA_PROXY_GATEWAY_GLOBAL_DOWNSTREAM_MAX_CONNECTIONS": "1", "KUMA_TRACING_OPENTELEMETRY_ENDPOINT": "otel-collector:4317", "KUMA_TRACING_OPENTELEMETRY_ENABLED": "true", diff --git a/pkg/core/bootstrap/bootstrap.go b/pkg/core/bootstrap/bootstrap.go index 27ef600b4ca1..b4b87cd296d8 100644 --- a/pkg/core/bootstrap/bootstrap.go +++ b/pkg/core/bootstrap/bootstrap.go @@ -49,6 +49,7 @@ import ( "github.com/kumahq/kuma/pkg/metrics" metrics_store "github.com/kumahq/kuma/pkg/metrics/store" "github.com/kumahq/kuma/pkg/multitenant" + "github.com/kumahq/kuma/pkg/plugins/policies/meshtrafficpermission/graph" "github.com/kumahq/kuma/pkg/plugins/resources/postgres/config" "github.com/kumahq/kuma/pkg/tokens/builtin" tokens_access "github.com/kumahq/kuma/pkg/tokens/builtin/access" @@ -500,6 +501,10 @@ func initializeConfigManager(builder *core_runtime.Builder) { } func initializeMeshCache(builder *core_runtime.Builder) error { + rsGraphBuilder := xds_context.AnyToAnyReachableServicesGraphBuilder + if builder.Config().Experimental.AutoReachableServices { + rsGraphBuilder = graph.Builder + } meshContextBuilder := xds_context.NewMeshContextBuilder( builder.ReadOnlyResourceManager(), xds_server.MeshResourceTypes(xds_server.HashMeshExcludedResources), @@ -508,6 +513,7 @@ func initializeMeshCache(builder *core_runtime.Builder) error { vips.NewPersistence(builder.ReadOnlyResourceManager(), builder.ConfigManager(), builder.Config().Experimental.UseTagFirstVirtualOutboundModel), builder.Config().DNSServer.Domain, builder.Config().DNSServer.ServiceVipPort, + rsGraphBuilder, ) meshSnapshotCache, err := mesh_cache.NewCache( diff --git a/pkg/plugins/policies/meshtrafficpermission/graph/graph_suite_test.go b/pkg/plugins/policies/meshtrafficpermission/graph/graph_suite_test.go new file mode 100644 index 000000000000..525c05d91dc3 --- /dev/null +++ b/pkg/plugins/policies/meshtrafficpermission/graph/graph_suite_test.go @@ -0,0 +1,11 @@ +package graph_test + +import ( + "testing" + + "github.com/kumahq/kuma/pkg/test" +) + +func TestGraph(t *testing.T) { + test.RunSpecs(t, "Graph Suite") +} diff --git a/pkg/plugins/policies/meshtrafficpermission/graph/reachable_services_graph.go b/pkg/plugins/policies/meshtrafficpermission/graph/reachable_services_graph.go new file mode 100644 index 000000000000..194f0427eda8 --- /dev/null +++ b/pkg/plugins/policies/meshtrafficpermission/graph/reachable_services_graph.go @@ -0,0 +1,176 @@ +package graph + +import ( + "golang.org/x/exp/maps" + + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/core" + "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" + core_model "github.com/kumahq/kuma/pkg/core/resources/model" + "github.com/kumahq/kuma/pkg/plugins/policies/core/matchers" + core_rules "github.com/kumahq/kuma/pkg/plugins/policies/core/rules" + mtp_api "github.com/kumahq/kuma/pkg/plugins/policies/meshtrafficpermission/api/v1alpha1" + "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/controllers" + "github.com/kumahq/kuma/pkg/xds/context" +) + +var log = core.Log.WithName("rs-graph") + +var SupportedTags = map[string]struct{}{ + controllers.KubeNamespaceTag: {}, + controllers.KubeServiceTag: {}, + controllers.KubePortTag: {}, +} + +type Graph struct { + rules map[string]core_rules.Rules +} + +func NewGraph() *Graph { + return &Graph{ + rules: map[string]core_rules.Rules{}, + } +} + +func (r *Graph) CanReach(fromTags map[string]string, toTags map[string]string) bool { + if _, crossMeshTagExist := toTags[mesh_proto.MeshTag]; crossMeshTagExist { + // we cannot compute graph for cross mesh, so it's better to allow the traffic + return true + } + rule := r.rules[toTags[mesh_proto.ServiceTag]].Compute(core_rules.SubsetFromTags(fromTags)) + if rule == nil { + return false + } + action := rule.Conf.(mtp_api.Conf).Action + return action == mtp_api.Allow || action == mtp_api.AllowWithShadowDeny +} + +func Builder(meshName string, resources context.Resources) context.ReachableServicesGraph { + services := BuildServices( + meshName, + resources.Dataplanes().Items, + resources.ExternalServices().Items, + resources.ZoneIngresses().Items, + ) + mtps := resources.ListOrEmpty(mtp_api.MeshTrafficPermissionType).(*mtp_api.MeshTrafficPermissionResourceList) + return BuildGraph(services, mtps.Items) +} + +// BuildServices we could just take result of xds_topology.VIPOutbounds, however it does not have a context of additional tags +func BuildServices( + meshName string, + dataplanes []*mesh.DataplaneResource, + externalServices []*mesh.ExternalServiceResource, + zoneIngresses []*mesh.ZoneIngressResource, +) map[string]mesh_proto.SingleValueTagSet { + services := map[string]mesh_proto.SingleValueTagSet{} + addSvc := func(tags map[string]string) { + svc := tags[mesh_proto.ServiceTag] + if _, ok := services[svc]; ok { + return + } + services[svc] = map[string]string{} + for tag := range SupportedTags { + if value := tags[tag]; value != "" { + services[svc][tag] = value + } + } + } + + for _, dp := range dataplanes { + for _, tagSet := range dp.Spec.SingleValueTagSets() { + addSvc(tagSet) + } + } + for _, zi := range zoneIngresses { + for _, availableSvc := range zi.Spec.GetAvailableServices() { + if meshName != availableSvc.Mesh { + continue + } + addSvc(availableSvc.Tags) + } + } + for _, es := range externalServices { + addSvc(es.Spec.Tags) + } + return services +} + +func BuildGraph(services map[string]mesh_proto.SingleValueTagSet, mtps []*mtp_api.MeshTrafficPermissionResource) *Graph { + resources := context.Resources{ + MeshLocalResources: map[core_model.ResourceType]core_model.ResourceList{ + mtp_api.MeshTrafficPermissionType: &mtp_api.MeshTrafficPermissionResourceList{ + Items: trimNotSupportedTags(mtps), + }, + }, + } + + graph := NewGraph() + + for service, tags := range services { + // build artificial dpp for matching + dp := mesh.NewDataplaneResource() + dpTags := maps.Clone(tags) + dpTags[mesh_proto.ServiceTag] = service + dp.Spec = &mesh_proto.Dataplane{ + Networking: &mesh_proto.Dataplane_Networking{ + Address: "1.1.1.1", + Inbound: []*mesh_proto.Dataplane_Networking_Inbound{ + { + Tags: dpTags, + Port: 1234, + }, + }, + }, + } + + matched, err := matchers.MatchedPolicies(mtp_api.MeshTrafficPermissionType, dp, resources) + if err != nil { + log.Error(err, "service could not be matched. It won't be reached by any other service", "service", service) + continue // it's better to ignore one service that to break the whole graph + } + + rl, ok := matched.FromRules.Rules[core_rules.InboundListener{ + Address: "1.1.1.1", + Port: 1234, + }] + if !ok { + continue + } + + graph.rules[service] = rl + } + + return graph +} + +// trimNotSupportedTags replaces tags present in subsets of top-level target ref. +// Because we need to do policy matching on services instead of individual proxies, we have to handle subsets in a special way. +// What we do is we only support subsets with predefined tags listed in SupportedTags. +// This assumes that tags listed in SupportedTags have the same value between all instances of a given service. +// Otherwise, we trim the tags making the target ref subset wider. +// +// Alternatively, we could have computed all common tags between instances of a given service and then allow subsets with those common tags. +// However, this would require calling this function for every service. +func trimNotSupportedTags(mtps []*mtp_api.MeshTrafficPermissionResource) []*mtp_api.MeshTrafficPermissionResource { + newMtps := make([]*mtp_api.MeshTrafficPermissionResource, len(mtps)) + for i, mtp := range mtps { + if len(mtp.Spec.TargetRef.Tags) > 0 { + filteredTags := map[string]string{} + for tag, val := range mtp.Spec.TargetRef.Tags { + if _, ok := SupportedTags[tag]; ok { + filteredTags[tag] = val + } + } + if len(filteredTags) != len(mtp.Spec.TargetRef.Tags) { + mtp = &mtp_api.MeshTrafficPermissionResource{ + Meta: mtp.Meta, + Spec: mtp.Spec.DeepCopy(), + } + mtp.Spec.TargetRef.Tags = filteredTags + } + } + newMtps[i] = mtp + } + return newMtps +} diff --git a/pkg/plugins/policies/meshtrafficpermission/graph/reachable_services_graph_test.go b/pkg/plugins/policies/meshtrafficpermission/graph/reachable_services_graph_test.go new file mode 100644 index 000000000000..abea40665fd2 --- /dev/null +++ b/pkg/plugins/policies/meshtrafficpermission/graph/reachable_services_graph_test.go @@ -0,0 +1,378 @@ +package graph_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + common_api "github.com/kumahq/kuma/api/common/v1alpha1" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" + "github.com/kumahq/kuma/pkg/plugins/policies/meshtrafficpermission/api/v1alpha1" + "github.com/kumahq/kuma/pkg/plugins/policies/meshtrafficpermission/graph" + "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/controllers" + "github.com/kumahq/kuma/pkg/test/resources/builders" + "github.com/kumahq/kuma/pkg/test/resources/samples" +) + +var _ = Describe("Reachable Services Graph", func() { + type testCase struct { + mtps []*v1alpha1.MeshTrafficPermissionResource + expectedFromAll map[string]struct{} + expectedConnections map[string]map[string]struct{} + } + + services := map[string]mesh_proto.SingleValueTagSet{ + "a": map[string]string{}, + "b": map[string]string{}, + "c": map[string]string{}, + "d": map[string]string{}, + } + + fromAllServices := map[string]struct{}{"a": {}, "b": {}, "c": {}, "d": {}} + + DescribeTable("should check reachability of the graph", + func(given testCase) { + // when + g := graph.BuildGraph(services, given.mtps) + + // then + for from := range services { + for to := range services { + _, fromAll := given.expectedFromAll[to] + _, conn := given.expectedConnections[from][to] + Expect(g.CanReach( + map[string]string{mesh_proto.ServiceTag: from}, + map[string]string{mesh_proto.ServiceTag: to}, + )).To(Equal(fromAll || conn)) + } + } + }, + Entry("allow all", testCase{ + mtps: []*v1alpha1.MeshTrafficPermissionResource{ + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefMesh()). + AddFrom(builders.TargetRefMesh(), v1alpha1.Allow). + Build(), + }, + expectedFromAll: fromAllServices, + }), + Entry("deny all", testCase{ + mtps: []*v1alpha1.MeshTrafficPermissionResource{ + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefMesh()). + AddFrom(builders.TargetRefMesh(), v1alpha1.Deny). + Build(), + }, + expectedFromAll: map[string]struct{}{}, + }), + Entry("no MeshTrafficPermissions", testCase{ + mtps: []*v1alpha1.MeshTrafficPermissionResource{}, + expectedFromAll: map[string]struct{}{}, + }), + Entry("one connection Allow", testCase{ + mtps: []*v1alpha1.MeshTrafficPermissionResource{ + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefService("b")). + AddFrom(builders.TargetRefService("a"), v1alpha1.Allow). + Build(), + }, + expectedConnections: map[string]map[string]struct{}{ + "a": {"b": {}}, + }, + }), + Entry("AllowWithShadowDeny is treated as Allow", testCase{ + mtps: []*v1alpha1.MeshTrafficPermissionResource{ + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefService("b")). + AddFrom(builders.TargetRefService("a"), v1alpha1.AllowWithShadowDeny). + Build(), + }, + expectedConnections: map[string]map[string]struct{}{ + "a": {"b": {}}, + }, + }), + Entry("multiple allowed connections", testCase{ + mtps: []*v1alpha1.MeshTrafficPermissionResource{ + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefService("b")). + AddFrom(builders.TargetRefService("a"), v1alpha1.Allow). + Build(), + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefService("c")). + AddFrom(builders.TargetRefService("b"), v1alpha1.AllowWithShadowDeny). + Build(), + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefService("d")). + AddFrom(builders.TargetRefMesh(), v1alpha1.Allow). + Build(), + }, + expectedFromAll: map[string]struct{}{ + "d": {}, + }, + expectedConnections: map[string]map[string]struct{}{ + "a": {"b": {}}, + "b": {"c": {}}, + }, + }), + Entry("all allowed except one connection", testCase{ + mtps: []*v1alpha1.MeshTrafficPermissionResource{ + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefMesh()). + AddFrom(builders.TargetRefMesh(), v1alpha1.Allow). + Build(), + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefService("b")). + AddFrom(builders.TargetRefService("a"), v1alpha1.Deny). + Build(), + }, + expectedFromAll: map[string]struct{}{ + "a": {}, + "c": {}, + "d": {}, + }, + expectedConnections: map[string]map[string]struct{}{ + "c": {"b": {}}, + "d": {"b": {}}, + "b": {"b": {}}, + }, + }), + Entry("allow all but one service has restrictive mesh traffic permission", testCase{ + mtps: []*v1alpha1.MeshTrafficPermissionResource{ + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefMesh()). + AddFrom(builders.TargetRefMesh(), v1alpha1.Allow). + Build(), + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefService("b")). + AddFrom(builders.TargetRefMesh(), v1alpha1.Deny). + AddFrom(builders.TargetRefService("a"), v1alpha1.Allow). + Build(), + }, + expectedFromAll: map[string]struct{}{ + "a": {}, + "c": {}, + "d": {}, + }, + expectedConnections: map[string]map[string]struct{}{ + "a": {"b": {}}, + }, + }), + Entry("top level target ref MeshSubset with unsupported predefined tags selects all", testCase{ + mtps: []*v1alpha1.MeshTrafficPermissionResource{ + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefMeshSubset("kuma.io/zone", "east")). + AddFrom(builders.TargetRefMesh(), v1alpha1.Allow). + Build(), + }, + expectedFromAll: fromAllServices, + }), + Entry("top level target ref MeshServiceSubset of unsupported predefined tags selects all instances of the service", testCase{ + mtps: []*v1alpha1.MeshTrafficPermissionResource{ + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefServiceSubset("a", "kuma.io/zone", "east")). + AddFrom(builders.TargetRefMesh(), v1alpha1.Allow). + Build(), + }, + expectedFromAll: map[string]struct{}{ + "a": {}, + }, + }), + Entry("equal subsets matching is preserved because of the names", testCase{ + mtps: []*v1alpha1.MeshTrafficPermissionResource{ + builders.MeshTrafficPermission(). + WithName("aaa"). + WithTargetRef(builders.TargetRefMeshSubset("kuma.io/zone", "east")). + AddFrom(builders.TargetRefMesh(), v1alpha1.Deny). + Build(), + builders.MeshTrafficPermission(). + WithName("bbb"). + WithTargetRef(builders.TargetRefMeshSubset("version", "v1")). + AddFrom(builders.TargetRefMesh(), v1alpha1.Allow). + Build(), + }, + expectedFromAll: fromAllServices, + }), + ) + + It("should work with service subsets in from", func() { + // given + services := map[string]mesh_proto.SingleValueTagSet{ + "a": map[string]string{}, + } + mtps := []*v1alpha1.MeshTrafficPermissionResource{ + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefMesh()). + AddFrom(builders.TargetRefServiceSubset("b", "version", "v1"), v1alpha1.Allow). + Build(), + } + + // when + graph := graph.BuildGraph(services, mtps) + + // then + Expect(graph.CanReach( + map[string]string{mesh_proto.ServiceTag: "b", "version": "v1"}, + map[string]string{mesh_proto.ServiceTag: "a"}, + )).To(BeTrue()) + Expect(graph.CanReach( + map[string]string{mesh_proto.ServiceTag: "b", "version": "v2"}, + map[string]string{mesh_proto.ServiceTag: "a"}, + )).To(BeFalse()) + }) + + It("should work with mesh subset in from", func() { + services := map[string]mesh_proto.SingleValueTagSet{ + "a": map[string]string{}, + } + mtps := []*v1alpha1.MeshTrafficPermissionResource{ + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefMesh()). + AddFrom(builders.TargetRefMeshSubset("kuma.io/zone", "east"), v1alpha1.Allow). + Build(), + } + + // when + graph := graph.BuildGraph(services, mtps) + + // then + Expect(graph.CanReach( + map[string]string{"kuma.io/zone": "east"}, + map[string]string{mesh_proto.ServiceTag: "a"}, + )).To(BeTrue()) + Expect(graph.CanReach( + map[string]string{"kuma.io/zone": "west"}, + map[string]string{mesh_proto.ServiceTag: "a"}, + )).To(BeFalse()) + Expect(graph.CanReach( + map[string]string{"othertag": "other"}, + map[string]string{mesh_proto.ServiceTag: "a"}, + )).To(BeFalse()) + }) + + It("should always allow cross mesh", func() { + // when + graph := graph.BuildGraph(nil, nil) + + // then + Expect(graph.CanReach( + map[string]string{mesh_proto.ServiceTag: "b"}, + map[string]string{mesh_proto.ServiceTag: "a", mesh_proto.MeshTag: "other"}, + )).To(BeTrue()) + }) + + DescribeTable("top level subset should work with predefined tags", + func(targetRef common_api.TargetRef) { + // given + services := map[string]mesh_proto.SingleValueTagSet{ + "a_kuma-demo_svc_1234": map[string]string{ + controllers.KubeNamespaceTag: "kuma-demo", + controllers.KubeServiceTag: "a", + controllers.KubePortTag: "1234", + }, + "b": map[string]string{}, + } + mtps := []*v1alpha1.MeshTrafficPermissionResource{ + builders.MeshTrafficPermission(). + WithTargetRef(targetRef). + AddFrom(builders.TargetRefMesh(), v1alpha1.Allow). + Build(), + } + + // when + graph := graph.BuildGraph(services, mtps) + + // then + Expect(graph.CanReach( + map[string]string{mesh_proto.ServiceTag: "b"}, + map[string]string{mesh_proto.ServiceTag: "a_kuma-demo_svc_1234"}, + )).To(BeTrue()) + Expect(graph.CanReach( + map[string]string{mesh_proto.ServiceTag: "a_kuma-demo_svc_1234"}, + map[string]string{mesh_proto.ServiceTag: "b"}, + )).To(BeFalse()) // it's not selected by top-level target ref + }, + Entry("MeshSubset by kube namespace", builders.TargetRefMeshSubset(controllers.KubeNamespaceTag, "kuma-demo")), + Entry("MeshSubset by kube service name", builders.TargetRefMeshSubset(controllers.KubeServiceTag, "a")), + Entry("MeshSubset by kube service port", builders.TargetRefMeshSubset(controllers.KubePortTag, "1234")), + Entry("MeshServiceSubset by kube namespace", builders.TargetRefServiceSubset("a_kuma-demo_svc_1234", controllers.KubeNamespaceTag, "kuma-demo")), + Entry("MeshServiceSubset by kube service name", builders.TargetRefServiceSubset("a_kuma-demo_svc_1234", controllers.KubeServiceTag, "a")), + Entry("MeshServiceSubset by kube service port", builders.TargetRefServiceSubset("a_kuma-demo_svc_1234", controllers.KubePortTag, "1234")), + ) + + It("should not modify MeshTrafficPermission passed to the func when replacing tags in subsets", func() { + // given + mtps := []*v1alpha1.MeshTrafficPermissionResource{ + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefMeshSubset("version", "v1")). + AddFrom(builders.TargetRefMesh(), v1alpha1.Allow). + Build(), + builders.MeshTrafficPermission(). + WithTargetRef(builders.TargetRefServiceSubset("a", "version", "v1")). + AddFrom(builders.TargetRefMesh(), v1alpha1.Allow). + Build(), + } + + // when + _ = graph.BuildGraph(services, mtps) + + // then + Expect(mtps[0].Spec.TargetRef.Tags).NotTo(BeNil()) + Expect(mtps[1].Spec.TargetRef.Tags).NotTo(BeNil()) + }) + + It("should build services adding supported tags and including external services", func() { + // given + tags := map[string]string{ + mesh_proto.ServiceTag: "a_kuma-demo_svc_1234", + controllers.KubeNamespaceTag: "kuma-demo", + controllers.KubeServiceTag: "a", + controllers.KubePortTag: "1234", + } + dpps := []*mesh.DataplaneResource{ + samples.DataplaneBackendBuilder(). + WithAddress("1.1.1.1"). + WithInboundOfTagsMap(tags). + Build(), + samples.DataplaneBackendBuilder(). + WithAddress("1.1.1.2"). + WithInboundOfTagsMap(tags). + Build(), + samples.DataplaneBackendBuilder(). + WithAddress("1.1.1.3"). + WithServices("b", "c"). + Build(), + } + es := []*mesh.ExternalServiceResource{ + { + Spec: &mesh_proto.ExternalService{ + Tags: map[string]string{ + mesh_proto.ServiceTag: "es-1", + }, + }, + }, + } + zis := []*mesh.ZoneIngressResource{ + builders.ZoneIngress(). + AddSimpleAvailableService("d"). + AddSimpleAvailableService("e"). + Build(), + } + + // when + services := graph.BuildServices("default", dpps, es, zis) + + // then + Expect(services).To(Equal(map[string]mesh_proto.SingleValueTagSet{ + "a_kuma-demo_svc_1234": map[string]string{ + controllers.KubeNamespaceTag: "kuma-demo", + controllers.KubeServiceTag: "a", + controllers.KubePortTag: "1234", + }, + "b": map[string]string{}, + "c": map[string]string{}, + "d": map[string]string{}, + "e": map[string]string{}, + "es-1": map[string]string{}, + })) + }) +}) diff --git a/pkg/plugins/runtime/gateway/suite_test.go b/pkg/plugins/runtime/gateway/suite_test.go index bfd458f24212..be01b438a52b 100644 --- a/pkg/plugins/runtime/gateway/suite_test.go +++ b/pkg/plugins/runtime/gateway/suite_test.go @@ -129,6 +129,7 @@ func MakeGeneratorContext(rt runtime.Runtime, key core_model.ResourceKey) (*xds_ vips.NewPersistence(rt.ReadOnlyResourceManager(), rt.ConfigManager(), false), rt.Config().DNSServer.Domain, rt.Config().DNSServer.ServiceVipPort, + xds_context.AnyToAnyReachableServicesGraphBuilder, ) meshCtx, err := meshCtxBuilder.Build(context.TODO(), key.Mesh) diff --git a/pkg/test/resources/builders/meshtrafficpermission_builder.go b/pkg/test/resources/builders/meshtrafficpermission_builder.go index d849ca966cfc..d1dd8c6b589e 100644 --- a/pkg/test/resources/builders/meshtrafficpermission_builder.go +++ b/pkg/test/resources/builders/meshtrafficpermission_builder.go @@ -38,11 +38,11 @@ func (m *MeshTrafficPermissionBuilder) WithTargetRef(targetRef common_api.Target return m } -func (m *MeshTrafficPermissionBuilder) AddFrom(targetRef common_api.TargetRef, action string) *MeshTrafficPermissionBuilder { +func (m *MeshTrafficPermissionBuilder) AddFrom(targetRef common_api.TargetRef, action mtp_proto.Action) *MeshTrafficPermissionBuilder { m.res.Spec.From = append(m.res.Spec.From, mtp_proto.From{ TargetRef: targetRef, Default: mtp_proto.Conf{ - Action: mtp_proto.Action(action), + Action: action, }, }) return m diff --git a/pkg/test/resources/builders/targetref_builder.go b/pkg/test/resources/builders/targetref_builder.go index f860f66b0eed..618a300a2605 100644 --- a/pkg/test/resources/builders/targetref_builder.go +++ b/pkg/test/resources/builders/targetref_builder.go @@ -10,6 +10,13 @@ func TargetRefMesh() common_api.TargetRef { } } +func TargetRefMeshSubset(kv ...string) common_api.TargetRef { + return common_api.TargetRef{ + Kind: "MeshSubset", + Tags: tagsKVToMap(kv), + } +} + func TargetRefService(name string) common_api.TargetRef { return common_api.TargetRef{ Kind: "MeshService", diff --git a/pkg/test/resources/builders/zoneingress_builder.go b/pkg/test/resources/builders/zoneingress_builder.go index 7628eefc2fc0..c9e693947ab7 100644 --- a/pkg/test/resources/builders/zoneingress_builder.go +++ b/pkg/test/resources/builders/zoneingress_builder.go @@ -24,6 +24,7 @@ func ZoneIngress() *ZoneIngressBuilder { Spec: &mesh_proto.ZoneIngress{ Networking: &mesh_proto.ZoneIngress_Networking{ Address: "127.0.0.1", + Port: 10000, }, }, }, @@ -91,3 +92,19 @@ func (b *ZoneIngressBuilder) WithAdvertisedPort(port uint32) *ZoneIngressBuilder b.res.Spec.Networking.AdvertisedPort = port return b } + +func (b *ZoneIngressBuilder) AddSimpleAvailableService(svc string) *ZoneIngressBuilder { + return b.AddAvailableService(&mesh_proto.ZoneIngress_AvailableService{ + Tags: map[string]string{ + mesh_proto.ServiceTag: svc, + }, + Instances: 1, + Mesh: "default", + ExternalService: false, + }) +} + +func (b *ZoneIngressBuilder) AddAvailableService(svc *mesh_proto.ZoneIngress_AvailableService) *ZoneIngressBuilder { + b.res.Spec.AvailableServices = append(b.res.Spec.AvailableServices, svc) + return b +} diff --git a/pkg/test/runtime/runtime.go b/pkg/test/runtime/runtime.go index 3e768ea44b37..e383eb4e1466 100644 --- a/pkg/test/runtime/runtime.go +++ b/pkg/test/runtime/runtime.go @@ -185,6 +185,7 @@ func initializeMeshCache(builder *core_runtime.Builder) error { vips.NewPersistence(builder.ReadOnlyResourceManager(), builder.ConfigManager(), builder.Config().Experimental.UseTagFirstVirtualOutboundModel), builder.Config().DNSServer.Domain, builder.Config().DNSServer.ServiceVipPort, + xds_context.AnyToAnyReachableServicesGraphBuilder, ) meshSnapshotCache, err := mesh_cache.NewCache( diff --git a/pkg/xds/cache/mesh/cache_test.go b/pkg/xds/cache/mesh/cache_test.go index 30f63c7c06ad..d2ec0c40c211 100644 --- a/pkg/xds/cache/mesh/cache_test.go +++ b/pkg/xds/cache/mesh/cache_test.go @@ -108,6 +108,7 @@ var _ = Describe("MeshSnapshot Cache", func() { vips.NewPersistence(core_manager.NewResourceManager(s), manager.NewConfigManager(s), false), "mesh", 80, + xds_context.AnyToAnyReachableServicesGraphBuilder, ) meshCache, err = mesh.NewCache( expiration, diff --git a/pkg/xds/context/context.go b/pkg/xds/context/context.go index 24f8d84ea9c6..4909b2c2a4f6 100644 --- a/pkg/xds/context/context.go +++ b/pkg/xds/context/context.go @@ -33,16 +33,17 @@ type ControlPlaneContext struct { // If there is an information that can be precomputed and shared between all data plane proxies // it should be put here. This way we can save CPU cycles of computing the same information. type MeshContext struct { - Hash string - Resource *core_mesh.MeshResource - Resources Resources - DataplanesByName map[string]*core_mesh.DataplaneResource - EndpointMap xds.EndpointMap - CrossMeshEndpoints map[xds.MeshName]xds.EndpointMap - VIPDomains []xds.VIPDomains - VIPOutbounds []*mesh_proto.Dataplane_Networking_Outbound - ServiceTLSReadiness map[string]bool - DataSourceLoader datasource.Loader + Hash string + Resource *core_mesh.MeshResource + Resources Resources + DataplanesByName map[string]*core_mesh.DataplaneResource + EndpointMap xds.EndpointMap + CrossMeshEndpoints map[xds.MeshName]xds.EndpointMap + VIPDomains []xds.VIPDomains + VIPOutbounds []*mesh_proto.Dataplane_Networking_Outbound + ServiceTLSReadiness map[string]bool + DataSourceLoader datasource.Loader + ReachableServicesGraph ReachableServicesGraph } func (mc *MeshContext) GetTracingBackend(tt *core_mesh.TrafficTraceResource) *mesh_proto.TracingBackend { diff --git a/pkg/xds/context/mesh_context_builder.go b/pkg/xds/context/mesh_context_builder.go index 64fa8b4ea316..ddaaf6095ee1 100644 --- a/pkg/xds/context/mesh_context_builder.go +++ b/pkg/xds/context/mesh_context_builder.go @@ -33,6 +33,7 @@ type meshContextBuilder struct { vipsPersistence *vips.Persistence topLevelDomain string vipPort uint32 + rsGraphBuilder ReachableServicesGraphBuilder } type MeshContextBuilder interface { @@ -53,6 +54,7 @@ func NewMeshContextBuilder( vipsPersistence *vips.Persistence, topLevelDomain string, vipPort uint32, + rsGraphBuilder ReachableServicesGraphBuilder, ) MeshContextBuilder { typeSet := map[core_model.ResourceType]struct{}{} for _, typ := range types { @@ -67,6 +69,7 @@ func NewMeshContextBuilder( vipsPersistence: vipsPersistence, topLevelDomain: topLevelDomain, vipPort: vipPort, + rsGraphBuilder: rsGraphBuilder, } } @@ -129,16 +132,17 @@ func (m *meshContextBuilder) BuildIfChanged(ctx context.Context, meshName string } return &MeshContext{ - Hash: newHash, - Resource: mesh, - Resources: resources, - DataplanesByName: dataplanesByName, - EndpointMap: endpointMap, - CrossMeshEndpoints: crossMeshEndpointMap, - VIPDomains: domains, - VIPOutbounds: outbounds, - ServiceTLSReadiness: m.resolveTLSReadiness(mesh, resources.ServiceInsights()), - DataSourceLoader: datasource.NewStaticLoader(resources.Secrets().Items), + Hash: newHash, + Resource: mesh, + Resources: resources, + DataplanesByName: dataplanesByName, + EndpointMap: endpointMap, + CrossMeshEndpoints: crossMeshEndpointMap, + VIPDomains: domains, + VIPOutbounds: outbounds, + ServiceTLSReadiness: m.resolveTLSReadiness(mesh, resources.ServiceInsights()), + DataSourceLoader: datasource.NewStaticLoader(resources.Secrets().Items), + ReachableServicesGraph: m.rsGraphBuilder(meshName, resources), }, nil } diff --git a/pkg/xds/context/reachable_services_graph.go b/pkg/xds/context/reachable_services_graph.go new file mode 100644 index 000000000000..f41cd9fa67af --- /dev/null +++ b/pkg/xds/context/reachable_services_graph.go @@ -0,0 +1,31 @@ +package context + +import mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + +// ReachableServicesGraph is a graph of services in the mesh. +// We can test whether the DPP of given tags can reach a service. +// This way we can trim the configuration for a DPP, so it won't include unnecessary configuration. +type ReachableServicesGraph interface { + CanReach(fromTags map[string]string, toTags map[string]string) bool +} + +func CanReachFromAny(graph ReachableServicesGraph, fromTagSets []mesh_proto.SingleValueTagSet, toTags map[string]string) bool { + for _, fromTags := range fromTagSets { + if graph.CanReach(fromTags, toTags) { + return true + } + } + return false +} + +type ReachableServicesGraphBuilder func(meshName string, resources Resources) ReachableServicesGraph + +type AnyToAnyReachableServicesGraph struct{} + +func (a AnyToAnyReachableServicesGraph) CanReach(map[string]string, map[string]string) bool { + return true +} + +func AnyToAnyReachableServicesGraphBuilder(string, Resources) ReachableServicesGraph { + return AnyToAnyReachableServicesGraph{} +} diff --git a/pkg/xds/server/v3/snapshot_generator_test.go b/pkg/xds/server/v3/snapshot_generator_test.go index a920ba003470..c04abb085ec7 100644 --- a/pkg/xds/server/v3/snapshot_generator_test.go +++ b/pkg/xds/server/v3/snapshot_generator_test.go @@ -116,6 +116,7 @@ var _ = Describe("GenerateSnapshot", func() { vips.NewPersistence(rm, config_manager.NewConfigManager(store), false), cfg.DNSServer.Domain, cfg.DNSServer.ServiceVipPort, + xds_context.AnyToAnyReachableServicesGraphBuilder, ) proxyBuilder = sync.DefaultDataplaneProxyBuilder(cfg, envoy_common.APIV3) diff --git a/pkg/xds/sync/dataplane_proxy_builder.go b/pkg/xds/sync/dataplane_proxy_builder.go index 0b73a668524e..b6ba7215cf17 100644 --- a/pkg/xds/sync/dataplane_proxy_builder.go +++ b/pkg/xds/sync/dataplane_proxy_builder.go @@ -116,11 +116,21 @@ func (p *DataplaneProxyBuilder) resolveVIPOutbounds(meshContext xds_context.Mesh for _, ob := range meshContext.VIPOutbounds { generatedVips[ob.Address] = true } + dpTagSets := dataplane.Spec.SingleValueTagSets() var outbounds []*mesh_proto.Dataplane_Networking_Outbound for _, outbound := range meshContext.VIPOutbounds { service := outbound.GetService() - if len(reachableServices) != 0 && !reachableServices[service] { - continue // ignore VIP outbound if reachableServices is defined and not specified + if len(reachableServices) != 0 { + if !reachableServices[service] { + // ignore VIP outbound if reachableServices is defined and not specified + // Reachable services takes precedence over reachable services graph. + continue + } + } else { + // static reachable services takes precedence over the graph + if !xds_context.CanReachFromAny(meshContext.ReachableServicesGraph, dpTagSets, outbound.Tags) { + continue + } } if dataplane.UsesInboundInterface(net.ParseIP(outbound.Address), outbound.Port) { // Skip overlapping outbound interface with inbound. diff --git a/pkg/xds/sync/dataplane_watchdog_test.go b/pkg/xds/sync/dataplane_watchdog_test.go index 6d75be292635..16554acafe05 100644 --- a/pkg/xds/sync/dataplane_watchdog_test.go +++ b/pkg/xds/sync/dataplane_watchdog_test.go @@ -76,6 +76,7 @@ var _ = Describe("Dataplane Watchdog", func() { vips.NewPersistence(resManager, config_manager.NewConfigManager(store), false), ".mesh", 80, + xds_context.AnyToAnyReachableServicesGraphBuilder, ) newMetrics, err := metrics.NewMetrics(zone) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/xds/sync/proxy_builder_test.go b/pkg/xds/sync/proxy_builder_test.go index d0fbf619e484..f84a06a28983 100644 --- a/pkg/xds/sync/proxy_builder_test.go +++ b/pkg/xds/sync/proxy_builder_test.go @@ -90,6 +90,7 @@ var _ = Describe("Proxy Builder", func() { vips.NewPersistence(builder.ReadOnlyResourceManager(), builder.ConfigManager(), false), builder.Config().DNSServer.Domain, builder.Config().DNSServer.ServiceVipPort, + xds_context.AnyToAnyReachableServicesGraphBuilder, ) metrics, err := core_metrics.NewMetrics("cache") Expect(err).ToNot(HaveOccurred()) diff --git a/test/e2e/reachableservices/auto_reachable_services_k8s.go b/test/e2e/reachableservices/auto_reachable_services_k8s.go new file mode 100644 index 000000000000..f8485ec84ca0 --- /dev/null +++ b/test/e2e/reachableservices/auto_reachable_services_k8s.go @@ -0,0 +1,94 @@ +package reachableservices + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + config_core "github.com/kumahq/kuma/pkg/config/core" + . "github.com/kumahq/kuma/test/framework" + "github.com/kumahq/kuma/test/framework/client" + "github.com/kumahq/kuma/test/framework/deployments/testserver" +) + +var k8sCluster Cluster + +var _ = E2EBeforeSuite(func() { + k8sCluster = NewK8sCluster(NewTestingT(), Kuma1, Silent) + + err := NewClusterSetup(). + Install(Kuma(config_core.Standalone, + WithEnv("KUMA_EXPERIMENTAL_AUTO_REACHABLE_SERVICES", "true"), + )). + Install(NamespaceWithSidecarInjection(TestNamespace)). + Install(MTLSMeshKubernetes("default")). + Install(testserver.Install(testserver.WithName("client-server"), testserver.WithMesh("default"))). + Install(testserver.Install(testserver.WithName("first-test-server"), testserver.WithMesh("default"))). + Install(testserver.Install(testserver.WithName("second-test-server"), testserver.WithMesh("default"))). + Setup(k8sCluster) + + Expect(err).ToNot(HaveOccurred()) + + E2EDeferCleanup(func() { + Expect(k8sCluster.DeleteKuma()).To(Succeed()) + Expect(k8sCluster.DismissCluster()).To(Succeed()) + }) +}) + +func AutoReachableServices() { + It("should not connect to non auto reachable service", func() { + // when + Expect(YamlK8s(fmt.Sprintf(` +apiVersion: kuma.io/v1alpha1 +kind: MeshTrafficPermission +metadata: + name: mtp1 + namespace: %s + labels: + kuma.io/mesh: default +spec: + targetRef: + kind: MeshService + name: first-test-server_kuma-test_svc_80 + from: + - targetRef: + kind: Mesh + default: + action: Deny + - targetRef: + kind: MeshService + name: client-server_kuma-test_svc_80 + default: + action: Allow +`, Config.KumaNamespace))(k8sCluster)).To(Succeed()) + + // then + Eventually(func(g Gomega) { + pod, err := PodNameOfApp(k8sCluster, "second-test-server", TestNamespace) + g.Expect(err).ToNot(HaveOccurred()) + stdout, err := k8sCluster.GetKumactlOptions().RunKumactlAndGetOutput("inspect", "dataplane", pod+"."+TestNamespace, "--type=clusters") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(stdout).To(Not(ContainSubstring("first-test-server_kuma-test_svc_80"))) + }, "30s", "1s").Should(Succeed()) + + Eventually(func(g Gomega) { + pod, err := PodNameOfApp(k8sCluster, "client-server", TestNamespace) + g.Expect(err).ToNot(HaveOccurred()) + stdout, err := k8sCluster.GetKumactlOptions().RunKumactlAndGetOutput("inspect", "dataplane", pod+"."+TestNamespace, "--type=clusters") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(stdout).To(ContainSubstring("first-test-server_kuma-test_svc_80")) + }, "30s", "1s").Should(Succeed()) + + Consistently(func(g Gomega) { + failures, err := client.CollectFailure( + k8sCluster, + "second-test-server", + "first-test-server:80", + client.FromKubernetesPod(TestNamespace, "second-test-server"), + ) + g.Expect(err).To(Not(HaveOccurred())) + g.Expect(failures.Exitcode).To(Equal(52)) + }, "30s", "1s").Should(Succeed()) + }) +} diff --git a/test/e2e/reachableservices/e2e_suite_test.go b/test/e2e/reachableservices/e2e_suite_test.go new file mode 100644 index 000000000000..e00026e7135f --- /dev/null +++ b/test/e2e/reachableservices/e2e_suite_test.go @@ -0,0 +1,16 @@ +package reachableservices_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + + "github.com/kumahq/kuma/pkg/test" + "github.com/kumahq/kuma/test/e2e/reachableservices" +) + +func TestE2E(t *testing.T) { + test.RunE2ESpecs(t, "E2E Auto Reachable Services Kubernetes Suite") +} + +var _ = Describe("Auto Reachable Services on Kubernetes", Label("job-1"), reachableservices.AutoReachableServices)