From 600ea98ab2bf9db20fc0525edadb9ce59c4fae07 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Thu, 2 Nov 2023 11:13:10 +0100 Subject: [PATCH 01/26] feat(kuma-cp): add locality aware implementation for egress Fixes: #8073 Signed-off-by: Marcin Skalski --- ...install-control-plane.defaults.golden.yaml | 1 - ...plane.gateway-api-present-not-enabled.yaml | 1 - ...all-control-plane.gateway-api-present.yaml | 1 - .../install-control-plane.with-helm-set.yaml | 1 - .../testdata/install-crds.all.golden.yaml | 1 - ...l-crds.experimental-gatewayapi.golden.yaml | 1 - .../kuma.io_meshloadbalancingstrategies.yaml | 1 - docs/generated/openapi.yaml | 1 - .../kuma.io_meshloadbalancingstrategies.yaml | 1 - pkg/core/plugins/interfaces.go | 2 +- pkg/plugins/policies/core/matchers/egress.go | 33 +- .../policies/core/matchers/egress_test.go | 4 +- pkg/plugins/policies/core/xds/clusters.go | 5 + pkg/plugins/policies/core/xds/endpoints.go | 16 +- .../api/v1alpha1/meshloadbalancingstrategy.go | 2 +- .../api/v1alpha1/schema.yaml | 1 - .../api/v1alpha1/validator.go | 4 +- .../api/v1alpha1/validator_test.go | 13 + .../api/v1alpha1/zz_generated.deepcopy.go | 8 +- .../kuma.io_meshloadbalancingstrategies.yaml | 1 - .../plugin/v1alpha1/locality_aware.go | 27 +- .../plugin/v1alpha1/plugin.go | 79 +- .../plugin/v1alpha1/plugin_test.go | 921 ++++-------------- .../plugin/v1alpha1/priority.go | 8 +- .../testdata/egress.clusters.golden.yaml | 3 + .../testdata/egress.endpoints.golden.yaml | 23 +- .../plugin/v1alpha1/plugin.go | 4 +- pkg/xds/sync/egress_proxy_builder.go | 46 +- .../locality_aware_multizone_hybrid_egress.go | 133 +++ .../e2e_env/multizone/multizone_suite_test.go | 1 + 30 files changed, 495 insertions(+), 848 deletions(-) create mode 100644 test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml index 47b3bcebc4e7..74eb289950ff 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml @@ -2086,7 +2086,6 @@ spec: type: array required: - type - - zones type: object required: - to diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.gateway-api-present-not-enabled.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.gateway-api-present-not-enabled.yaml index 47b3bcebc4e7..74eb289950ff 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.gateway-api-present-not-enabled.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.gateway-api-present-not-enabled.yaml @@ -2086,7 +2086,6 @@ spec: type: array required: - type - - zones type: object required: - to diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.gateway-api-present.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.gateway-api-present.yaml index 788407d8ad2b..e6daaf0e42d0 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.gateway-api-present.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.gateway-api-present.yaml @@ -2290,7 +2290,6 @@ spec: type: array required: - type - - zones type: object required: - to diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.with-helm-set.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.with-helm-set.yaml index 4cb1efe840fa..e02966d6f560 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.with-helm-set.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.with-helm-set.yaml @@ -2106,7 +2106,6 @@ spec: type: array required: - type - - zones type: object required: - to diff --git a/app/kumactl/cmd/install/testdata/install-crds.all.golden.yaml b/app/kumactl/cmd/install/testdata/install-crds.all.golden.yaml index cfcb7276470d..40216002b63f 100644 --- a/app/kumactl/cmd/install/testdata/install-crds.all.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-crds.all.golden.yaml @@ -3391,7 +3391,6 @@ spec: type: array required: - type - - zones type: object required: - to diff --git a/app/kumactl/cmd/install/testdata/install-crds.experimental-gatewayapi.golden.yaml b/app/kumactl/cmd/install/testdata/install-crds.experimental-gatewayapi.golden.yaml index 9873e32e9f76..98d49d337c1d 100644 --- a/app/kumactl/cmd/install/testdata/install-crds.experimental-gatewayapi.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-crds.experimental-gatewayapi.golden.yaml @@ -3595,7 +3595,6 @@ spec: type: array required: - type - - zones type: object required: - to diff --git a/deployments/charts/kuma/crds/kuma.io_meshloadbalancingstrategies.yaml b/deployments/charts/kuma/crds/kuma.io_meshloadbalancingstrategies.yaml index 042a6e0590e8..0d4a6e3dd563 100644 --- a/deployments/charts/kuma/crds/kuma.io_meshloadbalancingstrategies.yaml +++ b/deployments/charts/kuma/crds/kuma.io_meshloadbalancingstrategies.yaml @@ -414,7 +414,6 @@ spec: type: array required: - type - - zones type: object required: - to diff --git a/docs/generated/openapi.yaml b/docs/generated/openapi.yaml index b572a0894f20..e70ae5e61777 100644 --- a/docs/generated/openapi.yaml +++ b/docs/generated/openapi.yaml @@ -4052,7 +4052,6 @@ components: type: array required: - type - - zones type: object required: - to diff --git a/docs/generated/raw/crds/kuma.io_meshloadbalancingstrategies.yaml b/docs/generated/raw/crds/kuma.io_meshloadbalancingstrategies.yaml index 042a6e0590e8..0d4a6e3dd563 100644 --- a/docs/generated/raw/crds/kuma.io_meshloadbalancingstrategies.yaml +++ b/docs/generated/raw/crds/kuma.io_meshloadbalancingstrategies.yaml @@ -414,7 +414,6 @@ spec: type: array required: - type - - zones type: object required: - to diff --git a/pkg/core/plugins/interfaces.go b/pkg/core/plugins/interfaces.go index b0d2939bbc0b..0a93f33d26e7 100644 --- a/pkg/core/plugins/interfaces.go +++ b/pkg/core/plugins/interfaces.go @@ -102,7 +102,7 @@ type EgressPolicyPlugin interface { PolicyPlugin // EgressMatchedPolicies returns all the policies of the plugins' type matching the external service that // should be applied on the zone egress. - EgressMatchedPolicies(*core_mesh.ExternalServiceResource, xds_context.Resources) (core_xds.TypedMatchingPolicies, error) + EgressMatchedPolicies(map[string]string, xds_context.Resources) (core_xds.TypedMatchingPolicies, error) } // ProxyPlugin a plugin to modify the proxy. This happens before any `PolicyPlugin` or any envoy generation. and it is applied both for Dataplanes and ZoneProxies diff --git a/pkg/plugins/policies/core/matchers/egress.go b/pkg/plugins/policies/core/matchers/egress.go index fd2c0d324d2d..2ef66803a842 100644 --- a/pkg/plugins/policies/core/matchers/egress.go +++ b/pkg/plugins/policies/core/matchers/egress.go @@ -7,14 +7,13 @@ import ( common_api "github.com/kumahq/kuma/api/common/v1alpha1" mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" - core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" core_model "github.com/kumahq/kuma/pkg/core/resources/model" core_xds "github.com/kumahq/kuma/pkg/core/xds" core_rules "github.com/kumahq/kuma/pkg/plugins/policies/core/rules" xds_context "github.com/kumahq/kuma/pkg/xds/context" ) -func EgressMatchedPolicies(rType core_model.ResourceType, es *core_mesh.ExternalServiceResource, resources xds_context.Resources) (core_xds.TypedMatchingPolicies, error) { +func EgressMatchedPolicies(rType core_model.ResourceType, tags map[string]string, resources xds_context.Resources) (core_xds.TypedMatchingPolicies, error) { policies := resources.ListOrEmpty(rType) if len(policies.GetItems()) == 0 { @@ -41,9 +40,9 @@ func EgressMatchedPolicies(rType core_model.ResourceType, es *core_mesh.External var fr core_rules.FromRules var err error if isFrom { - fr, err = processFromRules(es, policies) + fr, err = processFromRules(tags, policies) } else { - fr, err = processToRules(es, policies) + fr, err = processToRules(tags, policies) } if err != nil { return core_xds.TypedMatchingPolicies{}, err @@ -56,14 +55,14 @@ func EgressMatchedPolicies(rType core_model.ResourceType, es *core_mesh.External } func processFromRules( - es *core_mesh.ExternalServiceResource, + tags map[string]string, rl core_model.ResourceList, ) (core_rules.FromRules, error) { matchedPolicies := []core_model.Resource{} for _, policy := range rl.GetItems() { spec := policy.GetSpec().(core_model.Policy) - if !externalServiceSelectedByTargetRef(spec.GetTargetRef(), es) { + if !serviceSelectedByTargetRef(spec.GetTargetRef(), tags) { continue } matchedPolicies = append(matchedPolicies, policy) @@ -113,11 +112,8 @@ func processFromRules( // disabled: true // // that's why processToRules() method produces FromRules for the Egress. -func processToRules( - es *core_mesh.ExternalServiceResource, - rl core_model.ResourceList, -) (core_rules.FromRules, error) { - matchedPolicies := []core_model.Resource{} +func processToRules(tags map[string]string, rl core_model.ResourceList) (core_rules.FromRules, error) { + var matchedPolicies []core_model.Resource for _, policy := range rl.GetItems() { spec := policy.GetSpec().(core_model.Policy) @@ -128,7 +124,7 @@ func processToRules( } for _, item := range to.GetToList() { - if externalServiceSelectedByTargetRef(item.GetTargetRef(), es) { + if serviceSelectedByTargetRef(item.GetTargetRef(), tags) { matchedPolicies = append(matchedPolicies, policy) } } @@ -136,10 +132,10 @@ func processToRules( sort.Sort(ByTargetRef(matchedPolicies)) - toList := []core_rules.PolicyItemWithMeta{} + var toList []core_rules.PolicyItemWithMeta for _, policy := range matchedPolicies { for _, item := range policy.GetSpec().(core_model.PolicyWithToList).GetToList() { - if !externalServiceSelectedByTargetRef(item.GetTargetRef(), es) { + if !serviceSelectedByTargetRef(item.GetTargetRef(), tags) { continue } // convert 'to' policyItem to 'from' policyItem @@ -175,17 +171,16 @@ func (a *artificialPolicyItem) GetDefault() interface{} { return a.conf } -func externalServiceSelectedByTargetRef(tr common_api.TargetRef, es *core_mesh.ExternalServiceResource) bool { +func serviceSelectedByTargetRef(tr common_api.TargetRef, tags map[string]string) bool { switch tr.Kind { case common_api.Mesh: return true case common_api.MeshSubset: - return mesh_proto.TagSelector(tr.Tags).Matches(es.Spec.GetTags()) + return mesh_proto.TagSelector(tr.Tags).Matches(tags) case common_api.MeshService: - return tr.Name == es.Spec.GetTags()[mesh_proto.ServiceTag] + return tr.Name == tags[mesh_proto.ServiceTag] case common_api.MeshServiceSubset: - return tr.Name == es.Spec.GetTags()[mesh_proto.ServiceTag] && - mesh_proto.TagSelector(tr.Tags).Matches(es.Spec.GetTags()) + return tr.Name == tags[mesh_proto.ServiceTag] && mesh_proto.TagSelector(tr.Tags).Matches(tags) } return false } diff --git a/pkg/plugins/policies/core/matchers/egress_test.go b/pkg/plugins/policies/core/matchers/egress_test.go index fc604c006696..cfa1e813643c 100644 --- a/pkg/plugins/policies/core/matchers/egress_test.go +++ b/pkg/plugins/policies/core/matchers/egress_test.go @@ -59,7 +59,7 @@ var _ = Describe("EgressMatchedPolicies", func() { resources, _ := readPolicies(given.policiesFile) // when - policies, err := matchers.EgressMatchedPolicies(policies_api.MeshTrafficPermissionType, es, resources) + policies, err := matchers.EgressMatchedPolicies(policies_api.MeshTrafficPermissionType, es.Spec.Tags, resources) Expect(err).ToNot(HaveOccurred()) // then @@ -76,7 +76,7 @@ var _ = Describe("EgressMatchedPolicies", func() { resources, _ := readPolicies(given.policiesFile) // when - policies, err := matchers.EgressMatchedPolicies(v1alpha1.MeshLoadBalancingStrategyType, es, resources) + policies, err := matchers.EgressMatchedPolicies(v1alpha1.MeshLoadBalancingStrategyType, es.Spec.Tags, resources) Expect(err).ToNot(HaveOccurred()) // then diff --git a/pkg/plugins/policies/core/xds/clusters.go b/pkg/plugins/policies/core/xds/clusters.go index 01671940f478..ff8f1d28e55a 100644 --- a/pkg/plugins/policies/core/xds/clusters.go +++ b/pkg/plugins/policies/core/xds/clusters.go @@ -10,6 +10,7 @@ import ( "github.com/kumahq/kuma/pkg/plugins/runtime/gateway/metadata" "github.com/kumahq/kuma/pkg/xds/envoy/tags" "github.com/kumahq/kuma/pkg/xds/generator" + "github.com/kumahq/kuma/pkg/xds/generator/egress" ) type Clusters struct { @@ -17,6 +18,7 @@ type Clusters struct { Outbound map[string]*envoy_cluster.Cluster OutboundSplit map[string][]*envoy_cluster.Cluster Gateway map[string]*envoy_cluster.Cluster + Egress map[string]*envoy_cluster.Cluster } func GatherClusters(rs *core_xds.ResourceSet) Clusters { @@ -25,6 +27,7 @@ func GatherClusters(rs *core_xds.ResourceSet) Clusters { Outbound: map[string]*envoy_cluster.Cluster{}, OutboundSplit: map[string][]*envoy_cluster.Cluster{}, Gateway: map[string]*envoy_cluster.Cluster{}, + Egress: map[string]*envoy_cluster.Cluster{}, } for _, res := range rs.Resources(envoy_resource.ClusterType) { cluster := res.Resource.(*envoy_cluster.Cluster) @@ -42,6 +45,8 @@ func GatherClusters(rs *core_xds.ResourceSet) Clusters { clusters.Inbound[cluster.Name] = cluster case metadata.OriginGateway: clusters.Gateway[cluster.Name] = cluster + case egress.OriginEgress: + clusters.Egress[cluster.Name] = cluster default: continue } diff --git a/pkg/plugins/policies/core/xds/endpoints.go b/pkg/plugins/policies/core/xds/endpoints.go index 6e9caccd67e7..779a2133d80e 100644 --- a/pkg/plugins/policies/core/xds/endpoints.go +++ b/pkg/plugins/policies/core/xds/endpoints.go @@ -38,8 +38,18 @@ func GatherEndpoints(rs *xds.ResourceSet) EndpointMap { return em } -func GatherEgressEndpoints(rs *xds.ResourceSet) map[string]*endpointv3.ClusterLoadAssignment { - em := map[string]*endpointv3.ClusterLoadAssignment{} +func GatherEgressEndpoints(rs *xds.ResourceSet) EndpointMap { + em := EndpointMap{} + for _, res := range rs.Resources(envoy_resource.EndpointType) { + if res.Origin != egress.OriginEgress { + continue + } + + cla := res.Resource.(*endpointv3.ClusterLoadAssignment) + serviceName := tags.ServiceFromClusterName(cla.ClusterName) + em[serviceName] = append(em[serviceName], cla) + } + for _, res := range rs.Resources(envoy_resource.ClusterType) { if res.Origin != egress.OriginEgress { continue @@ -47,7 +57,7 @@ func GatherEgressEndpoints(rs *xds.ResourceSet) map[string]*endpointv3.ClusterLo cluster := res.Resource.(*clusterv3.Cluster) if cluster.LoadAssignment != nil { - em[cluster.Name] = cluster.LoadAssignment + em[cluster.Name] = append(em[cluster.Name], cluster.LoadAssignment) } } return em diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/meshloadbalancingstrategy.go b/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/meshloadbalancingstrategy.go index bf8dee6026f2..3181111180a5 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/meshloadbalancingstrategy.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/meshloadbalancingstrategy.go @@ -88,7 +88,7 @@ type FromZone struct { type ToZone struct { // Type defines how target zones will be picked from available zones Type ToZoneType `json:"type"` - Zones []string `json:"zones"` + Zones *[]string `json:"zones,omitempty"` } type FailoverThreshold struct { diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/schema.yaml b/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/schema.yaml index f4b0712c2066..5d5d28efed4a 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/schema.yaml +++ b/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/schema.yaml @@ -279,7 +279,6 @@ properties: type: array required: - type - - zones type: object required: - to diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/validator.go b/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/validator.go index 438c6451f631..3e95168a6ec2 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/validator.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/validator.go @@ -111,11 +111,11 @@ func validateCrossZone(crossZone *CrossZone) validators.ValidationError { toZonesPath := path.Field("to").Field("zones") switch failover.To.Type { case Any, None: - if len(failover.To.Zones) > 0 { + if failover.To.Zones != nil && len(*failover.To.Zones) > 0 { verr.AddViolationAt(toZonesPath, fmt.Sprintf("must be empty when type is %s", failover.To.Type)) } case AnyExcept, Only: - if len(failover.To.Zones) == 0 { + if failover.To.Zones == nil || len(*failover.To.Zones) == 0 { verr.AddViolationAt(toZonesPath, fmt.Sprintf("must not be empty when type is %s", failover.To.Type)) } default: diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/validator_test.go b/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/validator_test.go index 5f1cdd3e9ada..e1a9fba581b2 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/validator_test.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/validator_test.go @@ -324,6 +324,14 @@ to: }, { Field: "spec.to[0].default.localityAwareness.crossZone.failover[4].to.zones", + Message: "must not be empty when type is Only", + }, + { + Field: "spec.to[0].default.localityAwareness.crossZone.failover[5].to.zones", + Message: "must not be empty when type is AnyExcept", + }, + { + Field: "spec.to[0].default.localityAwareness.crossZone.failover[6].to.zones", Message: "must not be empty when type is AnyExcept", }, }, ` @@ -355,9 +363,14 @@ to: - to: type: Only zones: [] + - to: + type: Only - to: type: AnyExcept zones: [] + - to: + type: AnyExcept + `), ) diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/zz_generated.deepcopy.go b/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/zz_generated.deepcopy.go index 9c16b1c18647..285527e5d695 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1/zz_generated.deepcopy.go @@ -531,8 +531,12 @@ func (in *ToZone) DeepCopyInto(out *ToZone) { *out = *in if in.Zones != nil { in, out := &in.Zones, &out.Zones - *out = make([]string, len(*in)) - copy(*out, *in) + *out = new([]string) + if **in != nil { + in, out := *in, *out + *out = make([]string, len(*in)) + copy(*out, *in) + } } } diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/k8s/crd/kuma.io_meshloadbalancingstrategies.yaml b/pkg/plugins/policies/meshloadbalancingstrategy/k8s/crd/kuma.io_meshloadbalancingstrategies.yaml index 042a6e0590e8..0d4a6e3dd563 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/k8s/crd/kuma.io_meshloadbalancingstrategies.yaml +++ b/pkg/plugins/policies/meshloadbalancingstrategy/k8s/crd/kuma.io_meshloadbalancingstrategies.yaml @@ -414,7 +414,6 @@ spec: type: array required: - type - - zones type: object required: - to diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go index 5ab2d9aebb25..3a5ab0382076 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go @@ -19,21 +19,18 @@ import ( const defaultOverprovisingFactor uint32 = 200 func ConfigureStaticEndpointsLocalityAware( - proxy *core_xds.Proxy, + tags mesh_proto.MultiValueTagSet, endpoints []*envoy_endpoint.ClusterLoadAssignment, cluster *envoy_cluster.Cluster, conf api.Conf, serviceName string, + localZone string, + apiVersion core_xds.APIVersion, ) error { - var localZone string - if tags := proxy.Dataplane.Spec.TagSet().Values(mesh_proto.ZoneTag); len(tags) > 0 { - localZone = tags[0] - } - if conf.LocalityAwareness != nil { for _, cla := range endpoints { if cla.ClusterName == serviceName { - loadAssignment, err := ConfigureEnpointLocalityAwareLb(proxy, &conf, cla, cla.ClusterName, localZone) + loadAssignment, err := ConfigureEndpointLocalityAwareLb(tags, &conf, cla, cla.ClusterName, localZone, apiVersion) if err != nil { return err } @@ -45,17 +42,18 @@ func ConfigureStaticEndpointsLocalityAware( } func ConfigureEndpointsLocalityAware( - proxy *core_xds.Proxy, + tags mesh_proto.MultiValueTagSet, endpoints []*envoy_endpoint.ClusterLoadAssignment, conf api.Conf, rs *core_xds.ResourceSet, serviceName string, localZone string, + apiVersion core_xds.APIVersion, ) error { if conf.LocalityAwareness != nil { for _, cla := range endpoints { if cla.ClusterName == serviceName { - loadAssignment, err := ConfigureEnpointLocalityAwareLb(proxy, &conf, cla, cla.ClusterName, localZone) + loadAssignment, err := ConfigureEndpointLocalityAwareLb(tags, &conf, cla, cla.ClusterName, localZone, apiVersion) if err != nil { return err } @@ -70,15 +68,16 @@ func ConfigureEndpointsLocalityAware( return nil } -func ConfigureEnpointLocalityAwareLb( - proxy *core_xds.Proxy, +func ConfigureEndpointLocalityAwareLb( + tags mesh_proto.MultiValueTagSet, conf *api.Conf, cla *envoy_endpoint.ClusterLoadAssignment, serviceName string, localZone string, + apiVersion core_xds.APIVersion, ) (proto.Message, error) { - localPriorityGroups, crossZonePriorityGroups := GetLocalityGroups(conf, proxy.Dataplane.Spec.TagSet(), localZone) - endpointsList := []core_xds.Endpoint{} + localPriorityGroups, crossZonePriorityGroups := GetLocalityGroups(conf, tags, localZone) + var endpointsList []core_xds.Endpoint for _, localityLbEndpoint := range cla.Endpoints { for _, lbEndpoint := range localityLbEndpoint.LbEndpoints { ed := createEndpoint(lbEndpoint, localZone) @@ -93,7 +92,7 @@ func ConfigureEnpointLocalityAwareLb( } } // TODO(lukidzi): use CLA Cache https://github.com/kumahq/kuma/issues/8121 - loadAssignment, err := envoy_endpoints.CreateClusterLoadAssignment(serviceName, endpointsList, proxy.APIVersion) + loadAssignment, err := envoy_endpoints.CreateClusterLoadAssignment(serviceName, endpointsList, apiVersion) if err != nil { return nil, err } diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin.go b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin.go index adc39cb503a8..1a12a570e3ee 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin.go @@ -6,7 +6,9 @@ import ( envoy_listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" envoy_route "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" envoy_hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + "github.com/kumahq/kuma/pkg/core" "github.com/pkg/errors" + "golang.org/x/exp/maps" mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" core_plugins "github.com/kumahq/kuma/pkg/core/plugins" @@ -32,12 +34,14 @@ func NewPlugin() core_plugins.Plugin { return &plugin{} } +var log = core.Log.WithName("MeshLoadBalance") + func (p plugin) MatchedPolicies(dataplane *core_mesh.DataplaneResource, resources xds_context.Resources) (core_xds.TypedMatchingPolicies, error) { return matchers.MatchedPolicies(api.MeshLoadBalancingStrategyType, dataplane, resources) } -func (p plugin) EgressMatchedPolicies(es *core_mesh.ExternalServiceResource, resources xds_context.Resources) (core_xds.TypedMatchingPolicies, error) { - return matchers.EgressMatchedPolicies(api.MeshLoadBalancingStrategyType, es, resources) +func (p plugin) EgressMatchedPolicies(tags map[string]string, resources xds_context.Resources) (core_xds.TypedMatchingPolicies, error) { + return matchers.EgressMatchedPolicies(api.MeshLoadBalancingStrategyType, tags, resources) } func (p plugin) Apply(rs *core_xds.ResourceSet, ctx xds_context.Context, proxy *core_xds.Proxy) error { @@ -99,7 +103,7 @@ func (p plugin) configureDPP( if err := p.configureCluster(cluster, conf); err != nil { return err } - if err := configureEndpoints(proxy, cluster, endpoints[serviceName], serviceName, conf, rs); err != nil { + if err := configureEndpoints(proxy.Dataplane.Spec.TagSet(), cluster, endpoints[serviceName], serviceName, conf, rs, proxy.Zone, proxy.APIVersion); err != nil { return errors.Wrapf(err, "failed to configure ClusterLoadAssignment for %s", serviceName) } } @@ -107,7 +111,7 @@ func (p plugin) configureDPP( if err := p.configureCluster(cluster, conf); err != nil { return err } - if err := configureEndpoints(proxy, cluster, endpoints[serviceName], cluster.Name, conf, rs); err != nil { + if err := configureEndpoints(proxy.Dataplane.Spec.TagSet(), cluster, endpoints[serviceName], cluster.Name, conf, rs, proxy.Zone, proxy.APIVersion); err != nil { return errors.Wrapf(err, "failed to configure ClusterLoadAssignment for %s", cluster.Name) } } @@ -117,25 +121,21 @@ func (p plugin) configureDPP( } func configureEndpoints( - proxy *core_xds.Proxy, + tags mesh_proto.MultiValueTagSet, cluster *envoy_cluster.Cluster, endpoints []*envoy_endpoint.ClusterLoadAssignment, serviceName string, conf api.Conf, rs *core_xds.ResourceSet, + localZone string, + apiVersion core_xds.APIVersion, ) error { - var localZone string - if tags := proxy.Dataplane.Spec.TagSet().Values(mesh_proto.ZoneTag); len(tags) > 0 { - localZone = tags[0] - } - - // ExternalServices have static endpoints if cluster.LoadAssignment != nil { - if err := ConfigureStaticEndpointsLocalityAware(proxy, endpoints, cluster, conf, serviceName); err != nil { + if err := ConfigureStaticEndpointsLocalityAware(tags, endpoints, cluster, conf, serviceName, localZone, apiVersion); err != nil { return err } } else { - if err := ConfigureEndpointsLocalityAware(proxy, endpoints, conf, rs, serviceName, localZone); err != nil { + if err := ConfigureEndpointsLocalityAware(tags, endpoints, conf, rs, serviceName, localZone, apiVersion); err != nil { return err } } @@ -201,7 +201,7 @@ func (p plugin) configureGateway( } serviceName := dest.Destination[mesh_proto.ServiceTag] - if err := configureEndpoints(proxy, cluster, endpoints[serviceName], serviceName, *conf, rs); err != nil { + if err := configureEndpoints(proxy.Dataplane.Spec.TagSet(), cluster, endpoints[serviceName], serviceName, *conf, rs, proxy.Zone, proxy.APIVersion); err != nil { return err } } @@ -212,38 +212,28 @@ func (p plugin) configureGateway( } func (p plugin) configureEgress(rs *core_xds.ResourceSet, proxy *core_xds.Proxy) error { - localityAwareExternalServices := map[string][]core_xds.ServiceName{} + endpoints := policies_xds.GatherEgressEndpoints(rs) + clusters := policies_xds.GatherClusters(rs) for _, mr := range proxy.ZoneEgressProxy.MeshResourcesList { - for es, dynamic := range mr.Dynamic { + for serviceName, dynamic := range mr.Dynamic { + meshName := mr.Mesh.GetMeta().GetName() policies, ok := dynamic[api.MeshLoadBalancingStrategyType] if !ok { continue } - if !p.isLocalityAware(policies.FromRules) { + rule := p.computeFrom(policies.FromRules) + if rule == nil { continue } + conf := rule.Conf.(api.Conf) - meshName := mr.Mesh.GetMeta().GetName() - localityAwareExternalServices[meshName] = append(localityAwareExternalServices[meshName], es) - } - } - - endpoints := policies_xds.GatherEgressEndpoints(rs) - zone := proxy.ZoneEgressProxy.ZoneEgressResource.Spec.GetZone() - - for meshName, externalServices := range localityAwareExternalServices { - for _, es := range externalServices { - clusterName := envoy_names.GetMeshClusterName(meshName, es) - cla, ok := endpoints[clusterName] - if !ok { - continue - } - for _, localityLbEndpoints := range cla.Endpoints { - if localityLbEndpoints.Locality != nil && localityLbEndpoints.Locality.Zone != zone { - localityLbEndpoints.Priority = 1 - } + clusterName := envoy_names.GetMeshClusterName(meshName, serviceName) + cluster, ok := clusters.Egress[clusterName] + err := configureEndpoints(mesh_proto.MultiValueTagSet{}, cluster, endpoints[clusterName], clusterName, conf, rs, proxy.Zone, proxy.APIVersion) + if err != nil { + return err } } } @@ -252,18 +242,13 @@ func (p plugin) configureEgress(rs *core_xds.ResourceSet, proxy *core_xds.Proxy) } // Zone egress is a single point for multiple clients. At this moment we don't support different -// configurations based on the client, that's why locality awareness is enabled if at least one -// client requires it to be enabled. -func (p plugin) isLocalityAware(fr core_rules.FromRules) bool { - for _, rules := range fr.Rules { - for _, r := range rules { - conf := r.Conf.(api.Conf) - if conf.LocalityAwareness == nil || !pointer.Deref(conf.LocalityAwareness.Disabled) { - return true - } - } +// configurations based on the client. That's why we are computing rules for MeshSubset +func (p plugin) computeFrom(fr core_rules.FromRules) *core_rules.Rule { + rules := maps.Values(fr.Rules) + if len(rules) == 0 { + return nil } - return false + return rules[0].Compute(core_rules.MeshSubset()) } func (p plugin) configureListener( diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin_test.go b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin_test.go index 91bb3513599f..de49f30ee600 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin_test.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin_test.go @@ -81,24 +81,8 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { Name: "backend", Origin: generator.OriginOutbound, Resource: endpoints.CreateClusterLoadAssignment("backend", []core_xds.Endpoint{ - { - Target: "192.168.1.1", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.2", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-2", - }, - Locality: &core_xds.Locality{Zone: "zone-2", Priority: 0}, - }, + createEndpointWith("zone-1", "192.168.1.1", map[string]string{}), + createEndpointWith("zone-2", "192.168.1.2", map[string]string{}), }), }, { @@ -107,77 +91,24 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { Resource: clusters.NewClusterBuilder(envoy_common.APIV3, "payment"). Configure(clusters.ProvidedEndpointCluster( false, - core_xds.Endpoint{ - Target: "192.168.0.1", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - core_xds.Endpoint{ - Target: "192.168.0.2", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-2", - }, - Locality: &core_xds.Locality{Zone: "zone-2", Priority: 0}, - }, + createEndpointWith("zone-1", "192.168.0.1", map[string]string{}), + createEndpointWith("zone-2", "192.168.0.2", map[string]string{}), )).MustBuild(), }, { - Name: "backend", - Origin: generator.OriginOutbound, - Resource: NewOutboundListenerBuilder(envoy_common.APIV3, "127.0.0.1", 27777, core_xds.SocketAddressProtocolTCP). - Configure(FilterChain(NewFilterChainBuilder(envoy_common.APIV3, envoy_common.AnonymousResource). - Configure(HttpConnectionManager("127.0.0.1:27777", false)). - Configure( - HttpOutboundRoute( - "backend", - envoy_common.Routes{{ - Clusters: []envoy_common.Cluster{envoy_common.NewCluster( - envoy_common.WithService("backend"), - envoy_common.WithWeight(100), - )}, - }}, - map[string]map[string]bool{ - "kuma.io/service": { - "backend": true, - }, - }, - ), - ), - )).MustBuild(), + Name: "backend", + Origin: generator.OriginOutbound, + Resource: backendListener(), }, { - Name: "payments", - Origin: generator.OriginOutbound, - Resource: NewOutboundListenerBuilder(envoy_common.APIV3, "127.0.0.1", 27778, core_xds.SocketAddressProtocolTCP). - Configure(FilterChain(NewFilterChainBuilder(envoy_common.APIV3, envoy_common.AnonymousResource). - Configure(HttpConnectionManager("127.0.0.1:27778", false)). - Configure( - HttpOutboundRoute( - "backend", - envoy_common.Routes{{ - Clusters: []envoy_common.Cluster{envoy_common.NewCluster( - envoy_common.WithService("payment"), - envoy_common.WithWeight(100), - )}, - }}, - map[string]map[string]bool{ - "kuma.io/service": { - "payment": true, - }, - }, - ), - ), - )).MustBuild(), + Name: "payments", + Origin: generator.OriginOutbound, + Resource: paymentsListener(), }, }, proxy: &core_xds.Proxy{ APIVersion: envoy_common.APIV3, + Zone: "zone-1", Dataplane: builders.Dataplane(). AddInboundOfTagsMap(map[string]string{ mesh_proto.ServiceTag: "backend", @@ -270,27 +201,12 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { MustBuild(), }, { - Name: "mesh-1:cla-for-eds-cluster", + Name: "mesh-1:eds-cluster", Origin: egress.OriginEgress, Resource: endpoints.CreateClusterLoadAssignment("mesh-1:eds-cluster", []core_xds.Endpoint{ - { - Target: "192.168.1.1", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.2", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-2", - }, - Locality: &core_xds.Locality{Zone: "zone-2", Priority: 0}, - }, + createEndpointWith("zone-1", "192.168.1.1", map[string]string{}), + createEndpointWith("zone-2", "192.168.1.2", map[string]string{}), + createEndpointWith("zone-3", "192.168.1.3", map[string]string{}), }), }, { @@ -299,33 +215,15 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { Resource: clusters.NewClusterBuilder(envoy_common.APIV3, "mesh-2:static-cluster"). Configure(clusters.ProvidedEndpointCluster( false, - core_xds.Endpoint{ - Target: "192.168.0.1", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - core_xds.Endpoint{ - Target: "192.168.0.2", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-2", - }, - Locality: &core_xds.Locality{Zone: "zone-2", Priority: 0}, - }, + createEndpointWith("zone-1", "192.168.0.1", map[string]string{}), + createEndpointWith("zone-2", "192.168.0.2", map[string]string{}), )).MustBuild(), }, }, proxy: &core_xds.Proxy{ APIVersion: envoy_common.APIV3, + Zone: "zone-1", ZoneEgressProxy: &core_xds.ZoneEgressProxy{ - ZoneEgressResource: &core_mesh.ZoneEgressResource{ - Spec: &mesh_proto.ZoneEgress{Zone: "zone-1"}, - }, MeshResourcesList: []*core_xds.MeshResources{ { Mesh: builders.Mesh().WithName("mesh-1").Build(), @@ -335,7 +233,27 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { FromRules: core_rules.FromRules{ Rules: map[core_rules.InboundListener]core_rules.Rules{ {}: { - {Conf: v1alpha1.Conf{LocalityAwareness: &v1alpha1.LocalityAwareness{Disabled: pointer.To(false)}}}, + {Conf: v1alpha1.Conf{LocalityAwareness: &v1alpha1.LocalityAwareness{ + Disabled: pointer.To(false), + CrossZone: &v1alpha1.CrossZone{ + Failover: []v1alpha1.Failover{ + { + From: &v1alpha1.FromZone{Zones: []string{"zone-1"}}, + To: v1alpha1.ToZone{ + Type: v1alpha1.Only, + Zones: &[]string{"zone-2"}, + }, + }, + { + From: &v1alpha1.FromZone{Zones: []string{"zone-1"}}, + To: v1alpha1.ToZone{ + Type: v1alpha1.Only, + Zones: &[]string{"zone-3"}, + }, + }, + }, + }, + }}}, }, }, }, @@ -351,7 +269,18 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { FromRules: core_rules.FromRules{ Rules: map[core_rules.InboundListener]core_rules.Rules{ {}: { - {Conf: v1alpha1.Conf{LocalityAwareness: &v1alpha1.LocalityAwareness{Disabled: pointer.To(false)}}}, + {Conf: v1alpha1.Conf{LocalityAwareness: &v1alpha1.LocalityAwareness{ + Disabled: pointer.To(false), + CrossZone: &v1alpha1.CrossZone{ + Failover: []v1alpha1.Failover{ + { + To: v1alpha1.ToZone{ + Type: v1alpha1.Any, + }, + }, + }, + }, + }}}, }, }, }, @@ -376,83 +305,14 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { Name: "backend", Origin: generator.OriginOutbound, Resource: endpoints.CreateClusterLoadAssignment("backend", []core_xds.Endpoint{ - { - Target: "192.168.1.1", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - - "k8s.io/node": "node1", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.2", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - "k8s.io/node": "node2", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.3", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - "k8s.io/az": "test", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.4", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - "k8s.io/region": "test", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.5", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-2", - }, - Locality: &core_xds.Locality{Zone: "zone-2", Priority: 0}, - }, - { - Target: "192.168.1.6", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-3", - }, - Locality: &core_xds.Locality{Zone: "zone-3", Priority: 0}, - }, - { - Target: "192.168.1.7", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-4", - }, - Locality: &core_xds.Locality{Zone: "zone-4", Priority: 0}, - }, - { - Target: "192.168.1.8", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-5", - }, - Locality: &core_xds.Locality{Zone: "zone-5", Priority: 0}, - }, + createEndpointWith("zone-1", "192.168.1.1", map[string]string{"k8s.io/node": "node1"}), + createEndpointWith("zone-1", "192.168.1.2", map[string]string{"k8s.io/node": "node2"}), + createEndpointWith("zone-1", "192.168.1.3", map[string]string{"k8s.io/az": "test"}), + createEndpointWith("zone-1", "192.168.1.4", map[string]string{"k8s.io/region": "test"}), + createEndpointWith("zone-2", "192.168.1.5", map[string]string{}), + createEndpointWith("zone-3", "192.168.1.6", map[string]string{}), + createEndpointWith("zone-4", "192.168.1.7", map[string]string{}), + createEndpointWith("zone-5", "192.168.1.8", map[string]string{}), }), }, { @@ -461,88 +321,25 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { Resource: clusters.NewClusterBuilder(envoy_common.APIV3, "payment"). Configure(clusters.ProvidedEndpointCluster( false, - core_xds.Endpoint{ - Target: "192.168.0.1", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - "k8s.io/node": "node1", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - core_xds.Endpoint{ - Target: "192.168.0.2", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - "k8s.io/node": "node2", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - core_xds.Endpoint{ - Target: "192.168.0.3", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-2", - }, - Locality: &core_xds.Locality{Zone: "zone-2", Priority: 0}, - }, + createEndpointWith("zone-1", "192.168.0.1", map[string]string{"k8s.io/node": "node1"}), + createEndpointWith("zone-1", "192.168.0.2", map[string]string{"k8s.io/node": "node2"}), + createEndpointWith("zone-2", "192.168.0.3", map[string]string{}), )).MustBuild(), }, { - Name: "backend", - Origin: generator.OriginOutbound, - Resource: NewOutboundListenerBuilder(envoy_common.APIV3, "127.0.0.1", 27777, core_xds.SocketAddressProtocolTCP). - Configure(FilterChain(NewFilterChainBuilder(envoy_common.APIV3, envoy_common.AnonymousResource). - Configure(HttpConnectionManager("127.0.0.1:27777", false)). - Configure( - HttpOutboundRoute( - "backend", - envoy_common.Routes{{ - Clusters: []envoy_common.Cluster{envoy_common.NewCluster( - envoy_common.WithService("backend"), - envoy_common.WithWeight(100), - )}, - }}, - map[string]map[string]bool{ - "kuma.io/service": { - "backend": true, - }, - }, - ), - ), - )).MustBuild(), + Name: "backend", + Origin: generator.OriginOutbound, + Resource: backendListener(), }, { - Name: "payments", - Origin: generator.OriginOutbound, - Resource: NewOutboundListenerBuilder(envoy_common.APIV3, "127.0.0.1", 27778, core_xds.SocketAddressProtocolTCP). - Configure(FilterChain(NewFilterChainBuilder(envoy_common.APIV3, envoy_common.AnonymousResource). - Configure(HttpConnectionManager("127.0.0.1:27778", false)). - Configure( - HttpOutboundRoute( - "backend", - envoy_common.Routes{{ - Clusters: []envoy_common.Cluster{envoy_common.NewCluster( - envoy_common.WithService("payment"), - envoy_common.WithWeight(100), - )}, - }}, - map[string]map[string]bool{ - "kuma.io/service": { - "payment": true, - }, - }, - ), - ), - )).MustBuild(), + Name: "payments", + Origin: generator.OriginOutbound, + Resource: paymentsListener(), }, }, proxy: &core_xds.Proxy{ APIVersion: envoy_common.APIV3, + Zone: "zone-1", Dataplane: builders.Dataplane(). AddInboundOfTagsMap(map[string]string{ mesh_proto.ServiceTag: "backend", @@ -598,7 +395,7 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { { To: v1alpha1.ToZone{ Type: v1alpha1.AnyExcept, - Zones: []string{"zone-3", "zone-4", "zone-5"}, + Zones: &[]string{"zone-3", "zone-4", "zone-5"}, }, }, { @@ -607,13 +404,13 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { }, To: v1alpha1.ToZone{ Type: v1alpha1.Only, - Zones: []string{"zone-3"}, + Zones: &[]string{"zone-3"}, }, }, { To: v1alpha1.ToZone{ Type: v1alpha1.Only, - Zones: []string{"zone-4"}, + Zones: &[]string{"zone-4"}, }, }, }, @@ -670,7 +467,7 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { { To: v1alpha1.ToZone{ Type: v1alpha1.Only, - Zones: []string{"zone-2"}, + Zones: &[]string{"zone-2"}, }, }, }, @@ -712,83 +509,14 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { Name: "backend", Origin: generator.OriginOutbound, Resource: endpoints.CreateClusterLoadAssignment("backend", []core_xds.Endpoint{ - { - Target: "192.168.1.1", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - - "k8s.io/node": "node1", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.2", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - "k8s.io/node": "node2", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.3", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - "k8s.io/az": "test", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.4", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - "k8s.io/region": "test", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.5", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-2", - }, - Locality: &core_xds.Locality{Zone: "zone-2", Priority: 0}, - }, - { - Target: "192.168.1.6", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-3", - }, - Locality: &core_xds.Locality{Zone: "zone-3", Priority: 0}, - }, - { - Target: "192.168.1.7", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-4", - }, - Locality: &core_xds.Locality{Zone: "zone-4", Priority: 0}, - }, - { - Target: "192.168.1.8", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-5", - }, - Locality: &core_xds.Locality{Zone: "zone-5", Priority: 0}, - }, + createEndpointWith("zone-1", "192.168.1.1", map[string]string{"k8s.io/node": "node1"}), + createEndpointWith("zone-1", "192.168.1.2", map[string]string{"k8s.io/node": "node2"}), + createEndpointWith("zone-1", "192.168.1.3", map[string]string{"k8s.io/az": "test"}), + createEndpointWith("zone-1", "192.168.1.4", map[string]string{"k8s.io/region": "test"}), + createEndpointWith("zone-2", "192.168.1.5", map[string]string{}), + createEndpointWith("zone-3", "192.168.1.6", map[string]string{}), + createEndpointWith("zone-4", "192.168.1.7", map[string]string{}), + createEndpointWith("zone-5", "192.168.1.8", map[string]string{}), }), }, { @@ -797,77 +525,24 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { Resource: clusters.NewClusterBuilder(envoy_common.APIV3, "payment"). Configure(clusters.ProvidedEndpointCluster( false, - core_xds.Endpoint{ - Target: "192.168.0.1", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - core_xds.Endpoint{ - Target: "192.168.0.2", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-2", - }, - Locality: &core_xds.Locality{Zone: "zone-2", Priority: 0}, - }, + createEndpointWith("zone-1", "192.168.0.1", map[string]string{}), + createEndpointWith("zone-2", "192.168.0.2", map[string]string{}), )).MustBuild(), }, { - Name: "backend", - Origin: generator.OriginOutbound, - Resource: NewOutboundListenerBuilder(envoy_common.APIV3, "127.0.0.1", 27777, core_xds.SocketAddressProtocolTCP). - Configure(FilterChain(NewFilterChainBuilder(envoy_common.APIV3, envoy_common.AnonymousResource). - Configure(HttpConnectionManager("127.0.0.1:27777", false)). - Configure( - HttpOutboundRoute( - "backend", - envoy_common.Routes{{ - Clusters: []envoy_common.Cluster{envoy_common.NewCluster( - envoy_common.WithService("backend"), - envoy_common.WithWeight(100), - )}, - }}, - map[string]map[string]bool{ - "kuma.io/service": { - "backend": true, - }, - }, - ), - ), - )).MustBuild(), + Name: "backend", + Origin: generator.OriginOutbound, + Resource: backendListener(), }, { - Name: "payments", - Origin: generator.OriginOutbound, - Resource: NewOutboundListenerBuilder(envoy_common.APIV3, "127.0.0.1", 27778, core_xds.SocketAddressProtocolTCP). - Configure(FilterChain(NewFilterChainBuilder(envoy_common.APIV3, envoy_common.AnonymousResource). - Configure(HttpConnectionManager("127.0.0.1:27778", false)). - Configure( - HttpOutboundRoute( - "backend", - envoy_common.Routes{{ - Clusters: []envoy_common.Cluster{envoy_common.NewCluster( - envoy_common.WithService("payment"), - envoy_common.WithWeight(100), - )}, - }}, - map[string]map[string]bool{ - "kuma.io/service": { - "payment": true, - }, - }, - ), - ), - )).MustBuild(), + Name: "payments", + Origin: generator.OriginOutbound, + Resource: paymentsListener(), }, }, proxy: &core_xds.Proxy{ APIVersion: envoy_common.APIV3, + Zone: "zone-1", Dataplane: builders.Dataplane(). AddInboundOfTagsMap(map[string]string{ mesh_proto.ServiceTag: "backend", @@ -984,83 +659,14 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { Name: "backend", Origin: generator.OriginOutbound, Resource: endpoints.CreateClusterLoadAssignment("backend", []core_xds.Endpoint{ - { - Target: "192.168.1.1", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - - "k8s.io/node": "node1", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.2", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - "k8s.io/node": "node2", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.3", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - "k8s.io/az": "test", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.4", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - "k8s.io/region": "test", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.5", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-2", - }, - Locality: &core_xds.Locality{Zone: "zone-2", Priority: 0}, - }, - { - Target: "192.168.1.6", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-3", - }, - Locality: &core_xds.Locality{Zone: "zone-3", Priority: 0}, - }, - { - Target: "192.168.1.7", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-4", - }, - Locality: &core_xds.Locality{Zone: "zone-4", Priority: 0}, - }, - { - Target: "192.168.1.8", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-5", - }, - Locality: &core_xds.Locality{Zone: "zone-5", Priority: 0}, - }, + createEndpointWith("zone-1", "192.168.1.1", map[string]string{"k8s.io/node": "node1"}), + createEndpointWith("zone-1", "192.168.1.2", map[string]string{"k8s.io/node": "node2"}), + createEndpointWith("zone-1", "192.168.1.3", map[string]string{"k8s.io/az": "test"}), + createEndpointWith("zone-1", "192.168.1.4", map[string]string{"k8s.io/region": "test"}), + createEndpointWith("zone-2", "192.168.1.5", map[string]string{}), + createEndpointWith("zone-3", "192.168.1.6", map[string]string{}), + createEndpointWith("zone-4", "192.168.1.7", map[string]string{}), + createEndpointWith("zone-5", "192.168.1.8", map[string]string{}), }), }, { @@ -1069,77 +675,24 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { Resource: clusters.NewClusterBuilder(envoy_common.APIV3, "payment"). Configure(clusters.ProvidedEndpointCluster( false, - core_xds.Endpoint{ - Target: "192.168.0.1", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - core_xds.Endpoint{ - Target: "192.168.0.2", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-2", - }, - Locality: &core_xds.Locality{Zone: "zone-2", Priority: 0}, - }, + createEndpointWith("zone-1", "192.168.0.1", map[string]string{}), + createEndpointWith("zone-2", "192.168.0.2", map[string]string{}), )).MustBuild(), }, { - Name: "backend", - Origin: generator.OriginOutbound, - Resource: NewOutboundListenerBuilder(envoy_common.APIV3, "127.0.0.1", 27777, core_xds.SocketAddressProtocolTCP). - Configure(FilterChain(NewFilterChainBuilder(envoy_common.APIV3, envoy_common.AnonymousResource). - Configure(HttpConnectionManager("127.0.0.1:27777", false)). - Configure( - HttpOutboundRoute( - "backend", - envoy_common.Routes{{ - Clusters: []envoy_common.Cluster{envoy_common.NewCluster( - envoy_common.WithService("backend"), - envoy_common.WithWeight(100), - )}, - }}, - map[string]map[string]bool{ - "kuma.io/service": { - "backend": true, - }, - }, - ), - ), - )).MustBuild(), + Name: "backend", + Origin: generator.OriginOutbound, + Resource: backendListener(), }, { - Name: "payments", - Origin: generator.OriginOutbound, - Resource: NewOutboundListenerBuilder(envoy_common.APIV3, "127.0.0.1", 27778, core_xds.SocketAddressProtocolTCP). - Configure(FilterChain(NewFilterChainBuilder(envoy_common.APIV3, envoy_common.AnonymousResource). - Configure(HttpConnectionManager("127.0.0.1:27778", false)). - Configure( - HttpOutboundRoute( - "backend", - envoy_common.Routes{{ - Clusters: []envoy_common.Cluster{envoy_common.NewCluster( - envoy_common.WithService("payment"), - envoy_common.WithWeight(100), - )}, - }}, - map[string]map[string]bool{ - "kuma.io/service": { - "payment": true, - }, - }, - ), - ), - )).MustBuild(), + Name: "payments", + Origin: generator.OriginOutbound, + Resource: paymentsListener(), }, }, proxy: &core_xds.Proxy{ APIVersion: envoy_common.APIV3, + Zone: "zone-1", Dataplane: builders.Dataplane(). AddInboundOfTagsMap(map[string]string{ mesh_proto.ServiceTag: "backend", @@ -1179,7 +732,7 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { { To: v1alpha1.ToZone{ Type: v1alpha1.AnyExcept, - Zones: []string{"zone-3", "zone-4", "zone-5"}, + Zones: &[]string{"zone-3", "zone-4", "zone-5"}, }, }, { @@ -1188,13 +741,13 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { }, To: v1alpha1.ToZone{ Type: v1alpha1.Only, - Zones: []string{"zone-3"}, + Zones: &[]string{"zone-3"}, }, }, { To: v1alpha1.ToZone{ Type: v1alpha1.Only, - Zones: []string{"zone-4"}, + Zones: &[]string{"zone-4"}, }, }, }, @@ -1272,137 +825,25 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { Name: "backend-bb38a94289f18fb9", Origin: generator.OriginOutbound, Resource: endpoints.CreateClusterLoadAssignment("backend-bb38a94289f18fb9", []core_xds.Endpoint{ - { - Target: "192.168.1.1", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - - "k8s.io/node": "node1", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.2", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - "k8s.io/node": "node2", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.3", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - "k8s.io/az": "test", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.4", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - "k8s.io/region": "test", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.5", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-2", - }, - Locality: &core_xds.Locality{Zone: "zone-2", Priority: 0}, - }, - { - Target: "192.168.1.6", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-3", - }, - Locality: &core_xds.Locality{Zone: "zone-3", Priority: 0}, - }, - { - Target: "192.168.1.7", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-4", - }, - Locality: &core_xds.Locality{Zone: "zone-4", Priority: 0}, - }, - { - Target: "192.168.1.8", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-5", - }, - Locality: &core_xds.Locality{Zone: "zone-5", Priority: 0}, - }, + createEndpointWith("zone-1", "192.168.1.1", map[string]string{"k8s.io/node": "node1"}), + createEndpointWith("zone-1", "192.168.1.2", map[string]string{"k8s.io/node": "node2"}), + createEndpointWith("zone-1", "192.168.1.3", map[string]string{"k8s.io/az": "test"}), + createEndpointWith("zone-1", "192.168.1.4", map[string]string{"k8s.io/region": "test"}), + createEndpointWith("zone-2", "192.168.1.5", map[string]string{}), + createEndpointWith("zone-3", "192.168.1.6", map[string]string{}), + createEndpointWith("zone-4", "192.168.1.7", map[string]string{}), + createEndpointWith("zone-5", "192.168.1.8", map[string]string{}), }), }, { Name: "backend-c72efb5be46fae6b", Origin: generator.OriginOutbound, Resource: endpoints.CreateClusterLoadAssignment("backend-c72efb5be46fae6b", []core_xds.Endpoint{ - { - Target: "192.168.1.1", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - - "k8s.io/node": "node1", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.2", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - "k8s.io/node": "node2", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - { - Target: "192.168.1.6", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-3", - }, - Locality: &core_xds.Locality{Zone: "zone-3", Priority: 0}, - }, - { - Target: "192.168.1.7", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-4", - }, - Locality: &core_xds.Locality{Zone: "zone-4", Priority: 0}, - }, - { - Target: "192.168.1.8", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-5", - }, - Locality: &core_xds.Locality{Zone: "zone-5", Priority: 0}, - }, + createEndpointWith("zone-1", "192.168.1.1", map[string]string{"k8s.io/node": "node1"}), + createEndpointWith("zone-1", "192.168.1.2", map[string]string{"k8s.io/node": "node2"}), + createEndpointWith("zone-3", "192.168.1.6", map[string]string{}), + createEndpointWith("zone-4", "192.168.1.7", map[string]string{}), + createEndpointWith("zone-5", "192.168.1.8", map[string]string{}), }), }, { @@ -1411,24 +852,8 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { Resource: clusters.NewClusterBuilder(envoy_common.APIV3, "payment"). Configure(clusters.ProvidedEndpointCluster( false, - core_xds.Endpoint{ - Target: "192.168.0.1", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-1", - }, - Locality: &core_xds.Locality{Zone: "zone-1", Priority: 0}, - }, - core_xds.Endpoint{ - Target: "192.168.0.2", - Port: 8080, - Tags: map[string]string{ - mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, - mesh_proto.ZoneTag: "zone-2", - }, - Locality: &core_xds.Locality{Zone: "zone-2", Priority: 0}, - }, + createEndpointWith("zone-1", "192.168.0.1", map[string]string{}), + createEndpointWith("zone-2", "192.168.0.2", map[string]string{}), )).MustBuild(), }, { @@ -1462,32 +887,14 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { )).MustBuild(), }, { - Name: "payments", - Origin: generator.OriginOutbound, - Resource: NewOutboundListenerBuilder(envoy_common.APIV3, "127.0.0.1", 27778, core_xds.SocketAddressProtocolTCP). - Configure(FilterChain(NewFilterChainBuilder(envoy_common.APIV3, envoy_common.AnonymousResource). - Configure(HttpConnectionManager("127.0.0.1:27778", false)). - Configure( - HttpOutboundRoute( - "backend", - envoy_common.Routes{{ - Clusters: []envoy_common.Cluster{envoy_common.NewCluster( - envoy_common.WithService("payment"), - envoy_common.WithWeight(100), - )}, - }}, - map[string]map[string]bool{ - "kuma.io/service": { - "payment": true, - }, - }, - ), - ), - )).MustBuild(), + Name: "payments", + Origin: generator.OriginOutbound, + Resource: paymentsListener(), }, }, proxy: &core_xds.Proxy{ APIVersion: envoy_common.APIV3, + Zone: "zone-1", Dataplane: builders.Dataplane(). AddInboundOfTagsMap(map[string]string{ mesh_proto.ServiceTag: "backend", @@ -1540,7 +947,7 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { { To: v1alpha1.ToZone{ Type: v1alpha1.AnyExcept, - Zones: []string{"zone-3", "zone-4", "zone-5"}, + Zones: &[]string{"zone-3", "zone-4", "zone-5"}, }, }, { @@ -1549,13 +956,13 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { }, To: v1alpha1.ToZone{ Type: v1alpha1.Only, - Zones: []string{"zone-3"}, + Zones: &[]string{"zone-3"}, }, }, { To: v1alpha1.ToZone{ Type: v1alpha1.Only, - Zones: []string{"zone-4"}, + Zones: &[]string{"zone-4"}, }, }, }, @@ -1706,3 +1113,67 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { }), ) }) + +func createEndpointWith(zone string, ip string, extraTags map[string]string) core_xds.Endpoint { + tags := map[string]string{ + mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP, + mesh_proto.ZoneTag: zone, + } + + for k, v := range extraTags { + tags[k] = v + } + + return core_xds.Endpoint{ + Target: ip, + Port: 8080, + Tags: tags, + Locality: &core_xds.Locality{Zone: zone, Priority: 0}, + } +} + +func paymentsListener() envoy_common.NamedResource { + return NewOutboundListenerBuilder(envoy_common.APIV3, "127.0.0.1", 27778, core_xds.SocketAddressProtocolTCP). + Configure(FilterChain(NewFilterChainBuilder(envoy_common.APIV3, envoy_common.AnonymousResource). + Configure(HttpConnectionManager("127.0.0.1:27778", false)). + Configure( + HttpOutboundRoute( + "backend", + envoy_common.Routes{{ + Clusters: []envoy_common.Cluster{envoy_common.NewCluster( + envoy_common.WithService("payment"), + envoy_common.WithWeight(100), + )}, + }}, + map[string]map[string]bool{ + "kuma.io/service": { + "payment": true, + }, + }, + ), + ), + )).MustBuild() +} + +func backendListener() envoy_common.NamedResource { + return NewOutboundListenerBuilder(envoy_common.APIV3, "127.0.0.1", 27777, core_xds.SocketAddressProtocolTCP). + Configure(FilterChain(NewFilterChainBuilder(envoy_common.APIV3, envoy_common.AnonymousResource). + Configure(HttpConnectionManager("127.0.0.1:27777", false)). + Configure( + HttpOutboundRoute( + "backend", + envoy_common.Routes{{ + Clusters: []envoy_common.Cluster{envoy_common.NewCluster( + envoy_common.WithService("backend"), + envoy_common.WithWeight(100), + )}, + }}, + map[string]map[string]bool{ + "kuma.io/service": { + "backend": true, + }, + }, + ), + ), + )).MustBuild() +} diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/priority.go b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/priority.go index cfddd081d6bc..8be733bbc386 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/priority.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/priority.go @@ -29,7 +29,7 @@ func GetLocalityGroups(conf *api.Conf, inboundTags mesh_proto.MultiValueTagSet, } func getLocalLbGroups(conf *api.Conf, inboundTags mesh_proto.MultiValueTagSet) []LocalLbGroup { - localGroups := []LocalLbGroup{} + var localGroups []LocalLbGroup if conf.LocalityAwareness.LocalZone != nil { rulesLen := len(conf.LocalityAwareness.LocalZone.AffinityTags) for i, tag := range conf.LocalityAwareness.LocalZone.AffinityTags { @@ -52,7 +52,7 @@ func getLocalLbGroups(conf *api.Conf, inboundTags mesh_proto.MultiValueTagSet) [ } func getCrossZoneLbGroups(conf *api.Conf, localZone string) []CrossZoneLbGroup { - crossZoneGroups := []CrossZoneLbGroup{} + var crossZoneGroups []CrossZoneLbGroup if conf.LocalityAwareness.CrossZone != nil && len(conf.LocalityAwareness.CrossZone.Failover) > 0 { // iterator starts from 0 while for remote zones we always set priority that favors local zone // we are using priority based on the rule position in the list so we increment it even if it doesn't match @@ -69,7 +69,7 @@ func getCrossZoneLbGroups(conf *api.Conf, localZone string) []CrossZoneLbGroup { case api.AnyExcept: lb.Type = api.AnyExcept zones := map[string]bool{} - for _, zone := range rule.To.Zones { + for _, zone := range *rule.To.Zones { zones[zone] = true } lb.Zones = zones @@ -77,7 +77,7 @@ func getCrossZoneLbGroups(conf *api.Conf, localZone string) []CrossZoneLbGroup { case api.Only: lb.Type = api.Only zones := map[string]bool{} - for _, zone := range rule.To.Zones { + for _, zone := range *rule.To.Zones { zones[zone] = true } lb.Zones = zones diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress.clusters.golden.yaml b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress.clusters.golden.yaml index b140c13b7ce7..a87156aed8fa 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress.clusters.golden.yaml +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress.clusters.golden.yaml @@ -30,6 +30,7 @@ resources: envoy.transport_socket_match: kuma.io/protocol: http kuma.io/zone: zone-1 + loadBalancingWeight: 1 locality: zone: zone-1 - lbEndpoints: @@ -49,5 +50,7 @@ resources: locality: zone: zone-2 priority: 1 + policy: + overprovisioningFactor: 200 name: mesh-2:static-cluster type: STATIC diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress.endpoints.golden.yaml b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress.endpoints.golden.yaml index eae6360c98f7..45b8f856ad1c 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress.endpoints.golden.yaml +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress.endpoints.golden.yaml @@ -1,5 +1,5 @@ resources: -- name: mesh-1:cla-for-eds-cluster +- name: mesh-1:eds-cluster resource: '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment clusterName: mesh-1:eds-cluster @@ -18,6 +18,7 @@ resources: envoy.transport_socket_match: kuma.io/protocol: http kuma.io/zone: zone-1 + loadBalancingWeight: 1 locality: zone: zone-1 - lbEndpoints: @@ -36,3 +37,23 @@ resources: kuma.io/zone: zone-2 locality: zone: zone-2 + priority: 1 + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.1.3 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + kuma.io/protocol: http + kuma.io/zone: zone-3 + envoy.transport_socket_match: + kuma.io/protocol: http + kuma.io/zone: zone-3 + locality: + zone: zone-3 + priority: 2 + policy: + overprovisioningFactor: 200 diff --git a/pkg/plugins/policies/meshtrafficpermission/plugin/v1alpha1/plugin.go b/pkg/plugins/policies/meshtrafficpermission/plugin/v1alpha1/plugin.go index 639bce483dc6..c33c7e3fd4cf 100644 --- a/pkg/plugins/policies/meshtrafficpermission/plugin/v1alpha1/plugin.go +++ b/pkg/plugins/policies/meshtrafficpermission/plugin/v1alpha1/plugin.go @@ -35,8 +35,8 @@ func (p plugin) MatchedPolicies(dataplane *core_mesh.DataplaneResource, resource return matchers.MatchedPolicies(api.MeshTrafficPermissionType, dataplane, resources) } -func (p plugin) EgressMatchedPolicies(es *core_mesh.ExternalServiceResource, resources xds_context.Resources) (core_xds.TypedMatchingPolicies, error) { - return matchers.EgressMatchedPolicies(api.MeshTrafficPermissionType, es, resources) +func (p plugin) EgressMatchedPolicies(tags map[string]string, resources xds_context.Resources) (core_xds.TypedMatchingPolicies, error) { + return matchers.EgressMatchedPolicies(api.MeshTrafficPermissionType, tags, resources) } func (p plugin) Apply(rs *core_xds.ResourceSet, ctx xds_context.Context, proxy *core_xds.Proxy) error { diff --git a/pkg/xds/sync/egress_proxy_builder.go b/pkg/xds/sync/egress_proxy_builder.go index 395ff3c73637..0d375dc4ebda 100644 --- a/pkg/xds/sync/egress_proxy_builder.go +++ b/pkg/xds/sync/egress_proxy_builder.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/pkg/core/faultinjections" "github.com/kumahq/kuma/pkg/core/permissions" core_plugins "github.com/kumahq/kuma/pkg/core/plugins" @@ -94,24 +95,21 @@ func (p *EgressProxyBuilder) Build( } for _, es := range externalServices { - policies := core_xds.PluginOriginatedPolicies{} - for name, plugin := range core_plugins.Plugins().PolicyPlugins() { - egressPlugin, ok := plugin.(core_plugins.EgressPolicyPlugin) - if !ok { - continue - } - res, err := egressPlugin.EgressMatchedPolicies(es, meshCtx.Resources) - if err != nil { - return nil, errors.Wrapf(err, "could not apply policy plugin %s", name) - } - if res.Type == "" { - return nil, errors.Errorf("matched policy didn't set type for policy plugin %s", name) - } - policies[res.Type] = res + policies, err := matchExternalPolicies(es.Spec.GetTags(), meshCtx.Resources) + if err != nil { + return nil, err } meshResources.Dynamic[es.Spec.GetService()] = policies } + for serviceName := range meshResources.EndpointMap { + policies, err := matchExternalPolicies(map[string]string{mesh_proto.ServiceTag: serviceName}, meshCtx.Resources) + if err != nil { + return nil, err + } + meshResources.Dynamic[serviceName] = policies + } + meshResourcesList = append(meshResourcesList, meshResources) } @@ -134,3 +132,23 @@ func (p *EgressProxyBuilder) Build( return proxy, nil } + +func matchExternalPolicies(tags map[string]string, resources xds_context.Resources) (core_xds.PluginOriginatedPolicies, error) { + policies := core_xds.PluginOriginatedPolicies{} + for name, plugin := range core_plugins.Plugins().PolicyPlugins() { + egressPlugin, ok := plugin.(core_plugins.EgressPolicyPlugin) + if !ok { + continue + } + res, err := egressPlugin.EgressMatchedPolicies(tags, resources) + if err != nil { + return nil, errors.Wrapf(err, "could not apply policy plugin %s", name) + } + if res.Type == "" { + return nil, errors.Errorf("matched policy didn't set type for policy plugin %s", name) + } + policies[res.Type] = res + } + + return policies, nil +} diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go new file mode 100644 index 000000000000..de6f0ab04418 --- /dev/null +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -0,0 +1,133 @@ +package localityawarelb + +import ( + "fmt" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/kumahq/kuma/pkg/test/resources/samples" + . "github.com/kumahq/kuma/test/framework" + "github.com/kumahq/kuma/test/framework/client" + "github.com/kumahq/kuma/test/framework/deployments/democlient" + "github.com/kumahq/kuma/test/framework/deployments/testserver" + "github.com/kumahq/kuma/test/framework/envs/multizone" +) + +func LocalityAwareLBEgress() { + const mesh = "locality-aware-lb-egress" + const namespace = "locality-aware-lb-egress" + + meshLoadBalancingStrategyDemoClient := fmt.Sprintf(` +type: MeshLoadBalancingStrategy +name: mlbs-1 +mesh: %s +spec: + targetRef: + kind: Mesh + to: + - targetRef: + kind: MeshService + name: test-server_locality-aware-lb-egress_svc_80 + default: + localityAwareness: + crossZone: + failover: + - from: + zones: ["kuma-4"] + to: + type: Only + zones: ["kuma-5"] + - from: + zones: ["kuma-4"] + to: + type: Only + zones: ["kuma-1-zone"] + - from: + zones: ["kuma-4"] + to: + type: Any`, mesh) + + BeforeAll(func() { + // Global + Expect(NewClusterSetup(). + Install(ResourceUniversal(samples.MeshMTLSBuilder().WithName(mesh).WithEgressRoutingEnabled().Build())). + Install(YamlUniversal(meshLoadBalancingStrategyDemoClient)). + Setup(multizone.Global)).To(Succeed()) + Expect(WaitForMesh(mesh, multizone.Zones())).To(Succeed()) + + // Universal Zone 4 + Expect(NewClusterSetup(). + Install(DemoClientUniversal( + "demo-client_locality-aware-lb_svc", + mesh, + WithTransparentProxy(true), + )). + Install(TestServerUniversal("test-server-zone-4", mesh, + WithServiceName("test-server_locality-aware-lb-egress_svc_80"), + WithArgs([]string{"echo", "--instance", "test-server-zone-4"}), + )). + Setup(multizone.UniZone1), + ).To(Succeed()) + + // Universal Zone 5 + Expect(NewClusterSetup(). + Install(DemoClientUniversal( + "demo-client_locality-aware-lb_svc", + mesh, + WithTransparentProxy(true), + )). + Install(TestServerUniversal("test-server-zone-5", mesh, + WithServiceName("test-server_locality-aware-lb-egress_svc_80"), + WithArgs([]string{"echo", "--instance", "test-server-zone-5"}), + )). + Setup(multizone.UniZone2), + ).To(Succeed()) + + // Kubernetes Zone 1 + Expect(NewClusterSetup(). + Install(NamespaceWithSidecarInjection(namespace)). + Install(democlient.Install(democlient.WithMesh(mesh), democlient.WithNamespace(namespace))). + Install(testserver.Install( + testserver.WithName("test-server"), + testserver.WithMesh(mesh), + testserver.WithNamespace(namespace), + testserver.WithEchoArgs("echo", "--instance", "test-server-zone-1"), + )). + Setup(multizone.KubeZone1)).ToNot(HaveOccurred()) + }) + + E2EAfterAll(func() { + Expect(multizone.UniZone1.DeleteMeshApps(mesh)).To(Succeed()) + Expect(multizone.UniZone2.DeleteMeshApps(mesh)).To(Succeed()) + Expect(multizone.KubeZone1.TriggerDeleteNamespace(namespace)).To(Succeed()) + Expect(multizone.Global.DeleteMesh(mesh)).To(Succeed()) + }) + + It("should route based on defined strategy with egress enabled", func() { + Eventually(func() (map[string]int, error) { + return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) + }, "30s", "5s").Should( + HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 100, 5)), + ) + + // when app with the highest weight is down + Expect(multizone.UniZone1.DeleteApp("test-server-zone-4")).To(Succeed()) + + // then traffic goes to the zone with the next priority, kuma-5 + Eventually(func() (map[string]int, error) { + return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) + }, "30s", "5s").Should( + HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 100, 5)), + ) + + // when zone kuma-5 is disabled + Expect(multizone.UniZone2.DeleteApp("test-server-zone-5")).To(Succeed()) + + // then traffic should go to k8s + Eventually(func() (map[string]int, error) { + return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) + }, "30s", "5s").Should( + HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 100, 5)), + ) + }) +} diff --git a/test/e2e_env/multizone/multizone_suite_test.go b/test/e2e_env/multizone/multizone_suite_test.go index 7258683aa9f5..232414e03ab8 100644 --- a/test/e2e_env/multizone/multizone_suite_test.go +++ b/test/e2e_env/multizone/multizone_suite_test.go @@ -70,4 +70,5 @@ var ( _ = Describe("MeshTimeout", meshtimeout.MeshTimeout, Ordered) _ = Describe("LocalityAwareness with MeshLoadBalancingStrategy", localityawarelb.LocalityAwarenessWithMeshLoadBalancingStrategy, Ordered) _ = Describe("Advanced LocalityAwareness with MeshLoadBalancingStrategy", localityawarelb.LocalityAwareLB, Ordered) + _ = Describe("Advanced LocalityAwareness with MeshLoadBalancingStrategy and Enabled Egress", localityawarelb.LocalityAwareLBEgress, Ordered) ) From 47008487d8cb6d0909911309f5855e4658df6fb2 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Thu, 2 Nov 2023 11:17:32 +0100 Subject: [PATCH 02/26] feat(kuma-cp): add locality aware implementation for egress Fixes: #8073 Signed-off-by: Marcin Skalski --- .../meshloadbalancingstrategy/plugin/v1alpha1/plugin.go | 6 +----- .../locality_aware_multizone_hybrid_egress.go | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin.go b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin.go index 1a12a570e3ee..07b5a7178e5e 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin.go @@ -6,7 +6,6 @@ import ( envoy_listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" envoy_route "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" envoy_hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" - "github.com/kumahq/kuma/pkg/core" "github.com/pkg/errors" "golang.org/x/exp/maps" @@ -34,8 +33,6 @@ func NewPlugin() core_plugins.Plugin { return &plugin{} } -var log = core.Log.WithName("MeshLoadBalance") - func (p plugin) MatchedPolicies(dataplane *core_mesh.DataplaneResource, resources xds_context.Resources) (core_xds.TypedMatchingPolicies, error) { return matchers.MatchedPolicies(api.MeshLoadBalancingStrategyType, dataplane, resources) } @@ -230,8 +227,7 @@ func (p plugin) configureEgress(rs *core_xds.ResourceSet, proxy *core_xds.Proxy) conf := rule.Conf.(api.Conf) clusterName := envoy_names.GetMeshClusterName(meshName, serviceName) - cluster, ok := clusters.Egress[clusterName] - err := configureEndpoints(mesh_proto.MultiValueTagSet{}, cluster, endpoints[clusterName], clusterName, conf, rs, proxy.Zone, proxy.APIVersion) + err := configureEndpoints(mesh_proto.MultiValueTagSet{}, clusters.Egress[clusterName], endpoints[clusterName], clusterName, conf, rs, proxy.Zone, proxy.APIVersion) if err != nil { return err } diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index de6f0ab04418..b572fd756f12 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -2,6 +2,7 @@ package localityawarelb import ( "fmt" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" From 4d829ae0d2d397d2d811893d2a39ed81cb7aa207 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Thu, 2 Nov 2023 14:57:04 +0100 Subject: [PATCH 03/26] feat(kuma-cp): add locality aware implementation for egress Fixes: #8073 Signed-off-by: Marcin Skalski --- .../locality_aware_multizone_hybrid_egress.go | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index b572fd756f12..86ca6fa6b841 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -2,16 +2,14 @@ package localityawarelb import ( "fmt" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/kumahq/kuma/pkg/test/resources/samples" . "github.com/kumahq/kuma/test/framework" "github.com/kumahq/kuma/test/framework/client" "github.com/kumahq/kuma/test/framework/deployments/democlient" "github.com/kumahq/kuma/test/framework/deployments/testserver" "github.com/kumahq/kuma/test/framework/envs/multizone" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) func LocalityAwareLBEgress() { @@ -52,7 +50,7 @@ spec: // Global Expect(NewClusterSetup(). Install(ResourceUniversal(samples.MeshMTLSBuilder().WithName(mesh).WithEgressRoutingEnabled().Build())). - Install(YamlUniversal(meshLoadBalancingStrategyDemoClient)). + // Install(YamlUniversal(meshLoadBalancingStrategyDemoClient)). Setup(multizone.Global)).To(Succeed()) Expect(WaitForMesh(mesh, multizone.Zones())).To(Succeed()) @@ -108,17 +106,30 @@ spec: Eventually(func() (map[string]int, error) { return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) }, "30s", "5s").Should( - HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 100, 5)), + And( + HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 66, 10)), + HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 17, 10)), + HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 17, 10)), + ), + ) + + // apply lb policy + Expect(multizone.Global.Install(YamlUniversal(meshLoadBalancingStrategyDemoClient))).To(Succeed()) + + Eventually(func() (map[string]int, error) { + return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) + }, "30s", "5s").Should( + HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 100, 10)), ) - // when app with the highest weight is down + // when app with the highest priority is down Expect(multizone.UniZone1.DeleteApp("test-server-zone-4")).To(Succeed()) // then traffic goes to the zone with the next priority, kuma-5 Eventually(func() (map[string]int, error) { return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) }, "30s", "5s").Should( - HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 100, 5)), + HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 100, 10)), ) // when zone kuma-5 is disabled @@ -128,7 +139,7 @@ spec: Eventually(func() (map[string]int, error) { return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) }, "30s", "5s").Should( - HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 100, 5)), + HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 100, 10)), ) }) } From 1565794bcbfc9ce6d9c2652d0dff13f6de13a80e Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Fri, 3 Nov 2023 11:48:40 +0100 Subject: [PATCH 04/26] feat(kuma-cp): add locality aware implementation for egress Fixes: #8237 Signed-off-by: Marcin Skalski --- pkg/core/plugins/interfaces.go | 2 +- .../plugin/v1alpha1/locality_aware.go | 26 +++++-- .../plugin/v1alpha1/plugin.go | 22 +++--- .../plugin/v1alpha1/priority.go | 4 +- .../locality_aware_multizone_hybrid_egress.go | 69 +++++++++++++++---- 5 files changed, 95 insertions(+), 28 deletions(-) diff --git a/pkg/core/plugins/interfaces.go b/pkg/core/plugins/interfaces.go index 0a93f33d26e7..5b2b0fc5cb32 100644 --- a/pkg/core/plugins/interfaces.go +++ b/pkg/core/plugins/interfaces.go @@ -102,7 +102,7 @@ type EgressPolicyPlugin interface { PolicyPlugin // EgressMatchedPolicies returns all the policies of the plugins' type matching the external service that // should be applied on the zone egress. - EgressMatchedPolicies(map[string]string, xds_context.Resources) (core_xds.TypedMatchingPolicies, error) + EgressMatchedPolicies(tags map[string]string, resources xds_context.Resources) (core_xds.TypedMatchingPolicies, error) } // ProxyPlugin a plugin to modify the proxy. This happens before any `PolicyPlugin` or any envoy generation. and it is applied both for Dataplanes and ZoneProxies diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go index 3a5ab0382076..68151c6fb002 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go @@ -2,7 +2,6 @@ package v1alpha1 import ( "fmt" - envoy_cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_endpoint "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" "google.golang.org/protobuf/proto" @@ -11,9 +10,10 @@ import ( mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" core_xds "github.com/kumahq/kuma/pkg/core/xds" api "github.com/kumahq/kuma/pkg/plugins/policies/meshloadbalancingstrategy/api/v1alpha1" + "github.com/kumahq/kuma/pkg/xds/cache/sha256" envoy_endpoints "github.com/kumahq/kuma/pkg/xds/envoy/endpoints" envoy_metadata "github.com/kumahq/kuma/pkg/xds/envoy/metadata/v3" - "github.com/kumahq/kuma/pkg/xds/generator" + "github.com/kumahq/kuma/pkg/xds/generator/egress" ) const defaultOverprovisingFactor uint32 = 200 @@ -26,11 +26,13 @@ func ConfigureStaticEndpointsLocalityAware( serviceName string, localZone string, apiVersion core_xds.APIVersion, + egressEnabled bool, + origin string, ) error { if conf.LocalityAwareness != nil { for _, cla := range endpoints { if cla.ClusterName == serviceName { - loadAssignment, err := ConfigureEndpointLocalityAwareLb(tags, &conf, cla, cla.ClusterName, localZone, apiVersion) + loadAssignment, err := ConfigureEndpointLocalityAwareLb(tags, &conf, cla, cla.ClusterName, localZone, apiVersion, egressEnabled, origin) if err != nil { return err } @@ -49,17 +51,19 @@ func ConfigureEndpointsLocalityAware( serviceName string, localZone string, apiVersion core_xds.APIVersion, + egressEnabled bool, + origin string, ) error { if conf.LocalityAwareness != nil { for _, cla := range endpoints { if cla.ClusterName == serviceName { - loadAssignment, err := ConfigureEndpointLocalityAwareLb(tags, &conf, cla, cla.ClusterName, localZone, apiVersion) + loadAssignment, err := ConfigureEndpointLocalityAwareLb(tags, &conf, cla, cla.ClusterName, localZone, apiVersion, egressEnabled, origin) if err != nil { return err } rs.Add(&core_xds.Resource{ Name: cla.ClusterName, - Origin: generator.OriginOutbound, // needs to be set so GatherEndpoints can get them + Origin: origin, // needs to be set so GatherEndpoints can get them Resource: loadAssignment, }) } @@ -75,6 +79,8 @@ func ConfigureEndpointLocalityAwareLb( serviceName string, localZone string, apiVersion core_xds.APIVersion, + egressEnabled bool, + origin string, ) (proto.Message, error) { localPriorityGroups, crossZonePriorityGroups := GetLocalityGroups(conf, tags, localZone) var endpointsList []core_xds.Endpoint @@ -87,6 +93,9 @@ func ConfigureEndpointLocalityAwareLb( configureLocalZoneEndpointLocality(localPriorityGroups, &ed, localZone) endpointsList = append(endpointsList, ed) } else if configureCrossZoneEndpointLocality(crossZonePriorityGroups, &ed, zoneName) { + if egressEnabled && origin != egress.OriginEgress { + ed.Locality = egressLocality(ed.Locality.Zone, ed.Locality.Priority) + } endpointsList = append(endpointsList, ed) } } @@ -179,3 +188,10 @@ func configureLocalZoneEndpointLocality(localPriorityGroups []LocalLbGroup, endp } } } + +func egressLocality(zone string, priority uint32) *core_xds.Locality { + return &core_xds.Locality{ + Zone: sha256.Hash(fmt.Sprintf("%s:%d", zone, priority)), + Priority: 1, + } +} diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin.go b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin.go index 07b5a7178e5e..895af61c0212 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin.go @@ -23,6 +23,8 @@ import ( xds_context "github.com/kumahq/kuma/pkg/xds/context" v3 "github.com/kumahq/kuma/pkg/xds/envoy/listeners/v3" envoy_names "github.com/kumahq/kuma/pkg/xds/envoy/names" + "github.com/kumahq/kuma/pkg/xds/generator" + "github.com/kumahq/kuma/pkg/xds/generator/egress" ) var _ core_plugins.EgressPolicyPlugin = &plugin{} @@ -56,11 +58,11 @@ func (p plugin) Apply(rs *core_xds.ResourceSet, ctx xds_context.Context, proxy * endpoints := policies_xds.GatherEndpoints(rs) routes := policies_xds.GatherRoutes(rs) - if err := p.configureGateway(proxy, policies.ToRules, listeners.Gateway, clusters.Gateway, routes.Gateway, endpoints, rs); err != nil { + if err := p.configureGateway(proxy, policies.ToRules, listeners.Gateway, clusters.Gateway, routes.Gateway, endpoints, rs, ctx.Mesh.Resource.ZoneEgressEnabled()); err != nil { return err } - return p.configureDPP(proxy, policies.ToRules, listeners, clusters, endpoints, rs) + return p.configureDPP(proxy, policies.ToRules, listeners, clusters, endpoints, rs, ctx.Mesh.Resource.ZoneEgressEnabled()) } func (p plugin) configureDPP( @@ -70,6 +72,7 @@ func (p plugin) configureDPP( clusters policies_xds.Clusters, endpoints policies_xds.EndpointMap, rs *core_xds.ResourceSet, + egressEnabled bool, ) error { serviceConfs := map[string]api.Conf{} @@ -100,7 +103,7 @@ func (p plugin) configureDPP( if err := p.configureCluster(cluster, conf); err != nil { return err } - if err := configureEndpoints(proxy.Dataplane.Spec.TagSet(), cluster, endpoints[serviceName], serviceName, conf, rs, proxy.Zone, proxy.APIVersion); err != nil { + if err := configureEndpoints(proxy.Dataplane.Spec.TagSet(), cluster, endpoints[serviceName], serviceName, conf, rs, proxy.Zone, proxy.APIVersion, egressEnabled, generator.OriginOutbound); err != nil { return errors.Wrapf(err, "failed to configure ClusterLoadAssignment for %s", serviceName) } } @@ -108,7 +111,7 @@ func (p plugin) configureDPP( if err := p.configureCluster(cluster, conf); err != nil { return err } - if err := configureEndpoints(proxy.Dataplane.Spec.TagSet(), cluster, endpoints[serviceName], cluster.Name, conf, rs, proxy.Zone, proxy.APIVersion); err != nil { + if err := configureEndpoints(proxy.Dataplane.Spec.TagSet(), cluster, endpoints[serviceName], cluster.Name, conf, rs, proxy.Zone, proxy.APIVersion, egressEnabled, generator.OriginOutbound); err != nil { return errors.Wrapf(err, "failed to configure ClusterLoadAssignment for %s", cluster.Name) } } @@ -126,13 +129,15 @@ func configureEndpoints( rs *core_xds.ResourceSet, localZone string, apiVersion core_xds.APIVersion, + egressEnabled bool, + origin string, ) error { if cluster.LoadAssignment != nil { - if err := ConfigureStaticEndpointsLocalityAware(tags, endpoints, cluster, conf, serviceName, localZone, apiVersion); err != nil { + if err := ConfigureStaticEndpointsLocalityAware(tags, endpoints, cluster, conf, serviceName, localZone, apiVersion, egressEnabled, origin); err != nil { return err } } else { - if err := ConfigureEndpointsLocalityAware(tags, endpoints, conf, rs, serviceName, localZone, apiVersion); err != nil { + if err := ConfigureEndpointsLocalityAware(tags, endpoints, conf, rs, serviceName, localZone, apiVersion, egressEnabled, origin); err != nil { return err } } @@ -157,6 +162,7 @@ func (p plugin) configureGateway( gatewayRoutes map[string]*envoy_route.RouteConfiguration, endpoints policies_xds.EndpointMap, rs *core_xds.ResourceSet, + egressEnabled bool, ) error { gatewayListenerInfos := gateway_plugin.ExtractGatewayListeners(proxy) if len(gatewayListenerInfos) == 0 { @@ -198,7 +204,7 @@ func (p plugin) configureGateway( } serviceName := dest.Destination[mesh_proto.ServiceTag] - if err := configureEndpoints(proxy.Dataplane.Spec.TagSet(), cluster, endpoints[serviceName], serviceName, *conf, rs, proxy.Zone, proxy.APIVersion); err != nil { + if err := configureEndpoints(proxy.Dataplane.Spec.TagSet(), cluster, endpoints[serviceName], serviceName, *conf, rs, proxy.Zone, proxy.APIVersion, egressEnabled, generator.OriginOutbound); err != nil { return err } } @@ -227,7 +233,7 @@ func (p plugin) configureEgress(rs *core_xds.ResourceSet, proxy *core_xds.Proxy) conf := rule.Conf.(api.Conf) clusterName := envoy_names.GetMeshClusterName(meshName, serviceName) - err := configureEndpoints(mesh_proto.MultiValueTagSet{}, clusters.Egress[clusterName], endpoints[clusterName], clusterName, conf, rs, proxy.Zone, proxy.APIVersion) + err := configureEndpoints(mesh_proto.MultiValueTagSet{}, clusters.Egress[clusterName], endpoints[clusterName], clusterName, conf, rs, proxy.Zone, proxy.APIVersion, true, egress.OriginEgress) if err != nil { return err } diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/priority.go b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/priority.go index 8be733bbc386..f02ab64baa43 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/priority.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/priority.go @@ -69,7 +69,7 @@ func getCrossZoneLbGroups(conf *api.Conf, localZone string) []CrossZoneLbGroup { case api.AnyExcept: lb.Type = api.AnyExcept zones := map[string]bool{} - for _, zone := range *rule.To.Zones { + for _, zone := range pointer.Deref(rule.To.Zones) { zones[zone] = true } lb.Zones = zones @@ -77,7 +77,7 @@ func getCrossZoneLbGroups(conf *api.Conf, localZone string) []CrossZoneLbGroup { case api.Only: lb.Type = api.Only zones := map[string]bool{} - for _, zone := range *rule.To.Zones { + for _, zone := range pointer.Deref(rule.To.Zones) { zones[zone] = true } lb.Zones = zones diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index 86ca6fa6b841..c688b24e2955 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -2,14 +2,16 @@ package localityawarelb import ( "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/kumahq/kuma/pkg/test/resources/samples" . "github.com/kumahq/kuma/test/framework" "github.com/kumahq/kuma/test/framework/client" "github.com/kumahq/kuma/test/framework/deployments/democlient" "github.com/kumahq/kuma/test/framework/deployments/testserver" "github.com/kumahq/kuma/test/framework/envs/multizone" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" ) func LocalityAwareLBEgress() { @@ -20,6 +22,36 @@ func LocalityAwareLBEgress() { type: MeshLoadBalancingStrategy name: mlbs-1 mesh: %s +spec: + targetRef: + kind: Mesh + to: + - targetRef: + kind: MeshService + name: test-server_locality-aware-lb-egress_svc_80 + default: + localityAwareness: + crossZone: + failover: + - from: + zones: ["kuma-4"] + to: + type: Only + zones: ["kuma-1-zone"] + - from: + zones: ["kuma-4"] + to: + type: Only + zones: ["kuma-5"] + - from: + zones: ["kuma-4"] + to: + type: Any`, mesh) + + meshLoadBalancingStrategyDemoClientChangedPriority := fmt.Sprintf(` +type: MeshLoadBalancingStrategy +name: mlbs-1 +mesh: %s spec: targetRef: kind: Mesh @@ -57,7 +89,7 @@ spec: // Universal Zone 4 Expect(NewClusterSetup(). Install(DemoClientUniversal( - "demo-client_locality-aware-lb_svc", + "demo-client_locality-aware-lb-egress_svc", mesh, WithTransparentProxy(true), )). @@ -71,7 +103,7 @@ spec: // Universal Zone 5 Expect(NewClusterSetup(). Install(DemoClientUniversal( - "demo-client_locality-aware-lb_svc", + "demo-client_locality-aware-lb-egress_svc", mesh, WithTransparentProxy(true), )). @@ -103,8 +135,9 @@ spec: }) It("should route based on defined strategy with egress enabled", func() { + // no lb priorities Eventually(func() (map[string]int, error) { - return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) + return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) }, "30s", "5s").Should( And( HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 66, 10)), @@ -114,30 +147,42 @@ spec: ) // apply lb policy + // kuma-4 - priority 0, kuma-1 - priority 1, kuma-5 - priority 2, Expect(multizone.Global.Install(YamlUniversal(meshLoadBalancingStrategyDemoClient))).To(Succeed()) Eventually(func() (map[string]int, error) { - return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) + return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) }, "30s", "5s").Should( HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 100, 10)), ) - // when app with the highest priority is down + // kill test-server in kuma-4 zone Expect(multizone.UniZone1.DeleteApp("test-server-zone-4")).To(Succeed()) - // then traffic goes to the zone with the next priority, kuma-5 + // traffic goes to kuma-1 zone + Eventually(func() (map[string]int, error) { + return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) + }, "30s", "5s").Should( + HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 100, 10)), + ) + + // apply lb policy with new priorities + // kuma-4 - priority 0, kuma-5 - priority 1, kuma-1 - priority 2, + Expect(multizone.Global.Install(YamlUniversal(meshLoadBalancingStrategyDemoClientChangedPriority))).To(Succeed()) + + // traffic goes to kuma-5 zone Eventually(func() (map[string]int, error) { - return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) + return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) }, "30s", "5s").Should( HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 100, 10)), ) - // when zone kuma-5 is disabled + // kill test-server from kuma-5 zone Expect(multizone.UniZone2.DeleteApp("test-server-zone-5")).To(Succeed()) - // then traffic should go to k8s + // then traffic should go to kuma-1 zone Eventually(func() (map[string]int, error) { - return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) + return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) }, "30s", "5s").Should( HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 100, 10)), ) From e51570843202e88eb4038c6420d71b37da679db3 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Fri, 3 Nov 2023 11:54:09 +0100 Subject: [PATCH 05/26] feat(kuma-cp): fix check Fixes: #8237 Signed-off-by: Marcin Skalski --- .../meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go index 68151c6fb002..e3aea045b654 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go @@ -2,6 +2,7 @@ package v1alpha1 import ( "fmt" + envoy_cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_endpoint "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" "google.golang.org/protobuf/proto" From c3a114ce302684972a1f94e86df8a8bae8445f6b Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Fri, 3 Nov 2023 11:58:43 +0100 Subject: [PATCH 06/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../localityawarelb/locality_aware_multizone_hybrid_egress.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index c688b24e2955..3ef92852533f 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -82,7 +82,6 @@ spec: // Global Expect(NewClusterSetup(). Install(ResourceUniversal(samples.MeshMTLSBuilder().WithName(mesh).WithEgressRoutingEnabled().Build())). - // Install(YamlUniversal(meshLoadBalancingStrategyDemoClient)). Setup(multizone.Global)).To(Succeed()) Expect(WaitForMesh(mesh, multizone.Zones())).To(Succeed()) From bcbaeb63f7c2fd2416699b0a6b29dd53cac15391 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Fri, 3 Nov 2023 13:36:45 +0100 Subject: [PATCH 07/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../plugin/v1alpha1/locality_aware.go | 34 +- .../plugin/v1alpha1/plugin_test.go | 296 +++++++++++++++++- .../egress_basic.clusters.golden.yaml | 53 ++++ .../egress_basic.endpoints.golden.yaml | 56 ++++ .../egress_basic.listeners.golden.yaml | 1 + ..._basic_egress_enabled.clusters.golden.yaml | 86 +++++ ...basic_egress_enabled.endpoints.golden.yaml | 151 +++++++++ ...basic_egress_enabled.listeners.golden.yaml | 79 +++++ 8 files changed, 746 insertions(+), 10 deletions(-) create mode 100644 pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress_basic.clusters.golden.yaml create mode 100644 pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress_basic.endpoints.golden.yaml create mode 100644 pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress_basic.listeners.golden.yaml create mode 100644 pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.clusters.golden.yaml create mode 100644 pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.endpoints.golden.yaml create mode 100644 pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.listeners.golden.yaml diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go index e3aea045b654..d653e7757f29 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go @@ -2,9 +2,11 @@ package v1alpha1 import ( "fmt" + "strings" envoy_cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_endpoint "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + "golang.org/x/exp/maps" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/wrapperspb" @@ -30,7 +32,7 @@ func ConfigureStaticEndpointsLocalityAware( egressEnabled bool, origin string, ) error { - if conf.LocalityAwareness != nil { + if conf.LocalityAwareness != nil && (conf.LocalityAwareness.LocalZone != nil || conf.LocalityAwareness.CrossZone != nil) { for _, cla := range endpoints { if cla.ClusterName == serviceName { loadAssignment, err := ConfigureEndpointLocalityAwareLb(tags, &conf, cla, cla.ClusterName, localZone, apiVersion, egressEnabled, origin) @@ -55,7 +57,7 @@ func ConfigureEndpointsLocalityAware( egressEnabled bool, origin string, ) error { - if conf.LocalityAwareness != nil { + if conf.LocalityAwareness != nil && (conf.LocalityAwareness.LocalZone != nil || conf.LocalityAwareness.CrossZone != nil) { for _, cla := range endpoints { if cla.ClusterName == serviceName { loadAssignment, err := ConfigureEndpointLocalityAwareLb(tags, &conf, cla, cla.ClusterName, localZone, apiVersion, egressEnabled, origin) @@ -90,13 +92,14 @@ func ConfigureEndpointLocalityAwareLb( ed := createEndpoint(lbEndpoint, localZone) zoneName := ed.Tags[mesh_proto.ZoneTag] + // nolint:gocritic if zoneName == localZone { configureLocalZoneEndpointLocality(localPriorityGroups, &ed, localZone) endpointsList = append(endpointsList, ed) + } else if egressEnabled && origin != egress.OriginEgress { + ed.Locality = egressLocality(crossZonePriorityGroups) + endpointsList = append(endpointsList, ed) } else if configureCrossZoneEndpointLocality(crossZonePriorityGroups, &ed, zoneName) { - if egressEnabled && origin != egress.OriginEgress { - ed.Locality = egressLocality(ed.Locality.Zone, ed.Locality.Priority) - } endpointsList = append(endpointsList, ed) } } @@ -190,9 +193,26 @@ func configureLocalZoneEndpointLocality(localPriorityGroups []LocalLbGroup, endp } } -func egressLocality(zone string, priority uint32) *core_xds.Locality { +func egressLocality(crossZoneGroups []CrossZoneLbGroup) *core_xds.Locality { + builder := strings.Builder{} + for _, group := range crossZoneGroups { + switch group.Type { + case api.Only: + zones := strings.Join(maps.Keys(group.Zones), ",") + builder.WriteString(fmt.Sprintf("%d:%s", group.Priority, zones)) + case api.Any: + builder.WriteString(fmt.Sprintf("%d:%s", group.Priority, group.Type)) + case api.AnyExcept: + zones := strings.Join(maps.Keys(group.Zones), ",") + builder.WriteString(fmt.Sprintf("%d:%s:%s", group.Priority, group.Type, zones)) + default: + continue + } + builder.WriteString(";") + } + return &core_xds.Locality{ - Zone: sha256.Hash(fmt.Sprintf("%s:%d", zone, priority)), + Zone: fmt.Sprintf("egress:%s", sha256.Hash(builder.String())), Priority: 1, } } diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin_test.go b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin_test.go index de49f30ee600..6e676c03768e 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin_test.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin_test.go @@ -47,6 +47,7 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { type testCase struct { resources []core_xds.Resource proxy *core_xds.Proxy + context xds_context.Context } DescribeTable("Apply to sidecar Dataplanes", func(given testCase) { @@ -56,10 +57,8 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { resources.Add(&r) } - context := xds_context.Context{} - plugin := plugin.NewPlugin().(core_plugins.PolicyPlugin) - Expect(plugin.Apply(resources, context, given.proxy)).To(Succeed()) + Expect(plugin.Apply(resources, given.context, given.proxy)).To(Succeed()) nameSplit := strings.Split(GinkgoT().Name(), " ") name := nameSplit[len(nameSplit)-1] @@ -292,6 +291,80 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { }, }, }), + Entry("egress_basic", testCase{ + resources: []core_xds.Resource{ + { + Name: "mesh-1:eds-cluster", + Origin: egress.OriginEgress, + Resource: clusters.NewClusterBuilder(envoy_common.APIV3, "mesh-1:eds-cluster"). + Configure(clusters.EdsCluster()). + MustBuild(), + }, + { + Name: "mesh-1:eds-cluster", + Origin: egress.OriginEgress, + Resource: endpoints.CreateClusterLoadAssignment("mesh-1:eds-cluster", []core_xds.Endpoint{ + createEndpointWith("zone-1", "192.168.1.1", map[string]string{}), + createEndpointWith("zone-2", "192.168.1.2", map[string]string{}), + createEndpointWith("zone-3", "192.168.1.3", map[string]string{}), + }), + }, + { + Name: "mesh-2:static-cluster", + Origin: egress.OriginEgress, + Resource: clusters.NewClusterBuilder(envoy_common.APIV3, "mesh-2:static-cluster"). + Configure(clusters.ProvidedEndpointCluster( + false, + createEndpointWith("zone-1", "192.168.0.1", map[string]string{}), + createEndpointWith("zone-2", "192.168.0.2", map[string]string{}), + )).MustBuild(), + }, + }, + proxy: &core_xds.Proxy{ + APIVersion: envoy_common.APIV3, + Zone: "zone-1", + ZoneEgressProxy: &core_xds.ZoneEgressProxy{ + MeshResourcesList: []*core_xds.MeshResources{ + { + Mesh: builders.Mesh().WithName("mesh-1").Build(), + Dynamic: core_xds.ExternalServiceDynamicPolicies{ + "eds-cluster": { + v1alpha1.MeshLoadBalancingStrategyType: core_xds.TypedMatchingPolicies{ + FromRules: core_rules.FromRules{ + Rules: map[core_rules.InboundListener]core_rules.Rules{ + {}: { + {Conf: v1alpha1.Conf{LocalityAwareness: &v1alpha1.LocalityAwareness{ + Disabled: pointer.To(false), + }}}, + }, + }, + }, + }, + }, + }, + }, + { + Mesh: builders.Mesh().WithName("mesh-2").Build(), + Dynamic: core_xds.ExternalServiceDynamicPolicies{ + "static-cluster": { + v1alpha1.MeshLoadBalancingStrategyType: core_xds.TypedMatchingPolicies{ + FromRules: core_rules.FromRules{ + Rules: map[core_rules.InboundListener]core_rules.Rules{ + {}: { + {Conf: v1alpha1.Conf{LocalityAwareness: &v1alpha1.LocalityAwareness{ + Disabled: pointer.To(false), + }}}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }), Entry("locality_aware_basic", testCase{ resources: []core_xds.Resource{ { @@ -496,6 +569,211 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { }, }, }), + Entry("locality_aware_basic_egress_enabled", testCase{ + resources: []core_xds.Resource{ + { + Name: "backend", + Origin: generator.OriginOutbound, + Resource: clusters.NewClusterBuilder(envoy_common.APIV3, "backend"). + Configure(clusters.EdsCluster()). + MustBuild(), + }, + { + Name: "backend", + Origin: generator.OriginOutbound, + Resource: endpoints.CreateClusterLoadAssignment("backend", []core_xds.Endpoint{ + createEndpointWith("zone-1", "192.168.1.1", map[string]string{"k8s.io/node": "node1"}), + createEndpointWith("zone-1", "192.168.1.2", map[string]string{"k8s.io/node": "node2"}), + createEndpointWith("zone-1", "192.168.1.3", map[string]string{"k8s.io/az": "test"}), + createEndpointWith("zone-1", "192.168.1.4", map[string]string{"k8s.io/region": "test"}), + createEndpointWith("zone-2", "192.168.1.5", map[string]string{}), + createEndpointWith("zone-3", "192.168.1.5", map[string]string{}), + createEndpointWith("zone-4", "192.168.1.5", map[string]string{}), + createEndpointWith("zone-5", "192.168.1.5", map[string]string{}), + }), + }, + { + Name: "payment", + Origin: generator.OriginOutbound, + Resource: clusters.NewClusterBuilder(envoy_common.APIV3, "payment"). + Configure(clusters.ProvidedEndpointCluster( + false, + createEndpointWith("zone-1", "192.168.0.1", map[string]string{"k8s.io/node": "node1"}), + createEndpointWith("zone-1", "192.168.0.2", map[string]string{"k8s.io/node": "node2"}), + createEndpointWith("zone-2", "192.168.0.5", map[string]string{}), + )).MustBuild(), + }, + { + Name: "backend", + Origin: generator.OriginOutbound, + Resource: backendListener(), + }, + { + Name: "payments", + Origin: generator.OriginOutbound, + Resource: paymentsListener(), + }, + }, + proxy: &core_xds.Proxy{ + APIVersion: envoy_common.APIV3, + Zone: "zone-1", + Dataplane: builders.Dataplane(). + AddInboundOfTagsMap(map[string]string{ + mesh_proto.ServiceTag: "backend", + mesh_proto.ZoneTag: "zone-1", + "k8s.io/node": "node1", + "k8s.io/az": "test", + "k8s.io/region": "test", + }). + AddOutbound( + builders.Outbound().WithAddress("127.0.0.1").WithPort(27777).WithTags(map[string]string{ + mesh_proto.ServiceTag: "backend", + mesh_proto.ProtocolTag: "http", + }), + ). + AddOutbound( + builders.Outbound().WithAddress("127.0.0.1").WithPort(27778).WithTags(map[string]string{ + mesh_proto.ServiceTag: "payment", + mesh_proto.ProtocolTag: "http", + }), + ). + Build(), + Policies: core_xds.MatchedPolicies{ + Dynamic: map[core_model.ResourceType]core_xds.TypedMatchingPolicies{ + v1alpha1.MeshLoadBalancingStrategyType: { + Type: v1alpha1.MeshLoadBalancingStrategyType, + ToRules: core_rules.ToRules{ + Rules: []*core_rules.Rule{ + { + Subset: core_rules.MeshService("backend"), + Conf: v1alpha1.Conf{ + LoadBalancer: &v1alpha1.LoadBalancer{ + Type: v1alpha1.RandomType, + }, + LocalityAwareness: &v1alpha1.LocalityAwareness{ + LocalZone: &v1alpha1.LocalZone{ + AffinityTags: []v1alpha1.AffinityTag{ + { + Key: "k8s.io/node", + Weight: pointer.To[uint32](9000), + }, + { + Key: "k8s.io/az", + Weight: pointer.To[uint32](900), + }, + { + Key: "k8s.io/region", + Weight: pointer.To[uint32](90), + }, + }, + }, + CrossZone: &v1alpha1.CrossZone{ + Failover: []v1alpha1.Failover{ + { + To: v1alpha1.ToZone{ + Type: v1alpha1.AnyExcept, + Zones: &[]string{"zone-3", "zone-4", "zone-5"}, + }, + }, + { + From: &v1alpha1.FromZone{ + Zones: []string{"zone-1"}, + }, + To: v1alpha1.ToZone{ + Type: v1alpha1.Only, + Zones: &[]string{"zone-3"}, + }, + }, + { + To: v1alpha1.ToZone{ + Type: v1alpha1.Only, + Zones: &[]string{"zone-4"}, + }, + }, + }, + }, + }, + }, + }, + { + Subset: core_rules.MeshService("payment"), + Conf: v1alpha1.Conf{ + LoadBalancer: &v1alpha1.LoadBalancer{ + Type: v1alpha1.RingHashType, + RingHash: &v1alpha1.RingHash{ + MinRingSize: pointer.To[uint32](100), + MaxRingSize: pointer.To[uint32](1000), + HashFunction: pointer.To(v1alpha1.MurmurHash2Type), + HashPolicies: &[]v1alpha1.HashPolicy{ + { + Type: v1alpha1.QueryParameterType, + QueryParameter: &v1alpha1.QueryParameter{ + Name: "queryparam", + }, + Terminal: pointer.To(true), + }, + { + Type: v1alpha1.ConnectionType, + Connection: &v1alpha1.Connection{ + SourceIP: pointer.To(true), + }, + Terminal: pointer.To(false), + }, + }, + }, + }, + LocalityAwareness: &v1alpha1.LocalityAwareness{ + LocalZone: &v1alpha1.LocalZone{ + AffinityTags: []v1alpha1.AffinityTag{ + { + Key: "k8s.io/node", + Weight: pointer.To[uint32](9000), + }, + { + Key: "k8s.io/az", + Weight: pointer.To[uint32](900), + }, + { + Key: "k8s.io/region", + Weight: pointer.To[uint32](90), + }, + }, + }, + CrossZone: &v1alpha1.CrossZone{ + Failover: []v1alpha1.Failover{ + { + To: v1alpha1.ToZone{ + Type: v1alpha1.Only, + Zones: &[]string{"zone-2"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + Routing: core_xds.Routing{ + OutboundTargets: map[core_xds.ServiceName][]core_xds.Endpoint{ + "backend": { + { + Tags: map[string]string{mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP}, + }, + }, + "payment": { + { + Tags: map[string]string{mesh_proto.ProtocolTag: core_mesh.ProtocolHTTP}, + }, + }, + }, + }, + }, + context: contextWithEgressEnabled(), + }), Entry("locality_aware_no_cross_zone", testCase{ resources: []core_xds.Resource{ { @@ -1177,3 +1455,15 @@ func backendListener() envoy_common.NamedResource { ), )).MustBuild() } + +func contextWithEgressEnabled() xds_context.Context { + return xds_context.Context{ + Mesh: xds_context.MeshContext{ + Resource: &core_mesh.MeshResource{ + Spec: &mesh_proto.Mesh{ + Routing: &mesh_proto.Routing{ZoneEgress: true}, + }, + }, + }, + } +} diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress_basic.clusters.golden.yaml b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress_basic.clusters.golden.yaml new file mode 100644 index 000000000000..b140c13b7ce7 --- /dev/null +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress_basic.clusters.golden.yaml @@ -0,0 +1,53 @@ +resources: +- name: mesh-1:eds-cluster + resource: + '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + altStatName: mesh-1_eds-cluster + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + name: mesh-1:eds-cluster + type: EDS +- name: mesh-2:static-cluster + resource: + '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + altStatName: mesh-2_static-cluster + loadAssignment: + clusterName: mesh-2:static-cluster + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.0.1 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + kuma.io/protocol: http + kuma.io/zone: zone-1 + envoy.transport_socket_match: + kuma.io/protocol: http + kuma.io/zone: zone-1 + locality: + zone: zone-1 + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.0.2 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + kuma.io/protocol: http + kuma.io/zone: zone-2 + envoy.transport_socket_match: + kuma.io/protocol: http + kuma.io/zone: zone-2 + locality: + zone: zone-2 + priority: 1 + name: mesh-2:static-cluster + type: STATIC diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress_basic.endpoints.golden.yaml b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress_basic.endpoints.golden.yaml new file mode 100644 index 000000000000..00b98395f6b0 --- /dev/null +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress_basic.endpoints.golden.yaml @@ -0,0 +1,56 @@ +resources: +- name: mesh-1:eds-cluster + resource: + '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment + clusterName: mesh-1:eds-cluster + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.1.1 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + kuma.io/protocol: http + kuma.io/zone: zone-1 + envoy.transport_socket_match: + kuma.io/protocol: http + kuma.io/zone: zone-1 + locality: + zone: zone-1 + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.1.2 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + kuma.io/protocol: http + kuma.io/zone: zone-2 + envoy.transport_socket_match: + kuma.io/protocol: http + kuma.io/zone: zone-2 + locality: + zone: zone-2 + priority: 1 + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.1.3 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + kuma.io/protocol: http + kuma.io/zone: zone-3 + envoy.transport_socket_match: + kuma.io/protocol: http + kuma.io/zone: zone-3 + locality: + zone: zone-3 + priority: 1 diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress_basic.listeners.golden.yaml b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress_basic.listeners.golden.yaml new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/egress_basic.listeners.golden.yaml @@ -0,0 +1 @@ +{} diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.clusters.golden.yaml b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.clusters.golden.yaml new file mode 100644 index 000000000000..cc6ceab978e4 --- /dev/null +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.clusters.golden.yaml @@ -0,0 +1,86 @@ +resources: +- name: backend + resource: + '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + commonLbConfig: + localityWeightedLbConfig: {} + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + lbPolicy: RANDOM + name: backend + type: EDS +- name: payment + resource: + '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + commonLbConfig: + localityWeightedLbConfig: {} + lbPolicy: RING_HASH + loadAssignment: + clusterName: payment + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.0.5 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + kuma.io/protocol: http + kuma.io/zone: zone-2 + envoy.transport_socket_match: + kuma.io/protocol: http + kuma.io/zone: zone-2 + locality: + zone: egress:JXegjD3U4x07ItT7LMc9tUM8zYPBfHgLOLcRJ73nzms= + priority: 1 + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.0.2 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + k8s.io/node: node2 + kuma.io/protocol: http + kuma.io/zone: zone-1 + envoy.transport_socket_match: + k8s.io/node: node2 + kuma.io/protocol: http + kuma.io/zone: zone-1 + loadBalancingWeight: 1 + locality: + zone: zone-1 + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.0.1 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + k8s.io/node: node1 + kuma.io/protocol: http + kuma.io/zone: zone-1 + envoy.transport_socket_match: + k8s.io/node: node1 + kuma.io/protocol: http + kuma.io/zone: zone-1 + loadBalancingWeight: 9000 + locality: + subZone: k8s.io/node=node1 + zone: zone-1 + policy: + overprovisioningFactor: 200 + name: payment + ringHashLbConfig: + hashFunction: MURMUR_HASH_2 + maximumRingSize: "1000" + minimumRingSize: "100" + type: STATIC diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.endpoints.golden.yaml b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.endpoints.golden.yaml new file mode 100644 index 000000000000..a9d83d0411ff --- /dev/null +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.endpoints.golden.yaml @@ -0,0 +1,151 @@ +resources: +- name: backend + resource: + '@type': type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment + clusterName: backend + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.1.5 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + kuma.io/protocol: http + kuma.io/zone: zone-2 + envoy.transport_socket_match: + kuma.io/protocol: http + kuma.io/zone: zone-2 + - endpoint: + address: + socketAddress: + address: 192.168.1.5 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + kuma.io/protocol: http + kuma.io/zone: zone-5 + envoy.transport_socket_match: + kuma.io/protocol: http + kuma.io/zone: zone-5 + locality: + zone: egress:+9ESnM2KXca5ii/1B4pp45do5Pqcju2lq/sjhaGwO2s= + priority: 1 + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.1.5 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + kuma.io/protocol: http + kuma.io/zone: zone-4 + envoy.transport_socket_match: + kuma.io/protocol: http + kuma.io/zone: zone-4 + locality: + zone: egress:P6QVbabe5FiNPDS8SPIE5m1v0pCRFs/Sk14W77rnZL4= + priority: 1 + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.1.5 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + kuma.io/protocol: http + kuma.io/zone: zone-3 + envoy.transport_socket_match: + kuma.io/protocol: http + kuma.io/zone: zone-3 + locality: + zone: egress:do0zBmmQuEx+oRYngyL1HNggzOvbEPvP/GC1d+rb8FQ= + priority: 1 + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.1.2 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + k8s.io/node: node2 + kuma.io/protocol: http + kuma.io/zone: zone-1 + envoy.transport_socket_match: + k8s.io/node: node2 + kuma.io/protocol: http + kuma.io/zone: zone-1 + loadBalancingWeight: 1 + locality: + zone: zone-1 + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.1.3 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + k8s.io/az: test + kuma.io/protocol: http + kuma.io/zone: zone-1 + envoy.transport_socket_match: + k8s.io/az: test + kuma.io/protocol: http + kuma.io/zone: zone-1 + loadBalancingWeight: 900 + locality: + subZone: k8s.io/az=test + zone: zone-1 + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.1.1 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + k8s.io/node: node1 + kuma.io/protocol: http + kuma.io/zone: zone-1 + envoy.transport_socket_match: + k8s.io/node: node1 + kuma.io/protocol: http + kuma.io/zone: zone-1 + loadBalancingWeight: 9000 + locality: + subZone: k8s.io/node=node1 + zone: zone-1 + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 192.168.1.4 + portValue: 8080 + metadata: + filterMetadata: + envoy.lb: + k8s.io/region: test + kuma.io/protocol: http + kuma.io/zone: zone-1 + envoy.transport_socket_match: + k8s.io/region: test + kuma.io/protocol: http + kuma.io/zone: zone-1 + loadBalancingWeight: 90 + locality: + subZone: k8s.io/region=test + zone: zone-1 + policy: + overprovisioningFactor: 200 diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.listeners.golden.yaml b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.listeners.golden.yaml new file mode 100644 index 000000000000..13c63bc279df --- /dev/null +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.listeners.golden.yaml @@ -0,0 +1,79 @@ +resources: +- name: backend + resource: + '@type': type.googleapis.com/envoy.config.listener.v3.Listener + address: + socketAddress: + address: 127.0.0.1 + portValue: 27777 + filterChains: + - filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + routeConfig: + name: outbound:backend + requestHeadersToAdd: + - header: + key: x-kuma-tags + value: '&kuma.io/service=backend&' + validateClusters: false + virtualHosts: + - domains: + - '*' + name: backend + routes: + - match: + prefix: / + route: + cluster: backend + timeout: 0s + statPrefix: "127_0_0_1_27777" + name: outbound:127.0.0.1:27777 + trafficDirection: OUTBOUND +- name: payments + resource: + '@type': type.googleapis.com/envoy.config.listener.v3.Listener + address: + socketAddress: + address: 127.0.0.1 + portValue: 27778 + filterChains: + - filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + routeConfig: + name: outbound:backend + requestHeadersToAdd: + - header: + key: x-kuma-tags + value: '&kuma.io/service=payment&' + validateClusters: false + virtualHosts: + - domains: + - '*' + name: backend + routes: + - match: + prefix: / + route: + cluster: payment + hashPolicy: + - queryParameter: + name: queryparam + terminal: true + - connectionProperties: + sourceIp: true + timeout: 0s + statPrefix: "127_0_0_1_27778" + name: outbound:127.0.0.1:27778 + trafficDirection: OUTBOUND From a267787239c55e03cba142a9a4e75afcd771f6d1 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Fri, 3 Nov 2023 15:19:20 +0100 Subject: [PATCH 08/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../plugin/v1alpha1/locality_aware.go | 14 ++++-- .../plugin/v1alpha1/plugin_test.go | 3 -- ..._basic_egress_enabled.clusters.golden.yaml | 2 +- ...basic_egress_enabled.endpoints.golden.yaml | 49 +------------------ 4 files changed, 11 insertions(+), 57 deletions(-) diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go index d653e7757f29..f0b2f27463df 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go @@ -2,6 +2,7 @@ package v1alpha1 import ( "fmt" + "sort" "strings" envoy_cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" @@ -198,13 +199,11 @@ func egressLocality(crossZoneGroups []CrossZoneLbGroup) *core_xds.Locality { for _, group := range crossZoneGroups { switch group.Type { case api.Only: - zones := strings.Join(maps.Keys(group.Zones), ",") - builder.WriteString(fmt.Sprintf("%d:%s", group.Priority, zones)) + builder.WriteString(fmt.Sprintf("%d:%s", group.Priority, strings.Join(sortedZones(group.Zones), ","))) case api.Any: builder.WriteString(fmt.Sprintf("%d:%s", group.Priority, group.Type)) case api.AnyExcept: - zones := strings.Join(maps.Keys(group.Zones), ",") - builder.WriteString(fmt.Sprintf("%d:%s:%s", group.Priority, group.Type, zones)) + builder.WriteString(fmt.Sprintf("%d:%s:%s", group.Priority, group.Type, strings.Join(sortedZones(group.Zones), ","))) default: continue } @@ -212,7 +211,12 @@ func egressLocality(crossZoneGroups []CrossZoneLbGroup) *core_xds.Locality { } return &core_xds.Locality{ - Zone: fmt.Sprintf("egress:%s", sha256.Hash(builder.String())), + Zone: fmt.Sprintf("egress_%s", sha256.Hash(builder.String())[:8]), Priority: 1, } } +func sortedZones(zones map[string]bool) []string { + keys := maps.Keys(zones) + sort.Strings(keys) + return keys +} diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin_test.go b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin_test.go index 6e676c03768e..90823379b687 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin_test.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/plugin_test.go @@ -587,9 +587,6 @@ var _ = Describe("MeshLoadBalancingStrategy", func() { createEndpointWith("zone-1", "192.168.1.3", map[string]string{"k8s.io/az": "test"}), createEndpointWith("zone-1", "192.168.1.4", map[string]string{"k8s.io/region": "test"}), createEndpointWith("zone-2", "192.168.1.5", map[string]string{}), - createEndpointWith("zone-3", "192.168.1.5", map[string]string{}), - createEndpointWith("zone-4", "192.168.1.5", map[string]string{}), - createEndpointWith("zone-5", "192.168.1.5", map[string]string{}), }), }, { diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.clusters.golden.yaml b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.clusters.golden.yaml index cc6ceab978e4..f8dfabcdd89a 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.clusters.golden.yaml +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.clusters.golden.yaml @@ -35,7 +35,7 @@ resources: kuma.io/protocol: http kuma.io/zone: zone-2 locality: - zone: egress:JXegjD3U4x07ItT7LMc9tUM8zYPBfHgLOLcRJ73nzms= + zone: egress_JXegjD3U priority: 1 - lbEndpoints: - endpoint: diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.endpoints.golden.yaml b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.endpoints.golden.yaml index a9d83d0411ff..0564faae9df9 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.endpoints.golden.yaml +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/testdata/locality_aware_basic_egress_enabled.endpoints.golden.yaml @@ -18,55 +18,8 @@ resources: envoy.transport_socket_match: kuma.io/protocol: http kuma.io/zone: zone-2 - - endpoint: - address: - socketAddress: - address: 192.168.1.5 - portValue: 8080 - metadata: - filterMetadata: - envoy.lb: - kuma.io/protocol: http - kuma.io/zone: zone-5 - envoy.transport_socket_match: - kuma.io/protocol: http - kuma.io/zone: zone-5 - locality: - zone: egress:+9ESnM2KXca5ii/1B4pp45do5Pqcju2lq/sjhaGwO2s= - priority: 1 - - lbEndpoints: - - endpoint: - address: - socketAddress: - address: 192.168.1.5 - portValue: 8080 - metadata: - filterMetadata: - envoy.lb: - kuma.io/protocol: http - kuma.io/zone: zone-4 - envoy.transport_socket_match: - kuma.io/protocol: http - kuma.io/zone: zone-4 - locality: - zone: egress:P6QVbabe5FiNPDS8SPIE5m1v0pCRFs/Sk14W77rnZL4= - priority: 1 - - lbEndpoints: - - endpoint: - address: - socketAddress: - address: 192.168.1.5 - portValue: 8080 - metadata: - filterMetadata: - envoy.lb: - kuma.io/protocol: http - kuma.io/zone: zone-3 - envoy.transport_socket_match: - kuma.io/protocol: http - kuma.io/zone: zone-3 locality: - zone: egress:do0zBmmQuEx+oRYngyL1HNggzOvbEPvP/GC1d+rb8FQ= + zone: egress_+9ESnM2K priority: 1 - lbEndpoints: - endpoint: From b51539b93351369d53b31f74e4be5df72fd23e93 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Fri, 3 Nov 2023 17:10:49 +0100 Subject: [PATCH 09/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go index f0b2f27463df..1125d38a52f8 100644 --- a/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go +++ b/pkg/plugins/policies/meshloadbalancingstrategy/plugin/v1alpha1/locality_aware.go @@ -215,6 +215,7 @@ func egressLocality(crossZoneGroups []CrossZoneLbGroup) *core_xds.Locality { Priority: 1, } } + func sortedZones(zones map[string]bool) []string { keys := maps.Keys(zones) sort.Strings(keys) From cb34eeb009465d0d940827f915a788a269177e6d Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Fri, 3 Nov 2023 17:52:04 +0100 Subject: [PATCH 10/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../locality_aware_multizone_hybrid_egress.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index 3ef92852533f..049d0d350804 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -137,11 +137,11 @@ spec: // no lb priorities Eventually(func() (map[string]int, error) { return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) - }, "30s", "5s").Should( + }, "1m", "10s").Should( And( - HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 66, 10)), - HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 17, 10)), - HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 17, 10)), + HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 66, 15)), + HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 17, 15)), + HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 17, 15)), ), ) @@ -151,7 +151,7 @@ spec: Eventually(func() (map[string]int, error) { return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) - }, "30s", "5s").Should( + }, "1m", "10s").Should( HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 100, 10)), ) @@ -161,7 +161,7 @@ spec: // traffic goes to kuma-1 zone Eventually(func() (map[string]int, error) { return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) - }, "30s", "5s").Should( + }, "1m", "10s").Should( HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 100, 10)), ) @@ -172,7 +172,7 @@ spec: // traffic goes to kuma-5 zone Eventually(func() (map[string]int, error) { return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) - }, "30s", "5s").Should( + }, "1m", "10s").Should( HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 100, 10)), ) @@ -182,7 +182,7 @@ spec: // then traffic should go to kuma-1 zone Eventually(func() (map[string]int, error) { return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) - }, "30s", "5s").Should( + }, "1m", "10s").Should( HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 100, 10)), ) }) From d8813b931ea34d06b7484a9c44a2c36cbe280124 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Fri, 3 Nov 2023 18:27:27 +0100 Subject: [PATCH 11/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../locality_aware_multizone_hybrid_egress.go | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index 049d0d350804..18488b51c64d 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -136,12 +136,12 @@ spec: It("should route based on defined strategy with egress enabled", func() { // no lb priorities Eventually(func() (map[string]int, error) { - return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) + return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(200)) }, "1m", "10s").Should( And( - HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 66, 15)), - HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 17, 15)), - HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 17, 15)), + HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 132, 15)), + HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 34, 15)), + HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 34, 15)), ), ) @@ -150,9 +150,9 @@ spec: Expect(multizone.Global.Install(YamlUniversal(meshLoadBalancingStrategyDemoClient))).To(Succeed()) Eventually(func() (map[string]int, error) { - return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) + return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(50)) }, "1m", "10s").Should( - HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 100, 10)), + HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 50, 10)), ) // kill test-server in kuma-4 zone @@ -160,9 +160,9 @@ spec: // traffic goes to kuma-1 zone Eventually(func() (map[string]int, error) { - return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) + return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(50)) }, "1m", "10s").Should( - HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 100, 10)), + HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 50, 10)), ) // apply lb policy with new priorities @@ -171,9 +171,9 @@ spec: // traffic goes to kuma-5 zone Eventually(func() (map[string]int, error) { - return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) + return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(50)) }, "1m", "10s").Should( - HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 100, 10)), + HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 50, 10)), ) // kill test-server from kuma-5 zone @@ -181,9 +181,9 @@ spec: // then traffic should go to kuma-1 zone Eventually(func() (map[string]int, error) { - return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(100)) + return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(50)) }, "1m", "10s").Should( - HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 100, 10)), + HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 50, 10)), ) }) } From dbdfc8f1ba9f7aced5340dc49e98f4cc1c32884e Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Sat, 4 Nov 2023 08:10:45 +0100 Subject: [PATCH 12/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../locality_aware_multizone_hybrid_egress.go | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index 18488b51c64d..c7207d32ef66 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -85,20 +85,6 @@ spec: Setup(multizone.Global)).To(Succeed()) Expect(WaitForMesh(mesh, multizone.Zones())).To(Succeed()) - // Universal Zone 4 - Expect(NewClusterSetup(). - Install(DemoClientUniversal( - "demo-client_locality-aware-lb-egress_svc", - mesh, - WithTransparentProxy(true), - )). - Install(TestServerUniversal("test-server-zone-4", mesh, - WithServiceName("test-server_locality-aware-lb-egress_svc_80"), - WithArgs([]string{"echo", "--instance", "test-server-zone-4"}), - )). - Setup(multizone.UniZone1), - ).To(Succeed()) - // Universal Zone 5 Expect(NewClusterSetup(). Install(DemoClientUniversal( @@ -124,6 +110,20 @@ spec: testserver.WithEchoArgs("echo", "--instance", "test-server-zone-1"), )). Setup(multizone.KubeZone1)).ToNot(HaveOccurred()) + + // Universal Zone 4 + Expect(NewClusterSetup(). + Install(DemoClientUniversal( + "demo-client_locality-aware-lb-egress_svc", + mesh, + WithTransparentProxy(true), + )). + Install(TestServerUniversal("test-server-zone-4", mesh, + WithServiceName("test-server_locality-aware-lb-egress_svc_80"), + WithArgs([]string{"echo", "--instance", "test-server-zone-4"}), + )). + Setup(multizone.UniZone1), + ).To(Succeed()) }) E2EAfterAll(func() { From b50c090cc2e7cf128bda9a0dee02e9ebb018574a Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Sat, 4 Nov 2023 08:45:36 +0100 Subject: [PATCH 13/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../locality_aware_multizone_hybrid_egress.go | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index c7207d32ef66..771d62f56757 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -2,6 +2,7 @@ package localityawarelb import ( "fmt" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -85,6 +86,18 @@ spec: Setup(multizone.Global)).To(Succeed()) Expect(WaitForMesh(mesh, multizone.Zones())).To(Succeed()) + // Kubernetes Zone 1 + Expect(NewClusterSetup(). + Install(NamespaceWithSidecarInjection(namespace)). + Install(democlient.Install(democlient.WithMesh(mesh), democlient.WithNamespace(namespace))). + Install(testserver.Install( + testserver.WithName("test-server"), + testserver.WithMesh(mesh), + testserver.WithNamespace(namespace), + testserver.WithEchoArgs("echo", "--instance", "test-server-zone-1"), + )). + Setup(multizone.KubeZone1)).ToNot(HaveOccurred()) + // Universal Zone 5 Expect(NewClusterSetup(). Install(DemoClientUniversal( @@ -99,18 +112,7 @@ spec: Setup(multizone.UniZone2), ).To(Succeed()) - // Kubernetes Zone 1 - Expect(NewClusterSetup(). - Install(NamespaceWithSidecarInjection(namespace)). - Install(democlient.Install(democlient.WithMesh(mesh), democlient.WithNamespace(namespace))). - Install(testserver.Install( - testserver.WithName("test-server"), - testserver.WithMesh(mesh), - testserver.WithNamespace(namespace), - testserver.WithEchoArgs("echo", "--instance", "test-server-zone-1"), - )). - Setup(multizone.KubeZone1)).ToNot(HaveOccurred()) - + time.Sleep(30 * time.Second) // Universal Zone 4 Expect(NewClusterSetup(). Install(DemoClientUniversal( From 77a722cdd7f54fb405b7b5a2b7ca0b2a1a7c2381 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Mon, 6 Nov 2023 10:38:48 +0100 Subject: [PATCH 14/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../locality_aware_multizone_hybrid_egress.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index 771d62f56757..58ab498df473 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -2,8 +2,6 @@ package localityawarelb import ( "fmt" - "time" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -112,7 +110,6 @@ spec: Setup(multizone.UniZone2), ).To(Succeed()) - time.Sleep(30 * time.Second) // Universal Zone 4 Expect(NewClusterSetup(). Install(DemoClientUniversal( @@ -139,7 +136,7 @@ spec: // no lb priorities Eventually(func() (map[string]int, error) { return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(200)) - }, "1m", "10s").Should( + }, "6m", "10s").Should( And( HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 132, 15)), HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 34, 15)), From ba4392cd1702b3aca101281149063cc25037c74e Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Mon, 6 Nov 2023 11:21:26 +0100 Subject: [PATCH 15/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../locality_aware_multizone_hybrid_egress.go | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index 58ab498df473..fde22923e346 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -2,14 +2,14 @@ package localityawarelb import ( "fmt" + "github.com/kumahq/kuma/test/framework/deployments/democlient" + "github.com/kumahq/kuma/test/framework/deployments/testserver" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/kumahq/kuma/pkg/test/resources/samples" . "github.com/kumahq/kuma/test/framework" "github.com/kumahq/kuma/test/framework/client" - "github.com/kumahq/kuma/test/framework/deployments/democlient" - "github.com/kumahq/kuma/test/framework/deployments/testserver" "github.com/kumahq/kuma/test/framework/envs/multizone" ) @@ -84,18 +84,6 @@ spec: Setup(multizone.Global)).To(Succeed()) Expect(WaitForMesh(mesh, multizone.Zones())).To(Succeed()) - // Kubernetes Zone 1 - Expect(NewClusterSetup(). - Install(NamespaceWithSidecarInjection(namespace)). - Install(democlient.Install(democlient.WithMesh(mesh), democlient.WithNamespace(namespace))). - Install(testserver.Install( - testserver.WithName("test-server"), - testserver.WithMesh(mesh), - testserver.WithNamespace(namespace), - testserver.WithEchoArgs("echo", "--instance", "test-server-zone-1"), - )). - Setup(multizone.KubeZone1)).ToNot(HaveOccurred()) - // Universal Zone 5 Expect(NewClusterSetup(). Install(DemoClientUniversal( @@ -123,6 +111,19 @@ spec: )). Setup(multizone.UniZone1), ).To(Succeed()) + + // Kubernetes Zone 1 + Expect(NewClusterSetup(). + Install(NamespaceWithSidecarInjection(namespace)). + Install(democlient.Install(democlient.WithMesh(mesh), democlient.WithNamespace(namespace))). + Install(testserver.Install( + testserver.WithName("test-server"), + testserver.WithMesh(mesh), + testserver.WithNamespace(namespace), + testserver.WithEchoArgs("echo", "--instance", "test-server-zone-1"), + )). + Setup(multizone.KubeZone1)).ToNot(HaveOccurred()) + }) E2EAfterAll(func() { @@ -168,6 +169,10 @@ spec: // kuma-4 - priority 0, kuma-5 - priority 1, kuma-1 - priority 2, Expect(multizone.Global.Install(YamlUniversal(meshLoadBalancingStrategyDemoClientChangedPriority))).To(Succeed()) + // TODO: should be removed after fixing egress imperfections + // egress config refresh workaround + Expect(multizone.UniZone1.GetApp("demo-client_locality-aware-lb-egress_svc").ReStart()).To(Succeed()) + // traffic goes to kuma-5 zone Eventually(func() (map[string]int, error) { return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(50)) From 72f3915682e3c0016e0f78c476b94b80abfb0928 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Mon, 6 Nov 2023 12:03:39 +0100 Subject: [PATCH 16/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../localityawarelb/locality_aware_multizone_hybrid_egress.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index fde22923e346..0c20459f1bf6 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -6,6 +6,7 @@ import ( "github.com/kumahq/kuma/test/framework/deployments/testserver" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "time" "github.com/kumahq/kuma/pkg/test/resources/samples" . "github.com/kumahq/kuma/test/framework" @@ -112,6 +113,7 @@ spec: Setup(multizone.UniZone1), ).To(Succeed()) + time.Sleep(30 * time.Second) // Kubernetes Zone 1 Expect(NewClusterSetup(). Install(NamespaceWithSidecarInjection(namespace)). From a57269134ef21e8a07e45f000cc668cbb7a92ff1 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Mon, 6 Nov 2023 12:45:16 +0100 Subject: [PATCH 17/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../locality_aware_multizone_hybrid_egress.go | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index 0c20459f1bf6..44219ad99578 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -2,16 +2,15 @@ package localityawarelb import ( "fmt" + "github.com/kumahq/kuma/pkg/test/resources/samples" + . "github.com/kumahq/kuma/test/framework" + "github.com/kumahq/kuma/test/framework/client" "github.com/kumahq/kuma/test/framework/deployments/democlient" "github.com/kumahq/kuma/test/framework/deployments/testserver" + "github.com/kumahq/kuma/test/framework/envs/multizone" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "time" - - "github.com/kumahq/kuma/pkg/test/resources/samples" - . "github.com/kumahq/kuma/test/framework" - "github.com/kumahq/kuma/test/framework/client" - "github.com/kumahq/kuma/test/framework/envs/multizone" ) func LocalityAwareLBEgress() { @@ -85,6 +84,18 @@ spec: Setup(multizone.Global)).To(Succeed()) Expect(WaitForMesh(mesh, multizone.Zones())).To(Succeed()) + // Kubernetes Zone 1 + Expect(NewClusterSetup(). + Install(NamespaceWithSidecarInjection(namespace)). + Install(democlient.Install(democlient.WithMesh(mesh), democlient.WithNamespace(namespace))). + Install(testserver.Install( + testserver.WithName("test-server"), + testserver.WithMesh(mesh), + testserver.WithNamespace(namespace), + testserver.WithEchoArgs("echo", "--instance", "test-server-zone-1"), + )). + Setup(multizone.KubeZone1)).ToNot(HaveOccurred()) + // Universal Zone 5 Expect(NewClusterSetup(). Install(DemoClientUniversal( @@ -100,6 +111,7 @@ spec: ).To(Succeed()) // Universal Zone 4 + time.Sleep(30 * time.Second) Expect(NewClusterSetup(). Install(DemoClientUniversal( "demo-client_locality-aware-lb-egress_svc", @@ -112,20 +124,6 @@ spec: )). Setup(multizone.UniZone1), ).To(Succeed()) - - time.Sleep(30 * time.Second) - // Kubernetes Zone 1 - Expect(NewClusterSetup(). - Install(NamespaceWithSidecarInjection(namespace)). - Install(democlient.Install(democlient.WithMesh(mesh), democlient.WithNamespace(namespace))). - Install(testserver.Install( - testserver.WithName("test-server"), - testserver.WithMesh(mesh), - testserver.WithNamespace(namespace), - testserver.WithEchoArgs("echo", "--instance", "test-server-zone-1"), - )). - Setup(multizone.KubeZone1)).ToNot(HaveOccurred()) - }) E2EAfterAll(func() { From 69ba4032ac4bcc9483c96d805036081a070ca587 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Mon, 6 Nov 2023 16:12:19 +0100 Subject: [PATCH 18/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- pkg/xds/sync/egress_proxy_builder.go | 6 +-- .../locality_aware_multizone_hybrid_egress.go | 24 +++++++++-- .../envoy_admin/clusters/clusters.go | 41 +++++++++++++++++++ test/framework/envoy_admin/interface.go | 2 + test/framework/envoy_admin/tunnel/k8s.go | 26 ++++++++++++ .../framework/envoy_admin/tunnel/universal.go | 26 ++++++++++++ 6 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 test/framework/envoy_admin/clusters/clusters.go diff --git a/pkg/xds/sync/egress_proxy_builder.go b/pkg/xds/sync/egress_proxy_builder.go index 0d375dc4ebda..1e7adee54e53 100644 --- a/pkg/xds/sync/egress_proxy_builder.go +++ b/pkg/xds/sync/egress_proxy_builder.go @@ -95,7 +95,7 @@ func (p *EgressProxyBuilder) Build( } for _, es := range externalServices { - policies, err := matchExternalPolicies(es.Spec.GetTags(), meshCtx.Resources) + policies, err := matchEgressPolicies(es.Spec.GetTags(), meshCtx.Resources) if err != nil { return nil, err } @@ -103,7 +103,7 @@ func (p *EgressProxyBuilder) Build( } for serviceName := range meshResources.EndpointMap { - policies, err := matchExternalPolicies(map[string]string{mesh_proto.ServiceTag: serviceName}, meshCtx.Resources) + policies, err := matchEgressPolicies(map[string]string{mesh_proto.ServiceTag: serviceName}, meshCtx.Resources) if err != nil { return nil, err } @@ -133,7 +133,7 @@ func (p *EgressProxyBuilder) Build( return proxy, nil } -func matchExternalPolicies(tags map[string]string, resources xds_context.Resources) (core_xds.PluginOriginatedPolicies, error) { +func matchEgressPolicies(tags map[string]string, resources xds_context.Resources) (core_xds.PluginOriginatedPolicies, error) { policies := core_xds.PluginOriginatedPolicies{} for name, plugin := range core_plugins.Plugins().PolicyPlugins() { egressPlugin, ok := plugin.(core_plugins.EgressPolicyPlugin) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index 44219ad99578..80fe4ebdcd31 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -7,15 +7,16 @@ import ( "github.com/kumahq/kuma/test/framework/client" "github.com/kumahq/kuma/test/framework/deployments/democlient" "github.com/kumahq/kuma/test/framework/deployments/testserver" + "github.com/kumahq/kuma/test/framework/envoy_admin" "github.com/kumahq/kuma/test/framework/envs/multizone" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "time" ) func LocalityAwareLBEgress() { const mesh = "locality-aware-lb-egress" const namespace = "locality-aware-lb-egress" + var egressTunnel envoy_admin.Tunnel meshLoadBalancingStrategyDemoClient := fmt.Sprintf(` type: MeshLoadBalancingStrategy @@ -110,8 +111,18 @@ spec: Setup(multizone.UniZone2), ).To(Succeed()) + // TODO: should be removed after fixing egress issues + // Workaround to wait for all remote zones to set up + egressTunnel = multizone.UniZone1.GetZoneEgressEnvoyTunnel() + Eventually(func() int { + egressClusters, err := egressTunnel.GetClusters() + Expect(err).ToNot(HaveOccurred()) + cluster := egressClusters.GetCluster(fmt.Sprintf("%s:test-server_locality-aware-lb-egress_svc_80", mesh)) + Expect(cluster).ToNot(BeNil()) + return len(cluster.HostStatuses) + }, "1m", "5s").Should(Equal(2)) + // Universal Zone 4 - time.Sleep(30 * time.Second) Expect(NewClusterSetup(). Install(DemoClientUniversal( "demo-client_locality-aware-lb-egress_svc", @@ -169,8 +180,15 @@ spec: // kuma-4 - priority 0, kuma-5 - priority 1, kuma-1 - priority 2, Expect(multizone.Global.Install(YamlUniversal(meshLoadBalancingStrategyDemoClientChangedPriority))).To(Succeed()) - // TODO: should be removed after fixing egress imperfections + // TODO: should be removed after fixing egress issues // egress config refresh workaround + Eventually(func() int { + egressClusters, err := egressTunnel.GetClusters() + Expect(err).ToNot(HaveOccurred()) + cluster := egressClusters.GetCluster(fmt.Sprintf("%s:test-server_locality-aware-lb-egress_svc_80", mesh)) + Expect(cluster).ToNot(BeNil()) + return cluster.GetPriorityForZone("kuma-5") + }, "1m", "5s").Should(Equal(1)) Expect(multizone.UniZone1.GetApp("demo-client_locality-aware-lb-egress_svc").ReStart()).To(Succeed()) // traffic goes to kuma-5 zone diff --git a/test/framework/envoy_admin/clusters/clusters.go b/test/framework/envoy_admin/clusters/clusters.go new file mode 100644 index 000000000000..54905acb9cd6 --- /dev/null +++ b/test/framework/envoy_admin/clusters/clusters.go @@ -0,0 +1,41 @@ +package clusters + +type ClusterItem struct { + Name string `json:"name"` + HostStatuses []HostStatus `json:"host_statuses"` +} + +type HostStatus struct { + Priority *int `json:"priority,omitempty"` + Locality *Locality `json:"locality,omitempty"` +} + +type Locality struct { + Zone string `json:"zone"` +} + +type Clusters struct { + Clusters []ClusterItem `json:"cluster_statuses"` +} + +func (c *Clusters) GetCluster(clusterName string) *ClusterItem { + for _, cluster := range c.Clusters { + if cluster.Name == clusterName { + return &cluster + } + } + return nil +} + +func (ci *ClusterItem) GetPriorityForZone(zone string) int { + for _, hs := range ci.HostStatuses { + if hs.Locality != nil && hs.Locality.Zone == zone { + if hs.Priority == nil { + return 0 + } else { + return *hs.Priority + } + } + } + return 0 +} diff --git a/test/framework/envoy_admin/interface.go b/test/framework/envoy_admin/interface.go index 07c59748dfc6..3d13771b7002 100644 --- a/test/framework/envoy_admin/interface.go +++ b/test/framework/envoy_admin/interface.go @@ -1,10 +1,12 @@ package envoy_admin import ( + "github.com/kumahq/kuma/test/framework/envoy_admin/clusters" "github.com/kumahq/kuma/test/framework/envoy_admin/stats" ) type Tunnel interface { GetStats(name string) (*stats.Stats, error) + GetClusters() (*clusters.Clusters, error) ResetCounters() error } diff --git a/test/framework/envoy_admin/tunnel/k8s.go b/test/framework/envoy_admin/tunnel/k8s.go index 64a490178ab8..602a64231855 100644 --- a/test/framework/envoy_admin/tunnel/k8s.go +++ b/test/framework/envoy_admin/tunnel/k8s.go @@ -3,6 +3,7 @@ package tunnel import ( "encoding/json" "fmt" + "github.com/kumahq/kuma/test/framework/envoy_admin/clusters" "net/http" "github.com/gruntwork-io/terratest/modules/testing" @@ -54,6 +55,31 @@ func (t *K8sTunnel) GetStats(name string) (*stats.Stats, error) { return &s, nil } +func (t *K8sTunnel) GetClusters() (*clusters.Clusters, error) { + url := fmt.Sprintf("http://%s/stats?format=json", t.endpoint) + + response, err := http.Post(url, "application/json", nil) // #nosec G107 -- make the url configurable is intended + if err != nil { + return nil, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return nil, errors.Errorf( + "got response with unexpected status code: %+q, Expected: %+q", + response.Status, + http.StatusText(http.StatusOK), + ) + } + + var c clusters.Clusters + if err := json.NewDecoder(response.Body).Decode(&c); err != nil { + return nil, err + } + + return &c, nil +} + func (t *K8sTunnel) ResetCounters() error { url := fmt.Sprintf("http://%s/reset_counters", t.endpoint) diff --git a/test/framework/envoy_admin/tunnel/universal.go b/test/framework/envoy_admin/tunnel/universal.go index 45ad7a568bec..a8d7f8183b2a 100644 --- a/test/framework/envoy_admin/tunnel/universal.go +++ b/test/framework/envoy_admin/tunnel/universal.go @@ -3,6 +3,7 @@ package tunnel import ( "encoding/json" "fmt" + "github.com/kumahq/kuma/test/framework/envoy_admin/clusters" "github.com/gruntwork-io/terratest/modules/testing" "github.com/pkg/errors" @@ -54,6 +55,31 @@ func (t *UniversalTunnel) GetStats(name string) (*stats.Stats, error) { return &s, nil } +func (t *UniversalTunnel) GetClusters() (*clusters.Clusters, error) { + url := "http://localhost:9901/clusters?format=json" + + sshArgs := []string{ + "curl", "--silent", "--max-time", "3", "--fail", url, + } + + app := ssh.NewApp("tunnel", "", t.verbose, t.port, nil, sshArgs) + + if err := app.Run(); err != nil { + return nil, err + } + + if app.Err() != "" { + return nil, errors.New(app.Err()) + } + + var c clusters.Clusters + if err := json.Unmarshal([]byte(app.Out()), &c); err != nil { + return nil, err + } + + return &c, nil +} + func (t *UniversalTunnel) ResetCounters() error { sshArgs := []string{ "curl", "--verbose", "--max-time", "3", "--fail", "-XPOST", From a493fa33bdeaf4d7dfafc7d374b81b0fc919380e Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Mon, 6 Nov 2023 16:22:13 +0100 Subject: [PATCH 19/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../locality_aware_multizone_hybrid_egress.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index 80fe4ebdcd31..1404d5564315 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -114,13 +114,13 @@ spec: // TODO: should be removed after fixing egress issues // Workaround to wait for all remote zones to set up egressTunnel = multizone.UniZone1.GetZoneEgressEnvoyTunnel() - Eventually(func() int { + Eventually(func(g Gomega) int { egressClusters, err := egressTunnel.GetClusters() - Expect(err).ToNot(HaveOccurred()) + g.Expect(err).ToNot(HaveOccurred()) cluster := egressClusters.GetCluster(fmt.Sprintf("%s:test-server_locality-aware-lb-egress_svc_80", mesh)) - Expect(cluster).ToNot(BeNil()) + g.Expect(cluster).ToNot(BeNil()) return len(cluster.HostStatuses) - }, "1m", "5s").Should(Equal(2)) + }, "2m", "5s").Should(Equal(2)) // Universal Zone 4 Expect(NewClusterSetup(). @@ -148,7 +148,7 @@ spec: // no lb priorities Eventually(func() (map[string]int, error) { return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(200)) - }, "6m", "10s").Should( + }, "2m", "10s").Should( And( HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 132, 15)), HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 34, 15)), @@ -182,13 +182,13 @@ spec: // TODO: should be removed after fixing egress issues // egress config refresh workaround - Eventually(func() int { + Eventually(func(g Gomega) int { egressClusters, err := egressTunnel.GetClusters() - Expect(err).ToNot(HaveOccurred()) + g.Expect(err).ToNot(HaveOccurred()) cluster := egressClusters.GetCluster(fmt.Sprintf("%s:test-server_locality-aware-lb-egress_svc_80", mesh)) - Expect(cluster).ToNot(BeNil()) + g.Expect(cluster).ToNot(BeNil()) return cluster.GetPriorityForZone("kuma-5") - }, "1m", "5s").Should(Equal(1)) + }, "2m", "5s").Should(Equal(1)) Expect(multizone.UniZone1.GetApp("demo-client_locality-aware-lb-egress_svc").ReStart()).To(Succeed()) // traffic goes to kuma-5 zone From 39bcc8e87e7a2e5792937dacaf2ca97422a345af Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Mon, 6 Nov 2023 16:45:40 +0100 Subject: [PATCH 20/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../locality_aware_multizone_hybrid_egress.go | 6 ++++-- test/framework/envoy_admin/tunnel/k8s.go | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index 1404d5564315..c08f1beb6ec9 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -2,6 +2,10 @@ package localityawarelb import ( "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/kumahq/kuma/pkg/test/resources/samples" . "github.com/kumahq/kuma/test/framework" "github.com/kumahq/kuma/test/framework/client" @@ -9,8 +13,6 @@ import ( "github.com/kumahq/kuma/test/framework/deployments/testserver" "github.com/kumahq/kuma/test/framework/envoy_admin" "github.com/kumahq/kuma/test/framework/envs/multizone" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" ) func LocalityAwareLBEgress() { diff --git a/test/framework/envoy_admin/tunnel/k8s.go b/test/framework/envoy_admin/tunnel/k8s.go index 602a64231855..adef9c38df07 100644 --- a/test/framework/envoy_admin/tunnel/k8s.go +++ b/test/framework/envoy_admin/tunnel/k8s.go @@ -3,13 +3,13 @@ package tunnel import ( "encoding/json" "fmt" - "github.com/kumahq/kuma/test/framework/envoy_admin/clusters" "net/http" "github.com/gruntwork-io/terratest/modules/testing" "github.com/pkg/errors" "github.com/kumahq/kuma/test/framework/envoy_admin" + "github.com/kumahq/kuma/test/framework/envoy_admin/clusters" "github.com/kumahq/kuma/test/framework/envoy_admin/stats" ) From 5fcd5cec0cf483a16c752daaa91a1ac3ae73aa05 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Mon, 6 Nov 2023 16:45:59 +0100 Subject: [PATCH 21/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- test/framework/envoy_admin/tunnel/universal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/envoy_admin/tunnel/universal.go b/test/framework/envoy_admin/tunnel/universal.go index a8d7f8183b2a..482499f810e0 100644 --- a/test/framework/envoy_admin/tunnel/universal.go +++ b/test/framework/envoy_admin/tunnel/universal.go @@ -3,12 +3,12 @@ package tunnel import ( "encoding/json" "fmt" - "github.com/kumahq/kuma/test/framework/envoy_admin/clusters" "github.com/gruntwork-io/terratest/modules/testing" "github.com/pkg/errors" "github.com/kumahq/kuma/test/framework/envoy_admin" + "github.com/kumahq/kuma/test/framework/envoy_admin/clusters" "github.com/kumahq/kuma/test/framework/envoy_admin/stats" "github.com/kumahq/kuma/test/framework/ssh" ) From 73d2511639c6b0aaccaf7bdfc97f9fe448b2051c Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Mon, 6 Nov 2023 17:21:23 +0100 Subject: [PATCH 22/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../locality_aware_multizone_hybrid_egress.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index c08f1beb6ec9..2fe6a0f37307 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -152,9 +152,9 @@ spec: return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(200)) }, "2m", "10s").Should( And( - HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 132, 15)), - HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 34, 15)), - HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 34, 15)), + HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 132, 20)), + HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 34, 20)), + HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 34, 20)), ), ) From fbd048d99d6077038fec144dd835e47717ce67bf Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Mon, 6 Nov 2023 19:37:31 +0100 Subject: [PATCH 23/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../localityawarelb/locality_aware_multizone_hybrid.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid.go index d29d9f68ef10..ad54667aa5f9 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid.go @@ -344,8 +344,8 @@ spec: ) }, "30s", "5s").Should( And( - HaveKeyWithValue(Equal(`test-server-mesh-route-zone-1`), BeNumerically("~", 50, 10)), - HaveKeyWithValue(Equal(`test-server-mesh-route-zone-5`), BeNumerically("~", 50, 10)), + HaveKeyWithValue(Equal(`test-server-mesh-route-zone-1`), BeNumerically("~", 50, 15)), + HaveKeyWithValue(Equal(`test-server-mesh-route-zone-5`), BeNumerically("~", 50, 15)), ), ) From c72c08f34b967ae232ba4e91ac7d3d46f563e1de Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Mon, 6 Nov 2023 20:59:12 +0100 Subject: [PATCH 24/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../locality_aware_multizone_hybrid_egress.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index 2fe6a0f37307..fd16f86d8759 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -149,12 +149,12 @@ spec: It("should route based on defined strategy with egress enabled", func() { // no lb priorities Eventually(func() (map[string]int, error) { - return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(200)) + return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(300)) }, "2m", "10s").Should( And( - HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 132, 20)), - HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 34, 20)), - HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 34, 20)), + HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 200, 20)), + HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 50, 20)), + HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 50, 20)), ), ) From c73a352c474718138ad37688eb81400221d9c697 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Mon, 6 Nov 2023 21:31:26 +0100 Subject: [PATCH 25/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../locality_aware_multizone_hybrid_egress.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go index fd16f86d8759..b01c06292c8c 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid_egress.go @@ -152,9 +152,9 @@ spec: return client.CollectResponsesByInstance(multizone.UniZone1, "demo-client_locality-aware-lb-egress_svc", "test-server_locality-aware-lb-egress_svc_80.mesh", client.WithNumberOfRequests(300)) }, "2m", "10s").Should( And( - HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 200, 20)), - HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 50, 20)), - HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 50, 20)), + HaveKeyWithValue(Equal(`test-server-zone-4`), BeNumerically("~", 200, 30)), + HaveKeyWithValue(Equal(`test-server-zone-5`), BeNumerically("~", 50, 30)), + HaveKeyWithValue(Equal(`test-server-zone-1`), BeNumerically("~", 50, 30)), ), ) From fa354d112ba77cd9a933bfa502ba931e162f1126 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Tue, 7 Nov 2023 09:59:44 +0100 Subject: [PATCH 26/26] feat(kuma-cp): review Fixes: #8237 Signed-off-by: Marcin Skalski --- .../localityawarelb/locality_aware_multizone_hybrid.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid.go b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid.go index 10f0d8e08e64..c7c7aa6867af 100644 --- a/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid.go +++ b/test/e2e_env/multizone/localityawarelb/locality_aware_multizone_hybrid.go @@ -368,10 +368,10 @@ spec: // then failedRequests, err := failRequestCount(multizone.KubeZone2, "demo-client-mesh-route") Expect(err).ToNot(HaveOccurred()) - Expect(failedRequests).To(BeNumerically("~", 100, 20)) + Expect(failedRequests).To(BeNumerically("~", 100, 25)) successRequests, err := successRequestCount(multizone.KubeZone2, "demo-client-mesh-route") Expect(err).ToNot(HaveOccurred()) - Expect(successRequests).To(BeNumerically("~", 100, 20)) + Expect(successRequests).To(BeNumerically("~", 100, 25)) }) }