pkg/*: address control plane implementation gaps

This change refactors the core control plane components
to address several gaps in the existing implementation.

At a high level, it resolves the following:

1. Traffic to a service with multiple ports is filtered and
   routed correctly based on the port the traffic is directed
   to (openservicemesh#3777). This was previously a blocker for users with
   multiple ports/protocols per k8s service object.

2. Builds mesh traffic policies for LDS/CDS/RDS configs
   inline consistently. This simplifies the code and makes
   it a lot easier to test config generation based on the
   state of the mesh (service, service accounts, SMI policies
   etc.). It also eliminates outbound policy merges that
   are unnecessary. Building config for different envoy configs
   inline will help when OSM moves to an event-driven config
   generation per proxy/group model, where an event can produce
   configuration using a nearly consistent view of the caches.

3. Allows the TargetPort for a service to be different than it's
   Port value. This is common in k8s to allow app authors to
   change the backend port without needing to update their client
   code.

4. Similar to egress, mesh RDS configs are isolated per port to
   avoid conflicts when the same host needs to be routed differently
   per port. This can happen when the same service exposes multiple
   HTTP ports that need to be routed differently.

5. Correctly generates hostnames for a service-port combination that
   clients can use to access the service.

7. Addresses bugs in endpoint discovery when the service has multiple
   ports. Correctly translates a cluster name to a MeshService to be
   able to retrieve its endpoints.

8. Implements SMI TrafficSplit for permissive mode for both HTTP and
   TCP traffic.

9. Removes and simplifies code that are not required, such as the APIs
   to retrieve port:protocol mappings for a service, and other redundant
   APIs.

10. E2e tests and unit tests for various scenarios: multiple ports per
    service, permissive mode traffic split, different TargetPort for
    service Port etc.

To understand this change better, it's important to note the following
fundamental changes made to address the existing implementation issues
noted above.

- A MeshService now carries information on the port, target port and
  protocol associated with the service. The implementation ensures
  the target port is always set for a service with an endpoint. It is
  necessary to disambiguate between the two because k8s allows routing
  traffic to a service whose Port and TargetPort are different. This
  is of significant importance when it comes to filtering and routing
  traffic on the client and server side. A k8s service with multiple
  ports is transposed to multiple MeshService representations within
  the OSM control plane. A MeshService is the source of truth while
  creating filtering and routing configs in Envoy as opposed to a k8s
  service. In other words, filter chains, clusters per service have
  a 1-1 mapping to a MeshService instance.

- RDS mesh inbound and outbound route configurations are built per
  port, similar to egress route configurations. This ensures there
  can be no conflicts when a service/FQDN needs to be routed differently
  based on the port.

- Endpoints for a MeshService is filtered by its TargetPort is set. This
  is required to endpoints not associated with the MeshService are not
  programmed for upstream clusters. This is required to support multiple
  ports per service.

Part of openservicemesh#4052
Resolves openservicemesh#3777
Resolves openservicemesh#2527

Signed-off-by: Shashank Ram <shashr2204@gmail.com>

pkg/catalog/ingress.go

@@ -140,7 +140,10 @@ func (mc *MeshCatalog) getIngressTrafficPolicy(svc service.MeshService) (*traffi
 		// Currently IngressBackend only supports a wildcard HTTP route. The
 		// 'Matches' field in the spec can be used to extend this to perform
 		// stricter enforcement.
-		backendCluster := getDefaultWeightedClusterForService(svc)
+		backendCluster := service.WeightedCluster{
+			ClusterName: service.ClusterName(svc.EnvoyLocalClusterName()),
+			Weight:      constants.ClusterWeightAcceptAll,
+		}
 		routingRule := &trafficpolicy.Rule{
 			Route: trafficpolicy.RouteWeightedClusters{
 				HTTPRouteMatch:   trafficpolicy.WildCardRouteMatch,
@@ -176,6 +179,11 @@ func (mc *MeshCatalog) getIngressTrafficPolicy(svc service.MeshService) (*traffi
 // getIngressTrafficPolicyFromK8s returns the ingress traffic policy for the given mesh service from the corresponding k8s Ingress resource
 // TODO: DEPRECATE once IngressBackend API is the default for configuring an ingress backend.
 func (mc *MeshCatalog) getIngressTrafficPolicyFromK8s(svc service.MeshService) (*trafficpolicy.IngressTrafficPolicy, error) {
+	if svc.Protocol != constants.ProtocolHTTP {
+		// Only HTTP ports can accept traffic using k8s Ingress
+		return nil, nil
+	}
+
 	httpRoutePolicies, err := mc.getIngressPoliciesFromK8s(svc)
 	if err != nil {
 		return nil, errors.Wrapf(err, "Error retrieving ingress HTTP routing policies for service %s from Kubernetes", svc)
@@ -186,43 +194,30 @@ func (mc *MeshCatalog) getIngressTrafficPolicyFromK8s(svc service.MeshService) (
 		return nil, nil
 	}
 
-	protocolToPortMap, err := mc.GetTargetPortToProtocolMappingForService(svc)
-	if err != nil {
-		return nil, errors.Wrapf(err, "Error retrieving port to protocol mapping for service %s", svc)
-	}
-
 	enableHTTPSIngress := mc.configurator.UseHTTPSIngress()
 	var trafficMatches []*trafficpolicy.IngressTrafficMatch
-	// Create protocol specific ingress filter chains per port to handle different ports serving different protocols
-	for port, appProtocol := range protocolToPortMap {
-		if appProtocol != constants.ProtocolHTTP {
-			// Only HTTP ports can accept traffic using k8s Ingress
-			continue
-		}
+	trafficMatch := &trafficpolicy.IngressTrafficMatch{
+		Port: uint32(svc.TargetPort),
+	}
 
-		trafficMatch := &trafficpolicy.IngressTrafficMatch{
-			Port: port,
-		}
+	if enableHTTPSIngress {
+		// Configure 2 taffic matches for HTTPS ingress (TLS):
+		// 1. Without SNI: to match clients that don't set the SNI
+		// 2. With SNI: to match clients that set the SNI
 
-		if enableHTTPSIngress {
-			// Configure 2 taffic matches for HTTPS ingress (TLS):
-			// 1. Without SNI: to match clients that don't set the SNI
-			// 2. With SNI: to match clients that set the SNI
-
-			trafficMatch.Name = fmt.Sprintf("ingress_%s_%d_%s", svc, port, constants.ProtocolHTTPS)
-			trafficMatch.Protocol = constants.ProtocolHTTPS
-			trafficMatch.SkipClientCertValidation = true
-			trafficMatches = append(trafficMatches, trafficMatch)
-
-			trafficMatchWithSNI := *trafficMatch
-			trafficMatchWithSNI.Name = fmt.Sprintf("ingress_%s_%d_%s_with_sni", svc, port, constants.ProtocolHTTPS)
-			trafficMatchWithSNI.ServerNames = []string{svc.ServerName()}
-			trafficMatches = append(trafficMatches, &trafficMatchWithSNI)
-		} else {
-			trafficMatch.Name = fmt.Sprintf("ingress_%s_%d_%s", svc, port, constants.ProtocolHTTP)
-			trafficMatch.Protocol = constants.ProtocolHTTP
-			trafficMatches = append(trafficMatches, trafficMatch)
-		}
+		trafficMatch.Name = fmt.Sprintf("ingress_%s_%d_%s", svc, svc.TargetPort, constants.ProtocolHTTPS)
+		trafficMatch.Protocol = constants.ProtocolHTTPS
+		trafficMatch.SkipClientCertValidation = true
+		trafficMatches = append(trafficMatches, trafficMatch)
+
+		trafficMatchWithSNI := *trafficMatch
+		trafficMatchWithSNI.Name = fmt.Sprintf("ingress_%s_%d_%s_with_sni", svc, svc.TargetPort, constants.ProtocolHTTPS)
+		trafficMatchWithSNI.ServerNames = []string{svc.ServerName()}
+		trafficMatches = append(trafficMatches, &trafficMatchWithSNI)
+	} else {
+		trafficMatch.Name = fmt.Sprintf("ingress_%s_%d_%s", svc, svc.TargetPort, constants.ProtocolHTTP)
+		trafficMatch.Protocol = constants.ProtocolHTTP
+		trafficMatches = append(trafficMatches, trafficMatch)
 	}
 
 	return &trafficpolicy.IngressTrafficPolicy{
@@ -271,7 +266,10 @@ func (mc *MeshCatalog) getIngressPoliciesNetworkingV1beta1(svc service.MeshServi
 		return inboundIngressPolicies, err
 	}
 
-	ingressWeightedCluster := getDefaultWeightedClusterForService(svc)
+	ingressWeightedCluster := service.WeightedCluster{
+		ClusterName: service.ClusterName(svc.EnvoyLocalClusterName()),
+		Weight:      constants.ClusterWeightAcceptAll,
+	}
 
 	for _, ingress := range ingresses {
 		if ingress.Spec.Backend != nil && ingress.Spec.Backend.ServiceName == svc.Name {
@@ -375,7 +373,10 @@ func (mc *MeshCatalog) getIngressPoliciesNetworkingV1(svc service.MeshService) (
 		return inboundIngressPolicies, err
 	}
 
-	ingressWeightedCluster := getDefaultWeightedClusterForService(svc)
+	ingressWeightedCluster := service.WeightedCluster{
+		ClusterName: service.ClusterName(svc.EnvoyLocalClusterName()),
+		Weight:      constants.ClusterWeightAcceptAll,
+	}
 
 	for _, ingress := range ingresses {
 		if ingress.Spec.DefaultBackend != nil && ingress.Spec.DefaultBackend.Service.Name == svc.Name {