diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index e4ab7d84f86..210b0bef278 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -790,6 +790,18 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + type: array required: - action type: object @@ -1918,6 +1930,17 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + type: array required: - action type: object diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 3f1a025027c..efe1bd22c6d 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -790,6 +790,18 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + type: array required: - action type: object @@ -1918,6 +1930,17 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + type: array required: - action type: object diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 2034cb3601b..7312dc9c748 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -790,6 +790,18 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + type: array required: - action type: object @@ -1918,6 +1930,17 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + type: array required: - action type: object diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index ecbce68ffb0..936da4df96d 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -790,6 +790,18 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + type: array required: - action type: object @@ -1918,6 +1930,17 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + type: array required: - action type: object diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 994023c84c0..154fef5ed7a 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -790,6 +790,18 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + type: array required: - action type: object @@ -1918,6 +1930,17 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + type: array required: - action type: object diff --git a/build/yamls/base/crds.yml b/build/yamls/base/crds.yml index f69807a46c4..303a1786573 100644 --- a/build/yamls/base/crds.yml +++ b/build/yamls/base/crds.yml @@ -991,6 +991,18 @@ spec: type: string fqdn: type: string + toServices: + type: array + items: + type: object + required: + - name + - namespace + properties: + name: + type: string + namespace: + type: string name: type: string enableLogging: @@ -1364,6 +1376,17 @@ spec: format: cidr fqdn: type: string + toServices: + type: array + items: + type: object + required: + - name + properties: + name: + type: string + namespace: + type: string name: type: string enableLogging: diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 93295281a8b..25f9da0fecf 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -40,6 +40,7 @@ import ( npl "antrea.io/antrea/pkg/agent/nodeportlocal" "antrea.io/antrea/pkg/agent/openflow" "antrea.io/antrea/pkg/agent/proxy" + proxytypes "antrea.io/antrea/pkg/agent/proxy/types" "antrea.io/antrea/pkg/agent/querier" "antrea.io/antrea/pkg/agent/route" "antrea.io/antrea/pkg/agent/stats" @@ -199,6 +200,11 @@ func run(o *Options) error { agentInitializer.GetWireGuardClient(), o.config.AntreaProxy.ProxyAll) + var groupCounters []proxytypes.GroupCounter + groupIDUpdates := make(chan string, 100) + v4GroupCounter := proxytypes.NewGroupCounter(false, groupIDUpdates) + v6GroupCounter := proxytypes.NewGroupCounter(true, groupIDUpdates) + var proxier proxy.Proxier if features.DefaultFeatureGate.Enabled(features.AntreaProxy) { v4Enabled := config.IsIPv4Enabled(nodeConfig, networkConfig.TrafficEncapMode) @@ -208,11 +214,14 @@ func run(o *Options) error { switch { case v4Enabled && v6Enabled: - proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient, routeClient, nodePortAddressesIPv4, nodePortAddressesIPv6, proxyAll, skipServices) + proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient, routeClient, nodePortAddressesIPv4, nodePortAddressesIPv6, proxyAll, skipServices, v4GroupCounter, v6GroupCounter) + groupCounters = append(groupCounters, v4GroupCounter, v6GroupCounter) case v4Enabled: - proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false, routeClient, nodePortAddressesIPv4, proxyAll, skipServices) + proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false, routeClient, nodePortAddressesIPv4, proxyAll, skipServices, v4GroupCounter) + groupCounters = append(groupCounters, v4GroupCounter) case v6Enabled: - proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true, routeClient, nodePortAddressesIPv6, proxyAll, skipServices) + proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true, routeClient, nodePortAddressesIPv6, proxyAll, skipServices, v6GroupCounter) + groupCounters = append(groupCounters, v6GroupCounter) default: return fmt.Errorf("at least one of IPv4 or IPv6 should be enabled") } @@ -247,6 +256,8 @@ func run(o *Options) error { ifaceStore, nodeConfig.Name, entityUpdates, + groupCounters, + groupIDUpdates, antreaPolicyEnabled, statusManagerEnabled, loggingEnabled, diff --git a/docs/antrea-network-policy.md b/docs/antrea-network-policy.md index fdb2b316549..38fecaebc97 100644 --- a/docs/antrea-network-policy.md +++ b/docs/antrea-network-policy.md @@ -14,6 +14,7 @@ - [ACNP with ClusterGroup reference](#acnp-with-clustergroup-reference) - [ACNP for complete Pod isolation in selected Namespaces](#acnp-for-complete-pod-isolation-in-selected-namespaces) - [ACNP for default Namespace isolation](#acnp-for-default-namespace-isolation) + - [ACNP for toServices rule](#acnp-for-toservices-rule) - [Behavior of to and from selectors](#behavior-of-to-and-from-selectors) - [Key differences from K8s NetworkPolicy](#key-differences-from-k8s-networkpolicy) - [kubectl commands for Antrea ClusterNetworkPolicy](#kubectl-commands-for-antrea-clusternetworkpolicy) @@ -32,6 +33,7 @@ - [K8s clusters with version 1.21 and above](#k8s-clusters-with-version-121-and-above) - [K8s clusters with version 1.20 and below](#k8s-clusters-with-version-120-and-below) - [FQDN based filtering](#fqdn-based-filtering) +- [toServices instruction](#toservices-instruction) - [RBAC](#rbac) - [Notes](#notes) @@ -331,6 +333,32 @@ spec: enableLogging: true ``` +#### ACNP for toServices rule + +```yaml +apiVersion: crd.antrea.io/v1alpha1 +kind: ClusterNetworkPolicy +metadata: + name: acnp-drop-to-services +spec: + priority: 5 + tier: securityops + appliedTo: + - podSelector: + matchLabels: + role: client + namespaceSelector: + matchLabels: + env: prod + egress: + - action: Drop + toServices: + - name: svcName + namespace: svcNamespace + name: DropToServices + enableLogging: true +``` + **spec**: The ClusterNetworkPolicy `spec` has all the information needed to define a cluster-wide security policy. @@ -408,6 +436,8 @@ a rule, it will be auto-generated by Antrea. The rule name auto-generation proce is the same as ingress rules. A ClusterGroup name can be set in the `group` field of a egress `to` section in place of stand-alone selectors to allow traffic to workloads/ipBlocks set in the ClusterGroup. +`toServices` field contains a list of combinations of Service Namespace and Service Name to match traffic to this Service. +More details can be found in the [toServices](#toservices-instruction) section. The [first example](#acnp-with-stand-alone-selectors) policy contains a single rule, which drops matched traffic on a single port, to the 10.0.10.0/24 subnet specified by the `ipBlock` field. The [second example](#acnp-with-clustergroup-reference) policy contains a single rule, which drops matched traffic on @@ -415,7 +445,11 @@ TCP port 5978 to all network endpoints selected by the "test-cg-with-ip-block" ClusterGroup. The [third example](#acnp-for-complete-pod-isolation-in-selected-namespaces) policy contains a single rule, which drops all egress traffic initiated by any Pod in Namespaces that have `app` set to -`no-network-access-required`. Note that an empty `To` in the egress rule means that +`no-network-access-required`. +The [fifth example](#acnp-for-toservices-rule) policy contains a single rule, +which drops traffic from "role: client" labeled Pods from "env: prod" labeled Namespaces to Service svcNamespace/svcName +via ClusterIP. +Note that an empty `to` + an empty `toServices` in the egress rule means that this rule matches all egress destinations. Egress `To` section also supports FQDN based filtering. This can be applied to exact FQDNs or wildcard expressions. More details can be found in the [FQDN](#fqdn-based-filtering) section. @@ -1021,6 +1055,17 @@ spec: - fqdn: "svcA.default.svc.cluster.local" ``` +## toServices instruction + +A combination of Service name and Service Namespace can be used in `toServices` to refer to a Service. +`toServices` match traffic based on the clusterIP, port and protocol of Services. So headless Service is not supported. +Since `toServices` represents a combination of IP+port, it can't be used with `to` or `ports`. Also, this match process +relies on the groupID assigned to the Service by AntreaProxy. So it can only be used when AntreaProxy is enabled. +This clusterIP based match has one caveat: directly access to the Endpoints of this Service is not controlled by +`toServices`. To control the access to the backend Endpoints, you could use `ClusterGroup` with `ServiceReference`. +Because `ClusterGroup` with `ServiceReference` is equivalent to a podSelector that selects all backend Endpoints Pods of +the Service referred in `ServiceReference`. + ## RBAC Antrea-native policy CRDs are meant for admins to manage the security of their diff --git a/pkg/agent/controller/networkpolicy/cache.go b/pkg/agent/controller/networkpolicy/cache.go index 0b1bc041f28..596fe8f23fc 100644 --- a/pkg/agent/controller/networkpolicy/cache.go +++ b/pkg/agent/controller/networkpolicy/cache.go @@ -32,6 +32,7 @@ import ( v1beta "antrea.io/antrea/pkg/apis/controlplane/v1beta2" crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" "antrea.io/antrea/pkg/querier" + "antrea.io/antrea/pkg/util/k8s" ) const ( @@ -39,6 +40,7 @@ const ( appliedToGroupIndex = "appliedToGroup" addressGroupIndex = "addressGroup" policyIndex = "policy" + toServicesIndex = "toServices" ) // rule is the struct stored in ruleCache, it contains necessary information @@ -156,6 +158,10 @@ type ruleCache struct { // entityUpdates is a channel for receiving entity (e.g. Pod) updates from CNIServer. entityUpdates <-chan antreatypes.EntityReference + + // groupIDUpdates is a channel for receiving groupID for Service is assigned + // or released events from groupCounters. + groupIDUpdates <-chan string } func (c *ruleCache) getNetworkPolicies(npFilter *querier.NetworkPolicyQueryFilter) []v1beta.NetworkPolicy { @@ -318,11 +324,23 @@ func policyIndexFunc(obj interface{}) ([]string, error) { return []string{string(rule.PolicyUID)}, nil } +// toServicesIndexFunc knows how to get NamespacedNames of Services referred in +// ToServices field of a *rule. It's provided to cache.Indexer to build an index of +// NetworkPolicy. +func toServicesIndexFunc(obj interface{}) ([]string, error) { + rule := obj.(*rule) + toSvcNamespacedName := sets.String{} + for _, svc := range rule.To.ToServices { + toSvcNamespacedName.Insert(k8s.NamespacedName(svc.Namespace, svc.Name)) + } + return toSvcNamespacedName.UnsortedList(), nil +} + // newRuleCache returns a new *ruleCache. -func newRuleCache(dirtyRuleHandler func(string), podUpdate <-chan antreatypes.EntityReference) *ruleCache { +func newRuleCache(dirtyRuleHandler func(string), podUpdate <-chan antreatypes.EntityReference, serviceGroupIDUpdate <-chan string) *ruleCache { rules := cache.NewIndexer( ruleKeyFunc, - cache.Indexers{addressGroupIndex: addressGroupIndexFunc, appliedToGroupIndex: appliedToGroupIndexFunc, policyIndex: policyIndexFunc}, + cache.Indexers{addressGroupIndex: addressGroupIndexFunc, appliedToGroupIndex: appliedToGroupIndexFunc, policyIndex: policyIndexFunc, toServicesIndex: toServicesIndexFunc}, ) cache := &ruleCache{ appliedToSetByGroup: make(map[string]v1beta.GroupMemberSet), @@ -331,8 +349,10 @@ func newRuleCache(dirtyRuleHandler func(string), podUpdate <-chan antreatypes.En rules: rules, dirtyRuleHandler: dirtyRuleHandler, entityUpdates: podUpdate, + groupIDUpdates: serviceGroupIDUpdate, } go cache.processEntityUpdates() + go cache.processGroupIDUpdates() return cache } @@ -364,6 +384,24 @@ func (c *ruleCache) processEntityUpdates() { } } +// processGroupIDUpdates is an infinite loop that takes Service groupID +// update events from the channel, finds out rules that refer this Service in +// ToServices field and use dirtyRuleHandler to re-queue these rules. +func (c *ruleCache) processGroupIDUpdates() { + for { + select { + case svcStr := <-c.groupIDUpdates: + toSvcRules, err := c.rules.ByIndex(toServicesIndex, svcStr) + if err != nil { + continue + } + for _, toSvcRule := range toSvcRules { + c.dirtyRuleHandler(toSvcRule.(*rule).ID) + } + } + } +} + // GetAddressGroupNum gets the number of AddressGroup. func (c *ruleCache) GetAddressGroupNum() int { c.addressSetLock.RLock() diff --git a/pkg/agent/controller/networkpolicy/cache_test.go b/pkg/agent/controller/networkpolicy/cache_test.go index 564a027eadf..7f47d9fc802 100644 --- a/pkg/agent/controller/networkpolicy/cache_test.go +++ b/pkg/agent/controller/networkpolicy/cache_test.go @@ -263,7 +263,8 @@ func TestRuleCacheAddAddressGroup(t *testing.T) { func newFakeRuleCache() (*ruleCache, *dirtyRuleRecorder, chan types.EntityReference) { recorder := newDirtyRuleRecorder() ch := make(chan types.EntityReference, 100) - c := newRuleCache(recorder.Record, ch) + ch2 := make(chan string, 100) + c := newRuleCache(recorder.Record, ch, ch2) return c, recorder, ch } diff --git a/pkg/agent/controller/networkpolicy/networkpolicy_controller.go b/pkg/agent/controller/networkpolicy/networkpolicy_controller.go index 84c271a94d1..65f23aa1ef1 100644 --- a/pkg/agent/controller/networkpolicy/networkpolicy_controller.go +++ b/pkg/agent/controller/networkpolicy/networkpolicy_controller.go @@ -33,6 +33,7 @@ import ( "antrea.io/antrea/pkg/agent/flowexporter/connections" "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/openflow" + proxytypes "antrea.io/antrea/pkg/agent/proxy/types" "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/apis/controlplane/v1beta2" "antrea.io/antrea/pkg/querier" @@ -110,6 +111,8 @@ func NewNetworkPolicyController(antreaClientGetter agent.AntreaClientProvider, ifaceStore interfacestore.InterfaceStore, nodeName string, entityUpdates <-chan types.EntityReference, + groupCounters []proxytypes.GroupCounter, + groupIDUpdates <-chan string, antreaPolicyEnabled bool, statusManagerEnabled bool, loggingEnabled bool, @@ -135,8 +138,8 @@ func NewNetworkPolicyController(antreaClientGetter agent.AntreaClientProvider, c.ofClient.RegisterPacketInHandler(uint8(openflow.PacketInReasonNP), "dnsresponse", c.fqdnController) } } - c.reconciler = newReconciler(ofClient, ifaceStore, idAllocator, c.fqdnController) - c.ruleCache = newRuleCache(c.enqueueRule, entityUpdates) + c.reconciler = newReconciler(ofClient, ifaceStore, idAllocator, c.fqdnController, groupCounters) + c.ruleCache = newRuleCache(c.enqueueRule, entityUpdates, groupIDUpdates) if statusManagerEnabled { c.statusManager = newStatusController(antreaClientGetter, nodeName, c.ruleCache) } diff --git a/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go b/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go index d0e8c7e5ba7..463a80d9dcb 100644 --- a/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go +++ b/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go @@ -32,6 +32,7 @@ import ( "k8s.io/component-base/metrics/legacyregistry" "antrea.io/antrea/pkg/agent/metrics" + proxytypes "antrea.io/antrea/pkg/agent/proxy/types" agenttypes "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/apis/controlplane/v1beta2" "antrea.io/antrea/pkg/client/clientset/versioned" @@ -52,7 +53,9 @@ func (g *antreaClientGetter) GetAntreaClient() (versioned.Interface, error) { func newTestController() (*Controller, *fake.Clientset, *mockReconciler) { clientset := &fake.Clientset{} ch := make(chan agenttypes.EntityReference, 100) - controller, _ := NewNetworkPolicyController(&antreaClientGetter{clientset}, nil, nil, "node1", ch, + ch2 := make(chan string, 100) + groupCounters := []proxytypes.GroupCounter{proxytypes.NewGroupCounter(false, ch2)} + controller, _ := NewNetworkPolicyController(&antreaClientGetter{clientset}, nil, nil, "node1", ch, groupCounters, ch2, true, true, true, nil, testAsyncDeleteInterval, "8.8.8.8:53") reconciler := newMockReconciler() controller.reconciler = reconciler diff --git a/pkg/agent/controller/networkpolicy/reconciler.go b/pkg/agent/controller/networkpolicy/reconciler.go index a003313df00..06a75ae4043 100644 --- a/pkg/agent/controller/networkpolicy/reconciler.go +++ b/pkg/agent/controller/networkpolicy/reconciler.go @@ -28,10 +28,12 @@ import ( "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/openflow" + proxytypes "antrea.io/antrea/pkg/agent/proxy/types" "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/apis/controlplane/v1beta2" binding "antrea.io/antrea/pkg/ovs/openflow" "antrea.io/antrea/pkg/util/ip" + "antrea.io/antrea/pkg/util/k8s" ) var ( @@ -150,15 +152,20 @@ type lastRealized struct { // the fqdn selector of this policy rule. It must be empty for policy rule // that is not egress and does not have toFQDN field. fqdnIPAddresses sets.String + // groupIDAddresses tracks the last realized set of groupIDs resolved for + // the toServices of this policy rule. It must be empty for policy rule + // that is not egress and does not have toServices field. + groupIDAddresses sets.Int64 } func newLastRealized(rule *CompletedRule) *lastRealized { return &lastRealized{ - ofIDs: map[servicesKey]uint32{}, - CompletedRule: rule, - podOFPorts: map[servicesKey]sets.Int32{}, - podIPs: nil, - fqdnIPAddresses: nil, + ofIDs: map[servicesKey]uint32{}, + CompletedRule: rule, + podOFPorts: map[servicesKey]sets.Int32{}, + podIPs: nil, + fqdnIPAddresses: nil, + groupIDAddresses: nil, } } @@ -198,6 +205,10 @@ type reconciler struct { // reconciler to register FQDN policy rules and query the IP addresses corresponded // to a FQDN. fqdnController *fqdnController + + // groupCounters is a list of GroupCounter for v4 and v6 env. reconciler uses these + // GroupCounters to get the groupIDs of a specific Service. + groupCounters []proxytypes.GroupCounter } // newReconciler returns a new *reconciler. @@ -205,6 +216,7 @@ func newReconciler(ofClient openflow.Client, ifaceStore interfacestore.InterfaceStore, idAllocator *idAllocator, fqdnController *fqdnController, + groupCounters []proxytypes.GroupCounter, ) *reconciler { priorityAssigners := map[uint8]*tablePriorityAssigner{} for _, table := range openflow.GetAntreaPolicyBaselineTierTables() { @@ -224,6 +236,7 @@ func newReconciler(ofClient openflow.Client, idAllocator: idAllocator, priorityAssigners: priorityAssigners, fqdnController: fqdnController, + groupCounters: groupCounters, } // Check if ofClient is nil or not to be compatible with unit tests. if ofClient != nil { @@ -412,6 +425,7 @@ func (r *reconciler) add(rule *CompletedRule, ofPriority *uint16, table uint8) e if r.fqdnController != nil { lastRealized.fqdnIPAddresses = nil } + lastRealized.groupIDAddresses = nil return err } // Record ofID only if its Openflow is installed successfully. @@ -485,7 +499,7 @@ func (r *reconciler) computeOFRulesForAdd(rule *CompletedRule, ofPriority *uint1 // isolated, so we create a PolicyRule with the original services if it doesn't exist. // If there are IPBlocks or Pods that cannot resolve any named port, they will share // this PolicyRule. Antrea policies do not need this default isolation. - if !rule.isAntreaNetworkPolicyRule() || len(rule.To.IPBlocks) > 0 || len(rule.To.FQDNs) > 0 { + if !rule.isAntreaNetworkPolicyRule() || len(rule.To.IPBlocks) > 0 || len(rule.To.FQDNs) > 0 || len(rule.To.ToServices) > 0 { svcKey := normalizeServices(rule.Services) ofRule, exists := ofRuleByServicesMap[svcKey] // Create a new Openflow rule if the group doesn't exist. @@ -521,6 +535,21 @@ func (r *reconciler) computeOFRulesForAdd(rule *CompletedRule, ofPriority *uint1 // If the rule installation fails, this will be reset lastRealized.fqdnIPAddresses = addressSet } + if len(rule.To.ToServices) > 0 { + var addresses []types.Address + addressSet := sets.NewInt64() + for _, svcRef := range rule.To.ToServices { + for _, groupCounter := range r.groupCounters { + for _, groupID := range groupCounter.GetAllGroupIDs(k8s.NamespacedName(svcRef.Namespace, svcRef.Name)) { + addresses = append(addresses, openflow.NewServiceGroupIDAddress(groupID)) + addressSet.Insert(int64(groupID)) + } + } + } + ofRule.To = append(ofRule.To, addresses...) + // If the rule installation fails, this will be reset. + lastRealized.groupIDAddresses = addressSet + } } } return ofRuleByServicesMap, lastRealized @@ -692,6 +721,27 @@ func (r *reconciler) update(lastRealized *lastRealized, newRule *CompletedRule, } } } + originalGroupIDAddressSet, newGroupIDAddressSet := sets.NewInt64(), sets.NewInt64() + if lastRealized.groupIDAddresses != nil { + originalGroupIDAddressSet = lastRealized.groupIDAddresses + } + if len(newRule.To.ToServices) > 0 { + for _, svcRef := range newRule.To.ToServices { + for _, groupCounter := range r.groupCounters { + for _, groupID := range groupCounter.GetAllGroupIDs(k8s.NamespacedName(svcRef.Namespace, svcRef.Name)) { + newGroupIDAddressSet.Insert(int64(groupID)) + } + } + } + addedGroupIDAddress := newGroupIDAddressSet.Difference(originalGroupIDAddressSet) + removedGroupIDAddress := originalGroupIDAddressSet.Difference(newGroupIDAddressSet) + for a := range addedGroupIDAddress { + addedTo = append(addedTo, openflow.NewServiceGroupIDAddress(binding.GroupIDType(a))) + } + for r := range removedGroupIDAddress { + deletedTo = append(deletedTo, openflow.NewServiceGroupIDAddress(binding.GroupIDType(r))) + } + } if err := r.updateOFRule(ofID, addedFrom, addedTo, deletedFrom, deletedTo, ofPriority); err != nil { return err } @@ -699,6 +749,8 @@ func (r *reconciler) update(lastRealized *lastRealized, newRule *CompletedRule, // Update the FQDN address set if rule installation succeeds. lastRealized.fqdnIPAddresses = newFQDNAddressSet } + // Update the groupID address set if rule installation succeeds. + lastRealized.groupIDAddresses = newGroupIDAddressSet // Delete valid servicesKey from staleOFIDs. delete(staleOFIDs, svcKey) } diff --git a/pkg/agent/controller/networkpolicy/reconciler_test.go b/pkg/agent/controller/networkpolicy/reconciler_test.go index e9993027962..ac4fc036694 100644 --- a/pkg/agent/controller/networkpolicy/reconciler_test.go +++ b/pkg/agent/controller/networkpolicy/reconciler_test.go @@ -29,6 +29,7 @@ import ( "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/openflow" openflowtest "antrea.io/antrea/pkg/agent/openflow/testing" + proxytypes "antrea.io/antrea/pkg/agent/proxy/types" "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/apis/controlplane/v1beta2" @@ -99,7 +100,9 @@ func newCIDR(cidrStr string) *net.IPNet { func newTestReconciler(t *testing.T, controller *gomock.Controller, ifaceStore interfacestore.InterfaceStore, ofClient *openflowtest.MockClient) *reconciler { f, _ := newMockFQDNController(t, controller, nil) - r := newReconciler(ofClient, ifaceStore, newIDAllocator(testAsyncDeleteInterval), f) + ch := make(chan string, 100) + groupCounters := []proxytypes.GroupCounter{proxytypes.NewGroupCounter(false, ch)} + r := newReconciler(ofClient, ifaceStore, newIDAllocator(testAsyncDeleteInterval), f, groupCounters) return r } diff --git a/pkg/agent/controller/networkpolicy/status_controller_test.go b/pkg/agent/controller/networkpolicy/status_controller_test.go index 8e1607febb6..37c9173fa4c 100644 --- a/pkg/agent/controller/networkpolicy/status_controller_test.go +++ b/pkg/agent/controller/networkpolicy/status_controller_test.go @@ -51,7 +51,7 @@ func (c *fakeNetworkPolicyControl) getNetworkPolicyStatus() *v1beta2.NetworkPoli } func newTestStatusController() (*StatusController, *ruleCache, *fakeNetworkPolicyControl) { - ruleCache := newRuleCache(func(s string) {}, make(<-chan types.EntityReference)) + ruleCache := newRuleCache(func(s string) {}, make(<-chan types.EntityReference), make(chan string, 100)) statusControl := &fakeNetworkPolicyControl{} statusController := newStatusController(nil, testNode1, ruleCache) statusController.statusControlInterface = statusControl diff --git a/pkg/agent/openflow/fields.go b/pkg/agent/openflow/fields.go index 31582dae681..59a6776613f 100644 --- a/pkg/agent/openflow/fields.go +++ b/pkg/agent/openflow/fields.go @@ -112,9 +112,13 @@ var ( // Field to cache the Egress conjunction ID hit by TraceFlow packet. TFEgressConjIDField = binding.NewRegField(5, 0, 31, "TFEgressConjunctionID") - // reg(N6XM_NX_REG6) + // reg6(NXM_NX_REG6) // Field to store the Ingress conjunction ID hit by TraceFlow packet. TFIngressConjIDField = binding.NewRegField(6, 0, 31, "TFIngressConjunctionID") + + // reg7(NXM_NX_REG7) + // Field to store the GroupID corresponding to the Service + ServiceGroupIDField = binding.NewRegField(7, 0, 31, "ServiceGroupID") ) // Fields using xxreg. diff --git a/pkg/agent/openflow/network_policy.go b/pkg/agent/openflow/network_policy.go index 4c926455025..0de01e0bf21 100644 --- a/pkg/agent/openflow/network_policy.go +++ b/pkg/agent/openflow/network_policy.go @@ -31,27 +31,28 @@ import ( ) var ( - MatchDstIP = types.NewMatchKey(binding.ProtocolIP, types.IPAddr, "nw_dst") - MatchSrcIP = types.NewMatchKey(binding.ProtocolIP, types.IPAddr, "nw_src") - MatchDstIPNet = types.NewMatchKey(binding.ProtocolIP, types.IPNetAddr, "nw_dst") - MatchSrcIPNet = types.NewMatchKey(binding.ProtocolIP, types.IPNetAddr, "nw_src") - MatchDstIPv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPAddr, "ipv6_dst") - MatchSrcIPv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPAddr, "ipv6_src") - MatchDstIPNetv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPNetAddr, "ipv6_dst") - MatchSrcIPNetv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPNetAddr, "ipv6_src") - MatchDstOFPort = types.NewMatchKey(binding.ProtocolIP, types.OFPortAddr, "reg1[0..31]") - MatchSrcOFPort = types.NewMatchKey(binding.ProtocolIP, types.OFPortAddr, "in_port") - MatchTCPDstPort = types.NewMatchKey(binding.ProtocolTCP, types.L4PortAddr, "tp_dst") - MatchTCPv6DstPort = types.NewMatchKey(binding.ProtocolTCPv6, types.L4PortAddr, "tp_dst") - MatchUDPDstPort = types.NewMatchKey(binding.ProtocolUDP, types.L4PortAddr, "tp_dst") - MatchUDPv6DstPort = types.NewMatchKey(binding.ProtocolUDPv6, types.L4PortAddr, "tp_dst") - MatchSCTPDstPort = types.NewMatchKey(binding.ProtocolSCTP, types.L4PortAddr, "tp_dst") - MatchSCTPv6DstPort = types.NewMatchKey(binding.ProtocolSCTPv6, types.L4PortAddr, "tp_dst") - MatchTCPSrcPort = types.NewMatchKey(binding.ProtocolTCP, types.L4PortAddr, "tp_src") - MatchTCPv6SrcPort = types.NewMatchKey(binding.ProtocolTCPv6, types.L4PortAddr, "tp_src") - MatchUDPSrcPort = types.NewMatchKey(binding.ProtocolUDP, types.L4PortAddr, "tp_src") - MatchUDPv6SrcPort = types.NewMatchKey(binding.ProtocolUDPv6, types.L4PortAddr, "tp_src") - Unsupported = types.NewMatchKey(binding.ProtocolIP, types.UnSupported, "unknown") + MatchDstIP = types.NewMatchKey(binding.ProtocolIP, types.IPAddr, "nw_dst") + MatchSrcIP = types.NewMatchKey(binding.ProtocolIP, types.IPAddr, "nw_src") + MatchDstIPNet = types.NewMatchKey(binding.ProtocolIP, types.IPNetAddr, "nw_dst") + MatchSrcIPNet = types.NewMatchKey(binding.ProtocolIP, types.IPNetAddr, "nw_src") + MatchDstIPv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPAddr, "ipv6_dst") + MatchSrcIPv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPAddr, "ipv6_src") + MatchDstIPNetv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPNetAddr, "ipv6_dst") + MatchSrcIPNetv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPNetAddr, "ipv6_src") + MatchDstOFPort = types.NewMatchKey(binding.ProtocolIP, types.OFPortAddr, "reg1[0..31]") + MatchSrcOFPort = types.NewMatchKey(binding.ProtocolIP, types.OFPortAddr, "in_port") + MatchTCPDstPort = types.NewMatchKey(binding.ProtocolTCP, types.L4PortAddr, "tp_dst") + MatchTCPv6DstPort = types.NewMatchKey(binding.ProtocolTCPv6, types.L4PortAddr, "tp_dst") + MatchUDPDstPort = types.NewMatchKey(binding.ProtocolUDP, types.L4PortAddr, "tp_dst") + MatchUDPv6DstPort = types.NewMatchKey(binding.ProtocolUDPv6, types.L4PortAddr, "tp_dst") + MatchSCTPDstPort = types.NewMatchKey(binding.ProtocolSCTP, types.L4PortAddr, "tp_dst") + MatchSCTPv6DstPort = types.NewMatchKey(binding.ProtocolSCTPv6, types.L4PortAddr, "tp_dst") + MatchTCPSrcPort = types.NewMatchKey(binding.ProtocolTCP, types.L4PortAddr, "tp_src") + MatchTCPv6SrcPort = types.NewMatchKey(binding.ProtocolTCPv6, types.L4PortAddr, "tp_src") + MatchUDPSrcPort = types.NewMatchKey(binding.ProtocolUDP, types.L4PortAddr, "tp_src") + MatchUDPv6SrcPort = types.NewMatchKey(binding.ProtocolUDPv6, types.L4PortAddr, "tp_src") + MatchServiceGroupID = types.NewMatchKey(binding.ProtocolIP, types.ServiceGroupIDAddr, "reg7[0..31]") + Unsupported = types.NewMatchKey(binding.ProtocolIP, types.UnSupported, "unknown") // metricFlowIdentifier is used to identify metric flows in metric table. // There could be other flows like default flow and Traceflow flows in the table. Only metric flows are supposed to @@ -164,6 +165,25 @@ func NewOFPortAddress(addr int32) *OFPortAddress { return &a } +type ServiceGroupIDAddress binding.GroupIDType + +func (a *ServiceGroupIDAddress) GetMatchKey(addrType types.AddressType) *types.MatchKey { + return MatchServiceGroupID +} + +func (a *ServiceGroupIDAddress) GetMatchValue() string { + return fmt.Sprintf("%d", uint32(*a)) +} + +func (a *ServiceGroupIDAddress) GetValue() interface{} { + return uint32(*a) +} + +func NewServiceGroupIDAddress(groupID binding.GroupIDType) *ServiceGroupIDAddress { + a := ServiceGroupIDAddress(groupID) + return &a +} + // ConjunctionNotFound is an error response when the specified policyRuleConjunction is not found from the local cache. type ConjunctionNotFound uint32 diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index be58e376416..6d3d4db2ede 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -1873,6 +1873,8 @@ func (c *client) addFlowMatch(fb binding.FlowBuilder, matchKey *types.MatchKey, if portValue.Value > 0 { fb = fb.MatchSrcPort(portValue.Value, portValue.Mask) } + case MatchServiceGroupID: + fb = fb.MatchRegFieldWithValue(ServiceGroupIDField, matchValue.(uint32)) } return fb } @@ -2255,7 +2257,9 @@ func (c *client) serviceLBFlow(groupID binding.GroupIDType, svcIP net.IP, svcPor flowBuilder = flowBuilder.Action().LoadRegMark(ServiceNeedSNATRegMark) } } - return flowBuilder.Action().Group(groupID).Done() + return flowBuilder. + Action().LoadToRegField(ServiceGroupIDField, uint32(groupID)). + Action().Group(groupID).Done() } // endpointDNATFlow generates the flow which transforms the Service Cluster IP diff --git a/pkg/agent/proxy/proxier.go b/pkg/agent/proxy/proxier.go index 0e4f708423a..dc15f7ad5db 100644 --- a/pkg/agent/proxy/proxier.go +++ b/pkg/agent/proxy/proxier.go @@ -790,7 +790,8 @@ func NewProxier( routeClient route.Interface, nodePortAddresses []net.IP, proxyAllEnabled bool, - skipServices []string) *proxier { + skipServices []string, + groupCounter types.GroupCounter) *proxier { recorder := record.NewBroadcaster().NewRecorder( runtime.NewScheme(), corev1.EventSource{Component: componentName, Host: hostname}, @@ -816,7 +817,7 @@ func NewProxier( endpointReferenceCounter: map[string]int{}, serviceStringMap: map[string]k8sproxy.ServicePortName{}, oversizeServiceSet: sets.NewString(), - groupCounter: types.NewGroupCounter(isIPv6), + groupCounter: groupCounter, ofClient: ofClient, routeClient: routeClient, nodePortAddresses: nodePortAddresses, @@ -875,13 +876,15 @@ func NewDualStackProxier( nodePortAddressesIPv4 []net.IP, nodePortAddressesIPv6 []net.IP, proxyAllEnabled bool, - skipServices []string) *metaProxierWrapper { + skipServices []string, + v4groupCounter types.GroupCounter, + v6groupCounter types.GroupCounter) *metaProxierWrapper { // Create an IPv4 instance of the single-stack proxier. - ipv4Proxier := NewProxier(hostname, informerFactory, ofClient, false, routeClient, nodePortAddressesIPv4, proxyAllEnabled, skipServices) + ipv4Proxier := NewProxier(hostname, informerFactory, ofClient, false, routeClient, nodePortAddressesIPv4, proxyAllEnabled, skipServices, v4groupCounter) // Create an IPv6 instance of the single-stack proxier. - ipv6Proxier := NewProxier(hostname, informerFactory, ofClient, true, routeClient, nodePortAddressesIPv6, proxyAllEnabled, skipServices) + ipv6Proxier := NewProxier(hostname, informerFactory, ofClient, true, routeClient, nodePortAddressesIPv6, proxyAllEnabled, skipServices, v6groupCounter) // Create a meta-proxier that dispatch calls between the two // single-stack proxier instances. diff --git a/pkg/agent/proxy/proxier_test.go b/pkg/agent/proxy/proxier_test.go index 9d2a53157f7..baf0bcc8515 100644 --- a/pkg/agent/proxy/proxier_test.go +++ b/pkg/agent/proxy/proxier_test.go @@ -120,7 +120,7 @@ func NewFakeProxier(routeClient route.Interface, ofClient openflow.Client, nodeP endpointsInstalledMap: types.EndpointsMap{}, endpointReferenceCounter: map[string]int{}, endpointsMap: types.EndpointsMap{}, - groupCounter: types.NewGroupCounter(isIPv6), + groupCounter: types.NewGroupCounter(isIPv6, make(chan string, 100)), ofClient: ofClient, routeClient: routeClient, serviceStringMap: map[string]k8sproxy.ServicePortName{}, diff --git a/pkg/agent/proxy/types/groupcounter.go b/pkg/agent/proxy/types/groupcounter.go index d4c537924d8..657f952240c 100644 --- a/pkg/agent/proxy/types/groupcounter.go +++ b/pkg/agent/proxy/types/groupcounter.go @@ -18,6 +18,8 @@ import ( "fmt" "sync" + "k8s.io/apimachinery/pkg/util/sets" + binding "antrea.io/antrea/pkg/ovs/openflow" k8sproxy "antrea.io/antrea/third_party/proxy" ) @@ -32,22 +34,26 @@ type GroupCounter interface { // Recycle removes a Service Group ID mapping. The recycled groupID can be // reused. Recycle(svcPortName k8sproxy.ServicePortName, isEndpointsLocal bool) bool + // GetAllGroupIDs gets all groupID related to a Service. + GetAllGroupIDs(svcNamespacedName string) []binding.GroupIDType } type groupCounter struct { mu sync.Mutex groupIDCounter binding.GroupIDType recycled []binding.GroupIDType + groupIDUpdates chan<- string - groupMap map[string]binding.GroupIDType + servicePortNamesMap map[string]sets.String + groupMap map[string]binding.GroupIDType } -func NewGroupCounter(isIPv6 bool) *groupCounter { +func NewGroupCounter(isIPv6 bool, groupIDUpdates chan<- string) *groupCounter { var groupIDCounter binding.GroupIDType if isIPv6 { groupIDCounter = 0x10000000 } - return &groupCounter{groupMap: map[string]binding.GroupIDType{}, groupIDCounter: groupIDCounter} + return &groupCounter{groupMap: map[string]binding.GroupIDType{}, groupIDCounter: groupIDCounter, groupIDUpdates: groupIDUpdates, servicePortNamesMap: map[string]sets.String{}} } func keyString(svcPortName k8sproxy.ServicePortName, isEndpointsLocal bool) string { @@ -58,6 +64,26 @@ func keyString(svcPortName k8sproxy.ServicePortName, isEndpointsLocal bool) stri return key } +func (c *groupCounter) updateServicePortNameMap(svcNamespacedName string, svcKeyString string) { + if _, ok := c.servicePortNamesMap[svcNamespacedName]; ok { + c.servicePortNamesMap[svcNamespacedName].Insert(svcKeyString) + } else { + keyStringSet := sets.NewString(svcKeyString) + c.servicePortNamesMap[svcNamespacedName] = keyStringSet + } +} + +func (c *groupCounter) deleteServicePortNameMap(svcNamespacedName string, svcKeyString string) { + keyStringSet, ok := c.servicePortNamesMap[svcNamespacedName] + if !ok { + return + } + keyStringSet.Delete(svcKeyString) + if keyStringSet.Len() == 0 { + delete(c.servicePortNamesMap, svcNamespacedName) + } +} + func (c *groupCounter) Get(svcPortName k8sproxy.ServicePortName, isEndpointsLocal bool) (binding.GroupIDType, bool) { c.mu.Lock() defer c.mu.Unlock() @@ -68,10 +94,14 @@ func (c *groupCounter) Get(svcPortName k8sproxy.ServicePortName, isEndpointsLoca id = c.recycled[len(c.recycled)-1] c.recycled = c.recycled[:len(c.recycled)-1] c.groupMap[key] = id + c.updateServicePortNameMap(svcPortName.NamespacedName.String(), key) + c.groupIDUpdates <- svcPortName.NamespacedName.String() return id, true } else { c.groupIDCounter += 1 c.groupMap[key] = c.groupIDCounter + c.updateServicePortNameMap(svcPortName.NamespacedName.String(), key) + c.groupIDUpdates <- svcPortName.NamespacedName.String() return c.groupIDCounter, true } } @@ -84,7 +114,21 @@ func (c *groupCounter) Recycle(svcPortName k8sproxy.ServicePortName, isEndpoints if id, ok := c.groupMap[key]; ok { delete(c.groupMap, key) c.recycled = append(c.recycled, id) + c.deleteServicePortNameMap(svcPortName.NamespacedName.String(), key) + c.groupIDUpdates <- svcPortName.NamespacedName.String() return true } return false } + +func (c *groupCounter) GetAllGroupIDs(svcNamespacedName string) []binding.GroupIDType { + c.mu.Lock() + defer c.mu.Unlock() + var ids []binding.GroupIDType + for _, key := range c.servicePortNamesMap[svcNamespacedName].UnsortedList() { + if id, ok := c.groupMap[key]; ok { + ids = append(ids, id) + } + } + return ids +} diff --git a/pkg/agent/types/networkpolicy.go b/pkg/agent/types/networkpolicy.go index 1343b3faa82..38d6e81d3fc 100644 --- a/pkg/agent/types/networkpolicy.go +++ b/pkg/agent/types/networkpolicy.go @@ -53,6 +53,7 @@ const ( IPNetAddr OFPortAddr L4PortAddr + ServiceGroupIDAddr UnSupported ) diff --git a/pkg/apis/controlplane/types.go b/pkg/apis/controlplane/types.go index cd38ff589be..a598d860c4e 100644 --- a/pkg/apis/controlplane/types.go +++ b/pkg/apis/controlplane/types.go @@ -273,6 +273,9 @@ type NetworkPolicyPeer struct { // A list of exact FQDN names or FQDN wildcard expressions. // This field can only be possibly set for NetworkPolicyPeer of egress rules. FQDNs []string + // A list of ServiceReference. + // This field can only be possibly set for NetworkPolicyPeer of egress rules. + ToServices []ServiceReference } // IPBlock describes a particular CIDR (Ex. "192.168.1.1/24"). The except entry describes CIDRs that should diff --git a/pkg/apis/controlplane/v1beta2/generated.pb.go b/pkg/apis/controlplane/v1beta2/generated.pb.go index b10d637a45d..ea273ba182b 100644 --- a/pkg/apis/controlplane/v1beta2/generated.pb.go +++ b/pkg/apis/controlplane/v1beta2/generated.pb.go @@ -896,124 +896,126 @@ func init() { } var fileDescriptor_fbaa7d016762fa1d = []byte{ - // 1867 bytes of a gzipped FileDescriptorProto + // 1892 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0x4d, 0x6c, 0x23, 0x49, - 0x15, 0x9e, 0xf6, 0x4f, 0x12, 0xbf, 0x38, 0x89, 0x53, 0xd9, 0x61, 0xcc, 0x32, 0xd8, 0xd9, 0xe6, - 0x47, 0x39, 0xb0, 0xed, 0x4d, 0x98, 0xdd, 0x1d, 0xd8, 0x1f, 0x14, 0x6f, 0x32, 0x91, 0xa5, 0x59, - 0xaf, 0xa9, 0x64, 0x35, 0x12, 0x62, 0x61, 0x3b, 0xdd, 0x65, 0xa7, 0x88, 0xdd, 0xd5, 0x74, 0x97, - 0xc3, 0x44, 0x48, 0x68, 0x11, 0x70, 0x58, 0x40, 0x82, 0x1b, 0x67, 0x4e, 0x5c, 0x38, 0x73, 0xe7, - 0x80, 0x34, 0xc7, 0x45, 0x80, 0xd8, 0x93, 0xc5, 0x18, 0x01, 0xe2, 0xc0, 0x8d, 0x53, 0xf6, 0x82, - 0xaa, 0xba, 0xfa, 0xd7, 0xc9, 0x64, 0x3c, 0xc9, 0x04, 0x09, 0xf6, 0x64, 0xf7, 0xab, 0xf7, 0xde, - 0xf7, 0x5e, 0xbd, 0xbf, 0xaa, 0x6e, 0x78, 0xdd, 0x74, 0xb8, 0x47, 0x4c, 0x83, 0xb2, 0x46, 0xf0, - 0xaf, 0xe1, 0x1e, 0xf6, 0x1a, 0xa6, 0x4b, 0xfd, 0x86, 0xc5, 0x1c, 0xee, 0xb1, 0xbe, 0xdb, 0x37, - 0x1d, 0xd2, 0x38, 0x5a, 0xdf, 0x27, 0xdc, 0xdc, 0x68, 0xf4, 0x88, 0x43, 0x3c, 0x93, 0x13, 0xdb, - 0x70, 0x3d, 0xc6, 0x19, 0x32, 0x02, 0xa9, 0x6f, 0x52, 0xa6, 0xfe, 0x19, 0xee, 0x61, 0xcf, 0x10, - 0xf2, 0x46, 0x52, 0xde, 0x50, 0xf2, 0xcf, 0xde, 0x3e, 0x1b, 0xcf, 0xe7, 0x26, 0xf7, 0x1b, 0x47, - 0xeb, 0x66, 0xdf, 0x3d, 0x30, 0xd7, 0xb3, 0x48, 0xcf, 0x3e, 0xdf, 0xa3, 0xfc, 0x60, 0xb8, 0x6f, - 0x58, 0x6c, 0xd0, 0xe8, 0xb1, 0x1e, 0x6b, 0x48, 0xf2, 0xfe, 0xb0, 0x2b, 0x9f, 0xe4, 0x83, 0xfc, - 0xa7, 0xd8, 0x6f, 0x1d, 0xde, 0xf6, 0x25, 0x8a, 0x4b, 0x07, 0xa6, 0x75, 0x40, 0x1d, 0xe2, 0x1d, - 0xc7, 0x58, 0x03, 0xc2, 0xcd, 0xc6, 0xd1, 0x24, 0x48, 0xe3, 0x2c, 0x29, 0x6f, 0xe8, 0x70, 0x3a, - 0x20, 0x13, 0x02, 0x2f, 0x9d, 0x27, 0xe0, 0x5b, 0x07, 0x64, 0x60, 0x4e, 0xc8, 0x7d, 0xf1, 0x2c, - 0xb9, 0x21, 0xa7, 0xfd, 0x06, 0x75, 0xb8, 0xcf, 0xbd, 0xac, 0x90, 0xfe, 0x0f, 0x0d, 0xca, 0x9b, - 0xb6, 0xed, 0x11, 0xdf, 0xdf, 0xf1, 0xd8, 0xd0, 0x45, 0xef, 0xc2, 0x9c, 0xf0, 0xc4, 0x36, 0xb9, - 0x59, 0xd5, 0x56, 0xb5, 0xb5, 0xf9, 0x8d, 0x17, 0x8c, 0x40, 0xb1, 0x91, 0x54, 0x1c, 0xc7, 0x44, - 0x70, 0x1b, 0x47, 0xeb, 0xc6, 0x5b, 0xfb, 0xdf, 0x22, 0x16, 0x7f, 0x93, 0x70, 0xb3, 0x89, 0x1e, - 0x8c, 0xea, 0xd7, 0xc6, 0xa3, 0x3a, 0xc4, 0x34, 0x1c, 0x69, 0x45, 0x43, 0x28, 0xf7, 0x04, 0xd4, - 0x9b, 0x64, 0xb0, 0x4f, 0x3c, 0xbf, 0x9a, 0x5b, 0xcd, 0xaf, 0xcd, 0x6f, 0xbc, 0x32, 0x65, 0xd8, - 0x8d, 0x9d, 0x58, 0x47, 0xf3, 0x19, 0x05, 0x58, 0x4e, 0x10, 0x7d, 0x9c, 0x82, 0xd1, 0xff, 0xa0, - 0x41, 0x25, 0xe9, 0xe9, 0x5d, 0xea, 0x73, 0xf4, 0xf5, 0x09, 0x6f, 0x8d, 0xc7, 0xf3, 0x56, 0x48, - 0x4b, 0x5f, 0x2b, 0x0a, 0x7a, 0x2e, 0xa4, 0x24, 0x3c, 0x35, 0xa1, 0x48, 0x39, 0x19, 0x84, 0x2e, - 0xbe, 0x3a, 0xad, 0x8b, 0x49, 0x73, 0x9b, 0x0b, 0x0a, 0xa8, 0xd8, 0x12, 0x2a, 0x71, 0xa0, 0x59, - 0x7f, 0x3f, 0x0f, 0xcb, 0x49, 0xb6, 0x8e, 0xc9, 0xad, 0x83, 0x2b, 0x08, 0xe2, 0x0f, 0x35, 0x58, - 0x36, 0x6d, 0x9b, 0xd8, 0x3b, 0x97, 0x1c, 0xca, 0x4f, 0x2a, 0x58, 0xe1, 0x55, 0x5a, 0x3b, 0x9e, - 0x04, 0x44, 0x3f, 0xd6, 0x60, 0xc5, 0x23, 0x03, 0x76, 0x94, 0x31, 0x24, 0x7f, 0x71, 0x43, 0x3e, - 0xa5, 0x0c, 0x59, 0xc1, 0x93, 0xfa, 0xf1, 0x69, 0xa0, 0xfa, 0x3f, 0x35, 0x58, 0xdc, 0x74, 0xdd, - 0x3e, 0x25, 0xf6, 0x1e, 0xfb, 0x1f, 0xaf, 0xa6, 0x3f, 0x6b, 0x80, 0xd2, 0xbe, 0x5e, 0x41, 0x3d, - 0x59, 0xe9, 0x7a, 0x7a, 0x7d, 0xea, 0x7a, 0x4a, 0x19, 0x7c, 0x46, 0x45, 0xfd, 0x24, 0x0f, 0x2b, - 0x69, 0xc6, 0x8f, 0x6b, 0xea, 0xbf, 0x57, 0x53, 0x1f, 0xe5, 0x60, 0xe5, 0x8d, 0xfe, 0xd0, 0xe7, - 0xc4, 0x4b, 0x19, 0xf9, 0xf4, 0xa3, 0xf1, 0x7d, 0x0d, 0x2a, 0xa4, 0xdb, 0x25, 0x16, 0xa7, 0x47, - 0xe4, 0x12, 0x83, 0x51, 0x55, 0xa8, 0x95, 0xed, 0x8c, 0x72, 0x3c, 0x01, 0x87, 0xbe, 0x07, 0xcb, - 0x11, 0xad, 0xd5, 0x69, 0xf6, 0x99, 0x75, 0x18, 0xc6, 0xe1, 0xc5, 0x69, 0x6d, 0x68, 0x75, 0xda, - 0x84, 0xc7, 0xa9, 0xb0, 0x9d, 0xd5, 0x8b, 0x27, 0xa1, 0xf4, 0xbf, 0x6b, 0x30, 0xbf, 0xdd, 0xfb, - 0x3f, 0x38, 0x1c, 0xfc, 0x5e, 0x83, 0xa5, 0x84, 0xa3, 0x57, 0xd0, 0xcb, 0xde, 0x4d, 0xf7, 0xb2, - 0xa9, 0x3d, 0x4c, 0x58, 0x7b, 0x46, 0x23, 0xfb, 0x69, 0x1e, 0x2a, 0x09, 0xae, 0xa0, 0x8b, 0xd9, - 0x00, 0x2c, 0xda, 0xf7, 0x4b, 0x8d, 0x61, 0x42, 0xef, 0xc7, 0x9d, 0xec, 0x94, 0x4e, 0xd6, 0x87, - 0x1b, 0xdb, 0xf7, 0x39, 0xf1, 0x1c, 0xb3, 0xbf, 0xed, 0x70, 0xca, 0x8f, 0x31, 0xe9, 0x12, 0x8f, - 0x38, 0x16, 0x41, 0xab, 0x50, 0x70, 0xcc, 0x01, 0x91, 0xe1, 0x28, 0x35, 0xcb, 0x4a, 0x75, 0xa1, - 0x6d, 0x0e, 0x08, 0x96, 0x2b, 0xa8, 0x01, 0x25, 0xf1, 0xeb, 0xbb, 0xa6, 0x45, 0xaa, 0x39, 0xc9, - 0xb6, 0xac, 0xd8, 0x4a, 0xed, 0x70, 0x01, 0xc7, 0x3c, 0xfa, 0x47, 0x1a, 0x54, 0x24, 0xfc, 0xa6, - 0xef, 0x33, 0x8b, 0x9a, 0x9c, 0x32, 0xe7, 0x6a, 0x46, 0x58, 0xc5, 0x54, 0x88, 0xca, 0xff, 0x27, - 0x9e, 0xd6, 0x52, 0x3a, 0xda, 0xa4, 0xb8, 0x6f, 0x6e, 0x66, 0xf4, 0xe3, 0x09, 0x44, 0xfd, 0xdf, - 0x39, 0x98, 0x4f, 0x6c, 0x3e, 0xba, 0x07, 0x79, 0x97, 0xd9, 0xca, 0xe7, 0xa9, 0x8f, 0xe1, 0x1d, - 0x66, 0xc7, 0x66, 0xcc, 0x8e, 0x47, 0xf5, 0xbc, 0xa0, 0x08, 0x8d, 0xe8, 0x07, 0x1a, 0x2c, 0x92, - 0x54, 0x54, 0x65, 0x74, 0xe6, 0x37, 0x76, 0xa6, 0xae, 0xe7, 0xd3, 0x73, 0xa3, 0x89, 0xc6, 0xa3, - 0xfa, 0x62, 0x66, 0x31, 0x03, 0x89, 0x3e, 0x0f, 0x79, 0xea, 0x06, 0x69, 0x5d, 0x6e, 0x3e, 0x23, - 0x0c, 0x6c, 0x75, 0xfc, 0x93, 0x51, 0xbd, 0xd4, 0xea, 0xa8, 0xbb, 0x01, 0x16, 0x0c, 0xe8, 0x1b, - 0x50, 0x74, 0x99, 0xc7, 0xfd, 0x6a, 0x41, 0x46, 0xe4, 0x4b, 0xd3, 0xda, 0x28, 0x32, 0xcd, 0xee, - 0x30, 0x8f, 0xc7, 0x1d, 0x47, 0x3c, 0xf9, 0x38, 0x50, 0xab, 0xff, 0x4a, 0x83, 0xc5, 0x74, 0xd4, - 0xd2, 0x89, 0xab, 0x9d, 0x9f, 0xb8, 0x51, 0x2d, 0xe4, 0xce, 0xac, 0x85, 0x26, 0xe4, 0x87, 0xd4, - 0xae, 0xe6, 0x25, 0xc3, 0x0b, 0x8a, 0x21, 0xff, 0x76, 0x6b, 0xeb, 0x64, 0x54, 0x7f, 0xee, 0xac, - 0x3b, 0x30, 0x3f, 0x76, 0x89, 0x6f, 0xbc, 0xdd, 0xda, 0xc2, 0x42, 0x58, 0xff, 0xad, 0x06, 0xb3, - 0x6a, 0xca, 0xa1, 0x7b, 0x50, 0xb0, 0xa8, 0xed, 0xa9, 0xec, 0x78, 0xc2, 0xb9, 0x1a, 0x19, 0xfa, - 0x46, 0x6b, 0x0b, 0x63, 0xa9, 0x10, 0xbd, 0x03, 0x33, 0xe4, 0xbe, 0x45, 0x5c, 0xae, 0x2a, 0xe0, - 0x09, 0x55, 0x2f, 0x2a, 0xd5, 0x33, 0xdb, 0x52, 0x19, 0x56, 0x4a, 0xf5, 0x2e, 0x14, 0x25, 0x03, - 0xfa, 0x0c, 0xe4, 0xa8, 0x2b, 0xcd, 0x2f, 0x37, 0x57, 0xc6, 0xa3, 0x7a, 0xae, 0xd5, 0x49, 0x07, - 0x3f, 0x47, 0x5d, 0x74, 0x1b, 0xca, 0xae, 0x47, 0xba, 0xf4, 0xfe, 0x5d, 0xe2, 0xf4, 0xf8, 0x81, - 0xdc, 0xdf, 0x62, 0x3c, 0x1b, 0x3b, 0x89, 0x35, 0x9c, 0xe2, 0xd4, 0xdf, 0xd7, 0xa0, 0x14, 0x45, - 0x5e, 0xc4, 0x47, 0x04, 0x5b, 0xc2, 0x15, 0x63, 0xb7, 0xc5, 0x1a, 0x96, 0x2b, 0x8f, 0x11, 0xc1, - 0xdb, 0x30, 0x27, 0xdf, 0x3e, 0x58, 0xac, 0xaf, 0xc2, 0x78, 0x33, 0x9c, 0x94, 0x1d, 0x45, 0x3f, - 0x49, 0xfc, 0xc7, 0x11, 0xb7, 0xfe, 0xaf, 0x3c, 0x2c, 0xb4, 0x09, 0xff, 0x0e, 0xf3, 0x0e, 0x3b, - 0xac, 0x4f, 0xad, 0xe3, 0x2b, 0xe8, 0x69, 0x5d, 0x28, 0x7a, 0xc3, 0x3e, 0x09, 0xfb, 0xd8, 0xe6, - 0xd4, 0x55, 0x93, 0xb4, 0x17, 0x0f, 0xfb, 0x24, 0xae, 0x1e, 0xf1, 0xe4, 0xe3, 0x40, 0x3d, 0x7a, - 0x0d, 0x96, 0xcc, 0xd4, 0xbd, 0x23, 0xa8, 0xe8, 0x92, 0x8c, 0xe9, 0x52, 0xfa, 0x4a, 0xe2, 0xe3, - 0x2c, 0x2f, 0x5a, 0x13, 0x9b, 0x4a, 0x99, 0x27, 0x7a, 0x50, 0x61, 0x55, 0x5b, 0xd3, 0x9a, 0xe5, - 0x60, 0x43, 0x03, 0x1a, 0x8e, 0x56, 0xd1, 0x2d, 0x28, 0x73, 0x4a, 0xbc, 0x70, 0xa5, 0x5a, 0x94, - 0xa1, 0xac, 0x88, 0x34, 0xd8, 0x4b, 0xd0, 0x71, 0x8a, 0x0b, 0xf9, 0x50, 0xf2, 0xd9, 0xd0, 0xb3, - 0x08, 0x26, 0xdd, 0xea, 0x8c, 0xdc, 0xe9, 0x3b, 0x17, 0xdb, 0x8a, 0xa8, 0xc7, 0x2d, 0x88, 0x6e, - 0xb0, 0x1b, 0x2a, 0xc7, 0x31, 0x8e, 0xfe, 0x27, 0x0d, 0x96, 0x53, 0x42, 0x57, 0x70, 0x32, 0xdb, - 0x4f, 0x9f, 0xcc, 0x5e, 0xbb, 0x90, 0x93, 0x67, 0x9c, 0xcd, 0xbe, 0x0b, 0x37, 0x52, 0x6c, 0x6d, - 0x66, 0x93, 0x5d, 0x6e, 0xf2, 0xa1, 0x8f, 0xbe, 0x00, 0x73, 0x0e, 0xb3, 0x49, 0x3b, 0x3e, 0x10, - 0x44, 0xc6, 0xb6, 0x15, 0x1d, 0x47, 0x1c, 0x68, 0x03, 0x40, 0xbd, 0xd2, 0xa3, 0xcc, 0x91, 0x25, - 0x97, 0x8f, 0xd3, 0x79, 0x27, 0x5a, 0xc1, 0x09, 0x2e, 0xfd, 0x8f, 0xd9, 0x4d, 0xed, 0x10, 0xe2, - 0xa1, 0x97, 0x61, 0xc1, 0x4c, 0xbc, 0x48, 0xf2, 0xab, 0x9a, 0x4c, 0xbe, 0xe5, 0xf1, 0xa8, 0xbe, - 0x90, 0x7c, 0xc3, 0xe4, 0xe3, 0x34, 0x1f, 0x22, 0x30, 0x47, 0x5d, 0x75, 0x37, 0x09, 0xb6, 0xec, - 0xe5, 0xe9, 0x1b, 0x9d, 0x94, 0x8f, 0x3d, 0x8d, 0x2e, 0x25, 0x91, 0x6a, 0x54, 0x87, 0x62, 0xf7, - 0xdb, 0xb6, 0x13, 0x16, 0x45, 0x49, 0xec, 0xe9, 0x9d, 0xaf, 0x6e, 0xb5, 0x7d, 0x1c, 0xd0, 0xc5, - 0x65, 0xe5, 0x13, 0xa7, 0x27, 0x18, 0x7a, 0x11, 0x0a, 0x62, 0x00, 0xa8, 0xfd, 0x7c, 0x2e, 0x6c, - 0x49, 0x7b, 0xc7, 0x2e, 0x39, 0x19, 0xd5, 0xd3, 0x9b, 0x21, 0x88, 0x58, 0xb2, 0x4f, 0x7d, 0xea, - 0x8a, 0x5a, 0x5f, 0xfe, 0xbc, 0xe1, 0x55, 0xb8, 0xc8, 0xf0, 0xfa, 0x65, 0x31, 0x13, 0x3f, 0xd1, - 0x46, 0xd0, 0xab, 0x50, 0xb2, 0xa9, 0x27, 0x2e, 0x70, 0xcc, 0x51, 0x8e, 0xd6, 0x42, 0x63, 0xb7, - 0xc2, 0x85, 0x93, 0xe4, 0x03, 0x8e, 0x05, 0x90, 0x05, 0x85, 0xae, 0xc7, 0x06, 0xea, 0xf4, 0x72, - 0xb1, 0x1e, 0x27, 0xd2, 0x29, 0x76, 0xfe, 0x8e, 0xc7, 0x06, 0x58, 0x2a, 0x47, 0xef, 0x40, 0x8e, - 0x33, 0xb9, 0x39, 0x97, 0x02, 0x01, 0x0a, 0x22, 0xb7, 0xc7, 0x70, 0x8e, 0x33, 0x91, 0x88, 0x3e, - 0xf1, 0x8e, 0xa8, 0x45, 0xc2, 0x13, 0xce, 0xd4, 0x89, 0xb8, 0x1b, 0xc8, 0xc7, 0x89, 0xa8, 0x08, - 0x3e, 0x8e, 0x54, 0x8b, 0x02, 0x75, 0x33, 0xad, 0x33, 0x9e, 0x5e, 0x13, 0xcd, 0xf6, 0x1e, 0xcc, - 0x98, 0x41, 0x4c, 0x66, 0x64, 0x4c, 0xbe, 0x22, 0x26, 0xf9, 0x66, 0x18, 0x8c, 0xf5, 0x47, 0x7c, - 0x2b, 0xf1, 0xec, 0xe8, 0xcb, 0x85, 0x21, 0x22, 0x1c, 0x08, 0x61, 0xa5, 0x0e, 0xbd, 0x02, 0x0b, - 0xc4, 0x31, 0xf7, 0xfb, 0xe4, 0x2e, 0xeb, 0xf5, 0xa8, 0xd3, 0xab, 0xce, 0xae, 0x6a, 0x6b, 0x73, - 0xcd, 0xeb, 0xca, 0x96, 0x85, 0xed, 0xe4, 0x22, 0x4e, 0xf3, 0x9e, 0x36, 0x6b, 0xe6, 0xa6, 0x98, - 0x35, 0x61, 0x9e, 0x97, 0xce, 0xca, 0x73, 0xfd, 0x67, 0x79, 0x40, 0xa9, 0x88, 0x89, 0xee, 0xe6, - 0x8b, 0xf3, 0xf2, 0x82, 0x93, 0x24, 0xab, 0xfe, 0x7d, 0x59, 0x93, 0x24, 0xf2, 0x3e, 0xbd, 0x9e, - 0xc6, 0x44, 0x2e, 0x94, 0xb9, 0x67, 0x76, 0xbb, 0xd4, 0x92, 0x56, 0xa9, 0xa4, 0x7f, 0xe9, 0x11, - 0x36, 0xc8, 0x0f, 0x49, 0x46, 0x14, 0x8e, 0xbd, 0x84, 0x74, 0x7c, 0x86, 0x4a, 0x52, 0x71, 0x0a, - 0x01, 0xbd, 0xa7, 0x41, 0x45, 0x4c, 0xf9, 0x24, 0x8b, 0xba, 0x86, 0x7e, 0xf9, 0xf1, 0x61, 0x71, - 0x46, 0x43, 0x7c, 0x27, 0xca, 0xae, 0xe0, 0x09, 0x34, 0xfd, 0x6f, 0x1a, 0xac, 0x4c, 0x44, 0x64, - 0x78, 0x15, 0x6f, 0xd2, 0xfa, 0x50, 0x14, 0xf3, 0x2a, 0x9c, 0x0e, 0x3b, 0x17, 0x8a, 0x75, 0x3c, - 0x29, 0xe3, 0xd1, 0x2a, 0x68, 0x3e, 0x0e, 0x40, 0xf4, 0xdf, 0x15, 0xa0, 0x12, 0x32, 0xf9, 0xbb, - 0xc3, 0xc1, 0xc0, 0xf4, 0xae, 0xe2, 0x94, 0xf8, 0x23, 0x0d, 0x96, 0x92, 0x59, 0x46, 0x23, 0x7f, - 0x9b, 0x17, 0xf2, 0x37, 0x08, 0xf4, 0x0d, 0x85, 0xbd, 0xd4, 0x4e, 0x43, 0xe0, 0x2c, 0x26, 0xfa, - 0xb5, 0x06, 0x37, 0x03, 0x14, 0xf5, 0xda, 0x34, 0x23, 0xa1, 0xb2, 0xee, 0x32, 0x8c, 0xfa, 0xac, - 0x32, 0xea, 0xe6, 0xe6, 0x23, 0xf0, 0xf0, 0x23, 0xad, 0x41, 0xbf, 0xd0, 0xe0, 0x7a, 0xc0, 0x90, - 0xb5, 0xb3, 0x70, 0x69, 0x76, 0x7e, 0x5a, 0xd9, 0x79, 0x7d, 0xf3, 0x34, 0x20, 0x7c, 0x3a, 0xbe, - 0x6e, 0x42, 0x39, 0x79, 0xf1, 0x7f, 0x1a, 0x2f, 0x69, 0x7e, 0xa3, 0xc1, 0xac, 0x1a, 0x30, 0xe8, - 0x56, 0xe2, 0x4e, 0x14, 0x40, 0x54, 0xcf, 0xbf, 0x0f, 0xa1, 0xb6, 0xba, 0x8d, 0xe5, 0xce, 0xc9, - 0xe9, 0x21, 0xa7, 0x7d, 0x23, 0xf8, 0x04, 0x6c, 0xb4, 0x1c, 0xfe, 0x96, 0xb7, 0xcb, 0x3d, 0xea, - 0xf4, 0x9a, 0x73, 0x99, 0xbb, 0xdb, 0xe7, 0x60, 0x96, 0x38, 0xf2, 0xa2, 0x27, 0xc7, 0x74, 0xb1, - 0x39, 0x3f, 0x1e, 0xd5, 0x67, 0xb7, 0x03, 0x12, 0x0e, 0xd7, 0x74, 0x02, 0x15, 0x65, 0xf7, 0xd3, - 0xdc, 0x9f, 0xe6, 0xf3, 0x0f, 0x1e, 0xd6, 0xae, 0x7d, 0xf0, 0xb0, 0x76, 0xed, 0xc3, 0x87, 0xb5, - 0x6b, 0xef, 0x8d, 0x6b, 0xda, 0x83, 0x71, 0x4d, 0xfb, 0x60, 0x5c, 0xd3, 0x3e, 0x1c, 0xd7, 0xb4, - 0xbf, 0x8c, 0x6b, 0xda, 0xcf, 0xff, 0x5a, 0xbb, 0xf6, 0xb5, 0x59, 0x15, 0xfa, 0xff, 0x04, 0x00, - 0x00, 0xff, 0xff, 0xbd, 0x86, 0x85, 0x11, 0x79, 0x20, 0x00, 0x00, + 0x15, 0x4e, 0xfb, 0x27, 0x89, 0x5f, 0x9c, 0xc4, 0xa9, 0xec, 0x30, 0x66, 0x19, 0xec, 0x6c, 0xf3, + 0xa3, 0x1c, 0xd8, 0xf6, 0x26, 0xcc, 0xee, 0x0c, 0xec, 0x0f, 0xc4, 0x9b, 0x4c, 0x64, 0x69, 0xd6, + 0x6b, 0x2a, 0x59, 0x8d, 0x84, 0x58, 0xd8, 0x4e, 0x77, 0xd9, 0x69, 0xd2, 0xee, 0x6a, 0xba, 0xcb, + 0x61, 0x22, 0x24, 0xb4, 0x08, 0x38, 0x2c, 0x20, 0xc1, 0x8d, 0x33, 0x27, 0x2e, 0x9c, 0xb9, 0x73, + 0x40, 0x1a, 0x71, 0x5a, 0x84, 0x10, 0x7b, 0xb2, 0x18, 0x23, 0x40, 0x1c, 0xb8, 0x71, 0xca, 0x5e, + 0x50, 0x55, 0x57, 0xff, 0x3a, 0x9e, 0x8c, 0x27, 0x99, 0x20, 0xb1, 0x7b, 0xb2, 0xbb, 0xea, 0xbd, + 0xf7, 0xbd, 0x57, 0xdf, 0xab, 0xf7, 0xaa, 0xba, 0xe1, 0x35, 0xdd, 0x61, 0x1e, 0xd1, 0x35, 0x8b, + 0x36, 0x82, 0x7f, 0x0d, 0xf7, 0xa8, 0xd7, 0xd0, 0x5d, 0xcb, 0x6f, 0x18, 0xd4, 0x61, 0x1e, 0xb5, + 0x5d, 0x5b, 0x77, 0x48, 0xe3, 0x78, 0xe3, 0x80, 0x30, 0x7d, 0xb3, 0xd1, 0x23, 0x0e, 0xf1, 0x74, + 0x46, 0x4c, 0xcd, 0xf5, 0x28, 0xa3, 0x48, 0x0b, 0xb4, 0xbe, 0x65, 0x51, 0xf9, 0x4f, 0x73, 0x8f, + 0x7a, 0x1a, 0xd7, 0xd7, 0x92, 0xfa, 0x9a, 0xd4, 0x7f, 0xf6, 0xf6, 0x64, 0x3c, 0x9f, 0xe9, 0xcc, + 0x6f, 0x1c, 0x6f, 0xe8, 0xb6, 0x7b, 0xa8, 0x6f, 0x64, 0x91, 0x9e, 0x7d, 0xbe, 0x67, 0xb1, 0xc3, + 0xc1, 0x81, 0x66, 0xd0, 0x7e, 0xa3, 0x47, 0x7b, 0xb4, 0x21, 0x86, 0x0f, 0x06, 0x5d, 0xf1, 0x24, + 0x1e, 0xc4, 0x3f, 0x29, 0x7e, 0xf3, 0xe8, 0xb6, 0x2f, 0x50, 0x5c, 0xab, 0xaf, 0x1b, 0x87, 0x96, + 0x43, 0xbc, 0x93, 0x18, 0xab, 0x4f, 0x98, 0xde, 0x38, 0x1e, 0x07, 0x69, 0x4c, 0xd2, 0xf2, 0x06, + 0x0e, 0xb3, 0xfa, 0x64, 0x4c, 0xe1, 0xa5, 0xf3, 0x14, 0x7c, 0xe3, 0x90, 0xf4, 0xf5, 0x31, 0xbd, + 0x2f, 0x4e, 0xd2, 0x1b, 0x30, 0xcb, 0x6e, 0x58, 0x0e, 0xf3, 0x99, 0x97, 0x55, 0x52, 0xff, 0xa9, + 0x40, 0x79, 0xcb, 0x34, 0x3d, 0xe2, 0xfb, 0xbb, 0x1e, 0x1d, 0xb8, 0xe8, 0x1d, 0x98, 0xe7, 0x91, + 0x98, 0x3a, 0xd3, 0xab, 0xca, 0x9a, 0xb2, 0xbe, 0xb0, 0xf9, 0x82, 0x16, 0x18, 0xd6, 0x92, 0x86, + 0x63, 0x4e, 0xb8, 0xb4, 0x76, 0xbc, 0xa1, 0xbd, 0x79, 0xf0, 0x6d, 0x62, 0xb0, 0x37, 0x08, 0xd3, + 0x9b, 0xe8, 0xc1, 0xb0, 0x3e, 0x33, 0x1a, 0xd6, 0x21, 0x1e, 0xc3, 0x91, 0x55, 0x34, 0x80, 0x72, + 0x8f, 0x43, 0xbd, 0x41, 0xfa, 0x07, 0xc4, 0xf3, 0xab, 0xb9, 0xb5, 0xfc, 0xfa, 0xc2, 0xe6, 0xcb, + 0x53, 0xd2, 0xae, 0xed, 0xc6, 0x36, 0x9a, 0xcf, 0x48, 0xc0, 0x72, 0x62, 0xd0, 0xc7, 0x29, 0x18, + 0xf5, 0x4f, 0x0a, 0x54, 0x92, 0x91, 0xde, 0xb5, 0x7c, 0x86, 0xbe, 0x31, 0x16, 0xad, 0xf6, 0x78, + 0xd1, 0x72, 0x6d, 0x11, 0x6b, 0x45, 0x42, 0xcf, 0x87, 0x23, 0x89, 0x48, 0x75, 0x28, 0x5a, 0x8c, + 0xf4, 0xc3, 0x10, 0x5f, 0x99, 0x36, 0xc4, 0xa4, 0xbb, 0xcd, 0x45, 0x09, 0x54, 0x6c, 0x71, 0x93, + 0x38, 0xb0, 0xac, 0xbe, 0x97, 0x87, 0x95, 0xa4, 0x58, 0x47, 0x67, 0xc6, 0xe1, 0x15, 0x90, 0xf8, + 0x23, 0x05, 0x56, 0x74, 0xd3, 0x24, 0xe6, 0xee, 0x25, 0x53, 0xf9, 0x49, 0x09, 0xcb, 0xa3, 0x4a, + 0x5b, 0xc7, 0xe3, 0x80, 0xe8, 0x27, 0x0a, 0xac, 0x7a, 0xa4, 0x4f, 0x8f, 0x33, 0x8e, 0xe4, 0x2f, + 0xee, 0xc8, 0xa7, 0xa4, 0x23, 0xab, 0x78, 0xdc, 0x3e, 0x3e, 0x0b, 0x54, 0xfd, 0x97, 0x02, 0x4b, + 0x5b, 0xae, 0x6b, 0x5b, 0xc4, 0xdc, 0xa7, 0xff, 0xe7, 0xbb, 0xe9, 0x2f, 0x0a, 0xa0, 0x74, 0xac, + 0x57, 0xb0, 0x9f, 0x8c, 0xf4, 0x7e, 0x7a, 0x6d, 0xea, 0xfd, 0x94, 0x72, 0x78, 0xc2, 0x8e, 0xfa, + 0x69, 0x1e, 0x56, 0xd3, 0x82, 0x1f, 0xef, 0xa9, 0xff, 0xdd, 0x9e, 0xfa, 0x30, 0x07, 0xab, 0xaf, + 0xdb, 0x03, 0x9f, 0x11, 0x2f, 0xe5, 0xe4, 0xd3, 0x67, 0xe3, 0x07, 0x0a, 0x54, 0x48, 0xb7, 0x4b, + 0x0c, 0x66, 0x1d, 0x93, 0x4b, 0x24, 0xa3, 0x2a, 0x51, 0x2b, 0x3b, 0x19, 0xe3, 0x78, 0x0c, 0x0e, + 0x7d, 0x1f, 0x56, 0xa2, 0xb1, 0x56, 0xa7, 0x69, 0x53, 0xe3, 0x28, 0xe4, 0xe1, 0xc5, 0x69, 0x7d, + 0x68, 0x75, 0xda, 0x84, 0xc5, 0xa9, 0xb0, 0x93, 0xb5, 0x8b, 0xc7, 0xa1, 0xd4, 0x7f, 0x28, 0xb0, + 0xb0, 0xd3, 0xfb, 0x08, 0x1c, 0x0e, 0xfe, 0xa8, 0xc0, 0x72, 0x22, 0xd0, 0x2b, 0xa8, 0x65, 0xef, + 0xa4, 0x6b, 0xd9, 0xd4, 0x11, 0x26, 0xbc, 0x9d, 0x50, 0xc8, 0x7e, 0x96, 0x87, 0x4a, 0x42, 0x2a, + 0xa8, 0x62, 0x26, 0x00, 0x8d, 0xd6, 0xfd, 0x52, 0x39, 0x4c, 0xd8, 0xfd, 0xb8, 0x92, 0x9d, 0x51, + 0xc9, 0x6c, 0xb8, 0xbe, 0x73, 0x9f, 0x11, 0xcf, 0xd1, 0xed, 0x1d, 0x87, 0x59, 0xec, 0x04, 0x93, + 0x2e, 0xf1, 0x88, 0x63, 0x10, 0xb4, 0x06, 0x05, 0x47, 0xef, 0x13, 0x41, 0x47, 0xa9, 0x59, 0x96, + 0xa6, 0x0b, 0x6d, 0xbd, 0x4f, 0xb0, 0x98, 0x41, 0x0d, 0x28, 0xf1, 0x5f, 0xdf, 0xd5, 0x0d, 0x52, + 0xcd, 0x09, 0xb1, 0x15, 0x29, 0x56, 0x6a, 0x87, 0x13, 0x38, 0x96, 0x51, 0x3f, 0x54, 0xa0, 0x22, + 0xe0, 0xb7, 0x7c, 0x9f, 0x1a, 0x96, 0xce, 0x2c, 0xea, 0x5c, 0x4d, 0x0b, 0xab, 0xe8, 0x12, 0x51, + 0xc6, 0xff, 0xc4, 0xdd, 0x5a, 0x68, 0x47, 0x8b, 0x14, 0xd7, 0xcd, 0xad, 0x8c, 0x7d, 0x3c, 0x86, + 0xa8, 0xfe, 0x27, 0x07, 0x0b, 0x89, 0xc5, 0x47, 0xf7, 0x20, 0xef, 0x52, 0x53, 0xc6, 0x3c, 0xf5, + 0x31, 0xbc, 0x43, 0xcd, 0xd8, 0x8d, 0xb9, 0xd1, 0xb0, 0x9e, 0xe7, 0x23, 0xdc, 0x22, 0xfa, 0xa1, + 0x02, 0x4b, 0x24, 0xc5, 0xaa, 0x60, 0x67, 0x61, 0x73, 0x77, 0xea, 0xfd, 0x7c, 0x76, 0x6e, 0x34, + 0xd1, 0x68, 0x58, 0x5f, 0xca, 0x4c, 0x66, 0x20, 0xd1, 0xe7, 0x21, 0x6f, 0xb9, 0x41, 0x5a, 0x97, + 0x9b, 0xcf, 0x70, 0x07, 0x5b, 0x1d, 0xff, 0x74, 0x58, 0x2f, 0xb5, 0x3a, 0xf2, 0x6e, 0x80, 0xb9, + 0x00, 0xfa, 0x26, 0x14, 0x5d, 0xea, 0x31, 0xbf, 0x5a, 0x10, 0x8c, 0x7c, 0x69, 0x5a, 0x1f, 0x79, + 0xa6, 0x99, 0x1d, 0xea, 0xb1, 0xb8, 0xe2, 0xf0, 0x27, 0x1f, 0x07, 0x66, 0xd5, 0x5f, 0x2b, 0xb0, + 0x94, 0x66, 0x2d, 0x9d, 0xb8, 0xca, 0xf9, 0x89, 0x1b, 0xed, 0x85, 0xdc, 0xc4, 0xbd, 0xd0, 0x84, + 0xfc, 0xc0, 0x32, 0xab, 0x79, 0x21, 0xf0, 0x82, 0x14, 0xc8, 0xbf, 0xd5, 0xda, 0x3e, 0x1d, 0xd6, + 0x9f, 0x9b, 0x74, 0x07, 0x66, 0x27, 0x2e, 0xf1, 0xb5, 0xb7, 0x5a, 0xdb, 0x98, 0x2b, 0xab, 0xbf, + 0x53, 0x60, 0x4e, 0x76, 0x39, 0x74, 0x0f, 0x0a, 0x86, 0x65, 0x7a, 0x32, 0x3b, 0x9e, 0xb0, 0xaf, + 0x46, 0x8e, 0xbe, 0xde, 0xda, 0xc6, 0x58, 0x18, 0x44, 0x6f, 0xc3, 0x2c, 0xb9, 0x6f, 0x10, 0x97, + 0xc9, 0x1d, 0xf0, 0x84, 0xa6, 0x97, 0xa4, 0xe9, 0xd9, 0x1d, 0x61, 0x0c, 0x4b, 0xa3, 0x6a, 0x17, + 0x8a, 0x42, 0x00, 0x7d, 0x06, 0x72, 0x96, 0x2b, 0xdc, 0x2f, 0x37, 0x57, 0x47, 0xc3, 0x7a, 0xae, + 0xd5, 0x49, 0x93, 0x9f, 0xb3, 0x5c, 0x74, 0x1b, 0xca, 0xae, 0x47, 0xba, 0xd6, 0xfd, 0xbb, 0xc4, + 0xe9, 0xb1, 0x43, 0xb1, 0xbe, 0xc5, 0xb8, 0x37, 0x76, 0x12, 0x73, 0x38, 0x25, 0xa9, 0xbe, 0xa7, + 0x40, 0x29, 0x62, 0x9e, 0xf3, 0xc3, 0xc9, 0x16, 0x70, 0xc5, 0x38, 0x6c, 0x3e, 0x87, 0xc5, 0xcc, + 0x63, 0x30, 0x78, 0x1b, 0xe6, 0xc5, 0xdb, 0x07, 0x83, 0xda, 0x92, 0xc6, 0x1b, 0x61, 0xa7, 0xec, + 0xc8, 0xf1, 0xd3, 0xc4, 0x7f, 0x1c, 0x49, 0xab, 0xff, 0xce, 0xc3, 0x62, 0x9b, 0xb0, 0xef, 0x52, + 0xef, 0xa8, 0x43, 0x6d, 0xcb, 0x38, 0xb9, 0x82, 0x9a, 0xd6, 0x85, 0xa2, 0x37, 0xb0, 0x49, 0x58, + 0xc7, 0xb6, 0xa6, 0xde, 0x35, 0x49, 0x7f, 0xf1, 0xc0, 0x26, 0xf1, 0xee, 0xe1, 0x4f, 0x3e, 0x0e, + 0xcc, 0xa3, 0x57, 0x61, 0x59, 0x4f, 0xdd, 0x3b, 0x82, 0x1d, 0x5d, 0x12, 0x9c, 0x2e, 0xa7, 0xaf, + 0x24, 0x3e, 0xce, 0xca, 0xa2, 0x75, 0xbe, 0xa8, 0x16, 0xf5, 0x78, 0x0d, 0x2a, 0xac, 0x29, 0xeb, + 0x4a, 0xb3, 0x1c, 0x2c, 0x68, 0x30, 0x86, 0xa3, 0x59, 0x74, 0x13, 0xca, 0xcc, 0x22, 0x5e, 0x38, + 0x53, 0x2d, 0x0a, 0x2a, 0x2b, 0x3c, 0x0d, 0xf6, 0x13, 0xe3, 0x38, 0x25, 0x85, 0x7c, 0x28, 0xf9, + 0x74, 0xe0, 0x19, 0x04, 0x93, 0x6e, 0x75, 0x56, 0xac, 0xf4, 0x9d, 0x8b, 0x2d, 0x45, 0x54, 0xe3, + 0x16, 0x79, 0x35, 0xd8, 0x0b, 0x8d, 0xe3, 0x18, 0x47, 0xfd, 0xb3, 0x02, 0x2b, 0x29, 0xa5, 0x2b, + 0x38, 0x99, 0x1d, 0xa4, 0x4f, 0x66, 0xaf, 0x5e, 0x28, 0xc8, 0x09, 0x67, 0xb3, 0xef, 0xc1, 0xf5, + 0x94, 0x58, 0x9b, 0x9a, 0x64, 0x8f, 0xe9, 0x6c, 0xe0, 0xa3, 0x2f, 0xc0, 0xbc, 0x43, 0x4d, 0xd2, + 0x8e, 0x0f, 0x04, 0x91, 0xb3, 0x6d, 0x39, 0x8e, 0x23, 0x09, 0xb4, 0x09, 0x20, 0x5f, 0xe9, 0x59, + 0xd4, 0x11, 0x5b, 0x2e, 0x1f, 0xa7, 0xf3, 0x6e, 0x34, 0x83, 0x13, 0x52, 0xea, 0x1f, 0x72, 0x99, + 0x45, 0xed, 0x10, 0xe2, 0xa1, 0x5b, 0xb0, 0xa8, 0x27, 0x5e, 0x24, 0xf9, 0x55, 0x45, 0x24, 0xdf, + 0xca, 0x68, 0x58, 0x5f, 0x4c, 0xbe, 0x61, 0xf2, 0x71, 0x5a, 0x0e, 0x11, 0x98, 0xb7, 0x5c, 0x79, + 0x37, 0x09, 0x96, 0xec, 0xd6, 0xf4, 0x85, 0x4e, 0xe8, 0xc7, 0x91, 0x46, 0x97, 0x92, 0xc8, 0x34, + 0xaa, 0x43, 0xb1, 0xfb, 0x1d, 0xd3, 0x09, 0x37, 0x45, 0x89, 0xaf, 0xe9, 0x9d, 0xaf, 0x6d, 0xb7, + 0x7d, 0x1c, 0x8c, 0x23, 0x06, 0xc0, 0xe8, 0x1e, 0xf1, 0x8e, 0x2d, 0x83, 0x84, 0x2d, 0xee, 0xab, + 0xd3, 0x7a, 0x22, 0xf5, 0x13, 0xfd, 0x37, 0x5c, 0xcc, 0xfd, 0xc8, 0x36, 0x4e, 0xe0, 0xf0, 0x2b, + 0xd2, 0x27, 0xce, 0x4e, 0x6b, 0xf4, 0x22, 0x14, 0x78, 0xdb, 0x91, 0x2c, 0x3e, 0x17, 0x16, 0xc2, + 0xfd, 0x13, 0x97, 0x9c, 0x0e, 0xeb, 0x69, 0x0a, 0xf8, 0x20, 0x16, 0xe2, 0x53, 0x9f, 0xf5, 0xa2, + 0x82, 0x9b, 0x3f, 0xaf, 0x65, 0x16, 0x2e, 0xd2, 0x32, 0x7f, 0x55, 0xcc, 0x64, 0x0d, 0x2f, 0x5e, + 0xe8, 0x15, 0x28, 0x99, 0x96, 0xc7, 0xaf, 0x8d, 0xd4, 0x91, 0x81, 0xd6, 0x42, 0x67, 0xb7, 0xc3, + 0x89, 0xd3, 0xe4, 0x03, 0x8e, 0x15, 0x90, 0x01, 0x85, 0xae, 0x47, 0xfb, 0xf2, 0xcc, 0x74, 0xb1, + 0xca, 0xca, 0x93, 0x38, 0x0e, 0xfe, 0x8e, 0x47, 0xfb, 0x58, 0x18, 0x47, 0x6f, 0x43, 0x8e, 0x51, + 0xb1, 0x38, 0x97, 0x02, 0x01, 0x12, 0x22, 0xb7, 0x4f, 0x71, 0x8e, 0x51, 0x9e, 0xfe, 0x7e, 0x3a, + 0xe9, 0x6e, 0x3d, 0x61, 0xd2, 0xc5, 0xe9, 0x1f, 0x65, 0x5a, 0x64, 0x9a, 0x97, 0x05, 0x37, 0x53, + 0xb0, 0xe3, 0x9e, 0x39, 0x56, 0xe2, 0xef, 0xc1, 0xac, 0x1e, 0x70, 0x32, 0x2b, 0x38, 0xf9, 0x0a, + 0x3f, 0x3f, 0x6c, 0x85, 0x64, 0x6c, 0x3c, 0xe2, 0x0b, 0x8d, 0x67, 0x46, 0xdf, 0x4b, 0x34, 0xce, + 0x70, 0xa0, 0x84, 0xa5, 0x39, 0xf4, 0x32, 0x2c, 0x12, 0x47, 0x3f, 0xb0, 0xc9, 0x5d, 0xda, 0xeb, + 0x59, 0x4e, 0xaf, 0x3a, 0xb7, 0xa6, 0xac, 0xcf, 0x37, 0xaf, 0x49, 0x5f, 0x16, 0x77, 0x92, 0x93, + 0x38, 0x2d, 0x7b, 0x56, 0x87, 0x9b, 0x9f, 0xa2, 0xc3, 0x85, 0x79, 0x5e, 0x9a, 0x94, 0xe7, 0xea, + 0xcf, 0xf3, 0x80, 0x52, 0x8c, 0xf1, 0x9a, 0xea, 0xf3, 0x53, 0xfa, 0xa2, 0x93, 0x1c, 0x96, 0x5d, + 0xe3, 0xb2, 0xfa, 0x57, 0x14, 0x7d, 0x7a, 0x3e, 0x8d, 0x89, 0x5c, 0x28, 0x33, 0x4f, 0xef, 0x76, + 0x2d, 0x43, 0x78, 0x25, 0x93, 0xfe, 0xa5, 0x47, 0xf8, 0x20, 0x3e, 0x5f, 0x69, 0x11, 0x1d, 0xfb, + 0x09, 0xed, 0xf8, 0xe4, 0x96, 0x1c, 0xc5, 0x29, 0x04, 0xf4, 0xae, 0x02, 0x15, 0x7e, 0xb6, 0x48, + 0x8a, 0xc8, 0xcb, 0xef, 0x97, 0x1f, 0x1f, 0x16, 0x67, 0x2c, 0xc4, 0x37, 0xb1, 0xec, 0x0c, 0x1e, + 0x43, 0x53, 0xff, 0xae, 0xc0, 0xea, 0x18, 0x23, 0x83, 0xab, 0x78, 0x7f, 0x67, 0x43, 0x91, 0x77, + 0xc9, 0xb0, 0x27, 0xed, 0x5e, 0x88, 0xeb, 0xb8, 0x3f, 0xc7, 0x0d, 0x9d, 0x8f, 0xf9, 0x38, 0x00, + 0x51, 0x7f, 0x5f, 0x80, 0x4a, 0x28, 0xe4, 0xef, 0x0d, 0xfa, 0x7d, 0xdd, 0xbb, 0x8a, 0xb3, 0xe9, + 0x8f, 0x15, 0x58, 0x4e, 0x66, 0x99, 0x15, 0xc5, 0xdb, 0xbc, 0x50, 0xbc, 0x01, 0xd1, 0xd7, 0x25, + 0xf6, 0x72, 0x3b, 0x0d, 0x81, 0xb3, 0x98, 0xe8, 0x37, 0x0a, 0xdc, 0x08, 0x50, 0xe4, 0xcb, 0xda, + 0x8c, 0x86, 0xcc, 0xba, 0xcb, 0x70, 0xea, 0xb3, 0xd2, 0xa9, 0x1b, 0x5b, 0x8f, 0xc0, 0xc3, 0x8f, + 0xf4, 0x06, 0xfd, 0x52, 0x81, 0x6b, 0x81, 0x40, 0xd6, 0xcf, 0xc2, 0xa5, 0xf9, 0xf9, 0x69, 0xe9, + 0xe7, 0xb5, 0xad, 0xb3, 0x80, 0xf0, 0xd9, 0xf8, 0xaa, 0x0e, 0xe5, 0xe4, 0xeb, 0x86, 0xa7, 0xf1, + 0x6a, 0xe8, 0xb7, 0x0a, 0xcc, 0xc9, 0x06, 0x83, 0x6e, 0x26, 0x6e, 0x62, 0x01, 0x44, 0xf5, 0xfc, + 0x5b, 0x18, 0x6a, 0xcb, 0x3b, 0x60, 0xee, 0x9c, 0x9c, 0x1e, 0x30, 0xcb, 0xd6, 0x82, 0x0f, 0xcf, + 0x5a, 0xcb, 0x61, 0x6f, 0x7a, 0x7b, 0xcc, 0xb3, 0x9c, 0x5e, 0x73, 0x3e, 0x73, 0x63, 0xfc, 0x1c, + 0xcc, 0x11, 0x47, 0x5c, 0x2f, 0x45, 0x9b, 0x2e, 0x36, 0x17, 0x46, 0xc3, 0xfa, 0xdc, 0x4e, 0x30, + 0x84, 0xc3, 0x39, 0x95, 0x40, 0x25, 0x7b, 0x3c, 0x7b, 0x0a, 0xeb, 0xd3, 0x7c, 0xfe, 0xc1, 0xc3, + 0xda, 0xcc, 0xfb, 0x0f, 0x6b, 0x33, 0x1f, 0x3c, 0xac, 0xcd, 0xbc, 0x3b, 0xaa, 0x29, 0x0f, 0x46, + 0x35, 0xe5, 0xfd, 0x51, 0x4d, 0xf9, 0x60, 0x54, 0x53, 0xfe, 0x3a, 0xaa, 0x29, 0xbf, 0xf8, 0x5b, + 0x6d, 0xe6, 0xeb, 0x73, 0x92, 0xfa, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xbe, 0xe6, 0x40, 0xcb, + 0xef, 0x20, 0x00, 0x00, } func (m *AddressGroup) Marshal() (dAtA []byte, err error) { @@ -2023,6 +2025,20 @@ func (m *NetworkPolicyPeer) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.ToServices) > 0 { + for iNdEx := len(m.ToServices) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.ToServices[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } + } if len(m.FQDNs) > 0 { for iNdEx := len(m.FQDNs) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.FQDNs[iNdEx]) @@ -2897,6 +2913,12 @@ func (m *NetworkPolicyPeer) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) } } + if len(m.ToServices) > 0 { + for _, e := range m.ToServices { + l = e.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + } return n } @@ -3405,10 +3427,16 @@ func (this *NetworkPolicyPeer) String() string { repeatedStringForIPBlocks += strings.Replace(strings.Replace(f.String(), "IPBlock", "IPBlock", 1), `&`, ``, 1) + "," } repeatedStringForIPBlocks += "}" + repeatedStringForToServices := "[]ServiceReference{" + for _, f := range this.ToServices { + repeatedStringForToServices += strings.Replace(strings.Replace(f.String(), "ServiceReference", "ServiceReference", 1), `&`, ``, 1) + "," + } + repeatedStringForToServices += "}" s := strings.Join([]string{`&NetworkPolicyPeer{`, `AddressGroups:` + fmt.Sprintf("%v", this.AddressGroups) + `,`, `IPBlocks:` + repeatedStringForIPBlocks + `,`, `FQDNs:` + fmt.Sprintf("%v", this.FQDNs) + `,`, + `ToServices:` + repeatedStringForToServices + `,`, `}`, }, "") return s @@ -6338,6 +6366,40 @@ func (m *NetworkPolicyPeer) Unmarshal(dAtA []byte) error { } m.FQDNs = append(m.FQDNs, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ToServices", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ToServices = append(m.ToServices, ServiceReference{}) + if err := m.ToServices[len(m.ToServices)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/pkg/apis/controlplane/v1beta2/generated.proto b/pkg/apis/controlplane/v1beta2/generated.proto index 63e9cdc9890..497c46b9337 100644 --- a/pkg/apis/controlplane/v1beta2/generated.proto +++ b/pkg/apis/controlplane/v1beta2/generated.proto @@ -232,6 +232,10 @@ message NetworkPolicyPeer { // A list of exact FQDN names or FQDN wildcard expressions. // This field can only be possibly set for NetworkPolicyPeer of egress rules. repeated string fqdns = 3; + + // A list of ServiceReference. + // This field can only be possibly set for NetworkPolicyPeer of egress rules. + repeated ServiceReference toServices = 4; } message NetworkPolicyReference { diff --git a/pkg/apis/controlplane/v1beta2/types.go b/pkg/apis/controlplane/v1beta2/types.go index e717fb01937..ec15a700e03 100644 --- a/pkg/apis/controlplane/v1beta2/types.go +++ b/pkg/apis/controlplane/v1beta2/types.go @@ -272,6 +272,9 @@ type NetworkPolicyPeer struct { // A list of exact FQDN names or FQDN wildcard expressions. // This field can only be possibly set for NetworkPolicyPeer of egress rules. FQDNs []string `json:"fqdns,omitempty" protobuf:"bytes,3,rep,name=fqdns"` + // A list of ServiceReference. + // This field can only be possibly set for NetworkPolicyPeer of egress rules. + ToServices []ServiceReference `json:"toServices,omitempty" protobuf:"bytes,4,rep,name=toServices"` } // IPBlock describes a particular CIDR (Ex. "192.168.1.1/24"). The except entry describes CIDRs that should diff --git a/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go b/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go index 21994ec2d0b..351f9e867fc 100644 --- a/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go +++ b/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go @@ -843,6 +843,7 @@ func autoConvert_v1beta2_NetworkPolicyPeer_To_controlplane_NetworkPolicyPeer(in out.AddressGroups = *(*[]string)(unsafe.Pointer(&in.AddressGroups)) out.IPBlocks = *(*[]controlplane.IPBlock)(unsafe.Pointer(&in.IPBlocks)) out.FQDNs = *(*[]string)(unsafe.Pointer(&in.FQDNs)) + out.ToServices = *(*[]controlplane.ServiceReference)(unsafe.Pointer(&in.ToServices)) return nil } @@ -855,6 +856,7 @@ func autoConvert_controlplane_NetworkPolicyPeer_To_v1beta2_NetworkPolicyPeer(in out.AddressGroups = *(*[]string)(unsafe.Pointer(&in.AddressGroups)) out.IPBlocks = *(*[]IPBlock)(unsafe.Pointer(&in.IPBlocks)) out.FQDNs = *(*[]string)(unsafe.Pointer(&in.FQDNs)) + out.ToServices = *(*[]ServiceReference)(unsafe.Pointer(&in.ToServices)) return nil } diff --git a/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go b/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go index b27a5afda11..17c4b00a365 100644 --- a/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go +++ b/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go @@ -692,6 +692,11 @@ func (in *NetworkPolicyPeer) DeepCopyInto(out *NetworkPolicyPeer) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.ToServices != nil { + in, out := &in.ToServices, &out.ToServices + *out = make([]ServiceReference, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/controlplane/zz_generated.deepcopy.go b/pkg/apis/controlplane/zz_generated.deepcopy.go index 6e81d5122bf..829e0756c2b 100644 --- a/pkg/apis/controlplane/zz_generated.deepcopy.go +++ b/pkg/apis/controlplane/zz_generated.deepcopy.go @@ -692,6 +692,11 @@ func (in *NetworkPolicyPeer) DeepCopyInto(out *NetworkPolicyPeer) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.ToServices != nil { + in, out := &in.ToServices, &out.ToServices + *out = make([]ServiceReference, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/crd/v1alpha1/types.go b/pkg/apis/crd/v1alpha1/types.go index 2c5e8f6928a..62114dcedb8 100644 --- a/pkg/apis/crd/v1alpha1/types.go +++ b/pkg/apis/crd/v1alpha1/types.go @@ -354,10 +354,17 @@ type Rule struct { // +optional From []NetworkPolicyPeer `json:"from"` // Rule is matched if traffic is intended for workloads selected by - // this field. If this field is empty or missing, this rule matches all - // destinations. + // this field. This field can't be used with ToServices. If this field + // and ToServices are both empty or missing this rule matches all destinations. // +optional To []NetworkPolicyPeer `json:"to"` + // Rule is matched if traffic is intended for a Service listed in this field. + // Currently only ClusterIP types Services are supported in this field. This field + // can only be used when AntreaProxy is enabled. This field can't be used with To + // or Ports. If this field and To are both empty or missing, this rule matches all + // destinations. + // +optional + ToServices []ServiceReference `json:"toServices,omitempty"` // Name describes the intention of this rule. // Name should be unique within the policy. // +optional @@ -457,6 +464,14 @@ type NetworkPolicyPort struct { EndPort *int32 `json:"endPort,omitempty"` } +// ServiceReference represents a reference to a v1.Service. +type ServiceReference struct { + // Name of the Service + Name string `json:"name"` + // Namespace of the Service + Namespace string `json:"namespace,omitempty"` +} + // RuleAction describes the action to be applied on traffic matching a rule. type RuleAction string diff --git a/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go index 870173cdd18..a2a892fd7df 100644 --- a/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go @@ -500,6 +500,11 @@ func (in *Rule) DeepCopyInto(out *Rule) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ToServices != nil { + in, out := &in.ToServices, &out.ToServices + *out = make([]ServiceReference, len(*in)) + copy(*out, *in) + } if in.AppliedTo != nil { in, out := &in.AppliedTo, &out.AppliedTo *out = make([]NetworkPolicyPeer, len(*in)) @@ -520,6 +525,22 @@ func (in *Rule) DeepCopy() *Rule { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceReference) DeepCopyInto(out *ServiceReference) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceReference. +func (in *ServiceReference) DeepCopy() *ServiceReference { + if in == nil { + return nil + } + out := new(ServiceReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Source) DeepCopyInto(out *Source) { *out = *in diff --git a/pkg/apiserver/openapi/zz_generated.openapi.go b/pkg/apiserver/openapi/zz_generated.openapi.go index 6cb88e92aa3..96844d0c14a 100644 --- a/pkg/apiserver/openapi/zz_generated.openapi.go +++ b/pkg/apiserver/openapi/zz_generated.openapi.go @@ -1340,11 +1340,25 @@ func schema_pkg_apis_controlplane_v1beta2_NetworkPolicyPeer(ref common.Reference }, }, }, + "toServices": { + SchemaProps: spec.SchemaProps{ + Description: "A list of ServiceReference. This field can only be possibly set for NetworkPolicyPeer of egress rules.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("antrea.io/antrea/pkg/apis/controlplane/v1beta2.ServiceReference"), + }, + }, + }, + }, + }, }, }, }, Dependencies: []string{ - "antrea.io/antrea/pkg/apis/controlplane/v1beta2.IPBlock"}, + "antrea.io/antrea/pkg/apis/controlplane/v1beta2.IPBlock", "antrea.io/antrea/pkg/apis/controlplane/v1beta2.ServiceReference"}, } } diff --git a/pkg/controller/networkpolicy/antreanetworkpolicy.go b/pkg/controller/networkpolicy/antreanetworkpolicy.go index 4b36190564d..eb0f4d5c54f 100644 --- a/pkg/controller/networkpolicy/antreanetworkpolicy.go +++ b/pkg/controller/networkpolicy/antreanetworkpolicy.go @@ -168,9 +168,15 @@ func (n *NetworkPolicyController) processAntreaNetworkPolicy(np *crdv1alpha1.Net appliedToGroupNamesForRule = append(appliedToGroupNamesForRule, atGroup) appliedToGroupNamesSet.Insert(atGroup) } + var peers *controlplane.NetworkPolicyPeer + if egressRule.ToServices != nil { + peers = n.svcRefToPeerForCRD(egressRule.ToServices, np.Namespace) + } else { + peers = n.toAntreaPeerForCRD(egressRule.To, np, controlplane.DirectionOut, namedPortExists) + } rules = append(rules, controlplane.NetworkPolicyRule{ Direction: controlplane.DirectionOut, - To: *n.toAntreaPeerForCRD(egressRule.To, np, controlplane.DirectionOut, namedPortExists), + To: *peers, Services: services, Name: egressRule.Name, Action: egressRule.Action, diff --git a/pkg/controller/networkpolicy/antreanetworkpolicy_test.go b/pkg/controller/networkpolicy/antreanetworkpolicy_test.go index d15a6a9fe0c..35d4f181c42 100644 --- a/pkg/controller/networkpolicy/antreanetworkpolicy_test.go +++ b/pkg/controller/networkpolicy/antreanetworkpolicy_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -35,6 +36,20 @@ var ( func TestProcessAntreaNetworkPolicy(t *testing.T) { p10 := float64(10) + svcA := v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc1", + Namespace: "ns5", + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Port: 80, + TargetPort: int80, + }, + }, + }, + } allowAction := crdv1alpha1.RuleActionAllow protocolTCP := controlplane.ProtocolTCP tests := []struct { @@ -379,10 +394,64 @@ func TestProcessAntreaNetworkPolicy(t *testing.T) { expectedAppliedToGroups: 1, expectedAddressGroups: 1, }, + { + name: "rules-with-to-services", + inputPolicy: &crdv1alpha1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns5", Name: "npE", UID: "uidE"}, + Spec: crdv1alpha1.NetworkPolicySpec{ + AppliedTo: []crdv1alpha1.NetworkPolicyPeer{ + {PodSelector: &selectorA}, + }, + Priority: p10, + Egress: []crdv1alpha1.Rule{ + { + ToServices: []crdv1alpha1.ServiceReference{ + { + Namespace: "ns5", + Name: "svc1", + }, + }, + Action: &allowAction, + }, + }, + }, + }, + expectedPolicy: &antreatypes.NetworkPolicy{ + UID: "uidE", + Name: "uidE", + SourceRef: &controlplane.NetworkPolicyReference{ + Type: controlplane.AntreaNetworkPolicy, + Namespace: "ns5", + Name: "npE", + UID: "uidE", + }, + Priority: &p10, + TierPriority: &DefaultTierPriority, + Rules: []controlplane.NetworkPolicyRule{ + { + Direction: controlplane.DirectionOut, + To: controlplane.NetworkPolicyPeer{ + ToServices: []controlplane.ServiceReference{ + { + Namespace: "ns5", + Name: "svc1", + }, + }, + }, + Priority: 0, + Action: &allowAction, + }, + }, + AppliedToGroups: []string{getNormalizedUID(toGroupSelector("ns5", &selectorA, nil, nil).NormalizedName)}, + }, + expectedAppliedToGroups: 1, + expectedAddressGroups: 0, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { _, c := newController() + c.serviceStore.Add(&svcA) assert.Equal(t, tt.expectedPolicy, c.processAntreaNetworkPolicy(tt.inputPolicy)) assert.Equal(t, tt.expectedAddressGroups, len(c.addressGroupStore.List())) assert.Equal(t, tt.expectedAppliedToGroups, len(c.appliedToGroupStore.List())) diff --git a/pkg/controller/networkpolicy/clusternetworkpolicy.go b/pkg/controller/networkpolicy/clusternetworkpolicy.go index 5f120194470..e0b9b0487c8 100644 --- a/pkg/controller/networkpolicy/clusternetworkpolicy.go +++ b/pkg/controller/networkpolicy/clusternetworkpolicy.go @@ -325,7 +325,11 @@ func (n *NetworkPolicyController) processClusterNetworkPolicy(cnp *crdv1alpha1.C } ruleATGNames := n.processClusterAppliedTo(ruleAppliedTos, atgNamesSet) klog.V(4).Infof("Adding a new cluster-level rule with appliedTos %v for %s", ruleATGNames, cnp.Name) - addRule(n.toAntreaPeerForCRD(clusterPeers, cnp, direction, namedPortExists), direction, ruleATGNames) + if cnpRule.ToServices != nil { + addRule(n.svcRefToPeerForCRD(cnpRule.ToServices, ""), direction, ruleATGNames) + } else { + addRule(n.toAntreaPeerForCRD(clusterPeers, cnp, direction, namedPortExists), direction, ruleATGNames) + } } if len(perNSPeers) > 0 { if len(cnp.Spec.AppliedTo) > 0 { diff --git a/pkg/controller/networkpolicy/clusternetworkpolicy_test.go b/pkg/controller/networkpolicy/clusternetworkpolicy_test.go index 9647f40a0ed..13635fa1861 100644 --- a/pkg/controller/networkpolicy/clusternetworkpolicy_test.go +++ b/pkg/controller/networkpolicy/clusternetworkpolicy_test.go @@ -51,6 +51,21 @@ func TestProcessClusterNetworkPolicy(t *testing.T) { }, } + svcA := v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svcA", + Namespace: "nsA", + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Port: 80, + TargetPort: int80, + }, + }, + }, + } + allowAction := crdv1alpha1.RuleActionAllow dropAction := crdv1alpha1.RuleActionDrop protocolTCP := controlplane.ProtocolTCP @@ -885,6 +900,58 @@ func TestProcessClusterNetworkPolicy(t *testing.T) { expectedAppliedToGroups: 2, expectedAddressGroups: 2, }, + { + name: "rule-with-to-service", + inputPolicy: &crdv1alpha1.ClusterNetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "cnpK", UID: "uidK"}, + Spec: crdv1alpha1.ClusterNetworkPolicySpec{ + AppliedTo: []crdv1alpha1.NetworkPolicyPeer{ + {PodSelector: &selectorA}, + }, + Priority: p10, + Egress: []crdv1alpha1.Rule{ + { + ToServices: []crdv1alpha1.ServiceReference{ + { + Namespace: "nsA", + Name: "svcA", + }, + }, + Action: &dropAction, + }, + }, + }, + }, + expectedPolicy: &antreatypes.NetworkPolicy{ + UID: "uidK", + Name: "uidK", + SourceRef: &controlplane.NetworkPolicyReference{ + Type: controlplane.AntreaClusterNetworkPolicy, + Name: "cnpK", + UID: "uidK", + }, + Priority: &p10, + TierPriority: &DefaultTierPriority, + Rules: []controlplane.NetworkPolicyRule{ + { + Direction: controlplane.DirectionOut, + To: controlplane.NetworkPolicyPeer{ + ToServices: []controlplane.ServiceReference{ + { + Namespace: "nsA", + Name: "svcA", + }, + }, + }, + Priority: 0, + Action: &dropAction, + }, + }, + AppliedToGroups: []string{getNormalizedUID(toGroupSelector("", &selectorA, nil, nil).NormalizedName)}, + }, + expectedAppliedToGroups: 1, + expectedAddressGroups: 0, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -893,6 +960,7 @@ func TestProcessClusterNetworkPolicy(t *testing.T) { c.cgStore.Add(&cgA) c.namespaceStore.Add(&nsA) c.namespaceStore.Add(&nsB) + c.serviceStore.Add(&svcA) if tt.inputPolicy.Spec.Tier != "" { c.tierStore.Add(&tierA) } @@ -907,6 +975,7 @@ func TestProcessClusterNetworkPolicy(t *testing.T) { assert.ElementsMatch(t, tt.expectedPolicy.PerNamespaceSelectors, actualPolicy.PerNamespaceSelectors) assert.ElementsMatch(t, tt.expectedPolicy.AppliedToGroups, actualPolicy.AppliedToGroups) assert.Equal(t, tt.expectedAppliedToGroups, len(c.appliedToGroupStore.List())) + assert.Equal(t, tt.expectedAddressGroups, len(c.addressGroupStore.List())) }) } } diff --git a/pkg/controller/networkpolicy/crd_utils.go b/pkg/controller/networkpolicy/crd_utils.go index ecfcbefbe80..ea283c5aa6e 100644 --- a/pkg/controller/networkpolicy/crd_utils.go +++ b/pkg/controller/networkpolicy/crd_utils.go @@ -134,6 +134,25 @@ func (n *NetworkPolicyController) toNamespacedPeerForCRD(peers []v1alpha1.Networ return &controlplane.NetworkPolicyPeer{AddressGroups: addressGroups} } +// svcRefToPeerForCRD creates an Antrea controlplane NetworkPolicyPeer from +// ServiceReference in ToServices field. For ANP, we will use the +// defaultNamespace(policy Namespace) as the Namespace of ServiceReference that +// doesn't set Namespace. +func (n *NetworkPolicyController) svcRefToPeerForCRD(svcRefs []v1alpha1.ServiceReference, defaultNamespace string) *controlplane.NetworkPolicyPeer { + var controlplaneSvcRefs []controlplane.ServiceReference + for _, svcRef := range svcRefs { + curNS := defaultNamespace + if svcRef.Namespace != "" { + curNS = svcRef.Namespace + } + controlplaneSvcRefs = append(controlplaneSvcRefs, controlplane.ServiceReference{ + Namespace: curNS, + Name: svcRef.Name, + }) + } + return &controlplane.NetworkPolicyPeer{ToServices: controlplaneSvcRefs} +} + // createAppliedToGroupForClusterGroupCRD creates an AppliedToGroup object corresponding to a // internal Group. If the AppliedToGroup already exists, it returns the key // otherwise it copies the internal Group contents to an AppliedToGroup resource and returns diff --git a/pkg/controller/networkpolicy/validate.go b/pkg/controller/networkpolicy/validate.go index c0974129b37..46482d43418 100644 --- a/pkg/controller/networkpolicy/validate.go +++ b/pkg/controller/networkpolicy/validate.go @@ -31,6 +31,7 @@ import ( crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" crdv1alpha2 "antrea.io/antrea/pkg/apis/crd/v1alpha2" "antrea.io/antrea/pkg/controller/networkpolicy/store" + "antrea.io/antrea/pkg/features" "antrea.io/antrea/pkg/util/env" ) @@ -482,6 +483,14 @@ func (v *antreaPolicyValidator) validatePeers(ingress, egress []crdv1alpha1.Rule } } for _, rule := range egress { + if rule.ToServices != nil { + if !features.DefaultFeatureGate.Enabled(features.AntreaProxy) { + return fmt.Sprintf("`toServices` can only be used when AntreaProxy is enabled"), false + } + if (rule.To != nil && len(rule.To) > 0) || rule.Ports != nil { + return fmt.Sprintf("`toServices` can't be used with `to` or `ports`"), false + } + } msg, isValid := checkPeers(rule.To) if !isValid { return msg, false diff --git a/test/e2e/antreapolicy_test.go b/test/e2e/antreapolicy_test.go index 6e22210f5fe..5c13bf2ee14 100644 --- a/test/e2e/antreapolicy_test.go +++ b/test/e2e/antreapolicy_test.go @@ -129,10 +129,12 @@ type TestStep struct { CustomProbes []*CustomProbe } -// fqdnTestStep is a single unit of testing spec for FQDN policy tests. -type fqdnTestStep struct { +// podToAddrTestStep is a single unit of testing the connectivity from a Pod to an +// arbitrary destination address. +type podToAddrTestStep struct { clientPod Pod - fqdnToQuery string + destAddr string + destPort int32 expectedConnectivity PodConnectivityMark } @@ -2368,25 +2370,29 @@ func testFQDNPolicy(t *testing.T) { builder.AddFQDNRule("*google.com", v1.ProtocolTCP, nil, nil, nil, "r1", nil, crdv1alpha1.RuleActionReject) builder.AddFQDNRule("wayfair.com", v1.ProtocolTCP, nil, nil, nil, "r2", nil, crdv1alpha1.RuleActionDrop) - testcases := []fqdnTestStep{ + testcases := []podToAddrTestStep{ { "x/a", "drive.google.com", + 80, Rejected, }, { "x/b", "maps.google.com", + 80, Rejected, }, { "y/a", "wayfair.com", + 80, Dropped, }, { "y/b", "facebook.com", + 80, Connected, }, } @@ -2394,14 +2400,14 @@ func testFQDNPolicy(t *testing.T) { failOnError(err, t) time.Sleep(networkPolicyDelay) for _, tc := range testcases { - log.Tracef("Probing: %s -> %s", tc.clientPod.PodName(), tc.fqdnToQuery) - connectivity, err := k8sUtils.ProbeEgress(tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.fqdnToQuery, 80, v1.ProtocolTCP) + log.Tracef("Probing: %s -> %s", tc.clientPod.PodName(), tc.destAddr) + connectivity, err := k8sUtils.ProbeEgress(tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.destAddr, tc.destPort, v1.ProtocolTCP) if err != nil { t.Errorf("failure -- could not complete probe: %v", err) } if connectivity != tc.expectedConnectivity { t.Errorf("failure -- wrong results for probe: Source %s/%s --> Dest %s connectivity: %v, expected: %v", - tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.fqdnToQuery, connectivity, tc.expectedConnectivity) + tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.destAddr, connectivity, tc.expectedConnectivity) } } // cleanup test resources @@ -2455,22 +2461,25 @@ func testFQDNPolicyInClusterService(t *testing.T) { k8sUtils.CreateOrUpdateACNP(acnp) failOnError(waitForResourceReady(acnp, timeout), t) - var testcases []fqdnTestStep + var testcases []podToAddrTestStep for _, service := range services { - eachServiceCases := []fqdnTestStep{ + eachServiceCases := []podToAddrTestStep{ { "y/b", svcDNSName(service), + 80, Rejected, }, { "z/c", svcDNSName(service), + 80, Dropped, }, { "x/c", svcDNSName(service), + 80, Connected, }, } @@ -2478,14 +2487,14 @@ func testFQDNPolicyInClusterService(t *testing.T) { } for _, tc := range testcases { - log.Tracef("Probing: %s -> %s", tc.clientPod.PodName(), tc.fqdnToQuery) - connectivity, err := k8sUtils.ProbeEgress(tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.fqdnToQuery, 80, v1.ProtocolTCP) + log.Tracef("Probing: %s -> %s", tc.clientPod.PodName(), tc.destAddr) + connectivity, err := k8sUtils.ProbeEgress(tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.destAddr, tc.destPort, v1.ProtocolTCP) if err != nil { t.Errorf("failure -- could not complete probe: %v", err) } if connectivity != tc.expectedConnectivity { t.Errorf("failure -- wrong results for probe: Source %s/%s --> Dest %s connectivity: %v, expected: %v", - tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.fqdnToQuery, connectivity, tc.expectedConnectivity) + tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.destAddr, connectivity, tc.expectedConnectivity) } } // cleanup test resources @@ -2498,6 +2507,83 @@ func testFQDNPolicyInClusterService(t *testing.T) { time.Sleep(networkPolicyDelay) } +func testToServices(t *testing.T) { + skipIfProxyDisabled(t) + var services []*v1.Service + if clusterInfo.podV4NetworkCIDR != "" { + ipv4Svc := k8sUtils.BuildService("ipv4-svc", "x", 81, 81, map[string]string{"pod": "a"}, nil) + ipv4Svc.Spec.IPFamilies = []v1.IPFamily{v1.IPv4Protocol} + services = append(services, ipv4Svc) + } + if clusterInfo.podV6NetworkCIDR != "" { + ipv6Svc := k8sUtils.BuildService("ipv6-svc", "x", 80, 80, map[string]string{"pod": "b"}, nil) + ipv6Svc.Spec.IPFamilies = []v1.IPFamily{v1.IPv6Protocol} + services = append(services, ipv6Svc) + } + + var svcRefs []crdv1alpha1.ServiceReference + var builtSvcs []*v1.Service + for _, service := range services { + builtSvc, _ := k8sUtils.CreateOrUpdateService(service) + failOnError(waitForResourceReady(service, timeout), t) + svcRefs = append(svcRefs, crdv1alpha1.ServiceReference{ + Name: service.Name, + Namespace: service.Namespace, + }) + builtSvcs = append(builtSvcs, builtSvc) + } + + builder := &ClusterNetworkPolicySpecBuilder{} + builder = builder.SetName("test-acnp-to-services"). + SetTier("application"). + SetPriority(1.0) + builder.AddToServicesRule(svcRefs, "svc", []ACNPAppliedToSpec{{NSSelector: map[string]string{"ns": "y"}}}, crdv1alpha1.RuleActionDrop) + time.Sleep(networkPolicyDelay) + + acnp := builder.Get() + k8sUtils.CreateOrUpdateACNP(acnp) + failOnError(waitForResourceReady(acnp, timeout), t) + + var testcases []podToAddrTestStep + for _, service := range builtSvcs { + eachServiceCases := []podToAddrTestStep{ + { + "y/b", + service.Spec.ClusterIP, + service.Spec.Ports[0].Port, + Dropped, + }, + { + "z/c", + service.Spec.ClusterIP, + service.Spec.Ports[0].Port, + Connected, + }, + } + testcases = append(testcases, eachServiceCases...) + } + + for _, tc := range testcases { + log.Tracef("Probing: %s -> %s", tc.clientPod.PodName(), tc.destAddr) + connectivity, err := k8sUtils.ProbeEgress(tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.destAddr, tc.destPort, v1.ProtocolTCP) + if err != nil { + t.Errorf("failure -- could not complete probe: %v", err) + } + if connectivity != tc.expectedConnectivity { + t.Errorf("failure -- wrong results for probe: Source %s/%s --> Dest %s:%d connectivity: %v, expected: %v", + tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.destAddr, tc.destPort, connectivity, tc.expectedConnectivity) + } + } + // cleanup test resources + failOnError(k8sUtils.DeleteACNP(builder.Name), t) + failOnError(waitForResourceDelete("", builder.Name, resourceACNP, timeout), t) + time.Sleep(networkPolicyDelay) + for _, service := range services { + failOnError(k8sUtils.DeleteService(service.Namespace, service.Name), t) + failOnError(waitForResourceDelete(service.Namespace, service.Name, resourceSVC, timeout), t) + } +} + // executeTests runs all the tests in testList and prints results func executeTests(t *testing.T, testList []*TestCase) { executeTestsWithData(t, testList, nil) @@ -2842,6 +2928,7 @@ func TestAntreaPolicy(t *testing.T) { t.Run("Case=ACNPNestedClusterGroup", func(t *testing.T) { testACNPNestedClusterGroupCreateAndUpdate(t, data) }) t.Run("Case=ACNPFQDNPolicy", func(t *testing.T) { testFQDNPolicy(t) }) t.Run("Case=FQDNPolicyInCluster", func(t *testing.T) { testFQDNPolicyInClusterService(t) }) + t.Run("Case=ACNPToServices", func(t *testing.T) { testToServices(t) }) }) // print results for reachability tests printResults() diff --git a/test/e2e/utils/cnpspecbuilder.go b/test/e2e/utils/cnpspecbuilder.go index c23294c3146..91788dec02b 100644 --- a/test/e2e/utils/cnpspecbuilder.go +++ b/test/e2e/utils/cnpspecbuilder.go @@ -274,6 +274,23 @@ func (b *ClusterNetworkPolicySpecBuilder) AddFQDNRule(fqdn string, return b } +func (b *ClusterNetworkPolicySpecBuilder) AddToServicesRule(svcRefs []crdv1alpha1.ServiceReference, + name string, ruleAppliedToSpecs []ACNPAppliedToSpec, action crdv1alpha1.RuleAction) *ClusterNetworkPolicySpecBuilder { + var appliedTos []crdv1alpha1.NetworkPolicyPeer + for _, at := range ruleAppliedToSpecs { + appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, at.NSSelector, at.PodSelectorMatchExp, at.NSSelectorMatchExp, at.Group)) + } + newRule := crdv1alpha1.Rule{ + To: make([]crdv1alpha1.NetworkPolicyPeer, 0), + ToServices: svcRefs, + Action: &action, + Name: name, + AppliedTo: appliedTos, + } + b.Spec.Egress = append(b.Spec.Egress, newRule) + return b +} + // AddEgressDNS mutates the nth policy rule to allow DNS, convenience method func (b *ClusterNetworkPolicySpecBuilder) WithEgressDNS() *ClusterNetworkPolicySpecBuilder { protocolUDP := v1.ProtocolUDP diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index d08572a5579..0392b5e8f1f 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -650,7 +650,7 @@ func expectedProxyServiceGroupAndFlows(gid uint32, svc svcConfig, endpointList [ svcFlows := expectTableFlows{tableID: 41, flows: []*ofTestUtils.ExpectFlow{ { MatchStr: fmt.Sprintf("priority=200,%s,reg4=0x10000/0x70000,nw_dst=%s,tp_dst=%d", string(svc.protocol), svc.ip.String(), svc.port), - ActStr: fmt.Sprintf("load:0x%x->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[19],group:%d", serviceLearnReg, gid), + ActStr: fmt.Sprintf("load:0x%x->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[19],load:0x%x->NXM_NX_REG7[],group:%d", serviceLearnReg, gid, gid), }, { MatchStr: fmt.Sprintf("priority=190,%s,reg4=0x30000/0x70000,nw_dst=%s,tp_dst=%d", string(svc.protocol), svc.ip.String(), svc.port),