From d955364d051d61c53f7a56615cbb244f19c53237 Mon Sep 17 00:00:00 2001 From: wgrayson Date: Thu, 2 Sep 2021 16:24:28 -0700 Subject: [PATCH 1/2] Add ToServices feature This PR adds ToServices feature, which allows users to apply an ACNP/ANP on a Service. This PR uses groupID assigned to the Service by AntreaProxy to match traffic, which means that ToServices can only be used when AntreaProxy is enabled and this Service must have at least one clusterIP. Also, this PR use groupID to match traffic, thus the policies will not be enforced when appliedTo workloads connect to Service Endpoints directly. In order to enforce policies on directly Endpoints traffic, one fallback is using ClusterGroup with ServiceReference added in PR #1797. What this PR did: 1. Add ToServices field in ACNP and ANP. 2. In ServiceLBTable, load OVS groupID to reg7. 3. Use OVS groupID of Service to do the conj match in the egress table. 4. Add a channel between proxier and networkpolicy controller for Service groupID update events. Signed-off-by: wgrayson --- build/yamls/antrea-aks.yml | 23 ++ build/yamls/antrea-eks.yml | 23 ++ build/yamls/antrea-gke.yml | 23 ++ build/yamls/antrea-ipsec.yml | 23 ++ build/yamls/antrea.yml | 23 ++ build/yamls/base/crds.yml | 23 ++ cmd/antrea-agent/agent.go | 17 +- docs/antrea-network-policy.md | 47 ++- pkg/agent/controller/networkpolicy/cache.go | 42 ++- .../controller/networkpolicy/cache_test.go | 3 +- .../networkpolicy/networkpolicy_controller.go | 7 +- .../networkpolicy_controller_test.go | 5 +- .../controller/networkpolicy/reconciler.go | 64 +++- .../networkpolicy/reconciler_test.go | 5 +- .../networkpolicy/status_controller_test.go | 2 +- pkg/agent/openflow/fields.go | 6 +- pkg/agent/openflow/network_policy.go | 62 ++-- pkg/agent/openflow/pipeline.go | 6 +- pkg/agent/proxy/proxier.go | 13 +- pkg/agent/proxy/proxier_test.go | 2 +- pkg/agent/proxy/types/groupcounter.go | 50 ++- pkg/agent/types/networkpolicy.go | 1 + pkg/apis/controlplane/types.go | 3 + pkg/apis/controlplane/v1beta2/generated.pb.go | 296 +++++++++++------- pkg/apis/controlplane/v1beta2/generated.proto | 4 + pkg/apis/controlplane/v1beta2/types.go | 3 + .../v1beta2/zz_generated.conversion.go | 2 + .../v1beta2/zz_generated.deepcopy.go | 5 + .../controlplane/zz_generated.deepcopy.go | 5 + pkg/apis/crd/v1alpha1/types.go | 19 +- .../crd/v1alpha1/zz_generated.deepcopy.go | 21 ++ pkg/apiserver/openapi/zz_generated.openapi.go | 16 +- .../networkpolicy/antreanetworkpolicy.go | 8 +- .../networkpolicy/antreanetworkpolicy_test.go | 69 ++++ .../networkpolicy/clusternetworkpolicy.go | 6 +- .../clusternetworkpolicy_test.go | 69 ++++ pkg/controller/networkpolicy/crd_utils.go | 19 ++ pkg/controller/networkpolicy/validate.go | 9 + test/e2e/antreapolicy_test.go | 111 ++++++- test/e2e/utils/cnpspecbuilder.go | 16 + test/integration/agent/openflow_test.go | 2 +- 41 files changed, 968 insertions(+), 185 deletions(-) diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index e4ab7d84f86..210b0bef278 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -790,6 +790,18 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + type: array required: - action type: object @@ -1918,6 +1930,17 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + type: array required: - action type: object diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 3f1a025027c..efe1bd22c6d 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -790,6 +790,18 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + type: array required: - action type: object @@ -1918,6 +1930,17 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + type: array required: - action type: object diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 2034cb3601b..7312dc9c748 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -790,6 +790,18 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + type: array required: - action type: object @@ -1918,6 +1930,17 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + type: array required: - action type: object diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index ecbce68ffb0..936da4df96d 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -790,6 +790,18 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + type: array required: - action type: object @@ -1918,6 +1930,17 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + type: array required: - action type: object diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 994023c84c0..154fef5ed7a 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -790,6 +790,18 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + type: array required: - action type: object @@ -1918,6 +1930,17 @@ spec: type: object type: object type: array + toServices: + items: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + type: array required: - action type: object diff --git a/build/yamls/base/crds.yml b/build/yamls/base/crds.yml index f69807a46c4..303a1786573 100644 --- a/build/yamls/base/crds.yml +++ b/build/yamls/base/crds.yml @@ -991,6 +991,18 @@ spec: type: string fqdn: type: string + toServices: + type: array + items: + type: object + required: + - name + - namespace + properties: + name: + type: string + namespace: + type: string name: type: string enableLogging: @@ -1364,6 +1376,17 @@ spec: format: cidr fqdn: type: string + toServices: + type: array + items: + type: object + required: + - name + properties: + name: + type: string + namespace: + type: string name: type: string enableLogging: diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 93295281a8b..25f9da0fecf 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -40,6 +40,7 @@ import ( npl "antrea.io/antrea/pkg/agent/nodeportlocal" "antrea.io/antrea/pkg/agent/openflow" "antrea.io/antrea/pkg/agent/proxy" + proxytypes "antrea.io/antrea/pkg/agent/proxy/types" "antrea.io/antrea/pkg/agent/querier" "antrea.io/antrea/pkg/agent/route" "antrea.io/antrea/pkg/agent/stats" @@ -199,6 +200,11 @@ func run(o *Options) error { agentInitializer.GetWireGuardClient(), o.config.AntreaProxy.ProxyAll) + var groupCounters []proxytypes.GroupCounter + groupIDUpdates := make(chan string, 100) + v4GroupCounter := proxytypes.NewGroupCounter(false, groupIDUpdates) + v6GroupCounter := proxytypes.NewGroupCounter(true, groupIDUpdates) + var proxier proxy.Proxier if features.DefaultFeatureGate.Enabled(features.AntreaProxy) { v4Enabled := config.IsIPv4Enabled(nodeConfig, networkConfig.TrafficEncapMode) @@ -208,11 +214,14 @@ func run(o *Options) error { switch { case v4Enabled && v6Enabled: - proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient, routeClient, nodePortAddressesIPv4, nodePortAddressesIPv6, proxyAll, skipServices) + proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient, routeClient, nodePortAddressesIPv4, nodePortAddressesIPv6, proxyAll, skipServices, v4GroupCounter, v6GroupCounter) + groupCounters = append(groupCounters, v4GroupCounter, v6GroupCounter) case v4Enabled: - proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false, routeClient, nodePortAddressesIPv4, proxyAll, skipServices) + proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false, routeClient, nodePortAddressesIPv4, proxyAll, skipServices, v4GroupCounter) + groupCounters = append(groupCounters, v4GroupCounter) case v6Enabled: - proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true, routeClient, nodePortAddressesIPv6, proxyAll, skipServices) + proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true, routeClient, nodePortAddressesIPv6, proxyAll, skipServices, v6GroupCounter) + groupCounters = append(groupCounters, v6GroupCounter) default: return fmt.Errorf("at least one of IPv4 or IPv6 should be enabled") } @@ -247,6 +256,8 @@ func run(o *Options) error { ifaceStore, nodeConfig.Name, entityUpdates, + groupCounters, + groupIDUpdates, antreaPolicyEnabled, statusManagerEnabled, loggingEnabled, diff --git a/docs/antrea-network-policy.md b/docs/antrea-network-policy.md index fdb2b316549..38fecaebc97 100644 --- a/docs/antrea-network-policy.md +++ b/docs/antrea-network-policy.md @@ -14,6 +14,7 @@ - [ACNP with ClusterGroup reference](#acnp-with-clustergroup-reference) - [ACNP for complete Pod isolation in selected Namespaces](#acnp-for-complete-pod-isolation-in-selected-namespaces) - [ACNP for default Namespace isolation](#acnp-for-default-namespace-isolation) + - [ACNP for toServices rule](#acnp-for-toservices-rule) - [Behavior of to and from selectors](#behavior-of-to-and-from-selectors) - [Key differences from K8s NetworkPolicy](#key-differences-from-k8s-networkpolicy) - [kubectl commands for Antrea ClusterNetworkPolicy](#kubectl-commands-for-antrea-clusternetworkpolicy) @@ -32,6 +33,7 @@ - [K8s clusters with version 1.21 and above](#k8s-clusters-with-version-121-and-above) - [K8s clusters with version 1.20 and below](#k8s-clusters-with-version-120-and-below) - [FQDN based filtering](#fqdn-based-filtering) +- [toServices instruction](#toservices-instruction) - [RBAC](#rbac) - [Notes](#notes) @@ -331,6 +333,32 @@ spec: enableLogging: true ``` +#### ACNP for toServices rule + +```yaml +apiVersion: crd.antrea.io/v1alpha1 +kind: ClusterNetworkPolicy +metadata: + name: acnp-drop-to-services +spec: + priority: 5 + tier: securityops + appliedTo: + - podSelector: + matchLabels: + role: client + namespaceSelector: + matchLabels: + env: prod + egress: + - action: Drop + toServices: + - name: svcName + namespace: svcNamespace + name: DropToServices + enableLogging: true +``` + **spec**: The ClusterNetworkPolicy `spec` has all the information needed to define a cluster-wide security policy. @@ -408,6 +436,8 @@ a rule, it will be auto-generated by Antrea. The rule name auto-generation proce is the same as ingress rules. A ClusterGroup name can be set in the `group` field of a egress `to` section in place of stand-alone selectors to allow traffic to workloads/ipBlocks set in the ClusterGroup. +`toServices` field contains a list of combinations of Service Namespace and Service Name to match traffic to this Service. +More details can be found in the [toServices](#toservices-instruction) section. The [first example](#acnp-with-stand-alone-selectors) policy contains a single rule, which drops matched traffic on a single port, to the 10.0.10.0/24 subnet specified by the `ipBlock` field. The [second example](#acnp-with-clustergroup-reference) policy contains a single rule, which drops matched traffic on @@ -415,7 +445,11 @@ TCP port 5978 to all network endpoints selected by the "test-cg-with-ip-block" ClusterGroup. The [third example](#acnp-for-complete-pod-isolation-in-selected-namespaces) policy contains a single rule, which drops all egress traffic initiated by any Pod in Namespaces that have `app` set to -`no-network-access-required`. Note that an empty `To` in the egress rule means that +`no-network-access-required`. +The [fifth example](#acnp-for-toservices-rule) policy contains a single rule, +which drops traffic from "role: client" labeled Pods from "env: prod" labeled Namespaces to Service svcNamespace/svcName +via ClusterIP. +Note that an empty `to` + an empty `toServices` in the egress rule means that this rule matches all egress destinations. Egress `To` section also supports FQDN based filtering. This can be applied to exact FQDNs or wildcard expressions. More details can be found in the [FQDN](#fqdn-based-filtering) section. @@ -1021,6 +1055,17 @@ spec: - fqdn: "svcA.default.svc.cluster.local" ``` +## toServices instruction + +A combination of Service name and Service Namespace can be used in `toServices` to refer to a Service. +`toServices` match traffic based on the clusterIP, port and protocol of Services. So headless Service is not supported. +Since `toServices` represents a combination of IP+port, it can't be used with `to` or `ports`. Also, this match process +relies on the groupID assigned to the Service by AntreaProxy. So it can only be used when AntreaProxy is enabled. +This clusterIP based match has one caveat: directly access to the Endpoints of this Service is not controlled by +`toServices`. To control the access to the backend Endpoints, you could use `ClusterGroup` with `ServiceReference`. +Because `ClusterGroup` with `ServiceReference` is equivalent to a podSelector that selects all backend Endpoints Pods of +the Service referred in `ServiceReference`. + ## RBAC Antrea-native policy CRDs are meant for admins to manage the security of their diff --git a/pkg/agent/controller/networkpolicy/cache.go b/pkg/agent/controller/networkpolicy/cache.go index 0b1bc041f28..596fe8f23fc 100644 --- a/pkg/agent/controller/networkpolicy/cache.go +++ b/pkg/agent/controller/networkpolicy/cache.go @@ -32,6 +32,7 @@ import ( v1beta "antrea.io/antrea/pkg/apis/controlplane/v1beta2" crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" "antrea.io/antrea/pkg/querier" + "antrea.io/antrea/pkg/util/k8s" ) const ( @@ -39,6 +40,7 @@ const ( appliedToGroupIndex = "appliedToGroup" addressGroupIndex = "addressGroup" policyIndex = "policy" + toServicesIndex = "toServices" ) // rule is the struct stored in ruleCache, it contains necessary information @@ -156,6 +158,10 @@ type ruleCache struct { // entityUpdates is a channel for receiving entity (e.g. Pod) updates from CNIServer. entityUpdates <-chan antreatypes.EntityReference + + // groupIDUpdates is a channel for receiving groupID for Service is assigned + // or released events from groupCounters. + groupIDUpdates <-chan string } func (c *ruleCache) getNetworkPolicies(npFilter *querier.NetworkPolicyQueryFilter) []v1beta.NetworkPolicy { @@ -318,11 +324,23 @@ func policyIndexFunc(obj interface{}) ([]string, error) { return []string{string(rule.PolicyUID)}, nil } +// toServicesIndexFunc knows how to get NamespacedNames of Services referred in +// ToServices field of a *rule. It's provided to cache.Indexer to build an index of +// NetworkPolicy. +func toServicesIndexFunc(obj interface{}) ([]string, error) { + rule := obj.(*rule) + toSvcNamespacedName := sets.String{} + for _, svc := range rule.To.ToServices { + toSvcNamespacedName.Insert(k8s.NamespacedName(svc.Namespace, svc.Name)) + } + return toSvcNamespacedName.UnsortedList(), nil +} + // newRuleCache returns a new *ruleCache. -func newRuleCache(dirtyRuleHandler func(string), podUpdate <-chan antreatypes.EntityReference) *ruleCache { +func newRuleCache(dirtyRuleHandler func(string), podUpdate <-chan antreatypes.EntityReference, serviceGroupIDUpdate <-chan string) *ruleCache { rules := cache.NewIndexer( ruleKeyFunc, - cache.Indexers{addressGroupIndex: addressGroupIndexFunc, appliedToGroupIndex: appliedToGroupIndexFunc, policyIndex: policyIndexFunc}, + cache.Indexers{addressGroupIndex: addressGroupIndexFunc, appliedToGroupIndex: appliedToGroupIndexFunc, policyIndex: policyIndexFunc, toServicesIndex: toServicesIndexFunc}, ) cache := &ruleCache{ appliedToSetByGroup: make(map[string]v1beta.GroupMemberSet), @@ -331,8 +349,10 @@ func newRuleCache(dirtyRuleHandler func(string), podUpdate <-chan antreatypes.En rules: rules, dirtyRuleHandler: dirtyRuleHandler, entityUpdates: podUpdate, + groupIDUpdates: serviceGroupIDUpdate, } go cache.processEntityUpdates() + go cache.processGroupIDUpdates() return cache } @@ -364,6 +384,24 @@ func (c *ruleCache) processEntityUpdates() { } } +// processGroupIDUpdates is an infinite loop that takes Service groupID +// update events from the channel, finds out rules that refer this Service in +// ToServices field and use dirtyRuleHandler to re-queue these rules. +func (c *ruleCache) processGroupIDUpdates() { + for { + select { + case svcStr := <-c.groupIDUpdates: + toSvcRules, err := c.rules.ByIndex(toServicesIndex, svcStr) + if err != nil { + continue + } + for _, toSvcRule := range toSvcRules { + c.dirtyRuleHandler(toSvcRule.(*rule).ID) + } + } + } +} + // GetAddressGroupNum gets the number of AddressGroup. func (c *ruleCache) GetAddressGroupNum() int { c.addressSetLock.RLock() diff --git a/pkg/agent/controller/networkpolicy/cache_test.go b/pkg/agent/controller/networkpolicy/cache_test.go index 564a027eadf..7f47d9fc802 100644 --- a/pkg/agent/controller/networkpolicy/cache_test.go +++ b/pkg/agent/controller/networkpolicy/cache_test.go @@ -263,7 +263,8 @@ func TestRuleCacheAddAddressGroup(t *testing.T) { func newFakeRuleCache() (*ruleCache, *dirtyRuleRecorder, chan types.EntityReference) { recorder := newDirtyRuleRecorder() ch := make(chan types.EntityReference, 100) - c := newRuleCache(recorder.Record, ch) + ch2 := make(chan string, 100) + c := newRuleCache(recorder.Record, ch, ch2) return c, recorder, ch } diff --git a/pkg/agent/controller/networkpolicy/networkpolicy_controller.go b/pkg/agent/controller/networkpolicy/networkpolicy_controller.go index 84c271a94d1..65f23aa1ef1 100644 --- a/pkg/agent/controller/networkpolicy/networkpolicy_controller.go +++ b/pkg/agent/controller/networkpolicy/networkpolicy_controller.go @@ -33,6 +33,7 @@ import ( "antrea.io/antrea/pkg/agent/flowexporter/connections" "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/openflow" + proxytypes "antrea.io/antrea/pkg/agent/proxy/types" "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/apis/controlplane/v1beta2" "antrea.io/antrea/pkg/querier" @@ -110,6 +111,8 @@ func NewNetworkPolicyController(antreaClientGetter agent.AntreaClientProvider, ifaceStore interfacestore.InterfaceStore, nodeName string, entityUpdates <-chan types.EntityReference, + groupCounters []proxytypes.GroupCounter, + groupIDUpdates <-chan string, antreaPolicyEnabled bool, statusManagerEnabled bool, loggingEnabled bool, @@ -135,8 +138,8 @@ func NewNetworkPolicyController(antreaClientGetter agent.AntreaClientProvider, c.ofClient.RegisterPacketInHandler(uint8(openflow.PacketInReasonNP), "dnsresponse", c.fqdnController) } } - c.reconciler = newReconciler(ofClient, ifaceStore, idAllocator, c.fqdnController) - c.ruleCache = newRuleCache(c.enqueueRule, entityUpdates) + c.reconciler = newReconciler(ofClient, ifaceStore, idAllocator, c.fqdnController, groupCounters) + c.ruleCache = newRuleCache(c.enqueueRule, entityUpdates, groupIDUpdates) if statusManagerEnabled { c.statusManager = newStatusController(antreaClientGetter, nodeName, c.ruleCache) } diff --git a/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go b/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go index d0e8c7e5ba7..463a80d9dcb 100644 --- a/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go +++ b/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go @@ -32,6 +32,7 @@ import ( "k8s.io/component-base/metrics/legacyregistry" "antrea.io/antrea/pkg/agent/metrics" + proxytypes "antrea.io/antrea/pkg/agent/proxy/types" agenttypes "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/apis/controlplane/v1beta2" "antrea.io/antrea/pkg/client/clientset/versioned" @@ -52,7 +53,9 @@ func (g *antreaClientGetter) GetAntreaClient() (versioned.Interface, error) { func newTestController() (*Controller, *fake.Clientset, *mockReconciler) { clientset := &fake.Clientset{} ch := make(chan agenttypes.EntityReference, 100) - controller, _ := NewNetworkPolicyController(&antreaClientGetter{clientset}, nil, nil, "node1", ch, + ch2 := make(chan string, 100) + groupCounters := []proxytypes.GroupCounter{proxytypes.NewGroupCounter(false, ch2)} + controller, _ := NewNetworkPolicyController(&antreaClientGetter{clientset}, nil, nil, "node1", ch, groupCounters, ch2, true, true, true, nil, testAsyncDeleteInterval, "8.8.8.8:53") reconciler := newMockReconciler() controller.reconciler = reconciler diff --git a/pkg/agent/controller/networkpolicy/reconciler.go b/pkg/agent/controller/networkpolicy/reconciler.go index a003313df00..06a75ae4043 100644 --- a/pkg/agent/controller/networkpolicy/reconciler.go +++ b/pkg/agent/controller/networkpolicy/reconciler.go @@ -28,10 +28,12 @@ import ( "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/openflow" + proxytypes "antrea.io/antrea/pkg/agent/proxy/types" "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/apis/controlplane/v1beta2" binding "antrea.io/antrea/pkg/ovs/openflow" "antrea.io/antrea/pkg/util/ip" + "antrea.io/antrea/pkg/util/k8s" ) var ( @@ -150,15 +152,20 @@ type lastRealized struct { // the fqdn selector of this policy rule. It must be empty for policy rule // that is not egress and does not have toFQDN field. fqdnIPAddresses sets.String + // groupIDAddresses tracks the last realized set of groupIDs resolved for + // the toServices of this policy rule. It must be empty for policy rule + // that is not egress and does not have toServices field. + groupIDAddresses sets.Int64 } func newLastRealized(rule *CompletedRule) *lastRealized { return &lastRealized{ - ofIDs: map[servicesKey]uint32{}, - CompletedRule: rule, - podOFPorts: map[servicesKey]sets.Int32{}, - podIPs: nil, - fqdnIPAddresses: nil, + ofIDs: map[servicesKey]uint32{}, + CompletedRule: rule, + podOFPorts: map[servicesKey]sets.Int32{}, + podIPs: nil, + fqdnIPAddresses: nil, + groupIDAddresses: nil, } } @@ -198,6 +205,10 @@ type reconciler struct { // reconciler to register FQDN policy rules and query the IP addresses corresponded // to a FQDN. fqdnController *fqdnController + + // groupCounters is a list of GroupCounter for v4 and v6 env. reconciler uses these + // GroupCounters to get the groupIDs of a specific Service. + groupCounters []proxytypes.GroupCounter } // newReconciler returns a new *reconciler. @@ -205,6 +216,7 @@ func newReconciler(ofClient openflow.Client, ifaceStore interfacestore.InterfaceStore, idAllocator *idAllocator, fqdnController *fqdnController, + groupCounters []proxytypes.GroupCounter, ) *reconciler { priorityAssigners := map[uint8]*tablePriorityAssigner{} for _, table := range openflow.GetAntreaPolicyBaselineTierTables() { @@ -224,6 +236,7 @@ func newReconciler(ofClient openflow.Client, idAllocator: idAllocator, priorityAssigners: priorityAssigners, fqdnController: fqdnController, + groupCounters: groupCounters, } // Check if ofClient is nil or not to be compatible with unit tests. if ofClient != nil { @@ -412,6 +425,7 @@ func (r *reconciler) add(rule *CompletedRule, ofPriority *uint16, table uint8) e if r.fqdnController != nil { lastRealized.fqdnIPAddresses = nil } + lastRealized.groupIDAddresses = nil return err } // Record ofID only if its Openflow is installed successfully. @@ -485,7 +499,7 @@ func (r *reconciler) computeOFRulesForAdd(rule *CompletedRule, ofPriority *uint1 // isolated, so we create a PolicyRule with the original services if it doesn't exist. // If there are IPBlocks or Pods that cannot resolve any named port, they will share // this PolicyRule. Antrea policies do not need this default isolation. - if !rule.isAntreaNetworkPolicyRule() || len(rule.To.IPBlocks) > 0 || len(rule.To.FQDNs) > 0 { + if !rule.isAntreaNetworkPolicyRule() || len(rule.To.IPBlocks) > 0 || len(rule.To.FQDNs) > 0 || len(rule.To.ToServices) > 0 { svcKey := normalizeServices(rule.Services) ofRule, exists := ofRuleByServicesMap[svcKey] // Create a new Openflow rule if the group doesn't exist. @@ -521,6 +535,21 @@ func (r *reconciler) computeOFRulesForAdd(rule *CompletedRule, ofPriority *uint1 // If the rule installation fails, this will be reset lastRealized.fqdnIPAddresses = addressSet } + if len(rule.To.ToServices) > 0 { + var addresses []types.Address + addressSet := sets.NewInt64() + for _, svcRef := range rule.To.ToServices { + for _, groupCounter := range r.groupCounters { + for _, groupID := range groupCounter.GetAllGroupIDs(k8s.NamespacedName(svcRef.Namespace, svcRef.Name)) { + addresses = append(addresses, openflow.NewServiceGroupIDAddress(groupID)) + addressSet.Insert(int64(groupID)) + } + } + } + ofRule.To = append(ofRule.To, addresses...) + // If the rule installation fails, this will be reset. + lastRealized.groupIDAddresses = addressSet + } } } return ofRuleByServicesMap, lastRealized @@ -692,6 +721,27 @@ func (r *reconciler) update(lastRealized *lastRealized, newRule *CompletedRule, } } } + originalGroupIDAddressSet, newGroupIDAddressSet := sets.NewInt64(), sets.NewInt64() + if lastRealized.groupIDAddresses != nil { + originalGroupIDAddressSet = lastRealized.groupIDAddresses + } + if len(newRule.To.ToServices) > 0 { + for _, svcRef := range newRule.To.ToServices { + for _, groupCounter := range r.groupCounters { + for _, groupID := range groupCounter.GetAllGroupIDs(k8s.NamespacedName(svcRef.Namespace, svcRef.Name)) { + newGroupIDAddressSet.Insert(int64(groupID)) + } + } + } + addedGroupIDAddress := newGroupIDAddressSet.Difference(originalGroupIDAddressSet) + removedGroupIDAddress := originalGroupIDAddressSet.Difference(newGroupIDAddressSet) + for a := range addedGroupIDAddress { + addedTo = append(addedTo, openflow.NewServiceGroupIDAddress(binding.GroupIDType(a))) + } + for r := range removedGroupIDAddress { + deletedTo = append(deletedTo, openflow.NewServiceGroupIDAddress(binding.GroupIDType(r))) + } + } if err := r.updateOFRule(ofID, addedFrom, addedTo, deletedFrom, deletedTo, ofPriority); err != nil { return err } @@ -699,6 +749,8 @@ func (r *reconciler) update(lastRealized *lastRealized, newRule *CompletedRule, // Update the FQDN address set if rule installation succeeds. lastRealized.fqdnIPAddresses = newFQDNAddressSet } + // Update the groupID address set if rule installation succeeds. + lastRealized.groupIDAddresses = newGroupIDAddressSet // Delete valid servicesKey from staleOFIDs. delete(staleOFIDs, svcKey) } diff --git a/pkg/agent/controller/networkpolicy/reconciler_test.go b/pkg/agent/controller/networkpolicy/reconciler_test.go index e9993027962..ac4fc036694 100644 --- a/pkg/agent/controller/networkpolicy/reconciler_test.go +++ b/pkg/agent/controller/networkpolicy/reconciler_test.go @@ -29,6 +29,7 @@ import ( "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/openflow" openflowtest "antrea.io/antrea/pkg/agent/openflow/testing" + proxytypes "antrea.io/antrea/pkg/agent/proxy/types" "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/apis/controlplane/v1beta2" @@ -99,7 +100,9 @@ func newCIDR(cidrStr string) *net.IPNet { func newTestReconciler(t *testing.T, controller *gomock.Controller, ifaceStore interfacestore.InterfaceStore, ofClient *openflowtest.MockClient) *reconciler { f, _ := newMockFQDNController(t, controller, nil) - r := newReconciler(ofClient, ifaceStore, newIDAllocator(testAsyncDeleteInterval), f) + ch := make(chan string, 100) + groupCounters := []proxytypes.GroupCounter{proxytypes.NewGroupCounter(false, ch)} + r := newReconciler(ofClient, ifaceStore, newIDAllocator(testAsyncDeleteInterval), f, groupCounters) return r } diff --git a/pkg/agent/controller/networkpolicy/status_controller_test.go b/pkg/agent/controller/networkpolicy/status_controller_test.go index 8e1607febb6..37c9173fa4c 100644 --- a/pkg/agent/controller/networkpolicy/status_controller_test.go +++ b/pkg/agent/controller/networkpolicy/status_controller_test.go @@ -51,7 +51,7 @@ func (c *fakeNetworkPolicyControl) getNetworkPolicyStatus() *v1beta2.NetworkPoli } func newTestStatusController() (*StatusController, *ruleCache, *fakeNetworkPolicyControl) { - ruleCache := newRuleCache(func(s string) {}, make(<-chan types.EntityReference)) + ruleCache := newRuleCache(func(s string) {}, make(<-chan types.EntityReference), make(chan string, 100)) statusControl := &fakeNetworkPolicyControl{} statusController := newStatusController(nil, testNode1, ruleCache) statusController.statusControlInterface = statusControl diff --git a/pkg/agent/openflow/fields.go b/pkg/agent/openflow/fields.go index 31582dae681..59a6776613f 100644 --- a/pkg/agent/openflow/fields.go +++ b/pkg/agent/openflow/fields.go @@ -112,9 +112,13 @@ var ( // Field to cache the Egress conjunction ID hit by TraceFlow packet. TFEgressConjIDField = binding.NewRegField(5, 0, 31, "TFEgressConjunctionID") - // reg(N6XM_NX_REG6) + // reg6(NXM_NX_REG6) // Field to store the Ingress conjunction ID hit by TraceFlow packet. TFIngressConjIDField = binding.NewRegField(6, 0, 31, "TFIngressConjunctionID") + + // reg7(NXM_NX_REG7) + // Field to store the GroupID corresponding to the Service + ServiceGroupIDField = binding.NewRegField(7, 0, 31, "ServiceGroupID") ) // Fields using xxreg. diff --git a/pkg/agent/openflow/network_policy.go b/pkg/agent/openflow/network_policy.go index 4c926455025..0de01e0bf21 100644 --- a/pkg/agent/openflow/network_policy.go +++ b/pkg/agent/openflow/network_policy.go @@ -31,27 +31,28 @@ import ( ) var ( - MatchDstIP = types.NewMatchKey(binding.ProtocolIP, types.IPAddr, "nw_dst") - MatchSrcIP = types.NewMatchKey(binding.ProtocolIP, types.IPAddr, "nw_src") - MatchDstIPNet = types.NewMatchKey(binding.ProtocolIP, types.IPNetAddr, "nw_dst") - MatchSrcIPNet = types.NewMatchKey(binding.ProtocolIP, types.IPNetAddr, "nw_src") - MatchDstIPv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPAddr, "ipv6_dst") - MatchSrcIPv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPAddr, "ipv6_src") - MatchDstIPNetv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPNetAddr, "ipv6_dst") - MatchSrcIPNetv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPNetAddr, "ipv6_src") - MatchDstOFPort = types.NewMatchKey(binding.ProtocolIP, types.OFPortAddr, "reg1[0..31]") - MatchSrcOFPort = types.NewMatchKey(binding.ProtocolIP, types.OFPortAddr, "in_port") - MatchTCPDstPort = types.NewMatchKey(binding.ProtocolTCP, types.L4PortAddr, "tp_dst") - MatchTCPv6DstPort = types.NewMatchKey(binding.ProtocolTCPv6, types.L4PortAddr, "tp_dst") - MatchUDPDstPort = types.NewMatchKey(binding.ProtocolUDP, types.L4PortAddr, "tp_dst") - MatchUDPv6DstPort = types.NewMatchKey(binding.ProtocolUDPv6, types.L4PortAddr, "tp_dst") - MatchSCTPDstPort = types.NewMatchKey(binding.ProtocolSCTP, types.L4PortAddr, "tp_dst") - MatchSCTPv6DstPort = types.NewMatchKey(binding.ProtocolSCTPv6, types.L4PortAddr, "tp_dst") - MatchTCPSrcPort = types.NewMatchKey(binding.ProtocolTCP, types.L4PortAddr, "tp_src") - MatchTCPv6SrcPort = types.NewMatchKey(binding.ProtocolTCPv6, types.L4PortAddr, "tp_src") - MatchUDPSrcPort = types.NewMatchKey(binding.ProtocolUDP, types.L4PortAddr, "tp_src") - MatchUDPv6SrcPort = types.NewMatchKey(binding.ProtocolUDPv6, types.L4PortAddr, "tp_src") - Unsupported = types.NewMatchKey(binding.ProtocolIP, types.UnSupported, "unknown") + MatchDstIP = types.NewMatchKey(binding.ProtocolIP, types.IPAddr, "nw_dst") + MatchSrcIP = types.NewMatchKey(binding.ProtocolIP, types.IPAddr, "nw_src") + MatchDstIPNet = types.NewMatchKey(binding.ProtocolIP, types.IPNetAddr, "nw_dst") + MatchSrcIPNet = types.NewMatchKey(binding.ProtocolIP, types.IPNetAddr, "nw_src") + MatchDstIPv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPAddr, "ipv6_dst") + MatchSrcIPv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPAddr, "ipv6_src") + MatchDstIPNetv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPNetAddr, "ipv6_dst") + MatchSrcIPNetv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPNetAddr, "ipv6_src") + MatchDstOFPort = types.NewMatchKey(binding.ProtocolIP, types.OFPortAddr, "reg1[0..31]") + MatchSrcOFPort = types.NewMatchKey(binding.ProtocolIP, types.OFPortAddr, "in_port") + MatchTCPDstPort = types.NewMatchKey(binding.ProtocolTCP, types.L4PortAddr, "tp_dst") + MatchTCPv6DstPort = types.NewMatchKey(binding.ProtocolTCPv6, types.L4PortAddr, "tp_dst") + MatchUDPDstPort = types.NewMatchKey(binding.ProtocolUDP, types.L4PortAddr, "tp_dst") + MatchUDPv6DstPort = types.NewMatchKey(binding.ProtocolUDPv6, types.L4PortAddr, "tp_dst") + MatchSCTPDstPort = types.NewMatchKey(binding.ProtocolSCTP, types.L4PortAddr, "tp_dst") + MatchSCTPv6DstPort = types.NewMatchKey(binding.ProtocolSCTPv6, types.L4PortAddr, "tp_dst") + MatchTCPSrcPort = types.NewMatchKey(binding.ProtocolTCP, types.L4PortAddr, "tp_src") + MatchTCPv6SrcPort = types.NewMatchKey(binding.ProtocolTCPv6, types.L4PortAddr, "tp_src") + MatchUDPSrcPort = types.NewMatchKey(binding.ProtocolUDP, types.L4PortAddr, "tp_src") + MatchUDPv6SrcPort = types.NewMatchKey(binding.ProtocolUDPv6, types.L4PortAddr, "tp_src") + MatchServiceGroupID = types.NewMatchKey(binding.ProtocolIP, types.ServiceGroupIDAddr, "reg7[0..31]") + Unsupported = types.NewMatchKey(binding.ProtocolIP, types.UnSupported, "unknown") // metricFlowIdentifier is used to identify metric flows in metric table. // There could be other flows like default flow and Traceflow flows in the table. Only metric flows are supposed to @@ -164,6 +165,25 @@ func NewOFPortAddress(addr int32) *OFPortAddress { return &a } +type ServiceGroupIDAddress binding.GroupIDType + +func (a *ServiceGroupIDAddress) GetMatchKey(addrType types.AddressType) *types.MatchKey { + return MatchServiceGroupID +} + +func (a *ServiceGroupIDAddress) GetMatchValue() string { + return fmt.Sprintf("%d", uint32(*a)) +} + +func (a *ServiceGroupIDAddress) GetValue() interface{} { + return uint32(*a) +} + +func NewServiceGroupIDAddress(groupID binding.GroupIDType) *ServiceGroupIDAddress { + a := ServiceGroupIDAddress(groupID) + return &a +} + // ConjunctionNotFound is an error response when the specified policyRuleConjunction is not found from the local cache. type ConjunctionNotFound uint32 diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index be58e376416..6d3d4db2ede 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -1873,6 +1873,8 @@ func (c *client) addFlowMatch(fb binding.FlowBuilder, matchKey *types.MatchKey, if portValue.Value > 0 { fb = fb.MatchSrcPort(portValue.Value, portValue.Mask) } + case MatchServiceGroupID: + fb = fb.MatchRegFieldWithValue(ServiceGroupIDField, matchValue.(uint32)) } return fb } @@ -2255,7 +2257,9 @@ func (c *client) serviceLBFlow(groupID binding.GroupIDType, svcIP net.IP, svcPor flowBuilder = flowBuilder.Action().LoadRegMark(ServiceNeedSNATRegMark) } } - return flowBuilder.Action().Group(groupID).Done() + return flowBuilder. + Action().LoadToRegField(ServiceGroupIDField, uint32(groupID)). + Action().Group(groupID).Done() } // endpointDNATFlow generates the flow which transforms the Service Cluster IP diff --git a/pkg/agent/proxy/proxier.go b/pkg/agent/proxy/proxier.go index 0e4f708423a..dc15f7ad5db 100644 --- a/pkg/agent/proxy/proxier.go +++ b/pkg/agent/proxy/proxier.go @@ -790,7 +790,8 @@ func NewProxier( routeClient route.Interface, nodePortAddresses []net.IP, proxyAllEnabled bool, - skipServices []string) *proxier { + skipServices []string, + groupCounter types.GroupCounter) *proxier { recorder := record.NewBroadcaster().NewRecorder( runtime.NewScheme(), corev1.EventSource{Component: componentName, Host: hostname}, @@ -816,7 +817,7 @@ func NewProxier( endpointReferenceCounter: map[string]int{}, serviceStringMap: map[string]k8sproxy.ServicePortName{}, oversizeServiceSet: sets.NewString(), - groupCounter: types.NewGroupCounter(isIPv6), + groupCounter: groupCounter, ofClient: ofClient, routeClient: routeClient, nodePortAddresses: nodePortAddresses, @@ -875,13 +876,15 @@ func NewDualStackProxier( nodePortAddressesIPv4 []net.IP, nodePortAddressesIPv6 []net.IP, proxyAllEnabled bool, - skipServices []string) *metaProxierWrapper { + skipServices []string, + v4groupCounter types.GroupCounter, + v6groupCounter types.GroupCounter) *metaProxierWrapper { // Create an IPv4 instance of the single-stack proxier. - ipv4Proxier := NewProxier(hostname, informerFactory, ofClient, false, routeClient, nodePortAddressesIPv4, proxyAllEnabled, skipServices) + ipv4Proxier := NewProxier(hostname, informerFactory, ofClient, false, routeClient, nodePortAddressesIPv4, proxyAllEnabled, skipServices, v4groupCounter) // Create an IPv6 instance of the single-stack proxier. - ipv6Proxier := NewProxier(hostname, informerFactory, ofClient, true, routeClient, nodePortAddressesIPv6, proxyAllEnabled, skipServices) + ipv6Proxier := NewProxier(hostname, informerFactory, ofClient, true, routeClient, nodePortAddressesIPv6, proxyAllEnabled, skipServices, v6groupCounter) // Create a meta-proxier that dispatch calls between the two // single-stack proxier instances. diff --git a/pkg/agent/proxy/proxier_test.go b/pkg/agent/proxy/proxier_test.go index 9d2a53157f7..baf0bcc8515 100644 --- a/pkg/agent/proxy/proxier_test.go +++ b/pkg/agent/proxy/proxier_test.go @@ -120,7 +120,7 @@ func NewFakeProxier(routeClient route.Interface, ofClient openflow.Client, nodeP endpointsInstalledMap: types.EndpointsMap{}, endpointReferenceCounter: map[string]int{}, endpointsMap: types.EndpointsMap{}, - groupCounter: types.NewGroupCounter(isIPv6), + groupCounter: types.NewGroupCounter(isIPv6, make(chan string, 100)), ofClient: ofClient, routeClient: routeClient, serviceStringMap: map[string]k8sproxy.ServicePortName{}, diff --git a/pkg/agent/proxy/types/groupcounter.go b/pkg/agent/proxy/types/groupcounter.go index d4c537924d8..657f952240c 100644 --- a/pkg/agent/proxy/types/groupcounter.go +++ b/pkg/agent/proxy/types/groupcounter.go @@ -18,6 +18,8 @@ import ( "fmt" "sync" + "k8s.io/apimachinery/pkg/util/sets" + binding "antrea.io/antrea/pkg/ovs/openflow" k8sproxy "antrea.io/antrea/third_party/proxy" ) @@ -32,22 +34,26 @@ type GroupCounter interface { // Recycle removes a Service Group ID mapping. The recycled groupID can be // reused. Recycle(svcPortName k8sproxy.ServicePortName, isEndpointsLocal bool) bool + // GetAllGroupIDs gets all groupID related to a Service. + GetAllGroupIDs(svcNamespacedName string) []binding.GroupIDType } type groupCounter struct { mu sync.Mutex groupIDCounter binding.GroupIDType recycled []binding.GroupIDType + groupIDUpdates chan<- string - groupMap map[string]binding.GroupIDType + servicePortNamesMap map[string]sets.String + groupMap map[string]binding.GroupIDType } -func NewGroupCounter(isIPv6 bool) *groupCounter { +func NewGroupCounter(isIPv6 bool, groupIDUpdates chan<- string) *groupCounter { var groupIDCounter binding.GroupIDType if isIPv6 { groupIDCounter = 0x10000000 } - return &groupCounter{groupMap: map[string]binding.GroupIDType{}, groupIDCounter: groupIDCounter} + return &groupCounter{groupMap: map[string]binding.GroupIDType{}, groupIDCounter: groupIDCounter, groupIDUpdates: groupIDUpdates, servicePortNamesMap: map[string]sets.String{}} } func keyString(svcPortName k8sproxy.ServicePortName, isEndpointsLocal bool) string { @@ -58,6 +64,26 @@ func keyString(svcPortName k8sproxy.ServicePortName, isEndpointsLocal bool) stri return key } +func (c *groupCounter) updateServicePortNameMap(svcNamespacedName string, svcKeyString string) { + if _, ok := c.servicePortNamesMap[svcNamespacedName]; ok { + c.servicePortNamesMap[svcNamespacedName].Insert(svcKeyString) + } else { + keyStringSet := sets.NewString(svcKeyString) + c.servicePortNamesMap[svcNamespacedName] = keyStringSet + } +} + +func (c *groupCounter) deleteServicePortNameMap(svcNamespacedName string, svcKeyString string) { + keyStringSet, ok := c.servicePortNamesMap[svcNamespacedName] + if !ok { + return + } + keyStringSet.Delete(svcKeyString) + if keyStringSet.Len() == 0 { + delete(c.servicePortNamesMap, svcNamespacedName) + } +} + func (c *groupCounter) Get(svcPortName k8sproxy.ServicePortName, isEndpointsLocal bool) (binding.GroupIDType, bool) { c.mu.Lock() defer c.mu.Unlock() @@ -68,10 +94,14 @@ func (c *groupCounter) Get(svcPortName k8sproxy.ServicePortName, isEndpointsLoca id = c.recycled[len(c.recycled)-1] c.recycled = c.recycled[:len(c.recycled)-1] c.groupMap[key] = id + c.updateServicePortNameMap(svcPortName.NamespacedName.String(), key) + c.groupIDUpdates <- svcPortName.NamespacedName.String() return id, true } else { c.groupIDCounter += 1 c.groupMap[key] = c.groupIDCounter + c.updateServicePortNameMap(svcPortName.NamespacedName.String(), key) + c.groupIDUpdates <- svcPortName.NamespacedName.String() return c.groupIDCounter, true } } @@ -84,7 +114,21 @@ func (c *groupCounter) Recycle(svcPortName k8sproxy.ServicePortName, isEndpoints if id, ok := c.groupMap[key]; ok { delete(c.groupMap, key) c.recycled = append(c.recycled, id) + c.deleteServicePortNameMap(svcPortName.NamespacedName.String(), key) + c.groupIDUpdates <- svcPortName.NamespacedName.String() return true } return false } + +func (c *groupCounter) GetAllGroupIDs(svcNamespacedName string) []binding.GroupIDType { + c.mu.Lock() + defer c.mu.Unlock() + var ids []binding.GroupIDType + for _, key := range c.servicePortNamesMap[svcNamespacedName].UnsortedList() { + if id, ok := c.groupMap[key]; ok { + ids = append(ids, id) + } + } + return ids +} diff --git a/pkg/agent/types/networkpolicy.go b/pkg/agent/types/networkpolicy.go index 1343b3faa82..38d6e81d3fc 100644 --- a/pkg/agent/types/networkpolicy.go +++ b/pkg/agent/types/networkpolicy.go @@ -53,6 +53,7 @@ const ( IPNetAddr OFPortAddr L4PortAddr + ServiceGroupIDAddr UnSupported ) diff --git a/pkg/apis/controlplane/types.go b/pkg/apis/controlplane/types.go index cd38ff589be..a598d860c4e 100644 --- a/pkg/apis/controlplane/types.go +++ b/pkg/apis/controlplane/types.go @@ -273,6 +273,9 @@ type NetworkPolicyPeer struct { // A list of exact FQDN names or FQDN wildcard expressions. // This field can only be possibly set for NetworkPolicyPeer of egress rules. FQDNs []string + // A list of ServiceReference. + // This field can only be possibly set for NetworkPolicyPeer of egress rules. + ToServices []ServiceReference } // IPBlock describes a particular CIDR (Ex. "192.168.1.1/24"). The except entry describes CIDRs that should diff --git a/pkg/apis/controlplane/v1beta2/generated.pb.go b/pkg/apis/controlplane/v1beta2/generated.pb.go index b10d637a45d..ea273ba182b 100644 --- a/pkg/apis/controlplane/v1beta2/generated.pb.go +++ b/pkg/apis/controlplane/v1beta2/generated.pb.go @@ -896,124 +896,126 @@ func init() { } var fileDescriptor_fbaa7d016762fa1d = []byte{ - // 1867 bytes of a gzipped FileDescriptorProto + // 1892 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0x4d, 0x6c, 0x23, 0x49, - 0x15, 0x9e, 0xf6, 0x4f, 0x12, 0xbf, 0x38, 0x89, 0x53, 0xd9, 0x61, 0xcc, 0x32, 0xd8, 0xd9, 0xe6, - 0x47, 0x39, 0xb0, 0xed, 0x4d, 0x98, 0xdd, 0x1d, 0xd8, 0x1f, 0x14, 0x6f, 0x32, 0x91, 0xa5, 0x59, - 0xaf, 0xa9, 0x64, 0x35, 0x12, 0x62, 0x61, 0x3b, 0xdd, 0x65, 0xa7, 0x88, 0xdd, 0xd5, 0x74, 0x97, - 0xc3, 0x44, 0x48, 0x68, 0x11, 0x70, 0x58, 0x40, 0x82, 0x1b, 0x67, 0x4e, 0x5c, 0x38, 0x73, 0xe7, - 0x80, 0x34, 0xc7, 0x45, 0x80, 0xd8, 0x93, 0xc5, 0x18, 0x01, 0xe2, 0xc0, 0x8d, 0x53, 0xf6, 0x82, - 0xaa, 0xba, 0xfa, 0xd7, 0xc9, 0x64, 0x3c, 0xc9, 0x04, 0x09, 0xf6, 0x64, 0xf7, 0xab, 0xf7, 0xde, - 0xf7, 0x5e, 0xbd, 0xbf, 0xaa, 0x6e, 0x78, 0xdd, 0x74, 0xb8, 0x47, 0x4c, 0x83, 0xb2, 0x46, 0xf0, - 0xaf, 0xe1, 0x1e, 0xf6, 0x1a, 0xa6, 0x4b, 0xfd, 0x86, 0xc5, 0x1c, 0xee, 0xb1, 0xbe, 0xdb, 0x37, - 0x1d, 0xd2, 0x38, 0x5a, 0xdf, 0x27, 0xdc, 0xdc, 0x68, 0xf4, 0x88, 0x43, 0x3c, 0x93, 0x13, 0xdb, - 0x70, 0x3d, 0xc6, 0x19, 0x32, 0x02, 0xa9, 0x6f, 0x52, 0xa6, 0xfe, 0x19, 0xee, 0x61, 0xcf, 0x10, - 0xf2, 0x46, 0x52, 0xde, 0x50, 0xf2, 0xcf, 0xde, 0x3e, 0x1b, 0xcf, 0xe7, 0x26, 0xf7, 0x1b, 0x47, - 0xeb, 0x66, 0xdf, 0x3d, 0x30, 0xd7, 0xb3, 0x48, 0xcf, 0x3e, 0xdf, 0xa3, 0xfc, 0x60, 0xb8, 0x6f, - 0x58, 0x6c, 0xd0, 0xe8, 0xb1, 0x1e, 0x6b, 0x48, 0xf2, 0xfe, 0xb0, 0x2b, 0x9f, 0xe4, 0x83, 0xfc, - 0xa7, 0xd8, 0x6f, 0x1d, 0xde, 0xf6, 0x25, 0x8a, 0x4b, 0x07, 0xa6, 0x75, 0x40, 0x1d, 0xe2, 0x1d, - 0xc7, 0x58, 0x03, 0xc2, 0xcd, 0xc6, 0xd1, 0x24, 0x48, 0xe3, 0x2c, 0x29, 0x6f, 0xe8, 0x70, 0x3a, - 0x20, 0x13, 0x02, 0x2f, 0x9d, 0x27, 0xe0, 0x5b, 0x07, 0x64, 0x60, 0x4e, 0xc8, 0x7d, 0xf1, 0x2c, - 0xb9, 0x21, 0xa7, 0xfd, 0x06, 0x75, 0xb8, 0xcf, 0xbd, 0xac, 0x90, 0xfe, 0x0f, 0x0d, 0xca, 0x9b, - 0xb6, 0xed, 0x11, 0xdf, 0xdf, 0xf1, 0xd8, 0xd0, 0x45, 0xef, 0xc2, 0x9c, 0xf0, 0xc4, 0x36, 0xb9, - 0x59, 0xd5, 0x56, 0xb5, 0xb5, 0xf9, 0x8d, 0x17, 0x8c, 0x40, 0xb1, 0x91, 0x54, 0x1c, 0xc7, 0x44, - 0x70, 0x1b, 0x47, 0xeb, 0xc6, 0x5b, 0xfb, 0xdf, 0x22, 0x16, 0x7f, 0x93, 0x70, 0xb3, 0x89, 0x1e, - 0x8c, 0xea, 0xd7, 0xc6, 0xa3, 0x3a, 0xc4, 0x34, 0x1c, 0x69, 0x45, 0x43, 0x28, 0xf7, 0x04, 0xd4, - 0x9b, 0x64, 0xb0, 0x4f, 0x3c, 0xbf, 0x9a, 0x5b, 0xcd, 0xaf, 0xcd, 0x6f, 0xbc, 0x32, 0x65, 0xd8, - 0x8d, 0x9d, 0x58, 0x47, 0xf3, 0x19, 0x05, 0x58, 0x4e, 0x10, 0x7d, 0x9c, 0x82, 0xd1, 0xff, 0xa0, - 0x41, 0x25, 0xe9, 0xe9, 0x5d, 0xea, 0x73, 0xf4, 0xf5, 0x09, 0x6f, 0x8d, 0xc7, 0xf3, 0x56, 0x48, - 0x4b, 0x5f, 0x2b, 0x0a, 0x7a, 0x2e, 0xa4, 0x24, 0x3c, 0x35, 0xa1, 0x48, 0x39, 0x19, 0x84, 0x2e, - 0xbe, 0x3a, 0xad, 0x8b, 0x49, 0x73, 0x9b, 0x0b, 0x0a, 0xa8, 0xd8, 0x12, 0x2a, 0x71, 0xa0, 0x59, - 0x7f, 0x3f, 0x0f, 0xcb, 0x49, 0xb6, 0x8e, 0xc9, 0xad, 0x83, 0x2b, 0x08, 0xe2, 0x0f, 0x35, 0x58, - 0x36, 0x6d, 0x9b, 0xd8, 0x3b, 0x97, 0x1c, 0xca, 0x4f, 0x2a, 0x58, 0xe1, 0x55, 0x5a, 0x3b, 0x9e, - 0x04, 0x44, 0x3f, 0xd6, 0x60, 0xc5, 0x23, 0x03, 0x76, 0x94, 0x31, 0x24, 0x7f, 0x71, 0x43, 0x3e, - 0xa5, 0x0c, 0x59, 0xc1, 0x93, 0xfa, 0xf1, 0x69, 0xa0, 0xfa, 0x3f, 0x35, 0x58, 0xdc, 0x74, 0xdd, - 0x3e, 0x25, 0xf6, 0x1e, 0xfb, 0x1f, 0xaf, 0xa6, 0x3f, 0x6b, 0x80, 0xd2, 0xbe, 0x5e, 0x41, 0x3d, - 0x59, 0xe9, 0x7a, 0x7a, 0x7d, 0xea, 0x7a, 0x4a, 0x19, 0x7c, 0x46, 0x45, 0xfd, 0x24, 0x0f, 0x2b, - 0x69, 0xc6, 0x8f, 0x6b, 0xea, 0xbf, 0x57, 0x53, 0x1f, 0xe5, 0x60, 0xe5, 0x8d, 0xfe, 0xd0, 0xe7, - 0xc4, 0x4b, 0x19, 0xf9, 0xf4, 0xa3, 0xf1, 0x7d, 0x0d, 0x2a, 0xa4, 0xdb, 0x25, 0x16, 0xa7, 0x47, - 0xe4, 0x12, 0x83, 0x51, 0x55, 0xa8, 0x95, 0xed, 0x8c, 0x72, 0x3c, 0x01, 0x87, 0xbe, 0x07, 0xcb, - 0x11, 0xad, 0xd5, 0x69, 0xf6, 0x99, 0x75, 0x18, 0xc6, 0xe1, 0xc5, 0x69, 0x6d, 0x68, 0x75, 0xda, - 0x84, 0xc7, 0xa9, 0xb0, 0x9d, 0xd5, 0x8b, 0x27, 0xa1, 0xf4, 0xbf, 0x6b, 0x30, 0xbf, 0xdd, 0xfb, - 0x3f, 0x38, 0x1c, 0xfc, 0x5e, 0x83, 0xa5, 0x84, 0xa3, 0x57, 0xd0, 0xcb, 0xde, 0x4d, 0xf7, 0xb2, - 0xa9, 0x3d, 0x4c, 0x58, 0x7b, 0x46, 0x23, 0xfb, 0x69, 0x1e, 0x2a, 0x09, 0xae, 0xa0, 0x8b, 0xd9, - 0x00, 0x2c, 0xda, 0xf7, 0x4b, 0x8d, 0x61, 0x42, 0xef, 0xc7, 0x9d, 0xec, 0x94, 0x4e, 0xd6, 0x87, - 0x1b, 0xdb, 0xf7, 0x39, 0xf1, 0x1c, 0xb3, 0xbf, 0xed, 0x70, 0xca, 0x8f, 0x31, 0xe9, 0x12, 0x8f, - 0x38, 0x16, 0x41, 0xab, 0x50, 0x70, 0xcc, 0x01, 0x91, 0xe1, 0x28, 0x35, 0xcb, 0x4a, 0x75, 0xa1, - 0x6d, 0x0e, 0x08, 0x96, 0x2b, 0xa8, 0x01, 0x25, 0xf1, 0xeb, 0xbb, 0xa6, 0x45, 0xaa, 0x39, 0xc9, - 0xb6, 0xac, 0xd8, 0x4a, 0xed, 0x70, 0x01, 0xc7, 0x3c, 0xfa, 0x47, 0x1a, 0x54, 0x24, 0xfc, 0xa6, - 0xef, 0x33, 0x8b, 0x9a, 0x9c, 0x32, 0xe7, 0x6a, 0x46, 0x58, 0xc5, 0x54, 0x88, 0xca, 0xff, 0x27, - 0x9e, 0xd6, 0x52, 0x3a, 0xda, 0xa4, 0xb8, 0x6f, 0x6e, 0x66, 0xf4, 0xe3, 0x09, 0x44, 0xfd, 0xdf, - 0x39, 0x98, 0x4f, 0x6c, 0x3e, 0xba, 0x07, 0x79, 0x97, 0xd9, 0xca, 0xe7, 0xa9, 0x8f, 0xe1, 0x1d, - 0x66, 0xc7, 0x66, 0xcc, 0x8e, 0x47, 0xf5, 0xbc, 0xa0, 0x08, 0x8d, 0xe8, 0x07, 0x1a, 0x2c, 0x92, - 0x54, 0x54, 0x65, 0x74, 0xe6, 0x37, 0x76, 0xa6, 0xae, 0xe7, 0xd3, 0x73, 0xa3, 0x89, 0xc6, 0xa3, - 0xfa, 0x62, 0x66, 0x31, 0x03, 0x89, 0x3e, 0x0f, 0x79, 0xea, 0x06, 0x69, 0x5d, 0x6e, 0x3e, 0x23, - 0x0c, 0x6c, 0x75, 0xfc, 0x93, 0x51, 0xbd, 0xd4, 0xea, 0xa8, 0xbb, 0x01, 0x16, 0x0c, 0xe8, 0x1b, - 0x50, 0x74, 0x99, 0xc7, 0xfd, 0x6a, 0x41, 0x46, 0xe4, 0x4b, 0xd3, 0xda, 0x28, 0x32, 0xcd, 0xee, - 0x30, 0x8f, 0xc7, 0x1d, 0x47, 0x3c, 0xf9, 0x38, 0x50, 0xab, 0xff, 0x4a, 0x83, 0xc5, 0x74, 0xd4, - 0xd2, 0x89, 0xab, 0x9d, 0x9f, 0xb8, 0x51, 0x2d, 0xe4, 0xce, 0xac, 0x85, 0x26, 0xe4, 0x87, 0xd4, - 0xae, 0xe6, 0x25, 0xc3, 0x0b, 0x8a, 0x21, 0xff, 0x76, 0x6b, 0xeb, 0x64, 0x54, 0x7f, 0xee, 0xac, - 0x3b, 0x30, 0x3f, 0x76, 0x89, 0x6f, 0xbc, 0xdd, 0xda, 0xc2, 0x42, 0x58, 0xff, 0xad, 0x06, 0xb3, - 0x6a, 0xca, 0xa1, 0x7b, 0x50, 0xb0, 0xa8, 0xed, 0xa9, 0xec, 0x78, 0xc2, 0xb9, 0x1a, 0x19, 0xfa, - 0x46, 0x6b, 0x0b, 0x63, 0xa9, 0x10, 0xbd, 0x03, 0x33, 0xe4, 0xbe, 0x45, 0x5c, 0xae, 0x2a, 0xe0, - 0x09, 0x55, 0x2f, 0x2a, 0xd5, 0x33, 0xdb, 0x52, 0x19, 0x56, 0x4a, 0xf5, 0x2e, 0x14, 0x25, 0x03, - 0xfa, 0x0c, 0xe4, 0xa8, 0x2b, 0xcd, 0x2f, 0x37, 0x57, 0xc6, 0xa3, 0x7a, 0xae, 0xd5, 0x49, 0x07, - 0x3f, 0x47, 0x5d, 0x74, 0x1b, 0xca, 0xae, 0x47, 0xba, 0xf4, 0xfe, 0x5d, 0xe2, 0xf4, 0xf8, 0x81, - 0xdc, 0xdf, 0x62, 0x3c, 0x1b, 0x3b, 0x89, 0x35, 0x9c, 0xe2, 0xd4, 0xdf, 0xd7, 0xa0, 0x14, 0x45, - 0x5e, 0xc4, 0x47, 0x04, 0x5b, 0xc2, 0x15, 0x63, 0xb7, 0xc5, 0x1a, 0x96, 0x2b, 0x8f, 0x11, 0xc1, - 0xdb, 0x30, 0x27, 0xdf, 0x3e, 0x58, 0xac, 0xaf, 0xc2, 0x78, 0x33, 0x9c, 0x94, 0x1d, 0x45, 0x3f, - 0x49, 0xfc, 0xc7, 0x11, 0xb7, 0xfe, 0xaf, 0x3c, 0x2c, 0xb4, 0x09, 0xff, 0x0e, 0xf3, 0x0e, 0x3b, - 0xac, 0x4f, 0xad, 0xe3, 0x2b, 0xe8, 0x69, 0x5d, 0x28, 0x7a, 0xc3, 0x3e, 0x09, 0xfb, 0xd8, 0xe6, - 0xd4, 0x55, 0x93, 0xb4, 0x17, 0x0f, 0xfb, 0x24, 0xae, 0x1e, 0xf1, 0xe4, 0xe3, 0x40, 0x3d, 0x7a, - 0x0d, 0x96, 0xcc, 0xd4, 0xbd, 0x23, 0xa8, 0xe8, 0x92, 0x8c, 0xe9, 0x52, 0xfa, 0x4a, 0xe2, 0xe3, - 0x2c, 0x2f, 0x5a, 0x13, 0x9b, 0x4a, 0x99, 0x27, 0x7a, 0x50, 0x61, 0x55, 0x5b, 0xd3, 0x9a, 0xe5, - 0x60, 0x43, 0x03, 0x1a, 0x8e, 0x56, 0xd1, 0x2d, 0x28, 0x73, 0x4a, 0xbc, 0x70, 0xa5, 0x5a, 0x94, - 0xa1, 0xac, 0x88, 0x34, 0xd8, 0x4b, 0xd0, 0x71, 0x8a, 0x0b, 0xf9, 0x50, 0xf2, 0xd9, 0xd0, 0xb3, - 0x08, 0x26, 0xdd, 0xea, 0x8c, 0xdc, 0xe9, 0x3b, 0x17, 0xdb, 0x8a, 0xa8, 0xc7, 0x2d, 0x88, 0x6e, - 0xb0, 0x1b, 0x2a, 0xc7, 0x31, 0x8e, 0xfe, 0x27, 0x0d, 0x96, 0x53, 0x42, 0x57, 0x70, 0x32, 0xdb, - 0x4f, 0x9f, 0xcc, 0x5e, 0xbb, 0x90, 0x93, 0x67, 0x9c, 0xcd, 0xbe, 0x0b, 0x37, 0x52, 0x6c, 0x6d, - 0x66, 0x93, 0x5d, 0x6e, 0xf2, 0xa1, 0x8f, 0xbe, 0x00, 0x73, 0x0e, 0xb3, 0x49, 0x3b, 0x3e, 0x10, - 0x44, 0xc6, 0xb6, 0x15, 0x1d, 0x47, 0x1c, 0x68, 0x03, 0x40, 0xbd, 0xd2, 0xa3, 0xcc, 0x91, 0x25, - 0x97, 0x8f, 0xd3, 0x79, 0x27, 0x5a, 0xc1, 0x09, 0x2e, 0xfd, 0x8f, 0xd9, 0x4d, 0xed, 0x10, 0xe2, - 0xa1, 0x97, 0x61, 0xc1, 0x4c, 0xbc, 0x48, 0xf2, 0xab, 0x9a, 0x4c, 0xbe, 0xe5, 0xf1, 0xa8, 0xbe, - 0x90, 0x7c, 0xc3, 0xe4, 0xe3, 0x34, 0x1f, 0x22, 0x30, 0x47, 0x5d, 0x75, 0x37, 0x09, 0xb6, 0xec, - 0xe5, 0xe9, 0x1b, 0x9d, 0x94, 0x8f, 0x3d, 0x8d, 0x2e, 0x25, 0x91, 0x6a, 0x54, 0x87, 0x62, 0xf7, - 0xdb, 0xb6, 0x13, 0x16, 0x45, 0x49, 0xec, 0xe9, 0x9d, 0xaf, 0x6e, 0xb5, 0x7d, 0x1c, 0xd0, 0xc5, - 0x65, 0xe5, 0x13, 0xa7, 0x27, 0x18, 0x7a, 0x11, 0x0a, 0x62, 0x00, 0xa8, 0xfd, 0x7c, 0x2e, 0x6c, - 0x49, 0x7b, 0xc7, 0x2e, 0x39, 0x19, 0xd5, 0xd3, 0x9b, 0x21, 0x88, 0x58, 0xb2, 0x4f, 0x7d, 0xea, - 0x8a, 0x5a, 0x5f, 0xfe, 0xbc, 0xe1, 0x55, 0xb8, 0xc8, 0xf0, 0xfa, 0x65, 0x31, 0x13, 0x3f, 0xd1, - 0x46, 0xd0, 0xab, 0x50, 0xb2, 0xa9, 0x27, 0x2e, 0x70, 0xcc, 0x51, 0x8e, 0xd6, 0x42, 0x63, 0xb7, - 0xc2, 0x85, 0x93, 0xe4, 0x03, 0x8e, 0x05, 0x90, 0x05, 0x85, 0xae, 0xc7, 0x06, 0xea, 0xf4, 0x72, - 0xb1, 0x1e, 0x27, 0xd2, 0x29, 0x76, 0xfe, 0x8e, 0xc7, 0x06, 0x58, 0x2a, 0x47, 0xef, 0x40, 0x8e, - 0x33, 0xb9, 0x39, 0x97, 0x02, 0x01, 0x0a, 0x22, 0xb7, 0xc7, 0x70, 0x8e, 0x33, 0x91, 0x88, 0x3e, - 0xf1, 0x8e, 0xa8, 0x45, 0xc2, 0x13, 0xce, 0xd4, 0x89, 0xb8, 0x1b, 0xc8, 0xc7, 0x89, 0xa8, 0x08, - 0x3e, 0x8e, 0x54, 0x8b, 0x02, 0x75, 0x33, 0xad, 0x33, 0x9e, 0x5e, 0x13, 0xcd, 0xf6, 0x1e, 0xcc, - 0x98, 0x41, 0x4c, 0x66, 0x64, 0x4c, 0xbe, 0x22, 0x26, 0xf9, 0x66, 0x18, 0x8c, 0xf5, 0x47, 0x7c, - 0x2b, 0xf1, 0xec, 0xe8, 0xcb, 0x85, 0x21, 0x22, 0x1c, 0x08, 0x61, 0xa5, 0x0e, 0xbd, 0x02, 0x0b, - 0xc4, 0x31, 0xf7, 0xfb, 0xe4, 0x2e, 0xeb, 0xf5, 0xa8, 0xd3, 0xab, 0xce, 0xae, 0x6a, 0x6b, 0x73, - 0xcd, 0xeb, 0xca, 0x96, 0x85, 0xed, 0xe4, 0x22, 0x4e, 0xf3, 0x9e, 0x36, 0x6b, 0xe6, 0xa6, 0x98, - 0x35, 0x61, 0x9e, 0x97, 0xce, 0xca, 0x73, 0xfd, 0x67, 0x79, 0x40, 0xa9, 0x88, 0x89, 0xee, 0xe6, - 0x8b, 0xf3, 0xf2, 0x82, 0x93, 0x24, 0xab, 0xfe, 0x7d, 0x59, 0x93, 0x24, 0xf2, 0x3e, 0xbd, 0x9e, - 0xc6, 0x44, 0x2e, 0x94, 0xb9, 0x67, 0x76, 0xbb, 0xd4, 0x92, 0x56, 0xa9, 0xa4, 0x7f, 0xe9, 0x11, - 0x36, 0xc8, 0x0f, 0x49, 0x46, 0x14, 0x8e, 0xbd, 0x84, 0x74, 0x7c, 0x86, 0x4a, 0x52, 0x71, 0x0a, - 0x01, 0xbd, 0xa7, 0x41, 0x45, 0x4c, 0xf9, 0x24, 0x8b, 0xba, 0x86, 0x7e, 0xf9, 0xf1, 0x61, 0x71, - 0x46, 0x43, 0x7c, 0x27, 0xca, 0xae, 0xe0, 0x09, 0x34, 0xfd, 0x6f, 0x1a, 0xac, 0x4c, 0x44, 0x64, - 0x78, 0x15, 0x6f, 0xd2, 0xfa, 0x50, 0x14, 0xf3, 0x2a, 0x9c, 0x0e, 0x3b, 0x17, 0x8a, 0x75, 0x3c, - 0x29, 0xe3, 0xd1, 0x2a, 0x68, 0x3e, 0x0e, 0x40, 0xf4, 0xdf, 0x15, 0xa0, 0x12, 0x32, 0xf9, 0xbb, - 0xc3, 0xc1, 0xc0, 0xf4, 0xae, 0xe2, 0x94, 0xf8, 0x23, 0x0d, 0x96, 0x92, 0x59, 0x46, 0x23, 0x7f, - 0x9b, 0x17, 0xf2, 0x37, 0x08, 0xf4, 0x0d, 0x85, 0xbd, 0xd4, 0x4e, 0x43, 0xe0, 0x2c, 0x26, 0xfa, - 0xb5, 0x06, 0x37, 0x03, 0x14, 0xf5, 0xda, 0x34, 0x23, 0xa1, 0xb2, 0xee, 0x32, 0x8c, 0xfa, 0xac, - 0x32, 0xea, 0xe6, 0xe6, 0x23, 0xf0, 0xf0, 0x23, 0xad, 0x41, 0xbf, 0xd0, 0xe0, 0x7a, 0xc0, 0x90, - 0xb5, 0xb3, 0x70, 0x69, 0x76, 0x7e, 0x5a, 0xd9, 0x79, 0x7d, 0xf3, 0x34, 0x20, 0x7c, 0x3a, 0xbe, - 0x6e, 0x42, 0x39, 0x79, 0xf1, 0x7f, 0x1a, 0x2f, 0x69, 0x7e, 0xa3, 0xc1, 0xac, 0x1a, 0x30, 0xe8, - 0x56, 0xe2, 0x4e, 0x14, 0x40, 0x54, 0xcf, 0xbf, 0x0f, 0xa1, 0xb6, 0xba, 0x8d, 0xe5, 0xce, 0xc9, - 0xe9, 0x21, 0xa7, 0x7d, 0x23, 0xf8, 0x04, 0x6c, 0xb4, 0x1c, 0xfe, 0x96, 0xb7, 0xcb, 0x3d, 0xea, - 0xf4, 0x9a, 0x73, 0x99, 0xbb, 0xdb, 0xe7, 0x60, 0x96, 0x38, 0xf2, 0xa2, 0x27, 0xc7, 0x74, 0xb1, - 0x39, 0x3f, 0x1e, 0xd5, 0x67, 0xb7, 0x03, 0x12, 0x0e, 0xd7, 0x74, 0x02, 0x15, 0x65, 0xf7, 0xd3, - 0xdc, 0x9f, 0xe6, 0xf3, 0x0f, 0x1e, 0xd6, 0xae, 0x7d, 0xf0, 0xb0, 0x76, 0xed, 0xc3, 0x87, 0xb5, - 0x6b, 0xef, 0x8d, 0x6b, 0xda, 0x83, 0x71, 0x4d, 0xfb, 0x60, 0x5c, 0xd3, 0x3e, 0x1c, 0xd7, 0xb4, - 0xbf, 0x8c, 0x6b, 0xda, 0xcf, 0xff, 0x5a, 0xbb, 0xf6, 0xb5, 0x59, 0x15, 0xfa, 0xff, 0x04, 0x00, - 0x00, 0xff, 0xff, 0xbd, 0x86, 0x85, 0x11, 0x79, 0x20, 0x00, 0x00, + 0x15, 0x4e, 0xfb, 0x27, 0x89, 0x5f, 0x9c, 0xc4, 0xa9, 0xec, 0x30, 0x66, 0x19, 0xec, 0x6c, 0xf3, + 0xa3, 0x1c, 0xd8, 0xf6, 0x26, 0xcc, 0xee, 0x0c, 0xec, 0x0f, 0xc4, 0x9b, 0x4c, 0x64, 0x69, 0xd6, + 0x6b, 0x2a, 0x59, 0x8d, 0x84, 0x58, 0xd8, 0x4e, 0x77, 0xd9, 0x69, 0xd2, 0xee, 0x6a, 0xba, 0xcb, + 0x61, 0x22, 0x24, 0xb4, 0x08, 0x38, 0x2c, 0x20, 0xc1, 0x8d, 0x33, 0x27, 0x2e, 0x9c, 0xb9, 0x73, + 0x40, 0x1a, 0x71, 0x5a, 0x84, 0x10, 0x7b, 0xb2, 0x18, 0x23, 0x40, 0x1c, 0xb8, 0x71, 0xca, 0x5e, + 0x50, 0x55, 0x57, 0xff, 0x3a, 0x9e, 0x8c, 0x27, 0x99, 0x20, 0xb1, 0x7b, 0xb2, 0xbb, 0xea, 0xbd, + 0xf7, 0xbd, 0x57, 0xdf, 0xab, 0xf7, 0xaa, 0xba, 0xe1, 0x35, 0xdd, 0x61, 0x1e, 0xd1, 0x35, 0x8b, + 0x36, 0x82, 0x7f, 0x0d, 0xf7, 0xa8, 0xd7, 0xd0, 0x5d, 0xcb, 0x6f, 0x18, 0xd4, 0x61, 0x1e, 0xb5, + 0x5d, 0x5b, 0x77, 0x48, 0xe3, 0x78, 0xe3, 0x80, 0x30, 0x7d, 0xb3, 0xd1, 0x23, 0x0e, 0xf1, 0x74, + 0x46, 0x4c, 0xcd, 0xf5, 0x28, 0xa3, 0x48, 0x0b, 0xb4, 0xbe, 0x65, 0x51, 0xf9, 0x4f, 0x73, 0x8f, + 0x7a, 0x1a, 0xd7, 0xd7, 0x92, 0xfa, 0x9a, 0xd4, 0x7f, 0xf6, 0xf6, 0x64, 0x3c, 0x9f, 0xe9, 0xcc, + 0x6f, 0x1c, 0x6f, 0xe8, 0xb6, 0x7b, 0xa8, 0x6f, 0x64, 0x91, 0x9e, 0x7d, 0xbe, 0x67, 0xb1, 0xc3, + 0xc1, 0x81, 0x66, 0xd0, 0x7e, 0xa3, 0x47, 0x7b, 0xb4, 0x21, 0x86, 0x0f, 0x06, 0x5d, 0xf1, 0x24, + 0x1e, 0xc4, 0x3f, 0x29, 0x7e, 0xf3, 0xe8, 0xb6, 0x2f, 0x50, 0x5c, 0xab, 0xaf, 0x1b, 0x87, 0x96, + 0x43, 0xbc, 0x93, 0x18, 0xab, 0x4f, 0x98, 0xde, 0x38, 0x1e, 0x07, 0x69, 0x4c, 0xd2, 0xf2, 0x06, + 0x0e, 0xb3, 0xfa, 0x64, 0x4c, 0xe1, 0xa5, 0xf3, 0x14, 0x7c, 0xe3, 0x90, 0xf4, 0xf5, 0x31, 0xbd, + 0x2f, 0x4e, 0xd2, 0x1b, 0x30, 0xcb, 0x6e, 0x58, 0x0e, 0xf3, 0x99, 0x97, 0x55, 0x52, 0xff, 0xa9, + 0x40, 0x79, 0xcb, 0x34, 0x3d, 0xe2, 0xfb, 0xbb, 0x1e, 0x1d, 0xb8, 0xe8, 0x1d, 0x98, 0xe7, 0x91, + 0x98, 0x3a, 0xd3, 0xab, 0xca, 0x9a, 0xb2, 0xbe, 0xb0, 0xf9, 0x82, 0x16, 0x18, 0xd6, 0x92, 0x86, + 0x63, 0x4e, 0xb8, 0xb4, 0x76, 0xbc, 0xa1, 0xbd, 0x79, 0xf0, 0x6d, 0x62, 0xb0, 0x37, 0x08, 0xd3, + 0x9b, 0xe8, 0xc1, 0xb0, 0x3e, 0x33, 0x1a, 0xd6, 0x21, 0x1e, 0xc3, 0x91, 0x55, 0x34, 0x80, 0x72, + 0x8f, 0x43, 0xbd, 0x41, 0xfa, 0x07, 0xc4, 0xf3, 0xab, 0xb9, 0xb5, 0xfc, 0xfa, 0xc2, 0xe6, 0xcb, + 0x53, 0xd2, 0xae, 0xed, 0xc6, 0x36, 0x9a, 0xcf, 0x48, 0xc0, 0x72, 0x62, 0xd0, 0xc7, 0x29, 0x18, + 0xf5, 0x4f, 0x0a, 0x54, 0x92, 0x91, 0xde, 0xb5, 0x7c, 0x86, 0xbe, 0x31, 0x16, 0xad, 0xf6, 0x78, + 0xd1, 0x72, 0x6d, 0x11, 0x6b, 0x45, 0x42, 0xcf, 0x87, 0x23, 0x89, 0x48, 0x75, 0x28, 0x5a, 0x8c, + 0xf4, 0xc3, 0x10, 0x5f, 0x99, 0x36, 0xc4, 0xa4, 0xbb, 0xcd, 0x45, 0x09, 0x54, 0x6c, 0x71, 0x93, + 0x38, 0xb0, 0xac, 0xbe, 0x97, 0x87, 0x95, 0xa4, 0x58, 0x47, 0x67, 0xc6, 0xe1, 0x15, 0x90, 0xf8, + 0x23, 0x05, 0x56, 0x74, 0xd3, 0x24, 0xe6, 0xee, 0x25, 0x53, 0xf9, 0x49, 0x09, 0xcb, 0xa3, 0x4a, + 0x5b, 0xc7, 0xe3, 0x80, 0xe8, 0x27, 0x0a, 0xac, 0x7a, 0xa4, 0x4f, 0x8f, 0x33, 0x8e, 0xe4, 0x2f, + 0xee, 0xc8, 0xa7, 0xa4, 0x23, 0xab, 0x78, 0xdc, 0x3e, 0x3e, 0x0b, 0x54, 0xfd, 0x97, 0x02, 0x4b, + 0x5b, 0xae, 0x6b, 0x5b, 0xc4, 0xdc, 0xa7, 0xff, 0xe7, 0xbb, 0xe9, 0x2f, 0x0a, 0xa0, 0x74, 0xac, + 0x57, 0xb0, 0x9f, 0x8c, 0xf4, 0x7e, 0x7a, 0x6d, 0xea, 0xfd, 0x94, 0x72, 0x78, 0xc2, 0x8e, 0xfa, + 0x69, 0x1e, 0x56, 0xd3, 0x82, 0x1f, 0xef, 0xa9, 0xff, 0xdd, 0x9e, 0xfa, 0x30, 0x07, 0xab, 0xaf, + 0xdb, 0x03, 0x9f, 0x11, 0x2f, 0xe5, 0xe4, 0xd3, 0x67, 0xe3, 0x07, 0x0a, 0x54, 0x48, 0xb7, 0x4b, + 0x0c, 0x66, 0x1d, 0x93, 0x4b, 0x24, 0xa3, 0x2a, 0x51, 0x2b, 0x3b, 0x19, 0xe3, 0x78, 0x0c, 0x0e, + 0x7d, 0x1f, 0x56, 0xa2, 0xb1, 0x56, 0xa7, 0x69, 0x53, 0xe3, 0x28, 0xe4, 0xe1, 0xc5, 0x69, 0x7d, + 0x68, 0x75, 0xda, 0x84, 0xc5, 0xa9, 0xb0, 0x93, 0xb5, 0x8b, 0xc7, 0xa1, 0xd4, 0x7f, 0x28, 0xb0, + 0xb0, 0xd3, 0xfb, 0x08, 0x1c, 0x0e, 0xfe, 0xa8, 0xc0, 0x72, 0x22, 0xd0, 0x2b, 0xa8, 0x65, 0xef, + 0xa4, 0x6b, 0xd9, 0xd4, 0x11, 0x26, 0xbc, 0x9d, 0x50, 0xc8, 0x7e, 0x96, 0x87, 0x4a, 0x42, 0x2a, + 0xa8, 0x62, 0x26, 0x00, 0x8d, 0xd6, 0xfd, 0x52, 0x39, 0x4c, 0xd8, 0xfd, 0xb8, 0x92, 0x9d, 0x51, + 0xc9, 0x6c, 0xb8, 0xbe, 0x73, 0x9f, 0x11, 0xcf, 0xd1, 0xed, 0x1d, 0x87, 0x59, 0xec, 0x04, 0x93, + 0x2e, 0xf1, 0x88, 0x63, 0x10, 0xb4, 0x06, 0x05, 0x47, 0xef, 0x13, 0x41, 0x47, 0xa9, 0x59, 0x96, + 0xa6, 0x0b, 0x6d, 0xbd, 0x4f, 0xb0, 0x98, 0x41, 0x0d, 0x28, 0xf1, 0x5f, 0xdf, 0xd5, 0x0d, 0x52, + 0xcd, 0x09, 0xb1, 0x15, 0x29, 0x56, 0x6a, 0x87, 0x13, 0x38, 0x96, 0x51, 0x3f, 0x54, 0xa0, 0x22, + 0xe0, 0xb7, 0x7c, 0x9f, 0x1a, 0x96, 0xce, 0x2c, 0xea, 0x5c, 0x4d, 0x0b, 0xab, 0xe8, 0x12, 0x51, + 0xc6, 0xff, 0xc4, 0xdd, 0x5a, 0x68, 0x47, 0x8b, 0x14, 0xd7, 0xcd, 0xad, 0x8c, 0x7d, 0x3c, 0x86, + 0xa8, 0xfe, 0x27, 0x07, 0x0b, 0x89, 0xc5, 0x47, 0xf7, 0x20, 0xef, 0x52, 0x53, 0xc6, 0x3c, 0xf5, + 0x31, 0xbc, 0x43, 0xcd, 0xd8, 0x8d, 0xb9, 0xd1, 0xb0, 0x9e, 0xe7, 0x23, 0xdc, 0x22, 0xfa, 0xa1, + 0x02, 0x4b, 0x24, 0xc5, 0xaa, 0x60, 0x67, 0x61, 0x73, 0x77, 0xea, 0xfd, 0x7c, 0x76, 0x6e, 0x34, + 0xd1, 0x68, 0x58, 0x5f, 0xca, 0x4c, 0x66, 0x20, 0xd1, 0xe7, 0x21, 0x6f, 0xb9, 0x41, 0x5a, 0x97, + 0x9b, 0xcf, 0x70, 0x07, 0x5b, 0x1d, 0xff, 0x74, 0x58, 0x2f, 0xb5, 0x3a, 0xf2, 0x6e, 0x80, 0xb9, + 0x00, 0xfa, 0x26, 0x14, 0x5d, 0xea, 0x31, 0xbf, 0x5a, 0x10, 0x8c, 0x7c, 0x69, 0x5a, 0x1f, 0x79, + 0xa6, 0x99, 0x1d, 0xea, 0xb1, 0xb8, 0xe2, 0xf0, 0x27, 0x1f, 0x07, 0x66, 0xd5, 0x5f, 0x2b, 0xb0, + 0x94, 0x66, 0x2d, 0x9d, 0xb8, 0xca, 0xf9, 0x89, 0x1b, 0xed, 0x85, 0xdc, 0xc4, 0xbd, 0xd0, 0x84, + 0xfc, 0xc0, 0x32, 0xab, 0x79, 0x21, 0xf0, 0x82, 0x14, 0xc8, 0xbf, 0xd5, 0xda, 0x3e, 0x1d, 0xd6, + 0x9f, 0x9b, 0x74, 0x07, 0x66, 0x27, 0x2e, 0xf1, 0xb5, 0xb7, 0x5a, 0xdb, 0x98, 0x2b, 0xab, 0xbf, + 0x53, 0x60, 0x4e, 0x76, 0x39, 0x74, 0x0f, 0x0a, 0x86, 0x65, 0x7a, 0x32, 0x3b, 0x9e, 0xb0, 0xaf, + 0x46, 0x8e, 0xbe, 0xde, 0xda, 0xc6, 0x58, 0x18, 0x44, 0x6f, 0xc3, 0x2c, 0xb9, 0x6f, 0x10, 0x97, + 0xc9, 0x1d, 0xf0, 0x84, 0xa6, 0x97, 0xa4, 0xe9, 0xd9, 0x1d, 0x61, 0x0c, 0x4b, 0xa3, 0x6a, 0x17, + 0x8a, 0x42, 0x00, 0x7d, 0x06, 0x72, 0x96, 0x2b, 0xdc, 0x2f, 0x37, 0x57, 0x47, 0xc3, 0x7a, 0xae, + 0xd5, 0x49, 0x93, 0x9f, 0xb3, 0x5c, 0x74, 0x1b, 0xca, 0xae, 0x47, 0xba, 0xd6, 0xfd, 0xbb, 0xc4, + 0xe9, 0xb1, 0x43, 0xb1, 0xbe, 0xc5, 0xb8, 0x37, 0x76, 0x12, 0x73, 0x38, 0x25, 0xa9, 0xbe, 0xa7, + 0x40, 0x29, 0x62, 0x9e, 0xf3, 0xc3, 0xc9, 0x16, 0x70, 0xc5, 0x38, 0x6c, 0x3e, 0x87, 0xc5, 0xcc, + 0x63, 0x30, 0x78, 0x1b, 0xe6, 0xc5, 0xdb, 0x07, 0x83, 0xda, 0x92, 0xc6, 0x1b, 0x61, 0xa7, 0xec, + 0xc8, 0xf1, 0xd3, 0xc4, 0x7f, 0x1c, 0x49, 0xab, 0xff, 0xce, 0xc3, 0x62, 0x9b, 0xb0, 0xef, 0x52, + 0xef, 0xa8, 0x43, 0x6d, 0xcb, 0x38, 0xb9, 0x82, 0x9a, 0xd6, 0x85, 0xa2, 0x37, 0xb0, 0x49, 0x58, + 0xc7, 0xb6, 0xa6, 0xde, 0x35, 0x49, 0x7f, 0xf1, 0xc0, 0x26, 0xf1, 0xee, 0xe1, 0x4f, 0x3e, 0x0e, + 0xcc, 0xa3, 0x57, 0x61, 0x59, 0x4f, 0xdd, 0x3b, 0x82, 0x1d, 0x5d, 0x12, 0x9c, 0x2e, 0xa7, 0xaf, + 0x24, 0x3e, 0xce, 0xca, 0xa2, 0x75, 0xbe, 0xa8, 0x16, 0xf5, 0x78, 0x0d, 0x2a, 0xac, 0x29, 0xeb, + 0x4a, 0xb3, 0x1c, 0x2c, 0x68, 0x30, 0x86, 0xa3, 0x59, 0x74, 0x13, 0xca, 0xcc, 0x22, 0x5e, 0x38, + 0x53, 0x2d, 0x0a, 0x2a, 0x2b, 0x3c, 0x0d, 0xf6, 0x13, 0xe3, 0x38, 0x25, 0x85, 0x7c, 0x28, 0xf9, + 0x74, 0xe0, 0x19, 0x04, 0x93, 0x6e, 0x75, 0x56, 0xac, 0xf4, 0x9d, 0x8b, 0x2d, 0x45, 0x54, 0xe3, + 0x16, 0x79, 0x35, 0xd8, 0x0b, 0x8d, 0xe3, 0x18, 0x47, 0xfd, 0xb3, 0x02, 0x2b, 0x29, 0xa5, 0x2b, + 0x38, 0x99, 0x1d, 0xa4, 0x4f, 0x66, 0xaf, 0x5e, 0x28, 0xc8, 0x09, 0x67, 0xb3, 0xef, 0xc1, 0xf5, + 0x94, 0x58, 0x9b, 0x9a, 0x64, 0x8f, 0xe9, 0x6c, 0xe0, 0xa3, 0x2f, 0xc0, 0xbc, 0x43, 0x4d, 0xd2, + 0x8e, 0x0f, 0x04, 0x91, 0xb3, 0x6d, 0x39, 0x8e, 0x23, 0x09, 0xb4, 0x09, 0x20, 0x5f, 0xe9, 0x59, + 0xd4, 0x11, 0x5b, 0x2e, 0x1f, 0xa7, 0xf3, 0x6e, 0x34, 0x83, 0x13, 0x52, 0xea, 0x1f, 0x72, 0x99, + 0x45, 0xed, 0x10, 0xe2, 0xa1, 0x5b, 0xb0, 0xa8, 0x27, 0x5e, 0x24, 0xf9, 0x55, 0x45, 0x24, 0xdf, + 0xca, 0x68, 0x58, 0x5f, 0x4c, 0xbe, 0x61, 0xf2, 0x71, 0x5a, 0x0e, 0x11, 0x98, 0xb7, 0x5c, 0x79, + 0x37, 0x09, 0x96, 0xec, 0xd6, 0xf4, 0x85, 0x4e, 0xe8, 0xc7, 0x91, 0x46, 0x97, 0x92, 0xc8, 0x34, + 0xaa, 0x43, 0xb1, 0xfb, 0x1d, 0xd3, 0x09, 0x37, 0x45, 0x89, 0xaf, 0xe9, 0x9d, 0xaf, 0x6d, 0xb7, + 0x7d, 0x1c, 0x8c, 0x23, 0x06, 0xc0, 0xe8, 0x1e, 0xf1, 0x8e, 0x2d, 0x83, 0x84, 0x2d, 0xee, 0xab, + 0xd3, 0x7a, 0x22, 0xf5, 0x13, 0xfd, 0x37, 0x5c, 0xcc, 0xfd, 0xc8, 0x36, 0x4e, 0xe0, 0xf0, 0x2b, + 0xd2, 0x27, 0xce, 0x4e, 0x6b, 0xf4, 0x22, 0x14, 0x78, 0xdb, 0x91, 0x2c, 0x3e, 0x17, 0x16, 0xc2, + 0xfd, 0x13, 0x97, 0x9c, 0x0e, 0xeb, 0x69, 0x0a, 0xf8, 0x20, 0x16, 0xe2, 0x53, 0x9f, 0xf5, 0xa2, + 0x82, 0x9b, 0x3f, 0xaf, 0x65, 0x16, 0x2e, 0xd2, 0x32, 0x7f, 0x55, 0xcc, 0x64, 0x0d, 0x2f, 0x5e, + 0xe8, 0x15, 0x28, 0x99, 0x96, 0xc7, 0xaf, 0x8d, 0xd4, 0x91, 0x81, 0xd6, 0x42, 0x67, 0xb7, 0xc3, + 0x89, 0xd3, 0xe4, 0x03, 0x8e, 0x15, 0x90, 0x01, 0x85, 0xae, 0x47, 0xfb, 0xf2, 0xcc, 0x74, 0xb1, + 0xca, 0xca, 0x93, 0x38, 0x0e, 0xfe, 0x8e, 0x47, 0xfb, 0x58, 0x18, 0x47, 0x6f, 0x43, 0x8e, 0x51, + 0xb1, 0x38, 0x97, 0x02, 0x01, 0x12, 0x22, 0xb7, 0x4f, 0x71, 0x8e, 0x51, 0x9e, 0xfe, 0x7e, 0x3a, + 0xe9, 0x6e, 0x3d, 0x61, 0xd2, 0xc5, 0xe9, 0x1f, 0x65, 0x5a, 0x64, 0x9a, 0x97, 0x05, 0x37, 0x53, + 0xb0, 0xe3, 0x9e, 0x39, 0x56, 0xe2, 0xef, 0xc1, 0xac, 0x1e, 0x70, 0x32, 0x2b, 0x38, 0xf9, 0x0a, + 0x3f, 0x3f, 0x6c, 0x85, 0x64, 0x6c, 0x3c, 0xe2, 0x0b, 0x8d, 0x67, 0x46, 0xdf, 0x4b, 0x34, 0xce, + 0x70, 0xa0, 0x84, 0xa5, 0x39, 0xf4, 0x32, 0x2c, 0x12, 0x47, 0x3f, 0xb0, 0xc9, 0x5d, 0xda, 0xeb, + 0x59, 0x4e, 0xaf, 0x3a, 0xb7, 0xa6, 0xac, 0xcf, 0x37, 0xaf, 0x49, 0x5f, 0x16, 0x77, 0x92, 0x93, + 0x38, 0x2d, 0x7b, 0x56, 0x87, 0x9b, 0x9f, 0xa2, 0xc3, 0x85, 0x79, 0x5e, 0x9a, 0x94, 0xe7, 0xea, + 0xcf, 0xf3, 0x80, 0x52, 0x8c, 0xf1, 0x9a, 0xea, 0xf3, 0x53, 0xfa, 0xa2, 0x93, 0x1c, 0x96, 0x5d, + 0xe3, 0xb2, 0xfa, 0x57, 0x14, 0x7d, 0x7a, 0x3e, 0x8d, 0x89, 0x5c, 0x28, 0x33, 0x4f, 0xef, 0x76, + 0x2d, 0x43, 0x78, 0x25, 0x93, 0xfe, 0xa5, 0x47, 0xf8, 0x20, 0x3e, 0x5f, 0x69, 0x11, 0x1d, 0xfb, + 0x09, 0xed, 0xf8, 0xe4, 0x96, 0x1c, 0xc5, 0x29, 0x04, 0xf4, 0xae, 0x02, 0x15, 0x7e, 0xb6, 0x48, + 0x8a, 0xc8, 0xcb, 0xef, 0x97, 0x1f, 0x1f, 0x16, 0x67, 0x2c, 0xc4, 0x37, 0xb1, 0xec, 0x0c, 0x1e, + 0x43, 0x53, 0xff, 0xae, 0xc0, 0xea, 0x18, 0x23, 0x83, 0xab, 0x78, 0x7f, 0x67, 0x43, 0x91, 0x77, + 0xc9, 0xb0, 0x27, 0xed, 0x5e, 0x88, 0xeb, 0xb8, 0x3f, 0xc7, 0x0d, 0x9d, 0x8f, 0xf9, 0x38, 0x00, + 0x51, 0x7f, 0x5f, 0x80, 0x4a, 0x28, 0xe4, 0xef, 0x0d, 0xfa, 0x7d, 0xdd, 0xbb, 0x8a, 0xb3, 0xe9, + 0x8f, 0x15, 0x58, 0x4e, 0x66, 0x99, 0x15, 0xc5, 0xdb, 0xbc, 0x50, 0xbc, 0x01, 0xd1, 0xd7, 0x25, + 0xf6, 0x72, 0x3b, 0x0d, 0x81, 0xb3, 0x98, 0xe8, 0x37, 0x0a, 0xdc, 0x08, 0x50, 0xe4, 0xcb, 0xda, + 0x8c, 0x86, 0xcc, 0xba, 0xcb, 0x70, 0xea, 0xb3, 0xd2, 0xa9, 0x1b, 0x5b, 0x8f, 0xc0, 0xc3, 0x8f, + 0xf4, 0x06, 0xfd, 0x52, 0x81, 0x6b, 0x81, 0x40, 0xd6, 0xcf, 0xc2, 0xa5, 0xf9, 0xf9, 0x69, 0xe9, + 0xe7, 0xb5, 0xad, 0xb3, 0x80, 0xf0, 0xd9, 0xf8, 0xaa, 0x0e, 0xe5, 0xe4, 0xeb, 0x86, 0xa7, 0xf1, + 0x6a, 0xe8, 0xb7, 0x0a, 0xcc, 0xc9, 0x06, 0x83, 0x6e, 0x26, 0x6e, 0x62, 0x01, 0x44, 0xf5, 0xfc, + 0x5b, 0x18, 0x6a, 0xcb, 0x3b, 0x60, 0xee, 0x9c, 0x9c, 0x1e, 0x30, 0xcb, 0xd6, 0x82, 0x0f, 0xcf, + 0x5a, 0xcb, 0x61, 0x6f, 0x7a, 0x7b, 0xcc, 0xb3, 0x9c, 0x5e, 0x73, 0x3e, 0x73, 0x63, 0xfc, 0x1c, + 0xcc, 0x11, 0x47, 0x5c, 0x2f, 0x45, 0x9b, 0x2e, 0x36, 0x17, 0x46, 0xc3, 0xfa, 0xdc, 0x4e, 0x30, + 0x84, 0xc3, 0x39, 0x95, 0x40, 0x25, 0x7b, 0x3c, 0x7b, 0x0a, 0xeb, 0xd3, 0x7c, 0xfe, 0xc1, 0xc3, + 0xda, 0xcc, 0xfb, 0x0f, 0x6b, 0x33, 0x1f, 0x3c, 0xac, 0xcd, 0xbc, 0x3b, 0xaa, 0x29, 0x0f, 0x46, + 0x35, 0xe5, 0xfd, 0x51, 0x4d, 0xf9, 0x60, 0x54, 0x53, 0xfe, 0x3a, 0xaa, 0x29, 0xbf, 0xf8, 0x5b, + 0x6d, 0xe6, 0xeb, 0x73, 0x92, 0xfa, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xbe, 0xe6, 0x40, 0xcb, + 0xef, 0x20, 0x00, 0x00, } func (m *AddressGroup) Marshal() (dAtA []byte, err error) { @@ -2023,6 +2025,20 @@ func (m *NetworkPolicyPeer) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.ToServices) > 0 { + for iNdEx := len(m.ToServices) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.ToServices[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } + } if len(m.FQDNs) > 0 { for iNdEx := len(m.FQDNs) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.FQDNs[iNdEx]) @@ -2897,6 +2913,12 @@ func (m *NetworkPolicyPeer) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) } } + if len(m.ToServices) > 0 { + for _, e := range m.ToServices { + l = e.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + } return n } @@ -3405,10 +3427,16 @@ func (this *NetworkPolicyPeer) String() string { repeatedStringForIPBlocks += strings.Replace(strings.Replace(f.String(), "IPBlock", "IPBlock", 1), `&`, ``, 1) + "," } repeatedStringForIPBlocks += "}" + repeatedStringForToServices := "[]ServiceReference{" + for _, f := range this.ToServices { + repeatedStringForToServices += strings.Replace(strings.Replace(f.String(), "ServiceReference", "ServiceReference", 1), `&`, ``, 1) + "," + } + repeatedStringForToServices += "}" s := strings.Join([]string{`&NetworkPolicyPeer{`, `AddressGroups:` + fmt.Sprintf("%v", this.AddressGroups) + `,`, `IPBlocks:` + repeatedStringForIPBlocks + `,`, `FQDNs:` + fmt.Sprintf("%v", this.FQDNs) + `,`, + `ToServices:` + repeatedStringForToServices + `,`, `}`, }, "") return s @@ -6338,6 +6366,40 @@ func (m *NetworkPolicyPeer) Unmarshal(dAtA []byte) error { } m.FQDNs = append(m.FQDNs, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ToServices", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ToServices = append(m.ToServices, ServiceReference{}) + if err := m.ToServices[len(m.ToServices)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/pkg/apis/controlplane/v1beta2/generated.proto b/pkg/apis/controlplane/v1beta2/generated.proto index 63e9cdc9890..497c46b9337 100644 --- a/pkg/apis/controlplane/v1beta2/generated.proto +++ b/pkg/apis/controlplane/v1beta2/generated.proto @@ -232,6 +232,10 @@ message NetworkPolicyPeer { // A list of exact FQDN names or FQDN wildcard expressions. // This field can only be possibly set for NetworkPolicyPeer of egress rules. repeated string fqdns = 3; + + // A list of ServiceReference. + // This field can only be possibly set for NetworkPolicyPeer of egress rules. + repeated ServiceReference toServices = 4; } message NetworkPolicyReference { diff --git a/pkg/apis/controlplane/v1beta2/types.go b/pkg/apis/controlplane/v1beta2/types.go index e717fb01937..ec15a700e03 100644 --- a/pkg/apis/controlplane/v1beta2/types.go +++ b/pkg/apis/controlplane/v1beta2/types.go @@ -272,6 +272,9 @@ type NetworkPolicyPeer struct { // A list of exact FQDN names or FQDN wildcard expressions. // This field can only be possibly set for NetworkPolicyPeer of egress rules. FQDNs []string `json:"fqdns,omitempty" protobuf:"bytes,3,rep,name=fqdns"` + // A list of ServiceReference. + // This field can only be possibly set for NetworkPolicyPeer of egress rules. + ToServices []ServiceReference `json:"toServices,omitempty" protobuf:"bytes,4,rep,name=toServices"` } // IPBlock describes a particular CIDR (Ex. "192.168.1.1/24"). The except entry describes CIDRs that should diff --git a/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go b/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go index 21994ec2d0b..351f9e867fc 100644 --- a/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go +++ b/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go @@ -843,6 +843,7 @@ func autoConvert_v1beta2_NetworkPolicyPeer_To_controlplane_NetworkPolicyPeer(in out.AddressGroups = *(*[]string)(unsafe.Pointer(&in.AddressGroups)) out.IPBlocks = *(*[]controlplane.IPBlock)(unsafe.Pointer(&in.IPBlocks)) out.FQDNs = *(*[]string)(unsafe.Pointer(&in.FQDNs)) + out.ToServices = *(*[]controlplane.ServiceReference)(unsafe.Pointer(&in.ToServices)) return nil } @@ -855,6 +856,7 @@ func autoConvert_controlplane_NetworkPolicyPeer_To_v1beta2_NetworkPolicyPeer(in out.AddressGroups = *(*[]string)(unsafe.Pointer(&in.AddressGroups)) out.IPBlocks = *(*[]IPBlock)(unsafe.Pointer(&in.IPBlocks)) out.FQDNs = *(*[]string)(unsafe.Pointer(&in.FQDNs)) + out.ToServices = *(*[]ServiceReference)(unsafe.Pointer(&in.ToServices)) return nil } diff --git a/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go b/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go index b27a5afda11..17c4b00a365 100644 --- a/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go +++ b/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go @@ -692,6 +692,11 @@ func (in *NetworkPolicyPeer) DeepCopyInto(out *NetworkPolicyPeer) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.ToServices != nil { + in, out := &in.ToServices, &out.ToServices + *out = make([]ServiceReference, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/controlplane/zz_generated.deepcopy.go b/pkg/apis/controlplane/zz_generated.deepcopy.go index 6e81d5122bf..829e0756c2b 100644 --- a/pkg/apis/controlplane/zz_generated.deepcopy.go +++ b/pkg/apis/controlplane/zz_generated.deepcopy.go @@ -692,6 +692,11 @@ func (in *NetworkPolicyPeer) DeepCopyInto(out *NetworkPolicyPeer) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.ToServices != nil { + in, out := &in.ToServices, &out.ToServices + *out = make([]ServiceReference, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/crd/v1alpha1/types.go b/pkg/apis/crd/v1alpha1/types.go index 2c5e8f6928a..d72531b251a 100644 --- a/pkg/apis/crd/v1alpha1/types.go +++ b/pkg/apis/crd/v1alpha1/types.go @@ -354,10 +354,17 @@ type Rule struct { // +optional From []NetworkPolicyPeer `json:"from"` // Rule is matched if traffic is intended for workloads selected by - // this field. If this field is empty or missing, this rule matches all - // destinations. + // this field. This field can't be used with ToServices. If this field + // and ToServices are both empty or missing this rule matches all destinations. // +optional To []NetworkPolicyPeer `json:"to"` + // Rule is matched if traffic is intended for a Service listed in this field. + // Currently only ClusterIP types Services are supported in this field. This field + // can only be used when AntreaProxy is enabled. This field can't be used with To + // or Ports. If this field and To are both empty or missing, this rule matches all + // destinations. + // +optional + ToServices []ServiceReference `json:"toServices"` // Name describes the intention of this rule. // Name should be unique within the policy. // +optional @@ -457,6 +464,14 @@ type NetworkPolicyPort struct { EndPort *int32 `json:"endPort,omitempty"` } +// ServiceReference represents a reference to a v1.Service. +type ServiceReference struct { + // Name of the Service + Name string `json:"name"` + // Namespace of the Service + Namespace string `json:"namespace,omitempty"` +} + // RuleAction describes the action to be applied on traffic matching a rule. type RuleAction string diff --git a/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go index 870173cdd18..a2a892fd7df 100644 --- a/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go @@ -500,6 +500,11 @@ func (in *Rule) DeepCopyInto(out *Rule) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ToServices != nil { + in, out := &in.ToServices, &out.ToServices + *out = make([]ServiceReference, len(*in)) + copy(*out, *in) + } if in.AppliedTo != nil { in, out := &in.AppliedTo, &out.AppliedTo *out = make([]NetworkPolicyPeer, len(*in)) @@ -520,6 +525,22 @@ func (in *Rule) DeepCopy() *Rule { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceReference) DeepCopyInto(out *ServiceReference) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceReference. +func (in *ServiceReference) DeepCopy() *ServiceReference { + if in == nil { + return nil + } + out := new(ServiceReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Source) DeepCopyInto(out *Source) { *out = *in diff --git a/pkg/apiserver/openapi/zz_generated.openapi.go b/pkg/apiserver/openapi/zz_generated.openapi.go index 6cb88e92aa3..96844d0c14a 100644 --- a/pkg/apiserver/openapi/zz_generated.openapi.go +++ b/pkg/apiserver/openapi/zz_generated.openapi.go @@ -1340,11 +1340,25 @@ func schema_pkg_apis_controlplane_v1beta2_NetworkPolicyPeer(ref common.Reference }, }, }, + "toServices": { + SchemaProps: spec.SchemaProps{ + Description: "A list of ServiceReference. This field can only be possibly set for NetworkPolicyPeer of egress rules.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("antrea.io/antrea/pkg/apis/controlplane/v1beta2.ServiceReference"), + }, + }, + }, + }, + }, }, }, }, Dependencies: []string{ - "antrea.io/antrea/pkg/apis/controlplane/v1beta2.IPBlock"}, + "antrea.io/antrea/pkg/apis/controlplane/v1beta2.IPBlock", "antrea.io/antrea/pkg/apis/controlplane/v1beta2.ServiceReference"}, } } diff --git a/pkg/controller/networkpolicy/antreanetworkpolicy.go b/pkg/controller/networkpolicy/antreanetworkpolicy.go index 4b36190564d..eb0f4d5c54f 100644 --- a/pkg/controller/networkpolicy/antreanetworkpolicy.go +++ b/pkg/controller/networkpolicy/antreanetworkpolicy.go @@ -168,9 +168,15 @@ func (n *NetworkPolicyController) processAntreaNetworkPolicy(np *crdv1alpha1.Net appliedToGroupNamesForRule = append(appliedToGroupNamesForRule, atGroup) appliedToGroupNamesSet.Insert(atGroup) } + var peers *controlplane.NetworkPolicyPeer + if egressRule.ToServices != nil { + peers = n.svcRefToPeerForCRD(egressRule.ToServices, np.Namespace) + } else { + peers = n.toAntreaPeerForCRD(egressRule.To, np, controlplane.DirectionOut, namedPortExists) + } rules = append(rules, controlplane.NetworkPolicyRule{ Direction: controlplane.DirectionOut, - To: *n.toAntreaPeerForCRD(egressRule.To, np, controlplane.DirectionOut, namedPortExists), + To: *peers, Services: services, Name: egressRule.Name, Action: egressRule.Action, diff --git a/pkg/controller/networkpolicy/antreanetworkpolicy_test.go b/pkg/controller/networkpolicy/antreanetworkpolicy_test.go index d15a6a9fe0c..35d4f181c42 100644 --- a/pkg/controller/networkpolicy/antreanetworkpolicy_test.go +++ b/pkg/controller/networkpolicy/antreanetworkpolicy_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -35,6 +36,20 @@ var ( func TestProcessAntreaNetworkPolicy(t *testing.T) { p10 := float64(10) + svcA := v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc1", + Namespace: "ns5", + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Port: 80, + TargetPort: int80, + }, + }, + }, + } allowAction := crdv1alpha1.RuleActionAllow protocolTCP := controlplane.ProtocolTCP tests := []struct { @@ -379,10 +394,64 @@ func TestProcessAntreaNetworkPolicy(t *testing.T) { expectedAppliedToGroups: 1, expectedAddressGroups: 1, }, + { + name: "rules-with-to-services", + inputPolicy: &crdv1alpha1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns5", Name: "npE", UID: "uidE"}, + Spec: crdv1alpha1.NetworkPolicySpec{ + AppliedTo: []crdv1alpha1.NetworkPolicyPeer{ + {PodSelector: &selectorA}, + }, + Priority: p10, + Egress: []crdv1alpha1.Rule{ + { + ToServices: []crdv1alpha1.ServiceReference{ + { + Namespace: "ns5", + Name: "svc1", + }, + }, + Action: &allowAction, + }, + }, + }, + }, + expectedPolicy: &antreatypes.NetworkPolicy{ + UID: "uidE", + Name: "uidE", + SourceRef: &controlplane.NetworkPolicyReference{ + Type: controlplane.AntreaNetworkPolicy, + Namespace: "ns5", + Name: "npE", + UID: "uidE", + }, + Priority: &p10, + TierPriority: &DefaultTierPriority, + Rules: []controlplane.NetworkPolicyRule{ + { + Direction: controlplane.DirectionOut, + To: controlplane.NetworkPolicyPeer{ + ToServices: []controlplane.ServiceReference{ + { + Namespace: "ns5", + Name: "svc1", + }, + }, + }, + Priority: 0, + Action: &allowAction, + }, + }, + AppliedToGroups: []string{getNormalizedUID(toGroupSelector("ns5", &selectorA, nil, nil).NormalizedName)}, + }, + expectedAppliedToGroups: 1, + expectedAddressGroups: 0, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { _, c := newController() + c.serviceStore.Add(&svcA) assert.Equal(t, tt.expectedPolicy, c.processAntreaNetworkPolicy(tt.inputPolicy)) assert.Equal(t, tt.expectedAddressGroups, len(c.addressGroupStore.List())) assert.Equal(t, tt.expectedAppliedToGroups, len(c.appliedToGroupStore.List())) diff --git a/pkg/controller/networkpolicy/clusternetworkpolicy.go b/pkg/controller/networkpolicy/clusternetworkpolicy.go index 5f120194470..e0b9b0487c8 100644 --- a/pkg/controller/networkpolicy/clusternetworkpolicy.go +++ b/pkg/controller/networkpolicy/clusternetworkpolicy.go @@ -325,7 +325,11 @@ func (n *NetworkPolicyController) processClusterNetworkPolicy(cnp *crdv1alpha1.C } ruleATGNames := n.processClusterAppliedTo(ruleAppliedTos, atgNamesSet) klog.V(4).Infof("Adding a new cluster-level rule with appliedTos %v for %s", ruleATGNames, cnp.Name) - addRule(n.toAntreaPeerForCRD(clusterPeers, cnp, direction, namedPortExists), direction, ruleATGNames) + if cnpRule.ToServices != nil { + addRule(n.svcRefToPeerForCRD(cnpRule.ToServices, ""), direction, ruleATGNames) + } else { + addRule(n.toAntreaPeerForCRD(clusterPeers, cnp, direction, namedPortExists), direction, ruleATGNames) + } } if len(perNSPeers) > 0 { if len(cnp.Spec.AppliedTo) > 0 { diff --git a/pkg/controller/networkpolicy/clusternetworkpolicy_test.go b/pkg/controller/networkpolicy/clusternetworkpolicy_test.go index 9647f40a0ed..13635fa1861 100644 --- a/pkg/controller/networkpolicy/clusternetworkpolicy_test.go +++ b/pkg/controller/networkpolicy/clusternetworkpolicy_test.go @@ -51,6 +51,21 @@ func TestProcessClusterNetworkPolicy(t *testing.T) { }, } + svcA := v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svcA", + Namespace: "nsA", + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Port: 80, + TargetPort: int80, + }, + }, + }, + } + allowAction := crdv1alpha1.RuleActionAllow dropAction := crdv1alpha1.RuleActionDrop protocolTCP := controlplane.ProtocolTCP @@ -885,6 +900,58 @@ func TestProcessClusterNetworkPolicy(t *testing.T) { expectedAppliedToGroups: 2, expectedAddressGroups: 2, }, + { + name: "rule-with-to-service", + inputPolicy: &crdv1alpha1.ClusterNetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "cnpK", UID: "uidK"}, + Spec: crdv1alpha1.ClusterNetworkPolicySpec{ + AppliedTo: []crdv1alpha1.NetworkPolicyPeer{ + {PodSelector: &selectorA}, + }, + Priority: p10, + Egress: []crdv1alpha1.Rule{ + { + ToServices: []crdv1alpha1.ServiceReference{ + { + Namespace: "nsA", + Name: "svcA", + }, + }, + Action: &dropAction, + }, + }, + }, + }, + expectedPolicy: &antreatypes.NetworkPolicy{ + UID: "uidK", + Name: "uidK", + SourceRef: &controlplane.NetworkPolicyReference{ + Type: controlplane.AntreaClusterNetworkPolicy, + Name: "cnpK", + UID: "uidK", + }, + Priority: &p10, + TierPriority: &DefaultTierPriority, + Rules: []controlplane.NetworkPolicyRule{ + { + Direction: controlplane.DirectionOut, + To: controlplane.NetworkPolicyPeer{ + ToServices: []controlplane.ServiceReference{ + { + Namespace: "nsA", + Name: "svcA", + }, + }, + }, + Priority: 0, + Action: &dropAction, + }, + }, + AppliedToGroups: []string{getNormalizedUID(toGroupSelector("", &selectorA, nil, nil).NormalizedName)}, + }, + expectedAppliedToGroups: 1, + expectedAddressGroups: 0, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -893,6 +960,7 @@ func TestProcessClusterNetworkPolicy(t *testing.T) { c.cgStore.Add(&cgA) c.namespaceStore.Add(&nsA) c.namespaceStore.Add(&nsB) + c.serviceStore.Add(&svcA) if tt.inputPolicy.Spec.Tier != "" { c.tierStore.Add(&tierA) } @@ -907,6 +975,7 @@ func TestProcessClusterNetworkPolicy(t *testing.T) { assert.ElementsMatch(t, tt.expectedPolicy.PerNamespaceSelectors, actualPolicy.PerNamespaceSelectors) assert.ElementsMatch(t, tt.expectedPolicy.AppliedToGroups, actualPolicy.AppliedToGroups) assert.Equal(t, tt.expectedAppliedToGroups, len(c.appliedToGroupStore.List())) + assert.Equal(t, tt.expectedAddressGroups, len(c.addressGroupStore.List())) }) } } diff --git a/pkg/controller/networkpolicy/crd_utils.go b/pkg/controller/networkpolicy/crd_utils.go index ecfcbefbe80..ea283c5aa6e 100644 --- a/pkg/controller/networkpolicy/crd_utils.go +++ b/pkg/controller/networkpolicy/crd_utils.go @@ -134,6 +134,25 @@ func (n *NetworkPolicyController) toNamespacedPeerForCRD(peers []v1alpha1.Networ return &controlplane.NetworkPolicyPeer{AddressGroups: addressGroups} } +// svcRefToPeerForCRD creates an Antrea controlplane NetworkPolicyPeer from +// ServiceReference in ToServices field. For ANP, we will use the +// defaultNamespace(policy Namespace) as the Namespace of ServiceReference that +// doesn't set Namespace. +func (n *NetworkPolicyController) svcRefToPeerForCRD(svcRefs []v1alpha1.ServiceReference, defaultNamespace string) *controlplane.NetworkPolicyPeer { + var controlplaneSvcRefs []controlplane.ServiceReference + for _, svcRef := range svcRefs { + curNS := defaultNamespace + if svcRef.Namespace != "" { + curNS = svcRef.Namespace + } + controlplaneSvcRefs = append(controlplaneSvcRefs, controlplane.ServiceReference{ + Namespace: curNS, + Name: svcRef.Name, + }) + } + return &controlplane.NetworkPolicyPeer{ToServices: controlplaneSvcRefs} +} + // createAppliedToGroupForClusterGroupCRD creates an AppliedToGroup object corresponding to a // internal Group. If the AppliedToGroup already exists, it returns the key // otherwise it copies the internal Group contents to an AppliedToGroup resource and returns diff --git a/pkg/controller/networkpolicy/validate.go b/pkg/controller/networkpolicy/validate.go index c0974129b37..0fe1b74444a 100644 --- a/pkg/controller/networkpolicy/validate.go +++ b/pkg/controller/networkpolicy/validate.go @@ -31,6 +31,7 @@ import ( crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" crdv1alpha2 "antrea.io/antrea/pkg/apis/crd/v1alpha2" "antrea.io/antrea/pkg/controller/networkpolicy/store" + "antrea.io/antrea/pkg/features" "antrea.io/antrea/pkg/util/env" ) @@ -482,6 +483,14 @@ func (v *antreaPolicyValidator) validatePeers(ingress, egress []crdv1alpha1.Rule } } for _, rule := range egress { + if rule.ToServices != nil { + if !features.DefaultFeatureGate.Enabled(features.AntreaProxy) { + return fmt.Sprintf("`toServices` can only be used when AntreaProxy is enabled"), false + } + if rule.To != nil || rule.Ports != nil { + return fmt.Sprintf("`toServices` can't be used with `to` or `ports`"), false + } + } msg, isValid := checkPeers(rule.To) if !isValid { return msg, false diff --git a/test/e2e/antreapolicy_test.go b/test/e2e/antreapolicy_test.go index 6e22210f5fe..5c13bf2ee14 100644 --- a/test/e2e/antreapolicy_test.go +++ b/test/e2e/antreapolicy_test.go @@ -129,10 +129,12 @@ type TestStep struct { CustomProbes []*CustomProbe } -// fqdnTestStep is a single unit of testing spec for FQDN policy tests. -type fqdnTestStep struct { +// podToAddrTestStep is a single unit of testing the connectivity from a Pod to an +// arbitrary destination address. +type podToAddrTestStep struct { clientPod Pod - fqdnToQuery string + destAddr string + destPort int32 expectedConnectivity PodConnectivityMark } @@ -2368,25 +2370,29 @@ func testFQDNPolicy(t *testing.T) { builder.AddFQDNRule("*google.com", v1.ProtocolTCP, nil, nil, nil, "r1", nil, crdv1alpha1.RuleActionReject) builder.AddFQDNRule("wayfair.com", v1.ProtocolTCP, nil, nil, nil, "r2", nil, crdv1alpha1.RuleActionDrop) - testcases := []fqdnTestStep{ + testcases := []podToAddrTestStep{ { "x/a", "drive.google.com", + 80, Rejected, }, { "x/b", "maps.google.com", + 80, Rejected, }, { "y/a", "wayfair.com", + 80, Dropped, }, { "y/b", "facebook.com", + 80, Connected, }, } @@ -2394,14 +2400,14 @@ func testFQDNPolicy(t *testing.T) { failOnError(err, t) time.Sleep(networkPolicyDelay) for _, tc := range testcases { - log.Tracef("Probing: %s -> %s", tc.clientPod.PodName(), tc.fqdnToQuery) - connectivity, err := k8sUtils.ProbeEgress(tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.fqdnToQuery, 80, v1.ProtocolTCP) + log.Tracef("Probing: %s -> %s", tc.clientPod.PodName(), tc.destAddr) + connectivity, err := k8sUtils.ProbeEgress(tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.destAddr, tc.destPort, v1.ProtocolTCP) if err != nil { t.Errorf("failure -- could not complete probe: %v", err) } if connectivity != tc.expectedConnectivity { t.Errorf("failure -- wrong results for probe: Source %s/%s --> Dest %s connectivity: %v, expected: %v", - tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.fqdnToQuery, connectivity, tc.expectedConnectivity) + tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.destAddr, connectivity, tc.expectedConnectivity) } } // cleanup test resources @@ -2455,22 +2461,25 @@ func testFQDNPolicyInClusterService(t *testing.T) { k8sUtils.CreateOrUpdateACNP(acnp) failOnError(waitForResourceReady(acnp, timeout), t) - var testcases []fqdnTestStep + var testcases []podToAddrTestStep for _, service := range services { - eachServiceCases := []fqdnTestStep{ + eachServiceCases := []podToAddrTestStep{ { "y/b", svcDNSName(service), + 80, Rejected, }, { "z/c", svcDNSName(service), + 80, Dropped, }, { "x/c", svcDNSName(service), + 80, Connected, }, } @@ -2478,14 +2487,14 @@ func testFQDNPolicyInClusterService(t *testing.T) { } for _, tc := range testcases { - log.Tracef("Probing: %s -> %s", tc.clientPod.PodName(), tc.fqdnToQuery) - connectivity, err := k8sUtils.ProbeEgress(tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.fqdnToQuery, 80, v1.ProtocolTCP) + log.Tracef("Probing: %s -> %s", tc.clientPod.PodName(), tc.destAddr) + connectivity, err := k8sUtils.ProbeEgress(tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.destAddr, tc.destPort, v1.ProtocolTCP) if err != nil { t.Errorf("failure -- could not complete probe: %v", err) } if connectivity != tc.expectedConnectivity { t.Errorf("failure -- wrong results for probe: Source %s/%s --> Dest %s connectivity: %v, expected: %v", - tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.fqdnToQuery, connectivity, tc.expectedConnectivity) + tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.destAddr, connectivity, tc.expectedConnectivity) } } // cleanup test resources @@ -2498,6 +2507,83 @@ func testFQDNPolicyInClusterService(t *testing.T) { time.Sleep(networkPolicyDelay) } +func testToServices(t *testing.T) { + skipIfProxyDisabled(t) + var services []*v1.Service + if clusterInfo.podV4NetworkCIDR != "" { + ipv4Svc := k8sUtils.BuildService("ipv4-svc", "x", 81, 81, map[string]string{"pod": "a"}, nil) + ipv4Svc.Spec.IPFamilies = []v1.IPFamily{v1.IPv4Protocol} + services = append(services, ipv4Svc) + } + if clusterInfo.podV6NetworkCIDR != "" { + ipv6Svc := k8sUtils.BuildService("ipv6-svc", "x", 80, 80, map[string]string{"pod": "b"}, nil) + ipv6Svc.Spec.IPFamilies = []v1.IPFamily{v1.IPv6Protocol} + services = append(services, ipv6Svc) + } + + var svcRefs []crdv1alpha1.ServiceReference + var builtSvcs []*v1.Service + for _, service := range services { + builtSvc, _ := k8sUtils.CreateOrUpdateService(service) + failOnError(waitForResourceReady(service, timeout), t) + svcRefs = append(svcRefs, crdv1alpha1.ServiceReference{ + Name: service.Name, + Namespace: service.Namespace, + }) + builtSvcs = append(builtSvcs, builtSvc) + } + + builder := &ClusterNetworkPolicySpecBuilder{} + builder = builder.SetName("test-acnp-to-services"). + SetTier("application"). + SetPriority(1.0) + builder.AddToServicesRule(svcRefs, "svc", []ACNPAppliedToSpec{{NSSelector: map[string]string{"ns": "y"}}}, crdv1alpha1.RuleActionDrop) + time.Sleep(networkPolicyDelay) + + acnp := builder.Get() + k8sUtils.CreateOrUpdateACNP(acnp) + failOnError(waitForResourceReady(acnp, timeout), t) + + var testcases []podToAddrTestStep + for _, service := range builtSvcs { + eachServiceCases := []podToAddrTestStep{ + { + "y/b", + service.Spec.ClusterIP, + service.Spec.Ports[0].Port, + Dropped, + }, + { + "z/c", + service.Spec.ClusterIP, + service.Spec.Ports[0].Port, + Connected, + }, + } + testcases = append(testcases, eachServiceCases...) + } + + for _, tc := range testcases { + log.Tracef("Probing: %s -> %s", tc.clientPod.PodName(), tc.destAddr) + connectivity, err := k8sUtils.ProbeEgress(tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.destAddr, tc.destPort, v1.ProtocolTCP) + if err != nil { + t.Errorf("failure -- could not complete probe: %v", err) + } + if connectivity != tc.expectedConnectivity { + t.Errorf("failure -- wrong results for probe: Source %s/%s --> Dest %s:%d connectivity: %v, expected: %v", + tc.clientPod.Namespace(), tc.clientPod.PodName(), tc.destAddr, tc.destPort, connectivity, tc.expectedConnectivity) + } + } + // cleanup test resources + failOnError(k8sUtils.DeleteACNP(builder.Name), t) + failOnError(waitForResourceDelete("", builder.Name, resourceACNP, timeout), t) + time.Sleep(networkPolicyDelay) + for _, service := range services { + failOnError(k8sUtils.DeleteService(service.Namespace, service.Name), t) + failOnError(waitForResourceDelete(service.Namespace, service.Name, resourceSVC, timeout), t) + } +} + // executeTests runs all the tests in testList and prints results func executeTests(t *testing.T, testList []*TestCase) { executeTestsWithData(t, testList, nil) @@ -2842,6 +2928,7 @@ func TestAntreaPolicy(t *testing.T) { t.Run("Case=ACNPNestedClusterGroup", func(t *testing.T) { testACNPNestedClusterGroupCreateAndUpdate(t, data) }) t.Run("Case=ACNPFQDNPolicy", func(t *testing.T) { testFQDNPolicy(t) }) t.Run("Case=FQDNPolicyInCluster", func(t *testing.T) { testFQDNPolicyInClusterService(t) }) + t.Run("Case=ACNPToServices", func(t *testing.T) { testToServices(t) }) }) // print results for reachability tests printResults() diff --git a/test/e2e/utils/cnpspecbuilder.go b/test/e2e/utils/cnpspecbuilder.go index c23294c3146..210f8c401f6 100644 --- a/test/e2e/utils/cnpspecbuilder.go +++ b/test/e2e/utils/cnpspecbuilder.go @@ -274,6 +274,22 @@ func (b *ClusterNetworkPolicySpecBuilder) AddFQDNRule(fqdn string, return b } +func (b *ClusterNetworkPolicySpecBuilder) AddToServicesRule(svcRefs []crdv1alpha1.ServiceReference, + name string, ruleAppliedToSpecs []ACNPAppliedToSpec, action crdv1alpha1.RuleAction) *ClusterNetworkPolicySpecBuilder { + var appliedTos []crdv1alpha1.NetworkPolicyPeer + for _, at := range ruleAppliedToSpecs { + appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, at.NSSelector, at.PodSelectorMatchExp, at.NSSelectorMatchExp, at.Group)) + } + newRule := crdv1alpha1.Rule{ + ToServices: svcRefs, + Action: &action, + Name: name, + AppliedTo: appliedTos, + } + b.Spec.Egress = append(b.Spec.Egress, newRule) + return b +} + // AddEgressDNS mutates the nth policy rule to allow DNS, convenience method func (b *ClusterNetworkPolicySpecBuilder) WithEgressDNS() *ClusterNetworkPolicySpecBuilder { protocolUDP := v1.ProtocolUDP diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index d08572a5579..0392b5e8f1f 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -650,7 +650,7 @@ func expectedProxyServiceGroupAndFlows(gid uint32, svc svcConfig, endpointList [ svcFlows := expectTableFlows{tableID: 41, flows: []*ofTestUtils.ExpectFlow{ { MatchStr: fmt.Sprintf("priority=200,%s,reg4=0x10000/0x70000,nw_dst=%s,tp_dst=%d", string(svc.protocol), svc.ip.String(), svc.port), - ActStr: fmt.Sprintf("load:0x%x->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[19],group:%d", serviceLearnReg, gid), + ActStr: fmt.Sprintf("load:0x%x->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[19],load:0x%x->NXM_NX_REG7[],group:%d", serviceLearnReg, gid, gid), }, { MatchStr: fmt.Sprintf("priority=190,%s,reg4=0x30000/0x70000,nw_dst=%s,tp_dst=%d", string(svc.protocol), svc.ip.String(), svc.port), From 59205cf2c5e7cf7e7b8a6e909bb62fcf592a4bf1 Mon Sep 17 00:00:00 2001 From: wgrayson Date: Wed, 20 Oct 2021 14:49:46 -0700 Subject: [PATCH 2/2] Make ToServices omitempty Signed-off-by: wgrayson --- pkg/apis/crd/v1alpha1/types.go | 2 +- pkg/controller/networkpolicy/validate.go | 2 +- test/e2e/utils/cnpspecbuilder.go | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/apis/crd/v1alpha1/types.go b/pkg/apis/crd/v1alpha1/types.go index d72531b251a..62114dcedb8 100644 --- a/pkg/apis/crd/v1alpha1/types.go +++ b/pkg/apis/crd/v1alpha1/types.go @@ -364,7 +364,7 @@ type Rule struct { // or Ports. If this field and To are both empty or missing, this rule matches all // destinations. // +optional - ToServices []ServiceReference `json:"toServices"` + ToServices []ServiceReference `json:"toServices,omitempty"` // Name describes the intention of this rule. // Name should be unique within the policy. // +optional diff --git a/pkg/controller/networkpolicy/validate.go b/pkg/controller/networkpolicy/validate.go index 0fe1b74444a..46482d43418 100644 --- a/pkg/controller/networkpolicy/validate.go +++ b/pkg/controller/networkpolicy/validate.go @@ -487,7 +487,7 @@ func (v *antreaPolicyValidator) validatePeers(ingress, egress []crdv1alpha1.Rule if !features.DefaultFeatureGate.Enabled(features.AntreaProxy) { return fmt.Sprintf("`toServices` can only be used when AntreaProxy is enabled"), false } - if rule.To != nil || rule.Ports != nil { + if (rule.To != nil && len(rule.To) > 0) || rule.Ports != nil { return fmt.Sprintf("`toServices` can't be used with `to` or `ports`"), false } } diff --git a/test/e2e/utils/cnpspecbuilder.go b/test/e2e/utils/cnpspecbuilder.go index 210f8c401f6..91788dec02b 100644 --- a/test/e2e/utils/cnpspecbuilder.go +++ b/test/e2e/utils/cnpspecbuilder.go @@ -281,6 +281,7 @@ func (b *ClusterNetworkPolicySpecBuilder) AddToServicesRule(svcRefs []crdv1alpha appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, at.NSSelector, at.PodSelectorMatchExp, at.NSSelectorMatchExp, at.Group)) } newRule := crdv1alpha1.Rule{ + To: make([]crdv1alpha1.NetworkPolicyPeer, 0), ToServices: svcRefs, Action: &action, Name: name,