From 97b8e582c272af03993faa00e76ed05a174d84a5 Mon Sep 17 00:00:00 2001 From: Hongliang Liu Date: Wed, 13 Apr 2022 15:14:25 +0800 Subject: [PATCH 1/2] Bump antrea.io/ofnet from v0.5.5 to v0.5.7 Fix #3583 Signed-off-by: Hongliang Liu --- go.mod | 4 ++-- go.sum | 9 ++++----- plugins/octant/go.sum | 5 ++--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index d0ac58a1bb0..b2920a29689 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module antrea.io/antrea go 1.17 require ( - antrea.io/libOpenflow v0.6.1 - antrea.io/ofnet v0.2.3 + antrea.io/libOpenflow v0.6.2 + antrea.io/ofnet v0.5.7 github.com/Mellanox/sriovnet v1.0.2 github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331 github.com/Microsoft/hcsshim v0.8.9 diff --git a/go.sum b/go.sum index dd5b13ae5ae..c9fe59e8716 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,7 @@ -antrea.io/libOpenflow v0.5.2/go.mod h1:CzEJZxDNAupiGxeL5VOw92PsxfyvehEAvE3PiC6gr8o= -antrea.io/libOpenflow v0.6.1 h1:RjYKz8WJTjerqu/J9ASygv+oF7Bu0jhIiqLnIIyZPhs= -antrea.io/libOpenflow v0.6.1/go.mod h1:CzEJZxDNAupiGxeL5VOw92PsxfyvehEAvE3PiC6gr8o= -antrea.io/ofnet v0.2.3 h1:wxXOqWaT5swtn9Ly6hV7pqvIgfmrr3aQfCGVQqHykr4= -antrea.io/ofnet v0.2.3/go.mod h1:jW4ICTvGjLO+Qr6GG/Glmjy34k6k/TfVlQhOm76UH84= +antrea.io/libOpenflow v0.6.2 h1:1JMSJ7Lp7yOhKybHey9VDtRI6JuIgkhUWJBX5GIFY9I= +antrea.io/libOpenflow v0.6.2/go.mod h1:CzEJZxDNAupiGxeL5VOw92PsxfyvehEAvE3PiC6gr8o= +antrea.io/ofnet v0.5.7 h1:x0q0lZqp05wu01gk1+S5S15FmIpmTGPhi/z/aIDmuMw= +antrea.io/ofnet v0.5.7/go.mod h1:8TJVF6MLe9/gZ/KbhGUvULs9/TxssepEaYEe+o1SEgs= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= diff --git a/plugins/octant/go.sum b/plugins/octant/go.sum index c85b029bb6b..286874fa82e 100644 --- a/plugins/octant/go.sum +++ b/plugins/octant/go.sum @@ -1,6 +1,5 @@ -antrea.io/libOpenflow v0.5.2/go.mod h1:CzEJZxDNAupiGxeL5VOw92PsxfyvehEAvE3PiC6gr8o= -antrea.io/libOpenflow v0.6.1/go.mod h1:CzEJZxDNAupiGxeL5VOw92PsxfyvehEAvE3PiC6gr8o= -antrea.io/ofnet v0.2.3/go.mod h1:jW4ICTvGjLO+Qr6GG/Glmjy34k6k/TfVlQhOm76UH84= +antrea.io/libOpenflow v0.6.2/go.mod h1:CzEJZxDNAupiGxeL5VOw92PsxfyvehEAvE3PiC6gr8o= +antrea.io/ofnet v0.5.7/go.mod h1:8TJVF6MLe9/gZ/KbhGUvULs9/TxssepEaYEe+o1SEgs= bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= From e7581b082108800e3bb3af0ffdea7186549d91d6 Mon Sep 17 00:00:00 2001 From: Hongliang Liu Date: Wed, 23 Mar 2022 16:52:59 +0800 Subject: [PATCH 2/2] Fix the issue of local probe bypassing flows on Windows When proxyAll is enabled, kube-proxy can be replaced by AntreaProxy, then Service traffic and non-Service traffic can be distinguished by ServiceCTMark and NotServiceCTMark. Service traffic with ServiceCTMark should not bypass Network Policies, and non-Service traffic generated by kubelet with NotServiceCTMark should bypass Network Policies. Signed-off-by: Hongliang Liu --- pkg/agent/openflow/client.go | 2 +- pkg/agent/openflow/fields.go | 6 ++-- pkg/agent/openflow/pipeline.go | 47 ++++++++++++++++--------- test/integration/agent/openflow_test.go | 2 +- 4 files changed, 36 insertions(+), 21 deletions(-) diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index ad577481ad4..17275acbd8f 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -713,7 +713,7 @@ func (c *client) InstallGatewayFlows() error { } // Add flow to ensure the liveness check packet could be forwarded correctly. - flows = append(flows, c.localProbeFlow(gatewayIPs, cookie.Default)...) + flows = append(flows, c.localProbeFlows(gatewayIPs, cookie.Default)...) flows = append(flows, c.l3FwdFlowToGateway(gatewayIPs, gatewayConfig.MAC, cookie.Default)...) if err := c.ofEntryOperations.AddAll(flows); err != nil { diff --git a/pkg/agent/openflow/fields.go b/pkg/agent/openflow/fields.go index e608cd1e3f5..15f0b1f07d4 100644 --- a/pkg/agent/openflow/fields.go +++ b/pkg/agent/openflow/fields.go @@ -155,8 +155,10 @@ var ( FromGatewayCTMark = binding.NewCTMark(ConnSourceCTMarkField, fromGatewayVal) FromBridgeCTMark = binding.NewCTMark(ConnSourceCTMarkField, fromBridgeVal) - // CTMark[4]: Mark to indicate DNAT is performed on the connection for Service. - ServiceCTMark = binding.NewOneBitCTMark(4) + // CTMark[4]: Marks to indicate whether DNAT is performed on the connection for Service. + // These CT marks are used in CtZone / CtZoneV6 and SNATCtZone / SNATCtZoneV6. + ServiceCTMark = binding.NewOneBitCTMark(4) + NotServiceCTMark = binding.NewOneBitZeroCTMark(4) ) // Fields using CT label. diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index 0095080af1b..e34b195f672 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -2046,34 +2046,47 @@ func (c *client) dnsPacketInFlow(conjunctionID uint32) binding.Flow { Done() } -// localProbeFlow generates the flow to forward locally generated packets to ConntrackCommitTable, bypassing ingress -// rules of Network Policies. The packets are sent by kubelet to probe the liveness/readiness of local Pods. -// On Linux and when OVS kernel datapath is used, it identifies locally generated packets by matching the -// HostLocalSourceMark, otherwise it matches the source IP. The difference is because: -// 1. On Windows, kube-proxy userspace mode is used, and currently there is no way to distinguish kubelet generated -// traffic from kube-proxy proxied traffic. +// localProbeFlows generates the flows to forward locally generated request packets to stageConntrack directly, bypassing +// ingress rules of Network Policies. The packets are sent by kubelet to probe the liveness/readiness of local Pods. +// On Linux and when OVS kernel datapath is used, the probe packets are identified by matching the HostLocalSourceMark. +// On Windows or when OVS userspace (netdev) datapath is used, we need a different approach because: +// 1. On Windows, kube-proxy userspace mode is used, and currently there is no way to distinguish kubelet generated traffic +// from kube-proxy proxied traffic. // 2. pkt_mark field is not properly supported for OVS userspace (netdev) datapath. -// Note that there is a defect in the latter way that NodePort Service access by external clients will be masqueraded as -// a local gateway IP to bypass Network Policies. See https://github.com/antrea-io/antrea/issues/280. -// TODO: Fix it after replacing kube-proxy with AntreaProxy. -func (c *client) localProbeFlow(localGatewayIPs []net.IP, category cookie.Category) []binding.Flow { +// When proxyAll is disabled, the probe packets are identified by matching the source IP is the Antrea gateway IP; +// otherwise, the packets are identified by matching both the Antrea gateway IP and NotServiceCTMark. Note that, when +// proxyAll is disabled, currently there is no way to distinguish kubelet generated traffic from kube-proxy proxied traffic +// only by matching the Antrea gateway IP. There is a defect that NodePort Service access by external clients will be +// masqueraded as the Antrea gateway IP to bypass NetworkPolicies. See https://github.com/antrea-io/antrea/issues/280. +func (c *client) localProbeFlows(localGatewayIPs []net.IP, category cookie.Category) []binding.Flow { var flows []binding.Flow if runtime.IsWindowsPlatform() || c.ovsDatapathType == ovsconfig.OVSDatapathNetdev { for _, ip := range localGatewayIPs { ipProtocol := getIPProtocol(ip) - flows = append(flows, IngressRuleTable.BuildFlow(priorityHigh). + fb := IngressRuleTable.BuildFlow(priorityHigh). MatchProtocol(ipProtocol). + MatchCTStateRpl(false). + MatchCTStateTrk(true). MatchSrcIP(ip). + Cookie(c.cookieAllocator.Request(category).Raw()) + if c.proxyAll { + fb = fb.MatchCTMark(NotServiceCTMark) + } + flows = append(flows, + fb.Action().GotoTable(ConntrackCommitTable.GetID()). + Done()) + } + } else { + for _, ipProtocol := range c.ipProtocols { + flows = append(flows, IngressRuleTable.BuildFlow(priorityHigh). + MatchProtocol(ipProtocol). + MatchCTStateRpl(false). + MatchCTStateTrk(true). + MatchPktMark(types.HostLocalSourceMark, &types.HostLocalSourceMark). Action().GotoTable(ConntrackCommitTable.GetID()). Cookie(c.cookieAllocator.Request(category).Raw()). Done()) } - } else { - flows = append(flows, IngressRuleTable.BuildFlow(priorityHigh). - MatchPktMark(types.HostLocalSourceMark, &types.HostLocalSourceMark). - Action().GotoTable(ConntrackCommitTable.GetID()). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done()) } return flows } diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index d887a232d30..7fc8d3dc154 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -1191,7 +1191,7 @@ func prepareGatewayFlows(gwIPs []net.IP, gwMAC net.HardwareAddr, vMAC net.Hardwa tableName: "IngressRule", flows: []*ofTestUtils.ExpectFlow{ { - MatchStr: fmt.Sprintf("priority=210,%s,%s=%s", ipProtoStr, nwSrcStr, gwIP.String()), + MatchStr: fmt.Sprintf("priority=210,ct_state=-rpl+trk,%s,%s=%s", ipProtoStr, nwSrcStr, gwIP.String()), ActStr: "goto_table:ConntrackCommit", }, },