Allow forwarding of all traffic inside the Kubernetes network

For Longhorn and other pods to function properly on the Kubernetes
"cni0" network, we should allow:
- all DNS traffic
- forwarding from pods to services
- forwarding between pods

But we may need to adjust these rules later as we continue developing
the Kubernetes clustering support...

Signed-off-by: Milan Lenco <milan@zededa.com>

pkg/pillar/dpcreconciler/linux.go

@@ -148,6 +148,10 @@ const (
 	// Network bridge used by Kubernetes CNI.
 	// Currently, this is hardcoded for the Flannel CNI plugin.
 	kubeCNIBridge = "cni0"
+	// CIDR used for IP allocation for K3s pods.
+	kubePodCIDR = "10.42.0.0/16"
+	// CIDR used for IP allocation for K3s services.
+	kubeSvcCIDR = "10.43.0.0/16"
 )
 
 // LinuxDpcReconciler is a DPC-reconciler for Linux network stack,
@@ -1679,22 +1683,39 @@ func (r *LinuxDpcReconciler) getIntendedACLs(
 		TargetOpts:  []string{"--set-mark", controlProtoMark("in_dhcp")},
 		Description: "Mark ingress DHCP traffic",
 	}
-	// Allow kubernetes DNS replies from an external server.
-	// XXX Maybe there is a better way to setup this, like using set-mark for outbound
-	// kubernetes DNS queries.
-	markDNS := iptables.Rule{
-		RuleLabel:   "Incoming DNS replies",
-		MatchOpts:   []string{"-p", "udp", "--sport", "domain"},
+	// Allow all traffic from Kubernetes pods to Kubernetes services.
+	// Note that traffic originating from another node is already D-NATed
+	// and will get marked with the kube_pod mark.
+	markKubeSvc := iptables.Rule{
+		RuleLabel:   "Kubernetes service mark",
+		MatchOpts:   []string{"-i", kubeCNIBridge, "-s", kubePodCIDR, "-d", kubeSvcCIDR},
 		Target:      "CONNMARK",
-		TargetOpts:  []string{"--set-mark", controlProtoMark("in_dns")},
-		Description: "Incoming DNS replies (used to allow kubernetes DNS replies from external server)",
+		TargetOpts:  []string{"--set-mark", controlProtoMark("kube_svc")},
+		Description: "Mark traffic from Kubernetes pods to Kubernetes services",
+	}
+	// Allow all traffic forwarded between Kubernetes pods.
+	markKubePod := iptables.Rule{
+		RuleLabel:   "Kubernetes pod mark",
+		MatchOpts:   []string{"-s", kubePodCIDR, "-d", kubePodCIDR},
+		Target:      "CONNMARK",
+		TargetOpts:  []string{"--set-mark", controlProtoMark("kube_pod")},
+		Description: "Mark all traffic directly forwarded between Kubernetes pods",
+	}
+	// Allow all DNS requests made from the Kubernetes network.
+	markKubeDNS := iptables.Rule{
+		RuleLabel:     "Kubernetes DNS mark",
+		MatchOpts:     []string{"-s", kubePodCIDR, "-p", "udp", "--dport", "domain"},
+		Target:        "CONNMARK",
+		TargetOpts:    []string{"--set-mark", controlProtoMark("kube_dns")},
+		AppliedBefore: []string{markKubeSvc.RuleLabel, markKubePod.RuleLabel},
+		Description:   "Mark DNS requests made from the Kubernetes network",
 	}
 
 	protoMarkV4Rules := []iptables.Rule{
 		markSSHAndGuacamole, markVnc, markIcmpV4, markDhcp,
 	}
 	if r.HVTypeKube {
-		protoMarkV4Rules = append(protoMarkV4Rules, markDNS)
+		protoMarkV4Rules = append(protoMarkV4Rules, markKubeDNS, markKubeSvc, markKubePod)
 	}
 	protoMarkV6Rules := []iptables.Rule{
 		markSSHAndGuacamole, markVnc, markIcmpV6,
@@ -1843,6 +1864,44 @@ func (r *LinuxDpcReconciler) getIntendedACLs(
 	intendedIPv4ACLs.PutItem(denyNonAppForwarding, nil)
 	denyNonAppForwarding.ForIPv6 = true
 	intendedIPv6ACLs.PutItem(denyNonAppForwarding, nil)
+	if r.HVTypeKube {
+		// Kubernetes network is an exception where we allow forwarding
+		// for most of the traffic.
+		allowKubeDNSForwarding := iptables.Rule{
+			RuleLabel: "Allow Kubernetes DNS forwarding",
+			Table:     "mangle",
+			ChainName: "FORWARD" + iptables.DeviceChainSuffix,
+			MatchOpts: []string{"--match", "connmark", "--mark",
+				controlProtoMark("kube_dns")},
+			Target:        "ACCEPT",
+			AppliedBefore: []string{denyNonAppForwarding.RuleLabel},
+			Description:   "Allow forwarding of DNS traffic inside the Kubernetes network",
+		}
+		intendedIPv4ACLs.PutItem(allowKubeDNSForwarding, nil)
+		allowKubeSvcForwarding := iptables.Rule{
+			RuleLabel: "Allow forwarding to Kubernetes services",
+			Table:     "mangle",
+			ChainName: "FORWARD" + iptables.DeviceChainSuffix,
+			MatchOpts: []string{"--match", "connmark", "--mark",
+				controlProtoMark("kube_svc")},
+			Target:        "ACCEPT",
+			AppliedBefore: []string{denyNonAppForwarding.RuleLabel},
+			Description: "Allow forwarding of all traffic from Kubernetes pods " +
+				"to Kubernetes services",
+		}
+		intendedIPv4ACLs.PutItem(allowKubeSvcForwarding, nil)
+		allowKubePodForwarding := iptables.Rule{
+			RuleLabel: "Allow forwarding between Kubernetes pods",
+			Table:     "mangle",
+			ChainName: "FORWARD" + iptables.DeviceChainSuffix,
+			MatchOpts: []string{"--match", "connmark", "--mark",
+				controlProtoMark("kube_pod")},
+			Target:        "ACCEPT",
+			AppliedBefore: []string{denyNonAppForwarding.RuleLabel},
+			Description:   "Allow forwarding of all traffic between Kubernetes pods",
+		}
+		intendedIPv4ACLs.PutItem(allowKubePodForwarding, nil)
+	}
 
 	// Mark all un-marked local traffic generated by local services.
 	outputRules := []iptables.Rule{

pkg/pillar/iptables/connmark.go

@@ -59,8 +59,12 @@ var ControlProtocolMarkingIDMap = map[string]uint32{
 	"app_http": 11,
 	// ICMPv6 traffic to and from an application
 	"app_icmpv6": 12,
-	// for Kubernetes DNS, allowing coreDNS to talk to external DNS servers
-	"in_dns": 13,
+	// DNS requests from Kubernetes pods to CoreDNS and from CoreDNS to external DNS servers.
+	"kube_dns": 13,
+	// Traffic from Kubernetes pods to Kubernetes services.
+	"kube_svc": 14,
+	// Traffic directly forwarded between Kubernetes pods (not via services).
+	"kube_pod": 15,
 }
 
 // GetConnmark : create connection mark corresponding to the given attributes.