From 28c5c3038ba3aa67ffb5f2dd8b767c968b81d7b0 Mon Sep 17 00:00:00 2001 From: Paul Greenberg Date: Sun, 15 Mar 2020 22:38:35 -0400 Subject: [PATCH] firewall: add nftables backend Resolves: #461 Signed-off-by: Paul Greenberg --- go.mod | 7 +- go.sum | 22 + plugins/meta/firewall/README.md | 204 +++++++ plugins/meta/firewall/firewall.go | 2 + .../meta/firewall/firewall_nftables_test.go | 443 +++++++++++++++ plugins/meta/firewall/nftables.go | 510 ++++++++++++++++++ 6 files changed, 1184 insertions(+), 4 deletions(-) create mode 100644 plugins/meta/firewall/firewall_nftables_test.go create mode 100644 plugins/meta/firewall/nftables.go diff --git a/go.mod b/go.mod index 2b4c4df34..7e974c44c 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4 // indirect github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c github.com/golang/protobuf v1.3.1 // indirect + github.com/google/nftables v0.0.0-20200316075819-7127d9d22474 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56 github.com/mattn/go-shellwords v1.0.3 github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b @@ -24,10 +25,8 @@ require ( github.com/sirupsen/logrus v1.0.6 // indirect github.com/stretchr/testify v1.3.0 // indirect github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf - github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc // indirect - golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941 // indirect - golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1 // indirect - golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f + github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc + golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect ) diff --git a/go.sum b/go.sum index ed3e97026..d190beb11 100644 --- a/go.sum +++ b/go.sum @@ -26,10 +26,20 @@ github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c h1:RBUpb2b14UnmRHNd2uH github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/nftables v0.0.0-20200316075819-7127d9d22474 h1:D6bN82zzK92ywYsE+Zjca7EHZCRZbcNTU3At7WdxQ+c= +github.com/google/nftables v0.0.0-20200316075819-7127d9d22474/go.mod h1:cfspEyr/Ap+JDIITA+N9a0ernqG0qZ4W1aqMRgDZa1g= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56 h1:742eGXur0715JMq73aD95/FU0XpVKXqNuTnEfXsLOYQ= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= +github.com/koneu/natend v0.0.0-20150829182554-ec0926ea948d h1:MFX8DxRnKMY/2M3H61iSsVbo/n3h0MWGmWNN1UViOU0= +github.com/koneu/natend v0.0.0-20150829182554-ec0926ea948d/go.mod h1:QHb4k4cr1fQikUahfcRVPcEXiUgFsdIstGqlurL0XL4= github.com/mattn/go-shellwords v1.0.3 h1:K/VxK7SZ+cvuPgFSLKi5QPI9Vr/ipOf4C1gN+ntueUk= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= +github.com/mdlayher/netlink v0.0.0-20191009155606-de872b0d824b h1:W3er9pI7mt2gOqOWzwvx20iJ8Akiqz1mUMTxU6wdvl8= +github.com/mdlayher/netlink v0.0.0-20191009155606-de872b0d824b/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b h1:Ey6yH0acn50T/v6CB75bGP4EMJqnv9WvnjN7oZaj+xE= github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a h1:KfNOeFvoAssuZLT7IntKZElKwi/5LRuxY71k+t6rfaM= @@ -49,10 +59,22 @@ github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrB github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941 h1:qBTHLajHecfu+xzRI9PqVDcqx7SdHj9d4B+EzSn3tAc= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1 h1:Y/KGZSOdz/2r0WJ9Mkmz6NJBusp0kiNx1Cn82lzJQ6w= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271 h1:N66aaryRB3Ax92gH0v3hp1QYZ3zWWCCUR/j8Ifh45Ss= +golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c h1:S/FtSvpNLtFBgjTqcKsRpsa6aVsI6iztaz1bQd9BJwE= +golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= diff --git a/plugins/meta/firewall/README.md b/plugins/meta/firewall/README.md index e53c0155b..6406e40c2 100644 --- a/plugins/meta/firewall/README.md +++ b/plugins/meta/firewall/README.md @@ -133,3 +133,207 @@ of the container as shown: - `-s 10.88.0.2 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT` - `-d 10.88.0.2 -j ACCEPT` +## nftables backend rule structure + +The prerequisite for the backend is the existence of `filter` table and +the existence of `FORWARD` chain in the table. + +A sample standalone config list (with the file extension `.conflist`) using +`nftables` backend might look like: + +```json +{ + "cniVersion": "0.4.0", + "name": "podman", + "plugins": [ + { + "type": "bridge", + "bridge": "cni-podman0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "ranges": [ + [ + { + "subnet": "192.168.100.0/24", + "gateway": "192.168.100.1" + } + ] + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + }, + { + "type": "firewall", + "backend": "nftables" + } + ] +} +``` + +Prior to the invocation of CNI `firewall` plugin, the `FORWARD` chain in `filter` +table might be configured be as follows: + +``` +table ip filter { + chain FORWARD { # handle 1 + type filter hook forward priority filter; policy drop; + log prefix "IPv4 FORWARD drop: " flags all # handle 28 + counter packets 0 bytes 0 drop # handle 29 + } +} +``` + +Subsequently, the plugin creates "non-base chain", e.g. `cnins-3-4026543850-dummy0` +and link it to `FORWARD` chain +via [`jump` instruction](https://wiki.nftables.org/wiki-nftables/index.php/Jumping_to_chain). + +``` +table ip filter { + chain FORWARD { # handle 1 + type filter hook forward priority filter; policy drop; + jump cnins-3-4026543850-dummy0 # handle 10 + log prefix "IPv4 FORWARD drop: " flags all # handle 28 + counter packets 0 bytes 0 drop # handle 29 + } + + chain cnins-3-4026543850-dummy0 { # handle 2 + oifname "dummy0" ip daddr 192.168.100.100 ct state established,related counter packets 0 bytes 0 accept # handle 3 + iifname "dummy0" ip saddr 192.168.100.100 counter packets 0 bytes 0 accept # handle 4 + iifname "dummy0" oifname "dummy0" counter packets 0 bytes 0 accept # handle 5 + } +} +``` + +The name of the chain is is prefixed with `CNINS-` and followed by `Dev` and `Ino` +of `Stat_t` struct. See [here](https://github.com/vishvananda/netns/blob/master/netns.go#L60) +for more information. + +Generally, the testing of nftables backend of this plugin begins with defining +the data structure the plugin would receive when processing a request. +In this example, the plugin received single interface `dummy0`, with IPv4 and +IPv6 addresses. + +```json +{ + "name": "test", + "type": "firewall", + "backend": "nftables", + "ifName": "dummy0", + "cniVersion": "0.4.0", + "prevResult": { + "interfaces": [ + { + "name": "dummy0" + } + ], + "ips": [ + { + "version": "4", + "address": "192.168.200.10/24", + "interface": 0 + }, + { + "version": "6", + "address": "2001:db8:1:2::1/64", + "interface": 0 + } + ] + } +} +``` + +Prior to running tests, the test harness does the following: + +1. creates `originalNS` namespace +2. adds `dummy0` interface to `originalNS` via Netlink +3. checks that the `dummy0` interface is available in the `originalNS` +4. creates `targetNS` namespace + +Upon the completion of the testing, the test harness does the following: + +1. closes `originalNS` namespace +2. closes `targetNS` namespace + +The tests in the harness start with `It()`. + +Generally, a test contains a number of input arguments. In the case of +"installs nftables rules, checks the rules exist, then cleans up on delete using v4.0.x", +the test has the following arguments: + +* container id: `dummy` +* the path to container namespace, i.e. `targetNS` +* the name of the interface +* the JSON payload containing a dummy request + +The test uses the same arguments and runs the following operations in +`originalNS` namespace: + +* `cmdAdd` +* `cmdCheck` +* `cmdDel` + +The operations correspond to the following functions: + +| **Operation** | **Function** | +| --- | --- | +| `cmdAdd` | `func (nb *nftBackend) Add(conf *FirewallNetConf, result *current.Result)` | +| `cmdCheck` | `func (nb *nftBackend) Del(conf *FirewallNetConf, result *current.Result)` | +| `cmdDel` | `func (nb *nftBackend) Check(conf *FirewallNetConf, result *current.Result)` | + +The following command triggers the testing of `firewall` plugin: + +```bash +sudo go test -v ./plugins/meta/firewall +``` + +At the outset, the test outputs dummy host and container namespaces: + +``` +Host Namespace: /var/run/netns/cnitest-ba94096d-68e1-90c0-5e0a-4acf3a8339cd +Container Namespace: /var/run/netns/cnitest-762bc306-9af5-5882-af95-e011590ce8d3 +``` + +The knowing the last part of the namespace path helps inspecting namespaces +with `sudo ip netns exec` command. For example, the following command +show `nftables` tables, chains, and rules. + +```bash + +$ sudo ip netns exec cnitest-ba94096d-68e1-90c0-5e0a-4acf3a8339cd nft list ruleset +table ip filter { + chain FORWARD { + type filter hook forward priority filter; policy drop; + jump cnins-3-4026550857-dummy0 + } + + chain cnins-3-4026550857-dummy0 { + oifname "dummy0" ip daddr 192.168.100.100 ct state established,related counter packets 0 bytes 0 accept + iifname "dummy0" ip saddr 192.168.100.100 counter packets 0 bytes 0 accept + iifname "dummy0" oifname "dummy0" counter packets 0 bytes 0 accept + } +} +table ip6 filter { + chain FORWARD { + type filter hook forward priority filter; policy drop; + jump cnins-3-4026550857-dummy0 + } + + chain cnins-3-4026550857-dummy0 { + oifname "dummy0" ip6 daddr 2001:db8:100:100::1 ct state established,related counter packets 0 bytes 0 accept + iifname "dummy0" ip6 saddr 2001:db8:100:100::1 counter packets 0 bytes 0 accept + iifname "dummy0" oifname "dummy0" counter packets 0 bytes 0 accept + } +} +``` diff --git a/plugins/meta/firewall/firewall.go b/plugins/meta/firewall/firewall.go index 875943beb..f33ac54dd 100644 --- a/plugins/meta/firewall/firewall.go +++ b/plugins/meta/firewall/firewall.go @@ -97,6 +97,8 @@ func getBackend(conf *FirewallNetConf) (FirewallBackend, error) { switch conf.Backend { case "iptables": return newIptablesBackend(conf) + case "nftables": + return newNftablesBackend(conf) case "firewalld": return newFirewalldBackend(conf) } diff --git a/plugins/meta/firewall/firewall_nftables_test.go b/plugins/meta/firewall/firewall_nftables_test.go new file mode 100644 index 000000000..d08046a08 --- /dev/null +++ b/plugins/meta/firewall/firewall_nftables_test.go @@ -0,0 +1,443 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/containernetworking/plugins/pkg/testutils" + + "fmt" + "github.com/google/nftables" + "github.com/vishvananda/netlink" + "path" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func getTestContainerID(s string) string { + _, containerID := path.Split(s) + return strings.ReplaceAll(containerID, "cnitest", "dummy") +} + +func validateNftRulesExist(bytes []byte) { + prevResult := getPrevResult(bytes) + + for _, ip := range prevResult.IPs { + Expect(ip).To(Equal(true)) + } +} + +func validateNftRulesCleanup(bytes []byte) { + prevResult := getPrevResult(bytes) + + for _, ip := range prevResult.IPs { + Expect(ip).To(Equal(true)) + } +} + +var _ = Describe("firewall plugin nftables backend v0.4.x", func() { + var originalNS, targetNS ns.NetNS + const IFNAME string = "dummy0" + + BeforeEach(func() { + // Create a new NetNS so we don't modify the host + var err error + originalNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + // fmt.Print("\n") + // fmt.Printf("Host Namespace: %s\n", originalNS.Path()) + + err = originalNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + + err = netlink.LinkAdd(&netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: IFNAME, + }, + }) + Expect(err).NotTo(HaveOccurred()) + _, err = netlink.LinkByName(IFNAME) + Expect(err).NotTo(HaveOccurred()) + + // Add netfilter connection + nftc := &nftables.Conn{} + defaultDropPolicy := nftables.ChainPolicyDrop + // Add IPv4 filter table + filter4Table := nftc.AddTable(&nftables.Table{ + Family: nftables.TableFamilyIPv4, + Name: "filter", + }) + Expect(filter4Table).NotTo(BeNil()) + // Add FORWARD chain in IPv4 filter table + forwardFilter4TableChain := nftc.AddChain(&nftables.Chain{ + Name: "FORWARD", + Table: filter4Table, + Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookForward, + Priority: nftables.ChainPriorityFilter, + Policy: &defaultDropPolicy, + }) + Expect(forwardFilter4TableChain).NotTo(BeNil()) + // Add IPv6 filter table + filter6Table := nftc.AddTable(&nftables.Table{ + Family: nftables.TableFamilyIPv6, + Name: "filter", + }) + Expect(filter6Table).NotTo(BeNil()) + // Add FORWARD chain in IPv6 filter table + forwardFilter6TableChain := nftc.AddChain(&nftables.Chain{ + Name: "FORWARD", + Table: filter6Table, + Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookForward, + Priority: nftables.ChainPriorityFilter, + Policy: &defaultDropPolicy, + }) + Expect(forwardFilter6TableChain).NotTo(BeNil()) + // Execute netfilter changes + err = nftc.Flush() + Expect(err).NotTo(HaveOccurred()) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + + targetNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + // fmt.Printf("Container Namespace: %s\n", targetNS.Path()) + _, nsName := path.Split(originalNS.Path()) + fmt.Printf("Debug: sudo ip netns exec %s nft --debug=netlink list ruleset\n", nsName) + }) + + AfterEach(func() { + Expect(originalNS.Close()).To(Succeed()) + Expect(targetNS.Close()).To(Succeed()) + }) + + It("fails when IP configuration version is invalid using v4.0.x", func() { + args := &skel.CmdArgs{ + ContainerID: getTestContainerID(targetNS.Path()), + Netns: targetNS.Path(), + IfName: IFNAME, + StdinData: []byte(`{ + "name": "test", + "type": "firewall", + "backend": "nftables", + "ifName": "dummy0", + "cniVersion": "0.4.0", + "prevResult": { + "interfaces": [ + {"name": "dummy0"} + ], + "ips": [ + { + "version": "4", + "address": "192.168.200.10/24", + "interface": 0 + }, + { + "version": "16", + "address": "2001:db8:1:2::1/64", + "interface": 0 + } + ] + } + }`), + } + + err := originalNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + + _, _, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).To(HaveOccurred()) + + return nil + }) + Expect(err).NotTo(HaveOccurred()) + + }) + + It("fails when IP configuration is not present using v4.0.x", func() { + args := &skel.CmdArgs{ + ContainerID: getTestContainerID(targetNS.Path()), + Netns: targetNS.Path(), + IfName: IFNAME, + StdinData: []byte(`{ + "name": "test", + "type": "firewall", + "backend": "nftables", + "ifName": "dummy0", + "cniVersion": "0.4.0", + "prevResult": { + "interfaces": [ + {"name": "dummy0"} + ] + } + }`), + } + + err := originalNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + + _, _, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).To(HaveOccurred()) + + return nil + }) + Expect(err).NotTo(HaveOccurred()) + + }) + + It("configures nftables for a single dual-stack interface using v4.0.x", func() { + args := &skel.CmdArgs{ + ContainerID: getTestContainerID(targetNS.Path()), + Netns: targetNS.Path(), + IfName: IFNAME, + StdinData: []byte(`{ + "name": "test", + "type": "firewall", + "backend": "nftables", + "ifName": "dummy0", + "cniVersion": "0.4.0", + "prevResult": { + "interfaces": [ + {"name": "dummy0"} + ], + "ips": [ + { + "version": "4", + "address": "192.168.200.10/24", + "interface": 0 + }, + { + "version": "6", + "address": "2001:db8:1:2::1/64", + "interface": 0 + } + ] + } + }`), + } + + err := originalNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + + r, _, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).NotTo(HaveOccurred()) + + _, err = current.GetResult(r) + Expect(err).NotTo(HaveOccurred()) + + err = testutils.CmdCheckWithArgs(args, func() error { + return cmdCheck(args) + }) + Expect(err).NotTo(HaveOccurred()) + //validateNftRulesExist(args.StdinData) + + err = testutils.CmdDelWithArgs(args, func() error { + return cmdDel(args) + }) + Expect(err).NotTo(HaveOccurred()) + //validateNftRulesCleanup(args.StdinData) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + }) + + It("configures nftables for two dual-stack interfaces using v4.0.x", func() { + args := &skel.CmdArgs{ + ContainerID: getTestContainerID(targetNS.Path()), + Netns: targetNS.Path(), + IfName: IFNAME, + StdinData: []byte(`{ + "name": "test", + "type": "firewall", + "backend": "nftables", + "ifName": "dummy0", + "cniVersion": "0.4.0", + "prevResult": { + "interfaces": [ + {"name": "dummy0"}, + {"name": "dummy1"} + ], + "ips": [ + { + "version": "4", + "address": "192.168.100.100/24", + "interface": 0 + }, + { + "version": "6", + "address": "2001:db8:100:100::1/64", + "interface": 0 + }, + { + "version": "4", + "address": "192.168.200.200/24", + "interface": 1 + }, + { + "version": "6", + "address": "2001:db8:200:200::1/64", + "interface": 1 + } + ] + } + }`), + } + + err := originalNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + + r, _, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).NotTo(HaveOccurred()) + + _, err = current.GetResult(r) + Expect(err).NotTo(HaveOccurred()) + + err = testutils.CmdCheckWithArgs(args, func() error { + return cmdCheck(args) + }) + Expect(err).NotTo(HaveOccurred()) + //validateNftRulesExist(args.StdinData) + + err = testutils.CmdDelWithArgs(args, func() error { + return cmdDel(args) + }) + Expect(err).NotTo(HaveOccurred()) + //validateNftRulesCleanup(args.StdinData) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + }) + + It("configures nftables for a single IPv4-only interface using v4.0.x", func() { + args := &skel.CmdArgs{ + ContainerID: getTestContainerID(targetNS.Path()), + Netns: targetNS.Path(), + IfName: IFNAME, + StdinData: []byte(`{ + "name": "test", + "type": "firewall", + "backend": "nftables", + "ifName": "dummy0", + "cniVersion": "0.4.0", + "prevResult": { + "interfaces": [ + {"name": "dummy0"} + ], + "ips": [ + { + "version": "4", + "address": "192.168.100.100/24", + "interface": 0 + } + ] + }}`), + } + + err := originalNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + + r, _, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).NotTo(HaveOccurred()) + + _, err = current.GetResult(r) + Expect(err).NotTo(HaveOccurred()) + + err = testutils.CmdCheckWithArgs(args, func() error { + return cmdCheck(args) + }) + Expect(err).NotTo(HaveOccurred()) + //validateNftRulesExist(args.StdinData) + + err = testutils.CmdDelWithArgs(args, func() error { + return cmdDel(args) + }) + Expect(err).NotTo(HaveOccurred()) + //validateNftRulesCleanup(args.StdinData) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + }) + + It("configures nftables for a single IPv6-only interface using v4.0.x", func() { + args := &skel.CmdArgs{ + ContainerID: getTestContainerID(targetNS.Path()), + Netns: targetNS.Path(), + IfName: IFNAME, + StdinData: []byte(`{ + "name": "test", + "type": "firewall", + "backend": "nftables", + "ifName": "dummy0", + "cniVersion": "0.4.0", + "prevResult": { + "interfaces": [ + {"name": "dummy0"} + ], + "ips": [ + { + "version": "6", + "address": "2001:db8:100:100::1/64", + "interface": 0 + } + ] + } + }`), + } + + err := originalNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + + r, _, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).NotTo(HaveOccurred()) + + _, err = current.GetResult(r) + Expect(err).NotTo(HaveOccurred()) + + err = testutils.CmdCheckWithArgs(args, func() error { + return cmdCheck(args) + }) + Expect(err).NotTo(HaveOccurred()) + //validateNftRulesExist(args.StdinData) + + err = testutils.CmdDelWithArgs(args, func() error { + return cmdDel(args) + }) + Expect(err).NotTo(HaveOccurred()) + //validateNftRulesCleanup(args.StdinData) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + }) + +}) diff --git a/plugins/meta/firewall/nftables.go b/plugins/meta/firewall/nftables.go new file mode 100644 index 000000000..8d6713485 --- /dev/null +++ b/plugins/meta/firewall/nftables.go @@ -0,0 +1,510 @@ +// Copyright 2018 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/google/nftables" + "github.com/google/nftables/expr" + "github.com/vishvananda/netns" + "strings" +) + +// nftBackend implements the FirewallBackend interface +var _ FirewallBackend = &nftBackend{} + +type nftBackend struct { + ns netns.NsHandle + conn *nftables.Conn + tables []*nftables.Table + chains []*nftables.Chain + targetTable string + targetChain string + targetInterfaces map[string]*nftTargetInterface +} + +type nftTargetInterface struct { + addrs []*nftTargetInterfaceAddress +} + +type nftTargetInterfaceAddress struct { + conf *current.IPConfig + table *nftables.Table + chain *nftables.Chain +} + +func newNftablesBackend(conf *FirewallNetConf) (FirewallBackend, error) { + backend := &nftBackend{ + targetTable: "filter", + targetChain: "FORWARD", + tables: []*nftables.Table{}, + chains: []*nftables.Chain{}, + } + + return backend, nil +} + +func (nb *nftBackend) getChainName(intf string) string { + name := "cni" + nb.ns.UniqueId() + intf + name = strings.ReplaceAll(name, "(", "-") + name = strings.ReplaceAll(name, ")", "-") + name = strings.ReplaceAll(name, ":", "-") + return strings.ToLower(name) +} + +func (nb *nftBackend) initConn() error { + if nb.conn != nil { + return nil + } + ns, err := netns.Get() + if err != nil { + return err + } + conn := &nftables.Conn{ + NetNS: int(ns), + } + + tables, err := conn.ListTables() + if err != nil { + return err + } + + for _, table := range tables { + if table == nil { + continue + } + if table.Name != nb.targetTable { + continue + } + if table.Family != nftables.TableFamilyIPv4 && table.Family != nftables.TableFamilyIPv6 { + continue + } + nb.tables = append(nb.tables, table) + } + + if len(nb.tables) == 0 { + return fmt.Errorf("nftables table %s not found", nb.targetTable) + } + + chains, err := conn.ListChains() + if err != nil { + return err + } + + for _, chain := range chains { + if chain == nil { + continue + } + if chain.Name != nb.targetChain { + continue + } + if chain.Table.Name != nb.targetTable { + continue + } + if chain.Table.Family != nftables.TableFamilyIPv4 && chain.Table.Family != nftables.TableFamilyIPv6 { + continue + } + nb.chains = append(nb.chains, chain) + } + + if len(nb.chains) == 0 { + return fmt.Errorf("nftables chain %s not found in %s table", nb.targetChain, nb.targetTable) + } + + nb.ns = ns + nb.conn = conn + return nil +} + +func (nb *nftBackend) addFilterChains() error { + if err := nb.initConn(); err != nil { + return err + } + + for intfName, targetInterface := range nb.targetInterfaces { + for _, addr := range targetInterface.addrs { + chainName := nb.getChainName(intfName) + + // Add a new chain + // defaultDropPolicy := nftables.ChainPolicyDrop + chain := nb.conn.AddChain(&nftables.Chain{ + Name: chainName, + Table: addr.table, + //Type: nftables.ChainTypeFilter, + //Hooknum: nftables.ChainHookForward, + //Priority: nftables.ChainPriorityFilter, + //Policy: &defaultDropPolicy, + }) + if err := nb.conn.Flush(); err != nil { + return fmt.Errorf( + "failed adding chain %s for address %v of interface %s", + chainName, addr.conf, intfName, + ) + } + + // Add rule for inbound traffic + // nft add rule oifname "dummy0" ip daddr 192.168.100.100 ct state established,related counter packets 0 bytes 0 accept + inboundInterfaceRule := &nftables.Rule{ + Table: addr.table, + Chain: chain, + Exprs: []expr.Any{}, + } + + // meta load oifname => reg 1 + // cmp eq reg 1 0x6d6d7564 0x00003079 0x00000000 0x00000000 + inboundInterfaceRule.Exprs = append(inboundInterfaceRule.Exprs, &expr.Meta{ + Key: expr.MetaKeyOIFNAME, + Register: 1, + }) + inboundInterfaceRule.Exprs = append(inboundInterfaceRule.Exprs, &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: getNftInterfaceName(intfName), + }) + + if addr.conf.Version == "6" { + // payload load 4b @ network header + 16 => reg 1 + // cmp eq reg 1 0xc8c8a8c0 + inboundInterfaceRule.Exprs = append(inboundInterfaceRule.Exprs, &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + // Offset: 8, + Offset: 24, + Len: 16, + }) + inboundInterfaceRule.Exprs = append(inboundInterfaceRule.Exprs, &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: addr.conf.Address.IP.To16(), + }) + } else { + // payload load 4b @ network header + 16 => reg 1 + // cmp eq reg 1 0x6464a8c0 ] + inboundInterfaceRule.Exprs = append(inboundInterfaceRule.Exprs, &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: 16, + Len: 4, + }) + inboundInterfaceRule.Exprs = append(inboundInterfaceRule.Exprs, &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: addr.conf.Address.IP.To4(), + }) + } + + // ct load state => reg 1 ] + // bitwise reg 1 = (reg=1 & 0x00000006 ) ^ 0x00000000 + // cmp neq reg 1 0x00000000 + inboundInterfaceRule.Exprs = append(inboundInterfaceRule.Exprs, &expr.Ct{ + Register: 1, + Key: expr.CtKeySTATE, + }) + inboundInterfaceRule.Exprs = append(inboundInterfaceRule.Exprs, &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Xor: []byte{0x0, 0x0, 0x0, 0x0}, + Mask: []byte("\x06\x00\x00\x00"), + Len: 4, + }) + inboundInterfaceRule.Exprs = append(inboundInterfaceRule.Exprs, &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: []byte{0x0, 0x0, 0x0, 0x0}, + }) + + // counter pkts 0 bytes 0 + inboundInterfaceRule.Exprs = append(inboundInterfaceRule.Exprs, &expr.Counter{}) + // immediate reg 0 accept + inboundInterfaceRule.Exprs = append(inboundInterfaceRule.Exprs, &expr.Verdict{ + Kind: expr.VerdictAccept, + }) + + nb.conn.AddRule(inboundInterfaceRule) + if err := nb.conn.Flush(); err != nil { + return fmt.Errorf( + "failed adding outbound traffic rule in table %s chain %s for address %v of interface %s", + addr.table.Name, chainName, addr.conf, intfName, + ) + } + + // Add rule for outbound traffic + // nft add rule ... iifname X ip saddr 1.1.1.1 counter packets 0 bytes 0 accept + outboundInterfaceRule := &nftables.Rule{ + Table: addr.table, + Chain: chain, + Exprs: []expr.Any{}, + } + + outboundInterfaceRule.Exprs = append(outboundInterfaceRule.Exprs, &expr.Meta{ + Key: expr.MetaKeyIIFNAME, + Register: 1, + }) + outboundInterfaceRule.Exprs = append(outboundInterfaceRule.Exprs, &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: getNftInterfaceName(intfName), + }) + + if addr.conf.Version == "6" { + outboundInterfaceRule.Exprs = append(outboundInterfaceRule.Exprs, &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: 8, + // Offset: 24, + Len: 16, + }) + outboundInterfaceRule.Exprs = append(outboundInterfaceRule.Exprs, &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: addr.conf.Address.IP.To16(), + }) + } else { + // payload load 4b @ network header + 12 => reg 1 + outboundInterfaceRule.Exprs = append(outboundInterfaceRule.Exprs, &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: 12, + Len: 4, + }) + // cmp eq reg 1 0x0245a8c0 + outboundInterfaceRule.Exprs = append(outboundInterfaceRule.Exprs, &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: addr.conf.Address.IP.To4(), + }) + } + + outboundInterfaceRule.Exprs = append(outboundInterfaceRule.Exprs, &expr.Counter{}) + outboundInterfaceRule.Exprs = append(outboundInterfaceRule.Exprs, &expr.Verdict{ + Kind: expr.VerdictAccept, + }) + + nb.conn.AddRule(outboundInterfaceRule) + if err := nb.conn.Flush(); err != nil { + return fmt.Errorf( + "failed adding outbound traffic rule in table %s chain %s for address %v of interface %s", + addr.table.Name, chainName, addr.conf, intfName, + ) + } + + // Add intra interface rule + // nft add rule iifname "dummy0" oifname "dummy0" counter packets 0 bytes 0 accept + intraInterfaceRule := &nftables.Rule{ + Table: addr.table, + Chain: chain, + Exprs: []expr.Any{}, + } + + // meta load iifname => reg 1 + // cmp eq reg 1 0x6d6d7564 0x00003079 0x00000000 0x00000000 + intraInterfaceRule.Exprs = append(intraInterfaceRule.Exprs, &expr.Meta{ + Key: expr.MetaKeyIIFNAME, + Register: 1, + }) + intraInterfaceRule.Exprs = append(intraInterfaceRule.Exprs, &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: getNftInterfaceName(intfName), + }) + + // meta load oifname => reg 2 + // cmp eq reg 2 0x6d6d7564 0x00003079 0x00000000 0x00000000 + intraInterfaceRule.Exprs = append(intraInterfaceRule.Exprs, &expr.Meta{ + Key: expr.MetaKeyOIFNAME, + Register: 1, + }) + intraInterfaceRule.Exprs = append(intraInterfaceRule.Exprs, &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: getNftInterfaceName(intfName), + }) + + // counter pkts 0 bytes 0 + intraInterfaceRule.Exprs = append(intraInterfaceRule.Exprs, &expr.Counter{}) + // immediate reg 0 accept + intraInterfaceRule.Exprs = append(intraInterfaceRule.Exprs, &expr.Verdict{ + Kind: expr.VerdictAccept, + }) + + nb.conn.AddRule(intraInterfaceRule) + if err := nb.conn.Flush(); err != nil { + return fmt.Errorf( + "failed adding intra-interface rule in table %s chain %s for address %v of interface %s", + addr.table.Name, chainName, addr.conf, intfName, + ) + } + + // Finally, add the jump to the above chain in FORWARD chain. + jumpRule := &nftables.Rule{ + Table: addr.table, + Chain: addr.chain, + Exprs: []expr.Any{}, + } + jumpRule.Exprs = append(jumpRule.Exprs, &expr.Verdict{ + Kind: expr.VerdictJump, + Chain: chainName, + }) + + nb.conn.AddRule(jumpRule) + if err := nb.conn.Flush(); err != nil { + return fmt.Errorf( + "failed adding jump rule in table %s chain %s for address %v of interface %s", + addr.table.Name, addr.chain.Name, addr.conf, intfName, + ) + } + + } + } + + return nil +} + +func (nb *nftBackend) validateFilters() error { + if err := nb.initConn(); err != nil { + return err + } + + // Check whether there is a filter table for IP address + // family used in IP configuration. + for intfName, targetInteface := range nb.targetInterfaces { + for _, addr := range targetInteface.addrs { + tableFound := false + chainFound := false + for _, table := range nb.tables { + if addr.conf.Version == "4" && table.Family != nftables.TableFamilyIPv4 { + continue + } + if addr.conf.Version == "6" && table.Family != nftables.TableFamilyIPv6 { + continue + } + tableFound = true + addr.table = table + } + if !tableFound { + return fmt.Errorf( + "failed to find %s table for interface %s with config %v", + nb.targetTable, intfName, addr.conf, + ) + } + for _, chain := range nb.chains { + if addr.conf.Version == "4" && chain.Table.Family != nftables.TableFamilyIPv4 { + continue + } + if addr.conf.Version == "6" && chain.Table.Family != nftables.TableFamilyIPv6 { + continue + } + chainFound = true + addr.chain = chain + } + if !chainFound { + return fmt.Errorf( + "failed to find %s chain for interface %s with config %v", + nb.targetChain, intfName, addr.conf, + ) + } + } + } + return nil +} + +func (nb *nftBackend) validateInput(result *current.Result) error { + + if len(result.Interfaces) == 0 { + return fmt.Errorf("the data passed to firewall plugin did not contain network interfaces") + } + + nb.targetInterfaces = make(map[string]*nftTargetInterface) + + intfMap := make(map[int]string) + for i, intf := range result.Interfaces { + if intf.Name == "" { + return fmt.Errorf("the data passed to firewall plugin has no bridge name, e.g. cnibr0") + } + if _, interfaceExists := nb.targetInterfaces[intf.Name]; interfaceExists { + return fmt.Errorf("found duplicate interface name %s", intf.Name) + } + targetInterface := &nftTargetInterface{ + addrs: []*nftTargetInterfaceAddress{}, + } + nb.targetInterfaces[intf.Name] = targetInterface + intfMap[i] = intf.Name + } + + if len(result.IPs) == 0 { + return fmt.Errorf("the data passed to firewall plugin has no IP addresses") + } + + for _, addr := range result.IPs { + if addr.Interface == nil { + return fmt.Errorf("the ip config interface is nil: %v", addr) + } + if _, interfaceExists := intfMap[*addr.Interface]; !interfaceExists { + return fmt.Errorf("the ip config points to non-existing interface: %v", addr) + } + intfName := intfMap[*addr.Interface] + targetInterface := nb.targetInterfaces[intfName] + targetInterfaceAddr := &nftTargetInterfaceAddress{ + conf: addr, + } + targetInterface.addrs = append(targetInterface.addrs, targetInterfaceAddr) + } + + for intf, targetInterface := range nb.targetInterfaces { + if targetInterface == nil { + return fmt.Errorf("interface %s has no associated IP information", intf) + } + if len(targetInterface.addrs) == 0 { + return fmt.Errorf("interface %s has no associated IP information", intf) + } + } + + for _, entry := range result.IPs { + if entry.Address.String() == "" { + return fmt.Errorf("the data passed to firewall plugin has empty IP address") + } + } + + return nil +} + +func (nb *nftBackend) Add(conf *FirewallNetConf, result *current.Result) error { + if err := nb.validateInput(result); err != nil { + return fmt.Errorf("nftBackend.Add() failed validation: %s", err) + } + if err := nb.validateFilters(); err != nil { + return fmt.Errorf("nftBackend.Add() failed parsing netfilter tables: %s", err) + } + + if err := nb.addFilterChains(); err != nil { + return fmt.Errorf("nftBackend.Add() failed adding netfilter chains: %s", err) + } + return nil +} + +func (nb *nftBackend) Del(conf *FirewallNetConf, result *current.Result) error { + return nil +} + +func (nb *nftBackend) Check(conf *FirewallNetConf, result *current.Result) error { + return nil +} + +func getNftInterfaceName(s string) []byte { + b := make([]byte, 16) + copy(b, []byte(s+"\x00")) + return b +}