NETOBSERV-200: Basic eBPF agent that fetches flows' information

Dockerfile

@@ -1,11 +1,12 @@
 # Build the manager binary
 FROM registry.access.redhat.com/ubi8/go-toolset:1.16.7-5 as builder
 ARG VERSION="unknown"
+ARG GOVERSION="1.17.8"
 
 WORKDIR /opt/app-root
 
 # TEMPORARY STEPS UNTIL ubi8 releases a go1.17 image
-RUN wget -q https://go.dev/dl/go1.17.8.linux-amd64.tar.gz && tar -xzf go1.17.8.linux-amd64.tar.gz
+RUN wget -q https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz && tar -xzf go$GOVERSION.linux-amd64.tar.gz
 ENV GOROOT /opt/app-root/go
 RUN mkdir -p /opt/app-root/gopath
 ENV GOPATH /opt/app-root/gopath

Makefile

@@ -12,6 +12,8 @@ IMAGE_TAG_BASE ?= quay.io/netobserv/netobserv-agent
 # Image URL to use all building/pushing image targets
 IMG ?= $(IMAGE_TAG_BASE):$(VERSION)
 
+LOCAL_GENERATOR_IMAGE ?= ebpf-generator:latest
+
 CILIUM_EBPF_VERSION := v0.8.1
 GOLANGCI_LINT_VERSION = v1.42.1
 
@@ -47,13 +49,22 @@ lint: prereqs
 	@echo "### Linting code"
 	golangci-lint run ./...
 
+# As generated artifacts are part of the code repo (pkg/ebpf package) you don't have to run this
+# for each build. Only when you change the C code inside the bpf folder
+# You might need to use the docker-generate target instead of this
 .PHONY: generate
 generate: export BPF_CLANG := $(CLANG)
 generate: export BPF_CFLAGS := $(CFLAGS)
 generate: prereqs
 	@echo "### Generating BPF Go bindings"
 	go generate ./pkg/...
 
+.PHONY: docker-generate
+docker-generate:
+	@echo "### Creating the container that generates the eBPF binaries"
+	docker build . -f scripts/Dockerfile_ebpf_generator -t $(LOCAL_GENERATOR_IMAGE)
+	docker run --rm -v $(shell pwd):/src $(LOCAL_GENERATOR_IMAGE)
+
 .PHONY: build
 build: prereqs fmt lint test compile
 

README.md

@@ -1,3 +1,38 @@
 # Network Observability Agent
 
 Network Observability Agent
+
+## How to compile
+
+```
+make build
+```
+
+## How to run
+
+```
+sudo bin/netobserv-agent
+```
+(Pod deployment will come soon)
+
+## Development receipts
+
+### How to regenerate the eBPF Kernel binaries
+
+The eBPF program is embedded into the `pkg/ebpf/bpf_*` generated files.
+This step is generally not needed unless you change the C code in the `bpf` folder.
+
+If you have Docker installed, you just need to run:
+
+```
+make docker-generate
+```
+
+If you can't install docker, you should locally install the following required packages:
+
+```
+dnf install -y kernel-devel make llvm clang glibc-devel.i686
+make generate
+```
+
+Tested in Fedora 35 and Red Hat Enterprise Linux 8.

bpf/flow.h

@@ -0,0 +1,46 @@
+#ifndef __FLOW_H__
+#define __FLOW_H__
+
+#define TC_ACT_OK 0
+#define TC_ACT_SHOT 2
+#define IP_MAX_LEN 16
+
+typedef __u8 u8;
+typedef __u16 u16;
+typedef __u32 u32;
+typedef __u64 u64;
+
+// L2 data link layer
+struct data_link {
+    u8 src_mac[ETH_ALEN];
+    u8 dst_mac[ETH_ALEN];
+};
+
+// L3 network layer
+struct network {
+    // todo: add protocol
+    // todo: support ipv6
+    u32 src_ip;
+    u32 dst_ip;
+};
+
+// L4 transport layer
+struct transport {
+    u16 src_port;
+    u16 dst_port;
+    u8 protocol;
+};
+
+// TODO: L5 session layer to bound flows to connections?
+
+// contents in this struct must match byte-by-byte with Go's pkc/flow/Record struct
+struct flow {
+    u16 protocol;
+    u8 direction;
+    struct data_link data_link;
+    struct network network;
+    struct transport transport;
+    u64 bytes;
+} __attribute__((packed));
+
+#endif

bpf/flows.c

@@ -1,79 +1,104 @@
-#include <vmlinux.h>
+#include <linux/ip.h>
+#include <linux/in.h>
+#include <linux/tcp.h>
+#include <linux/udp.h>
+#include <linux/bpf.h>
+#include <linux/types.h>
+#include <linux/if_ether.h>
 #include <bpf_helpers.h>
+#include <bpf_endian.h>
+#include "flow.h"
 
-#define TC_ACT_OK 0
-#define TC_ACT_SHOT 2
+#define DISCARD 1
+#define SUBMIT 0
 
-// definitions to not having to include arpa/inet.h version of 32-bit in the compiler
-// TODO: to unload even more the Kernel space, we can do this calculation in the Go userspace
-#define htons(x)                       \
-    ((u16)((((u16)(x)&0xff00U) >> 8) | \
-           (((u16)(x)&0x00ffU) << 8)))
-
-#define htonl(x)                            \
-    ((u32)((((u32)(x)&0xff000000U) >> 24) | \
-           (((u32)(x)&0x00ff0000U) >> 8) |  \
-           (((u32)(x)&0x0000ff00U) << 8) |  \
-           (((u32)(x)&0x000000ffU) << 24)))
+// according to field 61 in https://www.iana.org/assignments/ipfix/ipfix.xhtml
+#define INGRESS 0
+#define EGRESS 1
 
+// TODO: for performance reasons, replace the ring buffer by a hashmap and
+// aggregate the flows here instead of the Go Accounter
 struct {
     __uint(type, BPF_MAP_TYPE_RINGBUF);
     __uint(max_entries, 1 << 24);
 } flows SEC(".maps");
 
-struct egress_t {
-    u32 src_ip;
-    u16 src_port;
-    u32 dst_ip;
-    u16 dst_port;
-    u8 protocol;
-    u64 bytes;
-} __attribute__((packed));
-
-SEC("tc/flow_parse")
-static inline int flow_parse(struct __sk_buff *skb) {
-    void *data = (void *)(long)skb->data;
-    void *data_end = (void *)(long)skb->data_end;
-    struct ethhdr *eth = data;
+// sets flow fields from IPv4 header information
+static inline int fill_iphdr(struct iphdr *ip, void *data_end, struct flow *flow) {
+    if ((void *)ip + sizeof(*ip) > data_end) {
+        return DISCARD;
+    }
 
-    if ((void *)eth + sizeof(*eth) <= data_end) {
-        struct iphdr *ip = data + sizeof(*eth);
-        if ((void *)ip + sizeof(*ip) <= data_end) {
+    flow->network.src_ip = __bpf_htonl(ip->saddr);
+    flow->network.dst_ip = __bpf_htonl(ip->daddr);
+    flow->transport.protocol = ip->protocol;
 
-            struct egress_t *event = bpf_ringbuf_reserve(&flows, sizeof(struct egress_t), 0);
-            if (!event) {
-                return TC_ACT_OK;
-            }
+    switch (ip->protocol) {
+    case IPPROTO_TCP: {
+        struct tcphdr *tcp = (void *)ip + sizeof(*ip);
+        if ((void *)tcp + sizeof(*tcp) <= data_end) {
+            flow->transport.src_port = __bpf_htons(tcp->source);
+            flow->transport.dst_port = __bpf_htons(tcp->dest);
+        }
+    } break;
+    case IPPROTO_UDP: {
+        struct udphdr *udp = (void *)ip + sizeof(*ip);
+        if ((void *)udp + sizeof(*udp) <= data_end) {
+            flow->transport.src_port = __bpf_htons(udp->source);
+            flow->transport.dst_port = __bpf_htons(udp->dest);
+        }
+    } break;
+    default:
+        break;
+    }
+    return SUBMIT;
+}
 
-            event->src_ip = htonl(ip->saddr);
-            event->dst_ip = htonl(ip->daddr);
-            event->protocol = ip->protocol;
+// sets flow fields from Ethernet header information
+static inline int fill_ethhdr(struct ethhdr *eth, void *data_end, struct flow *flow) {
+    if ((void *)eth + sizeof(*eth) > data_end) {
+        return DISCARD;
+    }
+    __builtin_memcpy(flow->data_link.dst_mac, eth->h_dest, ETH_ALEN);
+    __builtin_memcpy(flow->data_link.src_mac, eth->h_source, ETH_ALEN);
+    flow->protocol = eth->h_proto;
+    // TODO: ETH_P_IPV6
+    if (flow->protocol == __bpf_constant_ntohs(ETH_P_IP)) {
+        struct iphdr *ip = (void *)eth + sizeof(*eth);
+        return fill_iphdr(ip, data_end, flow);
+    }
+    return SUBMIT;
+}
 
-            switch (ip->protocol) {
-            case IPPROTO_TCP: {
-                struct tcphdr *tcp = (void *)ip + sizeof(*ip);
-                if ((void *)tcp + sizeof(*tcp) <= data_end) {
-                    event->src_port = htons(tcp->source);
-                    event->dst_port = htons(tcp->dest);
-                }
-            } break;
-            case IPPROTO_UDP: {
-                struct udphdr *udp = (void *)ip + sizeof(*ip);
-                if ((void *)udp + sizeof(*udp) <= data_end) {
-                    event->src_port = htons(udp->source);
-                    event->dst_port = htons(udp->dest);
-                }
-            } break;
-            default:
-                break;
-            }
-            event->bytes = skb->len;
+// parses flow information for a given direction (ingress/egress)
+static inline int flow_parse(struct __sk_buff *skb, u8 direction) {
+    void *data = (void *)(long)skb->data;
+    void *data_end = (void *)(long)skb->data_end;
 
-            bpf_ringbuf_submit(event, 0);
-        }
+    struct flow *flow = bpf_ringbuf_reserve(&flows, sizeof(struct flow), 0);
+    if (!flow) {
+        return TC_ACT_OK;
     }
 
+    struct ethhdr *eth = data;
+    if (fill_ethhdr(eth, data_end, flow) == DISCARD) {
+        bpf_ringbuf_discard(flow, 0);
+    } else {
+        flow->direction = direction;
+        flow->bytes = skb->len;
+        bpf_ringbuf_submit(flow, 0);
+    }
     return TC_ACT_OK;
 }
 
+SEC("tc/ingress_flow_parse")
+static inline int ingress_flow_parse(struct __sk_buff *skb) {
+    return flow_parse(skb, INGRESS);
+}
+
+SEC("tc/egress_flow_parse")
+static inline int egress_flow_parse(struct __sk_buff *skb) {
+    return flow_parse(skb, EGRESS);
+}
+
 char __license[] SEC("license") = "GPL";

bpf/headers/bpf_endian.h

@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __BPF_ENDIAN__
+#define __BPF_ENDIAN__
+
+#include <linux/swab.h>
+
+/* LLVM's BPF target selects the endianness of the CPU
+ * it compiles on, or the user specifies (bpfel/bpfeb),
+ * respectively. The used __BYTE_ORDER__ is defined by
+ * the compiler, we cannot rely on __BYTE_ORDER from
+ * libc headers, since it doesn't reflect the actual
+ * requested byte order.
+ *
+ * Note, LLVM's BPF target has different __builtin_bswapX()
+ * semantics. It does map to BPF_ALU | BPF_END | BPF_TO_BE
+ * in bpfel and bpfeb case, which means below, that we map
+ * to cpu_to_be16(). We could use it unconditionally in BPF
+ * case, but better not rely on it, so that this header here
+ * can be used from application and BPF program side, which
+ * use different targets.
+ */
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+# define __bpf_ntohs(x)			__builtin_bswap16(x)
+# define __bpf_htons(x)			__builtin_bswap16(x)
+# define __bpf_constant_ntohs(x)	___constant_swab16(x)
+# define __bpf_constant_htons(x)	___constant_swab16(x)
+# define __bpf_ntohl(x)			__builtin_bswap32(x)
+# define __bpf_htonl(x)			__builtin_bswap32(x)
+# define __bpf_constant_ntohl(x)	___constant_swab32(x)
+# define __bpf_constant_htonl(x)	___constant_swab32(x)
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+# define __bpf_ntohs(x)			(x)
+# define __bpf_htons(x)			(x)
+# define __bpf_constant_ntohs(x)	(x)
+# define __bpf_constant_htons(x)	(x)
+# define __bpf_ntohl(x)			(x)
+# define __bpf_htonl(x)			(x)
+# define __bpf_constant_ntohl(x)	(x)
+# define __bpf_constant_htonl(x)	(x)
+#else
+# error "Fix your compiler's __BYTE_ORDER__?!"
+#endif
+
+#define bpf_htons(x)				\
+	(__builtin_constant_p(x) ?		\
+	 __bpf_constant_htons(x) : __bpf_htons(x))
+#define bpf_ntohs(x)				\
+	(__builtin_constant_p(x) ?		\
+	 __bpf_constant_ntohs(x) : __bpf_ntohs(x))
+#define bpf_htonl(x)				\
+	(__builtin_constant_p(x) ?		\
+	 __bpf_constant_htonl(x) : __bpf_htonl(x))
+#define bpf_ntohl(x)				\
+	(__builtin_constant_p(x) ?		\
+	 __bpf_constant_ntohl(x) : __bpf_ntohl(x))
+
+#endif /* __BPF_ENDIAN__ */