Skip to content

Commit

Permalink
bpf, xdp: Add bpf_link-based XDP attachment API
Browse files Browse the repository at this point in the history 
Add bpf_link-based API (bpf_xdp_link) to attach BPF XDP program through
BPF_LINK_CREATE command.

bpf_xdp_link is mutually exclusive with direct BPF program attachment,
previous BPF program should be detached prior to attempting to create a new
bpf_xdp_link attachment (for a given XDP mode). Once BPF link is attached, it
can't be replaced by other BPF program attachment or link attachment. It will
be detached only when the last BPF link FD is closed.

bpf_xdp_link will be auto-detached when net_device is shutdown, similarly to
how other BPF links behave (cgroup, flow_dissector). At that point bpf_link
will become defunct, but won't be destroyed until last FD is closed.

Signed-off-by: Andrii Nakryiko <andriin@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20200722064603.3350758-5-andriin@fb.com
  • Loading branch information
@anakryiko
anakryiko authored and Alexei Starovoitov committed Jul 26, 2020
1 parent d4baa93 commit aa8d3a7
Show file tree
Hide file tree
Showing 4 changed files with 178 additions and 7 deletions.
4 changes: 4 additions & 0 deletions include/linux/netdevice.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -888,6 +888,7 @@ struct bpf_prog_offload_ops;
struct netlink_ext_ack;
struct xdp_umem;
struct xdp_dev_bulk_queue;
struct bpf_xdp_link;

enum bpf_xdp_mode {
XDP_MODE_SKB = 0,
Expand All @@ -898,6 +899,7 @@ enum bpf_xdp_mode {

struct bpf_xdp_entity {
struct bpf_prog *prog;
struct bpf_xdp_link *link;
};

struct netdev_bpf {
Expand Down Expand Up @@ -3831,7 +3833,9 @@ struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf);
int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
int fd, int expected_fd, u32 flags);
int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode);

int xdp_umem_query(struct net_device *dev, u16 queue_id);

int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
Expand Down
7 changes: 6 additions & 1 deletion include/uapi/linux/bpf.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,7 @@ enum bpf_attach_type {
BPF_CGROUP_INET_SOCK_RELEASE,
BPF_XDP_CPUMAP,
BPF_SK_LOOKUP,
BPF_XDP,
__MAX_BPF_ATTACH_TYPE
};

Expand All @@ -242,6 +243,7 @@ enum bpf_link_type {
BPF_LINK_TYPE_CGROUP = 3,
BPF_LINK_TYPE_ITER = 4,
BPF_LINK_TYPE_NETNS = 5,
BPF_LINK_TYPE_XDP = 6,

MAX_BPF_LINK_TYPE,
};
Expand Down Expand Up @@ -614,7 +616,10 @@ union bpf_attr {

struct { /* struct used by BPF_LINK_CREATE command */
__u32 prog_fd; /* eBPF program to attach */
__u32 target_fd; /* object to attach to */
union {
__u32 target_fd; /* object to attach to */
__u32 target_ifindex; /* target ifindex */
};
__u32 attach_type; /* attach type */
__u32 flags; /* extra flags */
} link_create;
Expand Down
5 changes: 5 additions & 0 deletions kernel/bpf/syscall.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2824,6 +2824,8 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type)
return BPF_PROG_TYPE_TRACING;
case BPF_SK_LOOKUP:
return BPF_PROG_TYPE_SK_LOOKUP;
case BPF_XDP:
return BPF_PROG_TYPE_XDP;
default:
return BPF_PROG_TYPE_UNSPEC;
}
Expand Down Expand Up @@ -3921,6 +3923,9 @@ static int link_create(union bpf_attr *attr)
case BPF_PROG_TYPE_SK_LOOKUP:
ret = netns_bpf_link_create(attr, prog);
break;
case BPF_PROG_TYPE_XDP:
ret = bpf_xdp_link_attach(attr, prog);
break;
default:
ret = -EINVAL;
}
Expand Down
169 changes: 163 additions & 6 deletions net/core/dev.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8716,6 +8716,12 @@ int dev_change_proto_down_generic(struct net_device *dev, bool proto_down)
}
EXPORT_SYMBOL(dev_change_proto_down_generic);

struct bpf_xdp_link {
struct bpf_link link;
struct net_device *dev; /* protected by rtnl_lock, no refcnt held */
int flags;
};

static enum bpf_xdp_mode dev_xdp_mode(u32 flags)
{
if (flags & XDP_FLAGS_HW_MODE)
Expand All @@ -8738,9 +8744,19 @@ static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode)
};
}

static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev,
enum bpf_xdp_mode mode)
{
return dev->xdp_state[mode].link;
}

static struct bpf_prog *dev_xdp_prog(struct net_device *dev,
enum bpf_xdp_mode mode)
{
struct bpf_xdp_link *link = dev_xdp_link(dev, mode);

if (link)
return link->link.prog;
return dev->xdp_state[mode].prog;
}

Expand All @@ -8751,9 +8767,17 @@ u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
return prog ? prog->aux->id : 0;
}

static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode,
struct bpf_xdp_link *link)
{
dev->xdp_state[mode].link = link;
dev->xdp_state[mode].prog = NULL;
}

static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode,
struct bpf_prog *prog)
{
dev->xdp_state[mode].link = NULL;
dev->xdp_state[mode].prog = prog;
}

Expand Down Expand Up @@ -8793,6 +8817,7 @@ static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,

static void dev_xdp_uninstall(struct net_device *dev)
{
struct bpf_xdp_link *link;
struct bpf_prog *prog;
enum bpf_xdp_mode mode;
bpf_op_t bpf_op;
Expand All @@ -8810,14 +8835,20 @@ static void dev_xdp_uninstall(struct net_device *dev)

WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));

bpf_prog_put(prog);
dev_xdp_set_prog(dev, mode, NULL);
/* auto-detach link from net device */
link = dev_xdp_link(dev, mode);
if (link)
link->dev = NULL;
else
bpf_prog_put(prog);

dev_xdp_set_link(dev, mode, NULL);
}
}

static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack,
struct bpf_prog *new_prog, struct bpf_prog *old_prog,
u32 flags)
struct bpf_xdp_link *link, struct bpf_prog *new_prog,
struct bpf_prog *old_prog, u32 flags)
{
struct bpf_prog *cur_prog;
enum bpf_xdp_mode mode;
Expand All @@ -8826,6 +8857,14 @@ static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack

ASSERT_RTNL();

/* either link or prog attachment, never both */
if (link && (new_prog || old_prog))
return -EINVAL;
/* link supports only XDP mode flags */
if (link && (flags & ~XDP_FLAGS_MODES)) {
NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment");
return -EINVAL;
}
/* just one XDP mode bit should be set, zero defaults to SKB mode */
if (hweight32(flags & XDP_FLAGS_MODES) > 1) {
NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set");
Expand All @@ -8838,7 +8877,18 @@ static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack
}

mode = dev_xdp_mode(flags);
/* can't replace attached link */
if (dev_xdp_link(dev, mode)) {
NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link");
return -EBUSY;
}

cur_prog = dev_xdp_prog(dev, mode);
/* can't replace attached prog with link */
if (link && cur_prog) {
NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link");
return -EBUSY;
}
if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) {
NL_SET_ERR_MSG(extack, "Active program does not match expected");
return -EEXIST;
Expand All @@ -8848,6 +8898,10 @@ static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack
return -EBUSY;
}

/* put effective new program into new_prog */
if (link)
new_prog = link->link.prog;

if (new_prog) {
bool offload = mode == XDP_MODE_HW;
enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB
Expand Down Expand Up @@ -8884,13 +8938,116 @@ static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack
return err;
}

dev_xdp_set_prog(dev, mode, new_prog);
if (link)
dev_xdp_set_link(dev, mode, link);
else
dev_xdp_set_prog(dev, mode, new_prog);
if (cur_prog)
bpf_prog_put(cur_prog);

return 0;
}

static int dev_xdp_attach_link(struct net_device *dev,
struct netlink_ext_ack *extack,
struct bpf_xdp_link *link)
{
return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags);
}

static int dev_xdp_detach_link(struct net_device *dev,
struct netlink_ext_ack *extack,
struct bpf_xdp_link *link)
{
enum bpf_xdp_mode mode;
bpf_op_t bpf_op;

ASSERT_RTNL();

mode = dev_xdp_mode(link->flags);
if (dev_xdp_link(dev, mode) != link)
return -EINVAL;

bpf_op = dev_xdp_bpf_op(dev, mode);
WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
dev_xdp_set_link(dev, mode, NULL);
return 0;
}

static void bpf_xdp_link_release(struct bpf_link *link)
{
struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);

rtnl_lock();

/* if racing with net_device's tear down, xdp_link->dev might be
* already NULL, in which case link was already auto-detached
*/
if (xdp_link->dev)
WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));

rtnl_unlock();
}

static void bpf_xdp_link_dealloc(struct bpf_link *link)
{
struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);

kfree(xdp_link);
}

static const struct bpf_link_ops bpf_xdp_link_lops = {
.release = bpf_xdp_link_release,
.dealloc = bpf_xdp_link_dealloc,
};

int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
{
struct net *net = current->nsproxy->net_ns;
struct bpf_link_primer link_primer;
struct bpf_xdp_link *link;
struct net_device *dev;
int err, fd;

dev = dev_get_by_index(net, attr->link_create.target_ifindex);
if (!dev)
return -EINVAL;

link = kzalloc(sizeof(*link), GFP_USER);
if (!link) {
err = -ENOMEM;
goto out_put_dev;
}

bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog);
link->dev = dev;
link->flags = attr->link_create.flags;

err = bpf_link_prime(&link->link, &link_primer);
if (err) {
kfree(link);
goto out_put_dev;
}

rtnl_lock();
err = dev_xdp_attach_link(dev, NULL, link);
rtnl_unlock();

if (err) {
bpf_link_cleanup(&link_primer);
goto out_put_dev;
}

fd = bpf_link_settle(&link_primer);
/* link itself doesn't hold dev's refcnt to not complicate shutdown */
dev_put(dev);
return fd;

out_put_dev:
dev_put(dev);
return err;
}

/**
* dev_change_xdp_fd - set or clear a bpf program for a device rx path
* @dev: device
Expand Down Expand Up @@ -8927,7 +9084,7 @@ int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
}
}

err = dev_xdp_attach(dev, extack, new_prog, old_prog, flags);
err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags);

err_out:
if (err && new_prog)
Expand Down

0 comments on commit aa8d3a7

Please sign in to comment.