From d06d98dbaa1d591046026dcbfbe10574dc44db64 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Fri, 21 Jun 2024 13:48:02 +0200 Subject: [PATCH 01/49] netdevice: convert private flags > BIT(31) to bitfields Make dev->priv_flags `u32` back and define bits higher than 31 as bitfield booleans as per Jakub's suggestion. This simplifies code which accesses these bits with no optimization loss (testb both before/after), allows to not extend &netdev_priv_flags each time, but also scales better as bits > 63 in the future would only add a new u64 to the structure with no complications, comparing to that extending ::priv_flags would require converting it to a bitmap. Note that I picked `unsigned long :1` to not lose any potential optimizations comparing to `bool :1` etc. Suggested-by: Jakub Kicinski Signed-off-by: Alexander Lobakin --- .../networking/net_cachelines/net_device.rst | 4 ++- .../ethernet/microchip/lan966x/lan966x_main.c | 2 +- drivers/net/macvlan.c | 3 +- drivers/net/vxlan/vxlan_core.c | 3 +- include/linux/netdevice.h | 30 ++++++++++++------- net/8021q/vlanproc.c | 4 +-- net/core/dev.c | 4 +-- net/core/dev_ioctl.c | 9 +++--- net/core/rtnetlink.c | 2 +- 9 files changed, 37 insertions(+), 24 deletions(-) diff --git a/Documentation/networking/net_cachelines/net_device.rst b/Documentation/networking/net_cachelines/net_device.rst index 70c4fb9d4e5ce0..d7ba48ff55595f 100644 --- a/Documentation/networking/net_cachelines/net_device.rst +++ b/Documentation/networking/net_cachelines/net_device.rst @@ -7,6 +7,7 @@ net_device struct fast path usage breakdown Type Name fastpath_tx_access fastpath_rx_access Comments ..struct ..net_device +unsigned_long:32 priv_flags read_mostly - __dev_queue_xmit(tx) char name[16] - - struct_netdev_name_node* name_node struct_dev_ifalias* ifalias @@ -23,7 +24,6 @@ struct_list_head ptype_specific struct adj_list unsigned_int flags read_mostly read_mostly __dev_queue_xmit,__dev_xmit_skb,ip6_output,__ip6_finish_output(tx);ip6_rcv_core(rx) xdp_features_t xdp_features -unsigned_long_long priv_flags read_mostly - __dev_queue_xmit(tx) struct_net_device_ops* netdev_ops read_mostly - netdev_core_pick_tx,netdev_start_xmit(tx) struct_xdp_metadata_ops* xdp_metadata_ops int ifindex - read_mostly ip6_rcv_core @@ -163,6 +163,8 @@ struct_lock_class_key* qdisc_tx_busylock bool proto_down unsigned:1 wol_enabled unsigned:1 threaded - - napi_poll(napi_enable,dev_set_threaded) +unsigned_long:1 see_all_hwtstamp_requests +unsigned_long:1 change_proto_down struct_list_head net_notifier_list struct_macsec_ops* macsec_ops struct_udp_tunnel_nic_info* udp_tunnel_nic_info diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c index ec672af12e2515..534d4716d5f7d4 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c @@ -816,7 +816,7 @@ static int lan966x_probe_port(struct lan966x *lan966x, u32 p, NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_TC; dev->hw_features |= NETIF_F_HW_TC; - dev->priv_flags |= IFF_SEE_ALL_HWTSTAMP_REQUESTS; + dev->see_all_hwtstamp_requests = true; dev->needed_headroom = IFH_LEN_BYTES; eth_hw_addr_gen(dev, lan966x->base_mac, p + 1); diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index 24298a33e0e948..b45f137f365e7a 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -1213,7 +1213,8 @@ void macvlan_common_setup(struct net_device *dev) dev->max_mtu = ETH_MAX_MTU; dev->priv_flags &= ~IFF_TX_SKB_SHARING; netif_keep_dst(dev); - dev->priv_flags |= IFF_UNICAST_FLT | IFF_CHANGE_PROTO_DOWN; + dev->priv_flags |= IFF_UNICAST_FLT; + dev->change_proto_down = true; dev->netdev_ops = &macvlan_netdev_ops; dev->needs_free_netdev = true; dev->priv_destructor = macvlan_dev_free; diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index ba59e92ab941de..c1dedda4e6ba92 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -3332,7 +3332,8 @@ static void vxlan_setup(struct net_device *dev) dev->hw_features |= NETIF_F_RXCSUM; dev->hw_features |= NETIF_F_GSO_SOFTWARE; netif_keep_dst(dev); - dev->priv_flags |= IFF_NO_QUEUE | IFF_CHANGE_PROTO_DOWN; + dev->priv_flags |= IFF_NO_QUEUE; + dev->change_proto_down = true; /* MTU range: 68 - 65535 */ dev->min_mtu = ETH_MIN_MTU; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 607009150b5fa2..7d4d2758ea99ce 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1608,7 +1608,8 @@ struct net_device_ops { * userspace; this means that the order of these flags can change * during any kernel release. * - * You should have a pretty good reason to be extending these flags. + * You should add bitfield booleans after either net_device::priv_flags + * (hotpath) or ::threaded (slowpath) instead of extending these flags. * * @IFF_802_1Q_VLAN: 802.1Q VLAN device * @IFF_EBRIDGE: Ethernet bridging device @@ -1647,10 +1648,6 @@ struct net_device_ops { * @IFF_NO_ADDRCONF: prevent ipv6 addrconf * @IFF_TX_SKB_NO_LINEAR: device/driver is capable of xmitting frames with * skb_headlen(skb) == 0 (data starts from frag0) - * @IFF_CHANGE_PROTO_DOWN: device supports setting carrier via IFLA_PROTO_DOWN - * @IFF_SEE_ALL_HWTSTAMP_REQUESTS: device wants to see calls to - * ndo_hwtstamp_set() for all timestamp requests regardless of source, - * even if those aren't HWTSTAMP_SOURCE_NETDEV. */ enum netdev_priv_flags { IFF_802_1Q_VLAN = 1<<0, @@ -1685,8 +1682,6 @@ enum netdev_priv_flags { IFF_L3MDEV_RX_HANDLER = 1<<29, IFF_NO_ADDRCONF = BIT_ULL(30), IFF_TX_SKB_NO_LINEAR = BIT_ULL(31), - IFF_CHANGE_PROTO_DOWN = BIT_ULL(32), - IFF_SEE_ALL_HWTSTAMP_REQUESTS = BIT_ULL(33), }; #define IFF_802_1Q_VLAN IFF_802_1Q_VLAN @@ -1750,6 +1745,11 @@ enum netdev_reg_state { * data with strictly "high-level" data, and it has to know about * almost every data structure used in the INET module. * + * @priv_flags_fast: both hotpath private flags as bits and as bitfield + * booleans combined, only to assert cacheline placement + * @priv_flags: flags invisible to userspace defined as bits, see + * enum netdev_priv_flags for the definitions + * * @name: This is the first field of the "visible" part of this structure * (i.e. as seen by users in the "Space.c" file). It is the name * of the interface. @@ -1816,8 +1816,6 @@ enum netdev_reg_state { * * @flags: Interface flags (a la BSD) * @xdp_features: XDP capability supported by the device - * @priv_flags: Like 'flags' but invisible to userspace, - * see if.h for the definitions * @gflags: Global flags ( kept as legacy ) * @priv_len: Size of the ->priv flexible array * @priv: Flexible array containing private data @@ -1990,6 +1988,12 @@ enum netdev_reg_state { * * @threaded: napi threaded mode is enabled * + * @see_all_hwtstamp_requests: device wants to see calls to + * ndo_hwtstamp_set() for all timestamp requests + * regardless of source, even if those aren't + * HWTSTAMP_SOURCE_NETDEV + * @change_proto_down: device supports setting carrier via IFLA_PROTO_DOWN + * * @net_notifier_list: List of per-net netdev notifier block * that follow this device when it is moved * to another network namespace. @@ -2040,7 +2044,9 @@ struct net_device { /* TX read-mostly hotpath */ __cacheline_group_begin(net_device_read_tx); - unsigned long long priv_flags; + struct_group(priv_flags_fast, + unsigned long priv_flags:32; + ); const struct net_device_ops *netdev_ops; const struct header_ops *header_ops; struct netdev_queue *_tx; @@ -2375,6 +2381,10 @@ struct net_device { bool proto_down; bool threaded; + /* priv_flags_slow, ungrouped to save space */ + unsigned long see_all_hwtstamp_requests:1; + unsigned long change_proto_down:1; + struct list_head net_notifier_list; #if IS_ENABLED(CONFIG_MACSEC) diff --git a/net/8021q/vlanproc.c b/net/8021q/vlanproc.c index 87b959da00cd38..fa67374bda494c 100644 --- a/net/8021q/vlanproc.c +++ b/net/8021q/vlanproc.c @@ -238,9 +238,9 @@ static int vlandev_seq_show(struct seq_file *seq, void *offset) stats = dev_get_stats(vlandev, &temp); seq_printf(seq, - "%s VID: %d REORDER_HDR: %i dev->priv_flags: %llx\n", + "%s VID: %d REORDER_HDR: %i dev->priv_flags: %x\n", vlandev->name, vlan->vlan_id, - (int)(vlan->flags & 1), vlandev->priv_flags); + (int)(vlan->flags & 1), (u32)vlandev->priv_flags); seq_printf(seq, fmt64, "total frames received", stats->rx_packets); seq_printf(seq, fmt64, "total bytes received", stats->rx_bytes); diff --git a/net/core/dev.c b/net/core/dev.c index 6ea1d20676fbe9..c0831853a5f89e 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -9271,7 +9271,7 @@ EXPORT_SYMBOL(netdev_port_same_parent_id); */ int dev_change_proto_down(struct net_device *dev, bool proto_down) { - if (!(dev->priv_flags & IFF_CHANGE_PROTO_DOWN)) + if (!dev->change_proto_down) return -EOPNOTSUPP; if (!netif_device_present(dev)) return -ENODEV; @@ -11896,7 +11896,7 @@ static struct pernet_operations __net_initdata default_device_ops = { static void __init net_dev_struct_check(void) { /* TX read-mostly hotpath */ - CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, priv_flags); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, priv_flags_fast); CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, netdev_ops); CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, header_ops); CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, _tx); diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c index 8592c052c0f4f3..473c437b6b5386 100644 --- a/net/core/dev_ioctl.c +++ b/net/core/dev_ioctl.c @@ -317,8 +317,7 @@ static int dev_get_hwtstamp(struct net_device *dev, struct ifreq *ifr) * should take precedence in front of hardware timestamping provided by the * netdev. If the netdev driver needs to perform specific actions even for PHY * timestamping to work properly (a switch port must trap the timestamped - * frames and not forward them), it must set IFF_SEE_ALL_HWTSTAMP_REQUESTS in - * dev->priv_flags. + * frames and not forward them), it must set dev->see_all_hwtstamp_requests. */ int dev_set_hwtstamp_phylib(struct net_device *dev, struct kernel_hwtstamp_config *cfg, @@ -332,13 +331,13 @@ int dev_set_hwtstamp_phylib(struct net_device *dev, cfg->source = phy_ts ? HWTSTAMP_SOURCE_PHYLIB : HWTSTAMP_SOURCE_NETDEV; - if (phy_ts && (dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS)) { + if (phy_ts && dev->see_all_hwtstamp_requests) { err = ops->ndo_hwtstamp_get(dev, &old_cfg); if (err) return err; } - if (!phy_ts || (dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS)) { + if (!phy_ts || dev->see_all_hwtstamp_requests) { err = ops->ndo_hwtstamp_set(dev, cfg, extack); if (err) { if (extack->_msg) @@ -347,7 +346,7 @@ int dev_set_hwtstamp_phylib(struct net_device *dev, } } - if (phy_ts && (dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS)) + if (phy_ts && dev->see_all_hwtstamp_requests) changed = kernel_hwtstamp_config_changed(&old_cfg, cfg); if (phy_ts) { diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 87e67194f24046..a2889806abb4a3 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -2724,7 +2724,7 @@ static int do_set_proto_down(struct net_device *dev, bool proto_down; int err; - if (!(dev->priv_flags & IFF_CHANGE_PROTO_DOWN)) { + if (!dev->change_proto_down) { NL_SET_ERR_MSG(extack, "Protodown not supported by device"); return -EOPNOTSUPP; } From 1c76f4664343ef2dd9130f7187a0921365b8bdaa Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Thu, 4 Apr 2024 13:29:12 +0200 Subject: [PATCH 02/49] netdev_features: remove unused __UNUSED_NETIF_F_1 NETIF_F_NO_CSUM was removed in 3.2-rc2 by commit 34324dc2bf27 ("net: remove NETIF_F_NO_CSUM feature bit") and became __UNUSED_NETIF_F_1. It's not used anywhere in the code. Remove this bit waste. It wasn't needed to rename the flag instead of removing it as netdev features are not uAPI/ABI. Ethtool passes their names and values separately with no fixed positions and the userspace Ethtool code doesn't have any hardcoded feature names/bits, so that new Ethtool will work on older kernels and vice versa. Signed-off-by: Alexander Lobakin --- include/linux/netdev_features.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h index 7c2d77d75a888c..44c428d62db4e5 100644 --- a/include/linux/netdev_features.h +++ b/include/linux/netdev_features.h @@ -14,7 +14,6 @@ typedef u64 netdev_features_t; enum { NETIF_F_SG_BIT, /* Scatter/gather IO. */ NETIF_F_IP_CSUM_BIT, /* Can checksum TCP/UDP over IPv4. */ - __UNUSED_NETIF_F_1, NETIF_F_HW_CSUM_BIT, /* Can checksum all the packets. */ NETIF_F_IPV6_CSUM_BIT, /* Can checksum TCP/UDP over IPV6 */ NETIF_F_HIGHDMA_BIT, /* Can DMA to high memory. */ From ad1f18e4598103bd60baf6649c30d065c9e3c931 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 25 Jun 2024 13:20:58 +0200 Subject: [PATCH 03/49] netdev_features: convert NETIF_F_LLTX to dev->lltx NETIF_F_LLTX can't be changed via Ethtool and is not a feature, rather an attribute, very similar to IFF_NO_QUEUE (and hot). Free one netdev_features_t bit and make it a "hot" private flag. Signed-off-by: Alexander Lobakin --- Documentation/networking/net_cachelines/net_device.rst | 1 + Documentation/networking/netdev-features.rst | 8 -------- Documentation/networking/netdevices.rst | 4 ++-- drivers/net/amt.c | 2 +- drivers/net/bareudp.c | 2 +- drivers/net/bonding/bond_main.c | 2 +- drivers/net/dummy.c | 3 ++- drivers/net/ethernet/chelsio/cxgb/cxgb2.c | 3 ++- drivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 3 ++- drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c | 3 ++- drivers/net/ethernet/mellanox/mlxsw/spectrum.c | 3 ++- drivers/net/ethernet/netronome/nfp/nfp_net_repr.c | 3 +-- drivers/net/ethernet/pasemi/pasemi_mac.c | 5 +++-- drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c | 2 +- drivers/net/ethernet/sfc/ef100_rep.c | 4 ++-- drivers/net/ethernet/tehuti/tehuti.c | 4 ++-- drivers/net/ethernet/tehuti/tehuti.h | 2 +- drivers/net/ethernet/toshiba/spider_net.c | 3 ++- drivers/net/geneve.c | 2 +- drivers/net/gtp.c | 2 +- drivers/net/hamradio/bpqether.c | 2 +- drivers/net/ipvlan/ipvlan_main.c | 3 ++- drivers/net/loopback.c | 2 +- drivers/net/macsec.c | 4 ++-- drivers/net/macvlan.c | 3 ++- drivers/net/net_failover.c | 2 +- drivers/net/netkit.c | 3 ++- drivers/net/nlmon.c | 4 ++-- drivers/net/ppp/ppp_generic.c | 2 +- drivers/net/rionet.c | 2 +- drivers/net/team/team_core.c | 2 +- drivers/net/tun.c | 5 +++-- drivers/net/veth.c | 2 +- drivers/net/vrf.c | 2 +- drivers/net/vsockmon.c | 4 ++-- drivers/net/vxlan/vxlan_core.c | 2 +- drivers/net/wireguard/device.c | 2 +- drivers/staging/octeon/ethernet.c | 2 +- include/linux/netdev_features.h | 5 +---- include/linux/netdevice.h | 9 ++++++--- lib/test_bpf.c | 3 +-- net/8021q/vlan_dev.c | 4 ++-- net/batman-adv/soft-interface.c | 2 +- net/bridge/br_device.c | 3 ++- net/core/net-sysfs.c | 3 +-- net/dsa/user.c | 3 ++- net/ethtool/common.c | 1 - net/hsr/hsr_device.c | 4 ++-- net/ipv4/ip_gre.c | 4 +++- net/ipv4/ip_vti.c | 2 +- net/ipv4/ipip.c | 2 +- net/ipv6/ip6_gre.c | 4 +++- net/ipv6/ip6_tunnel.c | 2 +- net/ipv6/sit.c | 2 +- net/l2tp/l2tp_eth.c | 2 +- net/openvswitch/vport-internal_dev.c | 9 +++++---- net/xfrm/xfrm_interface_core.c | 2 +- 57 files changed, 91 insertions(+), 84 deletions(-) diff --git a/Documentation/networking/net_cachelines/net_device.rst b/Documentation/networking/net_cachelines/net_device.rst index d7ba48ff55595f..9b2b3aeaa4a999 100644 --- a/Documentation/networking/net_cachelines/net_device.rst +++ b/Documentation/networking/net_cachelines/net_device.rst @@ -8,6 +8,7 @@ net_device struct fast path usage breakdown Type Name fastpath_tx_access fastpath_rx_access Comments ..struct ..net_device unsigned_long:32 priv_flags read_mostly - __dev_queue_xmit(tx) +unsigned_long:1 lltx read_mostly - HARD_TX_LOCK,HARD_TX_TRYLOCK,HARD_TX_UNLOCK(tx) char name[16] - - struct_netdev_name_node* name_node struct_dev_ifalias* ifalias diff --git a/Documentation/networking/netdev-features.rst b/Documentation/networking/netdev-features.rst index d7b15bb64deb63..f29d982ebf5d3b 100644 --- a/Documentation/networking/netdev-features.rst +++ b/Documentation/networking/netdev-features.rst @@ -139,14 +139,6 @@ chained skbs (skb->next/prev list). Features contained in NETIF_F_SOFT_FEATURES are features of networking stack. Driver should not change behaviour based on them. - * LLTX driver (deprecated for hardware drivers) - -NETIF_F_LLTX is meant to be used by drivers that don't need locking at all, -e.g. software tunnels. - -This is also used in a few legacy drivers that implement their -own locking, don't use it for new (hardware) drivers. - * netns-local device NETIF_F_NETNS_LOCAL is set for devices that are not allowed to move between diff --git a/Documentation/networking/netdevices.rst b/Documentation/networking/netdevices.rst index c2476917a6c37d..857c9784f87efc 100644 --- a/Documentation/networking/netdevices.rst +++ b/Documentation/networking/netdevices.rst @@ -258,11 +258,11 @@ ndo_get_stats: ndo_start_xmit: Synchronization: __netif_tx_lock spinlock. - When the driver sets NETIF_F_LLTX in dev->features this will be + When the driver sets dev->lltx this will be called without holding netif_tx_lock. In this case the driver has to lock by itself when needed. The locking there should also properly protect against - set_rx_mode. WARNING: use of NETIF_F_LLTX is deprecated. + set_rx_mode. WARNING: use of dev->lltx is deprecated. Don't use it for new drivers. Context: Process with BHs disabled or BH (timer), diff --git a/drivers/net/amt.c b/drivers/net/amt.c index 6d15ab3bfbbcac..921bbfd72a3868 100644 --- a/drivers/net/amt.c +++ b/drivers/net/amt.c @@ -3098,7 +3098,7 @@ static void amt_link_setup(struct net_device *dev) dev->hard_header_len = 0; dev->addr_len = 0; dev->priv_flags |= IFF_NO_QUEUE; - dev->features |= NETIF_F_LLTX; + dev->lltx = true; dev->features |= NETIF_F_GSO_SOFTWARE; dev->features |= NETIF_F_NETNS_LOCAL; dev->hw_features |= NETIF_F_SG | NETIF_F_HW_CSUM; diff --git a/drivers/net/bareudp.c b/drivers/net/bareudp.c index d5c56ca91b7716..6f4de883e872f8 100644 --- a/drivers/net/bareudp.c +++ b/drivers/net/bareudp.c @@ -553,7 +553,6 @@ static void bareudp_setup(struct net_device *dev) SET_NETDEV_DEVTYPE(dev, &bareudp_type); dev->features |= NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_FRAGLIST; dev->features |= NETIF_F_RXCSUM; - dev->features |= NETIF_F_LLTX; dev->features |= NETIF_F_GSO_SOFTWARE; dev->hw_features |= NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_FRAGLIST; dev->hw_features |= NETIF_F_RXCSUM; @@ -566,6 +565,7 @@ static void bareudp_setup(struct net_device *dev) dev->type = ARPHRD_NONE; netif_keep_dst(dev); dev->priv_flags |= IFF_NO_QUEUE; + dev->lltx = true; dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST; dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; } diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index af9ddd3902cc96..48fb5e0f8bfdc5 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -5889,7 +5889,7 @@ void bond_setup(struct net_device *bond_dev) #endif /* CONFIG_XFRM_OFFLOAD */ /* don't acquire bond device's netif_tx_lock when transmitting */ - bond_dev->features |= NETIF_F_LLTX; + bond_dev->lltx = true; /* By default, we declare the bond to be fully * VLAN hardware accelerated capable. Special diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c index d29b5d7af0d72b..e9c5e1e11fa02d 100644 --- a/drivers/net/dummy.c +++ b/drivers/net/dummy.c @@ -109,9 +109,10 @@ static void dummy_setup(struct net_device *dev) dev->flags |= IFF_NOARP; dev->flags &= ~IFF_MULTICAST; dev->priv_flags |= IFF_LIVE_ADDR_CHANGE | IFF_NO_QUEUE; + dev->lltx = true; dev->features |= NETIF_F_SG | NETIF_F_FRAGLIST; dev->features |= NETIF_F_GSO_SOFTWARE; - dev->features |= NETIF_F_HW_CSUM | NETIF_F_HIGHDMA | NETIF_F_LLTX; + dev->features |= NETIF_F_HW_CSUM | NETIF_F_HIGHDMA; dev->features |= NETIF_F_GSO_ENCAP_ALL; dev->hw_features |= dev->features; dev->hw_enc_features |= dev->features; diff --git a/drivers/net/ethernet/chelsio/cxgb/cxgb2.c b/drivers/net/ethernet/chelsio/cxgb/cxgb2.c index 7d7d3e0098df2e..3b7068832f95ef 100644 --- a/drivers/net/ethernet/chelsio/cxgb/cxgb2.c +++ b/drivers/net/ethernet/chelsio/cxgb/cxgb2.c @@ -1034,7 +1034,8 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) netdev->hw_features |= NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_RXCSUM; netdev->features |= NETIF_F_SG | NETIF_F_IP_CSUM | - NETIF_F_RXCSUM | NETIF_F_LLTX | NETIF_F_HIGHDMA; + NETIF_F_RXCSUM | NETIF_F_HIGHDMA; + netdev->lltx = true; if (vlan_tso_capable(adapter)) { netdev->features |= diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c index cfe6b57b1da0e4..d3929d236df525 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c @@ -229,7 +229,7 @@ static int dpaa_netdev_init(struct net_device *net_dev, net_dev->max_mtu = dpaa_get_max_mtu(); net_dev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | - NETIF_F_LLTX | NETIF_F_RXHASH); + NETIF_F_RXHASH); net_dev->hw_features |= NETIF_F_SG | NETIF_F_HIGHDMA; /* The kernels enables GSO automatically, if we declare NETIF_F_SG. @@ -239,6 +239,7 @@ static int dpaa_netdev_init(struct net_device *net_dev, net_dev->features |= NETIF_F_RXCSUM; net_dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; + net_dev->lltx = true; /* we do not want shared skbs on TX */ net_dev->priv_flags &= ~IFF_TX_SKB_SHARING; diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c index 6866807973daa1..29886a8ba73f33 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c @@ -4594,12 +4594,13 @@ static int dpaa2_eth_netdev_init(struct net_device *net_dev) net_dev->priv_flags |= supported; net_dev->priv_flags &= ~not_supported; + net_dev->lltx = true; /* Features */ net_dev->features = NETIF_F_RXCSUM | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | NETIF_F_HIGHDMA | - NETIF_F_LLTX | NETIF_F_HW_TC | NETIF_F_TSO; + NETIF_F_HW_TC | NETIF_F_TSO; net_dev->gso_max_segs = DPAA2_ETH_ENQUEUE_MAX_FDS; net_dev->hw_features = net_dev->features; net_dev->xdp_features = NETDEV_XDP_ACT_BASIC | diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index f064789f324058..44d6e125bd6f00 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -1676,9 +1676,10 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u16 local_port, netif_carrier_off(dev); - dev->features |= NETIF_F_NETNS_LOCAL | NETIF_F_LLTX | NETIF_F_SG | + dev->features |= NETIF_F_NETNS_LOCAL | NETIF_F_SG | NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_TC; dev->hw_features |= NETIF_F_HW_TC | NETIF_F_LOOPBACK; + dev->lltx = true; dev->min_mtu = ETH_MIN_MTU; dev->max_mtu = MLXSW_PORT_MAX_MTU - MLXSW_PORT_ETH_FRAME_HDR; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c index eee0bfc41074ee..227e7a5d712e37 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c @@ -248,7 +248,6 @@ nfp_repr_fix_features(struct net_device *netdev, netdev_features_t features) features = netdev_intersect_features(features, lower_features); features |= old_features & (NETIF_F_SOFT_FEATURES | NETIF_F_HW_TC); - features |= NETIF_F_LLTX; return features; } @@ -386,7 +385,7 @@ int nfp_repr_init(struct nfp_app *app, struct net_device *netdev, netif_set_tso_max_segs(netdev, NFP_NET_LSO_MAX_SEGS); netdev->priv_flags |= IFF_NO_QUEUE | IFF_DISABLE_NETPOLL; - netdev->features |= NETIF_F_LLTX; + netdev->lltx = true; if (nfp_app_has_tc(app)) { netdev->features |= NETIF_F_HW_TC; diff --git a/drivers/net/ethernet/pasemi/pasemi_mac.c b/drivers/net/ethernet/pasemi/pasemi_mac.c index 62ba269da90265..cb4e12df7719df 100644 --- a/drivers/net/ethernet/pasemi/pasemi_mac.c +++ b/drivers/net/ethernet/pasemi/pasemi_mac.c @@ -1699,8 +1699,9 @@ pasemi_mac_probe(struct pci_dev *pdev, const struct pci_device_id *ent) netif_napi_add(dev, &mac->napi, pasemi_mac_poll); - dev->features = NETIF_F_IP_CSUM | NETIF_F_LLTX | NETIF_F_SG | - NETIF_F_HIGHDMA | NETIF_F_GSO; + dev->features = NETIF_F_IP_CSUM | NETIF_F_SG | NETIF_F_HIGHDMA | + NETIF_F_GSO; + dev->lltx = true; mac->dma_pdev = pci_get_device(PCI_VENDOR_ID_PASEMI, 0xa007, NULL); if (!mac->dma_pdev) { diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c index f1e40aade127bf..4f0ddcedfa9790 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c @@ -286,7 +286,7 @@ void rmnet_vnd_setup(struct net_device *rmnet_dev) rmnet_dev->needs_free_netdev = true; rmnet_dev->ethtool_ops = &rmnet_ethtool_ops; - rmnet_dev->features |= NETIF_F_LLTX; + rmnet_dev->lltx = true; /* This perm addr will be used as interface identifier by IPv6 */ rmnet_dev->addr_assign_type = NET_ADDR_RANDOM; diff --git a/drivers/net/ethernet/sfc/ef100_rep.c b/drivers/net/ethernet/sfc/ef100_rep.c index 0b3083ef0eada4..e923e1796369d0 100644 --- a/drivers/net/ethernet/sfc/ef100_rep.c +++ b/drivers/net/ethernet/sfc/ef100_rep.c @@ -233,8 +233,8 @@ static struct efx_rep *efx_ef100_rep_create_netdev(struct efx_nic *efx, net_dev->ethtool_ops = &efx_ef100_rep_ethtool_ops; net_dev->min_mtu = EFX_MIN_MTU; net_dev->max_mtu = EFX_MAX_MTU; - net_dev->features |= NETIF_F_LLTX; - net_dev->hw_features |= NETIF_F_LLTX; + net_dev->lltx = true; + return efv; fail1: free_netdev(net_dev); diff --git a/drivers/net/ethernet/tehuti/tehuti.c b/drivers/net/ethernet/tehuti/tehuti.c index ede5f7890fb4b0..fc77f424f90bb1 100644 --- a/drivers/net/ethernet/tehuti/tehuti.c +++ b/drivers/net/ethernet/tehuti/tehuti.c @@ -1671,7 +1671,7 @@ static netdev_tx_t bdx_tx_transmit(struct sk_buff *skb, #endif #ifdef BDX_LLTX - netif_trans_update(ndev); /* NETIF_F_LLTX driver :( */ + netif_trans_update(ndev); /* dev->lltx driver :( */ #endif ndev->stats.tx_packets++; ndev->stats.tx_bytes += skb->len; @@ -2019,7 +2019,7 @@ bdx_probe(struct pci_dev *pdev, const struct pci_device_id *ent) * set multicast list callback has to use priv->tx_lock. */ #ifdef BDX_LLTX - ndev->features |= NETIF_F_LLTX; + ndev->lltx = true; #endif /* MTU range: 60 - 16384 */ ndev->min_mtu = ETH_ZLEN; diff --git a/drivers/net/ethernet/tehuti/tehuti.h b/drivers/net/ethernet/tehuti/tehuti.h index 909e7296cecfa4..47a2d3e5f8ede6 100644 --- a/drivers/net/ethernet/tehuti/tehuti.h +++ b/drivers/net/ethernet/tehuti/tehuti.h @@ -260,7 +260,7 @@ struct bdx_priv { int tx_update_mark; int tx_noupd; #endif - spinlock_t tx_lock; /* NETIF_F_LLTX mode */ + spinlock_t tx_lock; /* dev->lltx mode */ /* rarely used */ u8 port; diff --git a/drivers/net/ethernet/toshiba/spider_net.c b/drivers/net/ethernet/toshiba/spider_net.c index 87e67121477cb5..a4937c18d7cb5b 100644 --- a/drivers/net/ethernet/toshiba/spider_net.c +++ b/drivers/net/ethernet/toshiba/spider_net.c @@ -2277,10 +2277,11 @@ spider_net_setup_netdev(struct spider_net_card *card) netdev->hw_features = NETIF_F_RXCSUM | NETIF_F_IP_CSUM; if (SPIDER_NET_RX_CSUM_DEFAULT) netdev->features |= NETIF_F_RXCSUM; - netdev->features |= NETIF_F_IP_CSUM | NETIF_F_LLTX; + netdev->features |= NETIF_F_IP_CSUM; /* some time: NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | * NETIF_F_HW_VLAN_CTAG_FILTER */ + netdev->lltx = true; /* MTU range: 64 - 2294 */ netdev->min_mtu = SPIDER_NET_MIN_MTU; diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c index 838e85ddec6710..7f611c74eb629b 100644 --- a/drivers/net/geneve.c +++ b/drivers/net/geneve.c @@ -1194,7 +1194,6 @@ static void geneve_setup(struct net_device *dev) SET_NETDEV_DEVTYPE(dev, &geneve_type); - dev->features |= NETIF_F_LLTX; dev->features |= NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_FRAGLIST; dev->features |= NETIF_F_RXCSUM; dev->features |= NETIF_F_GSO_SOFTWARE; @@ -1215,6 +1214,7 @@ static void geneve_setup(struct net_device *dev) netif_keep_dst(dev); dev->priv_flags &= ~IFF_TX_SKB_SHARING; dev->priv_flags |= IFF_LIVE_ADDR_CHANGE | IFF_NO_QUEUE; + dev->lltx = true; eth_hw_addr_random(dev); } diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index 427b91aca50d3a..1a4afb216a0151 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -1353,7 +1353,7 @@ static void gtp_link_setup(struct net_device *dev) dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; dev->priv_flags |= IFF_NO_QUEUE; - dev->features |= NETIF_F_LLTX; + dev->lltx = true; netif_keep_dst(dev); dev->needed_headroom = LL_MAX_HEADER + GTP_IPV4_MAXLEN; diff --git a/drivers/net/hamradio/bpqether.c b/drivers/net/hamradio/bpqether.c index 83a16d10eedbc7..bac1bb69d63a11 100644 --- a/drivers/net/hamradio/bpqether.c +++ b/drivers/net/hamradio/bpqether.c @@ -458,7 +458,7 @@ static void bpq_setup(struct net_device *dev) dev->needs_free_netdev = true; dev->flags = 0; - dev->features = NETIF_F_LLTX; /* Allow recursion */ + dev->lltx = true; /* Allow recursion */ #if IS_ENABLED(CONFIG_AX25) dev->header_ops = &ax25_header_ops; diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c index 094f44dac5c85f..ee2c3cf4df365e 100644 --- a/drivers/net/ipvlan/ipvlan_main.c +++ b/drivers/net/ipvlan/ipvlan_main.c @@ -114,7 +114,7 @@ static void ipvlan_port_destroy(struct net_device *dev) NETIF_F_GSO_ROBUST | NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL) #define IPVLAN_ALWAYS_ON \ - (IPVLAN_ALWAYS_ON_OFLOADS | NETIF_F_LLTX | NETIF_F_VLAN_CHALLENGED) + (IPVLAN_ALWAYS_ON_OFLOADS | NETIF_F_VLAN_CHALLENGED) #define IPVLAN_FEATURES \ (NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST | \ @@ -141,6 +141,7 @@ static int ipvlan_init(struct net_device *dev) dev->vlan_features = phy_dev->vlan_features & IPVLAN_FEATURES; dev->vlan_features |= IPVLAN_ALWAYS_ON_OFLOADS; dev->hw_enc_features |= dev->features; + dev->lltx = true; netif_inherit_tso_max(dev, phy_dev); dev->hard_header_len = phy_dev->hard_header_len; diff --git a/drivers/net/loopback.c b/drivers/net/loopback.c index 2b486e7c749ca3..bf857782be0ff3 100644 --- a/drivers/net/loopback.c +++ b/drivers/net/loopback.c @@ -171,6 +171,7 @@ static void gen_lo_setup(struct net_device *dev, dev->type = ARPHRD_LOOPBACK; /* 0x0001*/ dev->flags = IFF_LOOPBACK; dev->priv_flags |= IFF_LIVE_ADDR_CHANGE | IFF_NO_QUEUE; + dev->lltx = true; netif_keep_dst(dev); dev->hw_features = NETIF_F_GSO_SOFTWARE; dev->features = NETIF_F_SG | NETIF_F_FRAGLIST @@ -179,7 +180,6 @@ static void gen_lo_setup(struct net_device *dev, | NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA - | NETIF_F_LLTX | NETIF_F_NETNS_LOCAL | NETIF_F_VLAN_CHALLENGED | NETIF_F_LOOPBACK; diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index 2da70bc3dd8695..12d1b205f6d117 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -3550,7 +3550,8 @@ static int macsec_dev_init(struct net_device *dev) return err; dev->features = real_dev->features & MACSEC_FEATURES; - dev->features |= NETIF_F_LLTX | NETIF_F_GSO_SOFTWARE; + dev->features |= NETIF_F_GSO_SOFTWARE; + dev->lltx = true; dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; macsec_set_head_tail_room(dev); @@ -3581,7 +3582,6 @@ static netdev_features_t macsec_fix_features(struct net_device *dev, features &= (real_dev->features & MACSEC_FEATURES) | NETIF_F_GSO_SOFTWARE | NETIF_F_SOFT_FEATURES; - features |= NETIF_F_LLTX; return features; } diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index b45f137f365e7a..cf18e66de142c9 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -900,7 +900,7 @@ static struct lock_class_key macvlan_netdev_addr_lock_key; (NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE | \ NETIF_F_GSO_ROBUST | NETIF_F_GSO_ENCAP_ALL) -#define ALWAYS_ON_FEATURES (ALWAYS_ON_OFFLOADS | NETIF_F_LLTX) +#define ALWAYS_ON_FEATURES ALWAYS_ON_OFFLOADS #define MACVLAN_FEATURES \ (NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST | \ @@ -932,6 +932,7 @@ static int macvlan_init(struct net_device *dev) dev->vlan_features = lowerdev->vlan_features & MACVLAN_FEATURES; dev->vlan_features |= ALWAYS_ON_OFFLOADS; dev->hw_enc_features |= dev->features; + dev->lltx = true; netif_inherit_tso_max(dev, lowerdev); dev->hard_header_len = lowerdev->hard_header_len; macvlan_set_lockdep_class(dev); diff --git a/drivers/net/net_failover.c b/drivers/net/net_failover.c index 963d8b4af28d79..06728385a35fb2 100644 --- a/drivers/net/net_failover.c +++ b/drivers/net/net_failover.c @@ -731,7 +731,7 @@ struct failover *net_failover_create(struct net_device *standby_dev) IFF_TX_SKB_SHARING); /* don't acquire failover netdev's netif_tx_lock when transmitting */ - failover_dev->features |= NETIF_F_LLTX; + failover_dev->lltx = true; /* Don't allow failover devices to change network namespaces. */ failover_dev->features |= NETIF_F_NETNS_LOCAL; diff --git a/drivers/net/netkit.c b/drivers/net/netkit.c index 16789cd446e9e4..79232f5cc0882f 100644 --- a/drivers/net/netkit.c +++ b/drivers/net/netkit.c @@ -255,11 +255,12 @@ static void netkit_setup(struct net_device *dev) dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; dev->priv_flags |= IFF_PHONY_HEADROOM; dev->priv_flags |= IFF_NO_QUEUE; + dev->lltx = true; dev->ethtool_ops = &netkit_ethtool_ops; dev->netdev_ops = &netkit_netdev_ops; - dev->features |= netkit_features | NETIF_F_LLTX; + dev->features |= netkit_features; dev->hw_features = netkit_features; dev->hw_enc_features = netkit_features; dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; diff --git a/drivers/net/nlmon.c b/drivers/net/nlmon.c index e5a0987a263e55..8bfd4ee5a8c4fb 100644 --- a/drivers/net/nlmon.c +++ b/drivers/net/nlmon.c @@ -63,13 +63,13 @@ static void nlmon_setup(struct net_device *dev) { dev->type = ARPHRD_NETLINK; dev->priv_flags |= IFF_NO_QUEUE; + dev->lltx = true; dev->netdev_ops = &nlmon_ops; dev->ethtool_ops = &nlmon_ethtool_ops; dev->needs_free_netdev = true; - dev->features = NETIF_F_SG | NETIF_F_FRAGLIST | - NETIF_F_HIGHDMA | NETIF_F_LLTX; + dev->features = NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HIGHDMA; dev->flags = IFF_NOARP; dev->pcpu_stat_type = NETDEV_PCPU_STAT_LSTATS; diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c index eb9acfcaeb0974..4b2971e2bf484a 100644 --- a/drivers/net/ppp/ppp_generic.c +++ b/drivers/net/ppp/ppp_generic.c @@ -1631,7 +1631,7 @@ static void ppp_setup(struct net_device *dev) dev->netdev_ops = &ppp_netdev_ops; SET_NETDEV_DEVTYPE(dev, &ppp_type); - dev->features |= NETIF_F_LLTX; + dev->lltx = true; dev->hard_header_len = PPP_HDRLEN; dev->mtu = PPP_MRU; diff --git a/drivers/net/rionet.c b/drivers/net/rionet.c index 4eececc945138b..318a0ef1af50da 100644 --- a/drivers/net/rionet.c +++ b/drivers/net/rionet.c @@ -515,7 +515,7 @@ static int rionet_setup_netdev(struct rio_mport *mport, struct net_device *ndev) /* MTU range: 68 - 4082 */ ndev->min_mtu = ETH_MIN_MTU; ndev->max_mtu = RIONET_MAX_MTU; - ndev->features = NETIF_F_LLTX; + ndev->lltx = true; SET_NETDEV_DEV(ndev, &mport->dev); ndev->ethtool_ops = &rionet_ethtool_ops; diff --git a/drivers/net/team/team_core.c b/drivers/net/team/team_core.c index ab1935a4aa2cd6..1d1bad3cedc2d7 100644 --- a/drivers/net/team/team_core.c +++ b/drivers/net/team/team_core.c @@ -2189,8 +2189,8 @@ static void team_setup(struct net_device *dev) * Let this up to underlay drivers. */ dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE; + dev->lltx = true; - dev->features |= NETIF_F_LLTX; dev->features |= NETIF_F_GRO; /* Don't allow team devices to change network namespaces. */ diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 9b24861464bc65..dede5a9020a5f2 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -990,10 +990,11 @@ static int tun_net_init(struct net_device *dev) dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST | TUN_USER_FEATURES | NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_STAG_TX; - dev->features = dev->hw_features | NETIF_F_LLTX; + dev->features = dev->hw_features; dev->vlan_features = dev->features & ~(NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_STAG_TX); + dev->lltx = true; tun->flags = (tun->flags & ~TUN_FEATURES) | (ifr->ifr_flags & TUN_FEATURES); @@ -1129,7 +1130,7 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) goto drop; } - /* NETIF_F_LLTX requires to do our own update of trans_start */ + /* dev->lltx requires to do our own update of trans_start */ queue = netdev_get_tx_queue(dev, txq); txq_trans_cond_update(queue); diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 426e68a950672e..925b45d993f169 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -1696,11 +1696,11 @@ static void veth_setup(struct net_device *dev) dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; dev->priv_flags |= IFF_NO_QUEUE; dev->priv_flags |= IFF_PHONY_HEADROOM; + dev->lltx = true; dev->netdev_ops = &veth_netdev_ops; dev->xdp_metadata_ops = &veth_xdp_metadata_ops; dev->ethtool_ops = &veth_ethtool_ops; - dev->features |= NETIF_F_LLTX; dev->features |= VETH_FEATURES; dev->vlan_features = dev->features & ~(NETIF_F_HW_VLAN_CTAG_TX | diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c index 9af316cdd8b3bc..fce065d0b5a0a7 100644 --- a/drivers/net/vrf.c +++ b/drivers/net/vrf.c @@ -1634,7 +1634,7 @@ static void vrf_setup(struct net_device *dev) eth_hw_addr_random(dev); /* don't acquire vrf device's netif_tx_lock when transmitting */ - dev->features |= NETIF_F_LLTX; + dev->lltx = true; /* don't allow vrf devices to change network namespaces. */ dev->features |= NETIF_F_NETNS_LOCAL; diff --git a/drivers/net/vsockmon.c b/drivers/net/vsockmon.c index 4c260074c091f1..53fb76d574c6e2 100644 --- a/drivers/net/vsockmon.c +++ b/drivers/net/vsockmon.c @@ -83,13 +83,13 @@ static void vsockmon_setup(struct net_device *dev) { dev->type = ARPHRD_VSOCKMON; dev->priv_flags |= IFF_NO_QUEUE; + dev->lltx = true; dev->netdev_ops = &vsockmon_ops; dev->ethtool_ops = &vsockmon_ethtool_ops; dev->needs_free_netdev = true; - dev->features = NETIF_F_SG | NETIF_F_FRAGLIST | - NETIF_F_HIGHDMA | NETIF_F_LLTX; + dev->features = NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HIGHDMA; dev->flags = IFF_NOARP; diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index c1dedda4e6ba92..e14166a0a52c39 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -3322,7 +3322,6 @@ static void vxlan_setup(struct net_device *dev) dev->needs_free_netdev = true; SET_NETDEV_DEVTYPE(dev, &vxlan_type); - dev->features |= NETIF_F_LLTX; dev->features |= NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_FRAGLIST; dev->features |= NETIF_F_RXCSUM; dev->features |= NETIF_F_GSO_SOFTWARE; @@ -3334,6 +3333,7 @@ static void vxlan_setup(struct net_device *dev) netif_keep_dst(dev); dev->priv_flags |= IFF_NO_QUEUE; dev->change_proto_down = true; + dev->lltx = true; /* MTU range: 68 - 65535 */ dev->min_mtu = ETH_MIN_MTU; diff --git a/drivers/net/wireguard/device.c b/drivers/net/wireguard/device.c index 3feb36ee5bfb44..45e9b908dbfb05 100644 --- a/drivers/net/wireguard/device.c +++ b/drivers/net/wireguard/device.c @@ -289,7 +289,7 @@ static void wg_setup(struct net_device *dev) dev->type = ARPHRD_NONE; dev->flags = IFF_POINTOPOINT | IFF_NOARP; dev->priv_flags |= IFF_NO_QUEUE; - dev->features |= NETIF_F_LLTX; + dev->lltx = true; dev->features |= WG_NETDEV_FEATURES; dev->hw_features |= WG_NETDEV_FEATURES; dev->hw_enc_features |= WG_NETDEV_FEATURES; diff --git a/drivers/staging/octeon/ethernet.c b/drivers/staging/octeon/ethernet.c index 9eee28f2940cf1..a5e99cc78a454f 100644 --- a/drivers/staging/octeon/ethernet.c +++ b/drivers/staging/octeon/ethernet.c @@ -425,7 +425,7 @@ int cvm_oct_common_init(struct net_device *dev) dev->features |= NETIF_F_SG | NETIF_F_IP_CSUM; /* We do our own locking, Linux doesn't need to */ - dev->features |= NETIF_F_LLTX; + dev->lltx = true; dev->ethtool_ops = &cvm_oct_ethtool_ops; cvm_oct_set_mac_filter(dev); diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h index 44c428d62db4e5..54d1578f6642aa 100644 --- a/include/linux/netdev_features.h +++ b/include/linux/netdev_features.h @@ -23,8 +23,6 @@ enum { NETIF_F_HW_VLAN_CTAG_FILTER_BIT,/* Receive filtering on VLAN CTAGs */ NETIF_F_VLAN_CHALLENGED_BIT, /* Device cannot handle VLAN packets */ NETIF_F_GSO_BIT, /* Enable software GSO. */ - NETIF_F_LLTX_BIT, /* LockLess TX - deprecated. Please */ - /* do not use LLTX in new drivers */ NETIF_F_NETNS_LOCAL_BIT, /* Does not change network namespaces */ NETIF_F_GRO_BIT, /* Generic receive offload */ NETIF_F_LRO_BIT, /* large receive offload */ @@ -119,7 +117,6 @@ enum { #define NETIF_F_HW_VLAN_CTAG_TX __NETIF_F(HW_VLAN_CTAG_TX) #define NETIF_F_IP_CSUM __NETIF_F(IP_CSUM) #define NETIF_F_IPV6_CSUM __NETIF_F(IPV6_CSUM) -#define NETIF_F_LLTX __NETIF_F(LLTX) #define NETIF_F_LOOPBACK __NETIF_F(LOOPBACK) #define NETIF_F_LRO __NETIF_F(LRO) #define NETIF_F_NETNS_LOCAL __NETIF_F(NETNS_LOCAL) @@ -192,7 +189,7 @@ static inline int find_next_netdev_feature(u64 feature, unsigned long start) /* Features valid for ethtool to change */ /* = all defined minus driver/device-class-related */ #define NETIF_F_NEVER_CHANGE (NETIF_F_VLAN_CHALLENGED | \ - NETIF_F_LLTX | NETIF_F_NETNS_LOCAL) + NETIF_F_NETNS_LOCAL) /* remember that ((t)1 << t_BITS) is undefined in C99 */ #define NETIF_F_ETHTOOL_BITS ((__NETIF_F_BIT(NETDEV_FEATURE_COUNT - 1) | \ diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 7d4d2758ea99ce..d10dd825b0ac35 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1749,6 +1749,8 @@ enum netdev_reg_state { * booleans combined, only to assert cacheline placement * @priv_flags: flags invisible to userspace defined as bits, see * enum netdev_priv_flags for the definitions + * @lltx: device supports lockless Tx. Mainly used by logical + * interfaces, such as tunnels * * @name: This is the first field of the "visible" part of this structure * (i.e. as seen by users in the "Space.c" file). It is the name @@ -2046,6 +2048,7 @@ struct net_device { __cacheline_group_begin(net_device_read_tx); struct_group(priv_flags_fast, unsigned long priv_flags:32; + unsigned long lltx:1; ); const struct net_device_ops *netdev_ops; const struct header_ops *header_ops; @@ -4459,7 +4462,7 @@ static inline void netif_tx_unlock_bh(struct net_device *dev) } #define HARD_TX_LOCK(dev, txq, cpu) { \ - if ((dev->features & NETIF_F_LLTX) == 0) { \ + if (!(dev)->lltx) { \ __netif_tx_lock(txq, cpu); \ } else { \ __netif_tx_acquire(txq); \ @@ -4467,12 +4470,12 @@ static inline void netif_tx_unlock_bh(struct net_device *dev) } #define HARD_TX_TRYLOCK(dev, txq) \ - (((dev->features & NETIF_F_LLTX) == 0) ? \ + (!(dev)->lltx ? \ __netif_tx_trylock(txq) : \ __netif_tx_acquire(txq)) #define HARD_TX_UNLOCK(dev, txq) { \ - if ((dev->features & NETIF_F_LLTX) == 0) { \ + if (!(dev)->lltx) { \ __netif_tx_unlock(txq); \ } else { \ __netif_tx_release(txq); \ diff --git a/lib/test_bpf.c b/lib/test_bpf.c index b7acc29bcc3be0..d50050c3709b5f 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -15077,8 +15077,7 @@ static struct skb_segment_test skb_segment_tests[] __initconst = { .build_skb = build_test_skb_linear_no_head_frag, .features = NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_GSO | - NETIF_F_LLTX | NETIF_F_GRO | - NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | + NETIF_F_GRO | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_STAG_TX } }; diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index 217be32426b58a..3ca485537d778a 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -569,7 +569,8 @@ static int vlan_dev_init(struct net_device *dev) if (real_dev->vlan_features & NETIF_F_HW_MACSEC) dev->hw_features |= NETIF_F_HW_MACSEC; - dev->features |= dev->hw_features | NETIF_F_LLTX; + dev->features |= dev->hw_features; + dev->lltx = true; netif_inherit_tso_max(dev, real_dev); if (dev->features & NETIF_F_VLAN_FEATURES) netdev_warn(real_dev, "VLAN features are set incorrectly. Q-in-Q configurations may not work correctly.\n"); @@ -655,7 +656,6 @@ static netdev_features_t vlan_dev_fix_features(struct net_device *dev, lower_features |= NETIF_F_HW_CSUM; features = netdev_intersect_features(features, lower_features); features |= old_features & (NETIF_F_SOFT_FEATURES | NETIF_F_GSO_SOFTWARE); - features |= NETIF_F_LLTX; return features; } diff --git a/net/batman-adv/soft-interface.c b/net/batman-adv/soft-interface.c index 30ecbc2ef1fd9c..e791a73ef901b1 100644 --- a/net/batman-adv/soft-interface.c +++ b/net/batman-adv/soft-interface.c @@ -1021,8 +1021,8 @@ static void batadv_softif_init_early(struct net_device *dev) dev->needs_free_netdev = true; dev->priv_destructor = batadv_softif_free; dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_NETNS_LOCAL; - dev->features |= NETIF_F_LLTX; dev->priv_flags |= IFF_NO_QUEUE; + dev->lltx = true; /* can't call min_mtu, because the needed variables * have not been initialized yet diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c index fb1115857e497b..a6d25113dfb185 100644 --- a/net/bridge/br_device.c +++ b/net/bridge/br_device.c @@ -487,8 +487,9 @@ void br_dev_setup(struct net_device *dev) dev->ethtool_ops = &br_ethtool_ops; SET_NETDEV_DEVTYPE(dev, &br_type); dev->priv_flags = IFF_EBRIDGE | IFF_NO_QUEUE; + dev->lltx = true; - dev->features = COMMON_FEATURES | NETIF_F_LLTX | NETIF_F_NETNS_LOCAL | + dev->features = COMMON_FEATURES | NETIF_F_NETNS_LOCAL | NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_STAG_TX; dev->hw_features = COMMON_FEATURES | NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_STAG_TX; diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index 0e2084ce7b7572..95fb10cc70bb92 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -1764,8 +1764,7 @@ static const struct kobj_type netdev_queue_ktype = { static bool netdev_uses_bql(const struct net_device *dev) { - if (dev->features & NETIF_F_LLTX || - dev->priv_flags & IFF_NO_QUEUE) + if (dev->lltx || (dev->priv_flags & IFF_NO_QUEUE)) return false; return IS_ENABLED(CONFIG_BQL); diff --git a/net/dsa/user.c b/net/dsa/user.c index f5adfa1d978a28..74eda9b30608e6 100644 --- a/net/dsa/user.c +++ b/net/dsa/user.c @@ -2642,11 +2642,12 @@ void dsa_user_setup_tagger(struct net_device *user) user->features = conduit->vlan_features | NETIF_F_HW_TC; user->hw_features |= NETIF_F_HW_TC; - user->features |= NETIF_F_LLTX; if (user->needed_tailroom) user->features &= ~(NETIF_F_SG | NETIF_F_FRAGLIST); if (ds->needs_standalone_vlan_filtering) user->features |= NETIF_F_HW_VLAN_CTAG_FILTER; + + user->lltx = true; } int dsa_user_suspend(struct net_device *user_dev) diff --git a/net/ethtool/common.c b/net/ethtool/common.c index 07032babd1b693..80f6ef1a0f4ec7 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -24,7 +24,6 @@ const char netdev_features_strings[NETDEV_FEATURE_COUNT][ETH_GSTRING_LEN] = { [NETIF_F_HW_VLAN_STAG_FILTER_BIT] = "rx-vlan-stag-filter", [NETIF_F_VLAN_CHALLENGED_BIT] = "vlan-challenged", [NETIF_F_GSO_BIT] = "tx-generic-segmentation", - [NETIF_F_LLTX_BIT] = "tx-lockless", [NETIF_F_NETNS_LOCAL_BIT] = "netns-local", [NETIF_F_GRO_BIT] = "rx-gro", [NETIF_F_GRO_HW_BIT] = "rx-gro-hw", diff --git a/net/hsr/hsr_device.c b/net/hsr/hsr_device.c index e4cc6b78dcfc40..d4c7830766621b 100644 --- a/net/hsr/hsr_device.c +++ b/net/hsr/hsr_device.c @@ -554,6 +554,8 @@ void hsr_dev_setup(struct net_device *dev) dev->netdev_ops = &hsr_device_ops; SET_NETDEV_DEVTYPE(dev, &hsr_type); dev->priv_flags |= IFF_NO_QUEUE | IFF_DISABLE_NETPOLL; + /* Prevent recursive tx locking */ + dev->lltx = true; dev->needs_free_netdev = true; @@ -563,8 +565,6 @@ void hsr_dev_setup(struct net_device *dev) dev->features = dev->hw_features; - /* Prevent recursive tx locking */ - dev->features |= NETIF_F_LLTX; /* VLAN on top of HSR needs testing and probably some work on * hsr_header_create() etc. */ diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c index ba205473522e4e..b54c41f3ae3cd1 100644 --- a/net/ipv4/ip_gre.c +++ b/net/ipv4/ip_gre.c @@ -996,7 +996,7 @@ static void __gre_tunnel_init(struct net_device *dev) tunnel->hlen = tunnel->tun_hlen + tunnel->encap_hlen; dev->needed_headroom = tunnel->hlen + sizeof(tunnel->parms.iph); - dev->features |= GRE_FEATURES | NETIF_F_LLTX; + dev->features |= GRE_FEATURES; dev->hw_features |= GRE_FEATURES; /* TCP offload with GRE SEQ is not supported, nor can we support 2 @@ -1010,6 +1010,8 @@ static void __gre_tunnel_init(struct net_device *dev) dev->features |= NETIF_F_GSO_SOFTWARE; dev->hw_features |= NETIF_F_GSO_SOFTWARE; + + dev->lltx = true; } static int ipgre_tunnel_init(struct net_device *dev) diff --git a/net/ipv4/ip_vti.c b/net/ipv4/ip_vti.c index 14536da9f5dc0f..f0b4419cef3493 100644 --- a/net/ipv4/ip_vti.c +++ b/net/ipv4/ip_vti.c @@ -443,7 +443,7 @@ static int vti_tunnel_init(struct net_device *dev) dev->flags = IFF_NOARP; dev->addr_len = 4; - dev->features |= NETIF_F_LLTX; + dev->lltx = true; netif_keep_dst(dev); return ip_tunnel_init(dev); diff --git a/net/ipv4/ipip.c b/net/ipv4/ipip.c index 923a2ef68c2f41..dc0db5895e0e06 100644 --- a/net/ipv4/ipip.c +++ b/net/ipv4/ipip.c @@ -378,7 +378,7 @@ static void ipip_tunnel_setup(struct net_device *dev) dev->type = ARPHRD_TUNNEL; dev->flags = IFF_NOARP; dev->addr_len = 4; - dev->features |= NETIF_F_LLTX; + dev->lltx = true; netif_keep_dst(dev); dev->features |= IPIP_FEATURES; diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index 3942bd2ade78b4..08beab638bda6a 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -1471,7 +1471,7 @@ static void ip6gre_tnl_init_features(struct net_device *dev) { struct ip6_tnl *nt = netdev_priv(dev); - dev->features |= GRE6_FEATURES | NETIF_F_LLTX; + dev->features |= GRE6_FEATURES; dev->hw_features |= GRE6_FEATURES; /* TCP offload with GRE SEQ is not supported, nor can we support 2 @@ -1485,6 +1485,8 @@ static void ip6gre_tnl_init_features(struct net_device *dev) dev->features |= NETIF_F_GSO_SOFTWARE; dev->hw_features |= NETIF_F_GSO_SOFTWARE; + + dev->lltx = true; } static int ip6gre_tunnel_init_common(struct net_device *dev) diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index 9dee0c12795540..472fbf524602e3 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -1846,7 +1846,7 @@ static void ip6_tnl_dev_setup(struct net_device *dev) dev->type = ARPHRD_TUNNEL6; dev->flags |= IFF_NOARP; dev->addr_len = sizeof(struct in6_addr); - dev->features |= NETIF_F_LLTX; + dev->lltx = true; dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; netif_keep_dst(dev); diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c index 83b195f09561d9..008bb84c3b59f6 100644 --- a/net/ipv6/sit.c +++ b/net/ipv6/sit.c @@ -1435,7 +1435,7 @@ static void ipip6_tunnel_setup(struct net_device *dev) dev->flags = IFF_NOARP; netif_keep_dst(dev); dev->addr_len = 4; - dev->features |= NETIF_F_LLTX; + dev->lltx = true; dev->features |= SIT_FEATURES; dev->hw_features |= SIT_FEATURES; dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; diff --git a/net/l2tp/l2tp_eth.c b/net/l2tp/l2tp_eth.c index 8ba00ad433c21b..3102976b34b14c 100644 --- a/net/l2tp/l2tp_eth.c +++ b/net/l2tp/l2tp_eth.c @@ -109,7 +109,7 @@ static void l2tp_eth_dev_setup(struct net_device *dev) SET_NETDEV_DEVTYPE(dev, &l2tpeth_type); ether_setup(dev); dev->priv_flags &= ~IFF_TX_SKB_SHARING; - dev->features |= NETIF_F_LLTX; + dev->lltx = true; dev->netdev_ops = &l2tp_eth_netdev_ops; dev->needs_free_netdev = true; } diff --git a/net/openvswitch/vport-internal_dev.c b/net/openvswitch/vport-internal_dev.c index 4b33133cbdfffa..3a369a31c5cc93 100644 --- a/net/openvswitch/vport-internal_dev.c +++ b/net/openvswitch/vport-internal_dev.c @@ -102,19 +102,20 @@ static void do_setup(struct net_device *netdev) netdev->priv_flags &= ~IFF_TX_SKB_SHARING; netdev->priv_flags |= IFF_LIVE_ADDR_CHANGE | IFF_OPENVSWITCH | IFF_NO_QUEUE; + netdev->lltx = true; netdev->needs_free_netdev = true; netdev->priv_destructor = NULL; netdev->ethtool_ops = &internal_dev_ethtool_ops; netdev->rtnl_link_ops = &internal_dev_link_ops; - netdev->features = NETIF_F_LLTX | NETIF_F_SG | NETIF_F_FRAGLIST | - NETIF_F_HIGHDMA | NETIF_F_HW_CSUM | - NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL; + netdev->features = NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HIGHDMA | + NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE | + NETIF_F_GSO_ENCAP_ALL; netdev->vlan_features = netdev->features; netdev->hw_enc_features = netdev->features; netdev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_STAG_TX; - netdev->hw_features = netdev->features & ~NETIF_F_LLTX; + netdev->hw_features = netdev->features; eth_hw_addr_random(netdev); } diff --git a/net/xfrm/xfrm_interface_core.c b/net/xfrm/xfrm_interface_core.c index e50e4bf993fa47..98f1e2b67c76b3 100644 --- a/net/xfrm/xfrm_interface_core.c +++ b/net/xfrm/xfrm_interface_core.c @@ -769,7 +769,7 @@ static int xfrmi_dev_init(struct net_device *dev) if (err) return err; - dev->features |= NETIF_F_LLTX; + dev->lltx = true; dev->features |= XFRMI_FEATURES; dev->hw_features |= XFRMI_FEATURES; From 1e66f4b213a2690705f5453d2478a7d386246e29 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 25 Jun 2024 13:26:23 +0200 Subject: [PATCH 04/49] netdev_features: convert NETIF_F_NETNS_LOCAL to dev->netns_local "Interface can't change network namespaces" is rather an attribute, not a feature, and it can't be changed via Ethtool. Make it a "cold" private flag instead of a netdev_feature and free one more bit. Signed-off-by: Alexander Lobakin --- Documentation/networking/net_cachelines/net_device.rst | 1 + Documentation/networking/netdev-features.rst | 7 ------- Documentation/networking/switchdev.rst | 4 ++-- drivers/net/amt.c | 2 +- drivers/net/bonding/bond_main.c | 6 +++--- drivers/net/ethernet/adi/adin1110.c | 2 +- drivers/net/ethernet/marvell/prestera/prestera_main.c | 3 ++- drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 4 ++-- drivers/net/ethernet/mellanox/mlx5/core/en_rep.c | 3 ++- drivers/net/ethernet/mellanox/mlxsw/spectrum.c | 5 +++-- drivers/net/ethernet/rocker/rocker_main.c | 3 ++- drivers/net/ethernet/ti/cpsw_new.c | 3 ++- drivers/net/loopback.c | 2 +- drivers/net/net_failover.c | 2 +- drivers/net/team/team_core.c | 6 +++--- drivers/net/vrf.c | 2 +- include/linux/netdev_features.h | 5 +---- include/linux/netdevice.h | 2 ++ net/batman-adv/soft-interface.c | 3 ++- net/bridge/br_device.c | 5 +++-- net/core/dev.c | 4 ++-- net/ethtool/common.c | 1 - net/hsr/hsr_device.c | 8 ++++---- net/ieee802154/6lowpan/core.c | 2 +- net/ieee802154/core.c | 10 +++++----- net/ipv4/ip_tunnel.c | 2 +- net/ipv4/ipmr.c | 2 +- net/ipv6/ip6_gre.c | 3 +-- net/ipv6/ip6_tunnel.c | 2 +- net/ipv6/ip6mr.c | 2 +- net/ipv6/sit.c | 2 +- net/openvswitch/vport-internal_dev.c | 2 +- net/wireless/core.c | 10 +++++----- tools/testing/selftests/net/forwarding/README | 2 +- 34 files changed, 60 insertions(+), 62 deletions(-) diff --git a/Documentation/networking/net_cachelines/net_device.rst b/Documentation/networking/net_cachelines/net_device.rst index 9b2b3aeaa4a999..e65ffdfc9e0aef 100644 --- a/Documentation/networking/net_cachelines/net_device.rst +++ b/Documentation/networking/net_cachelines/net_device.rst @@ -166,6 +166,7 @@ unsigned:1 wol_enabled unsigned:1 threaded - - napi_poll(napi_enable,dev_set_threaded) unsigned_long:1 see_all_hwtstamp_requests unsigned_long:1 change_proto_down +unsigned_long:1 netns_local struct_list_head net_notifier_list struct_macsec_ops* macsec_ops struct_udp_tunnel_nic_info* udp_tunnel_nic_info diff --git a/Documentation/networking/netdev-features.rst b/Documentation/networking/netdev-features.rst index f29d982ebf5d3b..5014f7cc1398ba 100644 --- a/Documentation/networking/netdev-features.rst +++ b/Documentation/networking/netdev-features.rst @@ -139,13 +139,6 @@ chained skbs (skb->next/prev list). Features contained in NETIF_F_SOFT_FEATURES are features of networking stack. Driver should not change behaviour based on them. - * netns-local device - -NETIF_F_NETNS_LOCAL is set for devices that are not allowed to move between -network namespaces (e.g. loopback). - -Don't use it in drivers. - * VLAN challenged NETIF_F_VLAN_CHALLENGED should be set for devices which can't cope with VLAN diff --git a/Documentation/networking/switchdev.rst b/Documentation/networking/switchdev.rst index 758f1dae3fce20..f355f0166f1b4f 100644 --- a/Documentation/networking/switchdev.rst +++ b/Documentation/networking/switchdev.rst @@ -137,10 +137,10 @@ would be sub-port 0 on port 1 on switch 1. Port Features ^^^^^^^^^^^^^ -NETIF_F_NETNS_LOCAL +dev->netns_local If the switchdev driver (and device) only supports offloading of the default -network namespace (netns), the driver should set this feature flag to prevent +network namespace (netns), the driver should set this private flag to prevent the port netdev from being moved out of the default netns. A netns-aware driver/device would not set this flag and be responsible for partitioning hardware to preserve netns containment. This means hardware cannot forward diff --git a/drivers/net/amt.c b/drivers/net/amt.c index 921bbfd72a3868..0433a0f36d1b4e 100644 --- a/drivers/net/amt.c +++ b/drivers/net/amt.c @@ -3099,8 +3099,8 @@ static void amt_link_setup(struct net_device *dev) dev->addr_len = 0; dev->priv_flags |= IFF_NO_QUEUE; dev->lltx = true; + dev->netns_local = true; dev->features |= NETIF_F_GSO_SOFTWARE; - dev->features |= NETIF_F_NETNS_LOCAL; dev->hw_features |= NETIF_F_SG | NETIF_F_HW_CSUM; dev->hw_features |= NETIF_F_FRAGLIST | NETIF_F_RXCSUM; dev->hw_features |= NETIF_F_GSO_SOFTWARE; diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 48fb5e0f8bfdc5..8caee8f4935fe0 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -5891,6 +5891,9 @@ void bond_setup(struct net_device *bond_dev) /* don't acquire bond device's netif_tx_lock when transmitting */ bond_dev->lltx = true; + /* Don't allow bond devices to change network namespaces. */ + bond_dev->netns_local = true; + /* By default, we declare the bond to be fully * VLAN hardware accelerated capable. Special * care is taken in the various xmit functions @@ -5898,9 +5901,6 @@ void bond_setup(struct net_device *bond_dev) * capable */ - /* Don't allow bond devices to change network namespaces. */ - bond_dev->features |= NETIF_F_NETNS_LOCAL; - bond_dev->hw_features = BOND_VLAN_FEATURES | NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER | diff --git a/drivers/net/ethernet/adi/adin1110.c b/drivers/net/ethernet/adi/adin1110.c index 0713f1e2c7f38b..3431a7e62b0d2b 100644 --- a/drivers/net/ethernet/adi/adin1110.c +++ b/drivers/net/ethernet/adi/adin1110.c @@ -1599,7 +1599,7 @@ static int adin1110_probe_netdevs(struct adin1110_priv *priv) netdev->netdev_ops = &adin1110_netdev_ops; netdev->ethtool_ops = &adin1110_ethtool_ops; netdev->priv_flags |= IFF_UNICAST_FLT; - netdev->features |= NETIF_F_NETNS_LOCAL; + netdev->netns_local = true; port_priv->phydev = get_phy_device(priv->mii_bus, i + 1, false); if (IS_ERR(port_priv->phydev)) { diff --git a/drivers/net/ethernet/marvell/prestera/prestera_main.c b/drivers/net/ethernet/marvell/prestera/prestera_main.c index 63ae01954dfc5e..22ca6ee9665eb4 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_main.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_main.c @@ -633,7 +633,8 @@ static int prestera_port_create(struct prestera_switch *sw, u32 id) if (err) goto err_dl_port_register; - dev->features |= NETIF_F_NETNS_LOCAL | NETIF_F_HW_TC; + dev->features |= NETIF_F_HW_TC; + dev->netns_local = true; dev->netdev_ops = &prestera_netdev_ops; dev->ethtool_ops = &prestera_ethtool_ops; SET_NETDEV_DEV(dev, sw->dev->dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 6f686fabed4462..4b78c0ca233c2d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -4403,9 +4403,9 @@ static netdev_features_t mlx5e_fix_features(struct net_device *netdev, if (mlx5e_is_uplink_rep(priv)) { features = mlx5e_fix_uplink_rep_features(netdev, features); - features |= NETIF_F_NETNS_LOCAL; + netdev->netns_local = true; } else { - features &= ~NETIF_F_NETNS_LOCAL; + netdev->netns_local = false; } mutex_unlock(&priv->state_lock); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 8790d57dc6dbf0..e2e3b8c33d186f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -898,7 +898,8 @@ static void mlx5e_build_rep_netdev(struct net_device *netdev, netdev->hw_features |= NETIF_F_RXCSUM; netdev->features |= netdev->hw_features; - netdev->features |= NETIF_F_NETNS_LOCAL; + + netdev->netns_local = true; } static int mlx5e_init_rep(struct mlx5_core_dev *mdev, diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index 44d6e125bd6f00..b9ffd7236aff81 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -1676,10 +1676,11 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u16 local_port, netif_carrier_off(dev); - dev->features |= NETIF_F_NETNS_LOCAL | NETIF_F_SG | - NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_TC; + dev->features |= NETIF_F_SG | NETIF_F_HW_VLAN_CTAG_FILTER | + NETIF_F_HW_TC; dev->hw_features |= NETIF_F_HW_TC | NETIF_F_LOOPBACK; dev->lltx = true; + dev->netns_local = true; dev->min_mtu = ETH_MIN_MTU; dev->max_mtu = MLXSW_PORT_MAX_MTU - MLXSW_PORT_ETH_FRAME_HDR; diff --git a/drivers/net/ethernet/rocker/rocker_main.c b/drivers/net/ethernet/rocker/rocker_main.c index e097ce3e69ea3b..84fa911c78db55 100644 --- a/drivers/net/ethernet/rocker/rocker_main.c +++ b/drivers/net/ethernet/rocker/rocker_main.c @@ -2575,7 +2575,8 @@ static int rocker_probe_port(struct rocker *rocker, unsigned int port_number) netif_napi_add(dev, &rocker_port->napi_rx, rocker_port_poll_rx); rocker_carrier_init(rocker_port); - dev->features |= NETIF_F_NETNS_LOCAL | NETIF_F_SG; + dev->features |= NETIF_F_SG; + dev->netns_local = true; /* MTU range: 68 - 9000 */ dev->min_mtu = ROCKER_PORT_MIN_MTU; diff --git a/drivers/net/ethernet/ti/cpsw_new.c b/drivers/net/ethernet/ti/cpsw_new.c index 2baa198ebfa090..557cc71b9dd22c 100644 --- a/drivers/net/ethernet/ti/cpsw_new.c +++ b/drivers/net/ethernet/ti/cpsw_new.c @@ -1407,7 +1407,8 @@ static int cpsw_create_ports(struct cpsw_common *cpsw) cpsw->slaves[i].ndev = ndev; ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | - NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_NETNS_LOCAL | NETIF_F_HW_TC; + NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_TC; + ndev->netns_local = true; ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | diff --git a/drivers/net/loopback.c b/drivers/net/loopback.c index bf857782be0ff3..1993b90b1a5f90 100644 --- a/drivers/net/loopback.c +++ b/drivers/net/loopback.c @@ -172,6 +172,7 @@ static void gen_lo_setup(struct net_device *dev, dev->flags = IFF_LOOPBACK; dev->priv_flags |= IFF_LIVE_ADDR_CHANGE | IFF_NO_QUEUE; dev->lltx = true; + dev->netns_local = true; netif_keep_dst(dev); dev->hw_features = NETIF_F_GSO_SOFTWARE; dev->features = NETIF_F_SG | NETIF_F_FRAGLIST @@ -180,7 +181,6 @@ static void gen_lo_setup(struct net_device *dev, | NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA - | NETIF_F_NETNS_LOCAL | NETIF_F_VLAN_CHALLENGED | NETIF_F_LOOPBACK; dev->ethtool_ops = eth_ops; diff --git a/drivers/net/net_failover.c b/drivers/net/net_failover.c index 06728385a35fb2..54c8b9d5b5fcf4 100644 --- a/drivers/net/net_failover.c +++ b/drivers/net/net_failover.c @@ -734,7 +734,7 @@ struct failover *net_failover_create(struct net_device *standby_dev) failover_dev->lltx = true; /* Don't allow failover devices to change network namespaces. */ - failover_dev->features |= NETIF_F_NETNS_LOCAL; + failover_dev->netns_local = true; failover_dev->hw_features = FAILOVER_VLAN_FEATURES | NETIF_F_HW_VLAN_CTAG_TX | diff --git a/drivers/net/team/team_core.c b/drivers/net/team/team_core.c index 1d1bad3cedc2d7..18191d5a8bd4d3 100644 --- a/drivers/net/team/team_core.c +++ b/drivers/net/team/team_core.c @@ -2191,10 +2191,10 @@ static void team_setup(struct net_device *dev) dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE; dev->lltx = true; - dev->features |= NETIF_F_GRO; - /* Don't allow team devices to change network namespaces. */ - dev->features |= NETIF_F_NETNS_LOCAL; + dev->netns_local = true; + + dev->features |= NETIF_F_GRO; dev->hw_features = TEAM_VLAN_FEATURES | NETIF_F_HW_VLAN_CTAG_RX | diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c index fce065d0b5a0a7..597a041476fafc 100644 --- a/drivers/net/vrf.c +++ b/drivers/net/vrf.c @@ -1637,7 +1637,7 @@ static void vrf_setup(struct net_device *dev) dev->lltx = true; /* don't allow vrf devices to change network namespaces. */ - dev->features |= NETIF_F_NETNS_LOCAL; + dev->netns_local = true; /* does not make sense for a VLAN to be added to a vrf device */ dev->features |= NETIF_F_VLAN_CHALLENGED; diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h index 54d1578f6642aa..3bacd4b1adc9b4 100644 --- a/include/linux/netdev_features.h +++ b/include/linux/netdev_features.h @@ -23,7 +23,6 @@ enum { NETIF_F_HW_VLAN_CTAG_FILTER_BIT,/* Receive filtering on VLAN CTAGs */ NETIF_F_VLAN_CHALLENGED_BIT, /* Device cannot handle VLAN packets */ NETIF_F_GSO_BIT, /* Enable software GSO. */ - NETIF_F_NETNS_LOCAL_BIT, /* Does not change network namespaces */ NETIF_F_GRO_BIT, /* Generic receive offload */ NETIF_F_LRO_BIT, /* large receive offload */ @@ -119,7 +118,6 @@ enum { #define NETIF_F_IPV6_CSUM __NETIF_F(IPV6_CSUM) #define NETIF_F_LOOPBACK __NETIF_F(LOOPBACK) #define NETIF_F_LRO __NETIF_F(LRO) -#define NETIF_F_NETNS_LOCAL __NETIF_F(NETNS_LOCAL) #define NETIF_F_NOCACHE_COPY __NETIF_F(NOCACHE_COPY) #define NETIF_F_NTUPLE __NETIF_F(NTUPLE) #define NETIF_F_RXCSUM __NETIF_F(RXCSUM) @@ -188,8 +186,7 @@ static inline int find_next_netdev_feature(u64 feature, unsigned long start) /* Features valid for ethtool to change */ /* = all defined minus driver/device-class-related */ -#define NETIF_F_NEVER_CHANGE (NETIF_F_VLAN_CHALLENGED | \ - NETIF_F_NETNS_LOCAL) +#define NETIF_F_NEVER_CHANGE NETIF_F_VLAN_CHALLENGED /* remember that ((t)1 << t_BITS) is undefined in C99 */ #define NETIF_F_ETHTOOL_BITS ((__NETIF_F_BIT(NETDEV_FEATURE_COUNT - 1) | \ diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index d10dd825b0ac35..76e41059328e10 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1995,6 +1995,7 @@ enum netdev_reg_state { * regardless of source, even if those aren't * HWTSTAMP_SOURCE_NETDEV * @change_proto_down: device supports setting carrier via IFLA_PROTO_DOWN + * @netns_local: interface can't change network namespaces * * @net_notifier_list: List of per-net netdev notifier block * that follow this device when it is moved @@ -2387,6 +2388,7 @@ struct net_device { /* priv_flags_slow, ungrouped to save space */ unsigned long see_all_hwtstamp_requests:1; unsigned long change_proto_down:1; + unsigned long netns_local:1; struct list_head net_notifier_list; diff --git a/net/batman-adv/soft-interface.c b/net/batman-adv/soft-interface.c index e791a73ef901b1..2758aba47a2f25 100644 --- a/net/batman-adv/soft-interface.c +++ b/net/batman-adv/soft-interface.c @@ -1020,9 +1020,10 @@ static void batadv_softif_init_early(struct net_device *dev) dev->netdev_ops = &batadv_netdev_ops; dev->needs_free_netdev = true; dev->priv_destructor = batadv_softif_free; - dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_NETNS_LOCAL; + dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; dev->priv_flags |= IFF_NO_QUEUE; dev->lltx = true; + dev->netns_local = true; /* can't call min_mtu, because the needed variables * have not been initialized yet diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c index a6d25113dfb185..26b79feb385d2d 100644 --- a/net/bridge/br_device.c +++ b/net/bridge/br_device.c @@ -488,9 +488,10 @@ void br_dev_setup(struct net_device *dev) SET_NETDEV_DEVTYPE(dev, &br_type); dev->priv_flags = IFF_EBRIDGE | IFF_NO_QUEUE; dev->lltx = true; + dev->netns_local = true; - dev->features = COMMON_FEATURES | NETIF_F_NETNS_LOCAL | - NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_STAG_TX; + dev->features = COMMON_FEATURES | NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_HW_VLAN_STAG_TX; dev->hw_features = COMMON_FEATURES | NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_STAG_TX; dev->vlan_features = COMMON_FEATURES; diff --git a/net/core/dev.c b/net/core/dev.c index c0831853a5f89e..ab2bad1f83a74e 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -11453,7 +11453,7 @@ int __dev_change_net_namespace(struct net_device *dev, struct net *net, /* Don't allow namespace local devices to be moved. */ err = -EINVAL; - if (dev->features & NETIF_F_NETNS_LOCAL) + if (dev->netns_local) goto out; /* Ensure the device has been registrered */ @@ -11835,7 +11835,7 @@ static void __net_exit default_device_exit_net(struct net *net) char fb_name[IFNAMSIZ]; /* Ignore unmoveable devices (i.e. loopback) */ - if (dev->features & NETIF_F_NETNS_LOCAL) + if (dev->netns_local) continue; /* Leave virtual devices for the generic cleanup */ diff --git a/net/ethtool/common.c b/net/ethtool/common.c index 80f6ef1a0f4ec7..32fc8440092622 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -24,7 +24,6 @@ const char netdev_features_strings[NETDEV_FEATURE_COUNT][ETH_GSTRING_LEN] = { [NETIF_F_HW_VLAN_STAG_FILTER_BIT] = "rx-vlan-stag-filter", [NETIF_F_VLAN_CHALLENGED_BIT] = "vlan-challenged", [NETIF_F_GSO_BIT] = "tx-generic-segmentation", - [NETIF_F_NETNS_LOCAL_BIT] = "netns-local", [NETIF_F_GRO_BIT] = "rx-gro", [NETIF_F_GRO_HW_BIT] = "rx-gro-hw", [NETIF_F_LRO_BIT] = "rx-lro", diff --git a/net/hsr/hsr_device.c b/net/hsr/hsr_device.c index d4c7830766621b..a06e790042e2e0 100644 --- a/net/hsr/hsr_device.c +++ b/net/hsr/hsr_device.c @@ -556,6 +556,10 @@ void hsr_dev_setup(struct net_device *dev) dev->priv_flags |= IFF_NO_QUEUE | IFF_DISABLE_NETPOLL; /* Prevent recursive tx locking */ dev->lltx = true; + /* Not sure about this. Taken from bridge code. netdevice.h says + * it means "Does not change network namespaces". + */ + dev->netns_local = true; dev->needs_free_netdev = true; @@ -569,10 +573,6 @@ void hsr_dev_setup(struct net_device *dev) * hsr_header_create() etc. */ dev->features |= NETIF_F_VLAN_CHALLENGED; - /* Not sure about this. Taken from bridge code. netdev_features.h says - * it means "Does not change network namespaces". - */ - dev->features |= NETIF_F_NETNS_LOCAL; } /* Return true if dev is a HSR master; return false otherwise. diff --git a/net/ieee802154/6lowpan/core.c b/net/ieee802154/6lowpan/core.c index 77b4e92027c5df..175efd860f7b4a 100644 --- a/net/ieee802154/6lowpan/core.c +++ b/net/ieee802154/6lowpan/core.c @@ -116,7 +116,7 @@ static void lowpan_setup(struct net_device *ldev) ldev->netdev_ops = &lowpan_netdev_ops; ldev->header_ops = &lowpan_header_ops; ldev->needs_free_netdev = true; - ldev->features |= NETIF_F_NETNS_LOCAL; + ldev->netns_local = true; } static int lowpan_validate(struct nlattr *tb[], struct nlattr *data[], diff --git a/net/ieee802154/core.c b/net/ieee802154/core.c index 60e8fff1347ee8..88adb04e407220 100644 --- a/net/ieee802154/core.c +++ b/net/ieee802154/core.c @@ -226,11 +226,11 @@ int cfg802154_switch_netns(struct cfg802154_registered_device *rdev, list_for_each_entry(wpan_dev, &rdev->wpan_dev_list, list) { if (!wpan_dev->netdev) continue; - wpan_dev->netdev->features &= ~NETIF_F_NETNS_LOCAL; + wpan_dev->netdev->netns_local = false; err = dev_change_net_namespace(wpan_dev->netdev, net, "wpan%d"); if (err) break; - wpan_dev->netdev->features |= NETIF_F_NETNS_LOCAL; + wpan_dev->netdev->netns_local = true; } if (err) { @@ -242,11 +242,11 @@ int cfg802154_switch_netns(struct cfg802154_registered_device *rdev, list) { if (!wpan_dev->netdev) continue; - wpan_dev->netdev->features &= ~NETIF_F_NETNS_LOCAL; + wpan_dev->netdev->netns_local = false; err = dev_change_net_namespace(wpan_dev->netdev, net, "wpan%d"); WARN_ON(err); - wpan_dev->netdev->features |= NETIF_F_NETNS_LOCAL; + wpan_dev->netdev->netns_local = true; } return err; @@ -291,7 +291,7 @@ static int cfg802154_netdev_notifier_call(struct notifier_block *nb, switch (state) { /* TODO NETDEV_DEVTYPE */ case NETDEV_REGISTER: - dev->features |= NETIF_F_NETNS_LOCAL; + dev->netns_local = true; wpan_dev->identifier = ++rdev->wpan_dev_id; list_add_rcu(&wpan_dev->list, &rdev->wpan_dev_list); rdev->devlist_generation++; diff --git a/net/ipv4/ip_tunnel.c b/net/ipv4/ip_tunnel.c index 5cffad42fe8ca6..023b0ee8bd8567 100644 --- a/net/ipv4/ip_tunnel.c +++ b/net/ipv4/ip_tunnel.c @@ -1161,7 +1161,7 @@ int ip_tunnel_init_net(struct net *net, unsigned int ip_tnl_net_id, * Allowing to move it to another netns is clearly unsafe. */ if (!IS_ERR(itn->fb_tunnel_dev)) { - itn->fb_tunnel_dev->features |= NETIF_F_NETNS_LOCAL; + itn->fb_tunnel_dev->netns_local = true; itn->fb_tunnel_dev->mtu = ip_tunnel_bind_dev(itn->fb_tunnel_dev); ip_tunnel_add(itn, netdev_priv(itn->fb_tunnel_dev)); itn->type = itn->fb_tunnel_dev->type; diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c index 6c750bd13dd8dd..9b22dad2c9a2a5 100644 --- a/net/ipv4/ipmr.c +++ b/net/ipv4/ipmr.c @@ -536,7 +536,7 @@ static void reg_vif_setup(struct net_device *dev) dev->flags = IFF_NOARP; dev->netdev_ops = ®_vif_netdev_ops; dev->needs_free_netdev = true; - dev->features |= NETIF_F_NETNS_LOCAL; + dev->netns_local = true; } static struct net_device *ipmr_reg_vif(struct net *net, struct mr_table *mrt) diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index 08beab638bda6a..235808cfec7050 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -1621,8 +1621,7 @@ static int __net_init ip6gre_init_net(struct net *net) /* FB netdevice is special: we have one, and only one per netns. * Allowing to move it to another netns is clearly unsafe. */ - ign->fb_tunnel_dev->features |= NETIF_F_NETNS_LOCAL; - + ign->fb_tunnel_dev->netns_local = true; ip6gre_fb_tunnel_init(ign->fb_tunnel_dev); ign->fb_tunnel_dev->rtnl_link_ops = &ip6gre_link_ops; diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index 472fbf524602e3..f3c966c5a23439 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -2256,7 +2256,7 @@ static int __net_init ip6_tnl_init_net(struct net *net) /* FB netdevice is special: we have one, and only one per netns. * Allowing to move it to another netns is clearly unsafe. */ - ip6n->fb_tnl_dev->features |= NETIF_F_NETNS_LOCAL; + ip6n->fb_tnl_dev->netns_local = true; err = ip6_fb_tnl_dev_init(ip6n->fb_tnl_dev); if (err < 0) diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index dd342e6ecf3f45..bfaf3100801f42 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -640,7 +640,7 @@ static void reg_vif_setup(struct net_device *dev) dev->flags = IFF_NOARP; dev->netdev_ops = ®_vif_netdev_ops; dev->needs_free_netdev = true; - dev->features |= NETIF_F_NETNS_LOCAL; + dev->netns_local = true; } static struct net_device *ip6mr_reg_vif(struct net *net, struct mr_table *mrt) diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c index 008bb84c3b59f6..a0428612f7d42a 100644 --- a/net/ipv6/sit.c +++ b/net/ipv6/sit.c @@ -1855,7 +1855,7 @@ static int __net_init sit_init_net(struct net *net) /* FB netdevice is special: we have one, and only one per netns. * Allowing to move it to another netns is clearly unsafe. */ - sitn->fb_tunnel_dev->features |= NETIF_F_NETNS_LOCAL; + sitn->fb_tunnel_dev->netns_local = true; err = register_netdev(sitn->fb_tunnel_dev); if (err) diff --git a/net/openvswitch/vport-internal_dev.c b/net/openvswitch/vport-internal_dev.c index 3a369a31c5cc93..5858d65ea1a9b3 100644 --- a/net/openvswitch/vport-internal_dev.c +++ b/net/openvswitch/vport-internal_dev.c @@ -149,7 +149,7 @@ static struct vport *internal_dev_create(const struct vport_parms *parms) /* Restrict bridge port to current netns. */ if (vport->port_no == OVSP_LOCAL) - vport->dev->features |= NETIF_F_NETNS_LOCAL; + vport->dev->netns_local = true; rtnl_lock(); err = register_netdevice(vport->dev); diff --git a/net/wireless/core.c b/net/wireless/core.c index 4d5d351bd0b51e..661adfc7764444 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -165,11 +165,11 @@ int cfg80211_switch_netns(struct cfg80211_registered_device *rdev, list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) { if (!wdev->netdev) continue; - wdev->netdev->features &= ~NETIF_F_NETNS_LOCAL; + wdev->netdev->netns_local = false; err = dev_change_net_namespace(wdev->netdev, net, "wlan%d"); if (err) break; - wdev->netdev->features |= NETIF_F_NETNS_LOCAL; + wdev->netdev->netns_local = true; } if (err) { @@ -181,11 +181,11 @@ int cfg80211_switch_netns(struct cfg80211_registered_device *rdev, list) { if (!wdev->netdev) continue; - wdev->netdev->features &= ~NETIF_F_NETNS_LOCAL; + wdev->netdev->netns_local = false; err = dev_change_net_namespace(wdev->netdev, net, "wlan%d"); WARN_ON(err); - wdev->netdev->features |= NETIF_F_NETNS_LOCAL; + wdev->netdev->netns_local = true; } return err; @@ -1473,7 +1473,7 @@ static int cfg80211_netdev_notifier_call(struct notifier_block *nb, SET_NETDEV_DEVTYPE(dev, &wiphy_type); wdev->netdev = dev; /* can only change netns with wiphy */ - dev->features |= NETIF_F_NETNS_LOCAL; + dev->netns_local = true; cfg80211_init_wdev(wdev); break; diff --git a/tools/testing/selftests/net/forwarding/README b/tools/testing/selftests/net/forwarding/README index 7fdb6a9ca543f1..a652429bfd53cd 100644 --- a/tools/testing/selftests/net/forwarding/README +++ b/tools/testing/selftests/net/forwarding/README @@ -6,7 +6,7 @@ to easily create and test complex environments. Unfortunately, these namespaces can not be used with actual switching ASICs, as their ports can not be migrated to other network namespaces -(NETIF_F_NETNS_LOCAL) and most of them probably do not support the +(dev->netns_local) and most of them probably do not support the L1-separation provided by namespaces. However, a similar kind of flexibility can be achieved by using VRFs and From 66bab8188fae3cf0b9ad75f23989cf95497bff7e Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 25 Jun 2024 13:28:22 +0200 Subject: [PATCH 05/49] netdev_features: convert NETIF_F_FCOE_MTU to dev->fcoe_mtu Ability to handle maximum FCoE frames of 2158 bytes can never be changed and thus more of an attribute, not a toggleable feature. Move it from netdev_features_t to "cold" priv flags (bitfield bool) and free yet another feature bit. Signed-off-by: Alexander Lobakin --- .../networking/net_cachelines/net_device.rst | 1 + drivers/net/ethernet/chelsio/cxgb4/cxgb4_fcoe.c | 6 ++---- drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c | 2 +- drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c | 4 ++-- drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c | 2 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 11 ++++------- drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c | 4 ++-- drivers/scsi/fcoe/fcoe.c | 4 ++-- include/linux/netdev_features.h | 5 +---- include/linux/netdevice.h | 2 ++ net/8021q/vlan_dev.c | 1 + net/ethtool/common.c | 1 - 12 files changed, 19 insertions(+), 24 deletions(-) diff --git a/Documentation/networking/net_cachelines/net_device.rst b/Documentation/networking/net_cachelines/net_device.rst index e65ffdfc9e0aef..c3bbf101a887cb 100644 --- a/Documentation/networking/net_cachelines/net_device.rst +++ b/Documentation/networking/net_cachelines/net_device.rst @@ -167,6 +167,7 @@ unsigned:1 threaded - unsigned_long:1 see_all_hwtstamp_requests unsigned_long:1 change_proto_down unsigned_long:1 netns_local +unsigned_long:1 fcoe_mtu struct_list_head net_notifier_list struct_macsec_ops* macsec_ops struct_udp_tunnel_nic_info* udp_tunnel_nic_info diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_fcoe.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_fcoe.c index 33b2c0c455093e..f6f745f5c02218 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_fcoe.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_fcoe.c @@ -81,8 +81,7 @@ int cxgb_fcoe_enable(struct net_device *netdev) netdev->features |= NETIF_F_FCOE_CRC; netdev->vlan_features |= NETIF_F_FCOE_CRC; - netdev->features |= NETIF_F_FCOE_MTU; - netdev->vlan_features |= NETIF_F_FCOE_MTU; + netdev->fcoe_mtu = true; netdev_features_change(netdev); @@ -112,8 +111,7 @@ int cxgb_fcoe_disable(struct net_device *netdev) netdev->features &= ~NETIF_F_FCOE_CRC; netdev->vlan_features &= ~NETIF_F_FCOE_CRC; - netdev->features &= ~NETIF_F_FCOE_MTU; - netdev->vlan_features &= ~NETIF_F_FCOE_MTU; + netdev->fcoe_mtu = false; netdev_features_change(netdev); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c index e85f7d2e88106e..f2709b10c2e5ab 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c @@ -317,7 +317,7 @@ static u8 ixgbe_dcbnl_set_all(struct net_device *netdev) int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN; #ifdef IXGBE_FCOE - if (adapter->netdev->features & NETIF_F_FCOE_MTU) + if (adapter->netdev->fcoe_mtu) max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE); #endif diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c index 18d63c8c2ff4df..955dced844a989 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c @@ -858,7 +858,7 @@ int ixgbe_fcoe_enable(struct net_device *netdev) /* enable FCoE and notify stack */ adapter->flags |= IXGBE_FLAG_FCOE_ENABLED; - netdev->features |= NETIF_F_FCOE_MTU; + netdev->fcoe_mtu = true; netdev_features_change(netdev); /* release existing queues and reallocate them */ @@ -898,7 +898,7 @@ int ixgbe_fcoe_disable(struct net_device *netdev) /* disable FCoE and notify stack */ adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED; - netdev->features &= ~NETIF_F_FCOE_MTU; + netdev->fcoe_mtu = false; netdev_features_change(netdev); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c index 0ee943db3dc92b..16fa621ce0ffb5 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c @@ -981,7 +981,7 @@ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter, set_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state); #ifdef IXGBE_FCOE - if (adapter->netdev->features & NETIF_F_FCOE_MTU) { + if (adapter->netdev->fcoe_mtu) { struct ixgbe_ring_feature *f; f = &adapter->ring_feature[RING_F_FCOE]; if ((rxr_idx >= f->offset) && diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 8057cef61f397e..8b8404d8c94606 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -5079,7 +5079,7 @@ static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter) netif_set_tso_max_size(adapter->netdev, 32768); #ifdef IXGBE_FCOE - if (adapter->netdev->features & NETIF_F_FCOE_MTU) + if (adapter->netdev->fcoe_mtu) max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE); #endif @@ -5136,8 +5136,7 @@ static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb) #ifdef IXGBE_FCOE /* FCoE traffic class uses FCOE jumbo frames */ - if ((dev->features & NETIF_F_FCOE_MTU) && - (tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) && + if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE && (pb == ixgbe_fcoe_get_tc(adapter))) tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; #endif @@ -5197,8 +5196,7 @@ static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter, int pb) #ifdef IXGBE_FCOE /* FCoE traffic class uses FCOE jumbo frames */ - if ((dev->features & NETIF_F_FCOE_MTU) && - (tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) && + if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE && (pb == netdev_get_prio_tc_map(dev, adapter->fcoe.up))) tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; #endif @@ -11096,8 +11094,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_FCOE_CRC; netdev->vlan_features |= NETIF_F_FSO | - NETIF_F_FCOE_CRC | - NETIF_F_FCOE_MTU; + NETIF_F_FCOE_CRC; } #endif /* IXGBE_FCOE */ if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c index fcfd0a075eee56..e71715f5da2287 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c @@ -495,7 +495,7 @@ static int ixgbe_set_vf_lpe(struct ixgbe_adapter *adapter, u32 max_frame, u32 vf int err = 0; #ifdef CONFIG_FCOE - if (dev->features & NETIF_F_FCOE_MTU) + if (dev->fcoe_mtu) pf_max_frame = max_t(int, pf_max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE); @@ -857,7 +857,7 @@ static void ixgbe_set_vf_rx_tx(struct ixgbe_adapter *adapter, int vf) int pf_max_frame = dev->mtu + ETH_HLEN; #if IS_ENABLED(CONFIG_FCOE) - if (dev->features & NETIF_F_FCOE_MTU) + if (dev->fcoe_mtu) pf_max_frame = max_t(int, pf_max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE); #endif /* CONFIG_FCOE */ diff --git a/drivers/scsi/fcoe/fcoe.c b/drivers/scsi/fcoe/fcoe.c index f1429f27017043..39aec710660cfa 100644 --- a/drivers/scsi/fcoe/fcoe.c +++ b/drivers/scsi/fcoe/fcoe.c @@ -722,7 +722,7 @@ static int fcoe_netdev_config(struct fc_lport *lport, struct net_device *netdev) * will return 0, so do this first. */ mfs = netdev->mtu; - if (netdev->features & NETIF_F_FCOE_MTU) { + if (netdev->fcoe_mtu) { mfs = FCOE_MTU; FCOE_NETDEV_DBG(netdev, "Supports FCOE_MTU of %d bytes\n", mfs); } @@ -1863,7 +1863,7 @@ static int fcoe_device_notification(struct notifier_block *notifier, case NETDEV_CHANGE: break; case NETDEV_CHANGEMTU: - if (netdev->features & NETIF_F_FCOE_MTU) + if (netdev->fcoe_mtu) break; mfs = netdev->mtu - (sizeof(struct fcoe_hdr) + sizeof(struct fcoe_crc_eof)); diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h index 3bacd4b1adc9b4..1e9c4da181af5f 100644 --- a/include/linux/netdev_features.h +++ b/include/linux/netdev_features.h @@ -55,7 +55,6 @@ enum { NETIF_F_FCOE_CRC_BIT, /* FCoE CRC32 */ NETIF_F_SCTP_CRC_BIT, /* SCTP checksum offload */ - NETIF_F_FCOE_MTU_BIT, /* Supports max FCoE MTU, 2158 bytes*/ NETIF_F_NTUPLE_BIT, /* N-tuple filters supported */ NETIF_F_RXHASH_BIT, /* Receive hashing offload */ NETIF_F_RXCSUM_BIT, /* Receive checksumming offload */ @@ -102,7 +101,6 @@ enum { #define __NETIF_F(name) __NETIF_F_BIT(NETIF_F_##name##_BIT) #define NETIF_F_FCOE_CRC __NETIF_F(FCOE_CRC) -#define NETIF_F_FCOE_MTU __NETIF_F(FCOE_MTU) #define NETIF_F_FRAGLIST __NETIF_F(FRAGLIST) #define NETIF_F_FSO __NETIF_F(FSO) #define NETIF_F_GRO __NETIF_F(GRO) @@ -207,8 +205,7 @@ static inline int find_next_netdev_feature(u64 feature, unsigned long start) #define NETIF_F_ALL_TSO (NETIF_F_TSO | NETIF_F_TSO6 | \ NETIF_F_TSO_ECN | NETIF_F_TSO_MANGLEID) -#define NETIF_F_ALL_FCOE (NETIF_F_FCOE_CRC | NETIF_F_FCOE_MTU | \ - NETIF_F_FSO) +#define NETIF_F_ALL_FCOE (NETIF_F_FCOE_CRC | NETIF_F_FSO) /* List of features with software fallbacks. */ #define NETIF_F_GSO_SOFTWARE (NETIF_F_ALL_TSO | NETIF_F_GSO_SCTP | \ diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 76e41059328e10..5d559c4cb0374c 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1996,6 +1996,7 @@ enum netdev_reg_state { * HWTSTAMP_SOURCE_NETDEV * @change_proto_down: device supports setting carrier via IFLA_PROTO_DOWN * @netns_local: interface can't change network namespaces + * @fcoe_mtu: device supports maximum FCoE MTU, 2158 bytes * * @net_notifier_list: List of per-net netdev notifier block * that follow this device when it is moved @@ -2389,6 +2390,7 @@ struct net_device { unsigned long see_all_hwtstamp_requests:1; unsigned long change_proto_down:1; unsigned long netns_local:1; + unsigned long fcoe_mtu:1; struct list_head net_notifier_list; diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index 3ca485537d778a..09b46b057ab297 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -571,6 +571,7 @@ static int vlan_dev_init(struct net_device *dev) dev->features |= dev->hw_features; dev->lltx = true; + dev->fcoe_mtu = true; netif_inherit_tso_max(dev, real_dev); if (dev->features & NETIF_F_VLAN_FEATURES) netdev_warn(real_dev, "VLAN features are set incorrectly. Q-in-Q configurations may not work correctly.\n"); diff --git a/net/ethtool/common.c b/net/ethtool/common.c index 32fc8440092622..9e92c3ce044aae 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -49,7 +49,6 @@ const char netdev_features_strings[NETDEV_FEATURE_COUNT][ETH_GSTRING_LEN] = { [NETIF_F_FCOE_CRC_BIT] = "tx-checksum-fcoe-crc", [NETIF_F_SCTP_CRC_BIT] = "tx-checksum-sctp", - [NETIF_F_FCOE_MTU_BIT] = "fcoe-mtu", [NETIF_F_NTUPLE_BIT] = "rx-ntuple-filter", [NETIF_F_RXHASH_BIT] = "rx-hashing", [NETIF_F_RXCSUM_BIT] = "rx-checksum", From 44c8d76aae84e0476e08ce67421a30a4f3f3fe78 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Fri, 5 Jul 2024 16:38:02 +0200 Subject: [PATCH 06/49] net: netdev_features: remove NETIF_F_ALL_FCOE NETIF_F_ALL_FCOE is used only in vlan_dev.c, 2 times. Now that it's only 2 bits, open-code it and remove the definition from netdev_features.h. Suggested-by: Jakub Kicinski Signed-off-by: Alexander Lobakin --- include/linux/netdev_features.h | 2 -- net/8021q/vlan_dev.c | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h index 1e9c4da181af5f..e41cd8af2a725b 100644 --- a/include/linux/netdev_features.h +++ b/include/linux/netdev_features.h @@ -205,8 +205,6 @@ static inline int find_next_netdev_feature(u64 feature, unsigned long start) #define NETIF_F_ALL_TSO (NETIF_F_TSO | NETIF_F_TSO6 | \ NETIF_F_TSO_ECN | NETIF_F_TSO_MANGLEID) -#define NETIF_F_ALL_FCOE (NETIF_F_FCOE_CRC | NETIF_F_FSO) - /* List of features with software fallbacks. */ #define NETIF_F_GSO_SOFTWARE (NETIF_F_ALL_TSO | NETIF_F_GSO_SCTP | \ NETIF_F_GSO_UDP_L4 | NETIF_F_GSO_FRAGLIST) diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index 09b46b057ab297..458040e8a0e0be 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -564,7 +564,7 @@ static int vlan_dev_init(struct net_device *dev) NETIF_F_FRAGLIST | NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL | NETIF_F_HIGHDMA | NETIF_F_SCTP_CRC | - NETIF_F_ALL_FCOE; + NETIF_F_FCOE_CRC | NETIF_F_FSO; if (real_dev->vlan_features & NETIF_F_HW_MACSEC) dev->hw_features |= NETIF_F_HW_MACSEC; @@ -576,7 +576,8 @@ static int vlan_dev_init(struct net_device *dev) if (dev->features & NETIF_F_VLAN_FEATURES) netdev_warn(real_dev, "VLAN features are set incorrectly. Q-in-Q configurations may not work correctly.\n"); - dev->vlan_features = real_dev->vlan_features & ~NETIF_F_ALL_FCOE; + dev->vlan_features = real_dev->vlan_features & + ~(NETIF_F_FCOE_CRC | NETIF_F_FSO); dev->hw_enc_features = vlan_tnl_features(real_dev); dev->mpls_features = real_dev->mpls_features; From 1b85df9171c46332be2338158b1d779e4b90aa62 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Fri, 12 Jul 2024 14:55:59 +0200 Subject: [PATCH 07/49] idpf: fix memory leaks and crashes while performing a soft reset The second tagged commit introduced a UAF, as it removed restoring q_vector->vport pointers after reinitializating the structures. This is due to that all queue allocation functions are performed here with the new temporary vport structure and those functions rewrite the backpointers to the vport. Then, this new struct is freed and the pointers start leading to nowhere. But generally speaking, the current logic is very fragile. It claims to be more reliable when the system is low on memory, but in fact, it consumes two times more memory as at the moment of running this function, there are two vports allocated with their queues and vectors. Moreover, it claims to prevent the driver from running into "bad state", but in fact, any error during the rebuild leaves the old vport in the partially allocated state. Finally, if the interface is down when the function is called, it always allocates a new queue set, but when the user decides to enable the interface later on, vport_open() allocates them once again, IOW there's a clear memory leak here. There's now oneliner way to fix this all. Instead, rewrite the function from scratch without playing with two vports and memcpy()s. Just perform everything on the current structure and do a minimum set of stuff needed to rebuild the vport. Don't allocate the queues at all, as vport_open(), no matter if it will be called here or during the next ifup, will do that for us. Fixes: 02cbfba1add5 ("idpf: add ethtool callbacks") Fixes: e4891e4687c8 ("idpf: split &idpf_queue into 4 strictly-typed queue structures") Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/idpf/idpf.h | 17 +- .../net/ethernet/intel/idpf/idpf_ethtool.c | 4 +- drivers/net/ethernet/intel/idpf/idpf_lib.c | 159 +++++++----------- 3 files changed, 62 insertions(+), 118 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf.h b/drivers/net/ethernet/intel/idpf/idpf.h index 2c31ad87587a4a..2a5a86e22fab54 100644 --- a/drivers/net/ethernet/intel/idpf/idpf.h +++ b/drivers/net/ethernet/intel/idpf/idpf.h @@ -205,20 +205,6 @@ struct idpf_dev_ops { struct idpf_reg_ops reg_ops; }; -/** - * enum idpf_vport_reset_cause - Vport soft reset causes - * @IDPF_SR_Q_CHANGE: Soft reset queue change - * @IDPF_SR_Q_DESC_CHANGE: Soft reset descriptor change - * @IDPF_SR_MTU_CHANGE: Soft reset MTU change - * @IDPF_SR_RSC_CHANGE: Soft reset RSC change - */ -enum idpf_vport_reset_cause { - IDPF_SR_Q_CHANGE, - IDPF_SR_Q_DESC_CHANGE, - IDPF_SR_MTU_CHANGE, - IDPF_SR_RSC_CHANGE, -}; - /** * enum idpf_vport_flags - Vport flags * @IDPF_VPORT_DEL_QUEUES: To send delete queues message @@ -821,8 +807,7 @@ void idpf_vf_dev_ops_init(struct idpf_adapter *adapter); int idpf_intr_req(struct idpf_adapter *adapter); void idpf_intr_rel(struct idpf_adapter *adapter); u16 idpf_get_max_tx_hdr_size(struct idpf_adapter *adapter); -int idpf_initiate_soft_reset(struct idpf_vport *vport, - enum idpf_vport_reset_cause reset_cause); +int idpf_initiate_soft_reset(struct idpf_vport *vport); void idpf_deinit_task(struct idpf_adapter *adapter); int idpf_req_rel_vector_indexes(struct idpf_adapter *adapter, u16 *q_vector_idxs, diff --git a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c index 3806ddd3ce4ab9..f6a7263ee9c9b8 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c +++ b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c @@ -271,7 +271,7 @@ static int idpf_set_channels(struct net_device *netdev, vport_config->user_config.num_req_tx_qs = num_req_tx_q; vport_config->user_config.num_req_rx_qs = num_req_rx_q; - err = idpf_initiate_soft_reset(vport, IDPF_SR_Q_CHANGE); + err = idpf_initiate_soft_reset(vport); if (err) { /* roll back queue change */ vport_config->user_config.num_req_tx_qs = num_txq; @@ -391,7 +391,7 @@ static int idpf_set_ringparam(struct net_device *netdev, IDPF_RX_BUFQ_DESC_COUNT(new_rx_count, vport->num_bufqs_per_qgrp); - err = idpf_initiate_soft_reset(vport, IDPF_SR_Q_DESC_CHANGE); + err = idpf_initiate_soft_reset(vport); unlock_mutex: idpf_vport_ctrl_unlock(netdev); diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index 5dbf2b4ba1b001..b053ce23552cc4 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -1335,9 +1335,8 @@ static void idpf_rx_init_buf_tail(struct idpf_vport *vport) /** * idpf_vport_open - Bring up a vport * @vport: vport to bring up - * @alloc_res: allocate queue resources */ -static int idpf_vport_open(struct idpf_vport *vport, bool alloc_res) +static int idpf_vport_open(struct idpf_vport *vport) { struct idpf_netdev_priv *np = netdev_priv(vport->netdev); struct idpf_adapter *adapter = vport->adapter; @@ -1350,11 +1349,13 @@ static int idpf_vport_open(struct idpf_vport *vport, bool alloc_res) /* we do not allow interface up just yet */ netif_carrier_off(vport->netdev); - if (alloc_res) { - err = idpf_vport_queues_alloc(vport); - if (err) - return err; - } + err = idpf_set_real_num_queues(vport); + if (err) + return err; + + err = idpf_vport_queues_alloc(vport); + if (err) + return err; err = idpf_vport_intr_alloc(vport); if (err) { @@ -1539,7 +1540,7 @@ void idpf_init_task(struct work_struct *work) np = netdev_priv(vport->netdev); np->state = __IDPF_VPORT_DOWN; if (test_and_clear_bit(IDPF_VPORT_UP_REQUESTED, vport_config->flags)) - idpf_vport_open(vport, true); + idpf_vport_open(vport); /* Spawn and return 'idpf_init_task' work queue until all the * default vports are created @@ -1838,115 +1839,73 @@ void idpf_vc_event_task(struct work_struct *work) } } +static int idpf_reconfig_queues(struct idpf_vport *vport) +{ + int ret; + + ret = idpf_vport_adjust_qs(vport); + if (ret) { + netdev_err(vport->netdev, + "Could not adjust the queue number for vport %u: %pe\n", + vport->idx, ERR_PTR(ret)); + return ret; + } + + ret = idpf_send_add_queues_msg(vport, vport->num_txq, + vport->num_complq, + vport->num_rxq, vport->num_bufq); + if (ret) { + netdev_err(vport->netdev, + "Could not add queues for vport %u: %pe\n", + vport->idx, ERR_PTR(ret)); + return ret; + } + + idpf_vport_calc_num_q_desc(vport); + idpf_vport_calc_num_q_groups(vport); + idpf_vport_alloc_vec_indexes(vport); + + return 0; +} + /** * idpf_initiate_soft_reset - Initiate a software reset * @vport: virtual port data struct - * @reset_cause: reason for the soft reset * * Soft reset only reallocs vport queue resources. Returns 0 on success, * negative on failure. */ -int idpf_initiate_soft_reset(struct idpf_vport *vport, - enum idpf_vport_reset_cause reset_cause) +int idpf_initiate_soft_reset(struct idpf_vport *vport) { - struct idpf_netdev_priv *np = netdev_priv(vport->netdev); - enum idpf_vport_state current_state = np->state; - struct idpf_adapter *adapter = vport->adapter; - struct idpf_vport *new_vport; + const struct idpf_netdev_priv *np = netdev_priv(vport->netdev); + bool vport_is_up = np->state == __IDPF_VPORT_UP; int err; - /* If the system is low on memory, we can end up in bad state if we - * free all the memory for queue resources and try to allocate them - * again. Instead, we can pre-allocate the new resources before doing - * anything and bailing if the alloc fails. - * - * Make a clone of the existing vport to mimic its current - * configuration, then modify the new structure with any requested - * changes. Once the allocation of the new resources is done, stop the - * existing vport and copy the configuration to the main vport. If an - * error occurred, the existing vport will be untouched. - * - */ - new_vport = kzalloc(sizeof(*vport), GFP_KERNEL); - if (!new_vport) - return -ENOMEM; - - /* This purposely avoids copying the end of the struct because it - * contains wait_queues and mutexes and other stuff we don't want to - * mess with. Nothing below should use those variables from new_vport - * and should instead always refer to them in vport if they need to. - */ - memcpy(new_vport, vport, offsetof(struct idpf_vport, link_speed_mbps)); - - /* Adjust resource parameters prior to reallocating resources */ - switch (reset_cause) { - case IDPF_SR_Q_CHANGE: - err = idpf_vport_adjust_qs(new_vport); - if (err) - goto free_vport; - break; - case IDPF_SR_Q_DESC_CHANGE: - /* Update queue parameters before allocating resources */ - idpf_vport_calc_num_q_desc(new_vport); - break; - case IDPF_SR_MTU_CHANGE: - case IDPF_SR_RSC_CHANGE: - break; - default: - dev_err(&adapter->pdev->dev, "Unhandled soft reset cause\n"); - err = -EINVAL; - goto free_vport; - } - - err = idpf_vport_queues_alloc(new_vport); - if (err) - goto free_vport; - if (current_state <= __IDPF_VPORT_DOWN) { - idpf_send_delete_queues_msg(vport); - } else { + if (vport_is_up) { set_bit(IDPF_VPORT_DEL_QUEUES, vport->flags); idpf_vport_stop(vport); + } else { + idpf_send_delete_queues_msg(vport); } idpf_deinit_rss(vport); - /* We're passing in vport here because we need its wait_queue - * to send a message and it should be getting all the vport - * config data out of the adapter but we need to be careful not - * to add code to add_queues to change the vport config within - * vport itself as it will be wiped with a memcpy later. - */ - err = idpf_send_add_queues_msg(vport, new_vport->num_txq, - new_vport->num_complq, - new_vport->num_rxq, - new_vport->num_bufq); - if (err) - goto err_reset; - /* Same comment as above regarding avoiding copying the wait_queues and - * mutexes applies here. We do not want to mess with those if possible. - */ - memcpy(vport, new_vport, offsetof(struct idpf_vport, link_speed_mbps)); - - if (reset_cause == IDPF_SR_Q_CHANGE) - idpf_vport_alloc_vec_indexes(vport); - - err = idpf_set_real_num_queues(vport); + err = idpf_reconfig_queues(vport); if (err) - goto err_reset; - - if (current_state == __IDPF_VPORT_UP) - err = idpf_vport_open(vport, false); + return err; - kfree(new_vport); + if (!vport_is_up) + return idpf_set_real_num_queues(vport); - return err; - -err_reset: - idpf_vport_queues_rel(new_vport); -free_vport: - kfree(new_vport); + err = idpf_vport_open(vport); + if (err) { + netdev_err(vport->netdev, + "Could not reopen the vport %u: %pe", vport->idx, + ERR_PTR(err)); + return err; + } - return err; + return 0; } /** @@ -2135,7 +2094,7 @@ static int idpf_set_features(struct net_device *netdev, if (changed & NETIF_F_GRO_HW) { netdev->features ^= NETIF_F_GRO_HW; - err = idpf_initiate_soft_reset(vport, IDPF_SR_RSC_CHANGE); + err = idpf_initiate_soft_reset(vport); if (err) goto unlock_mutex; } @@ -2171,7 +2130,7 @@ static int idpf_open(struct net_device *netdev) idpf_vport_ctrl_lock(netdev); vport = idpf_netdev_to_vport(netdev); - err = idpf_vport_open(vport, true); + err = idpf_vport_open(vport); idpf_vport_ctrl_unlock(netdev); @@ -2195,7 +2154,7 @@ static int idpf_change_mtu(struct net_device *netdev, int new_mtu) WRITE_ONCE(netdev->mtu, new_mtu); - err = idpf_initiate_soft_reset(vport, IDPF_SR_MTU_CHANGE); + err = idpf_initiate_soft_reset(vport); idpf_vport_ctrl_unlock(netdev); From e2cb9d70dcb7689b84e35d54b72111b6e372348e Mon Sep 17 00:00:00 2001 From: Michal Kubiak Date: Fri, 12 Jul 2024 15:52:52 +0200 Subject: [PATCH 08/49] idpf: fix memleak in vport interrupt configuration The initialization of vport interrupt consists of two functions: 1) idpf_vport_intr_init() where a generic configuration is done 2) idpf_vport_intr_req_irq() where the irq for each q_vector is requested. The first function used to create a base name for each interrupt using "kasprintf()" call. Unfortunately, although that call allocated memory for a text buffer, that memory was never released. Fix this by removing creating the interrupt base name in 1). Instead, always create a full interrupt name in the function 2), because there is no need to create a base name separately, considering that the function 2) is never called out of idpf_vport_intr_init() context. Fixes: d4d558718266 ("idpf: initialize interrupts and enable vport") Cc: stable@vger.kernel.org # 6.7 Signed-off-by: Michal Kubiak Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/idpf/idpf_txrx.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index af2879f03b8df7..5c38cc87895e56 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -3780,13 +3780,15 @@ void idpf_vport_intr_update_itr_ena_irq(struct idpf_q_vector *q_vector) /** * idpf_vport_intr_req_irq - get MSI-X vectors from the OS for the vport * @vport: main vport structure - * @basename: name for the vector */ -static int idpf_vport_intr_req_irq(struct idpf_vport *vport, char *basename) +static int idpf_vport_intr_req_irq(struct idpf_vport *vport) { struct idpf_adapter *adapter = vport->adapter; + const char *drv_name, *if_name, *vec_name; int vector, err, irq_num, vidx; - const char *vec_name; + + drv_name = dev_driver_string(&vport->adapter->pdev->dev); + if_name = netdev_name(vport->netdev); for (vector = 0; vector < vport->num_q_vectors; vector++) { struct idpf_q_vector *q_vector = &vport->q_vectors[vector]; @@ -3804,8 +3806,8 @@ static int idpf_vport_intr_req_irq(struct idpf_vport *vport, char *basename) else continue; - name = kasprintf(GFP_KERNEL, "%s-%s-%d", basename, vec_name, - vidx); + name = kasprintf(GFP_KERNEL, "%s-%s-%s-%d", drv_name, if_name, + vec_name, vidx); err = request_irq(irq_num, idpf_vport_intr_clean_queues, 0, name, q_vector); @@ -4326,7 +4328,6 @@ int idpf_vport_intr_alloc(struct idpf_vport *vport) */ int idpf_vport_intr_init(struct idpf_vport *vport) { - char *int_name; int err; err = idpf_vport_intr_init_vec_idx(vport); @@ -4340,11 +4341,7 @@ int idpf_vport_intr_init(struct idpf_vport *vport) if (err) goto unroll_vectors_alloc; - int_name = kasprintf(GFP_KERNEL, "%s-%s", - dev_driver_string(&vport->adapter->pdev->dev), - vport->netdev->name); - - err = idpf_vport_intr_req_irq(vport, int_name); + err = idpf_vport_intr_req_irq(vport); if (err) goto unroll_vectors_alloc; From 5f5e2bd972d616ab2da2ff132192332c227d239a Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Fri, 12 Jul 2024 16:10:10 +0200 Subject: [PATCH 09/49] idpf: fix UAFs when destroying the queues The second tagged commit started sometimes (very rarely, but possible) throwing WARNs from net/core/page_pool.c:page_pool_disable_direct_recycling(). Turned out idpf frees interrupt vectors with embedded NAPIs *before* freeing the queues making page_pools' NAPI pointers lead to freed memory before these pools are destroyed by libeth. It's not clear whether there are other accesses to the freed vectors when destroying the queues, but anyway, we usually free queue/interrupt vectors only when the queues are destroyed and the NAPIs are guaranteed to not be referenced anywhere. Invert the allocation and freeing logic making queue/interrupt vectors be allocated first and freed last. Vectors don't require queues to be present, so this is safe. Additionally, this change allows to remove that useless queue->q_vector pointer cleanup, as vectors are still valid when freeing the queues (+ both are freed within one function, so it's not clear why nullify the pointers at all). Fixes: 1c325aac10a8 ("idpf: configure resources for TX queues") Fixes: 90912f9f4f2d ("idpf: convert header split mode to libeth + napi_build_skb()") Reported-by: Michal Kubiak Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/idpf/idpf_lib.c | 24 ++++++++++----------- drivers/net/ethernet/intel/idpf/idpf_txrx.c | 24 +-------------------- 2 files changed, 13 insertions(+), 35 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index b053ce23552cc4..10c76d6838d391 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -900,8 +900,8 @@ static void idpf_vport_stop(struct idpf_vport *vport) vport->link_up = false; idpf_vport_intr_deinit(vport); - idpf_vport_intr_rel(vport); idpf_vport_queues_rel(vport); + idpf_vport_intr_rel(vport); np->state = __IDPF_VPORT_DOWN; } @@ -1353,43 +1353,43 @@ static int idpf_vport_open(struct idpf_vport *vport) if (err) return err; - err = idpf_vport_queues_alloc(vport); - if (err) - return err; - err = idpf_vport_intr_alloc(vport); if (err) { dev_err(&adapter->pdev->dev, "Failed to allocate interrupts for vport %u: %d\n", vport->vport_id, err); - goto queues_rel; + return err; } + err = idpf_vport_queues_alloc(vport); + if (err) + goto intr_rel; + err = idpf_vport_queue_ids_init(vport); if (err) { dev_err(&adapter->pdev->dev, "Failed to initialize queue ids for vport %u: %d\n", vport->vport_id, err); - goto intr_rel; + goto queues_rel; } err = idpf_vport_intr_init(vport); if (err) { dev_err(&adapter->pdev->dev, "Failed to initialize interrupts for vport %u: %d\n", vport->vport_id, err); - goto intr_rel; + goto queues_rel; } err = idpf_rx_bufs_init_all(vport); if (err) { dev_err(&adapter->pdev->dev, "Failed to initialize RX buffers for vport %u: %d\n", vport->vport_id, err); - goto intr_rel; + goto queues_rel; } err = idpf_queue_reg_init(vport); if (err) { dev_err(&adapter->pdev->dev, "Failed to initialize queue registers for vport %u: %d\n", vport->vport_id, err); - goto intr_rel; + goto queues_rel; } idpf_rx_init_buf_tail(vport); @@ -1456,10 +1456,10 @@ static int idpf_vport_open(struct idpf_vport *vport) idpf_send_map_unmap_queue_vector_msg(vport, false); intr_deinit: idpf_vport_intr_deinit(vport); -intr_rel: - idpf_vport_intr_rel(vport); queues_rel: idpf_vport_queues_rel(vport); +intr_rel: + idpf_vport_intr_rel(vport); return err; } diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 5c38cc87895e56..a66a42f9355c65 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -3576,9 +3576,7 @@ static void idpf_vport_intr_napi_dis_all(struct idpf_vport *vport) */ void idpf_vport_intr_rel(struct idpf_vport *vport) { - int i, j, v_idx; - - for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) { + for (u32 v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) { struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx]; kfree(q_vector->complq); @@ -3593,26 +3591,6 @@ void idpf_vport_intr_rel(struct idpf_vport *vport) free_cpumask_var(q_vector->affinity_mask); } - /* Clean up the mapping of queues to vectors */ - for (i = 0; i < vport->num_rxq_grp; i++) { - struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; - - if (idpf_is_queue_model_split(vport->rxq_model)) - for (j = 0; j < rx_qgrp->splitq.num_rxq_sets; j++) - rx_qgrp->splitq.rxq_sets[j]->rxq.q_vector = NULL; - else - for (j = 0; j < rx_qgrp->singleq.num_rxq; j++) - rx_qgrp->singleq.rxqs[j]->q_vector = NULL; - } - - if (idpf_is_queue_model_split(vport->txq_model)) - for (i = 0; i < vport->num_txq_grp; i++) - vport->txq_grps[i].complq->q_vector = NULL; - else - for (i = 0; i < vport->num_txq_grp; i++) - for (j = 0; j < vport->txq_grps[i].num_txq; j++) - vport->txq_grps[i].txqs[j]->q_vector = NULL; - kfree(vport->q_vectors); vport->q_vectors = NULL; } From f0145fbde0c766b6c121acdff3ea8d18707496a4 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Mon, 12 Feb 2024 17:53:00 +0100 Subject: [PATCH 10/49] unroll: add generic loop unroll helpers There are cases when we need to explicitly unroll loops. For example, cache operations, filling DMA descriptors on very high speeds etc. Make MIPS' unroll header a generic one to have "unroll always" macro, which would work on any compiler and system, and add compiler-specific attribute macros. Example usage: #define UNROLL_BATCH 8 unrolled_count(UNROLL_BATCH) for (u32 i = 0; i < UNROLL_BATCH; i++) op(var, i); Not that sometimes the compilers won't unroll loops if they think that would have worse optimization and perf than with a loop, and that unroll attributes are available only starting GCC 8. In this case, you can still use unrolled_call(UNROLL_BATCH, op), which works in the range of [1...32] iterations. For better unrolling/parallelization, don't have any variables that interfere between iterations except for the iterator itself. Co-developed-by: Jose E. Marchesi # pragmas Signed-off-by: Jose E. Marchesi Co-developed-by: Paul Burton # unrolled_call() Signed-off-by: Paul Burton Signed-off-by: Alexander Lobakin --- arch/mips/include/asm/r4kcache.h | 5 +- arch/mips/include/asm/unroll.h | 75 -------------- drivers/net/ethernet/intel/i40e/i40e_xsk.c | 7 +- drivers/net/ethernet/intel/i40e/i40e_xsk.h | 8 -- drivers/net/ethernet/intel/ice/ice_xsk.c | 7 +- drivers/net/ethernet/intel/ice/ice_xsk.h | 8 -- include/linux/unroll.h | 112 +++++++++++++++++++++ 7 files changed, 125 insertions(+), 97 deletions(-) delete mode 100644 arch/mips/include/asm/unroll.h create mode 100644 include/linux/unroll.h diff --git a/arch/mips/include/asm/r4kcache.h b/arch/mips/include/asm/r4kcache.h index da1cd1bbdbc5d1..1004baba6ac972 100644 --- a/arch/mips/include/asm/r4kcache.h +++ b/arch/mips/include/asm/r4kcache.h @@ -13,6 +13,7 @@ #define _ASM_R4KCACHE_H #include +#include #include #include @@ -22,7 +23,6 @@ #include #include #include -#include extern void r5k_sc_init(void); extern void rm7k_sc_init(void); @@ -194,7 +194,8 @@ static inline void invalidate_tcache_page(unsigned long addr) #define cache_unroll(times, insn, op, addr, lsize) do { \ int i = 0; \ - unroll(times, _cache_op, insn, op, (addr) + (i++ * (lsize))); \ + unrolled_call(times, _cache_op, insn, op, \ + (addr) + (i++ * (lsize))); \ } while (0) /* build blast_xxx, blast_xxx_page, blast_xxx_page_indexed */ diff --git a/arch/mips/include/asm/unroll.h b/arch/mips/include/asm/unroll.h deleted file mode 100644 index 6f4ac854b12d28..00000000000000 --- a/arch/mips/include/asm/unroll.h +++ /dev/null @@ -1,75 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef __ASM_UNROLL_H__ -#define __ASM_UNROLL_H__ - -/* - * Explicitly unroll a loop, for use in cases where doing so is performance - * critical. - * - * Ideally we'd rely upon the compiler to provide this but there's no commonly - * available means to do so. For example GCC's "#pragma GCC unroll" - * functionality would be ideal but is only available from GCC 8 onwards. Using - * -funroll-loops is an option but GCC tends to make poor choices when - * compiling our string functions. -funroll-all-loops leads to massive code - * bloat, even if only applied to the string functions. - */ -#define unroll(times, fn, ...) do { \ - extern void bad_unroll(void) \ - __compiletime_error("Unsupported unroll"); \ - \ - /* \ - * We can't unroll if the number of iterations isn't \ - * compile-time constant. Unfortunately clang versions \ - * up until 8.0 tend to miss obvious constants & cause \ - * this check to fail, even though they go on to \ - * generate reasonable code for the switch statement, \ - * so we skip the sanity check for those compilers. \ - */ \ - BUILD_BUG_ON(!__builtin_constant_p(times)); \ - \ - switch (times) { \ - case 32: fn(__VA_ARGS__); fallthrough; \ - case 31: fn(__VA_ARGS__); fallthrough; \ - case 30: fn(__VA_ARGS__); fallthrough; \ - case 29: fn(__VA_ARGS__); fallthrough; \ - case 28: fn(__VA_ARGS__); fallthrough; \ - case 27: fn(__VA_ARGS__); fallthrough; \ - case 26: fn(__VA_ARGS__); fallthrough; \ - case 25: fn(__VA_ARGS__); fallthrough; \ - case 24: fn(__VA_ARGS__); fallthrough; \ - case 23: fn(__VA_ARGS__); fallthrough; \ - case 22: fn(__VA_ARGS__); fallthrough; \ - case 21: fn(__VA_ARGS__); fallthrough; \ - case 20: fn(__VA_ARGS__); fallthrough; \ - case 19: fn(__VA_ARGS__); fallthrough; \ - case 18: fn(__VA_ARGS__); fallthrough; \ - case 17: fn(__VA_ARGS__); fallthrough; \ - case 16: fn(__VA_ARGS__); fallthrough; \ - case 15: fn(__VA_ARGS__); fallthrough; \ - case 14: fn(__VA_ARGS__); fallthrough; \ - case 13: fn(__VA_ARGS__); fallthrough; \ - case 12: fn(__VA_ARGS__); fallthrough; \ - case 11: fn(__VA_ARGS__); fallthrough; \ - case 10: fn(__VA_ARGS__); fallthrough; \ - case 9: fn(__VA_ARGS__); fallthrough; \ - case 8: fn(__VA_ARGS__); fallthrough; \ - case 7: fn(__VA_ARGS__); fallthrough; \ - case 6: fn(__VA_ARGS__); fallthrough; \ - case 5: fn(__VA_ARGS__); fallthrough; \ - case 4: fn(__VA_ARGS__); fallthrough; \ - case 3: fn(__VA_ARGS__); fallthrough; \ - case 2: fn(__VA_ARGS__); fallthrough; \ - case 1: fn(__VA_ARGS__); fallthrough; \ - case 0: break; \ - \ - default: \ - /* \ - * Either the iteration count is unreasonable \ - * or we need to add more cases above. \ - */ \ - bad_unroll(); \ - break; \ - } \ -} while (0) - -#endif /* __ASM_UNROLL_H__ */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c index 4e885df789ef4d..794279495dd087 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c +++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c @@ -2,7 +2,10 @@ /* Copyright(c) 2018 Intel Corporation. */ #include +#include + #include + #include "i40e_txrx_common.h" #include "i40e_xsk.h" @@ -551,9 +554,9 @@ static void i40e_xmit_pkt_batch(struct i40e_ring *xdp_ring, struct xdp_desc *des u16 ntu = xdp_ring->next_to_use; struct i40e_tx_desc *tx_desc; dma_addr_t dma; - u32 i; - loop_unrolled_for(i = 0; i < PKTS_PER_BATCH; i++) { + unrolled_count(PKTS_PER_BATCH) + for (u32 i = 0; i < PKTS_PER_BATCH; i++) { u32 cmd = I40E_TX_DESC_CMD_ICRC | xsk_is_eop_desc(&desc[i]); dma = xsk_buff_raw_get_dma(xdp_ring->xsk_pool, desc[i].addr); diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.h b/drivers/net/ethernet/intel/i40e/i40e_xsk.h index ef156fad52f262..2622b6331715a9 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_xsk.h +++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.h @@ -14,14 +14,6 @@ */ #define PKTS_PER_BATCH 4 -#ifdef __clang__ -#define loop_unrolled_for _Pragma("clang loop unroll_count(4)") for -#elif __GNUC__ >= 8 -#define loop_unrolled_for _Pragma("GCC unroll 4") for -#else -#define loop_unrolled_for for -#endif - struct i40e_ring; struct i40e_vsi; struct net_device; diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c index a65955eb23c0bd..12d3592fe19de1 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.c +++ b/drivers/net/ethernet/intel/ice/ice_xsk.c @@ -2,8 +2,11 @@ /* Copyright (c) 2019, Intel Corporation. */ #include +#include + #include #include + #include "ice.h" #include "ice_base.h" #include "ice_type.h" @@ -994,9 +997,9 @@ static void ice_xmit_pkt_batch(struct ice_tx_ring *xdp_ring, struct xdp_desc *de { u16 ntu = xdp_ring->next_to_use; struct ice_tx_desc *tx_desc; - u32 i; - loop_unrolled_for(i = 0; i < PKTS_PER_BATCH; i++) { + unrolled_count(PKTS_PER_BATCH) + for (u32 i = 0; i < PKTS_PER_BATCH; i++) { dma_addr_t dma; dma = xsk_buff_raw_get_dma(xdp_ring->xsk_pool, descs[i].addr); diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.h b/drivers/net/ethernet/intel/ice/ice_xsk.h index 6fa181f080ef13..982dce2f187f30 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.h +++ b/drivers/net/ethernet/intel/ice/ice_xsk.h @@ -7,14 +7,6 @@ #define PKTS_PER_BATCH 8 -#ifdef __clang__ -#define loop_unrolled_for _Pragma("clang loop unroll_count(8)") for -#elif __GNUC__ >= 8 -#define loop_unrolled_for _Pragma("GCC unroll 8") for -#else -#define loop_unrolled_for for -#endif - struct ice_vsi; #ifdef CONFIG_XDP_SOCKETS diff --git a/include/linux/unroll.h b/include/linux/unroll.h new file mode 100644 index 00000000000000..1a7a0208bd72c0 --- /dev/null +++ b/include/linux/unroll.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2024 Intel Corporation */ + +#ifndef __LINUX_UNROLL_H +#define __LINUX_UNROLL_H + +#include + +#ifdef CONFIG_CC_IS_CLANG +#define __pick_unrolled(x, y) _Pragma(#x) +#elif CONFIG_GCC_VERSION >= 80000 +#define __pick_unrolled(x, y) _Pragma(#y) +#else +#define __pick_unrolled(x, y) /* not supported */ +#endif + +/** + * unrolled - loop attributes to ask the compiler to unroll it + * + * Usage: + * #define BATCH 4 + * unrolled_count(BATCH) + * for (u32 i = 0; i < BATCH; i++) + * // loop body without cross-iteration dependencies + * + * This is only a hint and the compiler is free to disable unrolling if it + * thinks the count is suboptimal and may hurt performance and/or hugely + * increase object code size. + * Not having any cross-iteration dependencies (i.e. when iter x + 1 depends + * on what iter x will do with variables) is not a strict requirement, but + * provides best performance and object code size. + * Available only on Clang and GCC 8.x onwards. + */ + +/* Ask the compiler to pick an optimal unroll count, Clang only */ +#define unrolled \ + __pick_unrolled(clang loop unroll(enable), /* nothing */) + +/* Unroll each @n iterations of a loop */ +#define unrolled_count(n) \ + __pick_unrolled(clang loop unroll_count(n), GCC unroll n) + +/* Unroll the whole loop */ +#define unrolled_full \ + __pick_unrolled(clang loop unroll(full), GCC unroll 65534) + +/* Never unroll a loop */ +#define unrolled_none \ + __pick_unrolled(clang loop unroll(disable), GCC unroll 1) + +/** + * unrolled_call - explicitly unroll a loop + * @times: number of times to call @fn, in range [1, 32] + * @fn: function to call repeatedly + * + * Usage: + * #define BATCH 4 + * #define func(ptr, i) // loop body without cross-iteration dependencies + * u32 i = 0; + * unrolled_call(BATCH, func, ptr, i++); + * + * Less convenient than unrolled* above, but available on every compiler + * and always performs unrolling even if the compiler wouldn't do that + * under an unrolled* hint due to his optimization decisions. + */ +#define unrolled_call(times, fn, ...) do { \ + static_assert(__builtin_constant_p(times)); \ + \ + switch (times) { \ + case 32: fn(__VA_ARGS__); fallthrough; \ + case 31: fn(__VA_ARGS__); fallthrough; \ + case 30: fn(__VA_ARGS__); fallthrough; \ + case 29: fn(__VA_ARGS__); fallthrough; \ + case 28: fn(__VA_ARGS__); fallthrough; \ + case 27: fn(__VA_ARGS__); fallthrough; \ + case 26: fn(__VA_ARGS__); fallthrough; \ + case 25: fn(__VA_ARGS__); fallthrough; \ + case 24: fn(__VA_ARGS__); fallthrough; \ + case 23: fn(__VA_ARGS__); fallthrough; \ + case 22: fn(__VA_ARGS__); fallthrough; \ + case 21: fn(__VA_ARGS__); fallthrough; \ + case 20: fn(__VA_ARGS__); fallthrough; \ + case 19: fn(__VA_ARGS__); fallthrough; \ + case 18: fn(__VA_ARGS__); fallthrough; \ + case 17: fn(__VA_ARGS__); fallthrough; \ + case 16: fn(__VA_ARGS__); fallthrough; \ + case 15: fn(__VA_ARGS__); fallthrough; \ + case 14: fn(__VA_ARGS__); fallthrough; \ + case 13: fn(__VA_ARGS__); fallthrough; \ + case 12: fn(__VA_ARGS__); fallthrough; \ + case 11: fn(__VA_ARGS__); fallthrough; \ + case 10: fn(__VA_ARGS__); fallthrough; \ + case 9: fn(__VA_ARGS__); fallthrough; \ + case 8: fn(__VA_ARGS__); fallthrough; \ + case 7: fn(__VA_ARGS__); fallthrough; \ + case 6: fn(__VA_ARGS__); fallthrough; \ + case 5: fn(__VA_ARGS__); fallthrough; \ + case 4: fn(__VA_ARGS__); fallthrough; \ + case 3: fn(__VA_ARGS__); fallthrough; \ + case 2: fn(__VA_ARGS__); fallthrough; \ + case 1: fn(__VA_ARGS__); break; \ + default: \ + /* \ + * Either the iteration count is unreasonable or we need \ + * to add more cases above. \ + */ \ + BUILD_BUG_ON_MSG(1, "Unsupported unroll count: " #times); \ + break; \ + } \ +} while (0) + +#endif /* __LINUX_UNROLL_H */ From 86249afd87767c82c9289c469646c98306cc01dc Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 14 Mar 2023 17:40:57 +0100 Subject: [PATCH 11/49] libeth: add common queue stats Define common structures, inline helpers and Ethtool helpers to collect, update and export the statistics (RQ, SQ, XDPSQ). Use u64_stats_t right from the start, as well as the corresponding helpers to ensure tear-free operations. For the NAPI parts of both Rx and Tx, also define small onstack containers to update them in polling loops and then sync the actual containers once a loop ends. In order to implement fully generic Netlink per-queue stats callbacks, &libeth_netdev_priv is introduced and is required to be embedded at the start of the driver's netdev_priv structure. Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/libeth/Makefile | 4 +- drivers/net/ethernet/intel/libeth/netdev.c | 157 +++++++++ drivers/net/ethernet/intel/libeth/priv.h | 21 ++ drivers/net/ethernet/intel/libeth/rx.c | 5 - drivers/net/ethernet/intel/libeth/stats.c | 357 +++++++++++++++++++++ include/net/libeth/netdev.h | 31 ++ include/net/libeth/stats.h | 145 +++++++++ include/net/libeth/types.h | 247 ++++++++++++++ 8 files changed, 961 insertions(+), 6 deletions(-) create mode 100644 drivers/net/ethernet/intel/libeth/netdev.c create mode 100644 drivers/net/ethernet/intel/libeth/priv.h create mode 100644 drivers/net/ethernet/intel/libeth/stats.c create mode 100644 include/net/libeth/netdev.h create mode 100644 include/net/libeth/stats.h create mode 100644 include/net/libeth/types.h diff --git a/drivers/net/ethernet/intel/libeth/Makefile b/drivers/net/ethernet/intel/libeth/Makefile index 52492b08113261..b30a2804554f04 100644 --- a/drivers/net/ethernet/intel/libeth/Makefile +++ b/drivers/net/ethernet/intel/libeth/Makefile @@ -3,4 +3,6 @@ obj-$(CONFIG_LIBETH) += libeth.o -libeth-y := rx.o +libeth-y += netdev.o +libeth-y += rx.o +libeth-y += stats.o diff --git a/drivers/net/ethernet/intel/libeth/netdev.c b/drivers/net/ethernet/intel/libeth/netdev.c new file mode 100644 index 00000000000000..6115472b3bb69a --- /dev/null +++ b/drivers/net/ethernet/intel/libeth/netdev.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2024 Intel Corporation */ + +#include +#include + +#include +#include + +#include "priv.h" + +/** + * __libeth_netdev_alloc - allocate a &net_device with libeth generic stats + * @priv: sizeof() of the private structure with embedded &libeth_netdev_priv + * @rqs: maximum number of Rx queues to be used + * @sqs: maximum number of Tx queues to be used + * @xdpsqs: maximum number of XDP Tx queues to be used + * + * Allocates a new &net_device and initializes the embedded &libeth_netdev_priv + * and the libeth generic stats for it. + * Use the non-underscored wrapper in drivers instead. + * + * Return: new &net_device on success, %NULL on error. + */ +struct net_device *__libeth_netdev_alloc(u32 priv, u32 rqs, u32 sqs, + u32 xdpsqs) +{ + struct net_device *dev; + + dev = alloc_etherdev_mqs(priv, sqs, rqs); + if (!dev) + return NULL; + + if (!libeth_stats_init_priv(dev, rqs, sqs, xdpsqs)) + goto err_netdev; + + return dev; + +err_netdev: + free_netdev(dev); + + return NULL; +} +EXPORT_SYMBOL_NS_GPL(__libeth_netdev_alloc, LIBETH); + +/** + * libeth_netdev_free - free a &net_device with libeth generic stats + * @dev: &net_device to free + * + * Deinitializes and frees the embedded &libeth_netdev_priv and the netdev + * itself, to be used if @dev was allocated using libeth_netdev_alloc(). + */ +void libeth_netdev_free(struct net_device *dev) +{ + libeth_stats_free_priv(dev); + free_netdev(dev); +} +EXPORT_SYMBOL_NS_GPL(libeth_netdev_free, LIBETH); + +/** + * __libeth_set_real_num_queues - set the actual number of queues in use + * @dev: &net_device to configure + * @rqs: actual number of Rx queues + * @sqs: actual number of Tx queues + * @xdpsqs: actual number of XDP Tx queues + * + * Sets the actual number of queues in use, to be called on ifup for netdevs + * allocated via libeth_netdev_alloc(). + * Use the non-underscored wrapper in drivers instead. + * + * Return: %0 on success, -errno on error. + */ +int __libeth_set_real_num_queues(struct net_device *dev, u32 rqs, u32 sqs, + u32 xdpsqs) +{ + struct libeth_netdev_priv *priv = netdev_priv(dev); + int ret; + + ret = netif_set_real_num_rx_queues(dev, rqs); + if (ret) + return ret; + + ret = netif_set_real_num_tx_queues(dev, sqs); + if (ret) + return ret; + + priv->curr_xdpsqs = xdpsqs; + + return 0; +} +EXPORT_SYMBOL_NS_GPL(__libeth_set_real_num_queues, LIBETH); + +/* Ethtool */ + +/** + * libeth_ethtool_get_sset_count - get the number of libeth generic stats + * @dev: libeth-driven &net_device + * @sset: ``ETH_SS_STATS`` only, for compatibility with Ethtool callbacks + * + * Can be used directly in ðtool_ops if the driver doesn't have HW-specific + * stats or called from the corresponding driver callback. + * + * Return: the number of stats/stringsets. + */ +int libeth_ethtool_get_sset_count(struct net_device *dev, int sset) +{ + if (sset != ETH_SS_STATS) + return 0; + + return libeth_stats_get_sset_count(dev); +} +EXPORT_SYMBOL_NS_GPL(libeth_ethtool_get_sset_count, LIBETH); + +/** + * libeth_ethtool_get_strings - get libeth generic stats strings/names + * @dev: libeth-driven &net_device + * @sset: ``ETH_SS_STATS`` only, for compatibility with Ethtool callbacks + * @data: container to fill with the stats names + * + * Can be used directly in ðtool_ops if the driver doesn't have HW-specific + * stats or called from the corresponding driver callback. + * Note that the function doesn't advance the @data pointer, so it's better to + * call it at the end to avoid code complication. + */ +void libeth_ethtool_get_strings(struct net_device *dev, u32 sset, u8 *data) +{ + if (sset != ETH_SS_STATS) + return; + + libeth_stats_get_strings(dev, data); +} +EXPORT_SYMBOL_NS_GPL(libeth_ethtool_get_strings, LIBETH); + +/** + * libeth_ethtool_get_stats - get libeth generic stats counters + * @dev: libeth-driven &net_device + * @stats: unused, for compatibility with Ethtool callbacks + * @data: container to fill with the stats counters + * + * Can be used directly in ðtool_ops if the driver doesn't have HW-specific + * stats or called from the corresponding driver callback. + * Note that the function doesn't advance the @data pointer, so it's better to + * call it at the end to avoid code complication. Anyhow, the order must be the + * same as in the ::get_strings() implementation. + */ +void libeth_ethtool_get_stats(struct net_device *dev, + struct ethtool_stats *stats, + u64 *data) +{ + libeth_stats_get_data(dev, data); +} +EXPORT_SYMBOL_NS_GPL(libeth_ethtool_get_stats, LIBETH); + +/* Module */ + +MODULE_DESCRIPTION("Common Ethernet library"); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/ethernet/intel/libeth/priv.h b/drivers/net/ethernet/intel/libeth/priv.h new file mode 100644 index 00000000000000..6455aab0311cf8 --- /dev/null +++ b/drivers/net/ethernet/intel/libeth/priv.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2024 Intel Corporation */ + +#ifndef __LIBETH_PRIV_H +#define __LIBETH_PRIV_H + +#include + +/* Stats */ + +struct net_device; + +bool libeth_stats_init_priv(struct net_device *dev, u32 rqs, u32 sqs, + u32 xdpsqs); +void libeth_stats_free_priv(const struct net_device *dev); + +int libeth_stats_get_sset_count(struct net_device *dev); +void libeth_stats_get_strings(struct net_device *dev, u8 *data); +void libeth_stats_get_data(struct net_device *dev, u64 *data); + +#endif /* __LIBETH_PRIV_H */ diff --git a/drivers/net/ethernet/intel/libeth/rx.c b/drivers/net/ethernet/intel/libeth/rx.c index f209266693185b..d31779bbfccd7c 100644 --- a/drivers/net/ethernet/intel/libeth/rx.c +++ b/drivers/net/ethernet/intel/libeth/rx.c @@ -252,8 +252,3 @@ void libeth_rx_pt_gen_hash_type(struct libeth_rx_pt *pt) pt->hash_type |= libeth_rx_pt_xdp_pl[pt->payload_layer]; } EXPORT_SYMBOL_NS_GPL(libeth_rx_pt_gen_hash_type, LIBETH); - -/* Module */ - -MODULE_DESCRIPTION("Common Ethernet library"); -MODULE_LICENSE("GPL"); diff --git a/drivers/net/ethernet/intel/libeth/stats.c b/drivers/net/ethernet/intel/libeth/stats.c new file mode 100644 index 00000000000000..ceac44d4ebfdf2 --- /dev/null +++ b/drivers/net/ethernet/intel/libeth/stats.c @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2024 Intel Corporation */ + +#include + +#include +#include + +#include "priv.h" + +/* Common */ + +static void libeth_stats_sync(u64 *base, u64 *sarr, + const struct u64_stats_sync *syncp, + const u64_stats_t *raw, u32 num) +{ + u32 start; + + do { + start = u64_stats_fetch_begin(syncp); + for (u32 i = 0; i < num; i++) + sarr[i] = u64_stats_read(&raw[i]); + } while (u64_stats_fetch_retry(syncp, start)); + + for (u32 i = 0; i < num; i++) + base[i] += sarr[i]; +} + +static void __libeth_stats_get_strings(u8 **data, u32 qid, const char *pfx, + const char * const *str, u32 num) +{ + for (u32 i = 0; i < num; i++) + ethtool_sprintf(data, "%s%u_%s", pfx, qid, str[i]); +} + +/* The following barely readable compression block defines, amidst others, + * exported libeth_{rq,sq,xdpsq}_stats_{,de}init() which must be called for + * each stats container embedded in a queue structure on ifup/ifdown + * correspondingly. Note that the @qid it takes is the networking stack + * queue ID, not a driver/device's internal one. + */ + +#define ___base(s) aligned_u64 s; +#define ___string(s) __stringify(s), + +#define LIBETH_STATS_DEFINE_HELPERS(pfx, PFX) \ +struct libeth_##pfx##_base_stats { \ + struct mutex lock; \ + \ + union { \ + struct { \ + LIBETH_DECLARE_##PFX##_STATS(___base); \ + }; \ + DECLARE_FLEX_ARRAY(aligned_u64, raw); \ + }; \ +}; \ +static const char * const libeth_##pfx##_stats_str[] = { \ + LIBETH_DECLARE_##PFX##_STATS(___string) \ +}; \ +static const u32 LIBETH_##PFX##_STATS_NUM = \ + ARRAY_SIZE(libeth_##pfx##_stats_str); \ + \ +static void libeth_##pfx##_stats_sync(u64 *base, \ + const struct libeth_##pfx##_stats *qs) \ +{ \ + u64 sarr[ARRAY_SIZE(libeth_##pfx##_stats_str)]; \ + \ + if (qs) \ + libeth_stats_sync(base, sarr, &qs->syncp, qs->raw, \ + LIBETH_##PFX##_STATS_NUM); \ +} \ + \ +void libeth_##pfx##_stats_init(const struct net_device *dev, \ + struct libeth_##pfx##_stats *stats, \ + u32 qid) \ +{ \ + const struct libeth_netdev_priv *priv = netdev_priv(dev); \ + \ + memset(stats, 0, sizeof(*stats)); \ + u64_stats_init(&stats->syncp); \ + \ + mutex_init(&priv->base_##pfx##s[qid].lock); \ + WRITE_ONCE(priv->live_##pfx##s[qid], stats); \ +} \ +EXPORT_SYMBOL_NS_GPL(libeth_##pfx##_stats_init, LIBETH); \ + \ +void libeth_##pfx##_stats_deinit(const struct net_device *dev, u32 qid) \ +{ \ + const struct libeth_netdev_priv *priv = netdev_priv(dev); \ + struct libeth_##pfx##_base_stats *base = &priv->base_##pfx##s[qid]; \ + \ + mutex_lock(&base->lock); \ + libeth_##pfx##_stats_sync(base->raw, \ + READ_ONCE(priv->live_##pfx##s[qid])); \ + mutex_unlock(&base->lock); \ + \ + WRITE_ONCE(priv->live_##pfx##s[qid], NULL); \ +} \ +EXPORT_SYMBOL_NS_GPL(libeth_##pfx##_stats_deinit, LIBETH); \ + \ +static void libeth_##pfx##_stats_get_strings(u8 **data, u32 num) \ +{ \ + for (u32 i = 0; i < num; i++) \ + __libeth_stats_get_strings(data, i, #pfx, \ + libeth_##pfx##_stats_str, \ + LIBETH_##PFX##_STATS_NUM); \ +} \ + \ +static void \ +__libeth_##pfx##_stats_get_data(u64 **data, \ + struct libeth_##pfx##_base_stats *base, \ + const struct libeth_##pfx##_stats *qs) \ +{ \ + mutex_lock(&base->lock); \ + memcpy(*data, base->raw, sizeof(*base)); \ + mutex_unlock(&base->lock); \ + \ + libeth_##pfx##_stats_sync(*data, qs); \ + *data += LIBETH_##PFX##_STATS_NUM; \ +} \ + \ +static void \ +libeth_##pfx##_stats_get_data(u64 **data, \ + const struct libeth_netdev_priv *priv) \ +{ \ + for (u32 i = 0; i < priv->last_##pfx##s; i++) { \ + const struct libeth_##pfx##_stats *qs; \ + \ + qs = READ_ONCE(priv->live_##pfx##s[i]); \ + __libeth_##pfx##_stats_get_data(data, \ + &priv->base_##pfx##s[i], \ + qs); \ + } \ +} + +LIBETH_STATS_DEFINE_HELPERS(rq, RQ); +LIBETH_STATS_DEFINE_HELPERS(sq, SQ); +LIBETH_STATS_DEFINE_HELPERS(xdpsq, XDPSQ); + +#undef ___base +#undef ___string + +/* Netlink stats. Exported fields have the same names as in the NL structs */ + +struct libeth_stats_export { + u16 li; + u16 gi; +}; + +#define LIBETH_STATS_EXPORT(lpfx, gpfx, field) { \ + .li = (offsetof(struct libeth_##lpfx##_stats, field) - \ + offsetof(struct libeth_##lpfx##_stats, raw)) / \ + sizeof_field(struct libeth_##lpfx##_stats, field), \ + .gi = offsetof(struct netdev_queue_stats_##gpfx, field) / \ + sizeof_field(struct netdev_queue_stats_##gpfx, field) \ +} +#define LIBETH_RQ_STATS_EXPORT(field) LIBETH_STATS_EXPORT(rq, rx, field) +#define LIBETH_SQ_STATS_EXPORT(field) LIBETH_STATS_EXPORT(sq, tx, field) + +static const struct libeth_stats_export libeth_rq_stats_export[] = { + LIBETH_RQ_STATS_EXPORT(bytes), + LIBETH_RQ_STATS_EXPORT(packets), + LIBETH_RQ_STATS_EXPORT(csum_unnecessary), + LIBETH_RQ_STATS_EXPORT(hw_gro_packets), + LIBETH_RQ_STATS_EXPORT(hw_gro_bytes), + LIBETH_RQ_STATS_EXPORT(alloc_fail), + LIBETH_RQ_STATS_EXPORT(csum_none), + LIBETH_RQ_STATS_EXPORT(csum_bad), +}; + +static const struct libeth_stats_export libeth_sq_stats_export[] = { + LIBETH_SQ_STATS_EXPORT(bytes), + LIBETH_SQ_STATS_EXPORT(packets), + LIBETH_SQ_STATS_EXPORT(csum_none), + LIBETH_SQ_STATS_EXPORT(needs_csum), + LIBETH_SQ_STATS_EXPORT(hw_gso_packets), + LIBETH_SQ_STATS_EXPORT(hw_gso_bytes), + LIBETH_SQ_STATS_EXPORT(stop), + LIBETH_SQ_STATS_EXPORT(wake), +}; + +#define libeth_stats_foreach_export(pfx, iter) \ + for (const struct libeth_stats_export *iter = \ + &libeth_##pfx##_stats_export[0]; \ + iter < &libeth_##pfx##_stats_export[ \ + ARRAY_SIZE(libeth_##pfx##_stats_export)]; \ + iter++) + +#define LIBETH_STATS_DEFINE_EXPORT(pfx, gpfx) \ +static void \ +libeth_get_queue_stats_##gpfx(struct net_device *dev, int idx, \ + struct netdev_queue_stats_##gpfx *stats) \ +{ \ + const struct libeth_netdev_priv *priv = netdev_priv(dev); \ + const struct libeth_##pfx##_stats *qs; \ + u64 *raw = (u64 *)stats; \ + u32 start; \ + \ + qs = READ_ONCE(priv->live_##pfx##s[idx]); \ + if (!qs) \ + return; \ + \ + do { \ + start = u64_stats_fetch_begin(&qs->syncp); \ + \ + libeth_stats_foreach_export(pfx, exp) \ + raw[exp->gi] = u64_stats_read(&qs->raw[exp->li]); \ + } while (u64_stats_fetch_retry(&qs->syncp, start)); \ +} \ + \ +static void \ +libeth_get_##pfx##_base_stats(const struct net_device *dev, \ + struct netdev_queue_stats_##gpfx *stats) \ +{ \ + const struct libeth_netdev_priv *priv = netdev_priv(dev); \ + u64 *raw = (u64 *)stats; \ + \ + memset(stats, 0, sizeof(*(stats))); \ + \ + for (u32 i = 0; i < dev->num_##gpfx##_queues; i++) { \ + struct libeth_##pfx##_base_stats *base = \ + &priv->base_##pfx##s[i]; \ + \ + mutex_lock(&base->lock); \ + \ + libeth_stats_foreach_export(pfx, exp) \ + raw[exp->gi] += base->raw[exp->li]; \ + \ + mutex_unlock(&base->lock); \ + } \ +} + +LIBETH_STATS_DEFINE_EXPORT(rq, rx); +LIBETH_STATS_DEFINE_EXPORT(sq, tx); + +static void libeth_get_base_stats(struct net_device *dev, + struct netdev_queue_stats_rx *rx, + struct netdev_queue_stats_tx *tx) +{ + libeth_get_rq_base_stats(dev, rx); + libeth_get_sq_base_stats(dev, tx); +} + +static const struct netdev_stat_ops libeth_netdev_stat_ops = { + .get_base_stats = libeth_get_base_stats, + .get_queue_stats_rx = libeth_get_queue_stats_rx, + .get_queue_stats_tx = libeth_get_queue_stats_tx, +}; + +/* Ethtool: base + live */ + +int libeth_stats_get_sset_count(struct net_device *dev) +{ + struct libeth_netdev_priv *priv = netdev_priv(dev); + + priv->last_rqs = dev->real_num_rx_queues; + priv->last_sqs = dev->real_num_tx_queues; + priv->last_xdpsqs = priv->curr_xdpsqs; + + return priv->last_rqs * LIBETH_RQ_STATS_NUM + + priv->last_sqs * LIBETH_SQ_STATS_NUM + + priv->last_xdpsqs * LIBETH_XDPSQ_STATS_NUM; +} + +void libeth_stats_get_strings(struct net_device *dev, u8 *data) +{ + const struct libeth_netdev_priv *priv = netdev_priv(dev); + + libeth_rq_stats_get_strings(&data, priv->last_rqs); + libeth_sq_stats_get_strings(&data, priv->last_sqs); + libeth_xdpsq_stats_get_strings(&data, priv->last_xdpsqs); +} + +void libeth_stats_get_data(struct net_device *dev, u64 *data) +{ + struct libeth_netdev_priv *priv = netdev_priv(dev); + + libeth_rq_stats_get_data(&data, priv); + libeth_sq_stats_get_data(&data, priv); + libeth_xdpsq_stats_get_data(&data, priv); + + priv->last_rqs = 0; + priv->last_sqs = 0; + priv->last_xdpsqs = 0; +} + +/* Private init */ + +bool libeth_stats_init_priv(struct net_device *dev, u32 rqs, u32 sqs, + u32 xdpsqs) +{ + struct libeth_netdev_priv *priv = netdev_priv(dev); + + priv->base_rqs = kvcalloc(rqs, sizeof(*priv->base_rqs), GFP_KERNEL); + if (!priv->base_rqs) + return false; + + priv->live_rqs = kvcalloc(rqs, sizeof(*priv->live_rqs), GFP_KERNEL); + if (!priv->live_rqs) + goto err_base_rqs; + + priv->base_sqs = kvcalloc(sqs, sizeof(*priv->base_sqs), GFP_KERNEL); + if (!priv->base_sqs) + goto err_live_rqs; + + priv->live_sqs = kvcalloc(sqs, sizeof(*priv->live_sqs), GFP_KERNEL); + if (!priv->live_sqs) + goto err_base_sqs; + + dev->stat_ops = &libeth_netdev_stat_ops; + + if (!xdpsqs) + return true; + + priv->base_xdpsqs = kvcalloc(xdpsqs, sizeof(*priv->base_xdpsqs), + GFP_KERNEL); + if (!priv->base_xdpsqs) + goto err_live_sqs; + + priv->live_xdpsqs = kvcalloc(xdpsqs, sizeof(*priv->live_xdpsqs), + GFP_KERNEL); + if (!priv->live_xdpsqs) + goto err_base_xdpsqs; + + priv->max_xdpsqs = xdpsqs; + + return true; + +err_base_xdpsqs: + kvfree(priv->base_xdpsqs); +err_live_sqs: + kvfree(priv->live_sqs); +err_base_sqs: + kvfree(priv->base_sqs); +err_live_rqs: + kvfree(priv->live_rqs); +err_base_rqs: + kvfree(priv->base_rqs); + + return false; +} + +void libeth_stats_free_priv(const struct net_device *dev) +{ + const struct libeth_netdev_priv *priv = netdev_priv(dev); + + kvfree(priv->base_rqs); + kvfree(priv->live_rqs); + kvfree(priv->base_sqs); + kvfree(priv->live_sqs); + + if (!priv->max_xdpsqs) + return; + + kvfree(priv->base_xdpsqs); + kvfree(priv->live_xdpsqs); +} diff --git a/include/net/libeth/netdev.h b/include/net/libeth/netdev.h new file mode 100644 index 00000000000000..22a07f0b16d7fc --- /dev/null +++ b/include/net/libeth/netdev.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2024 Intel Corporation */ + +#ifndef __LIBETH_NETDEV_H +#define __LIBETH_NETDEV_H + +#include + +struct ethtool_stats; + +struct net_device *__libeth_netdev_alloc(u32 priv, u32 rqs, u32 sqs, + u32 xdpsqs); +void libeth_netdev_free(struct net_device *dev); + +int __libeth_set_real_num_queues(struct net_device *dev, u32 rqs, u32 sqs, + u32 xdpsqs); + +#define libeth_netdev_alloc(priv, rqs, sqs, ...) \ + __libeth_netdev_alloc(priv, rqs, sqs, (__VA_ARGS__ + 0)) +#define libeth_set_real_num_queues(dev, rqs, sqs, ...) \ + __libeth_set_real_num_queues(dev, rqs, sqs, (__VA_ARGS__ + 0)) + +/* Ethtool */ + +int libeth_ethtool_get_sset_count(struct net_device *dev, int sset); +void libeth_ethtool_get_strings(struct net_device *dev, u32 sset, u8 *data); +void libeth_ethtool_get_stats(struct net_device *dev, + struct ethtool_stats *stats, + u64 *data); + +#endif /* __LIBETH_NETDEV_H */ diff --git a/include/net/libeth/stats.h b/include/net/libeth/stats.h new file mode 100644 index 00000000000000..1f2dd63418f2d2 --- /dev/null +++ b/include/net/libeth/stats.h @@ -0,0 +1,145 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2024 Intel Corporation */ + +#ifndef __LIBETH_STATS_H +#define __LIBETH_STATS_H + +#include +#include + +#include + +/* Common */ + +/** + * __libeth_stats_inc_one - safely increment one stats structure counter + * @s: queue stats structure to update (&libeth_rq_stats etc.) + * @f: name of the field to increment + * @n: name of the temporary variable, result of __UNIQUE_ID() + * + * To be used on exception or slow paths -- allocation fails, queue stops etc. + * Use the non-underscored wrapper in drivers instead. + */ +#define __libeth_stats_inc_one(s, f, n) do { \ + typeof(*(s)) *n = (s); \ + \ + u64_stats_update_begin(&n->syncp); \ + u64_stats_inc(&n->f); \ + u64_stats_update_end(&n->syncp); \ +} while (0) +#define libeth_stats_inc_one(s, f) \ + __libeth_stats_inc_one(s, f, __UNIQUE_ID(qs_)) + +/** + * __libeth_stats_add_frags - update the frags counter if needed + * @s: onstack stats structure to update (&libeth_rq_napi_stats etc.) + * @frags: number of frags processed + * @uf: name of the temporary variable, result of __UNIQUE_ID() + * + * Update the frags counter if @frags > 1, do nothing for non-SG frames. + * Use the non-underscored wrapper in drivers instead. + */ +#define __libeth_stats_add_frags(s, frags, uf) do { \ + u32 uf = (frags); \ + \ + if (uf > 1) \ + (s)->fragments += uf; \ +} while (0) +#define libeth_stats_add_frags(s, frags) \ + __libeth_stats_add_frags(s, frags, __UNIQUE_ID(frags_)) + +#define ___libeth_stats_add(qs, ss, group, uq, us, ur) do { \ + typeof(*(qs)) *uq = (qs); \ + u64_stats_t *ur = (typeof(ur))&uq->group; \ + typeof(*(ss)) *us = (ss); \ + \ + static_assert(sizeof(uq->group) == sizeof(*us) * 2); \ + u64_stats_update_begin(&uq->syncp); \ + \ + unrolled_count(__alignof(*uq) / sizeof(*uq->raw)) \ + for (u32 i = 0; i < sizeof(*us) / sizeof(*us->raw); i++) \ + u64_stats_add(&ur[i], us->raw[i]); \ + \ + u64_stats_update_end(&uq->syncp); \ +} while (0) +#define __libeth_stats_add(qs, ss, group) \ + ___libeth_stats_add(qs, ss, group, __UNIQUE_ID(qs_), \ + __UNIQUE_ID(ss_), __UNIQUE_ID(raw_)) + +/* The following barely readable compression block defines the following + * entities to be used in drivers: + * + * &libeth_rq_napi_stats - onstack stats container for RQ NAPI polling + * libeth_rq_napi_stats_add() - add RQ onstack stats to the queue container + * &libeth_sq_napi_stats - onstack stats container for SQ completion polling + * libeth_sq_napi_stats_add() - add SQ onstack stats to the queue container + * &libeth_sq_xmit_stats - onstack stats container for ::ndo_start_xmit() + * libeth_sq_xmit_stats_add() - add SQ xmit stats to the queue container + * &libeth_xdpsq_napi_stats - onstack stats container for XDPSQ polling + * libeth_xdpsq_napi_stats_add() - add XDPSQ stats to the queue container + * + * During the NAPI poll loop or any other hot function, the "hot" counters + * get updated on the stack only. Then at the end, the corresponding _add() + * is called to quickly add them to the stats container embedded into the + * queue structure using __libeth_stats_add(). + * The onstack counters are of type u32, thus it is assumed that one + * polling/sending cycle can't go above ``U32_MAX`` for any of them. + */ + +#define ___stack(s) u32 s; + +#define LIBETH_STATS_DEFINE_STACK(pfx, PFX, type, TYPE) \ +struct libeth_##pfx##_##type##_stats { \ + union { \ + struct { \ + LIBETH_DECLARE_##PFX##_##TYPE##_STATS(___stack); \ + }; \ + DECLARE_FLEX_ARRAY(u32, raw); \ + }; \ +}; \ + \ +static inline void \ +libeth_##pfx##_##type##_stats_add(struct libeth_##pfx##_stats *qs, \ + const struct libeth_##pfx##_##type##_stats \ + *ss) \ +{ \ + __libeth_stats_add(qs, ss, type); \ +} + +#define LIBETH_STATS_DECLARE_HELPERS(pfx) \ +void libeth_##pfx##_stats_init(const struct net_device *dev, \ + struct libeth_##pfx##_stats *stats, \ + u32 qid); \ +void libeth_##pfx##_stats_deinit(const struct net_device *dev, u32 qid) + +LIBETH_STATS_DEFINE_STACK(rq, RQ, napi, NAPI); +LIBETH_STATS_DECLARE_HELPERS(rq); + +LIBETH_STATS_DEFINE_STACK(sq, SQ, napi, NAPI); +LIBETH_STATS_DEFINE_STACK(sq, SQ, xmit, XMIT); + +/** + * libeth_sq_xmit_stats_csum - convert skb csum status to the SQ xmit stats + * @ss: onstack SQ xmit stats to increment + * @skb: &sk_buff to process stats for + * + * To be called from ::ndo_start_xmit() to account whether checksum offload + * was enabled when sending this @skb. + */ +static inline void libeth_sq_xmit_stats_csum(struct libeth_sq_xmit_stats *ss, + const struct sk_buff *skb) +{ + if (skb->ip_summed == CHECKSUM_PARTIAL) + ss->needs_csum++; + else + ss->csum_none++; +} + +LIBETH_STATS_DECLARE_HELPERS(sq); + +LIBETH_STATS_DEFINE_STACK(xdpsq, XDPSQ, napi, NAPI); +LIBETH_STATS_DECLARE_HELPERS(xdpsq); + +#undef ___stack + +#endif /* __LIBETH_STATS_H */ diff --git a/include/net/libeth/types.h b/include/net/libeth/types.h new file mode 100644 index 00000000000000..2437d68aae2190 --- /dev/null +++ b/include/net/libeth/types.h @@ -0,0 +1,247 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2024 Intel Corporation */ + +#ifndef __LIBETH_TYPES_H +#define __LIBETH_TYPES_H + +#include + +/** + * struct libeth_netdev_priv - libeth netdev private structure + * @curr_xdpsqs: current number of XDPSQs in use + * @max_xdpsqs: maximum number of XDPSQs this netdev has + * @last_rqs: number of RQs last time Ethtool stats were requested + * @last_sqs: number of SQs last time Ethtool stats were requested + * @last_xdpsqs: number of XDPSQ last time Ethtool stats were requested + * @base_rqs: per-queue RQ stats containers with the netdev lifetime + * @base_sqs: per-queue SQ stats containers with the netdev lifetime + * @base_xdpsqs: per-queue XDPSQ stats containers with the netdev lifetime + * @live_rqs: pointers to the current driver's embedded RQ stats + * @live_sqs: pointers to the current driver's embedded SQ stats + * @live_xdpsqs: pointers to the current driver's embedded XDPSQ stats + * + * The structure must be placed strictly at the beginning of driver's netdev + * private structure if it uses libeth generic stats, as libeth uses + * netdev_priv() to access it. The structure is private to libeth and + * shouldn't be accessed from drivers directly. + */ +struct libeth_netdev_priv { + u32 curr_xdpsqs; + u32 max_xdpsqs; + + u16 last_rqs; + u16 last_sqs; + u16 last_xdpsqs; + + struct libeth_rq_base_stats *base_rqs; + struct libeth_sq_base_stats *base_sqs; + struct libeth_xdpsq_base_stats *base_xdpsqs; + + const struct libeth_rq_stats **live_rqs; + const struct libeth_sq_stats **live_sqs; + const struct libeth_xdpsq_stats **live_xdpsqs; + + /* Driver's private data, ____cacheline_aligned */ +} ____cacheline_aligned; + +/** + * libeth_netdev_priv_assert - assert the layout of driver's netdev priv struct + * @t: typeof() of driver's netdev private structure + * @f: name of the embedded &libeth_netdev_priv inside @t + * + * Make sure &libeth_netdev_priv is placed strictly at the beginning of + * driver's private structure, so that libeth can use netdev_priv() to + * access it. + * To be called right after driver's netdev private struct declaration. + */ +#define libeth_netdev_priv_assert(t, f) \ + static_assert(__same_type(struct libeth_netdev_priv, \ + typeof_member(t, f)) && !offsetof(t, f)) + +/* Stats. '[NL]' means it's exported to the Netlink per-queue stats */ + +/* Use 32-byte alignment to reduce false sharing. The first ~4 fields usually + * are the hottest and the stats update helpers are unrolled by this count. + */ +#define __libeth_stats_aligned \ + __aligned(__cmp(min, 4 * sizeof(u64_stats_t), SMP_CACHE_BYTES)) + +/* Align queue stats counters naturally in case they aren't */ +#define __libeth_u64_stats_t \ + u64_stats_t __aligned(sizeof(u64_stats_t)) + +#define ___live(s) __libeth_u64_stats_t s; + +/* Rx per-queue stats: + * + * napi: "hot" counters, updated in bulks from NAPI polling loops: + * bytes: bytes received on this queue [NL] + * packets: packets received on this queue [NL] + * fragments: number of processed descriptors carrying only a fragment + * csum_unnecessary: number of frames the device checked the checksum for [NL] + * hsplit: number of frames the device performed the header split for + * hsplit_linear: number of frames placed entirely to the header buffer + * hw_gro_packets: number of frames the device did HW GRO for [NL] + * hw_gro_bytes: bytes for all HW GROed frames [NL] + * + * fail: "slow"/error counters, incremented by one when occured: + * alloc_fail: number of FQE (Rx buffer) allocation fails [NL] + * dma_errs: number of hardware Rx DMA errors + * csum_none: number of frames the device didn't check the checksum for [NL] + * csum_bad: number of frames with invalid checksum [NL] + * hsplit_errs: number of header split errors (header buffer overflows etc.) + * build_fail: number of napi_build_skb() fails + * + * &libeth_rq_stats must be embedded into the corresponding queue structure. + */ + +#define LIBETH_DECLARE_RQ_NAPI_STATS(act) \ + act(bytes) \ + act(packets) \ + act(fragments) \ + act(csum_unnecessary) \ + act(hsplit) \ + act(hsplit_linear) \ + act(hw_gro_packets) \ + act(hw_gro_bytes) + +#define LIBETH_DECLARE_RQ_FAIL_STATS(act) \ + act(alloc_fail) \ + act(dma_errs) \ + act(csum_none) \ + act(csum_bad) \ + act(hsplit_errs) \ + act(build_fail) + +#define LIBETH_DECLARE_RQ_STATS(act) \ + LIBETH_DECLARE_RQ_NAPI_STATS(act) \ + LIBETH_DECLARE_RQ_FAIL_STATS(act) + +struct libeth_rq_stats { + struct u64_stats_sync syncp; + + union { + struct { + struct_group(napi, + LIBETH_DECLARE_RQ_NAPI_STATS(___live); + ); + LIBETH_DECLARE_RQ_FAIL_STATS(___live); + }; + DECLARE_FLEX_ARRAY(__libeth_u64_stats_t, raw); + }; +} __libeth_stats_aligned; + +/* Tx per-queue stats: + * + * napi: "hot" counters, updated in bulks from NAPI polling loops: + * bytes: bytes sent from this queue [NL] + * packets: packets sent from this queue [NL] + * + * xmit: "hot" counters, updated in bulks from ::ndo_start_xmit(): + * fragments: number of descriptors carrying only a fragment + * csum_none: number of frames sent w/o checksum offload [NL] + * needs_csum: number of frames sent with checksum offload [NL] + * hw_gso_packets: number of frames sent with segmentation offload [NL] + * tso: number of frames sent with TCP segmentation offload + * uso: number of frames sent with UDP L4 segmentation offload + * hw_gso_bytes: total bytes for HW GSOed frames [NL] + * + * fail: "slow"/error counters, incremented by one when occured: + * linearized: number of non-linear skbs linearized due to HW limits + * dma_map_errs: number of DMA mapping errors + * drops: number of skbs dropped by ::ndo_start_xmit() + * busy: number of xmit failures due to the queue being full + * stop: number of times the queue was stopped by the driver [NL] + * wake: number of times the queue was started after being stopped [NL] + * + * &libeth_sq_stats must be embedded into the corresponding queue structure. + */ + +#define LIBETH_DECLARE_SQ_NAPI_STATS(act) \ + act(bytes) \ + act(packets) + +#define LIBETH_DECLARE_SQ_XMIT_STATS(act) \ + act(fragments) \ + act(csum_none) \ + act(needs_csum) \ + act(hw_gso_packets) \ + act(tso) \ + act(uso) \ + act(hw_gso_bytes) + +#define LIBETH_DECLARE_SQ_FAIL_STATS(act) \ + act(linearized) \ + act(dma_map_errs) \ + act(drops) \ + act(busy) \ + act(stop) \ + act(wake) + +#define LIBETH_DECLARE_SQ_STATS(act) \ + LIBETH_DECLARE_SQ_NAPI_STATS(act) \ + LIBETH_DECLARE_SQ_XMIT_STATS(act) \ + LIBETH_DECLARE_SQ_FAIL_STATS(act) + +struct libeth_sq_stats { + struct u64_stats_sync syncp; + + union { + struct { + struct_group(napi, + LIBETH_DECLARE_SQ_NAPI_STATS(___live); + ); + struct_group(xmit, + LIBETH_DECLARE_SQ_XMIT_STATS(___live); + ); + LIBETH_DECLARE_SQ_FAIL_STATS(___live); + }; + DECLARE_FLEX_ARRAY(__libeth_u64_stats_t, raw); + }; +} __libeth_stats_aligned; + +/* XDP Tx per-queue stats: + * + * napi: "hot" counters, updated in bulks from NAPI polling loops: + * bytes: bytes sent from this queue + * packets: packets sent from this queue + * fragments: number of descriptors carrying only a fragment + * + * fail: "slow"/error counters, incremented by one when occured: + * dma_map_errs: number of DMA mapping errors + * drops: number of frags dropped due to the queue being full + * busy: number of xmit failures due to the queue being full + * + * &libeth_xdpsq_stats must be embedded into the corresponding queue structure. + */ + +#define LIBETH_DECLARE_XDPSQ_NAPI_STATS(act) \ + LIBETH_DECLARE_SQ_NAPI_STATS(act) \ + act(fragments) + +#define LIBETH_DECLARE_XDPSQ_FAIL_STATS(act) \ + act(dma_map_errs) \ + act(drops) \ + act(busy) + +#define LIBETH_DECLARE_XDPSQ_STATS(act) \ + LIBETH_DECLARE_XDPSQ_NAPI_STATS(act) \ + LIBETH_DECLARE_XDPSQ_FAIL_STATS(act) + +struct libeth_xdpsq_stats { + struct u64_stats_sync syncp; + + union { + struct { + struct_group(napi, + LIBETH_DECLARE_XDPSQ_NAPI_STATS(___live); + ); + LIBETH_DECLARE_XDPSQ_FAIL_STATS(___live); + }; + DECLARE_FLEX_ARRAY(__libeth_u64_stats_t, raw); + }; +} __libeth_stats_aligned; + +#undef ___live + +#endif /* __LIBETH_TYPES_H */ From 7c49fd3da4e97686c3fd4e31b947910a0386fc0c Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Wed, 13 Dec 2023 12:36:08 +0100 Subject: [PATCH 12/49] libie: add Tx buffer completion helpers Software-side Tx buffers for storing DMA, frame size, skb pointers etc. are pretty much generic and every driver defines them the same way. The same can be said for software Tx completions -- same napi_consume_skb()s and all that... Add a couple simple wrappers for doing that to stop repeating the old tale at least within the Intel code. Drivers are free to use 'priv' member at the end of the structure. Signed-off-by: Alexander Lobakin --- include/net/libeth/tx.h | 127 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 include/net/libeth/tx.h diff --git a/include/net/libeth/tx.h b/include/net/libeth/tx.h new file mode 100644 index 00000000000000..f868309dcbda6c --- /dev/null +++ b/include/net/libeth/tx.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2024 Intel Corporation */ + +#ifndef __LIBETH_TX_H +#define __LIBETH_TX_H + +#include + +/* Tx buffer completion */ + +/** + * enum libeth_sqe_type - type of &libeth_sqe to act on Tx completion + * @LIBETH_SQE_EMPTY: unused/empty, no action required + * @LIBETH_SQE_CTX: context descriptor with empty SQE, no action required + * @LIBETH_SQE_SLAB: kmalloc-allocated buffer, unmap and kfree() + * @LIBETH_SQE_FRAG: mapped skb frag, only unmap DMA + * @LIBETH_SQE_SKB: &sk_buff, unmap and napi_consume_skb(), update stats + */ +enum libeth_sqe_type { + LIBETH_SQE_EMPTY = 0U, + LIBETH_SQE_CTX, + LIBETH_SQE_SLAB, + LIBETH_SQE_FRAG, + LIBETH_SQE_SKB, +}; + +/** + * struct libeth_sqe - represents a Send Queue Element / Tx buffer + * @type: type of the buffer, see the enum above + * @rs_idx: index of the last buffer from the batch this one was sent + * @raw: slab buffer to free via kfree() + * @skb: &sk_buff to consume + * @dma: DMA address to unmap + * @len: length of the mapped region to unmap + * @nr_frags: number of frags in the frame this buffer belongs to + * @packets: number of physical packets sent for this frame + * @bytes: number of physical bytes sent for this frame + * @priv: driver-private scratchpad + */ +struct libeth_sqe { + enum libeth_sqe_type type:32; + u32 rs_idx; + + union { + void *raw; + struct sk_buff *skb; + }; + + DEFINE_DMA_UNMAP_ADDR(dma); + DEFINE_DMA_UNMAP_LEN(len); + + u32 nr_frags; + u32 packets; + u32 bytes; + + unsigned long priv; +} __aligned_largest; + +/** + * LIBETH_SQE_CHECK_PRIV - check the driver's private SQE data + * @p: type or name of the object the driver wants to fit into &libeth_sqe + * + * Make sure the driver's private data fits into libeth_sqe::priv. To be used + * right after its declaration. + */ +#define LIBETH_SQE_CHECK_PRIV(p) \ + static_assert(sizeof(p) <= sizeof_field(struct libeth_sqe, priv)) + +/** + * struct libeth_cq_pp - completion queue poll params + * @dev: &device to perform DMA unmapping + * @ss: onstack NAPI stats to fill + * @napi: whether it's called from the NAPI context + * + * libeth uses this structure to access objects needed for performing full + * Tx complete operation without passing lots of arguments and change the + * prototypes each time a new one is added. + */ +struct libeth_cq_pp { + struct device *dev; + struct libeth_sq_napi_stats *ss; + + bool napi; +}; + +/** + * libeth_tx_complete - perform Tx completion for one SQE + * @sqe: SQE to complete + * @cp: poll params + * + * Do Tx complete for all the types of buffers, incl. freeing, unmapping, + * updating the stats etc. + */ +static inline void libeth_tx_complete(struct libeth_sqe *sqe, + const struct libeth_cq_pp *cp) +{ + switch (sqe->type) { + case LIBETH_SQE_EMPTY: + return; + case LIBETH_SQE_SKB: + case LIBETH_SQE_FRAG: + case LIBETH_SQE_SLAB: + dma_unmap_page(cp->dev, dma_unmap_addr(sqe, dma), + dma_unmap_len(sqe, len), DMA_TO_DEVICE); + break; + default: + break; + } + + switch (sqe->type) { + case LIBETH_SQE_SKB: + cp->ss->packets += sqe->packets; + cp->ss->bytes += sqe->bytes; + + napi_consume_skb(sqe->skb, cp->napi); + break; + case LIBETH_SQE_SLAB: + kfree(sqe->raw); + break; + default: + break; + } + + sqe->type = LIBETH_SQE_EMPTY; +} + +#endif /* __LIBETH_TX_H */ From c385b1f768ab90b7f0773df2a181f59b3526a59c Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Wed, 13 Dec 2023 13:39:50 +0100 Subject: [PATCH 13/49] idpf: convert to libie Tx buffer completion &idpf_tx_buffer is almost identical to the previous generations, as well as the way it's handled. Moreover, relying on dma_unmap_addr() and !!buf->skb instead of explicit defining of buffer's type was never good. Use the newly added libie helpers to do it properly and reduce the copy-paste around the Tx code. Signed-off-by: Alexander Lobakin --- .../ethernet/intel/idpf/idpf_singleq_txrx.c | 82 +++----- drivers/net/ethernet/intel/idpf/idpf_txrx.c | 195 ++++++------------ drivers/net/ethernet/intel/idpf/idpf_txrx.h | 50 +---- 3 files changed, 101 insertions(+), 226 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c index fe64febf7436f4..98f26a4b835f47 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c @@ -2,6 +2,7 @@ /* Copyright (C) 2023 Intel Corporation */ #include +#include #include "idpf.h" @@ -224,6 +225,7 @@ static void idpf_tx_singleq_map(struct idpf_tx_queue *tx_q, /* record length, and DMA address */ dma_unmap_len_set(tx_buf, len, size); dma_unmap_addr_set(tx_buf, dma, dma); + tx_buf->type = LIBETH_SQE_FRAG; /* align size to end of page */ max_data += -dma & (IDPF_TX_MAX_READ_REQ_SIZE - 1); @@ -245,6 +247,8 @@ static void idpf_tx_singleq_map(struct idpf_tx_queue *tx_q, i = 0; } + tx_q->tx_buf[i].type = LIBETH_SQE_EMPTY; + dma += max_data; size -= max_data; @@ -282,13 +286,13 @@ static void idpf_tx_singleq_map(struct idpf_tx_queue *tx_q, tx_desc->qw1 = idpf_tx_singleq_build_ctob(td_cmd, offsets, size, td_tag); - IDPF_SINGLEQ_BUMP_RING_IDX(tx_q, i); + first->type = LIBETH_SQE_SKB; + first->rs_idx = i; - /* set next_to_watch value indicating a packet is present */ - first->next_to_watch = tx_desc; + IDPF_SINGLEQ_BUMP_RING_IDX(tx_q, i); nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); - netdev_tx_sent_queue(nq, first->bytecount); + netdev_tx_sent_queue(nq, first->bytes); idpf_tx_buf_hw_update(tx_q, i, netdev_xmit_more()); } @@ -306,8 +310,7 @@ idpf_tx_singleq_get_ctx_desc(struct idpf_tx_queue *txq) struct idpf_base_tx_ctx_desc *ctx_desc; int ntu = txq->next_to_use; - memset(&txq->tx_buf[ntu], 0, sizeof(struct idpf_tx_buf)); - txq->tx_buf[ntu].ctx_entry = true; + txq->tx_buf[ntu].type = LIBETH_SQE_CTX; ctx_desc = &txq->base_ctx[ntu]; @@ -396,11 +399,11 @@ netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, first->skb = skb; if (tso) { - first->gso_segs = offload.tso_segs; - first->bytecount = skb->len + ((first->gso_segs - 1) * offload.tso_hdr_len); + first->packets = offload.tso_segs; + first->bytes = skb->len + ((first->packets - 1) * offload.tso_hdr_len); } else { - first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); - first->gso_segs = 1; + first->bytes = max_t(unsigned int, skb->len, ETH_ZLEN); + first->packets = 1; } idpf_tx_singleq_map(tx_q, first, &offload); @@ -420,10 +423,15 @@ netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, static bool idpf_tx_singleq_clean(struct idpf_tx_queue *tx_q, int napi_budget, int *cleaned) { - unsigned int total_bytes = 0, total_pkts = 0; + struct libeth_sq_napi_stats ss = { }; struct idpf_base_tx_desc *tx_desc; u32 budget = tx_q->clean_budget; s16 ntc = tx_q->next_to_clean; + struct libeth_cq_pp cp = { + .dev = tx_q->dev, + .ss = &ss, + .napi = napi_budget, + }; struct idpf_netdev_priv *np; struct idpf_tx_buf *tx_buf; struct netdev_queue *nq; @@ -441,47 +449,23 @@ static bool idpf_tx_singleq_clean(struct idpf_tx_queue *tx_q, int napi_budget, * such. We can skip this descriptor since there is no buffer * to clean. */ - if (tx_buf->ctx_entry) { - /* Clear this flag here to avoid stale flag values when - * this buffer is used for actual data in the future. - * There are cases where the tx_buf struct / the flags - * field will not be cleared before being reused. - */ - tx_buf->ctx_entry = false; + if (unlikely(tx_buf->type <= LIBETH_SQE_CTX)) { + tx_buf->type = LIBETH_SQE_EMPTY; goto fetch_next_txq_desc; } - /* if next_to_watch is not set then no work pending */ - eop_desc = (struct idpf_base_tx_desc *)tx_buf->next_to_watch; - if (!eop_desc) - break; - - /* prevent any other reads prior to eop_desc */ + /* prevent any other reads prior to type */ smp_rmb(); + eop_desc = &tx_q->base_tx[tx_buf->rs_idx]; + /* if the descriptor isn't done, no work yet to do */ if (!(eop_desc->qw1 & cpu_to_le64(IDPF_TX_DESC_DTYPE_DESC_DONE))) break; - /* clear next_to_watch to prevent false hangs */ - tx_buf->next_to_watch = NULL; - /* update the statistics for this packet */ - total_bytes += tx_buf->bytecount; - total_pkts += tx_buf->gso_segs; - - napi_consume_skb(tx_buf->skb, napi_budget); - - /* unmap skb header data */ - dma_unmap_single(tx_q->dev, - dma_unmap_addr(tx_buf, dma), - dma_unmap_len(tx_buf, len), - DMA_TO_DEVICE); - - /* clear tx_buf data */ - tx_buf->skb = NULL; - dma_unmap_len_set(tx_buf, len, 0); + libeth_tx_complete(tx_buf, &cp); /* unmap remaining buffers */ while (tx_desc != eop_desc) { @@ -495,13 +479,7 @@ static bool idpf_tx_singleq_clean(struct idpf_tx_queue *tx_q, int napi_budget, } /* unmap any remaining paged data */ - if (dma_unmap_len(tx_buf, len)) { - dma_unmap_page(tx_q->dev, - dma_unmap_addr(tx_buf, dma), - dma_unmap_len(tx_buf, len), - DMA_TO_DEVICE); - dma_unmap_len_set(tx_buf, len, 0); - } + libeth_tx_complete(tx_buf, &cp); } /* update budget only if we did something */ @@ -521,11 +499,11 @@ static bool idpf_tx_singleq_clean(struct idpf_tx_queue *tx_q, int napi_budget, ntc += tx_q->desc_count; tx_q->next_to_clean = ntc; - *cleaned += total_pkts; + *cleaned += ss.packets; u64_stats_update_begin(&tx_q->stats_sync); - u64_stats_add(&tx_q->q_stats.packets, total_pkts); - u64_stats_add(&tx_q->q_stats.bytes, total_bytes); + u64_stats_add(&tx_q->q_stats.packets, ss.packets); + u64_stats_add(&tx_q->q_stats.bytes, ss.bytes); u64_stats_update_end(&tx_q->stats_sync); np = netdev_priv(tx_q->netdev); @@ -533,7 +511,7 @@ static bool idpf_tx_singleq_clean(struct idpf_tx_queue *tx_q, int napi_budget, dont_wake = np->state != __IDPF_VPORT_UP || !netif_carrier_ok(tx_q->netdev); - __netif_txq_completed_wake(nq, total_pkts, total_bytes, + __netif_txq_completed_wake(nq, ss.packets, ss.bytes, IDPF_DESC_UNUSED(tx_q), IDPF_TX_WAKE_THRESH, dont_wake); diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index a66a42f9355c65..0c24684b7883e5 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -2,10 +2,19 @@ /* Copyright (C) 2023 Intel Corporation */ #include +#include #include "idpf.h" #include "idpf_virtchnl.h" +struct idpf_tx_stash { + struct hlist_node hlist; + struct libeth_sqe buf; +}; + +#define idpf_tx_buf_compl_tag(buf) (*(int *)&(buf)->priv) +LIBETH_SQE_CHECK_PRIV(int); + static bool idpf_chk_linearize(struct sk_buff *skb, unsigned int max_bufs, unsigned int count); @@ -60,41 +69,18 @@ void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue) } } -/** - * idpf_tx_buf_rel - Release a Tx buffer - * @tx_q: the queue that owns the buffer - * @tx_buf: the buffer to free - */ -static void idpf_tx_buf_rel(struct idpf_tx_queue *tx_q, - struct idpf_tx_buf *tx_buf) -{ - if (tx_buf->skb) { - if (dma_unmap_len(tx_buf, len)) - dma_unmap_single(tx_q->dev, - dma_unmap_addr(tx_buf, dma), - dma_unmap_len(tx_buf, len), - DMA_TO_DEVICE); - dev_kfree_skb_any(tx_buf->skb); - } else if (dma_unmap_len(tx_buf, len)) { - dma_unmap_page(tx_q->dev, - dma_unmap_addr(tx_buf, dma), - dma_unmap_len(tx_buf, len), - DMA_TO_DEVICE); - } - - tx_buf->next_to_watch = NULL; - tx_buf->skb = NULL; - tx_buf->compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG; - dma_unmap_len_set(tx_buf, len, 0); -} - /** * idpf_tx_buf_rel_all - Free any empty Tx buffers * @txq: queue to be cleaned */ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) { + struct libeth_sq_napi_stats ss = { }; struct idpf_buf_lifo *buf_stack; + struct libeth_cq_pp cp = { + .dev = txq->dev, + .ss = &ss, + }; u16 i; /* Buffers already cleared, nothing to do */ @@ -103,7 +89,7 @@ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) /* Free all the Tx buffer sk_buffs */ for (i = 0; i < txq->desc_count; i++) - idpf_tx_buf_rel(txq, &txq->tx_buf[i]); + libeth_tx_complete(&txq->tx_buf[i], &cp); kfree(txq->tx_buf); txq->tx_buf = NULL; @@ -203,10 +189,6 @@ static int idpf_tx_buf_alloc_all(struct idpf_tx_queue *tx_q) if (!tx_q->tx_buf) return -ENOMEM; - /* Initialize tx_bufs with invalid completion tags */ - for (i = 0; i < tx_q->desc_count; i++) - tx_q->tx_buf[i].compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG; - if (!idpf_queue_has(FLOW_SCH_EN, tx_q)) return 0; @@ -1655,37 +1637,6 @@ static void idpf_tx_handle_sw_marker(struct idpf_tx_queue *tx_q) wake_up(&vport->sw_marker_wq); } -/** - * idpf_tx_splitq_clean_hdr - Clean TX buffer resources for header portion of - * packet - * @tx_q: tx queue to clean buffer from - * @tx_buf: buffer to be cleaned - * @cleaned: pointer to stats struct to track cleaned packets/bytes - * @napi_budget: Used to determine if we are in netpoll - */ -static void idpf_tx_splitq_clean_hdr(struct idpf_tx_queue *tx_q, - struct idpf_tx_buf *tx_buf, - struct idpf_cleaned_stats *cleaned, - int napi_budget) -{ - napi_consume_skb(tx_buf->skb, napi_budget); - - if (dma_unmap_len(tx_buf, len)) { - dma_unmap_single(tx_q->dev, - dma_unmap_addr(tx_buf, dma), - dma_unmap_len(tx_buf, len), - DMA_TO_DEVICE); - - dma_unmap_len_set(tx_buf, len, 0); - } - - /* clear tx_buf data */ - tx_buf->skb = NULL; - - cleaned->bytes += tx_buf->bytecount; - cleaned->packets += tx_buf->gso_segs; -} - /** * idpf_tx_clean_stashed_bufs - clean bufs that were stored for * out of order completions @@ -1701,23 +1652,20 @@ static void idpf_tx_clean_stashed_bufs(struct idpf_tx_queue *txq, { struct idpf_tx_stash *stash; struct hlist_node *tmp_buf; + struct libeth_cq_pp cp = { + .dev = txq->dev, + .ss = cleaned, + .napi = budget, + }; /* Buffer completion */ hash_for_each_possible_safe(txq->stash->sched_buf_hash, stash, tmp_buf, hlist, compl_tag) { - if (unlikely(stash->buf.compl_tag != (int)compl_tag)) + if (unlikely(idpf_tx_buf_compl_tag(&stash->buf) != + (int)compl_tag)) continue; - if (stash->buf.skb) { - idpf_tx_splitq_clean_hdr(txq, &stash->buf, cleaned, - budget); - } else if (dma_unmap_len(&stash->buf, len)) { - dma_unmap_page(txq->dev, - dma_unmap_addr(&stash->buf, dma), - dma_unmap_len(&stash->buf, len), - DMA_TO_DEVICE); - dma_unmap_len_set(&stash->buf, len, 0); - } + libeth_tx_complete(&stash->buf, &cp); /* Push shadow buf back onto stack */ idpf_buf_lifo_push(&txq->stash->buf_stack, stash); @@ -1737,8 +1685,7 @@ static int idpf_stash_flow_sch_buffers(struct idpf_tx_queue *txq, { struct idpf_tx_stash *stash; - if (unlikely(!dma_unmap_addr(tx_buf, dma) && - !dma_unmap_len(tx_buf, len))) + if (unlikely(tx_buf->type <= LIBETH_SQE_CTX)) return 0; stash = idpf_buf_lifo_pop(&txq->stash->buf_stack); @@ -1751,20 +1698,18 @@ static int idpf_stash_flow_sch_buffers(struct idpf_tx_queue *txq, /* Store buffer params in shadow buffer */ stash->buf.skb = tx_buf->skb; - stash->buf.bytecount = tx_buf->bytecount; - stash->buf.gso_segs = tx_buf->gso_segs; + stash->buf.bytes = tx_buf->bytes; + stash->buf.packets = tx_buf->packets; + stash->buf.type = tx_buf->type; dma_unmap_addr_set(&stash->buf, dma, dma_unmap_addr(tx_buf, dma)); dma_unmap_len_set(&stash->buf, len, dma_unmap_len(tx_buf, len)); - stash->buf.compl_tag = tx_buf->compl_tag; + idpf_tx_buf_compl_tag(&stash->buf) = idpf_tx_buf_compl_tag(tx_buf); /* Add buffer to buf_hash table to be freed later */ hash_add(txq->stash->sched_buf_hash, &stash->hlist, - stash->buf.compl_tag); - - memset(tx_buf, 0, sizeof(struct idpf_tx_buf)); + idpf_tx_buf_compl_tag(&stash->buf)); - /* Reinitialize buf_id portion of tag */ - tx_buf->compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG; + tx_buf->type = LIBETH_SQE_EMPTY; return 0; } @@ -1806,6 +1751,11 @@ static void idpf_tx_splitq_clean(struct idpf_tx_queue *tx_q, u16 end, union idpf_tx_flex_desc *next_pending_desc = NULL; union idpf_tx_flex_desc *tx_desc; s16 ntc = tx_q->next_to_clean; + struct libeth_cq_pp cp = { + .dev = tx_q->dev, + .ss = cleaned, + .napi = napi_budget, + }; struct idpf_tx_buf *tx_buf; tx_desc = &tx_q->flex_tx[ntc]; @@ -1821,13 +1771,10 @@ static void idpf_tx_splitq_clean(struct idpf_tx_queue *tx_q, u16 end, * invalid completion tag since no buffer was used. We can * skip this descriptor since there is no buffer to clean. */ - if (unlikely(tx_buf->compl_tag == IDPF_SPLITQ_TX_INVAL_COMPL_TAG)) + if (tx_buf->type <= LIBETH_SQE_CTX) goto fetch_next_txq_desc; - eop_desc = (union idpf_tx_flex_desc *)tx_buf->next_to_watch; - - /* clear next_to_watch to prevent false hangs */ - tx_buf->next_to_watch = NULL; + eop_desc = &tx_q->flex_tx[tx_buf->rs_idx]; if (descs_only) { if (idpf_stash_flow_sch_buffers(tx_q, tx_buf)) @@ -1844,8 +1791,7 @@ static void idpf_tx_splitq_clean(struct idpf_tx_queue *tx_q, u16 end, } } } else { - idpf_tx_splitq_clean_hdr(tx_q, tx_buf, cleaned, - napi_budget); + libeth_tx_complete(tx_buf, &cp); /* unmap remaining buffers */ while (tx_desc != eop_desc) { @@ -1853,13 +1799,7 @@ static void idpf_tx_splitq_clean(struct idpf_tx_queue *tx_q, u16 end, tx_desc, tx_buf); /* unmap any remaining paged data */ - if (dma_unmap_len(tx_buf, len)) { - dma_unmap_page(tx_q->dev, - dma_unmap_addr(tx_buf, dma), - dma_unmap_len(tx_buf, len), - DMA_TO_DEVICE); - dma_unmap_len_set(tx_buf, len, 0); - } + libeth_tx_complete(tx_buf, &cp); } } @@ -1901,24 +1841,20 @@ static bool idpf_tx_clean_buf_ring(struct idpf_tx_queue *txq, u16 compl_tag, u16 idx = compl_tag & txq->compl_tag_bufid_m; struct idpf_tx_buf *tx_buf = NULL; u16 ntc = txq->next_to_clean; + struct libeth_cq_pp cp = { + .dev = txq->dev, + .ss = cleaned, + .napi = budget, + }; u16 num_descs_cleaned = 0; u16 orig_idx = idx; tx_buf = &txq->tx_buf[idx]; + if (unlikely(tx_buf->type <= LIBETH_SQE_CTX)) + return false; - while (tx_buf->compl_tag == (int)compl_tag) { - if (tx_buf->skb) { - idpf_tx_splitq_clean_hdr(txq, tx_buf, cleaned, budget); - } else if (dma_unmap_len(tx_buf, len)) { - dma_unmap_page(txq->dev, - dma_unmap_addr(tx_buf, dma), - dma_unmap_len(tx_buf, len), - DMA_TO_DEVICE); - dma_unmap_len_set(tx_buf, len, 0); - } - - memset(tx_buf, 0, sizeof(struct idpf_tx_buf)); - tx_buf->compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG; + while (idpf_tx_buf_compl_tag(tx_buf) == (int)compl_tag) { + libeth_tx_complete(tx_buf, &cp); num_descs_cleaned++; idpf_tx_clean_buf_ring_bump_ntc(txq, idx, tx_buf); @@ -2307,6 +2243,12 @@ unsigned int idpf_tx_desc_count_required(struct idpf_tx_queue *txq, void idpf_tx_dma_map_error(struct idpf_tx_queue *txq, struct sk_buff *skb, struct idpf_tx_buf *first, u16 idx) { + struct libeth_sq_napi_stats ss = { }; + struct libeth_cq_pp cp = { + .dev = txq->dev, + .ss = &ss, + }; + u64_stats_update_begin(&txq->stats_sync); u64_stats_inc(&txq->q_stats.dma_map_errs); u64_stats_update_end(&txq->stats_sync); @@ -2316,7 +2258,7 @@ void idpf_tx_dma_map_error(struct idpf_tx_queue *txq, struct sk_buff *skb, struct idpf_tx_buf *tx_buf; tx_buf = &txq->tx_buf[idx]; - idpf_tx_buf_rel(txq, tx_buf); + libeth_tx_complete(tx_buf, &cp); if (tx_buf == first) break; if (idx == 0) @@ -2405,7 +2347,8 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, if (dma_mapping_error(tx_q->dev, dma)) return idpf_tx_dma_map_error(tx_q, skb, first, i); - tx_buf->compl_tag = params->compl_tag; + idpf_tx_buf_compl_tag(tx_buf) = params->compl_tag; + tx_buf->type = LIBETH_SQE_FRAG; /* record length, and DMA address */ dma_unmap_len_set(tx_buf, len, size); @@ -2479,8 +2422,7 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, * simply pass over these holes and finish cleaning the * rest of the packet. */ - memset(&tx_q->tx_buf[i], 0, sizeof(struct idpf_tx_buf)); - tx_q->tx_buf[i].compl_tag = params->compl_tag; + tx_q->tx_buf[i].type = LIBETH_SQE_EMPTY; /* Adjust the DMA offset and the remaining size of the * fragment. On the first iteration of this loop, @@ -2525,19 +2467,19 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, /* record SW timestamp if HW timestamp is not available */ skb_tx_timestamp(skb); + first->type = LIBETH_SQE_SKB; + /* write last descriptor with RS and EOP bits */ + first->rs_idx = i; td_cmd |= params->eop_cmd; idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, size); i = idpf_tx_splitq_bump_ntu(tx_q, i); - /* set next_to_watch value indicating a packet is present */ - first->next_to_watch = tx_desc; - tx_q->txq_grp->num_completions_pending++; /* record bytecount for BQL */ nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); - netdev_tx_sent_queue(nq, first->bytecount); + netdev_tx_sent_queue(nq, first->bytes); idpf_tx_buf_hw_update(tx_q, i, netdev_xmit_more()); } @@ -2737,8 +2679,7 @@ idpf_tx_splitq_get_ctx_desc(struct idpf_tx_queue *txq) struct idpf_flex_tx_ctx_desc *desc; int i = txq->next_to_use; - memset(&txq->tx_buf[i], 0, sizeof(struct idpf_tx_buf)); - txq->tx_buf[i].compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG; + txq->tx_buf[i].type = LIBETH_SQE_CTX; /* grab the next descriptor */ desc = &txq->flex_ctx[i]; @@ -2822,12 +2763,12 @@ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, first->skb = skb; if (tso) { - first->gso_segs = tx_params.offload.tso_segs; - first->bytecount = skb->len + - ((first->gso_segs - 1) * tx_params.offload.tso_hdr_len); + first->packets = tx_params.offload.tso_segs; + first->bytes = skb->len + + ((first->packets - 1) * tx_params.offload.tso_hdr_len); } else { - first->gso_segs = 1; - first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); + first->packets = 1; + first->bytes = max_t(unsigned int, skb->len, ETH_ZLEN); } if (idpf_queue_has(FLOW_SCH_EN, tx_q)) { diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index 6215dbee554651..fa87754c73402d 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -131,7 +131,6 @@ do { \ (txq)->num_completions_pending - (txq)->complq->num_completions) #define IDPF_TX_SPLITQ_COMPL_TAG_WIDTH 16 -#define IDPF_SPLITQ_TX_INVAL_COMPL_TAG -1 /* Adjust the generation for the completion tag and wrap if necessary */ #define IDPF_TX_ADJ_COMPL_TAG_GEN(txq) \ ((++(txq)->compl_tag_cur_gen) >= (txq)->compl_tag_gen_max ? \ @@ -149,47 +148,7 @@ union idpf_tx_flex_desc { struct idpf_flex_tx_sched_desc flow; /* flow based scheduling */ }; -/** - * struct idpf_tx_buf - * @next_to_watch: Next descriptor to clean - * @skb: Pointer to the skb - * @dma: DMA address - * @len: DMA length - * @bytecount: Number of bytes - * @gso_segs: Number of GSO segments - * @compl_tag: Splitq only, unique identifier for a buffer. Used to compare - * with completion tag returned in buffer completion event. - * Because the completion tag is expected to be the same in all - * data descriptors for a given packet, and a single packet can - * span multiple buffers, we need this field to track all - * buffers associated with this completion tag independently of - * the buf_id. The tag consists of a N bit buf_id and M upper - * order "generation bits". See compl_tag_bufid_m and - * compl_tag_gen_s in struct idpf_queue. We'll use a value of -1 - * to indicate the tag is not valid. - * @ctx_entry: Singleq only. Used to indicate the corresponding entry - * in the descriptor ring was used for a context descriptor and - * this buffer entry should be skipped. - */ -struct idpf_tx_buf { - void *next_to_watch; - struct sk_buff *skb; - DEFINE_DMA_UNMAP_ADDR(dma); - DEFINE_DMA_UNMAP_LEN(len); - unsigned int bytecount; - unsigned short gso_segs; - - union { - int compl_tag; - - bool ctx_entry; - }; -}; - -struct idpf_tx_stash { - struct hlist_node hlist; - struct idpf_tx_buf buf; -}; +#define idpf_tx_buf libeth_sqe /** * struct idpf_buf_lifo - LIFO for managing OOO completions @@ -496,10 +455,7 @@ struct idpf_tx_queue_stats { u64_stats_t dma_map_errs; }; -struct idpf_cleaned_stats { - u32 packets; - u32 bytes; -}; +#define idpf_cleaned_stats libeth_sq_napi_stats #define IDPF_ITR_DYNAMIC 1 #define IDPF_ITR_MAX 0x1FE0 @@ -688,7 +644,7 @@ struct idpf_tx_queue { void *desc_ring; }; - struct idpf_tx_buf *tx_buf; + struct libeth_sqe *tx_buf; struct idpf_txq_group *txq_grp; struct device *dev; void __iomem *tail; From 9194f241ddb3596a68af11fdaf210138fb4898a2 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Mon, 15 Apr 2024 17:49:43 +0200 Subject: [PATCH 14/49] netdevice: add netdev_tx_reset_subqueue() shorthand Add a shorthand similar to other net*_subqueue() helpers for resetting the queue by its index w/o obtaining &netdev_tx_queue beforehand manually. Signed-off-by: Alexander Lobakin --- include/linux/netdevice.h | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 5d559c4cb0374c..301e6afd07803a 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3594,6 +3594,17 @@ static inline void netdev_tx_reset_queue(struct netdev_queue *q) #endif } +/** + * netdev_tx_reset_subqueue - reset the BQL stats and state of a netdev queue + * @dev: network device + * @qid: stack index of the queue to reset + */ +static inline void netdev_tx_reset_subqueue(const struct net_device *dev, + u32 qid) +{ + netdev_tx_reset_queue(netdev_get_tx_queue(dev, qid)); +} + /** * netdev_reset_queue - reset the packets and bytes count of a network device * @dev_queue: network device @@ -3603,7 +3614,7 @@ static inline void netdev_tx_reset_queue(struct netdev_queue *q) */ static inline void netdev_reset_queue(struct net_device *dev_queue) { - netdev_tx_reset_queue(netdev_get_tx_queue(dev_queue, 0)); + netdev_tx_reset_subqueue(dev_queue, 0); } /** From d0bd302ffa466c5f27d82f9727a68b593ae9bf8b Mon Sep 17 00:00:00 2001 From: Joshua Hay Date: Wed, 20 Mar 2024 15:19:31 +0100 Subject: [PATCH 15/49] idpf: refactor Tx completion routines This patch adds a mechanism to guard against stashing partial packets into the hash table. This makes the driver more robust, leads to more efficient decision making when cleaning. Doon't stash partial packets. This can happen when an RE completion is received in flow scheduling mode, or when an out of order RS completion is received. The first buffer with the skb is stashed, but some or all of its frags are not because the stack is out of reserve buffers. This leaves the ring in a weird state since the frags are still on the ring. Use the field to track the number of fragments/ tx_bufs representing the packet. The clean routines check to make sure there are enough reserve buffers on the stack before stashing any part of the packet. If there are not, next_to_clean is left pointing to the first buffer of the packet that failed to be stashed. This leaves the whole packet on the ring, and the next time around, cleaning will start from this packet. An RS completion is still expected for this packet in either case. So instead of being cleaned from the hash table, it will be cleaned from the ring directly. This should all still be fine since the DESC_UNUSED and BUFS_UNUSED will reflect the state of the ring. If we ever fall below the thresholds, the TXQ will still be stopped, giving the completion queue time to catch up. This may lead to stopping the queue more frequently, but it guarantees the TX ring will always be in a good state. Also, always use the idpf_tx_splitq_clean function to clean descriptors, i.e. use it from clean_buf_ring as well. This way we avoid duplicating the logic and make sure we're using the same reserve buffers guard rail. This does require a switch from the s16 next_to_clean overflow descriptor ring wrap calculation to u16 and the normal ring size check. Signed-off-by: Joshua Hay Signed-off-by: Alexander Lobakin --- .../ethernet/intel/idpf/idpf_singleq_txrx.c | 24 +-- drivers/net/ethernet/intel/idpf/idpf_txrx.c | 163 +++++++++++------- drivers/net/ethernet/intel/idpf/idpf_txrx.h | 6 +- 3 files changed, 117 insertions(+), 76 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c index 98f26a4b835f47..947d3ff9677c1e 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c @@ -239,15 +239,16 @@ static void idpf_tx_singleq_map(struct idpf_tx_queue *tx_q, offsets, max_data, td_tag); - tx_desc++; - i++; - - if (i == tx_q->desc_count) { + if (unlikely(++i == tx_q->desc_count)) { + tx_buf = &tx_q->tx_buf[0]; tx_desc = &tx_q->base_tx[0]; i = 0; + } else { + tx_buf++; + tx_desc++; } - tx_q->tx_buf[i].type = LIBETH_SQE_EMPTY; + tx_buf->type = LIBETH_SQE_EMPTY; dma += max_data; size -= max_data; @@ -261,12 +262,14 @@ static void idpf_tx_singleq_map(struct idpf_tx_queue *tx_q, tx_desc->qw1 = idpf_tx_singleq_build_ctob(td_cmd, offsets, size, td_tag); - tx_desc++; - i++; - if (i == tx_q->desc_count) { + if (unlikely(++i == tx_q->desc_count)) { + tx_buf = &tx_q->tx_buf[0]; tx_desc = &tx_q->base_tx[0]; i = 0; + } else { + tx_buf++; + tx_desc++; } size = skb_frag_size(frag); @@ -274,8 +277,6 @@ static void idpf_tx_singleq_map(struct idpf_tx_queue *tx_q, dma = skb_frag_dma_map(tx_q->dev, frag, 0, size, DMA_TO_DEVICE); - - tx_buf = &tx_q->tx_buf[i]; } skb_tx_timestamp(first->skb); @@ -454,6 +455,9 @@ static bool idpf_tx_singleq_clean(struct idpf_tx_queue *tx_q, int napi_budget, goto fetch_next_txq_desc; } + if (unlikely(tx_buf->type != LIBETH_SQE_SKB)) + break; + /* prevent any other reads prior to type */ smp_rmb(); diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 0c24684b7883e5..49f9f07ce3e9d4 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -12,8 +12,8 @@ struct idpf_tx_stash { struct libeth_sqe buf; }; -#define idpf_tx_buf_compl_tag(buf) (*(int *)&(buf)->priv) -LIBETH_SQE_CHECK_PRIV(int); +#define idpf_tx_buf_compl_tag(buf) (*(u32 *)&(buf)->priv) +LIBETH_SQE_CHECK_PRIV(u32); static bool idpf_chk_linearize(struct sk_buff *skb, unsigned int max_bufs, unsigned int count); @@ -77,11 +77,13 @@ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) { struct libeth_sq_napi_stats ss = { }; struct idpf_buf_lifo *buf_stack; + struct idpf_tx_stash *stash; struct libeth_cq_pp cp = { .dev = txq->dev, .ss = &ss, }; - u16 i; + struct hlist_node *tmp; + u16 i, tag; /* Buffers already cleared, nothing to do */ if (!txq->tx_buf) @@ -101,6 +103,19 @@ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) if (!buf_stack->bufs) return; + /* If a TX timeout occurred, there are potentially still bufs in the + * hash table, free them here. + */ + hash_for_each_safe(txq->stash->sched_buf_hash, tag, tmp, stash, + hlist) { + if (!stash) + continue; + + libeth_tx_complete(&stash->buf, &cp); + hash_del(&stash->hlist); + idpf_buf_lifo_push(buf_stack, stash); + } + for (i = 0; i < buf_stack->size; i++) kfree(buf_stack->bufs[i]); @@ -117,6 +132,7 @@ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) static void idpf_tx_desc_rel(struct idpf_tx_queue *txq) { idpf_tx_buf_rel_all(txq); + netdev_tx_reset_subqueue(txq->netdev, txq->idx); if (!txq->desc_ring) return; @@ -1661,16 +1677,14 @@ static void idpf_tx_clean_stashed_bufs(struct idpf_tx_queue *txq, /* Buffer completion */ hash_for_each_possible_safe(txq->stash->sched_buf_hash, stash, tmp_buf, hlist, compl_tag) { - if (unlikely(idpf_tx_buf_compl_tag(&stash->buf) != - (int)compl_tag)) + if (unlikely(idpf_tx_buf_compl_tag(&stash->buf) != compl_tag)) continue; + hash_del(&stash->hlist); libeth_tx_complete(&stash->buf, &cp); /* Push shadow buf back onto stack */ idpf_buf_lifo_push(&txq->stash->buf_stack, stash); - - hash_del(&stash->hlist); } } @@ -1701,6 +1715,7 @@ static int idpf_stash_flow_sch_buffers(struct idpf_tx_queue *txq, stash->buf.bytes = tx_buf->bytes; stash->buf.packets = tx_buf->packets; stash->buf.type = tx_buf->type; + stash->buf.nr_frags = tx_buf->nr_frags; dma_unmap_addr_set(&stash->buf, dma, dma_unmap_addr(tx_buf, dma)); dma_unmap_len_set(&stash->buf, len, dma_unmap_len(tx_buf, len)); idpf_tx_buf_compl_tag(&stash->buf) = idpf_tx_buf_compl_tag(tx_buf); @@ -1716,9 +1731,8 @@ static int idpf_stash_flow_sch_buffers(struct idpf_tx_queue *txq, #define idpf_tx_splitq_clean_bump_ntc(txq, ntc, desc, buf) \ do { \ - (ntc)++; \ - if (unlikely(!(ntc))) { \ - ntc -= (txq)->desc_count; \ + if (unlikely(++(ntc) == (txq)->desc_count)) { \ + ntc = 0; \ buf = (txq)->tx_buf; \ desc = &(txq)->flex_tx[0]; \ } else { \ @@ -1742,59 +1756,65 @@ do { \ * Separate packet completion events will be reported on the completion queue, * and the buffers will be cleaned separately. The stats are not updated from * this function when using flow-based scheduling. + * + * Furthermore, in flow scheduling mode, check to make sure there are enough + * reserve buffers to stash the packet. If there are not, return early, which + * will leave next_to_clean pointing to the packet that failed to be stashed. + * Return false in this scenario. Otherwise, return true. */ -static void idpf_tx_splitq_clean(struct idpf_tx_queue *tx_q, u16 end, +static bool idpf_tx_splitq_clean(struct idpf_tx_queue *tx_q, u16 end, int napi_budget, struct idpf_cleaned_stats *cleaned, bool descs_only) { union idpf_tx_flex_desc *next_pending_desc = NULL; union idpf_tx_flex_desc *tx_desc; - s16 ntc = tx_q->next_to_clean; + u32 ntc = tx_q->next_to_clean; struct libeth_cq_pp cp = { .dev = tx_q->dev, .ss = cleaned, .napi = napi_budget, }; struct idpf_tx_buf *tx_buf; + bool clean_complete = true; tx_desc = &tx_q->flex_tx[ntc]; next_pending_desc = &tx_q->flex_tx[end]; tx_buf = &tx_q->tx_buf[ntc]; - ntc -= tx_q->desc_count; while (tx_desc != next_pending_desc) { - union idpf_tx_flex_desc *eop_desc; + u32 eop_idx; /* If this entry in the ring was used as a context descriptor, - * it's corresponding entry in the buffer ring will have an - * invalid completion tag since no buffer was used. We can - * skip this descriptor since there is no buffer to clean. + * it's corresponding entry in the buffer ring is reserved. We + * can skip this descriptor since there is no buffer to clean. */ if (tx_buf->type <= LIBETH_SQE_CTX) goto fetch_next_txq_desc; - eop_desc = &tx_q->flex_tx[tx_buf->rs_idx]; + if (unlikely(tx_buf->type != LIBETH_SQE_SKB)) + break; + + eop_idx = tx_buf->rs_idx; if (descs_only) { - if (idpf_stash_flow_sch_buffers(tx_q, tx_buf)) + if (IDPF_TX_BUF_RSV_UNUSED(tx_q) < tx_buf->nr_frags) { + clean_complete = false; goto tx_splitq_clean_out; + } + + idpf_stash_flow_sch_buffers(tx_q, tx_buf); - while (tx_desc != eop_desc) { + while (ntc != eop_idx) { idpf_tx_splitq_clean_bump_ntc(tx_q, ntc, tx_desc, tx_buf); - - if (dma_unmap_len(tx_buf, len)) { - if (idpf_stash_flow_sch_buffers(tx_q, - tx_buf)) - goto tx_splitq_clean_out; - } + idpf_stash_flow_sch_buffers(tx_q, tx_buf); } } else { libeth_tx_complete(tx_buf, &cp); /* unmap remaining buffers */ - while (tx_desc != eop_desc) { + while (ntc != eop_idx) { idpf_tx_splitq_clean_bump_ntc(tx_q, ntc, tx_desc, tx_buf); @@ -1808,8 +1828,9 @@ static void idpf_tx_splitq_clean(struct idpf_tx_queue *tx_q, u16 end, } tx_splitq_clean_out: - ntc += tx_q->desc_count; tx_q->next_to_clean = ntc; + + return clean_complete; } #define idpf_tx_clean_buf_ring_bump_ntc(txq, ntc, buf) \ @@ -1840,48 +1861,60 @@ static bool idpf_tx_clean_buf_ring(struct idpf_tx_queue *txq, u16 compl_tag, { u16 idx = compl_tag & txq->compl_tag_bufid_m; struct idpf_tx_buf *tx_buf = NULL; - u16 ntc = txq->next_to_clean; struct libeth_cq_pp cp = { .dev = txq->dev, .ss = cleaned, .napi = budget, }; - u16 num_descs_cleaned = 0; - u16 orig_idx = idx; + u16 ntc, orig_idx = idx; tx_buf = &txq->tx_buf[idx]; - if (unlikely(tx_buf->type <= LIBETH_SQE_CTX)) + + if (unlikely(tx_buf->type <= LIBETH_SQE_CTX || + idpf_tx_buf_compl_tag(tx_buf) != compl_tag)) return false; - while (idpf_tx_buf_compl_tag(tx_buf) == (int)compl_tag) { + if (tx_buf->type == LIBETH_SQE_SKB) libeth_tx_complete(tx_buf, &cp); - num_descs_cleaned++; + idpf_tx_clean_buf_ring_bump_ntc(txq, idx, tx_buf); + + while (idpf_tx_buf_compl_tag(tx_buf) == compl_tag) { + libeth_tx_complete(tx_buf, &cp); idpf_tx_clean_buf_ring_bump_ntc(txq, idx, tx_buf); } - /* If we didn't clean anything on the ring for this completion, there's - * nothing more to do. - */ - if (unlikely(!num_descs_cleaned)) - return false; - - /* Otherwise, if we did clean a packet on the ring directly, it's safe - * to assume that the descriptors starting from the original - * next_to_clean up until the previously cleaned packet can be reused. - * Therefore, we will go back in the ring and stash any buffers still - * in the ring into the hash table to be cleaned later. + /* It's possible the packet we just cleaned was an out of order + * completion, which means we can we can stash the buffers starting + * from the original next_to_clean and reuse the descriptors. We need + * to compare the descriptor ring next_to_clean packet's "first" buffer + * to the "first" buffer of the packet we just cleaned to determine if + * this is the case. Howevever, next_to_clean can point to either a + * reserved buffer that corresponds to a context descriptor used for the + * next_to_clean packet (TSO packet) or the "first" buffer (single + * packet). The orig_idx from the packet we just cleaned will always + * point to the "first" buffer. If next_to_clean points to a reserved + * buffer, let's bump ntc once and start the comparison from there. */ + ntc = txq->next_to_clean; tx_buf = &txq->tx_buf[ntc]; - while (tx_buf != &txq->tx_buf[orig_idx]) { - idpf_stash_flow_sch_buffers(txq, tx_buf); + + if (tx_buf->type == LIBETH_SQE_CTX) idpf_tx_clean_buf_ring_bump_ntc(txq, ntc, tx_buf); - } - /* Finally, update next_to_clean to reflect the work that was just done - * on the ring, if any. If the packet was only cleaned from the hash - * table, the ring will not be impacted, therefore we should not touch - * next_to_clean. The updated idx is used here + /* If ntc still points to a different "first" buffer, clean the + * descriptor ring and stash all of the buffers for later cleaning. If + * we cannot stash all of the buffers, next_to_clean will point to the + * "first" buffer of the packet that could not be stashed and cleaning + * will start there next time. + */ + if (unlikely(tx_buf != &txq->tx_buf[orig_idx] && + !idpf_tx_splitq_clean(txq, orig_idx, budget, cleaned, + true))) + return true; + + /* Otherwise, update next_to_clean to reflect the cleaning that was + * done above. */ txq->next_to_clean = idx; @@ -1909,7 +1942,8 @@ static void idpf_tx_handle_rs_completion(struct idpf_tx_queue *txq, if (!idpf_queue_has(FLOW_SCH_EN, txq)) { u16 head = le16_to_cpu(desc->q_head_compl_tag.q_head); - return idpf_tx_splitq_clean(txq, head, budget, cleaned, false); + idpf_tx_splitq_clean(txq, head, budget, cleaned, false); + return; } compl_tag = le16_to_cpu(desc->q_head_compl_tag.compl_tag); @@ -2337,6 +2371,7 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, dma = dma_map_single(tx_q->dev, skb->data, size, DMA_TO_DEVICE); tx_buf = first; + first->nr_frags = 0; params->compl_tag = (tx_q->compl_tag_cur_gen << tx_q->compl_tag_gen_s) | i; @@ -2347,6 +2382,7 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, if (dma_mapping_error(tx_q->dev, dma)) return idpf_tx_dma_map_error(tx_q, skb, first, i); + first->nr_frags++; idpf_tx_buf_compl_tag(tx_buf) = params->compl_tag; tx_buf->type = LIBETH_SQE_FRAG; @@ -2402,14 +2438,15 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, max_data); - tx_desc++; - i++; - - if (i == tx_q->desc_count) { + if (unlikely(++i == tx_q->desc_count)) { + tx_buf = tx_q->tx_buf; tx_desc = &tx_q->flex_tx[0]; i = 0; tx_q->compl_tag_cur_gen = IDPF_TX_ADJ_COMPL_TAG_GEN(tx_q); + } else { + tx_buf++; + tx_desc++; } /* Since this packet has a buffer that is going to span @@ -2422,7 +2459,7 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, * simply pass over these holes and finish cleaning the * rest of the packet. */ - tx_q->tx_buf[i].type = LIBETH_SQE_EMPTY; + tx_buf->type = LIBETH_SQE_EMPTY; /* Adjust the DMA offset and the remaining size of the * fragment. On the first iteration of this loop, @@ -2446,13 +2483,15 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, break; idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, size); - tx_desc++; - i++; - if (i == tx_q->desc_count) { + if (unlikely(++i == tx_q->desc_count)) { + tx_buf = tx_q->tx_buf; tx_desc = &tx_q->flex_tx[0]; i = 0; tx_q->compl_tag_cur_gen = IDPF_TX_ADJ_COMPL_TAG_GEN(tx_q); + } else { + tx_buf++; + tx_desc++; } size = skb_frag_size(frag); @@ -2460,8 +2499,6 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, dma = skb_frag_dma_map(tx_q->dev, frag, 0, size, DMA_TO_DEVICE); - - tx_buf = &tx_q->tx_buf[i]; } /* record SW timestamp if HW timestamp is not available */ diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index fa87754c73402d..2478f71adb95a1 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -127,7 +127,7 @@ do { \ */ #define IDPF_TX_COMPLQ_PENDING(txq) \ (((txq)->num_completions_pending >= (txq)->complq->num_completions ? \ - 0 : U64_MAX) + \ + 0 : U32_MAX) + \ (txq)->num_completions_pending - (txq)->complq->num_completions) #define IDPF_TX_SPLITQ_COMPL_TAG_WIDTH 16 @@ -787,7 +787,7 @@ struct idpf_compl_queue { u32 next_to_use; u32 next_to_clean; - u32 num_completions; + aligned_u64 num_completions; __cacheline_group_end_aligned(read_write); __cacheline_group_begin_aligned(cold); @@ -919,7 +919,7 @@ struct idpf_txq_group { struct idpf_compl_queue *complq; - u32 num_completions_pending; + aligned_u64 num_completions_pending; }; static inline int idpf_q_vector_to_mem(const struct idpf_q_vector *q_vector) From cc9628bf3039bf4fb2b0e2dc41ccac13fc19f8fd Mon Sep 17 00:00:00 2001 From: Michal Kubiak Date: Fri, 22 Mar 2024 13:19:07 +0100 Subject: [PATCH 16/49] idpf: fix netdev Tx queue stop/wake netif_txq_maybe_stop() returns -1, 0, or 1, while idpf_tx_maybe_stop_common() says it returns 0 or -EBUSY. As a result, there sometimes are Tx queue timeout warnings despite that the queue is empty or there is at least enough space to restart it. Make idpf_tx_maybe_stop_common() inline and returning true or false, handling the return of netif_txq_maybe_stop() properly. Use a correct goto in idpf_tx_maybe_stop_splitq() to avoid stopping the queue or incrementing the stops counter twice. Fixes: 6818c4d5b3c2 ("idpf: add splitq start_xmit") Fixes: a5ab9ee0df0b ("idpf: add singleq start_xmit and napi poll") Signed-off-by: Michal Kubiak Signed-off-by: Alexander Lobakin --- .../ethernet/intel/idpf/idpf_singleq_txrx.c | 4 +++ drivers/net/ethernet/intel/idpf/idpf_txrx.c | 35 +++++-------------- drivers/net/ethernet/intel/idpf/idpf_txrx.h | 9 ++++- 3 files changed, 21 insertions(+), 27 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c index 947d3ff9677c1e..5ba360abbe66e9 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c @@ -375,6 +375,10 @@ netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, IDPF_TX_DESCS_FOR_CTX)) { idpf_tx_buf_hw_update(tx_q, tx_q->next_to_use, false); + u64_stats_update_begin(&tx_q->stats_sync); + u64_stats_inc(&tx_q->q_stats.q_busy); + u64_stats_update_end(&tx_q->stats_sync); + return NETDEV_TX_BUSY; } diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 49f9f07ce3e9d4..9e4bbaa9c492de 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -2127,29 +2127,6 @@ void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc, desc->flow.qw1.compl_tag = cpu_to_le16(params->compl_tag); } -/** - * idpf_tx_maybe_stop_common - 1st level check for common Tx stop conditions - * @tx_q: the queue to be checked - * @size: number of descriptors we want to assure is available - * - * Returns 0 if stop is not needed - */ -int idpf_tx_maybe_stop_common(struct idpf_tx_queue *tx_q, unsigned int size) -{ - struct netdev_queue *nq; - - if (likely(IDPF_DESC_UNUSED(tx_q) >= size)) - return 0; - - u64_stats_update_begin(&tx_q->stats_sync); - u64_stats_inc(&tx_q->q_stats.q_busy); - u64_stats_update_end(&tx_q->stats_sync); - - nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); - - return netif_txq_maybe_stop(nq, IDPF_DESC_UNUSED(tx_q), size, size); -} - /** * idpf_tx_maybe_stop_splitq - 1st level check for Tx splitq stop conditions * @tx_q: the queue to be checked @@ -2161,7 +2138,7 @@ static int idpf_tx_maybe_stop_splitq(struct idpf_tx_queue *tx_q, unsigned int descs_needed) { if (idpf_tx_maybe_stop_common(tx_q, descs_needed)) - goto splitq_stop; + goto out; /* If there are too many outstanding completions expected on the * completion queue, stop the TX queue to give the device some time to @@ -2180,10 +2157,12 @@ static int idpf_tx_maybe_stop_splitq(struct idpf_tx_queue *tx_q, return 0; splitq_stop: + netif_stop_subqueue(tx_q->netdev, tx_q->idx); + +out: u64_stats_update_begin(&tx_q->stats_sync); u64_stats_inc(&tx_q->q_stats.q_busy); u64_stats_update_end(&tx_q->stats_sync); - netif_stop_subqueue(tx_q->netdev, tx_q->idx); return -EBUSY; } @@ -2206,7 +2185,11 @@ void idpf_tx_buf_hw_update(struct idpf_tx_queue *tx_q, u32 val, nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); tx_q->next_to_use = val; - idpf_tx_maybe_stop_common(tx_q, IDPF_TX_DESC_NEEDED); + if (idpf_tx_maybe_stop_common(tx_q, IDPF_TX_DESC_NEEDED)) { + u64_stats_update_begin(&tx_q->stats_sync); + u64_stats_inc(&tx_q->q_stats.q_busy); + u64_stats_update_end(&tx_q->stats_sync); + } /* Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index 2478f71adb95a1..df3574ac58c293 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -1020,7 +1020,6 @@ void idpf_tx_dma_map_error(struct idpf_tx_queue *txq, struct sk_buff *skb, struct idpf_tx_buf *first, u16 ring_idx); unsigned int idpf_tx_desc_count_required(struct idpf_tx_queue *txq, struct sk_buff *skb); -int idpf_tx_maybe_stop_common(struct idpf_tx_queue *tx_q, unsigned int size); void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue); netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, struct idpf_tx_queue *tx_q); @@ -1029,4 +1028,12 @@ bool idpf_rx_singleq_buf_hw_alloc_all(struct idpf_rx_queue *rxq, u16 cleaned_count); int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off); +static inline bool idpf_tx_maybe_stop_common(struct idpf_tx_queue *tx_q, + u32 needed) +{ + return !netif_subqueue_maybe_stop(tx_q->netdev, tx_q->idx, + IDPF_DESC_UNUSED(tx_q), + needed, needed); +} + #endif /* !_IDPF_TXRX_H_ */ From 6cae557c4cca7834374eb11279ad87a441b1d6de Mon Sep 17 00:00:00 2001 From: Joshua Hay Date: Fri, 15 Dec 2023 20:37:21 +0100 Subject: [PATCH 17/49] idpf: enable WB_ON_ITR Tell hardware to write back completed descriptors even when interrupts are disabled. Otherwise, descriptors might not be written back until the hardware can flush a full cacheline of descriptors. This can cause unnecessary delays when traffic is light (or even trigger Tx queue timeout). The example scenario to reproduce the Tx timeout if the fix is not applied: - configure at least 2 Tx queues to be assigned to the same q_vector, - generate a huge Tx traffic on the first Tx queue - try to send a few packets using the second Tx queue. In such a case Tx timeout will appear on the second Tx queue because no completion descriptors are written back for that queue while interrupts are disabled due to NAPI polling. The patch is necessary to start work on the AF_XDP implementation for the idpf driver, because there may be a case where a regular LAN Tx queue and an XDP queue share the same NAPI. Fixes: c2d548cad150 ("idpf: add TX splitq napi poll support") Fixes: a5ab9ee0df0b ("idpf: add singleq start_xmit and napi poll") Reviewed-by: Przemek Kitszel Reviewed-by: Alexander Lobakin Signed-off-by: Joshua Hay Co-developed-by: Michal Kubiak Signed-off-by: Michal Kubiak --- drivers/net/ethernet/intel/idpf/idpf_dev.c | 2 ++ .../ethernet/intel/idpf/idpf_singleq_txrx.c | 6 ++++- drivers/net/ethernet/intel/idpf/idpf_txrx.c | 7 ++++- drivers/net/ethernet/intel/idpf/idpf_txrx.h | 27 ++++++++++++++++++- drivers/net/ethernet/intel/idpf/idpf_vf_dev.c | 2 ++ 5 files changed, 41 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf_dev.c b/drivers/net/ethernet/intel/idpf/idpf_dev.c index 3df9935685e962..6c913a703df647 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_dev.c +++ b/drivers/net/ethernet/intel/idpf/idpf_dev.c @@ -97,8 +97,10 @@ static int idpf_intr_reg_init(struct idpf_vport *vport) intr->dyn_ctl = idpf_get_reg_addr(adapter, reg_vals[vec_id].dyn_ctl_reg); intr->dyn_ctl_intena_m = PF_GLINT_DYN_CTL_INTENA_M; + intr->dyn_ctl_intena_msk_m = PF_GLINT_DYN_CTL_INTENA_MSK_M; intr->dyn_ctl_itridx_s = PF_GLINT_DYN_CTL_ITR_INDX_S; intr->dyn_ctl_intrvl_s = PF_GLINT_DYN_CTL_INTERVAL_S; + intr->dyn_ctl_wb_on_itr_m = PF_GLINT_DYN_CTL_WB_ON_ITR_M; spacing = IDPF_ITR_IDX_SPACING(reg_vals[vec_id].itrn_index_spacing, IDPF_PF_ITR_IDX_SPACING); diff --git a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c index 5ba360abbe66e9..dfd7cf1d9aa0ad 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c @@ -1120,8 +1120,10 @@ int idpf_vport_singleq_napi_poll(struct napi_struct *napi, int budget) &work_done); /* If work not completed, return budget and polling will return */ - if (!clean_complete) + if (!clean_complete) { + idpf_vport_intr_set_wb_on_itr(q_vector); return budget; + } work_done = min_t(int, work_done, budget - 1); @@ -1130,6 +1132,8 @@ int idpf_vport_singleq_napi_poll(struct napi_struct *napi, int budget) */ if (likely(napi_complete_done(napi, work_done))) idpf_vport_intr_update_itr_ena_irq(q_vector); + else + idpf_vport_intr_set_wb_on_itr(q_vector); return work_done; } diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 9e4bbaa9c492de..536e41a4432809 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -3710,6 +3710,7 @@ void idpf_vport_intr_update_itr_ena_irq(struct idpf_q_vector *q_vector) /* net_dim() updates ITR out-of-band using a work item */ idpf_net_dim(q_vector); + q_vector->wb_on_itr = false; intval = idpf_vport_intr_buildreg_itr(q_vector, IDPF_NO_ITR_UPDATE_IDX, 0); @@ -4012,8 +4013,10 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget, &work_done); /* If work not completed, return budget and polling will return */ - if (!clean_complete) + if (!clean_complete) { + idpf_vport_intr_set_wb_on_itr(q_vector); return budget; + } work_done = min_t(int, work_done, budget - 1); @@ -4022,6 +4025,8 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) */ if (likely(napi_complete_done(napi, work_done))) idpf_vport_intr_update_itr_ena_irq(q_vector); + else + idpf_vport_intr_set_wb_on_itr(q_vector); /* Switch to poll mode in the tear-down path after sending disable * queues virtchnl message, as the interrupts will be disabled after diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index df3574ac58c293..b4a87f8661a8af 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -349,9 +349,11 @@ struct idpf_vec_regs { * struct idpf_intr_reg * @dyn_ctl: Dynamic control interrupt register * @dyn_ctl_intena_m: Mask for dyn_ctl interrupt enable + * @dyn_ctl_intena_msk_m: Mask for dyn_ctl interrupt enable mask * @dyn_ctl_itridx_s: Register bit offset for ITR index * @dyn_ctl_itridx_m: Mask for ITR index * @dyn_ctl_intrvl_s: Register bit offset for ITR interval + * @dyn_ctl_wb_on_itr_m: Mask for WB on ITR feature * @rx_itr: RX ITR register * @tx_itr: TX ITR register * @icr_ena: Interrupt cause register offset @@ -360,9 +362,11 @@ struct idpf_vec_regs { struct idpf_intr_reg { void __iomem *dyn_ctl; u32 dyn_ctl_intena_m; + u32 dyn_ctl_intena_msk_m; u32 dyn_ctl_itridx_s; u32 dyn_ctl_itridx_m; u32 dyn_ctl_intrvl_s; + u32 dyn_ctl_wb_on_itr_m; void __iomem *rx_itr; void __iomem *tx_itr; void __iomem *icr_ena; @@ -383,6 +387,7 @@ struct idpf_intr_reg { * @intr_reg: See struct idpf_intr_reg * @napi: napi handler * @total_events: Number of interrupts processed + * @wb_on_itr: WB on ITR enabled or not * @tx_dim: Data for TX net_dim algorithm * @tx_itr_value: TX interrupt throttling rate * @tx_intr_mode: Dynamic ITR or not @@ -413,6 +418,7 @@ struct idpf_q_vector { __cacheline_group_begin_aligned(read_write); struct napi_struct napi; u16 total_events; + bool wb_on_itr; struct dim tx_dim; u16 tx_itr_value; @@ -431,7 +437,7 @@ struct idpf_q_vector { cpumask_var_t affinity_mask; __cacheline_group_end_aligned(cold); }; -libeth_cacheline_set_assert(struct idpf_q_vector, 104, +libeth_cacheline_set_assert(struct idpf_q_vector, 112, 424 + 2 * sizeof(struct dim), 8 + sizeof(cpumask_var_t)); @@ -989,6 +995,25 @@ static inline void idpf_tx_splitq_build_desc(union idpf_tx_flex_desc *desc, idpf_tx_splitq_build_flow_desc(desc, params, td_cmd, size); } +/** + * idpf_vport_intr_set_wb_on_itr - enable descriptor writeback on disabled interrupts + * @q_vector: pointer to queue vector struct + */ +static inline void idpf_vport_intr_set_wb_on_itr(struct idpf_q_vector *q_vector) +{ + struct idpf_intr_reg *reg; + + if (q_vector->wb_on_itr) + return; + + q_vector->wb_on_itr = true; + reg = &q_vector->intr_reg; + + writel(reg->dyn_ctl_wb_on_itr_m | reg->dyn_ctl_intena_msk_m | + (IDPF_NO_ITR_UPDATE_IDX << reg->dyn_ctl_itridx_s), + reg->dyn_ctl); +} + int idpf_vport_singleq_napi_poll(struct napi_struct *napi, int budget); void idpf_vport_init_num_qs(struct idpf_vport *vport, struct virtchnl2_create_vport *vport_msg); diff --git a/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c b/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c index 629cb5cb7c9fc1..99b8dbaf4225c5 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c +++ b/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c @@ -97,7 +97,9 @@ static int idpf_vf_intr_reg_init(struct idpf_vport *vport) intr->dyn_ctl = idpf_get_reg_addr(adapter, reg_vals[vec_id].dyn_ctl_reg); intr->dyn_ctl_intena_m = VF_INT_DYN_CTLN_INTENA_M; + intr->dyn_ctl_intena_msk_m = VF_INT_DYN_CTLN_INTENA_MSK_M; intr->dyn_ctl_itridx_s = VF_INT_DYN_CTLN_ITR_INDX_S; + intr->dyn_ctl_wb_on_itr_m = VF_INT_DYN_CTLN_WB_ON_ITR_M; spacing = IDPF_ITR_IDX_SPACING(reg_vals[vec_id].itrn_index_spacing, IDPF_VF_ITR_IDX_SPACING); From 56c1d9b98af27ec30b8c563ee699dc7075020dcf Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 16 Apr 2024 10:52:30 +0200 Subject: [PATCH 18/49] idpf: switch do libeth generic statistics Fully reimplement idpf's per-queue stats using the libeth infra. Embed &libeth_netdev_priv to the beginning of &idpf_netdev_priv(), call the necessary init/deinit helpers and the corresponding Ethtool helpers. Update hotpath counters such as hsplit and tso/gso using the onstack containers instead of direct accesses to queue->stats. Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/idpf/idpf.h | 21 +- .../net/ethernet/intel/idpf/idpf_ethtool.c | 498 ++---------------- drivers/net/ethernet/intel/idpf/idpf_lib.c | 21 +- .../ethernet/intel/idpf/idpf_singleq_txrx.c | 76 +-- drivers/net/ethernet/intel/idpf/idpf_txrx.c | 172 +++--- drivers/net/ethernet/intel/idpf/idpf_txrx.h | 54 +- .../net/ethernet/intel/idpf/idpf_virtchnl.c | 37 +- drivers/net/ethernet/intel/idpf/virtchnl2.h | 33 +- 8 files changed, 232 insertions(+), 680 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf.h b/drivers/net/ethernet/intel/idpf/idpf.h index 2a5a86e22fab54..88e36cc29ae726 100644 --- a/drivers/net/ethernet/intel/idpf/idpf.h +++ b/drivers/net/ethernet/intel/idpf/idpf.h @@ -138,6 +138,7 @@ enum idpf_vport_state { /** * struct idpf_netdev_priv - Struct to store vport back pointer + * @priv: libeth private structure * @adapter: Adapter back pointer * @vport: Vport back pointer * @vport_id: Vport identifier @@ -147,6 +148,8 @@ enum idpf_vport_state { * @stats_lock: Lock to protect stats update */ struct idpf_netdev_priv { + struct libeth_netdev_priv priv; + struct idpf_adapter *adapter; struct idpf_vport *vport; u32 vport_id; @@ -155,6 +158,7 @@ struct idpf_netdev_priv { struct rtnl_link_stats64 netstats; spinlock_t stats_lock; }; +libeth_netdev_priv_assert(struct idpf_netdev_priv, priv); /** * struct idpf_reset_reg - Reset register offsets/masks @@ -218,19 +222,6 @@ enum idpf_vport_flags { IDPF_VPORT_FLAGS_NBITS, }; -struct idpf_port_stats { - struct u64_stats_sync stats_sync; - u64_stats_t rx_hw_csum_err; - u64_stats_t rx_hsplit; - u64_stats_t rx_hsplit_hbo; - u64_stats_t rx_bad_descs; - u64_stats_t tx_linearize; - u64_stats_t tx_busy; - u64_stats_t tx_drops; - u64_stats_t tx_dma_map_errs; - struct virtchnl2_vport_stats vport_stats; -}; - /** * struct idpf_vport - Handle for netdevices and queue resources * @num_txq: Number of allocated TX queues @@ -271,7 +262,7 @@ struct idpf_port_stats { * @default_mac_addr: device will give a default MAC to use * @rx_itr_profile: RX profiles for Dynamic Interrupt Moderation * @tx_itr_profile: TX profiles for Dynamic Interrupt Moderation - * @port_stats: per port csum, header split, and other offload stats + * @vport_stats: vport stats reported by HW * @link_up: True if link is up * @link_speed_mbps: Link speed in mbps * @sw_marker_wq: workqueue for marker packets @@ -314,7 +305,7 @@ struct idpf_vport { u8 default_mac_addr[ETH_ALEN]; u16 rx_itr_profile[IDPF_DIM_PROFILE_SLOTS]; u16 tx_itr_profile[IDPF_DIM_PROFILE_SLOTS]; - struct idpf_port_stats port_stats; + struct virtchnl2_vport_stats vport_stats; bool link_up; u32 link_speed_mbps; diff --git a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c index f6a7263ee9c9b8..c08a137059e3c1 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c +++ b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (C) 2023 Intel Corporation */ +#include + #include "idpf.h" /** @@ -399,172 +401,37 @@ static int idpf_set_ringparam(struct net_device *netdev, return err; } -/** - * struct idpf_stats - definition for an ethtool statistic - * @stat_string: statistic name to display in ethtool -S output - * @sizeof_stat: the sizeof() the stat, must be no greater than sizeof(u64) - * @stat_offset: offsetof() the stat from a base pointer - * - * This structure defines a statistic to be added to the ethtool stats buffer. - * It defines a statistic as offset from a common base pointer. Stats should - * be defined in constant arrays using the IDPF_STAT macro, with every element - * of the array using the same _type for calculating the sizeof_stat and - * stat_offset. - * - * The @sizeof_stat is expected to be sizeof(u8), sizeof(u16), sizeof(u32) or - * sizeof(u64). Other sizes are not expected and will produce a WARN_ONCE from - * the idpf_add_ethtool_stat() helper function. - * - * The @stat_string is interpreted as a format string, allowing formatted - * values to be inserted while looping over multiple structures for a given - * statistics array. Thus, every statistic string in an array should have the - * same type and number of format specifiers, to be formatted by variadic - * arguments to the idpf_add_stat_string() helper function. - */ -struct idpf_stats { - char stat_string[ETH_GSTRING_LEN]; - int sizeof_stat; - int stat_offset; -}; - -/* Helper macro to define an idpf_stat structure with proper size and type. - * Use this when defining constant statistics arrays. Note that @_type expects - * only a type name and is used multiple times. - */ -#define IDPF_STAT(_type, _name, _stat) { \ - .stat_string = _name, \ - .sizeof_stat = sizeof_field(_type, _stat), \ - .stat_offset = offsetof(_type, _stat) \ -} - -/* Helper macros for defining some statistics related to queues */ -#define IDPF_RX_QUEUE_STAT(_name, _stat) \ - IDPF_STAT(struct idpf_rx_queue, _name, _stat) -#define IDPF_TX_QUEUE_STAT(_name, _stat) \ - IDPF_STAT(struct idpf_tx_queue, _name, _stat) - -/* Stats associated with a Tx queue */ -static const struct idpf_stats idpf_gstrings_tx_queue_stats[] = { - IDPF_TX_QUEUE_STAT("pkts", q_stats.packets), - IDPF_TX_QUEUE_STAT("bytes", q_stats.bytes), - IDPF_TX_QUEUE_STAT("lso_pkts", q_stats.lso_pkts), -}; - -/* Stats associated with an Rx queue */ -static const struct idpf_stats idpf_gstrings_rx_queue_stats[] = { - IDPF_RX_QUEUE_STAT("pkts", q_stats.packets), - IDPF_RX_QUEUE_STAT("bytes", q_stats.bytes), - IDPF_RX_QUEUE_STAT("rx_gro_hw_pkts", q_stats.rsc_pkts), -}; - -#define IDPF_TX_QUEUE_STATS_LEN ARRAY_SIZE(idpf_gstrings_tx_queue_stats) -#define IDPF_RX_QUEUE_STATS_LEN ARRAY_SIZE(idpf_gstrings_rx_queue_stats) - -#define IDPF_PORT_STAT(_name, _stat) \ - IDPF_STAT(struct idpf_vport, _name, _stat) - -static const struct idpf_stats idpf_gstrings_port_stats[] = { - IDPF_PORT_STAT("rx-csum_errors", port_stats.rx_hw_csum_err), - IDPF_PORT_STAT("rx-hsplit", port_stats.rx_hsplit), - IDPF_PORT_STAT("rx-hsplit_hbo", port_stats.rx_hsplit_hbo), - IDPF_PORT_STAT("rx-bad_descs", port_stats.rx_bad_descs), - IDPF_PORT_STAT("tx-skb_drops", port_stats.tx_drops), - IDPF_PORT_STAT("tx-dma_map_errs", port_stats.tx_dma_map_errs), - IDPF_PORT_STAT("tx-linearized_pkts", port_stats.tx_linearize), - IDPF_PORT_STAT("tx-busy_events", port_stats.tx_busy), - IDPF_PORT_STAT("rx-unicast_pkts", port_stats.vport_stats.rx_unicast), - IDPF_PORT_STAT("rx-multicast_pkts", port_stats.vport_stats.rx_multicast), - IDPF_PORT_STAT("rx-broadcast_pkts", port_stats.vport_stats.rx_broadcast), - IDPF_PORT_STAT("rx-unknown_protocol", port_stats.vport_stats.rx_unknown_protocol), - IDPF_PORT_STAT("tx-unicast_pkts", port_stats.vport_stats.tx_unicast), - IDPF_PORT_STAT("tx-multicast_pkts", port_stats.vport_stats.tx_multicast), - IDPF_PORT_STAT("tx-broadcast_pkts", port_stats.vport_stats.tx_broadcast), +static const char * const idpf_gstrings_port_stats[] = { + "rx_bytes", + "rx_unicast", + "rx_multicast", + "rx_broadcast", + "rx_discards", + "rx_errors", + "rx_unknown_protocol", + "tx_bytes", + "tx_unicast", + "tx_multicast", + "tx_broadcast", + "tx_discards", + "tx_errors", + "rx_invalid_frame_length", + "rx_overflow_drop", }; +static_assert(ARRAY_SIZE(idpf_gstrings_port_stats) == + sizeof_field(struct virtchnl2_vport_stats, counters) / + sizeof(__le64)); #define IDPF_PORT_STATS_LEN ARRAY_SIZE(idpf_gstrings_port_stats) -/** - * __idpf_add_qstat_strings - copy stat strings into ethtool buffer - * @p: ethtool supplied buffer - * @stats: stat definitions array - * @size: size of the stats array - * @type: stat type - * @idx: stat index - * - * Format and copy the strings described by stats into the buffer pointed at - * by p. - */ -static void __idpf_add_qstat_strings(u8 **p, const struct idpf_stats *stats, - const unsigned int size, const char *type, - unsigned int idx) -{ - unsigned int i; - - for (i = 0; i < size; i++) - ethtool_sprintf(p, "%s_q-%u_%s", - type, idx, stats[i].stat_string); -} - -/** - * idpf_add_qstat_strings - Copy queue stat strings into ethtool buffer - * @p: ethtool supplied buffer - * @stats: stat definitions array - * @type: stat type - * @idx: stat idx - * - * Format and copy the strings described by the const static stats value into - * the buffer pointed at by p. - * - * The parameter @stats is evaluated twice, so parameters with side effects - * should be avoided. Additionally, stats must be an array such that - * ARRAY_SIZE can be called on it. - */ -#define idpf_add_qstat_strings(p, stats, type, idx) \ - __idpf_add_qstat_strings(p, stats, ARRAY_SIZE(stats), type, idx) - /** * idpf_add_stat_strings - Copy port stat strings into ethtool buffer * @p: ethtool buffer - * @stats: struct to copy from - * @size: size of stats array to copy from */ -static void idpf_add_stat_strings(u8 **p, const struct idpf_stats *stats, - const unsigned int size) +static void idpf_add_stat_strings(u8 **p) { - unsigned int i; - - for (i = 0; i < size; i++) - ethtool_puts(p, stats[i].stat_string); -} - -/** - * idpf_get_stat_strings - Get stat strings - * @netdev: network interface device structure - * @data: buffer for string data - * - * Builds the statistics string table - */ -static void idpf_get_stat_strings(struct net_device *netdev, u8 *data) -{ - struct idpf_netdev_priv *np = netdev_priv(netdev); - struct idpf_vport_config *vport_config; - unsigned int i; - - idpf_add_stat_strings(&data, idpf_gstrings_port_stats, - IDPF_PORT_STATS_LEN); - - vport_config = np->adapter->vport_config[np->vport_idx]; - /* It's critical that we always report a constant number of strings and - * that the strings are reported in the same order regardless of how - * many queues are actually in use. - */ - for (i = 0; i < vport_config->max_q.max_txq; i++) - idpf_add_qstat_strings(&data, idpf_gstrings_tx_queue_stats, - "tx", i); - - for (i = 0; i < vport_config->max_q.max_rxq; i++) - idpf_add_qstat_strings(&data, idpf_gstrings_rx_queue_stats, - "rx", i); + for (u32 i = 0; i < IDPF_PORT_STATS_LEN; i++) + ethtool_puts(p, idpf_gstrings_port_stats[i]); } /** @@ -579,7 +446,8 @@ static void idpf_get_strings(struct net_device *netdev, u32 sset, u8 *data) { switch (sset) { case ETH_SS_STATS: - idpf_get_stat_strings(netdev, data); + idpf_add_stat_strings(&data); + libeth_ethtool_get_strings(netdev, sset, data); break; default: break; @@ -595,146 +463,15 @@ static void idpf_get_strings(struct net_device *netdev, u32 sset, u8 *data) */ static int idpf_get_sset_count(struct net_device *netdev, int sset) { - struct idpf_netdev_priv *np = netdev_priv(netdev); - struct idpf_vport_config *vport_config; - u16 max_txq, max_rxq; + u32 count; if (sset != ETH_SS_STATS) return -EINVAL; - vport_config = np->adapter->vport_config[np->vport_idx]; - /* This size reported back here *must* be constant throughout the - * lifecycle of the netdevice, i.e. we must report the maximum length - * even for queues that don't technically exist. This is due to the - * fact that this userspace API uses three separate ioctl calls to get - * stats data but has no way to communicate back to userspace when that - * size has changed, which can typically happen as a result of changing - * number of queues. If the number/order of stats change in the middle - * of this call chain it will lead to userspace crashing/accessing bad - * data through buffer under/overflow. - */ - max_txq = vport_config->max_q.max_txq; - max_rxq = vport_config->max_q.max_rxq; - - return IDPF_PORT_STATS_LEN + (IDPF_TX_QUEUE_STATS_LEN * max_txq) + - (IDPF_RX_QUEUE_STATS_LEN * max_rxq); -} - -/** - * idpf_add_one_ethtool_stat - copy the stat into the supplied buffer - * @data: location to store the stat value - * @pstat: old stat pointer to copy from - * @stat: the stat definition - * - * Copies the stat data defined by the pointer and stat structure pair into - * the memory supplied as data. If the pointer is null, data will be zero'd. - */ -static void idpf_add_one_ethtool_stat(u64 *data, const void *pstat, - const struct idpf_stats *stat) -{ - char *p; - - if (!pstat) { - /* Ensure that the ethtool data buffer is zero'd for any stats - * which don't have a valid pointer. - */ - *data = 0; - return; - } + count = IDPF_PORT_STATS_LEN; + count += libeth_ethtool_get_sset_count(netdev, sset); - p = (char *)pstat + stat->stat_offset; - switch (stat->sizeof_stat) { - case sizeof(u64): - *data = *((u64 *)p); - break; - case sizeof(u32): - *data = *((u32 *)p); - break; - case sizeof(u16): - *data = *((u16 *)p); - break; - case sizeof(u8): - *data = *((u8 *)p); - break; - default: - WARN_ONCE(1, "unexpected stat size for %s", - stat->stat_string); - *data = 0; - } -} - -/** - * idpf_add_queue_stats - copy queue statistics into supplied buffer - * @data: ethtool stats buffer - * @q: the queue to copy - * @type: type of the queue - * - * Queue statistics must be copied while protected by u64_stats_fetch_begin, - * so we can't directly use idpf_add_ethtool_stats. Assumes that queue stats - * are defined in idpf_gstrings_queue_stats. If the queue pointer is null, - * zero out the queue stat values and update the data pointer. Otherwise - * safely copy the stats from the queue into the supplied buffer and update - * the data pointer when finished. - * - * This function expects to be called while under rcu_read_lock(). - */ -static void idpf_add_queue_stats(u64 **data, const void *q, - enum virtchnl2_queue_type type) -{ - const struct u64_stats_sync *stats_sync; - const struct idpf_stats *stats; - unsigned int start; - unsigned int size; - unsigned int i; - - if (type == VIRTCHNL2_QUEUE_TYPE_RX) { - size = IDPF_RX_QUEUE_STATS_LEN; - stats = idpf_gstrings_rx_queue_stats; - stats_sync = &((const struct idpf_rx_queue *)q)->stats_sync; - } else { - size = IDPF_TX_QUEUE_STATS_LEN; - stats = idpf_gstrings_tx_queue_stats; - stats_sync = &((const struct idpf_tx_queue *)q)->stats_sync; - } - - /* To avoid invalid statistics values, ensure that we keep retrying - * the copy until we get a consistent value according to - * u64_stats_fetch_retry. - */ - do { - start = u64_stats_fetch_begin(stats_sync); - for (i = 0; i < size; i++) - idpf_add_one_ethtool_stat(&(*data)[i], q, &stats[i]); - } while (u64_stats_fetch_retry(stats_sync, start)); - - /* Once we successfully copy the stats in, update the data pointer */ - *data += size; -} - -/** - * idpf_add_empty_queue_stats - Add stats for a non-existent queue - * @data: pointer to data buffer - * @qtype: type of data queue - * - * We must report a constant length of stats back to userspace regardless of - * how many queues are actually in use because stats collection happens over - * three separate ioctls and there's no way to notify userspace the size - * changed between those calls. This adds empty to data to the stats since we - * don't have a real queue to refer to for this stats slot. - */ -static void idpf_add_empty_queue_stats(u64 **data, u16 qtype) -{ - unsigned int i; - int stats_len; - - if (qtype == VIRTCHNL2_QUEUE_TYPE_RX) - stats_len = IDPF_RX_QUEUE_STATS_LEN; - else - stats_len = IDPF_TX_QUEUE_STATS_LEN; - - for (i = 0; i < stats_len; i++) - (*data)[i] = 0; - *data += stats_len; + return count; } /** @@ -744,116 +481,15 @@ static void idpf_add_empty_queue_stats(u64 **data, u16 qtype) */ static void idpf_add_port_stats(struct idpf_vport *vport, u64 **data) { - unsigned int size = IDPF_PORT_STATS_LEN; - unsigned int start; - unsigned int i; - - do { - start = u64_stats_fetch_begin(&vport->port_stats.stats_sync); - for (i = 0; i < size; i++) - idpf_add_one_ethtool_stat(&(*data)[i], vport, - &idpf_gstrings_port_stats[i]); - } while (u64_stats_fetch_retry(&vport->port_stats.stats_sync, start)); - - *data += size; -} + u64 *stats = *data; -/** - * idpf_collect_queue_stats - accumulate various per queue stats - * into port level stats - * @vport: pointer to vport struct - **/ -static void idpf_collect_queue_stats(struct idpf_vport *vport) -{ - struct idpf_port_stats *pstats = &vport->port_stats; - int i, j; + memcpy(stats, &vport->vport_stats.counters, + sizeof(vport->vport_stats.counters)); - /* zero out port stats since they're actually tracked in per - * queue stats; this is only for reporting - */ - u64_stats_update_begin(&pstats->stats_sync); - u64_stats_set(&pstats->rx_hw_csum_err, 0); - u64_stats_set(&pstats->rx_hsplit, 0); - u64_stats_set(&pstats->rx_hsplit_hbo, 0); - u64_stats_set(&pstats->rx_bad_descs, 0); - u64_stats_set(&pstats->tx_linearize, 0); - u64_stats_set(&pstats->tx_busy, 0); - u64_stats_set(&pstats->tx_drops, 0); - u64_stats_set(&pstats->tx_dma_map_errs, 0); - u64_stats_update_end(&pstats->stats_sync); - - for (i = 0; i < vport->num_rxq_grp; i++) { - struct idpf_rxq_group *rxq_grp = &vport->rxq_grps[i]; - u16 num_rxq; - - if (idpf_is_queue_model_split(vport->rxq_model)) - num_rxq = rxq_grp->splitq.num_rxq_sets; - else - num_rxq = rxq_grp->singleq.num_rxq; - - for (j = 0; j < num_rxq; j++) { - u64 hw_csum_err, hsplit, hsplit_hbo, bad_descs; - struct idpf_rx_queue_stats *stats; - struct idpf_rx_queue *rxq; - unsigned int start; - - if (idpf_is_queue_model_split(vport->rxq_model)) - rxq = &rxq_grp->splitq.rxq_sets[j]->rxq; - else - rxq = rxq_grp->singleq.rxqs[j]; - - if (!rxq) - continue; - - do { - start = u64_stats_fetch_begin(&rxq->stats_sync); - - stats = &rxq->q_stats; - hw_csum_err = u64_stats_read(&stats->hw_csum_err); - hsplit = u64_stats_read(&stats->hsplit_pkts); - hsplit_hbo = u64_stats_read(&stats->hsplit_buf_ovf); - bad_descs = u64_stats_read(&stats->bad_descs); - } while (u64_stats_fetch_retry(&rxq->stats_sync, start)); - - u64_stats_update_begin(&pstats->stats_sync); - u64_stats_add(&pstats->rx_hw_csum_err, hw_csum_err); - u64_stats_add(&pstats->rx_hsplit, hsplit); - u64_stats_add(&pstats->rx_hsplit_hbo, hsplit_hbo); - u64_stats_add(&pstats->rx_bad_descs, bad_descs); - u64_stats_update_end(&pstats->stats_sync); - } - } + for (u32 i = 0; i < IDPF_PORT_STATS_LEN; i++) + le64_to_cpus(&stats[i]); - for (i = 0; i < vport->num_txq_grp; i++) { - struct idpf_txq_group *txq_grp = &vport->txq_grps[i]; - - for (j = 0; j < txq_grp->num_txq; j++) { - u64 linearize, qbusy, skb_drops, dma_map_errs; - struct idpf_tx_queue *txq = txq_grp->txqs[j]; - struct idpf_tx_queue_stats *stats; - unsigned int start; - - if (!txq) - continue; - - do { - start = u64_stats_fetch_begin(&txq->stats_sync); - - stats = &txq->q_stats; - linearize = u64_stats_read(&stats->linearize); - qbusy = u64_stats_read(&stats->q_busy); - skb_drops = u64_stats_read(&stats->skb_drops); - dma_map_errs = u64_stats_read(&stats->dma_map_errs); - } while (u64_stats_fetch_retry(&txq->stats_sync, start)); - - u64_stats_update_begin(&pstats->stats_sync); - u64_stats_add(&pstats->tx_linearize, linearize); - u64_stats_add(&pstats->tx_busy, qbusy); - u64_stats_add(&pstats->tx_drops, skb_drops); - u64_stats_add(&pstats->tx_dma_map_errs, dma_map_errs); - u64_stats_update_end(&pstats->stats_sync); - } - } + *data += IDPF_PORT_STATS_LEN; } /** @@ -869,12 +505,7 @@ static void idpf_get_ethtool_stats(struct net_device *netdev, u64 *data) { struct idpf_netdev_priv *np = netdev_priv(netdev); - struct idpf_vport_config *vport_config; struct idpf_vport *vport; - unsigned int total = 0; - unsigned int i, j; - bool is_splitq; - u16 qtype; idpf_vport_ctrl_lock(netdev); vport = idpf_netdev_to_vport(netdev); @@ -887,63 +518,8 @@ static void idpf_get_ethtool_stats(struct net_device *netdev, rcu_read_lock(); - idpf_collect_queue_stats(vport); idpf_add_port_stats(vport, &data); - - for (i = 0; i < vport->num_txq_grp; i++) { - struct idpf_txq_group *txq_grp = &vport->txq_grps[i]; - - qtype = VIRTCHNL2_QUEUE_TYPE_TX; - - for (j = 0; j < txq_grp->num_txq; j++, total++) { - struct idpf_tx_queue *txq = txq_grp->txqs[j]; - - if (!txq) - idpf_add_empty_queue_stats(&data, qtype); - else - idpf_add_queue_stats(&data, txq, qtype); - } - } - - vport_config = vport->adapter->vport_config[vport->idx]; - /* It is critical we provide a constant number of stats back to - * userspace regardless of how many queues are actually in use because - * there is no way to inform userspace the size has changed between - * ioctl calls. This will fill in any missing stats with zero. - */ - for (; total < vport_config->max_q.max_txq; total++) - idpf_add_empty_queue_stats(&data, VIRTCHNL2_QUEUE_TYPE_TX); - total = 0; - - is_splitq = idpf_is_queue_model_split(vport->rxq_model); - - for (i = 0; i < vport->num_rxq_grp; i++) { - struct idpf_rxq_group *rxq_grp = &vport->rxq_grps[i]; - u16 num_rxq; - - qtype = VIRTCHNL2_QUEUE_TYPE_RX; - - if (is_splitq) - num_rxq = rxq_grp->splitq.num_rxq_sets; - else - num_rxq = rxq_grp->singleq.num_rxq; - - for (j = 0; j < num_rxq; j++, total++) { - struct idpf_rx_queue *rxq; - - if (is_splitq) - rxq = &rxq_grp->splitq.rxq_sets[j]->rxq; - else - rxq = rxq_grp->singleq.rxqs[j]; - if (!rxq) - idpf_add_empty_queue_stats(&data, qtype); - else - idpf_add_queue_stats(&data, rxq, qtype); - } - } - - for (; total < vport_config->max_q.max_rxq; total++) - idpf_add_empty_queue_stats(&data, VIRTCHNL2_QUEUE_TYPE_RX); + libeth_ethtool_get_stats(netdev, stats, data); rcu_read_unlock(); diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index 10c76d6838d391..3089def8d4536c 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (C) 2023 Intel Corporation */ +#include + #include "idpf.h" #include "idpf_virtchnl.h" @@ -739,9 +741,9 @@ static int idpf_cfg_netdev(struct idpf_vport *vport) return idpf_init_mac_addr(vport, netdev); } - netdev = alloc_etherdev_mqs(sizeof(struct idpf_netdev_priv), - vport_config->max_q.max_txq, - vport_config->max_q.max_rxq); + netdev = libeth_netdev_alloc(sizeof(struct idpf_netdev_priv), + vport_config->max_q.max_rxq, + vport_config->max_q.max_txq); if (!netdev) return -ENOMEM; @@ -756,7 +758,7 @@ static int idpf_cfg_netdev(struct idpf_vport *vport) err = idpf_init_mac_addr(vport, netdev); if (err) { - free_netdev(vport->netdev); + libeth_netdev_free(vport->netdev); vport->netdev = NULL; return err; @@ -945,7 +947,7 @@ static void idpf_decfg_netdev(struct idpf_vport *vport) vport->rx_ptype_lkup = NULL; unregister_netdev(vport->netdev); - free_netdev(vport->netdev); + libeth_netdev_free(vport->netdev); vport->netdev = NULL; adapter->netdevs[vport->idx] = NULL; @@ -1274,13 +1276,8 @@ static void idpf_restore_features(struct idpf_vport *vport) */ static int idpf_set_real_num_queues(struct idpf_vport *vport) { - int err; - - err = netif_set_real_num_rx_queues(vport->netdev, vport->num_rxq); - if (err) - return err; - - return netif_set_real_num_tx_queues(vport->netdev, vport->num_txq); + return libeth_set_real_num_queues(vport->netdev, vport->num_rxq, + vport->num_txq); } /** diff --git a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c index dfd7cf1d9aa0ad..c6ce819b352894 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c @@ -337,10 +337,6 @@ static void idpf_tx_singleq_build_ctx_desc(struct idpf_tx_queue *txq, qw1 |= FIELD_PREP(IDPF_TXD_CTX_QW1_TSO_LEN_M, offload->tso_len); qw1 |= FIELD_PREP(IDPF_TXD_CTX_QW1_MSS_M, offload->mss); - - u64_stats_update_begin(&txq->stats_sync); - u64_stats_inc(&txq->q_stats.lso_pkts); - u64_stats_update_end(&txq->stats_sync); } desc->qw0.tunneling_params = cpu_to_le32(offload->cd_tunneling); @@ -361,6 +357,7 @@ netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, struct idpf_tx_queue *tx_q) { struct idpf_tx_offload_params offload = { }; + struct libeth_sq_xmit_stats ss = { }; struct idpf_tx_buf *first; unsigned int count; __be16 protocol; @@ -374,10 +371,7 @@ netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, count + IDPF_TX_DESCS_PER_CACHE_LINE + IDPF_TX_DESCS_FOR_CTX)) { idpf_tx_buf_hw_update(tx_q, tx_q->next_to_use, false); - - u64_stats_update_begin(&tx_q->stats_sync); - u64_stats_inc(&tx_q->q_stats.q_busy); - u64_stats_update_end(&tx_q->stats_sync); + libeth_stats_inc_one(&tx_q->stats, busy); return NETDEV_TX_BUSY; } @@ -388,7 +382,7 @@ netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, else if (protocol == htons(ETH_P_IPV6)) offload.tx_flags |= IDPF_TX_FLAGS_IPV6; - tso = idpf_tso(skb, &offload); + tso = idpf_tso(skb, &offload, &ss); if (tso < 0) goto out_drop; @@ -412,6 +406,10 @@ netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, } idpf_tx_singleq_map(tx_q, first, &offload); + libeth_stats_add_frags(&ss, count); + libeth_sq_xmit_stats_csum(&ss, skb); + libeth_sq_xmit_stats_add(&tx_q->stats, &ss); + return NETDEV_TX_OK; out_drop: @@ -508,20 +506,18 @@ static bool idpf_tx_singleq_clean(struct idpf_tx_queue *tx_q, int napi_budget, tx_q->next_to_clean = ntc; *cleaned += ss.packets; - - u64_stats_update_begin(&tx_q->stats_sync); - u64_stats_add(&tx_q->q_stats.packets, ss.packets); - u64_stats_add(&tx_q->q_stats.bytes, ss.bytes); - u64_stats_update_end(&tx_q->stats_sync); + libeth_sq_napi_stats_add(&tx_q->stats, &ss); np = netdev_priv(tx_q->netdev); nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); dont_wake = np->state != __IDPF_VPORT_UP || !netif_carrier_ok(tx_q->netdev); - __netif_txq_completed_wake(nq, ss.packets, ss.bytes, - IDPF_DESC_UNUSED(tx_q), IDPF_TX_WAKE_THRESH, - dont_wake); + if (!__netif_txq_completed_wake(nq, ss.packets, ss.bytes, + IDPF_DESC_UNUSED(tx_q), + IDPF_TX_WAKE_THRESH, + dont_wake)) + libeth_stats_inc_one(&tx_q->stats, wake); return !!budget; } @@ -590,23 +586,25 @@ static bool idpf_rx_singleq_is_non_eop(const union virtchnl2_rx_desc *rx_desc) * @skb: skb currently being received and modified * @csum_bits: checksum bits from descriptor * @decoded: the packet type decoded by hardware + * @rs: RQ polling onstack stats * * skb->protocol must be set before this function is called */ static void idpf_rx_singleq_csum(struct idpf_rx_queue *rxq, struct sk_buff *skb, struct idpf_rx_csum_decoded csum_bits, - struct libeth_rx_pt decoded) + struct libeth_rx_pt decoded, + struct libeth_rq_napi_stats *rs) { bool ipv4, ipv6; /* check if Rx checksum is enabled */ if (!libeth_rx_pt_has_checksum(rxq->netdev, decoded)) - return; + goto none; /* check if HW has decoded the packet and checksum */ if (unlikely(!csum_bits.l3l4p)) - return; + goto none; ipv4 = libeth_rx_pt_get_ip_ver(decoded) == LIBETH_RX_PT_OUTER_IPV4; ipv6 = libeth_rx_pt_get_ip_ver(decoded) == LIBETH_RX_PT_OUTER_IPV6; @@ -619,7 +617,7 @@ static void idpf_rx_singleq_csum(struct idpf_rx_queue *rxq, * headers as indicated by setting IPV6EXADD bit */ if (unlikely(ipv6 && csum_bits.ipv6exadd)) - return; + goto none; /* check for L4 errors and handle packets that were not able to be * checksummed due to arrival speed @@ -634,7 +632,7 @@ static void idpf_rx_singleq_csum(struct idpf_rx_queue *rxq, * speed, in this case the stack can compute the csum. */ if (unlikely(csum_bits.pprs)) - return; + goto none; /* If there is an outer header present that might contain a checksum * we need to bump the checksum level by 1 to reflect the fact that @@ -644,12 +642,16 @@ static void idpf_rx_singleq_csum(struct idpf_rx_queue *rxq, skb->csum_level = 1; skb->ip_summed = CHECKSUM_UNNECESSARY; + rs->csum_unnecessary++; + + return; + +none: + libeth_stats_inc_one(&rxq->stats, csum_none); return; checksum_fail: - u64_stats_update_begin(&rxq->stats_sync); - u64_stats_inc(&rxq->q_stats.hw_csum_err); - u64_stats_update_end(&rxq->stats_sync); + libeth_stats_inc_one(&rxq->stats, csum_bad); } /** @@ -786,6 +788,7 @@ static void idpf_rx_singleq_flex_hash(struct idpf_rx_queue *rx_q, * @skb: pointer to current skb being populated * @rx_desc: descriptor for skb * @ptype: packet type + * @rs: RQ polling onstack stats * * This function checks the ring, descriptor, and packet information in * order to populate the hash, checksum, VLAN, protocol, and @@ -795,7 +798,8 @@ static void idpf_rx_singleq_process_skb_fields(struct idpf_rx_queue *rx_q, struct sk_buff *skb, const union virtchnl2_rx_desc *rx_desc, - u16 ptype) + u16 ptype, + struct libeth_rq_napi_stats *rs) { struct libeth_rx_pt decoded = rx_q->rx_ptype_lkup[ptype]; struct idpf_rx_csum_decoded csum_bits; @@ -812,7 +816,7 @@ idpf_rx_singleq_process_skb_fields(struct idpf_rx_queue *rx_q, csum_bits = idpf_rx_singleq_flex_csum(rx_desc); } - idpf_rx_singleq_csum(rx_q, skb, csum_bits, decoded); + idpf_rx_singleq_csum(rx_q, skb, csum_bits, decoded, rs); skb_record_rx_queue(skb, rx_q->idx); } @@ -958,14 +962,14 @@ idpf_rx_singleq_extract_fields(const struct idpf_rx_queue *rx_q, */ static int idpf_rx_singleq_clean(struct idpf_rx_queue *rx_q, int budget) { - unsigned int total_rx_bytes = 0, total_rx_pkts = 0; + struct libeth_rq_napi_stats rs = { }; struct sk_buff *skb = rx_q->skb; u16 ntc = rx_q->next_to_clean; u16 cleaned_count = 0; bool failure = false; /* Process Rx packets bounded by budget */ - while (likely(total_rx_pkts < (unsigned int)budget)) { + while (likely(rs.packets < budget)) { struct idpf_rx_extracted fields = { }; union virtchnl2_rx_desc *rx_desc; struct idpf_rx_buf *rx_buf; @@ -1030,18 +1034,19 @@ static int idpf_rx_singleq_clean(struct idpf_rx_queue *rx_q, int budget) } /* probably a little skewed due to removing CRC */ - total_rx_bytes += skb->len; + rs.bytes += skb->len; /* protocol */ idpf_rx_singleq_process_skb_fields(rx_q, skb, - rx_desc, fields.rx_ptype); + rx_desc, fields.rx_ptype, + &rs); /* send completed skb up the stack */ napi_gro_receive(rx_q->pp->p.napi, skb); skb = NULL; /* update budget accounting */ - total_rx_pkts++; + rs.packets++; } rx_q->skb = skb; @@ -1052,13 +1057,10 @@ static int idpf_rx_singleq_clean(struct idpf_rx_queue *rx_q, int budget) if (cleaned_count) failure = idpf_rx_singleq_buf_hw_alloc_all(rx_q, cleaned_count); - u64_stats_update_begin(&rx_q->stats_sync); - u64_stats_add(&rx_q->q_stats.packets, total_rx_pkts); - u64_stats_add(&rx_q->q_stats.bytes, total_rx_bytes); - u64_stats_update_end(&rx_q->stats_sync); + libeth_rq_napi_stats_add(&rx_q->stats, &rs); /* guarantee a trip back through this routine if there was a failure */ - return failure ? budget : (int)total_rx_pkts; + return failure ? budget : rs.packets; } /** diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 536e41a4432809..ea8914a42899da 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -132,6 +132,8 @@ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) static void idpf_tx_desc_rel(struct idpf_tx_queue *txq) { idpf_tx_buf_rel_all(txq); + + libeth_sq_stats_deinit(txq->netdev, txq->idx); netdev_tx_reset_subqueue(txq->netdev, txq->idx); if (!txq->desc_ring) @@ -265,6 +267,8 @@ static int idpf_tx_desc_alloc(const struct idpf_vport *vport, tx_q->next_to_clean = 0; idpf_queue_set(GEN_CHK, tx_q); + libeth_sq_stats_init(vport->netdev, &tx_q->stats, tx_q->idx); + return 0; err_alloc: @@ -480,6 +484,8 @@ static void idpf_rx_desc_rel(struct idpf_rx_queue *rxq, struct device *dev, if (!idpf_is_queue_model_split(model)) idpf_rx_buf_rel_all(rxq); + libeth_rq_stats_deinit(rxq->netdev, rxq->idx); + rxq->next_to_alloc = 0; rxq->next_to_clean = 0; rxq->next_to_use = 0; @@ -880,6 +886,8 @@ static int idpf_rx_desc_alloc(const struct idpf_vport *vport, rxq->next_to_use = 0; idpf_queue_set(GEN_CHK, rxq); + libeth_rq_stats_init(vport->netdev, &rxq->stats, rxq->idx); + return 0; } @@ -1663,7 +1671,7 @@ static void idpf_tx_handle_sw_marker(struct idpf_tx_queue *tx_q) */ static void idpf_tx_clean_stashed_bufs(struct idpf_tx_queue *txq, u16 compl_tag, - struct idpf_cleaned_stats *cleaned, + struct libeth_sq_napi_stats *cleaned, int budget) { struct idpf_tx_stash *stash; @@ -1764,7 +1772,7 @@ do { \ */ static bool idpf_tx_splitq_clean(struct idpf_tx_queue *tx_q, u16 end, int napi_budget, - struct idpf_cleaned_stats *cleaned, + struct libeth_sq_napi_stats *cleaned, bool descs_only) { union idpf_tx_flex_desc *next_pending_desc = NULL; @@ -1856,7 +1864,7 @@ do { \ * this completion tag. */ static bool idpf_tx_clean_buf_ring(struct idpf_tx_queue *txq, u16 compl_tag, - struct idpf_cleaned_stats *cleaned, + struct libeth_sq_napi_stats *cleaned, int budget) { u16 idx = compl_tag & txq->compl_tag_bufid_m; @@ -1934,7 +1942,7 @@ static bool idpf_tx_clean_buf_ring(struct idpf_tx_queue *txq, u16 compl_tag, */ static void idpf_tx_handle_rs_completion(struct idpf_tx_queue *txq, struct idpf_splitq_tx_compl_desc *desc, - struct idpf_cleaned_stats *cleaned, + struct libeth_sq_napi_stats *cleaned, int budget) { u16 compl_tag; @@ -1978,7 +1986,7 @@ static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget, ntc -= complq->desc_count; do { - struct idpf_cleaned_stats cleaned_stats = { }; + struct libeth_sq_napi_stats cleaned_stats = { }; struct idpf_tx_queue *tx_q; int rel_tx_qid; u16 hw_head; @@ -2024,13 +2032,10 @@ static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget, goto fetch_next_desc; } - u64_stats_update_begin(&tx_q->stats_sync); - u64_stats_add(&tx_q->q_stats.packets, cleaned_stats.packets); - u64_stats_add(&tx_q->q_stats.bytes, cleaned_stats.bytes); + libeth_sq_napi_stats_add(&tx_q->stats, &cleaned_stats); tx_q->cleaned_pkts += cleaned_stats.packets; tx_q->cleaned_bytes += cleaned_stats.bytes; complq->num_completions++; - u64_stats_update_end(&tx_q->stats_sync); fetch_next_desc: tx_desc++; @@ -2073,9 +2078,12 @@ static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget, np->state != __IDPF_VPORT_UP || !netif_carrier_ok(tx_q->netdev); /* Check if the TXQ needs to and can be restarted */ - __netif_txq_completed_wake(nq, tx_q->cleaned_pkts, tx_q->cleaned_bytes, - IDPF_DESC_UNUSED(tx_q), IDPF_TX_WAKE_THRESH, - dont_wake); + if (!__netif_txq_completed_wake(nq, tx_q->cleaned_pkts, + tx_q->cleaned_bytes, + IDPF_DESC_UNUSED(tx_q), + IDPF_TX_WAKE_THRESH, + dont_wake)) + libeth_stats_inc_one(&tx_q->stats, wake); /* Reset cleaned stats for the next time this queue is * cleaned @@ -2158,11 +2166,8 @@ static int idpf_tx_maybe_stop_splitq(struct idpf_tx_queue *tx_q, splitq_stop: netif_stop_subqueue(tx_q->netdev, tx_q->idx); - out: - u64_stats_update_begin(&tx_q->stats_sync); - u64_stats_inc(&tx_q->q_stats.q_busy); - u64_stats_update_end(&tx_q->stats_sync); + libeth_stats_inc_one(&tx_q->stats, stop); return -EBUSY; } @@ -2185,11 +2190,8 @@ void idpf_tx_buf_hw_update(struct idpf_tx_queue *tx_q, u32 val, nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); tx_q->next_to_use = val; - if (idpf_tx_maybe_stop_common(tx_q, IDPF_TX_DESC_NEEDED)) { - u64_stats_update_begin(&tx_q->stats_sync); - u64_stats_inc(&tx_q->q_stats.q_busy); - u64_stats_update_end(&tx_q->stats_sync); - } + if (idpf_tx_maybe_stop_common(tx_q, IDPF_TX_DESC_NEEDED)) + libeth_stats_inc_one(&tx_q->stats, stop); /* Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only @@ -2242,9 +2244,7 @@ unsigned int idpf_tx_desc_count_required(struct idpf_tx_queue *txq, return 0; count = idpf_size_to_txd_count(skb->len); - u64_stats_update_begin(&txq->stats_sync); - u64_stats_inc(&txq->q_stats.linearize); - u64_stats_update_end(&txq->stats_sync); + libeth_stats_inc_one(&txq->stats, linearized); } return count; @@ -2266,9 +2266,7 @@ void idpf_tx_dma_map_error(struct idpf_tx_queue *txq, struct sk_buff *skb, .ss = &ss, }; - u64_stats_update_begin(&txq->stats_sync); - u64_stats_inc(&txq->q_stats.dma_map_errs); - u64_stats_update_end(&txq->stats_sync); + libeth_stats_inc_one(&txq->stats, dma_map_errs); /* clear dma mappings for failed tx_buf map */ for (;;) { @@ -2508,11 +2506,13 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, * idpf_tso - computes mss and TSO length to prepare for TSO * @skb: pointer to skb * @off: pointer to struct that holds offload parameters + * @ss: SQ xmit onstack stats * * Returns error (negative) if TSO was requested but cannot be applied to the * given skb, 0 if TSO does not apply to the given skb, or 1 otherwise. */ -int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off) +int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off, + struct libeth_sq_xmit_stats *ss) { const struct skb_shared_info *shinfo; union { @@ -2559,6 +2559,8 @@ int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off) csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen)); off->tso_hdr_len = __tcp_hdrlen(l4.tcp) + l4_start; + + ss->tso++; break; case SKB_GSO_UDP_L4: csum_replace_by_diff(&l4.udp->check, @@ -2566,6 +2568,8 @@ int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off) /* compute length of segmentation header */ off->tso_hdr_len = sizeof(struct udphdr) + l4_start; l4.udp->len = htons(shinfo->gso_size + sizeof(struct udphdr)); + + ss->uso++; break; default: return -EINVAL; @@ -2577,6 +2581,9 @@ int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off) off->tx_flags |= IDPF_TX_FLAGS_TSO; + ss->hw_gso_packets++; + ss->hw_gso_bytes += skb->len; + return 1; } @@ -2715,10 +2722,7 @@ idpf_tx_splitq_get_ctx_desc(struct idpf_tx_queue *txq) */ netdev_tx_t idpf_tx_drop_skb(struct idpf_tx_queue *tx_q, struct sk_buff *skb) { - u64_stats_update_begin(&tx_q->stats_sync); - u64_stats_inc(&tx_q->q_stats.skb_drops); - u64_stats_update_end(&tx_q->stats_sync); - + libeth_stats_inc_one(&tx_q->stats, drops); idpf_tx_buf_hw_update(tx_q, tx_q->next_to_use, false); dev_kfree_skb(skb); @@ -2737,6 +2741,7 @@ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, struct idpf_tx_queue *tx_q) { struct idpf_tx_splitq_params tx_params = { }; + struct libeth_sq_xmit_stats ss = { }; struct idpf_tx_buf *first; unsigned int count; int tso; @@ -2745,7 +2750,7 @@ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, if (unlikely(!count)) return idpf_tx_drop_skb(tx_q, skb); - tso = idpf_tso(skb, &tx_params.offload); + tso = idpf_tso(skb, &tx_params.offload, &ss); if (unlikely(tso < 0)) return idpf_tx_drop_skb(tx_q, skb); @@ -2753,6 +2758,7 @@ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, count += (IDPF_TX_DESCS_PER_CACHE_LINE + tso); if (idpf_tx_maybe_stop_splitq(tx_q, count)) { idpf_tx_buf_hw_update(tx_q, tx_q->next_to_use, false); + libeth_stats_inc_one(&tx_q->stats, busy); return NETDEV_TX_BUSY; } @@ -2772,10 +2778,6 @@ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, cpu_to_le16(tx_params.offload.mss & IDPF_TXD_FLEX_CTX_MSS_RT_M); ctx_desc->tso.qw0.hdr_len = tx_params.offload.tso_hdr_len; - - u64_stats_update_begin(&tx_q->stats_sync); - u64_stats_inc(&tx_q->q_stats.lso_pkts); - u64_stats_update_end(&tx_q->stats_sync); } /* record the location of the first descriptor for this packet */ @@ -2817,6 +2819,10 @@ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, idpf_tx_splitq_map(tx_q, &tx_params, first); + libeth_stats_add_frags(&ss, count); + libeth_sq_xmit_stats_csum(&ss, skb); + libeth_sq_xmit_stats_add(&tx_q->stats, &ss); + return NETDEV_TX_OK; } @@ -2885,41 +2891,44 @@ idpf_rx_hash(const struct idpf_rx_queue *rxq, struct sk_buff *skb, * @skb: pointer to current skb being populated * @csum_bits: checksum fields extracted from the descriptor * @decoded: Decoded Rx packet type related fields + * @rs: RQ polling onstack stats * * skb->protocol must be set before this function is called */ static void idpf_rx_csum(struct idpf_rx_queue *rxq, struct sk_buff *skb, struct idpf_rx_csum_decoded csum_bits, - struct libeth_rx_pt decoded) + struct libeth_rx_pt decoded, + struct libeth_rq_napi_stats *rs) { bool ipv4, ipv6; /* check if Rx checksum is enabled */ if (!libeth_rx_pt_has_checksum(rxq->netdev, decoded)) - return; + goto none; /* check if HW has decoded the packet and checksum */ if (unlikely(!csum_bits.l3l4p)) - return; + goto none; ipv4 = libeth_rx_pt_get_ip_ver(decoded) == LIBETH_RX_PT_OUTER_IPV4; ipv6 = libeth_rx_pt_get_ip_ver(decoded) == LIBETH_RX_PT_OUTER_IPV6; if (unlikely(ipv4 && (csum_bits.ipe || csum_bits.eipe))) - goto checksum_fail; + goto bad; if (unlikely(ipv6 && csum_bits.ipv6exadd)) - return; + goto none; /* check for L4 errors and handle packets that were not able to be * checksummed */ if (unlikely(csum_bits.l4e)) - goto checksum_fail; + goto bad; if (csum_bits.raw_csum_inv || decoded.inner_prot == LIBETH_RX_PT_INNER_SCTP) { skb->ip_summed = CHECKSUM_UNNECESSARY; + rs->csum_unnecessary++; return; } @@ -2928,10 +2937,12 @@ static void idpf_rx_csum(struct idpf_rx_queue *rxq, struct sk_buff *skb, return; -checksum_fail: - u64_stats_update_begin(&rxq->stats_sync); - u64_stats_inc(&rxq->q_stats.hw_csum_err); - u64_stats_update_end(&rxq->stats_sync); +none: + libeth_stats_inc_one(&rxq->stats, csum_none); + return; + +bad: + libeth_stats_inc_one(&rxq->stats, csum_bad); } /** @@ -2973,6 +2984,7 @@ idpf_rx_splitq_extract_csum_bits(const struct virtchnl2_rx_flex_desc_adv_nic_3 * * @skb : pointer to current skb being populated * @rx_desc: Receive descriptor * @decoded: Decoded Rx packet type related fields + * @rs: RQ polling onstack stats * * Return 0 on success and error code on failure * @@ -2981,7 +2993,8 @@ idpf_rx_splitq_extract_csum_bits(const struct virtchnl2_rx_flex_desc_adv_nic_3 * */ static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc, - struct libeth_rx_pt decoded) + struct libeth_rx_pt decoded, + struct libeth_rq_napi_stats *rs) { u16 rsc_segments, rsc_seg_len; bool ipv4, ipv6; @@ -3033,9 +3046,8 @@ static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, tcp_gro_complete(skb); - u64_stats_update_begin(&rxq->stats_sync); - u64_stats_inc(&rxq->q_stats.rsc_pkts); - u64_stats_update_end(&rxq->stats_sync); + rs->hw_gro_packets++; + rs->hw_gro_bytes += skb->len; return 0; } @@ -3045,6 +3057,7 @@ static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, * @rxq: Rx descriptor ring packet is being transacted on * @skb: pointer to current skb being populated * @rx_desc: Receive descriptor + * @rs: RQ polling onstack stats * * This function checks the ring, descriptor, and packet information in * order to populate the hash, checksum, protocol, and @@ -3052,7 +3065,8 @@ static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, */ static int idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, - const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc) + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc, + struct libeth_rq_napi_stats *rs) { struct idpf_rx_csum_decoded csum_bits; struct libeth_rx_pt decoded; @@ -3069,10 +3083,10 @@ idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, if (le16_get_bits(rx_desc->hdrlen_flags, VIRTCHNL2_RX_FLEX_DESC_ADV_RSC_M)) - return idpf_rx_rsc(rxq, skb, rx_desc, decoded); + return idpf_rx_rsc(rxq, skb, rx_desc, decoded, rs); csum_bits = idpf_rx_splitq_extract_csum_bits(rx_desc); - idpf_rx_csum(rxq, skb, csum_bits, decoded); + idpf_rx_csum(rxq, skb, csum_bits, decoded, rs); skb_record_rx_queue(skb, rxq->idx); @@ -3203,13 +3217,13 @@ static bool idpf_rx_splitq_is_eop(struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_de */ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) { - int total_rx_bytes = 0, total_rx_pkts = 0; struct idpf_buf_queue *rx_bufq = NULL; + struct libeth_rq_napi_stats rs = { }; struct sk_buff *skb = rxq->skb; u16 ntc = rxq->next_to_clean; /* Process Rx packets bounded by budget */ - while (likely(total_rx_pkts < budget)) { + while (likely(rs.packets < budget)) { struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc; struct libeth_fqe *hdr, *rx_buf = NULL; struct idpf_sw_queue *refillq = NULL; @@ -3239,9 +3253,7 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) rx_desc->rxdid_ucast); if (rxdid != VIRTCHNL2_RXDID_2_FLEX_SPLITQ) { IDPF_RX_BUMP_NTC(rxq, ntc); - u64_stats_update_begin(&rxq->stats_sync); - u64_stats_inc(&rxq->q_stats.bad_descs); - u64_stats_update_end(&rxq->stats_sync); + libeth_stats_inc_one(&rxq->stats, dma_errs); continue; } @@ -3283,9 +3295,7 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) hdr_len = idpf_rx_hsplit_wa(hdr, rx_buf, pkt_len); pkt_len -= hdr_len; - u64_stats_update_begin(&rxq->stats_sync); - u64_stats_inc(&rxq->q_stats.hsplit_buf_ovf); - u64_stats_update_end(&rxq->stats_sync); + libeth_stats_inc_one(&rxq->stats, hsplit_errs); } if (libeth_rx_sync_for_cpu(hdr, hdr_len)) { @@ -3293,13 +3303,14 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) if (!skb) break; - u64_stats_update_begin(&rxq->stats_sync); - u64_stats_inc(&rxq->q_stats.hsplit_pkts); - u64_stats_update_end(&rxq->stats_sync); + rs.hsplit++; } hdr->page = NULL; + if (!pkt_len) + rs.hsplit_linear++; + payload: if (!libeth_rx_sync_for_cpu(rx_buf, pkt_len)) goto skip_data; @@ -3330,10 +3341,11 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) } /* probably a little skewed due to removing CRC */ - total_rx_bytes += skb->len; + rs.bytes += skb->len; /* protocol */ - if (unlikely(idpf_rx_process_skb_fields(rxq, skb, rx_desc))) { + if (unlikely(idpf_rx_process_skb_fields(rxq, skb, rx_desc, + &rs))) { dev_kfree_skb_any(skb); skb = NULL; continue; @@ -3344,19 +3356,15 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) skb = NULL; /* update budget accounting */ - total_rx_pkts++; + rs.packets++; } rxq->next_to_clean = ntc; rxq->skb = skb; - u64_stats_update_begin(&rxq->stats_sync); - u64_stats_add(&rxq->q_stats.packets, total_rx_pkts); - u64_stats_add(&rxq->q_stats.bytes, total_rx_bytes); - u64_stats_update_end(&rxq->stats_sync); + libeth_rq_napi_stats_add(&rxq->stats, &rs); - /* guarantee a trip back through this routine if there was a failure */ - return total_rx_pkts; + return rs.packets; } /** @@ -3666,10 +3674,10 @@ static void idpf_net_dim(struct idpf_q_vector *q_vector) unsigned int start; do { - start = u64_stats_fetch_begin(&txq->stats_sync); - packets += u64_stats_read(&txq->q_stats.packets); - bytes += u64_stats_read(&txq->q_stats.bytes); - } while (u64_stats_fetch_retry(&txq->stats_sync, start)); + start = u64_stats_fetch_begin(&txq->stats.syncp); + packets += u64_stats_read(&txq->stats.packets); + bytes += u64_stats_read(&txq->stats.bytes); + } while (u64_stats_fetch_retry(&txq->stats.syncp, start)); } idpf_update_dim_sample(q_vector, &dim_sample, &q_vector->tx_dim, @@ -3685,10 +3693,10 @@ static void idpf_net_dim(struct idpf_q_vector *q_vector) unsigned int start; do { - start = u64_stats_fetch_begin(&rxq->stats_sync); - packets += u64_stats_read(&rxq->q_stats.packets); - bytes += u64_stats_read(&rxq->q_stats.bytes); - } while (u64_stats_fetch_retry(&rxq->stats_sync, start)); + start = u64_stats_fetch_begin(&rxq->stats.syncp); + packets += u64_stats_read(&rxq->stats.packets); + bytes += u64_stats_read(&rxq->stats.bytes); + } while (u64_stats_fetch_retry(&rxq->stats.syncp, start)); } idpf_update_dim_sample(q_vector, &dim_sample, &q_vector->rx_dim, diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index b4a87f8661a8af..8188f5cb418baa 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -143,6 +144,8 @@ do { \ #define IDPF_TX_FLAGS_IPV6 BIT(2) #define IDPF_TX_FLAGS_TUNNEL BIT(3) +struct libeth_sq_xmit_stats; + union idpf_tx_flex_desc { struct idpf_flex_tx_desc q; /* queue based scheduling */ struct idpf_flex_tx_sched_desc flow; /* flow based scheduling */ @@ -441,28 +444,6 @@ libeth_cacheline_set_assert(struct idpf_q_vector, 112, 424 + 2 * sizeof(struct dim), 8 + sizeof(cpumask_var_t)); -struct idpf_rx_queue_stats { - u64_stats_t packets; - u64_stats_t bytes; - u64_stats_t rsc_pkts; - u64_stats_t hw_csum_err; - u64_stats_t hsplit_pkts; - u64_stats_t hsplit_buf_ovf; - u64_stats_t bad_descs; -}; - -struct idpf_tx_queue_stats { - u64_stats_t packets; - u64_stats_t bytes; - u64_stats_t lso_pkts; - u64_stats_t linearize; - u64_stats_t q_busy; - u64_stats_t skb_drops; - u64_stats_t dma_map_errs; -}; - -#define idpf_cleaned_stats libeth_sq_napi_stats - #define IDPF_ITR_DYNAMIC 1 #define IDPF_ITR_MAX 0x1FE0 #define IDPF_ITR_20K 0x0032 @@ -508,10 +489,9 @@ struct idpf_txq_stash { * @next_to_use: Next descriptor to use * @next_to_clean: Next descriptor to clean * @next_to_alloc: RX buffer to allocate at - * @skb: Pointer to the skb * @truesize: data buffer truesize in singleq - * @stats_sync: See struct u64_stats_sync - * @q_stats: See union idpf_rx_queue_stats + * @skb: Pointer to the skb + * @stats: per-queue RQ stats * @q_id: Queue id * @size: Length of descriptor ring in bytes * @dma: Physical address of ring @@ -551,15 +531,14 @@ struct idpf_rx_queue { __cacheline_group_end_aligned(read_mostly); __cacheline_group_begin_aligned(read_write); - u16 next_to_use; - u16 next_to_clean; - u16 next_to_alloc; + u32 next_to_use; + u32 next_to_clean; + u32 next_to_alloc; - struct sk_buff *skb; u32 truesize; + struct sk_buff *skb; - struct u64_stats_sync stats_sync; - struct idpf_rx_queue_stats q_stats; + struct libeth_rq_stats stats; __cacheline_group_end_aligned(read_write); __cacheline_group_begin_aligned(cold); @@ -576,7 +555,7 @@ struct idpf_rx_queue { __cacheline_group_end_aligned(cold); }; libeth_cacheline_set_assert(struct idpf_rx_queue, 64, - 80 + sizeof(struct u64_stats_sync), + 32 + sizeof(struct libeth_rq_stats), 32); /** @@ -633,8 +612,7 @@ libeth_cacheline_set_assert(struct idpf_rx_queue, 64, * @compl_tag_bufid_m: Completion tag buffer id mask * @compl_tag_cur_gen: Used to keep track of current completion tag generation * @compl_tag_gen_max: To determine when compl_tag_cur_gen should be reset - * @stats_sync: See struct u64_stats_sync - * @q_stats: See union idpf_tx_queue_stats + * @stats: per-queue SQ stats * @q_id: Queue id * @size: Length of descriptor ring in bytes * @dma: Physical address of ring @@ -682,8 +660,7 @@ struct idpf_tx_queue { u16 compl_tag_cur_gen; u16 compl_tag_gen_max; - struct u64_stats_sync stats_sync; - struct idpf_tx_queue_stats q_stats; + struct libeth_sq_stats stats; __cacheline_group_end_aligned(read_write); __cacheline_group_begin_aligned(cold); @@ -695,7 +672,7 @@ struct idpf_tx_queue { __cacheline_group_end_aligned(cold); }; libeth_cacheline_set_assert(struct idpf_tx_queue, 64, - 88 + sizeof(struct u64_stats_sync), + 32 + sizeof(struct libeth_sq_stats), 24); /** @@ -1051,7 +1028,8 @@ netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, netdev_tx_t idpf_tx_start(struct sk_buff *skb, struct net_device *netdev); bool idpf_rx_singleq_buf_hw_alloc_all(struct idpf_rx_queue *rxq, u16 cleaned_count); -int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off); +int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off, + struct libeth_sq_xmit_stats *ss); static inline bool idpf_tx_maybe_stop_common(struct idpf_tx_queue *tx_q, u32 needed) diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c index 70986e12da28e3..3fcc8ac70b7449 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c @@ -2292,47 +2292,44 @@ int idpf_send_set_sriov_vfs_msg(struct idpf_adapter *adapter, u16 num_vfs) */ int idpf_send_get_stats_msg(struct idpf_vport *vport) { + struct virtchnl2_vport_stats *stats_msg = &vport->vport_stats; struct idpf_netdev_priv *np = netdev_priv(vport->netdev); struct rtnl_link_stats64 *netstats = &np->netstats; - struct virtchnl2_vport_stats stats_msg = {}; struct idpf_vc_xn_params xn_params = {}; ssize_t reply_sz; - /* Don't send get_stats message if the link is down */ if (np->state <= __IDPF_VPORT_DOWN) return 0; - stats_msg.vport_id = cpu_to_le32(vport->vport_id); + stats_msg->vport_id = cpu_to_le32(vport->vport_id); xn_params.vc_op = VIRTCHNL2_OP_GET_STATS; - xn_params.send_buf.iov_base = &stats_msg; - xn_params.send_buf.iov_len = sizeof(stats_msg); + xn_params.send_buf.iov_base = stats_msg; + xn_params.send_buf.iov_len = sizeof(*stats_msg); xn_params.recv_buf = xn_params.send_buf; xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); if (reply_sz < 0) return reply_sz; - if (reply_sz < sizeof(stats_msg)) + if (reply_sz < sizeof(*stats_msg)) return -EIO; spin_lock_bh(&np->stats_lock); - netstats->rx_packets = le64_to_cpu(stats_msg.rx_unicast) + - le64_to_cpu(stats_msg.rx_multicast) + - le64_to_cpu(stats_msg.rx_broadcast); - netstats->tx_packets = le64_to_cpu(stats_msg.tx_unicast) + - le64_to_cpu(stats_msg.tx_multicast) + - le64_to_cpu(stats_msg.tx_broadcast); - netstats->rx_bytes = le64_to_cpu(stats_msg.rx_bytes); - netstats->tx_bytes = le64_to_cpu(stats_msg.tx_bytes); - netstats->rx_errors = le64_to_cpu(stats_msg.rx_errors); - netstats->tx_errors = le64_to_cpu(stats_msg.tx_errors); - netstats->rx_dropped = le64_to_cpu(stats_msg.rx_discards); - netstats->tx_dropped = le64_to_cpu(stats_msg.tx_discards); - - vport->port_stats.vport_stats = stats_msg; + netstats->rx_packets = le64_to_cpu(stats_msg->rx_unicast) + + le64_to_cpu(stats_msg->rx_multicast) + + le64_to_cpu(stats_msg->rx_broadcast); + netstats->tx_packets = le64_to_cpu(stats_msg->tx_unicast) + + le64_to_cpu(stats_msg->tx_multicast) + + le64_to_cpu(stats_msg->tx_broadcast); + netstats->rx_bytes = le64_to_cpu(stats_msg->rx_bytes); + netstats->tx_bytes = le64_to_cpu(stats_msg->tx_bytes); + netstats->rx_errors = le64_to_cpu(stats_msg->rx_errors); + netstats->tx_errors = le64_to_cpu(stats_msg->tx_errors); + netstats->rx_dropped = le64_to_cpu(stats_msg->rx_discards); + netstats->tx_dropped = le64_to_cpu(stats_msg->tx_discards); spin_unlock_bh(&np->stats_lock); diff --git a/drivers/net/ethernet/intel/idpf/virtchnl2.h b/drivers/net/ethernet/intel/idpf/virtchnl2.h index 63deb120359cfa..19d62cfc17bea0 100644 --- a/drivers/net/ethernet/intel/idpf/virtchnl2.h +++ b/drivers/net/ethernet/intel/idpf/virtchnl2.h @@ -1021,6 +1021,7 @@ VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_get_ptype_info); * struct virtchnl2_vport_stats - Vport statistics. * @vport_id: Vport id. * @pad: Padding. + * @counters: grouped counters for bulk operations * @rx_bytes: Received bytes. * @rx_unicast: Received unicast packets. * @rx_multicast: Received multicast packets. @@ -1045,21 +1046,23 @@ VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_get_ptype_info); struct virtchnl2_vport_stats { __le32 vport_id; u8 pad[4]; - __le64 rx_bytes; - __le64 rx_unicast; - __le64 rx_multicast; - __le64 rx_broadcast; - __le64 rx_discards; - __le64 rx_errors; - __le64 rx_unknown_protocol; - __le64 tx_bytes; - __le64 tx_unicast; - __le64 tx_multicast; - __le64 tx_broadcast; - __le64 tx_discards; - __le64 tx_errors; - __le64 rx_invalid_frame_length; - __le64 rx_overflow_drop; + struct_group(counters, + __le64 rx_bytes; + __le64 rx_unicast; + __le64 rx_multicast; + __le64 rx_broadcast; + __le64 rx_discards; + __le64 rx_errors; + __le64 rx_unknown_protocol; + __le64 tx_bytes; + __le64 tx_unicast; + __le64 tx_multicast; + __le64 tx_broadcast; + __le64 tx_discards; + __le64 tx_errors; + __le64 rx_invalid_frame_length; + __le64 rx_overflow_drop; + ); }; VIRTCHNL2_CHECK_STRUCT_LEN(128, virtchnl2_vport_stats); From 1db8f949f4e117bc41fd2dc1e6fa24ab3798e096 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Mon, 11 Dec 2023 18:17:12 +0100 Subject: [PATCH 19/49] bpf, xdp: constify some bpf_prog * function arguments In lots of places, bpf_prog pointer is used only for tracing or other stuff that doesn't modify the structure itself. Same for net_device. Address at least some of them and add `const` attributes there. The object code didn't change, but that may prevent unwanted data modifications and also allow more helpers to have const arguments. Signed-off-by: Alexander Lobakin --- include/linux/bpf.h | 12 ++++++------ include/linux/filter.h | 9 +++++---- include/linux/netdevice.h | 6 +++--- include/linux/skbuff.h | 3 ++- kernel/bpf/devmap.c | 8 ++++---- net/core/dev.c | 10 +++++----- net/core/filter.c | 29 ++++++++++++++++------------- net/core/skbuff.c | 2 +- 8 files changed, 42 insertions(+), 37 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 4f1d4a97b9d14d..0c038babb21b04 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2502,10 +2502,10 @@ int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf, int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, struct bpf_map *map, bool exclude_ingress); int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, - struct bpf_prog *xdp_prog); + const struct bpf_prog *xdp_prog); int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, - struct bpf_prog *xdp_prog, struct bpf_map *map, - bool exclude_ingress); + const struct bpf_prog *xdp_prog, + struct bpf_map *map, bool exclude_ingress); void __cpu_map_flush(struct list_head *flush_list); int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf, @@ -2769,15 +2769,15 @@ struct sk_buff; static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, - struct bpf_prog *xdp_prog) + const struct bpf_prog *xdp_prog) { return 0; } static inline int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, - struct bpf_prog *xdp_prog, struct bpf_map *map, - bool exclude_ingress) + const struct bpf_prog *xdp_prog, + struct bpf_map *map, bool exclude_ingress) { return 0; } diff --git a/include/linux/filter.h b/include/linux/filter.h index b6672ff61407fc..c70fdb3b496cdf 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1168,17 +1168,18 @@ static inline int xdp_ok_fwd_dev(const struct net_device *fwd, * This does not appear to be a real limitation for existing software. */ int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb, - struct xdp_buff *xdp, struct bpf_prog *prog); + struct xdp_buff *xdp, const struct bpf_prog *prog); int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, - struct bpf_prog *prog); + const struct bpf_prog *prog); int xdp_do_redirect_frame(struct net_device *dev, struct xdp_buff *xdp, struct xdp_frame *xdpf, - struct bpf_prog *prog); + const struct bpf_prog *prog); void xdp_do_flush(void); -void bpf_warn_invalid_xdp_action(struct net_device *dev, struct bpf_prog *prog, u32 act); +void bpf_warn_invalid_xdp_action(const struct net_device *dev, + const struct bpf_prog *prog, u32 act); #ifdef CONFIG_INET struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk, diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 301e6afd07803a..649ccbb5203df2 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3900,9 +3900,9 @@ static inline void dev_consume_skb_any(struct sk_buff *skb) } u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, - struct bpf_prog *xdp_prog); -void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog); -int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff **pskb); + const struct bpf_prog *xdp_prog); +void generic_xdp_tx(struct sk_buff *skb, const struct bpf_prog *xdp_prog); +int do_xdp_generic(const struct bpf_prog *xdp_prog, struct sk_buff **pskb); int netif_rx(struct sk_buff *skb); int __netif_rx(struct sk_buff *skb); diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 9c29bdd5596d89..08106a5653f3ac 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3529,7 +3529,8 @@ static inline struct page *skb_frag_page(const skb_frag_t *frag) int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, unsigned int headroom); int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, - struct bpf_prog *prog); + const struct bpf_prog *prog); + /** * skb_frag_address - gets the address of the data contained in a paged fragment * @frag: the paged fragment buffer diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index 9e0e3b0a18e406..f634b87aa0fa47 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -675,7 +675,7 @@ int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, } int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, - struct bpf_prog *xdp_prog) + const struct bpf_prog *xdp_prog) { int err; @@ -698,7 +698,7 @@ int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, static int dev_map_redirect_clone(struct bpf_dtab_netdev *dst, struct sk_buff *skb, - struct bpf_prog *xdp_prog) + const struct bpf_prog *xdp_prog) { struct sk_buff *nskb; int err; @@ -717,8 +717,8 @@ static int dev_map_redirect_clone(struct bpf_dtab_netdev *dst, } int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, - struct bpf_prog *xdp_prog, struct bpf_map *map, - bool exclude_ingress) + const struct bpf_prog *xdp_prog, + struct bpf_map *map, bool exclude_ingress) { struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); struct bpf_dtab_netdev *dst, *last_dst = NULL; diff --git a/net/core/dev.c b/net/core/dev.c index ab2bad1f83a74e..a8d24d4b5d4a57 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4920,7 +4920,7 @@ static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb) } u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, - struct bpf_prog *xdp_prog) + const struct bpf_prog *xdp_prog) { void *orig_data, *orig_data_end, *hard_start; struct netdev_rx_queue *rxqueue; @@ -5022,7 +5022,7 @@ u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, } static int -netif_skb_check_for_xdp(struct sk_buff **pskb, struct bpf_prog *prog) +netif_skb_check_for_xdp(struct sk_buff **pskb, const struct bpf_prog *prog) { struct sk_buff *skb = *pskb; int err, hroom, troom; @@ -5046,7 +5046,7 @@ netif_skb_check_for_xdp(struct sk_buff **pskb, struct bpf_prog *prog) static u32 netif_receive_generic_xdp(struct sk_buff **pskb, struct xdp_buff *xdp, - struct bpf_prog *xdp_prog) + const struct bpf_prog *xdp_prog) { struct sk_buff *skb = *pskb; u32 mac_len, act = XDP_DROP; @@ -5099,7 +5099,7 @@ static u32 netif_receive_generic_xdp(struct sk_buff **pskb, * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX * queues, so they do not have this starvation issue. */ -void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog) +void generic_xdp_tx(struct sk_buff *skb, const struct bpf_prog *xdp_prog) { struct net_device *dev = skb->dev; struct netdev_queue *txq; @@ -5124,7 +5124,7 @@ void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog) static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key); -int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff **pskb) +int do_xdp_generic(const struct bpf_prog *xdp_prog, struct sk_buff **pskb) { struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; diff --git a/net/core/filter.c b/net/core/filter.c index 4cf1d34f761728..21ba2fa3dfacd3 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -4340,9 +4340,9 @@ u32 xdp_master_redirect(struct xdp_buff *xdp) EXPORT_SYMBOL_GPL(xdp_master_redirect); static inline int __xdp_do_redirect_xsk(struct bpf_redirect_info *ri, - struct net_device *dev, + const struct net_device *dev, struct xdp_buff *xdp, - struct bpf_prog *xdp_prog) + const struct bpf_prog *xdp_prog) { enum bpf_map_type map_type = ri->map_type; void *fwd = ri->tgt_value; @@ -4363,10 +4363,10 @@ static inline int __xdp_do_redirect_xsk(struct bpf_redirect_info *ri, return err; } -static __always_inline int __xdp_do_redirect_frame(struct bpf_redirect_info *ri, - struct net_device *dev, - struct xdp_frame *xdpf, - struct bpf_prog *xdp_prog) +static __always_inline int +__xdp_do_redirect_frame(struct bpf_redirect_info *ri, struct net_device *dev, + struct xdp_frame *xdpf, + const struct bpf_prog *xdp_prog) { enum bpf_map_type map_type = ri->map_type; void *fwd = ri->tgt_value; @@ -4435,7 +4435,7 @@ static __always_inline int __xdp_do_redirect_frame(struct bpf_redirect_info *ri, } int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, - struct bpf_prog *xdp_prog) + const struct bpf_prog *xdp_prog) { struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); enum bpf_map_type map_type = ri->map_type; @@ -4449,7 +4449,8 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, EXPORT_SYMBOL_GPL(xdp_do_redirect); int xdp_do_redirect_frame(struct net_device *dev, struct xdp_buff *xdp, - struct xdp_frame *xdpf, struct bpf_prog *xdp_prog) + struct xdp_frame *xdpf, + const struct bpf_prog *xdp_prog) { struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); enum bpf_map_type map_type = ri->map_type; @@ -4464,9 +4465,9 @@ EXPORT_SYMBOL_GPL(xdp_do_redirect_frame); static int xdp_do_generic_redirect_map(struct net_device *dev, struct sk_buff *skb, struct xdp_buff *xdp, - struct bpf_prog *xdp_prog, void *fwd, - enum bpf_map_type map_type, u32 map_id, - u32 flags) + const struct bpf_prog *xdp_prog, + void *fwd, enum bpf_map_type map_type, + u32 map_id, u32 flags) { struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); struct bpf_map *map; @@ -4520,7 +4521,8 @@ static int xdp_do_generic_redirect_map(struct net_device *dev, } int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb, - struct xdp_buff *xdp, struct bpf_prog *xdp_prog) + struct xdp_buff *xdp, + const struct bpf_prog *xdp_prog) { struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); enum bpf_map_type map_type = ri->map_type; @@ -9039,7 +9041,8 @@ static bool xdp_is_valid_access(int off, int size, return __is_valid_xdp_access(off, size); } -void bpf_warn_invalid_xdp_action(struct net_device *dev, struct bpf_prog *prog, u32 act) +void bpf_warn_invalid_xdp_action(const struct net_device *dev, + const struct bpf_prog *prog, u32 act) { const u32 act_max = XDP_REDIRECT; diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 83f8cd8aa2d16a..3f60544df5a7f2 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1005,7 +1005,7 @@ int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, EXPORT_SYMBOL(skb_pp_cow_data); int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, - struct bpf_prog *prog) + const struct bpf_prog *prog) { if (!prog->aux->xdp_has_frags) return -EINVAL; From afcb93dd47e7050460ed36aa64033ee36667468a Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Wed, 6 Dec 2023 12:11:29 +0100 Subject: [PATCH 20/49] xdp, xsk: constify read-only arguments of some static inline helpers Lots of read-only helpers for &xdp_buff and &xdp_frame, such as getting the frame length, skb_shared_info etc., don't have their arguments marked with `const` for no reason. Add the missing annotations to leave less place for mistakes and more for optimization. Signed-off-by: Alexander Lobakin --- include/net/xdp.h | 29 +++++++++++++++++------------ include/net/xdp_sock_drv.h | 11 ++++++----- include/net/xsk_buff_pool.h | 2 +- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/include/net/xdp.h b/include/net/xdp.h index e6770dd40c917e..197808df1ee12e 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -88,7 +88,7 @@ struct xdp_buff { u32 flags; /* supported values defined in xdp_buff_flags */ }; -static __always_inline bool xdp_buff_has_frags(struct xdp_buff *xdp) +static __always_inline bool xdp_buff_has_frags(const struct xdp_buff *xdp) { return !!(xdp->flags & XDP_FLAGS_HAS_FRAGS); } @@ -103,7 +103,8 @@ static __always_inline void xdp_buff_clear_frags_flag(struct xdp_buff *xdp) xdp->flags &= ~XDP_FLAGS_HAS_FRAGS; } -static __always_inline bool xdp_buff_is_frag_pfmemalloc(struct xdp_buff *xdp) +static __always_inline bool +xdp_buff_is_frag_pfmemalloc(const struct xdp_buff *xdp) { return !!(xdp->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC); } @@ -144,15 +145,16 @@ xdp_prepare_buff(struct xdp_buff *xdp, unsigned char *hard_start, SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) static inline struct skb_shared_info * -xdp_get_shared_info_from_buff(struct xdp_buff *xdp) +xdp_get_shared_info_from_buff(const struct xdp_buff *xdp) { return (struct skb_shared_info *)xdp_data_hard_end(xdp); } -static __always_inline unsigned int xdp_get_buff_len(struct xdp_buff *xdp) +static __always_inline unsigned int +xdp_get_buff_len(const struct xdp_buff *xdp) { unsigned int len = xdp->data_end - xdp->data; - struct skb_shared_info *sinfo; + const struct skb_shared_info *sinfo; if (likely(!xdp_buff_has_frags(xdp))) goto out; @@ -177,12 +179,13 @@ struct xdp_frame { u32 flags; /* supported values defined in xdp_buff_flags */ }; -static __always_inline bool xdp_frame_has_frags(struct xdp_frame *frame) +static __always_inline bool xdp_frame_has_frags(const struct xdp_frame *frame) { return !!(frame->flags & XDP_FLAGS_HAS_FRAGS); } -static __always_inline bool xdp_frame_is_frag_pfmemalloc(struct xdp_frame *frame) +static __always_inline bool +xdp_frame_is_frag_pfmemalloc(const struct xdp_frame *frame) { return !!(frame->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC); } @@ -201,7 +204,7 @@ static __always_inline void xdp_frame_bulk_init(struct xdp_frame_bulk *bq) } static inline struct skb_shared_info * -xdp_get_shared_info_from_frame(struct xdp_frame *frame) +xdp_get_shared_info_from_frame(const struct xdp_frame *frame) { void *data_hard_start = frame->data - frame->headroom - sizeof(*frame); @@ -249,7 +252,8 @@ int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp); struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf); static inline -void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp) +void xdp_convert_frame_to_buff(const struct xdp_frame *frame, + struct xdp_buff *xdp) { xdp->data_hard_start = frame->data - frame->headroom - sizeof(*frame); xdp->data = frame->data; @@ -260,7 +264,7 @@ void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp) } static inline -int xdp_update_frame_from_buff(struct xdp_buff *xdp, +int xdp_update_frame_from_buff(const struct xdp_buff *xdp, struct xdp_frame *xdp_frame) { int metasize, headroom; @@ -317,9 +321,10 @@ void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq); void xdp_return_frame_bulk(struct xdp_frame *xdpf, struct xdp_frame_bulk *bq); -static __always_inline unsigned int xdp_get_frame_len(struct xdp_frame *xdpf) +static __always_inline unsigned int +xdp_get_frame_len(const struct xdp_frame *xdpf) { - struct skb_shared_info *sinfo; + const struct skb_shared_info *sinfo; unsigned int len = xdpf->len; if (likely(!xdp_frame_has_frags(xdpf))) diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index 0a5dca2b2b3f6b..dcd469d258405d 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -101,7 +101,7 @@ static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool) return xp_alloc(pool); } -static inline bool xsk_is_eop_desc(struct xdp_desc *desc) +static inline bool xsk_is_eop_desc(const struct xdp_desc *desc) { return !xp_mb_desc(desc); } @@ -143,7 +143,7 @@ static inline void xsk_buff_add_frag(struct xdp_buff *xdp) list_add_tail(&frag->xskb_list_node, &frag->pool->xskb_list); } -static inline struct xdp_buff *xsk_buff_get_frag(struct xdp_buff *first) +static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first) { struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp); struct xdp_buff *ret = NULL; @@ -200,7 +200,8 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) XDP_TXMD_FLAGS_CHECKSUM | \ 0) -static inline bool xsk_buff_valid_tx_metadata(struct xsk_tx_metadata *meta) +static inline bool +xsk_buff_valid_tx_metadata(const struct xsk_tx_metadata *meta) { return !(meta->flags & ~XDP_TXMD_FLAGS_VALID); } @@ -337,7 +338,7 @@ static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool) return NULL; } -static inline bool xsk_is_eop_desc(struct xdp_desc *desc) +static inline bool xsk_is_eop_desc(const struct xdp_desc *desc) { return false; } @@ -360,7 +361,7 @@ static inline void xsk_buff_add_frag(struct xdp_buff *xdp) { } -static inline struct xdp_buff *xsk_buff_get_frag(struct xdp_buff *first) +static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first) { return NULL; } diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h index bacb33f1e3e581..0442ba8dafa46a 100644 --- a/include/net/xsk_buff_pool.h +++ b/include/net/xsk_buff_pool.h @@ -185,7 +185,7 @@ static inline bool xp_desc_crosses_non_contig_pg(struct xsk_buff_pool *pool, !(pool->dma_pages[addr >> PAGE_SHIFT] & XSK_NEXT_PG_CONTIG_MASK); } -static inline bool xp_mb_desc(struct xdp_desc *desc) +static inline bool xp_mb_desc(const struct xdp_desc *desc) { return desc->options & XDP_PKT_CONTD; } From da6f0f1dfbd1147bc8b3e6341c21789f3a73fdac Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 19 Dec 2023 12:35:55 +0100 Subject: [PATCH 21/49] xdp: allow attaching already registered memory model to xdp_rxq_info One may need to register memory model separately from xdp_rxq_info. One simple example may be XDP test run code, but in general, it might be useful when memory model registering is managed by one layer and then XDP RxQ info by a different one. Allow such scenarios by adding a simple helper which "attaches" an already registered memory model to the desired xdp_rxq_info. As this is mostly needed for Page Pool, add a special function to do that for a &page_pool pointer. Signed-off-by: Alexander Lobakin --- include/net/xdp.h | 32 +++++++++++++++++++++++++++ net/core/xdp.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/include/net/xdp.h b/include/net/xdp.h index 197808df1ee12e..3e748bb916d353 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -356,6 +356,38 @@ void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq); int xdp_reg_mem_model(struct xdp_mem_info *mem, enum xdp_mem_type type, void *allocator); void xdp_unreg_mem_model(struct xdp_mem_info *mem); +int xdp_reg_page_pool(struct page_pool *pool); +void xdp_unreg_page_pool(const struct page_pool *pool); +void xdp_rxq_info_attach_page_pool(struct xdp_rxq_info *xdp_rxq, + const struct page_pool *pool); + +/** + * xdp_rxq_info_attach_mem_model - attach a registered mem info to an RxQ info + * @xdp_rxq: XDP RxQ info to attach the memory info to + * @mem: already registered memory info + * + * If a driver registers its memory providers manually, it must use this + * function instead of xdp_rxq_info_reg_mem_model(). + */ +static inline void +xdp_rxq_info_attach_mem_model(struct xdp_rxq_info *xdp_rxq, + const struct xdp_mem_info *mem) +{ + xdp_rxq->mem = *mem; +} + +/** + * xdp_rxq_info_detach_mem_model - detach a registered mem info from RxQ info + * @xdp_rxq: XDP RxQ info to detach the memory info from + * + * If a driver registers its memory providers manually and then attaches it + * via xdp_rxq_info_attach_mem_model(), it must call this function before + * xdp_rxq_info_unreg(). + */ +static inline void xdp_rxq_info_detach_mem_model(struct xdp_rxq_info *xdp_rxq) +{ + xdp_rxq->mem = (struct xdp_mem_info){ }; +} /* Drivers not supporting XDP metadata can use this helper, which * rejects any room expansion for metadata as a result. diff --git a/net/core/xdp.c b/net/core/xdp.c index bcc5551c6424bd..bd2aa340baad9b 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -365,6 +365,62 @@ int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model); +/** + * xdp_reg_page_pool - register a &page_pool as a memory provider for XDP + * @pool: &page_pool to register + * + * Can be used to register pools manually without connecting to any XDP RxQ + * info, so that the XDP layer will be aware of them. Then, they can be + * attached to an RxQ info manually via xdp_rxq_info_attach_page_pool(). + * + * Return: %0 on success, -errno on error. + */ +int xdp_reg_page_pool(struct page_pool *pool) +{ + struct xdp_mem_info mem; + + return xdp_reg_mem_model(&mem, MEM_TYPE_PAGE_POOL, pool); +} +EXPORT_SYMBOL_GPL(xdp_reg_page_pool); + +/** + * xdp_unreg_page_pool - unregister a &page_pool from the memory providers list + * @pool: &page_pool to unregister + * + * A shorthand for manual unregistering page pools. If the pool was previously + * attached to an RxQ info, it must be detached first. + */ +void xdp_unreg_page_pool(const struct page_pool *pool) +{ + struct xdp_mem_info mem = { + .type = MEM_TYPE_PAGE_POOL, + .id = pool->xdp_mem_id, + }; + + xdp_unreg_mem_model(&mem); +} +EXPORT_SYMBOL_GPL(xdp_unreg_page_pool); + +/** + * xdp_rxq_info_attach_page_pool - attach a registered pool to an RxQ info + * @xdp_rxq: XDP RxQ info to attach the pool to + * @pool: pool to attach + * + * If the pool was registered manually, this function must be called instead + * of xdp_rxq_info_reg_mem_model() to connect it to an RxQ info. + */ +void xdp_rxq_info_attach_page_pool(struct xdp_rxq_info *xdp_rxq, + const struct page_pool *pool) +{ + struct xdp_mem_info mem = { + .type = MEM_TYPE_PAGE_POOL, + .id = pool->xdp_mem_id, + }; + + xdp_rxq_info_attach_mem_model(xdp_rxq, &mem); +} +EXPORT_SYMBOL_GPL(xdp_rxq_info_attach_page_pool); + /* XDP RX runs under NAPI protection, and in different delivery error * scenarios (e.g. queue full), it is possible to return the xdp_frame * while still leveraging this protection. The @napi_direct boolean From 1859320c497fd5eeda132f6aad4d0f09cfb1e123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= Date: Tue, 20 Feb 2024 22:03:38 +0100 Subject: [PATCH 22/49] net: Register system page pool as an XDP memory model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To make the system page pool usable as a source for allocating XDP frames, we need to register it with xdp_reg_mem_model(), so that page return works correctly. This is done in preparation for using the system page pool for the XDP live frame mode in BPF_TEST_RUN; for the same reason, make the per-cpu variable non-static so we can access it from the test_run code as well. Reviewed-by: Alexander Lobakin Tested-by: Alexander Lobakin Signed-off-by: Toke Høiland-Jørgensen --- include/linux/netdevice.h | 1 + net/core/dev.c | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 649ccbb5203df2..d509e9c2b00ac6 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3276,6 +3276,7 @@ struct softnet_data { }; DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data); +DECLARE_PER_CPU(struct page_pool *, system_page_pool); #ifndef CONFIG_PREEMPT_RT static inline int dev_recursion_level(void) diff --git a/net/core/dev.c b/net/core/dev.c index a8d24d4b5d4a57..f9956244ca57fb 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -458,7 +458,7 @@ EXPORT_PER_CPU_SYMBOL(softnet_data); * PP consumers must pay attention to run APIs in the appropriate context * (e.g. NAPI context). */ -static DEFINE_PER_CPU(struct page_pool *, system_page_pool); +DEFINE_PER_CPU(struct page_pool *, system_page_pool); #ifdef CONFIG_LOCKDEP /* @@ -11969,11 +11969,18 @@ static int net_page_pool_create(int cpuid) .nid = cpu_to_mem(cpuid), }; struct page_pool *pp_ptr; + int err; pp_ptr = page_pool_create_percpu(&page_pool_params, cpuid); if (IS_ERR(pp_ptr)) return -ENOMEM; + err = xdp_reg_page_pool(pp_ptr); + if (err) { + page_pool_destroy(pp_ptr); + return err; + } + per_cpu(system_page_pool, cpuid) = pp_ptr; #endif return 0; @@ -12101,13 +12108,13 @@ static int __init net_dev_init(void) out: if (rc < 0) { for_each_possible_cpu(i) { - struct page_pool *pp_ptr; + const struct page_pool *pp_ptr; pp_ptr = per_cpu(system_page_pool, i); if (!pp_ptr) continue; - page_pool_destroy(pp_ptr); + xdp_unreg_page_pool(pp_ptr); per_cpu(system_page_pool, i) = NULL; } } From 2cd561a4dc283f55ecfa22c5a83f4807473a64ba Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Mon, 5 Feb 2024 18:10:58 +0100 Subject: [PATCH 23/49] page_pool: make page_pool_put_page_bulk() actually handle array of pages Currently, page_pool_put_page_bulk() indeed takes an array of pointers to the data, not pages, despite the name. As one side effect, when you're freeing frags from &skb_shared_info, xdp_return_frame_bulk() converts page pointers to virtual addresses and then page_pool_put_page_bulk() converts them back. Make page_pool_put_page_bulk() actually handle array of pages. Pass frags directly and use virt_to_page() when freeing xdpf->data, so that the PP core will then get the compound head and take care of the rest. Signed-off-by: Alexander Lobakin --- include/net/page_pool/types.h | 8 ++++---- include/net/xdp.h | 2 +- net/core/page_pool.c | 6 +++--- net/core/xdp.c | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h index 50569fed7868ea..946a50702e4297 100644 --- a/include/net/page_pool/types.h +++ b/include/net/page_pool/types.h @@ -240,8 +240,8 @@ void page_pool_disable_direct_recycling(struct page_pool *pool); void page_pool_destroy(struct page_pool *pool); void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *), const struct xdp_mem_info *mem); -void page_pool_put_page_bulk(struct page_pool *pool, void **data, - int count); +void page_pool_put_page_bulk(struct page_pool *pool, struct page **data, + u32 count); #else static inline void page_pool_destroy(struct page_pool *pool) { @@ -253,8 +253,8 @@ static inline void page_pool_use_xdp_mem(struct page_pool *pool, { } -static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data, - int count) +static inline void page_pool_put_page_bulk(struct page_pool *pool, + struct page **data, u32 count) { } #endif diff --git a/include/net/xdp.h b/include/net/xdp.h index 3e748bb916d353..4416cd4b508658 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -194,7 +194,7 @@ xdp_frame_is_frag_pfmemalloc(const struct xdp_frame *frame) struct xdp_frame_bulk { int count; void *xa; - void *q[XDP_BULK_QUEUE_SIZE]; + struct page *q[XDP_BULK_QUEUE_SIZE]; }; static __always_inline void xdp_frame_bulk_init(struct xdp_frame_bulk *bq) diff --git a/net/core/page_pool.c b/net/core/page_pool.c index 2abe6e919224d9..627f6054d9625d 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -809,8 +809,8 @@ EXPORT_SYMBOL(page_pool_put_unrefed_page); * Please note the caller must not use data area after running * page_pool_put_page_bulk(), as this function overwrites it. */ -void page_pool_put_page_bulk(struct page_pool *pool, void **data, - int count) +void page_pool_put_page_bulk(struct page_pool *pool, struct page **data, + u32 count) { int i, bulk_len = 0; bool allow_direct; @@ -819,7 +819,7 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data, allow_direct = page_pool_napi_local(pool); for (i = 0; i < count; i++) { - netmem_ref netmem = page_to_netmem(virt_to_head_page(data[i])); + netmem_ref netmem = page_to_netmem(compound_head(data[i])); /* It is not the last user for the page frag case */ if (!page_pool_is_last_ref(netmem)) diff --git a/net/core/xdp.c b/net/core/xdp.c index bd2aa340baad9b..779e646f347b27 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -556,12 +556,12 @@ void xdp_return_frame_bulk(struct xdp_frame *xdpf, for (i = 0; i < sinfo->nr_frags; i++) { skb_frag_t *frag = &sinfo->frags[i]; - bq->q[bq->count++] = skb_frag_address(frag); + bq->q[bq->count++] = skb_frag_page(frag); if (bq->count == XDP_BULK_QUEUE_SIZE) xdp_flush_frame_bulk(bq); } } - bq->q[bq->count++] = xdpf->data; + bq->q[bq->count++] = virt_to_page(xdpf->data); } EXPORT_SYMBOL_GPL(xdp_return_frame_bulk); From 23c4f5c9c919248b41971444aa1c39f99d6912d3 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Thu, 7 Dec 2023 15:16:52 +0100 Subject: [PATCH 24/49] page_pool: allow mixing PPs within one bulk The main reason for this change was to allow mixing pages from different &page_pools within one &xdp_buff/&xdp_frame. Why not? Adjust xdp_return_frame_bulk() and page_pool_put_page_bulk(), so that they won't be tied to a particular pool. Let the latter splice the bulk when it encounters a page whichs PP is different and flush it recursively. This greatly optimizes xdp_return_frame_bulk(): no more hashtable lookups. Also make xdp_flush_frame_bulk() inline, as it's just one if + function call + one u32 read, not worth extending the call ladder. Signed-off-by: Alexander Lobakin --- include/net/page_pool/types.h | 7 +++-- include/net/xdp.h | 16 +++++++++--- net/core/page_pool.c | 49 ++++++++++++++++++++++++++++++----- net/core/xdp.c | 29 +-------------------- 4 files changed, 58 insertions(+), 43 deletions(-) diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h index 946a50702e4297..ff9b6c7019cb70 100644 --- a/include/net/page_pool/types.h +++ b/include/net/page_pool/types.h @@ -240,8 +240,7 @@ void page_pool_disable_direct_recycling(struct page_pool *pool); void page_pool_destroy(struct page_pool *pool); void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *), const struct xdp_mem_info *mem); -void page_pool_put_page_bulk(struct page_pool *pool, struct page **data, - u32 count); +void page_pool_put_page_bulk(struct page **data, u32 count, bool rec); #else static inline void page_pool_destroy(struct page_pool *pool) { @@ -253,8 +252,8 @@ static inline void page_pool_use_xdp_mem(struct page_pool *pool, { } -static inline void page_pool_put_page_bulk(struct page_pool *pool, - struct page **data, u32 count) +static inline void page_pool_put_page_bulk(struct page **data, u32 count, + bool rec) { } #endif diff --git a/include/net/xdp.h b/include/net/xdp.h index 4416cd4b508658..49f5965134354a 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -11,6 +11,8 @@ #include #include /* skb_shared_info */ +#include + /** * DOC: XDP RX-queue information * @@ -193,14 +195,12 @@ xdp_frame_is_frag_pfmemalloc(const struct xdp_frame *frame) #define XDP_BULK_QUEUE_SIZE 16 struct xdp_frame_bulk { int count; - void *xa; struct page *q[XDP_BULK_QUEUE_SIZE]; }; static __always_inline void xdp_frame_bulk_init(struct xdp_frame_bulk *bq) { - /* bq->count will be zero'ed when bq->xa gets updated */ - bq->xa = NULL; + bq->count = 0; } static inline struct skb_shared_info * @@ -317,10 +317,18 @@ void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct, void xdp_return_frame(struct xdp_frame *xdpf); void xdp_return_frame_rx_napi(struct xdp_frame *xdpf); void xdp_return_buff(struct xdp_buff *xdp); -void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq); void xdp_return_frame_bulk(struct xdp_frame *xdpf, struct xdp_frame_bulk *bq); +static inline void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq) +{ + if (unlikely(!bq->count)) + return; + + page_pool_put_page_bulk(bq->q, bq->count, false); + bq->count = 0; +} + static __always_inline unsigned int xdp_get_frame_len(const struct xdp_frame *xdpf) { diff --git a/net/core/page_pool.c b/net/core/page_pool.c index 627f6054d9625d..127e14ccb5f4bd 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -794,11 +794,22 @@ void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page, } EXPORT_SYMBOL(page_pool_put_unrefed_page); +static void page_pool_bulk_splice(struct xdp_frame_bulk *bulk, + struct page *page) +{ + bulk->q[bulk->count++] = page; + + if (unlikely(bulk->count == ARRAY_SIZE(bulk->q))) { + page_pool_put_page_bulk(bulk->q, ARRAY_SIZE(bulk->q), true); + bulk->count = 0; + } +} + /** * page_pool_put_page_bulk() - release references on multiple pages - * @pool: pool from which pages were allocated * @data: array holding page pointers * @count: number of pages in @data + * @rec: whether it's called recursively by itself * * Tries to refill a number of pages into the ptr_ring cache holding ptr_ring * producer lock. If the ptr_ring is full, page_pool_put_page_bulk() @@ -809,21 +820,42 @@ EXPORT_SYMBOL(page_pool_put_unrefed_page); * Please note the caller must not use data area after running * page_pool_put_page_bulk(), as this function overwrites it. */ -void page_pool_put_page_bulk(struct page_pool *pool, struct page **data, - u32 count) +void page_pool_put_page_bulk(struct page **data, u32 count, bool rec) { + struct page_pool *pool = NULL; + struct xdp_frame_bulk sub; int i, bulk_len = 0; bool allow_direct; bool in_softirq; - allow_direct = page_pool_napi_local(pool); + xdp_frame_bulk_init(&sub); for (i = 0; i < count; i++) { - netmem_ref netmem = page_to_netmem(compound_head(data[i])); + struct page *page; + netmem_ref netmem; + + if (!rec) { + page = compound_head(data[i]); + netmem = page_to_netmem(page); - /* It is not the last user for the page frag case */ - if (!page_pool_is_last_ref(netmem)) + /* It is not the last user for the page frag case */ + if (!page_pool_is_last_ref(netmem)) + continue; + } else { + page = data[i]; + netmem = page_to_netmem(page); + } + + if (unlikely(!pool)) { + pool = page->pp; + allow_direct = page_pool_napi_local(pool); + } else if (page->pp != pool) { + /* If the page belongs to a different page_pool, + * splice the array and handle it recursively. + */ + page_pool_bulk_splice(&sub, page); continue; + } netmem = __page_pool_put_page(pool, netmem, -1, allow_direct); /* Approved for bulk recycling in ptr_ring cache */ @@ -831,6 +863,9 @@ void page_pool_put_page_bulk(struct page_pool *pool, struct page **data, data[bulk_len++] = (__force void *)netmem; } + if (sub.count) + page_pool_put_page_bulk(sub.q, sub.count, true); + if (!bulk_len) return; diff --git a/net/core/xdp.c b/net/core/xdp.c index 779e646f347b27..0fde1bb54192e5 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -508,46 +508,19 @@ EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi); * xdp_frame_bulk is usually stored/allocated on the function * call-stack to avoid locking penalties. */ -void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq) -{ - struct xdp_mem_allocator *xa = bq->xa; - - if (unlikely(!xa || !bq->count)) - return; - - page_pool_put_page_bulk(xa->page_pool, bq->q, bq->count); - /* bq->xa is not cleared to save lookup, if mem.id same in next bulk */ - bq->count = 0; -} -EXPORT_SYMBOL_GPL(xdp_flush_frame_bulk); /* Must be called with rcu_read_lock held */ void xdp_return_frame_bulk(struct xdp_frame *xdpf, struct xdp_frame_bulk *bq) { - struct xdp_mem_info *mem = &xdpf->mem; - struct xdp_mem_allocator *xa; - - if (mem->type != MEM_TYPE_PAGE_POOL) { + if (xdpf->mem.type != MEM_TYPE_PAGE_POOL) { xdp_return_frame(xdpf); return; } - xa = bq->xa; - if (unlikely(!xa)) { - xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params); - bq->count = 0; - bq->xa = xa; - } - if (bq->count == XDP_BULK_QUEUE_SIZE) xdp_flush_frame_bulk(bq); - if (unlikely(mem->id != xa->mem.id)) { - xdp_flush_frame_bulk(bq); - bq->xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params); - } - if (unlikely(xdp_frame_has_frags(xdpf))) { struct skb_shared_info *sinfo; int i; From 92b506a3b519c294630fb18cd8759dab66819e32 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Thu, 7 Dec 2023 15:16:52 +0100 Subject: [PATCH 25/49] xdp: get rid of xdp_frame::mem.id Initially, xdp_frame::mem.id was used to search for the corresponding &page_pool to return the page correctly. However, after that struct page now contains a direct pointer to its PP, further keeping of this field makes no sense. xdp_return_frame_bulk() still uses it to do a lookup, but this is rather a leftover. Remove xdp_frame::mem and replace it with ::mem_type, as only memory type still matters and we need to know it to be able to free the frame correctly. As a cute side effect, we can now make every scalar field in &xdp_frame of 4 byte width, speeding up accesses to them. Signed-off-by: Alexander Lobakin --- .../net/ethernet/freescale/dpaa/dpaa_eth.c | 2 +- drivers/net/veth.c | 4 +-- include/net/xdp.h | 14 +++++----- kernel/bpf/cpumap.c | 2 +- net/bpf/test_run.c | 2 +- net/core/filter.c | 12 ++++---- net/core/xdp.c | 28 +++++++++---------- 7 files changed, 32 insertions(+), 32 deletions(-) diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c index d3929d236df525..e9a69dd7d0b7e9 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c @@ -2259,7 +2259,7 @@ static int dpaa_a050385_wa_xdpf(struct dpaa_priv *priv, new_xdpf->len = xdpf->len; new_xdpf->headroom = priv->tx_headroom; new_xdpf->frame_sz = DPAA_BP_RAW_SIZE; - new_xdpf->mem.type = MEM_TYPE_PAGE_ORDER0; + new_xdpf->mem_type = MEM_TYPE_PAGE_ORDER0; /* Release the initial buffer */ xdp_return_frame_rx_napi(xdpf); diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 925b45d993f169..7adaf54ec90b8e 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -634,7 +634,7 @@ static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq, break; case XDP_TX: orig_frame = *frame; - xdp->rxq->mem = frame->mem; + xdp->rxq->mem.type = frame->mem_type; if (unlikely(veth_xdp_tx(rq, xdp, bq) < 0)) { trace_xdp_exception(rq->dev, xdp_prog, act); frame = &orig_frame; @@ -646,7 +646,7 @@ static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq, goto xdp_xmit; case XDP_REDIRECT: orig_frame = *frame; - xdp->rxq->mem = frame->mem; + xdp->rxq->mem.type = frame->mem_type; if (xdp_do_redirect(rq->dev, xdp, xdp_prog)) { frame = &orig_frame; stats->rx_drops++; diff --git a/include/net/xdp.h b/include/net/xdp.h index 49f5965134354a..c4b408d2266972 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -169,13 +169,13 @@ xdp_get_buff_len(const struct xdp_buff *xdp) struct xdp_frame { void *data; - u16 len; - u16 headroom; + u32 len; + u32 headroom; u32 metasize; /* uses lower 8-bits */ /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time, - * while mem info is valid on remote CPU. + * while mem_type is valid on remote CPU. */ - struct xdp_mem_info mem; + enum xdp_mem_type mem_type:32; struct net_device *dev_rx; /* used by cpumap */ u32 frame_sz; u32 flags; /* supported values defined in xdp_buff_flags */ @@ -306,13 +306,13 @@ struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp) if (unlikely(xdp_update_frame_from_buff(xdp, xdp_frame) < 0)) return NULL; - /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */ - xdp_frame->mem = xdp->rxq->mem; + /* rxq only valid until napi_schedule ends, convert to xdp_mem_type */ + xdp_frame->mem_type = xdp->rxq->mem.type; return xdp_frame; } -void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct, +void __xdp_return(void *data, enum xdp_mem_type mem_type, bool napi_direct, struct xdp_buff *xdp); void xdp_return_frame(struct xdp_frame *xdpf); void xdp_return_frame_rx_napi(struct xdp_frame *xdpf); diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index fbdf5a1aabfe4f..6227dd7e5d3d35 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -190,7 +190,7 @@ static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu, int err; rxq.dev = xdpf->dev_rx; - rxq.mem = xdpf->mem; + rxq.mem.type = xdpf->mem_type; /* TODO: report queue_index to xdp_rxq_info */ xdp_convert_frame_to_buff(xdpf, &xdp); diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 6d7a442ceb89be..eac959b04fa9f3 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -153,7 +153,7 @@ static void xdp_test_run_init_page(netmem_ref netmem, void *arg) new_ctx->data = new_ctx->data_meta + meta_len; xdp_update_frame_from_buff(new_ctx, frm); - frm->mem = new_ctx->rxq->mem; + frm->mem_type = new_ctx->rxq->mem.type; memcpy(&head->orig_ctx, new_ctx, sizeof(head->orig_ctx)); } diff --git a/net/core/filter.c b/net/core/filter.c index 21ba2fa3dfacd3..f9a8015ae8fc93 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -4111,13 +4111,13 @@ static int bpf_xdp_frags_increase_tail(struct xdp_buff *xdp, int offset) } static void bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink, - struct xdp_mem_info *mem_info, bool release) + enum xdp_mem_type mem_type, bool release) { struct xdp_buff *zc_frag = xsk_buff_get_tail(xdp); if (release) { xsk_buff_del_tail(zc_frag); - __xdp_return(NULL, mem_info, false, zc_frag); + __xdp_return(NULL, mem_type, false, zc_frag); } else { zc_frag->data_end -= shrink; } @@ -4126,18 +4126,18 @@ static void bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink, static bool bpf_xdp_shrink_data(struct xdp_buff *xdp, skb_frag_t *frag, int shrink) { - struct xdp_mem_info *mem_info = &xdp->rxq->mem; + enum xdp_mem_type mem_type = xdp->rxq->mem.type; bool release = skb_frag_size(frag) == shrink; - if (mem_info->type == MEM_TYPE_XSK_BUFF_POOL) { - bpf_xdp_shrink_data_zc(xdp, shrink, mem_info, release); + if (mem_type == MEM_TYPE_XSK_BUFF_POOL) { + bpf_xdp_shrink_data_zc(xdp, shrink, mem_type, release); goto out; } if (release) { struct page *page = skb_frag_page(frag); - __xdp_return(page_address(page), mem_info, false, NULL); + __xdp_return(page_address(page), mem_type, false, NULL); } out: diff --git a/net/core/xdp.c b/net/core/xdp.c index 0fde1bb54192e5..b1b426a9b146a5 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -427,12 +427,12 @@ EXPORT_SYMBOL_GPL(xdp_rxq_info_attach_page_pool); * is used for those calls sites. Thus, allowing for faster recycling * of xdp_frames/pages in those cases. */ -void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct, +void __xdp_return(void *data, enum xdp_mem_type mem_type, bool napi_direct, struct xdp_buff *xdp) { struct page *page; - switch (mem->type) { + switch (mem_type) { case MEM_TYPE_PAGE_POOL: page = virt_to_head_page(data); if (napi_direct && xdp_return_frame_no_direct()) @@ -455,7 +455,7 @@ void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct, break; default: /* Not possible, checked in xdp_rxq_info_reg_mem_model() */ - WARN(1, "Incorrect XDP memory type (%d) usage", mem->type); + WARN(1, "Incorrect XDP memory type (%d) usage", mem_type); break; } } @@ -472,10 +472,10 @@ void xdp_return_frame(struct xdp_frame *xdpf) for (i = 0; i < sinfo->nr_frags; i++) { struct page *page = skb_frag_page(&sinfo->frags[i]); - __xdp_return(page_address(page), &xdpf->mem, false, NULL); + __xdp_return(page_address(page), xdpf->mem_type, false, NULL); } out: - __xdp_return(xdpf->data, &xdpf->mem, false, NULL); + __xdp_return(xdpf->data, xdpf->mem_type, false, NULL); } EXPORT_SYMBOL_GPL(xdp_return_frame); @@ -491,10 +491,10 @@ void xdp_return_frame_rx_napi(struct xdp_frame *xdpf) for (i = 0; i < sinfo->nr_frags; i++) { struct page *page = skb_frag_page(&sinfo->frags[i]); - __xdp_return(page_address(page), &xdpf->mem, true, NULL); + __xdp_return(page_address(page), xdpf->mem_type, true, NULL); } out: - __xdp_return(xdpf->data, &xdpf->mem, true, NULL); + __xdp_return(xdpf->data, xdpf->mem_type, true, NULL); } EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi); @@ -513,7 +513,7 @@ EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi); void xdp_return_frame_bulk(struct xdp_frame *xdpf, struct xdp_frame_bulk *bq) { - if (xdpf->mem.type != MEM_TYPE_PAGE_POOL) { + if (xdpf->mem_type != MEM_TYPE_PAGE_POOL) { xdp_return_frame(xdpf); return; } @@ -550,10 +550,11 @@ void xdp_return_buff(struct xdp_buff *xdp) for (i = 0; i < sinfo->nr_frags; i++) { struct page *page = skb_frag_page(&sinfo->frags[i]); - __xdp_return(page_address(page), &xdp->rxq->mem, true, xdp); + __xdp_return(page_address(page), xdp->rxq->mem.type, true, + xdp); } out: - __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp); + __xdp_return(xdp->data, xdp->rxq->mem.type, true, xdp); } EXPORT_SYMBOL_GPL(xdp_return_buff); @@ -599,7 +600,7 @@ struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp) xdpf->headroom = 0; xdpf->metasize = metasize; xdpf->frame_sz = PAGE_SIZE; - xdpf->mem.type = MEM_TYPE_PAGE_ORDER0; + xdpf->mem_type = MEM_TYPE_PAGE_ORDER0; xsk_buff_free(xdp); return xdpf; @@ -669,7 +670,7 @@ struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf, * - RX ring dev queue index (skb_record_rx_queue) */ - if (xdpf->mem.type == MEM_TYPE_PAGE_POOL) + if (xdpf->mem_type == MEM_TYPE_PAGE_POOL) skb_mark_for_recycle(skb); /* Allow SKB to reuse area used by xdp_frame */ @@ -716,8 +717,7 @@ struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf) nxdpf = addr; nxdpf->data = addr + headroom; nxdpf->frame_sz = PAGE_SIZE; - nxdpf->mem.type = MEM_TYPE_PAGE_ORDER0; - nxdpf->mem.id = 0; + nxdpf->mem_type = MEM_TYPE_PAGE_ORDER0; return nxdpf; } From d5f42872648c87d517ba405721024d249a9db5d7 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Wed, 6 Dec 2023 12:43:46 +0100 Subject: [PATCH 26/49] xdp: add generic xdp_buff_add_frag() The code piece which would attach a frag to &xdp_buff is almost identical across the drivers supporting XDP multi-buffer on Rx. Make it a generic elegant onelner. Also, I see lots of drivers calculating frags_truesize as `xdp->frame_sz * nr_frags`. I can't say this is fully correct, since frags might be backed by chunks of different sizes, especially with stuff like the header split. Even page_pool_alloc() can give you two different truesizes on two subsequent requests to allocate the same buffer size. Add a field to &skb_shared_info (unionized as there's no free slot currently on x6_64) to track the "true" truesize. It can be used later when updating an skb. Signed-off-by: Alexander Lobakin --- include/linux/skbuff.h | 15 ++++++--- include/net/xdp.h | 75 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 08106a5653f3ac..f01fa8ebd8fd0f 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -607,11 +607,18 @@ struct skb_shared_info { * Warning : all fields before dataref are cleared in __alloc_skb() */ atomic_t dataref; - unsigned int xdp_frags_size; - /* Intermediate layers must ensure that destructor_arg - * remains valid until skb destructor */ - void * destructor_arg; + union { + struct { + u32 xdp_frags_size; + u32 xdp_frags_truesize; + }; + + /* Intermediate layers must ensure that destructor_arg + * remains valid until skb destructor. + */ + void *destructor_arg; + }; /* must be last field, see pskb_expand_head() */ skb_frag_t frags[MAX_SKB_FRAGS]; diff --git a/include/net/xdp.h b/include/net/xdp.h index c4b408d2266972..1e14b70c9e573c 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -167,6 +167,73 @@ xdp_get_buff_len(const struct xdp_buff *xdp) return len; } +/** + * __xdp_buff_add_frag - attach a frag to an &xdp_buff + * @xdp: XDP buffer to attach the frag to + * @page: page containing the frag + * @offset: page offset at which the frag starts + * @size: size of the frag + * @truesize: truesize (page / page frag size) of the frag + * + * Attach a frag to an XDP buffer. If it currently has no frags attached, + * initialize the related fields, otherwise check that the frag number + * didn't reach the limit of ``MAX_SKB_FRAGS``. + * The function doesn't check/update the pfmemalloc bit. Please use the + * non-underscored wrapper in drivers. + * + * Return: true on success, false if there's no space for the frag in + * the shared info struct. + */ +static inline bool __xdp_buff_add_frag(struct xdp_buff *xdp, struct page *page, + u32 offset, u32 size, u32 truesize) +{ + struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); + + if (!xdp_buff_has_frags(xdp)) { + sinfo->nr_frags = 0; + + sinfo->xdp_frags_size = 0; + sinfo->xdp_frags_truesize = 0; + + xdp_buff_set_frags_flag(xdp); + } + + if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) + return false; + + __skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++, page, offset, + size); + sinfo->xdp_frags_size += size; + sinfo->xdp_frags_truesize += truesize; + + return true; +} + +/** + * xdp_buff_add_frag - attach a frag to an &xdp_buff + * @xdp: XDP buffer to attach the frag to + * @page: page containing the frag + * @offset: page offset at which the frag starts + * @size: size of the frag + * @truesize: truesize (page / page frag size) of the frag + * + * Version of __xdp_buff_add_frag() which takes care of the pfmemalloc bit. + * + * Return: true on success, false if there's no space for the frag in + * the shared info struct. + */ +static inline bool xdp_buff_add_frag(struct xdp_buff *xdp, struct page *page, + u32 offset, u32 size, u32 truesize) +{ + if (!__xdp_buff_add_frag(xdp, page, offset, size, truesize)) + return false; + + if (unlikely(page_is_pfmemalloc(page))) + xdp_buff_set_frag_pfmemalloc(xdp); + + return true; +} + struct xdp_frame { void *data; u32 len; @@ -230,7 +297,13 @@ xdp_update_skb_shared_info(struct sk_buff *skb, u8 nr_frags, unsigned int size, unsigned int truesize, bool pfmemalloc) { - skb_shinfo(skb)->nr_frags = nr_frags; + struct skb_shared_info *sinfo = skb_shinfo(skb); + + sinfo->nr_frags = nr_frags; + /* ``destructor_arg`` is unionized with ``xdp_frags_{,true}size``, + * reset it after that these fields aren't used anymore. + */ + sinfo->destructor_arg = NULL; skb->len += size; skb->data_len += size; From 3f7575896d5b9e93a604a5adfd2c0e656c525b7b Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Wed, 6 Dec 2023 13:25:46 +0100 Subject: [PATCH 27/49] xdp: add generic xdp_build_skb_from_buff() The code which builds an skb from an &xdp_buff keeps multiplying itself around the drivers with almost no changes. Let's try to stop that by adding a generic function. There's __xdp_build_skb_from_frame() already, so just convert it to take &xdp_buff instead, while making the original one a wrapper. The original one always took an already allocated skb, allow both variants here -- if no skb passed, which is expected when calling from a driver, pick one via napi_build_skb(). Signed-off-by: Alexander Lobakin --- include/net/xdp.h | 1 + net/core/xdp.c | 55 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/include/net/xdp.h b/include/net/xdp.h index 1e14b70c9e573c..e8e2278f92bea7 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -315,6 +315,7 @@ xdp_update_skb_shared_info(struct sk_buff *skb, u8 nr_frags, void xdp_warn(const char *msg, const char *func, const int line); #define XDP_WARN(msg) xdp_warn(msg, __func__, __LINE__) +struct sk_buff *xdp_build_skb_from_buff(const struct xdp_buff *xdp); struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp); struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf, struct sk_buff *skb, diff --git a/net/core/xdp.c b/net/core/xdp.c index b1b426a9b146a5..de089bee600567 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -624,6 +624,61 @@ int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp) } EXPORT_SYMBOL_GPL(xdp_alloc_skb_bulk); +/** + * xdp_build_skb_from_buff - create an skb from an &xdp_buff + * @xdp: &xdp_buff to convert to an skb + * + * Perform common operations to create a new skb to pass up the stack from + * an &xdp_buff: allocate an skb head from the NAPI percpu cache, initialize + * skb data pointers and offsets, set the recycle bit if the buff is PP-backed, + * Rx queue index, protocol and update frags info. + * + * Return: new &sk_buff on success, %NULL on error. + */ +struct sk_buff *xdp_build_skb_from_buff(const struct xdp_buff *xdp) +{ + const struct xdp_rxq_info *rxq = xdp->rxq; + const struct skb_shared_info *sinfo; + struct sk_buff *skb; + u32 nr_frags = 0; + int metalen; + + if (unlikely(xdp_buff_has_frags(xdp))) { + sinfo = xdp_get_shared_info_from_buff(xdp); + nr_frags = sinfo->nr_frags; + } + + skb = napi_build_skb(xdp->data_hard_start, xdp->frame_sz); + if (unlikely(!skb)) + return NULL; + + skb_reserve(skb, xdp->data - xdp->data_hard_start); + __skb_put(skb, xdp->data_end - xdp->data); + + metalen = xdp->data - xdp->data_meta; + if (metalen > 0) + skb_metadata_set(skb, metalen); + + if (rxq->mem.type == MEM_TYPE_PAGE_POOL && is_page_pool_compiled_in()) + skb_mark_for_recycle(skb); + + skb_record_rx_queue(skb, rxq->queue_index); + + if (unlikely(nr_frags)) { + u32 ts; + + ts = sinfo->xdp_frags_truesize ? : nr_frags * xdp->frame_sz; + xdp_update_skb_shared_info(skb, nr_frags, + sinfo->xdp_frags_size, ts, + xdp_buff_is_frag_pfmemalloc(xdp)); + } + + skb->protocol = eth_type_trans(skb, rxq->dev); + + return skb; +} +EXPORT_SYMBOL_GPL(xdp_build_skb_from_buff); + struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf, struct sk_buff *skb, struct net_device *dev) From 1b659d29702d9bb678abb3d3b57b7ee7a184293f Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Wed, 31 Jan 2024 11:47:26 +0100 Subject: [PATCH 28/49] xsk: allow attaching XSk pool via xdp_rxq_info_reg_mem_model() When you register an XSk pool as XDP Rxq info memory model, you then need to manually attach it after the registration. Let the user combine both actions into one by just passing a pointer to the pool directly to xdp_rxq_info_reg_mem_model(), which will take care of calling xsk_pool_set_rxq_info(). This looks similar to how a &page_pool gets registered and reduce repeating driver code. Signed-off-by: Alexander Lobakin --- net/core/xdp.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/net/core/xdp.c b/net/core/xdp.c index de089bee600567..2c1757e6401bb2 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -358,6 +358,9 @@ int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, if (IS_ERR(xdp_alloc)) return PTR_ERR(xdp_alloc); + if (type == MEM_TYPE_XSK_BUFF_POOL && allocator) + xsk_pool_set_rxq_info(allocator, xdp_rxq); + if (trace_mem_connect_enabled() && xdp_alloc) trace_mem_connect(xdp_alloc, xdp_rxq); return 0; From bddf7b11eb481cdeb6a235714fae681f26f305b4 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 30 Jan 2024 16:48:39 +0100 Subject: [PATCH 29/49] xsk: make xsk_buff_add_frag really add a frag via __xdp_buff_add_frag() Currently, xsk_buff_add_frag() only adds a frag to the pool linked list, not doing anythig with the &xdp_buff. The drivers do that manually and the logic is the same. Make it really add an skb frag, just like xdp_buff_add_frag() does that, and freeing frags on error if needed. This allows to remove repeating code from i40e and ice and not add the same code again and again. Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/i40e/i40e_xsk.c | 30 ++------------------ drivers/net/ethernet/intel/ice/ice_xsk.c | 32 ++-------------------- include/net/xdp_sock_drv.h | 18 ++++++++++-- 3 files changed, 20 insertions(+), 60 deletions(-) diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c index 794279495dd087..7d894520617542 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c +++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c @@ -398,32 +398,6 @@ static void i40e_handle_xdp_result_zc(struct i40e_ring *rx_ring, WARN_ON_ONCE(1); } -static int -i40e_add_xsk_frag(struct i40e_ring *rx_ring, struct xdp_buff *first, - struct xdp_buff *xdp, const unsigned int size) -{ - struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(first); - - if (!xdp_buff_has_frags(first)) { - sinfo->nr_frags = 0; - sinfo->xdp_frags_size = 0; - xdp_buff_set_frags_flag(first); - } - - if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) { - xsk_buff_free(first); - return -ENOMEM; - } - - __skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++, - virt_to_page(xdp->data_hard_start), - XDP_PACKET_HEADROOM, size); - sinfo->xdp_frags_size += size; - xsk_buff_add_frag(xdp); - - return 0; -} - /** * i40e_clean_rx_irq_zc - Consumes Rx packets from the hardware ring * @rx_ring: Rx ring @@ -489,8 +463,10 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) if (!first) first = bi; - else if (i40e_add_xsk_frag(rx_ring, first, bi, size)) + else if (!xsk_buff_add_frag(first, bi)) { + xsk_buff_free(first); break; + } if (++next_to_process == count) next_to_process = 0; diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c index 12d3592fe19de1..653e0d9444361f 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.c +++ b/drivers/net/ethernet/intel/ice/ice_xsk.c @@ -792,35 +792,6 @@ ice_run_xdp_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, return result; } -static int -ice_add_xsk_frag(struct ice_rx_ring *rx_ring, struct xdp_buff *first, - struct xdp_buff *xdp, const unsigned int size) -{ - struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(first); - - if (!size) - return 0; - - if (!xdp_buff_has_frags(first)) { - sinfo->nr_frags = 0; - sinfo->xdp_frags_size = 0; - xdp_buff_set_frags_flag(first); - } - - if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) { - xsk_buff_free(first); - return -ENOMEM; - } - - __skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++, - virt_to_page(xdp->data_hard_start), - XDP_PACKET_HEADROOM, size); - sinfo->xdp_frags_size += size; - xsk_buff_add_frag(xdp); - - return 0; -} - /** * ice_clean_rx_irq_zc - consumes packets from the hardware ring * @rx_ring: AF_XDP Rx ring @@ -884,7 +855,8 @@ int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, int budget) if (!first) { first = xdp; - } else if (ice_add_xsk_frag(rx_ring, first, xdp, size)) { + } else if (likely(size) && !xsk_buff_add_frag(first, xdp)) { + xsk_buff_free(first); break; } diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index dcd469d258405d..6fe0e4c032e42c 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -136,11 +136,21 @@ static inline void xsk_buff_free(struct xdp_buff *xdp) xp_free(xskb); } -static inline void xsk_buff_add_frag(struct xdp_buff *xdp) +static inline bool xsk_buff_add_frag(struct xdp_buff *head, + struct xdp_buff *xdp) { - struct xdp_buff_xsk *frag = container_of(xdp, struct xdp_buff_xsk, xdp); + const void *data = xdp->data; + struct xdp_buff_xsk *frag; + + if (!__xdp_buff_add_frag(head, virt_to_page(data), + offset_in_page(data), xdp->data_end - data, + xdp->frame_sz)) + return false; + frag = container_of(xdp, struct xdp_buff_xsk, xdp); list_add_tail(&frag->xskb_list_node, &frag->pool->xskb_list); + + return true; } static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first) @@ -357,8 +367,10 @@ static inline void xsk_buff_free(struct xdp_buff *xdp) { } -static inline void xsk_buff_add_frag(struct xdp_buff *xdp) +static inline bool xsk_buff_add_frag(struct xdp_buff *head, + struct xdp_buff *xdp) { + return false; } static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first) From 5f5c62c75ff713a8cd5d8057218a1e0354ebbed1 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Fri, 22 Dec 2023 19:59:40 +0100 Subject: [PATCH 30/49] xsk: add generic XSk &xdp_buff -> skb conversion Same as with converting &xdp_buff to skb on Rx, the code which allocates a new skb and copies the XSk frame there is identical across the drivers, so make it generic. This includes copying all the frags if they are present in the original buff. System percpu Page Pools help here a lot: when available, allocate pages from there instead of the MM layer. This greatly improves XDP_PASS performance on XSk: instead of page_alloc() + page_free(), the net core recycles the same pages, so the only overhead left is memcpy()s. Note that the passed buff gets freed if the conversion is done w/o any error, assuming you don't need this buffer after you convert it to an skb. Signed-off-by: Alexander Lobakin --- include/net/xdp.h | 1 + net/core/xdp.c | 138 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/include/net/xdp.h b/include/net/xdp.h index e8e2278f92bea7..d7b4e0b3be08c8 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -316,6 +316,7 @@ void xdp_warn(const char *msg, const char *func, const int line); #define XDP_WARN(msg) xdp_warn(msg, __func__, __LINE__) struct sk_buff *xdp_build_skb_from_buff(const struct xdp_buff *xdp); +struct sk_buff *xdp_build_skb_from_zc(struct xdp_buff *xdp); struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp); struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf, struct sk_buff *skb, diff --git a/net/core/xdp.c b/net/core/xdp.c index 2c1757e6401bb2..316b39816c8c3e 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -22,6 +22,8 @@ #include #include +#include "dev.h" + #define REG_STATE_NEW 0x0 #define REG_STATE_REGISTERED 0x1 #define REG_STATE_UNREGISTERED 0x2 @@ -682,6 +684,142 @@ struct sk_buff *xdp_build_skb_from_buff(const struct xdp_buff *xdp) } EXPORT_SYMBOL_GPL(xdp_build_skb_from_buff); +/** + * xdp_copy_frags_from_zc - copy the frags from an XSk buff to an skb + * @skb: skb to copy frags to + * @xdp: XSk &xdp_buff from which the frags will be copied + * @pp: &page_pool backing page allocation, if available + * + * Copy all frags from an XSk &xdp_buff to an skb to pass it up the stack. + * Allocate a new page / page frag for each frag, copy it and attach to + * the skb. + * + * Return: true on success, false on page allocation fail. + */ +static noinline bool xdp_copy_frags_from_zc(struct sk_buff *skb, + const struct xdp_buff *xdp, + struct page_pool *pp) +{ + const struct skb_shared_info *xinfo; + struct skb_shared_info *sinfo; + u32 nr_frags, ts; + + xinfo = xdp_get_shared_info_from_buff(xdp); + nr_frags = xinfo->nr_frags; + sinfo = skb_shinfo(skb); + +#if IS_ENABLED(CONFIG_PAGE_POOL) + ts = 0; +#else + ts = xinfo->xdp_frags_truesize ? : nr_frags * xdp->frame_sz; +#endif + + for (u32 i = 0; i < nr_frags; i++) { + u32 len = skb_frag_size(&xinfo->frags[i]); + void *data; +#if IS_ENABLED(CONFIG_PAGE_POOL) + u32 truesize = len; + + data = page_pool_dev_alloc_va(pp, &truesize); + ts += truesize; +#else + data = napi_alloc_frag(len); +#endif + if (unlikely(!data)) + return false; + + memcpy(data, skb_frag_address(&xinfo->frags[i]), + LARGEST_ALIGN(len)); + __skb_fill_page_desc(skb, sinfo->nr_frags++, + virt_to_page(data), + offset_in_page(data), len); + } + + xdp_update_skb_shared_info(skb, nr_frags, xinfo->xdp_frags_size, + ts, false); + + return true; +} + +/** + * xdp_build_skb_from_zc - create an skb from an XSk &xdp_buff + * @xdp: source XSk buff + * + * Similar to xdp_build_skb_from_buff(), but for XSk frames. Allocate an skb + * head, new page for the head, copy the data and initialize the skb fields. + * If there are frags, allocate new pages for them and copy. + * If Page Pool is available, the function allocates memory from the system + * percpu pools to try recycling the pages, otherwise it uses the NAPI page + * frag caches. + * If new skb was built successfully, @xdp is returned to XSk pool's freelist. + * On error, it remains untouched and the caller must take care of this. + * + * Return: new &sk_buff on success, %NULL on error. + */ +struct sk_buff *xdp_build_skb_from_zc(struct xdp_buff *xdp) +{ + const struct xdp_rxq_info *rxq = xdp->rxq; + u32 len = xdp->data_end - xdp->data_meta; + struct page_pool *pp; + struct sk_buff *skb; + int metalen; +#if IS_ENABLED(CONFIG_PAGE_POOL) + u32 truesize; + void *data; + + pp = this_cpu_read(system_page_pool); + truesize = xdp->frame_sz; + + data = page_pool_dev_alloc_va(pp, &truesize); + if (unlikely(!data)) + return NULL; + + skb = napi_build_skb(data, truesize); + if (unlikely(!skb)) { + page_pool_free_va(pp, data, true); + return NULL; + } + + skb_mark_for_recycle(skb); + skb_reserve(skb, xdp->data_meta - xdp->data_hard_start); +#else /* !CONFIG_PAGE_POOL */ + struct napi_struct *napi; + + pp = NULL; + napi = napi_by_id(rxq->napi_id); + if (likely(napi)) + skb = napi_alloc_skb(napi, len); + else + skb = __netdev_alloc_skb_ip_align(rxq->dev, len, + GFP_ATOMIC | __GFP_NOWARN); + if (unlikely(!skb)) + return NULL; +#endif /* !CONFIG_PAGE_POOL */ + + memcpy(__skb_put(skb, len), xdp->data_meta, LARGEST_ALIGN(len)); + + metalen = xdp->data - xdp->data_meta; + if (metalen > 0) { + skb_metadata_set(skb, metalen); + __skb_pull(skb, metalen); + } + + skb_record_rx_queue(skb, rxq->queue_index); + + if (unlikely(xdp_buff_has_frags(xdp)) && + unlikely(!xdp_copy_frags_from_zc(skb, xdp, pp))) { + napi_consume_skb(skb, true); + return NULL; + } + + xsk_buff_free(xdp); + + skb->protocol = eth_type_trans(skb, rxq->dev); + + return skb; +} +EXPORT_SYMBOL_GPL(xdp_build_skb_from_zc); + struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf, struct sk_buff *skb, struct net_device *dev) From 39081746872c8787ce8fc32cd09af7e2ee8d1bff Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Mon, 25 Mar 2024 17:49:50 +0100 Subject: [PATCH 31/49] xsk: add helper to get &xdp_desc's DMA and meta pointer in one go Currently, when you send an XSk frame without metadata, you need to do the following: * call external xsk_buff_raw_get_dma(); * call inline xsk_buff_get_metadata(), which calls external xsk_buff_raw_get_data() and then do some inline checks. This effectively means that the following piece: addr = pool->unaligned ? xp_unaligned_add_offset_to_addr(addr) : addr; is done twice per frame, plus you have 2 external calls per frame, plus this: meta = pool->addrs + addr - pool->tx_metadata_len; if (unlikely(!xsk_buff_valid_tx_metadata(meta))) is always inlined, even if there's no meta or it's invalid. Add xsk_buff_raw_get_ctx() (xp_raw_get_ctx() to be precise) to do that in one go. It returns a small structure with 2 fields: DMA address, filled unconditionally, and metadata pointer, valid only if it's present. The address correction is performed only once and you also have only 1 external call per XSk frame, which does all the calculations and checks outside of your hotpath. You only need to check `if (ctx.meta)` for the metadata presence. Signed-off-by: Alexander Lobakin --- include/net/xdp_sock_drv.h | 23 +++++++++++++++++++++ include/net/xsk_buff_pool.h | 8 ++++++++ net/xdp/xsk_buff_pool.c | 40 +++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index 6fe0e4c032e42c..dfd97b079363f1 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -205,6 +205,23 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) return xp_raw_get_data(pool, addr); } +/** + * xsk_buff_raw_get_ctx - get &xdp_desc context + * @pool: XSk buff pool desc address belongs to + * @addr: desc address (from userspace) + * + * Wrapper for xp_raw_get_ctx() to be used in drivers, see its kdoc for + * details. + * + * Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata + * pointer, if it is present and valid (initialized to %NULL otherwise). + */ +static inline struct xdp_desc_ctx +xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) +{ + return xp_raw_get_ctx(pool, addr); +} + #define XDP_TXMD_FLAGS_VALID ( \ XDP_TXMD_FLAGS_TIMESTAMP | \ XDP_TXMD_FLAGS_CHECKSUM | \ @@ -402,6 +419,12 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) return NULL; } +static inline struct xdp_desc_ctx +xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) +{ + return (struct xdp_desc_ctx){ }; +} + static inline bool xsk_buff_valid_tx_metadata(struct xsk_tx_metadata *meta) { return false; diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h index 0442ba8dafa46a..e50918b6283ed4 100644 --- a/include/net/xsk_buff_pool.h +++ b/include/net/xsk_buff_pool.h @@ -143,6 +143,14 @@ u32 xp_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max); bool xp_can_alloc(struct xsk_buff_pool *pool, u32 count); void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr); dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr); + +struct xdp_desc_ctx { + dma_addr_t dma; + struct xsk_tx_metadata *meta; +}; + +struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr); + static inline dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb) { return xskb->dma; diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c index c0e0204b963045..4636f071b4143e 100644 --- a/net/xdp/xsk_buff_pool.c +++ b/net/xdp/xsk_buff_pool.c @@ -687,3 +687,43 @@ dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr) (addr & ~PAGE_MASK); } EXPORT_SYMBOL(xp_raw_get_dma); + +/** + * xp_raw_get_ctx - get &xdp_desc context + * @pool: XSk buff pool desc address belongs to + * @addr: desc address (from userspace) + * + * Helper for getting desc's DMA address and metadata pointer, if present. + * Saves one call on hotpath, double calculation of the actual address, + * and inline checks for metadata presense and sanity. + * Please use xsk_buff_raw_get_ctx() in drivers instead. + * + * Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata + * pointer, if it is present and valid (initialized to %NULL otherwise). + */ +struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) +{ + struct xsk_tx_metadata *meta; + struct xdp_desc_ctx ret; + + addr = pool->unaligned ? xp_unaligned_add_offset_to_addr(addr) : addr; + ret = (typeof(ret)){ + /* Same logic as in xp_raw_get_dma() */ + .dma = (pool->dma_pages[addr >> PAGE_SHIFT] & + ~XSK_NEXT_PG_CONTIG_MASK) + (addr & ~PAGE_MASK), + }; + + if (!pool->tx_metadata_len) + goto out; + + /* Same logic as in xp_raw_get_data() + xsk_buff_get_metadata() */ + meta = pool->addrs + addr - pool->tx_metadata_len; + if (unlikely(!xsk_buff_valid_tx_metadata(meta))) + goto out; + + ret.meta = meta; + +out: + return ret; +} +EXPORT_SYMBOL(xp_raw_get_ctx); From 2b9703f5232807e1346a6b16bc608c0680af21c3 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Fri, 5 Apr 2024 17:23:24 +0200 Subject: [PATCH 32/49] skbuff: allow 2-4-argument skb_frag_dma_map() skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag), DMA_TO_DEVICE) is repeated across dozens of drivers and really wants a shorthand. Add a macro which will count args and handle all possible number from 2 to 5. Semantics: skb_frag_dma_map(dev, frag) -> __skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag), DMA_TO_DEVICE) skb_frag_dma_map(dev, frag, offset) -> __skb_frag_dma_map(dev, frag, offset, skb_frag_size(frag) - offset, DMA_TO_DEVICE) skb_frag_dma_map(dev, frag, offset, size) -> __skb_frag_dma_map(dev, frag, offset, size, DMA_TO_DEVICE) skb_frag_dma_map(dev, frag, offset, size, dir) -> __skb_frag_dma_map(dev, frag, offset, size, dir) No object code size changes for the existing callers. Users passing less arguments also won't have bigger size comparing to the full equivalent call. Signed-off-by: Alexander Lobakin --- include/linux/skbuff.h | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index f01fa8ebd8fd0f..dceacf3691c98a 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3580,7 +3580,7 @@ static inline void skb_frag_page_copy(skb_frag_t *fragto, bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio); /** - * skb_frag_dma_map - maps a paged fragment via the DMA API + * __skb_frag_dma_map - maps a paged fragment via the DMA API * @dev: the device to map the fragment to * @frag: the paged fragment to map * @offset: the offset within the fragment (starting at the @@ -3590,15 +3590,36 @@ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio); * * Maps the page associated with @frag to @device. */ -static inline dma_addr_t skb_frag_dma_map(struct device *dev, - const skb_frag_t *frag, - size_t offset, size_t size, - enum dma_data_direction dir) +static inline dma_addr_t __skb_frag_dma_map(struct device *dev, + const skb_frag_t *frag, + size_t offset, size_t size, + enum dma_data_direction dir) { return dma_map_page(dev, skb_frag_page(frag), skb_frag_off(frag) + offset, size, dir); } +#define __skb_frag_dma_map1(dev, frag, offset, uf, uo) ({ \ + const skb_frag_t *uf = (frag); \ + size_t uo = (offset); \ + \ + __skb_frag_dma_map(dev, uf, uo, skb_frag_size(uf) - uo, \ + DMA_TO_DEVICE); \ +}) +#define _skb_frag_dma_map1(dev, frag, offset) \ + __skb_frag_dma_map1(dev, frag, offset, __UNIQUE_ID(frag_), \ + __UNIQUE_ID(offset_)) +#define _skb_frag_dma_map0(dev, frag) \ + _skb_frag_dma_map1(dev, frag, 0) +#define _skb_frag_dma_map2(dev, frag, offset, size) \ + __skb_frag_dma_map(dev, frag, offset, size, DMA_TO_DEVICE) +#define _skb_frag_dma_map3(dev, frag, offset, size, dir) \ + __skb_frag_dma_map(dev, frag, offset, size, dir) + +#define skb_frag_dma_map(dev, frag, ...) \ + CONCATENATE(_skb_frag_dma_map, \ + COUNT_ARGS(__VA_ARGS__))(dev, frag, ##__VA_ARGS__) + static inline struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask) { From ae28117519466b92cde82c68039836fb63d9b959 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Fri, 15 Dec 2023 17:35:38 +0100 Subject: [PATCH 33/49] jump_label: export static_key_slow_{inc,dec}_cpuslocked() Sometimes, there's a need to modify a lot of static keys or modify the same key multiple times in a loop. In that case, it seems more optimal to lock cpu_read_lock once and then call _cpuslocked() variants. The enable/disable functions are already exported, the refcounted counterparts however are not. Fix that to allow modules to save some cycles. Signed-off-by: Alexander Lobakin --- kernel/jump_label.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 3218fa5688b939..aec11c1aebe16e 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -177,6 +177,7 @@ bool static_key_slow_inc_cpuslocked(struct static_key *key) jump_label_unlock(); return true; } +EXPORT_SYMBOL_GPL(static_key_slow_inc_cpuslocked); bool static_key_slow_inc(struct static_key *key) { @@ -304,6 +305,7 @@ void static_key_slow_dec_cpuslocked(struct static_key *key) STATIC_KEY_CHECK_USE(key); __static_key_slow_dec_cpuslocked(key); } +EXPORT_SYMBOL_GPL(static_key_slow_dec_cpuslocked); void __static_key_slow_dec_deferred(struct static_key *key, struct delayed_work *work, From 35d394755b77c35b9a3964e08ace8fa77840802f Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Fri, 8 Dec 2023 13:53:51 +0100 Subject: [PATCH 34/49] libeth: support native XDP and register memory model Expand libeth's Page Pool functionality by adding native XDP support. This means picking the appropriate headroom and DMA direction. Also, register all the created &page_pools as XDP memory models. A driver then can call xdp_rxq_info_attach_page_pool() when registering its RxQ info. Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/libeth/rx.c | 21 +++++++++++++++------ include/net/libeth/rx.h | 6 +++++- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/drivers/net/ethernet/intel/libeth/rx.c b/drivers/net/ethernet/intel/libeth/rx.c index d31779bbfccd7c..5c65efbc527a9a 100644 --- a/drivers/net/ethernet/intel/libeth/rx.c +++ b/drivers/net/ethernet/intel/libeth/rx.c @@ -68,7 +68,7 @@ static u32 libeth_rx_hw_len_truesize(const struct page_pool_params *pp, static bool libeth_rx_page_pool_params(struct libeth_fq *fq, struct page_pool_params *pp) { - pp->offset = LIBETH_SKB_HEADROOM; + pp->offset = fq->xdp ? LIBETH_XDP_HEADROOM : LIBETH_SKB_HEADROOM; /* HW-writeable / syncable length per one page */ pp->max_len = LIBETH_RX_PAGE_LEN(pp->offset); @@ -155,11 +155,12 @@ int libeth_rx_fq_create(struct libeth_fq *fq, struct napi_struct *napi) .dev = napi->dev->dev.parent, .netdev = napi->dev, .napi = napi, - .dma_dir = DMA_FROM_DEVICE, }; struct libeth_fqe *fqes; struct page_pool *pool; - bool ret; + int ret; + + pp.dma_dir = fq->xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; if (!fq->hsplit) ret = libeth_rx_page_pool_params(fq, &pp); @@ -173,18 +174,26 @@ int libeth_rx_fq_create(struct libeth_fq *fq, struct napi_struct *napi) return PTR_ERR(pool); fqes = kvcalloc_node(fq->count, sizeof(*fqes), GFP_KERNEL, fq->nid); - if (!fqes) + if (!fqes) { + ret = -ENOMEM; goto err_buf; + } + + ret = xdp_reg_page_pool(pool); + if (ret) + goto err_mem; fq->fqes = fqes; fq->pp = pool; return 0; +err_mem: + kvfree(fqes); err_buf: page_pool_destroy(pool); - return -ENOMEM; + return ret; } EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_create, LIBETH); @@ -194,8 +203,8 @@ EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_create, LIBETH); */ void libeth_rx_fq_destroy(struct libeth_fq *fq) { + xdp_unreg_page_pool(fq->pp); kvfree(fq->fqes); - page_pool_destroy(fq->pp); } EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_destroy, LIBETH); diff --git a/include/net/libeth/rx.h b/include/net/libeth/rx.h index 43574bd6612f0e..148be5cd822e92 100644 --- a/include/net/libeth/rx.h +++ b/include/net/libeth/rx.h @@ -13,8 +13,10 @@ /* Space reserved in front of each frame */ #define LIBETH_SKB_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN) +#define LIBETH_XDP_HEADROOM (ALIGN(XDP_PACKET_HEADROOM, NET_SKB_PAD) + \ + NET_IP_ALIGN) /* Maximum headroom for worst-case calculations */ -#define LIBETH_MAX_HEADROOM LIBETH_SKB_HEADROOM +#define LIBETH_MAX_HEADROOM LIBETH_XDP_HEADROOM /* Link layer / L2 overhead: Ethernet, 2 VLAN tags (C + S), FCS */ #define LIBETH_RX_LL_LEN (ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN) /* Maximum supported L2-L4 header length */ @@ -66,6 +68,7 @@ enum libeth_fqe_type { * @count: number of descriptors/buffers the queue has * @type: type of the buffers this queue has * @hsplit: flag whether header split is enabled + * @xdp: flag indicating whether XDP is enabled * @buf_len: HW-writeable length per each buffer * @nid: ID of the closest NUMA node with memory */ @@ -81,6 +84,7 @@ struct libeth_fq { /* Cold fields */ enum libeth_fqe_type type:2; bool hsplit:1; + bool xdp:1; u32 buf_len; int nid; From 6f307ba5f6cfb795e8c1e980c44462b351da5e12 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Mon, 11 Dec 2023 12:27:39 +0100 Subject: [PATCH 35/49] libeth: add a couple of XDP helpers (libeth_xdp) "Couple" is a bit humbly... Add the following functionality to libeth: * XDP shared queues managing * XDP_TX bulk sending infra * .ndo_xdp_xmit() infra * adding buffers to &xdp_buff * running XDP prog and managing its verdict * completing XDP Tx buffers * ^ repeat everything for XSk Suggested-by: Maciej Fijalkowski # lots of stuff Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/libeth/Kconfig | 6 + drivers/net/ethernet/intel/libeth/Makefile | 6 + drivers/net/ethernet/intel/libeth/priv.h | 28 + drivers/net/ethernet/intel/libeth/rx.c | 2 +- drivers/net/ethernet/intel/libeth/tx.c | 39 + drivers/net/ethernet/intel/libeth/xdp.c | 427 +++++ drivers/net/ethernet/intel/libeth/xsk.c | 264 +++ include/net/libeth/stats.h | 1 - include/net/libeth/tx.h | 34 +- include/net/libeth/types.h | 158 +- include/net/libeth/xdp.h | 1776 ++++++++++++++++++++ include/net/libeth/xsk.h | 680 ++++++++ 12 files changed, 3416 insertions(+), 5 deletions(-) create mode 100644 drivers/net/ethernet/intel/libeth/tx.c create mode 100644 drivers/net/ethernet/intel/libeth/xdp.c create mode 100644 drivers/net/ethernet/intel/libeth/xsk.c create mode 100644 include/net/libeth/xdp.h create mode 100644 include/net/libeth/xsk.h diff --git a/drivers/net/ethernet/intel/libeth/Kconfig b/drivers/net/ethernet/intel/libeth/Kconfig index 480293b71dbc37..80e99d0445998f 100644 --- a/drivers/net/ethernet/intel/libeth/Kconfig +++ b/drivers/net/ethernet/intel/libeth/Kconfig @@ -7,3 +7,9 @@ config LIBETH help libeth is a common library containing routines shared between several drivers, but not yet promoted to the generic kernel API. + +config LIBETH_XDP + tristate + select LIBETH + help + XDP and XSk helpers based on libeth hotpath management. diff --git a/drivers/net/ethernet/intel/libeth/Makefile b/drivers/net/ethernet/intel/libeth/Makefile index b30a2804554f04..9183cfa7d87f7a 100644 --- a/drivers/net/ethernet/intel/libeth/Makefile +++ b/drivers/net/ethernet/intel/libeth/Makefile @@ -6,3 +6,9 @@ obj-$(CONFIG_LIBETH) += libeth.o libeth-y += netdev.o libeth-y += rx.o libeth-y += stats.o +libeth-y += tx.o + +obj-$(CONFIG_LIBETH_XDP) += libeth_xdp.o + +libeth_xdp-y += xdp.o +libeth_xdp-y += xsk.o diff --git a/drivers/net/ethernet/intel/libeth/priv.h b/drivers/net/ethernet/intel/libeth/priv.h index 6455aab0311cf8..86f236e6533741 100644 --- a/drivers/net/ethernet/intel/libeth/priv.h +++ b/drivers/net/ethernet/intel/libeth/priv.h @@ -18,4 +18,32 @@ int libeth_stats_get_sset_count(struct net_device *dev); void libeth_stats_get_strings(struct net_device *dev, u8 *data); void libeth_stats_get_data(struct net_device *dev, u64 *data); +/* XDP */ + +enum xdp_action; +struct libeth_xdp_buff; +struct libeth_xdp_tx_frame; +struct skb_shared_info; +struct xdp_frame_bulk; + +extern const struct xsk_tx_metadata_ops libeth_xsktmo_slow; + +void libeth_xsk_tx_return_bulk(const struct libeth_xdp_tx_frame *bq, + u32 count); +u32 libeth_xsk_prog_exception(struct libeth_xdp_buff *xdp, enum xdp_action act, + int ret); + +struct libeth_xdp_ops { + void (*bulk)(const struct skb_shared_info *sinfo, + struct xdp_frame_bulk *bq, bool frags); + void (*xsk)(struct libeth_xdp_buff *xdp); +}; + +void libeth_attach_xdp(const struct libeth_xdp_ops *ops); + +static inline void libeth_detach_xdp(void) +{ + libeth_attach_xdp(NULL); +} + #endif /* __LIBETH_PRIV_H */ diff --git a/drivers/net/ethernet/intel/libeth/rx.c b/drivers/net/ethernet/intel/libeth/rx.c index 5c65efbc527a9a..63ffa48e2f989f 100644 --- a/drivers/net/ethernet/intel/libeth/rx.c +++ b/drivers/net/ethernet/intel/libeth/rx.c @@ -214,7 +214,7 @@ EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_destroy, LIBETH); * * To be used on exceptions or rare cases not requiring fast inline recycling. */ -void libeth_rx_recycle_slow(struct page *page) +void __cold libeth_rx_recycle_slow(struct page *page) { page_pool_recycle_direct(page->pp, page); } diff --git a/drivers/net/ethernet/intel/libeth/tx.c b/drivers/net/ethernet/intel/libeth/tx.c new file mode 100644 index 00000000000000..dc8df216c92286 --- /dev/null +++ b/drivers/net/ethernet/intel/libeth/tx.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2024 Intel Corporation */ + +#include + +#include "priv.h" + +/* Tx buffer completion */ + +DEFINE_STATIC_CALL_NULL(bulk, libeth_xdp_return_buff_bulk); +DEFINE_STATIC_CALL_NULL(xsk, libeth_xsk_buff_free_slow); + +/** + * libeth_tx_complete_any - perform Tx completion for one SQE of any type + * @sqe: Tx buffer to complete + * @cp: polling params + * + * Can be used to complete both regular and XDP SQEs, for example when + * destroying queues. + * When libeth_xdp is not loaded, XDPSQEs won't be handled. + */ +void libeth_tx_complete_any(struct libeth_sqe *sqe, struct libeth_cq_pp *cp) +{ + if (sqe->type >= __LIBETH_SQE_XDP_START) + __libeth_xdp_complete_tx(sqe, cp, static_call(bulk), + static_call(xsk)); + else + libeth_tx_complete(sqe, cp); +} +EXPORT_SYMBOL_NS_GPL(libeth_tx_complete_any, LIBETH); + +/* Module */ + +void libeth_attach_xdp(const struct libeth_xdp_ops *ops) +{ + static_call_update(bulk, ops ? ops->bulk : NULL); + static_call_update(xsk, ops ? ops->xsk : NULL); +} +EXPORT_SYMBOL_NS_GPL(libeth_attach_xdp, LIBETH); diff --git a/drivers/net/ethernet/intel/libeth/xdp.c b/drivers/net/ethernet/intel/libeth/xdp.c new file mode 100644 index 00000000000000..9487bf99f09365 --- /dev/null +++ b/drivers/net/ethernet/intel/libeth/xdp.c @@ -0,0 +1,427 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2024 Intel Corporation */ + +#include + +#include "priv.h" + +/* XDPSQ sharing */ + +DEFINE_STATIC_KEY_FALSE(libeth_xdpsq_share); +EXPORT_SYMBOL_NS_GPL(libeth_xdpsq_share, LIBETH_XDP); + +void __libeth_xdpsq_get(struct libeth_xdpsq_lock *lock, + const struct net_device *dev) +{ + bool warn; + + spin_lock_init(&lock->lock); + lock->share = true; + + warn = !static_key_enabled(&libeth_xdpsq_share); + static_branch_inc_cpuslocked(&libeth_xdpsq_share); + + if (warn && net_ratelimit()) + netdev_warn(dev, "XDPSQ sharing enabled, possible XDP Tx slowdown\n"); +} +EXPORT_SYMBOL_NS_GPL(__libeth_xdpsq_get, LIBETH_XDP); + +void __libeth_xdpsq_put(struct libeth_xdpsq_lock *lock, + const struct net_device *dev) +{ + static_branch_dec_cpuslocked(&libeth_xdpsq_share); + + if (!static_key_enabled(&libeth_xdpsq_share) && net_ratelimit()) + netdev_notice(dev, "XDPSQ sharing disabled\n"); + + lock->share = false; +} +EXPORT_SYMBOL_NS_GPL(__libeth_xdpsq_put, LIBETH_XDP); + +void __libeth_xdpsq_lock(struct libeth_xdpsq_lock *lock) +{ + spin_lock(&lock->lock); +} +EXPORT_SYMBOL_NS_GPL(__libeth_xdpsq_lock, LIBETH_XDP); + +void __libeth_xdpsq_unlock(struct libeth_xdpsq_lock *lock) +{ + spin_unlock(&lock->lock); +} +EXPORT_SYMBOL_NS_GPL(__libeth_xdpsq_unlock, LIBETH_XDP); + +/* XDPSQ clean-up timers */ + +/** + * libeth_xdpsq_init_timer - initialize an XDPSQ clean-up timer + * @timer: timer to initialize + * @xdpsq: queue this timer belongs to + * @lock: corresponding XDPSQ lock + * @poll: queue polling/completion function + * + * XDPSQ clean-up timers must be set up before using at the queue configuration + * time. Set the required pointers and the cleaning callback. + */ +void libeth_xdpsq_init_timer(struct libeth_xdpsq_timer *timer, void *xdpsq, + struct libeth_xdpsq_lock *lock, + void (*poll)(struct work_struct *work)) +{ + timer->xdpsq = xdpsq; + timer->lock = lock; + + INIT_DELAYED_WORK(&timer->dwork, poll); +} +EXPORT_SYMBOL_NS_GPL(libeth_xdpsq_init_timer, LIBETH_XDP); + +/* ``XDP_TX`` bulking */ + +static void __cold +libeth_xdp_tx_return_one(const struct libeth_xdp_tx_frame *frm) +{ + if (frm->len_fl & LIBETH_XDP_TX_MULTI) + libeth_xdp_return_frags(frm->data + frm->soff, true); + + libeth_xdp_return_va(frm->data, true); +} + +static void __cold +libeth_xdp_tx_return_bulk(const struct libeth_xdp_tx_frame *bq, u32 count) +{ + for (u32 i = 0; i < count; i++) { + const struct libeth_xdp_tx_frame *frm = &bq[i]; + + if (!(frm->len_fl & LIBETH_XDP_TX_FIRST)) + continue; + + libeth_xdp_tx_return_one(frm); + } +} + +static void __cold libeth_trace_xdp_exception(const struct net_device *dev, + const struct bpf_prog *prog, + u32 act) +{ + trace_xdp_exception(dev, prog, act); +} + +/** + * libeth_xdp_tx_exception - handle Tx exceptions of XDP frames + * @bq: XDP Tx frame bulk + * @sent: number of frames sent successfully (from this bulk) + * @flags: internal libeth_xdp flags (XSk, .ndo_xdp_xmit etc.) + * + * Cold helper used by __libeth_xdp_tx_flush_bulk(), do not call directly. + * Reports XDP Tx exceptions, frees the frames that won't be sent or adjust + * the Tx bulk to try again later. + */ +void __cold libeth_xdp_tx_exception(struct libeth_xdp_tx_bulk *bq, u32 sent, + u32 flags) +{ + const struct libeth_xdp_tx_frame *pos = &bq->bulk[sent]; + u32 left = bq->count - sent; + + if (!(flags & LIBETH_XDP_TX_NDO)) + libeth_trace_xdp_exception(bq->dev, bq->prog, XDP_TX); + + if (!(flags & LIBETH_XDP_TX_DROP)) { + memmove(bq->bulk, pos, left * sizeof(*bq->bulk)); + bq->count = left; + + return; + } + + if (flags & LIBETH_XDP_TX_XSK) + libeth_xsk_tx_return_bulk(pos, left); + else if (!(flags & LIBETH_XDP_TX_NDO)) + libeth_xdp_tx_return_bulk(pos, left); + else + libeth_xdp_xmit_return_bulk(pos, left, bq->dev); + + bq->count = 0; +} +EXPORT_SYMBOL_NS_GPL(libeth_xdp_tx_exception, LIBETH_XDP); + +/* .ndo_xdp_xmit() implementation */ + +u32 __cold libeth_xdp_xmit_return_bulk(const struct libeth_xdp_tx_frame *bq, + u32 count, const struct net_device *dev) +{ + u32 n = 0; + + for (u32 i = 0; i < count; i++) { + const struct libeth_xdp_tx_frame *frm = &bq[i]; + dma_addr_t dma; + + if (frm->flags & LIBETH_XDP_TX_FIRST) + dma = *libeth_xdp_xmit_frame_dma(frm->xdpf); + else + dma = dma_unmap_addr(frm, dma); + + dma_unmap_page(dev->dev.parent, dma, dma_unmap_len(frm, len), + DMA_TO_DEVICE); + + /* Actual xdp_frames are freed by the core */ + n += !!(frm->flags & LIBETH_XDP_TX_FIRST); + } + + return n; +} +EXPORT_SYMBOL_NS_GPL(libeth_xdp_xmit_return_bulk, LIBETH_XDP); + +/* Rx polling path */ + +/** + * libeth_xdp_load_stash - recreate an &xdp_buff from a libeth_xdp buffer stash + * @dst: target &libeth_xdp_buff to initialize + * @src: source stash + * + * External helper used by libeth_xdp_init_buff(), do not call directly. + * Recreate an onstack &libeth_xdp_buff using the stash saved earlier. + * The only field untouched (rxq) is initialized later in the + * abovementioned function. + */ +void libeth_xdp_load_stash(struct libeth_xdp_buff *dst, + const struct libeth_xdp_buff_stash *src) +{ + dst->data = src->data; + dst->base.data_end = src->data + src->len; + dst->base.data_meta = src->data; + dst->base.data_hard_start = src->data - src->headroom; + + dst->base.frame_sz = src->frame_sz; + dst->base.flags = src->flags; +} +EXPORT_SYMBOL_NS_GPL(libeth_xdp_load_stash, LIBETH_XDP); + +/** + * libeth_xdp_save_stash - convert an &xdp_buff to a libeth_xdp buffer stash + * @dst: target &libeth_xdp_buff_stash to initialize + * @src: source XDP buffer + * + * External helper used by libeth_xdp_save_buff(), do not call directly. + * Use the fields from the passed XDP buffer to initialize the stash on the + * queue, so that a partially received frame can be finished later during + * the next NAPI poll. + */ +void libeth_xdp_save_stash(struct libeth_xdp_buff_stash *dst, + const struct libeth_xdp_buff *src) +{ + dst->data = src->data; + dst->headroom = src->data - src->base.data_hard_start; + dst->len = src->base.data_end - src->data; + + dst->frame_sz = src->base.frame_sz; + dst->flags = src->base.flags; + + WARN_ON_ONCE(dst->flags != src->base.flags); +} +EXPORT_SYMBOL_NS_GPL(libeth_xdp_save_stash, LIBETH_XDP); + +void __libeth_xdp_return_stash(struct libeth_xdp_buff_stash *stash) +{ + LIBETH_XDP_ONSTACK_BUFF(xdp); + + libeth_xdp_load_stash(xdp, stash); + libeth_xdp_return_buff_slow(xdp); + + stash->data = NULL; +} +EXPORT_SYMBOL_NS_GPL(__libeth_xdp_return_stash, LIBETH_XDP); + +/** + * libeth_xdp_return_buff_slow - free a &libeth_xdp_buff + * @xdp: buffer to free/return + * + * Slowpath version of libeth_xdp_return_buff() to be called on exceptions, + * queue clean-ups etc., without unwanted inlining. + */ +void __cold libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp) +{ + libeth_xdp_return_buff(xdp); +} +EXPORT_SYMBOL_NS_GPL(libeth_xdp_return_buff_slow, LIBETH_XDP); + +/** + * libeth_xdp_buff_add_frag - add a frag to an XDP buffer + * @xdp: head XDP buffer + * @fqe: Rx buffer containing the frag + * @len: frag length reported by HW + * + * External helper used by libeth_xdp_process_buff(), do not call directly. + * Frees both head and frag buffers on error. + * + * Return: true success, false on error (no space for a new frag). + */ +bool libeth_xdp_buff_add_frag(struct libeth_xdp_buff *xdp, + const struct libeth_fqe *fqe, + u32 len) +{ + struct page *page = fqe->page; + + if (!xdp_buff_add_frag(&xdp->base, page, + fqe->offset + page->pp->p.offset, + len, fqe->truesize)) + goto recycle; + + return true; + +recycle: + libeth_rx_recycle_slow(page); + libeth_xdp_return_buff_slow(xdp); + + return false; +} +EXPORT_SYMBOL_NS_GPL(libeth_xdp_buff_add_frag, LIBETH_XDP); + +/** + * libeth_xdp_prog_exception - handle XDP prog exceptions + * @bq: XDP Tx bulk + * @xdp: buffer to process + * @act: original XDP prog verdict + * @ret: error code if redirect failed + * + * External helper used by __libeth_xdp_run_prog() and + * __libeth_xsk_run_prog_slow(), do not call directly. + * Reports invalid @act, XDP exception trace event and frees the buffer. + * + * Return: libeth_xdp XDP prog verdict. + */ +u32 __cold libeth_xdp_prog_exception(const struct libeth_xdp_tx_bulk *bq, + struct libeth_xdp_buff *xdp, + enum xdp_action act, int ret) +{ + if (act > XDP_REDIRECT) + bpf_warn_invalid_xdp_action(bq->dev, bq->prog, act); + + libeth_trace_xdp_exception(bq->dev, bq->prog, act); + + if (xdp->base.rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) + return libeth_xsk_prog_exception(xdp, act, ret); + + libeth_xdp_return_buff_slow(xdp); + + return LIBETH_XDP_DROP; +} +EXPORT_SYMBOL_NS_GPL(libeth_xdp_prog_exception, LIBETH_XDP); + +/* Tx buffer completion */ + +static void libeth_xdp_put_page_bulk(struct page *page, + struct xdp_frame_bulk *bq) +{ + if (unlikely(bq->count == XDP_BULK_QUEUE_SIZE)) + xdp_flush_frame_bulk(bq); + + bq->q[bq->count++] = page; +} + +/** + * libeth_xdp_return_buff_bulk - free &xdp_buff as part of a bulk + * @sinfo: shared info corresponding to the buffer + * @bq: XDP frame bulk to store the buffer + * @frags: whether the buffer has frags + * + * Same as xdp_return_frame_bulk(), but for &libeth_xdp_buff, speeds up Tx + * completion of ``XDP_TX`` buffers and allows to free them in same bulks + * with &xdp_frame buffers. + */ +void libeth_xdp_return_buff_bulk(const struct skb_shared_info *sinfo, + struct xdp_frame_bulk *bq, bool frags) +{ + if (!frags) + goto head; + + for (u32 i = 0; i < sinfo->nr_frags; i++) + libeth_xdp_put_page_bulk(skb_frag_page(&sinfo->frags[i]), bq); + +head: + libeth_xdp_put_page_bulk(virt_to_page(sinfo), bq); +} +EXPORT_SYMBOL_NS_GPL(libeth_xdp_return_buff_bulk, LIBETH_XDP); + +/* Misc */ + +/** + * libeth_xdp_queue_threshold - calculate XDP queue clean/refill threshold + * @count: number of descriptors in the queue + * + * The threshold is the limit at which RQs start to refill (when the number of + * empty buffers exceeds it) and SQs get cleaned up (when the number of free + * descriptors goes below it). To speed up hotpath processing, threshold is + * always pow-2, closest to 1/4 of the queue length. + * Don't call it on hotpath, calculate and cache the threshold during the + * queue initialization. + * + * Return: the calculated threshold. + */ +u32 libeth_xdp_queue_threshold(u32 count) +{ + u32 quarter, low, high; + + if (likely(is_power_of_2(count))) + return count >> 2; + + quarter = DIV_ROUND_CLOSEST(count, 4); + low = rounddown_pow_of_two(quarter); + high = roundup_pow_of_two(quarter); + + return high - quarter <= quarter - low ? high : low; +} +EXPORT_SYMBOL_NS_GPL(libeth_xdp_queue_threshold, LIBETH_XDP); + +/** + * __libeth_xdp_set_features - set XDP features for a netdev + * @dev: &net_device to configure + * @xmo: XDP metadata ops (Rx hints) + * @zc_segs: maximum number of S/G frags the HW can transmit + * @tmo: XSk Tx metadata ops (Tx hints) + * + * Set all the features libeth_xdp supports. Only the first argument is + * necessary; without the third one (zero), XSk support won't be advertised. + * Use the non-underscored versions in drivers instead. + */ +void __libeth_xdp_set_features(struct net_device *dev, + const struct xdp_metadata_ops *xmo, + u32 zc_segs, + const struct xsk_tx_metadata_ops *tmo) +{ + xdp_set_features_flag(dev, + NETDEV_XDP_ACT_BASIC | + NETDEV_XDP_ACT_REDIRECT | + NETDEV_XDP_ACT_NDO_XMIT | + (zc_segs ? NETDEV_XDP_ACT_XSK_ZEROCOPY : 0) | + NETDEV_XDP_ACT_RX_SG | + NETDEV_XDP_ACT_NDO_XMIT_SG); + dev->xdp_metadata_ops = xmo; + + tmo = tmo == libeth_xsktmo ? &libeth_xsktmo_slow : tmo; + + dev->xdp_zc_max_segs = zc_segs ? : 1; + dev->xsk_tx_metadata_ops = zc_segs ? tmo : NULL; +} +EXPORT_SYMBOL_NS_GPL(__libeth_xdp_set_features, LIBETH_XDP); + +/* Module */ + +static const struct libeth_xdp_ops xdp_ops __initconst = { + .bulk = libeth_xdp_return_buff_bulk, + .xsk = libeth_xsk_buff_free_slow, +}; + +static int __init libeth_xdp_module_init(void) +{ + libeth_attach_xdp(&xdp_ops); + + return 0; +} +module_init(libeth_xdp_module_init); + +static void __exit libeth_xdp_module_exit(void) +{ + libeth_detach_xdp(); +} +module_exit(libeth_xdp_module_exit); + +MODULE_DESCRIPTION("Common Ethernet library - XDP infra"); +MODULE_IMPORT_NS(LIBETH); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/ethernet/intel/libeth/xsk.c b/drivers/net/ethernet/intel/libeth/xsk.c new file mode 100644 index 00000000000000..eccb66a644f083 --- /dev/null +++ b/drivers/net/ethernet/intel/libeth/xsk.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2024 Intel Corporation */ + +#include + +#include "priv.h" + +/* ``XDP_TX`` bulking */ + +void __cold libeth_xsk_tx_return_bulk(const struct libeth_xdp_tx_frame *bq, + u32 count) +{ + for (u32 i = 0; i < count; i++) + libeth_xsk_buff_free_slow(bq[i].xsk); +} + +/* XSk TMO */ + +const struct xsk_tx_metadata_ops libeth_xsktmo_slow = { + .tmo_request_checksum = libeth_xsktmo_req_csum, +}; + +/* Rx polling path */ + +/** + * libeth_xsk_buff_free_slow - free an XSk Rx buffer + * @xdp: buffer to free + * + * Slowpath version of xsk_buff_free() to be used on exceptions, cleanups etc. + * to avoid unwanted inlining. + */ +void libeth_xsk_buff_free_slow(struct libeth_xdp_buff *xdp) +{ + xsk_buff_free(&xdp->base); +} +EXPORT_SYMBOL_NS_GPL(libeth_xsk_buff_free_slow, LIBETH_XDP); + +/** + * libeth_xsk_buff_add_frag - add a frag to an XSk Rx buffer + * @head: head buffer + * @xdp: frag buffer + * + * External helper used by libeth_xsk_process_buff(), do not call directly. + * Frees both main and frag buffers on error. + * + * Return: main buffer with attached frag on success, %NULL on error (no space + * for a new frag). + */ +struct libeth_xdp_buff *libeth_xsk_buff_add_frag(struct libeth_xdp_buff *head, + struct libeth_xdp_buff *xdp) +{ + if (!xsk_buff_add_frag(&head->base, &xdp->base)) + goto free; + + return head; + +free: + libeth_xsk_buff_free_slow(xdp); + libeth_xsk_buff_free_slow(head); + + return NULL; +} +EXPORT_SYMBOL_NS_GPL(libeth_xsk_buff_add_frag, LIBETH_XDP); + +/** + * libeth_xsk_buff_stats_frags - update onstack RQ stats with XSk frags info + * @rs: onstack stats to update + * @xdp: buffer to account + * + * External helper used by __libeth_xsk_run_pass(), do not call directly. + * Adds buffer's frags count and total len to the onstack stats. + */ +void libeth_xsk_buff_stats_frags(struct libeth_rq_napi_stats *rs, + const struct libeth_xdp_buff *xdp) +{ + libeth_xdp_buff_stats_frags(rs, xdp); +} +EXPORT_SYMBOL_NS_GPL(libeth_xsk_buff_stats_frags, LIBETH_XDP); + +/** + * __libeth_xsk_run_prog_slow - process the non-``XDP_REDIRECT`` verdicts + * @xdp: buffer to process + * @bq: Tx bulk for queueing on ``XDP_TX`` + * @act: verdict to process + * @ret: error code if ``XDP_REDIRECT`` failed + * + * External helper used by __libeth_xsk_run_prog(), do not call directly. + * ``XDP_REDIRECT`` is the most common and hottest verdict on XSk, thus + * it is processed inline. The rest goes here for out-of-line processing, + * together with redirect errors. + * + * Return: libeth_xdp XDP prog verdict. + */ +u32 __libeth_xsk_run_prog_slow(struct libeth_xdp_buff *xdp, + const struct libeth_xdp_tx_bulk *bq, + enum xdp_action act, int ret) +{ + switch (act) { + case XDP_DROP: + xsk_buff_free(&xdp->base); + + return LIBETH_XDP_DROP; + case XDP_TX: + return LIBETH_XDP_TX; + case XDP_PASS: + return LIBETH_XDP_PASS; + default: + break; + } + + return libeth_xdp_prog_exception(bq, xdp, act, ret); +} +EXPORT_SYMBOL_NS_GPL(__libeth_xsk_run_prog_slow, LIBETH_XDP); + +/** + * libeth_xsk_prog_exception - handle XDP prog exceptions on XSk + * @xdp: buffer to process + * @act: verdict returned by the prog + * @ret: error code if ``XDP_REDIRECT`` failed + * + * Internal. Frees the buffer and, if the queue uses XSk wakeups, stop the + * current NAPI poll when there are no free buffers left. + * + * Return: libeth_xdp's XDP prog verdict. + */ +u32 __cold libeth_xsk_prog_exception(struct libeth_xdp_buff *xdp, + enum xdp_action act, int ret) +{ + const struct xdp_buff_xsk *xsk; + u32 __ret = LIBETH_XDP_DROP; + + if (act != XDP_REDIRECT) + goto drop; + + xsk = container_of(&xdp->base, typeof(*xsk), xdp); + if (xsk_uses_need_wakeup(xsk->pool) && ret == -ENOBUFS) + __ret = LIBETH_XDP_ABORTED; + +drop: + libeth_xsk_buff_free_slow(xdp); + + return __ret; +} + +/* Refill */ + +/** + * libeth_xskfq_create - create an XSkFQ + * @fq: fill queue to initialize + * + * Allocates the FQEs and initializes the fields used by libeth_xdp: number + * of buffers to refill, refill threshold and buffer len. + * + * Return: %0 on success, -errno otherwise. + */ +int libeth_xskfq_create(struct libeth_xskfq *fq) +{ + fq->fqes = kvcalloc_node(fq->count, sizeof(*fq->fqes), GFP_KERNEL, + fq->nid); + if (!fq->fqes) + return -ENOMEM; + + fq->pending = fq->count; + fq->thresh = libeth_xdp_queue_threshold(fq->count); + fq->buf_len = xsk_pool_get_rx_frame_size(fq->pool); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(libeth_xskfq_create, LIBETH_XDP); + +/** + * libeth_xskfq_destroy - destroy an XSkFQ + * @fq: fill queue to destroy + * + * Zeroes the used fields and frees the FQEs array. + */ +void libeth_xskfq_destroy(struct libeth_xskfq *fq) +{ + fq->buf_len = 0; + fq->thresh = 0; + fq->pending = 0; + + kvfree(fq->fqes); +} +EXPORT_SYMBOL_NS_GPL(libeth_xskfq_destroy, LIBETH_XDP); + +/* .ndo_xsk_wakeup */ + +static void libeth_xsk_napi_sched(void *info) +{ + __napi_schedule_irqoff(info); +} + +/** + * libeth_xsk_init_wakeup - initialize libeth XSk wakeup structure + * @csd: struct to initialize + * @napi: NAPI corresponding to this queue + * + * libeth_xdp uses inter-processor interrupts to perform XSk wakeups. In order + * to do that, the corresponding CSDs must be initialized when creating the + * queues. + */ +void libeth_xsk_init_wakeup(call_single_data_t *csd, struct napi_struct *napi) +{ + INIT_CSD(csd, libeth_xsk_napi_sched, napi); +} +EXPORT_SYMBOL_NS_GPL(libeth_xsk_init_wakeup, LIBETH_XDP); + +/** + * libeth_xsk_wakeup - perform an XSk wakeup + * @csd: CSD corresponding to the queue + * @qid: the stack queue index + * + * Try to mark the NAPI as missed first, so that it could be rescheduled. + * If it's not, schedule it on the corresponding CPU using IPIs (or directly + * if already running on it). + */ +void libeth_xsk_wakeup(call_single_data_t *csd, u32 qid) +{ + struct napi_struct *napi = csd->info; + + if (napi_if_scheduled_mark_missed(napi) || + unlikely(!napi_schedule_prep(napi))) + return; + + if (qid != raw_smp_processor_id()) + smp_call_function_single_async(qid, csd); + else + __napi_schedule(napi); +} +EXPORT_SYMBOL_NS_GPL(libeth_xsk_wakeup, LIBETH_XDP); + +/* Pool setup */ + +#define LIBETH_XSK_DMA_ATTR \ + (DMA_ATTR_WEAK_ORDERING | DMA_ATTR_SKIP_CPU_SYNC) + +/** + * libeth_xsk_setup_pool - setup or destroy an XSk pool for a queue + * @dev: target &net_device + * @qid: stack queue index to configure + * @enable: whether to enable or disable the pool + * + * Check that @qid is valid and then map or unmap the pool. + * + * Return: %0 on success, -errno otherwise. + */ +int libeth_xsk_setup_pool(struct net_device *dev, u32 qid, bool enable) +{ + struct xsk_buff_pool *pool; + + pool = xsk_get_pool_from_qid(dev, qid); + if (!pool) + return -EINVAL; + + if (enable) + return xsk_pool_dma_map(pool, dev->dev.parent, + LIBETH_XSK_DMA_ATTR); + else + xsk_pool_dma_unmap(pool, LIBETH_XSK_DMA_ATTR); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(libeth_xsk_setup_pool, LIBETH_XDP); diff --git a/include/net/libeth/stats.h b/include/net/libeth/stats.h index 1f2dd63418f2d2..aa5221e408aedc 100644 --- a/include/net/libeth/stats.h +++ b/include/net/libeth/stats.h @@ -4,7 +4,6 @@ #ifndef __LIBETH_STATS_H #define __LIBETH_STATS_H -#include #include #include diff --git a/include/net/libeth/tx.h b/include/net/libeth/tx.h index f868309dcbda6c..043111c46407c7 100644 --- a/include/net/libeth/tx.h +++ b/include/net/libeth/tx.h @@ -10,11 +10,17 @@ /** * enum libeth_sqe_type - type of &libeth_sqe to act on Tx completion - * @LIBETH_SQE_EMPTY: unused/empty, no action required + * @LIBETH_SQE_EMPTY: unused/empty OR XDP_TX/XSk frame, no action required * @LIBETH_SQE_CTX: context descriptor with empty SQE, no action required * @LIBETH_SQE_SLAB: kmalloc-allocated buffer, unmap and kfree() * @LIBETH_SQE_FRAG: mapped skb frag, only unmap DMA * @LIBETH_SQE_SKB: &sk_buff, unmap and napi_consume_skb(), update stats + * @__LIBETH_SQE_XDP_START: separator between skb and XDP types + * @LIBETH_SQE_XDP_TX: &skb_shared_info, libeth_xdp_return_buff_bulk(), stats + * @LIBETH_SQE_XDP_XMIT: &xdp_frame, unmap and xdp_return_frame_bulk(), stats + * @LIBETH_SQE_XDP_XMIT_FRAG: &xdp_frame frag, only unmap DMA + * @LIBETH_SQE_XSK_TX: &libeth_xdp_buff on XSk queue, xsk_buff_free(), stats + * @LIBETH_SQE_XSK_TX_FRAG: &libeth_xdp_buff frag on XSk queue, xsk_buff_free() */ enum libeth_sqe_type { LIBETH_SQE_EMPTY = 0U, @@ -22,6 +28,13 @@ enum libeth_sqe_type { LIBETH_SQE_SLAB, LIBETH_SQE_FRAG, LIBETH_SQE_SKB, + + __LIBETH_SQE_XDP_START, + LIBETH_SQE_XDP_TX = __LIBETH_SQE_XDP_START, + LIBETH_SQE_XDP_XMIT, + LIBETH_SQE_XDP_XMIT_FRAG, + LIBETH_SQE_XSK_TX, + LIBETH_SQE_XSK_TX_FRAG, }; /** @@ -30,6 +43,9 @@ enum libeth_sqe_type { * @rs_idx: index of the last buffer from the batch this one was sent * @raw: slab buffer to free via kfree() * @skb: &sk_buff to consume + * @sinfo: skb shared info of an XDP_TX frame + * @xdpf: XDP frame from ::ndo_xdp_xmit() + * @xsk: XSk Rx frame from XDP_TX action * @dma: DMA address to unmap * @len: length of the mapped region to unmap * @nr_frags: number of frags in the frame this buffer belongs to @@ -44,6 +60,9 @@ struct libeth_sqe { union { void *raw; struct sk_buff *skb; + struct skb_shared_info *sinfo; + struct xdp_frame *xdpf; + struct libeth_xdp_buff *xsk; }; DEFINE_DMA_UNMAP_ADDR(dma); @@ -69,7 +88,10 @@ struct libeth_sqe { /** * struct libeth_cq_pp - completion queue poll params * @dev: &device to perform DMA unmapping + * @bq: XDP frame bulk to combine return operations * @ss: onstack NAPI stats to fill + * @xss: onstack XDPSQ NAPI stats to fill + * @xdp_tx: number of XDP-not-XSk frames processed * @napi: whether it's called from the NAPI context * * libeth uses this structure to access objects needed for performing full @@ -78,7 +100,13 @@ struct libeth_sqe { */ struct libeth_cq_pp { struct device *dev; - struct libeth_sq_napi_stats *ss; + struct xdp_frame_bulk *bq; + + union { + struct libeth_sq_napi_stats *ss; + struct libeth_xdpsq_napi_stats *xss; + }; + u32 xdp_tx; bool napi; }; @@ -124,4 +152,6 @@ static inline void libeth_tx_complete(struct libeth_sqe *sqe, sqe->type = LIBETH_SQE_EMPTY; } +void libeth_tx_complete_any(struct libeth_sqe *sqe, struct libeth_cq_pp *cp); + #endif /* __LIBETH_TX_H */ diff --git a/include/net/libeth/types.h b/include/net/libeth/types.h index 2437d68aae2190..2b699e17958367 100644 --- a/include/net/libeth/types.h +++ b/include/net/libeth/types.h @@ -4,7 +4,7 @@ #ifndef __LIBETH_TYPES_H #define __LIBETH_TYPES_H -#include +#include /** * struct libeth_netdev_priv - libeth netdev private structure @@ -244,4 +244,160 @@ struct libeth_xdpsq_stats { #undef ___live +/* XDP */ + +/* &xdp_buff_xsk is the largest structure &libeth_xdp_buff gets casted to, + * pick maximum pointer-compatible alignment. + */ +#define __libeth_xdp_buff_aligned \ + __aligned(IS_ALIGNED(sizeof(struct xdp_buff_xsk), 16) ? 16 : \ + IS_ALIGNED(sizeof(struct xdp_buff_xsk), 8) ? 8 : \ + sizeof(long)) + +/** + * struct libeth_xdp_buff - libeth extension over &xdp_buff + * @base: main &xdp_buff + * @data: shortcut for @base.data + * @desc: RQ descriptor containing metadata for this buffer + * @priv: driver-private scratchspace + * + * The main reason for this is to have a pointer to the descriptor to be able + * to quickly get frame metadata from xdpmo and driver buff-to-xdp callbacks + * (as well as bigger alignment). + * Pointer/layout-compatible with &xdp_buff and &xdp_buff_xsk. + */ +struct libeth_xdp_buff { + union { + struct xdp_buff base; + void *data; + }; + + const void *desc; + unsigned long priv[] __libeth_xdp_buff_aligned; +} __libeth_xdp_buff_aligned; +static_assert(offsetof(struct libeth_xdp_buff, data) == + offsetof(struct xdp_buff_xsk, xdp.data)); +static_assert(offsetof(struct libeth_xdp_buff, desc) == + offsetof(struct xdp_buff_xsk, cb)); +static_assert(IS_ALIGNED(sizeof(struct xdp_buff_xsk), + __alignof(struct libeth_xdp_buff))); + +#define __libeth_xdp_psz0(...) +#define __libeth_xdp_psz1(...) sizeof(__VA_ARGS__) +#define __libeth_xdp_priv_sz(...) \ + CONCATENATE(__libeth_xdp_psz, COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__) + +#define LIBETH_XDP_PRIV_SZ(sz) \ + (ALIGN(sz, __alignof(struct libeth_xdp_buff)) / sizeof(long)) + +/* Performs XSK_CHECK_PRIV_TYPE() */ +#define LIBETH_XDP_ASSERT_PRIV_SZ(sz) \ + static_assert(offsetofend(struct xdp_buff_xsk, cb) >= \ + struct_size_t(struct libeth_xdp_buff, priv, \ + LIBETH_XDP_PRIV_SZ(sz))) + +#define ___LIBETH_XDP_DECLARE_BUFF(name, up, ...) \ + union { \ + u8 up[struct_size_t(struct libeth_xdp_buff, priv, \ + LIBETH_XDP_PRIV_SZ(__VA_ARGS__ + 0))]; \ + struct libeth_xdp_buff name; \ + \ + LIBETH_XDP_ASSERT_PRIV_SZ(__VA_ARGS__ + 0); \ + } +#define ___LIBETH_XDP_ONSTACK_BUFF(name, ...) \ + _DEFINE_FLEX(struct libeth_xdp_buff, name, priv, \ + LIBETH_XDP_PRIV_SZ(__VA_ARGS__ + 0), /* no init */); \ + LIBETH_XDP_ASSERT_PRIV_SZ(__VA_ARGS__ + 0) + +/** + * __LIBETH_XDP_DECLARE_BUFF - declare a &libeth_xdp_buff inside a structure + * @name: name of the field to declare + * @...: sizeof() of the driver-private data + * + * This group of helpers already performs checks for the private data size + * and reserves space for it. + */ +#define __LIBETH_XDP_DECLARE_BUFF(name, ...) \ + ___LIBETH_XDP_DECLARE_BUFF(name, __UNIQUE_ID(raw_), ##__VA_ARGS__) +/** + * __LIBETH_XDP_ONSTACK_BUFF - declare a &libeth_xdp_buff on the stack + * @name: name of the variable to declare + * @...: sizeof() of the driver-private data + */ +#define __LIBETH_XDP_ONSTACK_BUFF(name, ...) \ + ___LIBETH_XDP_ONSTACK_BUFF(name, ##__VA_ARGS__) + +/** + * LIBETH_XDP_DECLARE_BUFF - declare a &libeth_xdp_buff inside a structure + * @name: name of the field to declare + * @...: type or variable name of the driver-private data + */ +#define LIBETH_XDP_DECLARE_BUFF(name, ...) \ + __LIBETH_XDP_DECLARE_BUFF(name, __libeth_xdp_priv_sz(__VA_ARGS__)) +/** + * LIBETH_XDP_ONSTACK_BUFF - declare a &libeth_xdp_buff on the stack + * @name: name of the variable to declare + * @...: type or variable name of the driver-private data + */ +#define LIBETH_XDP_ONSTACK_BUFF(name, ...) \ + __LIBETH_XDP_ONSTACK_BUFF(name, __libeth_xdp_priv_sz(__VA_ARGS__)) + +/* The following structures should be embedded into driver's queue structure + * and passed to the libeth_xdp helpers, never used directly. + */ + +/* XDPSQ sharing */ + +/** + * struct libeth_xdpsq_lock - locking primitive for sharing XDPSQs + * @lock: spinlock for locking the queue + * @share: whether this particular queue is shared + */ +struct libeth_xdpsq_lock { + spinlock_t lock; + bool share; +}; + +/* XDPSQ clean-up timers */ + +/** + * struct libeth_xdpsq_timer - timer for cleaning up XDPSQs w/o interrupts + * @xdpsq: queue this timer belongs to + * @lock: lock for the queue + * @dwork: work performing cleanups + * + * XDPSQs not using interrupts but lazy cleaning, i.e. only when there's no + * space for sending the current queued frame/bulk, must fire up timers to + * make sure there are no stale buffers to free. + */ +struct libeth_xdpsq_timer { + void *xdpsq; + struct libeth_xdpsq_lock *lock; + + struct delayed_work dwork; +}; + +/* Rx polling path */ + +/** + * struct libeth_xdp_buff_stash - struct for stashing &xdp_buff on a queue + * @data: pointer to the start of the frame, xdp_buff.data + * @headroom: frame headroom, xdp_buff.data - xdp_buff.data_hard_start + * @len: frame linear space length, xdp_buff.data_end - xdp_buff.data + * @frame_sz: truesize occupied by the frame, xdp_buff.frame_sz + * @flags: xdp_buff.flags + * + * &xdp_buff is 56 bytes long on x64, &libeth_xdp_buff is 64 bytes. This + * structure carries only necessary fields to save/restore a partially built + * frame on the queue structure to finish it during the next NAPI poll. + */ +struct libeth_xdp_buff_stash { + void *data; + u16 headroom; + u16 len; + + u32 frame_sz:24; + enum xdp_buff_flags flags:8; +} __aligned_largest; + #endif /* __LIBETH_TYPES_H */ diff --git a/include/net/libeth/xdp.h b/include/net/libeth/xdp.h new file mode 100644 index 00000000000000..2c4aa4320af128 --- /dev/null +++ b/include/net/libeth/xdp.h @@ -0,0 +1,1776 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2024 Intel Corporation */ + +#ifndef __LIBETH_XDP_H +#define __LIBETH_XDP_H + +#include + +#include +#include + +/* Defined as bits to be able to use them as a mask */ +enum { + LIBETH_XDP_PASS = 0U, + LIBETH_XDP_DROP = BIT(0), + LIBETH_XDP_ABORTED = BIT(1), + LIBETH_XDP_TX = BIT(2), + LIBETH_XDP_REDIRECT = BIT(3), +}; + +/* XDPSQ sharing */ + +DECLARE_STATIC_KEY_FALSE(libeth_xdpsq_share); + +/** + * libeth_xdpsq_num - calculate optimal number of XDPSQs for this device + sys + * @rxq: current number of active Rx queues + * @txq: current number of active Tx queues + * @max: maximum number of Tx queues + * + * Each RQ must have its own XDPSQ for XSk pairs, each CPU must have own XDPSQ + * for lockless sending (``XDP_TX``, .ndo_xdp_xmit()). Cap the maximum of these + * two with the number of SQs the device can have (minus used ones). + * + * Return: number of XDP Tx queues the device needs to use. + */ +static inline u32 libeth_xdpsq_num(u32 rxq, u32 txq, u32 max) +{ + return min(max(nr_cpu_ids, rxq), max - txq); +} + +/** + * libeth_xdpsq_shared - whether XDPSQs can be shared between several CPUs + * @num: number of active XDPSQs + * + * Return: true if there's no 1:1 XDPSQ/CPU association, false otherwise. + */ +static inline bool libeth_xdpsq_shared(u32 num) +{ + return num < nr_cpu_ids; +} + +/** + * libeth_xdpsq_id - get XDPSQ index corresponding to this CPU + * @qid: number of active XDPSQs + * + * Helper for libeth_xdp routines, do not use in drivers directly. + * + * Return: XDPSQ index needs to be used on this CPU. + */ +static inline u32 libeth_xdpsq_id(u32 qid) +{ + u32 ret = raw_smp_processor_id(); + + if (static_branch_unlikely(&libeth_xdpsq_share) && + libeth_xdpsq_shared(qid)) + ret %= qid; + + return ret; +} + +void __libeth_xdpsq_get(struct libeth_xdpsq_lock *lock, + const struct net_device *dev); +void __libeth_xdpsq_put(struct libeth_xdpsq_lock *lock, + const struct net_device *dev); + +#define libeth_xdpsq_get_start cpus_read_lock +#define libeth_xdpsq_get_end cpus_read_unlock + +/** + * libeth_xdpsq_get - initialize a &libeth_xdpsq_lock + * @lock: lock to initialize + * @dev: netdev which this lock belongs to + * @share: whether XDPSQs can be shared + * + * Must be called only inside a libeth_xdpsq_get_{start,put}() block. + * Tracks the current XDPSQ association and enables the static lock + * if needed. + */ +static inline void libeth_xdpsq_get(struct libeth_xdpsq_lock *lock, + const struct net_device *dev, + bool share) +{ + if (unlikely(share)) + __libeth_xdpsq_get(lock, dev); +} + +/** + * libeth_xdpsq_put - deinitialize a &libeth_xdpsq_lock + * @lock: lock to deinitialize + * @dev: netdev which this lock belongs to + * + * Must be called only inside a libeth_xdpsq_get_{start,put}() block. + * Tracks the current XDPSQ association and disables the static lock + * if needed. + */ +static inline void libeth_xdpsq_put(struct libeth_xdpsq_lock *lock, + const struct net_device *dev) +{ + if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share) + __libeth_xdpsq_put(lock, dev); +} + +void __acquires(&lock->lock) +__libeth_xdpsq_lock(struct libeth_xdpsq_lock *lock); +void __releases(&lock->lock) +__libeth_xdpsq_unlock(struct libeth_xdpsq_lock *lock); + +/** + * libeth_xdpsq_lock - grab a &libeth_xdpsq_lock if needed + * @lock: lock to take + * + * Touches the underlying spinlock only if the static key is enabled + * and the queue itself is marked as shareable. + */ +static inline void libeth_xdpsq_lock(struct libeth_xdpsq_lock *lock) +{ + if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share) + __libeth_xdpsq_lock(lock); +} + +/** + * libeth_xdpsq_unlock - free a &libeth_xdpsq_lock if needed + * @lock: lock to free + * + * Touches the underlying spinlock only if the static key is enabled + * and the queue itself is marked as shareable. + */ +static inline void libeth_xdpsq_unlock(struct libeth_xdpsq_lock *lock) +{ + if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share) + __libeth_xdpsq_unlock(lock); +} + +/* XDPSQ clean-up timers */ + +void libeth_xdpsq_init_timer(struct libeth_xdpsq_timer *timer, void *xdpsq, + struct libeth_xdpsq_lock *lock, + void (*poll)(struct work_struct *work)); + +/** + * libeth_xdpsq_deinit_timer - deinitialize a &libeth_xdpsq_timer + * @timer: timer to deinitialize + * + * Flush and disable the underlying workqueue. + */ +static inline void libeth_xdpsq_deinit_timer(struct libeth_xdpsq_timer *timer) +{ + cancel_delayed_work_sync(&timer->dwork); +} + +/** + * libeth_xdpsq_queue_timer - run a &libeth_xdpsq_timer + * @timer: timer to queue + * + * Should be called after the queue was filled and the transmission was run + * to complete the pending buffers if no further sending will be done in a + * second (-> lazy cleaning won't happen). + * If the timer was already run, it will be requeued back to one second + * timeout again. + */ +static inline void libeth_xdpsq_queue_timer(struct libeth_xdpsq_timer *timer) +{ + mod_delayed_work_on(raw_smp_processor_id(), system_bh_highpri_wq, + &timer->dwork, HZ); +} + +/** + * libeth_xdpsq_run_timer - wrapper to run a queue clean-up on a timer event + * @work: workqueue belonging to the corresponding timer + * @poll: driver-specific completion queue poll function + * + * Run the polling function on the locked queue and requeue the timer if + * there's more work to do. + * Designed to be used via LIBETH_XDP_DEFINE_TIMER() below. + */ +static __always_inline void +libeth_xdpsq_run_timer(struct work_struct *work, + u32 (*poll)(void *xdpsq, u32 budget)) +{ + struct libeth_xdpsq_timer *timer = container_of(work, typeof(*timer), + dwork.work); + + libeth_xdpsq_lock(timer->lock); + + if (poll(timer->xdpsq, U32_MAX)) + libeth_xdpsq_queue_timer(timer); + + libeth_xdpsq_unlock(timer->lock); +} + +/* Common Tx bits */ + +/** + * enum - libeth_xdp internal Tx flags + * @LIBETH_XDP_TX_BULK: one bulk size at which it will be flushed to the queue + * @LIBETH_XDP_TX_BATCH: batch size for which the queue fill loop is unrolled + * @LIBETH_XDP_TX_DROP: indicates the send function must drop frames not sent + * @LIBETH_XDP_TX_NDO: whether the send function is called from .ndo_xdp_xmit() + * @LIBETH_XDP_TX_XSK: whether the function is called for ``XDP_TX`` for XSk + */ +enum { + LIBETH_XDP_TX_BULK = DEV_MAP_BULK_SIZE, + LIBETH_XDP_TX_BATCH = 8, + + LIBETH_XDP_TX_DROP = BIT(0), + LIBETH_XDP_TX_NDO = BIT(1), + LIBETH_XDP_TX_XSK = BIT(2), +}; + +/** + * enum - &libeth_xdp_tx_frame and &libeth_xdp_tx_desc flags + * @LIBETH_XDP_TX_LEN: only for ``XDP_TX``, [15:0] of ::len_fl is actual length + * @LIBETH_XDP_TX_TSTAMP: for XSk xmit, enable tstamp offload + * @LIBETH_XDP_TX_CSUM: for XSk xmit, enable checksum offload + * @LIBETH_XDP_TX_XSKMD: for XSk xmit, mask of the metadata bits + * @LIBETH_XDP_TX_FIRST: indicates the frag is the first one of the frame + * @LIBETH_XDP_TX_LAST: whether the frag is the last one of the frame + * @LIBETH_XDP_TX_MULTI: whether the frame contains several frags + * @LIBETH_XDP_TX_FLAGS: only for ``XDP_TX``, [31:16] of ::len_fl is flags + */ +enum { + LIBETH_XDP_TX_LEN = GENMASK(15, 0), + + LIBETH_XDP_TX_TSTAMP = XDP_TXMD_FLAGS_TIMESTAMP, + LIBETH_XDP_TX_CSUM = XDP_TXMD_FLAGS_CHECKSUM, + LIBETH_XDP_TX_XSKMD = LIBETH_XDP_TX_LEN, + + LIBETH_XDP_TX_FIRST = BIT(16), + LIBETH_XDP_TX_LAST = BIT(17), + LIBETH_XDP_TX_MULTI = BIT(18), + + LIBETH_XDP_TX_FLAGS = GENMASK(31, 16), +}; + +/** + * struct libeth_xdp_tx_frame - represents one XDP Tx element + * @data: frame start pointer for ``XDP_TX`` + * @len_fl: ``XDP_TX``, combined flags [31:16] and len [15:0] field for speed + * @soff: ``XDP_TX``, offset from @data to the start of &skb_shared_info + * @frag: one (non-head) frag for ``XDP_TX`` + * @xdpf: &xdp_frame for the head frag for .ndo_xdp_xmit() + * @dma: DMA address of the non-head frag for .ndo_xdp_xmit() + * @xsk: ``XDP_TX`` for XSk, XDP buffer for any frag + * @len: frag length for XSk ``XDP_TX`` and .ndo_xdp_xmit() + * @flags: Tx flags for the above + * @opts: combined @len + @flags for the above for speed + * @desc: XSk xmit descriptor for direct casting + */ +struct libeth_xdp_tx_frame { + union { + /* ``XDP_TX`` */ + struct { + void *data; + u32 len_fl; + u32 soff; + }; + + /* ``XDP_TX`` frag */ + skb_frag_t frag; + + /* .ndo_xdp_xmit(), XSk ``XDP_TX`` */ + struct { + union { + struct xdp_frame *xdpf; + dma_addr_t dma; + + struct libeth_xdp_buff *xsk; + }; + union { + struct { + u32 len; + u32 flags; + }; + aligned_u64 opts; + }; + }; + + /* XSk xmit */ + struct xdp_desc desc; + }; +} __aligned(sizeof(struct xdp_desc)); +static_assert(offsetof(struct libeth_xdp_tx_frame, frag.len) == + offsetof(struct libeth_xdp_tx_frame, len_fl)); +static_assert(sizeof(struct libeth_xdp_tx_frame) == sizeof(struct xdp_desc)); + +/** + * struct libeth_xdp_tx_bulk - XDP Tx frame bulk for bulk sending + * @prog: corresponding active XDP program, %NULL for .ndo_xdp_xmit() + * @dev: &net_device which the frames are transmitted on + * @xdpsq: shortcut to the corresponding driver-specific XDPSQ structure + * @act_mask: Rx only, mask of all the XDP prog verdicts for that NAPI session + * @count: current number of frames in @bulk + * @bulk: array of queued frames for bulk Tx + * + * All XDP Tx operations except XSk xmit queue each frame to the bulk first + * and flush it when @count reaches the array end. Bulk is always placed on + * the stack for performance. One bulk element contains all the data necessary + * for sending a frame and then freeing it on completion. + * For XSk xmit, Tx descriptor array from &xsk_buff_pool is casted directly + * to &libeth_xdp_tx_frame as they are compatible and the bulk structure is + * not used. + */ +struct libeth_xdp_tx_bulk { + const struct bpf_prog *prog; + struct net_device *dev; + void *xdpsq; + + u32 act_mask; + u32 count; + struct libeth_xdp_tx_frame bulk[LIBETH_XDP_TX_BULK]; +} __aligned(sizeof(struct libeth_xdp_tx_frame)); + +/** + * struct libeth_xdpsq - abstraction for an XDPSQ + * @pool: XSk buffer pool for XSk ``XDP_TX`` and xmit + * @sqes: array of Tx buffers from the actual queue struct + * @descs: opaque pointer to the HW descriptor array + * @ntu: pointer to the next free descriptor index + * @count: number of descriptors on that queue + * @pending: pointer to the number of sent-not-completed descs on that queue + * @xdp_tx: pointer to the above, but only for non-XSk-xmit frames + * @lock: corresponding XDPSQ lock + * + * Abstraction for driver-independent implementation of Tx. Placed on the stack + * and filled by the driver before the transmission, so that the generic + * functions can access and modify driver-specific resources. + */ +struct libeth_xdpsq { + struct xsk_buff_pool *pool; + struct libeth_sqe *sqes; + void *descs; + + u32 *ntu; + u32 count; + + u32 *pending; + u32 *xdp_tx; + struct libeth_xdpsq_lock *lock; +}; + +/** + * struct libeth_xdp_tx_desc - abstraction for an XDP Tx descriptor + * @addr: DMA address of the frame + * @len: length of the frame + * @flags: XDP Tx flags + * @opts: combined @len + @flags for speed + * + * Filled by the generic functions and then passed to driver-specific functions + * to fill a HW Tx descriptor, always placed on the [function] stack. + */ +struct libeth_xdp_tx_desc { + dma_addr_t addr; + union { + struct { + u32 len; + u32 flags; + }; + aligned_u64 opts; + }; +} __aligned_largest; + +/** + * libeth_xdp_ptr_to_priv - convert a pointer to a libeth_xdp u64 priv + * @ptr: pointer to convert + * + * The main sending function passes private data as the largest scalar, u64. + * Use this helper when you want to pass a pointer there. + */ +#define libeth_xdp_ptr_to_priv(ptr) ({ \ + typecheck_pointer(ptr); \ + ((u64)(uintptr_t)(ptr)); \ +}) +/** + * libeth_xdp_priv_to_ptr - convert a libeth_xdp u64 priv to a pointer + * @priv: private data to convert + * + * The main sending function passes private data as the largest scalar, u64. + * Use this helper when your callback takes this u64 and you want to convert + * it back to a pointer. + */ +#define libeth_xdp_priv_to_ptr(priv) ({ \ + static_assert(__same_type(priv, u64)); \ + ((const void *)(uintptr_t)(priv)); \ +}) + +/* On 64-bit systems, assigning one u64 is faster than two u32s. When ::len + * occupies lowest 32 bits (LE), whole ::opts can be assigned directly instead. + */ +#ifdef __LITTLE_ENDIAN +#define __LIBETH_WORD_ACCESS 1 +#endif +#ifdef __LIBETH_WORD_ACCESS +#define __libeth_xdp_tx_len(flen, ...) \ + .opts = ((flen) | FIELD_PREP(GENMASK_ULL(63, 32), (__VA_ARGS__ + 0))) +#else +#define __libeth_xdp_tx_len(flen, ...) \ + .len = (flen), .flags = (__VA_ARGS__ + 0) +#endif + +/** + * libeth_xdp_tx_xmit_bulk - main XDP Tx function + * @bulk: array of frames to send + * @xdpsq: pointer to the driver-specific XDPSQ struct + * @n: number of frames to send + * @unroll: whether to unroll the queue filling loop for speed + * @priv: driver-specific private data + * @prep: callback for cleaning the queue and filling abstract &libeth_xdpsq + * @fill: internal callback for filling &libeth_sqe and &libeth_xdp_tx_desc + * @xmit: callback for filling a HW descriptor with the frame info + * + * Internal abstraction for placing @n XDP Tx frames on the HW XDPSQ. Used for + * all types of frames: ``XDP_TX``, .ndo_xdp_xmit(), XSk ``XDP_TX`` and XSk + * xmit. + * @prep must lock the queue as this function releases it at the end. @unroll + * greatly increases the object code size, but also greatly increases XSk xmit + * performance; for other types of frames, it's not enabled. + * The compilers inline all those onstack abstractions to direct data accesses. + * + * Return: number of frames actually placed on the queue, <= @n. The function + * can't fail, but can send less frames if there's no enough free descriptors + * available. The actual free space is returned by @prep from the driver. + */ +static __always_inline u32 +libeth_xdp_tx_xmit_bulk(const struct libeth_xdp_tx_frame *bulk, void *xdpsq, + u32 n, bool unroll, u64 priv, + u32 (*prep)(void *xdpsq, struct libeth_xdpsq *sq), + struct libeth_xdp_tx_desc + (*fill)(struct libeth_xdp_tx_frame frm, u32 i, + const struct libeth_xdpsq *sq, u64 priv), + void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, u64 priv)) +__releases(sq.lock) +{ + u32 this, batched, off = 0; + struct libeth_xdpsq sq; + u32 ntu, i = 0; + + n = min(n, prep(xdpsq, &sq)); + if (unlikely(!n)) + goto unlock; + + ntu = *sq.ntu; + + this = sq.count - ntu; + if (likely(this > n)) + this = n; + +again: + if (!unroll) + goto linear; + + batched = ALIGN_DOWN(this, LIBETH_XDP_TX_BATCH); + + for ( ; i < off + batched; i += LIBETH_XDP_TX_BATCH) { + u32 base = ntu + i - off; + + unrolled_count(LIBETH_XDP_TX_BATCH) + for (u32 j = 0; j < LIBETH_XDP_TX_BATCH; j++) + xmit(fill(bulk[i + j], base + j, &sq, priv), + base + j, &sq, priv); + } + + if (batched < this) { +linear: + for ( ; i < off + this; i++) + xmit(fill(bulk[i], ntu + i - off, &sq, priv), + ntu + i - off, &sq, priv); + } + + ntu += this; + if (likely(ntu < sq.count)) + goto out; + + ntu = 0; + + if (i < n) { + this = n - i; + off = i; + + goto again; + } + +out: + *sq.ntu = ntu; + *sq.pending += n; + if (sq.xdp_tx) + *sq.xdp_tx += n; + +unlock: + libeth_xdpsq_unlock(sq.lock); + + return n; +} + +/* ``XDP_TX`` bulking */ + +void libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp); + +/** + * libeth_xdp_tx_queue_head - internal helper for queueing one ``XDP_TX`` head + * @bq: XDP Tx bulk to queue the head frag to + * @xdp: XDP buffer with the head to queue + * + * Return: false if it's the only frag of the frame, true if it's an S/G frame. + */ +static inline bool libeth_xdp_tx_queue_head(struct libeth_xdp_tx_bulk *bq, + const struct libeth_xdp_buff *xdp) +{ + const struct xdp_buff *base = &xdp->base; + + bq->bulk[bq->count++] = (typeof(*bq->bulk)){ + .data = xdp->data, + .len_fl = (base->data_end - xdp->data) | LIBETH_XDP_TX_FIRST, + .soff = xdp_data_hard_end(base) - xdp->data, + }; + + if (!xdp_buff_has_frags(base)) + return false; + + bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_MULTI; + + return true; +} + +/** + * libeth_xdp_tx_queue_frag - internal helper for queueing one ``XDP_TX`` frag + * @bq: XDP Tx bulk to queue the frag to + * @frag: frag to queue + */ +static inline void libeth_xdp_tx_queue_frag(struct libeth_xdp_tx_bulk *bq, + const skb_frag_t *frag) +{ + bq->bulk[bq->count++].frag = *frag; +} + +/** + * libeth_xdp_tx_queue_bulk - internal helper for queueing one ``XDP_TX`` frame + * @bq: XDP Tx bulk to queue the frame to + * @xdp: XDP buffer to queue + * @flush_bulk: driver callback to flush the bulk to the HW queue + * + * Return: true on success, false on flush error. + */ +static __always_inline bool +libeth_xdp_tx_queue_bulk(struct libeth_xdp_tx_bulk *bq, + struct libeth_xdp_buff *xdp, + bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq, + u32 flags)) +{ + const struct skb_shared_info *sinfo; + bool ret = true; + u32 nr_frags; + + if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && + unlikely(!flush_bulk(bq, 0))) { + libeth_xdp_return_buff_slow(xdp); + return false; + } + + if (!libeth_xdp_tx_queue_head(bq, xdp)) + goto out; + + sinfo = xdp_get_shared_info_from_buff(&xdp->base); + nr_frags = sinfo->nr_frags; + + for (u32 i = 0; i < nr_frags; i++) { + if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && + unlikely(!flush_bulk(bq, 0))) { + ret = false; + break; + } + + libeth_xdp_tx_queue_frag(bq, &sinfo->frags[i]); + }; + +out: + bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_LAST; + xdp->data = NULL; + + return ret; +} + +#define __libeth_xdp_tx_fill_stats(sqe, desc, sinfo, ue, ud, us) do { \ + const struct libeth_xdp_tx_desc *ud = (desc); \ + const struct skb_shared_info *us; \ + struct libeth_sqe *ue = (sqe); \ + \ + ue->nr_frags = 1; \ + ue->bytes = ud->len; \ + \ + if (ud->flags & LIBETH_XDP_TX_MULTI) { \ + us = (sinfo); \ + ue->nr_frags += us->nr_frags; \ + ue->bytes += us->xdp_frags_size; \ + } \ +} while (0) +/** + * libeth_xdp_tx_fill_stats - fill a &libeth_sqe with ``XDP_TX`` frame stats + * @sqe: SQ element to fill + * @desc: libeth_xdp Tx descriptor + * @sinfo: &skb_shared_info for this frame + * + * Internal helper for filling an SQE with the frame stats, do not use in + * drivers. Fills the number of frags and bytes for this frame. + */ +#define libeth_xdp_tx_fill_stats(sqe, desc, sinfo) \ + __libeth_xdp_tx_fill_stats(sqe, desc, sinfo, __UNIQUE_ID(sqe_), \ + __UNIQUE_ID(desc_), __UNIQUE_ID(sinfo_)) + +/** + * libeth_xdp_tx_fill_buf - internal helper to fill one ``XDP_TX`` &libeth_sqe + * @frm: XDP Tx frame from the bulk + * @i: index on the HW queue + * @sq: XDPSQ abstraction for the queue + * @priv: private data + * + * Return: XDP Tx descriptor with the synced DMA and other info to pass to + * the driver callback. + */ +static inline struct libeth_xdp_tx_desc +libeth_xdp_tx_fill_buf(struct libeth_xdp_tx_frame frm, u32 i, + const struct libeth_xdpsq *sq, u64 priv) +{ + struct libeth_xdp_tx_desc desc; + struct skb_shared_info *sinfo; + skb_frag_t *frag = &frm.frag; + struct libeth_sqe *sqe; + + if (frm.len_fl & LIBETH_XDP_TX_FIRST) { + sinfo = frm.data + frm.soff; + skb_frag_fill_page_desc(frag, virt_to_page(frm.data), + offset_in_page(frm.data), + frm.len_fl); + } else { + sinfo = NULL; + } + + desc = (typeof(desc)){ + .addr = page_pool_get_dma_addr(skb_frag_page(frag)) + + skb_frag_off(frag), + .len = skb_frag_size(frag) & LIBETH_XDP_TX_LEN, + .flags = skb_frag_size(frag) & LIBETH_XDP_TX_FLAGS, + }; + + dma_sync_single_for_device(skb_frag_page(frag)->pp->p.dev, desc.addr, + desc.len, DMA_BIDIRECTIONAL); + + if (!sinfo) + return desc; + + sqe = &sq->sqes[i]; + sqe->type = LIBETH_SQE_XDP_TX; + sqe->sinfo = sinfo; + libeth_xdp_tx_fill_stats(sqe, &desc, sinfo); + + return desc; +} + +void libeth_xdp_tx_exception(struct libeth_xdp_tx_bulk *bq, u32 sent, + u32 flags); + +/** + * __libeth_xdp_tx_flush_bulk - internal helper to flush one XDP Tx bulk + * @bq: bulk to flush + * @flags: XDP TX flags (.ndo_xdp_xmit(), XSk etc.) + * @prep: driver-specific callback to prepare the queue for sending + * @fill: libeth_xdp callback to fill &libeth_sqe and &libeth_xdp_tx_desc + * @xmit: driver callback to fill a HW descriptor + * + * Internal abstraction to create bulk flush functions for drivers. Used for + * everything except XSk xmit. + * + * Return: true if anything was sent, false otherwise. + */ +static __always_inline bool +__libeth_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags, + u32 (*prep)(void *xdpsq, struct libeth_xdpsq *sq), + struct libeth_xdp_tx_desc + (*fill)(struct libeth_xdp_tx_frame frm, u32 i, + const struct libeth_xdpsq *sq, u64 priv), + void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, + u64 priv)) +{ + u32 sent, drops; + int err = 0; + + sent = libeth_xdp_tx_xmit_bulk(bq->bulk, bq->xdpsq, + min(bq->count, LIBETH_XDP_TX_BULK), + false, 0, prep, fill, xmit); + drops = bq->count - sent; + + if (unlikely(drops)) { + libeth_xdp_tx_exception(bq, sent, flags); + err = -ENXIO; + } else { + bq->count = 0; + } + + trace_xdp_bulk_tx(bq->dev, sent, drops, err); + + return likely(sent); +} + +/** + * libeth_xdp_tx_flush_bulk - wrapper to define flush of one ``XDP_TX`` bulk + * @bq: bulk to flush + * @flags: Tx flags, see above + * @prep: driver callback to prepare the queue + * @xmit: driver callback to fill a HW descriptor + * + * Use via LIBETH_XDP_DEFINE_FLUSH_TX() to define an ``XDP_TX`` driver + * callback. + */ +#define libeth_xdp_tx_flush_bulk(bq, flags, prep, xmit) \ + __libeth_xdp_tx_flush_bulk(bq, flags, prep, libeth_xdp_tx_fill_buf, \ + xmit) + +/* .ndo_xdp_xmit() implementation */ + +static inline void __libeth_xdp_xmit_init_bulk(struct libeth_xdp_tx_bulk *bq, + struct net_device *dev, + void *xdpsq) +{ + bq->dev = dev; + bq->xdpsq = xdpsq; + bq->count = 0; +} + +/** + * libeth_xdp_xmit_init_bulk - internal helper to initialize bulk for XDP xmit + * @bq: bulk to initialize + * @dev: target &net_device + * @xdpsqs: array of driver-specific XDPSQ structs + * @num: number of active XDPSQs (the above array length) + */ +#define libeth_xdp_xmit_init_bulk(bq, dev, xdpsqs, num) \ + __libeth_xdp_xmit_init_bulk(bq, dev, (xdpsqs)[libeth_xdpsq_id(num)]) + +static inline void *__libeth_xdp_xmit_frame_dma(const struct xdp_frame *xdpf) +{ + void *addr = (void *)(xdpf + 1); + + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && + __alignof(*xdpf) < sizeof(dma_addr_t)) + addr = PTR_ALIGN(addr, sizeof(dma_addr_t)); + + return addr; +} + +/** + * libeth_xdp_xmit_frame_dma - internal helper to access DMA of an &xdp_frame + * @xf: pointer to the XDP frame + * + * There's no place in &libeth_xdp_tx_frame to store DMA address for an + * &xdp_frame head. The headroom is used then, the address is placed right + * after the frame struct, naturally aligned. + * + * Return: pointer to the DMA address to use. + */ +#define libeth_xdp_xmit_frame_dma(xf) \ + _Generic((xf), \ + const struct xdp_frame *: \ + (const dma_addr_t *)__libeth_xdp_xmit_frame_dma(xf), \ + struct xdp_frame *: \ + (dma_addr_t *)__libeth_xdp_xmit_frame_dma(xf) \ + ) + +/** + * libeth_xdp_xmit_queue_head - internal helper for queueing one XDP xmit head + * @bq: XDP Tx bulk to queue the head frag to + * @xdpf: XDP frame with the head to queue + * @dev: device to perform DMA mapping + * + * Return: ``LIBETH_XDP_DROP`` on DMA mapping error, + * ``LIBETH_XDP_PASS`` if it's the only frag in the frame, + * ``LIBETH_XDP_TX`` if it's an S/G frame. + */ +static inline u32 libeth_xdp_xmit_queue_head(struct libeth_xdp_tx_bulk *bq, + struct xdp_frame *xdpf, + struct device *dev) +{ + dma_addr_t dma; + + dma = dma_map_single(dev, xdpf->data, xdpf->len, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma)) + return LIBETH_XDP_DROP; + + *libeth_xdp_xmit_frame_dma(xdpf) = dma; + + bq->bulk[bq->count++] = (typeof(*bq->bulk)){ + .xdpf = xdpf, + __libeth_xdp_tx_len(xdpf->len, LIBETH_XDP_TX_FIRST), + }; + + if (!xdp_frame_has_frags(xdpf)) + return LIBETH_XDP_PASS; + + bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_MULTI; + + return LIBETH_XDP_TX; +} + +/** + * libeth_xdp_xmit_queue_frag - internal helper for queueing one XDP xmit frag + * @bq: XDP Tx bulk to queue the frag to + * @frag: frag to queue + * @dev: device to perform DMA mapping + * + * Return: true on success, false on DMA mapping error. + */ +static inline bool libeth_xdp_xmit_queue_frag(struct libeth_xdp_tx_bulk *bq, + const skb_frag_t *frag, + struct device *dev) +{ + dma_addr_t dma; + + dma = skb_frag_dma_map(dev, frag); + if (dma_mapping_error(dev, dma)) + return false; + + bq->bulk[bq->count++] = (typeof(*bq->bulk)){ + .dma = dma, + __libeth_xdp_tx_len(skb_frag_size(frag)), + }; + + return true; +} + +/** + * libeth_xdp_xmit_queue_bulk - internal helper for queueing one XDP xmit frame + * @bq: XDP Tx bulk to queue the frame to + * @xdpf: XDP frame to queue + * @flush_bulk: driver callback to flush the bulk to the HW queue + * + * Return: ``LIBETH_XDP_TX`` on success, + * ``LIBETH_XDP_DROP`` if the frame should be dropped by the stack, + * ``LIBETH_XDP_ABORTED`` if the frame will be dropped by libeth_xdp. + */ +static __always_inline u32 +libeth_xdp_xmit_queue_bulk(struct libeth_xdp_tx_bulk *bq, + struct xdp_frame *xdpf, + bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq, + u32 flags)) +{ + u32 head, nr_frags, i, ret = LIBETH_XDP_TX; + struct device *dev = bq->dev->dev.parent; + const struct skb_shared_info *sinfo; + + if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && + unlikely(!flush_bulk(bq, LIBETH_XDP_TX_NDO))) + return LIBETH_XDP_DROP; + + head = libeth_xdp_xmit_queue_head(bq, xdpf, dev); + if (head == LIBETH_XDP_PASS) + goto out; + else if (head == LIBETH_XDP_DROP) + return LIBETH_XDP_DROP; + + sinfo = xdp_get_shared_info_from_frame(xdpf); + nr_frags = sinfo->nr_frags; + + for (i = 0; i < nr_frags; i++) { + if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && + unlikely(!flush_bulk(bq, LIBETH_XDP_TX_NDO))) + break; + + if (!libeth_xdp_xmit_queue_frag(bq, &sinfo->frags[i], dev)) + break; + }; + + if (unlikely(i < nr_frags)) + ret = LIBETH_XDP_ABORTED; + +out: + bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_LAST; + + return ret; +} + +/** + * libeth_xdp_xmit_fill_buf - internal helper to fill one XDP xmit &libeth_sqe + * @frm: XDP Tx frame from the bulk + * @i: index on the HW queue + * @sq: XDPSQ abstraction for the queue + * @priv: private data + * + * Return: XDP Tx descriptor with the mapped DMA and other info to pass to + * the driver callback. + */ +static inline struct libeth_xdp_tx_desc +libeth_xdp_xmit_fill_buf(struct libeth_xdp_tx_frame frm, u32 i, + const struct libeth_xdpsq *sq, u64 priv) +{ + struct libeth_xdp_tx_desc desc; + struct libeth_sqe *sqe; + struct xdp_frame *xdpf; + + if (frm.flags & LIBETH_XDP_TX_FIRST) { + xdpf = frm.xdpf; + desc.addr = *libeth_xdp_xmit_frame_dma(xdpf); + } else { + xdpf = NULL; + desc.addr = frm.dma; + } + desc.opts = frm.opts; + + sqe = &sq->sqes[i]; + dma_unmap_addr_set(sqe, dma, desc.addr); + dma_unmap_len_set(sqe, len, desc.len); + + if (!xdpf) { + sqe->type = LIBETH_SQE_XDP_XMIT_FRAG; + return desc; + } + + sqe->type = LIBETH_SQE_XDP_XMIT; + sqe->xdpf = xdpf; + libeth_xdp_tx_fill_stats(sqe, &desc, + xdp_get_shared_info_from_frame(xdpf)); + + return desc; +} + +/** + * libeth_xdp_xmit_flush_bulk - wrapper to define flush of one XDP xmit bulk + * @bq: bulk to flush + * @flags: Tx flags, see __libeth_xdp_tx_flush_bulk() + * @prep: driver callback to prepare the queue + * @xmit: driver callback to fill a HW descriptor + * + * Use via LIBETH_XDP_DEFINE_FLUSH_XMIT() to define an XDP xmit driver + * callback. + */ +#define libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit) \ + __libeth_xdp_tx_flush_bulk(bq, (flags) | LIBETH_XDP_TX_NDO, prep, \ + libeth_xdp_xmit_fill_buf, xmit); + +u32 libeth_xdp_xmit_return_bulk(const struct libeth_xdp_tx_frame *bq, + u32 count, const struct net_device *dev); + +/** + * __libeth_xdp_xmit_do_bulk - internal function to implement .ndo_xdp_xmit() + * @bq: XDP Tx bulk to queue frames to + * @frames: XDP frames passed by the stack + * @n: number of frames + * @flags: flags passed by the stack + * @flush_bulk: driver callback to flush an XDP xmit bulk + * @finalize: driver callback to finalize sending XDP Tx frames on the queue + * + * Please use the non-underscored version in drivers. + * Perform common checks, map the frags and queue them to the bulk, then flush + * the bulk to the XDPSQ. If requested by the stack, finalize the queue. + * + * Return: number of frames send or -errno on error. + */ +static __always_inline int +__libeth_xdp_xmit_do_bulk(struct libeth_xdp_tx_bulk *bq, + struct xdp_frame **frames, u32 n, u32 flags, + bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq, + u32 flags), + void (*finalize)(void *xdpsq, bool sent, bool flush)) +{ + u32 nxmit = 0; + + if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) + return -EINVAL; + + for (u32 i = 0; likely(i < n); i++) { + u32 ret; + + ret = libeth_xdp_xmit_queue_bulk(bq, frames[i], flush_bulk); + if (unlikely(ret != LIBETH_XDP_TX)) { + nxmit += ret == LIBETH_XDP_ABORTED; + break; + } else { + nxmit++; + } + } + + if (bq->count) { + flush_bulk(bq, LIBETH_XDP_TX_NDO); + if (unlikely(bq->count)) + nxmit -= libeth_xdp_xmit_return_bulk(bq->bulk, + bq->count, + bq->dev); + } + + finalize(bq->xdpsq, nxmit, flags & XDP_XMIT_FLUSH); + + return nxmit; +} + +#define _libeth_xdp_xmit_do_bulk(d, n, fr, f, xqs, nqs, fl, fin, ub, ur, un) \ +({ \ + u32 un = (nqs); \ + int ur; \ + \ + if (likely(un)) { \ + struct libeth_xdp_tx_bulk ub; \ + \ + libeth_xdp_xmit_init_bulk(&ub, d, xqs, un); \ + ur = __libeth_xdp_xmit_do_bulk(&ub, fr, n, f, fl, fin); \ + } else { \ + ur = -ENXIO; \ + } \ + \ + ur; \ +}) +/** + * libeth_xdp_xmit_do_bulk - implement full .ndo_xdp_xmit() in driver + * @dev: target &net_device + * @n: number of frames to send + * @fr: XDP frames to send + * @f: flags passed by the stack + * @xqs: array of XDPSQs driver structs + * @nqs: number of active XDPSQs, the above array length + * @fl: driver callback to flush an XDP xmit bulk + * @fin: driver cabback to finalize the queue + * + * If the driver has active XDPSQs, perform common checks and send the frames. + * Finalize the queue, if requested. + * + * Return: number of frames sent or -errno on error. + */ +#define libeth_xdp_xmit_do_bulk(dev, n, fr, f, xqs, nqs, fl, fin) \ + _libeth_xdp_xmit_do_bulk(dev, n, fr, f, xqs, nqs, fl, fin, \ + __UNIQUE_ID(bq_), __UNIQUE_ID(ret_), \ + __UNIQUE_ID(nqs_)) + +/* Rx polling path */ + +#define __libeth_xdp_tx_init_bulk(bq, pr, d, xdpsqs, num, xsk, ub, un) do { \ + typeof(bq) ub = (bq); \ + u32 un = (num); \ + \ + if (un || (xsk)) { \ + ub->prog = rcu_dereference(pr); \ + ub->dev = (d); \ + ub->xdpsq = (xdpsqs)[libeth_xdpsq_id(un)]; \ + } else { \ + ub->prog = NULL; \ + } \ + \ + ub->act_mask = 0; \ + ub->count = 0; \ +} while (0) +/** + * libeth_xdp_tx_init_bulk - initialize an XDP Tx bulk for Rx NAPI poll + * @bq: bulk to initialize + * @prog: RCU pointer to the XDP program (can be %NULL) + * @dev: target &net_device + * @xdpsqs: array of driver XDPSQ structs + * @num: number of active XDPSQs, the above array length + * + * Should be called on an onstack XDP Tx bulk before the NAPI polling loop. + * Initializes all the needed fields to run libeth_xdp functions. If @num == 0, + * assumes XDP is not enabled. + * Do not use for XSk, it has its own optimized helper. + */ +#define libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num) \ + __libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, false, \ + __UNIQUE_ID(bq_), __UNIQUE_ID(nqs_)) + +void libeth_xdp_load_stash(struct libeth_xdp_buff *dst, + const struct libeth_xdp_buff_stash *src); +void libeth_xdp_save_stash(struct libeth_xdp_buff_stash *dst, + const struct libeth_xdp_buff *src); +void __libeth_xdp_return_stash(struct libeth_xdp_buff_stash *stash); + +/** + * libeth_xdp_init_buff - initialize a &libeth_xdp_buff for Rx NAPI poll + * @dst: onstack buffer to initialize + * @src: XDP buffer stash placed on the queue + * @rxq: registered &xdp_rxq_info corresponding to this queue + * + * Should be called before the main NAPI polling loop. Loads the content of + * the previously saved stash or initalizes the buffer from scratch. + * Do not use for XSk. + */ +static inline void +libeth_xdp_init_buff(struct libeth_xdp_buff *dst, + const struct libeth_xdp_buff_stash *src, + struct xdp_rxq_info *rxq) +{ + if (likely(!src->data)) + dst->data = NULL; + else + libeth_xdp_load_stash(dst, src); + + dst->base.rxq = rxq; +} + +/** + * libeth_xdp_save_buff - save a partially built buffer on a queue + * @dst: XDP buffer stash placed on the queue + * @src: onstack buffer to save + * + * Should be called after the main NAPI polling loop. If the loop exited before + * the buffer was finished, saves its content on the queue, so that it can be + * completed during the next poll. Otherwise, clears the stash. + */ +static inline void libeth_xdp_save_buff(struct libeth_xdp_buff_stash *dst, + const struct libeth_xdp_buff *src) +{ + if (likely(!src->data)) + dst->data = NULL; + else + libeth_xdp_save_stash(dst, src); +} + +/** + * libeth_xdp_return_stash - free an XDP buffer stash from a queue + * @stash: stash to free + * + * If the queue is about to be destroyed, but it still has an incompleted + * buffer stash, this helper should be called to free it. + */ +static inline void libeth_xdp_return_stash(struct libeth_xdp_buff_stash *stash) +{ + if (stash->data) + __libeth_xdp_return_stash(stash); +} + +static inline void libeth_xdp_return_va(const void *data, bool napi) +{ + struct page *page = virt_to_page(data); + + page_pool_put_full_page(page->pp, page, napi); +} + +static inline void libeth_xdp_return_frags(const struct skb_shared_info *sinfo, + bool napi) +{ + for (u32 i = 0; i < sinfo->nr_frags; i++) { + struct page *page = skb_frag_page(&sinfo->frags[i]); + + page_pool_put_full_page(page->pp, page, napi); + } +} + +/** + * libeth_xdp_return_buff - free/recycle a &libeth_xdp_buff + * @xdp: buffer to free + * + * Hotpath helper to free a &libeth_xdp_buff. Comparing to xdp_return_buff(), + * it's faster as it gets inlined and always assumes order-0 pages and safe + * direct recycling. Zeroes @xdp->data to avoid UAFs. + */ +static inline void libeth_xdp_return_buff(struct libeth_xdp_buff *xdp) +{ + if (!xdp_buff_has_frags(&xdp->base)) + goto out; + + libeth_xdp_return_frags(xdp_get_shared_info_from_buff(&xdp->base), + true); + +out: + libeth_xdp_return_va(xdp->data, true); + xdp->data = NULL; +} + +bool libeth_xdp_buff_add_frag(struct libeth_xdp_buff *xdp, + const struct libeth_fqe *fqe, + u32 len); + +/** + * libeth_xdp_prepare_buff - fill a &libeth_xdp_buff with a head FQE data + * @xdp: XDP buffer to attach the head to + * @fqe: FQE containing the head buffer + * @len: buffer len passed from HW + * + * Internal, use libeth_xdp_process_buff() instead. Initializes XDP buffer + * head with the Rx buffer data: data pointer, length, headroom, and + * truesize/tailroom. Zeroes the flags. + * Uses faster single u64 write instead of per-field access. + */ +static inline void libeth_xdp_prepare_buff(struct libeth_xdp_buff *xdp, + const struct libeth_fqe *fqe, + u32 len) +{ + const struct page *page = fqe->page; + +#ifdef __LIBETH_WORD_ACCESS + static_assert(offsetofend(typeof(xdp->base), flags) - + offsetof(typeof(xdp->base), frame_sz) == + sizeof(u64)); + + *(u64 *)&xdp->base.frame_sz = fqe->truesize; +#else + xdp_init_buff(&xdp->base, fqe->truesize, xdp->base.rxq); +#endif + xdp_prepare_buff(&xdp->base, page_address(page) + fqe->offset, + page->pp->p.offset, len, true); +} + +/** + * libeth_xdp_process_buff - attach an Rx buffer to a &libeth_xdp_buff + * @xdp: XDP buffer to attach the Rx buffer to + * @fqe: Rx buffer to process + * @len: received data length from the descriptor + * + * If the XDP buffer is empty, attaches the Rx buffer as head and initializes + * the required fields. Otherwise, attaches the buffer as a frag. + * Already performs DMA sync-for-CPU and frame start prefetch + * (for head buffers only). + * + * Return: true on succes, false if the descriptor must be skipped (empty or + * no space for a new frag). + */ +static inline bool libeth_xdp_process_buff(struct libeth_xdp_buff *xdp, + const struct libeth_fqe *fqe, + u32 len) +{ + if (!libeth_rx_sync_for_cpu(fqe, len)) + return false; + + if (xdp->data) + return libeth_xdp_buff_add_frag(xdp, fqe, len); + + libeth_xdp_prepare_buff(xdp, fqe, len); + + prefetch(xdp->data); + + return true; +} + +/** + * libeth_xdp_buff_stats_frags - update onstack RQ stats with XDP frags info + * @ss: onstack stats to update + * @xdp: buffer to account + * + * Internal helper used by __libeth_xdp_run_pass(), do not call directly. + * Adds buffer's frags count and total len to the onstack stats. + */ +static inline void +libeth_xdp_buff_stats_frags(struct libeth_rq_napi_stats *ss, + const struct libeth_xdp_buff *xdp) +{ + const struct skb_shared_info *sinfo; + + sinfo = xdp_get_shared_info_from_buff(&xdp->base); + ss->bytes += sinfo->xdp_frags_size; + ss->fragments += sinfo->nr_frags + 1; +} + +u32 libeth_xdp_prog_exception(const struct libeth_xdp_tx_bulk *bq, + struct libeth_xdp_buff *xdp, + enum xdp_action act, int ret); + +/** + * __libeth_xdp_run_prog - run XDP program on an XDP buffer + * @xdp: XDP buffer to run the prog on + * @bq: buffer bulk for ``XDP_TX`` queueing + * + * Internal inline abstraction to run XDP program. Handles ``XDP_DROP`` + * and ``XDP_REDIRECT`` only, the rest is processed levels up. + * Reports an XDP prog exception on errors. + * + * Return: libeth_xdp prog verdict depending on the prog's verdict. + */ +static __always_inline u32 +__libeth_xdp_run_prog(struct libeth_xdp_buff *xdp, + const struct libeth_xdp_tx_bulk *bq) +{ + enum xdp_action act; + + act = bpf_prog_run_xdp(bq->prog, &xdp->base); + if (unlikely(act < XDP_DROP || act > XDP_REDIRECT)) + goto out; + + switch (act) { + case XDP_PASS: + return LIBETH_XDP_PASS; + case XDP_DROP: + libeth_xdp_return_buff(xdp); + + return LIBETH_XDP_DROP; + case XDP_TX: + return LIBETH_XDP_TX; + case XDP_REDIRECT: + if (unlikely(xdp_do_redirect(bq->dev, &xdp->base, bq->prog))) + break; + + xdp->data = NULL; + + return LIBETH_XDP_REDIRECT; + default: + break; + } + +out: + return libeth_xdp_prog_exception(bq, xdp, act, 0); +} + +/** + * __libeth_xdp_run_flush - run XDP program and handle ``XDP_TX`` verdict + * @xdp: XDP buffer to run the prog on + * @bq: buffer bulk for ``XDP_TX`` queueing + * @run: internal callback for running XDP program + * @queue: internal callback for queuing ``XDP_TX`` frame + * @flush_bulk: driver callback for flushing a bulk + * + * Internal inline abstraction to run XDP program and additionally handle + * ``XDP_TX`` verdict. Used by both XDP and XSk, hence @run and @queue. + * Do not use directly. + * + * Return: libeth_xdp prog verdict depending on the prog's verdict. + */ +static __always_inline u32 +__libeth_xdp_run_flush(struct libeth_xdp_buff *xdp, + struct libeth_xdp_tx_bulk *bq, + u32 (*run)(struct libeth_xdp_buff *xdp, + const struct libeth_xdp_tx_bulk *bq), + bool (*queue)(struct libeth_xdp_tx_bulk *bq, + struct libeth_xdp_buff *xdp, + bool (*flush_bulk) + (struct libeth_xdp_tx_bulk *bq, + u32 flags)), + bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq, + u32 flags)) +{ + u32 act; + + act = run(xdp, bq); + if (act == LIBETH_XDP_TX && unlikely(!queue(bq, xdp, flush_bulk))) + act = LIBETH_XDP_DROP; + + bq->act_mask |= act; + + return act; +} + +/** + * libeth_xdp_run_prog - run XDP program (non-XSk path) and handle all verdicts + * @xdp: XDP buffer to process + * @bq: XDP Tx bulk to queue ``XDP_TX`` buffers + * @fl: driver ``XDP_TX`` bulk flush callback + * + * Run the attached XDP program and handle all possible verdicts. XSk has its + * own version. + * Prefer using it via LIBETH_XDP_DEFINE_RUN{,_PASS,_PROG}(). + * + * Return: true if the buffer should be passed up the stack, false if the poll + * should go to the next buffer. + */ +#define libeth_xdp_run_prog(xdp, bq, fl) \ + (__libeth_xdp_run_flush(xdp, bq, __libeth_xdp_run_prog, \ + libeth_xdp_tx_queue_bulk, \ + fl) == LIBETH_XDP_PASS) + +/** + * __libeth_xdp_run_pass - helper to run XDP program and handle the result + * @xdp: XDP buffer to process + * @bq: XDP Tx bulk to queue ``XDP_TX`` frames + * @napi: NAPI to build an skb and pass it up the stack + * @rs: onstack libeth RQ stats + * @md: metadata that should be filled to the XDP buffer + * @prep: callback for filling the metadata + * @run: driver wrapper to run XDP program + * @populate: driver callback to populate an skb with the HW descriptor data + * + * Inline abstraction that does the following (non-XSk path): + * 1) adds frame size and frag number (if needed) to the onstack stats; + * 2) fills the descriptor metadata to the onstack &libeth_xdp_buff + * 3) runs XDP program if present; + * 4) handles all possible verdicts; + * 5) on ``XDP_PASS`, builds an skb from the buffer; + * 6) populates it with the descriptor metadata; + * 7) passes it up the stack. + * + * In most cases, number 2 means just writing the pointer to the HW descriptor + * to the XDP buffer. If so, please use LIBETH_XDP_DEFINE_RUN{,_PASS}() + * wrappers to build a driver function. + */ +static __always_inline void +__libeth_xdp_run_pass(struct libeth_xdp_buff *xdp, + struct libeth_xdp_tx_bulk *bq, struct napi_struct *napi, + struct libeth_rq_napi_stats *rs, const void *md, + void (*prep)(struct libeth_xdp_buff *xdp, + const void *md), + bool (*run)(struct libeth_xdp_buff *xdp, + struct libeth_xdp_tx_bulk *bq), + bool (*populate)(struct sk_buff *skb, + const struct libeth_xdp_buff *xdp, + struct libeth_rq_napi_stats *rs)) +{ + struct sk_buff *skb; + + rs->bytes += xdp->base.data_end - xdp->data; + rs->packets++; + + if (xdp_buff_has_frags(&xdp->base)) + libeth_xdp_buff_stats_frags(rs, xdp); + + if (prep && (!__builtin_constant_p(!!md) || md)) + prep(xdp, md); + + if (!bq || !run || !bq->prog) + goto build; + + if (!run(xdp, bq)) + return; + +build: + skb = xdp_build_skb_from_buff(&xdp->base); + if (unlikely(!skb)) { + libeth_xdp_return_buff_slow(xdp); + return; + } + + xdp->data = NULL; + + if (unlikely(!populate(skb, xdp, rs))) { + napi_consume_skb(skb, true); + return; + } + + napi_gro_receive(napi, skb); +} + +static inline void libeth_xdp_prep_desc(struct libeth_xdp_buff *xdp, + const void *desc) +{ + xdp->desc = desc; +} + +/** + * libeth_xdp_run_pass - helper to run XDP program and handle the result + * @xdp: XDP buffer to process + * @bq: XDP Tx bulk to queue ``XDP_TX`` frames + * @napi: NAPI to build an skb and pass it up the stack + * @ss: onstack libeth RQ stats + * @desc: pointer to the HW descriptor for that frame + * @run: driver wrapper to run XDP program + * @populate: driver callback to populate an skb with the HW descriptor data + * + * Wrapper around the underscored version when "fill the descriptor metadata" + * means just writing the pointer to the HW descriptor as @xdp->desc. + */ +#define libeth_xdp_run_pass(xdp, bq, napi, ss, desc, run, populate) \ + __libeth_xdp_run_pass(xdp, bq, napi, ss, desc, libeth_xdp_prep_desc, \ + run, populate) + +static __always_inline void +__libeth_xdp_finalize_rx(struct libeth_xdp_tx_bulk *bq, u32 flags, + bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq, + u32 flags), + void (*finalize)(void *xdpsq, bool sent, bool flush)) +{ + if (bq->act_mask & LIBETH_XDP_TX) { + if (bq->count) + flush_bulk(bq, flags | LIBETH_XDP_TX_DROP); + finalize(bq->xdpsq, true, true); + } + if (bq->act_mask & LIBETH_XDP_REDIRECT) + xdp_do_flush(); +} + +/** + * libeth_xdp_finalize_rx - finalize XDPSQ after a NAPI polling loop (non-XSk) + * @bq: ``XDP_TX`` frame bulk + * @flush: driver callback to flush the bulk + * @finalize: driver callback to start sending the frames and run the timer + * + * Flush the bulk if there are frames left to send, kick the queue and flush + * the XDP maps. + */ +#define libeth_xdp_finalize_rx(bq, flush, finalize) \ + __libeth_xdp_finalize_rx(bq, 0, flush, finalize) + +/* Helpers to reduce boilerplate code in drivers. + * + * Typical driver Rx flow would be (excl. bulk and buff init, frag attach): + * LIBETH_XDP_DEFINE_START(); + * LIBETH_XDP_DEFINE_FLUSH_TX(static driver_xdp_flush_tx, driver_xdp_tx_prep, + * driver_xdp_xmit); + * LIBETH_XDP_DEFINE_RUN(static driver_xdp_run, driver_xdp_run_prog, + * driver_xdp_flush_tx, driver_populate_skb); + * LIBETH_XDP_DEFINE_FINALIZE(static driver_xdp_finalize_rx, + * driver_xdp_flush_tx, driver_xdp_finalize_sq); + * LIBETH_XDP_DEFINE_END(); + * This will build a set of 4 static functions. The compiler is free to decide + * whether to inline them. + * Then, in the NAPI polling function: + * while (packets < budget) { + * // ... + * driver_xdp_run(xdp, &bq, napi, &rs, desc); + * } + * driver_xdp_finalize_rx(&bq); + */ + +#define LIBETH_XDP_DEFINE_START() \ + __diag_push(); \ + __diag_ignore(GCC, 8, "-Wold-style-declaration", \ + "Allow specifying \'static\' after the return type") + +/** + * LIBETH_XDP_DEFINE_TIMER - define a driver XDPSQ cleanup timer callback + * @name: name of the function to define + * @poll: Tx polling/completion function + */ +#define LIBETH_XDP_DEFINE_TIMER(name, poll) \ +void name(struct work_struct *work) \ +{ \ + libeth_xdpsq_run_timer(work, poll); \ +} + +#define __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, pfx) \ +bool name(struct libeth_xdp_tx_bulk *bq, u32 flags) \ +{ \ + return libeth_##pfx##_tx_flush_bulk(bq, flags, prep, xmit); \ +} +/** + * LIBETH_XDP_DEFINE_FLUSH_TX - define a driver ``XDP_TX`` bulk flush function + * @name: name of the function to define + * @prep: driver callback to clean an XDPSQ + * @xmit: driver callback to write a HW Tx descriptor + */ +#define LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit) \ + __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, xdp) + +/** + * LIBETH_XDP_DEFINE_FLUSH_XMIT - define a driver XDP xmit bulk flush function + * @name: name of the function to define + * @prep: driver callback to clean an XDPSQ + * @xmit: driver callback to write a HW Tx descriptor + */ +#define LIBETH_XDP_DEFINE_FLUSH_XMIT(name, prep, xmit) \ +bool name(struct libeth_xdp_tx_bulk *bq, u32 flags) \ +{ \ + return libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit); \ +} + +#define __LIBETH_XDP_DEFINE_RUN_PROG(name, flush, pfx) \ +name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq) \ +{ \ + return libeth_##pfx##_run_prog(xdp, bq, flush); \ +} +/** + * LIBETH_XDP_DEFINE_RUN_PROG - define a driver XDP program run function + * @name: name of the function to define + * @flush: driver callback to flush an ``XDP_TX`` bulk + */ +#define LIBETH_XDP_DEFINE_RUN_PROG(name, flush) \ + bool __LIBETH_XDP_DEFINE_RUN_PROG(name, flush, xdp) + +#define __LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate, pfx) \ +name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq, \ + struct napi_struct *napi, struct libeth_rq_napi_stats *ss, \ + const void *desc) \ +{ \ + return libeth_##pfx##_run_pass(xdp, bq, napi, ss, desc, run, \ + populate); \ +} +/** + * LIBETH_XDP_DEFINE_RUN_PASS - define a driver buffer process + pass function + * @name: name of the function to define + * @run: driver callback to run XDP program (above) + * @populate: driver callback to fill an skb with HW descriptor info + */ +#define LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate) \ + void __LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate, xdp) + +#define __LIBETH_XDP_DEFINE_RUN(name, run, flush, populate, pfx) \ + LIBETH_##pfx##_DEFINE_RUN_PROG(static run, flush); \ + LIBETH_##pfx##_DEFINE_RUN_PASS(name, run, populate) +/** + * LIBETH_XDP_DEFINE_RUN - define a driver buffer process, run + pass function + * @name: name of the function to define + * @run: name of the XDP prog run function to define + * @flush: driver callback to flush an ``XDP_TX`` bulk + * @populate: driver callback to fill an skb with HW descriptor info + */ +#define LIBETH_XDP_DEFINE_RUN(name, run, flush, populate) \ + __LIBETH_XDP_DEFINE_RUN(name, run, flush, populate, XDP) + +#define __LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize, pfx) \ +void name(struct libeth_xdp_tx_bulk *bq) \ +{ \ + libeth_##pfx##_finalize_rx(bq, flush, finalize); \ +} +/** + * LIBETH_XDP_DEFINE_FINALIZE - define a driver Rx NAPI poll finalize function + * @name: name of the function to define + * @flush: driver callback to flush an ``XDP_TX`` bulk + * @finalize: driver callback to finalize an XDPSQ and run the timer + */ +#define LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize) \ + __LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize, xdp) + +#define LIBETH_XDP_DEFINE_END() __diag_pop() + +/* XMO */ + +/** + * libeth_xdp_buff_to_rq - get RQ pointer from an XDP buffer pointer + * @xdp: &libeth_xdp_buff corresponding to the queue + * @type: typeof() of the driver Rx queue structure + * @member: name of &xdp_rxq_info inside @type + * + * Often times, pointer to the RQ is needed when reading/filling metadata from + * HW descriptors. The helper can be used to quickly jump from an XDP buffer + * to the queue corresponding to its &xdp_rxq_info without introducing + * additional fields (&libeth_xdp_buff is precisely 1 cacheline long on x64). + */ +#define libeth_xdp_buff_to_rq(xdp, type, member) \ + container_of_const((xdp)->base.rxq, type, member) + +/** + * libeth_xdpmo_rx_hash - convert &libeth_rx_pt to an XDP RSS hash metadata + * @hash: pointer to the variable to write the hash to + * @rss_type: pointer to the variable to write the hash type to + * @val: hash value from the HW descriptor + * @pt: libeth parsed packet type + * + * Handle zeroed/non-available hash and convert libeth parsed packet type to + * the corresponding XDP RSS hash type. To be called at the end of + * xdp_metadata_ops idpf_xdpmo::xmo_rx_hash() implementation. + * Note that if the driver doesn't use a constant packet type lookup table but + * generates it at runtime, it must call libeth_rx_pt_gen_hash_type(pt) to + * generate XDP RSS hash type for each packet type. + * + * Return: 0 on success, -ENODATA when the hash is not available. + */ +static inline int libeth_xdpmo_rx_hash(u32 *hash, + enum xdp_rss_hash_type *rss_type, + u32 val, struct libeth_rx_pt pt) +{ + if (unlikely(!val)) + return -ENODATA; + + *hash = val; + *rss_type = pt.hash_type; + + return 0; +} + +/* Tx buffer completion */ + +void libeth_xdp_return_buff_bulk(const struct skb_shared_info *sinfo, + struct xdp_frame_bulk *bq, bool frags); +void libeth_xsk_buff_free_slow(struct libeth_xdp_buff *xdp); + +/** + * __libeth_xdp_complete_tx - complete a sent XDPSQE + * @sqe: SQ element / Tx buffer to complete + * @cp: Tx polling/completion params + * @bulk: internal callback to bulk-free ``XDP_TX`` buffers + * @xsk: internal callback to free XSk ``XDP_TX`` buffers + * + * Use the non-underscored version in drivers instead. This one is shared + * internally with libeth_tx_complete_any(). + * Complete an XDPSQE of any type of XDP frame. This includes DMA unmapping + * when needed, buffer freeing, stats update, and SQE invalidating. + */ +static __always_inline void +__libeth_xdp_complete_tx(struct libeth_sqe *sqe, struct libeth_cq_pp *cp, + typeof(libeth_xdp_return_buff_bulk) bulk, + typeof(libeth_xsk_buff_free_slow) xsk) +{ + enum libeth_sqe_type type = sqe->type; + + switch (type) { + case LIBETH_SQE_EMPTY: + return; + case LIBETH_SQE_XDP_XMIT: + case LIBETH_SQE_XDP_XMIT_FRAG: + dma_unmap_page(cp->dev, dma_unmap_addr(sqe, dma), + dma_unmap_len(sqe, len), DMA_TO_DEVICE); + break; + default: + break; + } + + switch (type) { + case LIBETH_SQE_XDP_TX: + bulk(sqe->sinfo, cp->bq, sqe->nr_frags != 1); + break; + case LIBETH_SQE_XDP_XMIT: + xdp_return_frame_bulk(sqe->xdpf, cp->bq); + break; + case LIBETH_SQE_XSK_TX: + case LIBETH_SQE_XSK_TX_FRAG: + xsk(sqe->xsk); + break; + default: + break; + } + + switch (type) { + case LIBETH_SQE_XDP_TX: + case LIBETH_SQE_XDP_XMIT: + case LIBETH_SQE_XSK_TX: + cp->xdp_tx -= sqe->nr_frags; + + cp->xss->packets++; + cp->xss->bytes += sqe->bytes; + break; + default: + break; + } + + sqe->type = LIBETH_SQE_EMPTY; +} + +static inline void libeth_xdp_complete_tx(struct libeth_sqe *sqe, + struct libeth_cq_pp *cp) +{ + __libeth_xdp_complete_tx(sqe, cp, libeth_xdp_return_buff_bulk, + libeth_xsk_buff_free_slow); +} + +/* Misc */ + +u32 libeth_xdp_queue_threshold(u32 count); +void __libeth_xdp_set_features(struct net_device *dev, + const struct xdp_metadata_ops *xmo, + u32 zc_segs, + const struct xsk_tx_metadata_ops *tmo); + +/** + * libeth_xdp_set_redirect - toggle the XDP redirect feature + * @dev: &net_device to configure + * @enable: whether XDP is enabled + * + * Use this when XDPSQs are not always available to dynamically enable and + * disable redirect feature. + */ +static inline void libeth_xdp_set_redirect(struct net_device *dev, bool enable) +{ + if (enable) + xdp_features_set_redirect_target(dev, true); + else + xdp_features_clear_redirect_target(dev); +} + +#define __libeth_xdp_feat0(dev) \ + __libeth_xdp_set_features(dev, NULL, 0, NULL) +#define __libeth_xdp_feat1(dev, xmo) \ + __libeth_xdp_set_features(dev, xmo, 0, NULL) +#define __libeth_xdp_feat2(dev, xmo, zc_segs) \ + __libeth_xdp_set_features(dev, xmo, zc_segs, NULL) +#define __libeth_xdp_feat3(dev, xmo, zc_segs, tmo) \ + __libeth_xdp_set_features(dev, xmo, zc_segs, tmo) + +#define libeth_xdp_set_features(dev, ...) \ + CONCATENATE(__libeth_xdp_feat, \ + COUNT_ARGS(__VA_ARGS__))(dev, ##__VA_ARGS__) +/** + * libeth_xdp_set_features_noredir - enable all libeth_xdp features w/o redir + * @dev: target &net_device + * @...: optional params, see __libeth_xdp_set_features() + * + * Enable everything except the XDP redirect feature, use when XDPSQs are not + * available right after netdev registration. + */ +#define libeth_xdp_set_features_noredir(dev, ...) do { \ + libeth_xdp_set_features(dev, ##__VA_ARGS__); \ + libeth_xdp_set_redirect(dev, false); \ +} while (0) + +#define libeth_xsktmo ((const void *)true) + +#endif /* __LIBETH_XDP_H */ diff --git a/include/net/libeth/xsk.h b/include/net/libeth/xsk.h new file mode 100644 index 00000000000000..0a2f7502ec2693 --- /dev/null +++ b/include/net/libeth/xsk.h @@ -0,0 +1,680 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2024 Intel Corporation */ + +#ifndef __LIBETH_XSK_H +#define __LIBETH_XSK_H + +#include +#include + +/* ``XDP_TXMD_FLAGS_VALID`` is defined only under ``CONFIG_XDP_SOCKETS`` */ +#ifdef XDP_TXMD_FLAGS_VALID +static_assert(XDP_TXMD_FLAGS_VALID <= LIBETH_XDP_TX_XSKMD); +#endif + +/* ``XDP_TX`` bulking */ + +/** + * libeth_xsk_tx_queue_head - internal helper for queueing XSk ``XDP_TX`` head + * @bq: XDP Tx bulk to queue the head frag to + * @xdp: XSk buffer with the head to queue + * + * Return: false if it's the only frag of the frame, true if it's an S/G frame. + */ +static inline bool libeth_xsk_tx_queue_head(struct libeth_xdp_tx_bulk *bq, + struct libeth_xdp_buff *xdp) +{ + bq->bulk[bq->count++] = (typeof(*bq->bulk)){ + .xsk = xdp, + __libeth_xdp_tx_len(xdp->base.data_end - xdp->data, + LIBETH_XDP_TX_FIRST), + }; + + if (likely(!xdp_buff_has_frags(&xdp->base))) + return false; + + bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_MULTI; + + return true; +} + +/** + * libeth_xsk_tx_queue_frag - internal helper for queueing XSk ``XDP_TX`` frag + * @bq: XDP Tx bulk to queue the frag to + * @frag: XSk frag to queue + */ +static inline void libeth_xsk_tx_queue_frag(struct libeth_xdp_tx_bulk *bq, + struct libeth_xdp_buff *frag) +{ + bq->bulk[bq->count++] = (typeof(*bq->bulk)){ + .xsk = frag, + __libeth_xdp_tx_len(frag->base.data_end - frag->data), + }; +} + +/** + * libeth_xsk_tx_queue_bulk - internal helper for queueing XSk ``XDP_TX`` frame + * @bq: XDP Tx bulk to queue the frame to + * @xdp: XSk buffer to queue + * @flush_bulk: driver callback to flush the bulk to the HW queue + * + * Return: true on success, false on flush error. + */ +static __always_inline bool +libeth_xsk_tx_queue_bulk(struct libeth_xdp_tx_bulk *bq, + struct libeth_xdp_buff *xdp, + bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq, + u32 flags)) +{ + bool ret = true; + + if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && + unlikely(!flush_bulk(bq, LIBETH_XDP_TX_XSK))) { + libeth_xsk_buff_free_slow(xdp); + return false; + } + + if (!libeth_xsk_tx_queue_head(bq, xdp)) + goto out; + + for (const struct libeth_xdp_buff *head = xdp; ; ) { + xdp = container_of(xsk_buff_get_frag(&head->base), + typeof(*xdp), base); + if (!xdp) + break; + + if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && + unlikely(!flush_bulk(bq, LIBETH_XDP_TX_XSK))) { + ret = false; + break; + } + + libeth_xsk_tx_queue_frag(bq, xdp); + }; + +out: + bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_LAST; + + return ret; +} + +/** + * libeth_xsk_tx_fill_buf - internal helper to fill XSk ``XDP_TX`` &libeth_sqe + * @frm: XDP Tx frame from the bulk + * @i: index on the HW queue + * @sq: XDPSQ abstraction for the queue + * @priv: private data + * + * Return: XDP Tx descriptor with the synced DMA and other info to pass to + * the driver callback. + */ +static inline struct libeth_xdp_tx_desc +libeth_xsk_tx_fill_buf(struct libeth_xdp_tx_frame frm, u32 i, + const struct libeth_xdpsq *sq, u64 priv) +{ + struct libeth_xdp_buff *xdp = frm.xsk; + struct libeth_xdp_tx_desc desc = { + .addr = xsk_buff_xdp_get_dma(&xdp->base), + .opts = frm.opts, + }; + struct libeth_sqe *sqe; + + xsk_buff_raw_dma_sync_for_device(sq->pool, desc.addr, desc.len); + + sqe = &sq->sqes[i]; + sqe->xsk = xdp; + + if (!(desc.flags & LIBETH_XDP_TX_FIRST)) { + sqe->type = LIBETH_SQE_XSK_TX_FRAG; + return desc; + } + + sqe->type = LIBETH_SQE_XSK_TX; + libeth_xdp_tx_fill_stats(sqe, &desc, + xdp_get_shared_info_from_buff(&xdp->base)); + + return desc; +} + +/** + * libeth_xsk_tx_flush_bulk - wrapper to define flush of an XSk ``XDP_TX`` bulk + * @bq: bulk to flush + * @flags: Tx flags, see __libeth_xdp_tx_flush_bulk() + * @prep: driver callback to prepare the queue + * @xmit: driver callback to fill a HW descriptor + * + * Use via LIBETH_XSK_DEFINE_FLUSH_TX() to define an XSk ``XDP_TX`` driver + * callback. + */ +#define libeth_xsk_tx_flush_bulk(bq, flags, prep, xmit) \ + __libeth_xdp_tx_flush_bulk(bq, (flags) | LIBETH_XDP_TX_XSK, prep, \ + libeth_xsk_tx_fill_buf, xmit) + +/* XSk TMO */ + +/** + * libeth_xsktmo_req_csum - XSk Tx metadata op to request checksum offload + * @csum_start: unused + * @csum_offset: unused + * @priv: &libeth_xdp_tx_desc from the filling helper + * + * Generic implementation of ::tmo_request_checksum. Works only when HW doesn't + * require filling checksum offsets and other parameters beside the checksum + * request bit. + * Consider using within @libeth_xsktmo unless the driver requires HW-specific + * callbacks. + */ +static inline void libeth_xsktmo_req_csum(u16 csum_start, u16 csum_offset, + void *priv) +{ + ((struct libeth_xdp_tx_desc *)priv)->flags |= LIBETH_XDP_TX_CSUM; +} + +/* Only to inline the callbacks below, use @libeth_xsktmo in drivers instead */ +static const struct xsk_tx_metadata_ops __libeth_xsktmo = { + .tmo_request_checksum = libeth_xsktmo_req_csum, +}; + +/** + * __libeth_xsk_xmit_fill_buf_md - internal helper to prepare XSk xmit w/meta + * @xdesc: &xdp_desc from the XSk buffer pool + * @sq: XDPSQ abstraction for the queue + * @priv: XSk Tx metadata ops + * + * Same as __libeth_xsk_xmit_fill_buf(), but requests metadata pointer and + * fills additional fields in &libeth_xdp_tx_desc to ask for metadata offload. + * + * Return: XDP Tx descriptor with the DMA, metadata request bits, and other + * info to pass to the driver callback. + */ +static __always_inline struct libeth_xdp_tx_desc +__libeth_xsk_xmit_fill_buf_md(const struct xdp_desc *xdesc, + const struct libeth_xdpsq *sq, + u64 priv) +{ + const struct xsk_tx_metadata_ops *tmo = libeth_xdp_priv_to_ptr(priv); + struct libeth_xdp_tx_desc desc; + struct xdp_desc_ctx ctx; + + ctx = xsk_buff_raw_get_ctx(sq->pool, xdesc->addr); + desc = (typeof(desc)){ + .addr = ctx.dma, + __libeth_xdp_tx_len(xdesc->len), + }; + + BUILD_BUG_ON(!__builtin_constant_p(tmo == libeth_xsktmo)); + tmo = tmo == libeth_xsktmo ? &__libeth_xsktmo : tmo; + + xsk_tx_metadata_request(ctx.meta, tmo, &desc); + + return desc; +} + +/* XSk xmit implementation */ + +/** + * __libeth_xsk_xmit_fill_buf - internal helper to prepare XSk xmit w/o meta + * @xdesc: &xdp_desc from the XSk buffer pool + * @sq: XDPSQ abstraction for the queue + * + * Return: XDP Tx descriptor with the DMA and other info to pass to + * the driver callback. + */ +static inline struct libeth_xdp_tx_desc +__libeth_xsk_xmit_fill_buf(const struct xdp_desc *xdesc, + const struct libeth_xdpsq *sq) +{ + return (struct libeth_xdp_tx_desc){ + .addr = xsk_buff_raw_get_dma(sq->pool, xdesc->addr), + __libeth_xdp_tx_len(xdesc->len), + }; +} + +/** + * libeth_xsk_xmit_fill_buf - internal helper to prepare an XSk xmit + * @frm: &xdp_desc from the XSk buffer pool + * @i: index on the HW queue + * @sq: XDPSQ abstraction for the queue + * @priv: XSk Tx metadata ops + * + * Depending on the metadata ops presence (determined at compile time), calls + * the quickest helper to build a libeth XDP Tx descriptor. + * + * Return: XDP Tx descriptor with the synced DMA, metadata request bits, + * and other info to pass to the driver callback. + */ +static __always_inline struct libeth_xdp_tx_desc +libeth_xsk_xmit_fill_buf(struct libeth_xdp_tx_frame frm, u32 i, + const struct libeth_xdpsq *sq, u64 priv) +{ + struct libeth_xdp_tx_desc desc; + + if (priv) + desc = __libeth_xsk_xmit_fill_buf_md(&frm.desc, sq, priv); + else + desc = __libeth_xsk_xmit_fill_buf(&frm.desc, sq); + + desc.flags |= xsk_is_eop_desc(&frm.desc) ? LIBETH_XDP_TX_LAST : 0; + + xsk_buff_raw_dma_sync_for_device(sq->pool, desc.addr, desc.len); + + return desc; +} + +/** + * libeth_xsk_xmit_do_bulk - send XSk xmit frames + * @pool: XSk buffer pool containing the frames to send + * @xdpsq: opaque pointer to driver's XDPSQ struct + * @budget: maximum number of frames can be sent + * @tmo: optional XSk Tx metadata ops + * @prep: driver callback to build a &libeth_xdpsq + * @xmit: driver callback to put frames to a HW queue + * @finalize: driver callback to start a transmisson + * + * Implements generic XSk xmit. Always turns on XSk Tx wakeup as it's assumed + * lazy cleaning is used and interrupts are disabled for the queue. + * HW descriptor filling is unrolled by ``LIBETH_XDP_TX_BATCH`` to optimize + * writes. + * Note that unlike other XDP Tx ops, the queue must be locked and cleaned + * prior to calling this function to already know available @budget. + * @prepare must only build a &libeth_xdpsq and return ``U32_MAX``. + * + * Return: false if @budget was exhausted, true otherwise. + */ +static __always_inline bool +libeth_xsk_xmit_do_bulk(struct xsk_buff_pool *pool, void *xdpsq, u32 budget, + const struct xsk_tx_metadata_ops *tmo, + u32 (*prep)(void *xdpsq, struct libeth_xdpsq *sq), + void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, u64 priv), + void (*finalize)(void *xdpsq, bool sent, bool flush)) +{ + const struct libeth_xdp_tx_frame *bulk; + bool wake; + u32 n; + + wake = xsk_uses_need_wakeup(pool); + if (wake) + xsk_clear_tx_need_wakeup(pool); + + n = xsk_tx_peek_release_desc_batch(pool, budget); + bulk = container_of(&pool->tx_descs[0], typeof(*bulk), desc); + + libeth_xdp_tx_xmit_bulk(bulk, xdpsq, n, true, + libeth_xdp_ptr_to_priv(tmo), prep, + libeth_xsk_xmit_fill_buf, xmit); + finalize(xdpsq, n, true); + + if (wake) + xsk_set_tx_need_wakeup(pool); + + return n < budget; +} + +/* Rx polling path */ + +/** + * libeth_xsk_tx_init_bulk - initialize an XDP Tx bulk for XSk Rx NAPI poll + * @bq: bulk to initialize + * @prog: RCU pointer to the XDP program (never %NULL) + * @dev: target &net_device + * @xdpsqs: array of driver XDPSQ structs + * @num: number of active XDPSQs, the above array length + * + * Should be called on an onstack XDP Tx bulk before the XSk NAPI polling loop. + * Initializes all the needed fields to run libeth_xdp functions. + * Never checks if @prog is %NULL or @num == 0 as XDP must always be enabled + * when hitting this path. + */ +#define libeth_xsk_tx_init_bulk(bq, prog, dev, xdpsqs, num) \ + __libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, true, \ + __UNIQUE_ID(bq_), __UNIQUE_ID(nqs_)) + +struct libeth_xdp_buff *libeth_xsk_buff_add_frag(struct libeth_xdp_buff *head, + struct libeth_xdp_buff *xdp); + +/** + * libeth_xsk_process_buff - attach an XSk Rx buffer to a &libeth_xdp_buff + * @head: head XSk buffer to attach the XSk buffer to (or %NULL) + * @xdp: XSk buffer to process + * @len: received data length from the descriptor + * + * If @head == %NULL, treats the XSk buffer as head and initializes + * the required fields. Otherwise, attaches the buffer as a frag. + * Already performs DMA sync-for-CPU and frame start prefetch + * (for head buffers only). + * + * Return: head XSk buffer on success or if the descriptor must be skipped + * (empty), %NULL if there is no space for a new frag. + */ +static inline struct libeth_xdp_buff * +libeth_xsk_process_buff(struct libeth_xdp_buff *head, + struct libeth_xdp_buff *xdp, u32 len) +{ + if (unlikely(!len)) { + libeth_xsk_buff_free_slow(xdp); + return head; + } + + xsk_buff_set_size(&xdp->base, len); + xsk_buff_dma_sync_for_cpu(&xdp->base); + + if (head) + return libeth_xsk_buff_add_frag(head, xdp); + + prefetch(xdp->data); + + return xdp; +} + +void libeth_xsk_buff_stats_frags(struct libeth_rq_napi_stats *rs, + const struct libeth_xdp_buff *xdp); + +u32 __libeth_xsk_run_prog_slow(struct libeth_xdp_buff *xdp, + const struct libeth_xdp_tx_bulk *bq, + enum xdp_action act, int ret); + +/** + * __libeth_xsk_run_prog - run XDP program on an XSk buffer + * @xdp: XSk buffer to run the prog on + * @bq: buffer bulk for ``XDP_TX`` queueing + * + * Internal inline abstraction to run XDP program on XSk Rx path. Handles + * only the most common ``XDP_REDIRECT`` inline, the rest is processed + * externally. + * Reports an XDP prog exception on errors. + * + * Return: libeth_xdp prog verdict depending on the prog's verdict. + */ +static __always_inline u32 +__libeth_xsk_run_prog(struct libeth_xdp_buff *xdp, + const struct libeth_xdp_tx_bulk *bq) +{ + enum xdp_action act; + int ret = 0; + + act = bpf_prog_run_xdp(bq->prog, &xdp->base); + if (unlikely(act != XDP_REDIRECT)) +rest: + return __libeth_xsk_run_prog_slow(xdp, bq, act, ret); + + ret = xdp_do_redirect(bq->dev, &xdp->base, bq->prog); + if (unlikely(ret)) + goto rest; + + return LIBETH_XDP_REDIRECT; +} + +/** + * libeth_xsk_run_prog - run XDP program on XSk path and handle all verdicts + * @xdp: XSk buffer to process + * @bq: XDP Tx bulk to queue ``XDP_TX`` buffers + * @fl: driver ``XDP_TX`` bulk flush callback + * + * Run the attached XDP program and handle all possible verdicts. + * Prefer using it via LIBETH_XSK_DEFINE_RUN{,_PASS,_PROG}(). + * + * Return: libeth_xdp prog verdict depending on the prog's verdict. + */ +#define libeth_xsk_run_prog(xdp, bq, fl) \ + __libeth_xdp_run_flush(xdp, bq, __libeth_xsk_run_prog, \ + libeth_xsk_tx_queue_bulk, fl) + +/** + * __libeth_xsk_run_pass - helper to run XDP program and handle the result + * @xdp: XSk buffer to process + * @bq: XDP Tx bulk to queue ``XDP_TX`` frames + * @napi: NAPI to build an skb and pass it up the stack + * @rs: onstack libeth RQ stats + * @md: metadata that should be filled to the XSk buffer + * @prep: callback for filling the metadata + * @run: driver wrapper to run XDP program + * @populate: driver callback to populate an skb with the HW descriptor data + * + * Inline abstraction, XSk's counterpart of __libeth_xdp_run_pass(), see its + * doc for details. + * + * Return: false if the polling loop must be exited due to lack of free + * buffers, true otherwise. + */ +static __always_inline bool +__libeth_xsk_run_pass(struct libeth_xdp_buff *xdp, + struct libeth_xdp_tx_bulk *bq, struct napi_struct *napi, + struct libeth_rq_napi_stats *rs, const void *md, + void (*prep)(struct libeth_xdp_buff *xdp, + const void *md), + u32 (*run)(struct libeth_xdp_buff *xdp, + struct libeth_xdp_tx_bulk *bq), + bool (*populate)(struct sk_buff *skb, + const struct libeth_xdp_buff *xdp, + struct libeth_rq_napi_stats *rs)) +{ + struct sk_buff *skb; + u32 act; + + rs->bytes += xdp->base.data_end - xdp->data; + rs->packets++; + + if (unlikely(xdp_buff_has_frags(&xdp->base))) + libeth_xsk_buff_stats_frags(rs, xdp); + + if (prep && (!__builtin_constant_p(!!md) || md)) + prep(xdp, md); + + act = run(xdp, bq); + if (unlikely(act == LIBETH_XDP_ABORTED)) + return false; + else if (likely(act != LIBETH_XDP_PASS)) + return true; + + skb = xdp_build_skb_from_zc(&xdp->base); + if (unlikely(!skb)) { + libeth_xsk_buff_free_slow(xdp); + return true; + } + + if (unlikely(!populate(skb, xdp, rs))) { + napi_consume_skb(skb, true); + return true; + } + + napi_gro_receive(napi, skb); + + return true; +} + +/** + * libeth_xsk_run_pass - helper to run XDP program and handle the result + * @xdp: XSk buffer to process + * @bq: XDP Tx bulk to queue ``XDP_TX`` frames + * @napi: NAPI to build an skb and pass it up the stack + * @rs: onstack libeth RQ stats + * @desc: pointer to the HW descriptor for that frame + * @run: driver wrapper to run XDP program + * @populate: driver callback to populate an skb with the HW descriptor data + * + * Wrapper around the underscored version when "fill the descriptor metadata" + * means just writing the pointer to the HW descriptor as @xdp->desc. + */ +#define libeth_xsk_run_pass(xdp, bq, napi, rs, desc, run, populate) \ + __libeth_xsk_run_pass(xdp, bq, napi, rs, desc, libeth_xdp_prep_desc, \ + run, populate) + +/** + * libeth_xsk_finalize_rx - finalize XDPSQ after an XSk NAPI polling loop + * @bq: ``XDP_TX`` frame bulk + * @flush: driver callback to flush the bulk + * @finalize: driver callback to start sending the frames and run the timer + * + * Flush the bulk if there are frames left to send, kick the queue and flush + * the XDP maps. + */ +#define libeth_xsk_finalize_rx(bq, flush, finalize) \ + __libeth_xdp_finalize_rx(bq, LIBETH_XDP_TX_XSK, flush, finalize) + +/* Helpers to reduce boilerplate code in drivers. + * + * Typical driver XSk Rx flow would be (excl. bulk and buff init, frag attach): + * LIBETH_XDP_DEFINE_START(); + * LIBETH_XSK_DEFINE_FLUSH_TX(static driver_xsk_flush_tx, driver_xsk_tx_prep, + * driver_xdp_xmit); + * LIBETH_XSK_DEFINE_RUN(static driver_xsk_run, driver_xsk_run_prog, + * driver_xsk_flush_tx, driver_populate_skb); + * LIBETH_XSK_DEFINE_FINALIZE(static driver_xsk_finalize_rx, + * driver_xsk_flush_tx, driver_xdp_finalize_sq); + * LIBETH_XDP_DEFINE_END(); + * This will build a set of 4 static functions. The compiler is free to decide + * whether to inline them. + * Then, in the NAPI polling function: + * while (packets < budget) { + * // ... + * if (!driver_xsk_run(xdp, &bq, napi, &rs, desc)) + * break; + * } + * driver_xsk_finalize_rx(&bq); + */ + +/** + * LIBETH_XSK_DEFINE_FLUSH_TX - define a driver XSk ``XDP_TX`` flush function + * @name: name of the function to define + * @prep: driver callback to clean an XDPSQ + * @xmit: driver callback to write a HW Tx descriptor + */ +#define LIBETH_XSK_DEFINE_FLUSH_TX(name, prep, xmit) \ + __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, xsk) + +/** + * LIBETH_XSK_DEFINE_RUN_PROG - define a driver XDP program run function + * @name: name of the function to define + * @flush: driver callback to flush an XSk ``XDP_TX`` bulk + */ +#define LIBETH_XSK_DEFINE_RUN_PROG(name, flush) \ + u32 __LIBETH_XDP_DEFINE_RUN_PROG(name, flush, xsk) + +/** + * LIBETH_XSK_DEFINE_RUN_PASS - define a driver buffer process + pass function + * @name: name of the function to define + * @run: driver callback to run XDP program (above) + * @populate: driver callback to fill an skb with HW descriptor info + */ +#define LIBETH_XSK_DEFINE_RUN_PASS(name, run, populate) \ + bool __LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate, xsk) + +/** + * LIBETH_XSK_DEFINE_RUN - define a driver buffer process, run + pass function + * @name: name of the function to define + * @run: name of the XDP prog run function to define + * @flush: driver callback to flush an XSk ``XDP_TX`` bulk + * @populate: driver callback to fill an skb with HW descriptor info + */ +#define LIBETH_XSK_DEFINE_RUN(name, run, flush, populate) \ + __LIBETH_XDP_DEFINE_RUN(name, run, flush, populate, XSK) + +/** + * LIBETH_XSK_DEFINE_FINALIZE - define a driver XSk NAPI poll finalize function + * @name: name of the function to define + * @flush: driver callback to flush an XSk ``XDP_TX`` bulk + * @finalize: driver callback to finalize an XDPSQ and run the timer + */ +#define LIBETH_XSK_DEFINE_FINALIZE(name, flush, finalize) \ + __LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize, xsk) + +/* Refilling */ + +/** + * struct libeth_xskfq - structure representing an XSk buffer (fill) queue + * @fp: hotpath part of the structure + * @pool: &xsk_buff_pool for buffer management + * @fqes: array of XSk buffer pointers + * @descs: opaque pointer to the HW descriptor array + * @ntu: index of the next buffer to poll + * @count: number of descriptors/buffers the queue has + * @pending: current number of XSkFQEs to refill + * @thresh: threshold below which the queue is refilled + * @buf_len: HW-writeable length per each buffer + * @nid: ID of the closest NUMA node with memory + */ +struct libeth_xskfq { + struct_group_tagged(libeth_xskfq_fp, fp, + struct xsk_buff_pool *pool; + struct libeth_xdp_buff **fqes; + void *descs; + + u32 ntu; + u32 count; + ); + + /* Cold fields */ + u32 pending; + u32 thresh; + + u32 buf_len; + int nid; +}; + +int libeth_xskfq_create(struct libeth_xskfq *fq); +void libeth_xskfq_destroy(struct libeth_xskfq *fq); + +/** + * libeth_xsk_buff_xdp_get_dma - get DMA address for an XSk &libeth_xdp_buff + * @xdp: buffer to get the DMA addr for + */ +#define libeth_xsk_buff_xdp_get_dma(xdp) \ + xsk_buff_xdp_get_dma(&(xdp)->base) + +/** + * libeth_xskfqe_alloc - allocate @n XSk Rx buffers + * @fq: hotpath part of the XSkFQ, usually onstack + * @n: number of buffers to allocate + * @fill: driver callback to write DMA addresses to HW descriptors + * + * Note that @fq->ntu gets updated, but ::pending must be recalculated + * by the caller. + * + * Return: number of buffers refilled. + */ +static __always_inline u32 +libeth_xskfqe_alloc(struct libeth_xskfq_fp *fq, u32 n, + void (*fill)(const struct libeth_xskfq_fp *fq, u32 i)) +{ + u32 this, ret, done = 0; + struct xdp_buff **xskb; + + this = fq->count - fq->ntu; + if (likely(this > n)) + this = n; + +again: + xskb = (typeof(xskb))&fq->fqes[fq->ntu]; + ret = xsk_buff_alloc_batch(fq->pool, xskb, this); + + for (u32 i = 0, ntu = fq->ntu; likely(i < ret); i++) + fill(fq, ntu + i); + + done += ret; + fq->ntu += ret; + + if (likely(fq->ntu < fq->count) || unlikely(ret < this)) + goto out; + + fq->ntu = 0; + + if (this < n) { + this = n - this; + goto again; + } + +out: + return done; +} + +/* .ndo_xsk_wakeup */ + +void libeth_xsk_init_wakeup(call_single_data_t *csd, struct napi_struct *napi); +void libeth_xsk_wakeup(call_single_data_t *csd, u32 qid); + +/* Pool setup */ + +int libeth_xsk_setup_pool(struct net_device *dev, u32 qid, bool enable); + +#endif /* __LIBETH_XSK_H */ From a6d14165b99c024de573c36b22327a348dd44477 Mon Sep 17 00:00:00 2001 From: Michal Kubiak Date: Fri, 20 Oct 2023 11:56:51 +0200 Subject: [PATCH 36/49] idpf: make complq cleaning dependent on scheduling mode Extend completion queue cleaning function to support queue-based scheduling mode needed for XDP queues. Add 4-byte descriptor for queue-based scheduling mode and perform some refactoring to extract the common code for both scheduling modes. Signed-off-by: Michal Kubiak Signed-off-by: Alexander Lobakin --- .../net/ethernet/intel/idpf/idpf_lan_txrx.h | 6 +- drivers/net/ethernet/intel/idpf/idpf_txrx.c | 262 +++++++++++------- drivers/net/ethernet/intel/idpf/idpf_txrx.h | 11 +- 3 files changed, 180 insertions(+), 99 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf_lan_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_lan_txrx.h index 8c7f8ef8f1a153..7f12c7f2e70ec6 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lan_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_lan_txrx.h @@ -186,13 +186,17 @@ struct idpf_base_tx_desc { __le64 qw1; /* type_cmd_offset_bsz_l2tag1 */ }; /* read used with buffer queues */ -struct idpf_splitq_tx_compl_desc { +struct idpf_splitq_4b_tx_compl_desc { /* qid=[10:0] comptype=[13:11] rsvd=[14] gen=[15] */ __le16 qid_comptype_gen; union { __le16 q_head; /* Queue head */ __le16 compl_tag; /* Completion tag */ } q_head_compl_tag; +}; /* writeback used with completion queues */ + +struct idpf_splitq_tx_compl_desc { + struct idpf_splitq_4b_tx_compl_desc common; u8 ts[3]; u8 rsvd; /* Reserved */ }; /* writeback used with completion queues */ diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index ea8914a42899da..3df15a8924c7dc 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -157,8 +157,8 @@ static void idpf_compl_desc_rel(struct idpf_compl_queue *complq) return; dma_free_coherent(complq->netdev->dev.parent, complq->size, - complq->comp, complq->dma); - complq->comp = NULL; + complq->desc_ring, complq->dma); + complq->desc_ring = NULL; complq->next_to_use = 0; complq->next_to_clean = 0; } @@ -287,12 +287,16 @@ static int idpf_tx_desc_alloc(const struct idpf_vport *vport, static int idpf_compl_desc_alloc(const struct idpf_vport *vport, struct idpf_compl_queue *complq) { - complq->size = array_size(complq->desc_count, sizeof(*complq->comp)); + u32 desc_size; - complq->comp = dma_alloc_coherent(complq->netdev->dev.parent, - complq->size, &complq->dma, - GFP_KERNEL); - if (!complq->comp) + desc_size = idpf_queue_has(FLOW_SCH_EN, complq) ? + sizeof(*complq->comp) : sizeof(*complq->comp_4b); + complq->size = array_size(complq->desc_count, desc_size); + + complq->desc_ring = dma_alloc_coherent(complq->netdev->dev.parent, + complq->size, &complq->dma, + GFP_KERNEL); + if (!complq->desc_ring) return -ENOMEM; complq->next_to_use = 0; @@ -1930,8 +1934,46 @@ static bool idpf_tx_clean_buf_ring(struct idpf_tx_queue *txq, u16 compl_tag, } /** - * idpf_tx_handle_rs_completion - clean a single packet and all of its buffers - * whether on the buffer ring or in the hash table + * idpf_parse_compl_desc - Parse the completion descriptor + * @desc: completion descriptor to be parsed + * @complq: completion queue containing the descriptor + * @txq: returns corresponding Tx queue for a given descriptor + * @gen_flag: current generation flag in the completion queue + * + * Returns completion type from descriptor or negative value in case of error: + * -ENODATA if there is no completion descriptor to be cleaned + * -EINVAL if no Tx queue has been found for the completion queue + */ +static int +idpf_parse_compl_desc(const struct idpf_splitq_4b_tx_compl_desc *desc, + const struct idpf_compl_queue *complq, + struct idpf_tx_queue **txq, bool gen_flag) +{ + struct idpf_tx_queue *target; + u32 rel_tx_qid, comptype; + + /* if the descriptor isn't done, no work yet to do */ + comptype = le16_to_cpu(desc->qid_comptype_gen); + if (!!(comptype & IDPF_TXD_COMPLQ_GEN_M) != gen_flag) + return -ENODATA; + + /* Find necessary info of TX queue to clean buffers */ + rel_tx_qid = FIELD_GET(IDPF_TXD_COMPLQ_QID_M, comptype); + target = likely(rel_tx_qid < complq->txq_grp->num_txq) ? + complq->txq_grp->txqs[rel_tx_qid] : NULL; + + if (!target) + return -EINVAL; + + *txq = target; + + /* Determine completion type */ + return FIELD_GET(IDPF_TXD_COMPLQ_COMPL_TYPE_M, comptype); +} + +/** + * idpf_tx_handle_rs_cmpl_qb - clean a single packet and all of its buffers + * whether the Tx queue is working in queue-based scheduling * @txq: Tx ring to clean * @desc: pointer to completion queue descriptor to extract completion * information from @@ -1940,21 +1982,33 @@ static bool idpf_tx_clean_buf_ring(struct idpf_tx_queue *txq, u16 compl_tag, * * Returns bytes/packets cleaned */ -static void idpf_tx_handle_rs_completion(struct idpf_tx_queue *txq, - struct idpf_splitq_tx_compl_desc *desc, - struct libeth_sq_napi_stats *cleaned, - int budget) +static void +idpf_tx_handle_rs_cmpl_qb(struct idpf_tx_queue *txq, + const struct idpf_splitq_4b_tx_compl_desc *desc, + struct libeth_sq_napi_stats *cleaned, int budget) { - u16 compl_tag; + u16 head = le16_to_cpu(desc->q_head_compl_tag.q_head); - if (!idpf_queue_has(FLOW_SCH_EN, txq)) { - u16 head = le16_to_cpu(desc->q_head_compl_tag.q_head); - - idpf_tx_splitq_clean(txq, head, budget, cleaned, false); - return; - } + idpf_tx_splitq_clean(txq, head, budget, cleaned, false); +} - compl_tag = le16_to_cpu(desc->q_head_compl_tag.compl_tag); +/** + * idpf_tx_handle_rs_cmpl_fb - clean a single packet and all of its buffers + * whether on the buffer ring or in the hash table (flow-based scheduling only) + * @txq: Tx ring to clean + * @desc: pointer to completion queue descriptor to extract completion + * information from + * @cleaned: pointer to stats struct to track cleaned packets/bytes + * @budget: Used to determine if we are in netpoll + * + * Returns bytes/packets cleaned + */ +static void +idpf_tx_handle_rs_cmpl_fb(struct idpf_tx_queue *txq, + const struct idpf_splitq_4b_tx_compl_desc *desc, + struct libeth_sq_napi_stats *cleaned,int budget) +{ + u16 compl_tag = le16_to_cpu(desc->q_head_compl_tag.compl_tag); /* If we didn't clean anything on the ring, this packet must be * in the hash table. Go clean it there. @@ -1963,6 +2017,64 @@ static void idpf_tx_handle_rs_completion(struct idpf_tx_queue *txq, idpf_tx_clean_stashed_bufs(txq, compl_tag, cleaned, budget); } +/** + * idpf_tx_finalize_complq - Finalize completion queue cleaning + * @complq: completion queue to finalize + * @ntc: next to complete index + * @gen_flag: current state of generation flag + * @cleaned: returns number of packets cleaned + */ +static void idpf_tx_finalize_complq(struct idpf_compl_queue *complq, int ntc, + bool gen_flag, int *cleaned) +{ + struct idpf_netdev_priv *np; + bool complq_ok = true; + int i; + + /* Store the state of the complq to be used later in deciding if a + * TXQ can be started again + */ + if (unlikely(IDPF_TX_COMPLQ_PENDING(complq->txq_grp) > + IDPF_TX_COMPLQ_OVERFLOW_THRESH(complq))) + complq_ok = false; + + np = netdev_priv(complq->netdev); + for (i = 0; i < complq->txq_grp->num_txq; ++i) { + struct idpf_tx_queue *tx_q = complq->txq_grp->txqs[i]; + struct netdev_queue *nq; + bool dont_wake; + + /* We didn't clean anything on this queue, move along */ + if (!tx_q->cleaned_bytes) + continue; + + *cleaned += tx_q->cleaned_pkts; + + /* Update BQL */ + nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); + + dont_wake = !complq_ok || IDPF_TX_BUF_RSV_LOW(tx_q) || + np->state != __IDPF_VPORT_UP || + !netif_carrier_ok(tx_q->netdev); + /* Check if the TXQ needs to and can be restarted */ + if (!__netif_txq_completed_wake(nq, tx_q->cleaned_pkts, + tx_q->cleaned_bytes, + IDPF_DESC_UNUSED(tx_q), + IDPF_TX_WAKE_THRESH, + dont_wake)) + libeth_stats_inc_one(&tx_q->stats, wake); + + /* Reset cleaned stats for the next time this queue is + * cleaned + */ + tx_q->cleaned_bytes = 0; + tx_q->cleaned_pkts = 0; + } + + complq->next_to_clean = ntc + complq->desc_count; + idpf_queue_assign(GEN_CHK, complq, gen_flag); +} + /** * idpf_tx_clean_complq - Reclaim resources on completion queue * @complq: Tx ring to clean @@ -1974,60 +2086,56 @@ static void idpf_tx_handle_rs_completion(struct idpf_tx_queue *txq, static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget, int *cleaned) { - struct idpf_splitq_tx_compl_desc *tx_desc; + struct idpf_splitq_4b_tx_compl_desc *tx_desc; s16 ntc = complq->next_to_clean; - struct idpf_netdev_priv *np; unsigned int complq_budget; - bool complq_ok = true; - int i; + bool flow, gen_flag; + u32 pos = ntc; + + flow = idpf_queue_has(FLOW_SCH_EN, complq); + gen_flag = idpf_queue_has(GEN_CHK, complq); complq_budget = complq->clean_budget; - tx_desc = &complq->comp[ntc]; + tx_desc = flow ? &complq->comp[pos].common : &complq->comp_4b[pos]; ntc -= complq->desc_count; do { struct libeth_sq_napi_stats cleaned_stats = { }; struct idpf_tx_queue *tx_q; - int rel_tx_qid; u16 hw_head; - u8 ctype; /* completion type */ - u16 gen; + int ctype; - /* if the descriptor isn't done, no work yet to do */ - gen = le16_get_bits(tx_desc->qid_comptype_gen, - IDPF_TXD_COMPLQ_GEN_M); - if (idpf_queue_has(GEN_CHK, complq) != gen) - break; - - /* Find necessary info of TX queue to clean buffers */ - rel_tx_qid = le16_get_bits(tx_desc->qid_comptype_gen, - IDPF_TXD_COMPLQ_QID_M); - if (rel_tx_qid >= complq->txq_grp->num_txq || - !complq->txq_grp->txqs[rel_tx_qid]) { - netdev_err(complq->netdev, "TxQ not found\n"); - goto fetch_next_desc; - } - tx_q = complq->txq_grp->txqs[rel_tx_qid]; - - /* Determine completion type */ - ctype = le16_get_bits(tx_desc->qid_comptype_gen, - IDPF_TXD_COMPLQ_COMPL_TYPE_M); + ctype = idpf_parse_compl_desc(tx_desc, complq, &tx_q, + gen_flag); switch (ctype) { case IDPF_TXD_COMPLT_RE: + if (unlikely(!flow)) + goto fetch_next_desc; + hw_head = le16_to_cpu(tx_desc->q_head_compl_tag.q_head); idpf_tx_splitq_clean(tx_q, hw_head, budget, &cleaned_stats, true); break; case IDPF_TXD_COMPLT_RS: - idpf_tx_handle_rs_completion(tx_q, tx_desc, - &cleaned_stats, budget); + if (flow) + idpf_tx_handle_rs_cmpl_fb(tx_q, tx_desc, + &cleaned_stats, + budget); + else + idpf_tx_handle_rs_cmpl_qb(tx_q, tx_desc, + &cleaned_stats, + budget); break; case IDPF_TXD_COMPLT_SW_MARKER: idpf_tx_handle_sw_marker(tx_q); break; + case -ENODATA: + goto exit_clean_complq; + case -EINVAL: + goto fetch_next_desc; default: - netdev_err(tx_q->netdev, + netdev_err(complq->netdev, "Unknown TX completion type: %d\n", ctype); goto fetch_next_desc; } @@ -2038,62 +2146,24 @@ static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget, complq->num_completions++; fetch_next_desc: - tx_desc++; + pos++; ntc++; if (unlikely(!ntc)) { ntc -= complq->desc_count; - tx_desc = &complq->comp[0]; - idpf_queue_change(GEN_CHK, complq); + pos = 0; + gen_flag = !gen_flag; } + tx_desc = flow ? &complq->comp[pos].common : + &complq->comp_4b[pos]; prefetch(tx_desc); /* update budget accounting */ complq_budget--; } while (likely(complq_budget)); - /* Store the state of the complq to be used later in deciding if a - * TXQ can be started again - */ - if (unlikely(IDPF_TX_COMPLQ_PENDING(complq->txq_grp) > - IDPF_TX_COMPLQ_OVERFLOW_THRESH(complq))) - complq_ok = false; - - np = netdev_priv(complq->netdev); - for (i = 0; i < complq->txq_grp->num_txq; ++i) { - struct idpf_tx_queue *tx_q = complq->txq_grp->txqs[i]; - struct netdev_queue *nq; - bool dont_wake; - - /* We didn't clean anything on this queue, move along */ - if (!tx_q->cleaned_bytes) - continue; - - *cleaned += tx_q->cleaned_pkts; - - /* Update BQL */ - nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); - - dont_wake = !complq_ok || IDPF_TX_BUF_RSV_LOW(tx_q) || - np->state != __IDPF_VPORT_UP || - !netif_carrier_ok(tx_q->netdev); - /* Check if the TXQ needs to and can be restarted */ - if (!__netif_txq_completed_wake(nq, tx_q->cleaned_pkts, - tx_q->cleaned_bytes, - IDPF_DESC_UNUSED(tx_q), - IDPF_TX_WAKE_THRESH, - dont_wake)) - libeth_stats_inc_one(&tx_q->stats, wake); - - /* Reset cleaned stats for the next time this queue is - * cleaned - */ - tx_q->cleaned_bytes = 0; - tx_q->cleaned_pkts = 0; - } - - ntc += complq->desc_count; - complq->next_to_clean = ntc; +exit_clean_complq: + idpf_tx_finalize_complq(complq, ntc, gen_flag, cleaned); return !!complq_budget; } diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index 8188f5cb418baa..c8760355fe1792 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -736,7 +736,9 @@ libeth_cacheline_set_assert(struct idpf_buf_queue, 64, 24, 32); /** * struct idpf_compl_queue - software structure representing a completion queue - * @comp: completion descriptor array + * @comp: 8-byte completion descriptor array + * @comp_4b: 4-byte completion descriptor array + * @desc_ring: virtual descriptor ring address * @txq_grp: See struct idpf_txq_group * @flags: See enum idpf_queue_flags_t * @desc_count: Number of descriptors @@ -756,7 +758,12 @@ libeth_cacheline_set_assert(struct idpf_buf_queue, 64, 24, 32); */ struct idpf_compl_queue { __cacheline_group_begin_aligned(read_mostly); - struct idpf_splitq_tx_compl_desc *comp; + union { + struct idpf_splitq_tx_compl_desc *comp; + struct idpf_splitq_4b_tx_compl_desc *comp_4b; + + void *desc_ring; + }; struct idpf_txq_group *txq_grp; DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); From 1ecd090d257237ae79345f4e885d69b2b9b97c29 Mon Sep 17 00:00:00 2001 From: Michal Kubiak Date: Fri, 23 Feb 2024 17:19:01 +0100 Subject: [PATCH 37/49] idpf: remove SW marker handling from NAPI SW marker descriptors on completion queues are used only when a queue is about to be destroyed. It's far from hotpath and handling it in the hotpath NAPI poll makes no sense. Instead, run a simple poller after a virtchnl message for destroying the queue is sent and wait for the replies. If replies for all of the queues are received, this means the synchronization is done correctly and we can go forth with stopping the link. Signed-off-by: Michal Kubiak Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/idpf/idpf.h | 7 +- drivers/net/ethernet/intel/idpf/idpf_lib.c | 2 - drivers/net/ethernet/intel/idpf/idpf_txrx.c | 108 +++++++++++------- drivers/net/ethernet/intel/idpf/idpf_txrx.h | 4 +- .../net/ethernet/intel/idpf/idpf_virtchnl.c | 34 ++---- 5 files changed, 80 insertions(+), 75 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf.h b/drivers/net/ethernet/intel/idpf/idpf.h index 88e36cc29ae726..c0109879d4db81 100644 --- a/drivers/net/ethernet/intel/idpf/idpf.h +++ b/drivers/net/ethernet/intel/idpf/idpf.h @@ -36,6 +36,7 @@ struct idpf_vport_max_q; #define IDPF_NUM_CHUNKS_PER_MSG(struct_sz, chunk_sz) \ ((IDPF_CTLQ_MAX_BUF_LEN - (struct_sz)) / (chunk_sz)) +#define IDPF_WAIT_FOR_MARKER_TIMEO 500 #define IDPF_MAX_WAIT 500 /* available message levels */ @@ -212,13 +213,10 @@ struct idpf_dev_ops { /** * enum idpf_vport_flags - Vport flags * @IDPF_VPORT_DEL_QUEUES: To send delete queues message - * @IDPF_VPORT_SW_MARKER: Indicate TX pipe drain software marker packets - * processing is done * @IDPF_VPORT_FLAGS_NBITS: Must be last */ enum idpf_vport_flags { IDPF_VPORT_DEL_QUEUES, - IDPF_VPORT_SW_MARKER, IDPF_VPORT_FLAGS_NBITS, }; @@ -265,7 +263,6 @@ enum idpf_vport_flags { * @vport_stats: vport stats reported by HW * @link_up: True if link is up * @link_speed_mbps: Link speed in mbps - * @sw_marker_wq: workqueue for marker packets */ struct idpf_vport { u16 num_txq; @@ -309,8 +306,6 @@ struct idpf_vport { bool link_up; u32 link_speed_mbps; - - wait_queue_head_t sw_marker_wq; }; /** diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index 3089def8d4536c..ee6768c1b6be42 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -1514,8 +1514,6 @@ void idpf_init_task(struct work_struct *work) index = vport->idx; vport_config = adapter->vport_config[index]; - init_waitqueue_head(&vport->sw_marker_wq); - spin_lock_init(&vport_config->mac_filter_list_lock); INIT_LIST_HEAD(&vport_config->user_config.mac_filter_list); diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 3df15a8924c7dc..2d616078b6c201 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -1639,32 +1639,6 @@ int idpf_vport_queues_alloc(struct idpf_vport *vport) return err; } -/** - * idpf_tx_handle_sw_marker - Handle queue marker packet - * @tx_q: tx queue to handle software marker - */ -static void idpf_tx_handle_sw_marker(struct idpf_tx_queue *tx_q) -{ - struct idpf_netdev_priv *priv = netdev_priv(tx_q->netdev); - struct idpf_vport *vport = priv->vport; - int i; - - idpf_queue_clear(SW_MARKER, tx_q); - /* Hardware must write marker packets to all queues associated with - * completion queues. So check if all queues received marker packets - */ - for (i = 0; i < vport->num_txq; i++) - /* If we're still waiting on any other TXQ marker completions, - * just return now since we cannot wake up the marker_wq yet. - */ - if (idpf_queue_has(SW_MARKER, vport->txqs[i])) - return; - - /* Drain complete */ - set_bit(IDPF_VPORT_SW_MARKER, vport->flags); - wake_up(&vport->sw_marker_wq); -} - /** * idpf_tx_clean_stashed_bufs - clean bufs that were stored for * out of order completions @@ -2017,6 +1991,19 @@ idpf_tx_handle_rs_cmpl_fb(struct idpf_tx_queue *txq, idpf_tx_clean_stashed_bufs(txq, compl_tag, cleaned, budget); } +/** + * idpf_tx_update_complq_indexes - update completion queue indexes + * @complq: completion queue being updated + * @ntc: current "next to clean" index value + * @gen_flag: current "generation" flag value + */ +static void idpf_tx_update_complq_indexes(struct idpf_compl_queue *complq, + int ntc, bool gen_flag) +{ + complq->next_to_clean = ntc + complq->desc_count; + idpf_queue_assign(GEN_CHK, complq, gen_flag); +} + /** * idpf_tx_finalize_complq - Finalize completion queue cleaning * @complq: completion queue to finalize @@ -2071,8 +2058,7 @@ static void idpf_tx_finalize_complq(struct idpf_compl_queue *complq, int ntc, tx_q->cleaned_pkts = 0; } - complq->next_to_clean = ntc + complq->desc_count; - idpf_queue_assign(GEN_CHK, complq, gen_flag); + idpf_tx_update_complq_indexes(complq, ntc, gen_flag); } /** @@ -2127,9 +2113,6 @@ static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget, &cleaned_stats, budget); break; - case IDPF_TXD_COMPLT_SW_MARKER: - idpf_tx_handle_sw_marker(tx_q); - break; case -ENODATA: goto exit_clean_complq; case -EINVAL: @@ -2168,6 +2151,59 @@ static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget, return !!complq_budget; } +/** + * idpf_wait_for_sw_marker_completion - wait for SW marker of disabled Tx queue + * @txq: disabled Tx queue + */ +void idpf_wait_for_sw_marker_completion(struct idpf_tx_queue *txq) +{ + struct idpf_compl_queue *complq = txq->txq_grp->complq; + struct idpf_splitq_4b_tx_compl_desc *tx_desc; + s16 ntc = complq->next_to_clean; + unsigned long timeout; + bool flow, gen_flag; + u32 pos = ntc; + + if (!idpf_queue_has(SW_MARKER, txq)) + return; + + flow = idpf_queue_has(FLOW_SCH_EN, complq); + gen_flag = idpf_queue_has(GEN_CHK, complq); + + timeout = jiffies + msecs_to_jiffies(IDPF_WAIT_FOR_MARKER_TIMEO); + tx_desc = flow ? &complq->comp[pos].common : &complq->comp_4b[pos]; + ntc -= complq->desc_count; + + do { + struct idpf_tx_queue *tx_q; + int ctype; + + ctype = idpf_parse_compl_desc(tx_desc, complq, &tx_q, + gen_flag); + if (ctype == IDPF_TXD_COMPLT_SW_MARKER) { + idpf_queue_clear(SW_MARKER, tx_q); + if (txq == tx_q) + break; + } else if (ctype == -ENODATA) { + usleep_range(500, 1000); + continue; + } + + pos++; + ntc++; + if (unlikely(!ntc)) { + ntc -= complq->desc_count; + pos = 0; + gen_flag = !gen_flag; + } + + tx_desc = flow ? &complq->comp[pos].common : + &complq->comp_4b[pos]; + prefetch(tx_desc); + } while (time_before(jiffies, timeout)); + + idpf_tx_update_complq_indexes(complq, ntc, gen_flag); +} /** * idpf_tx_splitq_build_ctb - populate command tag and size for queue * based scheduling descriptors @@ -4106,15 +4142,7 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) else idpf_vport_intr_set_wb_on_itr(q_vector); - /* Switch to poll mode in the tear-down path after sending disable - * queues virtchnl message, as the interrupts will be disabled after - * that - */ - if (unlikely(q_vector->num_txq && idpf_queue_has(POLL_MODE, - q_vector->tx[0]))) - return budget; - else - return work_done; + return work_done; } /** diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index c8760355fe1792..e4c3c802b3f08a 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -308,7 +308,6 @@ struct idpf_ptype_state { * bit and Q_RFL_GEN is the SW bit. * @__IDPF_Q_FLOW_SCH_EN: Enable flow scheduling * @__IDPF_Q_SW_MARKER: Used to indicate TX queue marker completions - * @__IDPF_Q_POLL_MODE: Enable poll mode * @__IDPF_Q_CRC_EN: enable CRC offload in singleq mode * @__IDPF_Q_HSPLIT_EN: enable header split on Rx (splitq) * @__IDPF_Q_FLAGS_NBITS: Must be last @@ -318,7 +317,6 @@ enum idpf_queue_flags_t { __IDPF_Q_RFL_GEN_CHK, __IDPF_Q_FLOW_SCH_EN, __IDPF_Q_SW_MARKER, - __IDPF_Q_POLL_MODE, __IDPF_Q_CRC_EN, __IDPF_Q_HSPLIT_EN, @@ -1038,6 +1036,8 @@ bool idpf_rx_singleq_buf_hw_alloc_all(struct idpf_rx_queue *rxq, int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off, struct libeth_sq_xmit_stats *ss); +void idpf_wait_for_sw_marker_completion(struct idpf_tx_queue *txq); + static inline bool idpf_tx_maybe_stop_common(struct idpf_tx_queue *tx_q, u32 needed) { diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c index 3fcc8ac70b7449..dba0ce23af6687 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c @@ -748,21 +748,17 @@ int idpf_recv_mb_msg(struct idpf_adapter *adapter) **/ static int idpf_wait_for_marker_event(struct idpf_vport *vport) { - int event; - int i; - - for (i = 0; i < vport->num_txq; i++) - idpf_queue_set(SW_MARKER, vport->txqs[i]); + bool markers_rcvd = true; - event = wait_event_timeout(vport->sw_marker_wq, - test_and_clear_bit(IDPF_VPORT_SW_MARKER, - vport->flags), - msecs_to_jiffies(500)); + for (u32 i = 0; i < vport->num_txq; i++) { + struct idpf_tx_queue *txq = vport->txqs[i]; - for (i = 0; i < vport->num_txq; i++) - idpf_queue_clear(POLL_MODE, vport->txqs[i]); + idpf_queue_set(SW_MARKER, txq); + idpf_wait_for_sw_marker_completion(txq); + markers_rcvd &= !idpf_queue_has(SW_MARKER, txq); + } - if (event) + if (markers_rcvd) return 0; dev_warn(&vport->adapter->pdev->dev, "Failed to receive marker packets\n"); @@ -1993,24 +1989,12 @@ int idpf_send_enable_queues_msg(struct idpf_vport *vport) */ int idpf_send_disable_queues_msg(struct idpf_vport *vport) { - int err, i; + int err; err = idpf_send_ena_dis_queues_msg(vport, false); if (err) return err; - /* switch to poll mode as interrupts will be disabled after disable - * queues virtchnl message is sent - */ - for (i = 0; i < vport->num_txq; i++) - idpf_queue_set(POLL_MODE, vport->txqs[i]); - - /* schedule the napi to receive all the marker packets */ - local_bh_disable(); - for (i = 0; i < vport->num_q_vectors; i++) - napi_schedule(&vport->q_vectors[i].napi); - local_bh_enable(); - return idpf_wait_for_marker_event(vport); } From 707e479a02d47a36d7ad9299c8561e45db1f848c Mon Sep 17 00:00:00 2001 From: Michal Kubiak Date: Tue, 3 Oct 2023 11:29:25 +0200 Subject: [PATCH 38/49] idpf: prepare structures to support xdp Extend basic structures of the driver (e.g. 'idpf_vport', 'idpf_*_queue', 'idpf_vport_user_config_data') by adding members necessary to support XDP. Add extra XDP Tx queues needed to support XDP_TX and XDP_REDIRECT actions without interfering a regular Tx traffic. Also add functions dedicated to support XDP initialization for Rx and Tx queues and call those functions from the existing algorithms of queues configuration. Signed-off-by: Michal Kubiak Co-developed-by: Alexander Lobakin Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/idpf/Kconfig | 2 +- drivers/net/ethernet/intel/idpf/Makefile | 2 + drivers/net/ethernet/intel/idpf/idpf.h | 18 ++ .../net/ethernet/intel/idpf/idpf_ethtool.c | 6 +- drivers/net/ethernet/intel/idpf/idpf_lib.c | 23 +- drivers/net/ethernet/intel/idpf/idpf_main.c | 1 + .../ethernet/intel/idpf/idpf_singleq_txrx.c | 8 +- drivers/net/ethernet/intel/idpf/idpf_txrx.c | 170 +++++++++++---- drivers/net/ethernet/intel/idpf/idpf_txrx.h | 90 ++++++-- .../net/ethernet/intel/idpf/idpf_virtchnl.c | 64 ++++-- drivers/net/ethernet/intel/idpf/xdp.c | 197 ++++++++++++++++++ drivers/net/ethernet/intel/idpf/xdp.h | 17 ++ 12 files changed, 515 insertions(+), 83 deletions(-) create mode 100644 drivers/net/ethernet/intel/idpf/xdp.c create mode 100644 drivers/net/ethernet/intel/idpf/xdp.h diff --git a/drivers/net/ethernet/intel/idpf/Kconfig b/drivers/net/ethernet/intel/idpf/Kconfig index 1addd663acad98..7207ee4dbae8bb 100644 --- a/drivers/net/ethernet/intel/idpf/Kconfig +++ b/drivers/net/ethernet/intel/idpf/Kconfig @@ -5,7 +5,7 @@ config IDPF tristate "Intel(R) Infrastructure Data Path Function Support" depends on PCI_MSI select DIMLIB - select LIBETH + select LIBETH_XDP help This driver supports Intel(R) Infrastructure Data Path Function devices. diff --git a/drivers/net/ethernet/intel/idpf/Makefile b/drivers/net/ethernet/intel/idpf/Makefile index 2ce01a0b58981d..c58abe6f8f5d4e 100644 --- a/drivers/net/ethernet/intel/idpf/Makefile +++ b/drivers/net/ethernet/intel/idpf/Makefile @@ -17,3 +17,5 @@ idpf-y := \ idpf_vf_dev.o idpf-$(CONFIG_IDPF_SINGLEQ) += idpf_singleq_txrx.o + +idpf-y += xdp.o diff --git a/drivers/net/ethernet/intel/idpf/idpf.h b/drivers/net/ethernet/intel/idpf/idpf.h index c0109879d4db81..0481c07f3ef0c9 100644 --- a/drivers/net/ethernet/intel/idpf/idpf.h +++ b/drivers/net/ethernet/intel/idpf/idpf.h @@ -276,6 +276,10 @@ struct idpf_vport { struct idpf_tx_queue **txqs; bool crc_enable; + bool xdpq_share; + u16 num_xdp_txq; + u16 xdp_txq_offset; + u16 num_rxq; u16 num_bufq; u32 rxq_desc_count; @@ -358,8 +362,10 @@ struct idpf_vport_user_config_data { struct idpf_rss_data rss_data; u16 num_req_tx_qs; u16 num_req_rx_qs; + u16 num_req_xdp_qs; u32 num_req_txq_desc; u32 num_req_rxq_desc; + struct xdp_attachment_info xdp; DECLARE_BITMAP(user_flags, __IDPF_USER_FLAGS_NBITS); struct list_head mac_filter_list; }; @@ -573,6 +579,18 @@ static inline int idpf_is_queue_model_split(u16 q_model) q_model == VIRTCHNL2_QUEUE_MODEL_SPLIT; } +/** + * idpf_xdp_is_prog_ena - check if there is an XDP program on adapter + * @vport: vport to check + */ +static inline bool idpf_xdp_is_prog_ena(const struct idpf_vport *vport) +{ + if (!vport->adapter) + return false; + + return !!vport->adapter->vport_config[vport->idx]->user_config.xdp.prog; +} + #define idpf_is_cap_ena(adapter, field, flag) \ idpf_is_capability_ena(adapter, false, field, flag) #define idpf_is_cap_ena_all(adapter, field, flag) \ diff --git a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c index c08a137059e3c1..d528b421e701d3 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c +++ b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c @@ -188,9 +188,11 @@ static void idpf_get_channels(struct net_device *netdev, { struct idpf_netdev_priv *np = netdev_priv(netdev); struct idpf_vport_config *vport_config; + const struct idpf_vport *vport; u16 num_txq, num_rxq; u16 combined; + vport = idpf_netdev_to_vport(netdev); vport_config = np->adapter->vport_config[np->vport_idx]; num_txq = vport_config->user_config.num_req_tx_qs; @@ -204,8 +206,8 @@ static void idpf_get_channels(struct net_device *netdev, ch->max_rx = vport_config->max_q.max_rxq; ch->max_tx = vport_config->max_q.max_txq; - ch->max_other = IDPF_MAX_MBXQ; - ch->other_count = IDPF_MAX_MBXQ; + ch->max_other = IDPF_MAX_MBXQ + vport->num_xdp_txq; + ch->other_count = IDPF_MAX_MBXQ + vport->num_xdp_txq; ch->combined_count = combined; ch->rx_count = num_rxq - combined; diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index ee6768c1b6be42..a96a01fdb33737 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -5,6 +5,7 @@ #include "idpf.h" #include "idpf_virtchnl.h" +#include "xdp.h" static const struct net_device_ops idpf_netdev_ops; @@ -743,7 +744,8 @@ static int idpf_cfg_netdev(struct idpf_vport *vport) netdev = libeth_netdev_alloc(sizeof(struct idpf_netdev_priv), vport_config->max_q.max_rxq, - vport_config->max_q.max_txq); + vport_config->max_q.max_txq, + IDPF_LARGE_MAX_Q); if (!netdev) return -ENOMEM; @@ -902,6 +904,7 @@ static void idpf_vport_stop(struct idpf_vport *vport) vport->link_up = false; idpf_vport_intr_deinit(vport); + idpf_xdp_rxq_info_deinit_all(vport); idpf_vport_queues_rel(vport); idpf_vport_intr_rel(vport); np->state = __IDPF_VPORT_DOWN; @@ -1277,7 +1280,8 @@ static void idpf_restore_features(struct idpf_vport *vport) static int idpf_set_real_num_queues(struct idpf_vport *vport) { return libeth_set_real_num_queues(vport->netdev, vport->num_rxq, - vport->num_txq); + vport->num_txq - vport->num_xdp_txq, + vport->num_xdp_txq); } /** @@ -1390,20 +1394,29 @@ static int idpf_vport_open(struct idpf_vport *vport) } idpf_rx_init_buf_tail(vport); + + err = idpf_xdp_rxq_info_init_all(vport); + if (err) { + netdev_err(vport->netdev, + "Failed to initialize XDP RxQ info for vport %u: %pe\n", + vport->vport_id, ERR_PTR(err)); + goto intr_deinit; + } + idpf_vport_intr_ena(vport); err = idpf_send_config_queues_msg(vport); if (err) { dev_err(&adapter->pdev->dev, "Failed to configure queues for vport %u, %d\n", vport->vport_id, err); - goto intr_deinit; + goto rxq_deinit; } err = idpf_send_map_unmap_queue_vector_msg(vport, true); if (err) { dev_err(&adapter->pdev->dev, "Failed to map queue vectors for vport %u: %d\n", vport->vport_id, err); - goto intr_deinit; + goto rxq_deinit; } err = idpf_send_enable_queues_msg(vport); @@ -1451,6 +1464,8 @@ static int idpf_vport_open(struct idpf_vport *vport) idpf_send_disable_queues_msg(vport); unmap_queue_vectors: idpf_send_map_unmap_queue_vector_msg(vport, false); +rxq_deinit: + idpf_xdp_rxq_info_deinit_all(vport); intr_deinit: idpf_vport_intr_deinit(vport); queues_rel: diff --git a/drivers/net/ethernet/intel/idpf/idpf_main.c b/drivers/net/ethernet/intel/idpf/idpf_main.c index db476b3314c8a5..28a573b0d3e69d 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_main.c +++ b/drivers/net/ethernet/intel/idpf/idpf_main.c @@ -9,6 +9,7 @@ MODULE_DESCRIPTION(DRV_SUMMARY); MODULE_IMPORT_NS(LIBETH); +MODULE_IMPORT_NS(LIBETH_XDP); MODULE_LICENSE("GPL"); /** diff --git a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c index c6ce819b352894..35a0a3740bcf75 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c @@ -599,7 +599,7 @@ static void idpf_rx_singleq_csum(struct idpf_rx_queue *rxq, bool ipv4, ipv6; /* check if Rx checksum is enabled */ - if (!libeth_rx_pt_has_checksum(rxq->netdev, decoded)) + if (!libeth_rx_pt_has_checksum(rxq->xdp_rxq.dev, decoded)) goto none; /* check if HW has decoded the packet and checksum */ @@ -742,7 +742,7 @@ static void idpf_rx_singleq_base_hash(struct idpf_rx_queue *rx_q, { u64 mask, qw1; - if (!libeth_rx_pt_has_hash(rx_q->netdev, decoded)) + if (!libeth_rx_pt_has_hash(rx_q->xdp_rxq.dev, decoded)) return; mask = VIRTCHNL2_RX_BASE_DESC_FLTSTAT_RSS_HASH_M; @@ -770,7 +770,7 @@ static void idpf_rx_singleq_flex_hash(struct idpf_rx_queue *rx_q, const union virtchnl2_rx_desc *rx_desc, struct libeth_rx_pt decoded) { - if (!libeth_rx_pt_has_hash(rx_q->netdev, decoded)) + if (!libeth_rx_pt_has_hash(rx_q->xdp_rxq.dev, decoded)) return; if (FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_STATUS0_RSS_VALID_M, @@ -805,7 +805,7 @@ idpf_rx_singleq_process_skb_fields(struct idpf_rx_queue *rx_q, struct idpf_rx_csum_decoded csum_bits; /* modifies the skb - consumes the enet header */ - skb->protocol = eth_type_trans(skb, rx_q->netdev); + skb->protocol = eth_type_trans(skb, rx_q->xdp_rxq.dev); /* Check if we're using base mode descriptor IDs */ if (rx_q->rxdids == VIRTCHNL2_RXDID_1_32B_BASE_M) { diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 2d616078b6c201..f3146881200690 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -1,11 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (C) 2023 Intel Corporation */ -#include -#include +#include #include "idpf.h" #include "idpf_virtchnl.h" +#include "xdp.h" struct idpf_tx_stash { struct hlist_node hlist; @@ -69,6 +69,29 @@ void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue) } } +static void idpf_sq_stats_init(const struct idpf_vport *vport, + struct idpf_tx_queue *txq) +{ + if (idpf_queue_has(XDP, txq)) + libeth_xdpsq_stats_init(vport->netdev, &txq->xstats, + txq->idx - vport->xdp_txq_offset); + else + libeth_sq_stats_init(vport->netdev, &txq->stats, txq->idx); +} + +static void idpf_sq_stats_deinit(const struct idpf_vport *vport, + struct idpf_tx_queue *txq) +{ + if (idpf_queue_has(XDP, txq)) { + u32 idx = txq->idx - vport->xdp_txq_offset; + + libeth_xdpsq_stats_deinit(txq->netdev, idx); + } else { + libeth_sq_stats_deinit(txq->netdev, txq->idx); + netdev_tx_reset_subqueue(txq->netdev, txq->idx); + } +} + /** * idpf_tx_buf_rel_all - Free any empty Tx buffers * @txq: queue to be cleaned @@ -78,8 +101,10 @@ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) struct libeth_sq_napi_stats ss = { }; struct idpf_buf_lifo *buf_stack; struct idpf_tx_stash *stash; + struct xdp_frame_bulk bq; struct libeth_cq_pp cp = { .dev = txq->dev, + .bq = &bq, .ss = &ss, }; struct hlist_node *tmp; @@ -89,9 +114,15 @@ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) if (!txq->tx_buf) return; + xdp_frame_bulk_init(&bq); + rcu_read_lock(); + /* Free all the Tx buffer sk_buffs */ for (i = 0; i < txq->desc_count; i++) - libeth_tx_complete(&txq->tx_buf[i], &cp); + libeth_tx_complete_any(&txq->tx_buf[i], &cp); + + xdp_flush_frame_bulk(&bq); + rcu_read_unlock(); kfree(txq->tx_buf); txq->tx_buf = NULL; @@ -125,16 +156,16 @@ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) /** * idpf_tx_desc_rel - Free Tx resources per queue + * @vport: virtual port the queue belongs to * @txq: Tx descriptor ring for a specific queue * * Free all transmit software resources */ -static void idpf_tx_desc_rel(struct idpf_tx_queue *txq) +static void idpf_tx_desc_rel(const struct idpf_vport *vport, + struct idpf_tx_queue *txq) { idpf_tx_buf_rel_all(txq); - - libeth_sq_stats_deinit(txq->netdev, txq->idx); - netdev_tx_reset_subqueue(txq->netdev, txq->idx); + idpf_sq_stats_deinit(vport, txq); if (!txq->desc_ring) return; @@ -180,7 +211,7 @@ static void idpf_tx_desc_rel_all(struct idpf_vport *vport) struct idpf_txq_group *txq_grp = &vport->txq_grps[i]; for (j = 0; j < txq_grp->num_txq; j++) - idpf_tx_desc_rel(txq_grp->txqs[j]); + idpf_tx_desc_rel(vport, txq_grp->txqs[j]); if (idpf_is_queue_model_split(vport->txq_model)) idpf_compl_desc_rel(txq_grp->complq); @@ -267,12 +298,12 @@ static int idpf_tx_desc_alloc(const struct idpf_vport *vport, tx_q->next_to_clean = 0; idpf_queue_set(GEN_CHK, tx_q); - libeth_sq_stats_init(vport->netdev, &tx_q->stats, tx_q->idx); + idpf_sq_stats_init(vport, tx_q); return 0; err_alloc: - idpf_tx_desc_rel(tx_q); + idpf_tx_desc_rel(vport, tx_q); return err; } @@ -334,7 +365,8 @@ static int idpf_tx_desc_alloc_all(struct idpf_vport *vport) goto err_out; } - if (!idpf_is_queue_model_split(vport->txq_model)) + if (!idpf_is_queue_model_split(vport->txq_model) || + idpf_queue_has(XDP, txq)) continue; txq->compl_tag_cur_gen = 0; @@ -469,12 +501,13 @@ static void idpf_rx_buf_rel_all(struct idpf_rx_queue *rxq) /** * idpf_rx_desc_rel - Free a specific Rx q resources * @rxq: queue to clean the resources from - * @dev: device to free DMA memory + * @dev: &net_device to free DMA memory * @model: single or split queue model * * Free a specific rx queue resources */ -static void idpf_rx_desc_rel(struct idpf_rx_queue *rxq, struct device *dev, +static void idpf_rx_desc_rel(struct idpf_rx_queue *rxq, + const struct net_device *dev, u32 model) { if (!rxq) @@ -488,7 +521,7 @@ static void idpf_rx_desc_rel(struct idpf_rx_queue *rxq, struct device *dev, if (!idpf_is_queue_model_split(model)) idpf_rx_buf_rel_all(rxq); - libeth_rq_stats_deinit(rxq->netdev, rxq->idx); + libeth_rq_stats_deinit(dev, rxq->idx); rxq->next_to_alloc = 0; rxq->next_to_clean = 0; @@ -496,7 +529,8 @@ static void idpf_rx_desc_rel(struct idpf_rx_queue *rxq, struct device *dev, if (!rxq->desc_ring) return; - dmam_free_coherent(dev, rxq->size, rxq->desc_ring, rxq->dma); + dmam_free_coherent(dev->dev.parent, rxq->size, rxq->desc_ring, + rxq->dma); rxq->desc_ring = NULL; } @@ -532,7 +566,7 @@ static void idpf_rx_desc_rel_bufq(struct idpf_buf_queue *bufq, */ static void idpf_rx_desc_rel_all(struct idpf_vport *vport) { - struct device *dev = &vport->adapter->pdev->dev; + struct net_device *dev = vport->netdev; struct idpf_rxq_group *rx_qgrp; u16 num_rxq; int i, j; @@ -562,7 +596,8 @@ static void idpf_rx_desc_rel_all(struct idpf_vport *vport) struct idpf_bufq_set *bufq_set = &rx_qgrp->splitq.bufq_sets[j]; - idpf_rx_desc_rel_bufq(&bufq_set->bufq, dev); + idpf_rx_desc_rel_bufq(&bufq_set->bufq, + dev->dev.parent); } } } @@ -594,6 +629,7 @@ static int idpf_rx_hdr_buf_alloc_all(struct idpf_buf_queue *bufq) struct libeth_fq fq = { .count = bufq->desc_count, .type = LIBETH_FQE_HDR, + .xdp = idpf_xdp_is_prog_ena(bufq->q_vector->vport), .nid = idpf_q_vector_to_mem(bufq->q_vector), }; int ret; @@ -793,6 +829,7 @@ static int idpf_rx_bufs_init(struct idpf_buf_queue *bufq, .count = bufq->desc_count, .type = type, .hsplit = idpf_queue_has(HSPLIT_EN, bufq), + .xdp = idpf_xdp_is_prog_ena(bufq->q_vector->vport), .nid = idpf_q_vector_to_mem(bufq->q_vector), }; int ret; @@ -1100,6 +1137,8 @@ void idpf_vport_queues_rel(struct idpf_vport *vport) { idpf_tx_desc_rel_all(vport); idpf_rx_desc_rel_all(vport); + + idpf_vport_xdpq_put(vport); idpf_vport_queue_grp_rel_all(vport); kfree(vport->txqs); @@ -1165,6 +1204,19 @@ void idpf_vport_init_num_qs(struct idpf_vport *vport, if (idpf_is_queue_model_split(vport->rxq_model)) vport->num_bufq = le16_to_cpu(vport_msg->num_rx_bufq); + if (idpf_xdp_is_prog_ena(vport)) { + vport->xdp_txq_offset = config_data->num_req_tx_qs; + vport->num_xdp_txq = le16_to_cpu(vport_msg->num_tx_q) - + vport->xdp_txq_offset; + vport->xdpq_share = libeth_xdpsq_shared(vport->num_xdp_txq); + } else { + vport->xdp_txq_offset = 0; + vport->num_xdp_txq = 0; + vport->xdpq_share = false; + } + + config_data->num_req_xdp_qs = vport->num_xdp_txq; + /* Adjust number of buffer queues per Rx queue group. */ if (!idpf_is_queue_model_split(vport->rxq_model)) { vport->num_bufqs_per_qgrp = 0; @@ -1236,9 +1288,10 @@ int idpf_vport_calc_total_qs(struct idpf_adapter *adapter, u16 vport_idx, int dflt_splitq_txq_grps = 0, dflt_singleq_txqs = 0; int dflt_splitq_rxq_grps = 0, dflt_singleq_rxqs = 0; u16 num_req_tx_qs = 0, num_req_rx_qs = 0; + struct idpf_vport_user_config_data *user; struct idpf_vport_config *vport_config; u16 num_txq_grps, num_rxq_grps; - u32 num_qs; + u32 num_qs, num_xdpq; vport_config = adapter->vport_config[vport_idx]; if (vport_config) { @@ -1286,6 +1339,29 @@ int idpf_vport_calc_total_qs(struct idpf_adapter *adapter, u16 vport_idx, vport_msg->num_rx_bufq = 0; } + if (!vport_config) + return 0; + + user = &vport_config->user_config; + user->num_req_rx_qs = le16_to_cpu(vport_msg->num_rx_q); + user->num_req_tx_qs = le16_to_cpu(vport_msg->num_tx_q); + + if (vport_config->user_config.xdp.prog) + /* As we now know new number of Rx and Tx queues, we can + * request additional Tx queues for XDP. + */ + num_xdpq = libeth_xdpsq_num(user->num_req_rx_qs, + user->num_req_tx_qs, + IDPF_LARGE_MAX_Q); + else + num_xdpq = 0; + + user->num_req_xdp_qs = num_xdpq; + + vport_msg->num_tx_q = cpu_to_le16(user->num_req_tx_qs + num_xdpq); + if (idpf_is_queue_model_split(le16_to_cpu(vport_msg->txq_model))) + vport_msg->num_tx_complq = vport_msg->num_tx_q; + return 0; } @@ -1335,14 +1411,13 @@ static void idpf_vport_calc_numq_per_grp(struct idpf_vport *vport, static void idpf_rxq_set_descids(const struct idpf_vport *vport, struct idpf_rx_queue *q) { - if (idpf_is_queue_model_split(vport->rxq_model)) { - q->rxdids = VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M; - } else { - if (vport->base_rxd) - q->rxdids = VIRTCHNL2_RXDID_1_32B_BASE_M; - else - q->rxdids = VIRTCHNL2_RXDID_2_FLEX_SQ_NIC_M; - } + if (idpf_is_queue_model_split(vport->rxq_model)) + return; + + if (vport->base_rxd) + q->rxdids = VIRTCHNL2_RXDID_1_32B_BASE_M; + else + q->rxdids = VIRTCHNL2_RXDID_2_FLEX_SQ_NIC_M; } /** @@ -1558,7 +1633,6 @@ static int idpf_rxq_group_alloc(struct idpf_vport *vport, u16 num_rxq) setup_rxq: q->desc_count = vport->rxq_desc_count; q->rx_ptype_lkup = vport->rx_ptype_lkup; - q->netdev = vport->netdev; q->bufq_sets = rx_qgrp->splitq.bufq_sets; q->idx = (i * num_rxq) + j; q->rx_buffer_low_watermark = IDPF_LOW_WATERMARK; @@ -1619,15 +1693,19 @@ int idpf_vport_queues_alloc(struct idpf_vport *vport) if (err) goto err_out; - err = idpf_tx_desc_alloc_all(vport); + err = idpf_vport_init_fast_path_txqs(vport); if (err) goto err_out; - err = idpf_rx_desc_alloc_all(vport); + err = idpf_vport_xdpq_get(vport); if (err) goto err_out; - err = idpf_vport_init_fast_path_txqs(vport); + err = idpf_tx_desc_alloc_all(vport); + if (err) + goto err_out; + + err = idpf_rx_desc_alloc_all(vport); if (err) goto err_out; @@ -2157,16 +2235,24 @@ static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget, */ void idpf_wait_for_sw_marker_completion(struct idpf_tx_queue *txq) { - struct idpf_compl_queue *complq = txq->txq_grp->complq; struct idpf_splitq_4b_tx_compl_desc *tx_desc; - s16 ntc = complq->next_to_clean; + struct idpf_compl_queue *complq; unsigned long timeout; bool flow, gen_flag; - u32 pos = ntc; + u32 pos; + s16 ntc; if (!idpf_queue_has(SW_MARKER, txq)) return; + if (idpf_queue_has(XDP, txq)) + complq = txq->complq; + else + complq = txq->txq_grp->complq; + + ntc = complq->next_to_clean; + pos = ntc; + flow = idpf_queue_has(FLOW_SCH_EN, complq); gen_flag = idpf_queue_has(GEN_CHK, complq); @@ -2941,10 +3027,11 @@ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, */ netdev_tx_t idpf_tx_start(struct sk_buff *skb, struct net_device *netdev) { - struct idpf_vport *vport = idpf_netdev_to_vport(netdev); + const struct idpf_vport *vport = idpf_netdev_to_vport(netdev); struct idpf_tx_queue *tx_q; - if (unlikely(skb_get_queue_mapping(skb) >= vport->num_txq)) { + if (unlikely(skb_get_queue_mapping(skb) >= + vport->num_txq - vport->num_xdp_txq)) { dev_kfree_skb_any(skb); return NETDEV_TX_OK; @@ -2981,7 +3068,7 @@ idpf_rx_hash(const struct idpf_rx_queue *rxq, struct sk_buff *skb, { u32 hash; - if (!libeth_rx_pt_has_hash(rxq->netdev, decoded)) + if (!libeth_rx_pt_has_hash(rxq->xdp_rxq.dev, decoded)) return; hash = le16_to_cpu(rx_desc->hash1) | @@ -3009,7 +3096,7 @@ static void idpf_rx_csum(struct idpf_rx_queue *rxq, struct sk_buff *skb, bool ipv4, ipv6; /* check if Rx checksum is enabled */ - if (!libeth_rx_pt_has_checksum(rxq->netdev, decoded)) + if (!libeth_rx_pt_has_checksum(rxq->xdp_rxq.dev, decoded)) goto none; /* check if HW has decoded the packet and checksum */ @@ -3185,7 +3272,7 @@ idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, /* process RSS/hash */ idpf_rx_hash(rxq, skb, rx_desc, decoded); - skb->protocol = eth_type_trans(skb, rxq->netdev); + skb->protocol = eth_type_trans(skb, rxq->xdp_rxq.dev); if (le16_get_bits(rx_desc->hdrlen_flags, VIRTCHNL2_RX_FLEX_DESC_ADV_RSC_M)) @@ -4159,6 +4246,13 @@ static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) struct idpf_txq_group *tx_qgrp; u32 i, qv_idx, q_index; + /* XDP Tx queues are handled within Rx loop, correct num_txq_grp so + * that it stores number of regular Tx queue groups. This way when we + * later assign Tx to qvector, we go only through regular Tx queues. + */ + if (idpf_xdp_is_prog_ena(vport)) + num_txq_grp -= vport->num_xdp_txq; + for (i = 0, qv_idx = 0; i < vport->num_rxq_grp; i++) { u16 num_rxq; diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index e4c3c802b3f08a..b862c16caf20fc 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -310,6 +310,8 @@ struct idpf_ptype_state { * @__IDPF_Q_SW_MARKER: Used to indicate TX queue marker completions * @__IDPF_Q_CRC_EN: enable CRC offload in singleq mode * @__IDPF_Q_HSPLIT_EN: enable header split on Rx (splitq) + * @__IDPF_Q_NOIRQ: queue is polling-driven and has no interrupt + * @__IDPF_Q_XDP: this is an XDP queue * @__IDPF_Q_FLAGS_NBITS: Must be last */ enum idpf_queue_flags_t { @@ -319,6 +321,8 @@ enum idpf_queue_flags_t { __IDPF_Q_SW_MARKER, __IDPF_Q_CRC_EN, __IDPF_Q_HSPLIT_EN, + __IDPF_Q_NOIRQ, + __IDPF_Q_XDP, __IDPF_Q_FLAGS_NBITS, }; @@ -476,18 +480,20 @@ struct idpf_txq_stash { * @napi: NAPI instance corresponding to this queue (splitq) * @rx_buf: See struct &libeth_fqe * @pp: Page pool pointer in singleq mode - * @netdev: &net_device corresponding to this queue * @tail: Tail offset. Used for both queue models single and split. * @flags: See enum idpf_queue_flags_t * @idx: For RX queue, it is used to index to total RX queue across groups and * used for skb reporting. * @desc_count: Number of descriptors + * @num_xdp_txq: total number of XDP Tx queues + * @xdpqs: shortcut for XDP Tx queues array * @rxdids: Supported RX descriptor ids + * @truesize: data buffer truesize in singleq * @rx_ptype_lkup: LUT of Rx ptypes + * @xdp_rxq: XDP queue info * @next_to_use: Next descriptor to use * @next_to_clean: Next descriptor to clean * @next_to_alloc: RX buffer to allocate at - * @truesize: data buffer truesize in singleq * @skb: Pointer to the skb * @stats: per-queue RQ stats * @q_id: Queue id @@ -517,15 +523,23 @@ struct idpf_rx_queue { struct page_pool *pp; }; }; - struct net_device *netdev; void __iomem *tail; DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); u16 idx; u16 desc_count; - u32 rxdids; + u32 num_xdp_txq; + union { + struct idpf_tx_queue **xdpqs; + struct { + u32 rxdids; + u32 truesize; + }; + }; const struct libeth_rx_pt *rx_ptype_lkup; + + struct xdp_rxq_info xdp_rxq; __cacheline_group_end_aligned(read_mostly); __cacheline_group_begin_aligned(read_write); @@ -533,7 +547,6 @@ struct idpf_rx_queue { u32 next_to_clean; u32 next_to_alloc; - u32 truesize; struct sk_buff *skb; struct libeth_rq_stats stats; @@ -552,7 +565,9 @@ struct idpf_rx_queue { u16 rx_max_pkt_size; __cacheline_group_end_aligned(cold); }; -libeth_cacheline_set_assert(struct idpf_rx_queue, 64, +libeth_cacheline_set_assert(struct idpf_rx_queue, + ALIGN(64, __alignof(struct xdp_rxq_info)) + + sizeof(struct xdp_rxq_info), 32 + sizeof(struct libeth_rq_stats), 32); @@ -565,6 +580,7 @@ libeth_cacheline_set_assert(struct idpf_rx_queue, 64, * @desc_ring: virtual descriptor ring address * @tx_buf: See struct idpf_tx_buf * @txq_grp: See struct idpf_txq_group + * @complq: corresponding completion queue in XDP mode * @dev: Device back pointer for DMA mapping * @tail: Tail offset. Used for both queue models single and split * @flags: See enum idpf_queue_flags_t @@ -592,6 +608,7 @@ libeth_cacheline_set_assert(struct idpf_rx_queue, 64, * -------------------------------- * * This gives us 8*8160 = 65280 possible unique values. + * @thresh: XDP queue cleaning threshold * @netdev: &net_device corresponding to this queue * @next_to_use: Next descriptor to use * @next_to_clean: Next descriptor to clean @@ -610,7 +627,12 @@ libeth_cacheline_set_assert(struct idpf_rx_queue, 64, * @compl_tag_bufid_m: Completion tag buffer id mask * @compl_tag_cur_gen: Used to keep track of current completion tag generation * @compl_tag_gen_max: To determine when compl_tag_cur_gen should be reset + * @pending: number of pending descriptors to send in QB + * @xdp_tx: number of pending &xdp_buff or &xdp_frame buffers + * @timer: timer for XDP Tx queue cleanup + * @xdp_lock: lock for XDP Tx queues sharing * @stats: per-queue SQ stats + * @xstats: per-queue XDPSQ stats * @q_id: Queue id * @size: Length of descriptor ring in bytes * @dma: Physical address of ring @@ -627,7 +649,10 @@ struct idpf_tx_queue { void *desc_ring; }; struct libeth_sqe *tx_buf; - struct idpf_txq_group *txq_grp; + union { + struct idpf_txq_group *txq_grp; + struct idpf_compl_queue *complq; + }; struct device *dev; void __iomem *tail; @@ -635,8 +660,13 @@ struct idpf_tx_queue { u16 idx; u16 desc_count; - u16 tx_min_pkt_len; - u16 compl_tag_gen_s; + union { + struct { + u16 tx_min_pkt_len; + u16 compl_tag_gen_s; + }; + u32 thresh; + }; struct net_device *netdev; __cacheline_group_end_aligned(read_mostly); @@ -646,19 +676,33 @@ struct idpf_tx_queue { u16 next_to_clean; union { - u32 cleaned_bytes; - u32 clean_budget; - }; - u16 cleaned_pkts; - - u16 tx_max_bufs; - struct idpf_txq_stash *stash; + struct { + union { + u32 cleaned_bytes; + u32 clean_budget; + }; + u16 cleaned_pkts; + + u16 tx_max_bufs; + struct idpf_txq_stash *stash; + + u16 compl_tag_bufid_m; + u16 compl_tag_cur_gen; + u16 compl_tag_gen_max; + }; + struct { + u32 pending; + u32 xdp_tx; - u16 compl_tag_bufid_m; - u16 compl_tag_cur_gen; - u16 compl_tag_gen_max; + struct libeth_xdpsq_timer *timer; + struct libeth_xdpsq_lock xdp_lock; + }; + }; - struct libeth_sq_stats stats; + union { + struct libeth_sq_stats stats; + struct libeth_xdpsq_stats xstats; + }; __cacheline_group_end_aligned(read_write); __cacheline_group_begin_aligned(cold); @@ -669,8 +713,12 @@ struct idpf_tx_queue { struct idpf_q_vector *q_vector; __cacheline_group_end_aligned(cold); }; +/* sizeof(spinlock_t) can take many values depending on the config, don't + * hardcode any assumptions. + */ libeth_cacheline_set_assert(struct idpf_tx_queue, 64, - 32 + sizeof(struct libeth_sq_stats), + 4 + offsetofend(struct idpf_tx_queue, stats) - + offsetofend(struct idpf_tx_queue, next_to_clean), 24); /** diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c index dba0ce23af6687..7bbab6671322df 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c @@ -1602,9 +1602,12 @@ static int idpf_send_config_rx_queues_msg(struct idpf_vport *vport) for (j = 0; j < num_rxq; j++, k++) { const struct idpf_bufq_set *sets; struct idpf_rx_queue *rxq; + u32 rxdids; if (!idpf_is_queue_model_split(vport->rxq_model)) { rxq = rx_qgrp->singleq.rxqs[j]; + rxdids = rxq->rxdids; + goto common_qi_fields; } @@ -1637,6 +1640,8 @@ static int idpf_send_config_rx_queues_msg(struct idpf_vport *vport) cpu_to_le16(rxq->rx_hbuf_size); } + rxdids = VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M; + common_qi_fields: qi[k].queue_id = cpu_to_le32(rxq->q_id); qi[k].model = cpu_to_le16(vport->rxq_model); @@ -1647,7 +1652,7 @@ static int idpf_send_config_rx_queues_msg(struct idpf_vport *vport) qi[k].data_buffer_size = cpu_to_le32(rxq->rx_buf_size); qi[k].qflags |= cpu_to_le16(VIRTCHNL2_RX_DESC_SIZE_32BYTE); - qi[k].desc_ids = cpu_to_le64(rxq->rxdids); + qi[k].desc_ids = cpu_to_le64(rxdids); } } @@ -1871,21 +1876,33 @@ int idpf_send_map_unmap_queue_vector_msg(struct idpf_vport *vport, bool map) struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; for (j = 0; j < tx_qgrp->num_txq; j++, k++) { + const struct idpf_tx_queue *txq = tx_qgrp->txqs[j]; + const struct idpf_q_vector *vec; + u32 v_idx, tx_itr_idx; + vqv[k].queue_type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); - vqv[k].queue_id = cpu_to_le32(tx_qgrp->txqs[j]->q_id); + vqv[k].queue_id = cpu_to_le32(txq->q_id); + + if (idpf_queue_has(NOIRQ, txq)) + vec = NULL; + else if (idpf_queue_has(XDP, txq)) + vec = txq->complq->q_vector; + else if (idpf_is_queue_model_split(vport->txq_model)) + vec = txq->txq_grp->complq->q_vector; + else + vec = txq->q_vector; - if (idpf_is_queue_model_split(vport->txq_model)) { - vqv[k].vector_id = - cpu_to_le16(tx_qgrp->complq->q_vector->v_idx); - vqv[k].itr_idx = - cpu_to_le32(tx_qgrp->complq->q_vector->tx_itr_idx); + if (vec) { + v_idx = vec->v_idx; + tx_itr_idx = vec->tx_itr_idx; } else { - vqv[k].vector_id = - cpu_to_le16(tx_qgrp->txqs[j]->q_vector->v_idx); - vqv[k].itr_idx = - cpu_to_le32(tx_qgrp->txqs[j]->q_vector->tx_itr_idx); + v_idx = 0; + tx_itr_idx = VIRTCHNL2_ITR_IDX_1; } + + vqv[k].vector_id = cpu_to_le16(v_idx); + vqv[k].itr_idx = cpu_to_le32(tx_itr_idx); } } @@ -1903,6 +1920,7 @@ int idpf_send_map_unmap_queue_vector_msg(struct idpf_vport *vport, bool map) for (j = 0; j < num_rxq; j++, k++) { struct idpf_rx_queue *rxq; + u32 v_idx, rx_itr_idx; if (idpf_is_queue_model_split(vport->rxq_model)) rxq = &rx_qgrp->splitq.rxq_sets[j]->rxq; @@ -1912,8 +1930,17 @@ int idpf_send_map_unmap_queue_vector_msg(struct idpf_vport *vport, bool map) vqv[k].queue_type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); vqv[k].queue_id = cpu_to_le32(rxq->q_id); - vqv[k].vector_id = cpu_to_le16(rxq->q_vector->v_idx); - vqv[k].itr_idx = cpu_to_le32(rxq->q_vector->rx_itr_idx); + + if (idpf_queue_has(NOIRQ, rxq)) { + v_idx = 0; + rx_itr_idx = VIRTCHNL2_ITR_IDX_0; + } else { + v_idx = rxq->q_vector->v_idx; + rx_itr_idx = rxq->q_vector->rx_itr_idx; + } + + vqv[k].vector_id = cpu_to_le16(v_idx); + vqv[k].itr_idx = cpu_to_le32(rx_itr_idx); } } @@ -3104,6 +3131,17 @@ int idpf_vport_alloc_vec_indexes(struct idpf_vport *vport) vec_info.default_vport = vport->default_vport; vec_info.index = vport->idx; + /* Additional XDP Tx queues share the q_vector with regular Tx and Rx + * queues to which they are assigned. Also, XDP shall request additional + * Tx queues via VIRTCHNL. Therefore, to avoid exceeding over + * "vport->q_vector_idxs array", do not request empty q_vectors + * for XDP Tx queues. + */ + if (idpf_xdp_is_prog_ena(vport)) + vec_info.num_req_vecs = max_t(u16, + vport->num_txq - vport->num_xdp_txq, + vport->num_rxq); + num_alloc_vecs = idpf_req_rel_vector_indexes(vport->adapter, vport->q_vector_idxs, &vec_info); diff --git a/drivers/net/ethernet/intel/idpf/xdp.c b/drivers/net/ethernet/intel/idpf/xdp.c new file mode 100644 index 00000000000000..c25f14dbcac026 --- /dev/null +++ b/drivers/net/ethernet/intel/idpf/xdp.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2024 Intel Corporation */ + +#include + +#include "idpf.h" +#include "xdp.h" + +static int idpf_rxq_for_each(const struct idpf_vport *vport, + int (*fn)(struct idpf_rx_queue *rxq, void *arg), + void *arg) +{ + bool splitq = idpf_is_queue_model_split(vport->rxq_model); + + if (!vport->rxq_grps) + return -ENETDOWN; + + for (u32 i = 0; i < vport->num_rxq_grp; i++) { + const struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; + u32 num_rxq; + + if (splitq) + num_rxq = rx_qgrp->splitq.num_rxq_sets; + else + num_rxq = rx_qgrp->singleq.num_rxq; + + for (u32 j = 0; j < num_rxq; j++) { + struct idpf_rx_queue *q; + int err; + + if (splitq) + q = &rx_qgrp->splitq.rxq_sets[j]->rxq; + else + q = rx_qgrp->singleq.rxqs[j]; + + err = fn(q, arg); + if (err) + return err; + } + } + + return 0; +} + +/** + * __idpf_xdp_rxq_info_init - Setup XDP RxQ info for a given Rx queue + * @rxq: Rx queue for which the resources are setup + * @arg: flag indicating if the HW works in split queue mode + * + * Return: 0 on success, negative on failure. + */ +static int __idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq, void *arg) +{ + const struct idpf_vport *vport = rxq->q_vector->vport; + bool split = idpf_is_queue_model_split(vport->rxq_model); + const struct page_pool *pp; + int err; + + err = __xdp_rxq_info_reg(&rxq->xdp_rxq, vport->netdev, rxq->idx, + rxq->q_vector->napi.napi_id, + rxq->rx_buf_size); + if (err) + return err; + + pp = split ? rxq->bufq_sets[0].bufq.pp : rxq->pp; + xdp_rxq_info_attach_page_pool(&rxq->xdp_rxq, pp); + + if (!split) + return 0; + + rxq->xdpqs = &vport->txqs[vport->xdp_txq_offset]; + rxq->num_xdp_txq = vport->num_xdp_txq; + + return 0; +} + +/** + * idpf_xdp_rxq_info_init_all - initialize RxQ info for all Rx queues in vport + * @vport: vport to setup the info + * + * Return: 0 on success, negative on failure. + */ +int idpf_xdp_rxq_info_init_all(const struct idpf_vport *vport) +{ + return idpf_rxq_for_each(vport, __idpf_xdp_rxq_info_init, NULL); +} + +/** + * __idpf_xdp_rxq_info_deinit - Deinit XDP RxQ info for a given Rx queue + * @rxq: Rx queue for which the resources are destroyed + * @arg: flag indicating if the HW works in split queue mode + * + * Return: always 0. + */ +static int __idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, void *arg) +{ + if (idpf_is_queue_model_split((size_t)arg)) { + rxq->xdpqs = NULL; + rxq->num_xdp_txq = 0; + } + + xdp_rxq_info_detach_mem_model(&rxq->xdp_rxq); + xdp_rxq_info_unreg(&rxq->xdp_rxq); + + return 0; +} + +/** + * idpf_xdp_rxq_info_deinit_all - deinit RxQ info for all Rx queues in vport + * @vport: vport to setup the info + */ +void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport) +{ + idpf_rxq_for_each(vport, __idpf_xdp_rxq_info_deinit, + (void *)(size_t)vport->rxq_model); +} + +int idpf_vport_xdpq_get(const struct idpf_vport *vport) +{ + struct libeth_xdpsq_timer **timers __free(kvfree) = NULL; + struct net_device *dev; + u32 sqs; + + if (!idpf_xdp_is_prog_ena(vport)) + return 0; + + timers = kvcalloc(vport->num_xdp_txq, sizeof(*timers), GFP_KERNEL); + if (!timers) + return -ENOMEM; + + for (u32 i = 0; i < vport->num_xdp_txq; i++) { + timers[i] = kzalloc_node(sizeof(*timers[i]), GFP_KERNEL, + cpu_to_mem(i)); + if (!timers[i]) { + for (int j = i - 1; j >= 0; j--) + kfree(timers[j]); + + return -ENOMEM; + } + } + + dev = vport->netdev; + sqs = vport->xdp_txq_offset; + + libeth_xdpsq_get_start(); + + for (u32 i = sqs; i < vport->num_txq; i++) { + struct idpf_tx_queue *xdpq = vport->txqs[i]; + + xdpq->complq = xdpq->txq_grp->complq; + + idpf_queue_clear(FLOW_SCH_EN, xdpq); + idpf_queue_clear(FLOW_SCH_EN, xdpq->complq); + idpf_queue_set(NOIRQ, xdpq); + idpf_queue_set(XDP, xdpq); + idpf_queue_set(XDP, xdpq->complq); + + xdpq->timer = timers[i - sqs]; + libeth_xdpsq_get(&xdpq->xdp_lock, dev, vport->xdpq_share); + + xdpq->pending = 0; + xdpq->xdp_tx = 0; + xdpq->thresh = libeth_xdp_queue_threshold(xdpq->desc_count); + } + + libeth_xdpsq_get_end(); + + return 0; +} + +void idpf_vport_xdpq_put(const struct idpf_vport *vport) +{ + struct net_device *dev; + u32 sqs; + + if (!idpf_xdp_is_prog_ena(vport)) + return; + + dev = vport->netdev; + sqs = vport->xdp_txq_offset; + + libeth_xdpsq_get_start(); + + for (u32 i = sqs; i < vport->num_txq; i++) { + struct idpf_tx_queue *xdpq = vport->txqs[i]; + + if (!idpf_queue_has_clear(XDP, xdpq)) + continue; + + libeth_xdpsq_put(&xdpq->xdp_lock, dev); + + kfree(xdpq->timer); + idpf_queue_clear(NOIRQ, xdpq); + } + + libeth_xdpsq_get_end(); +} diff --git a/drivers/net/ethernet/intel/idpf/xdp.h b/drivers/net/ethernet/intel/idpf/xdp.h new file mode 100644 index 00000000000000..8ace8384f34842 --- /dev/null +++ b/drivers/net/ethernet/intel/idpf/xdp.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2024 Intel Corporation */ + +#ifndef _IDPF_XDP_H_ +#define _IDPF_XDP_H_ + +#include + +struct idpf_vport; + +int idpf_xdp_rxq_info_init_all(const struct idpf_vport *vport); +void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport); + +int idpf_vport_xdpq_get(const struct idpf_vport *vport); +void idpf_vport_xdpq_put(const struct idpf_vport *vport); + +#endif /* _IDPF_XDP_H_ */ From 8a6f7e052799cce861f0878dc6d6b8bd5871aec1 Mon Sep 17 00:00:00 2001 From: Michal Kubiak Date: Wed, 4 Oct 2023 17:49:07 +0200 Subject: [PATCH 39/49] idpf: implement XDP_SETUP_PROG in ndo_bpf for splitq Implement loading the XDP program using ndo_bpf callback for splitq and XDP_SETUP_PROG parameter. Add functions for stopping, reconfiguring and restarting all queues when needed. Also, implement the XDP hot swap mechanism when the existing XDP program is replaced by another one (without a necessity of reconfiguring anything). Signed-off-by: Michal Kubiak Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/idpf/idpf_lib.c | 1 + drivers/net/ethernet/intel/idpf/idpf_txrx.c | 6 ++ drivers/net/ethernet/intel/idpf/idpf_txrx.h | 4 +- drivers/net/ethernet/intel/idpf/xdp.c | 92 +++++++++++++++++++++ drivers/net/ethernet/intel/idpf/xdp.h | 7 ++ 5 files changed, 109 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index a96a01fdb33737..335b0deb26f116 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -2332,4 +2332,5 @@ static const struct net_device_ops idpf_netdev_ops = { .ndo_get_stats64 = idpf_get_stats64, .ndo_set_features = idpf_set_features, .ndo_tx_timeout = idpf_tx_timeout, + .ndo_bpf = idpf_xdp, }; diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index f3146881200690..b485c8578eda16 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -1135,6 +1135,8 @@ static void idpf_vport_queue_grp_rel_all(struct idpf_vport *vport) */ void idpf_vport_queues_rel(struct idpf_vport *vport) { + idpf_copy_xdp_prog_to_qs(vport, NULL); + idpf_tx_desc_rel_all(vport); idpf_rx_desc_rel_all(vport); @@ -1687,6 +1689,7 @@ static int idpf_vport_queue_grp_alloc_all(struct idpf_vport *vport) */ int idpf_vport_queues_alloc(struct idpf_vport *vport) { + struct bpf_prog *prog; int err; err = idpf_vport_queue_grp_alloc_all(vport); @@ -1709,6 +1712,9 @@ int idpf_vport_queues_alloc(struct idpf_vport *vport) if (err) goto err_out; + prog = vport->adapter->vport_config[vport->idx]->user_config.xdp.prog; + idpf_copy_xdp_prog_to_qs(vport, prog); + return 0; err_out: diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index b862c16caf20fc..8597307acead74 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -478,6 +478,7 @@ struct idpf_txq_stash { * @desc_ring: virtual descriptor ring address * @bufq_sets: Pointer to the array of buffer queues in splitq mode * @napi: NAPI instance corresponding to this queue (splitq) + * @xdp_prog: attached XDP program * @rx_buf: See struct &libeth_fqe * @pp: Page pool pointer in singleq mode * @tail: Tail offset. Used for both queue models single and split. @@ -517,13 +518,14 @@ struct idpf_rx_queue { struct { struct idpf_bufq_set *bufq_sets; struct napi_struct *napi; + struct bpf_prog __rcu *xdp_prog; }; struct { struct libeth_fqe *rx_buf; struct page_pool *pp; + void __iomem *tail; }; }; - void __iomem *tail; DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); u16 idx; diff --git a/drivers/net/ethernet/intel/idpf/xdp.c b/drivers/net/ethernet/intel/idpf/xdp.c index c25f14dbcac026..9b656b98dbe76b 100644 --- a/drivers/net/ethernet/intel/idpf/xdp.c +++ b/drivers/net/ethernet/intel/idpf/xdp.c @@ -4,6 +4,7 @@ #include #include "idpf.h" +#include "idpf_virtchnl.h" #include "xdp.h" static int idpf_rxq_for_each(const struct idpf_vport *vport, @@ -115,6 +116,33 @@ void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport) (void *)(size_t)vport->rxq_model); } +static int idpf_xdp_rxq_assign_prog(struct idpf_rx_queue *rxq, void *arg) +{ + struct mutex *lock = &rxq->q_vector->vport->adapter->vport_ctrl_lock; + struct bpf_prog *prog = arg; + struct bpf_prog *old; + + if (prog) + bpf_prog_inc(prog); + + old = rcu_replace_pointer(rxq->xdp_prog, prog, lockdep_is_held(lock)); + if (old) + bpf_prog_put(old); + + return 0; +} + +/** + * idpf_copy_xdp_prog_to_qs - set pointers to xdp program for each Rx queue + * @vport: vport to setup XDP for + * @xdp_prog: XDP program that should be copied to all Rx queues + */ +void idpf_copy_xdp_prog_to_qs(const struct idpf_vport *vport, + struct bpf_prog *xdp_prog) +{ + idpf_rxq_for_each(vport, idpf_xdp_rxq_assign_prog, xdp_prog); +} + int idpf_vport_xdpq_get(const struct idpf_vport *vport) { struct libeth_xdpsq_timer **timers __free(kvfree) = NULL; @@ -195,3 +223,67 @@ void idpf_vport_xdpq_put(const struct idpf_vport *vport) libeth_xdpsq_get_end(); } + +/** + * idpf_xdp_setup_prog - Add or remove XDP eBPF program + * @vport: vport to setup XDP for + * @xdp: XDP program and extack + */ +static int +idpf_xdp_setup_prog(struct idpf_vport *vport, struct netdev_bpf *xdp) +{ + struct bpf_prog *prog = xdp->prog; + struct xdp_attachment_info *info; + bool reconfig; + int ret; + + info = &vport->adapter->vport_config[vport->idx]->user_config.xdp; + reconfig = !!info->prog != !!prog; + + xdp_attachment_setup(info, xdp); + + if (!reconfig) { + idpf_copy_xdp_prog_to_qs(vport, prog); + return 0; + } + + ret = idpf_initiate_soft_reset(vport); + if (ret) { + NL_SET_ERR_MSG_MOD(xdp->extack, + "Could not reopen the vport after XDP setup"); + return ret; + } + + return 0; +} + +/** + * idpf_xdp - implements XDP handler + * @dev: netdevice + * @xdp: XDP command + */ +int idpf_xdp(struct net_device *dev, struct netdev_bpf *xdp) +{ + struct idpf_vport *vport; + int ret; + + idpf_vport_ctrl_lock(dev); + vport = idpf_netdev_to_vport(dev); + + if (!idpf_is_queue_model_split(vport->txq_model)) + goto notsupp; + + switch (xdp->command) { + case XDP_SETUP_PROG: + ret = idpf_xdp_setup_prog(vport, xdp); + break; + default: +notsupp: + ret = -EOPNOTSUPP; + break; + } + + idpf_vport_ctrl_unlock(dev); + + return ret; +} diff --git a/drivers/net/ethernet/intel/idpf/xdp.h b/drivers/net/ethernet/intel/idpf/xdp.h index 8ace8384f34842..a72a7638a6ea5c 100644 --- a/drivers/net/ethernet/intel/idpf/xdp.h +++ b/drivers/net/ethernet/intel/idpf/xdp.h @@ -6,12 +6,19 @@ #include +struct bpf_prog; struct idpf_vport; +struct net_device; +struct netdev_bpf; int idpf_xdp_rxq_info_init_all(const struct idpf_vport *vport); void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport); +void idpf_copy_xdp_prog_to_qs(const struct idpf_vport *vport, + struct bpf_prog *xdp_prog); int idpf_vport_xdpq_get(const struct idpf_vport *vport); void idpf_vport_xdpq_put(const struct idpf_vport *vport); +int idpf_xdp(struct net_device *dev, struct netdev_bpf *xdp); + #endif /* _IDPF_XDP_H_ */ From 7aa5b1a79d2a46e6a0835e66b3c14cd476615ebd Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Wed, 6 Dec 2023 18:22:13 +0100 Subject: [PATCH 40/49] idpf: use generic functions to build xdp_buff and skb In preparation of XDP support, move from having skb as the main frame container during the Rx polling to &xdp_buff. This allows to use generic and libie helpers for building an XDP buffer and changes the logics: now we try to allocate an skb only when we processed all the descriptors related to the frame. Store &libeth_xdp_stash instead of the skb pointer on the Rx queue. It's only 8 bytes wider and there's a place to fit it in. Signed-off-by: Alexander Lobakin --- .../ethernet/intel/idpf/idpf_singleq_txrx.c | 93 ++++++------ drivers/net/ethernet/intel/idpf/idpf_txrx.c | 142 +++++------------- drivers/net/ethernet/intel/idpf/idpf_txrx.h | 7 +- 3 files changed, 79 insertions(+), 163 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c index 35a0a3740bcf75..2e5a19f83d4873 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c @@ -1,8 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (C) 2023 Intel Corporation */ -#include -#include +#include #include "idpf.h" @@ -782,7 +781,7 @@ static void idpf_rx_singleq_flex_hash(struct idpf_rx_queue *rx_q, } /** - * idpf_rx_singleq_process_skb_fields - Populate skb header fields from Rx + * __idpf_rx_singleq_process_skb_fields - Populate skb header fields from Rx * descriptor * @rx_q: Rx ring being processed * @skb: pointer to current skb being populated @@ -795,18 +794,15 @@ static void idpf_rx_singleq_flex_hash(struct idpf_rx_queue *rx_q, * other fields within the skb. */ static void -idpf_rx_singleq_process_skb_fields(struct idpf_rx_queue *rx_q, - struct sk_buff *skb, - const union virtchnl2_rx_desc *rx_desc, - u16 ptype, - struct libeth_rq_napi_stats *rs) +__idpf_rx_singleq_process_skb_fields(struct idpf_rx_queue *rx_q, + struct sk_buff *skb, + const union virtchnl2_rx_desc *rx_desc, + u16 ptype, + struct libeth_rq_napi_stats *rs) { struct libeth_rx_pt decoded = rx_q->rx_ptype_lkup[ptype]; struct idpf_rx_csum_decoded csum_bits; - /* modifies the skb - consumes the enet header */ - skb->protocol = eth_type_trans(skb, rx_q->xdp_rxq.dev); - /* Check if we're using base mode descriptor IDs */ if (rx_q->rxdids == VIRTCHNL2_RXDID_1_32B_BASE_M) { idpf_rx_singleq_base_hash(rx_q, skb, rx_desc, decoded); @@ -817,7 +813,6 @@ idpf_rx_singleq_process_skb_fields(struct idpf_rx_queue *rx_q, } idpf_rx_singleq_csum(rx_q, skb, csum_bits, decoded, rs); - skb_record_rx_queue(skb, rx_q->idx); } /** @@ -953,6 +948,32 @@ idpf_rx_singleq_extract_fields(const struct idpf_rx_queue *rx_q, idpf_rx_singleq_extract_flex_fields(rx_desc, fields); } +static bool +idpf_rx_singleq_process_skb_fields(struct sk_buff *skb, + const struct libeth_xdp_buff *xdp, + struct libeth_rq_napi_stats *rs) +{ + struct idpf_rx_extracted fields; + struct idpf_rx_queue *rxq; + + rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq); + + idpf_rx_singleq_extract_fields(rxq, xdp->desc, &fields); + __idpf_rx_singleq_process_skb_fields(rxq, skb, xdp->desc, + fields.rx_ptype, rs); + + return true; +} + +static void idpf_xdp_run_pass(struct libeth_xdp_buff *xdp, + struct napi_struct *napi, + struct libeth_rq_napi_stats *rs, + const union virtchnl2_rx_desc *desc) +{ + libeth_xdp_run_pass(xdp, NULL, napi, rs, desc, NULL, + idpf_rx_singleq_process_skb_fields); +} + /** * idpf_rx_singleq_clean - Reclaim resources after receive completes * @rx_q: rx queue to clean @@ -963,11 +984,13 @@ idpf_rx_singleq_extract_fields(const struct idpf_rx_queue *rx_q, static int idpf_rx_singleq_clean(struct idpf_rx_queue *rx_q, int budget) { struct libeth_rq_napi_stats rs = { }; - struct sk_buff *skb = rx_q->skb; u16 ntc = rx_q->next_to_clean; + LIBETH_XDP_ONSTACK_BUFF(xdp); u16 cleaned_count = 0; bool failure = false; + libeth_xdp_init_buff(xdp, &rx_q->xdp, &rx_q->xdp_rxq); + /* Process Rx packets bounded by budget */ while (likely(rs.packets < budget)) { struct idpf_rx_extracted fields = { }; @@ -996,62 +1019,30 @@ static int idpf_rx_singleq_clean(struct idpf_rx_queue *rx_q, int budget) idpf_rx_singleq_extract_fields(rx_q, rx_desc, &fields); rx_buf = &rx_q->rx_buf[ntc]; - if (!libeth_rx_sync_for_cpu(rx_buf, fields.size)) - goto skip_data; - - if (skb) - idpf_rx_add_frag(rx_buf, skb, fields.size); - else - skb = idpf_rx_build_skb(rx_buf, fields.size); - - /* exit if we failed to retrieve a buffer */ - if (!skb) - break; - -skip_data: + libeth_xdp_process_buff(xdp, rx_buf, fields.size); rx_buf->page = NULL; IDPF_SINGLEQ_BUMP_RING_IDX(rx_q, ntc); cleaned_count++; /* skip if it is non EOP desc */ - if (idpf_rx_singleq_is_non_eop(rx_desc) || unlikely(!skb)) + if (idpf_rx_singleq_is_non_eop(rx_desc) || + unlikely(!xdp->data)) continue; #define IDPF_RXD_ERR_S FIELD_PREP(VIRTCHNL2_RX_BASE_DESC_QW1_ERROR_M, \ VIRTCHNL2_RX_BASE_DESC_ERROR_RXE_M) if (unlikely(idpf_rx_singleq_test_staterr(rx_desc, IDPF_RXD_ERR_S))) { - dev_kfree_skb_any(skb); - skb = NULL; + libeth_xdp_return_buff_slow(xdp); continue; } - /* pad skb if needed (to make valid ethernet frame) */ - if (eth_skb_pad(skb)) { - skb = NULL; - continue; - } - - /* probably a little skewed due to removing CRC */ - rs.bytes += skb->len; - - /* protocol */ - idpf_rx_singleq_process_skb_fields(rx_q, skb, - rx_desc, fields.rx_ptype, - &rs); - - /* send completed skb up the stack */ - napi_gro_receive(rx_q->pp->p.napi, skb); - skb = NULL; - - /* update budget accounting */ - rs.packets++; + idpf_xdp_run_pass(xdp, rx_q->pp->p.napi, &rs, rx_desc); } - rx_q->skb = skb; - rx_q->next_to_clean = ntc; + libeth_xdp_save_buff(&rx_q->xdp, xdp); page_pool_nid_changed(rx_q->pp, numa_mem_id()); if (cleaned_count) diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index b485c8578eda16..63c0c20e13b682 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -513,10 +513,7 @@ static void idpf_rx_desc_rel(struct idpf_rx_queue *rxq, if (!rxq) return; - if (rxq->skb) { - dev_kfree_skb_any(rxq->skb); - rxq->skb = NULL; - } + libeth_xdp_return_stash(&rxq->xdp); if (!idpf_is_queue_model_split(model)) idpf_rx_buf_rel_all(rxq); @@ -3252,7 +3249,7 @@ static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, } /** - * idpf_rx_process_skb_fields - Populate skb header fields from Rx descriptor + * __idpf_rx_process_skb_fields - Populate skb header fields from Rx descriptor * @rxq: Rx descriptor ring packet is being transacted on * @skb: pointer to current skb being populated * @rx_desc: Receive descriptor @@ -3263,9 +3260,9 @@ static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, * other fields within the skb. */ static int -idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, - const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc, - struct libeth_rq_napi_stats *rs) +__idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc, + struct libeth_rq_napi_stats *rs) { struct idpf_rx_csum_decoded csum_bits; struct libeth_rx_pt decoded; @@ -3278,8 +3275,6 @@ idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, /* process RSS/hash */ idpf_rx_hash(rxq, skb, rx_desc, decoded); - skb->protocol = eth_type_trans(skb, rxq->xdp_rxq.dev); - if (le16_get_bits(rx_desc->hdrlen_flags, VIRTCHNL2_RX_FLEX_DESC_ADV_RSC_M)) return idpf_rx_rsc(rxq, skb, rx_desc, decoded, rs); @@ -3287,28 +3282,27 @@ idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, csum_bits = idpf_rx_splitq_extract_csum_bits(rx_desc); idpf_rx_csum(rxq, skb, csum_bits, decoded, rs); - skb_record_rx_queue(skb, rxq->idx); - return 0; } -/** - * idpf_rx_add_frag - Add contents of Rx buffer to sk_buff as a frag - * @rx_buf: buffer containing page to add - * @skb: sk_buff to place the data into - * @size: packet length from rx_desc - * - * This function will add the data contained in rx_buf->page to the skb. - * It will just attach the page as a frag to the skb. - * The function will then update the page offset. - */ -void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb, - unsigned int size) +static bool idpf_rx_process_skb_fields(struct sk_buff *skb, + const struct libeth_xdp_buff *xdp, + struct libeth_rq_napi_stats *ss) { - u32 hr = rx_buf->page->pp->p.offset; + struct idpf_rx_queue *rxq; + + rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq); - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buf->page, - rx_buf->offset + hr, size, rx_buf->truesize); + return !__idpf_rx_process_skb_fields(rxq, skb, xdp->desc, ss); +} + +static void +idpf_xdp_run_pass(struct libeth_xdp_buff *xdp, struct napi_struct *napi, + struct libeth_rq_napi_stats *ss, + const struct virtchnl2_rx_flex_desc_adv_nic_3 *desc) +{ + libeth_xdp_run_pass(xdp, NULL, napi, ss, desc, NULL, + idpf_rx_process_skb_fields); } /** @@ -3346,35 +3340,6 @@ static u32 idpf_rx_hsplit_wa(const struct libeth_fqe *hdr, return copy; } -/** - * idpf_rx_build_skb - Allocate skb and populate it from header buffer - * @buf: Rx buffer to pull data from - * @size: the length of the packet - * - * This function allocates an skb. It then populates it with the page data from - * the current receive descriptor, taking care to set up the skb correctly. - */ -struct sk_buff *idpf_rx_build_skb(const struct libeth_fqe *buf, u32 size) -{ - u32 hr = buf->page->pp->p.offset; - struct sk_buff *skb; - void *va; - - va = page_address(buf->page) + buf->offset; - prefetch(va + hr); - - skb = napi_build_skb(va, buf->truesize); - if (unlikely(!skb)) - return NULL; - - skb_mark_for_recycle(skb); - - skb_reserve(skb, hr); - __skb_put(skb, size); - - return skb; -} - /** * idpf_rx_splitq_test_staterr - tests bits in Rx descriptor * status and error fields @@ -3418,9 +3383,11 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) { struct idpf_buf_queue *rx_bufq = NULL; struct libeth_rq_napi_stats rs = { }; - struct sk_buff *skb = rxq->skb; + LIBETH_XDP_ONSTACK_BUFF(xdp); u16 ntc = rxq->next_to_clean; + libeth_xdp_init_buff(xdp, &rxq->xdp, &rxq->xdp_rxq); + /* Process Rx packets bounded by budget */ while (likely(rs.packets < budget)) { struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc; @@ -3436,11 +3403,6 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) /* get the Rx desc from Rx queue based on 'next_to_clean' */ rx_desc = &rxq->rx[ntc].flex_adv_nic_3_wb; - /* This memory barrier is needed to keep us from reading - * any other fields out of the rx_desc - */ - dma_rmb(); - /* if the descriptor isn't done, no work yet to do */ gen_id = le16_get_bits(rx_desc->pktlen_gen_bufq_id, VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_M); @@ -3448,6 +3410,11 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) if (idpf_queue_has(GEN_CHK, rxq) != gen_id) break; + /* This memory barrier is needed to keep us from reading + * any other fields out of the rx_desc + */ + dma_rmb(); + rxdid = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_RXDID_M, rx_desc->rxdid_ucast); if (rxdid != VIRTCHNL2_RXDID_2_FLEX_SPLITQ) { @@ -3490,20 +3457,15 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) hdr = &rx_bufq->hdr_buf[buf_id]; - if (unlikely(!hdr_len && !skb)) { + if (unlikely(!hdr_len && !xdp->data)) { hdr_len = idpf_rx_hsplit_wa(hdr, rx_buf, pkt_len); pkt_len -= hdr_len; libeth_stats_inc_one(&rxq->stats, hsplit_errs); } - if (libeth_rx_sync_for_cpu(hdr, hdr_len)) { - skb = idpf_rx_build_skb(hdr, hdr_len); - if (!skb) - break; - + if (libeth_xdp_process_buff(xdp, hdr, hdr_len)) rs.hsplit++; - } hdr->page = NULL; @@ -3511,56 +3473,22 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) rs.hsplit_linear++; payload: - if (!libeth_rx_sync_for_cpu(rx_buf, pkt_len)) - goto skip_data; - - if (skb) - idpf_rx_add_frag(rx_buf, skb, pkt_len); - else - skb = idpf_rx_build_skb(rx_buf, pkt_len); - - /* exit if we failed to retrieve a buffer */ - if (!skb) - break; - -skip_data: + libeth_xdp_process_buff(xdp, rx_buf, pkt_len); rx_buf->page = NULL; idpf_rx_post_buf_refill(refillq, buf_id); IDPF_RX_BUMP_NTC(rxq, ntc); /* skip if it is non EOP desc */ - if (!idpf_rx_splitq_is_eop(rx_desc) || unlikely(!skb)) + if (!idpf_rx_splitq_is_eop(rx_desc) || unlikely(!xdp->data)) continue; - /* pad skb if needed (to make valid ethernet frame) */ - if (eth_skb_pad(skb)) { - skb = NULL; - continue; - } - - /* probably a little skewed due to removing CRC */ - rs.bytes += skb->len; - - /* protocol */ - if (unlikely(idpf_rx_process_skb_fields(rxq, skb, rx_desc, - &rs))) { - dev_kfree_skb_any(skb); - skb = NULL; - continue; - } - - /* send completed skb up the stack */ - napi_gro_receive(rxq->napi, skb); - skb = NULL; - - /* update budget accounting */ - rs.packets++; + idpf_xdp_run_pass(xdp, rxq->napi, &rs, rx_desc); } rxq->next_to_clean = ntc; + libeth_xdp_save_buff(&rxq->xdp, xdp); - rxq->skb = skb; libeth_rq_napi_stats_add(&rxq->stats, &rs); return rs.packets; diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index 8597307acead74..ead036dc226b9d 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -495,7 +495,7 @@ struct idpf_txq_stash { * @next_to_use: Next descriptor to use * @next_to_clean: Next descriptor to clean * @next_to_alloc: RX buffer to allocate at - * @skb: Pointer to the skb + * @xdp: XDP buffer with the current frame * @stats: per-queue RQ stats * @q_id: Queue id * @size: Length of descriptor ring in bytes @@ -549,7 +549,7 @@ struct idpf_rx_queue { u32 next_to_clean; u32 next_to_alloc; - struct sk_buff *skb; + struct libeth_xdp_buff_stash xdp; struct libeth_rq_stats stats; __cacheline_group_end_aligned(read_write); @@ -1066,9 +1066,6 @@ int idpf_config_rss(struct idpf_vport *vport); int idpf_init_rss(struct idpf_vport *vport); void idpf_deinit_rss(struct idpf_vport *vport); int idpf_rx_bufs_init_all(struct idpf_vport *vport); -void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb, - unsigned int size); -struct sk_buff *idpf_rx_build_skb(const struct libeth_fqe *buf, u32 size); void idpf_tx_buf_hw_update(struct idpf_tx_queue *tx_q, u32 val, bool xmit_more); unsigned int idpf_size_to_txd_count(unsigned int size); From 242d36e579af207c3330a9ead56222e4d165c564 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 12 Dec 2023 17:01:52 +0100 Subject: [PATCH 41/49] idpf: add support for XDP on Rx Use libeth XDP infra to support running XDP program on Rx polling. This includes all of the possible verdicts/actions. XDP Tx queues are cleaned only in "lazy" mode when there are less than 1/4 free descriptors left on the ring. libeth helper macros to define driver-specific XDP functions make sure the compiler could uninline them when needed. Use __LIBETH_WORD_ACCESS to parse descriptors more efficiently when applicable. It really gives some good boosts and code size reduction on x86_64. Co-developed-by: Michal Kubiak Signed-off-by: Michal Kubiak Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/idpf/idpf_lib.c | 2 + drivers/net/ethernet/intel/idpf/idpf_txrx.c | 23 +-- drivers/net/ethernet/intel/idpf/idpf_txrx.h | 6 +- drivers/net/ethernet/intel/idpf/xdp.c | 157 +++++++++++++++++++- drivers/net/ethernet/intel/idpf/xdp.h | 100 ++++++++++++- 5 files changed, 267 insertions(+), 21 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index 335b0deb26f116..c0a7cee345d16e 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -830,6 +830,8 @@ static int idpf_cfg_netdev(struct idpf_vport *vport) netdev->features |= dflt_features; netdev->hw_features |= dflt_features | offloads; netdev->hw_enc_features |= dflt_features | offloads; + idpf_xdp_set_features(vport); + idpf_set_ethtool_ops(netdev); SET_NETDEV_DEV(netdev, &adapter->pdev->dev); diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 63c0c20e13b682..e9bb70fa311a2f 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (C) 2023 Intel Corporation */ -#include - #include "idpf.h" #include "idpf_virtchnl.h" #include "xdp.h" @@ -3296,14 +3294,12 @@ static bool idpf_rx_process_skb_fields(struct sk_buff *skb, return !__idpf_rx_process_skb_fields(rxq, skb, xdp->desc, ss); } -static void -idpf_xdp_run_pass(struct libeth_xdp_buff *xdp, struct napi_struct *napi, - struct libeth_rq_napi_stats *ss, - const struct virtchnl2_rx_flex_desc_adv_nic_3 *desc) -{ - libeth_xdp_run_pass(xdp, NULL, napi, ss, desc, NULL, - idpf_rx_process_skb_fields); -} +LIBETH_XDP_DEFINE_START(); +LIBETH_XDP_DEFINE_RUN(static idpf_xdp_run_pass, idpf_xdp_run_prog, + idpf_xdp_tx_flush_bulk, idpf_rx_process_skb_fields); +LIBETH_XDP_DEFINE_FINALIZE(static idpf_xdp_finalize_rx, idpf_xdp_tx_flush_bulk, + idpf_xdp_tx_finalize); +LIBETH_XDP_DEFINE_END(); /** * idpf_rx_hsplit_wa - handle header buffer overflows and split errors @@ -3383,9 +3379,12 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) { struct idpf_buf_queue *rx_bufq = NULL; struct libeth_rq_napi_stats rs = { }; + struct libeth_xdp_tx_bulk bq; LIBETH_XDP_ONSTACK_BUFF(xdp); u16 ntc = rxq->next_to_clean; + libeth_xdp_tx_init_bulk(&bq, rxq->xdp_prog, rxq->xdp_rxq.dev, + rxq->xdpqs, rxq->num_xdp_txq); libeth_xdp_init_buff(xdp, &rxq->xdp, &rxq->xdp_rxq); /* Process Rx packets bounded by budget */ @@ -3483,11 +3482,13 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) if (!idpf_rx_splitq_is_eop(rx_desc) || unlikely(!xdp->data)) continue; - idpf_xdp_run_pass(xdp, rxq->napi, &rs, rx_desc); + idpf_xdp_run_pass(xdp, &bq, rxq->napi, &rs, rx_desc); } rxq->next_to_clean = ntc; + libeth_xdp_save_buff(&rxq->xdp, xdp); + idpf_xdp_finalize_rx(&bq); libeth_rq_napi_stats_add(&rxq->stats, &rs); diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index ead036dc226b9d..1b4a150b337e44 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -674,8 +674,8 @@ struct idpf_tx_queue { __cacheline_group_end_aligned(read_mostly); __cacheline_group_begin_aligned(read_write); - u16 next_to_use; - u16 next_to_clean; + u32 next_to_use; + u32 next_to_clean; union { struct { @@ -719,7 +719,7 @@ struct idpf_tx_queue { * hardcode any assumptions. */ libeth_cacheline_set_assert(struct idpf_tx_queue, 64, - 4 + offsetofend(struct idpf_tx_queue, stats) - + 8 + offsetofend(struct idpf_tx_queue, stats) - offsetofend(struct idpf_tx_queue, next_to_clean), 24); diff --git a/drivers/net/ethernet/intel/idpf/xdp.c b/drivers/net/ethernet/intel/idpf/xdp.c index 9b656b98dbe76b..4eccfa30229ca6 100644 --- a/drivers/net/ethernet/intel/idpf/xdp.c +++ b/drivers/net/ethernet/intel/idpf/xdp.c @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (C) 2024 Intel Corporation */ -#include - #include "idpf.h" #include "idpf_virtchnl.h" #include "xdp.h" @@ -143,6 +141,8 @@ void idpf_copy_xdp_prog_to_qs(const struct idpf_vport *vport, idpf_rxq_for_each(vport, idpf_xdp_rxq_assign_prog, xdp_prog); } +static void idpf_xdp_tx_timer(struct work_struct *work); + int idpf_vport_xdpq_get(const struct idpf_vport *vport) { struct libeth_xdpsq_timer **timers __free(kvfree) = NULL; @@ -185,6 +185,8 @@ int idpf_vport_xdpq_get(const struct idpf_vport *vport) xdpq->timer = timers[i - sqs]; libeth_xdpsq_get(&xdpq->xdp_lock, dev, vport->xdpq_share); + libeth_xdpsq_init_timer(xdpq->timer, xdpq, &xdpq->xdp_lock, + idpf_xdp_tx_timer); xdpq->pending = 0; xdpq->xdp_tx = 0; @@ -215,6 +217,7 @@ void idpf_vport_xdpq_put(const struct idpf_vport *vport) if (!idpf_queue_has_clear(XDP, xdpq)) continue; + libeth_xdpsq_deinit_timer(xdpq->timer); libeth_xdpsq_put(&xdpq->xdp_lock, dev); kfree(xdpq->timer); @@ -224,6 +227,156 @@ void idpf_vport_xdpq_put(const struct idpf_vport *vport) libeth_xdpsq_get_end(); } +static int +idpf_xdp_parse_compl_desc(const struct idpf_splitq_4b_tx_compl_desc *desc, + bool gen) +{ + u32 val; + +#ifdef __LIBETH_WORD_ACCESS + val = *(const u32 *)desc; +#else + val = ((u32)le16_to_cpu(desc->q_head_compl_tag.q_head) << 16) | + le16_to_cpu(desc->qid_comptype_gen); +#endif + if (!!(val & IDPF_TXD_COMPLQ_GEN_M) != gen) + return -ENODATA; + + if (unlikely((val & GENMASK(IDPF_TXD_COMPLQ_GEN_S - 1, 0)) != + FIELD_PREP(IDPF_TXD_COMPLQ_COMPL_TYPE_M, + IDPF_TXD_COMPLT_RS))) + return -EINVAL; + + return upper_16_bits(val); +} + +static u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget) +{ + struct idpf_compl_queue *cq = xdpsq->complq; + u32 tx_ntc = xdpsq->next_to_clean; + u32 tx_cnt = xdpsq->desc_count; + u32 ntc = cq->next_to_clean; + u32 cnt = cq->desc_count; + u32 done_frames; + bool gen; + + gen = idpf_queue_has(GEN_CHK, cq); + + for (done_frames = 0; done_frames < budget; ) { + int ret; + + ret = idpf_xdp_parse_compl_desc(&cq->comp_4b[ntc], gen); + if (ret >= 0) { + done_frames = ret > tx_ntc ? ret - tx_ntc : + ret + tx_cnt - tx_ntc; + goto next; + } + + switch (ret) { + case -ENODATA: + goto out; + case -EINVAL: + break; + } + +next: + if (unlikely(++ntc == cnt)) { + ntc = 0; + gen = !gen; + idpf_queue_change(GEN_CHK, cq); + } + } + +out: + cq->next_to_clean = ntc; + + return done_frames; +} + +/** + * idpf_clean_xdp_irq - Reclaim a batch of TX resources from completed XDP_TX + * @_xdpq: XDP Tx queue + * @budget: maximum number of descriptors to clean + * + * Returns number of cleaned descriptors. + */ +static u32 idpf_clean_xdp_irq(void *_xdpq, u32 budget) +{ + struct libeth_xdpsq_napi_stats ss = { }; + struct idpf_tx_queue *xdpq = _xdpq; + u32 tx_ntc = xdpq->next_to_clean; + u32 tx_cnt = xdpq->desc_count; + struct xdp_frame_bulk bq; + struct libeth_cq_pp cp = { + .dev = xdpq->dev, + .bq = &bq, + .xss = &ss, + .napi = true, + }; + u32 done_frames; + + done_frames = idpf_xdpsq_poll(xdpq, budget); + if (unlikely(!done_frames)) + return 0; + + xdp_frame_bulk_init(&bq); + + for (u32 i = 0; likely(i < done_frames); i++) { + libeth_xdp_complete_tx(&xdpq->tx_buf[tx_ntc], &cp); + + if (unlikely(++tx_ntc == tx_cnt)) + tx_ntc = 0; + } + + xdp_flush_frame_bulk(&bq); + + xdpq->next_to_clean = tx_ntc; + xdpq->pending -= done_frames; + xdpq->xdp_tx -= cp.xdp_tx; + + libeth_xdpsq_napi_stats_add(&xdpq->xstats, &ss); + + return done_frames; +} + +static u32 idpf_xdp_tx_prep(void *_xdpq, struct libeth_xdpsq *sq) +{ + struct idpf_tx_queue *xdpq = _xdpq; + u32 free; + + libeth_xdpsq_lock(&xdpq->xdp_lock); + + free = xdpq->desc_count - xdpq->pending; + if (unlikely(free < xdpq->thresh)) + free += idpf_clean_xdp_irq(xdpq, NAPI_POLL_WEIGHT); + + *sq = (struct libeth_xdpsq){ + .sqes = xdpq->tx_buf, + .descs = xdpq->desc_ring, + .count = xdpq->desc_count, + .lock = &xdpq->xdp_lock, + .ntu = &xdpq->next_to_use, + .pending = &xdpq->pending, + .xdp_tx = &xdpq->xdp_tx, + }; + + return free; +} + +LIBETH_XDP_DEFINE_START(); +LIBETH_XDP_DEFINE_TIMER(static idpf_xdp_tx_timer, idpf_clean_xdp_irq); +LIBETH_XDP_DEFINE_FLUSH_TX(idpf_xdp_tx_flush_bulk, idpf_xdp_tx_prep, + idpf_xdp_tx_xmit); +LIBETH_XDP_DEFINE_END(); + +void idpf_xdp_set_features(const struct idpf_vport *vport) +{ + if (!idpf_is_queue_model_split(vport->rxq_model)) + return; + + libeth_xdp_set_features_noredir(vport->netdev); +} + /** * idpf_xdp_setup_prog - Add or remove XDP eBPF program * @vport: vport to setup XDP for diff --git a/drivers/net/ethernet/intel/idpf/xdp.h b/drivers/net/ethernet/intel/idpf/xdp.h index a72a7638a6ea5c..fde85528a3151f 100644 --- a/drivers/net/ethernet/intel/idpf/xdp.h +++ b/drivers/net/ethernet/intel/idpf/xdp.h @@ -4,12 +4,9 @@ #ifndef _IDPF_XDP_H_ #define _IDPF_XDP_H_ -#include +#include -struct bpf_prog; -struct idpf_vport; -struct net_device; -struct netdev_bpf; +#include "idpf_txrx.h" int idpf_xdp_rxq_info_init_all(const struct idpf_vport *vport); void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport); @@ -19,6 +16,99 @@ void idpf_copy_xdp_prog_to_qs(const struct idpf_vport *vport, int idpf_vport_xdpq_get(const struct idpf_vport *vport); void idpf_vport_xdpq_put(const struct idpf_vport *vport); +bool idpf_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags); + +/** + * idpf_xdp_tx_xmit - produce a single HW Tx descriptor out of XDP desc + * @desc: XDP descriptor to pull the DMA address and length from + * @i: descriptor index on the queue to fill + * @sq: XDP queue to produce the HW Tx descriptor on + * @priv: &xsk_tx_metadata_ops on XSk xmit or %NULL + */ +static inline void idpf_xdp_tx_xmit(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, u64 priv) +{ + struct idpf_flex_tx_desc *tx_desc = sq->descs; + u32 cmd; + + cmd = FIELD_PREP(IDPF_FLEX_TXD_QW1_DTYPE_M, + IDPF_TX_DESC_DTYPE_FLEX_L2TAG1_L2TAG2); + if (desc.flags & LIBETH_XDP_TX_LAST) + cmd |= FIELD_PREP(IDPF_FLEX_TXD_QW1_CMD_M, + IDPF_TX_DESC_CMD_EOP); + if (priv && (desc.flags & LIBETH_XDP_TX_CSUM)) + cmd |= FIELD_PREP(IDPF_FLEX_TXD_QW1_CMD_M, + IDPF_TX_FLEX_DESC_CMD_CS_EN); + + tx_desc = &tx_desc[i]; + tx_desc->buf_addr = cpu_to_le64(desc.addr); +#ifdef __LIBETH_WORD_ACCESS + *(u64 *)&tx_desc->qw1 = ((u64)desc.len << 48) | cmd; +#else + tx_desc->qw1.buf_size = cpu_to_le16(desc.len); + tx_desc->qw1.cmd_dtype = cpu_to_le16(cmd); +#endif +} + +/** + * idpf_set_rs_bit - set RS bit on last produced descriptor + * @xdpq: XDP queue to produce the HW Tx descriptors on + */ +static inline void idpf_set_rs_bit(const struct idpf_tx_queue *xdpq) +{ + u32 ntu, cmd; + + ntu = xdpq->next_to_use; + if (unlikely(!ntu)) + ntu = xdpq->desc_count; + + cmd = FIELD_PREP(IDPF_FLEX_TXD_QW1_CMD_M, IDPF_TX_DESC_CMD_RS); +#ifdef __LIBETH_WORD_ACCESS + *(u64 *)&xdpq->flex_tx[ntu - 1].q.qw1 |= cmd; +#else + xdpq->flex_tx[ntu - 1].q.qw1.cmd_dtype |= cpu_to_le16(cmd); +#endif +} + +/** + * idpf_xdpq_update_tail - update the XDP Tx queue tail register + * @xdpq: XDP Tx queue + */ +static inline void idpf_xdpq_update_tail(const struct idpf_tx_queue *xdpq) +{ + dma_wmb(); + writel_relaxed(xdpq->next_to_use, xdpq->tail); +} + +/** + * idpf_xdp_tx_finalize - Update RS bit and bump XDP Tx tail + * @_xdpq: XDP Tx queue + * @sent: whether any frames were sent + * @flush: whether to update RS bit and the tail register + * + * This function bumps XDP Tx tail and should be called when a batch of packets + * has been processed in the napi loop. + */ +static inline void idpf_xdp_tx_finalize(void *_xdpq, bool sent, bool flush) +{ + struct idpf_tx_queue *xdpq = _xdpq; + + if ((!flush || unlikely(!sent)) && + likely(xdpq->desc_count != xdpq->pending)) + return; + + libeth_xdpsq_lock(&xdpq->xdp_lock); + + idpf_set_rs_bit(xdpq); + idpf_xdpq_update_tail(xdpq); + + libeth_xdpsq_queue_timer(xdpq->timer); + + libeth_xdpsq_unlock(&xdpq->xdp_lock); +} + +void idpf_xdp_set_features(const struct idpf_vport *vport); + int idpf_xdp(struct net_device *dev, struct netdev_bpf *xdp); #endif /* _IDPF_XDP_H_ */ From 7e0b8efb97ab3e7a3813bde74e0c70f0a23cffc1 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Mon, 30 Oct 2023 16:10:26 +0100 Subject: [PATCH 42/49] idpf: add support for .ndo_xdp_xmit() Use libeth XDP infra to implement .ndo_xdp_xmit() in idpf. The Tx callbacks are reused from XDP_TX code. XDP redirect target feature is set/cleared depending on the XDP prog presence, as for now we still don't allocate XDP Tx queues when there's no program. Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/idpf/idpf_lib.c | 1 + drivers/net/ethernet/intel/idpf/xdp.c | 29 ++++++++++++++++++++++ drivers/net/ethernet/intel/idpf/xdp.h | 2 ++ 3 files changed, 32 insertions(+) diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index c0a7cee345d16e..2bc9e9358e30e0 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -2335,4 +2335,5 @@ static const struct net_device_ops idpf_netdev_ops = { .ndo_set_features = idpf_set_features, .ndo_tx_timeout = idpf_tx_timeout, .ndo_bpf = idpf_xdp, + .ndo_xdp_xmit = idpf_xdp_xmit, }; diff --git a/drivers/net/ethernet/intel/idpf/xdp.c b/drivers/net/ethernet/intel/idpf/xdp.c index 4eccfa30229ca6..5e11bdcbac8611 100644 --- a/drivers/net/ethernet/intel/idpf/xdp.c +++ b/drivers/net/ethernet/intel/idpf/xdp.c @@ -367,8 +367,35 @@ LIBETH_XDP_DEFINE_START(); LIBETH_XDP_DEFINE_TIMER(static idpf_xdp_tx_timer, idpf_clean_xdp_irq); LIBETH_XDP_DEFINE_FLUSH_TX(idpf_xdp_tx_flush_bulk, idpf_xdp_tx_prep, idpf_xdp_tx_xmit); +LIBETH_XDP_DEFINE_FLUSH_XMIT(static idpf_xdp_xmit_flush_bulk, idpf_xdp_tx_prep, + idpf_xdp_tx_xmit); LIBETH_XDP_DEFINE_END(); +/** + * idpf_xdp_xmit - submit packets to xdp ring for transmission + * @dev: netdev + * @n: number of xdp frames to be transmitted + * @frames: xdp frames to be transmitted + * @flags: transmit flags + * + * Return: number of frames successfully sent or -errno on error. + */ +int idpf_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, + u32 flags) +{ + const struct idpf_netdev_priv *np = netdev_priv(dev); + const struct idpf_vport *vport = np->vport; + + if (unlikely(!netif_carrier_ok(dev) || !vport->link_up)) + return -ENETDOWN; + + return libeth_xdp_xmit_do_bulk(dev, n, frames, flags, + &vport->txqs[vport->xdp_txq_offset], + vport->num_xdp_txq, + idpf_xdp_xmit_flush_bulk, + idpf_xdp_tx_finalize); +} + void idpf_xdp_set_features(const struct idpf_vport *vport) { if (!idpf_is_queue_model_split(vport->rxq_model)) @@ -400,6 +427,8 @@ idpf_xdp_setup_prog(struct idpf_vport *vport, struct netdev_bpf *xdp) return 0; } + libeth_xdp_set_redirect(vport->netdev, prog); + ret = idpf_initiate_soft_reset(vport); if (ret) { NL_SET_ERR_MSG_MOD(xdp->extack, diff --git a/drivers/net/ethernet/intel/idpf/xdp.h b/drivers/net/ethernet/intel/idpf/xdp.h index fde85528a3151f..a2ac1b2f334f09 100644 --- a/drivers/net/ethernet/intel/idpf/xdp.h +++ b/drivers/net/ethernet/intel/idpf/xdp.h @@ -110,5 +110,7 @@ static inline void idpf_xdp_tx_finalize(void *_xdpq, bool sent, bool flush) void idpf_xdp_set_features(const struct idpf_vport *vport); int idpf_xdp(struct net_device *dev, struct netdev_bpf *xdp); +int idpf_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, + u32 flags); #endif /* _IDPF_XDP_H_ */ From 74ed675a7a13e37248b54814fa3e85fcb662fe9c Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Fri, 8 Mar 2024 12:54:08 +0100 Subject: [PATCH 43/49] idpf: add XDP RSS hash hint Add &xdp_metadata_ops with a callback to get RSS hash hint from the descriptor. Declare the splitq 32-byte descriptor as 4 u64s to parse them more efficiently when possible. Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/idpf/xdp.c | 28 +++++++++++- drivers/net/ethernet/intel/idpf/xdp.h | 64 +++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/intel/idpf/xdp.c b/drivers/net/ethernet/intel/idpf/xdp.c index 5e11bdcbac8611..11ef3216bef58e 100644 --- a/drivers/net/ethernet/intel/idpf/xdp.c +++ b/drivers/net/ethernet/intel/idpf/xdp.c @@ -396,12 +396,38 @@ int idpf_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, idpf_xdp_tx_finalize); } +static int idpf_xdpmo_rx_hash(const struct xdp_md *ctx, u32 *hash, + enum xdp_rss_hash_type *rss_type) +{ + const struct libeth_xdp_buff *xdp = (typeof(xdp))ctx; + const struct idpf_rx_queue *rxq; + struct idpf_xdp_rx_desc desc; + struct libeth_rx_pt pt; + + rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq); + + idpf_xdp_get_qw0(&desc, xdp->desc); + + pt = rxq->rx_ptype_lkup[idpf_xdp_rx_pt(&desc)]; + if (!libeth_rx_pt_has_hash(rxq->xdp_rxq.dev, pt)) + return -ENODATA; + + idpf_xdp_get_qw2(&desc, xdp->desc); + + return libeth_xdpmo_rx_hash(hash, rss_type, idpf_xdp_rx_hash(&desc), + pt); +} + +static const struct xdp_metadata_ops idpf_xdpmo = { + .xmo_rx_hash = idpf_xdpmo_rx_hash, +}; + void idpf_xdp_set_features(const struct idpf_vport *vport) { if (!idpf_is_queue_model_split(vport->rxq_model)) return; - libeth_xdp_set_features_noredir(vport->netdev); + libeth_xdp_set_features_noredir(vport->netdev, &idpf_xdpmo); } /** diff --git a/drivers/net/ethernet/intel/idpf/xdp.h b/drivers/net/ethernet/intel/idpf/xdp.h index a2ac1b2f334f09..52783a5c8e0fb2 100644 --- a/drivers/net/ethernet/intel/idpf/xdp.h +++ b/drivers/net/ethernet/intel/idpf/xdp.h @@ -107,6 +107,70 @@ static inline void idpf_xdp_tx_finalize(void *_xdpq, bool sent, bool flush) libeth_xdpsq_unlock(&xdpq->xdp_lock); } +struct idpf_xdp_rx_desc { + aligned_u64 qw0; +#define IDPF_XDP_RX_BUFQ BIT_ULL(47) +#define IDPF_XDP_RX_GEN BIT_ULL(46) +#define IDPF_XDP_RX_LEN GENMASK_ULL(45, 32) +#define IDPF_XDP_RX_PT GENMASK_ULL(25, 16) + + aligned_u64 qw1; +#define IDPF_XDP_RX_BUF GENMASK_ULL(47, 32) +#define IDPF_XDP_RX_EOP BIT_ULL(1) + + aligned_u64 qw2; +#define IDPF_XDP_RX_HASH GENMASK_ULL(31, 0) + + aligned_u64 qw3; +} __aligned(4 * sizeof(u64)); +static_assert(sizeof(struct idpf_xdp_rx_desc) == + sizeof(struct virtchnl2_rx_flex_desc_adv_nic_3)); + +#define idpf_xdp_rx_bufq(desc) !!((desc)->qw0 & IDPF_XDP_RX_BUFQ) +#define idpf_xdp_rx_gen(desc) !!((desc)->qw0 & IDPF_XDP_RX_GEN) +#define idpf_xdp_rx_len(desc) FIELD_GET(IDPF_XDP_RX_LEN, (desc)->qw0) +#define idpf_xdp_rx_pt(desc) FIELD_GET(IDPF_XDP_RX_PT, (desc)->qw0) +#define idpf_xdp_rx_buf(desc) FIELD_GET(IDPF_XDP_RX_BUF, (desc)->qw1) +#define idpf_xdp_rx_eop(desc) !!((desc)->qw1 & IDPF_XDP_RX_EOP) +#define idpf_xdp_rx_hash(desc) FIELD_GET(IDPF_XDP_RX_HASH, (desc)->qw2) + +static inline void +idpf_xdp_get_qw0(struct idpf_xdp_rx_desc *desc, + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rxd) +{ +#ifdef __LIBETH_WORD_ACCESS + desc->qw0 = ((const typeof(desc))rxd)->qw0; +#else + desc->qw0 = ((u64)le16_to_cpu(rxd->pktlen_gen_bufq_id) << 32) | + ((u64)le16_to_cpu(rxd->ptype_err_fflags0) << 16); +#endif +} + +static inline void +idpf_xdp_get_qw1(struct idpf_xdp_rx_desc *desc, + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rxd) +{ +#ifdef __LIBETH_WORD_ACCESS + desc->qw1 = ((const typeof(desc))rxd)->qw1; +#else + desc->qw1 = ((u64)le16_to_cpu(rxd->buf_id) << 32) | + rxd->status_err0_qw1; +#endif +} + +static inline void +idpf_xdp_get_qw2(struct idpf_xdp_rx_desc *desc, + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rxd) +{ +#ifdef __LIBETH_WORD_ACCESS + desc->qw2 = ((const typeof(desc))rxd)->qw2; +#else + desc->qw2 = ((u64)rxd->hash3 << 24) | + ((u64)rxd->ff2_mirrid_hash2.hash2 << 16) | + le16_to_cpu(rxd->hash1); +#endif +} + void idpf_xdp_set_features(const struct idpf_vport *vport); int idpf_xdp(struct net_device *dev, struct netdev_bpf *xdp); From dd01c331de15f3b38c70fb497bfc9333a8f2096a Mon Sep 17 00:00:00 2001 From: Michal Kubiak Date: Fri, 17 Nov 2023 14:39:59 +0100 Subject: [PATCH 44/49] idpf: add vc functions to manage selected queues Implement VC functions dedicated to enabling, disabling and configuring randomly selected queues. Also, refactor the existing implementation to make the code more modular. Introduce new generic functions for sending VC messages consisting of chunks, in order to isolate the sending algorithm and its implementation for specific VC messages. Finally, rewrite the function for mapping queues to q_vectors using the new modular approach to avoid copying the code that implements the VC message sending algorithm. Signed-off-by: Michal Kubiak Co-developed-by: Alexander Lobakin Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/idpf/idpf_txrx.c | 1 + drivers/net/ethernet/intel/idpf/idpf_txrx.h | 5 +- .../net/ethernet/intel/idpf/idpf_virtchnl.c | 1158 +++++++++++------ .../net/ethernet/intel/idpf/idpf_virtchnl.h | 32 +- 4 files changed, 762 insertions(+), 434 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index e9bb70fa311a2f..4515c968bd074b 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -1472,6 +1472,7 @@ static int idpf_txq_group_alloc(struct idpf_vport *vport, u16 num_txq) q->tx_min_pkt_len = idpf_get_min_tx_pkt_len(adapter); q->netdev = vport->netdev; q->txq_grp = tx_qgrp; + q->rel_q_id = j; if (!split) { q->clean_budget = vport->compln_clean_budget; diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index 1b4a150b337e44..beaf09c85f5fe4 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -639,6 +639,7 @@ libeth_cacheline_set_assert(struct idpf_rx_queue, * @size: Length of descriptor ring in bytes * @dma: Physical address of ring * @q_vector: Backreference to associated vector + * @rel_q_id: relative virtchnl queue index */ struct idpf_tx_queue { __cacheline_group_begin_aligned(read_mostly); @@ -713,6 +714,8 @@ struct idpf_tx_queue { dma_addr_t dma; struct idpf_q_vector *q_vector; + + u32 rel_q_id; __cacheline_group_end_aligned(cold); }; /* sizeof(spinlock_t) can take many values depending on the config, don't @@ -721,7 +724,7 @@ struct idpf_tx_queue { libeth_cacheline_set_assert(struct idpf_tx_queue, 64, 8 + offsetofend(struct idpf_tx_queue, stats) - offsetofend(struct idpf_tx_queue, next_to_clean), - 24); + 32); /** * struct idpf_buf_queue - software structure representing a buffer queue diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c index 7bbab6671322df..ec3f3a9eb8f956 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c @@ -740,6 +740,124 @@ int idpf_recv_mb_msg(struct idpf_adapter *adapter) return err; } +struct idpf_chunked_msg_params { + u32 vc_op; + int timeout_ms; + + u32 (*prepare_msg) + (const struct idpf_vport *vport, + void *buf, const void *pos, u32 num); + const void *chunks; + u32 num_chunks; + u32 chunk_sz; + u32 config_sz; +}; + +struct idpf_queue_set *idpf_alloc_queue_set(struct idpf_vport *vport, u32 num) +{ + struct idpf_queue_set *qp; + + qp = kzalloc(struct_size(qp, qs, num), GFP_KERNEL); + if (!qp) + return NULL; + + qp->vport = vport; + qp->num = num; + + return qp; +} + +/** + * idpf_send_chunked_msg - Send a VC message consisting of chunks. + * @vport: virtual port data structure + * @params: message params + * + * Helper function for preparing the message describing queues to be enabled + * or disabled. + * Returns the total size of the prepared message. + */ +static int idpf_send_chunked_msg(struct idpf_vport *vport, + const struct idpf_chunked_msg_params *params) +{ + struct idpf_vc_xn_params xn_params = { + .vc_op = params->vc_op, + .timeout_ms = params->timeout_ms ? : + IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + }; + const void *pos = params->chunks; + u32 num_chunks, num_msgs, buf_sz; + void *buf __free(kfree) = NULL; + u32 totqs = params->num_chunks; + + num_chunks = min(IDPF_NUM_CHUNKS_PER_MSG(params->config_sz, + params->chunk_sz), totqs); + num_msgs = DIV_ROUND_UP(totqs, num_chunks); + + buf_sz = params->config_sz + num_chunks * params->chunk_sz; + buf = kzalloc(buf_sz, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + xn_params.send_buf.iov_base = buf; + + for (u32 i = 0; i < num_msgs; i++) { + ssize_t reply_sz; + + memset(buf, 0, buf_sz); + xn_params.send_buf.iov_len = buf_sz; + + if (params->prepare_msg(vport, buf, pos, num_chunks) != buf_sz) + return -EINVAL; + + reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); + if (reply_sz < 0) + return reply_sz; + + pos += num_chunks * params->chunk_sz; + totqs -= num_chunks; + + num_chunks = min(num_chunks, totqs); + buf_sz = params->config_sz + num_chunks * params->chunk_sz; + } + + return 0; +} + +/** + * idpf_wait_for_marker_event_set - wait for software marker response + * for selected tx queues + * @qs: set of the Tx queues + * + * Return: 0 success, negative on failure. + */ +static int idpf_wait_for_marker_event_set(const struct idpf_queue_set *qs) +{ + struct idpf_tx_queue *txq; + bool markers_rcvd = true; + + for (u32 i = 0; i < qs->num; i++) { + switch (qs->qs[i].type) { + case VIRTCHNL2_QUEUE_TYPE_TX: + txq = qs->qs[i].txq; + + idpf_queue_set(SW_MARKER, txq); + idpf_wait_for_sw_marker_completion(txq); + markers_rcvd &= !idpf_queue_has(SW_MARKER, txq); + break; + default: + break; + } + } + + if (!markers_rcvd) { + netdev_warn(qs->vport->netdev, + "Failed to receive marker packets\n"); + return -ETIMEDOUT; + } + + return 0; +} + /** * idpf_wait_for_marker_event - wait for software marker response * @vport: virtual port data structure @@ -748,22 +866,18 @@ int idpf_recv_mb_msg(struct idpf_adapter *adapter) **/ static int idpf_wait_for_marker_event(struct idpf_vport *vport) { - bool markers_rcvd = true; + struct idpf_queue_set *qs __free(kfree) = NULL; - for (u32 i = 0; i < vport->num_txq; i++) { - struct idpf_tx_queue *txq = vport->txqs[i]; + qs = idpf_alloc_queue_set(vport, vport->num_txq); + if (!qs) + return -ENOMEM; - idpf_queue_set(SW_MARKER, txq); - idpf_wait_for_sw_marker_completion(txq); - markers_rcvd &= !idpf_queue_has(SW_MARKER, txq); + for (u32 i = 0; i < qs->num; i++) { + qs->qs[i].type = VIRTCHNL2_QUEUE_TYPE_TX; + qs->qs[i].txq = vport->txqs[i]; } - if (markers_rcvd) - return 0; - - dev_warn(&vport->adapter->pdev->dev, "Failed to receive marker packets\n"); - - return -ETIMEDOUT; + return idpf_wait_for_marker_event_set(qs); } /** @@ -1424,6 +1538,135 @@ int idpf_send_disable_vport_msg(struct idpf_vport *vport) return reply_sz < 0 ? reply_sz : 0; } +/** + * idpf_fill_txq_config_chunk - Fill the chunk describing a Tx queue + * @vport: virtual port data structure + * @q: tx queue to be inserted into VC chunk + * @qi: pointer to the buffer containing the VC chunk + */ +static void idpf_fill_txq_config_chunk(const struct idpf_vport *vport, + const struct idpf_tx_queue *q, + struct virtchnl2_txq_info *qi) +{ + u32 val; + + qi->queue_id = cpu_to_le32(q->q_id); + qi->model = cpu_to_le16(vport->txq_model); + qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); + qi->ring_len = cpu_to_le16(q->desc_count); + qi->dma_ring_addr = cpu_to_le64(q->dma); + qi->relative_queue_id = cpu_to_le16(q->rel_q_id); + + if (!idpf_is_queue_model_split(vport->txq_model)) { + qi->sched_mode = cpu_to_le16(VIRTCHNL2_TXQ_SCHED_MODE_QUEUE); + return; + } + + if (idpf_queue_has(XDP, q)) + val = q->complq->q_id; + else + val = q->txq_grp->complq->q_id; + + qi->tx_compl_queue_id = cpu_to_le16(val); + + if (idpf_queue_has(FLOW_SCH_EN, q)) + val = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; + else + val = VIRTCHNL2_TXQ_SCHED_MODE_QUEUE; + + qi->sched_mode = cpu_to_le16(val); +} + +/** + * idpf_fill_complq_config_chunk - Fill the chunk describing a completion queue + * @vport: virtual port data structure + * @q: completion queue to be inserted into VC chunk + * @qi: pointer to the buffer containing the VC chunk + */ +static void idpf_fill_complq_config_chunk(const struct idpf_vport *vport, + const struct idpf_compl_queue *q, + struct virtchnl2_txq_info *qi) +{ + u32 val; + + qi->queue_id = cpu_to_le32(q->q_id); + qi->model = cpu_to_le16(vport->txq_model); + qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); + qi->ring_len = cpu_to_le16(q->desc_count); + qi->dma_ring_addr = cpu_to_le64(q->dma); + + if (idpf_queue_has(FLOW_SCH_EN, q)) + val = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; + else + val = VIRTCHNL2_TXQ_SCHED_MODE_QUEUE; + + qi->sched_mode = cpu_to_le16(val); +} + +/** + * idpf_prepare_cfg_txqs_msg - Prepare message to configure selected tx queues. + * @vport: virtual port data structure + * @buf: buffer containing the message + * @pos: pointer to the first chunk describing the tx queue + * @num_chunks: number of chunks in the message + * + * Helper function for preparing the message describing configuration of + * tx queues. + * Returns the total size of the prepared message. + */ +static u32 idpf_prepare_cfg_txqs_msg(const struct idpf_vport *vport, + void *buf, const void *pos, + u32 num_chunks) +{ + struct virtchnl2_config_tx_queues *ctq = buf; + + ctq->vport_id = cpu_to_le32(vport->vport_id); + ctq->num_qinfo = cpu_to_le16(num_chunks); + memcpy(ctq->qinfo, pos, num_chunks * sizeof(*ctq->qinfo)); + + return struct_size(ctq, qinfo, num_chunks); +} + +/** + * idpf_send_config_tx_queue_set_msg - Send virtchnl config tx queues + * message for selected tx queues only. + * @qs: params of the Tx queues + * + * Send config queues virtchnl message for queues contained in 'qs' array. + * The 'qs' array can contain tx queues (or completion queues) only. + * + * Return: 0 on success, -errno on failure. + */ +static int +idpf_send_config_tx_queue_set_msg(const struct idpf_queue_set *qs) +{ + struct virtchnl2_txq_info *qi __free(kfree) = NULL; + struct idpf_chunked_msg_params params = { + .vc_op = VIRTCHNL2_OP_CONFIG_TX_QUEUES, + .prepare_msg = idpf_prepare_cfg_txqs_msg, + .config_sz = sizeof(struct virtchnl2_config_tx_queues), + .chunk_sz = sizeof(*qi), + }; + + qi = kcalloc(qs->num, sizeof(*qi), GFP_KERNEL); + if (!qi) + return -ENOMEM; + + params.chunks = qi; + + for (u32 i = 0; i < qs->num; i++) { + if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_TX) + idpf_fill_txq_config_chunk(qs->vport, qs->qs[i].txq, + &qi[params.num_chunks++]); + else if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION) + idpf_fill_complq_config_chunk(qs->vport, + qs->qs[i].complq, + &qi[params.num_chunks++]); + } + + return idpf_send_chunked_msg(qs->vport, ¶ms); +} + /** * idpf_send_config_tx_queues_msg - Send virtchnl config tx queues message * @vport: virtual port data structure @@ -1433,114 +1676,178 @@ int idpf_send_disable_vport_msg(struct idpf_vport *vport) */ static int idpf_send_config_tx_queues_msg(struct idpf_vport *vport) { - struct virtchnl2_config_tx_queues *ctq __free(kfree) = NULL; - struct virtchnl2_txq_info *qi __free(kfree) = NULL; - struct idpf_vc_xn_params xn_params = {}; - u32 config_sz, chunk_sz, buf_sz; - int totqs, num_msgs, num_chunks; - ssize_t reply_sz; - int i, k = 0; + struct idpf_queue_set *qs __free(kfree) = NULL; + u32 totqs = vport->num_txq + vport->num_complq; + u32 k = 0; - totqs = vport->num_txq + vport->num_complq; - qi = kcalloc(totqs, sizeof(struct virtchnl2_txq_info), GFP_KERNEL); - if (!qi) + qs = idpf_alloc_queue_set(vport, totqs); + if (!qs) return -ENOMEM; /* Populate the queue info buffer with all queue context info */ - for (i = 0; i < vport->num_txq_grp; i++) { - struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; - int j, sched_mode; - - for (j = 0; j < tx_qgrp->num_txq; j++, k++) { - qi[k].queue_id = - cpu_to_le32(tx_qgrp->txqs[j]->q_id); - qi[k].model = - cpu_to_le16(vport->txq_model); - qi[k].type = - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); - qi[k].ring_len = - cpu_to_le16(tx_qgrp->txqs[j]->desc_count); - qi[k].dma_ring_addr = - cpu_to_le64(tx_qgrp->txqs[j]->dma); - if (idpf_is_queue_model_split(vport->txq_model)) { - struct idpf_tx_queue *q = tx_qgrp->txqs[j]; - - qi[k].tx_compl_queue_id = - cpu_to_le16(tx_qgrp->complq->q_id); - qi[k].relative_queue_id = cpu_to_le16(j); - - if (idpf_queue_has(FLOW_SCH_EN, q)) - qi[k].sched_mode = - cpu_to_le16(VIRTCHNL2_TXQ_SCHED_MODE_FLOW); - else - qi[k].sched_mode = - cpu_to_le16(VIRTCHNL2_TXQ_SCHED_MODE_QUEUE); - } else { - qi[k].sched_mode = - cpu_to_le16(VIRTCHNL2_TXQ_SCHED_MODE_QUEUE); - } - } - - if (!idpf_is_queue_model_split(vport->txq_model)) - continue; - - qi[k].queue_id = cpu_to_le32(tx_qgrp->complq->q_id); - qi[k].model = cpu_to_le16(vport->txq_model); - qi[k].type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); - qi[k].ring_len = cpu_to_le16(tx_qgrp->complq->desc_count); - qi[k].dma_ring_addr = cpu_to_le64(tx_qgrp->complq->dma); + for (u32 i = 0; i < vport->num_txq_grp; i++) { + const struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; - if (idpf_queue_has(FLOW_SCH_EN, tx_qgrp->complq)) - sched_mode = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; - else - sched_mode = VIRTCHNL2_TXQ_SCHED_MODE_QUEUE; - qi[k].sched_mode = cpu_to_le16(sched_mode); + for (u32 j = 0; j < tx_qgrp->num_txq; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX; + qs->qs[k++].txq = tx_qgrp->txqs[j]; + } - k++; + if (idpf_is_queue_model_split(vport->txq_model)) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; + qs->qs[k++].complq = tx_qgrp->complq; + } } /* Make sure accounting agrees */ if (k != totqs) return -EINVAL; - /* Chunk up the queue contexts into multiple messages to avoid - * sending a control queue message buffer that is too large + return idpf_send_config_tx_queue_set_msg(qs); +} + +/** + * idpf_fill_rxq_config_chunk - Fill the chunk describing an rx queue. + * @vport: virtual port data structure + * @q: rx queue to be inserted into VC chunk + * @qi: pointer to the buffer containing the VC chunk + */ +static void idpf_fill_rxq_config_chunk(const struct idpf_vport *vport, + struct idpf_rx_queue *q, + struct virtchnl2_rxq_info *qi) +{ + const struct idpf_bufq_set *sets; + + qi->queue_id = cpu_to_le32(q->q_id); + qi->model = cpu_to_le16(vport->rxq_model); + qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); + qi->ring_len = cpu_to_le16(q->desc_count); + qi->dma_ring_addr = cpu_to_le64(q->dma); + qi->max_pkt_size = cpu_to_le32(q->rx_max_pkt_size); + qi->rx_buffer_low_watermark = cpu_to_le16(q->rx_buffer_low_watermark); + qi->qflags = cpu_to_le16(VIRTCHNL2_RX_DESC_SIZE_32BYTE); + if (idpf_is_feature_ena(vport, NETIF_F_GRO_HW)) + qi->qflags |= cpu_to_le16(VIRTCHNL2_RXQ_RSC); + + if (!idpf_is_queue_model_split(vport->rxq_model)) { + qi->data_buffer_size = cpu_to_le32(q->rx_buf_size); + qi->desc_ids = cpu_to_le64(q->rxdids); + + return; + } + + sets = q->bufq_sets; + + /* In splitq mode, RXQ buffer size should be set to that of the first + * buffer queue associated with this RXQ. */ - config_sz = sizeof(struct virtchnl2_config_tx_queues); - chunk_sz = sizeof(struct virtchnl2_txq_info); + q->rx_buf_size = sets[0].bufq.rx_buf_size; + qi->data_buffer_size = cpu_to_le32(q->rx_buf_size); - num_chunks = min_t(u32, IDPF_NUM_CHUNKS_PER_MSG(config_sz, chunk_sz), - totqs); - num_msgs = DIV_ROUND_UP(totqs, num_chunks); + qi->rx_bufq1_id = cpu_to_le16(sets[0].bufq.q_id); + if (vport->num_bufqs_per_qgrp > IDPF_SINGLE_BUFQ_PER_RXQ_GRP) { + qi->bufq2_ena = IDPF_BUFQ2_ENA; + qi->rx_bufq2_id = cpu_to_le16(sets[1].bufq.q_id); + } - buf_sz = struct_size(ctq, qinfo, num_chunks); - ctq = kzalloc(buf_sz, GFP_KERNEL); - if (!ctq) - return -ENOMEM; + q->rx_hbuf_size = sets[0].bufq.rx_hbuf_size; - xn_params.vc_op = VIRTCHNL2_OP_CONFIG_TX_QUEUES; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; + if (idpf_queue_has(HSPLIT_EN, q)) { + qi->qflags |= cpu_to_le16(VIRTCHNL2_RXQ_HDR_SPLIT); + qi->hdr_buffer_size = cpu_to_le16(q->rx_hbuf_size); + } - for (i = 0, k = 0; i < num_msgs; i++) { - memset(ctq, 0, buf_sz); - ctq->vport_id = cpu_to_le32(vport->vport_id); - ctq->num_qinfo = cpu_to_le16(num_chunks); - memcpy(ctq->qinfo, &qi[k], chunk_sz * num_chunks); + qi->desc_ids = cpu_to_le64(VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M); +} - xn_params.send_buf.iov_base = ctq; - xn_params.send_buf.iov_len = buf_sz; - reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; +/** + * idpf_fill_bufq_config_chunk - Fill the chunk describing a buf queue. + * @vport: virtual port data structure + * @q: rx or buffer queue to be inserted into VC chunk + * @qi: pointer to the buffer containing the VC chunk + */ +static void idpf_fill_bufq_config_chunk(const struct idpf_vport *vport, + const struct idpf_buf_queue *q, + struct virtchnl2_rxq_info *qi) +{ + qi->queue_id = cpu_to_le32(q->q_id); + qi->model = cpu_to_le16(vport->rxq_model); + qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); + qi->ring_len = cpu_to_le16(q->desc_count); + qi->dma_ring_addr = cpu_to_le64(q->dma); + qi->data_buffer_size = cpu_to_le32(q->rx_buf_size); + qi->rx_buffer_low_watermark = cpu_to_le16(q->rx_buffer_low_watermark); + qi->desc_ids = cpu_to_le64(VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M); + qi->buffer_notif_stride = IDPF_RX_BUF_STRIDE; + if (idpf_is_feature_ena(vport, NETIF_F_GRO_HW)) + qi->qflags = cpu_to_le16(VIRTCHNL2_RXQ_RSC); - k += num_chunks; - totqs -= num_chunks; - num_chunks = min(num_chunks, totqs); - /* Recalculate buffer size */ - buf_sz = struct_size(ctq, qinfo, num_chunks); + if (idpf_queue_has(HSPLIT_EN, q)) { + qi->qflags |= cpu_to_le16(VIRTCHNL2_RXQ_HDR_SPLIT); + qi->hdr_buffer_size = cpu_to_le16(q->rx_hbuf_size); } +} - return 0; +/** + * idpf_prepare_cfg_rxqs_msg - Prepare message to configure selected rx queues. + * @vport: virtual port data structure + * @buf: buffer containing the message + * @pos: pointer to the first chunk describing the rx queue + * @num_chunks: number of chunks in the message + * + * Helper function for preparing the message describing configuration of + * Rx queues. + * Returns the total size of the prepared message. + */ +static u32 idpf_prepare_cfg_rxqs_msg(const struct idpf_vport *vport, + void *buf, const void *pos, + u32 num_chunks) +{ + struct virtchnl2_config_rx_queues *crq = buf; + + crq->vport_id = cpu_to_le32(vport->vport_id); + crq->num_qinfo = cpu_to_le16(num_chunks); + memcpy(crq->qinfo, pos, num_chunks * sizeof(*crq->qinfo)); + + return struct_size(crq, qinfo, num_chunks); +} + +/** + * idpf_send_config_rx_queue_set_msg - Send virtchnl config rx queues + * message for selected rx queues only. + * @qs: params of the Rx queues + * + * Send config queues virtchnl message for queues contained in 'qs' array. + * The 'qs' array can contain Rx queues (or buffer queues) only. + * + * Return: 0 on success, -errno on failure. + */ +static int idpf_send_config_rx_queue_set_msg(const struct idpf_queue_set *qs) +{ + struct virtchnl2_rxq_info *qi __free(kfree) = NULL; + struct idpf_chunked_msg_params params = { + .vc_op = VIRTCHNL2_OP_CONFIG_RX_QUEUES, + .prepare_msg = idpf_prepare_cfg_rxqs_msg, + .config_sz = sizeof(struct virtchnl2_config_rx_queues), + .chunk_sz = sizeof(*qi), + }; + + qi = kcalloc(qs->num, sizeof(*qi), GFP_KERNEL); + if (!qi) + return -ENOMEM; + + params.chunks = qi; + + for (u32 i = 0; i < qs->num; i++) { + if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_RX) + idpf_fill_rxq_config_chunk(qs->vport, qs->qs[i].rxq, + &qi[params.num_chunks++]); + else if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_RX_BUFFER) + idpf_fill_bufq_config_chunk(qs->vport, qs->qs[i].bufq, + &qi[params.num_chunks++]); + } + + return idpf_send_chunked_msg(qs->vport, ¶ms); } /** @@ -1552,107 +1859,41 @@ static int idpf_send_config_tx_queues_msg(struct idpf_vport *vport) */ static int idpf_send_config_rx_queues_msg(struct idpf_vport *vport) { - struct virtchnl2_config_rx_queues *crq __free(kfree) = NULL; - struct virtchnl2_rxq_info *qi __free(kfree) = NULL; - struct idpf_vc_xn_params xn_params = {}; - u32 config_sz, chunk_sz, buf_sz; - int totqs, num_msgs, num_chunks; - ssize_t reply_sz; - int i, k = 0; + bool splitq = idpf_is_queue_model_split(vport->rxq_model); + struct idpf_queue_set *qs __free(kfree) = NULL; + u32 totqs = vport->num_rxq + vport->num_bufq; + u32 k = 0; - totqs = vport->num_rxq + vport->num_bufq; - qi = kcalloc(totqs, sizeof(struct virtchnl2_rxq_info), GFP_KERNEL); - if (!qi) + qs = idpf_alloc_queue_set(vport, totqs); + if (!qs) return -ENOMEM; /* Populate the queue info buffer with all queue context info */ - for (i = 0; i < vport->num_rxq_grp; i++) { - struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; - u16 num_rxq; - int j; - - if (!idpf_is_queue_model_split(vport->rxq_model)) - goto setup_rxqs; - - for (j = 0; j < vport->num_bufqs_per_qgrp; j++, k++) { - struct idpf_buf_queue *bufq = - &rx_qgrp->splitq.bufq_sets[j].bufq; - - qi[k].queue_id = cpu_to_le32(bufq->q_id); - qi[k].model = cpu_to_le16(vport->rxq_model); - qi[k].type = - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); - qi[k].desc_ids = cpu_to_le64(VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M); - qi[k].ring_len = cpu_to_le16(bufq->desc_count); - qi[k].dma_ring_addr = cpu_to_le64(bufq->dma); - qi[k].data_buffer_size = cpu_to_le32(bufq->rx_buf_size); - qi[k].buffer_notif_stride = IDPF_RX_BUF_STRIDE; - qi[k].rx_buffer_low_watermark = - cpu_to_le16(bufq->rx_buffer_low_watermark); - if (idpf_is_feature_ena(vport, NETIF_F_GRO_HW)) - qi[k].qflags |= cpu_to_le16(VIRTCHNL2_RXQ_RSC); - } + for (u32 i = 0; i < vport->num_rxq_grp; i++) { + const struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; + u32 num_rxq; -setup_rxqs: - if (idpf_is_queue_model_split(vport->rxq_model)) - num_rxq = rx_qgrp->splitq.num_rxq_sets; - else + if (!splitq) { num_rxq = rx_qgrp->singleq.num_rxq; + goto rxq; + } - for (j = 0; j < num_rxq; j++, k++) { - const struct idpf_bufq_set *sets; - struct idpf_rx_queue *rxq; - u32 rxdids; - - if (!idpf_is_queue_model_split(vport->rxq_model)) { - rxq = rx_qgrp->singleq.rxqs[j]; - rxdids = rxq->rxdids; - - goto common_qi_fields; - } - - rxq = &rx_qgrp->splitq.rxq_sets[j]->rxq; - sets = rxq->bufq_sets; + for (u32 j = 0; j < vport->num_bufqs_per_qgrp; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; + qs->qs[k++].bufq = &rx_qgrp->splitq.bufq_sets[j].bufq; + } - /* In splitq mode, RXQ buffer size should be - * set to that of the first buffer queue - * associated with this RXQ. - */ - rxq->rx_buf_size = sets[0].bufq.rx_buf_size; + num_rxq = rx_qgrp->splitq.num_rxq_sets; - qi[k].rx_bufq1_id = cpu_to_le16(sets[0].bufq.q_id); - if (vport->num_bufqs_per_qgrp > IDPF_SINGLE_BUFQ_PER_RXQ_GRP) { - qi[k].bufq2_ena = IDPF_BUFQ2_ENA; - qi[k].rx_bufq2_id = - cpu_to_le16(sets[1].bufq.q_id); - } - qi[k].rx_buffer_low_watermark = - cpu_to_le16(rxq->rx_buffer_low_watermark); - if (idpf_is_feature_ena(vport, NETIF_F_GRO_HW)) - qi[k].qflags |= cpu_to_le16(VIRTCHNL2_RXQ_RSC); - - rxq->rx_hbuf_size = sets[0].bufq.rx_hbuf_size; - - if (idpf_queue_has(HSPLIT_EN, rxq)) { - qi[k].qflags |= - cpu_to_le16(VIRTCHNL2_RXQ_HDR_SPLIT); - qi[k].hdr_buffer_size = - cpu_to_le16(rxq->rx_hbuf_size); - } +rxq: + for (u32 j = 0; j < num_rxq; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX; - rxdids = VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M; - -common_qi_fields: - qi[k].queue_id = cpu_to_le32(rxq->q_id); - qi[k].model = cpu_to_le16(vport->rxq_model); - qi[k].type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); - qi[k].ring_len = cpu_to_le16(rxq->desc_count); - qi[k].dma_ring_addr = cpu_to_le64(rxq->dma); - qi[k].max_pkt_size = cpu_to_le32(rxq->rx_max_pkt_size); - qi[k].data_buffer_size = cpu_to_le32(rxq->rx_buf_size); - qi[k].qflags |= - cpu_to_le16(VIRTCHNL2_RX_DESC_SIZE_32BYTE); - qi[k].desc_ids = cpu_to_le64(rxdids); + if (splitq) + qs->qs[k++].rxq = + &rx_qgrp->splitq.rxq_sets[j]->rxq; + else + qs->qs[k++].rxq = rx_qgrp->singleq.rxqs[j]; } } @@ -1660,339 +1901,396 @@ static int idpf_send_config_rx_queues_msg(struct idpf_vport *vport) if (k != totqs) return -EINVAL; - /* Chunk up the queue contexts into multiple messages to avoid - * sending a control queue message buffer that is too large - */ - config_sz = sizeof(struct virtchnl2_config_rx_queues); - chunk_sz = sizeof(struct virtchnl2_rxq_info); + return idpf_send_config_rx_queue_set_msg(qs); +} - num_chunks = min_t(u32, IDPF_NUM_CHUNKS_PER_MSG(config_sz, chunk_sz), - totqs); - num_msgs = DIV_ROUND_UP(totqs, num_chunks); +/** + * idpf_prepare_ena_dis_qs_msg - Prepare message to enable/disable selected + * queues. + * @vport: virtual port data structure + * @buf: buffer containing the message + * @pos: pointer to the first chunk describing the queue + * @num_chunks: number of chunks in the message + * + * Helper function for preparing the message describing queues to be enabled + * or disabled. + * Returns the total size of the prepared message. + */ +static u32 idpf_prepare_ena_dis_qs_msg(const struct idpf_vport *vport, + void *buf, const void *pos, + u32 num_chunks) +{ + struct virtchnl2_del_ena_dis_queues *eq = buf; + + eq->vport_id = cpu_to_le32(vport->vport_id); + eq->chunks.num_chunks = cpu_to_le16(num_chunks); + memcpy(eq->chunks.chunks, pos, + num_chunks * sizeof(*eq->chunks.chunks)); + + return struct_size(eq, chunks.chunks, num_chunks); +} - buf_sz = struct_size(crq, qinfo, num_chunks); - crq = kzalloc(buf_sz, GFP_KERNEL); - if (!crq) +/** + * idpf_send_ena_dis_queue_set_msg - Send virtchnl enable or disable + * queues message for selected queues only + * @qs: params of the queues + * @en: whether to enable or disable queues + * + * Send enable or disable queues virtchnl message for queues contained + * in 'qs' array. + * The 'qs' array can contain pointers to both rx and tx queues. + * + * Return: 0 on success, -errno on failure. + */ +static int idpf_send_ena_dis_queue_set_msg(const struct idpf_queue_set *qs, + bool en) +{ + struct virtchnl2_queue_chunk *qc __free(kfree) = NULL; + struct idpf_chunked_msg_params params = { + .vc_op = en ? VIRTCHNL2_OP_ENABLE_QUEUES : + VIRTCHNL2_OP_DISABLE_QUEUES, + .timeout_ms = en ? 0 : IDPF_VC_XN_MIN_TIMEOUT_MSEC, + .prepare_msg = idpf_prepare_ena_dis_qs_msg, + .config_sz = sizeof(struct virtchnl2_del_ena_dis_queues), + .chunk_sz = sizeof(*qc), + .num_chunks = qs->num, + }; + + qc = kcalloc(qs->num, sizeof(*qc), GFP_KERNEL); + if (!qc) return -ENOMEM; - xn_params.vc_op = VIRTCHNL2_OP_CONFIG_RX_QUEUES; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; + params.chunks = qc; - for (i = 0, k = 0; i < num_msgs; i++) { - memset(crq, 0, buf_sz); - crq->vport_id = cpu_to_le32(vport->vport_id); - crq->num_qinfo = cpu_to_le16(num_chunks); - memcpy(crq->qinfo, &qi[k], chunk_sz * num_chunks); + for (u32 i = 0; i < qs->num; i++) { + const struct idpf_queue_ptr *q = &qs->qs[i]; + u32 qid; - xn_params.send_buf.iov_base = crq; - xn_params.send_buf.iov_len = buf_sz; - reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; + qc[i].type = cpu_to_le32(q->type); + qc[i].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); - k += num_chunks; - totqs -= num_chunks; - num_chunks = min(num_chunks, totqs); - /* Recalculate buffer size */ - buf_sz = struct_size(crq, qinfo, num_chunks); + switch (q->type) { + case VIRTCHNL2_QUEUE_TYPE_RX: + qid = q->rxq->q_id; + break; + case VIRTCHNL2_QUEUE_TYPE_TX: + qid = q->txq->q_id; + break; + case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: + qid = q->bufq->q_id; + break; + case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: + qid = q->complq->q_id; + break; + default: + return -EINVAL; + } + + qc[i].start_queue_id = cpu_to_le32(qid); } - return 0; + return idpf_send_chunked_msg(qs->vport, ¶ms); } /** * idpf_send_ena_dis_queues_msg - Send virtchnl enable or disable * queues message * @vport: virtual port data structure - * @ena: if true enable, false disable + * @en: whether to enable or disable queues * - * Send enable or disable queues virtchnl message. Returns 0 on success, - * negative on failure. + * Return: 0 on success, -errno on failure. */ -static int idpf_send_ena_dis_queues_msg(struct idpf_vport *vport, bool ena) +static int idpf_send_ena_dis_queues_msg(struct idpf_vport *vport, bool en) { - struct virtchnl2_del_ena_dis_queues *eq __free(kfree) = NULL; - struct virtchnl2_queue_chunk *qc __free(kfree) = NULL; - u32 num_msgs, num_chunks, num_txq, num_rxq, num_q; - struct idpf_vc_xn_params xn_params = {}; - struct virtchnl2_queue_chunks *qcs; - u32 config_sz, chunk_sz, buf_sz; - ssize_t reply_sz; - int i, j, k = 0; + struct idpf_queue_set *qs __free(kfree) = NULL; + u32 num_txq, num_q, k = 0; + bool split; num_txq = vport->num_txq + vport->num_complq; - num_rxq = vport->num_rxq + vport->num_bufq; - num_q = num_txq + num_rxq; - buf_sz = sizeof(struct virtchnl2_queue_chunk) * num_q; - qc = kzalloc(buf_sz, GFP_KERNEL); - if (!qc) + num_q = num_txq + vport->num_rxq + vport->num_bufq; + + qs = idpf_alloc_queue_set(vport, num_q); + if (!qs) return -ENOMEM; - for (i = 0; i < vport->num_txq_grp; i++) { - struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; + split = idpf_is_queue_model_split(vport->txq_model); - for (j = 0; j < tx_qgrp->num_txq; j++, k++) { - qc[k].type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); - qc[k].start_queue_id = cpu_to_le32(tx_qgrp->txqs[j]->q_id); - qc[k].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); - } - } - if (vport->num_txq != k) - return -EINVAL; + for (u32 i = 0; i < vport->num_txq_grp; i++) { + const struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; - if (!idpf_is_queue_model_split(vport->txq_model)) - goto setup_rx; + for (u32 j = 0; j < tx_qgrp->num_txq; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX; + qs->qs[k++].txq = tx_qgrp->txqs[j]; + } - for (i = 0; i < vport->num_txq_grp; i++, k++) { - struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; + if (!split) + continue; - qc[k].type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); - qc[k].start_queue_id = cpu_to_le32(tx_qgrp->complq->q_id); - qc[k].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; + qs->qs[k++].complq = tx_qgrp->complq; } - if (vport->num_complq != (k - vport->num_txq)) + + if (k != num_txq) return -EINVAL; -setup_rx: - for (i = 0; i < vport->num_rxq_grp; i++) { - struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; + split = idpf_is_queue_model_split(vport->rxq_model); - if (idpf_is_queue_model_split(vport->rxq_model)) + for (u32 i = 0; i < vport->num_rxq_grp; i++) { + const struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; + u32 num_rxq; + + if (split) num_rxq = rx_qgrp->splitq.num_rxq_sets; else num_rxq = rx_qgrp->singleq.num_rxq; - for (j = 0; j < num_rxq; j++, k++) { - if (idpf_is_queue_model_split(vport->rxq_model)) { - qc[k].start_queue_id = - cpu_to_le32(rx_qgrp->splitq.rxq_sets[j]->rxq.q_id); - qc[k].type = - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); - } else { - qc[k].start_queue_id = - cpu_to_le32(rx_qgrp->singleq.rxqs[j]->q_id); - qc[k].type = - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); - } - qc[k].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); - } - } - if (vport->num_rxq != k - (vport->num_txq + vport->num_complq)) - return -EINVAL; - - if (!idpf_is_queue_model_split(vport->rxq_model)) - goto send_msg; + for (u32 j = 0; j < num_rxq; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX; - for (i = 0; i < vport->num_rxq_grp; i++) { - struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; + if (split) + qs->qs[k++].rxq = + &rx_qgrp->splitq.rxq_sets[j]->rxq; + else + qs->qs[k++].rxq = rx_qgrp->singleq.rxqs[j]; + } - for (j = 0; j < vport->num_bufqs_per_qgrp; j++, k++) { - const struct idpf_buf_queue *q; + if (!split) + continue; - q = &rx_qgrp->splitq.bufq_sets[j].bufq; - qc[k].type = - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); - qc[k].start_queue_id = cpu_to_le32(q->q_id); - qc[k].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); + for (u32 j = 0; j < vport->num_bufqs_per_qgrp; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; + qs->qs[k++].bufq = &rx_qgrp->splitq.bufq_sets[j].bufq; } } - if (vport->num_bufq != k - (vport->num_txq + - vport->num_complq + - vport->num_rxq)) - return -EINVAL; - -send_msg: - /* Chunk up the queue info into multiple messages */ - config_sz = sizeof(struct virtchnl2_del_ena_dis_queues); - chunk_sz = sizeof(struct virtchnl2_queue_chunk); - - num_chunks = min_t(u32, IDPF_NUM_CHUNKS_PER_MSG(config_sz, chunk_sz), - num_q); - num_msgs = DIV_ROUND_UP(num_q, num_chunks); - buf_sz = struct_size(eq, chunks.chunks, num_chunks); - eq = kzalloc(buf_sz, GFP_KERNEL); - if (!eq) - return -ENOMEM; + if (k != num_q) + return -EINVAL; - if (ena) { - xn_params.vc_op = VIRTCHNL2_OP_ENABLE_QUEUES; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - } else { - xn_params.vc_op = VIRTCHNL2_OP_DISABLE_QUEUES; - xn_params.timeout_ms = IDPF_VC_XN_MIN_TIMEOUT_MSEC; - } + return idpf_send_ena_dis_queue_set_msg(qs, en); +} - for (i = 0, k = 0; i < num_msgs; i++) { - memset(eq, 0, buf_sz); - eq->vport_id = cpu_to_le32(vport->vport_id); - eq->chunks.num_chunks = cpu_to_le16(num_chunks); - qcs = &eq->chunks; - memcpy(qcs->chunks, &qc[k], chunk_sz * num_chunks); +/** + * idpf_prep_map_unmap_queue_set_vector_msg - Prepare message to map or unmap + * queue set to the interrupt vector. + * @vport: virtual port data structure + * @buf: buffer containing the message + * @pos: pointer to the first chunk describing the vector mapping + * @num_chunks: number of chunks in the message + * + * Helper function for preparing the message describing mapping queues to + * q_vectors. + * Returns the total size of the prepared message. + */ +static u32 +idpf_prep_map_unmap_queue_set_vector_msg(const struct idpf_vport *vport, + void *buf, const void *pos, + u32 num_chunks) +{ + struct virtchnl2_queue_vector_maps *vqvm = buf; - xn_params.send_buf.iov_base = eq; - xn_params.send_buf.iov_len = buf_sz; - reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; + vqvm->vport_id = cpu_to_le32(vport->vport_id); + vqvm->num_qv_maps = cpu_to_le16(num_chunks); + memcpy(vqvm->qv_maps, pos, num_chunks * sizeof(*vqvm->qv_maps)); - k += num_chunks; - num_q -= num_chunks; - num_chunks = min(num_chunks, num_q); - /* Recalculate buffer size */ - buf_sz = struct_size(eq, chunks.chunks, num_chunks); - } - - return 0; + return struct_size(vqvm, qv_maps, num_chunks); } /** - * idpf_send_map_unmap_queue_vector_msg - Send virtchnl map or unmap queue - * vector message - * @vport: virtual port data structure + * idpf_send_map_unmap_queue_set_vector_msg - Send virtchnl map or unmap + * queue set vector message + * @qs: params of the queues * @map: true for map and false for unmap * - * Send map or unmap queue vector virtchnl message. Returns 0 on success, - * negative on failure. + * Return: 0 on success, -errno on failure. */ -int idpf_send_map_unmap_queue_vector_msg(struct idpf_vport *vport, bool map) +static int +idpf_send_map_unmap_queue_set_vector_msg(const struct idpf_queue_set *qs, + bool map) { - struct virtchnl2_queue_vector_maps *vqvm __free(kfree) = NULL; struct virtchnl2_queue_vector *vqv __free(kfree) = NULL; - struct idpf_vc_xn_params xn_params = {}; - u32 config_sz, chunk_sz, buf_sz; - u32 num_msgs, num_chunks, num_q; - ssize_t reply_sz; - int i, j, k = 0; - - num_q = vport->num_txq + vport->num_rxq; + struct idpf_chunked_msg_params params = { + .vc_op = map ? VIRTCHNL2_OP_MAP_QUEUE_VECTOR : + VIRTCHNL2_OP_UNMAP_QUEUE_VECTOR, + .timeout_ms = map ? 0 : IDPF_VC_XN_MIN_TIMEOUT_MSEC, + .prepare_msg = idpf_prep_map_unmap_queue_set_vector_msg, + .config_sz = sizeof(struct virtchnl2_queue_vector_maps), + .chunk_sz = sizeof(*vqv), + .num_chunks = qs->num, + }; + bool split; - buf_sz = sizeof(struct virtchnl2_queue_vector) * num_q; - vqv = kzalloc(buf_sz, GFP_KERNEL); + vqv = kcalloc(qs->num, sizeof(*vqv), GFP_KERNEL); if (!vqv) return -ENOMEM; - for (i = 0; i < vport->num_txq_grp; i++) { - struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; + params.chunks = vqv; + + split = idpf_is_queue_model_split(qs->vport->txq_model); - for (j = 0; j < tx_qgrp->num_txq; j++, k++) { - const struct idpf_tx_queue *txq = tx_qgrp->txqs[j]; - const struct idpf_q_vector *vec; - u32 v_idx, tx_itr_idx; + for (u32 i = 0; i < qs->num; i++) { + const struct idpf_queue_ptr *q = &qs->qs[i]; + const struct idpf_q_vector *vec; + u32 qid, v_idx, itr_idx; - vqv[k].queue_type = - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); - vqv[k].queue_id = cpu_to_le32(txq->q_id); + vqv[i].queue_type = cpu_to_le32(q->type); - if (idpf_queue_has(NOIRQ, txq)) + switch (q->type) { + case VIRTCHNL2_QUEUE_TYPE_RX: + qid = q->rxq->q_id; + + if (idpf_queue_has(NOIRQ, q->rxq)) vec = NULL; - else if (idpf_queue_has(XDP, txq)) - vec = txq->complq->q_vector; - else if (idpf_is_queue_model_split(vport->txq_model)) - vec = txq->txq_grp->complq->q_vector; else - vec = txq->q_vector; + vec = q->rxq->q_vector; if (vec) { v_idx = vec->v_idx; - tx_itr_idx = vec->tx_itr_idx; + itr_idx = vec->rx_itr_idx; } else { v_idx = 0; - tx_itr_idx = VIRTCHNL2_ITR_IDX_1; + itr_idx = VIRTCHNL2_ITR_IDX_0; } + break; + case VIRTCHNL2_QUEUE_TYPE_TX: + qid = q->txq->q_id; + + if (idpf_queue_has(NOIRQ, q->txq)) + vec = NULL; + else if (idpf_queue_has(XDP, q->txq)) + vec = q->txq->complq->q_vector; + else if (split) + vec = q->txq->txq_grp->complq->q_vector; + else + vec = q->txq->q_vector; - vqv[k].vector_id = cpu_to_le16(v_idx); - vqv[k].itr_idx = cpu_to_le32(tx_itr_idx); + if (vec) { + v_idx = vec->v_idx; + itr_idx = vec->tx_itr_idx; + } else { + v_idx = 0; + itr_idx = VIRTCHNL2_ITR_IDX_1; + } + break; + default: + return -EINVAL; } + + vqv[i].queue_id = cpu_to_le32(qid); + vqv[i].vector_id = cpu_to_le16(v_idx); + vqv[i].itr_idx = cpu_to_le32(itr_idx); } - if (vport->num_txq != k) + return idpf_send_chunked_msg(qs->vport, ¶ms); +} + +/** + * idpf_send_map_unmap_queue_vector_msg - Send virtchnl map or unmap queue + * vector message + * @vport: virtual port data structure + * @map: true for map and false for unmap + * + * Send map or unmap queue vector virtchnl message. Returns 0 on success, + * negative on failure. + */ +int idpf_send_map_unmap_queue_vector_msg(struct idpf_vport *vport, bool map) +{ + struct idpf_queue_set *qs __free(kfree) = NULL; + u32 num_q = vport->num_txq + vport->num_rxq; + u32 k = 0; + + qs = idpf_alloc_queue_set(vport, num_q); + if (!qs) + return -ENOMEM; + + for (u32 i = 0; i < vport->num_txq_grp; i++) { + const struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; + + for (u32 j = 0; j < tx_qgrp->num_txq; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX; + qs->qs[k++].txq = tx_qgrp->txqs[j]; + } + } + + if (k != vport->num_txq) return -EINVAL; - for (i = 0; i < vport->num_rxq_grp; i++) { - struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; - u16 num_rxq; + for (u32 i = 0; i < vport->num_rxq_grp; i++) { + const struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; + u32 num_rxq; if (idpf_is_queue_model_split(vport->rxq_model)) num_rxq = rx_qgrp->splitq.num_rxq_sets; else num_rxq = rx_qgrp->singleq.num_rxq; - for (j = 0; j < num_rxq; j++, k++) { - struct idpf_rx_queue *rxq; - u32 v_idx, rx_itr_idx; + for (u32 j = 0; j < num_rxq; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX; if (idpf_is_queue_model_split(vport->rxq_model)) - rxq = &rx_qgrp->splitq.rxq_sets[j]->rxq; + qs->qs[k++].rxq = + &rx_qgrp->splitq.rxq_sets[j]->rxq; else - rxq = rx_qgrp->singleq.rxqs[j]; - - vqv[k].queue_type = - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); - vqv[k].queue_id = cpu_to_le32(rxq->q_id); - - if (idpf_queue_has(NOIRQ, rxq)) { - v_idx = 0; - rx_itr_idx = VIRTCHNL2_ITR_IDX_0; - } else { - v_idx = rxq->q_vector->v_idx; - rx_itr_idx = rxq->q_vector->rx_itr_idx; - } - - vqv[k].vector_id = cpu_to_le16(v_idx); - vqv[k].itr_idx = cpu_to_le32(rx_itr_idx); + qs->qs[k++].rxq = rx_qgrp->singleq.rxqs[j]; } } - if (idpf_is_queue_model_split(vport->txq_model)) { - if (vport->num_rxq != k - vport->num_complq) - return -EINVAL; - } else { - if (vport->num_rxq != k - vport->num_txq) - return -EINVAL; - } + if (k != num_q) + return -EINVAL; - /* Chunk up the vector info into multiple messages */ - config_sz = sizeof(struct virtchnl2_queue_vector_maps); - chunk_sz = sizeof(struct virtchnl2_queue_vector); + return idpf_send_map_unmap_queue_set_vector_msg(qs, map); +} - num_chunks = min_t(u32, IDPF_NUM_CHUNKS_PER_MSG(config_sz, chunk_sz), - num_q); - num_msgs = DIV_ROUND_UP(num_q, num_chunks); +/** + * idpf_send_enable_queue_set_msg - send enable queues virtchnl message for + * selected queues only + * @qs: params of the queues + * + * Will send enable queues virtchnl message for queues contained in 'qs' table. + * + * Return: 0 on success, -errno on failure. + */ +int idpf_send_enable_queue_set_msg(const struct idpf_queue_set *qs) +{ + return idpf_send_ena_dis_queue_set_msg(qs, true); +} - buf_sz = struct_size(vqvm, qv_maps, num_chunks); - vqvm = kzalloc(buf_sz, GFP_KERNEL); - if (!vqvm) - return -ENOMEM; +/** + * idpf_send_disable_queue_set_msg - send disable queues virtchnl message + * for selected queues only + * @qs: params of the queues + * + * Return: 0 on success, -errno on failure. + */ +int idpf_send_disable_queue_set_msg(const struct idpf_queue_set *qs) +{ + int err; - if (map) { - xn_params.vc_op = VIRTCHNL2_OP_MAP_QUEUE_VECTOR; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - } else { - xn_params.vc_op = VIRTCHNL2_OP_UNMAP_QUEUE_VECTOR; - xn_params.timeout_ms = IDPF_VC_XN_MIN_TIMEOUT_MSEC; - } + err = idpf_send_ena_dis_queue_set_msg(qs, false); + if (err) + return err; - for (i = 0, k = 0; i < num_msgs; i++) { - memset(vqvm, 0, buf_sz); - xn_params.send_buf.iov_base = vqvm; - xn_params.send_buf.iov_len = buf_sz; - vqvm->vport_id = cpu_to_le32(vport->vport_id); - vqvm->num_qv_maps = cpu_to_le16(num_chunks); - memcpy(vqvm->qv_maps, &vqv[k], chunk_sz * num_chunks); + return idpf_wait_for_marker_event_set(qs); +} - reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; +/** + * idpf_send_config_queue_set_msg - Send virtchnl config queues message for + * selected queues only. + * @qs: params of the queues + * + * Send config queues virtchnl message for queues contained in 'qs' array. + * The 'qs' array can contain both RX or TX queues. + * + * Return: 0 on success, -errno on failure. + */ +int idpf_send_config_queue_set_msg(const struct idpf_queue_set *qs) +{ + int err; - k += num_chunks; - num_q -= num_chunks; - num_chunks = min(num_chunks, num_q); - /* Recalculate buffer size */ - buf_sz = struct_size(vqvm, qv_maps, num_chunks); - } + err = idpf_send_config_tx_queue_set_msg(qs); + if (err) + return err; - return 0; + return idpf_send_config_rx_queue_set_msg(qs); } /** diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h index 83da5d8da56bf2..f04053681a4da9 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h @@ -4,6 +4,8 @@ #ifndef _IDPF_VIRTCHNL_H_ #define _IDPF_VIRTCHNL_H_ +#include "virtchnl2.h" + struct idpf_adapter; struct idpf_netdev_priv; struct idpf_vec_regs; @@ -25,6 +27,33 @@ int idpf_recv_mb_msg(struct idpf_adapter *adapter); int idpf_send_mb_msg(struct idpf_adapter *adapter, u32 op, u16 msg_size, u8 *msg, u16 cookie); +struct idpf_queue_ptr { + enum virtchnl2_queue_type type; + union { + struct idpf_rx_queue *rxq; + struct idpf_tx_queue *txq; + struct idpf_buf_queue *bufq; + struct idpf_compl_queue *complq; + }; +}; + +struct idpf_queue_set { + struct idpf_vport *vport; + + u32 num; + struct idpf_queue_ptr qs[] __counted_by(num); +}; + +struct idpf_queue_set *idpf_alloc_queue_set(struct idpf_vport *vport, u32 num); + +int idpf_send_enable_queue_set_msg(const struct idpf_queue_set *qs); +int idpf_send_disable_queue_set_msg(const struct idpf_queue_set *qs); +int idpf_send_config_queue_set_msg(const struct idpf_queue_set *qs); + +int idpf_send_disable_queues_msg(struct idpf_vport *vport); +int idpf_send_config_queues_msg(struct idpf_vport *vport); +int idpf_send_enable_queues_msg(struct idpf_vport *vport); + void idpf_vport_init(struct idpf_vport *vport, struct idpf_vport_max_q *max_q); u32 idpf_get_vport_id(struct idpf_vport *vport); int idpf_send_create_vport_msg(struct idpf_adapter *adapter, @@ -41,9 +70,6 @@ void idpf_vport_dealloc_max_qs(struct idpf_adapter *adapter, int idpf_send_add_queues_msg(const struct idpf_vport *vport, u16 num_tx_q, u16 num_complq, u16 num_rx_q, u16 num_rx_bufq); int idpf_send_delete_queues_msg(struct idpf_vport *vport); -int idpf_send_enable_queues_msg(struct idpf_vport *vport); -int idpf_send_disable_queues_msg(struct idpf_vport *vport); -int idpf_send_config_queues_msg(struct idpf_vport *vport); int idpf_vport_alloc_vec_indexes(struct idpf_vport *vport); int idpf_get_vec_ids(struct idpf_adapter *adapter, From 62a153fbacfc74926e969511a74d50ab2c2792d2 Mon Sep 17 00:00:00 2001 From: Michal Kubiak Date: Mon, 27 Nov 2023 16:41:20 +0100 Subject: [PATCH 45/49] idpf: add XSk pool initialization Add functionality to setup an XSk buffer pool, including ability to stop, reconfig and start only selected queues, not the whole device. Pool DMA mapping is managed by libeth. Signed-off-by: Michal Kubiak Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/idpf/Makefile | 1 + drivers/net/ethernet/intel/idpf/idpf.h | 11 + .../net/ethernet/intel/idpf/idpf_ethtool.c | 8 +- drivers/net/ethernet/intel/idpf/idpf_txrx.c | 318 ++++++++++++++++++ drivers/net/ethernet/intel/idpf/idpf_txrx.h | 7 + drivers/net/ethernet/intel/idpf/xdp.c | 14 + drivers/net/ethernet/intel/idpf/xdp.h | 2 + drivers/net/ethernet/intel/idpf/xsk.c | 49 +++ drivers/net/ethernet/intel/idpf/xsk.h | 14 + 9 files changed, 420 insertions(+), 4 deletions(-) create mode 100644 drivers/net/ethernet/intel/idpf/xsk.c create mode 100644 drivers/net/ethernet/intel/idpf/xsk.h diff --git a/drivers/net/ethernet/intel/idpf/Makefile b/drivers/net/ethernet/intel/idpf/Makefile index c58abe6f8f5d4e..a03eb4451c57b9 100644 --- a/drivers/net/ethernet/intel/idpf/Makefile +++ b/drivers/net/ethernet/intel/idpf/Makefile @@ -19,3 +19,4 @@ idpf-y := \ idpf-$(CONFIG_IDPF_SINGLEQ) += idpf_singleq_txrx.o idpf-y += xdp.o +idpf-y += xsk.o diff --git a/drivers/net/ethernet/intel/idpf/idpf.h b/drivers/net/ethernet/intel/idpf/idpf.h index 0481c07f3ef0c9..0c17ea921f26c0 100644 --- a/drivers/net/ethernet/intel/idpf/idpf.h +++ b/drivers/net/ethernet/intel/idpf/idpf.h @@ -801,6 +801,17 @@ static inline void idpf_vport_ctrl_unlock(struct net_device *netdev) mutex_unlock(&np->adapter->vport_ctrl_lock); } +/** + * idpf_vport_ctrl_is_locked - Check if vport control lock is taken + * @netdev: Network interface device structure + */ +static inline bool idpf_vport_ctrl_is_locked(struct net_device *netdev) +{ + struct idpf_netdev_priv *np = netdev_priv(netdev); + + return mutex_is_locked(&np->adapter->vport_ctrl_lock); +} + void idpf_statistics_task(struct work_struct *work); void idpf_init_task(struct work_struct *work); void idpf_service_task(struct work_struct *work); diff --git a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c index d528b421e701d3..eda2266c7c296e 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c +++ b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c @@ -535,8 +535,8 @@ static void idpf_get_ethtool_stats(struct net_device *netdev, * * returns pointer to rx vector */ -static struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport, - int q_num) +struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport, + u32 q_num) { int q_grp, q_idx; @@ -556,8 +556,8 @@ static struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport, * * returns pointer to tx vector */ -static struct idpf_q_vector *idpf_find_txq_vec(const struct idpf_vport *vport, - int q_num) +struct idpf_q_vector *idpf_find_txq_vec(const struct idpf_vport *vport, + u32 q_num) { int q_grp; diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 4515c968bd074b..ca0fc9d465ad94 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -1017,6 +1017,324 @@ static int idpf_rx_desc_alloc_all(struct idpf_vport *vport) return err; } +/** + * idpf_init_queue_set - initialize all queues in a set + * @qs: set of queues to initialize + * + * Return: 0 if success, -errno otherwise. + */ +static int idpf_init_queue_set(const struct idpf_queue_set *qs) +{ + const struct idpf_vport *vport = qs->vport; + bool splitq; + int err; + + splitq = idpf_is_queue_model_split(vport->rxq_model); + + for (u32 i = 0; i < qs->num; i++) { + const struct idpf_queue_ptr *q = &qs->qs[i]; + struct idpf_buf_queue *bufq; + + switch (q->type) { + case VIRTCHNL2_QUEUE_TYPE_RX: + err = idpf_rx_desc_alloc(vport, q->rxq); + if (err) + break; + + err = idpf_xdp_rxq_info_init(q->rxq); + if (err) + break; + + if (!splitq) + err = idpf_rx_bufs_init_singleq(q->rxq); + + break; + case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: + bufq = q->bufq; + + err = idpf_bufq_desc_alloc(vport, bufq); + if (err) + break; + + for (u32 j = 0; j < bufq->q_vector->num_bufq; j++) { + struct idpf_buf_queue * const *bufqs; + enum libeth_fqe_type type; + u32 ts; + + bufqs = bufq->q_vector->bufq; + if (bufqs[j] != bufq) + continue; + + if (j) { + type = LIBETH_FQE_SHORT; + ts = bufqs[j - 1]->truesize >> 1; + } else { + type = LIBETH_FQE_MTU; + ts = 0; + } + + bufq->truesize = ts; + + err = idpf_rx_bufs_init(bufq, type); + break; + } + + break; + case VIRTCHNL2_QUEUE_TYPE_TX: + err = idpf_tx_desc_alloc(vport, q->txq); + break; + case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: + err = idpf_compl_desc_alloc(vport, q->complq); + break; + default: + continue; + } + + if (err) + return err; + } + + return 0; +} + +/** + * idpf_clean_queue_set - clean all queues in a set + * @qs: set of queues to clean + */ +static void idpf_clean_queue_set(const struct idpf_queue_set *qs) +{ + const struct idpf_vport *vport = qs->vport; + const struct net_device *dev = vport->netdev; + + for (u32 i = 0; i < qs->num; i++) { + const struct idpf_queue_ptr *q = &qs->qs[i]; + + switch (q->type) { + case VIRTCHNL2_QUEUE_TYPE_RX: + idpf_xdp_rxq_info_deinit(q->rxq, vport->rxq_model); + idpf_rx_desc_rel(q->rxq, dev, vport->rxq_model); + break; + case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: + idpf_rx_desc_rel_bufq(q->bufq, dev->dev.parent); + break; + case VIRTCHNL2_QUEUE_TYPE_TX: + idpf_tx_desc_rel(vport, q->txq); + + if (!idpf_queue_has(XDP, q->txq)) + q->txq->txq_grp->num_completions_pending = 0; + + writel(q->txq->next_to_use, q->txq->tail); + break; + case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: + idpf_compl_desc_rel(q->complq); + q->complq->num_completions = 0; + break; + default: + break; + } + } +} + +/** + * idpf_qvec_ena_irq - Enable IRQ for given queue vector + * @qv: queue vector + */ +static void idpf_qvec_ena_irq(struct idpf_q_vector *qv) +{ + if (qv->num_txq) { + u32 itr; + + if (IDPF_ITR_IS_DYNAMIC(qv->tx_intr_mode)) + itr = qv->vport->tx_itr_profile[qv->tx_dim.profile_ix]; + else + itr = qv->tx_itr_value; + + idpf_vport_intr_write_itr(qv, itr, true); + } + + if (qv->num_rxq) { + u32 itr; + + if (IDPF_ITR_IS_DYNAMIC(qv->rx_intr_mode)) + itr = qv->vport->rx_itr_profile[qv->rx_dim.profile_ix]; + else + itr = qv->rx_itr_value; + + idpf_vport_intr_write_itr(qv, itr, false); + } + + if (qv->num_txq || qv->num_rxq) + idpf_vport_intr_update_itr_ena_irq(qv); +} + +/** + * idpf_vector_to_queue_set - create a queue set associated with a given + * queue vector. + * @qv: queue vector corresponding to the queue pair + * + * Returns a pointer to a dynamically allocated array of pointers to all + * queues associated with a given queue vector (@qv). + * Please note that the caller is responsible to free the memory allocated + * by this function using kfree(). + * + * Return: &idpf_queue_set on success, %NULL in case of error. + */ +static struct idpf_queue_set * +idpf_vector_to_queue_set(struct idpf_q_vector *qv) +{ + bool xdp = qv->vport->xdp_txq_offset; + struct idpf_vport *vport = qv->vport; + struct idpf_queue_set *qs; + u32 num; + + num = qv->num_rxq + qv->num_bufq + qv->num_txq + qv->num_complq; + num += xdp ? qv->num_rxq * 2 : 0; + if (!num) + return NULL; + + qs = idpf_alloc_queue_set(vport, num); + if (!qs) + return NULL; + + num = 0; + + for (u32 i = 0; i < qv->num_bufq; i++) { + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; + qs->qs[num++].bufq = qv->bufq[i]; + } + + for (u32 i = 0; i < qv->num_rxq; i++) { + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_RX; + qs->qs[num++].rxq = qv->rx[i]; + } + + for (u32 i = 0; i < qv->num_txq; i++) { + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX; + qs->qs[num++].txq = qv->tx[i]; + } + + for (u32 i = 0; i < qv->num_complq; i++) { + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; + qs->qs[num++].complq = qv->complq[i]; + } + + if (!vport->xdp_txq_offset) + goto finalize; + + if (xdp) { + for (u32 i = 0; i < qv->num_rxq; i++) { + u32 idx = vport->xdp_txq_offset + qv->rx[i]->idx; + + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX; + qs->qs[num++].txq = vport->txqs[idx]; + + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; + qs->qs[num++].complq = vport->txqs[idx]->complq; + } + } + +finalize: + if (num != qs->num) { + kfree(qs); + return NULL; + } + + return qs; +} + +/** + * idpf_qp_disable - disable queues associated with a queue pair + * @qs: queue set to be disabled + * @qid: index of queue pair to disable + * + * Return: 0 on success, -errno on failure. + */ +static int idpf_qp_disable(struct idpf_queue_set *qs, u32 qid) +{ + struct idpf_vport *vport = qs->vport; + struct idpf_q_vector *q_vector; + int err; + + q_vector = idpf_find_rxq_vec(vport, qid); + netif_stop_subqueue(vport->netdev, qid); + + err = idpf_send_disable_queue_set_msg(qs); + if (err) { + netdev_err(vport->netdev, "Could not disable queues for index %d, error = %d\n", + qid, err); + return err; + } + + writel(0, q_vector->intr_reg.dyn_ctl); + napi_disable(&q_vector->napi); + + idpf_clean_queue_set(qs); + + return 0; +} + +/** + * idpf_qp_enable - enable queues associated with a queue pair + * @qs: queue set to enable + * @qid: index of queue pair to disable + * + * Return: 0 on success, -errno on failure. + */ +static int idpf_qp_enable(struct idpf_queue_set *qs, u32 qid) +{ + struct idpf_vport *vport = qs->vport; + struct idpf_q_vector *q_vector; + int err; + + q_vector = idpf_find_rxq_vec(vport, qid); + + err = idpf_init_queue_set(qs); + if (err) { + netdev_err(vport->netdev, "Could not initialize queues for index %d, error = %d\n", + qid, err); + return err; + } + + napi_enable(&q_vector->napi); + idpf_qvec_ena_irq(q_vector); + + err = idpf_send_config_queue_set_msg(qs); + if (err) { + netdev_err(vport->netdev, "Could not configure queues for index %d, error = %d\n", + qid, err); + return err; + } + + err = idpf_send_enable_queue_set_msg(qs); + if (err) { + netdev_err(vport->netdev, "Could not enable queues for index %d, error = %d\n", + qid, err); + return err; + } + + netif_start_subqueue(vport->netdev, qid); + + return 0; +} + +int idpf_qp_restart(struct idpf_vport *vport, u32 qid) +{ + struct idpf_q_vector *q_vector = idpf_find_rxq_vec(vport, qid); + struct idpf_queue_set *qs __free(kfree) = NULL; + int ret; + + if (idpf_find_txq_vec(vport, qid) != q_vector) + return -EINVAL; + + qs = idpf_vector_to_queue_set(q_vector); + if (!qs) + return -ENOMEM; + + ret = idpf_qp_disable(qs, qid); + + return ret ? : idpf_qp_enable(qs, qid); +} + /** * idpf_txq_group_rel - Release all resources for txq groups * @vport: vport to release txq groups on diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index beaf09c85f5fe4..6b460f8708822e 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -1069,6 +1069,13 @@ int idpf_config_rss(struct idpf_vport *vport); int idpf_init_rss(struct idpf_vport *vport); void idpf_deinit_rss(struct idpf_vport *vport); int idpf_rx_bufs_init_all(struct idpf_vport *vport); + +struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport, + u32 q_num); +struct idpf_q_vector *idpf_find_txq_vec(const struct idpf_vport *vport, + u32 q_num); +int idpf_qp_restart(struct idpf_vport *vport, u32 qid); + void idpf_tx_buf_hw_update(struct idpf_tx_queue *tx_q, u32 val, bool xmit_more); unsigned int idpf_size_to_txd_count(unsigned int size); diff --git a/drivers/net/ethernet/intel/idpf/xdp.c b/drivers/net/ethernet/intel/idpf/xdp.c index 11ef3216bef58e..4dd5aae69ed349 100644 --- a/drivers/net/ethernet/intel/idpf/xdp.c +++ b/drivers/net/ethernet/intel/idpf/xdp.c @@ -4,6 +4,7 @@ #include "idpf.h" #include "idpf_virtchnl.h" #include "xdp.h" +#include "xsk.h" static int idpf_rxq_for_each(const struct idpf_vport *vport, int (*fn)(struct idpf_rx_queue *rxq, void *arg), @@ -73,6 +74,11 @@ static int __idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq, void *arg) return 0; } +int idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq) +{ + return __idpf_xdp_rxq_info_init(rxq, NULL); +} + /** * idpf_xdp_rxq_info_init_all - initialize RxQ info for all Rx queues in vport * @vport: vport to setup the info @@ -104,6 +110,11 @@ static int __idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, void *arg) return 0; } +void idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, u32 model) +{ + __idpf_xdp_rxq_info_deinit(rxq, (void *)(size_t)model); +} + /** * idpf_xdp_rxq_info_deinit_all - deinit RxQ info for all Rx queues in vport * @vport: vport to setup the info @@ -485,6 +496,9 @@ int idpf_xdp(struct net_device *dev, struct netdev_bpf *xdp) case XDP_SETUP_PROG: ret = idpf_xdp_setup_prog(vport, xdp); break; + case XDP_SETUP_XSK_POOL: + ret = idpf_xsk_pool_setup(vport, xdp); + break; default: notsupp: ret = -EOPNOTSUPP; diff --git a/drivers/net/ethernet/intel/idpf/xdp.h b/drivers/net/ethernet/intel/idpf/xdp.h index 52783a5c8e0fb2..02a42775d77cc3 100644 --- a/drivers/net/ethernet/intel/idpf/xdp.h +++ b/drivers/net/ethernet/intel/idpf/xdp.h @@ -8,7 +8,9 @@ #include "idpf_txrx.h" +int idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq); int idpf_xdp_rxq_info_init_all(const struct idpf_vport *vport); +void idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, u32 model); void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport); void idpf_copy_xdp_prog_to_qs(const struct idpf_vport *vport, struct bpf_prog *xdp_prog); diff --git a/drivers/net/ethernet/intel/idpf/xsk.c b/drivers/net/ethernet/intel/idpf/xsk.c new file mode 100644 index 00000000000000..df30eef6a99ac6 --- /dev/null +++ b/drivers/net/ethernet/intel/idpf/xsk.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2024 Intel Corporation */ + +#include + +#include "idpf.h" +#include "xsk.h" + +/** + * idpf_xsk_pool_setup - set up an XSk pool + * @vport: current vport of interest + * @bpf: pointer to the pool data + * + * Return: 0 on success, -errno on failure. + */ +int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *bpf) +{ + struct xsk_buff_pool *pool = bpf->xsk.pool; + u32 qid = bpf->xsk.queue_id; + bool restart; + int ret; + + ret = libeth_xsk_setup_pool(vport->netdev, qid, pool); + if (ret) { + NL_SET_ERR_MSG_FMT_MOD(bpf->extack, + "%s: failed to configure XSk pool for pair %u: %pe", + netdev_name(vport->netdev), qid, &ret); + return ret; + } + + restart = netif_running(vport->netdev) && idpf_xdp_is_prog_ena(vport); + if (!restart) + return 0; + + ret = idpf_qp_restart(vport, qid); + if (ret) { + NL_SET_ERR_MSG_FMT_MOD(bpf->extack, + "%s: failed to reconfigure queue pair %u: %pe", + netdev_name(vport->netdev), qid, &ret); + goto err_dis; + } + + return 0; + +err_dis: + libeth_xsk_setup_pool(vport->netdev, qid, false); + + return ret; +} diff --git a/drivers/net/ethernet/intel/idpf/xsk.h b/drivers/net/ethernet/intel/idpf/xsk.h new file mode 100644 index 00000000000000..7168fd7655eb7f --- /dev/null +++ b/drivers/net/ethernet/intel/idpf/xsk.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2024 Intel Corporation */ + +#ifndef _IDPF_XSK_H_ +#define _IDPF_XSK_H_ + +#include + +struct idpf_vport; +struct netdev_bpf; + +int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *xdp); + +#endif /* !_IDPF_XSK_H_ */ From af817d92c69dfa4464e2d61fc29ce4de3ddc19d5 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Thu, 7 Dec 2023 13:13:36 +0100 Subject: [PATCH 46/49] idpf: implement Tx path for AF_XDP Implement Tx handling for AF_XDP feature in zero-copy mode using the libeth (libeth_xdp) XSk infra. When the NAPI poll is called, XSk Tx queues are polled first, before regular Tx and Rx. They're generally faster to serve and have higher priority comparing to regular traffic. Co-developed-by: Michal Kubiak Signed-off-by: Michal Kubiak Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/idpf/idpf_txrx.c | 150 ++++++++--- drivers/net/ethernet/intel/idpf/idpf_txrx.h | 14 +- drivers/net/ethernet/intel/idpf/xdp.c | 2 +- drivers/net/ethernet/intel/idpf/xdp.h | 1 + drivers/net/ethernet/intel/idpf/xsk.c | 261 ++++++++++++++++++++ drivers/net/ethernet/intel/idpf/xsk.h | 9 + 6 files changed, 402 insertions(+), 35 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index ca0fc9d465ad94..5906b2f7c4a01a 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -4,6 +4,7 @@ #include "idpf.h" #include "idpf_virtchnl.h" #include "xdp.h" +#include "xsk.h" struct idpf_tx_stash { struct hlist_node hlist; @@ -90,14 +91,9 @@ static void idpf_sq_stats_deinit(const struct idpf_vport *vport, } } -/** - * idpf_tx_buf_rel_all - Free any empty Tx buffers - * @txq: queue to be cleaned - */ -static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) +static void idpf_tx_buf_clean(struct idpf_tx_queue *txq) { struct libeth_sq_napi_stats ss = { }; - struct idpf_buf_lifo *buf_stack; struct idpf_tx_stash *stash; struct xdp_frame_bulk bq; struct libeth_cq_pp cp = { @@ -106,31 +102,17 @@ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) .ss = &ss, }; struct hlist_node *tmp; - u16 i, tag; - - /* Buffers already cleared, nothing to do */ - if (!txq->tx_buf) - return; + u16 tag; xdp_frame_bulk_init(&bq); rcu_read_lock(); /* Free all the Tx buffer sk_buffs */ - for (i = 0; i < txq->desc_count; i++) + for (u32 i = 0; i < txq->desc_count; i++) libeth_tx_complete_any(&txq->tx_buf[i], &cp); - xdp_flush_frame_bulk(&bq); - rcu_read_unlock(); - - kfree(txq->tx_buf); - txq->tx_buf = NULL; - if (!idpf_queue_has(FLOW_SCH_EN, txq)) - return; - - buf_stack = &txq->stash->buf_stack; - if (!buf_stack->bufs) - return; + goto unlock; /* If a TX timeout occurred, there are potentially still bufs in the * hash table, free them here. @@ -142,10 +124,42 @@ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) libeth_tx_complete(&stash->buf, &cp); hash_del(&stash->hlist); - idpf_buf_lifo_push(buf_stack, stash); + idpf_buf_lifo_push(&txq->stash->buf_stack, stash); } - for (i = 0; i < buf_stack->size; i++) +unlock: + xdp_flush_frame_bulk(&bq); + rcu_read_unlock(); +} + +/** + * idpf_tx_buf_rel_all - Free any empty Tx buffers + * @txq: queue to be cleaned + */ +static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) +{ + struct idpf_buf_lifo *buf_stack; + + /* Buffers already cleared, nothing to do */ + if (!txq->tx_buf) + return; + + if (idpf_queue_has(XSK, txq)) + idpf_xsk_clean_xdpq(txq); + else + idpf_tx_buf_clean(txq); + + kfree(txq->tx_buf); + txq->tx_buf = NULL; + + if (!idpf_queue_has(FLOW_SCH_EN, txq)) + return; + + buf_stack = &txq->stash->buf_stack; + if (!buf_stack->bufs) + return; + + for (u32 i = 0; i < buf_stack->size; i++) kfree(buf_stack->bufs[i]); kfree(buf_stack->bufs); @@ -163,7 +177,9 @@ static void idpf_tx_desc_rel(const struct idpf_vport *vport, struct idpf_tx_queue *txq) { idpf_tx_buf_rel_all(txq); + idpf_sq_stats_deinit(vport, txq); + idpf_xsk_clear_queue(txq, VIRTCHNL2_QUEUE_TYPE_TX); if (!txq->desc_ring) return; @@ -182,6 +198,8 @@ static void idpf_tx_desc_rel(const struct idpf_vport *vport, */ static void idpf_compl_desc_rel(struct idpf_compl_queue *complq) { + idpf_xsk_clear_queue(complq, VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); + if (!complq->comp) return; @@ -296,6 +314,7 @@ static int idpf_tx_desc_alloc(const struct idpf_vport *vport, tx_q->next_to_clean = 0; idpf_queue_set(GEN_CHK, tx_q); + idpf_xsk_setup_queue(vport, tx_q, VIRTCHNL2_QUEUE_TYPE_TX); idpf_sq_stats_init(vport, tx_q); return 0; @@ -332,6 +351,9 @@ static int idpf_compl_desc_alloc(const struct idpf_vport *vport, complq->next_to_clean = 0; idpf_queue_set(GEN_CHK, complq); + idpf_xsk_setup_queue(vport, complq, + VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); + return 0; } @@ -1182,13 +1204,13 @@ static void idpf_qvec_ena_irq(struct idpf_q_vector *qv) static struct idpf_queue_set * idpf_vector_to_queue_set(struct idpf_q_vector *qv) { - bool xdp = qv->vport->xdp_txq_offset; + bool xdp = qv->vport->xdp_txq_offset && !qv->num_xsksq; struct idpf_vport *vport = qv->vport; struct idpf_queue_set *qs; u32 num; num = qv->num_rxq + qv->num_bufq + qv->num_txq + qv->num_complq; - num += xdp ? qv->num_rxq * 2 : 0; + num += xdp ? qv->num_rxq * 2 : qv->num_xsksq * 2; if (!num) return NULL; @@ -1231,6 +1253,14 @@ idpf_vector_to_queue_set(struct idpf_q_vector *qv) qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; qs->qs[num++].complq = vport->txqs[idx]->complq; } + } else { + for (u32 i = 0; i < qv->num_xsksq; i++) { + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX; + qs->qs[num++].txq = qv->xsksq[i]; + + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; + qs->qs[num++].complq = qv->xsksq[i]->complq; + } } finalize: @@ -1270,6 +1300,9 @@ static int idpf_qp_disable(struct idpf_queue_set *qs, u32 qid) idpf_clean_queue_set(qs); + kfree(q_vector->xsksq); + q_vector->num_xsksq = 0; + return 0; } @@ -1295,6 +1328,29 @@ static int idpf_qp_enable(struct idpf_queue_set *qs, u32 qid) return err; } + if (!vport->xdp_txq_offset) + goto napi; + + q_vector->xsksq = kcalloc(DIV_ROUND_UP(vport->num_rxq_grp, + vport->num_q_vectors), + sizeof(*q_vector->xsksq), GFP_KERNEL); + if (!q_vector->xsksq) + return -ENOMEM; + + for (u32 i = 0; i < qs->num; i++) { + struct idpf_queue_ptr *q = &qs->qs[i]; + + if (q->type != VIRTCHNL2_QUEUE_TYPE_TX) + continue; + + if (!idpf_queue_has(XSK, q->txq)) + continue; + + q->txq->q_vector = q_vector; + q_vector->xsksq[q_vector->num_xsksq++] = q->txq; + } + +napi: napi_enable(&q_vector->napi); idpf_qvec_ena_irq(q_vector); @@ -3954,7 +4010,7 @@ static irqreturn_t idpf_vport_intr_clean_queues(int __always_unused irq, struct idpf_q_vector *q_vector = (struct idpf_q_vector *)data; q_vector->total_events++; - napi_schedule(&q_vector->napi); + napi_schedule_irqoff(&q_vector->napi); return IRQ_HANDLED; } @@ -3995,6 +4051,8 @@ void idpf_vport_intr_rel(struct idpf_vport *vport) for (u32 v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) { struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx]; + kfree(q_vector->xsksq); + q_vector->xsksq = NULL; kfree(q_vector->complq); q_vector->complq = NULL; kfree(q_vector->bufq); @@ -4454,7 +4512,7 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) { struct idpf_q_vector *q_vector = container_of(napi, struct idpf_q_vector, napi); - bool clean_complete; + bool clean_complete = true; int work_done = 0; /* Handle case where we are called by netpoll with a budget of 0 */ @@ -4464,8 +4522,13 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) return 0; } - clean_complete = idpf_rx_splitq_clean_all(q_vector, budget, &work_done); - clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget, &work_done); + for (u32 i = 0; i < q_vector->num_xsksq; i++) + clean_complete &= idpf_xsk_xmit(q_vector->xsksq[i]); + + clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget, + &work_done); + clean_complete &= idpf_rx_splitq_clean_all(q_vector, budget, + &work_done); /* If work not completed, return budget and polling will return */ if (!clean_complete) { @@ -4478,7 +4541,7 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) /* Exit the polling mode, but don't re-enable interrupts if stack might * poll us due to busy-polling */ - if (likely(napi_complete_done(napi, work_done))) + if (napi_complete_done(napi, work_done)) idpf_vport_intr_update_itr_ena_irq(q_vector); else idpf_vport_intr_set_wb_on_itr(q_vector); @@ -4578,6 +4641,20 @@ static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) qv_idx++; } + + for (i = 0; i < vport->num_xdp_txq; i++) { + struct idpf_tx_queue *xdpsq; + struct idpf_q_vector *qv; + + xdpsq = vport->txqs[vport->xdp_txq_offset + i]; + if (!idpf_queue_has(XSK, xdpsq)) + continue; + + qv = idpf_find_rxq_vec(vport, i); + + xdpsq->q_vector = qv; + qv->xsksq[qv->num_xsksq++] = xdpsq; + } } /** @@ -4708,6 +4785,15 @@ int idpf_vport_intr_alloc(struct idpf_vport *vport) GFP_KERNEL); if (!q_vector->complq) goto error; + + if (!vport->xdp_txq_offset) + continue; + + q_vector->xsksq = kcalloc(rxqs_per_vector, + sizeof(*q_vector->xsksq), + GFP_KERNEL); + if (!q_vector->xsksq) + goto error; } return 0; diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index 6b460f8708822e..c5a1c361f6823b 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -313,6 +313,7 @@ struct idpf_ptype_state { * @__IDPF_Q_NOIRQ: queue is polling-driven and has no interrupt * @__IDPF_Q_XDP: this is an XDP queue * @__IDPF_Q_FLAGS_NBITS: Must be last + * @__IDPF_Q_XSK: Queue used to handle the AF_XDP socket */ enum idpf_queue_flags_t { __IDPF_Q_GEN_CHK, @@ -323,6 +324,7 @@ enum idpf_queue_flags_t { __IDPF_Q_HSPLIT_EN, __IDPF_Q_NOIRQ, __IDPF_Q_XDP, + __IDPF_Q_XSK, __IDPF_Q_FLAGS_NBITS, }; @@ -385,10 +387,12 @@ struct idpf_intr_reg { * @num_txq: Number of TX queues * @num_bufq: Number of buffer queues * @num_complq: number of completion queues + * @num_xsksq: number of XSk send queues * @rx: Array of RX queues to service * @tx: Array of TX queues to service * @bufq: Array of buffer queues to service * @complq: array of completion queues + * @xsksq: array of XSk send queues * @intr_reg: See struct idpf_intr_reg * @napi: napi handler * @total_events: Number of interrupts processed @@ -412,10 +416,12 @@ struct idpf_q_vector { u16 num_txq; u16 num_bufq; u16 num_complq; + u16 num_xsksq; struct idpf_rx_queue **rx; struct idpf_tx_queue **tx; struct idpf_buf_queue **bufq; struct idpf_compl_queue **complq; + struct idpf_tx_queue **xsksq; struct idpf_intr_reg intr_reg; __cacheline_group_end_aligned(read_mostly); @@ -442,7 +448,7 @@ struct idpf_q_vector { cpumask_var_t affinity_mask; __cacheline_group_end_aligned(cold); }; -libeth_cacheline_set_assert(struct idpf_q_vector, 112, +libeth_cacheline_set_assert(struct idpf_q_vector, 128, 424 + 2 * sizeof(struct dim), 8 + sizeof(cpumask_var_t)); @@ -583,6 +589,7 @@ libeth_cacheline_set_assert(struct idpf_rx_queue, * @tx_buf: See struct idpf_tx_buf * @txq_grp: See struct idpf_txq_group * @complq: corresponding completion queue in XDP mode + * @pool: corresponding XSk pool if installed * @dev: Device back pointer for DMA mapping * @tail: Tail offset. Used for both queue models single and split * @flags: See enum idpf_queue_flags_t @@ -656,7 +663,10 @@ struct idpf_tx_queue { struct idpf_txq_group *txq_grp; struct idpf_compl_queue *complq; }; - struct device *dev; + union { + struct device *dev; + struct xsk_buff_pool *pool; + }; void __iomem *tail; DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); diff --git a/drivers/net/ethernet/intel/idpf/xdp.c b/drivers/net/ethernet/intel/idpf/xdp.c index 4dd5aae69ed349..5f0aa702a617b4 100644 --- a/drivers/net/ethernet/intel/idpf/xdp.c +++ b/drivers/net/ethernet/intel/idpf/xdp.c @@ -261,7 +261,7 @@ idpf_xdp_parse_compl_desc(const struct idpf_splitq_4b_tx_compl_desc *desc, return upper_16_bits(val); } -static u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget) +u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget) { struct idpf_compl_queue *cq = xdpsq->complq; u32 tx_ntc = xdpsq->next_to_clean; diff --git a/drivers/net/ethernet/intel/idpf/xdp.h b/drivers/net/ethernet/intel/idpf/xdp.h index 02a42775d77cc3..9f76dd8b05ea6c 100644 --- a/drivers/net/ethernet/intel/idpf/xdp.h +++ b/drivers/net/ethernet/intel/idpf/xdp.h @@ -18,6 +18,7 @@ void idpf_copy_xdp_prog_to_qs(const struct idpf_vport *vport, int idpf_vport_xdpq_get(const struct idpf_vport *vport); void idpf_vport_xdpq_put(const struct idpf_vport *vport); +u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget); bool idpf_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags); /** diff --git a/drivers/net/ethernet/intel/idpf/xsk.c b/drivers/net/ethernet/intel/idpf/xsk.c index df30eef6a99ac6..427e1a2b3c076c 100644 --- a/drivers/net/ethernet/intel/idpf/xsk.c +++ b/drivers/net/ethernet/intel/idpf/xsk.c @@ -4,8 +4,269 @@ #include #include "idpf.h" +#include "xdp.h" #include "xsk.h" +static void idpf_xsk_tx_timer(struct work_struct *work); + +static void idpf_xsk_setup_txq(const struct idpf_vport *vport, + struct idpf_tx_queue *txq) +{ + struct xsk_buff_pool *pool; + u32 qid; + + idpf_queue_clear(XSK, txq); + + if (!idpf_queue_has(XDP, txq)) + return; + + qid = txq->idx - vport->xdp_txq_offset; + + pool = xsk_get_pool_from_qid(vport->netdev, qid); + if (!pool || !pool->dev) + return; + + txq->pool = pool; + libeth_xdpsq_init_timer(txq->timer, txq, &txq->xdp_lock, + idpf_xsk_tx_timer); + + idpf_queue_assign(NOIRQ, txq, xsk_uses_need_wakeup(pool)); + idpf_queue_set(XSK, txq); +} + +static void idpf_xsk_setup_complq(const struct idpf_vport *vport, + struct idpf_compl_queue *complq) +{ + const struct xsk_buff_pool *pool; + u32 qid; + + idpf_queue_clear(XSK, complq); + + if (!idpf_queue_has(XDP, complq)) + return; + + qid = complq->txq_grp->txqs[0]->idx - vport->xdp_txq_offset; + + pool = xsk_get_pool_from_qid(vport->netdev, qid); + if (!pool || !pool->dev) + return; + + idpf_queue_set(XSK, complq); +} + +/** + * idpf_xsk_setup_queue - set xsk_pool pointer from netdev to the queue structure + * @vport: vport this queue belongs to + * @q: queue to use + * @type: queue type + * + * Assigns pointer to xsk_pool field in queue struct if it is supported in + * netdev, NULL otherwise. + */ +void idpf_xsk_setup_queue(const struct idpf_vport *vport, void *q, + enum virtchnl2_queue_type type) +{ + if (!idpf_xdp_is_prog_ena(vport)) + return; + + switch (type) { + case VIRTCHNL2_QUEUE_TYPE_TX: + idpf_xsk_setup_txq(vport, q); + break; + case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: + idpf_xsk_setup_complq(vport, q); + break; + default: + break; + } +} + +void idpf_xsk_clear_queue(void *q, enum virtchnl2_queue_type type) +{ + struct idpf_compl_queue *complq; + struct idpf_tx_queue *txq; + + switch (type) { + case VIRTCHNL2_QUEUE_TYPE_TX: + txq = q; + if (!idpf_queue_has_clear(XSK, txq)) + return; + + idpf_queue_set(NOIRQ, txq); + txq->dev = txq->netdev->dev.parent; + break; + case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: + complq = q; + idpf_queue_clear(XSK, complq); + break; + default: + break; + } +} + +/** + * idpf_xsk_clean_xdpq - clean an XSk Tx queue + * @xdpq: XSk Tx queue + */ +void idpf_xsk_clean_xdpq(struct idpf_tx_queue *xdpq) +{ + struct libeth_xdpsq_napi_stats ss = { }; + u32 ntc = xdpq->next_to_clean; + struct xdp_frame_bulk bq; + struct libeth_cq_pp cp = { + .dev = xdpq->pool->dev, + .bq = &bq, + .xss = &ss, + }; + u32 xsk_frames = 0; + + xdp_frame_bulk_init(&bq); + rcu_read_lock(); + + while (ntc != xdpq->next_to_use) { + struct libeth_sqe *sqe = &xdpq->tx_buf[ntc]; + + if (sqe->type) + libeth_xdp_complete_tx(sqe, &cp); + else + xsk_frames++; + + if (unlikely(++ntc == xdpq->desc_count)) + ntc = 0; + } + + xdp_flush_frame_bulk(&bq); + rcu_read_unlock(); + + if (xsk_frames) + xsk_tx_completed(xdpq->pool, xsk_frames); +} + +static noinline u32 idpf_xsksq_complete_slow(struct idpf_tx_queue *xdpsq, + u32 done) +{ + struct libeth_xdpsq_napi_stats ss = { }; + u32 ntc = xdpsq->next_to_clean; + u32 cnt = xdpsq->desc_count; + struct xdp_frame_bulk bq; + struct libeth_cq_pp cp = { + .dev = xdpsq->pool->dev, + .bq = &bq, + .xss = &ss, + .napi = true, + }; + u32 xsk_frames = 0; + + xdp_frame_bulk_init(&bq); + + for (u32 i = 0; likely(i < done); i++) { + struct libeth_sqe *sqe = &xdpsq->tx_buf[ntc]; + + if (sqe->type) + libeth_xdp_complete_tx(sqe, &cp); + else + xsk_frames++; + + if (unlikely(++ntc == cnt)) + ntc = 0; + } + + xdp_flush_frame_bulk(&bq); + + xdpsq->next_to_clean = ntc; + xdpsq->xdp_tx -= cp.xdp_tx; + + libeth_xdpsq_napi_stats_add(&xdpsq->xstats, &ss); + + return xsk_frames; +} + +/** + * idpf_clean_xdp_irq_zc - produce AF_XDP descriptors to CQ + * @_xdpq: XDP Tx queue + * @budget: maximum number of descriptors to clean + * + * Return: number of cleaned descriptors. + */ +static __always_inline u32 idpf_clean_xdp_irq_zc(void *_xdpq, u32 budget) +{ + struct idpf_tx_queue *xdpq = _xdpq; + u32 tx_ntc = xdpq->next_to_clean; + u32 tx_cnt = xdpq->desc_count; + u32 done_frames; + u32 xsk_frames; + + done_frames = idpf_xdpsq_poll(xdpq, budget); + if (unlikely(!done_frames)) + return 0; + + if (likely(!xdpq->xdp_tx)) { + tx_ntc += done_frames; + if (tx_ntc >= tx_cnt) + tx_ntc -= tx_cnt; + + xdpq->next_to_clean = tx_ntc; + xsk_frames = done_frames; + + goto finalize; + } + + xsk_frames = idpf_xsksq_complete_slow(xdpq, done_frames); + if (xsk_frames) +finalize: + xsk_tx_completed(xdpq->pool, xsk_frames); + + xdpq->pending -= done_frames; + + return done_frames; +} + +static u32 idpf_xsk_xmit_prep(void *_xdpq, struct libeth_xdpsq *sq) +{ + struct idpf_tx_queue *xdpq = _xdpq; + + *sq = (struct libeth_xdpsq){ + .pool = xdpq->pool, + .sqes = xdpq->tx_buf, + .descs = xdpq->desc_ring, + .count = xdpq->desc_count, + .lock = &xdpq->xdp_lock, + .ntu = &xdpq->next_to_use, + .pending = &xdpq->pending, + }; + + /* The queue is cleaned, the budget is already known, optimize out + * the second min() by passing the type limit. + */ + return U32_MAX; +} + +/** + * idpf_xsk_xmit - send XSk frames + * @xsksq: XSk queue to produce the HW Tx descriptors on + * + * Return: %true if there's no more work to be done, %false otherwise. + */ +bool idpf_xsk_xmit(struct idpf_tx_queue *xsksq) +{ + u32 free; + + libeth_xdpsq_lock(&xsksq->xdp_lock); + + free = xsksq->desc_count - xsksq->pending; + if (unlikely(free < xsksq->thresh)) + free += idpf_clean_xdp_irq_zc(xsksq, xsksq->thresh); + + return libeth_xsk_xmit_do_bulk(xsksq->pool, xsksq, + min(free, xsksq->thresh), + libeth_xsktmo, idpf_xsk_xmit_prep, + idpf_xdp_tx_xmit, idpf_xdp_tx_finalize); +} + +LIBETH_XDP_DEFINE_START(); +LIBETH_XDP_DEFINE_TIMER(static idpf_xsk_tx_timer, idpf_clean_xdp_irq_zc); +LIBETH_XDP_DEFINE_END(); + /** * idpf_xsk_pool_setup - set up an XSk pool * @vport: current vport of interest diff --git a/drivers/net/ethernet/intel/idpf/xsk.h b/drivers/net/ethernet/intel/idpf/xsk.h index 7168fd7655eb7f..ae1ffc57e37842 100644 --- a/drivers/net/ethernet/intel/idpf/xsk.h +++ b/drivers/net/ethernet/intel/idpf/xsk.h @@ -6,9 +6,18 @@ #include +enum virtchnl2_queue_type; +struct idpf_tx_queue; struct idpf_vport; struct netdev_bpf; +void idpf_xsk_setup_queue(const struct idpf_vport *vport, void *q, + enum virtchnl2_queue_type type); +void idpf_xsk_clear_queue(void *q, enum virtchnl2_queue_type type); + +void idpf_xsk_clean_xdpq(struct idpf_tx_queue *xdpq); +bool idpf_xsk_xmit(struct idpf_tx_queue *xsksq); + int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *xdp); #endif /* !_IDPF_XSK_H_ */ From 3ce0d15425f5774bbc3d16332d57c0e495a179e0 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 12 Dec 2023 15:00:56 +0100 Subject: [PATCH 47/49] idpf: implement Rx path for AF_XDP Implement Rx packet processing specific to AF_XDP ZC using the libeth XSk infra. Initialize queue registers before allocating buffers to avoid redundant ifs when updating the queue tail. Co-developed-by: Michal Kubiak Signed-off-by: Michal Kubiak Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/idpf/idpf_lib.c | 8 +- drivers/net/ethernet/intel/idpf/idpf_txrx.c | 32 +- drivers/net/ethernet/intel/idpf/idpf_txrx.h | 37 ++- drivers/net/ethernet/intel/idpf/xdp.c | 24 +- drivers/net/ethernet/intel/idpf/xsk.c | 345 ++++++++++++++++++++ drivers/net/ethernet/intel/idpf/xsk.h | 6 + 6 files changed, 432 insertions(+), 20 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index 2bc9e9358e30e0..cdc90750e29127 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -1381,16 +1381,16 @@ static int idpf_vport_open(struct idpf_vport *vport) goto queues_rel; } - err = idpf_rx_bufs_init_all(vport); + err = idpf_queue_reg_init(vport); if (err) { - dev_err(&adapter->pdev->dev, "Failed to initialize RX buffers for vport %u: %d\n", + dev_err(&adapter->pdev->dev, "Failed to initialize queue registers for vport %u: %d\n", vport->vport_id, err); goto queues_rel; } - err = idpf_queue_reg_init(vport); + err = idpf_rx_bufs_init_all(vport); if (err) { - dev_err(&adapter->pdev->dev, "Failed to initialize queue registers for vport %u: %d\n", + dev_err(&adapter->pdev->dev, "Failed to initialize RX buffers for vport %u: %d\n", vport->vport_id, err); goto queues_rel; } diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 5906b2f7c4a01a..7f7613730e420b 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -484,6 +484,11 @@ static void idpf_rx_buf_rel_bufq(struct idpf_buf_queue *bufq) if (!bufq->buf) return; + if (idpf_queue_has(XSK, bufq)) { + idpf_xsk_buf_rel(bufq); + return; + } + /* Free all the bufs allocated and given to hw on Rx queue */ for (u32 i = 0; i < bufq->desc_count; i++) idpf_rx_page_rel(&bufq->buf[i]); @@ -533,12 +538,14 @@ static void idpf_rx_desc_rel(struct idpf_rx_queue *rxq, if (!rxq) return; - libeth_xdp_return_stash(&rxq->xdp); + if (!idpf_queue_has(XSK, rxq)) + libeth_xdp_return_stash(&rxq->xdp); if (!idpf_is_queue_model_split(model)) idpf_rx_buf_rel_all(rxq); libeth_rq_stats_deinit(dev, rxq->idx); + idpf_xsk_clear_queue(rxq, VIRTCHNL2_QUEUE_TYPE_RX); rxq->next_to_alloc = 0; rxq->next_to_clean = 0; @@ -563,6 +570,7 @@ static void idpf_rx_desc_rel_bufq(struct idpf_buf_queue *bufq, return; idpf_rx_buf_rel_bufq(bufq); + idpf_xsk_clear_queue(bufq, VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); bufq->next_to_alloc = 0; bufq->next_to_clean = 0; @@ -851,6 +859,9 @@ static int idpf_rx_bufs_init(struct idpf_buf_queue *bufq, }; int ret; + if (idpf_queue_has(XSK, bufq)) + return idpf_xsk_bufs_init(bufq); + ret = libeth_rx_fq_create(&fq, &bufq->q_vector->napi); if (ret) return ret; @@ -944,6 +955,7 @@ static int idpf_rx_desc_alloc(const struct idpf_vport *vport, rxq->next_to_use = 0; idpf_queue_set(GEN_CHK, rxq); + idpf_xsk_setup_queue(vport, rxq, VIRTCHNL2_QUEUE_TYPE_RX); libeth_rq_stats_init(vport->netdev, &rxq->stats, rxq->idx); return 0; @@ -974,6 +986,8 @@ static int idpf_bufq_desc_alloc(const struct idpf_vport *vport, idpf_queue_set(GEN_CHK, bufq); + idpf_xsk_setup_queue(vport, bufq, VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); + return 0; } @@ -3658,9 +3672,9 @@ __idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, return 0; } -static bool idpf_rx_process_skb_fields(struct sk_buff *skb, - const struct libeth_xdp_buff *xdp, - struct libeth_rq_napi_stats *ss) +bool idpf_rx_process_skb_fields(struct sk_buff *skb, + const struct libeth_xdp_buff *xdp, + struct libeth_rq_napi_stats *ss) { struct idpf_rx_queue *rxq; @@ -4487,7 +4501,9 @@ static bool idpf_rx_splitq_clean_all(struct idpf_q_vector *q_vec, int budget, struct idpf_rx_queue *rxq = q_vec->rx[i]; int pkts_cleaned_per_q; - pkts_cleaned_per_q = idpf_rx_splitq_clean(rxq, budget_per_q); + pkts_cleaned_per_q = idpf_queue_has(XSK, rxq) ? + idpf_clean_rx_irq_zc(rxq, budget_per_q) : + idpf_rx_splitq_clean(rxq, budget_per_q); /* if we clean as many as budgeted, we must not be done */ if (pkts_cleaned_per_q >= budget_per_q) clean_complete = false; @@ -4497,8 +4513,10 @@ static bool idpf_rx_splitq_clean_all(struct idpf_q_vector *q_vec, int budget, nid = numa_mem_id(); - for (i = 0; i < q_vec->num_bufq; i++) - idpf_rx_clean_refillq_all(q_vec->bufq[i], nid); + for (i = 0; i < q_vec->num_bufq; i++) { + if (!idpf_queue_has(XSK, q_vec->bufq[i])) + idpf_rx_clean_refillq_all(q_vec->bufq[i], nid); + } return clean_complete; } diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index c5a1c361f6823b..5a4f47adef69e3 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -144,6 +144,7 @@ do { \ #define IDPF_TX_FLAGS_IPV6 BIT(2) #define IDPF_TX_FLAGS_TUNNEL BIT(3) +struct libeth_rq_napi_stats; struct libeth_sq_xmit_stats; union idpf_tx_flex_desc { @@ -502,6 +503,8 @@ struct idpf_txq_stash { * @next_to_clean: Next descriptor to clean * @next_to_alloc: RX buffer to allocate at * @xdp: XDP buffer with the current frame + * @xsk: current XDP buffer in XSk mode + * @pool: XSk pool if installed * @stats: per-queue RQ stats * @q_id: Queue id * @size: Length of descriptor ring in bytes @@ -555,7 +558,13 @@ struct idpf_rx_queue { u32 next_to_clean; u32 next_to_alloc; - struct libeth_xdp_buff_stash xdp; + union { + struct libeth_xdp_buff_stash xdp; + struct { + struct libeth_xdp_buff *xsk; + struct xsk_buff_pool *pool; + }; + }; struct libeth_rq_stats stats; __cacheline_group_end_aligned(read_write); @@ -739,16 +748,20 @@ libeth_cacheline_set_assert(struct idpf_tx_queue, 64, /** * struct idpf_buf_queue - software structure representing a buffer queue * @split_buf: buffer descriptor array - * @hdr_buf: &libeth_fqe for header buffers - * @hdr_pp: &page_pool for header buffers * @buf: &libeth_fqe for data buffers * @pp: &page_pool for data buffers + * @xsk_buf: &xdp_buff for XSk Rx buffers + * @pool: &xsk_buff_pool on XSk queues + * @hdr_buf: &libeth_fqe for header buffers + * @hdr_pp: &page_pool for header buffers * @tail: Tail offset * @flags: See enum idpf_queue_flags_t * @desc_count: Number of descriptors + * @thresh: refill threshold in XSk * @next_to_use: Next descriptor to use * @next_to_clean: Next descriptor to clean * @next_to_alloc: RX buffer to allocate at + * @pending: number of buffers to refill (Xsk) * @hdr_truesize: truesize for buffer headers * @truesize: truesize for data buffers * @q_id: Queue id @@ -762,14 +775,24 @@ libeth_cacheline_set_assert(struct idpf_tx_queue, 64, struct idpf_buf_queue { __cacheline_group_begin_aligned(read_mostly); struct virtchnl2_splitq_rx_buf_desc *split_buf; + union { + struct { + struct libeth_fqe *buf; + struct page_pool *pp; + }; + struct { + struct libeth_xdp_buff **xsk_buf; + struct xsk_buff_pool *pool; + }; + }; struct libeth_fqe *hdr_buf; struct page_pool *hdr_pp; - struct libeth_fqe *buf; - struct page_pool *pp; void __iomem *tail; DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); u32 desc_count; + + u32 thresh; __cacheline_group_end_aligned(read_mostly); __cacheline_group_begin_aligned(read_write); @@ -777,6 +800,7 @@ struct idpf_buf_queue { u32 next_to_clean; u32 next_to_alloc; + u32 pending; u32 hdr_truesize; u32 truesize; __cacheline_group_end_aligned(read_write); @@ -1100,6 +1124,9 @@ netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, netdev_tx_t idpf_tx_start(struct sk_buff *skb, struct net_device *netdev); bool idpf_rx_singleq_buf_hw_alloc_all(struct idpf_rx_queue *rxq, u16 cleaned_count); +bool idpf_rx_process_skb_fields(struct sk_buff *skb, + const struct libeth_xdp_buff *xdp, + struct libeth_rq_napi_stats *ss); int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off, struct libeth_sq_xmit_stats *ss); diff --git a/drivers/net/ethernet/intel/idpf/xdp.c b/drivers/net/ethernet/intel/idpf/xdp.c index 5f0aa702a617b4..ccb8c7e8e23861 100644 --- a/drivers/net/ethernet/intel/idpf/xdp.c +++ b/drivers/net/ethernet/intel/idpf/xdp.c @@ -53,7 +53,6 @@ static int __idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq, void *arg) { const struct idpf_vport *vport = rxq->q_vector->vport; bool split = idpf_is_queue_model_split(vport->rxq_model); - const struct page_pool *pp; int err; err = __xdp_rxq_info_reg(&rxq->xdp_rxq, vport->netdev, rxq->idx, @@ -62,8 +61,18 @@ static int __idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq, void *arg) if (err) return err; - pp = split ? rxq->bufq_sets[0].bufq.pp : rxq->pp; - xdp_rxq_info_attach_page_pool(&rxq->xdp_rxq, pp); + if (idpf_queue_has(XSK, rxq)) { + err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, + MEM_TYPE_XSK_BUFF_POOL, + rxq->pool); + if (err) + goto unreg; + } else { + const struct page_pool *pp; + + pp = split ? rxq->bufq_sets[0].bufq.pp : rxq->pp; + xdp_rxq_info_attach_page_pool(&rxq->xdp_rxq, pp); + } if (!split) return 0; @@ -72,6 +81,11 @@ static int __idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq, void *arg) rxq->num_xdp_txq = vport->num_xdp_txq; return 0; + +unreg: + xdp_rxq_info_unreg(&rxq->xdp_rxq); + + return err; } int idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq) @@ -104,7 +118,9 @@ static int __idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, void *arg) rxq->num_xdp_txq = 0; } - xdp_rxq_info_detach_mem_model(&rxq->xdp_rxq); + if (!idpf_queue_has(XSK, rxq)) + xdp_rxq_info_detach_mem_model(&rxq->xdp_rxq); + xdp_rxq_info_unreg(&rxq->xdp_rxq); return 0; diff --git a/drivers/net/ethernet/intel/idpf/xsk.c b/drivers/net/ethernet/intel/idpf/xsk.c index 427e1a2b3c076c..5c19c3a6be028e 100644 --- a/drivers/net/ethernet/intel/idpf/xsk.c +++ b/drivers/net/ethernet/intel/idpf/xsk.c @@ -9,6 +9,47 @@ static void idpf_xsk_tx_timer(struct work_struct *work); +static void idpf_xsk_setup_rxq(const struct idpf_vport *vport, + struct idpf_rx_queue *rxq) +{ + struct xsk_buff_pool *pool; + + pool = xsk_get_pool_from_qid(vport->netdev, rxq->idx); + if (!pool || !pool->dev || !xsk_buff_can_alloc(pool, 1)) + return; + + rxq->pool = pool; + + idpf_queue_set(XSK, rxq); +} + +static void idpf_xsk_setup_bufq(const struct idpf_vport *vport, + struct idpf_buf_queue *bufq) +{ + struct xsk_buff_pool *pool; + u32 qid = U32_MAX; + + for (u32 i = 0; i < vport->num_rxq_grp; i++) { + const struct idpf_rxq_group *grp = &vport->rxq_grps[i]; + + for (u32 j = 0; j < vport->num_bufqs_per_qgrp; j++) { + if (&grp->splitq.bufq_sets[j].bufq == bufq) { + qid = grp->splitq.rxq_sets[0]->rxq.idx; + goto setup; + } + } + } + +setup: + pool = xsk_get_pool_from_qid(vport->netdev, qid); + if (!pool || !pool->dev || !xsk_buff_can_alloc(pool, 1)) + return; + + bufq->pool = pool; + + idpf_queue_set(XSK, bufq); +} + static void idpf_xsk_setup_txq(const struct idpf_vport *vport, struct idpf_tx_queue *txq) { @@ -70,6 +111,12 @@ void idpf_xsk_setup_queue(const struct idpf_vport *vport, void *q, return; switch (type) { + case VIRTCHNL2_QUEUE_TYPE_RX: + idpf_xsk_setup_rxq(vport, q); + break; + case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: + idpf_xsk_setup_bufq(vport, q); + break; case VIRTCHNL2_QUEUE_TYPE_TX: idpf_xsk_setup_txq(vport, q); break; @@ -84,9 +131,25 @@ void idpf_xsk_setup_queue(const struct idpf_vport *vport, void *q, void idpf_xsk_clear_queue(void *q, enum virtchnl2_queue_type type) { struct idpf_compl_queue *complq; + struct idpf_buf_queue *bufq; + struct idpf_rx_queue *rxq; struct idpf_tx_queue *txq; switch (type) { + case VIRTCHNL2_QUEUE_TYPE_RX: + rxq = q; + if (!idpf_queue_has_clear(XSK, rxq)) + return; + + rxq->pool = NULL; + break; + case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: + bufq = q; + if (!idpf_queue_has_clear(XSK, bufq)) + return; + + bufq->pool = NULL; + break; case VIRTCHNL2_QUEUE_TYPE_TX: txq = q; if (!idpf_queue_has_clear(XSK, txq)) @@ -104,6 +167,124 @@ void idpf_xsk_clear_queue(void *q, enum virtchnl2_queue_type type) } } +static void idpf_xskfqe_init(const struct libeth_xskfq_fp *fq, u32 i) +{ + struct virtchnl2_splitq_rx_buf_desc *desc = fq->descs; + + desc = &desc[i]; +#ifdef __LIBETH_WORD_ACCESS + *(u64 *)&desc->qword0 = i; +#else + desc->qword0.buf_id = cpu_to_le16(i); +#endif + desc->pkt_addr = cpu_to_le64(libeth_xsk_buff_xdp_get_dma(fq->fqes[i])); +} + +/** + * __idpf_alloc_rx_buffers_zc - allocate a number of Rx buffers + * @bufq: buffer queue + * @count: The number of buffers to allocate + * + * Place the @count of descriptors onto buffer queue. Handle the queue wrap + * for case where space from next_to_use up to the end of ring is less + * than @count. Finally do a tail bump. + * + * Returns true if all allocations were successful, false if any fail. + */ +static bool __idpf_alloc_rx_buffers_zc(struct idpf_buf_queue *bufq, u32 count) +{ + struct libeth_xskfq_fp fq = { + .pool = bufq->pool, + .fqes = bufq->xsk_buf, + .descs = bufq->split_buf, + .ntu = bufq->next_to_use, + .count = bufq->desc_count, + }; + u32 done; + + done = libeth_xskfqe_alloc(&fq, count, idpf_xskfqe_init); + writel(fq.ntu, bufq->tail); + + bufq->next_to_use = fq.ntu; + bufq->pending -= done; + + return done == count; +} + +/** + * idpf_alloc_rx_buffers_zc - allocate a number of Rx buffers + * @bufq: buffer queue + * + * Wrapper for internal allocation routine; figure out how many tail + * bumps should take place based on the given threshold + * + * Returns true if all calls to internal alloc routine succeeded + */ +static bool idpf_alloc_rx_buffers_zc(struct idpf_buf_queue *bufq) +{ + u32 count, rx_thresh = bufq->thresh; + + count = ALIGN_DOWN(bufq->pending - 1, rx_thresh); + + for (u32 i = 0; i < count; i += rx_thresh) { + if (unlikely(!__idpf_alloc_rx_buffers_zc(bufq, rx_thresh))) + return false; + } + + return true; +} + +/** + * idpf_xsk_bufs_init - allocate a number of Rx buffers with logs + * @bufq: buffer queue + * + * Wrapper for internal allocation routine; Prints out logs, if allocation + * did not go as expected + */ +int idpf_xsk_bufs_init(struct idpf_buf_queue *bufq) +{ + struct libeth_xskfq fq = { + .pool = bufq->pool, + .count = bufq->desc_count, + .nid = idpf_q_vector_to_mem(bufq->q_vector), + }; + int ret; + + ret = libeth_xskfq_create(&fq); + if (ret) + return ret; + + bufq->xsk_buf = fq.fqes; + bufq->pending = fq.pending; + bufq->thresh = fq.thresh; + bufq->rx_buf_size = fq.buf_len; + + if (!idpf_alloc_rx_buffers_zc(bufq)) + netdev_err(bufq->pool->netdev, + "failed to allocate XSk buffers for qid %d\n", + bufq->pool->queue_id); + + bufq->next_to_alloc = bufq->next_to_use; + + idpf_queue_clear(HSPLIT_EN, bufq); + bufq->rx_hbuf_size = 0; + + return 0; +} + +void idpf_xsk_buf_rel(struct idpf_buf_queue *bufq) +{ + struct libeth_xskfq fq = { + .fqes = bufq->xsk_buf, + }; + + libeth_xskfq_destroy(&fq); + + bufq->rx_buf_size = fq.buf_len; + bufq->thresh = fq.thresh; + bufq->pending = fq.pending; +} + /** * idpf_xsk_clean_xdpq - clean an XSk Tx queue * @xdpq: XSk Tx queue @@ -221,6 +402,31 @@ static __always_inline u32 idpf_clean_xdp_irq_zc(void *_xdpq, u32 budget) return done_frames; } +static u32 idpf_xsk_tx_prep(void *_xdpq, struct libeth_xdpsq *sq) +{ + struct idpf_tx_queue *xdpq = _xdpq; + u32 free; + + libeth_xdpsq_lock(&xdpq->xdp_lock); + + free = xdpq->desc_count - xdpq->pending; + if (unlikely(free < xdpq->thresh)) + free += idpf_clean_xdp_irq_zc(xdpq, NAPI_POLL_WEIGHT); + + *sq = (struct libeth_xdpsq){ + .pool = xdpq->pool, + .sqes = xdpq->tx_buf, + .descs = xdpq->desc_ring, + .count = xdpq->desc_count, + .lock = &xdpq->xdp_lock, + .ntu = &xdpq->next_to_use, + .pending = &xdpq->pending, + .xdp_tx = &xdpq->xdp_tx, + }; + + return free; +} + static u32 idpf_xsk_xmit_prep(void *_xdpq, struct libeth_xdpsq *sq) { struct idpf_tx_queue *xdpq = _xdpq; @@ -265,8 +471,137 @@ bool idpf_xsk_xmit(struct idpf_tx_queue *xsksq) LIBETH_XDP_DEFINE_START(); LIBETH_XDP_DEFINE_TIMER(static idpf_xsk_tx_timer, idpf_clean_xdp_irq_zc); +LIBETH_XSK_DEFINE_FLUSH_TX(static idpf_xsk_tx_flush_bulk, idpf_xsk_tx_prep, + idpf_xdp_tx_xmit); +LIBETH_XSK_DEFINE_RUN(static idpf_xsk_run_pass, idpf_xsk_run_prog, + idpf_xsk_tx_flush_bulk, idpf_rx_process_skb_fields); +LIBETH_XSK_DEFINE_FINALIZE(static idpf_xsk_finalize_rx, idpf_xsk_tx_flush_bulk, + idpf_xdp_tx_finalize); LIBETH_XDP_DEFINE_END(); +struct idpf_xsk_refill_set { + struct { + struct idpf_buf_queue *q; + u32 buf_id; + u32 pending; + } bufqs[IDPF_MAX_BUFQS_PER_RXQ_GRP]; +}; + +static bool idpf_xsk_refill(const struct idpf_xsk_refill_set *set) +{ + bool ret = true; + + for (u32 i = 0; i < ARRAY_SIZE(set->bufqs); i++) { + struct idpf_buf_queue *bufq = set->bufqs[i].q; + u32 ntc; + + if (!bufq) + continue; + + ntc = set->bufqs[i].buf_id; + if (unlikely(++ntc == bufq->desc_count)) + ntc = 0; + + bufq->next_to_clean = ntc; + bufq->pending += set->bufqs[i].pending; + + if (bufq->pending > bufq->thresh) + ret &= idpf_alloc_rx_buffers_zc(bufq); + } + + return ret; +} + +/** + * idpf_clean_rx_irq_zc - consumes packets from the hardware queue + * @rxq: AF_XDP Rx queue + * @budget: NAPI budget + * + * Returns number of processed packets on success, remaining budget on failure. + */ +int idpf_clean_rx_irq_zc(struct idpf_rx_queue *rxq, u32 budget) +{ + struct libeth_rq_napi_stats rs = { }; + struct idpf_xsk_refill_set set = { }; + struct libeth_xdp_tx_bulk bq; + bool wake, gen, fail = false; + u32 ntc = rxq->next_to_clean; + struct libeth_xdp_buff *xdp; + u32 cnt = rxq->desc_count; + + wake = xsk_uses_need_wakeup(rxq->pool); + if (wake) + xsk_clear_rx_need_wakeup(rxq->pool); + + gen = idpf_queue_has(GEN_CHK, rxq); + + libeth_xsk_tx_init_bulk(&bq, rxq->xdp_prog, rxq->xdp_rxq.dev, + rxq->xdpqs, rxq->num_xdp_txq); + xdp = rxq->xsk; + + while (likely(rs.packets < budget)) { + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc; + struct idpf_xdp_rx_desc desc; + struct idpf_buf_queue *bufq; + u32 bufq_id, buf_id; + + rx_desc = &rxq->rx[ntc].flex_adv_nic_3_wb; + + idpf_xdp_get_qw0(&desc, rx_desc); + if (idpf_xdp_rx_gen(&desc) != gen) + break; + + dma_rmb(); + + bufq_id = idpf_xdp_rx_bufq(&desc); + bufq = set.bufqs[bufq_id].q; + if (!bufq) { + bufq = &rxq->bufq_sets[bufq_id].bufq; + set.bufqs[bufq_id].q = bufq; + } + + idpf_xdp_get_qw1(&desc, rx_desc); + buf_id = idpf_xdp_rx_buf(&desc); + + set.bufqs[bufq_id].buf_id = buf_id; + set.bufqs[bufq_id].pending++; + + xdp = libeth_xsk_process_buff(xdp, bufq->xsk_buf[buf_id], + idpf_xdp_rx_len(&desc)); + + if (unlikely(++ntc == cnt)) { + ntc = 0; + gen = !gen; + idpf_queue_change(GEN_CHK, rxq); + } + + if (!idpf_xdp_rx_eop(&desc) || unlikely(!xdp)) + continue; + + fail = !idpf_xsk_run_pass(xdp, &bq, rxq->napi, &rs, rx_desc); + xdp = NULL; + + if (fail) + break; + } + + rxq->next_to_clean = ntc; + rxq->xsk = xdp; + + idpf_xsk_finalize_rx(&bq); + fail |= !idpf_xsk_refill(&set); + + libeth_rq_napi_stats_add(&rxq->stats, &rs); + + if (!wake) + return unlikely(fail) ? budget : rs.packets; + + if (unlikely(fail)) + xsk_set_rx_need_wakeup(rxq->pool); + + return rs.packets; +} + /** * idpf_xsk_pool_setup - set up an XSk pool * @vport: current vport of interest @@ -281,6 +616,16 @@ int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *bpf) bool restart; int ret; + if (pool && !IS_ALIGNED(xsk_pool_get_rx_frame_size(pool), + LIBETH_RX_BUF_STRIDE)) { + NL_SET_ERR_MSG_FMT_MOD(bpf->extack, + "%s: HW doesn't support frames sizes not aligned to %u (qid %u: %u)", + netdev_name(vport->netdev), + LIBETH_RX_BUF_STRIDE, qid, + xsk_pool_get_rx_frame_size(pool)); + return -EINVAL; + } + ret = libeth_xsk_setup_pool(vport->netdev, qid, pool); if (ret) { NL_SET_ERR_MSG_FMT_MOD(bpf->extack, diff --git a/drivers/net/ethernet/intel/idpf/xsk.h b/drivers/net/ethernet/intel/idpf/xsk.h index ae1ffc57e37842..d9f028d688cd86 100644 --- a/drivers/net/ethernet/intel/idpf/xsk.h +++ b/drivers/net/ethernet/intel/idpf/xsk.h @@ -7,6 +7,8 @@ #include enum virtchnl2_queue_type; +struct idpf_buf_queue; +struct idpf_rx_queue; struct idpf_tx_queue; struct idpf_vport; struct netdev_bpf; @@ -15,7 +17,11 @@ void idpf_xsk_setup_queue(const struct idpf_vport *vport, void *q, enum virtchnl2_queue_type type); void idpf_xsk_clear_queue(void *q, enum virtchnl2_queue_type type); +int idpf_xsk_bufs_init(struct idpf_buf_queue *bufq); +void idpf_xsk_buf_rel(struct idpf_buf_queue *bufq); void idpf_xsk_clean_xdpq(struct idpf_tx_queue *xdpq); + +int idpf_clean_rx_irq_zc(struct idpf_rx_queue *rxq, u32 budget); bool idpf_xsk_xmit(struct idpf_tx_queue *xsksq); int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *xdp); From de4b645ad3e11a2e659fc022733cc794274a94e4 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Wed, 6 Dec 2023 14:31:14 +0100 Subject: [PATCH 48/49] idpf: enable XSk features and ndo_xsk_wakeup Now that AF_XDP functionality is fully implemented, advertise XSk XDP feature and add .ndo_xsk_wakeup() callback to be able to use it with this driver. Co-developed-by: Michal Kubiak Signed-off-by: Michal Kubiak Signed-off-by: Alexander Lobakin --- drivers/net/ethernet/intel/idpf/idpf_lib.c | 2 ++ drivers/net/ethernet/intel/idpf/idpf_txrx.c | 3 ++ drivers/net/ethernet/intel/idpf/idpf_txrx.h | 9 +++-- drivers/net/ethernet/intel/idpf/xdp.c | 4 ++- drivers/net/ethernet/intel/idpf/xsk.c | 37 +++++++++++++++++++++ drivers/net/ethernet/intel/idpf/xsk.h | 4 +++ 6 files changed, 55 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index cdc90750e29127..ef551179bd47a2 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -6,6 +6,7 @@ #include "idpf.h" #include "idpf_virtchnl.h" #include "xdp.h" +#include "xsk.h" static const struct net_device_ops idpf_netdev_ops; @@ -2336,4 +2337,5 @@ static const struct net_device_ops idpf_netdev_ops = { .ndo_tx_timeout = idpf_tx_timeout, .ndo_bpf = idpf_xdp, .ndo_xdp_xmit = idpf_xdp_xmit, + .ndo_xsk_wakeup = idpf_xsk_wakeup, }; diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 7f7613730e420b..a15e0056635cb8 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -1360,6 +1360,8 @@ static int idpf_qp_enable(struct idpf_queue_set *qs, u32 qid) if (!idpf_queue_has(XSK, q->txq)) continue; + idpf_xsk_init_wakeup(q_vector); + q->txq->q_vector = q_vector; q_vector->xsksq[q_vector->num_xsksq++] = q->txq; } @@ -4669,6 +4671,7 @@ static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) continue; qv = idpf_find_rxq_vec(vport, i); + idpf_xsk_init_wakeup(qv); xdpsq->q_vector = qv; qv->xsksq[qv->num_xsksq++] = xdpsq; diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index 5a4f47adef69e3..729c064977425a 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -395,9 +395,10 @@ struct idpf_intr_reg { * @complq: array of completion queues * @xsksq: array of XSk send queues * @intr_reg: See struct idpf_intr_reg - * @napi: napi handler + * @csd: XSk wakeup CSD * @total_events: Number of interrupts processed * @wb_on_itr: WB on ITR enabled or not + * @napi: napi handler * @tx_dim: Data for TX net_dim algorithm * @tx_itr_value: TX interrupt throttling rate * @tx_intr_mode: Dynamic ITR or not @@ -428,9 +429,11 @@ struct idpf_q_vector { __cacheline_group_end_aligned(read_mostly); __cacheline_group_begin_aligned(read_write); - struct napi_struct napi; + call_single_data_t csd; + u16 total_events; bool wb_on_itr; + struct napi_struct napi; struct dim tx_dim; u16 tx_itr_value; @@ -450,7 +453,7 @@ struct idpf_q_vector { __cacheline_group_end_aligned(cold); }; libeth_cacheline_set_assert(struct idpf_q_vector, 128, - 424 + 2 * sizeof(struct dim), + 456 + 2 * sizeof(struct dim), 8 + sizeof(cpumask_var_t)); #define IDPF_ITR_DYNAMIC 1 diff --git a/drivers/net/ethernet/intel/idpf/xdp.c b/drivers/net/ethernet/intel/idpf/xdp.c index ccb8c7e8e23861..19342b4ae0b5c0 100644 --- a/drivers/net/ethernet/intel/idpf/xdp.c +++ b/drivers/net/ethernet/intel/idpf/xdp.c @@ -454,7 +454,9 @@ void idpf_xdp_set_features(const struct idpf_vport *vport) if (!idpf_is_queue_model_split(vport->rxq_model)) return; - libeth_xdp_set_features_noredir(vport->netdev, &idpf_xdpmo); + libeth_xdp_set_features_noredir(vport->netdev, &idpf_xdpmo, + idpf_get_max_tx_bufs(vport->adapter), + libeth_xsktmo); } /** diff --git a/drivers/net/ethernet/intel/idpf/xsk.c b/drivers/net/ethernet/intel/idpf/xsk.c index 5c19c3a6be028e..7ef6dd407b798b 100644 --- a/drivers/net/ethernet/intel/idpf/xsk.c +++ b/drivers/net/ethernet/intel/idpf/xsk.c @@ -167,6 +167,11 @@ void idpf_xsk_clear_queue(void *q, enum virtchnl2_queue_type type) } } +void idpf_xsk_init_wakeup(struct idpf_q_vector *qv) +{ + libeth_xsk_init_wakeup(&qv->csd, &qv->napi); +} + static void idpf_xskfqe_init(const struct libeth_xskfq_fp *fq, u32 i) { struct virtchnl2_splitq_rx_buf_desc *desc = fq->descs; @@ -653,3 +658,35 @@ int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *bpf) return ret; } + +/** + * idpf_xsk_wakeup - wake up a queue pair + * @dev: network device + * @qid: queue to wake up + * @flags: wakeup flags + * + * Return: 0 on success, -errno otherwise. + */ +int idpf_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) +{ + const struct idpf_netdev_priv *np = netdev_priv(dev); + const struct idpf_vport *vport = np->vport; + struct idpf_q_vector *q_vector; + + if (unlikely(idpf_vport_ctrl_is_locked(dev))) + return -EBUSY; + + if (unlikely(!vport->link_up)) + return -ENETDOWN; + + if (unlikely(!vport->num_xdp_txq)) + return -ENXIO; + + q_vector = idpf_find_rxq_vec(vport, qid); + if (unlikely(!q_vector->xsksq)) + return -ENXIO; + + libeth_xsk_wakeup(&q_vector->csd, qid); + + return 0; +} diff --git a/drivers/net/ethernet/intel/idpf/xsk.h b/drivers/net/ethernet/intel/idpf/xsk.h index d9f028d688cd86..d8e31462677c45 100644 --- a/drivers/net/ethernet/intel/idpf/xsk.h +++ b/drivers/net/ethernet/intel/idpf/xsk.h @@ -8,14 +8,17 @@ enum virtchnl2_queue_type; struct idpf_buf_queue; +struct idpf_q_vector; struct idpf_rx_queue; struct idpf_tx_queue; struct idpf_vport; +struct net_device; struct netdev_bpf; void idpf_xsk_setup_queue(const struct idpf_vport *vport, void *q, enum virtchnl2_queue_type type); void idpf_xsk_clear_queue(void *q, enum virtchnl2_queue_type type); +void idpf_xsk_init_wakeup(struct idpf_q_vector *qv); int idpf_xsk_bufs_init(struct idpf_buf_queue *bufq); void idpf_xsk_buf_rel(struct idpf_buf_queue *bufq); @@ -25,5 +28,6 @@ int idpf_clean_rx_irq_zc(struct idpf_rx_queue *rxq, u32 budget); bool idpf_xsk_xmit(struct idpf_tx_queue *xsksq); int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *xdp); +int idpf_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags); #endif /* !_IDPF_XSK_H_ */ From 04fdca7137e65a3fee89295a3bbb38423a8b6f02 Mon Sep 17 00:00:00 2001 From: Michal Kubiak Date: Thu, 18 Jul 2024 17:36:54 +0200 Subject: [PATCH 49/49] idpf-linux: block changing ring params while af_xdp is active Changing ring parameters, especially ring size, should not be modified while AF_XDP socket is assigned to any Rx ring. Implement a function for checking all Rx queues for AF_XDP socket assign and block changing queue parameters if at least one Rx queue has AF_XDP socket. Signed-off-by: Michal Kubiak --- drivers/net/ethernet/intel/idpf/idpf_ethtool.c | 11 +++++++++++ drivers/net/ethernet/intel/idpf/xsk.c | 18 ++++++++++++++++++ drivers/net/ethernet/intel/idpf/xsk.h | 2 ++ 3 files changed, 31 insertions(+) diff --git a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c index eda2266c7c296e..2858dffba47f5f 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c +++ b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c @@ -4,6 +4,7 @@ #include #include "idpf.h" +#include "xsk.h" /** * idpf_get_rxnfc - command to get RX flow classification rules @@ -342,6 +343,16 @@ static int idpf_set_ringparam(struct net_device *netdev, idpf_vport_ctrl_lock(netdev); vport = idpf_netdev_to_vport(netdev); + /* If there is a AF_XDP UMEM attached to any of Rx queues, + * disallow changing the number of descriptors -- regardless + * if the netdev is running or not. + */ + if (idpf_xsk_any_rxq_ena(vport)) { + netdev_err(netdev, + "Cannot modify ring parameters while AF_XDP UMEM is attached to any Rx queue\n"); + return -EBUSY; + } + idx = vport->idx; if (ring->tx_pending < IDPF_MIN_TXQ_DESC) { diff --git a/drivers/net/ethernet/intel/idpf/xsk.c b/drivers/net/ethernet/intel/idpf/xsk.c index 7ef6dd407b798b..6fcd6a4be7ce5a 100644 --- a/drivers/net/ethernet/intel/idpf/xsk.c +++ b/drivers/net/ethernet/intel/idpf/xsk.c @@ -690,3 +690,21 @@ int idpf_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) return 0; } + +/** + * idpf_xsk_any_rxq_ena - Checks if Rx queues have AF_XDP buff pool attached + * @vport: vport to be checked + * + * Returns true if any of the Rx queues has an AF_XDP buff pool attached + */ +bool idpf_xsk_any_rxq_ena(struct idpf_vport *vport) +{ + int i; + + for (i = 0; i < vport->num_rxq; i++) { + if (xsk_get_pool_from_qid(vport->netdev, i)) + return true; + } + + return false; +} diff --git a/drivers/net/ethernet/intel/idpf/xsk.h b/drivers/net/ethernet/intel/idpf/xsk.h index d8e31462677c45..1f8e11ca6beed8 100644 --- a/drivers/net/ethernet/intel/idpf/xsk.h +++ b/drivers/net/ethernet/intel/idpf/xsk.h @@ -30,4 +30,6 @@ bool idpf_xsk_xmit(struct idpf_tx_queue *xsksq); int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *xdp); int idpf_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags); +bool idpf_xsk_any_rxq_ena(struct idpf_vport *vport); + #endif /* !_IDPF_XSK_H_ */