Merge branch 'net-create-dynamic-software-irq-moderation-library'

Andy Gospodarek says:

====================
net: create dynamic software irq moderation library

This converts the dynamic interrupt moderation library from the mlx5e
driver into a library so it can be used by any driver.  The penultimate
patch in this set adds support for this new dynamic interrupt moderation
library in the bnxt_en driver and the last patch creates an entry in the
MAINTAINERS file for this library.

The main purpose of this code is to allow an administrator to make sure
that default coalesce settings are optimized for low latency, but
quickly adapt to handle high throughput/bulk traffic by altering how
much time passes before popping an interrupt.

For any new driver the following changes would be needed to use this
library:

- add elements in ring struct to track items needed by this library
- create function that can be called to actually set coalesce settings
  for the driver

Credit to Rob Rice and Lee Reed for doing some of the initial proof of
concept and testing for this patch and Tal Gilboa and Or Gerlitz for
their comments, etc on this set.

v4: Fix build breakage for VF representers noticed by kbuild test robot.
Thanks for being so courteous, kbuild test robot!

v3: bnxt_en fix from Michael Chan, comment suggestion from Vasundhara
Volam, and small mlx5e header file fix from Tal Gilboa.

v2: Spelling fixes from Stephen Hemminger, bnxt_en suggestions from
Michael Chan, spelling and formatting fixes from Or Gerlitz, and
spelling and mlx5e changes suggested by Tal Gilboa.
====================

Signed-off-by: David S. Miller <davem@davemloft.net>

MAINTAINERS

@@ -4942,6 +4942,11 @@ S: Maintained
 F: lib/dynamic_debug.c
 F: include/linux/dynamic_debug.h
 
+DYNAMIC INTERRUPT MODERATION
+M: Tal Gilboa <talgi@mellanox.com>
+S: Maintained
+F: include/linux/net_dim.h
+
 DZ DECSTATION DZ11 SERIAL DRIVER
 M: "Maciej W. Rozycki" <macro@linux-mips.org>
 S: Maintained

drivers/net/ethernet/broadcom/bnxt/Makefile

@@ -1,4 +1,4 @@
 obj-$(CONFIG_BNXT) += bnxt_en.o
 
-bnxt_en-y := bnxt.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_vfr.o bnxt_devlink.o
+bnxt_en-y := bnxt.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_vfr.o bnxt_devlink.o bnxt_dim.o
 bnxt_en-$(CONFIG_BNXT_FLOWER_OFFLOAD) += bnxt_tc.o

drivers/net/ethernet/broadcom/bnxt/bnxt.c

@@ -1645,6 +1645,8 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
  rxr->rx_next_cons = NEXT_RX(cons);
 
 next_rx_no_prod:
+ cpr->rx_packets += 1;
+ cpr->rx_bytes += len;
  *raw_cons = tmp_raw_cons;
 
  return rc;
@@ -1802,6 +1804,7 @@ static irqreturn_t bnxt_msix(int irq, void *dev_instance)
  struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
  u32 cons = RING_CMP(cpr->cp_raw_cons);
 
+ cpr->event_ctr++;
  prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
  napi_schedule(&bnapi->napi);
  return IRQ_HANDLED;
@@ -2025,6 +2028,15 @@ static int bnxt_poll(struct napi_struct *napi, int budget)
  break;
  }
  }
+ if (bp->flags & BNXT_FLAG_DIM) {
+ struct net_dim_sample dim_sample;
+
+ net_dim_sample(cpr->event_ctr,
+ cpr->rx_packets,
+ cpr->rx_bytes,
+ &dim_sample);
+ net_dim(&cpr->dim, dim_sample);
+ }
  mmiowb();
  return work_done;
 }
@@ -2617,6 +2629,8 @@ static void bnxt_init_cp_rings(struct bnxt *bp)
  struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
 
  ring->fw_ring_id = INVALID_HW_RING_ID;
+ cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
+ cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
  }
 }
 
@@ -4593,6 +4607,36 @@ static void bnxt_hwrm_set_coal_params(struct bnxt_coal *hw_coal,
  req->flags = cpu_to_le16(flags);
 }
 
+int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
+{
+ struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
+ struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
+ struct bnxt_coal coal;
+ unsigned int grp_idx;
+
+ /* Tick values in micro seconds.
+ * 1 coal_buf x bufs_per_record = 1 completion record.
+ */
+ memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
+
+ coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
+ coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
+
+ if (!bnapi->rx_ring)
+ return -ENODEV;
+
+ bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
+ HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
+
+ bnxt_hwrm_set_coal_params(&coal, &req_rx);
+
+ grp_idx = bnapi->index;
+ req_rx.ring_id = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
+
+ return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
+ HWRM_CMD_TIMEOUT);
+}
+
 int bnxt_hwrm_set_coal(struct bnxt *bp)
 {
  int i, rc = 0;
@@ -5715,7 +5759,13 @@ static void bnxt_enable_napi(struct bnxt *bp)
  int i;
 
  for (i = 0; i < bp->cp_nr_rings; i++) {
+ struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
  bp->bnapi[i]->in_reset = false;
+
+ if (bp->bnapi[i]->rx_ring) {
+ INIT_WORK(&cpr->dim.work, bnxt_dim_work);
+ cpr->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
+ }
  napi_enable(&bp->bnapi[i]->napi);
  }
 }

drivers/net/ethernet/broadcom/bnxt/bnxt.h

@@ -24,6 +24,7 @@
 #include <net/dst_metadata.h>
 #include <net/switchdev.h>
 #include <net/xdp.h>
+#include <linux/net_dim.h>
 
 struct tx_bd {
  __le32 tx_bd_len_flags_type;
@@ -608,6 +609,17 @@ struct bnxt_tx_ring_info {
  struct bnxt_ring_struct tx_ring_struct;
 };
 
+struct bnxt_coal {
+ u16 coal_ticks;
+ u16 coal_ticks_irq;
+ u16 coal_bufs;
+ u16 coal_bufs_irq;
+ /* RING_IDLE enabled when coal ticks < idle_thresh */
+ u16 idle_thresh;
+ u8 bufs_per_record;
+ u8 budget;
+};
+
 struct bnxt_tpa_info {
  void *data;
  u8 *data_ptr;
@@ -672,6 +684,13 @@ struct bnxt_cp_ring_info {
  u32 cp_raw_cons;
  void __iomem *cp_doorbell;
 
+ struct bnxt_coal rx_ring_coal;
+ u64 rx_packets;
+ u64 rx_bytes;
+ u64 event_ctr;
+
+ struct net_dim dim;
+
  struct tx_cmp *cp_desc_ring[MAX_CP_PAGES];
 
  dma_addr_t cp_desc_mapping[MAX_CP_PAGES];
@@ -946,17 +965,6 @@ struct bnxt_test_info {
 #define BNXT_CAG_REG_LEGACY_INT_STATUS 0x4014
 #define BNXT_CAG_REG_BASE 0x300000
 
-struct bnxt_coal {
- u16 coal_ticks;
- u16 coal_ticks_irq;
- u16 coal_bufs;
- u16 coal_bufs_irq;
- /* RING_IDLE enabled when coal ticks < idle_thresh */
- u16 idle_thresh;
- u8 bufs_per_record;
- u8 budget;
-};
-
 struct bnxt_tc_flow_stats {
  u64 packets;
  u64 bytes;
@@ -1128,6 +1136,7 @@ struct bnxt {
  #define BNXT_FLAG_DOUBLE_DB 0x400000
  #define BNXT_FLAG_FW_DCBX_AGENT 0x800000
  #define BNXT_FLAG_CHIP_NITRO_A0 0x1000000
+ #define BNXT_FLAG_DIM 0x2000000
 
  #define BNXT_FLAG_ALL_CONFIG_FEATS (BNXT_FLAG_TPA | \
  BNXT_FLAG_RFS | \
@@ -1425,4 +1434,7 @@ int bnxt_setup_mq_tc(struct net_device *dev, u8 tc);
 int bnxt_get_max_rings(struct bnxt *, int *, int *, bool);
 void bnxt_restore_pf_fw_resources(struct bnxt *bp);
 int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr);
+void bnxt_dim_work(struct work_struct *work);
+int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi);
+
 #endif

drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c

@@ -0,0 +1,32 @@
+/* Broadcom NetXtreme-C/E network driver.
+ *
+ * Copyright (c) 2017-2018 Broadcom Limited
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/net_dim.h>
+#include "bnxt_hsi.h"
+#include "bnxt.h"
+
+void bnxt_dim_work(struct work_struct *work)
+{
+ struct net_dim *dim = container_of(work, struct net_dim,
+ work);
+ struct bnxt_cp_ring_info *cpr = container_of(dim,
+ struct bnxt_cp_ring_info,
+ dim);
+ struct bnxt_napi *bnapi = container_of(cpr,
+ struct bnxt_napi,
+ cp_ring);
+ struct net_dim_cq_moder cur_profile = net_dim_get_profile(dim->mode,
+ dim->profile_ix);
+
+ cpr->rx_ring_coal.coal_ticks = cur_profile.usec;
+ cpr->rx_ring_coal.coal_bufs = cur_profile.pkts;
+
+ bnxt_hwrm_set_ring_coal(bnapi->bp, bnapi);
+ dim->state = NET_DIM_START_MEASURE;
+}

drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c

@@ -49,6 +49,8 @@ static int bnxt_get_coalesce(struct net_device *dev,
 
  memset(coal, 0, sizeof(*coal));
 
+ coal->use_adaptive_rx_coalesce = bp->flags & BNXT_FLAG_DIM;
+
  hw_coal = &bp->rx_coal;
  mult = hw_coal->bufs_per_record;
  coal->rx_coalesce_usecs = hw_coal->coal_ticks;
@@ -77,6 +79,15 @@ static int bnxt_set_coalesce(struct net_device *dev,
  int rc = 0;
  u16 mult;
 
+ if (coal->use_adaptive_rx_coalesce) {
+ bp->flags |= BNXT_FLAG_DIM;
+ } else {
+ if (bp->flags & BNXT_FLAG_DIM) {
+ bp->flags &= ~(BNXT_FLAG_DIM);
+ goto reset_coalesce;
+ }
+ }
+
  hw_coal = &bp->rx_coal;
  mult = hw_coal->bufs_per_record;
  hw_coal->coal_ticks = coal->rx_coalesce_usecs;
@@ -104,6 +115,7 @@ static int bnxt_set_coalesce(struct net_device *dev,
  update_stats = true;
  }
 
+reset_coalesce:
  if (netif_running(dev)) {
  if (update_stats) {
  rc = bnxt_close_nic(bp, true, false);

drivers/net/ethernet/mellanox/mlx5/core/Makefile

@@ -14,7 +14,7 @@ mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cmd.o fpga/core.o fpga/conn.o fpga/sdk.o \
  fpga/ipsec.o
 
 mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \
- en_tx.o en_rx.o en_rx_am.o en_txrx.o en_stats.o vxlan.o \
+ en_tx.o en_rx.o en_dim.o en_txrx.o en_stats.o vxlan.o \
  en_arfs.o en_fs_ethtool.o en_selftest.o
 
 mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o

drivers/net/ethernet/mellanox/mlx5/core/en.h

@@ -47,6 +47,7 @@
 #include <linux/rhashtable.h>
 #include <net/switchdev.h>
 #include <net/xdp.h>
+#include <linux/net_dim.h>
 #include "wq.h"
 #include "mlx5_core.h"
 #include "en_stats.h"
@@ -227,12 +228,6 @@ enum mlx5e_priv_flag {
 #define MLX5E_MAX_BW_ALLOC 100 /* Max percentage of BW allocation */
 #endif
 
-struct mlx5e_cq_moder {
- u16 usec;
- u16 pkts;
- u8 cq_period_mode;
-};
-
 struct mlx5e_params {
  u8 log_sq_size;
  u8 rq_wq_type;
@@ -243,8 +238,8 @@ struct mlx5e_params {
  u16 num_channels;
  u8 num_tc;
  bool rx_cqe_compress_def;
- struct mlx5e_cq_moder rx_cq_moderation;
- struct mlx5e_cq_moder tx_cq_moderation;
+ struct net_dim_cq_moder rx_cq_moderation;
+ struct net_dim_cq_moder tx_cq_moderation;
  bool lro_en;
  u32 lro_wqe_sz;
  u16 tx_max_inline;
@@ -254,7 +249,7 @@ struct mlx5e_params {
  u32 indirection_rqt[MLX5E_INDIR_RQT_SIZE];
  bool vlan_strip_disable;
  bool scatter_fcs_en;
- bool rx_am_enabled;
+ bool rx_dim_enabled;
  u32 lro_timeout;
  u32 pflags;
  struct bpf_prog *xdp_prog;
@@ -473,32 +468,6 @@ struct mlx5e_mpw_info {
  u16 skbs_frags[MLX5_MPWRQ_PAGES_PER_WQE];
 };
 
-struct mlx5e_rx_am_stats {
- int ppms; /* packets per msec */
- int bpms; /* bytes per msec */
- int epms; /* events per msec */
-};
-
-struct mlx5e_rx_am_sample {
- ktime_t time;
- u32 pkt_ctr;
- u32 byte_ctr;
- u16 event_ctr;
-};
-
-struct mlx5e_rx_am { /* Adaptive Moderation */
- u8 state;
- struct mlx5e_rx_am_stats prev_stats;
- struct mlx5e_rx_am_sample start_sample;
- struct work_struct work;
- u8 profile_ix;
- u8 mode;
- u8 tune_state;
- u8 steps_right;
- u8 steps_left;
- u8 tired;
-};
-
 /* a single cache unit is capable to serve one napi call (for non-striding rq)
  * or a MPWQE (for striding rq).
  */
@@ -559,7 +528,7 @@ struct mlx5e_rq {
  unsigned long state;
  int ix;
 
- struct mlx5e_rx_am am; /* Adaptive Moderation */
+ struct net_dim  dim; /* Dynamic Interrupt Moderation */
 
  /* XDP */
  struct bpf_prog *xdp_prog;
@@ -865,10 +834,6 @@ void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);
 void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
 void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi);
 
-void mlx5e_rx_am(struct mlx5e_rq *rq);
-void mlx5e_rx_am_work(struct work_struct *work);
-struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode);
-
 void mlx5e_update_stats(struct mlx5e_priv *priv, bool full);
 
 int mlx5e_create_flow_steering(struct mlx5e_priv *priv);
@@ -1115,4 +1080,5 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
  struct mlx5e_params *params,
  u16 max_channels);
 u8 mlx5e_params_calculate_tx_min_inline(struct mlx5_core_dev *mdev);
+void mlx5e_rx_dim_work(struct work_struct *work);
 #endif /* __MLX5_EN_H__ */