From 98dd1edffc6cd1a2ff3b18596f7c88f6c082fd2e Mon Sep 17 00:00:00 2001 From: Andy Gospodarek Date: Tue, 9 Jan 2018 16:06:12 -0500 Subject: [PATCH 01/10] net/mlx5e: Move interrupt moderation structs to new file Create new header file to prepare to move code that handles irq moderation to a library that lives in a header file. Signed-off-by: Andy Gospodarek Acked-by: Tal Gilboa Acked-by: Saeed Mahameed Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 33 +-------- .../net/ethernet/mellanox/mlx5/core/en_dim.h | 69 +++++++++++++++++++ 2 files changed, 70 insertions(+), 32 deletions(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_dim.h diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 72bab8d3f4b0..f692a66e42ef 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -50,6 +50,7 @@ #include "wq.h" #include "mlx5_core.h" #include "en_stats.h" +#include "en_dim.h" #define MLX5_SET_CFG(p, f, v) MLX5_SET(create_flow_group_in, p, f, v) @@ -227,12 +228,6 @@ enum mlx5e_priv_flag { #define MLX5E_MAX_BW_ALLOC 100 /* Max percentage of BW allocation */ #endif -struct mlx5e_cq_moder { - u16 usec; - u16 pkts; - u8 cq_period_mode; -}; - struct mlx5e_params { u8 log_sq_size; u8 rq_wq_type; @@ -473,32 +468,6 @@ struct mlx5e_mpw_info { u16 skbs_frags[MLX5_MPWRQ_PAGES_PER_WQE]; }; -struct mlx5e_rx_am_stats { - int ppms; /* packets per msec */ - int bpms; /* bytes per msec */ - int epms; /* events per msec */ -}; - -struct mlx5e_rx_am_sample { - ktime_t time; - u32 pkt_ctr; - u32 byte_ctr; - u16 event_ctr; -}; - -struct mlx5e_rx_am { /* Adaptive Moderation */ - u8 state; - struct mlx5e_rx_am_stats prev_stats; - struct mlx5e_rx_am_sample start_sample; - struct work_struct work; - u8 profile_ix; - u8 mode; - u8 tune_state; - u8 steps_right; - u8 steps_left; - u8 tired; -}; - /* a single cache unit is capable to serve one napi call (for non-striding rq) * or a MPWQE (for striding rq). */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h new file mode 100644 index 000000000000..9eeaa11e3e74 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. + * Copyright (c) 2017-2018, Broadcom Limited + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MLX5_AM_H +#define MLX5_AM_H + +struct mlx5e_cq_moder { + u16 usec; + u16 pkts; + u8 cq_period_mode; +}; + +struct mlx5e_rx_am_sample { + ktime_t time; + u32 pkt_ctr; + u32 byte_ctr; + u16 event_ctr; +}; + +struct mlx5e_rx_am_stats { + int ppms; /* packets per msec */ + int bpms; /* bytes per msec */ + int epms; /* events per msec */ +}; + +struct mlx5e_rx_am { /* Adaptive Moderation */ + u8 state; + struct mlx5e_rx_am_stats prev_stats; + struct mlx5e_rx_am_sample start_sample; + struct work_struct work; + u8 profile_ix; + u8 mode; + u8 tune_state; + u8 steps_right; + u8 steps_left; + u8 tired; +}; + +#endif /* MLX5_AM_H */ From f58ee099f32e3bfc19f1cf308c7cd89bb82467fe Mon Sep 17 00:00:00 2001 From: Andy Gospodarek Date: Tue, 9 Jan 2018 16:06:13 -0500 Subject: [PATCH 02/10] net/mlx5e: Move interrupt moderation forward declarations Move these to newly created file to prepare to move these functions to a library. Signed-off-by: Andy Gospodarek Acked-by: Tal Gilboa Acked-by: Saeed Mahameed Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 4 ---- drivers/net/ethernet/mellanox/mlx5/core/en_dim.h | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index f692a66e42ef..470dcf5b2cc8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -834,10 +834,6 @@ void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix); void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix); void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi); -void mlx5e_rx_am(struct mlx5e_rq *rq); -void mlx5e_rx_am_work(struct work_struct *work); -struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode); - void mlx5e_update_stats(struct mlx5e_priv *priv, bool full); int mlx5e_create_flow_steering(struct mlx5e_priv *priv); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h index 9eeaa11e3e74..2031a21bc4f2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h @@ -66,4 +66,9 @@ struct mlx5e_rx_am { /* Adaptive Moderation */ u8 tired; }; +struct mlx5e_rq; +void mlx5e_rx_am(struct mlx5e_rq *rq); +void mlx5e_rx_am_work(struct work_struct *work); +struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode); + #endif /* MLX5_AM_H */ From 138968e9979f0c4afb7cc86b8ddca8bf46911771 Mon Sep 17 00:00:00 2001 From: Andy Gospodarek Date: Tue, 9 Jan 2018 16:06:14 -0500 Subject: [PATCH 03/10] net/mlx5e: Remove rq references in mlx5e_rx_am This makes mlx5e_am_sample more generic so that it can be called easily from a driver that does not use the same data structure to store these values in a single structure. Signed-off-by: Andy Gospodarek Acked-by: Tal Gilboa Acked-by: Saeed Mahameed Signed-off-by: David S. Miller --- .../net/ethernet/mellanox/mlx5/core/en_dim.h | 6 +++-- .../ethernet/mellanox/mlx5/core/en_rx_am.c | 22 +++++++++++-------- .../net/ethernet/mellanox/mlx5/core/en_txrx.c | 5 ++++- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h index 2031a21bc4f2..7d5499a3d738 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h @@ -66,8 +66,10 @@ struct mlx5e_rx_am { /* Adaptive Moderation */ u8 tired; }; -struct mlx5e_rq; -void mlx5e_rx_am(struct mlx5e_rq *rq); +void mlx5e_rx_am(struct mlx5e_rx_am *am, + u16 event_ctr, + u64 packets, + u64 bytes); void mlx5e_rx_am_work(struct work_struct *work); struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c index e401d9d245f3..1630076844ef 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c @@ -264,13 +264,15 @@ static bool mlx5e_am_decision(struct mlx5e_rx_am_stats *curr_stats, return am->profile_ix != prev_ix; } -static void mlx5e_am_sample(struct mlx5e_rq *rq, +static void mlx5e_am_sample(u16 event_ctr, + u64 packets, + u64 bytes, struct mlx5e_rx_am_sample *s) { s->time = ktime_get(); - s->pkt_ctr = rq->stats.packets; - s->byte_ctr = rq->stats.bytes; - s->event_ctr = rq->cq.event_ctr; + s->pkt_ctr = packets; + s->byte_ctr = bytes; + s->event_ctr = event_ctr; } #define MLX5E_AM_NEVENTS 64 @@ -309,20 +311,22 @@ void mlx5e_rx_am_work(struct work_struct *work) am->state = MLX5E_AM_START_MEASURE; } -void mlx5e_rx_am(struct mlx5e_rq *rq) +void mlx5e_rx_am(struct mlx5e_rx_am *am, + u16 event_ctr, + u64 packets, + u64 bytes) { - struct mlx5e_rx_am *am = &rq->am; struct mlx5e_rx_am_sample end_sample; struct mlx5e_rx_am_stats curr_stats; u16 nevents; switch (am->state) { case MLX5E_AM_MEASURE_IN_PROGRESS: - nevents = BIT_GAP(BITS_PER_TYPE(u16), rq->cq.event_ctr, + nevents = BIT_GAP(BITS_PER_TYPE(u16), event_ctr, am->start_sample.event_ctr); if (nevents < MLX5E_AM_NEVENTS) break; - mlx5e_am_sample(rq, &end_sample); + mlx5e_am_sample(event_ctr, packets, bytes, &end_sample); mlx5e_am_calc_stats(&am->start_sample, &end_sample, &curr_stats); if (mlx5e_am_decision(&curr_stats, am)) { @@ -332,7 +336,7 @@ void mlx5e_rx_am(struct mlx5e_rq *rq) } /* fall through */ case MLX5E_AM_START_MEASURE: - mlx5e_am_sample(rq, &am->start_sample); + mlx5e_am_sample(event_ctr, packets, bytes, &am->start_sample); am->state = MLX5E_AM_MEASURE_IN_PROGRESS; break; case MLX5E_AM_APPLY_NEW_PROFILE: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c index ab92298eafc3..18491693bf14 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c @@ -79,7 +79,10 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) mlx5e_cq_arm(&c->sq[i].cq); if (MLX5E_TEST_BIT(c->rq.state, MLX5E_RQ_STATE_AM)) - mlx5e_rx_am(&c->rq); + mlx5e_rx_am(&c->rq.am, + c->rq.cq.event_ctr, + c->rq.stats.packets, + c->rq.stats.bytes); mlx5e_cq_arm(&c->rq.cq); mlx5e_cq_arm(&c->icosq.cq); From f5e7f67d9bc90d2556e131c44d83a14f85cee9c3 Mon Sep 17 00:00:00 2001 From: Andy Gospodarek Date: Tue, 9 Jan 2018 16:06:15 -0500 Subject: [PATCH 04/10] net/mlx5e: Move AM logic enums More movement to help make this code more generic. Signed-off-by: Andy Gospodarek Acked-by: Tal Gilboa Acked-by: Saeed Mahameed Signed-off-by: David S. Miller --- .../net/ethernet/mellanox/mlx5/core/en_dim.h | 26 +++++++++++++++++++ .../ethernet/mellanox/mlx5/core/en_rx_am.c | 25 ------------------ 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h index 7d5499a3d738..a1497bab7154 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h @@ -66,6 +66,32 @@ struct mlx5e_rx_am { /* Adaptive Moderation */ u8 tired; }; +/* Adaptive moderation logic */ +enum { + MLX5E_AM_START_MEASURE, + MLX5E_AM_MEASURE_IN_PROGRESS, + MLX5E_AM_APPLY_NEW_PROFILE, +}; + +enum { + MLX5E_AM_PARKING_ON_TOP, + MLX5E_AM_PARKING_TIRED, + MLX5E_AM_GOING_RIGHT, + MLX5E_AM_GOING_LEFT, +}; + +enum { + MLX5E_AM_STATS_WORSE, + MLX5E_AM_STATS_SAME, + MLX5E_AM_STATS_BETTER, +}; + +enum { + MLX5E_AM_STEPPED, + MLX5E_AM_TOO_TIRED, + MLX5E_AM_ON_EDGE, +}; + void mlx5e_rx_am(struct mlx5e_rx_am *am, u16 event_ctr, u64 packets, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c index 1630076844ef..337dd6024487 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c @@ -82,31 +82,6 @@ struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode) return mlx5e_am_get_profile(rx_cq_period_mode, default_profile_ix); } -/* Adaptive moderation logic */ -enum { - MLX5E_AM_START_MEASURE, - MLX5E_AM_MEASURE_IN_PROGRESS, - MLX5E_AM_APPLY_NEW_PROFILE, -}; - -enum { - MLX5E_AM_PARKING_ON_TOP, - MLX5E_AM_PARKING_TIRED, - MLX5E_AM_GOING_RIGHT, - MLX5E_AM_GOING_LEFT, -}; - -enum { - MLX5E_AM_STATS_WORSE, - MLX5E_AM_STATS_SAME, - MLX5E_AM_STATS_BETTER, -}; - -enum { - MLX5E_AM_STEPPED, - MLX5E_AM_TOO_TIRED, - MLX5E_AM_ON_EDGE, -}; static bool mlx5e_am_on_top(struct mlx5e_rx_am *am) { From b9c872f2316d27e2a323ffb410d706d680508093 Mon Sep 17 00:00:00 2001 From: Andy Gospodarek Date: Tue, 9 Jan 2018 16:06:16 -0500 Subject: [PATCH 05/10] net/mlx5e: Move generic functions to new file These functions were identified as ones that could be made generic and used by multiple drivers. Most of the contents of en_rx_am.c are moved to net_dim.c. Signed-off-by: Andy Gospodarek Acked-by: Tal Gilboa Acked-by: Saeed Mahameed Signed-off-by: David S. Miller --- .../net/ethernet/mellanox/mlx5/core/Makefile | 4 +- drivers/net/ethernet/mellanox/mlx5/core/en.h | 2 +- .../net/ethernet/mellanox/mlx5/core/en_dim.c | 47 +++++++++++++++++++ .../mlx5/core/{en_rx_am.c => net_dim.c} | 19 ++------ .../mlx5/core/{en_dim.h => net_dim.h} | 3 +- 5 files changed, 55 insertions(+), 20 deletions(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_dim.c rename drivers/net/ethernet/mellanox/mlx5/core/{en_rx_am.c => net_dim.c} (94%) rename drivers/net/ethernet/mellanox/mlx5/core/{en_dim.h => net_dim.h} (95%) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 19b21b40ab07..b46b6de2819b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -14,8 +14,8 @@ mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cmd.o fpga/core.o fpga/conn.o fpga/sdk.o \ fpga/ipsec.o mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \ - en_tx.o en_rx.o en_rx_am.o en_txrx.o en_stats.o vxlan.o \ - en_arfs.o en_fs_ethtool.o en_selftest.o + en_tx.o en_rx.o en_dim.o en_txrx.o en_stats.o vxlan.o \ + en_arfs.o en_fs_ethtool.o en_selftest.o net_dim.o mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 470dcf5b2cc8..d13fa1e8a40b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -50,7 +50,7 @@ #include "wq.h" #include "mlx5_core.h" #include "en_stats.h" -#include "en_dim.h" +#include "net_dim.h" #define MLX5_SET_CFG(p, f, v) MLX5_SET(create_flow_group_in, p, f, v) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c new file mode 100644 index 000000000000..57e7e4c434ef --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016, Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "en.h" + +void mlx5e_rx_am_work(struct work_struct *work) +{ + struct mlx5e_rx_am *am = container_of(work, struct mlx5e_rx_am, + work); + struct mlx5e_rq *rq = container_of(am, struct mlx5e_rq, am); + struct mlx5e_cq_moder cur_profile = mlx5e_am_get_profile(am->mode, + am->profile_ix); + + mlx5_core_modify_cq_moderation(rq->mdev, &rq->cq.mcq, + cur_profile.usec, cur_profile.pkts); + + am->state = MLX5E_AM_START_MEASURE; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c b/drivers/net/ethernet/mellanox/mlx5/core/net_dim.c similarity index 94% rename from drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c rename to drivers/net/ethernet/mellanox/mlx5/core/net_dim.c index 337dd6024487..9fafe1da78a1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/net_dim.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, Mellanox Technologies. All rights reserved. + * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -32,11 +33,11 @@ #include "en.h" +#define MLX5E_PARAMS_AM_NUM_PROFILES 5 /* Adaptive moderation profiles */ #define MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256 #define MLX5E_RX_AM_DEF_PROFILE_CQE 1 #define MLX5E_RX_AM_DEF_PROFILE_EQE 1 -#define MLX5E_PARAMS_AM_NUM_PROFILES 5 /* All profiles sizes must be MLX5E_PARAMS_AM_NUM_PROFILES */ #define MLX5_AM_EQE_PROFILES { \ @@ -61,7 +62,7 @@ profile[MLX5_CQ_PERIOD_NUM_MODES][MLX5E_PARAMS_AM_NUM_PROFILES] = { MLX5_AM_CQE_PROFILES, }; -static inline struct mlx5e_cq_moder mlx5e_am_get_profile(u8 cq_period_mode, int ix) +struct mlx5e_cq_moder mlx5e_am_get_profile(u8 cq_period_mode, int ix) { struct mlx5e_cq_moder cq_moder; @@ -82,7 +83,6 @@ struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode) return mlx5e_am_get_profile(rx_cq_period_mode, default_profile_ix); } - static bool mlx5e_am_on_top(struct mlx5e_rx_am *am) { switch (am->tune_state) { @@ -273,19 +273,6 @@ static void mlx5e_am_calc_stats(struct mlx5e_rx_am_sample *start, delta_us); } -void mlx5e_rx_am_work(struct work_struct *work) -{ - struct mlx5e_rx_am *am = container_of(work, struct mlx5e_rx_am, - work); - struct mlx5e_rq *rq = container_of(am, struct mlx5e_rq, am); - struct mlx5e_cq_moder cur_profile = profile[am->mode][am->profile_ix]; - - mlx5_core_modify_cq_moderation(rq->mdev, &rq->cq.mcq, - cur_profile.usec, cur_profile.pkts); - - am->state = MLX5E_AM_START_MEASURE; -} - void mlx5e_rx_am(struct mlx5e_rx_am *am, u16 event_ctr, u64 packets, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h b/drivers/net/ethernet/mellanox/mlx5/core/net_dim.h similarity index 95% rename from drivers/net/ethernet/mellanox/mlx5/core/en_dim.h rename to drivers/net/ethernet/mellanox/mlx5/core/net_dim.h index a1497bab7154..b03a48fc1540 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/net_dim.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. - * Copyright (c) 2017-2018, Broadcom Limited + * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -98,5 +98,6 @@ void mlx5e_rx_am(struct mlx5e_rx_am *am, u64 bytes); void mlx5e_rx_am_work(struct work_struct *work); struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode); +struct mlx5e_cq_moder mlx5e_am_get_profile(u8 cq_period_mode, int ix); #endif /* MLX5_AM_H */ From 9a31742531018c0b6ab1bccff97109c8020ad86e Mon Sep 17 00:00:00 2001 From: Andy Gospodarek Date: Tue, 9 Jan 2018 16:06:17 -0500 Subject: [PATCH 06/10] net/mlx5e: Change Mellanox references in DIM code Change all appropriate mlx5_am* and MLX5_AM* references to net_dim and NET_DIM, respectively, in code that handles dynamic interrupt moderation. Also change all references from 'am' to 'dim' when used as local variables and add generic profile references. Signed-off-by: Andy Gospodarek Acked-by: Tal Gilboa Acked-by: Saeed Mahameed Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 9 +- .../net/ethernet/mellanox/mlx5/core/en_dim.c | 14 +- .../ethernet/mellanox/mlx5/core/en_ethtool.c | 6 +- .../net/ethernet/mellanox/mlx5/core/en_main.c | 40 ++- .../net/ethernet/mellanox/mlx5/core/en_rep.c | 2 +- .../net/ethernet/mellanox/mlx5/core/en_txrx.c | 8 +- .../net/ethernet/mellanox/mlx5/core/net_dim.c | 286 +++++++++--------- .../net/ethernet/mellanox/mlx5/core/net_dim.h | 63 ++-- 8 files changed, 226 insertions(+), 202 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index d13fa1e8a40b..349918c4ff73 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -238,8 +238,8 @@ struct mlx5e_params { u16 num_channels; u8 num_tc; bool rx_cqe_compress_def; - struct mlx5e_cq_moder rx_cq_moderation; - struct mlx5e_cq_moder tx_cq_moderation; + struct net_dim_cq_moder rx_cq_moderation; + struct net_dim_cq_moder tx_cq_moderation; bool lro_en; u32 lro_wqe_sz; u16 tx_max_inline; @@ -249,7 +249,7 @@ struct mlx5e_params { u32 indirection_rqt[MLX5E_INDIR_RQT_SIZE]; bool vlan_strip_disable; bool scatter_fcs_en; - bool rx_am_enabled; + bool rx_dim_enabled; u32 lro_timeout; u32 pflags; struct bpf_prog *xdp_prog; @@ -528,7 +528,7 @@ struct mlx5e_rq { unsigned long state; int ix; - struct mlx5e_rx_am am; /* Adaptive Moderation */ + struct net_dim dim; /* Dynamic Interrupt Moderation */ /* XDP */ struct bpf_prog *xdp_prog; @@ -1080,4 +1080,5 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params, u16 max_channels); u8 mlx5e_params_calculate_tx_min_inline(struct mlx5_core_dev *mdev); +void mlx5e_rx_dim_work(struct work_struct *work); #endif /* __MLX5_EN_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c index 57e7e4c434ef..c2612fbf8189 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c @@ -32,16 +32,16 @@ #include "en.h" -void mlx5e_rx_am_work(struct work_struct *work) +void mlx5e_rx_dim_work(struct work_struct *work) { - struct mlx5e_rx_am *am = container_of(work, struct mlx5e_rx_am, - work); - struct mlx5e_rq *rq = container_of(am, struct mlx5e_rq, am); - struct mlx5e_cq_moder cur_profile = mlx5e_am_get_profile(am->mode, - am->profile_ix); + struct net_dim *dim = container_of(work, struct net_dim, + work); + struct mlx5e_rq *rq = container_of(dim, struct mlx5e_rq, dim); + struct net_dim_cq_moder cur_profile = net_dim_get_profile(dim->mode, + dim->profile_ix); mlx5_core_modify_cq_moderation(rq->mdev, &rq->cq.mcq, cur_profile.usec, cur_profile.pkts); - am->state = MLX5E_AM_START_MEASURE; + dim->state = NET_DIM_START_MEASURE; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 1554780d1810..bd5af7f37198 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -465,7 +465,7 @@ int mlx5e_ethtool_get_coalesce(struct mlx5e_priv *priv, coal->rx_max_coalesced_frames = priv->channels.params.rx_cq_moderation.pkts; coal->tx_coalesce_usecs = priv->channels.params.tx_cq_moderation.usec; coal->tx_max_coalesced_frames = priv->channels.params.tx_cq_moderation.pkts; - coal->use_adaptive_rx_coalesce = priv->channels.params.rx_am_enabled; + coal->use_adaptive_rx_coalesce = priv->channels.params.rx_dim_enabled; return 0; } @@ -519,7 +519,7 @@ int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv, new_channels.params.tx_cq_moderation.pkts = coal->tx_max_coalesced_frames; new_channels.params.rx_cq_moderation.usec = coal->rx_coalesce_usecs; new_channels.params.rx_cq_moderation.pkts = coal->rx_max_coalesced_frames; - new_channels.params.rx_am_enabled = !!coal->use_adaptive_rx_coalesce; + new_channels.params.rx_dim_enabled = !!coal->use_adaptive_rx_coalesce; if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { priv->channels.params = new_channels.params; @@ -527,7 +527,7 @@ int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv, } /* we are opened */ - reset = !!coal->use_adaptive_rx_coalesce != priv->channels.params.rx_am_enabled; + reset = !!coal->use_adaptive_rx_coalesce != priv->channels.params.rx_dim_enabled; if (!reset) { mlx5e_set_priv_channels_coalesce(priv, coal); priv->channels.params = new_channels.params; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 539bd1d24396..a5f82bbfa7dc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -677,8 +677,17 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, wqe->data.lkey = rq->mkey_be; } - INIT_WORK(&rq->am.work, mlx5e_rx_am_work); - rq->am.mode = params->rx_cq_moderation.cq_period_mode; + INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work); + + switch (params->rx_cq_moderation.cq_period_mode) { + case MLX5_CQ_PERIOD_MODE_START_FROM_CQE: + rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE; + break; + case MLX5_CQ_PERIOD_MODE_START_FROM_EQE: + default: + rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; + } + rq->page_cache.head = 0; rq->page_cache.tail = 0; @@ -925,7 +934,7 @@ static int mlx5e_open_rq(struct mlx5e_channel *c, if (err) goto err_destroy_rq; - if (params->rx_am_enabled) + if (params->rx_dim_enabled) c->rq.state |= BIT(MLX5E_RQ_STATE_AM); return 0; @@ -958,7 +967,7 @@ static void mlx5e_deactivate_rq(struct mlx5e_rq *rq) static void mlx5e_close_rq(struct mlx5e_rq *rq) { - cancel_work_sync(&rq->am.work); + cancel_work_sync(&rq->dim.work); mlx5e_destroy_rq(rq); mlx5e_free_rx_descs(rq); mlx5e_free_rq(rq); @@ -1571,7 +1580,7 @@ static void mlx5e_destroy_cq(struct mlx5e_cq *cq) } static int mlx5e_open_cq(struct mlx5e_channel *c, - struct mlx5e_cq_moder moder, + struct net_dim_cq_moder moder, struct mlx5e_cq_param *param, struct mlx5e_cq *cq) { @@ -1753,7 +1762,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, struct mlx5e_channel_param *cparam, struct mlx5e_channel **cp) { - struct mlx5e_cq_moder icocq_moder = {0, 0}; + struct net_dim_cq_moder icocq_moder = {0, 0}; struct net_device *netdev = priv->netdev; int cpu = mlx5e_get_cpu(priv, ix); struct mlx5e_channel *c; @@ -2005,7 +2014,7 @@ static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv, mlx5e_build_common_cq_param(priv, param); - param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE; + param->cq_period_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; } static void mlx5e_build_icosq_param(struct mlx5e_priv *priv, @@ -4047,9 +4056,18 @@ void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode) params->rx_cq_moderation.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE; - if (params->rx_am_enabled) - params->rx_cq_moderation = - mlx5e_am_get_def_profile(cq_period_mode); + if (params->rx_dim_enabled) { + switch (cq_period_mode) { + case MLX5_CQ_PERIOD_MODE_START_FROM_CQE: + params->rx_cq_moderation = + net_dim_get_def_profile(NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE); + break; + case MLX5_CQ_PERIOD_MODE_START_FROM_EQE: + default: + params->rx_cq_moderation = + net_dim_get_def_profile(NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE); + } + } MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER, params->rx_cq_moderation.cq_period_mode == @@ -4111,7 +4129,7 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev, cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? MLX5_CQ_PERIOD_MODE_START_FROM_CQE : MLX5_CQ_PERIOD_MODE_START_FROM_EQE; - params->rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation); + params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); mlx5e_set_rx_cq_mode_params(params, cq_period_mode); mlx5e_set_tx_cq_mode_params(params, cq_period_mode); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index c6a77f8e99a4..4d1b0ff4b6e4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -884,7 +884,7 @@ static void mlx5e_build_rep_params(struct mlx5_core_dev *mdev, params->rq_wq_type = MLX5_WQ_TYPE_LINKED_LIST; params->log_rq_size = MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE; - params->rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation); + params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); mlx5e_set_rx_cq_mode_params(params, cq_period_mode); params->tx_max_inline = mlx5e_get_max_inline_cap(mdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c index 18491693bf14..a1c94fdc7174 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c @@ -79,10 +79,10 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) mlx5e_cq_arm(&c->sq[i].cq); if (MLX5E_TEST_BIT(c->rq.state, MLX5E_RQ_STATE_AM)) - mlx5e_rx_am(&c->rq.am, - c->rq.cq.event_ctr, - c->rq.stats.packets, - c->rq.stats.bytes); + net_dim(&c->rq.dim, + c->rq.cq.event_ctr, + c->rq.stats.packets, + c->rq.stats.bytes); mlx5e_cq_arm(&c->rq.cq); mlx5e_cq_arm(&c->icosq.cq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/net_dim.c b/drivers/net/ethernet/mellanox/mlx5/core/net_dim.c index 9fafe1da78a1..decb3707d134 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/net_dim.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/net_dim.c @@ -33,22 +33,22 @@ #include "en.h" -#define MLX5E_PARAMS_AM_NUM_PROFILES 5 +#define NET_DIM_PARAMS_NUM_PROFILES 5 /* Adaptive moderation profiles */ -#define MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256 -#define MLX5E_RX_AM_DEF_PROFILE_CQE 1 -#define MLX5E_RX_AM_DEF_PROFILE_EQE 1 - -/* All profiles sizes must be MLX5E_PARAMS_AM_NUM_PROFILES */ -#define MLX5_AM_EQE_PROFILES { \ - {1, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {8, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {64, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {128, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {256, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ +#define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256 +#define NET_DIM_DEF_PROFILE_CQE 1 +#define NET_DIM_DEF_PROFILE_EQE 1 + +/* All profiles sizes must be NET_PARAMS_DIM_NUM_PROFILES */ +#define NET_DIM_EQE_PROFILES { \ + {1, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {8, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {64, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ } -#define MLX5_AM_CQE_PROFILES { \ +#define NET_DIM_CQE_PROFILES { \ {2, 256}, \ {8, 128}, \ {16, 64}, \ @@ -56,193 +56,193 @@ {64, 64} \ } -static const struct mlx5e_cq_moder -profile[MLX5_CQ_PERIOD_NUM_MODES][MLX5E_PARAMS_AM_NUM_PROFILES] = { - MLX5_AM_EQE_PROFILES, - MLX5_AM_CQE_PROFILES, +static const struct net_dim_cq_moder +profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = { + NET_DIM_EQE_PROFILES, + NET_DIM_CQE_PROFILES, }; -struct mlx5e_cq_moder mlx5e_am_get_profile(u8 cq_period_mode, int ix) +struct net_dim_cq_moder net_dim_get_profile(u8 cq_period_mode, int ix) { - struct mlx5e_cq_moder cq_moder; + struct net_dim_cq_moder cq_moder; cq_moder = profile[cq_period_mode][ix]; cq_moder.cq_period_mode = cq_period_mode; return cq_moder; } -struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode) +struct net_dim_cq_moder net_dim_get_def_profile(u8 rx_cq_period_mode) { int default_profile_ix; - if (rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE) - default_profile_ix = MLX5E_RX_AM_DEF_PROFILE_CQE; - else /* MLX5_CQ_PERIOD_MODE_START_FROM_EQE */ - default_profile_ix = MLX5E_RX_AM_DEF_PROFILE_EQE; + if (rx_cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE) + default_profile_ix = NET_DIM_DEF_PROFILE_CQE; + else /* NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE */ + default_profile_ix = NET_DIM_DEF_PROFILE_EQE; - return mlx5e_am_get_profile(rx_cq_period_mode, default_profile_ix); + return net_dim_get_profile(rx_cq_period_mode, default_profile_ix); } -static bool mlx5e_am_on_top(struct mlx5e_rx_am *am) +static bool net_dim_on_top(struct net_dim *dim) { - switch (am->tune_state) { - case MLX5E_AM_PARKING_ON_TOP: - case MLX5E_AM_PARKING_TIRED: + switch (dim->tune_state) { + case NET_DIM_PARKING_ON_TOP: + case NET_DIM_PARKING_TIRED: return true; - case MLX5E_AM_GOING_RIGHT: - return (am->steps_left > 1) && (am->steps_right == 1); - default: /* MLX5E_AM_GOING_LEFT */ - return (am->steps_right > 1) && (am->steps_left == 1); + case NET_DIM_GOING_RIGHT: + return (dim->steps_left > 1) && (dim->steps_right == 1); + default: /* NET_DIM_GOING_LEFT */ + return (dim->steps_right > 1) && (dim->steps_left == 1); } } -static void mlx5e_am_turn(struct mlx5e_rx_am *am) +static void net_dim_turn(struct net_dim *dim) { - switch (am->tune_state) { - case MLX5E_AM_PARKING_ON_TOP: - case MLX5E_AM_PARKING_TIRED: + switch (dim->tune_state) { + case NET_DIM_PARKING_ON_TOP: + case NET_DIM_PARKING_TIRED: break; - case MLX5E_AM_GOING_RIGHT: - am->tune_state = MLX5E_AM_GOING_LEFT; - am->steps_left = 0; + case NET_DIM_GOING_RIGHT: + dim->tune_state = NET_DIM_GOING_LEFT; + dim->steps_left = 0; break; - case MLX5E_AM_GOING_LEFT: - am->tune_state = MLX5E_AM_GOING_RIGHT; - am->steps_right = 0; + case NET_DIM_GOING_LEFT: + dim->tune_state = NET_DIM_GOING_RIGHT; + dim->steps_right = 0; break; } } -static int mlx5e_am_step(struct mlx5e_rx_am *am) +static int net_dim_step(struct net_dim *dim) { - if (am->tired == (MLX5E_PARAMS_AM_NUM_PROFILES * 2)) - return MLX5E_AM_TOO_TIRED; + if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2)) + return NET_DIM_TOO_TIRED; - switch (am->tune_state) { - case MLX5E_AM_PARKING_ON_TOP: - case MLX5E_AM_PARKING_TIRED: + switch (dim->tune_state) { + case NET_DIM_PARKING_ON_TOP: + case NET_DIM_PARKING_TIRED: break; - case MLX5E_AM_GOING_RIGHT: - if (am->profile_ix == (MLX5E_PARAMS_AM_NUM_PROFILES - 1)) - return MLX5E_AM_ON_EDGE; - am->profile_ix++; - am->steps_right++; + case NET_DIM_GOING_RIGHT: + if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1)) + return NET_DIM_ON_EDGE; + dim->profile_ix++; + dim->steps_right++; break; - case MLX5E_AM_GOING_LEFT: - if (am->profile_ix == 0) - return MLX5E_AM_ON_EDGE; - am->profile_ix--; - am->steps_left++; + case NET_DIM_GOING_LEFT: + if (dim->profile_ix == 0) + return NET_DIM_ON_EDGE; + dim->profile_ix--; + dim->steps_left++; break; } - am->tired++; - return MLX5E_AM_STEPPED; + dim->tired++; + return NET_DIM_STEPPED; } -static void mlx5e_am_park_on_top(struct mlx5e_rx_am *am) +static void net_dim_park_on_top(struct net_dim *dim) { - am->steps_right = 0; - am->steps_left = 0; - am->tired = 0; - am->tune_state = MLX5E_AM_PARKING_ON_TOP; + dim->steps_right = 0; + dim->steps_left = 0; + dim->tired = 0; + dim->tune_state = NET_DIM_PARKING_ON_TOP; } -static void mlx5e_am_park_tired(struct mlx5e_rx_am *am) +static void net_dim_park_tired(struct net_dim *dim) { - am->steps_right = 0; - am->steps_left = 0; - am->tune_state = MLX5E_AM_PARKING_TIRED; + dim->steps_right = 0; + dim->steps_left = 0; + dim->tune_state = NET_DIM_PARKING_TIRED; } -static void mlx5e_am_exit_parking(struct mlx5e_rx_am *am) +static void net_dim_exit_parking(struct net_dim *dim) { - am->tune_state = am->profile_ix ? MLX5E_AM_GOING_LEFT : - MLX5E_AM_GOING_RIGHT; - mlx5e_am_step(am); + dim->tune_state = dim->profile_ix ? NET_DIM_GOING_LEFT : + NET_DIM_GOING_RIGHT; + net_dim_step(dim); } #define IS_SIGNIFICANT_DIFF(val, ref) \ (((100 * abs((val) - (ref))) / (ref)) > 10) /* more than 10% difference */ -static int mlx5e_am_stats_compare(struct mlx5e_rx_am_stats *curr, - struct mlx5e_rx_am_stats *prev) +static int net_dim_stats_compare(struct net_dim_stats *curr, + struct net_dim_stats *prev) { if (!prev->bpms) - return curr->bpms ? MLX5E_AM_STATS_BETTER : - MLX5E_AM_STATS_SAME; + return curr->bpms ? NET_DIM_STATS_BETTER : + NET_DIM_STATS_SAME; if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms)) - return (curr->bpms > prev->bpms) ? MLX5E_AM_STATS_BETTER : - MLX5E_AM_STATS_WORSE; + return (curr->bpms > prev->bpms) ? NET_DIM_STATS_BETTER : + NET_DIM_STATS_WORSE; if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms)) - return (curr->ppms > prev->ppms) ? MLX5E_AM_STATS_BETTER : - MLX5E_AM_STATS_WORSE; + return (curr->ppms > prev->ppms) ? NET_DIM_STATS_BETTER : + NET_DIM_STATS_WORSE; if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms)) - return (curr->epms < prev->epms) ? MLX5E_AM_STATS_BETTER : - MLX5E_AM_STATS_WORSE; + return (curr->epms < prev->epms) ? NET_DIM_STATS_BETTER : + NET_DIM_STATS_WORSE; - return MLX5E_AM_STATS_SAME; + return NET_DIM_STATS_SAME; } -static bool mlx5e_am_decision(struct mlx5e_rx_am_stats *curr_stats, - struct mlx5e_rx_am *am) +static bool net_dim_decision(struct net_dim_stats *curr_stats, + struct net_dim *dim) { - int prev_state = am->tune_state; - int prev_ix = am->profile_ix; + int prev_state = dim->tune_state; + int prev_ix = dim->profile_ix; int stats_res; int step_res; - switch (am->tune_state) { - case MLX5E_AM_PARKING_ON_TOP: - stats_res = mlx5e_am_stats_compare(curr_stats, &am->prev_stats); - if (stats_res != MLX5E_AM_STATS_SAME) - mlx5e_am_exit_parking(am); + switch (dim->tune_state) { + case NET_DIM_PARKING_ON_TOP: + stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); + if (stats_res != NET_DIM_STATS_SAME) + net_dim_exit_parking(dim); break; - case MLX5E_AM_PARKING_TIRED: - am->tired--; - if (!am->tired) - mlx5e_am_exit_parking(am); + case NET_DIM_PARKING_TIRED: + dim->tired--; + if (!dim->tired) + net_dim_exit_parking(dim); break; - case MLX5E_AM_GOING_RIGHT: - case MLX5E_AM_GOING_LEFT: - stats_res = mlx5e_am_stats_compare(curr_stats, &am->prev_stats); - if (stats_res != MLX5E_AM_STATS_BETTER) - mlx5e_am_turn(am); + case NET_DIM_GOING_RIGHT: + case NET_DIM_GOING_LEFT: + stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); + if (stats_res != NET_DIM_STATS_BETTER) + net_dim_turn(dim); - if (mlx5e_am_on_top(am)) { - mlx5e_am_park_on_top(am); + if (net_dim_on_top(dim)) { + net_dim_park_on_top(dim); break; } - step_res = mlx5e_am_step(am); + step_res = net_dim_step(dim); switch (step_res) { - case MLX5E_AM_ON_EDGE: - mlx5e_am_park_on_top(am); + case NET_DIM_ON_EDGE: + net_dim_park_on_top(dim); break; - case MLX5E_AM_TOO_TIRED: - mlx5e_am_park_tired(am); + case NET_DIM_TOO_TIRED: + net_dim_park_tired(dim); break; } break; } - if ((prev_state != MLX5E_AM_PARKING_ON_TOP) || - (am->tune_state != MLX5E_AM_PARKING_ON_TOP)) - am->prev_stats = *curr_stats; + if ((prev_state != NET_DIM_PARKING_ON_TOP) || + (dim->tune_state != NET_DIM_PARKING_ON_TOP)) + dim->prev_stats = *curr_stats; - return am->profile_ix != prev_ix; + return dim->profile_ix != prev_ix; } -static void mlx5e_am_sample(u16 event_ctr, - u64 packets, - u64 bytes, - struct mlx5e_rx_am_sample *s) +static void net_dim_sample(u16 event_ctr, + u64 packets, + u64 bytes, + struct net_dim_sample *s) { s->time = ktime_get(); s->pkt_ctr = packets; @@ -250,13 +250,13 @@ static void mlx5e_am_sample(u16 event_ctr, s->event_ctr = event_ctr; } -#define MLX5E_AM_NEVENTS 64 +#define NET_DIM_NEVENTS 64 #define BITS_PER_TYPE(type) (sizeof(type) * BITS_PER_BYTE) #define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1)) -static void mlx5e_am_calc_stats(struct mlx5e_rx_am_sample *start, - struct mlx5e_rx_am_sample *end, - struct mlx5e_rx_am_stats *curr_stats) +static void net_dim_calc_stats(struct net_dim_sample *start, + struct net_dim_sample *end, + struct net_dim_stats *curr_stats) { /* u32 holds up to 71 minutes, should be enough */ u32 delta_us = ktime_us_delta(end->time, start->time); @@ -269,39 +269,39 @@ static void mlx5e_am_calc_stats(struct mlx5e_rx_am_sample *start, curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us); curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us); - curr_stats->epms = DIV_ROUND_UP(MLX5E_AM_NEVENTS * USEC_PER_MSEC, + curr_stats->epms = DIV_ROUND_UP(NET_DIM_NEVENTS * USEC_PER_MSEC, delta_us); } -void mlx5e_rx_am(struct mlx5e_rx_am *am, - u16 event_ctr, - u64 packets, - u64 bytes) +void net_dim(struct net_dim *dim, + u16 event_ctr, + u64 packets, + u64 bytes) { - struct mlx5e_rx_am_sample end_sample; - struct mlx5e_rx_am_stats curr_stats; + struct net_dim_sample end_sample; + struct net_dim_stats curr_stats; u16 nevents; - switch (am->state) { - case MLX5E_AM_MEASURE_IN_PROGRESS: + switch (dim->state) { + case NET_DIM_MEASURE_IN_PROGRESS: nevents = BIT_GAP(BITS_PER_TYPE(u16), event_ctr, - am->start_sample.event_ctr); - if (nevents < MLX5E_AM_NEVENTS) + dim->start_sample.event_ctr); + if (nevents < NET_DIM_NEVENTS) break; - mlx5e_am_sample(event_ctr, packets, bytes, &end_sample); - mlx5e_am_calc_stats(&am->start_sample, &end_sample, - &curr_stats); - if (mlx5e_am_decision(&curr_stats, am)) { - am->state = MLX5E_AM_APPLY_NEW_PROFILE; - schedule_work(&am->work); + net_dim_sample(event_ctr, packets, bytes, &end_sample); + net_dim_calc_stats(&dim->start_sample, &end_sample, + &curr_stats); + if (net_dim_decision(&curr_stats, dim)) { + dim->state = NET_DIM_APPLY_NEW_PROFILE; + schedule_work(&dim->work); break; } /* fall through */ - case MLX5E_AM_START_MEASURE: - mlx5e_am_sample(event_ctr, packets, bytes, &am->start_sample); - am->state = MLX5E_AM_MEASURE_IN_PROGRESS; + case NET_DIM_START_MEASURE: + net_dim_sample(event_ctr, packets, bytes, &dim->start_sample); + dim->state = NET_DIM_MEASURE_IN_PROGRESS; break; - case MLX5E_AM_APPLY_NEW_PROFILE: + case NET_DIM_APPLY_NEW_PROFILE: break; } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/net_dim.h b/drivers/net/ethernet/mellanox/mlx5/core/net_dim.h index b03a48fc1540..f0a70cd39a2a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/net_dim.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/net_dim.h @@ -31,32 +31,32 @@ * SOFTWARE. */ -#ifndef MLX5_AM_H -#define MLX5_AM_H +#ifndef NET_DIM_H +#define NET_DIM_H -struct mlx5e_cq_moder { +struct net_dim_cq_moder { u16 usec; u16 pkts; u8 cq_period_mode; }; -struct mlx5e_rx_am_sample { +struct net_dim_sample { ktime_t time; u32 pkt_ctr; u32 byte_ctr; u16 event_ctr; }; -struct mlx5e_rx_am_stats { +struct net_dim_stats { int ppms; /* packets per msec */ int bpms; /* bytes per msec */ int epms; /* events per msec */ }; -struct mlx5e_rx_am { /* Adaptive Moderation */ +struct net_dim { /* Adaptive Moderation */ u8 state; - struct mlx5e_rx_am_stats prev_stats; - struct mlx5e_rx_am_sample start_sample; + struct net_dim_stats prev_stats; + struct net_dim_sample start_sample; struct work_struct work; u8 profile_ix; u8 mode; @@ -66,38 +66,43 @@ struct mlx5e_rx_am { /* Adaptive Moderation */ u8 tired; }; +enum { + NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0, + NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1, + NET_DIM_CQ_PERIOD_NUM_MODES +}; + /* Adaptive moderation logic */ enum { - MLX5E_AM_START_MEASURE, - MLX5E_AM_MEASURE_IN_PROGRESS, - MLX5E_AM_APPLY_NEW_PROFILE, + NET_DIM_START_MEASURE, + NET_DIM_MEASURE_IN_PROGRESS, + NET_DIM_APPLY_NEW_PROFILE, }; enum { - MLX5E_AM_PARKING_ON_TOP, - MLX5E_AM_PARKING_TIRED, - MLX5E_AM_GOING_RIGHT, - MLX5E_AM_GOING_LEFT, + NET_DIM_PARKING_ON_TOP, + NET_DIM_PARKING_TIRED, + NET_DIM_GOING_RIGHT, + NET_DIM_GOING_LEFT, }; enum { - MLX5E_AM_STATS_WORSE, - MLX5E_AM_STATS_SAME, - MLX5E_AM_STATS_BETTER, + NET_DIM_STATS_WORSE, + NET_DIM_STATS_SAME, + NET_DIM_STATS_BETTER, }; enum { - MLX5E_AM_STEPPED, - MLX5E_AM_TOO_TIRED, - MLX5E_AM_ON_EDGE, + NET_DIM_STEPPED, + NET_DIM_TOO_TIRED, + NET_DIM_ON_EDGE, }; -void mlx5e_rx_am(struct mlx5e_rx_am *am, - u16 event_ctr, - u64 packets, - u64 bytes); -void mlx5e_rx_am_work(struct work_struct *work); -struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode); -struct mlx5e_cq_moder mlx5e_am_get_profile(u8 cq_period_mode, int ix); +void net_dim(struct net_dim *dim, + u16 event_ctr, + u64 packets, + u64 bytes); +struct net_dim_cq_moder net_dim_get_def_profile(u8 rx_cq_period_mode); +struct net_dim_cq_moder net_dim_get_profile(u8 cq_period_mode, int ix); -#endif /* MLX5_AM_H */ +#endif /* NET_DIM_H */ From 4c4dbb4a7363ce56c77cf8eb2090f67c1b0aa2cc Mon Sep 17 00:00:00 2001 From: Andy Gospodarek Date: Tue, 9 Jan 2018 16:06:18 -0500 Subject: [PATCH 07/10] net/mlx5e: Move dynamic interrupt coalescing code to include/linux This move allows drivers to add private structure elements to track the number of packets, bytes, and interrupts events per ring. A driver also defines a workqueue handler to act on this collected data once per poll and modify the coalescing parameters per ring. Signed-off-by: Andy Gospodarek Acked-by: Tal Gilboa Acked-by: Saeed Mahameed Signed-off-by: David S. Miller --- .../net/ethernet/mellanox/mlx5/core/Makefile | 2 +- drivers/net/ethernet/mellanox/mlx5/core/en.h | 2 +- .../net/ethernet/mellanox/mlx5/core/en_dim.c | 1 + .../net/ethernet/mellanox/mlx5/core/net_dim.h | 108 ---------------- .../core/net_dim.c => include/linux/net_dim.h | 118 ++++++++++++++---- 5 files changed, 97 insertions(+), 134 deletions(-) delete mode 100644 drivers/net/ethernet/mellanox/mlx5/core/net_dim.h rename drivers/net/ethernet/mellanox/mlx5/core/net_dim.c => include/linux/net_dim.h (75%) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index b46b6de2819b..c805769d92a9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -15,7 +15,7 @@ mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cmd.o fpga/core.o fpga/conn.o fpga/sdk.o \ mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \ en_tx.o en_rx.o en_dim.o en_txrx.o en_stats.o vxlan.o \ - en_arfs.o en_fs_ethtool.o en_selftest.o net_dim.o + en_arfs.o en_fs_ethtool.o en_selftest.o mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 349918c4ff73..d629da213511 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -47,10 +47,10 @@ #include #include #include +#include #include "wq.h" #include "mlx5_core.h" #include "en_stats.h" -#include "net_dim.h" #define MLX5_SET_CFG(p, f, v) MLX5_SET(create_flow_group_in, p, f, v) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c index c2612fbf8189..602851ab5b14 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c @@ -30,6 +30,7 @@ * SOFTWARE. */ +#include #include "en.h" void mlx5e_rx_dim_work(struct work_struct *work) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/net_dim.h b/drivers/net/ethernet/mellanox/mlx5/core/net_dim.h deleted file mode 100644 index f0a70cd39a2a..000000000000 --- a/drivers/net/ethernet/mellanox/mlx5/core/net_dim.h +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. - * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef NET_DIM_H -#define NET_DIM_H - -struct net_dim_cq_moder { - u16 usec; - u16 pkts; - u8 cq_period_mode; -}; - -struct net_dim_sample { - ktime_t time; - u32 pkt_ctr; - u32 byte_ctr; - u16 event_ctr; -}; - -struct net_dim_stats { - int ppms; /* packets per msec */ - int bpms; /* bytes per msec */ - int epms; /* events per msec */ -}; - -struct net_dim { /* Adaptive Moderation */ - u8 state; - struct net_dim_stats prev_stats; - struct net_dim_sample start_sample; - struct work_struct work; - u8 profile_ix; - u8 mode; - u8 tune_state; - u8 steps_right; - u8 steps_left; - u8 tired; -}; - -enum { - NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0, - NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1, - NET_DIM_CQ_PERIOD_NUM_MODES -}; - -/* Adaptive moderation logic */ -enum { - NET_DIM_START_MEASURE, - NET_DIM_MEASURE_IN_PROGRESS, - NET_DIM_APPLY_NEW_PROFILE, -}; - -enum { - NET_DIM_PARKING_ON_TOP, - NET_DIM_PARKING_TIRED, - NET_DIM_GOING_RIGHT, - NET_DIM_GOING_LEFT, -}; - -enum { - NET_DIM_STATS_WORSE, - NET_DIM_STATS_SAME, - NET_DIM_STATS_BETTER, -}; - -enum { - NET_DIM_STEPPED, - NET_DIM_TOO_TIRED, - NET_DIM_ON_EDGE, -}; - -void net_dim(struct net_dim *dim, - u16 event_ctr, - u64 packets, - u64 bytes); -struct net_dim_cq_moder net_dim_get_def_profile(u8 rx_cq_period_mode); -struct net_dim_cq_moder net_dim_get_profile(u8 cq_period_mode, int ix); - -#endif /* NET_DIM_H */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/net_dim.c b/include/linux/net_dim.h similarity index 75% rename from drivers/net/ethernet/mellanox/mlx5/core/net_dim.c rename to include/linux/net_dim.h index decb3707d134..741510fb74cb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/net_dim.c +++ b/include/linux/net_dim.h @@ -31,7 +31,74 @@ * SOFTWARE. */ -#include "en.h" +#ifndef NET_DIM_H +#define NET_DIM_H + +#include + +struct net_dim_cq_moder { + u16 usec; + u16 pkts; + u8 cq_period_mode; +}; + +struct net_dim_sample { + ktime_t time; + u32 pkt_ctr; + u32 byte_ctr; + u16 event_ctr; +}; + +struct net_dim_stats { + int ppms; /* packets per msec */ + int bpms; /* bytes per msec */ + int epms; /* events per msec */ +}; + +struct net_dim { /* Adaptive Moderation */ + u8 state; + struct net_dim_stats prev_stats; + struct net_dim_sample start_sample; + struct work_struct work; + u8 profile_ix; + u8 mode; + u8 tune_state; + u8 steps_right; + u8 steps_left; + u8 tired; +}; + +enum { + NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0, + NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1, + NET_DIM_CQ_PERIOD_NUM_MODES +}; + +/* Adaptive moderation logic */ +enum { + NET_DIM_START_MEASURE, + NET_DIM_MEASURE_IN_PROGRESS, + NET_DIM_APPLY_NEW_PROFILE, +}; + +enum { + NET_DIM_PARKING_ON_TOP, + NET_DIM_PARKING_TIRED, + NET_DIM_GOING_RIGHT, + NET_DIM_GOING_LEFT, +}; + +enum { + NET_DIM_STATS_WORSE, + NET_DIM_STATS_SAME, + NET_DIM_STATS_BETTER, +}; + +enum { + NET_DIM_STEPPED, + NET_DIM_TOO_TIRED, + NET_DIM_ON_EDGE, +}; #define NET_DIM_PARAMS_NUM_PROFILES 5 /* Adaptive moderation profiles */ @@ -62,7 +129,8 @@ profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = { NET_DIM_CQE_PROFILES, }; -struct net_dim_cq_moder net_dim_get_profile(u8 cq_period_mode, int ix) +static inline struct net_dim_cq_moder net_dim_get_profile(u8 cq_period_mode, + int ix) { struct net_dim_cq_moder cq_moder; @@ -71,7 +139,7 @@ struct net_dim_cq_moder net_dim_get_profile(u8 cq_period_mode, int ix) return cq_moder; } -struct net_dim_cq_moder net_dim_get_def_profile(u8 rx_cq_period_mode) +static inline struct net_dim_cq_moder net_dim_get_def_profile(u8 rx_cq_period_mode) { int default_profile_ix; @@ -83,7 +151,7 @@ struct net_dim_cq_moder net_dim_get_def_profile(u8 rx_cq_period_mode) return net_dim_get_profile(rx_cq_period_mode, default_profile_ix); } -static bool net_dim_on_top(struct net_dim *dim) +static inline bool net_dim_on_top(struct net_dim *dim) { switch (dim->tune_state) { case NET_DIM_PARKING_ON_TOP: @@ -96,7 +164,7 @@ static bool net_dim_on_top(struct net_dim *dim) } } -static void net_dim_turn(struct net_dim *dim) +static inline void net_dim_turn(struct net_dim *dim) { switch (dim->tune_state) { case NET_DIM_PARKING_ON_TOP: @@ -113,7 +181,7 @@ static void net_dim_turn(struct net_dim *dim) } } -static int net_dim_step(struct net_dim *dim) +static inline int net_dim_step(struct net_dim *dim) { if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2)) return NET_DIM_TOO_TIRED; @@ -140,7 +208,7 @@ static int net_dim_step(struct net_dim *dim) return NET_DIM_STEPPED; } -static void net_dim_park_on_top(struct net_dim *dim) +static inline void net_dim_park_on_top(struct net_dim *dim) { dim->steps_right = 0; dim->steps_left = 0; @@ -148,14 +216,14 @@ static void net_dim_park_on_top(struct net_dim *dim) dim->tune_state = NET_DIM_PARKING_ON_TOP; } -static void net_dim_park_tired(struct net_dim *dim) +static inline void net_dim_park_tired(struct net_dim *dim) { dim->steps_right = 0; dim->steps_left = 0; dim->tune_state = NET_DIM_PARKING_TIRED; } -static void net_dim_exit_parking(struct net_dim *dim) +static inline void net_dim_exit_parking(struct net_dim *dim) { dim->tune_state = dim->profile_ix ? NET_DIM_GOING_LEFT : NET_DIM_GOING_RIGHT; @@ -165,8 +233,8 @@ static void net_dim_exit_parking(struct net_dim *dim) #define IS_SIGNIFICANT_DIFF(val, ref) \ (((100 * abs((val) - (ref))) / (ref)) > 10) /* more than 10% difference */ -static int net_dim_stats_compare(struct net_dim_stats *curr, - struct net_dim_stats *prev) +static inline int net_dim_stats_compare(struct net_dim_stats *curr, + struct net_dim_stats *prev) { if (!prev->bpms) return curr->bpms ? NET_DIM_STATS_BETTER : @@ -187,8 +255,8 @@ static int net_dim_stats_compare(struct net_dim_stats *curr, return NET_DIM_STATS_SAME; } -static bool net_dim_decision(struct net_dim_stats *curr_stats, - struct net_dim *dim) +static inline bool net_dim_decision(struct net_dim_stats *curr_stats, + struct net_dim *dim) { int prev_state = dim->tune_state; int prev_ix = dim->profile_ix; @@ -239,10 +307,10 @@ static bool net_dim_decision(struct net_dim_stats *curr_stats, return dim->profile_ix != prev_ix; } -static void net_dim_sample(u16 event_ctr, - u64 packets, - u64 bytes, - struct net_dim_sample *s) +static inline void net_dim_sample(u16 event_ctr, + u64 packets, + u64 bytes, + struct net_dim_sample *s) { s->time = ktime_get(); s->pkt_ctr = packets; @@ -254,9 +322,9 @@ static void net_dim_sample(u16 event_ctr, #define BITS_PER_TYPE(type) (sizeof(type) * BITS_PER_BYTE) #define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1)) -static void net_dim_calc_stats(struct net_dim_sample *start, - struct net_dim_sample *end, - struct net_dim_stats *curr_stats) +static inline void net_dim_calc_stats(struct net_dim_sample *start, + struct net_dim_sample *end, + struct net_dim_stats *curr_stats) { /* u32 holds up to 71 minutes, should be enough */ u32 delta_us = ktime_us_delta(end->time, start->time); @@ -273,10 +341,10 @@ static void net_dim_calc_stats(struct net_dim_sample *start, delta_us); } -void net_dim(struct net_dim *dim, - u16 event_ctr, - u64 packets, - u64 bytes) +static inline void net_dim(struct net_dim *dim, + u16 event_ctr, + u64 packets, + u64 bytes) { struct net_dim_sample end_sample; struct net_dim_stats curr_stats; @@ -305,3 +373,5 @@ void net_dim(struct net_dim *dim, break; } } + +#endif /* NET_DIM_H */ From 8115b750dbcbeb3aa412f18b7a8b0aee4d35b079 Mon Sep 17 00:00:00 2001 From: Andy Gospodarek Date: Tue, 9 Jan 2018 16:06:19 -0500 Subject: [PATCH 08/10] net/dim: use struct net_dim_sample as arg to net_dim Simplify the arguments net_dim() by formatting them into a struct net_dim_sample before calling the function. Signed-off-by: Andy Gospodarek Suggested-by: Tal Gilboa Acked-by: Tal Gilboa Acked-by: Saeed Mahameed Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c | 13 ++++++++----- include/linux/net_dim.h | 10 +++------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c index a1c94fdc7174..f292bb346985 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c @@ -78,11 +78,14 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) for (i = 0; i < c->num_tc; i++) mlx5e_cq_arm(&c->sq[i].cq); - if (MLX5E_TEST_BIT(c->rq.state, MLX5E_RQ_STATE_AM)) - net_dim(&c->rq.dim, - c->rq.cq.event_ctr, - c->rq.stats.packets, - c->rq.stats.bytes); + if (MLX5E_TEST_BIT(c->rq.state, MLX5E_RQ_STATE_AM)) { + struct net_dim_sample dim_sample; + net_dim_sample(c->rq.cq.event_ctr, + c->rq.stats.packets, + c->rq.stats.bytes, + &dim_sample); + net_dim(&c->rq.dim, dim_sample); + } mlx5e_cq_arm(&c->rq.cq); mlx5e_cq_arm(&c->icosq.cq); diff --git a/include/linux/net_dim.h b/include/linux/net_dim.h index 741510fb74cb..1c7e45016120 100644 --- a/include/linux/net_dim.h +++ b/include/linux/net_dim.h @@ -342,21 +342,18 @@ static inline void net_dim_calc_stats(struct net_dim_sample *start, } static inline void net_dim(struct net_dim *dim, - u16 event_ctr, - u64 packets, - u64 bytes) + struct net_dim_sample end_sample) { - struct net_dim_sample end_sample; struct net_dim_stats curr_stats; u16 nevents; switch (dim->state) { case NET_DIM_MEASURE_IN_PROGRESS: - nevents = BIT_GAP(BITS_PER_TYPE(u16), event_ctr, + nevents = BIT_GAP(BITS_PER_TYPE(u16), + end_sample.event_ctr, dim->start_sample.event_ctr); if (nevents < NET_DIM_NEVENTS) break; - net_dim_sample(event_ctr, packets, bytes, &end_sample); net_dim_calc_stats(&dim->start_sample, &end_sample, &curr_stats); if (net_dim_decision(&curr_stats, dim)) { @@ -366,7 +363,6 @@ static inline void net_dim(struct net_dim *dim, } /* fall through */ case NET_DIM_START_MEASURE: - net_dim_sample(event_ctr, packets, bytes, &dim->start_sample); dim->state = NET_DIM_MEASURE_IN_PROGRESS; break; case NET_DIM_APPLY_NEW_PROFILE: From 6a8788f25625eab31ffa624da2db758ecae6151d Mon Sep 17 00:00:00 2001 From: Andy Gospodarek Date: Tue, 9 Jan 2018 16:06:20 -0500 Subject: [PATCH 09/10] bnxt_en: add support for software dynamic interrupt moderation This implements the changes needed for the bnxt_en driver to add support for dynamic interrupt moderation per ring. This does add additional counters in the receive path, but testing shows that any additional instructions are offset by throughput gain when the default configuration is for low latency. Signed-off-by: Andy Gospodarek Acked-by: Michael Chan Signed-off-by: David S. Miller --- drivers/net/ethernet/broadcom/bnxt/Makefile | 2 +- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 50 +++++++++++++++++++ drivers/net/ethernet/broadcom/bnxt/bnxt.h | 34 +++++++++---- drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c | 32 ++++++++++++ .../net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 12 +++++ 5 files changed, 118 insertions(+), 12 deletions(-) create mode 100644 drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c diff --git a/drivers/net/ethernet/broadcom/bnxt/Makefile b/drivers/net/ethernet/broadcom/bnxt/Makefile index 59c8ec9c1cad..7c560d545c03 100644 --- a/drivers/net/ethernet/broadcom/bnxt/Makefile +++ b/drivers/net/ethernet/broadcom/bnxt/Makefile @@ -1,4 +1,4 @@ obj-$(CONFIG_BNXT) += bnxt_en.o -bnxt_en-y := bnxt.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_vfr.o bnxt_devlink.o +bnxt_en-y := bnxt.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_vfr.o bnxt_devlink.o bnxt_dim.o bnxt_en-$(CONFIG_BNXT_FLOWER_OFFLOAD) += bnxt_tc.o diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 89c3c8760a78..cf6ebf1e324b 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -1645,6 +1645,8 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons, rxr->rx_next_cons = NEXT_RX(cons); next_rx_no_prod: + cpr->rx_packets += 1; + cpr->rx_bytes += len; *raw_cons = tmp_raw_cons; return rc; @@ -1802,6 +1804,7 @@ static irqreturn_t bnxt_msix(int irq, void *dev_instance) struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; u32 cons = RING_CMP(cpr->cp_raw_cons); + cpr->event_ctr++; prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); napi_schedule(&bnapi->napi); return IRQ_HANDLED; @@ -2025,6 +2028,15 @@ static int bnxt_poll(struct napi_struct *napi, int budget) break; } } + if (bp->flags & BNXT_FLAG_DIM) { + struct net_dim_sample dim_sample; + + net_dim_sample(cpr->event_ctr, + cpr->rx_packets, + cpr->rx_bytes, + &dim_sample); + net_dim(&cpr->dim, dim_sample); + } mmiowb(); return work_done; } @@ -2617,6 +2629,8 @@ static void bnxt_init_cp_rings(struct bnxt *bp) struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; ring->fw_ring_id = INVALID_HW_RING_ID; + cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; + cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; } } @@ -4593,6 +4607,36 @@ static void bnxt_hwrm_set_coal_params(struct bnxt_coal *hw_coal, req->flags = cpu_to_le16(flags); } +int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) +{ + struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}; + struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; + struct bnxt_coal coal; + unsigned int grp_idx; + + /* Tick values in micro seconds. + * 1 coal_buf x bufs_per_record = 1 completion record. + */ + memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); + + coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; + coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; + + if (!bnapi->rx_ring) + return -ENODEV; + + bnxt_hwrm_cmd_hdr_init(bp, &req_rx, + HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); + + bnxt_hwrm_set_coal_params(&coal, &req_rx); + + grp_idx = bnapi->index; + req_rx.ring_id = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); + + return hwrm_send_message(bp, &req_rx, sizeof(req_rx), + HWRM_CMD_TIMEOUT); +} + int bnxt_hwrm_set_coal(struct bnxt *bp) { int i, rc = 0; @@ -5715,7 +5759,13 @@ static void bnxt_enable_napi(struct bnxt *bp) int i; for (i = 0; i < bp->cp_nr_rings; i++) { + struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; bp->bnapi[i]->in_reset = false; + + if (bp->bnapi[i]->rx_ring) { + INIT_WORK(&cpr->dim.work, bnxt_dim_work); + cpr->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; + } napi_enable(&bp->bnapi[i]->napi); } } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index 2d268fc26f5e..89887a88b1bd 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -24,6 +24,7 @@ #include #include #include +#include struct tx_bd { __le32 tx_bd_len_flags_type; @@ -608,6 +609,17 @@ struct bnxt_tx_ring_info { struct bnxt_ring_struct tx_ring_struct; }; +struct bnxt_coal { + u16 coal_ticks; + u16 coal_ticks_irq; + u16 coal_bufs; + u16 coal_bufs_irq; + /* RING_IDLE enabled when coal ticks < idle_thresh */ + u16 idle_thresh; + u8 bufs_per_record; + u8 budget; +}; + struct bnxt_tpa_info { void *data; u8 *data_ptr; @@ -672,6 +684,13 @@ struct bnxt_cp_ring_info { u32 cp_raw_cons; void __iomem *cp_doorbell; + struct bnxt_coal rx_ring_coal; + u64 rx_packets; + u64 rx_bytes; + u64 event_ctr; + + struct net_dim dim; + struct tx_cmp *cp_desc_ring[MAX_CP_PAGES]; dma_addr_t cp_desc_mapping[MAX_CP_PAGES]; @@ -946,17 +965,6 @@ struct bnxt_test_info { #define BNXT_CAG_REG_LEGACY_INT_STATUS 0x4014 #define BNXT_CAG_REG_BASE 0x300000 -struct bnxt_coal { - u16 coal_ticks; - u16 coal_ticks_irq; - u16 coal_bufs; - u16 coal_bufs_irq; - /* RING_IDLE enabled when coal ticks < idle_thresh */ - u16 idle_thresh; - u8 bufs_per_record; - u8 budget; -}; - struct bnxt_tc_flow_stats { u64 packets; u64 bytes; @@ -1128,6 +1136,7 @@ struct bnxt { #define BNXT_FLAG_DOUBLE_DB 0x400000 #define BNXT_FLAG_FW_DCBX_AGENT 0x800000 #define BNXT_FLAG_CHIP_NITRO_A0 0x1000000 + #define BNXT_FLAG_DIM 0x2000000 #define BNXT_FLAG_ALL_CONFIG_FEATS (BNXT_FLAG_TPA | \ BNXT_FLAG_RFS | \ @@ -1425,4 +1434,7 @@ int bnxt_setup_mq_tc(struct net_device *dev, u8 tc); int bnxt_get_max_rings(struct bnxt *, int *, int *, bool); void bnxt_restore_pf_fw_resources(struct bnxt *bp); int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr); +void bnxt_dim_work(struct work_struct *work); +int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi); + #endif diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c new file mode 100644 index 000000000000..408dd190331e --- /dev/null +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c @@ -0,0 +1,32 @@ +/* Broadcom NetXtreme-C/E network driver. + * + * Copyright (c) 2017-2018 Broadcom Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + */ + +#include +#include "bnxt_hsi.h" +#include "bnxt.h" + +void bnxt_dim_work(struct work_struct *work) +{ + struct net_dim *dim = container_of(work, struct net_dim, + work); + struct bnxt_cp_ring_info *cpr = container_of(dim, + struct bnxt_cp_ring_info, + dim); + struct bnxt_napi *bnapi = container_of(cpr, + struct bnxt_napi, + cp_ring); + struct net_dim_cq_moder cur_profile = net_dim_get_profile(dim->mode, + dim->profile_ix); + + cpr->rx_ring_coal.coal_ticks = cur_profile.usec; + cpr->rx_ring_coal.coal_bufs = cur_profile.pkts; + + bnxt_hwrm_set_ring_coal(bnapi->bp, bnapi); + dim->state = NET_DIM_START_MEASURE; +} diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index fe7599f404bf..1801582076be 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -49,6 +49,8 @@ static int bnxt_get_coalesce(struct net_device *dev, memset(coal, 0, sizeof(*coal)); + coal->use_adaptive_rx_coalesce = bp->flags & BNXT_FLAG_DIM; + hw_coal = &bp->rx_coal; mult = hw_coal->bufs_per_record; coal->rx_coalesce_usecs = hw_coal->coal_ticks; @@ -77,6 +79,15 @@ static int bnxt_set_coalesce(struct net_device *dev, int rc = 0; u16 mult; + if (coal->use_adaptive_rx_coalesce) { + bp->flags |= BNXT_FLAG_DIM; + } else { + if (bp->flags & BNXT_FLAG_DIM) { + bp->flags &= ~(BNXT_FLAG_DIM); + goto reset_coalesce; + } + } + hw_coal = &bp->rx_coal; mult = hw_coal->bufs_per_record; hw_coal->coal_ticks = coal->rx_coalesce_usecs; @@ -104,6 +115,7 @@ static int bnxt_set_coalesce(struct net_device *dev, update_stats = true; } +reset_coalesce: if (netif_running(dev)) { if (update_stats) { rc = bnxt_close_nic(bp, true, false); From f4e5f0ea7cf4913733fff75c78e4320ed16b60aa Mon Sep 17 00:00:00 2001 From: Andy Gospodarek Date: Tue, 9 Jan 2018 16:06:21 -0500 Subject: [PATCH 10/10] MAINTAINERS: add entry for Dynamic Interrupt Moderation Signed-off-by: Andy Gospodarek Signed-off-by: Tal Gilboa Acked-by: Saeed Mahameed Signed-off-by: David S. Miller --- MAINTAINERS | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index f9af20ccee73..e22ca0ae995d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4942,6 +4942,11 @@ S: Maintained F: lib/dynamic_debug.c F: include/linux/dynamic_debug.h +DYNAMIC INTERRUPT MODERATION +M: Tal Gilboa +S: Maintained +F: include/linux/net_dim.h + DZ DECSTATION DZ11 SERIAL DRIVER M: "Maciej W. Rozycki" S: Maintained