Skip to content

Commit f92e186

Browse files
dthompsodavem330
dthompso
authored and
davem330
committed
Add Mellanox BlueField Gigabit Ethernet driver
This patch adds build and driver logic for the "mlxbf_gige" Ethernet driver from Mellanox Technologies. The second generation BlueField SoC from Mellanox supports an out-of-band GigaBit Ethernet management port to the Arm subsystem. This driver supports TCP/IP network connectivity for that port, and provides back-end routines to handle basic ethtool requests. The driver interfaces to the Gigabit Ethernet block of BlueField SoC via MMIO accesses to registers, which contain control information or pointers describing transmit and receive resources. There is a single transmit queue, and the port supports transmit ring sizes of 4 to 256 entries. There is a single receive queue, and the port supports receive ring sizes of 32 to 32K entries. The transmit and receive rings are allocated from DMA coherent memory. There is a 16-bit producer and consumer index per ring to denote software ownership and hardware ownership, respectively. The main driver logic such as probe(), remove(), and netdev ops are in "mlxbf_gige_main.c". Logic in "mlxbf_gige_rx.c" and "mlxbf_gige_tx.c" handles the packet processing for receive and transmit respectively. The logic in "mlxbf_gige_ethtool.c" supports the handling of some basic ethtool requests: get driver info, get ring parameters, get registers, and get statistics. The logic in "mlxbf_gige_mdio.c" is the driver controlling the Mellanox BlueField hardware that interacts with a PHY device via MDIO/MDC pins. This driver does the following: - At driver probe time, it configures several BlueField MDIO parameters such as sample rate, full drive, voltage and MDC - It defines functions to read and write MDIO registers and registers the MDIO bus. - It defines the phy interrupt handler reporting a link up/down status change - This driver's probe is invoked from the main driver logic while the phy interrupt handler is registered in ndo_open. Driver limitations - Only supports 1Gbps speed - Only supports GMII protocol - Supports maximum packet size of 2KB - Does not support scatter-gather buffering Testing - Successful build of kernel for ARM64, ARM32, X86_64 - Tested ARM64 build on FastModels & Palladium - Tested ARM64 build on several Mellanox boards that are built with the BlueField-2 SoC. The testing includes coverage in the areas of networking (e.g. ping, iperf, ifconfig, route), file transfers (e.g. SCP), and various ethtool options relevant to this driver. Signed-off-by: David Thompson <davthompson@nvidia.com> Signed-off-by: Asmaa Mnebhi <asmaa@nvidia.com> Reviewed-by: Liming Sun <limings@nvidia.com> Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent 19938ba commit f92e186

13 files changed

+2028
-0
lines changed

‎drivers/net/ethernet/mellanox/Kconfig

+1
Original file line numberDiff line numberDiff line change
@@ -22,5 +22,6 @@ source "drivers/net/ethernet/mellanox/mlx4/Kconfig"
2222
source "drivers/net/ethernet/mellanox/mlx5/core/Kconfig"
2323
source "drivers/net/ethernet/mellanox/mlxsw/Kconfig"
2424
source "drivers/net/ethernet/mellanox/mlxfw/Kconfig"
25+
source "drivers/net/ethernet/mellanox/mlxbf_gige/Kconfig"
2526

2627
endif # NET_VENDOR_MELLANOX

‎drivers/net/ethernet/mellanox/Makefile

+1
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,4 @@ obj-$(CONFIG_MLX4_CORE) += mlx4/
77
obj-$(CONFIG_MLX5_CORE) += mlx5/core/
88
obj-$(CONFIG_MLXSW_CORE) += mlxsw/
99
obj-$(CONFIG_MLXFW) += mlxfw/
10+
obj-$(CONFIG_MLXBF_GIGE) += mlxbf_gige/

‎drivers/net/ethernet/mellanox/mlxbf_gige/Kconfig

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause
2+
#
3+
# Mellanox GigE driver configuration
4+
#
5+
6+
config MLXBF_GIGE
7+
tristate "Mellanox Technologies BlueField Gigabit Ethernet support"
8+
depends on (ARM64 && ACPI) || COMPILE_TEST
9+
select PHYLIB
10+
help
11+
The second generation BlueField SoC from Mellanox Technologies
12+
supports an out-of-band Gigabit Ethernet management port to the
13+
Arm subsystem.

‎drivers/net/ethernet/mellanox/mlxbf_gige/Makefile

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause
2+
3+
obj-$(CONFIG_MLXBF_GIGE) += mlxbf_gige.o
4+
5+
mlxbf_gige-y := mlxbf_gige_ethtool.o \
6+
mlxbf_gige_gpio.o \
7+
mlxbf_gige_intr.o \
8+
mlxbf_gige_main.o \
9+
mlxbf_gige_mdio.o \
10+
mlxbf_gige_rx.o \
11+
mlxbf_gige_tx.o

‎drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige.h

+190
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
/* SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause */
2+
3+
/* Header file for Gigabit Ethernet driver for Mellanox BlueField SoC
4+
* - this file contains software data structures and any chip-specific
5+
* data structures (e.g. TX WQE format) that are memory resident.
6+
*
7+
* Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES
8+
*/
9+
10+
#ifndef __MLXBF_GIGE_H__
11+
#define __MLXBF_GIGE_H__
12+
13+
#include <linux/io-64-nonatomic-lo-hi.h>
14+
#include <linux/irqreturn.h>
15+
#include <linux/netdevice.h>
16+
#include <linux/irq.h>
17+
18+
/* The silicon design supports a maximum RX ring size of
19+
* 32K entries. Based on current testing this maximum size
20+
* is not required to be supported. Instead the RX ring
21+
* will be capped at a realistic value of 1024 entries.
22+
*/
23+
#define MLXBF_GIGE_MIN_RXQ_SZ 32
24+
#define MLXBF_GIGE_MAX_RXQ_SZ 1024
25+
#define MLXBF_GIGE_DEFAULT_RXQ_SZ 128
26+
27+
#define MLXBF_GIGE_MIN_TXQ_SZ 4
28+
#define MLXBF_GIGE_MAX_TXQ_SZ 256
29+
#define MLXBF_GIGE_DEFAULT_TXQ_SZ 128
30+
31+
#define MLXBF_GIGE_DEFAULT_BUF_SZ 2048
32+
33+
#define MLXBF_GIGE_DMA_PAGE_SZ 4096
34+
#define MLXBF_GIGE_DMA_PAGE_SHIFT 12
35+
36+
/* There are four individual MAC RX filters. Currently
37+
* two of them are being used: one for the broadcast MAC
38+
* (index 0) and one for local MAC (index 1)
39+
*/
40+
#define MLXBF_GIGE_BCAST_MAC_FILTER_IDX 0
41+
#define MLXBF_GIGE_LOCAL_MAC_FILTER_IDX 1
42+
43+
/* Define for broadcast MAC literal */
44+
#define BCAST_MAC_ADDR 0xFFFFFFFFFFFF
45+
46+
/* There are three individual interrupts:
47+
* 1) Errors, "OOB" interrupt line
48+
* 2) Receive Packet, "OOB_LLU" interrupt line
49+
* 3) LLU and PLU Events, "OOB_PLU" interrupt line
50+
*/
51+
#define MLXBF_GIGE_ERROR_INTR_IDX 0
52+
#define MLXBF_GIGE_RECEIVE_PKT_INTR_IDX 1
53+
#define MLXBF_GIGE_LLU_PLU_INTR_IDX 2
54+
#define MLXBF_GIGE_PHY_INT_N 3
55+
56+
#define MLXBF_GIGE_MDIO_DEFAULT_PHY_ADDR 0x3
57+
58+
#define MLXBF_GIGE_DEFAULT_PHY_INT_GPIO 12
59+
60+
struct mlxbf_gige_stats {
61+
u64 hw_access_errors;
62+
u64 tx_invalid_checksums;
63+
u64 tx_small_frames;
64+
u64 tx_index_errors;
65+
u64 sw_config_errors;
66+
u64 sw_access_errors;
67+
u64 rx_truncate_errors;
68+
u64 rx_mac_errors;
69+
u64 rx_din_dropped_pkts;
70+
u64 tx_fifo_full;
71+
u64 rx_filter_passed_pkts;
72+
u64 rx_filter_discard_pkts;
73+
};
74+
75+
struct mlxbf_gige {
76+
void __iomem *base;
77+
void __iomem *llu_base;
78+
void __iomem *plu_base;
79+
struct device *dev;
80+
struct net_device *netdev;
81+
struct platform_device *pdev;
82+
void __iomem *mdio_io;
83+
struct mii_bus *mdiobus;
84+
void __iomem *gpio_io;
85+
struct irq_domain *irqdomain;
86+
u32 phy_int_gpio_mask;
87+
spinlock_t lock; /* for packet processing indices */
88+
spinlock_t gpio_lock; /* for GPIO bus access */
89+
u16 rx_q_entries;
90+
u16 tx_q_entries;
91+
u64 *tx_wqe_base;
92+
dma_addr_t tx_wqe_base_dma;
93+
u64 *tx_wqe_next;
94+
u64 *tx_cc;
95+
dma_addr_t tx_cc_dma;
96+
dma_addr_t *rx_wqe_base;
97+
dma_addr_t rx_wqe_base_dma;
98+
u64 *rx_cqe_base;
99+
dma_addr_t rx_cqe_base_dma;
100+
u16 tx_pi;
101+
u16 prev_tx_ci;
102+
u64 error_intr_count;
103+
u64 rx_intr_count;
104+
u64 llu_plu_intr_count;
105+
struct sk_buff *rx_skb[MLXBF_GIGE_MAX_RXQ_SZ];
106+
struct sk_buff *tx_skb[MLXBF_GIGE_MAX_TXQ_SZ];
107+
int error_irq;
108+
int rx_irq;
109+
int llu_plu_irq;
110+
int phy_irq;
111+
int hw_phy_irq;
112+
bool promisc_enabled;
113+
u8 valid_polarity;
114+
struct napi_struct napi;
115+
struct mlxbf_gige_stats stats;
116+
};
117+
118+
/* Rx Work Queue Element definitions */
119+
#define MLXBF_GIGE_RX_WQE_SZ 8
120+
121+
/* Rx Completion Queue Element definitions */
122+
#define MLXBF_GIGE_RX_CQE_SZ 8
123+
#define MLXBF_GIGE_RX_CQE_PKT_LEN_MASK GENMASK(10, 0)
124+
#define MLXBF_GIGE_RX_CQE_VALID_MASK GENMASK(11, 11)
125+
#define MLXBF_GIGE_RX_CQE_PKT_STATUS_MASK GENMASK(15, 12)
126+
#define MLXBF_GIGE_RX_CQE_PKT_STATUS_MAC_ERR GENMASK(12, 12)
127+
#define MLXBF_GIGE_RX_CQE_PKT_STATUS_TRUNCATED GENMASK(13, 13)
128+
#define MLXBF_GIGE_RX_CQE_CHKSUM_MASK GENMASK(31, 16)
129+
130+
/* Tx Work Queue Element definitions */
131+
#define MLXBF_GIGE_TX_WQE_SZ_QWORDS 2
132+
#define MLXBF_GIGE_TX_WQE_SZ 16
133+
#define MLXBF_GIGE_TX_WQE_PKT_LEN_MASK GENMASK(10, 0)
134+
#define MLXBF_GIGE_TX_WQE_UPDATE_MASK GENMASK(31, 31)
135+
#define MLXBF_GIGE_TX_WQE_CHKSUM_LEN_MASK GENMASK(42, 32)
136+
#define MLXBF_GIGE_TX_WQE_CHKSUM_START_MASK GENMASK(55, 48)
137+
#define MLXBF_GIGE_TX_WQE_CHKSUM_OFFSET_MASK GENMASK(63, 56)
138+
139+
/* Macro to return packet length of specified TX WQE */
140+
#define MLXBF_GIGE_TX_WQE_PKT_LEN(tx_wqe_addr) \
141+
(*((tx_wqe_addr) + 1) & MLXBF_GIGE_TX_WQE_PKT_LEN_MASK)
142+
143+
/* Tx Completion Count */
144+
#define MLXBF_GIGE_TX_CC_SZ 8
145+
146+
/* List of resources in ACPI table */
147+
enum mlxbf_gige_res {
148+
MLXBF_GIGE_RES_MAC,
149+
MLXBF_GIGE_RES_MDIO9,
150+
MLXBF_GIGE_RES_GPIO0,
151+
MLXBF_GIGE_RES_LLU,
152+
MLXBF_GIGE_RES_PLU
153+
};
154+
155+
/* Version of register data returned by mlxbf_gige_get_regs() */
156+
#define MLXBF_GIGE_REGS_VERSION 1
157+
158+
int mlxbf_gige_mdio_probe(struct platform_device *pdev,
159+
struct mlxbf_gige *priv);
160+
void mlxbf_gige_mdio_remove(struct mlxbf_gige *priv);
161+
irqreturn_t mlxbf_gige_mdio_handle_phy_interrupt(int irq, void *dev_id);
162+
void mlxbf_gige_mdio_enable_phy_int(struct mlxbf_gige *priv);
163+
164+
void mlxbf_gige_set_mac_rx_filter(struct mlxbf_gige *priv,
165+
unsigned int index, u64 dmac);
166+
void mlxbf_gige_get_mac_rx_filter(struct mlxbf_gige *priv,
167+
unsigned int index, u64 *dmac);
168+
void mlxbf_gige_enable_promisc(struct mlxbf_gige *priv);
169+
void mlxbf_gige_disable_promisc(struct mlxbf_gige *priv);
170+
int mlxbf_gige_rx_init(struct mlxbf_gige *priv);
171+
void mlxbf_gige_rx_deinit(struct mlxbf_gige *priv);
172+
int mlxbf_gige_tx_init(struct mlxbf_gige *priv);
173+
void mlxbf_gige_tx_deinit(struct mlxbf_gige *priv);
174+
bool mlxbf_gige_handle_tx_complete(struct mlxbf_gige *priv);
175+
netdev_tx_t mlxbf_gige_start_xmit(struct sk_buff *skb,
176+
struct net_device *netdev);
177+
struct sk_buff *mlxbf_gige_alloc_skb(struct mlxbf_gige *priv,
178+
unsigned int map_len,
179+
dma_addr_t *buf_dma,
180+
enum dma_data_direction dir);
181+
int mlxbf_gige_request_irqs(struct mlxbf_gige *priv);
182+
void mlxbf_gige_free_irqs(struct mlxbf_gige *priv);
183+
int mlxbf_gige_poll(struct napi_struct *napi, int budget);
184+
extern const struct ethtool_ops mlxbf_gige_ethtool_ops;
185+
void mlxbf_gige_update_tx_wqe_next(struct mlxbf_gige *priv);
186+
187+
int mlxbf_gige_gpio_init(struct platform_device *pdev, struct mlxbf_gige *priv);
188+
void mlxbf_gige_gpio_free(struct mlxbf_gige *priv);
189+
190+
#endif /* !defined(__MLXBF_GIGE_H__) */

‎drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c

+137
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
// SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause
2+
3+
/* Ethtool support for Mellanox Gigabit Ethernet driver
4+
*
5+
* Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES
6+
*/
7+
8+
#include <linux/phy.h>
9+
10+
#include "mlxbf_gige.h"
11+
#include "mlxbf_gige_regs.h"
12+
13+
/* Start of struct ethtool_ops functions */
14+
static int mlxbf_gige_get_regs_len(struct net_device *netdev)
15+
{
16+
return MLXBF_GIGE_MMIO_REG_SZ;
17+
}
18+
19+
static void mlxbf_gige_get_regs(struct net_device *netdev,
20+
struct ethtool_regs *regs, void *p)
21+
{
22+
struct mlxbf_gige *priv = netdev_priv(netdev);
23+
24+
regs->version = MLXBF_GIGE_REGS_VERSION;
25+
26+
/* Read entire MMIO register space and store results
27+
* into the provided buffer. Each 64-bit word is converted
28+
* to big-endian to make the output more readable.
29+
*
30+
* NOTE: by design, a read to an offset without an existing
31+
* register will be acknowledged and return zero.
32+
*/
33+
memcpy_fromio(p, priv->base, MLXBF_GIGE_MMIO_REG_SZ);
34+
}
35+
36+
static void mlxbf_gige_get_ringparam(struct net_device *netdev,
37+
struct ethtool_ringparam *ering)
38+
{
39+
struct mlxbf_gige *priv = netdev_priv(netdev);
40+
41+
ering->rx_max_pending = MLXBF_GIGE_MAX_RXQ_SZ;
42+
ering->tx_max_pending = MLXBF_GIGE_MAX_TXQ_SZ;
43+
ering->rx_pending = priv->rx_q_entries;
44+
ering->tx_pending = priv->tx_q_entries;
45+
}
46+
47+
static const struct {
48+
const char string[ETH_GSTRING_LEN];
49+
} mlxbf_gige_ethtool_stats_keys[] = {
50+
{ "hw_access_errors" },
51+
{ "tx_invalid_checksums" },
52+
{ "tx_small_frames" },
53+
{ "tx_index_errors" },
54+
{ "sw_config_errors" },
55+
{ "sw_access_errors" },
56+
{ "rx_truncate_errors" },
57+
{ "rx_mac_errors" },
58+
{ "rx_din_dropped_pkts" },
59+
{ "tx_fifo_full" },
60+
{ "rx_filter_passed_pkts" },
61+
{ "rx_filter_discard_pkts" },
62+
};
63+
64+
static int mlxbf_gige_get_sset_count(struct net_device *netdev, int stringset)
65+
{
66+
if (stringset != ETH_SS_STATS)
67+
return -EOPNOTSUPP;
68+
return ARRAY_SIZE(mlxbf_gige_ethtool_stats_keys);
69+
}
70+
71+
static void mlxbf_gige_get_strings(struct net_device *netdev, u32 stringset,
72+
u8 *buf)
73+
{
74+
if (stringset != ETH_SS_STATS)
75+
return;
76+
memcpy(buf, &mlxbf_gige_ethtool_stats_keys,
77+
sizeof(mlxbf_gige_ethtool_stats_keys));
78+
}
79+
80+
static void mlxbf_gige_get_ethtool_stats(struct net_device *netdev,
81+
struct ethtool_stats *estats,
82+
u64 *data)
83+
{
84+
struct mlxbf_gige *priv = netdev_priv(netdev);
85+
86+
/* Fill data array with interface statistics
87+
*
88+
* NOTE: the data writes must be in
89+
* sync with the strings shown in
90+
* the mlxbf_gige_ethtool_stats_keys[] array
91+
*
92+
* NOTE2: certain statistics below are zeroed upon
93+
* port disable, so the calculation below
94+
* must include the "cached" value of the stat
95+
* plus the value read directly from hardware.
96+
* Cached statistics are currently:
97+
* rx_din_dropped_pkts
98+
* rx_filter_passed_pkts
99+
* rx_filter_discard_pkts
100+
*/
101+
*data++ = priv->stats.hw_access_errors;
102+
*data++ = priv->stats.tx_invalid_checksums;
103+
*data++ = priv->stats.tx_small_frames;
104+
*data++ = priv->stats.tx_index_errors;
105+
*data++ = priv->stats.sw_config_errors;
106+
*data++ = priv->stats.sw_access_errors;
107+
*data++ = priv->stats.rx_truncate_errors;
108+
*data++ = priv->stats.rx_mac_errors;
109+
*data++ = (priv->stats.rx_din_dropped_pkts +
110+
readq(priv->base + MLXBF_GIGE_RX_DIN_DROP_COUNTER));
111+
*data++ = priv->stats.tx_fifo_full;
112+
*data++ = (priv->stats.rx_filter_passed_pkts +
113+
readq(priv->base + MLXBF_GIGE_RX_PASS_COUNTER_ALL));
114+
*data++ = (priv->stats.rx_filter_discard_pkts +
115+
readq(priv->base + MLXBF_GIGE_RX_DISC_COUNTER_ALL));
116+
}
117+
118+
static void mlxbf_gige_get_pauseparam(struct net_device *netdev,
119+
struct ethtool_pauseparam *pause)
120+
{
121+
pause->autoneg = AUTONEG_DISABLE;
122+
pause->rx_pause = 1;
123+
pause->tx_pause = 1;
124+
}
125+
126+
const struct ethtool_ops mlxbf_gige_ethtool_ops = {
127+
.get_link = ethtool_op_get_link,
128+
.get_ringparam = mlxbf_gige_get_ringparam,
129+
.get_regs_len = mlxbf_gige_get_regs_len,
130+
.get_regs = mlxbf_gige_get_regs,
131+
.get_strings = mlxbf_gige_get_strings,
132+
.get_sset_count = mlxbf_gige_get_sset_count,
133+
.get_ethtool_stats = mlxbf_gige_get_ethtool_stats,
134+
.nway_reset = phy_ethtool_nway_reset,
135+
.get_pauseparam = mlxbf_gige_get_pauseparam,
136+
.get_link_ksettings = phy_ethtool_get_link_ksettings,
137+
};

0 commit comments

Comments
 (0)