Skip to content

Commit

Permalink
net: mvpp2: replace TX coalescing interrupts with hrtimer
Browse files Browse the repository at this point in the history 
The PP2 controller is capable of per-CPU TX processing, which means there are
per-CPU banked register sets and queues. Current version of the driver supports
TX packet coalescing - once on given CPU sent packets amount reaches a threshold
value, an IRQ occurs. However, there is a single interrupt line responsible for
CPU0/1 TX and RX events (the latter is not per-CPU, the hardware does not
support RSS).

When the top-half executes the interrupt cause is not known. This is why in
NAPI poll function, along with RX processing, IRQ cause register on both
CPU's is accessed in order to determine on which of them the TX coalescing
threshold might have been reached. Thus the egress processing and releasing the
buffers is able to take place on the corresponding CPU. Hitherto approach lead
to an illegal usage of on_each_cpu function in softirq context.

The problem is solved by resigning from TX coalescing interrupts and separating
egress finalization from NAPI processing. For that purpose a method of using
hrtimer is introduced. In main transmit function (mvpp2_tx) buffers are released
once a software coalescing threshold is reached. In case not all the data is
processed a timer is set on this CPU - in its interrupt context a tasklet is
scheduled in which all queues are processed. At once only one timer per-CPU can
be running, which is controlled by a dedicated flag.

This commit removes TX processing from NAPI polling function, disables hardware
coalescing and enables hrtimer with tasklet, using new per-CPU port structure
(mvpp2_port_pcpu).

Signed-off-by: Marcin Wojtas <mw@semihalf.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
@wojtas-marcin @davem330
wojtas-marcin authored and davem330 committed Aug 10, 2015
1 parent 71ce391 commit edc660f
Showing 1 changed file with 130 additions and 47 deletions.
177 changes: 130 additions & 47 deletions drivers/net/ethernet/marvell/mvpp2.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
#include <linux/of_address.h>
#include <linux/phy.h>
#include <linux/clk.h>
#include <linux/hrtimer.h>
#include <linux/ktime.h>
#include <uapi/linux/ppp_defs.h>
#include <net/ip.h>
#include <net/ipv6.h>
Expand Down Expand Up @@ -299,6 +301,7 @@

/* Coalescing */
#define MVPP2_TXDONE_COAL_PKTS_THRESH 15
#define MVPP2_TXDONE_HRTIMER_PERIOD_NS 1000000UL
#define MVPP2_RX_COAL_PKTS 32
#define MVPP2_RX_COAL_USEC 100

Expand Down Expand Up @@ -660,6 +663,14 @@ struct mvpp2_pcpu_stats {
u64 tx_bytes;
};

/* Per-CPU port control */
struct mvpp2_port_pcpu {
struct hrtimer tx_done_timer;
bool timer_scheduled;
/* Tasklet for egress finalization */
struct tasklet_struct tx_done_tasklet;
};

struct mvpp2_port {
u8 id;

Expand All @@ -679,6 +690,9 @@ struct mvpp2_port {
u32 pending_cause_rx;
struct napi_struct napi;

/* Per-CPU port control */
struct mvpp2_port_pcpu __percpu *pcpu;

/* Flags */
unsigned long flags;

Expand Down Expand Up @@ -3798,7 +3812,6 @@ static void mvpp2_interrupts_unmask(void *arg)

mvpp2_write(port->priv, MVPP2_ISR_RX_TX_MASK_REG(port->id),
(MVPP2_CAUSE_MISC_SUM_MASK |
MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK |
MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK));
}

Expand Down Expand Up @@ -4374,23 +4387,6 @@ static void mvpp2_rx_time_coal_set(struct mvpp2_port *port,
rxq->time_coal = usec;
}

/* Set threshold for TX_DONE pkts coalescing */
static void mvpp2_tx_done_pkts_coal_set(void *arg)
{
struct mvpp2_port *port = arg;
int queue;
u32 val;

for (queue = 0; queue < txq_number; queue++) {
struct mvpp2_tx_queue *txq = port->txqs[queue];

val = (txq->done_pkts_coal << MVPP2_TRANSMITTED_THRESH_OFFSET) &
MVPP2_TRANSMITTED_THRESH_MASK;
mvpp2_write(port->priv, MVPP2_TXQ_NUM_REG, txq->id);
mvpp2_write(port->priv, MVPP2_TXQ_THRESH_REG, val);
}
}

/* Free Tx queue skbuffs */
static void mvpp2_txq_bufs_free(struct mvpp2_port *port,
struct mvpp2_tx_queue *txq,
Expand Down Expand Up @@ -4425,7 +4421,7 @@ static inline struct mvpp2_rx_queue *mvpp2_get_rx_queue(struct mvpp2_port *port,
static inline struct mvpp2_tx_queue *mvpp2_get_tx_queue(struct mvpp2_port *port,
u32 cause)
{
int queue = fls(cause >> 16) - 1;
int queue = fls(cause) - 1;

return port->txqs[queue];
}
Expand All @@ -4452,6 +4448,29 @@ static void mvpp2_txq_done(struct mvpp2_port *port, struct mvpp2_tx_queue *txq,
netif_tx_wake_queue(nq);
}

static unsigned int mvpp2_tx_done(struct mvpp2_port *port, u32 cause)
{
struct mvpp2_tx_queue *txq;
struct mvpp2_txq_pcpu *txq_pcpu;
unsigned int tx_todo = 0;

while (cause) {
txq = mvpp2_get_tx_queue(port, cause);
if (!txq)
break;

txq_pcpu = this_cpu_ptr(txq->pcpu);

if (txq_pcpu->count) {
mvpp2_txq_done(port, txq, txq_pcpu);
tx_todo += txq_pcpu->count;
}

cause &= ~(1 << txq->log_id);
}
return tx_todo;
}

/* Rx/Tx queue initialization/cleanup methods */

/* Allocate and initialize descriptors for aggr TXQ */
Expand Down Expand Up @@ -4812,7 +4831,6 @@ static int mvpp2_setup_txqs(struct mvpp2_port *port)
goto err_cleanup;
}

on_each_cpu(mvpp2_tx_done_pkts_coal_set, port, 1);
on_each_cpu(mvpp2_txq_sent_counter_clear, port, 1);
return 0;

Expand Down Expand Up @@ -4894,6 +4912,49 @@ static void mvpp2_link_event(struct net_device *dev)
}
}

static void mvpp2_timer_set(struct mvpp2_port_pcpu *port_pcpu)
{
ktime_t interval;

if (!port_pcpu->timer_scheduled) {
port_pcpu->timer_scheduled = true;
interval = ktime_set(0, MVPP2_TXDONE_HRTIMER_PERIOD_NS);
hrtimer_start(&port_pcpu->tx_done_timer, interval,
HRTIMER_MODE_REL_PINNED);
}
}

static void mvpp2_tx_proc_cb(unsigned long data)
{
struct net_device *dev = (struct net_device *)data;
struct mvpp2_port *port = netdev_priv(dev);
struct mvpp2_port_pcpu *port_pcpu = this_cpu_ptr(port->pcpu);
unsigned int tx_todo, cause;

if (!netif_running(dev))
return;
port_pcpu->timer_scheduled = false;

/* Process all the Tx queues */
cause = (1 << txq_number) - 1;
tx_todo = mvpp2_tx_done(port, cause);

/* Set the timer in case not all the packets were processed */
if (tx_todo)
mvpp2_timer_set(port_pcpu);
}

static enum hrtimer_restart mvpp2_hr_timer_cb(struct hrtimer *timer)
{
struct mvpp2_port_pcpu *port_pcpu = container_of(timer,
struct mvpp2_port_pcpu,
tx_done_timer);

tasklet_schedule(&port_pcpu->tx_done_tasklet);

return HRTIMER_NORESTART;
}

/* Main RX/TX processing routines */

/* Display more error info */
Expand Down Expand Up @@ -5262,6 +5323,17 @@ static int mvpp2_tx(struct sk_buff *skb, struct net_device *dev)
dev_kfree_skb_any(skb);
}

/* Finalize TX processing */
if (txq_pcpu->count >= txq->done_pkts_coal)
mvpp2_txq_done(port, txq, txq_pcpu);

/* Set the timer in case not all frags were processed */
if (txq_pcpu->count <= frags && txq_pcpu->count > 0) {
struct mvpp2_port_pcpu *port_pcpu = this_cpu_ptr(port->pcpu);

mvpp2_timer_set(port_pcpu);
}

return NETDEV_TX_OK;
}

Expand All @@ -5275,10 +5347,11 @@ static inline void mvpp2_cause_error(struct net_device *dev, int cause)
netdev_err(dev, "tx fifo underrun error\n");
}

static void mvpp2_txq_done_percpu(void *arg)
static int mvpp2_poll(struct napi_struct *napi, int budget)
{
struct mvpp2_port *port = arg;
u32 cause_rx_tx, cause_tx, cause_misc;
u32 cause_rx_tx, cause_rx, cause_misc;
int rx_done = 0;
struct mvpp2_port *port = netdev_priv(napi->dev);

/* Rx/Tx cause register
*
Expand All @@ -5292,7 +5365,7 @@ static void mvpp2_txq_done_percpu(void *arg)
*/
cause_rx_tx = mvpp2_read(port->priv,
MVPP2_ISR_RX_TX_CAUSE_REG(port->id));
cause_tx = cause_rx_tx & MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK;
cause_rx_tx &= ~MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK;
cause_misc = cause_rx_tx & MVPP2_CAUSE_MISC_SUM_MASK;

if (cause_misc) {
Expand All @@ -5304,26 +5377,6 @@ static void mvpp2_txq_done_percpu(void *arg)
cause_rx_tx & ~MVPP2_CAUSE_MISC_SUM_MASK);
}

/* Release TX descriptors */
if (cause_tx) {
struct mvpp2_tx_queue *txq = mvpp2_get_tx_queue(port, cause_tx);
struct mvpp2_txq_pcpu *txq_pcpu = this_cpu_ptr(txq->pcpu);

if (txq_pcpu->count)
mvpp2_txq_done(port, txq, txq_pcpu);
}
}

static int mvpp2_poll(struct napi_struct *napi, int budget)
{
u32 cause_rx_tx, cause_rx;
int rx_done = 0;
struct mvpp2_port *port = netdev_priv(napi->dev);

on_each_cpu(mvpp2_txq_done_percpu, port, 1);

cause_rx_tx = mvpp2_read(port->priv,
MVPP2_ISR_RX_TX_CAUSE_REG(port->id));
cause_rx = cause_rx_tx & MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK;

/* Process RX packets */
Expand Down Expand Up @@ -5568,6 +5621,8 @@ static int mvpp2_open(struct net_device *dev)
static int mvpp2_stop(struct net_device *dev)
{
struct mvpp2_port *port = netdev_priv(dev);
struct mvpp2_port_pcpu *port_pcpu;
int cpu;

mvpp2_stop_dev(port);
mvpp2_phy_disconnect(port);
Expand All @@ -5576,6 +5631,13 @@ static int mvpp2_stop(struct net_device *dev)
on_each_cpu(mvpp2_interrupts_mask, port, 1);

free_irq(port->irq, port);
for_each_present_cpu(cpu) {
port_pcpu = per_cpu_ptr(port->pcpu, cpu);

hrtimer_cancel(&port_pcpu->tx_done_timer);
port_pcpu->timer_scheduled = false;
tasklet_kill(&port_pcpu->tx_done_tasklet);
}
mvpp2_cleanup_rxqs(port);
mvpp2_cleanup_txqs(port);

Expand Down Expand Up @@ -5791,7 +5853,6 @@ static int mvpp2_ethtool_set_coalesce(struct net_device *dev,
txq->done_pkts_coal = c->tx_max_coalesced_frames;
}

on_each_cpu(mvpp2_tx_done_pkts_coal_set, port, 1);
return 0;
}

Expand Down Expand Up @@ -6042,6 +6103,7 @@ static int mvpp2_port_probe(struct platform_device *pdev,
{
struct device_node *phy_node;
struct mvpp2_port *port;
struct mvpp2_port_pcpu *port_pcpu;
struct net_device *dev;
struct resource *res;
const char *dt_mac_addr;
Expand All @@ -6051,7 +6113,7 @@ static int mvpp2_port_probe(struct platform_device *pdev,
int features;
int phy_mode;
int priv_common_regs_num = 2;
int err, i;
int err, i, cpu;

dev = alloc_etherdev_mqs(sizeof(struct mvpp2_port), txq_number,
rxq_number);
Expand Down Expand Up @@ -6142,6 +6204,24 @@ static int mvpp2_port_probe(struct platform_device *pdev,
}
mvpp2_port_power_up(port);

port->pcpu = alloc_percpu(struct mvpp2_port_pcpu);
if (!port->pcpu) {
err = -ENOMEM;
goto err_free_txq_pcpu;
}

for_each_present_cpu(cpu) {
port_pcpu = per_cpu_ptr(port->pcpu, cpu);

hrtimer_init(&port_pcpu->tx_done_timer, CLOCK_MONOTONIC,
HRTIMER_MODE_REL_PINNED);
port_pcpu->tx_done_timer.function = mvpp2_hr_timer_cb;
port_pcpu->timer_scheduled = false;

tasklet_init(&port_pcpu->tx_done_tasklet, mvpp2_tx_proc_cb,
(unsigned long)dev);
}

netif_napi_add(dev, &port->napi, mvpp2_poll, NAPI_POLL_WEIGHT);
features = NETIF_F_SG | NETIF_F_IP_CSUM;
dev->features = features | NETIF_F_RXCSUM;
Expand All @@ -6151,7 +6231,7 @@ static int mvpp2_port_probe(struct platform_device *pdev,
err = register_netdev(dev);
if (err < 0) {
dev_err(&pdev->dev, "failed to register netdev\n");
goto err_free_txq_pcpu;
goto err_free_port_pcpu;
}
netdev_info(dev, "Using %s mac address %pM\n", mac_from, dev->dev_addr);

Expand All @@ -6160,6 +6240,8 @@ static int mvpp2_port_probe(struct platform_device *pdev,
priv->port_list[id] = port;
return 0;

err_free_port_pcpu:
free_percpu(port->pcpu);
err_free_txq_pcpu:
for (i = 0; i < txq_number; i++)
free_percpu(port->txqs[i]->pcpu);
Expand All @@ -6178,6 +6260,7 @@ static void mvpp2_port_remove(struct mvpp2_port *port)
int i;

unregister_netdev(port->dev);
free_percpu(port->pcpu);
free_percpu(port->stats);
for (i = 0; i < txq_number; i++)
free_percpu(port->txqs[i]->pcpu);
Expand Down

0 comments on commit edc660f

Please sign in to comment.