enic(4): fix down/up, MTU changes and more

ifconfig down/up cycles was not working.  Fix that which is required
to support MTU changes.  Now doing ifconfig enic0 mtu 3000 for example
works.  If the MTU is changes in the VIC HW configuration, that is not
reflected in and the OS reports the default 1500.  I need to look at
that but changing it via ifconfig works.  So this is different then
what Linux does.

Change TX interrupt allocation to be in this driver.  Change the admin
interrupt count to 2.  This make multiple queues work but need to be
done as pairs so if the VIC has more TX or RX queues setup in the
VIC configuration it will use the lesser value.

While updating the TX interrupt also add support for devcmd2.

Enable checksum offloading.

PR:	282095

sys/dev/enic/cq_desc.h

@@ -44,14 +44,6 @@ struct cq_desc {
 #define CQ_DESC_COMP_NDX_BITS    12
 #define CQ_DESC_COMP_NDX_MASK    ((1 << CQ_DESC_COMP_NDX_BITS) - 1)
 
-static inline void cq_color_enc(struct cq_desc *desc, const u8 color)
-{
-	if (color)
-		desc->type_color |=  (1 << CQ_DESC_COLOR_SHIFT);
-	else
-		desc->type_color &= ~(1 << CQ_DESC_COLOR_SHIFT);
-}
-
 static inline void cq_desc_enc(struct cq_desc *desc,
 	const u8 type, const u8 color, const u16 q_number,
 	const u16 completed_index)
@@ -87,11 +79,4 @@ static inline void cq_desc_dec(const struct cq_desc *desc_arg,
 		CQ_DESC_COMP_NDX_MASK;
 }
 
-static inline void cq_color_dec(const struct cq_desc *desc_arg, u8 *color)
-{
-	volatile const struct cq_desc *desc = desc_arg;
-
-	*color = (desc->type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK;
-}
-
 #endif /* _CQ_DESC_H_ */

sys/dev/enic/enic.h

@@ -108,13 +108,13 @@ struct vnic_res {
 #define ENIC_DEFAULT_VXLAN_PORT		4789
 
 /*
- * Interrupt 0: LSC and errors
  * Interrupt 1: rx queue 0
  * Interrupt 2: rx queue 1
  * ...
+ * Interrupt x: LSC and errors
  */
 #define ENICPMD_LSC_INTR_OFFSET 0
-#define ENICPMD_RXQ_INTR_OFFSET 1
+#define ENICPMD_RXQ_INTR_OFFSET 0
 
 #include "vnic_devcmd.h"
 
@@ -152,6 +152,9 @@ struct vnic_dev {
 	u64 args[VNIC_DEVCMD_NARGS];
 	int in_reset;
 	struct vnic_intr_coal_timer_info intr_coal_timer_info;
+	struct devcmd2_controller *devcmd2;
+	int (*devcmd_rtn)(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd,
+	    int wait);
 	void *(*alloc_consistent)(void *priv, size_t size,
 	    bus_addr_t *dma_handle, struct iflib_dma_info *res, u8 *name);
 	void (*free_consistent)(void *priv, size_t size, void *vaddr,
@@ -175,6 +178,28 @@ struct intr_queue {
 	struct enic_softc *softc;
 };
 
+#define ENIC_MAX_LINK_SPEEDS		3
+#define ENIC_LINK_SPEED_10G		10000
+#define ENIC_LINK_SPEED_4G		4000
+#define ENIC_LINK_40G_INDEX		2
+#define ENIC_LINK_10G_INDEX		1
+#define ENIC_LINK_4G_INDEX		0
+#define ENIC_RX_COALESCE_RANGE_END	125
+#define ENIC_AIC_TS_BREAK		100
+
+struct enic_rx_coal {
+	u32 small_pkt_range_start;
+	u32 large_pkt_range_start;
+	u32 range_end;
+	u32 use_adaptive_rx_coalesce;
+};
+
+/* Store only the lower range.  Higher range is given by fw. */
+struct enic_intr_mod_range {
+	u32 small_pkt_range_start;
+	u32 large_pkt_range_start;
+};
+
 struct enic {
 	struct enic *next;
 	struct rte_pci_device *pdev;
@@ -267,6 +292,9 @@ struct enic {
 	uint64_t tx_offload_mask; /* PKT_TX flags accepted */
 	struct enic_softc *softc;
 	int port_mtu;
+	struct enic_rx_coal rx_coalesce_setting;
+	u32 rx_coalesce_usecs;
+	u32 tx_coalesce_usecs;
 };
 
 struct enic_softc {
@@ -307,11 +335,6 @@ struct enic_softc {
 
 /* Per-instance private data structure */
 
-static inline unsigned int enic_vnic_rq_count(struct enic *enic)
-{
-	return enic->rq_count;
-}
-
 static inline unsigned int enic_cq_rq(struct enic *enic, unsigned int rq)
 {
 	return rq;
@@ -322,21 +345,6 @@ static inline unsigned int enic_cq_wq(struct enic *enic, unsigned int wq)
 	return enic->rq_count + wq;
 }
 
-static inline uint32_t
-enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
-{
-	uint32_t d = i0 + i1;
-	d -= (d >= n_descriptors) ? n_descriptors : 0;
-	return d;
-}
-
-static inline uint32_t
-enic_ring_sub(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
-{
-	int32_t d = i1 - i0;
-	return (uint32_t)((d < 0) ? ((int32_t)n_descriptors + d) : d);
-}
-
 static inline uint32_t
 enic_ring_incr(uint32_t n_descriptors, uint32_t idx)
 {
@@ -346,34 +354,14 @@ enic_ring_incr(uint32_t n_descriptors, uint32_t idx)
 	return idx;
 }
 
-void enic_free_wq(void *txq);
-int enic_alloc_intr_resources(struct enic *enic);
 int enic_setup_finish(struct enic *enic);
-int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
-		  unsigned int socket_id, uint16_t nb_desc);
 void enic_start_wq(struct enic *enic, uint16_t queue_idx);
 int enic_stop_wq(struct enic *enic, uint16_t queue_idx);
 void enic_start_rq(struct enic *enic, uint16_t queue_idx);
-void enic_free_rq(void *rxq);
-int enic_set_vnic_res(struct enic *enic);
-int enic_init_rss_nic_cfg(struct enic *enic);
-int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu);
-int enic_set_vlan_strip(struct enic *enic);
+int enic_stop_rq(struct enic *enic, uint16_t queue_idx);
+void enic_dev_disable(struct enic *enic);
 int enic_enable(struct enic *enic);
 int enic_disable(struct enic *enic);
-void enic_remove(struct enic *enic);
-int enic_get_link_status(struct enic *enic);
-void enic_dev_stats_clear(struct enic *enic);
-void enic_add_packet_filter(struct enic *enic);
-int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr);
-int enic_del_mac_address(struct enic *enic, int mac_index);
-unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq);
-
-void enic_post_wq_index(struct vnic_wq *wq);
-int enic_probe(struct enic *enic);
-int enic_clsf_init(struct enic *enic);
-void enic_clsf_destroy(struct enic *enic);
-int enic_set_mtu(struct enic *enic, uint16_t new_mtu);
 int enic_link_update(struct enic *enic);
 bool enic_use_vector_rx_handler(struct enic *enic);
 void enic_fdir_info(struct enic *enic);

sys/dev/enic/enic_res.c

@@ -95,11 +95,11 @@ int enic_get_vnic_config(struct enic *enic)
 
 	dev_info(enic_get_dev(enic),
 		"vNIC MAC addr %02x:%02x:%02x:%02x:%02x:%02x "
-		"wq/rq %d/%d mtu d, max mtu:%d\n",
+		"wq/rq %d/%d mtu %d, max mtu:%d\n",
 		enic->mac_addr[0], enic->mac_addr[1], enic->mac_addr[2],
 		enic->mac_addr[3], enic->mac_addr[4], enic->mac_addr[5],
 		c->wq_desc_count, c->rq_desc_count,
-		 /* enic->rte_dev->data->mtu, */ enic->max_mtu);
+		c->mtu, enic->max_mtu);
 	dev_info(enic_get_dev(enic), "vNIC csum tx/rx %s/%s "
 		"rss %s intr mode %s type %s timer %d usec "
 		"loopback tag 0x%04x\n",

sys/dev/enic/enic_res.h

@@ -67,7 +67,5 @@ int enic_set_nic_cfg(struct enic *enic, u8 rss_default_cpu, u8 rss_hash_type,
 	u8 ig_vlan_strip_en);
 void enic_get_res_counts(struct enic *enic);
 void enic_init_vnic_resources(struct enic *enic);
-int enic_alloc_vnic_resources(struct enic *);
-void enic_free_vnic_resources(struct enic *);
 
 #endif /* _ENIC_RES_H_ */

sys/dev/enic/enic_txrx.c

@@ -103,6 +103,7 @@ enic_isc_txd_encap(void *vsc, if_pkt_info_t pi)
 
 	softc = vsc;
 	enic = &softc->enic;
+	if_softc_ctx_t scctx = softc->scctx;
 
 	wq = &enic->wq[pi->ipi_qsidx];
 	nsegs = pi->ipi_nsegs;
@@ -112,6 +113,9 @@ enic_isc_txd_encap(void *vsc, if_pkt_info_t pi)
 	head_idx = wq->head_idx;
 	desc_count = wq->ring.desc_count;
 
+	if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0)
+		offload_mode |= WQ_ENET_OFFLOAD_MODE_CSUM;
+
 	for (i = 0; i < nsegs; i++) {
 		eop = 0;
 		cq = 0;
@@ -320,7 +324,7 @@ enic_isc_rxd_flush(void *vsc, uint16_t rxqid, uint8_t flid, qidx_t pidx)
 static int
 enic_legacy_intr(void *xsc)
 {
-	return -1;
+	return (1);
 }
 
 static inline void
@@ -375,7 +379,7 @@ enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, u8 type,
 
 	vnic_wq_service(&enic->wq[q_number], cq_desc,
 			completed_index, NULL, opaque);
-	return 0;
+	return (0);
 }
 
 static void
@@ -384,7 +388,7 @@ vnic_rq_service(struct vnic_rq *rq, struct cq_desc *cq_desc,
     void(*buf_service)(struct vnic_rq *rq, struct cq_desc *cq_desc,
     /* struct vnic_rq_buf * *buf, */ int skipped, void *opaque), void *opaque)
 {
-
+	if_softc_ctx_t scctx;
 	if_rxd_info_t ri = (if_rxd_info_t) opaque;
 	u8 type, color, eop, sop, ingress_port, vlan_stripped;
 	u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof;
@@ -396,6 +400,8 @@ vnic_rq_service(struct vnic_rq *rq, struct cq_desc *cq_desc,
 	int cqidx;
 	if_rxd_frag_t frag;
 
+	scctx = rq->vdev->softc->scctx;
+
 	cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc,
 	    &type, &color, &q_number, &completed_index,
 	    &ingress_port, &fcoe, &eop, &sop, &rss_type,
@@ -419,6 +425,11 @@ vnic_rq_service(struct vnic_rq *rq, struct cq_desc *cq_desc,
 	ri->iri_cidx = cqidx;
 	ri->iri_nfrags = 1;
 	ri->iri_len = bytes_written;
+
+	if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0)
+		if (!csum_not_calc && (tcp_udp_csum_ok || ipv4_csum_ok)) {
+			ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID);
+		}
 }
 
 static int
@@ -431,7 +442,7 @@ enic_rq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc,
 	vnic_rq_service(&enic->rq[ri->iri_qsidx], cq_desc, completed_index,
 	    VNIC_RQ_RETURN_DESC, NULL, /* enic_rq_indicate_buf, */ opaque);
 
-	return 0;
+	return (0);
 }
 
 void
@@ -468,10 +479,8 @@ enic_stop_wq(struct enic *enic, uint16_t queue_idx)
 	int ret;
 
 	ret = vnic_wq_disable(&enic->wq[queue_idx]);
-	if (ret)
-		return ret;
 
-	return 0;
+	return (ret);
 }
 
 void
@@ -483,3 +492,19 @@ enic_start_rq(struct enic *enic, uint16_t queue_idx)
 	vnic_rq_enable(rq);
 	enic_initial_post_rx(enic, rq);
 }
+
+int
+enic_stop_rq(struct enic *enic, uint16_t queue_idx)
+{
+	int ret;
+
+	ret = vnic_rq_disable(&enic->rq[queue_idx]);
+
+	return (ret);
+}
+
+
+void
+enic_dev_disable(struct enic *enic) {
+	vnic_dev_disable(enic->vdev);
+}