IB/core: Define 'ib' and 'roce' rdma_ah_attr types

rdma_ah_attr can now be either ib or roce allowing
core components to use one type or the other and also
to define attributes unique to a specific type. struct
ib_ah is also initialized with the type when its first
created. This ensures that calls such as modify_ah
dont modify the type of the address handle attribute.

Reviewed-by: Ira Weiny <ira.weiny@intel.com>
Reviewed-by: Don Hiatt <don.hiatt@intel.com>
Reviewed-by: Sean Hefty <sean.hefty@intel.com>
Reviewed-by: Niranjana Vishwanathapura <niranjana.vishwanathapura@intel.com>
Signed-off-by: Dasaratharaman Chandramouli <dasaratharaman.chandramouli@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

drivers/infiniband/core/cm.c

@@ -1761,7 +1761,9 @@ static int cm_req_handler(struct cm_work *work)
 	cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
 	cm_format_paths_from_req(req_msg, &work->path[0], &work->path[1]);
 
-	memcpy(work->path[0].dmac, cm_id_priv->av.ah_attr.dmac, ETH_ALEN);
+	if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
+		memcpy(work->path[0].dmac, cm_id_priv->av.ah_attr.roce.dmac,
+		       ETH_ALEN);
 	grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr);
 	work->path[0].hop_limit = grh->hop_limit;
 	ret = ib_get_cached_gid(work->port->cm_dev->ib_device,

drivers/infiniband/core/multicast.c

@@ -743,6 +743,7 @@ int ib_init_ah_from_mcmember(struct ib_device *device, u8 port_num,
 		return ret;
 
 	memset(ah_attr, 0, sizeof *ah_attr);
+	ah_attr->type = rdma_ah_find_type(device, port_num);
 
 	rdma_ah_set_dlid(ah_attr, be16_to_cpu(rec->mlid));
 	rdma_ah_set_sl(ah_attr, rec->sl);

drivers/infiniband/core/sa_query.c

@@ -1108,6 +1108,7 @@ int ib_init_ah_from_path(struct ib_device *device, u8 port_num,
 	struct net_device *ndev = NULL;
 
 	memset(ah_attr, 0, sizeof *ah_attr);
+	ah_attr->type = rdma_ah_find_type(device, port_num);
 
 	rdma_ah_set_dlid(ah_attr, be16_to_cpu(rec->dlid));
 	rdma_ah_set_sl(ah_attr, rec->sl);
@@ -1192,7 +1193,7 @@ int ib_init_ah_from_path(struct ib_device *device, u8 port_num,
 	}
 
 	if (use_roce)
-		memcpy(ah_attr->dmac, rec->dmac, ETH_ALEN);
+		memcpy(ah_attr->roce.dmac, rec->dmac, ETH_ALEN);
 
 	return 0;
 }
@@ -2029,6 +2030,8 @@ static void update_sm_ah(struct work_struct *work)
 		pr_err("Couldn't find index for default PKey\n");
 
 	memset(&ah_attr, 0, sizeof(ah_attr));
+	ah_attr.type = rdma_ah_find_type(port->agent->device,
+					 port->port_num);
 	rdma_ah_set_dlid(&ah_attr, port_attr.sm_lid);
 	rdma_ah_set_sl(&ah_attr, port_attr.sm_sl);
 	rdma_ah_set_port_num(&ah_attr, port->port_num);

drivers/infiniband/core/user_mad.c

@@ -491,6 +491,8 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,
 	}
 
 	memset(&ah_attr, 0, sizeof ah_attr);
+	ah_attr.type = rdma_ah_find_type(file->port->ib_dev,
+					 file->port->port_num);
 	rdma_ah_set_dlid(&ah_attr, be16_to_cpu(packet->mad.hdr.lid));
 	rdma_ah_set_sl(&ah_attr, packet->mad.hdr.sl);
 	rdma_ah_set_path_bits(&ah_attr, packet->mad.hdr.path_bits);

drivers/infiniband/core/uverbs_cmd.c

@@ -1954,6 +1954,8 @@ static int modify_qp(struct ib_uverbs_file *file,
 	attr->alt_timeout	  = cmd->base.alt_timeout;
 	attr->rate_limit	  = cmd->rate_limit;
 
+	attr->ah_attr.type = rdma_ah_find_type(qp->device,
+					       cmd->base.dest.port_num);
 	if (cmd->base.dest.is_global) {
 		rdma_ah_set_grh(&attr->ah_attr, NULL,
 				cmd->base.dest.flow_label,
@@ -1971,6 +1973,8 @@ static int modify_qp(struct ib_uverbs_file *file,
 	rdma_ah_set_port_num(&attr->ah_attr,
 			     cmd->base.dest.port_num);
 
+	attr->alt_ah_attr.type = rdma_ah_find_type(qp->device,
+						   cmd->base.dest.port_num);
 	if (cmd->base.alt_dest.is_global) {
 		rdma_ah_set_grh(&attr->alt_ah_attr, NULL,
 				cmd->base.alt_dest.flow_label,
@@ -2551,6 +2555,7 @@ ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file,
 		goto err;
 	}
 
+	attr.type = rdma_ah_find_type(ib_dev, cmd.attr.port_num);
 	rdma_ah_set_dlid(&attr, cmd.attr.dlid);
 	rdma_ah_set_sl(&attr, cmd.attr.sl);
 	rdma_ah_set_path_bits(&attr, cmd.attr.src_path_bits);

drivers/infiniband/core/verbs.c

@@ -321,6 +321,7 @@ struct ib_ah *rdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr)
 		ah->device  = pd->device;
 		ah->pd      = pd;
 		ah->uobject = NULL;
+		ah->type    = ah_attr->type;
 		atomic_inc(&pd->usecnt);
 	}
 
@@ -464,6 +465,7 @@ int ib_init_ah_from_wc(struct ib_device *device, u8 port_num,
 	union ib_gid sgid;
 
 	memset(ah_attr, 0, sizeof *ah_attr);
+	ah_attr->type = rdma_ah_find_type(device, port_num);
 	if (rdma_cap_eth_ah(device, port_num)) {
 		if (wc->wc_flags & IB_WC_WITH_NETWORK_HDR_TYPE)
 			net_type = wc->network_hdr_type;
@@ -494,7 +496,7 @@ int ib_init_ah_from_wc(struct ib_device *device, u8 port_num,
 			return -ENODEV;
 
 		ret = rdma_addr_find_l2_eth_by_grh(&dgid, &sgid,
-						   ah_attr->dmac,
+						   ah_attr->roce.dmac,
 						   wc->wc_flags & IB_WC_WITH_VLAN ?
 						   NULL : &vlan_id,
 						   &if_index, &hoplimit);
@@ -571,6 +573,9 @@ EXPORT_SYMBOL(ib_create_ah_from_wc);
 
 int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr)
 {
+	if (ah->type != ah_attr->type)
+		return -EINVAL;
+
 	return ah->device->modify_ah ?
 		ah->device->modify_ah(ah, ah_attr) :
 		-ENOSYS;
@@ -1207,14 +1212,14 @@ int ib_resolve_eth_dmac(struct ib_device *device,
 	if (!rdma_is_port_valid(device, rdma_ah_get_port_num(ah_attr)))
 		return -EINVAL;
 
-	if (!rdma_cap_eth_ah(device, rdma_ah_get_port_num(ah_attr)))
+	if (ah_attr->type != RDMA_AH_ATTR_TYPE_ROCE)
 		return 0;
 
 	grh = rdma_ah_retrieve_grh(ah_attr);
 
 	if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw)) {
 		rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw,
-				ah_attr->dmac);
+				ah_attr->roce.dmac);
 	} else {
 		union ib_gid		sgid;
 		struct ib_gid_attr	sgid_attr;
@@ -1236,7 +1241,7 @@ int ib_resolve_eth_dmac(struct ib_device *device,
 
 		ret =
 		rdma_addr_find_l2_eth_by_grh(&sgid, &grh->dgid,
-					     ah_attr->dmac,
+					     ah_attr->roce.dmac,
 					     NULL, &ifindex, &hop_limit);
 
 		dev_put(sgid_attr.ndev);

drivers/infiniband/hw/bnxt_re/ib_verbs.c

@@ -597,7 +597,7 @@ struct ib_ah *bnxt_re_create_ah(struct ib_pd *ib_pd,
 			break;
 		}
 		rc = rdma_addr_find_l2_eth_by_grh(&sgid, &grh->dgid,
-						  ah_attr->dmac, &vlan_tag,
+						  ah_attr->roce.dmac, &vlan_tag,
 						  &sgid_attr.ndev->ifindex,
 						  NULL);
 		if (rc) {
@@ -606,7 +606,7 @@ struct ib_ah *bnxt_re_create_ah(struct ib_pd *ib_pd,
 		}
 	}
 
-	memcpy(ah->qplib_ah.dmac, ah_attr->dmac, ETH_ALEN);
+	memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN);
 	rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah);
 	if (rc) {
 		dev_err(rdev_to_dev(rdev), "Failed to allocate HW AH");
@@ -644,8 +644,9 @@ int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
 {
 	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 
+	ah_attr->type = ib_ah->type;
 	rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl);
-	memcpy(ah_attr->dmac, ah->qplib_ah.dmac, ETH_ALEN);
+	memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN);
 	rdma_ah_set_grh(ah_attr, NULL, 0,
 			ah->qplib_ah.host_sgid_index,
 			0, ah->qplib_ah.traffic_class);
@@ -1280,7 +1281,8 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 		qp->qplib_qp.ah.hop_limit = grh->hop_limit;
 		qp->qplib_qp.ah.traffic_class = grh->traffic_class;
 		qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr);
-		ether_addr_copy(qp->qplib_qp.ah.dmac, qp_attr->ah_attr.dmac);
+		ether_addr_copy(qp->qplib_qp.ah.dmac,
+				qp_attr->ah_attr.roce.dmac);
 
 		status = ib_get_cached_gid(&rdev->ibdev, 1,
 					   grh->sgid_index,
@@ -1423,13 +1425,14 @@ int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 	qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp.access);
 	qp_attr->pkey_index = qplib_qp.pkey_index;
 	qp_attr->qkey = qplib_qp.qkey;
+	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
 	rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp.ah.flow_label,
 			qplib_qp.ah.host_sgid_index,
 			qplib_qp.ah.hop_limit,
 			qplib_qp.ah.traffic_class);
 	rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp.ah.dgid.data);
 	rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp.ah.sl);
-	ether_addr_copy(qp_attr->ah_attr.dmac, qplib_qp.ah.dmac);
+	ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp.ah.dmac);
 	qp_attr->path_mtu = __to_ib_mtu(qplib_qp.path_mtu);
 	qp_attr->timeout = qplib_qp.timeout;
 	qp_attr->retry_cnt = qplib_qp.retry_cnt;

drivers/infiniband/hw/hfi1/verbs.c

@@ -1511,8 +1511,12 @@ struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid)
 	struct rdma_ah_attr attr;
 	struct ib_ah *ah = ERR_PTR(-EINVAL);
 	struct rvt_qp *qp0;
+	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
+	struct hfi1_devdata *dd = dd_from_ppd(ppd);
+	u8 port_num = ppd->port;
 
 	memset(&attr, 0, sizeof(attr));
+	attr.type = rdma_ah_find_type(&dd->verbs_dev.rdi.ibdev, port_num);
 	rdma_ah_set_dlid(&attr, dlid);
 	rdma_ah_set_port_num(&attr, ppd_from_ibp(ibp)->port);
 	rcu_read_lock();

drivers/infiniband/hw/hns/hns_roce_hw_v1.c

@@ -2738,7 +2738,7 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 			goto out;
 		}
 
-		dmac = (u8 *)attr->ah_attr.dmac;
+		dmac = (u8 *)attr->ah_attr.roce.dmac;
 
 		context->sq_rq_bt_l = (u32)(dma_handle);
 		roce_set_field(context->qpc_bytes_24,

drivers/infiniband/hw/mlx4/ah.c

@@ -96,7 +96,7 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd,
 		is_mcast = 1;
 		rdma_get_mcast_mac(&in6, ah->av.eth.mac);
 	} else {
-		memcpy(ah->av.eth.mac, ah_attr->dmac, ETH_ALEN);
+		memcpy(ah->av.eth.mac, ah_attr->roce.dmac, ETH_ALEN);
 	}
 	ret = ib_get_cached_gid(pd->device, rdma_ah_get_port_num(ah_attr),
 				grh->sgid_index, &sgid, &gid_attr);
@@ -154,9 +154,7 @@ struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
 	if (!ah)
 		return ERR_PTR(-ENOMEM);
 
-	if (rdma_port_get_link_layer(pd->device,
-				     rdma_ah_get_port_num(ah_attr)) ==
-	    IB_LINK_LAYER_ETHERNET) {
+	if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
 		if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) {
 			ret = ERR_PTR(-EINVAL);
 		} else {
@@ -182,30 +180,29 @@ struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
 int mlx4_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
 {
 	struct mlx4_ib_ah *ah = to_mah(ibah);
-	enum rdma_link_layer ll;
+	int port_num = be32_to_cpu(ah->av.ib.port_pd) >> 24;
 
 	memset(ah_attr, 0, sizeof *ah_attr);
-	rdma_ah_set_port_num(ah_attr,
-			     be32_to_cpu(ah->av.ib.port_pd) >> 24);
-	ll = rdma_port_get_link_layer(ibah->device,
-				      rdma_ah_get_port_num(ah_attr));
-	if (ll == IB_LINK_LAYER_ETHERNET)
+	ah_attr->type = ibah->type;
+
+	if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
+		rdma_ah_set_dlid(ah_attr, 0);
 		rdma_ah_set_sl(ah_attr,
 			       be32_to_cpu(ah->av.eth.sl_tclass_flowlabel)
 			       >> 29);
-	else
+	} else {
+		rdma_ah_set_dlid(ah_attr, be16_to_cpu(ah->av.ib.dlid));
 		rdma_ah_set_sl(ah_attr,
 			       be32_to_cpu(ah->av.ib.sl_tclass_flowlabel)
 			       >> 28);
+	}
 
-	rdma_ah_set_dlid(ah_attr, (ll == IB_LINK_LAYER_INFINIBAND) ?
-			 be16_to_cpu(ah->av.ib.dlid) : 0);
+	rdma_ah_set_port_num(ah_attr, port_num);
 	if (ah->av.ib.stat_rate)
 		rdma_ah_set_static_rate(ah_attr,
 					ah->av.ib.stat_rate -
 					MLX4_STAT_RATE_OFFSET);
 	rdma_ah_set_path_bits(ah_attr, ah->av.ib.g_slid & 0x7F);
-
 	if (mlx4_ib_ah_grh_present(ah)) {
 		u32 tc_fl = be32_to_cpu(ah->av.ib.sl_tclass_flowlabel);
 

drivers/infiniband/hw/mlx4/mad.c

@@ -196,6 +196,7 @@ static void update_sm_ah(struct mlx4_ib_dev *dev, u8 port_num, u16 lid, u8 sl)
 		return;
 
 	memset(&ah_attr, 0, sizeof ah_attr);
+	ah_attr.type = rdma_ah_find_type(&dev->ib_dev, port_num);
 	rdma_ah_set_dlid(&ah_attr, lid);
 	rdma_ah_set_sl(&ah_attr, sl);
 	rdma_ah_set_port_num(&ah_attr, port_num);
@@ -555,6 +556,7 @@ int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port,
 	/* create ah. Just need an empty one with the port num for the post send.
 	 * The driver will set the force loopback bit in post_send */
 	memset(&attr, 0, sizeof attr);
+	attr.type = rdma_ah_find_type(&dev->ib_dev, port);
 
 	rdma_ah_set_port_num(&attr, port);
 	if (is_eth) {

drivers/infiniband/hw/mlx4/qp.c

@@ -1388,8 +1388,6 @@ static int _mlx4_set_path(struct mlx4_ib_dev *dev,
 			  u64 smac, u16 vlan_tag, struct mlx4_qp_path *path,
 			  struct mlx4_roce_smac_vlan_info *smac_info, u8 port)
 {
-	int is_eth = rdma_port_get_link_layer(&dev->ib_dev, port) ==
-		IB_LINK_LAYER_ETHERNET;
 	int vidx;
 	int smac_index;
 	int err;
@@ -1426,7 +1424,7 @@ static int _mlx4_set_path(struct mlx4_ib_dev *dev,
 		memcpy(path->rgid, grh->dgid.raw, 16);
 	}
 
-	if (is_eth) {
+	if (ah->type == RDMA_AH_ATTR_TYPE_ROCE) {
 		if (!(rdma_ah_get_ah_flags(ah) & IB_AH_GRH))
 			return -1;
 
@@ -1490,7 +1488,7 @@ static int _mlx4_set_path(struct mlx4_ib_dev *dev,
 		} else {
 			smac_index = smac_info->smac_index;
 		}
-		memcpy(path->dmac, ah->dmac, 6);
+		memcpy(path->dmac, ah->roce.dmac, 6);
 		path->ackto = MLX4_IB_LINK_TYPE_ETH;
 		/* put MAC table smac index for IBoE */
 		path->grh_mylmc = (u8) (smac_index) | 0x80;
@@ -3402,23 +3400,19 @@ static void to_rdma_ah_attr(struct mlx4_ib_dev *ibdev,
 			    struct mlx4_qp_path *path)
 {
 	struct mlx4_dev *dev = ibdev->dev;
-	int is_eth;
 	u8 port_num = path->sched_queue & 0x40 ? 2 : 1;
 
 	memset(ah_attr, 0, sizeof(*ah_attr));
-	rdma_ah_set_port_num(ah_attr, port_num);
-
+	ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, port_num);
 	if (port_num == 0 || port_num > dev->caps.num_ports)
 		return;
 
-	is_eth = rdma_port_get_link_layer(&ibdev->ib_dev,
-					  rdma_ah_get_port_num(ah_attr)) ==
-			IB_LINK_LAYER_ETHERNET;
-	if (is_eth)
+	if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE)
 		rdma_ah_set_sl(ah_attr, ((path->sched_queue >> 3) & 0x7) |
 			       ((path->sched_queue & 4) << 1));
 	else
 		rdma_ah_set_sl(ah_attr, (path->sched_queue >> 2) & 0xf);
+	rdma_ah_set_port_num(ah_attr, port_num);
 
 	rdma_ah_set_dlid(ah_attr, be16_to_cpu(path->rlid));
 	rdma_ah_set_path_bits(ah_attr, path->grh_mylmc & 0x7f);