Find max WR

sydidelot · Mar 20, 2024 · 359ca1c · 359ca1c
1 parent 17e1fd0
commit 359ca1c
Show file tree

Hide file tree

Showing 3 changed files with 94 additions and 4 deletions.
diff --git a/prov/verbs/src/verbs_info.c b/prov/verbs/src/verbs_info.c
@@ -463,15 +463,19 @@ static inline int vrb_get_qp_cap(struct ibv_context *ctx,
 	       info->rx_attr->size &&
 	       info->rx_attr->iov_limit);
 
-	init_attr.cap.max_send_wr = MIN(vrb_gl_data.def_tx_size,
-					info->tx_attr->size);
+	init_attr.cap.max_send_wr =
+		vrb_adjust_max_send_wr(pd, ctx, qp_type,
+				       MIN(vrb_gl_data.def_tx_size,
+					   info->tx_attr->size));
 	init_attr.cap.max_send_sge = MIN(vrb_gl_data.def_tx_iov_limit,
 					 info->tx_attr->iov_limit);
 
 	if (qp_type != IBV_QPT_XRC_SEND) {
 		init_attr.recv_cq = cq;
-		init_attr.cap.max_recv_wr = MIN(vrb_gl_data.def_rx_size,
-						info->rx_attr->size);
+		init_attr.cap.max_recv_wr =
+			vrb_adjust_max_recv_wr(pd, ctx, qp_type,
+					       MIN(vrb_gl_data.def_rx_size,
+						   info->rx_attr->size));
 		init_attr.cap.max_recv_sge = MIN(vrb_gl_data.def_rx_iov_limit,
 						 info->rx_attr->iov_limit);
 	}

diff --git a/prov/verbs/src/verbs_init.c b/prov/verbs/src/verbs_init.c
@@ -461,6 +461,88 @@ void vrb_set_rnr_timer(struct ibv_qp *qp)
 	vrb_dbg_query_qp_attr(qp);
 }
 
+int vrb_adjust_max_send_wr(struct ibv_pd *pd, struct ibv_context *context,
+			   enum ibv_qp_type qp_type, int send_wr)
+{
+	struct ibv_qp_init_attr qp_attr;
+	struct ibv_qp *qp = NULL;
+	struct ibv_cq *cq;
+
+	cq = ibv_create_cq(context, 1, NULL, NULL, 0);
+	assert(cq);
+
+	memset(&qp_attr, 0, sizeof(qp_attr));
+	qp_attr.send_cq = cq;
+	qp_attr.qp_type = qp_type;
+	qp_attr.cap.max_send_sge = 1;
+	if (qp_type != IBV_QPT_XRC_SEND) {
+		qp_attr.recv_cq = cq;
+		qp_attr.cap.max_recv_wr = 1;
+		qp_attr.cap.max_recv_sge = 1;
+	}
+	qp_attr.sq_sig_all = 1;
+
+	while (send_wr > 1)
+	{
+		qp_attr.cap.max_send_wr = send_wr;
+		qp = ibv_create_qp(pd, &qp_attr);
+		if (qp) {
+			send_wr = qp_attr.cap.max_send_wr;
+			ibv_destroy_qp(qp);
+			break;
+		} else {
+			send_wr /= 2;
+		}
+	}
+
+	if (cq)
+		ibv_destroy_cq(cq);
+
+	return send_wr;
+}
+
+int vrb_adjust_max_recv_wr(struct ibv_pd *pd, struct ibv_context *context,
+			   enum ibv_qp_type qp_type, int recv_wr)
+{
+	struct ibv_qp_init_attr qp_attr;
+	struct ibv_qp *qp = NULL;
+	struct ibv_cq *cq;
+
+	if (qp_type == IBV_QPT_XRC_SEND)
+		return recv_wr;
+
+	cq = ibv_create_cq(context, 1, NULL, NULL, 0);
+	assert(cq);
+
+	memset(&qp_attr, 0, sizeof(qp_attr));
+	qp_attr.send_cq = cq;
+	qp_attr.qp_type = qp_type;
+	qp_attr.cap.max_send_wr = 1;
+	qp_attr.cap.max_send_sge = 1;
+	qp_attr.recv_cq = cq;
+	qp_attr.cap.max_recv_wr = 1;
+	qp_attr.cap.max_recv_sge = 1;
+	qp_attr.sq_sig_all = 1;
+
+	while (recv_wr > 1)
+	{
+		qp_attr.cap.max_recv_wr = recv_wr;
+		qp = ibv_create_qp(pd, &qp_attr);
+		if (qp) {
+			recv_wr = qp_attr.cap.max_recv_wr;
+			ibv_destroy_qp(qp);
+			break;
+		} else {
+			recv_wr /= 2;
+		}
+	}
+
+	if (cq)
+		ibv_destroy_cq(cq);
+
+	return recv_wr;
+}
+
 int vrb_find_max_inline(struct ibv_pd *pd, struct ibv_context *context,
 			   enum ibv_qp_type qp_type)
 {

diff --git a/prov/verbs/src/verbs_ofi.h b/prov/verbs/src/verbs_ofi.h
@@ -878,6 +878,10 @@ int vrb_query_atomic(struct fid_domain *domain_fid, enum fi_datatype datatype,
 			uint64_t flags);
 void vrb_set_rnr_timer(struct ibv_qp *qp);
 void vrb_cleanup_cq(struct vrb_ep *cur_ep);
+int vrb_adjust_max_send_wr(struct ibv_pd *pd, struct ibv_context *context,
+			   enum ibv_qp_type qp_type, int send_wr);
+int vrb_adjust_max_recv_wr(struct ibv_pd *pd, struct ibv_context *context,
+			   enum ibv_qp_type qp_type, int recv_wr);
 int vrb_find_max_inline(struct ibv_pd *pd, struct ibv_context *context,
 			   enum ibv_qp_type qp_type);