providers/mana: Create RNIC CQs

Implement creation and destruction of RNIC cqs.

Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>

providers/mana/cq.c

@@ -20,89 +20,112 @@
 
 #include "mana.h"
 
-DECLARE_DRV_CMD(mana_create_cq, IB_USER_VERBS_CMD_CREATE_CQ, mana_ib_create_cq,
-		empty);
+#define INITIALIZED_OWNER_BIT(log2_num_entries) (1UL << (log2_num_entries))
+
+DECLARE_DRV_CMD(mana_create_cq, IB_USER_VERBS_CMD_CREATE_CQ,
+		mana_ib_create_cq, mana_ib_create_cq_resp);
 
 struct ibv_cq *mana_create_cq(struct ibv_context *context, int cqe,
 			      struct ibv_comp_channel *channel, int comp_vector)
 {
 	struct mana_context *ctx = to_mctx(context);
-	struct mana_cq *cq;
-	struct mana_create_cq cmd = {};
 	struct mana_create_cq_resp resp = {};
 	struct mana_ib_create_cq *cmd_drv;
-	int cq_size;
+	struct mana_create_cq cmd = {};
+	struct mana_cq *cq;
+	uint16_t flags = 0;
+	size_t cq_size;
 	int ret;
 
-	if (!ctx->extern_alloc.alloc || !ctx->extern_alloc.free) {
-		/*
-		 * This version of driver doesn't support allocating buffers
-		 * in rdma-core.
-		 */
-		verbs_err(verbs_get_ctx(context),
-			  "Allocating core buffers for CQ is not supported\n");
-		errno = EINVAL;
-		return NULL;
-	}
-
 	cq = calloc(1, sizeof(*cq));
 	if (!cq)
 		return NULL;
 
 	cq_size = align_hw_size(cqe * COMP_ENTRY_SIZE);
-
-	cq->buf = ctx->extern_alloc.alloc(cq_size, ctx->extern_alloc.data);
+	cq->db_page = ctx->db_page;
+	list_head_init(&cq->send_qp_list);
+	list_head_init(&cq->recv_qp_list);
+	pthread_spin_init(&cq->lock, PTHREAD_PROCESS_PRIVATE);
+
+	cq->buf_external = ctx->extern_alloc.alloc && ctx->extern_alloc.free;
+	if (!cq->buf_external)
+		flags |= MANA_IB_CREATE_RNIC_CQ;
+
+	if (cq->buf_external)
+		cq->buf = ctx->extern_alloc.alloc(cq_size, ctx->extern_alloc.data);
+	else
+		cq->buf = mana_alloc_mem(cq_size);
 	if (!cq->buf) {
 		errno = ENOMEM;
 		goto free_cq;
 	}
-	cq->cqe = cqe;
+
+	if (flags & MANA_IB_CREATE_RNIC_CQ)
+		cq->cqe = cq_size / COMP_ENTRY_SIZE;
+	else
+		cq->cqe = cqe; // to preserve old behaviour for DPDK
+	cq->head = INITIALIZED_OWNER_BIT(ilog32(cq->cqe) - 1);
+	cq->last_armed_head = cq->head - 1;
+	cq->ready_wcs = 0;
 
 	cmd_drv = &cmd.drv_payload;
 	cmd_drv->buf_addr = (uintptr_t)cq->buf;
+	cmd_drv->flags = flags;
+	resp.cqid = UINT32_MAX;
 
 	ret = ibv_cmd_create_cq(context, cq->cqe, channel, comp_vector,
 				&cq->ibcq, &cmd.ibv_cmd, sizeof(cmd),
 				&resp.ibv_resp, sizeof(resp));
 
 	if (ret) {
 		verbs_err(verbs_get_ctx(context), "Failed to Create CQ\n");
-		ctx->extern_alloc.free(cq->buf, ctx->extern_alloc.data);
 		errno = ret;
-		goto free_cq;
+		goto free_mem;
+	}
+
+	if (flags & MANA_IB_CREATE_RNIC_CQ) {
+		cq->cqid = resp.cqid;
+		if (cq->cqid == UINT32_MAX) {
+			errno = ENODEV;
+			goto destroy_cq;
+		}
 	}
 
 	return &cq->ibcq;
 
+destroy_cq:
+	ibv_cmd_destroy_cq(&cq->ibcq);
+free_mem:
+	if (cq->buf_external)
+		ctx->extern_alloc.free(cq->buf, ctx->extern_alloc.data);
+	else
+		munmap(cq->buf, cq_size);
 free_cq:
 	free(cq);
 	return NULL;
 }
 
 int mana_destroy_cq(struct ibv_cq *ibcq)
 {
-	int ret;
 	struct mana_cq *cq = container_of(ibcq, struct mana_cq, ibcq);
 	struct mana_context *ctx = to_mctx(ibcq->context);
+	int ret;
 
-	if (!ctx->extern_alloc.free) {
-		/*
-		 * This version of driver doesn't support allocating buffers
-		 * in rdma-core. It's not possible to reach the code here.
-		 */
-		verbs_err(verbs_get_ctx(ibcq->context),
-			  "Invalid external context in destroy CQ\n");
-		return -EINVAL;
-	}
-
+	pthread_spin_lock(&cq->lock);
 	ret = ibv_cmd_destroy_cq(ibcq);
 	if (ret) {
 		verbs_err(verbs_get_ctx(ibcq->context),
 			  "Failed to Destroy CQ\n");
+		pthread_spin_unlock(&cq->lock);
 		return ret;
 	}
+	pthread_spin_destroy(&cq->lock);
+
+	if (cq->buf_external)
+		ctx->extern_alloc.free(cq->buf, ctx->extern_alloc.data);
+	else
+		munmap(cq->buf, cq->cqe * COMP_ENTRY_SIZE);
 
-	ctx->extern_alloc.free(cq->buf, ctx->extern_alloc.data);
 	free(cq);
 
 	return ret;

providers/mana/mana.c

@@ -35,6 +35,18 @@ struct mana_context *to_mctx(struct ibv_context *ibctx)
 	return container_of(ibctx, struct mana_context, ibv_ctx.context);
 }
 
+void *mana_alloc_mem(uint32_t size)
+{
+	void *buf;
+
+	buf = mmap(NULL, size, PROT_READ | PROT_WRITE,
+		   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+
+	if (buf == MAP_FAILED)
+		return NULL;
+	return buf;
+}
+
 int mana_query_device_ex(struct ibv_context *context,
 			 const struct ibv_query_device_ex_input *input,
 			 struct ibv_device_attr_ex *attr, size_t attr_size)

providers/mana/mana.h

@@ -77,9 +77,19 @@ struct mana_wq {
 struct mana_cq {
 	struct ibv_cq ibcq;
 	uint32_t cqe;
+	uint32_t cqid;
 	void *buf;
 
-	uint32_t cqid;
+	pthread_spinlock_t lock;
+	uint32_t head;
+	uint32_t last_armed_head;
+	uint32_t ready_wcs;
+	void *db_page;
+	/* list of qp's that use this cq for send completions */
+	struct list_head send_qp_list;
+	/* list of qp's that use this cq for recv completions */
+	struct list_head recv_qp_list;
+	bool buf_external;
 };
 
 struct mana_device {
@@ -98,6 +108,8 @@ struct mana_parent_domain {
 
 struct mana_context *to_mctx(struct ibv_context *ibctx);
 
+void *mana_alloc_mem(uint32_t size);
+
 int mana_query_device_ex(struct ibv_context *context,
 			 const struct ibv_query_device_ex_input *input,
 			 struct ibv_device_attr_ex *attr, size_t attr_size);