Merge pull request #1461 from TaranovK/for-next/cq_upstream

providers/mana: RC QPs

providers/mana/CMakeLists.txt

@@ -4,6 +4,8 @@ rdma_shared_provider(mana libmana.map
   manadv.c
   qp.c
   wq.c
+  cq.c
+  wr.c
 )
 
 publish_headers(infiniband

providers/mana/cq.c

@@ -0,0 +1,362 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright (c) 2024, Microsoft Corporation. All rights reserved.
+ */
+
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <stdatomic.h>
+#include <util/compiler.h>
+#include <util/util.h>
+#include <sys/mman.h>
+
+#include <infiniband/driver.h>
+
+#include <infiniband/kern-abi.h>
+#include <rdma/mana-abi.h>
+#include <kernel-abi/mana-abi.h>
+
+#include "mana.h"
+#include "gdma.h"
+#include "doorbells.h"
+#include "rollback.h"
+#define INITIALIZED_OWNER_BIT(log2_num_entries) (1UL << (log2_num_entries))
+
+DECLARE_DRV_CMD(mana_create_cq, IB_USER_VERBS_CMD_CREATE_CQ,
+		mana_ib_create_cq, mana_ib_create_cq_resp);
+
+struct ibv_cq *mana_create_cq(struct ibv_context *context, int cqe,
+			      struct ibv_comp_channel *channel, int comp_vector)
+{
+	struct mana_context *ctx = to_mctx(context);
+	struct mana_create_cq_resp resp = {};
+	struct mana_ib_create_cq *cmd_drv;
+	struct mana_create_cq cmd = {};
+	struct mana_cq *cq;
+	uint16_t flags = 0;
+	size_t cq_size;
+	int ret;
+
+	cq = calloc(1, sizeof(*cq));
+	if (!cq)
+		return NULL;
+
+	cq_size = align_hw_size(cqe * COMP_ENTRY_SIZE);
+	cq->db_page = ctx->db_page;
+	list_head_init(&cq->send_qp_list);
+	list_head_init(&cq->recv_qp_list);
+	pthread_spin_init(&cq->lock, PTHREAD_PROCESS_PRIVATE);
+
+	cq->buf_external = ctx->extern_alloc.alloc && ctx->extern_alloc.free;
+	if (!cq->buf_external)
+		flags |= MANA_IB_CREATE_RNIC_CQ;
+
+	if (cq->buf_external)
+		cq->buf = ctx->extern_alloc.alloc(cq_size, ctx->extern_alloc.data);
+	else
+		cq->buf = mana_alloc_mem(cq_size);
+	if (!cq->buf) {
+		errno = ENOMEM;
+		goto free_cq;
+	}
+
+	if (flags & MANA_IB_CREATE_RNIC_CQ)
+		cq->cqe = cq_size / COMP_ENTRY_SIZE;
+	else
+		cq->cqe = cqe; // to preserve old behaviour for DPDK
+	cq->head = INITIALIZED_OWNER_BIT(ilog32(cq->cqe) - 1);
+	cq->last_armed_head = cq->head - 1;
+	cq->ready_wcs = 0;
+
+	cmd_drv = &cmd.drv_payload;
+	cmd_drv->buf_addr = (uintptr_t)cq->buf;
+	cmd_drv->flags = flags;
+	resp.cqid = UINT32_MAX;
+
+	ret = ibv_cmd_create_cq(context, cq->cqe, channel, comp_vector,
+				&cq->ibcq, &cmd.ibv_cmd, sizeof(cmd),
+				&resp.ibv_resp, sizeof(resp));
+
+	if (ret) {
+		verbs_err(verbs_get_ctx(context), "Failed to Create CQ\n");
+		errno = ret;
+		goto free_mem;
+	}
+
+	if (flags & MANA_IB_CREATE_RNIC_CQ) {
+		cq->cqid = resp.cqid;
+		if (cq->cqid == UINT32_MAX) {
+			errno = ENODEV;
+			goto destroy_cq;
+		}
+	}
+
+	return &cq->ibcq;
+
+destroy_cq:
+	ibv_cmd_destroy_cq(&cq->ibcq);
+free_mem:
+	if (cq->buf_external)
+		ctx->extern_alloc.free(cq->buf, ctx->extern_alloc.data);
+	else
+		munmap(cq->buf, cq_size);
+free_cq:
+	free(cq);
+	return NULL;
+}
+
+int mana_destroy_cq(struct ibv_cq *ibcq)
+{
+	struct mana_cq *cq = container_of(ibcq, struct mana_cq, ibcq);
+	struct mana_context *ctx = to_mctx(ibcq->context);
+	int ret;
+
+	pthread_spin_lock(&cq->lock);
+	ret = ibv_cmd_destroy_cq(ibcq);
+	if (ret) {
+		verbs_err(verbs_get_ctx(ibcq->context),
+			  "Failed to Destroy CQ\n");
+		pthread_spin_unlock(&cq->lock);
+		return ret;
+	}
+	pthread_spin_destroy(&cq->lock);
+
+	if (cq->buf_external)
+		ctx->extern_alloc.free(cq->buf, ctx->extern_alloc.data);
+	else
+		munmap(cq->buf, cq->cqe * COMP_ENTRY_SIZE);
+
+	free(cq);
+
+	return ret;
+}
+
+int mana_arm_cq(struct ibv_cq *ibcq, int solicited)
+{
+	struct mana_cq *cq = container_of(ibcq, struct mana_cq, ibcq);
+
+	if (solicited)
+		return -EOPNOTSUPP;
+	if (cq->cqid == UINT32_MAX)
+		return -EINVAL;
+
+	gdma_ring_cq_doorbell(cq);
+	return 0;
+}
+
+static inline uint32_t handle_rc_requester_cqe(struct mana_qp *qp, struct gdma_cqe *cqe)
+{
+	struct mana_gdma_queue *recv_queue = &qp->rc_qp.queues[USER_RC_RECV_QUEUE_REQUESTER];
+	struct mana_gdma_queue *send_queue = &qp->rc_qp.queues[USER_RC_SEND_QUEUE_REQUESTER];
+	uint32_t syndrome = cqe->rdma_cqe.rc_armed_completion.syndrome;
+	uint32_t psn = cqe->rdma_cqe.rc_armed_completion.psn;
+	struct rc_sq_shadow_wqe *shadow_wqe;
+	uint32_t wcs = 0;
+
+	if (!IB_IS_ACK(syndrome))
+		return 0;
+
+	if (!PSN_GT(psn, qp->rc_qp.sq_highest_completed_psn))
+		return 0;
+
+	qp->rc_qp.sq_highest_completed_psn = psn;
+
+	if (!PSN_LT(psn, qp->rc_qp.sq_psn))
+		return 0;
+
+	while ((shadow_wqe = (struct rc_sq_shadow_wqe *)
+		shadow_queue_get_next_to_complete(&qp->shadow_sq)) != NULL) {
+		if (PSN_LT(psn, shadow_wqe->end_psn))
+			break;
+
+		send_queue->cons_idx += shadow_wqe->header.posted_wqe_size_in_bu;
+		send_queue->cons_idx &= GDMA_QUEUE_OFFSET_MASK;
+
+		recv_queue->cons_idx += shadow_wqe->read_posted_wqe_size_in_bu;
+		recv_queue->cons_idx &= GDMA_QUEUE_OFFSET_MASK;
+
+		uint32_t offset = shadow_wqe->header.unmasked_queue_offset +
+				  shadow_wqe->header.posted_wqe_size_in_bu;
+		mana_ib_update_shared_mem_left_offset(qp, offset & GDMA_QUEUE_OFFSET_MASK);
+
+		shadow_queue_advance_next_to_complete(&qp->shadow_sq);
+		if (shadow_wqe->header.flags != MANA_NO_SIGNAL_WC)
+			wcs++;
+	}
+
+	uint32_t prev_psn = PSN_DEC(qp->rc_qp.sq_psn);
+
+	if (qp->rc_qp.sq_highest_completed_psn == prev_psn)
+		gdma_arm_normal_cqe(recv_queue, qp->rc_qp.sq_psn);
+	else
+		gdma_arm_normal_cqe(recv_queue, prev_psn);
+
+	return wcs;
+}
+
+static inline uint32_t handle_rc_responder_cqe(struct mana_qp *qp, struct gdma_cqe *cqe)
+{
+	struct mana_gdma_queue *recv_queue = &qp->rc_qp.queues[USER_RC_RECV_QUEUE_RESPONDER];
+	struct rc_rq_shadow_wqe *shadow_wqe;
+
+	shadow_wqe = (struct rc_rq_shadow_wqe *)shadow_queue_get_next_to_complete(&qp->shadow_rq);
+	if (!shadow_wqe)
+		return 0;
+
+	uint32_t offset_cqe = cqe->rdma_cqe.rc_recv.rx_wqe_offset / GDMA_WQE_ALIGNMENT_UNIT_SIZE;
+	uint32_t offset_wqe = shadow_wqe->header.unmasked_queue_offset & GDMA_QUEUE_OFFSET_MASK;
+
+	if (offset_cqe != offset_wqe)
+		return 0;
+
+	shadow_wqe->byte_len = cqe->rdma_cqe.rc_recv.msg_len;
+	shadow_wqe->imm_or_rkey = cqe->rdma_cqe.rc_recv.imm_data;
+
+	switch (cqe->rdma_cqe.cqe_type) {
+	case CQE_TYPE_RC_WRITE_IMM:
+		shadow_wqe->header.opcode = IBV_WC_RECV_RDMA_WITH_IMM;
+		SWITCH_FALLTHROUGH;
+	case CQE_TYPE_RC_SEND_IMM:
+		shadow_wqe->header.flags |= IBV_WC_WITH_IMM;
+		break;
+	case CQE_TYPE_RC_SEND_INV:
+		shadow_wqe->header.flags |= IBV_WC_WITH_INV;
+		break;
+	default:
+		break;
+	}
+
+	recv_queue->cons_idx += shadow_wqe->header.posted_wqe_size_in_bu;
+	recv_queue->cons_idx &= GDMA_QUEUE_OFFSET_MASK;
+
+	shadow_queue_advance_next_to_complete(&qp->shadow_rq);
+	return 1;
+}
+
+static inline uint32_t mana_handle_cqe(struct mana_context *ctx, struct gdma_cqe *cqe)
+{
+	struct mana_qp *qp;
+
+	if (cqe->is_sq) // impossible for rc
+		return 0;
+
+	qp = mana_get_qp_from_rq(ctx, cqe->wqid);
+	if (!qp)
+		return 0;
+
+	if (cqe->rdma_cqe.cqe_type == CQE_TYPE_ARMED_CMPL)
+		return handle_rc_requester_cqe(qp, cqe);
+	else
+		return handle_rc_responder_cqe(qp, cqe);
+}
+
+static inline int gdma_read_cqe(struct mana_cq *cq, struct gdma_cqe *cqe)
+{
+	uint32_t new_entry_owner_bits;
+	uint32_t old_entry_owner_bits;
+	struct gdma_cqe *current_cqe;
+	uint32_t owner_bits;
+
+	current_cqe = ((struct gdma_cqe *)cq->buf) + (cq->head % cq->cqe);
+	new_entry_owner_bits = (cq->head / cq->cqe) & CQ_OWNER_MASK;
+	old_entry_owner_bits = (cq->head / cq->cqe - 1) & CQ_OWNER_MASK;
+	owner_bits = current_cqe->owner_bits;
+
+	if (owner_bits == old_entry_owner_bits)
+		return 0; /* no new entry */
+	if (owner_bits != new_entry_owner_bits)
+		return -1; /*overflow detected*/
+
+	udma_from_device_barrier();
+	*cqe = *current_cqe;
+	cq->head++;
+	return 1;
+}
+
+static void fill_verbs_from_shadow_wqe(struct mana_qp *qp, struct ibv_wc *wc,
+				       const struct shadow_wqe_header *shadow_wqe)
+{
+	const struct rc_rq_shadow_wqe *rc_wqe = (const struct rc_rq_shadow_wqe *)shadow_wqe;
+
+	wc->wr_id = shadow_wqe->wr_id;
+	wc->status = IBV_WC_SUCCESS;
+	wc->opcode = shadow_wqe->opcode;
+	wc->vendor_err = 0;
+	wc->wc_flags = shadow_wqe->flags;
+	wc->qp_num = qp->ibqp.qp.qp_num;
+	wc->pkey_index = 0;
+
+	if (shadow_wqe->opcode & IBV_WC_RECV) {
+		wc->byte_len = rc_wqe->byte_len;
+		wc->imm_data = htobe32(rc_wqe->imm_or_rkey);
+	}
+}
+
+static int mana_process_completions(struct mana_cq *cq, int nwc, struct ibv_wc *wc)
+{
+	struct shadow_wqe_header *shadow_wqe;
+	struct mana_qp *qp;
+	int wc_index = 0;
+
+	/* process send shadow queue completions  */
+	list_for_each(&cq->send_qp_list, qp, send_cq_node) {
+		while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq))
+				!= NULL) {
+			if (wc_index >= nwc && shadow_wqe->flags != MANA_NO_SIGNAL_WC)
+				goto out;
+
+			if (shadow_wqe->flags != MANA_NO_SIGNAL_WC) {
+				fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
+				wc_index++;
+			}
+			shadow_queue_advance_consumer(&qp->shadow_sq);
+		}
+	}
+
+	/* process recv shadow queue completions */
+	list_for_each(&cq->recv_qp_list, qp, recv_cq_node) {
+		while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq))
+				!= NULL) {
+			if (wc_index >= nwc)
+				goto out;
+
+			fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
+			wc_index++;
+			shadow_queue_advance_consumer(&qp->shadow_rq);
+		}
+	}
+
+out:
+	return wc_index;
+}
+
+int mana_poll_cq(struct ibv_cq *ibcq, int nwc, struct ibv_wc *wc)
+{
+	struct mana_cq *cq = container_of(ibcq, struct mana_cq, ibcq);
+	struct mana_context *ctx = to_mctx(ibcq->context);
+	struct gdma_cqe gdma_cqe;
+	int num_polled = 0;
+	int ret;
+
+	pthread_spin_lock(&cq->lock);
+
+	while (cq->ready_wcs < nwc) {
+		ret = gdma_read_cqe(cq, &gdma_cqe);
+		if (ret < 0) {
+			num_polled = -1;
+			goto out;
+		}
+		if (ret == 0)
+			break;
+		cq->ready_wcs += mana_handle_cqe(ctx, &gdma_cqe);
+	}
+
+	num_polled = mana_process_completions(cq, nwc, wc);
+	cq->ready_wcs -= num_polled;
+out:
+	pthread_spin_unlock(&cq->lock);
+
+	return num_polled;
+}