From d47badd85bb4287a151c7a2082f983578e543835 Mon Sep 17 00:00:00 2001
From: Konstantin Taranov <kotaranov@microsoft.com>
Date: Tue, 14 May 2024 17:10:51 +0200
Subject: [PATCH] providers/mana: Add shadow queues

Implement shadow queues to store pending WQEs.

Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
---
 providers/mana/mana.h         |   1 +
 providers/mana/shadow_queue.h | 145 ++++++++++++++++++++++++++++++++++
 2 files changed, 146 insertions(+)
 create mode 100644 providers/mana/shadow_queue.h

diff --git a/providers/mana/mana.h b/providers/mana/mana.h
index 0d64d92ad..783afb312 100644
--- a/providers/mana/mana.h
+++ b/providers/mana/mana.h
@@ -7,6 +7,7 @@
 #define _MANA_H_
 
 #include "manadv.h"
+#include "shadow_queue.h"
 
 #define COMP_ENTRY_SIZE 64
 #define MANA_IB_TOEPLITZ_HASH_KEY_SIZE_IN_BYTES 40
diff --git a/providers/mana/shadow_queue.h b/providers/mana/shadow_queue.h
new file mode 100644
index 000000000..dfd9edcc2
--- /dev/null
+++ b/providers/mana/shadow_queue.h
@@ -0,0 +1,145 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/*
+ * Copyright (c) 2024, Microsoft Corporation. All rights reserved.
+ */
+
+#ifndef _SHADOW_QUEUE_H_
+#define _SHADOW_QUEUE_H_
+
+#include <stdio.h>
+#include <linux/types.h>
+#include <endian.h>
+#include <infiniband/verbs.h>
+#include <sys/mman.h>
+#include <util/util.h>
+
+struct shadow_wqe_header {
+	uint8_t opcode; // see enum ibv_wc_opcode
+	uint8_t flags; // see enum ibv_wc_flags
+	uint16_t posted_wqe_size_in_bu;
+	uint32_t unmasked_queue_offset;
+	uint64_t wr_id;
+};
+
+struct rc_sq_shadow_wqe {
+	struct  shadow_wqe_header header;
+	uint32_t end_psn;
+	uint32_t read_posted_wqe_size_in_bu;
+};
+
+struct rc_rq_shadow_wqe {
+	struct shadow_wqe_header header;
+	uint32_t byte_len;
+	uint32_t imm_or_rkey;
+};
+
+struct shadow_queue {
+	uint64_t prod_idx;
+	uint64_t cons_idx;
+	uint64_t next_to_complete_idx;
+	uint32_t length;
+	uint32_t stride;
+	void *buffer;
+};
+
+static inline void reset_shadow_queue(struct shadow_queue *queue)
+{
+	queue->prod_idx = 0;
+	queue->cons_idx = 0;
+	queue->next_to_complete_idx = 0;
+}
+
+static inline int create_shadow_queue(struct shadow_queue *queue, uint32_t length, uint32_t stride)
+{
+	length = roundup_pow_of_two(length);
+	stride = align(stride, 8);
+
+	void *buffer = mmap(NULL, stride * length, PROT_READ | PROT_WRITE,
+			MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+
+	if (buffer == MAP_FAILED)
+		return -1;
+
+	queue->length = length;
+	queue->stride = stride;
+	reset_shadow_queue(queue);
+	queue->buffer = buffer;
+	return 0;
+}
+
+static inline void destroy_shadow_queue(struct shadow_queue *queue)
+{
+	if (queue->buffer) {
+		munmap(queue->buffer, queue->stride * queue->length);
+		queue->buffer = NULL;
+	}
+}
+
+static inline struct shadow_wqe_header *
+shadow_queue_get_element(const struct shadow_queue *queue, uint64_t unmasked_index)
+{
+	uint32_t index = unmasked_index & (queue->length - 1);
+
+	return (struct shadow_wqe_header *)((uint8_t *)queue->buffer + index * queue->stride);
+}
+
+static inline bool shadow_queue_full(struct shadow_queue *queue)
+{
+	return (queue->prod_idx - queue->cons_idx) >= queue->length;
+}
+
+static inline struct shadow_wqe_header *
+shadow_queue_producer_entry(struct shadow_queue *queue)
+{
+	return shadow_queue_get_element(queue, queue->prod_idx);
+}
+
+static inline void shadow_queue_advance_producer(struct shadow_queue *queue)
+{
+	queue->prod_idx++;
+}
+
+static inline void shadow_queue_retreat_producer(struct shadow_queue *queue)
+{
+	queue->prod_idx--;
+}
+
+static inline void shadow_queue_advance_consumer(struct shadow_queue *queue)
+{
+	queue->cons_idx++;
+}
+
+static inline bool shadow_queue_empty(struct shadow_queue *queue)
+{
+	return queue->prod_idx == queue->cons_idx;
+}
+
+static inline uint32_t shadow_queue_get_pending_wqe_count(struct shadow_queue *queue)
+{
+	return (uint32_t)(queue->prod_idx - queue->next_to_complete_idx);
+}
+
+static inline struct shadow_wqe_header *
+shadow_queue_get_next_to_consume(const struct shadow_queue *queue)
+{
+	if (queue->cons_idx == queue->next_to_complete_idx)
+		return NULL;
+
+	return shadow_queue_get_element(queue, queue->cons_idx);
+}
+
+static inline struct shadow_wqe_header *
+shadow_queue_get_next_to_complete(struct shadow_queue *queue)
+{
+	if (queue->next_to_complete_idx == queue->prod_idx)
+		return NULL;
+
+	return shadow_queue_get_element(queue, queue->next_to_complete_idx);
+}
+
+static inline void shadow_queue_advance_next_to_complete(struct shadow_queue *queue)
+{
+	queue->next_to_complete_idx++;
+}
+
+#endif //_SHADOW_QUEUE_H_