examples: make example-09 use seperate receive CQ

Signed-off-by: Xiao Yang <yangx.jy@fujitsu.com>

examples/09-flush-to-persistent-GPSPM/client.c

@@ -185,10 +185,17 @@ main(int argc, char *argv[])
 	if ((ret = client_peer_via_address(addr, &peer)))
 		goto err_free;
 
-	/* establish a new connection to a server listening at addr:port */
-	if ((ret = client_connect(peer, addr, port, NULL, NULL, &conn)))
+	struct rpma_conn_cfg *cfg = NULL;
+	if ((ret = rpma_conn_cfg_new(&cfg)))
 		goto err_peer_delete;
 
+	if ((ret = rpma_conn_cfg_set_rcq_size(cfg, RCQ_SIZE)))
+		goto err_cfg_delete;
+
+	/* establish a new connection to a server listening at addr:port */
+	if ((ret = client_connect(peer, addr, port, cfg, NULL, &conn)))
+		goto err_cfg_delete;
+
 	/* register the memory RDMA write */
 	if ((ret = rpma_mr_reg(peer, mr_ptr, mr_size,
 			RPMA_MR_USAGE_WRITE_SRC, &src_mr)))
@@ -254,16 +261,46 @@ main(int argc, char *argv[])
 
 	/* send the flush message */
 	if ((ret = rpma_send(conn, msg_mr, SEND_OFFSET, flush_req_size,
-			RPMA_F_COMPLETION_ON_ERROR, NULL)))
+			RPMA_F_COMPLETION_ALWAYS, NULL)))
 		goto err_mr_remote_delete;
 
-	/* wait for the completion to be ready */
-	if ((ret = rpma_conn_completion_wait(conn)))
+	/* wait for the send completion to be ready */
+	struct rpma_cq *cq = NULL;
+	if ((ret = rpma_conn_get_cq(conn, &cq)))
+		goto err_mr_remote_delete;
+	if ((ret = rpma_cq_wait(cq)))
 		goto err_mr_remote_delete;
-	if ((ret = rpma_conn_completion_get(conn, &cmpl)))
+	if ((ret = rpma_cq_get_completion(cq, &cmpl)))
 		goto err_mr_remote_delete;
 
-	/* validate the completion */
+	/* validate the send completion */
+	if (cmpl.op_status != IBV_WC_SUCCESS) {
+		ret = -1;
+		(void) fprintf(stderr, "rpma_send() failed: %s\n",
+				ibv_wc_status_str(cmpl.op_status));
+		goto err_mr_remote_delete;
+	}
+
+	if (cmpl.op != RPMA_OP_SEND) {
+		ret = -1;
+		(void) fprintf(stderr,
+				"unexpected cmpl.op value "
+				"(0x%" PRIXPTR " != 0x%" PRIXPTR ")\n",
+				(uintptr_t)cmpl.op,
+				(uintptr_t)RPMA_OP_SEND);
+		goto err_mr_remote_delete;
+	}
+
+	/* wait for the receive completion to be ready */
+	struct rpma_cq *rcq = NULL;
+	if ((ret = rpma_conn_get_rcq(conn, &rcq)))
+		goto err_mr_remote_delete;
+	if ((ret = rpma_cq_wait(rcq)))
+		goto err_mr_remote_delete;
+	if ((ret = rpma_cq_get_completion(rcq, &cmpl)))
+		goto err_mr_remote_delete;
+
+	/* validate the receive completion */
 	if (cmpl.op_status != IBV_WC_SUCCESS) {
 		ret = -1;
 		(void) fprintf(stderr, "rpma_recv() failed: %s\n",
@@ -322,6 +359,9 @@ main(int argc, char *argv[])
 err_conn_disconnect:
 	(void) common_disconnect_and_wait_for_conn_close(&conn);
 
+err_cfg_delete:
+	(void) rpma_conn_cfg_delete(&cfg);
+
 err_peer_delete:
 	/* delete the peer */
 	(void) rpma_peer_delete(&peer);

examples/09-flush-to-persistent-GPSPM/flush-to-persistent-GPSPM.h

@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause */
 /* Copyright 2020, Intel Corporation */
+/* Copyright 2021, Fujitsu */
 
 /*
  * flush-to-persistent-GSPSPM.h -- a common declarations for the 09 example
@@ -13,4 +14,6 @@
 #define SEND_OFFSET	0
 #define RECV_OFFSET	(SEND_OFFSET + MSG_SIZE_MAX)
 
+#define RCQ_SIZE	1
+
 #endif /* EXAMPLES_FLUSH_2_PMEM_GPSPM */

examples/09-flush-to-persistent-GPSPM/server.c

@@ -198,6 +198,13 @@ main(int argc, char *argv[])
 	if ((ret = rpma_mr_get_descriptor(mr, &data.descriptors[0])))
 		goto err_mr_dereg;
 
+	struct rpma_conn_cfg *cfg = NULL;
+	if ((ret = rpma_conn_cfg_new(&cfg)))
+		goto err_mr_dereg;
+
+	if ((ret = rpma_conn_cfg_set_rcq_size(cfg, RCQ_SIZE)))
+		goto err_cfg_delete;
+
 	/*
 	 * Wait for an incoming connection request, accept it and wait for its
 	 * establishment.
@@ -207,7 +214,7 @@ main(int argc, char *argv[])
 	pdata.len = sizeof(struct common_data);
 
 	/* receive an incoming connection request */
-	if ((ret = rpma_ep_next_conn_req(ep, NULL, &req)))
+	if ((ret = rpma_ep_next_conn_req(ep, cfg, &req)))
 		goto err_mr_dereg;
 
 	/* prepare buffer for a flush request */
@@ -232,19 +239,33 @@ main(int argc, char *argv[])
 	if (ret)
 		goto err_conn_delete;
 
-	/* wait for the completion to be ready */
-	if ((ret = rpma_conn_completion_wait(conn)))
+	/* wait for the receive completion to be ready */
+	struct rpma_cq *rcq = NULL;
+	if ((ret = rpma_conn_get_rcq(conn, &rcq)))
+		goto err_conn_delete;
+	if ((ret = rpma_cq_wait(rcq)))
 		goto err_conn_delete;
-	if ((ret = rpma_conn_completion_get(conn, &cmpl)))
+	if ((ret = rpma_cq_get_completion(rcq, &cmpl)))
 		goto err_conn_delete;
 
+	/* validate the receive completion */
 	if (cmpl.op_status != IBV_WC_SUCCESS) {
 		ret = -1;
 		(void) fprintf(stderr, "rpma_recv() failed: %s\n",
 				ibv_wc_status_str(cmpl.op_status));
 		goto err_conn_delete;
 	}
 
+	if (cmpl.op != RPMA_OP_RECV) {
+		ret = -1;
+		(void) fprintf(stderr,
+				"unexpected cmpl.op value "
+				"(0x%" PRIXPTR " != 0x%" PRIXPTR ")\n",
+				(uintptr_t)cmpl.op,
+				(uintptr_t)RPMA_OP_RECV);
+		goto err_conn_delete;
+	}
+
 	/* unpack a flush request from the received buffer */
 	flush_req = gpspm_flush_request__unpack(NULL, cmpl.byte_len, recv_ptr);
 	if (flush_req == NULL) {
@@ -282,13 +303,16 @@ main(int argc, char *argv[])
 			RPMA_F_COMPLETION_ALWAYS, NULL)))
 		goto err_conn_delete;
 
-	/* wait for the completion to be ready */
-	if ((ret = rpma_conn_completion_wait(conn)))
+	/* wait for the send completion to be ready */
+	struct rpma_cq *cq = NULL;
+	if ((ret = rpma_conn_get_cq(conn, &cq)))
+		goto err_conn_delete;
+	if ((ret = rpma_cq_wait(cq)))
 		goto err_conn_delete;
-	if ((ret = rpma_conn_completion_get(conn, &cmpl)))
+	if ((ret = rpma_cq_get_completion(cq, &cmpl)))
 		goto err_conn_delete;
 
-	/* validate the completion */
+	/* validate the send completion */
 	if (cmpl.op_status != IBV_WC_SUCCESS) {
 		ret = -1;
 		(void) fprintf(stderr, "rpma_send() failed: %s\n",
@@ -322,6 +346,9 @@ main(int argc, char *argv[])
 err_req_delete:
 	(void) rpma_conn_req_delete(&req);
 
+err_cfg_delete:
+	(void) rpma_conn_cfg_delete(&cfg);
+
 err_mr_dereg:
 	(void) rpma_mr_dereg(&msg_mr);
 	(void) rpma_mr_dereg(&mr);