Merge pull request #856 from yangx-jy/rpma_write_with_imm

Implement rpma write with immediate data

doc/manuals_3.txt

@@ -56,3 +56,4 @@ rpma_utils_get_ibv_context.3
 rpma_utils_ibv_context_is_odp_capable.3
 rpma_write.3
 rpma_write_atomic.3
+rpma_write_with_imm.3

examples/11-write-with-imm/CMakeLists.txt

@@ -0,0 +1,44 @@
+#
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 Fujitsu
+#
+
+cmake_minimum_required(VERSION 3.3)
+project(write-with-imm C)
+
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
+	${CMAKE_SOURCE_DIR}/../cmake
+	${CMAKE_SOURCE_DIR}/../../cmake)
+
+include(${CMAKE_SOURCE_DIR}/../../cmake/functions.cmake)
+# set LIBRT_LIBRARIES if linking with librt is required
+check_if_librt_is_required()
+
+find_package(PkgConfig QUIET)
+
+if(PKG_CONFIG_FOUND)
+	pkg_check_modules(LIBRPMA librpma)
+	pkg_check_modules(LIBIBVERBS libibverbs)
+endif()
+if(NOT LIBRPMA_FOUND)
+	find_package(LIBRPMA REQUIRED librpma)
+endif()
+if(NOT LIBIBVERBS_FOUND)
+	find_package(LIBIBVERBS REQUIRED libibverbs)
+endif()
+
+link_directories(${LIBRPMA_LIBRARY_DIRS} ${LIBIBVERBS_LIBRARY_DIRS})
+
+function(add_example name)
+	set(srcs ${ARGN})
+	add_executable(${name} ${srcs})
+	target_include_directories(${name}
+		PRIVATE
+			${LIBRPMA_INCLUDE_DIRS}
+			${LIBIBVERBS_INCLUDE_DIRS}
+			../common)
+	target_link_libraries(${name} rpma ${LIBIBVERBS_LIBRARIES} ${LIBRT_LIBRARIES})
+endfunction()
+
+add_example(server server.c ../common/common-conn.c)
+add_example(client client.c ../common/common-conn.c)

examples/11-write-with-imm/README.md

@@ -0,0 +1,19 @@
+Example of rpma write with immediate data
+===
+
+The rpma write with immediate data implements two parts of the process:
+- The client connects to the server and writes a message with immediate
+data to the server.
+- The server receives an immediate data from the client.
+The immediate data has the same value as the message written by the
+client so the immediate data is compared with the expected message.
+
+## Usage
+
+```bash
+[user@server]$ ./server $server_address $port
+```
+
+```bash
+[user@client]$ ./client $server_address $port $imm
+```

examples/11-write-with-imm/client.c

@@ -0,0 +1,154 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/* Copyright (c) 2021 Fujitsu */
+
+/*
+ * client.c -- a client of the write-with-imm example
+ *
+ * Please see README.md for a detailed description of this example.
+ */
+
+#include <librpma.h>
+#include <limits.h>
+#include <inttypes.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "common-conn.h"
+
+#define USAGE_STR "usage: %s <server_address> <port> <imm>\n"
+
+int
+main(int argc, char *argv[])
+{
+	/* validate parameters */
+	if (argc < 4) {
+		fprintf(stderr, USAGE_STR, argv[0]);
+		return -1;
+	}
+
+	/* configure logging thresholds to see more details */
+	rpma_log_set_threshold(RPMA_LOG_THRESHOLD, RPMA_LOG_LEVEL_INFO);
+	rpma_log_set_threshold(RPMA_LOG_THRESHOLD_AUX, RPMA_LOG_LEVEL_INFO);
+
+	/* read common parameters */
+	char *addr = argv[1];
+	char *port = argv[2];
+
+	uint64_t imm = strtoul(argv[3], NULL, 10);
+	if (imm == ULONG_MAX && errno == ERANGE) {
+		fprintf(stderr, "strtoul() overflowed\n");
+		return -1;
+	}
+
+	if (imm > UINT32_MAX) {
+		fprintf(stderr,
+			"the provided immediate data is too big(%lu > %u)\n",
+			imm, UINT32_MAX);
+		return -1;
+	}
+
+	/* RPMA resources - general */
+	struct rpma_peer *peer = NULL;
+	struct rpma_mr_local *src_mr = NULL;
+	struct rpma_mr_remote *dst_mr = NULL;
+	struct rpma_conn *conn = NULL;
+	struct rpma_completion cmpl;
+	int ret;
+
+	/* prepare memory */
+	uint32_t *src = malloc_aligned(KILOBYTE);
+	if (!src)
+		return -1;
+
+	/*
+	 * lookup an ibv_context via the address and create a new peer using it
+	 */
+	ret = client_peer_via_address(addr, &peer);
+	if (ret)
+		goto err_mr_free;
+
+	/* register the memory */
+	ret = rpma_mr_reg(peer, src, KILOBYTE, RPMA_MR_USAGE_WRITE_SRC,
+			&src_mr);
+	if (ret)
+		goto err_peer_delete;
+
+	/* establish a new connection to a server */
+	ret = client_connect(peer, addr, port, NULL, &conn);
+	if (ret)
+		goto err_mr_dereg;
+
+	/* obtain the remote memory description */
+	struct rpma_conn_private_data pdata;
+	ret = rpma_conn_get_private_data(conn, &pdata);
+	if (ret)
+		goto err_conn_disconnect;
+	if (pdata.len < sizeof(struct common_data)) {
+		fprintf(stderr,
+			"received connection's private data is too small (%u < %zu)\n",
+			pdata.len, sizeof(struct common_data));
+		ret = -1;
+		goto err_conn_disconnect;
+	}
+
+	struct common_data *dst_data = pdata.ptr;
+	ret = rpma_mr_remote_from_descriptor(&dst_data->descriptors[0],
+			dst_data->mr_desc_size, &dst_mr);
+	if (ret)
+		goto err_conn_disconnect;
+
+	/* write a message with immediate data to the server */
+	memcpy(src, (uint32_t *)&imm, sizeof(uint32_t));
+	fprintf(stdout, "write a value %u with immediate data %u\n",
+		*src, (uint32_t)imm);
+	ret = rpma_write_with_imm(conn, dst_mr, dst_data->data_offset, src_mr,
+			0, KILOBYTE, RPMA_F_COMPLETION_ALWAYS, (uint32_t)imm,
+			NULL);
+	if (ret)
+		goto err_mr_remote_delete;
+
+	/* prepare completions, get one and validate it */
+	ret = rpma_conn_completion_wait(conn);
+	if (ret)
+		goto err_mr_remote_delete;
+
+	ret = rpma_conn_completion_get(conn, &cmpl);
+	if (ret)
+		goto err_mr_remote_delete;
+
+	if (cmpl.op_status != IBV_WC_SUCCESS) {
+		fprintf(stderr,
+			"an unexpected completion: %s\n",
+			ibv_wc_status_str(cmpl.op_status));
+		ret = -1;
+		goto err_mr_remote_delete;
+	}
+
+	if (cmpl.op != RPMA_OP_WRITE) {
+		fprintf(stderr,
+			"an unexpected type of operation (%d != %d)\n",
+			cmpl.op, RPMA_OP_WRITE);
+		ret = -1;
+	}
+
+err_mr_remote_delete:
+	/* delete the remote memory region's structure */
+	(void) rpma_mr_remote_delete(&dst_mr);
+
+err_conn_disconnect:
+	(void) common_disconnect_and_wait_for_conn_close(&conn);
+
+err_mr_dereg:
+	/* deregister the memory regions */
+	(void) rpma_mr_dereg(&src_mr);
+
+err_peer_delete:
+	/* delete the peer object */
+	(void) rpma_peer_delete(&peer);
+
+err_mr_free:
+	/* free the memory */
+	free(src);
+
+	return ret;
+}