From 8adce02da00062ad954f715a0d6cfb64200928dc Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 9 Sep 2022 15:10:16 +0200 Subject: [PATCH] example R: add the librpma example In this example we want to introduce to you a basic way of accessing Persistent Memory when it is installed in a remote system. Remote Persistent Memory (or RPMem for short) is a way of doing this via a network by making use of the Remote Direct Memory Access (RDMA) technology. This example is an introduction to RPMem. We will guide you briefly through setting up all required hardware and software components and verifying whether the connection works properly. Having that, we will show you how to access Persistent Memory on a remote system using the librpma library. After completing this example you will know: - what is RPMem and what it is good for, - how RPMem is different comparing to PMem, - what hardware and software components are required to start using RPMem, - how to verify whether the RDMA network works properly, - how to use librpma API to: - establish a connection, - prepare memory for remote manipulation, - manipulate memory on the remote system, - assure persistency of stores to the remote system. Step-by-step you will: - test the connection on the basic level using ping, - test the connection RDMA capabilities using rping, - review an application focusing on establishing a connection, - review an application reading remote system's memory and writing it back in the persistent manner. Co-authored-by: Oksana Salyk Co-authored-by: Jan M Michalski Signed-off-by: Lukasz Dorau --- examples/R/.gitignore | 1 + examples/R/CMakeLists.txt | 50 +++++ examples/R/README.md | 32 +++ examples/R/build_main.sh | 10 + examples/R/build_simple.sh | 10 + examples/R/client.c | 275 ++++++++++++++++++++++++++ examples/R/cmake/FindLIBIBVERBS.cmake | 20 ++ examples/R/cmake/FindLIBPMEM.cmake | 20 ++ examples/R/cmake/FindLIBRPMA.cmake | 20 ++ examples/R/common.c | 167 ++++++++++++++++ examples/R/common.h | 53 +++++ examples/R/run_main.sh | 21 ++ examples/R/run_simple.sh | 14 ++ examples/R/run_test_ping.sh | 8 + examples/R/run_test_rping.sh | 10 + examples/R/server.c | 203 +++++++++++++++++++ examples/R/simple_client.c | 78 ++++++++ examples/R/simple_server.c | 91 +++++++++ 18 files changed, 1083 insertions(+) create mode 100644 examples/R/.gitignore create mode 100644 examples/R/CMakeLists.txt create mode 100644 examples/R/README.md create mode 100755 examples/R/build_main.sh create mode 100755 examples/R/build_simple.sh create mode 100644 examples/R/client.c create mode 100644 examples/R/cmake/FindLIBIBVERBS.cmake create mode 100644 examples/R/cmake/FindLIBPMEM.cmake create mode 100644 examples/R/cmake/FindLIBRPMA.cmake create mode 100644 examples/R/common.c create mode 100644 examples/R/common.h create mode 100755 examples/R/run_main.sh create mode 100755 examples/R/run_simple.sh create mode 100755 examples/R/run_test_ping.sh create mode 100755 examples/R/run_test_rping.sh create mode 100644 examples/R/server.c create mode 100644 examples/R/simple_client.c create mode 100644 examples/R/simple_server.c diff --git a/examples/R/.gitignore b/examples/R/.gitignore new file mode 100644 index 0000000..567609b --- /dev/null +++ b/examples/R/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/examples/R/CMakeLists.txt b/examples/R/CMakeLists.txt new file mode 100644 index 0000000..67838cc --- /dev/null +++ b/examples/R/CMakeLists.txt @@ -0,0 +1,50 @@ +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2020-2021, Intel Corporation +# Copyright 2021, Fujitsu +# + +cmake_minimum_required(VERSION 3.3) +project(exampleR C) + +set(LIBPMEM_REQUIRED_VERSION 1.6) + +# append -Wall and -Werror to CMAKE_C_FLAGS +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") + +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/cmake) + +find_package(PkgConfig QUIET) + +if(PKG_CONFIG_FOUND) + pkg_check_modules(LIBRPMA librpma) + pkg_check_modules(LIBIBVERBS libibverbs) + pkg_check_modules(LIBPMEM libpmem>=${LIBPMEM_REQUIRED_VERSION}) +endif() +if(NOT LIBRPMA_FOUND) + find_package(LIBRPMA REQUIRED librpma) +endif() +if(NOT LIBIBVERBS_FOUND) + find_package(LIBIBVERBS REQUIRED libibverbs) +endif() +if(NOT LIBPMEM_FOUND) + find_package(LIBPMEM ${LIBPMEM_REQUIRED_VERSION} REQUIRED libpmem) +endif() + +link_directories(${LIBRPMA_LIBRARY_DIRS}) + +function(add_example name) + set(srcs ${ARGN}) + add_executable(${name} ${srcs}) + target_include_directories(${name} PRIVATE ${LIBRPMA_INCLUDE_DIRS} ${LIBPMEM_INCLUDE_DIRS}) + target_link_libraries(${name} rpma ${LIBIBVERBS_LIBRARIES} ${LIBRT_LIBRARIES} ${LIBPMEM_LIBRARIES}) +endfunction() + +add_example(server server.c common.c) +add_example(client client.c common.c) +add_example(simple_server simple_server.c common.c) +add_example(simple_client simple_client.c common.c) + +add_custom_target(config_softroce + COMMAND ${CMAKE_SOURCE_DIR}/tools/config_softroce.sh) diff --git a/examples/R/README.md b/examples/R/README.md new file mode 100644 index 0000000..4107076 --- /dev/null +++ b/examples/R/README.md @@ -0,0 +1,32 @@ +Example of performing an RPMA read, write and flush to persistence operations +=== + +This example implements both sides of an RDMA connection: +- The server prepares a local persistent memory and exposes the memory description +along with other parameters required to perform an RDMA read, write and flush operations. +After the connection is established, the server waits for the client to disconnect. +- The client allocates memory from DRAM and registers it as a reading destination +and writing source. After the connection is established the client receives +the server's memory regions registered as a reading source and a writing destination. +The client performs the RDMA read from the remote memory region to the local memory region, +then it writes new data to the local memory region and performs the RDMA write +from the local memory region to the remote memory region followed by the RPMA flush. + +**Note**: The server requires a unique argument in order to use +a different part of persistent memory which is shared by all the server instances +running on the same . is also used to pick a unique TCP port. + +**Note**: For the sake of this example, the memory region being written to and +the server's peer configuration are transferred via the connection's private +data. In general, it can be transferred via an out-of-band or the in-band +channel. + +## Usage + +```bash +[user@server]$ ./server $server_address $port ${pmem-path} ${user-id} +``` + +```bash +[user@client]$ ./client $server_address $port +``` diff --git a/examples/R/build_main.sh b/examples/R/build_main.sh new file mode 100755 index 0000000..48379c9 --- /dev/null +++ b/examples/R/build_main.sh @@ -0,0 +1,10 @@ +#!/bin/bash -ex +# +# The shell commands to build the main example. +# + +mkdir -p build +cd build +cmake .. +make server +make client diff --git a/examples/R/build_simple.sh b/examples/R/build_simple.sh new file mode 100755 index 0000000..e9f9173 --- /dev/null +++ b/examples/R/build_simple.sh @@ -0,0 +1,10 @@ +#!/bin/bash -ex +# +# The shell commands to build the simple_* example. +# + +mkdir -p build +cd build +cmake .. +make simple_client +make simple_server diff --git a/examples/R/client.c b/examples/R/client.c new file mode 100644 index 0000000..8ef60f2 --- /dev/null +++ b/examples/R/client.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright 2020-2021, Intel Corporation */ +/* Copyright 2021, Fujitsu */ + +/* + * client.c -- a client of the "read, write and flush to persistent domain" example + */ + +#include +#include +#include +#include +#include "common.h" + +#define USAGE_STR "usage: %s \n" +#define FLUSH_ID (void *)0xF01D /* a random identifier */ + +static const char *hello_str = "Hello world!"; + +/* + * malloc_aligned -- allocate an aligned chunk of memory + */ +void * +malloc_aligned(size_t size) +{ + long pagesize = sysconf(_SC_PAGESIZE); + if (pagesize < 0) { + perror("sysconf"); + return NULL; + } + + /* allocate a page size aligned local memory pool */ + void *mem; + int ret = posix_memalign(&mem, (size_t)pagesize, size); + if (ret) { + (void) fprintf(stderr, "Client: error: posix_memalign: %s\n", + strerror(ret)); + return NULL; + } + + /* zero the allocated memory */ + memset(mem, 0, size); + + return mem; +} + +int +main(int argc, char *argv[]) +{ + /* validate parameters */ + if (argc < 3) { + fprintf(stderr, USAGE_STR, argv[0]); + exit(-1); + } + + /* configure logging thresholds to see more details */ + rpma_log_set_threshold(RPMA_LOG_THRESHOLD, RPMA_LOG_LEVEL_INFO); + rpma_log_set_threshold(RPMA_LOG_THRESHOLD_AUX, RPMA_LOG_LEVEL_INFO); + + /* read common parameters */ + char *addr = argv[1]; + char *port = argv[2]; + int ret; + + /* resources - memory region */ + void *local_mr_ptr; + size_t local_mr_size; + size_t local_offset = 0; + struct rpma_mr_remote *remote_mr = NULL; + size_t remote_size = 0; + size_t remote_offset = 0; + struct rpma_mr_local *local_mr = NULL; + struct ibv_wc wc; + + /* RPMA resources */ + struct rpma_peer_cfg *pcfg = NULL; + struct rpma_peer *peer = NULL; + struct rpma_conn *conn = NULL; + bool direct_write_to_pmem = false; + + local_mr_size = KILOBYTE; + local_mr_ptr = malloc_aligned(local_mr_size); + if (local_mr_ptr == NULL) + return -1; + + /* + * lookup an ibv_context via the address and create a new peer using it + */ + ret = common_client_peer_via_address(addr, &peer); + if (ret) + goto err_free; + + /* establish a new connection to a server listening at addr:port */ + ret = common_client_connect(peer, addr, port, NULL, NULL, &conn); + if (ret) + goto err_peer_delete; + + /* register the memory for RDMA read and write operations */ + ret = rpma_mr_reg(peer, local_mr_ptr, local_mr_size, + (RPMA_MR_USAGE_READ_DST | RPMA_MR_USAGE_WRITE_SRC), + &local_mr); + if (ret) + goto err_conn_disconnect; + + /* obtain the remote side resources description */ + struct rpma_conn_private_data pdata; + ret = rpma_conn_get_private_data(conn, &pdata); + if (ret != 0 || pdata.len < sizeof(struct common_data)) + goto err_mr_dereg; + + /* + * Create a remote peer configuration structure from the received + * descriptor and apply it to the current connection. + */ + struct common_data *remote_data = pdata.ptr; + ret = rpma_peer_cfg_from_descriptor( + &remote_data->descriptors[remote_data->mr_desc_size], + remote_data->pcfg_desc_size, &pcfg); + if (ret) + goto err_mr_dereg; + ret = rpma_peer_cfg_get_direct_write_to_pmem(pcfg, + &direct_write_to_pmem); + ret |= rpma_conn_apply_remote_peer_cfg(conn, pcfg); + (void) rpma_peer_cfg_delete(&pcfg); + /* either get or apply failed */ + if (ret) + goto err_mr_dereg; + + /* verify the Direct Write to PMem support */ + if (!direct_write_to_pmem) { + (void) fprintf(stderr, + "Client: error: the server does not support Direct Write to PMem\n"); + goto err_mr_dereg; + } + + /* + * Create a remote memory registration structure from the received + * descriptor. + */ + ret = rpma_mr_remote_from_descriptor(&remote_data->descriptors[0], + remote_data->mr_desc_size, &remote_mr); + if (ret) + goto err_mr_dereg; + + /* get the remote memory region size */ + ret = rpma_mr_remote_get_size(remote_mr, &remote_size); + if (ret) { + goto err_mr_remote_delete; + } else if (remote_size < KILOBYTE) { + fprintf(stderr, + "Client: error: remote memory region size too small for writing the data of the assumed size (%zu < %d)\n", + remote_size, KILOBYTE); + goto err_mr_remote_delete; + } + + /* read the initial value */ + size_t len = (local_mr_size < remote_size) ? local_mr_size : remote_size; + ret = rpma_read(conn, local_mr, local_offset, + remote_mr, remote_offset, len, + RPMA_F_COMPLETION_ALWAYS, NULL); + if (ret) + goto err_mr_remote_delete; + + /* get the connection's main CQ */ + struct rpma_cq *cq = NULL; + ret = rpma_conn_get_cq(conn, &cq); + if (ret) + goto err_mr_remote_delete; + + /* wait for the completion to be ready */ + ret = rpma_cq_wait(cq); + if (ret) + goto err_mr_remote_delete; + + /* wait for a completion of the RDMA read */ + ret = rpma_cq_get_wc(cq, 1, &wc, NULL); + if (ret) + goto err_mr_remote_delete; + + if (wc.status != IBV_WC_SUCCESS) { + ret = -1; + (void) fprintf(stderr, "Client: error: rpma_read() failed: %s\n", + ibv_wc_status_str(wc.status)); + goto err_mr_remote_delete; + } + + if (wc.opcode != IBV_WC_RDMA_READ) { + ret = -1; + (void) fprintf(stderr, "Client: error: unexpected wc.opcode value (%d != %d)\n", + wc.opcode, IBV_WC_RDMA_READ); + goto err_mr_remote_delete; + } + + (void) fprintf(stdout, "Client: read the initial content of the remote (server's) persistent memory: %s\n", + (char *)local_mr_ptr + local_offset); + + /* write the next value */ + strncpy(local_mr_ptr, hello_str, KILOBYTE); + (void) printf("Client: writing to the remote (server's) persistent memory: %s\n", + (char *)local_mr_ptr); + + ret = rpma_write(conn, remote_mr, remote_offset, + local_mr, local_offset, KILOBYTE, + RPMA_F_COMPLETION_ON_ERROR, NULL); + if (ret) { + (void) fprintf(stderr, "Client: error: rpma_write() failed: %s\n", + rpma_err_2str(ret)); + goto err_mr_remote_delete; + } + + (void) printf("Client: flushing the remote data to the persistent domain...\n"); + + ret = rpma_flush(conn, remote_mr, remote_offset, KILOBYTE, + RPMA_FLUSH_TYPE_PERSISTENT, + RPMA_F_COMPLETION_ALWAYS, FLUSH_ID); + if (ret) { + (void) fprintf(stderr, "Client: error: rpma_flush() failed: %s\n", + rpma_err_2str(ret)); + goto err_mr_remote_delete; + } + + /* get the connection's main CQ */ + ret = rpma_conn_get_cq(conn, &cq); + if (ret) + goto err_mr_remote_delete; + + /* wait for the completion to be ready */ + ret = rpma_cq_wait(cq); + if (ret) + goto err_mr_remote_delete; + + /* wait for a completion of the RDMA read */ + ret = rpma_cq_get_wc(cq, 1, &wc, NULL); + if (ret) + goto err_mr_remote_delete; + + if (wc.wr_id != (uintptr_t)FLUSH_ID) { + ret = -1; + (void) fprintf(stderr, + "Client: error: unexpected wc.wr_id value " + "(0x%" PRIXPTR " != 0x%" PRIXPTR ")\n", + (uintptr_t)wc.wr_id, (uintptr_t)FLUSH_ID); + goto err_mr_remote_delete; + } + if (wc.status != IBV_WC_SUCCESS) { + ret = -1; + (void) fprintf(stderr, "Client: error: rpma_flush() failed: %s\n", + ibv_wc_status_str(wc.status)); + goto err_mr_remote_delete; + } + +err_mr_remote_delete: + /* delete the remote memory region's structure */ + (void) rpma_mr_remote_delete(&remote_mr); + +err_mr_dereg: + /* deregister the memory region */ + (void) rpma_mr_dereg(&local_mr); + +err_conn_disconnect: + /* + * Disconnect, wait for RPMA_CONN_CLOSED + * and delete the connection structure. + */ + (void) common_disconnect_and_wait_for_conn_close(&conn); + +err_peer_delete: + /* delete the peer */ + (void) rpma_peer_delete(&peer); + +err_free: + free(local_mr_ptr); + + return ret; +} diff --git a/examples/R/cmake/FindLIBIBVERBS.cmake b/examples/R/cmake/FindLIBIBVERBS.cmake new file mode 100644 index 0000000..cd26480 --- /dev/null +++ b/examples/R/cmake/FindLIBIBVERBS.cmake @@ -0,0 +1,20 @@ +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2020-2021, Intel Corporation +# + +message(STATUS "Checking for module 'libibverbs' w/o PkgConfig") + +find_library(LIBIBVERBS_LIBRARY NAMES libibverbs.so libibverbs ibverbs) +set(LIBIBVERBS_LIBRARIES ${LIBIBVERBS_LIBRARY}) + +if(LIBIBVERBS_LIBRARY) + message(STATUS " Found libibverbs w/o PkgConfig") +else() + set(MSG_NOT_FOUND "libibverbs NOT found (set CMAKE_PREFIX_PATH to point the location)") + if(LIBIBVERBS_FIND_REQUIRED) + message(FATAL_ERROR ${MSG_NOT_FOUND}) + else() + message(WARNING ${MSG_NOT_FOUND}) + endif() +endif() diff --git a/examples/R/cmake/FindLIBPMEM.cmake b/examples/R/cmake/FindLIBPMEM.cmake new file mode 100644 index 0000000..0d2c06f --- /dev/null +++ b/examples/R/cmake/FindLIBPMEM.cmake @@ -0,0 +1,20 @@ +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2020-2021, Intel Corporation +# + +message(STATUS "Checking for module 'libpmem' w/o PkgConfig") + +find_library(LIBPMEM_LIBRARY NAMES libpmem.so libpmem pmem) +set(LIBPMEM_LIBRARIES ${LIBPMEM_LIBRARY}) + +if(LIBPMEM_LIBRARY) + message(STATUS " Found libpmem w/o PkgConfig") +else() + set(MSG_NOT_FOUND "libpmem NOT found (set CMAKE_PREFIX_PATH to point the location)") + if(LIBPMEM_FIND_REQUIRED) + message(FATAL_ERROR ${MSG_NOT_FOUND}) + else() + message(WARNING ${MSG_NOT_FOUND}) + endif() +endif() diff --git a/examples/R/cmake/FindLIBRPMA.cmake b/examples/R/cmake/FindLIBRPMA.cmake new file mode 100644 index 0000000..99a208c --- /dev/null +++ b/examples/R/cmake/FindLIBRPMA.cmake @@ -0,0 +1,20 @@ +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2020-2021, Intel Corporation +# + +message(STATUS "Checking for module 'librpma' w/o PkgConfig") + +find_library(LIBRPMA_LIBRARY NAMES librpma.so librpma rpma) +set(LIBRPMA_LIBRARIES ${LIBRPMA_LIBRARY}) + +if(LIBRPMA_LIBRARY) + message(STATUS " Found librpma w/o PkgConfig") +else() + set(MSG_NOT_FOUND "librpma NOT found (set CMAKE_PREFIX_PATH to point the location)") + if(LIBRPMA_FIND_REQUIRED) + message(FATAL_ERROR ${MSG_NOT_FOUND}) + else() + message(WARNING ${MSG_NOT_FOUND}) + endif() +endif() diff --git a/examples/R/common.c b/examples/R/common.c new file mode 100644 index 0000000..7536e08 --- /dev/null +++ b/examples/R/common.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright 2020-2021, Intel Corporation */ + +/* + * common.c -- common functions used by the example R + */ + +#include +#include +#include + +#include "common.h" + +/* + * common_peer_via_address -- create a new RPMA peer based on ibv_context + * received by the provided address + */ +int +common_peer_via_address(const char *addr, enum rpma_util_ibv_context_type type, + struct rpma_peer **peer_ptr) +{ + struct ibv_context *dev = NULL; + + int ret = rpma_utils_get_ibv_context(addr, type, &dev); + if (ret) + return ret; + + /* create a new peer object */ + return rpma_peer_new(dev, peer_ptr); +} + +/* + * common_client_connect -- establish a new connection to a server listening at + * addr:port + */ +int +common_client_connect(struct rpma_peer *peer, const char *addr, + const char *port, struct rpma_conn_cfg *cfg, + struct rpma_conn_private_data *pdata, + struct rpma_conn **conn_ptr) +{ + struct rpma_conn_req *req = NULL; + enum rpma_conn_event conn_event = RPMA_CONN_UNDEFINED; + + /* create a connection request */ + int ret = rpma_conn_req_new(peer, addr, port, cfg, &req); + if (ret) + return ret; + + /* connect the connection request and obtain the connection object */ + ret = rpma_conn_req_connect(&req, pdata, conn_ptr); + if (ret) { + (void) rpma_conn_req_delete(&req); + return ret; + } + + /* wait for the connection to establish */ + ret = rpma_conn_next_event(*conn_ptr, &conn_event); + if (ret) { + goto err_conn_delete; + } else if (conn_event != RPMA_CONN_ESTABLISHED) { + fprintf(stderr, + "rpma_conn_next_event returned an unexpected event: %s\n", + rpma_utils_conn_event_2str(conn_event)); + ret = -1; + goto err_conn_delete; + } + + return 0; + +err_conn_delete: + (void) rpma_conn_delete(conn_ptr); + + return ret; +} + +/* + * common_server_accept_connection -- wait for an incoming connection request, + * accept it and wait for its establishment + */ +int +common_server_accept_connection(struct rpma_ep *ep, struct rpma_conn_cfg *cfg, + struct rpma_conn_private_data *pdata, + struct rpma_conn **conn_ptr) +{ + struct rpma_conn_req *req = NULL; + enum rpma_conn_event conn_event = RPMA_CONN_UNDEFINED; + + /* receive an incoming connection request */ + int ret = rpma_ep_next_conn_req(ep, cfg, &req); + if (ret) + return ret; + + /* + * connect / accept the connection request and obtain the connection + * object + */ + ret = rpma_conn_req_connect(&req, pdata, conn_ptr); + if (ret) + return ret; + + /* wait for the connection to be established */ + ret = rpma_conn_next_event(*conn_ptr, &conn_event); + if (!ret && conn_event != RPMA_CONN_ESTABLISHED) { + fprintf(stderr, "rpma_conn_next_event returned an unexpected event: %s\n", + rpma_utils_conn_event_2str(conn_event)); + ret = -1; + } + + if (ret) + (void) rpma_conn_delete(conn_ptr); + + return ret; +} + +/* + * common_wait_for_conn_close_verbose -- wait for RPMA_CONN_CLOSED and print + * an error message on error + */ +static inline int +common_wait_for_conn_close_verbose(struct rpma_conn *conn) +{ + enum rpma_conn_event conn_event = RPMA_CONN_UNDEFINED; + + /* wait for the connection to be closed */ + int ret = rpma_conn_next_event(conn, &conn_event); + if (!ret && conn_event != RPMA_CONN_CLOSED) { + fprintf(stderr, + "rpma_conn_next_event returned an unexpected event: %s\n", + rpma_utils_conn_event_2str(conn_event)); + } + + return ret; +} + +/* + * common_wait_for_conn_close_and_disconnect -- wait for RPMA_CONN_CLOSED, + * disconnect and delete the connection structure + */ +int +common_wait_for_conn_close_and_disconnect(struct rpma_conn **conn_ptr) +{ + int ret = 0; + ret |= common_wait_for_conn_close_verbose(*conn_ptr); + ret |= rpma_conn_disconnect(*conn_ptr); + ret |= rpma_conn_delete(conn_ptr); + + return ret; +} + +/* + * common_disconnect_and_wait_for_conn_close -- disconnect, wait for + * RPMA_CONN_CLOSED and delete the connection structure + */ +int +common_disconnect_and_wait_for_conn_close(struct rpma_conn **conn_ptr) +{ + int ret = 0; + + ret |= rpma_conn_disconnect(*conn_ptr); + if (ret == 0) + ret |= common_wait_for_conn_close_verbose(*conn_ptr); + + ret |= rpma_conn_delete(conn_ptr); + + return ret; +} diff --git a/examples/R/common.h b/examples/R/common.h new file mode 100644 index 0000000..4f5b0f0 --- /dev/null +++ b/examples/R/common.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright 2020-2021, Intel Corporation */ + +/* + * common.h -- common functions declarations for the example R + */ + +#ifndef PMEMHACKATHON_COMMON_H +#define PMEMHACKATHON_COMMON_H + +#include +#include + +#define KILOBYTE 1024 + +/* + * Limited by the maximum length of the private data + * for rdma_connect() in case of RDMA_PS_TCP (28 bytes). + */ +#define DESCRIPTORS_MAX_SIZE 26 + +struct common_data { + uint8_t mr_desc_size; /* size of mr_desc in descriptors[] */ + uint8_t pcfg_desc_size; /* size of pcfg_desc in descriptors[] */ + /* buffer containing mr_desc and pcfg_desc */ + char descriptors[DESCRIPTORS_MAX_SIZE]; +}; + +int common_peer_via_address(const char *addr, + enum rpma_util_ibv_context_type type, + struct rpma_peer **peer_ptr); + +#define common_client_peer_via_address(addr, peer_ptr) \ + common_peer_via_address(addr, RPMA_UTIL_IBV_CONTEXT_REMOTE, \ + peer_ptr) + +#define common_server_peer_via_address(addr, peer_ptr) \ + common_peer_via_address(addr, RPMA_UTIL_IBV_CONTEXT_LOCAL, \ + peer_ptr) + +int common_client_connect(struct rpma_peer *peer, const char *addr, + const char *port, struct rpma_conn_cfg *cfg, + struct rpma_conn_private_data *pdata, + struct rpma_conn **conn_ptr); + +int common_server_accept_connection(struct rpma_ep *ep, + struct rpma_conn_cfg *cfg, struct rpma_conn_private_data *pdata, + struct rpma_conn **conn_ptr); + +int common_wait_for_conn_close_and_disconnect(struct rpma_conn **conn_ptr); +int common_disconnect_and_wait_for_conn_close(struct rpma_conn **conn_ptr); + +#endif /* PMEMHACKATHON_COMMON_H */ diff --git a/examples/R/run_main.sh b/examples/R/run_main.sh new file mode 100755 index 0000000..4656661 --- /dev/null +++ b/examples/R/run_main.sh @@ -0,0 +1,21 @@ +#!/bin/bash -ex +# +# The shell commands to run the main example. +# + +if [ ! -c /${DEV_DAX} ]; then + echo "Error: \"${DEV_DAX}\" device DAX not found (DEV_DAX environment variable)" + exit 1 +fi + +echo "Device DAX: ${DEV_DAX}" + +SERVER_IP=$RPMA_SOFT_ROCE_IP +SERVER_PORT=$((62000 + $PMEMUSER_ID)) + +cd build +# usage: ./server +./server $SERVER_IP $SERVER_PORT $DEV_DAX $PMEMUSER_ID & +sleep 1 +# usage: ./client +./client $SERVER_IP $SERVER_PORT diff --git a/examples/R/run_simple.sh b/examples/R/run_simple.sh new file mode 100755 index 0000000..706e877 --- /dev/null +++ b/examples/R/run_simple.sh @@ -0,0 +1,14 @@ +#!/bin/bash -ex +# +# The shell commands to run the simple_* example. +# + +SERVER_IP=$RPMA_SOFT_ROCE_IP +SERVER_PORT=$((62000 + $PMEMUSER_ID)) + +cd build +# usage: ./simple_server +./simple_server $SERVER_IP $SERVER_PORT & +sleep 1 +# usage: ./simple_client +./simple_client $SERVER_IP $SERVER_PORT diff --git a/examples/R/run_test_ping.sh b/examples/R/run_test_ping.sh new file mode 100755 index 0000000..1242f0b --- /dev/null +++ b/examples/R/run_test_ping.sh @@ -0,0 +1,8 @@ +#!/bin/bash -ex +# +# The shell commands to run the basic ping test. +# + +SERVER_IP=$RPMA_SOFT_ROCE_IP + +ping -c5 $SERVER_IP diff --git a/examples/R/run_test_rping.sh b/examples/R/run_test_rping.sh new file mode 100755 index 0000000..fa98f04 --- /dev/null +++ b/examples/R/run_test_rping.sh @@ -0,0 +1,10 @@ +#!/bin/bash -ex +# +# The shell commands to run the rping test. +# + +SERVER_IP=$RPMA_SOFT_ROCE_IP + +rping -s -a $SERVER_IP -C 10 & +sleep 1 +rping -c -a $SERVER_IP -C 10 -v diff --git a/examples/R/server.c b/examples/R/server.c new file mode 100644 index 0000000..4dd7ac3 --- /dev/null +++ b/examples/R/server.c @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright 2020-2021, Intel Corporation */ + +/* + * server.c -- a server of the "read, write and flush to persistent domain" example + */ + +#include +#include +#include +#include "common.h" + +#define USAGE_STR "usage: %s \n" +#define INIT_STR "This is the initial server string." +#define MAX_USERS 200 +#define PMEM_MIN_SIZE (MAX_USERS * 4 * KILOBYTE) /* 4KiB for each user */ + +#define ERROR_ON_WRONG_USAGE(cond) \ + do { \ + if (cond) { \ + fprintf(stderr, USAGE_STR, argv[0]); \ + exit(-1); \ + } \ + } while (0) + +int +main(int argc, char *argv[]) +{ + int ret; + + /* configure logging thresholds to see more details */ + rpma_log_set_threshold(RPMA_LOG_THRESHOLD, RPMA_LOG_LEVEL_INFO); + rpma_log_set_threshold(RPMA_LOG_THRESHOLD_AUX, RPMA_LOG_LEVEL_INFO); + + /* validate parameters */ + ERROR_ON_WRONG_USAGE(argc < 5); + + /* read parameters */ + char *addr = argv[1]; + char *port = argv[2]; + char *path = argv[3]; + int user_id = atoi(argv[4]); + + ERROR_ON_WRONG_USAGE(user_id < 1 || user_id > MAX_USERS); + + /* resources - PMem */ + char *pmem_ptr; + size_t pmem_size; + int is_pmem; + + /* map the file */ + pmem_ptr = pmem_map_file(path, 0 /* len */, 0 /* flags */, + 0 /* mode */, &pmem_size, &is_pmem); + if (pmem_ptr == NULL) { + (void) fprintf(stderr, "Server: error: pmem_map_file() for %s failed\n", path); + return -1; + } + + /* pmem is expected */ + if (!is_pmem) { + (void) fprintf(stderr, "Server: error: %s is not an actual PMEM\n", path); + (void) pmem_unmap(pmem_ptr, pmem_size); + return -1; + } + + /* check if PMem has minimum required size */ + if (pmem_size < PMEM_MIN_SIZE) { + (void) fprintf(stderr, "Server: error: %s too small (%zu < %u)\n", + path, pmem_size, PMEM_MIN_SIZE); + (void) pmem_unmap(pmem_ptr, pmem_size); + return -1; + } + + /* resources - memory region */ + struct rpma_mr_local *mr = NULL; + /* separate 4KiB of PMem for each user */ + size_t mr_size = 4 * KILOBYTE; + char *mr_ptr = pmem_ptr + (user_id - 1) * mr_size; + + /* RPMA resources */ + struct rpma_peer_cfg *pcfg = NULL; + struct rpma_peer *peer = NULL; + struct rpma_ep *ep = NULL; + struct rpma_conn *conn = NULL; + + /* set and print the initial content */ + strncpy(mr_ptr, INIT_STR, mr_size); + (void) printf("Server: the initial content of the server's persistent memory: %s\n", + mr_ptr); + /* + * Flush the output buffer in order to assure the initial value + * will be really close to the top of the console + * in the case of mixing the client and the server output. + */ + (void) fflush(stdout); + + /* create a peer configuration structure */ + ret = rpma_peer_cfg_new(&pcfg); + if (ret) + goto err_unmap; + + /* configure peer's direct write to pmem support */ + ret = rpma_peer_cfg_set_direct_write_to_pmem(pcfg, true); + if (ret) { + (void) rpma_peer_cfg_delete(&pcfg); + goto err_unmap; + } + + /* + * lookup an ibv_context via the address and create a new peer using it + */ + ret = common_server_peer_via_address(addr, &peer); + if (ret) + goto err_pcfg_delete; + + /* start a listening endpoint at addr:port */ + ret = rpma_ep_listen(peer, addr, port, &ep); + if (ret) + goto err_peer_delete; + + /* register the memory */ + ret = rpma_mr_reg(peer, mr_ptr, mr_size, + RPMA_MR_USAGE_READ_SRC | RPMA_MR_USAGE_WRITE_DST | + (RPMA_MR_USAGE_FLUSH_TYPE_PERSISTENT | + RPMA_MR_USAGE_FLUSH_TYPE_VISIBILITY), + &mr); + if (ret) + goto err_ep_shutdown; + + /* get size of the memory region's descriptor */ + size_t mr_desc_size; + ret = rpma_mr_get_descriptor_size(mr, &mr_desc_size); + if (ret) + goto err_mr_dereg; + + /* get size of the peer config descriptor */ + size_t pcfg_desc_size; + ret = rpma_peer_cfg_get_descriptor_size(pcfg, &pcfg_desc_size); + if (ret) + goto err_mr_dereg; + + /* calculate data for the client write */ + struct common_data data = {0}; + data.mr_desc_size = mr_desc_size; + data.pcfg_desc_size = pcfg_desc_size; + + /* get the memory region's descriptor */ + ret = rpma_mr_get_descriptor(mr, &data.descriptors[0]); + if (ret) + goto err_mr_dereg; + + /* + * Get the peer's configuration descriptor. + * The pcfg_desc descriptor is saved in the `descriptors[]` array + * just after the mr_desc descriptor. + */ + ret = rpma_peer_cfg_get_descriptor(pcfg, + &data.descriptors[mr_desc_size]); + if (ret) + goto err_mr_dereg; + + /* + * Wait for an incoming connection request, accept it and wait for its + * establishment. + */ + struct rpma_conn_private_data pdata; + pdata.ptr = &data; + pdata.len = sizeof(struct common_data); + ret = common_server_accept_connection(ep, NULL, &pdata, &conn); + if (ret) + goto err_mr_dereg; + + /* + * Wait for RPMA_CONN_CLOSED, disconnect and delete the connection + * structure. + */ + ret = common_wait_for_conn_close_and_disconnect(&conn); + if (ret) + goto err_mr_dereg; + + (void) printf("Server: read a new data from the server's persistent memory: %s\n", + mr_ptr); + +err_mr_dereg: + /* deregister the memory region */ + (void) rpma_mr_dereg(&mr); + +err_ep_shutdown: + /* shutdown the endpoint */ + (void) rpma_ep_shutdown(&ep); + +err_peer_delete: + /* delete the peer object */ + (void) rpma_peer_delete(&peer); + +err_pcfg_delete: + (void) rpma_peer_cfg_delete(&pcfg); + +err_unmap: + (void) pmem_unmap(mr_ptr, mr_size); + + return ret; +} diff --git a/examples/R/simple_client.c b/examples/R/simple_client.c new file mode 100644 index 0000000..ce118bf --- /dev/null +++ b/examples/R/simple_client.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright 2021, Intel Corporation */ + +/* + * simple_client.c -- a simple connection client + */ + +#include +#include +#include "common.h" + +#define USAGE_STR "usage: %s \n" + +int +main(int argc, char *argv[]) +{ + /* validate parameters */ + if (argc < 3) { + fprintf(stderr, USAGE_STR, argv[0]); + exit(-1); + } + + /* configure logging thresholds to see more details */ + rpma_log_set_threshold(RPMA_LOG_THRESHOLD, RPMA_LOG_LEVEL_INFO); + rpma_log_set_threshold(RPMA_LOG_THRESHOLD_AUX, RPMA_LOG_LEVEL_INFO); + + /* read common parameters */ + char *addr = argv[1]; + char *port = argv[2]; + int ret; + + /* RPMA resources */ + struct rpma_peer *peer = NULL; + struct rpma_conn *conn = NULL; + + /* + * lookup an ibv_context via the address and create a new peer using it + */ + ret = common_client_peer_via_address(addr, &peer); + if (ret) + return ret; + + /* + * Prepare a connection's private data. + * The length of the private data for rdma_connect() + * in case of RDMA_PS_TCP is limited to 28 bytes. + */ + const char *msg = "Hello server! I'm client :)"; + struct rpma_conn_private_data pdata; + pdata.ptr = (void *)msg; + pdata.len = (strlen(msg) + 1) * sizeof(char); + + /* establish a new connection to a server listening at addr:port */ + ret = common_client_connect(peer, addr, port, NULL, &pdata, &conn); + if (ret) + goto err_peer_delete; + + /* here you can use the newly established connection */ + (void) rpma_conn_get_private_data(conn, &pdata); + if (pdata.ptr) { + char *msg = pdata.ptr; + fprintf(stdout, "Client: received the message: %s\n", msg); + } else { + fprintf(stdout, "Client: no message received\n"); + } + + /* + * Disconnect, wait for RPMA_CONN_CLOSED + * and delete the connection structure. + */ + ret = common_disconnect_and_wait_for_conn_close(&conn); + +err_peer_delete: + /* delete the peer object */ + (void) rpma_peer_delete(&peer); + + return ret; +} diff --git a/examples/R/simple_server.c b/examples/R/simple_server.c new file mode 100644 index 0000000..5dee302 --- /dev/null +++ b/examples/R/simple_server.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright 2021, Intel Corporation */ + +/* + * simple_server.c -- a simple connection server + */ + +#include +#include +#include "common.h" + +#define USAGE_STR "usage: %s \n" + +int +main(int argc, char *argv[]) +{ + /* validate parameters */ + if (argc < 3) { + fprintf(stderr, USAGE_STR, argv[0]); + exit(-1); + } + + /* configure logging thresholds to see more details */ + rpma_log_set_threshold(RPMA_LOG_THRESHOLD, RPMA_LOG_LEVEL_INFO); + rpma_log_set_threshold(RPMA_LOG_THRESHOLD_AUX, RPMA_LOG_LEVEL_INFO); + + /* read common parameters */ + char *addr = argv[1]; + char *port = argv[2]; + int ret; + + /* RPMA resources */ + struct rpma_peer *peer = NULL; + struct rpma_ep *ep = NULL; + struct rpma_conn *conn = NULL; + + /* + * lookup an ibv_context via the address and create a new peer using it + */ + ret = common_server_peer_via_address(addr, &peer); + if (ret) + return ret; + + /* start a listening endpoint at addr:port */ + ret = rpma_ep_listen(peer, addr, port, &ep); + if (ret) + goto err_peer_delete; + + /* + * Prepare a connection's private data. + * The length of the private data for rdma_connect() + * in case of RDMA_PS_TCP is limited to 28 bytes. + */ + const char *msg = "Hello client! I'm server :)"; + struct rpma_conn_private_data pdata; + pdata.ptr = (void *)msg; + pdata.len = (strlen(msg) + 1) * sizeof(char); + + /* + * Wait for an incoming connection request, accept it and wait for its + * establishment. + */ + ret = common_server_accept_connection(ep, NULL, &pdata, &conn); + if (ret) + goto err_ep_shutdown; + + /* here you can use the newly established connection */ + (void) rpma_conn_get_private_data(conn, &pdata); + if (pdata.ptr) { + char *msg = pdata.ptr; + fprintf(stdout, "Server: received the message: %s\n", msg); + } else { + fprintf(stdout, "Server: no message received\n"); + } + + /* + * Wait for RPMA_CONN_CLOSED, disconnect and delete the connection + * structure. + */ + ret = common_wait_for_conn_close_and_disconnect(&conn); + +err_ep_shutdown: + /* shutdown the endpoint */ + (void) rpma_ep_shutdown(&ep); + +err_peer_delete: + /* delete the peer object */ + (void) rpma_peer_delete(&peer); + + return ret; +}