diff --git a/examples/R/.gitignore b/examples/R/.gitignore new file mode 100644 index 0000000..567609b --- /dev/null +++ b/examples/R/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/examples/R/CMakeLists.txt b/examples/R/CMakeLists.txt new file mode 100644 index 0000000..67838cc --- /dev/null +++ b/examples/R/CMakeLists.txt @@ -0,0 +1,50 @@ +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2020-2021, Intel Corporation +# Copyright 2021, Fujitsu +# + +cmake_minimum_required(VERSION 3.3) +project(exampleR C) + +set(LIBPMEM_REQUIRED_VERSION 1.6) + +# append -Wall and -Werror to CMAKE_C_FLAGS +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") + +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/cmake) + +find_package(PkgConfig QUIET) + +if(PKG_CONFIG_FOUND) + pkg_check_modules(LIBRPMA librpma) + pkg_check_modules(LIBIBVERBS libibverbs) + pkg_check_modules(LIBPMEM libpmem>=${LIBPMEM_REQUIRED_VERSION}) +endif() +if(NOT LIBRPMA_FOUND) + find_package(LIBRPMA REQUIRED librpma) +endif() +if(NOT LIBIBVERBS_FOUND) + find_package(LIBIBVERBS REQUIRED libibverbs) +endif() +if(NOT LIBPMEM_FOUND) + find_package(LIBPMEM ${LIBPMEM_REQUIRED_VERSION} REQUIRED libpmem) +endif() + +link_directories(${LIBRPMA_LIBRARY_DIRS}) + +function(add_example name) + set(srcs ${ARGN}) + add_executable(${name} ${srcs}) + target_include_directories(${name} PRIVATE ${LIBRPMA_INCLUDE_DIRS} ${LIBPMEM_INCLUDE_DIRS}) + target_link_libraries(${name} rpma ${LIBIBVERBS_LIBRARIES} ${LIBRT_LIBRARIES} ${LIBPMEM_LIBRARIES}) +endfunction() + +add_example(server server.c common.c) +add_example(client client.c common.c) +add_example(simple_server simple_server.c common.c) +add_example(simple_client simple_client.c common.c) + +add_custom_target(config_softroce + COMMAND ${CMAKE_SOURCE_DIR}/tools/config_softroce.sh) diff --git a/examples/R/README.md b/examples/R/README.md new file mode 100644 index 0000000..4107076 --- /dev/null +++ b/examples/R/README.md @@ -0,0 +1,32 @@ +Example of performing an RPMA read, write and flush to persistence operations +=== + +This example implements both sides of an RDMA connection: +- The server prepares a local persistent memory and exposes the memory description +along with other parameters required to perform an RDMA read, write and flush operations. +After the connection is established, the server waits for the client to disconnect. +- The client allocates memory from DRAM and registers it as a reading destination +and writing source. After the connection is established the client receives +the server's memory regions registered as a reading source and a writing destination. +The client performs the RDMA read from the remote memory region to the local memory region, +then it writes new data to the local memory region and performs the RDMA write +from the local memory region to the remote memory region followed by the RPMA flush. + +**Note**: The server requires a unique argument in order to use +a different part of persistent memory which is shared by all the server instances +running on the same . is also used to pick a unique TCP port. + +**Note**: For the sake of this example, the memory region being written to and +the server's peer configuration are transferred via the connection's private +data. In general, it can be transferred via an out-of-band or the in-band +channel. + +## Usage + +```bash +[user@server]$ ./server $server_address $port ${pmem-path} ${user-id} +``` + +```bash +[user@client]$ ./client $server_address $port +``` diff --git a/examples/R/build_main.sh b/examples/R/build_main.sh new file mode 100755 index 0000000..48379c9 --- /dev/null +++ b/examples/R/build_main.sh @@ -0,0 +1,10 @@ +#!/bin/bash -ex +# +# The shell commands to build the main example. +# + +mkdir -p build +cd build +cmake .. +make server +make client diff --git a/examples/R/build_simple.sh b/examples/R/build_simple.sh new file mode 100755 index 0000000..e9f9173 --- /dev/null +++ b/examples/R/build_simple.sh @@ -0,0 +1,10 @@ +#!/bin/bash -ex +# +# The shell commands to build the simple_* example. +# + +mkdir -p build +cd build +cmake .. +make simple_client +make simple_server diff --git a/examples/R/client.c b/examples/R/client.c new file mode 100644 index 0000000..8ef60f2 --- /dev/null +++ b/examples/R/client.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright 2020-2021, Intel Corporation */ +/* Copyright 2021, Fujitsu */ + +/* + * client.c -- a client of the "read, write and flush to persistent domain" example + */ + +#include +#include +#include +#include +#include "common.h" + +#define USAGE_STR "usage: %s \n" +#define FLUSH_ID (void *)0xF01D /* a random identifier */ + +static const char *hello_str = "Hello world!"; + +/* + * malloc_aligned -- allocate an aligned chunk of memory + */ +void * +malloc_aligned(size_t size) +{ + long pagesize = sysconf(_SC_PAGESIZE); + if (pagesize < 0) { + perror("sysconf"); + return NULL; + } + + /* allocate a page size aligned local memory pool */ + void *mem; + int ret = posix_memalign(&mem, (size_t)pagesize, size); + if (ret) { + (void) fprintf(stderr, "Client: error: posix_memalign: %s\n", + strerror(ret)); + return NULL; + } + + /* zero the allocated memory */ + memset(mem, 0, size); + + return mem; +} + +int +main(int argc, char *argv[]) +{ + /* validate parameters */ + if (argc < 3) { + fprintf(stderr, USAGE_STR, argv[0]); + exit(-1); + } + + /* configure logging thresholds to see more details */ + rpma_log_set_threshold(RPMA_LOG_THRESHOLD, RPMA_LOG_LEVEL_INFO); + rpma_log_set_threshold(RPMA_LOG_THRESHOLD_AUX, RPMA_LOG_LEVEL_INFO); + + /* read common parameters */ + char *addr = argv[1]; + char *port = argv[2]; + int ret; + + /* resources - memory region */ + void *local_mr_ptr; + size_t local_mr_size; + size_t local_offset = 0; + struct rpma_mr_remote *remote_mr = NULL; + size_t remote_size = 0; + size_t remote_offset = 0; + struct rpma_mr_local *local_mr = NULL; + struct ibv_wc wc; + + /* RPMA resources */ + struct rpma_peer_cfg *pcfg = NULL; + struct rpma_peer *peer = NULL; + struct rpma_conn *conn = NULL; + bool direct_write_to_pmem = false; + + local_mr_size = KILOBYTE; + local_mr_ptr = malloc_aligned(local_mr_size); + if (local_mr_ptr == NULL) + return -1; + + /* + * lookup an ibv_context via the address and create a new peer using it + */ + ret = common_client_peer_via_address(addr, &peer); + if (ret) + goto err_free; + + /* establish a new connection to a server listening at addr:port */ + ret = common_client_connect(peer, addr, port, NULL, NULL, &conn); + if (ret) + goto err_peer_delete; + + /* register the memory for RDMA read and write operations */ + ret = rpma_mr_reg(peer, local_mr_ptr, local_mr_size, + (RPMA_MR_USAGE_READ_DST | RPMA_MR_USAGE_WRITE_SRC), + &local_mr); + if (ret) + goto err_conn_disconnect; + + /* obtain the remote side resources description */ + struct rpma_conn_private_data pdata; + ret = rpma_conn_get_private_data(conn, &pdata); + if (ret != 0 || pdata.len < sizeof(struct common_data)) + goto err_mr_dereg; + + /* + * Create a remote peer configuration structure from the received + * descriptor and apply it to the current connection. + */ + struct common_data *remote_data = pdata.ptr; + ret = rpma_peer_cfg_from_descriptor( + &remote_data->descriptors[remote_data->mr_desc_size], + remote_data->pcfg_desc_size, &pcfg); + if (ret) + goto err_mr_dereg; + ret = rpma_peer_cfg_get_direct_write_to_pmem(pcfg, + &direct_write_to_pmem); + ret |= rpma_conn_apply_remote_peer_cfg(conn, pcfg); + (void) rpma_peer_cfg_delete(&pcfg); + /* either get or apply failed */ + if (ret) + goto err_mr_dereg; + + /* verify the Direct Write to PMem support */ + if (!direct_write_to_pmem) { + (void) fprintf(stderr, + "Client: error: the server does not support Direct Write to PMem\n"); + goto err_mr_dereg; + } + + /* + * Create a remote memory registration structure from the received + * descriptor. + */ + ret = rpma_mr_remote_from_descriptor(&remote_data->descriptors[0], + remote_data->mr_desc_size, &remote_mr); + if (ret) + goto err_mr_dereg; + + /* get the remote memory region size */ + ret = rpma_mr_remote_get_size(remote_mr, &remote_size); + if (ret) { + goto err_mr_remote_delete; + } else if (remote_size < KILOBYTE) { + fprintf(stderr, + "Client: error: remote memory region size too small for writing the data of the assumed size (%zu < %d)\n", + remote_size, KILOBYTE); + goto err_mr_remote_delete; + } + + /* read the initial value */ + size_t len = (local_mr_size < remote_size) ? local_mr_size : remote_size; + ret = rpma_read(conn, local_mr, local_offset, + remote_mr, remote_offset, len, + RPMA_F_COMPLETION_ALWAYS, NULL); + if (ret) + goto err_mr_remote_delete; + + /* get the connection's main CQ */ + struct rpma_cq *cq = NULL; + ret = rpma_conn_get_cq(conn, &cq); + if (ret) + goto err_mr_remote_delete; + + /* wait for the completion to be ready */ + ret = rpma_cq_wait(cq); + if (ret) + goto err_mr_remote_delete; + + /* wait for a completion of the RDMA read */ + ret = rpma_cq_get_wc(cq, 1, &wc, NULL); + if (ret) + goto err_mr_remote_delete; + + if (wc.status != IBV_WC_SUCCESS) { + ret = -1; + (void) fprintf(stderr, "Client: error: rpma_read() failed: %s\n", + ibv_wc_status_str(wc.status)); + goto err_mr_remote_delete; + } + + if (wc.opcode != IBV_WC_RDMA_READ) { + ret = -1; + (void) fprintf(stderr, "Client: error: unexpected wc.opcode value (%d != %d)\n", + wc.opcode, IBV_WC_RDMA_READ); + goto err_mr_remote_delete; + } + + (void) fprintf(stdout, "Client: read the initial content of the remote (server's) persistent memory: %s\n", + (char *)local_mr_ptr + local_offset); + + /* write the next value */ + strncpy(local_mr_ptr, hello_str, KILOBYTE); + (void) printf("Client: writing to the remote (server's) persistent memory: %s\n", + (char *)local_mr_ptr); + + ret = rpma_write(conn, remote_mr, remote_offset, + local_mr, local_offset, KILOBYTE, + RPMA_F_COMPLETION_ON_ERROR, NULL); + if (ret) { + (void) fprintf(stderr, "Client: error: rpma_write() failed: %s\n", + rpma_err_2str(ret)); + goto err_mr_remote_delete; + } + + (void) printf("Client: flushing the remote data to the persistent domain...\n"); + + ret = rpma_flush(conn, remote_mr, remote_offset, KILOBYTE, + RPMA_FLUSH_TYPE_PERSISTENT, + RPMA_F_COMPLETION_ALWAYS, FLUSH_ID); + if (ret) { + (void) fprintf(stderr, "Client: error: rpma_flush() failed: %s\n", + rpma_err_2str(ret)); + goto err_mr_remote_delete; + } + + /* get the connection's main CQ */ + ret = rpma_conn_get_cq(conn, &cq); + if (ret) + goto err_mr_remote_delete; + + /* wait for the completion to be ready */ + ret = rpma_cq_wait(cq); + if (ret) + goto err_mr_remote_delete; + + /* wait for a completion of the RDMA read */ + ret = rpma_cq_get_wc(cq, 1, &wc, NULL); + if (ret) + goto err_mr_remote_delete; + + if (wc.wr_id != (uintptr_t)FLUSH_ID) { + ret = -1; + (void) fprintf(stderr, + "Client: error: unexpected wc.wr_id value " + "(0x%" PRIXPTR " != 0x%" PRIXPTR ")\n", + (uintptr_t)wc.wr_id, (uintptr_t)FLUSH_ID); + goto err_mr_remote_delete; + } + if (wc.status != IBV_WC_SUCCESS) { + ret = -1; + (void) fprintf(stderr, "Client: error: rpma_flush() failed: %s\n", + ibv_wc_status_str(wc.status)); + goto err_mr_remote_delete; + } + +err_mr_remote_delete: + /* delete the remote memory region's structure */ + (void) rpma_mr_remote_delete(&remote_mr); + +err_mr_dereg: + /* deregister the memory region */ + (void) rpma_mr_dereg(&local_mr); + +err_conn_disconnect: + /* + * Disconnect, wait for RPMA_CONN_CLOSED + * and delete the connection structure. + */ + (void) common_disconnect_and_wait_for_conn_close(&conn); + +err_peer_delete: + /* delete the peer */ + (void) rpma_peer_delete(&peer); + +err_free: + free(local_mr_ptr); + + return ret; +} diff --git a/examples/R/cmake/FindLIBIBVERBS.cmake b/examples/R/cmake/FindLIBIBVERBS.cmake new file mode 100644 index 0000000..cd26480 --- /dev/null +++ b/examples/R/cmake/FindLIBIBVERBS.cmake @@ -0,0 +1,20 @@ +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2020-2021, Intel Corporation +# + +message(STATUS "Checking for module 'libibverbs' w/o PkgConfig") + +find_library(LIBIBVERBS_LIBRARY NAMES libibverbs.so libibverbs ibverbs) +set(LIBIBVERBS_LIBRARIES ${LIBIBVERBS_LIBRARY}) + +if(LIBIBVERBS_LIBRARY) + message(STATUS " Found libibverbs w/o PkgConfig") +else() + set(MSG_NOT_FOUND "libibverbs NOT found (set CMAKE_PREFIX_PATH to point the location)") + if(LIBIBVERBS_FIND_REQUIRED) + message(FATAL_ERROR ${MSG_NOT_FOUND}) + else() + message(WARNING ${MSG_NOT_FOUND}) + endif() +endif() diff --git a/examples/R/cmake/FindLIBPMEM.cmake b/examples/R/cmake/FindLIBPMEM.cmake new file mode 100644 index 0000000..0d2c06f --- /dev/null +++ b/examples/R/cmake/FindLIBPMEM.cmake @@ -0,0 +1,20 @@ +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2020-2021, Intel Corporation +# + +message(STATUS "Checking for module 'libpmem' w/o PkgConfig") + +find_library(LIBPMEM_LIBRARY NAMES libpmem.so libpmem pmem) +set(LIBPMEM_LIBRARIES ${LIBPMEM_LIBRARY}) + +if(LIBPMEM_LIBRARY) + message(STATUS " Found libpmem w/o PkgConfig") +else() + set(MSG_NOT_FOUND "libpmem NOT found (set CMAKE_PREFIX_PATH to point the location)") + if(LIBPMEM_FIND_REQUIRED) + message(FATAL_ERROR ${MSG_NOT_FOUND}) + else() + message(WARNING ${MSG_NOT_FOUND}) + endif() +endif() diff --git a/examples/R/cmake/FindLIBRPMA.cmake b/examples/R/cmake/FindLIBRPMA.cmake new file mode 100644 index 0000000..99a208c --- /dev/null +++ b/examples/R/cmake/FindLIBRPMA.cmake @@ -0,0 +1,20 @@ +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2020-2021, Intel Corporation +# + +message(STATUS "Checking for module 'librpma' w/o PkgConfig") + +find_library(LIBRPMA_LIBRARY NAMES librpma.so librpma rpma) +set(LIBRPMA_LIBRARIES ${LIBRPMA_LIBRARY}) + +if(LIBRPMA_LIBRARY) + message(STATUS " Found librpma w/o PkgConfig") +else() + set(MSG_NOT_FOUND "librpma NOT found (set CMAKE_PREFIX_PATH to point the location)") + if(LIBRPMA_FIND_REQUIRED) + message(FATAL_ERROR ${MSG_NOT_FOUND}) + else() + message(WARNING ${MSG_NOT_FOUND}) + endif() +endif() diff --git a/examples/R/common.c b/examples/R/common.c new file mode 100644 index 0000000..7536e08 --- /dev/null +++ b/examples/R/common.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright 2020-2021, Intel Corporation */ + +/* + * common.c -- common functions used by the example R + */ + +#include +#include +#include + +#include "common.h" + +/* + * common_peer_via_address -- create a new RPMA peer based on ibv_context + * received by the provided address + */ +int +common_peer_via_address(const char *addr, enum rpma_util_ibv_context_type type, + struct rpma_peer **peer_ptr) +{ + struct ibv_context *dev = NULL; + + int ret = rpma_utils_get_ibv_context(addr, type, &dev); + if (ret) + return ret; + + /* create a new peer object */ + return rpma_peer_new(dev, peer_ptr); +} + +/* + * common_client_connect -- establish a new connection to a server listening at + * addr:port + */ +int +common_client_connect(struct rpma_peer *peer, const char *addr, + const char *port, struct rpma_conn_cfg *cfg, + struct rpma_conn_private_data *pdata, + struct rpma_conn **conn_ptr) +{ + struct rpma_conn_req *req = NULL; + enum rpma_conn_event conn_event = RPMA_CONN_UNDEFINED; + + /* create a connection request */ + int ret = rpma_conn_req_new(peer, addr, port, cfg, &req); + if (ret) + return ret; + + /* connect the connection request and obtain the connection object */ + ret = rpma_conn_req_connect(&req, pdata, conn_ptr); + if (ret) { + (void) rpma_conn_req_delete(&req); + return ret; + } + + /* wait for the connection to establish */ + ret = rpma_conn_next_event(*conn_ptr, &conn_event); + if (ret) { + goto err_conn_delete; + } else if (conn_event != RPMA_CONN_ESTABLISHED) { + fprintf(stderr, + "rpma_conn_next_event returned an unexpected event: %s\n", + rpma_utils_conn_event_2str(conn_event)); + ret = -1; + goto err_conn_delete; + } + + return 0; + +err_conn_delete: + (void) rpma_conn_delete(conn_ptr); + + return ret; +} + +/* + * common_server_accept_connection -- wait for an incoming connection request, + * accept it and wait for its establishment + */ +int +common_server_accept_connection(struct rpma_ep *ep, struct rpma_conn_cfg *cfg, + struct rpma_conn_private_data *pdata, + struct rpma_conn **conn_ptr) +{ + struct rpma_conn_req *req = NULL; + enum rpma_conn_event conn_event = RPMA_CONN_UNDEFINED; + + /* receive an incoming connection request */ + int ret = rpma_ep_next_conn_req(ep, cfg, &req); + if (ret) + return ret; + + /* + * connect / accept the connection request and obtain the connection + * object + */ + ret = rpma_conn_req_connect(&req, pdata, conn_ptr); + if (ret) + return ret; + + /* wait for the connection to be established */ + ret = rpma_conn_next_event(*conn_ptr, &conn_event); + if (!ret && conn_event != RPMA_CONN_ESTABLISHED) { + fprintf(stderr, "rpma_conn_next_event returned an unexpected event: %s\n", + rpma_utils_conn_event_2str(conn_event)); + ret = -1; + } + + if (ret) + (void) rpma_conn_delete(conn_ptr); + + return ret; +} + +/* + * common_wait_for_conn_close_verbose -- wait for RPMA_CONN_CLOSED and print + * an error message on error + */ +static inline int +common_wait_for_conn_close_verbose(struct rpma_conn *conn) +{ + enum rpma_conn_event conn_event = RPMA_CONN_UNDEFINED; + + /* wait for the connection to be closed */ + int ret = rpma_conn_next_event(conn, &conn_event); + if (!ret && conn_event != RPMA_CONN_CLOSED) { + fprintf(stderr, + "rpma_conn_next_event returned an unexpected event: %s\n", + rpma_utils_conn_event_2str(conn_event)); + } + + return ret; +} + +/* + * common_wait_for_conn_close_and_disconnect -- wait for RPMA_CONN_CLOSED, + * disconnect and delete the connection structure + */ +int +common_wait_for_conn_close_and_disconnect(struct rpma_conn **conn_ptr) +{ + int ret = 0; + ret |= common_wait_for_conn_close_verbose(*conn_ptr); + ret |= rpma_conn_disconnect(*conn_ptr); + ret |= rpma_conn_delete(conn_ptr); + + return ret; +} + +/* + * common_disconnect_and_wait_for_conn_close -- disconnect, wait for + * RPMA_CONN_CLOSED and delete the connection structure + */ +int +common_disconnect_and_wait_for_conn_close(struct rpma_conn **conn_ptr) +{ + int ret = 0; + + ret |= rpma_conn_disconnect(*conn_ptr); + if (ret == 0) + ret |= common_wait_for_conn_close_verbose(*conn_ptr); + + ret |= rpma_conn_delete(conn_ptr); + + return ret; +} diff --git a/examples/R/common.h b/examples/R/common.h new file mode 100644 index 0000000..4f5b0f0 --- /dev/null +++ b/examples/R/common.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright 2020-2021, Intel Corporation */ + +/* + * common.h -- common functions declarations for the example R + */ + +#ifndef PMEMHACKATHON_COMMON_H +#define PMEMHACKATHON_COMMON_H + +#include +#include + +#define KILOBYTE 1024 + +/* + * Limited by the maximum length of the private data + * for rdma_connect() in case of RDMA_PS_TCP (28 bytes). + */ +#define DESCRIPTORS_MAX_SIZE 26 + +struct common_data { + uint8_t mr_desc_size; /* size of mr_desc in descriptors[] */ + uint8_t pcfg_desc_size; /* size of pcfg_desc in descriptors[] */ + /* buffer containing mr_desc and pcfg_desc */ + char descriptors[DESCRIPTORS_MAX_SIZE]; +}; + +int common_peer_via_address(const char *addr, + enum rpma_util_ibv_context_type type, + struct rpma_peer **peer_ptr); + +#define common_client_peer_via_address(addr, peer_ptr) \ + common_peer_via_address(addr, RPMA_UTIL_IBV_CONTEXT_REMOTE, \ + peer_ptr) + +#define common_server_peer_via_address(addr, peer_ptr) \ + common_peer_via_address(addr, RPMA_UTIL_IBV_CONTEXT_LOCAL, \ + peer_ptr) + +int common_client_connect(struct rpma_peer *peer, const char *addr, + const char *port, struct rpma_conn_cfg *cfg, + struct rpma_conn_private_data *pdata, + struct rpma_conn **conn_ptr); + +int common_server_accept_connection(struct rpma_ep *ep, + struct rpma_conn_cfg *cfg, struct rpma_conn_private_data *pdata, + struct rpma_conn **conn_ptr); + +int common_wait_for_conn_close_and_disconnect(struct rpma_conn **conn_ptr); +int common_disconnect_and_wait_for_conn_close(struct rpma_conn **conn_ptr); + +#endif /* PMEMHACKATHON_COMMON_H */ diff --git a/examples/R/run_main.sh b/examples/R/run_main.sh new file mode 100755 index 0000000..4656661 --- /dev/null +++ b/examples/R/run_main.sh @@ -0,0 +1,21 @@ +#!/bin/bash -ex +# +# The shell commands to run the main example. +# + +if [ ! -c /${DEV_DAX} ]; then + echo "Error: \"${DEV_DAX}\" device DAX not found (DEV_DAX environment variable)" + exit 1 +fi + +echo "Device DAX: ${DEV_DAX}" + +SERVER_IP=$RPMA_SOFT_ROCE_IP +SERVER_PORT=$((62000 + $PMEMUSER_ID)) + +cd build +# usage: ./server +./server $SERVER_IP $SERVER_PORT $DEV_DAX $PMEMUSER_ID & +sleep 1 +# usage: ./client +./client $SERVER_IP $SERVER_PORT diff --git a/examples/R/run_simple.sh b/examples/R/run_simple.sh new file mode 100755 index 0000000..706e877 --- /dev/null +++ b/examples/R/run_simple.sh @@ -0,0 +1,14 @@ +#!/bin/bash -ex +# +# The shell commands to run the simple_* example. +# + +SERVER_IP=$RPMA_SOFT_ROCE_IP +SERVER_PORT=$((62000 + $PMEMUSER_ID)) + +cd build +# usage: ./simple_server +./simple_server $SERVER_IP $SERVER_PORT & +sleep 1 +# usage: ./simple_client +./simple_client $SERVER_IP $SERVER_PORT diff --git a/examples/R/run_test_ping.sh b/examples/R/run_test_ping.sh new file mode 100755 index 0000000..1242f0b --- /dev/null +++ b/examples/R/run_test_ping.sh @@ -0,0 +1,8 @@ +#!/bin/bash -ex +# +# The shell commands to run the basic ping test. +# + +SERVER_IP=$RPMA_SOFT_ROCE_IP + +ping -c5 $SERVER_IP diff --git a/examples/R/run_test_rping.sh b/examples/R/run_test_rping.sh new file mode 100755 index 0000000..fa98f04 --- /dev/null +++ b/examples/R/run_test_rping.sh @@ -0,0 +1,10 @@ +#!/bin/bash -ex +# +# The shell commands to run the rping test. +# + +SERVER_IP=$RPMA_SOFT_ROCE_IP + +rping -s -a $SERVER_IP -C 10 & +sleep 1 +rping -c -a $SERVER_IP -C 10 -v diff --git a/examples/R/server.c b/examples/R/server.c new file mode 100644 index 0000000..4dd7ac3 --- /dev/null +++ b/examples/R/server.c @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright 2020-2021, Intel Corporation */ + +/* + * server.c -- a server of the "read, write and flush to persistent domain" example + */ + +#include +#include +#include +#include "common.h" + +#define USAGE_STR "usage: %s \n" +#define INIT_STR "This is the initial server string." +#define MAX_USERS 200 +#define PMEM_MIN_SIZE (MAX_USERS * 4 * KILOBYTE) /* 4KiB for each user */ + +#define ERROR_ON_WRONG_USAGE(cond) \ + do { \ + if (cond) { \ + fprintf(stderr, USAGE_STR, argv[0]); \ + exit(-1); \ + } \ + } while (0) + +int +main(int argc, char *argv[]) +{ + int ret; + + /* configure logging thresholds to see more details */ + rpma_log_set_threshold(RPMA_LOG_THRESHOLD, RPMA_LOG_LEVEL_INFO); + rpma_log_set_threshold(RPMA_LOG_THRESHOLD_AUX, RPMA_LOG_LEVEL_INFO); + + /* validate parameters */ + ERROR_ON_WRONG_USAGE(argc < 5); + + /* read parameters */ + char *addr = argv[1]; + char *port = argv[2]; + char *path = argv[3]; + int user_id = atoi(argv[4]); + + ERROR_ON_WRONG_USAGE(user_id < 1 || user_id > MAX_USERS); + + /* resources - PMem */ + char *pmem_ptr; + size_t pmem_size; + int is_pmem; + + /* map the file */ + pmem_ptr = pmem_map_file(path, 0 /* len */, 0 /* flags */, + 0 /* mode */, &pmem_size, &is_pmem); + if (pmem_ptr == NULL) { + (void) fprintf(stderr, "Server: error: pmem_map_file() for %s failed\n", path); + return -1; + } + + /* pmem is expected */ + if (!is_pmem) { + (void) fprintf(stderr, "Server: error: %s is not an actual PMEM\n", path); + (void) pmem_unmap(pmem_ptr, pmem_size); + return -1; + } + + /* check if PMem has minimum required size */ + if (pmem_size < PMEM_MIN_SIZE) { + (void) fprintf(stderr, "Server: error: %s too small (%zu < %u)\n", + path, pmem_size, PMEM_MIN_SIZE); + (void) pmem_unmap(pmem_ptr, pmem_size); + return -1; + } + + /* resources - memory region */ + struct rpma_mr_local *mr = NULL; + /* separate 4KiB of PMem for each user */ + size_t mr_size = 4 * KILOBYTE; + char *mr_ptr = pmem_ptr + (user_id - 1) * mr_size; + + /* RPMA resources */ + struct rpma_peer_cfg *pcfg = NULL; + struct rpma_peer *peer = NULL; + struct rpma_ep *ep = NULL; + struct rpma_conn *conn = NULL; + + /* set and print the initial content */ + strncpy(mr_ptr, INIT_STR, mr_size); + (void) printf("Server: the initial content of the server's persistent memory: %s\n", + mr_ptr); + /* + * Flush the output buffer in order to assure the initial value + * will be really close to the top of the console + * in the case of mixing the client and the server output. + */ + (void) fflush(stdout); + + /* create a peer configuration structure */ + ret = rpma_peer_cfg_new(&pcfg); + if (ret) + goto err_unmap; + + /* configure peer's direct write to pmem support */ + ret = rpma_peer_cfg_set_direct_write_to_pmem(pcfg, true); + if (ret) { + (void) rpma_peer_cfg_delete(&pcfg); + goto err_unmap; + } + + /* + * lookup an ibv_context via the address and create a new peer using it + */ + ret = common_server_peer_via_address(addr, &peer); + if (ret) + goto err_pcfg_delete; + + /* start a listening endpoint at addr:port */ + ret = rpma_ep_listen(peer, addr, port, &ep); + if (ret) + goto err_peer_delete; + + /* register the memory */ + ret = rpma_mr_reg(peer, mr_ptr, mr_size, + RPMA_MR_USAGE_READ_SRC | RPMA_MR_USAGE_WRITE_DST | + (RPMA_MR_USAGE_FLUSH_TYPE_PERSISTENT | + RPMA_MR_USAGE_FLUSH_TYPE_VISIBILITY), + &mr); + if (ret) + goto err_ep_shutdown; + + /* get size of the memory region's descriptor */ + size_t mr_desc_size; + ret = rpma_mr_get_descriptor_size(mr, &mr_desc_size); + if (ret) + goto err_mr_dereg; + + /* get size of the peer config descriptor */ + size_t pcfg_desc_size; + ret = rpma_peer_cfg_get_descriptor_size(pcfg, &pcfg_desc_size); + if (ret) + goto err_mr_dereg; + + /* calculate data for the client write */ + struct common_data data = {0}; + data.mr_desc_size = mr_desc_size; + data.pcfg_desc_size = pcfg_desc_size; + + /* get the memory region's descriptor */ + ret = rpma_mr_get_descriptor(mr, &data.descriptors[0]); + if (ret) + goto err_mr_dereg; + + /* + * Get the peer's configuration descriptor. + * The pcfg_desc descriptor is saved in the `descriptors[]` array + * just after the mr_desc descriptor. + */ + ret = rpma_peer_cfg_get_descriptor(pcfg, + &data.descriptors[mr_desc_size]); + if (ret) + goto err_mr_dereg; + + /* + * Wait for an incoming connection request, accept it and wait for its + * establishment. + */ + struct rpma_conn_private_data pdata; + pdata.ptr = &data; + pdata.len = sizeof(struct common_data); + ret = common_server_accept_connection(ep, NULL, &pdata, &conn); + if (ret) + goto err_mr_dereg; + + /* + * Wait for RPMA_CONN_CLOSED, disconnect and delete the connection + * structure. + */ + ret = common_wait_for_conn_close_and_disconnect(&conn); + if (ret) + goto err_mr_dereg; + + (void) printf("Server: read a new data from the server's persistent memory: %s\n", + mr_ptr); + +err_mr_dereg: + /* deregister the memory region */ + (void) rpma_mr_dereg(&mr); + +err_ep_shutdown: + /* shutdown the endpoint */ + (void) rpma_ep_shutdown(&ep); + +err_peer_delete: + /* delete the peer object */ + (void) rpma_peer_delete(&peer); + +err_pcfg_delete: + (void) rpma_peer_cfg_delete(&pcfg); + +err_unmap: + (void) pmem_unmap(mr_ptr, mr_size); + + return ret; +} diff --git a/examples/R/simple_client.c b/examples/R/simple_client.c new file mode 100644 index 0000000..ce118bf --- /dev/null +++ b/examples/R/simple_client.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright 2021, Intel Corporation */ + +/* + * simple_client.c -- a simple connection client + */ + +#include +#include +#include "common.h" + +#define USAGE_STR "usage: %s \n" + +int +main(int argc, char *argv[]) +{ + /* validate parameters */ + if (argc < 3) { + fprintf(stderr, USAGE_STR, argv[0]); + exit(-1); + } + + /* configure logging thresholds to see more details */ + rpma_log_set_threshold(RPMA_LOG_THRESHOLD, RPMA_LOG_LEVEL_INFO); + rpma_log_set_threshold(RPMA_LOG_THRESHOLD_AUX, RPMA_LOG_LEVEL_INFO); + + /* read common parameters */ + char *addr = argv[1]; + char *port = argv[2]; + int ret; + + /* RPMA resources */ + struct rpma_peer *peer = NULL; + struct rpma_conn *conn = NULL; + + /* + * lookup an ibv_context via the address and create a new peer using it + */ + ret = common_client_peer_via_address(addr, &peer); + if (ret) + return ret; + + /* + * Prepare a connection's private data. + * The length of the private data for rdma_connect() + * in case of RDMA_PS_TCP is limited to 28 bytes. + */ + const char *msg = "Hello server! I'm client :)"; + struct rpma_conn_private_data pdata; + pdata.ptr = (void *)msg; + pdata.len = (strlen(msg) + 1) * sizeof(char); + + /* establish a new connection to a server listening at addr:port */ + ret = common_client_connect(peer, addr, port, NULL, &pdata, &conn); + if (ret) + goto err_peer_delete; + + /* here you can use the newly established connection */ + (void) rpma_conn_get_private_data(conn, &pdata); + if (pdata.ptr) { + char *msg = pdata.ptr; + fprintf(stdout, "Client: received the message: %s\n", msg); + } else { + fprintf(stdout, "Client: no message received\n"); + } + + /* + * Disconnect, wait for RPMA_CONN_CLOSED + * and delete the connection structure. + */ + ret = common_disconnect_and_wait_for_conn_close(&conn); + +err_peer_delete: + /* delete the peer object */ + (void) rpma_peer_delete(&peer); + + return ret; +} diff --git a/examples/R/simple_server.c b/examples/R/simple_server.c new file mode 100644 index 0000000..5dee302 --- /dev/null +++ b/examples/R/simple_server.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright 2021, Intel Corporation */ + +/* + * simple_server.c -- a simple connection server + */ + +#include +#include +#include "common.h" + +#define USAGE_STR "usage: %s \n" + +int +main(int argc, char *argv[]) +{ + /* validate parameters */ + if (argc < 3) { + fprintf(stderr, USAGE_STR, argv[0]); + exit(-1); + } + + /* configure logging thresholds to see more details */ + rpma_log_set_threshold(RPMA_LOG_THRESHOLD, RPMA_LOG_LEVEL_INFO); + rpma_log_set_threshold(RPMA_LOG_THRESHOLD_AUX, RPMA_LOG_LEVEL_INFO); + + /* read common parameters */ + char *addr = argv[1]; + char *port = argv[2]; + int ret; + + /* RPMA resources */ + struct rpma_peer *peer = NULL; + struct rpma_ep *ep = NULL; + struct rpma_conn *conn = NULL; + + /* + * lookup an ibv_context via the address and create a new peer using it + */ + ret = common_server_peer_via_address(addr, &peer); + if (ret) + return ret; + + /* start a listening endpoint at addr:port */ + ret = rpma_ep_listen(peer, addr, port, &ep); + if (ret) + goto err_peer_delete; + + /* + * Prepare a connection's private data. + * The length of the private data for rdma_connect() + * in case of RDMA_PS_TCP is limited to 28 bytes. + */ + const char *msg = "Hello client! I'm server :)"; + struct rpma_conn_private_data pdata; + pdata.ptr = (void *)msg; + pdata.len = (strlen(msg) + 1) * sizeof(char); + + /* + * Wait for an incoming connection request, accept it and wait for its + * establishment. + */ + ret = common_server_accept_connection(ep, NULL, &pdata, &conn); + if (ret) + goto err_ep_shutdown; + + /* here you can use the newly established connection */ + (void) rpma_conn_get_private_data(conn, &pdata); + if (pdata.ptr) { + char *msg = pdata.ptr; + fprintf(stdout, "Server: received the message: %s\n", msg); + } else { + fprintf(stdout, "Server: no message received\n"); + } + + /* + * Wait for RPMA_CONN_CLOSED, disconnect and delete the connection + * structure. + */ + ret = common_wait_for_conn_close_and_disconnect(&conn); + +err_ep_shutdown: + /* shutdown the endpoint */ + (void) rpma_ep_shutdown(&ep); + +err_peer_delete: + /* delete the peer object */ + (void) rpma_peer_delete(&peer); + + return ret; +}