rpma: mmap() memory for rpma_mr_reg() in rpma_flush_apm_new()

The memory region passed to rpma_mr_reg() (ibv_mr_reg())
cannot be allocated from the heap (using malloc()
or posix_memalign()), but should be mapped using mmap().

Rationale:

If ibv_fork_init() was called, the memory region
passed to ibv_mr_reg() is marked by this function
with flag “do not copy on fork”
and after having called fork(), the child process
does not receive this range of virtual addresses.
If this memory region was allocated from the heap,
the child process receives a corrupted heap
with a “hole” of inaccessible addresses inside.
A memory allocator knows nothing about this “hole”
and if it tries to access (read or write)
that range of virtual addresses, it causes a segfault.

Ref: #751

src/flush.c

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: BSD-3-Clause
-/* Copyright 2020, Intel Corporation */
+/* Copyright 2020-2021, Intel Corporation */
 
 /*
  * flush.c -- librpma flush-related implementations
@@ -9,6 +9,7 @@
 #include <stddef.h>
 #include <stdlib.h>
 #include <unistd.h>
+#include <sys/mman.h>
 
 #ifdef TEST_MOCK_ALLOC
 #include "cmocka_alloc.h"
@@ -40,6 +41,7 @@ struct rpma_flush_internal {
 
 struct flush_apm {
 	void *raw; /* buffer for read-after-write memory region */
+	size_t mmap_size; /* size of the mmap()'ed memory */
 	struct rpma_mr_local *raw_mr; /* read-after-write memory region */
 };
 
@@ -51,6 +53,8 @@ struct flush_apm {
 static int
 rpma_flush_apm_new(struct rpma_peer *peer, struct rpma_flush *flush)
 {
+	int ret;
+
 	/* a memory registration has to be page-aligned */
 	long pagesize = sysconf(_SC_PAGESIZE);
 	if (pagesize < 0) {
@@ -59,29 +63,32 @@ rpma_flush_apm_new(struct rpma_peer *peer, struct rpma_flush *flush)
 		return RPMA_E_PROVIDER;
 	}
 
+	size_t mmap_size = (size_t)pagesize;
+
 	/* allocate memory for the read-after-write buffer (RAW) */
-	void *raw = NULL;
-	int ret = posix_memalign(&raw, (size_t)pagesize, RAW_SIZE);
-	if (ret)
+	void *raw = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
+			MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+	if (raw == MAP_FAILED)
 		return RPMA_E_NOMEM;
 
 	/* register the RAW buffer */
 	struct rpma_mr_local *raw_mr = NULL;
 	ret = rpma_mr_reg(peer, raw, RAW_SIZE, RPMA_MR_USAGE_READ_DST, &raw_mr);
 	if (ret) {
-		free(raw);
+		(void) munmap(raw, mmap_size);
 		return ret;
 	}
 
 	struct flush_apm *flush_apm = malloc(sizeof(struct flush_apm));
 	if (flush_apm == NULL) {
 		(void) rpma_mr_dereg(&raw_mr);
-		free(raw);
+		(void) munmap(raw, mmap_size);
 		return RPMA_E_NOMEM;
 	}
 
 	flush_apm->raw = raw;
 	flush_apm->raw_mr = raw_mr;
+	flush_apm->mmap_size = mmap_size;
 
 	struct rpma_flush_internal *flush_internal =
 			(struct rpma_flush_internal *)flush;
@@ -102,12 +109,18 @@ rpma_flush_apm_delete(struct rpma_flush *flush)
 			(struct rpma_flush_internal *)flush;
 	struct flush_apm *flush_apm =
 			(struct flush_apm *)flush_internal->context;
-	int ret = rpma_mr_dereg(&flush_apm->raw_mr);
 
-	free(flush_apm->raw);
+	int ret_dereg = rpma_mr_dereg(&flush_apm->raw_mr);
+	int ret_unmap = munmap(flush_apm->raw, flush_apm->mmap_size);
 	free(flush_apm);
 
-	return ret;
+	if (ret_dereg)
+		return ret_dereg;
+
+	if (ret_unmap)
+		return RPMA_E_INVAL;
+
+	return 0;
 }
 
 /*

src/flush.h

@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause */
-/* Copyright 2020, Intel Corporation */
+/* Copyright 2020-2021, Intel Corporation */
 
 /*
  * flush.h -- librpma flush-related internal definitions
@@ -24,7 +24,7 @@ struct rpma_flush {
  * ERRORS
  * rpma_flush_new() can fail with the following errors:
  *
- * - RPMA_E_NOMEM - out of memory
+ * - RPMA_E_NOMEM - out of memory (mmap() failed)
  * - RPMA_E_PROVIDER - sysconf() or ibv_reg_mr() failed
  */
 int rpma_flush_new(struct rpma_peer *peer, struct rpma_flush **flush_ptr);
@@ -33,7 +33,8 @@ int rpma_flush_new(struct rpma_peer *peer, struct rpma_flush **flush_ptr);
  * ERRORS
  * rpma_flush_delete() can fail with the following error:
  *
- * - RPMA_E_PROVIDER ibv_dereg_mr() failed
+ * - RPMA_E_PROVIDER - ibv_dereg_mr() failed
+ * - RPMA_E_INVAL - munmap() failed
  */
 int rpma_flush_delete(struct rpma_flush **flush_ptr);
 

src/include/librpma.h

@@ -1637,7 +1637,7 @@ int rpma_conn_disconnect(struct rpma_conn *conn);
  *
  * ERRORS
  * rpma_conn_delete() can fail with the following errors:
- * - RPMA_E_INVAL - conn_ptr is NULL
+ * - RPMA_E_INVAL - conn_ptr is NULL or munmap() failed
  * - RPMA_E_PROVIDER - ibv_destroy_cq() or rdma_destroy_id() failed
  *
  * SEE ALSO

tests/integration/common/mocks.c

@@ -1,12 +1,14 @@
 // SPDX-License-Identifier: BSD-3-Clause
-/* Copyright 2020, Intel Corporation */
+/* Copyright 2020-2021, Intel Corporation */
 
 /*
  * mocks.c -- common mocks for integration tests
  */
 
 #include <string.h>
 #include <librpma.h>
+#include <sys/types.h>
+#include <sys/mman.h>
 
 #include "cmocka_headers.h"
 #include "conn_cfg.h"
@@ -758,3 +760,47 @@ create_descriptor(void *desc,
 
 	return 0;
 }
+
+/*
+ * __wrap_mmap -- mmap() mock
+ */
+void *
+__wrap_mmap(void *__addr, size_t __len, int __prot,
+		int __flags, int __fd, off_t __offset)
+{
+	void *ret = mock_type(void *);
+	if (ret != (void *)MOCK_OK)
+		return MAP_FAILED;
+
+	struct mmap_args *args = mock_type(struct mmap_args *);
+
+	void *memptr = __real__test_malloc(__len);
+
+	/*
+	 * Save the address and length of the allocated memory
+	 * in order to verify it later.
+	 */
+	args->addr = memptr;
+	args->len = __len;
+
+	return memptr;
+}
+
+/*
+ * __wrap_munmap -- munmap() mock
+ */
+int
+__wrap_munmap(void *__addr, size_t __len)
+{
+	struct mmap_args *args = mock_type(struct mmap_args *);
+	assert_ptr_equal(__addr, args->addr);
+	assert_int_equal(__len, args->len);
+
+	test_free(__addr);
+
+	errno = mock_type(int);
+	if (errno)
+		return -1;
+
+	return 0;
+}

tests/integration/common/mocks.h

@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause */
-/* Copyright 2020, Intel Corporation */
+/* Copyright 2020-2021, Intel Corporation */
 
 /*
  * mocks.h -- common mocks for integration tests
@@ -62,6 +62,11 @@ struct posix_memalign_args {
 	void *ptr;
 };
 
+struct mmap_args {
+	void *addr;
+	size_t len;
+};
+
 #ifdef ON_DEMAND_PAGING_SUPPORTED
 int ibv_query_device_ex_mock(struct ibv_context *context,
 		const struct ibv_query_device_ex_input *input,

tests/integration/example-01-connection/CMakeLists.txt

@@ -1,6 +1,6 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2020, Intel Corporation
+# Copyright 2020-2021, Intel Corporation
 #
 
 include(../../cmake/ctest_helpers.cmake)
@@ -34,7 +34,7 @@ target_include_directories(${TARGET} PRIVATE
 
 set_target_properties(${TARGET}
        PROPERTIES
-       LINK_FLAGS "-Wl,--wrap=_test_malloc,--wrap=posix_memalign,--wrap=fprintf")
+       LINK_FLAGS "-Wl,--wrap=_test_malloc,--wrap=mmap,--wrap=munmap,--wrap=fprintf")
 
 target_compile_definitions(${TARGET}
        PUBLIC TEST_MOCK_MAIN

tests/integration/example-01-connection/example-01-connection.c

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: BSD-3-Clause
-/* Copyright 2020, Intel Corporation */
+/* Copyright 2020-2021, Intel Corporation */
 
 /*
  * example-01-connection.c -- connection integration tests
@@ -92,13 +92,14 @@ test_client__success(void **unused)
 	expect_value(rdma_migrate_id, channel, MOCK_EVCH);
 	will_return(rdma_migrate_id, MOCK_OK);
 
-	struct posix_memalign_args allocated_raw = {0};
-	will_return(__wrap_posix_memalign, &allocated_raw);
+	struct mmap_args allocated_raw = {0};
+	will_return(__wrap_mmap, MOCK_OK);
+	will_return(__wrap_mmap, &allocated_raw);
 
 	expect_value(ibv_reg_mr, pd, MOCK_IBV_PD);
 	expect_value(ibv_reg_mr, length, MOCK_RAW_SIZE);
 	expect_value(ibv_reg_mr, access, IBV_ACCESS_LOCAL_WRITE);
-	will_return(ibv_reg_mr, &allocated_raw.ptr);
+	will_return(ibv_reg_mr, &allocated_raw.addr);
 	will_return(ibv_reg_mr, MOCK_MR);
 
 	expect_value(rdma_connect, id, &id);
@@ -133,6 +134,8 @@ test_client__success(void **unused)
 	/* configure mocks for rpma_conn_delete() */
 	expect_value(ibv_dereg_mr, mr, MOCK_MR);
 	will_return(ibv_dereg_mr, MOCK_OK);
+	will_return(__wrap_munmap, &allocated_raw);
+	will_return(__wrap_munmap, MOCK_OK);
 
 	expect_value(rdma_destroy_qp, id, &id);
 
@@ -244,13 +247,14 @@ test_server__success(void **unused)
 	expect_value(rdma_migrate_id, channel, MOCK_EVCH);
 	will_return(rdma_migrate_id, MOCK_OK);
 
-	struct posix_memalign_args allocated_raw = {0};
-	will_return(__wrap_posix_memalign, &allocated_raw);
+	struct mmap_args allocated_raw = {0};
+	will_return(__wrap_mmap, MOCK_OK);
+	will_return(__wrap_mmap, &allocated_raw);
 
 	expect_value(ibv_reg_mr, pd, MOCK_IBV_PD);
 	expect_value(ibv_reg_mr, length, MOCK_RAW_SIZE);
 	expect_value(ibv_reg_mr, access, IBV_ACCESS_LOCAL_WRITE);
-	will_return(ibv_reg_mr, &allocated_raw.ptr);
+	will_return(ibv_reg_mr, &allocated_raw.addr);
 	will_return(ibv_reg_mr, MOCK_MR);
 
 	/* configure mocks for rpma_conn_next_event() */
@@ -283,6 +287,8 @@ test_server__success(void **unused)
 	/* configure mocks for rpma_conn_delete() */
 	expect_value(ibv_dereg_mr, mr, MOCK_MR);
 	will_return(ibv_dereg_mr, MOCK_OK);
+	will_return(__wrap_munmap, &allocated_raw);
+	will_return(__wrap_munmap, MOCK_OK);
 
 	expect_value(rdma_destroy_qp, id, &id);
 

tests/integration/example-02-read-to-volatile/CMakeLists.txt

@@ -1,6 +1,6 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2020, Intel Corporation
+# Copyright 2020-2021, Intel Corporation
 #
 
 include(../../cmake/ctest_helpers.cmake)
@@ -35,7 +35,7 @@ target_include_directories(${TARGET} PRIVATE
 
 set_target_properties(${TARGET}
        PROPERTIES
-       LINK_FLAGS "-Wl,--wrap=_test_malloc,--wrap=posix_memalign,--wrap=fprintf")
+       LINK_FLAGS "-Wl,--wrap=_test_malloc,--wrap=posix_memalign,--wrap=fprintf,--wrap=mmap,--wrap=munmap")
 
 target_compile_definitions(${TARGET}
        PUBLIC TEST_MOCK_MAIN

tests/integration/example-02-read-to-volatile/example-02-read-to-volatile.c

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: BSD-3-Clause
-/* Copyright 2020, Intel Corporation */
+/* Copyright 2020-2021, Intel Corporation */
 
 /*
  * example-02-read-to-volatile.c -- 'read to volatile' integration tests
@@ -107,13 +107,14 @@ test_client__success(void **unused)
 	expect_value(rdma_migrate_id, channel, MOCK_EVCH);
 	will_return(rdma_migrate_id, MOCK_OK);
 
-	struct posix_memalign_args allocated_raw = {0};
-	will_return(__wrap_posix_memalign, &allocated_raw);
+	struct mmap_args allocated_raw = {0};
+	will_return(__wrap_mmap, MOCK_OK);
+	will_return(__wrap_mmap, &allocated_raw);
 
 	expect_value(ibv_reg_mr, pd, MOCK_IBV_PD);
 	expect_value(ibv_reg_mr, length, MOCK_RAW_SIZE);
 	expect_value(ibv_reg_mr, access, IBV_ACCESS_LOCAL_WRITE);
-	will_return(ibv_reg_mr, &allocated_raw.ptr);
+	will_return(ibv_reg_mr, &allocated_raw.addr);
 	will_return(ibv_reg_mr, MOCK_MR_RAW);
 
 	expect_value(rdma_connect, id, &Cm_id);
@@ -180,6 +181,8 @@ test_client__success(void **unused)
 	/* configure mocks for rpma_conn_delete() */
 	expect_value(ibv_dereg_mr, mr, MOCK_MR_RAW);
 	will_return(ibv_dereg_mr, MOCK_OK);
+	will_return(__wrap_munmap, &allocated_raw);
+	will_return(__wrap_munmap, MOCK_OK);
 
 	expect_value(rdma_destroy_qp, id, &Cm_id);
 
@@ -304,13 +307,14 @@ test_server__success(void **unused)
 	expect_value(rdma_migrate_id, channel, MOCK_EVCH);
 	will_return(rdma_migrate_id, MOCK_OK);
 
-	struct posix_memalign_args allocated_raw = {0};
-	will_return(__wrap_posix_memalign, &allocated_raw);
+	struct mmap_args allocated_raw = {0};
+	will_return(__wrap_mmap, MOCK_OK);
+	will_return(__wrap_mmap, &allocated_raw);
 
 	expect_value(ibv_reg_mr, pd, MOCK_IBV_PD);
 	expect_value(ibv_reg_mr, length, MOCK_RAW_SIZE);
 	expect_value(ibv_reg_mr, access, IBV_ACCESS_LOCAL_WRITE);
-	will_return(ibv_reg_mr, &allocated_raw.ptr);
+	will_return(ibv_reg_mr, &allocated_raw.addr);
 	will_return(ibv_reg_mr, MOCK_MR_RAW);
 
 	/* configure mocks for rpma_conn_next_event() */
@@ -343,6 +347,8 @@ test_server__success(void **unused)
 	/* configure mocks for rpma_conn_delete() */
 	expect_value(ibv_dereg_mr, mr, MOCK_MR_RAW);
 	will_return(ibv_dereg_mr, MOCK_OK);
+	will_return(__wrap_munmap, &allocated_raw);
+	will_return(__wrap_munmap, MOCK_OK);
 
 	expect_value(rdma_destroy_qp, id, &Cm_id);
 

tests/integration/example-04-write-to-persistent/CMakeLists.txt

@@ -1,6 +1,6 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2020, Intel Corporation
+# Copyright 2020-2021, Intel Corporation
 #
 
 include(../../cmake/ctest_helpers.cmake)
@@ -35,7 +35,7 @@ target_include_directories(${TARGET} PRIVATE
 
 set_target_properties(${TARGET}
        PROPERTIES
-       LINK_FLAGS "-Wl,--wrap=_test_malloc,--wrap=posix_memalign,--wrap=fprintf")
+       LINK_FLAGS "-Wl,--wrap=_test_malloc,--wrap=posix_memalign,--wrap=fprintf,--wrap=mmap,--wrap=munmap")
 
 target_compile_definitions(${TARGET}
        PUBLIC TEST_MOCK_MAIN