Skip to content
This repository has been archived by the owner on Sep 5, 2023. It is now read-only.

Commit

Permalink
rpma: mmap() memory for rpma_mr_reg() in rpma_flush_apm_new()
Browse files Browse the repository at this point in the history
Ref: #751
  • Loading branch information
ldorau committed Feb 10, 2021
1 parent b48194c commit d9217ed
Show file tree
Hide file tree
Showing 16 changed files with 189 additions and 55 deletions.
31 changes: 22 additions & 9 deletions src/flush.c
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// SPDX-License-Identifier: BSD-3-Clause
/* Copyright 2020, Intel Corporation */
/* Copyright 2020-2021, Intel Corporation */

/*
* flush.c -- librpma flush-related implementations
Expand All @@ -9,6 +9,7 @@
#include <stddef.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/mman.h>

#ifdef TEST_MOCK_ALLOC
#include "cmocka_alloc.h"
Expand Down Expand Up @@ -40,6 +41,7 @@ struct rpma_flush_internal {

struct flush_apm {
void *raw; /* buffer for read-after-write memory region */
size_t mmap_size; /* size of the mmap()'ed memory */
struct rpma_mr_local *raw_mr; /* read-after-write memory region */
};

Expand All @@ -51,6 +53,8 @@ struct flush_apm {
static int
rpma_flush_apm_new(struct rpma_peer *peer, struct rpma_flush *flush)
{
int ret;

/* a memory registration has to be page-aligned */
long pagesize = sysconf(_SC_PAGESIZE);
if (pagesize < 0) {
Expand All @@ -59,29 +63,32 @@ rpma_flush_apm_new(struct rpma_peer *peer, struct rpma_flush *flush)
return RPMA_E_PROVIDER;
}

size_t mmap_size = (size_t)pagesize;

/* allocate memory for the read-after-write buffer (RAW) */
void *raw = NULL;
int ret = posix_memalign(&raw, (size_t)pagesize, RAW_SIZE);
if (ret)
void *raw = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (raw == MAP_FAILED)
return RPMA_E_NOMEM;

/* register the RAW buffer */
struct rpma_mr_local *raw_mr = NULL;
ret = rpma_mr_reg(peer, raw, RAW_SIZE, RPMA_MR_USAGE_READ_DST, &raw_mr);
if (ret) {
free(raw);
(void) munmap(raw, mmap_size);
return ret;
}

struct flush_apm *flush_apm = malloc(sizeof(struct flush_apm));
if (flush_apm == NULL) {
(void) rpma_mr_dereg(&raw_mr);
free(raw);
(void) munmap(raw, mmap_size);
return RPMA_E_NOMEM;
}

flush_apm->raw = raw;
flush_apm->raw_mr = raw_mr;
flush_apm->mmap_size = mmap_size;

struct rpma_flush_internal *flush_internal =
(struct rpma_flush_internal *)flush;
Expand All @@ -102,12 +109,18 @@ rpma_flush_apm_delete(struct rpma_flush *flush)
(struct rpma_flush_internal *)flush;
struct flush_apm *flush_apm =
(struct flush_apm *)flush_internal->context;
int ret = rpma_mr_dereg(&flush_apm->raw_mr);

free(flush_apm->raw);
int ret_dereg = rpma_mr_dereg(&flush_apm->raw_mr);
int ret_unmap = munmap(flush_apm->raw, flush_apm->mmap_size);
free(flush_apm);

return ret;
if (ret_dereg)
return ret_dereg;

if (ret_unmap)
return RPMA_E_INVAL;

return 0;
}

/*
Expand Down
7 changes: 4 additions & 3 deletions src/flush.h
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright 2020, Intel Corporation */
/* Copyright 2020-2021, Intel Corporation */

/*
* flush.h -- librpma flush-related internal definitions
Expand All @@ -24,7 +24,7 @@ struct rpma_flush {
* ERRORS
* rpma_flush_new() can fail with the following errors:
*
* - RPMA_E_NOMEM - out of memory
* - RPMA_E_NOMEM - out of memory (mmap() failed)
* - RPMA_E_PROVIDER - sysconf() or ibv_reg_mr() failed
*/
int rpma_flush_new(struct rpma_peer *peer, struct rpma_flush **flush_ptr);
Expand All @@ -33,7 +33,8 @@ int rpma_flush_new(struct rpma_peer *peer, struct rpma_flush **flush_ptr);
* ERRORS
* rpma_flush_delete() can fail with the following error:
*
* - RPMA_E_PROVIDER ibv_dereg_mr() failed
* - RPMA_E_PROVIDER - ibv_dereg_mr() failed
* - RPMA_E_INVAL - munmap() failed
*/
int rpma_flush_delete(struct rpma_flush **flush_ptr);

Expand Down
38 changes: 37 additions & 1 deletion tests/integration/common/mocks.c
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
// SPDX-License-Identifier: BSD-3-Clause
/* Copyright 2020, Intel Corporation */
/* Copyright 2020-2021, Intel Corporation */

/*
* mocks.c -- common mocks for integration tests
*/

#include <string.h>
#include <librpma.h>
#include <sys/types.h>
#include <sys/mman.h>

#include "cmocka_headers.h"
#include "conn_cfg.h"
Expand Down Expand Up @@ -758,3 +760,37 @@ create_descriptor(void *desc,

return 0;
}

/*
* __wrap_mmap -- mmap() mock
*/
void *
__wrap_mmap(void *__addr, size_t __len, int __prot,
int __flags, int __fd, off_t __offset)
{
void *err = mock_type(void *);
if (err == MAP_FAILED)
return MAP_FAILED;

struct mmap_args *args = mock_type(struct mmap_args *);

void *memptr = __real__test_malloc(__len);

/* save the address of the allocated memory to verify it later */
args->ptr = memptr;

return memptr;
}

/*
* __wrap_munmap -- munmap() mock
*/
int
__wrap_munmap(void *__addr, size_t __len)
{
(void) __len; /* unsused */

test_free(__addr);

return 0;
}
6 changes: 5 additions & 1 deletion tests/integration/common/mocks.h
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright 2020, Intel Corporation */
/* Copyright 2020-2021, Intel Corporation */

/*
* mocks.h -- common mocks for integration tests
Expand Down Expand Up @@ -62,6 +62,10 @@ struct posix_memalign_args {
void *ptr;
};

struct mmap_args {
void *ptr;
};

#ifdef ON_DEMAND_PAGING_SUPPORTED
int ibv_query_device_ex_mock(struct ibv_context *context,
const struct ibv_query_device_ex_input *input,
Expand Down
4 changes: 2 additions & 2 deletions tests/integration/example-01-connection/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#
# SPDX-License-Identifier: BSD-3-Clause
# Copyright 2020, Intel Corporation
# Copyright 2020-2021, Intel Corporation
#

include(../../cmake/ctest_helpers.cmake)
Expand Down Expand Up @@ -34,7 +34,7 @@ target_include_directories(${TARGET} PRIVATE

set_target_properties(${TARGET}
PROPERTIES
LINK_FLAGS "-Wl,--wrap=_test_malloc,--wrap=posix_memalign,--wrap=fprintf")
LINK_FLAGS "-Wl,--wrap=_test_malloc,--wrap=mmap,--wrap=munmap,--wrap=fprintf")

target_compile_definitions(${TARGET}
PUBLIC TEST_MOCK_MAIN
Expand Down
12 changes: 7 additions & 5 deletions tests/integration/example-01-connection/example-01-connection.c
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// SPDX-License-Identifier: BSD-3-Clause
/* Copyright 2020, Intel Corporation */
/* Copyright 2020-2021, Intel Corporation */

/*
* example-01-connection.c -- connection integration tests
Expand Down Expand Up @@ -92,8 +92,9 @@ test_client__success(void **unused)
expect_value(rdma_migrate_id, channel, MOCK_EVCH);
will_return(rdma_migrate_id, MOCK_OK);

struct posix_memalign_args allocated_raw = {0};
will_return(__wrap_posix_memalign, &allocated_raw);
struct mmap_args allocated_raw = {0};
will_return(__wrap_mmap, &allocated_raw);
will_return(__wrap_mmap, &allocated_raw);

expect_value(ibv_reg_mr, pd, MOCK_IBV_PD);
expect_value(ibv_reg_mr, length, MOCK_RAW_SIZE);
Expand Down Expand Up @@ -244,8 +245,9 @@ test_server__success(void **unused)
expect_value(rdma_migrate_id, channel, MOCK_EVCH);
will_return(rdma_migrate_id, MOCK_OK);

struct posix_memalign_args allocated_raw = {0};
will_return(__wrap_posix_memalign, &allocated_raw);
struct mmap_args allocated_raw = {0};
will_return(__wrap_mmap, &allocated_raw);
will_return(__wrap_mmap, &allocated_raw);

expect_value(ibv_reg_mr, pd, MOCK_IBV_PD);
expect_value(ibv_reg_mr, length, MOCK_RAW_SIZE);
Expand Down
4 changes: 2 additions & 2 deletions tests/integration/example-02-read-to-volatile/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#
# SPDX-License-Identifier: BSD-3-Clause
# Copyright 2020, Intel Corporation
# Copyright 2020-2021, Intel Corporation
#

include(../../cmake/ctest_helpers.cmake)
Expand Down Expand Up @@ -35,7 +35,7 @@ target_include_directories(${TARGET} PRIVATE

set_target_properties(${TARGET}
PROPERTIES
LINK_FLAGS "-Wl,--wrap=_test_malloc,--wrap=posix_memalign,--wrap=fprintf")
LINK_FLAGS "-Wl,--wrap=_test_malloc,--wrap=posix_memalign,--wrap=fprintf,--wrap=mmap,--wrap=munmap")

target_compile_definitions(${TARGET}
PUBLIC TEST_MOCK_MAIN
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// SPDX-License-Identifier: BSD-3-Clause
/* Copyright 2020, Intel Corporation */
/* Copyright 2020-2021, Intel Corporation */

/*
* example-02-read-to-volatile.c -- 'read to volatile' integration tests
Expand Down Expand Up @@ -107,8 +107,9 @@ test_client__success(void **unused)
expect_value(rdma_migrate_id, channel, MOCK_EVCH);
will_return(rdma_migrate_id, MOCK_OK);

struct posix_memalign_args allocated_raw = {0};
will_return(__wrap_posix_memalign, &allocated_raw);
struct mmap_args allocated_raw = {0};
will_return(__wrap_mmap, &allocated_raw);
will_return(__wrap_mmap, &allocated_raw);

expect_value(ibv_reg_mr, pd, MOCK_IBV_PD);
expect_value(ibv_reg_mr, length, MOCK_RAW_SIZE);
Expand Down Expand Up @@ -304,8 +305,9 @@ test_server__success(void **unused)
expect_value(rdma_migrate_id, channel, MOCK_EVCH);
will_return(rdma_migrate_id, MOCK_OK);

struct posix_memalign_args allocated_raw = {0};
will_return(__wrap_posix_memalign, &allocated_raw);
struct mmap_args allocated_raw = {0};
will_return(__wrap_mmap, &allocated_raw);
will_return(__wrap_mmap, &allocated_raw);

expect_value(ibv_reg_mr, pd, MOCK_IBV_PD);
expect_value(ibv_reg_mr, length, MOCK_RAW_SIZE);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#
# SPDX-License-Identifier: BSD-3-Clause
# Copyright 2020, Intel Corporation
# Copyright 2020-2021, Intel Corporation
#

include(../../cmake/ctest_helpers.cmake)
Expand Down Expand Up @@ -35,7 +35,7 @@ target_include_directories(${TARGET} PRIVATE

set_target_properties(${TARGET}
PROPERTIES
LINK_FLAGS "-Wl,--wrap=_test_malloc,--wrap=posix_memalign,--wrap=fprintf")
LINK_FLAGS "-Wl,--wrap=_test_malloc,--wrap=posix_memalign,--wrap=fprintf,--wrap=mmap,--wrap=munmap")

target_compile_definitions(${TARGET}
PUBLIC TEST_MOCK_MAIN
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// SPDX-License-Identifier: BSD-3-Clause
/* Copyright 2020, Intel Corporation */
/* Copyright 2020-2021, Intel Corporation */

/*
* example-04-write-to-persistent.c -- 'write to persistent' integration tests
Expand Down Expand Up @@ -111,8 +111,9 @@ test_client__success(void **unused)
will_return(rdma_migrate_id, MOCK_OK);

/* allocate memory for the rpma_flush_apm_new */
struct posix_memalign_args flush = {0};
will_return(__wrap_posix_memalign, &flush);
struct mmap_args flush = {0};
will_return(__wrap_mmap, &flush);
will_return(__wrap_mmap, &flush);

expect_value(ibv_reg_mr, pd, MOCK_IBV_PD);
expect_value(ibv_reg_mr, length, MOCK_RAW_SIZE);
Expand Down Expand Up @@ -341,8 +342,9 @@ test_server__success(void **unused)
expect_value(rdma_migrate_id, channel, MOCK_EVCH);
will_return(rdma_migrate_id, MOCK_OK);

struct posix_memalign_args allocated_raw = {0};
will_return(__wrap_posix_memalign, &allocated_raw);
struct mmap_args allocated_raw = {0};
will_return(__wrap_mmap, &allocated_raw);
will_return(__wrap_mmap, &allocated_raw);

expect_value(ibv_reg_mr, pd, MOCK_IBV_PD);
expect_value(ibv_reg_mr, length, MOCK_RAW_SIZE);
Expand Down
38 changes: 37 additions & 1 deletion tests/unit/common/mocks-stdlib.c
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
// SPDX-License-Identifier: BSD-3-Clause
/* Copyright 2020, Intel Corporation */
/* Copyright 2020-2021, Intel Corporation */

/*
* mocks-stdlib.c -- stdlib mocks
*/

#include <errno.h>
#include <stddef.h>
#include <sys/types.h>
#include <sys/mman.h>

#include "cmocka_headers.h"
#include "mocks-stdlib.h"
Expand Down Expand Up @@ -47,3 +49,37 @@ __wrap_posix_memalign(void **memptr, size_t alignment, size_t size)

return 0;
}

/*
* __wrap_mmap -- mmap() mock
*/
void *
__wrap_mmap(void *__addr, size_t __len, int __prot,
int __flags, int __fd, off_t __offset)
{
void *err = mock_type(void *);
if (err == MAP_FAILED)
return MAP_FAILED;

struct mmap_args *args = mock_type(struct mmap_args *);

void *memptr = __real__test_malloc(__len);

/* save the address of the allocated memory to verify it later */
args->ptr = memptr;

return memptr;
}

/*
* __wrap_munmap -- munmap() mock
*/
int
__wrap_munmap(void *__addr, size_t __len)
{
(void) __len; /* unsused */

test_free(__addr);

return 0;
}
Loading

0 comments on commit d9217ed

Please sign in to comment.