From 81e7eb3fd15f63bb99b64e85796829415943390c Mon Sep 17 00:00:00 2001 From: Piotr Balcer Date: Fri, 15 Feb 2019 09:14:00 +0100 Subject: [PATCH 1/2] storage: introduce shm/pmem backed datapool abstraction This patch creates a datapool abstraction for managing large amounts of contiguous memory, and adds two implementations: - shm, backed by shared memory allocated through cc_alloc - pmem, backed by memory-mapped file managed through libpmem The default implementation, datapool_shm, currently does not differ from simply managing memory through cc_alloc/cc_free. The datapool_pmem implementation, which can be enabled through -DUSE_PMEM=ON, allows for durable (persistent) implementations of the storage layer as it attempts to recover the previous content of the managed memory. --- CMakeLists.txt | 13 ++ deps/ccommon/include/cc_mm.h | 4 + deps/ccommon/src/cc_mm.c | 11 ++ src/CMakeLists.txt | 1 + src/datapool/CMakeLists.txt | 9 ++ src/datapool/datapool.h | 12 ++ src/datapool/datapool_pmem.c | 227 +++++++++++++++++++++++++++++++++ src/datapool/datapool_shm.c | 43 +++++++ test/CMakeLists.txt | 3 + test/datapool/CMakeLists.txt | 12 ++ test/datapool/check_datapool.c | 89 +++++++++++++ 11 files changed, 424 insertions(+) create mode 100644 src/datapool/CMakeLists.txt create mode 100644 src/datapool/datapool.h create mode 100644 src/datapool/datapool_pmem.c create mode 100644 src/datapool/datapool_shm.c create mode 100644 test/datapool/CMakeLists.txt create mode 100644 test/datapool/check_datapool.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 4fc9f9097..dc2795aec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,6 +47,7 @@ option(TARGET_RESPCLI "build resp-cli binary" ON) option(HAVE_RUST "build features written in rust" OFF) option(RUST_USE_MUSL "build rust deps against musl" OFF) option(BUILD_AND_INSTALL_CHECK "build our own version of check and link against it" OFF) +option(USE_PMEM "build persistent memory features" OFF) option(COVERAGE "code coverage" OFF) @@ -126,7 +127,10 @@ add_subdirectory(${CCOMMON_SOURCE_DIR} ${PROJECT_BINARY_DIR}/ccommon) include(FindPackageHandleStandardArgs) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake") +find_package(PkgConfig QUIET) + find_package(Check) + if(NOT CHECK_FOUND) message(WARNING "Check is required to build and run tests") endif(NOT CHECK_FOUND) @@ -137,6 +141,15 @@ if(CHECK_FOUND) endif(NOT CHECK_WORKING) endif(CHECK_FOUND) +if (USE_PMEM) + if(PKG_CONFIG_FOUND) + pkg_check_modules(LIBPMEM REQUIRED libpmem>=1.0) + else() + find_package(LIBPMEM REQUIRED 1.0) + endif() + link_directories(${LIBPMEM_LIBRARY_DIRS}) +endif(USE_PMEM) + find_package(Threads) if(TARGET_CDB) diff --git a/deps/ccommon/include/cc_mm.h b/deps/ccommon/include/cc_mm.h index 2d982196e..a9effa23b 100644 --- a/deps/ccommon/include/cc_mm.h +++ b/deps/ccommon/include/cc_mm.h @@ -66,6 +66,9 @@ extern "C" { #define cc_munmap(_p, _s) \ _cc_munmap(_p, (size_t)(_s), __FILE__, __LINE__) +#define cc_alloc_usable_size(_p) \ + _cc_alloc_usable_size(_p, __FILE__, __LINE__) + void * _cc_alloc(size_t size, const char *name, int line); void * _cc_zalloc(size_t size, const char *name, int line); void * _cc_calloc(size_t nmemb, size_t size, const char *name, int line); @@ -74,6 +77,7 @@ void * _cc_realloc_move(void *ptr, size_t size, const char *name, int line); void _cc_free(void *ptr, const char *name, int line); void * _cc_mmap(size_t size, const char *name, int line); int _cc_munmap(void *p, size_t size, const char *name, int line); +size_t _cc_alloc_usable_size(void *ptr, const char *name, int line); #ifdef __cplusplus } diff --git a/deps/ccommon/src/cc_mm.c b/deps/ccommon/src/cc_mm.c index 6a0554289..ef4ed349a 100644 --- a/deps/ccommon/src/cc_mm.c +++ b/deps/ccommon/src/cc_mm.c @@ -27,6 +27,10 @@ /* TODO(yao): detect OS in one place and use one variable everywhere */ #if defined(__APPLE__) && defined(__MACH__) # define MAP_ANONYMOUS MAP_ANON +#include +#define malloc_usable_size malloc_size +#else +#include #endif void * @@ -167,3 +171,10 @@ _cc_munmap(void *p, size_t size, const char *name, int line) return status; } + +size_t +_cc_alloc_usable_size(void *ptr, const char *name, int line) +{ + log_vverb("malloc_usable_size(%p) @ %s:%d", ptr, name, line); + return malloc_usable_size(ptr); +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8c85de1e2..2e1203334 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,6 +5,7 @@ add_subdirectory(protocol ${PROJECT_BINARY_DIR}/protocol) add_subdirectory(storage ${PROJECT_BINARY_DIR}/storage) add_subdirectory(time ${PROJECT_BINARY_DIR}/time) add_subdirectory(util ${PROJECT_BINARY_DIR}/util) +add_subdirectory(datapool ${PROJECT_BINARY_DIR}/datapool) # executables add_custom_target(service) diff --git a/src/datapool/CMakeLists.txt b/src/datapool/CMakeLists.txt new file mode 100644 index 000000000..4148d3e6d --- /dev/null +++ b/src/datapool/CMakeLists.txt @@ -0,0 +1,9 @@ +add_library(datapool datapool.h) + +if(USE_PMEM) + target_sources(datapool PRIVATE datapool_pmem.c) + target_include_directories(datapool PRIVATE ${LIBPMEM_INCLUDE_DIRS}) + target_link_libraries(datapool ${LIBPMEM_LIBRARIES}) +else() + target_sources(datapool PRIVATE datapool_shm.c) +endif(USE_PMEM) diff --git a/src/datapool/datapool.h b/src/datapool/datapool.h new file mode 100644 index 000000000..02951f2db --- /dev/null +++ b/src/datapool/datapool.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +struct datapool; + +struct datapool *datapool_open(const char *path, size_t size, + int *fresh); +void datapool_close(struct datapool *pool); + +void *datapool_addr(struct datapool *pool); +size_t datapool_size(struct datapool *pool); diff --git a/src/datapool/datapool_pmem.c b/src/datapool/datapool_pmem.c new file mode 100644 index 000000000..cb4fd72bc --- /dev/null +++ b/src/datapool/datapool_pmem.c @@ -0,0 +1,227 @@ +/* + * File-backed datapool. + * Retains its contents if the pool has been closed correctly. + * + */ +#include "datapool.h" + +#include +#include + +#include +#include +#include + +#define DATAPOOL_SIGNATURE ("PELIKAN") /* 8 bytes */ +#define DATAPOOL_SIGNATURE_LEN (sizeof(DATAPOOL_SIGNATURE)) + +/* + * Size of the data pool header. + * Big enough to fit all necessary metadata, but most of this size is left + * unused for future expansion. + */ +#define DATAPOOL_HEADER_LEN 4096 +#define DATAPOOL_VERSION 1 + +#define DATAPOOL_FLAG_DIRTY (1 << 0) +#define DATAPOOL_VALID_FLAGS (DATAPOOL_FLAG_DIRTY) + +/* + * Header at the beginning of the file, it's verified every time the pool is + * opened. + */ +struct datapool_header { + uint8_t signature[DATAPOOL_SIGNATURE_LEN]; + uint64_t version; + uint64_t size; + uint64_t flags; + uint8_t unused[DATAPOOL_HEADER_LEN - 32]; +}; + +struct datapool { + void *addr; + + struct datapool_header *hdr; + void *user_addr; + size_t mapped_len; + int is_pmem; + int file_backed; +}; + +static void +datapool_sync_hdr(struct datapool *pool) +{ + int ret = pmem_msync(pool->hdr, DATAPOOL_HEADER_LEN); + ASSERT(ret == 0); +} + +static void +datapool_sync(struct datapool *pool) +{ + int ret = pmem_msync(pool->addr, pool->mapped_len); + ASSERT(ret == 0); +} + +static bool +datapool_valid(struct datapool *pool) +{ + if (memcmp(pool->hdr->signature, + DATAPOOL_SIGNATURE, DATAPOOL_SIGNATURE_LEN) != 0) { + log_info("no signature found in datapool"); + return false; + } + + if (pool->hdr->version != DATAPOOL_VERSION) { + log_info("incompatible datapool version (is: %d, expecting: %d)", + pool->hdr->version, DATAPOOL_SIGNATURE); + return false; + } + + if (pool->hdr->size == 0) { + log_error("datapool has 0 size"); + return false; + } + + if (pool->hdr->size > pool->mapped_len) { + log_error("datapool has invalid size (is: %d, expecting: %d)", + pool->mapped_len, pool->hdr->size); + return false; + } + + if (pool->hdr->flags & ~DATAPOOL_VALID_FLAGS) { + log_error("datapool has invalid flags set"); + return false; + } + + if (pool->hdr->flags & DATAPOOL_FLAG_DIRTY) { + log_info("datapool has a valid header but is dirty"); + return false; + } + + return true; +} + +static void +datapool_initialize(struct datapool *pool) +{ + log_info("initializing fresh datapool"); + + /* 1. clear the header from any leftovers */ + memset(pool->hdr, 0, DATAPOOL_HEADER_LEN); + datapool_sync_hdr(pool); + + /* 2. fill in the data */ + pool->hdr->version = DATAPOOL_VERSION; + pool->hdr->size = pool->mapped_len; + pool->hdr->flags = 0; + datapool_sync_hdr(pool); + + /* 3. set the signature */ + memcpy(pool->hdr->signature, DATAPOOL_SIGNATURE, DATAPOOL_SIGNATURE_LEN); + datapool_sync_hdr(pool); +} + +static void +datapool_flag_set(struct datapool *pool, int flag) +{ + pool->hdr->flags |= flag; + datapool_sync_hdr(pool); +} + +static void +datapool_flag_clear(struct datapool *pool, int flag) +{ + pool->hdr->flags &= ~flag; + datapool_sync_hdr(pool); +} + +/* + * Opens, and if necessary initializes, a datapool that resides in the given + * file. If no file is provided, the pool is allocated through cc_zalloc. + * + * The the datapool to retain its contents, the datapool_close() call must + * finish successfully. + */ +struct datapool * +datapool_open(const char *path, size_t size, int *fresh) +{ + struct datapool *pool = cc_alloc(sizeof(*pool)); + if (pool == NULL) { + log_error("unable to create allocate memory for pmem mapping"); + goto err_alloc; + } + + size_t map_size = size + sizeof(struct datapool_header); + + if (path == NULL) { /* fallback to DRAM if pmem is not configured */ + pool->addr = cc_zalloc(map_size); + pool->mapped_len = map_size; + pool->is_pmem = 0; + pool->file_backed = 0; + } else { + pool->addr = pmem_map_file(path, map_size, PMEM_FILE_CREATE, 0600, + &pool->mapped_len, &pool->is_pmem); + pool->file_backed = 1; + } + + if (pool->addr == NULL) { + log_error(path == NULL ? strerror(errno) : pmem_errormsg()); + goto err_map; + } + + log_info("mapped datapool %s with size %llu, is_pmem: %d", + path, pool->mapped_len, pool->is_pmem); + + pool->hdr = pool->addr; + pool->user_addr = (uint8_t *)pool->addr + sizeof(struct datapool_header); + + if (fresh) { + *fresh = 0; + } + + if (!datapool_valid(pool)) { + if (fresh) { + *fresh = 1; + } + + datapool_initialize(pool); + } + + datapool_flag_set(pool, DATAPOOL_FLAG_DIRTY); + + return pool; + +err_map: + cc_free(pool); +err_alloc: + return NULL; +} + +void +datapool_close(struct datapool *pool) +{ + datapool_sync(pool); + datapool_flag_clear(pool, DATAPOOL_FLAG_DIRTY); + + if (pool->file_backed) { + int ret = pmem_unmap(pool->addr, pool->mapped_len); + ASSERT(ret == 0); + } else { + cc_free(pool->addr); + } + + cc_free(pool); +} + +void * +datapool_addr(struct datapool *pool) +{ + return pool->user_addr; +} + +size_t +datapool_size(struct datapool *pool) +{ + return pool->mapped_len - sizeof(struct datapool_header); +} + diff --git a/src/datapool/datapool_shm.c b/src/datapool/datapool_shm.c new file mode 100644 index 000000000..8e1933300 --- /dev/null +++ b/src/datapool/datapool_shm.c @@ -0,0 +1,43 @@ +/* + * Anonymous shared memory backed datapool. + * Loses all its contents after closing. + */ +#include "datapool.h" + +#include +#include + +struct datapool * +datapool_open(const char *path, size_t size, int *fresh) +{ + if (path != NULL) { + log_warn("attempted to open a file-based data pool without" + "pmem features enabled"); + return NULL; + } + + if (fresh) { + *fresh = 1; + } + + return cc_zalloc(size); +} + +void +datapool_close(struct datapool *pool) +{ + cc_free(pool); +} + +void * +datapool_addr(struct datapool *pool) +{ + return pool; +} + +size_t +datapool_size(struct datapool *pool) +{ + return cc_alloc_usable_size(pool); +} + diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1583bedd3..ecc5257af 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -7,5 +7,8 @@ add_subdirectory(protocol) add_subdirectory(hotkey) add_subdirectory(storage) add_subdirectory(time) +if(USE_PMEM) + add_subdirectory(datapool) +endif() add_subdirectory(integration) diff --git a/test/datapool/CMakeLists.txt b/test/datapool/CMakeLists.txt new file mode 100644 index 000000000..2d8d65efa --- /dev/null +++ b/test/datapool/CMakeLists.txt @@ -0,0 +1,12 @@ +set(suite datapool) +set(test_name check_${suite}) + +set(source check_${suite}.c) + +add_executable(${test_name} ${source}) +target_link_libraries(${test_name} ${suite}) +target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES}) +target_link_libraries(${test_name} datapool) + +add_dependencies(check ${test_name}) +add_test(${test_name} ${test_name}) diff --git a/test/datapool/check_datapool.c b/test/datapool/check_datapool.c new file mode 100644 index 000000000..8bd649be2 --- /dev/null +++ b/test/datapool/check_datapool.c @@ -0,0 +1,89 @@ +#include + +#include + +#include +#include +#include +#include + +#define SUITE_NAME "datapool" +#define DEBUG_LOG SUITE_NAME ".log" +#define TEST_DATAFILE "./datapool.pelikan" +#define TEST_DATASIZE (1 << 20) + +/* + * tests + */ +START_TEST(test_datapool) +{ + int fresh = 0; + struct datapool *pool = datapool_open(TEST_DATAFILE, TEST_DATASIZE, &fresh); + ck_assert_ptr_nonnull(pool); + size_t s = datapool_size(pool); + ck_assert_int_ge(s, TEST_DATASIZE); + ck_assert_int_eq(fresh, 1); + ck_assert_ptr_nonnull(datapool_addr(pool)); + datapool_close(pool); + + pool = datapool_open(TEST_DATAFILE, TEST_DATASIZE, &fresh); + ck_assert_ptr_nonnull(pool); + ck_assert_int_eq(s, datapool_size(pool)); + ck_assert_int_eq(fresh, 0); + datapool_close(pool); +} +END_TEST + +START_TEST(test_devzero) +{ + int fresh = 0; + struct datapool *pool = datapool_open(NULL, TEST_DATASIZE, &fresh); + ck_assert_ptr_nonnull(pool); + size_t s = datapool_size(pool); + ck_assert_int_ge(s, TEST_DATASIZE); + ck_assert_int_eq(fresh, 1); + ck_assert_ptr_nonnull(datapool_addr(pool)); + datapool_close(pool); + + pool = datapool_open(NULL, TEST_DATASIZE, &fresh); + ck_assert_ptr_nonnull(pool); + ck_assert_int_eq(s, datapool_size(pool)); + ck_assert_int_eq(fresh, 1); + datapool_close(pool); +} +END_TEST + +/* + * test suite + */ +static Suite * +datapool_suite(void) +{ + Suite *s = suite_create(SUITE_NAME); + + TCase *tc_pool = tcase_create("pool"); + tcase_add_test(tc_pool, test_datapool); + tcase_add_test(tc_pool, test_devzero); + + suite_add_tcase(s, tc_pool); + + return s; +} + +int +main(void) +{ + int nfail; + + Suite *suite = datapool_suite(); + SRunner *srunner = srunner_create(suite); + srunner_set_fork_status(srunner, CK_NOFORK); + srunner_set_log(srunner, DEBUG_LOG); + srunner_run_all(srunner, CK_ENV); /* set CK_VEBOSITY in ENV to customize */ + nfail = srunner_ntests_failed(srunner); + srunner_free(srunner); + + unlink(TEST_DATAFILE); + + return (nfail == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} From 2a2d3c6da53bd6ff7f539e28c2212831cb3e5239 Mon Sep 17 00:00:00 2001 From: Piotr Balcer Date: Fri, 15 Feb 2019 09:15:18 +0100 Subject: [PATCH 2/2] cuckoo: use datapool for cuckoo data storage This patch uses the datapool interface to allow for durable operations of the cuckoo storage engine. To enable durable storage, provide a path to a file in "cuckoo_datapool" config option. If no file is provided, the datapool falls back to shared memory. --- src/storage/cuckoo/CMakeLists.txt | 1 + src/storage/cuckoo/cuckoo.c | 11 +- src/storage/cuckoo/cuckoo.h | 4 +- test/storage/CMakeLists.txt | 3 + test/storage/cuckoo_pmem/CMakeLists.txt | 13 ++ test/storage/cuckoo_pmem/check_cuckoo_pmem.c | 216 +++++++++++++++++++ 6 files changed, 244 insertions(+), 4 deletions(-) create mode 100644 test/storage/cuckoo_pmem/CMakeLists.txt create mode 100644 test/storage/cuckoo_pmem/check_cuckoo_pmem.c diff --git a/src/storage/cuckoo/CMakeLists.txt b/src/storage/cuckoo/CMakeLists.txt index d675ad3fc..5e196c5c6 100644 --- a/src/storage/cuckoo/CMakeLists.txt +++ b/src/storage/cuckoo/CMakeLists.txt @@ -1 +1,2 @@ add_library(cuckoo cuckoo.c) +target_link_libraries(cuckoo datapool) diff --git a/src/storage/cuckoo/cuckoo.c b/src/storage/cuckoo/cuckoo.c index 688bff018..6ee181776 100644 --- a/src/storage/cuckoo/cuckoo.c +++ b/src/storage/cuckoo/cuckoo.c @@ -5,6 +5,8 @@ #include #include +#include + /* TODO(yao): make D and iv[] configurable */ #include #include @@ -37,6 +39,7 @@ static uint32_t iv[D] = { 0x4dd2be0a }; +static struct datapool *pool; /* data pool mapping for the hash table */ static void* ds; /* data store is also the hash table */ static size_t item_size = CUCKOO_ITEM_SIZE; static uint32_t max_nitem = CUCKOO_NITEM; @@ -279,11 +282,13 @@ cuckoo_setup(cuckoo_options_st *options, cuckoo_metrics_st *metrics) } hash_size = item_size * max_nitem; - ds = cc_zalloc(hash_size); - if (ds == NULL) { + pool = datapool_open(option_str(&options->cuckoo_datapool), + hash_size, NULL); + if (pool == NULL) { log_crit("cuckoo data store allocation failed"); exit(EX_CONFIG); } + ds = datapool_addr(pool); cuckoo_init = true; } @@ -296,7 +301,7 @@ cuckoo_teardown(void) if (!cuckoo_init) { log_warn("%s has never been setup", CUCKOO_MODULE_NAME); } else { - cc_free(ds); + datapool_close(pool); } cuckoo_metrics = NULL; diff --git a/src/storage/cuckoo/cuckoo.h b/src/storage/cuckoo/cuckoo.h index a9a2cecf7..c42a187be 100644 --- a/src/storage/cuckoo/cuckoo.h +++ b/src/storage/cuckoo/cuckoo.h @@ -17,6 +17,7 @@ #define CUCKOO_NITEM 1024 #define CUCKOO_POLICY CUCKOO_POLICY_RANDOM #define CUCKOO_MAX_TTL (30 * 24 * 60 * 60) /* 30 days */ +#define CUCKOO_DATAPOOL NULL /* name type default description */ #define CUCKOO_OPTION(ACTION) \ @@ -25,7 +26,8 @@ ACTION( cuckoo_item_size, OPTION_TYPE_UINT, CUCKOO_ITEM_SIZE, "item size (inclusive)" )\ ACTION( cuckoo_nitem, OPTION_TYPE_UINT, CUCKOO_NITEM, "# items allocated" )\ ACTION( cuckoo_policy, OPTION_TYPE_UINT, CUCKOO_POLICY, "evict policy" )\ - ACTION( cuckoo_max_ttl, OPTION_TYPE_UINT, CUCKOO_MAX_TTL, "max ttl in seconds" ) + ACTION( cuckoo_max_ttl, OPTION_TYPE_UINT, CUCKOO_MAX_TTL, "max ttl in seconds" )\ + ACTION( cuckoo_datapool, OPTION_TYPE_STR, CUCKOO_DATAPOOL, "path to data pool" ) typedef struct { CUCKOO_OPTION(OPTION_DECLARE) diff --git a/test/storage/CMakeLists.txt b/test/storage/CMakeLists.txt index c06eede35..9553cd5c7 100644 --- a/test/storage/CMakeLists.txt +++ b/test/storage/CMakeLists.txt @@ -1,2 +1,5 @@ add_subdirectory(cuckoo) add_subdirectory(slab) +if(USE_PMEM) + add_subdirectory(cuckoo_pmem) +endif(USE_PMEM) diff --git a/test/storage/cuckoo_pmem/CMakeLists.txt b/test/storage/cuckoo_pmem/CMakeLists.txt new file mode 100644 index 000000000..8d5084496 --- /dev/null +++ b/test/storage/cuckoo_pmem/CMakeLists.txt @@ -0,0 +1,13 @@ +set(suite cuckoo_pmem) +set(test_name check_${suite}) + +set(source check_${suite}.c) + +add_executable(${test_name} ${source}) +target_link_libraries(${test_name} cuckoo) +target_link_libraries(${test_name} time) +target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES}) +target_link_libraries(${test_name} pthread m) + +add_dependencies(check ${test_name}) +add_test(${test_name} ${test_name}) diff --git a/test/storage/cuckoo_pmem/check_cuckoo_pmem.c b/test/storage/cuckoo_pmem/check_cuckoo_pmem.c new file mode 100644 index 000000000..c1c93117a --- /dev/null +++ b/test/storage/cuckoo_pmem/check_cuckoo_pmem.c @@ -0,0 +1,216 @@ +#include +#include + +#include