Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add fixed length str2int / asin str2int memory-mappable hashmap #295

Merged
merged 4 commits into from
Aug 29, 2024
Merged

add fixed length str2int / asin str2int memory-mappable hashmap #295

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a0a3fd8
add fixed length str2int / asin str2int memory-mappable hashmap
vpung Aug 6, 2024
d3442dd
rename asin_str2int to fixed_len_10_str2int
vpung Aug 12, 2024
720fdf1
Merge branch 'mainline' into ankerl-mmap
vpung Aug 27, 2024
a54ac74
Merge branch 'mainline' into ankerl-mmap
vpung Aug 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/style_type_check_cfg/.flake8
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
[flake8]
ignore = E203,E501,W605,F541
extend-ignore = E203,E501,W605,F541
max_line_length = 100
18 changes: 17 additions & 1 deletion pecos/core/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2070,12 +2070,20 @@ def link_mmap_hashmap_methods(self):
Specify C-lib's Memory-mappable Hashmap methods arguments and return types.
"""
fn_prefix = "mmap_hashmap"
map_type_list = ["str2int", "int2int"]
map_type_list = ["str2int", "fixed_len_str2int", "fixed_len_10_str2int", "int2int"]
key_args_dict = {
"str2int": [
c_char_p, # pointer of key string
c_uint32, # length of key string
],
"fixed_len_str2int": [
c_char_p, # pointer of key string
c_uint32, # length of key string
],
"fixed_len_10_str2int": [
c_char_p, # pointer of key string
c_uint32, # length of key string
],
"int2int": [
c_uint64, # key int64
],
Expand All @@ -2085,6 +2093,14 @@ def link_mmap_hashmap_methods(self):
c_void_p, # List of pointer of key string
POINTER(c_uint32), # List of length of key string
],
"fixed_len_str2int": [
c_void_p, # List of pointer of key string
POINTER(c_uint32), # List of length of key string
],
"fixed_len_10_str2int": [
c_void_p, # List of pointer of key string
POINTER(c_uint32), # List of length of key string
],
"int2int": [
POINTER(c_uint64), # List of key int64
],
Expand Down
24 changes: 23 additions & 1 deletion pecos/core/libpecos.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -661,28 +661,36 @@ extern "C" {

// ==== C Interface of Memory-mappable Hashmap ====

typedef pecos::mmap_hashmap::Str2IntMap mmap_hashmap_str2int;
typedef pecos::mmap_hashmap::Str2IntMap<pecos::mmap_hashmap::details_::AnkerlStr2IntMmapableVector> mmap_hashmap_str2int;
typedef pecos::mmap_hashmap::Str2IntMap<pecos::mmap_hashmap::details_::AnkerlFixedLenStr2IntMmapableVector> mmap_hashmap_fixed_len_str2int;
typedef pecos::mmap_hashmap::Str2IntMap<pecos::mmap_hashmap::details_::AnkerlFixedLen10Str2IntMmapableVector> mmap_hashmap_fixed_len_10_str2int;
typedef pecos::mmap_hashmap::Int2IntMap mmap_hashmap_int2int;

// New
#define MMAP_MAP_NEW(SUFFIX) \
void* mmap_hashmap_new_ ## SUFFIX () { \
return static_cast<void*>(new mmap_hashmap_ ## SUFFIX()); }
MMAP_MAP_NEW(str2int)
MMAP_MAP_NEW(fixed_len_str2int)
MMAP_MAP_NEW(fixed_len_10_str2int)
MMAP_MAP_NEW(int2int)

// Destruct
#define MMAP_MAP_DESTRUCT(SUFFIX) \
void mmap_hashmap_destruct_ ## SUFFIX (void* map_ptr) { \
delete static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr); }
MMAP_MAP_DESTRUCT(str2int)
MMAP_MAP_DESTRUCT(fixed_len_str2int)
MMAP_MAP_DESTRUCT(fixed_len_10_str2int)
MMAP_MAP_DESTRUCT(int2int)

// Save
#define MMAP_MAP_SAVE(SUFFIX) \
void mmap_hashmap_save_ ## SUFFIX (void* map_ptr, const char* map_dir) { \
static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr)->save(map_dir); }
MMAP_MAP_SAVE(str2int)
MMAP_MAP_SAVE(fixed_len_str2int)
MMAP_MAP_SAVE(fixed_len_10_str2int)
MMAP_MAP_SAVE(int2int)

// Load
Expand All @@ -692,13 +700,17 @@ extern "C" {
map_ptr->load(map_dir, lazy_load); \
return static_cast<void *>(map_ptr); }
MMAP_MAP_LOAD(str2int)
MMAP_MAP_LOAD(fixed_len_str2int)
MMAP_MAP_LOAD(fixed_len_10_str2int)
MMAP_MAP_LOAD(int2int)

// Size
#define MMAP_MAP_SIZE(SUFFIX) \
size_t mmap_hashmap_size_ ## SUFFIX (void* map_ptr) { \
return static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr)->size(); }
MMAP_MAP_SIZE(str2int)
MMAP_MAP_SIZE(fixed_len_str2int)
MMAP_MAP_SIZE(fixed_len_10_str2int)
MMAP_MAP_SIZE(int2int)

// Insert
Expand All @@ -707,32 +719,42 @@ extern "C" {
void mmap_hashmap_insert_ ## SUFFIX (void* map_ptr, KEY, uint64_t val) { \
static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr)->insert(FUNC_CALL_KEY, val); }
MMAP_MAP_INSERT(str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_INSERT(fixed_len_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_INSERT(fixed_len_10_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_INSERT(int2int, uint64_t key, key)

// Get
#define MMAP_MAP_GET(SUFFIX, KEY, FUNC_CALL_KEY) \
uint64_t mmap_hashmap_get_ ## SUFFIX (void* map_ptr, KEY) { \
return static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr)->get(FUNC_CALL_KEY); }
MMAP_MAP_GET(str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_GET(fixed_len_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_GET(fixed_len_10_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_GET(int2int, uint64_t key, key)

#define MMAP_MAP_GET_W_DEFAULT(SUFFIX, KEY, FUNC_CALL_KEY) \
uint64_t mmap_hashmap_get_w_default_ ## SUFFIX (void* map_ptr, KEY, uint64_t def_val) { \
return static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr)->get_w_default(FUNC_CALL_KEY, def_val); }
MMAP_MAP_GET_W_DEFAULT(str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_GET_W_DEFAULT(fixed_len_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_GET_W_DEFAULT(fixed_len_10_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_GET_W_DEFAULT(int2int, uint64_t key, key)

#define MMAP_MAP_BATCH_GET_W_DEFAULT(SUFFIX, KEY, FUNC_CALL_KEY) \
void mmap_hashmap_batch_get_w_default_ ## SUFFIX (void* map_ptr, const uint32_t n_key, KEY, uint64_t def_val, uint64_t* vals, const int threads) { \
static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr)->batch_get_w_default(n_key, FUNC_CALL_KEY, def_val, vals, threads); }
MMAP_MAP_BATCH_GET_W_DEFAULT(str2int, KEY_SINGLE_ARG(const char* const* keys, const uint32_t* keys_lens), KEY_SINGLE_ARG(keys, keys_lens))
MMAP_MAP_BATCH_GET_W_DEFAULT(fixed_len_str2int, KEY_SINGLE_ARG(const char* const* keys, const uint32_t* keys_lens), KEY_SINGLE_ARG(keys, keys_lens))
MMAP_MAP_BATCH_GET_W_DEFAULT(fixed_len_10_str2int, KEY_SINGLE_ARG(const char* const* keys, const uint32_t* keys_lens), KEY_SINGLE_ARG(keys, keys_lens))
MMAP_MAP_BATCH_GET_W_DEFAULT(int2int, const uint64_t* key, key)

// Contains
#define MMAP_MAP_CONTAINS(SUFFIX, KEY, FUNC_CALL_KEY) \
bool mmap_hashmap_contains_ ## SUFFIX (void* map_ptr, KEY) { \
return static_cast<mmap_hashmap_ ## SUFFIX *>(map_ptr)->contains(FUNC_CALL_KEY); }
MMAP_MAP_CONTAINS(str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_CONTAINS(fixed_len_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_CONTAINS(fixed_len_10_str2int, KEY_SINGLE_ARG(const char* key, uint32_t key_len), KEY_SINGLE_ARG(key, key_len))
MMAP_MAP_CONTAINS(int2int, uint64_t key, key)


Expand Down
Loading
Loading