From ce2d56bd9ddc2678e4b4752be516dd43b3fd51a4 Mon Sep 17 00:00:00 2001 From: Change72 Date: Wed, 3 Jan 2024 10:58:56 -0700 Subject: [PATCH 01/22] KVCache V1.0 --- .gitignore | 9 + CMakeLists.txt | 2 +- cmake/DetectOptions.cmake | 15 + examples/cache/build_cache.sh | 110 ++++++++ source/adios2/CMakeLists.txt | 15 +- source/adios2/engine/bp5/BP5Reader.cpp | 170 +++++++++++- source/adios2/engine/bp5/BP5Reader.h | 8 +- source/adios2/toolkit/cache/KVCacheCommon.h | 64 +++++ source/adios2/toolkit/cache/KVCacheCommon.tcc | 176 ++++++++++++ source/adios2/toolkit/cache/QueryBox.h | 261 ++++++++++++++++++ thirdparty/KWSys/CMakeLists.txt | 1 + 11 files changed, 826 insertions(+), 5 deletions(-) create mode 100644 examples/cache/build_cache.sh create mode 100644 source/adios2/toolkit/cache/KVCacheCommon.h create mode 100644 source/adios2/toolkit/cache/KVCacheCommon.tcc create mode 100644 source/adios2/toolkit/cache/QueryBox.h diff --git a/.gitignore b/.gitignore index 35a8d5ab87..e2f2ed3121 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,12 @@ CMakeSettings.json # Python wheels stuff *.egg-info/ + +# CMake generated files +build-*/ +cmake-build-*/ +.idea/ +.vscode/ + +# redis dump files +*.rdb \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 5fa8697873..0c54c69d57 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -265,7 +265,7 @@ set(ADIOS2_CONFIG_OPTS DataMan DataSpaces HDF5 HDF5_VOL MHS SST Fortran MPI Python PIP Blosc2 BZip2 LIBPRESSIO MGARD MGARD_MDR PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem UCX ZeroMQ Profiling Endian_Reverse Derived_Variable AWSSDK XRootD GPU_Support CUDA Kokkos - Kokkos_CUDA Kokkos_HIP Kokkos_SYCL Campaign + Kokkos_CUDA Kokkos_HIP Kokkos_SYCL Campaign KVCACHE ) GenerateADIOSHeaderConfig(${ADIOS2_CONFIG_OPTS}) diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 84553d0de1..1561752642 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -606,6 +606,21 @@ elseif(ADIOS2_USE_Campaign) endif() endif() +# KVCache +if(ADIOS2_USE_Cache STREQUAL AUTO) + find_package(hiredis REQUIRED) + if (hiredis_FOUND) + message(STATUS "hiredis found. Turn on KVCache") + set(ADIOS2_HAVE_KVCACHE TRUE) + endif() +elseif(ADIOS2_USE_Cache) + find_package(hiredis REQUIRED) + if (hiredis_FOUND) + message(STATUS "hiredis found. Turn on KVCache") + set(ADIOS2_HAVE_KVCACHE TRUE) + endif() +endif() + # Multithreading find_package(Threads REQUIRED) diff --git a/examples/cache/build_cache.sh b/examples/cache/build_cache.sh new file mode 100644 index 0000000000..00edb612e5 --- /dev/null +++ b/examples/cache/build_cache.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +# This script is to build the remote server with cache enabled. +# Attention: hiredis cannot be installed by apt-get, since the default version is too old (libhiredis0.14 (= 0.14.1-2)). +# We need to build it from source code. You can also use the following scripts to install hiredis (v1.2.0). + +# sample usage: in project home directory: +# source examples/cache/build_cache.sh --build +# source examples/cache/build_cache.sh --start +# source examples/cache/build_cache.sh --stop + +if [ -z ${BUILD_DIR} ] +then + BUILD_DIR=${PWD}/build-cache-test +fi + +if [ ! -d ${BUILD_DIR} ] +then + mkdir -p ${BUILD_DIR} +fi + +SW_DIR=${BUILD_DIR}/sw +if [ ! -d ${SW_DIR} ] +then + mkdir -p ${SW_DIR} +fi + +build_cache() { + # redis - in-memory data structure store + redis_dir=${SW_DIR}/redis + if [ ! -d ${redis_dir} ] + then + git clone https://github.com/redis/redis.git ${redis_dir} + cd ${redis_dir} + git checkout tags/7.2.3 + # cannot accleerate by 'make -j8'. It will cause error. + make + + # hiredis - C client library to connect Redis server + cd ${redis_dir}/deps/hiredis + mkdir build && cd build + cmake .. -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hiredis + make -j32 + make install + fi + + cd ${BUILD_DIR} + cmake .. -DADIOS2_USE_Cache=ON \ + -DADIOS2_USE_Python=ON \ + -DCMAKE_PREFIX_PATH=${SW_DIR} \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2 + + make -j32 + make install + cd ${BUILD_DIR}/../ +} + +start_services() { + echo "Starting redis server and setting environment variables..." + export DoRemote=1 + export useKVCache=1 + export PYTHONPATH=${SW_DIR}/adios2/local/lib/python3.10/dist-packages/ + export LD_LIBRARY_PATH=${SW_DIR}/adios2/lib:${SW_DIR}/hiredis/lib:$LD_LIBRARY_PATH + nohup ${SW_DIR}/redis/src/redis-server > ${SW_DIR}redis_server.log 2>&1 & + nohup ${SW_DIR}/adios2/bin/adios2_remote_server > ${SW_DIR}remote_server.log 2>&1 & + sleep 5 + nohup ${SW_DIR}/redis/src/redis-cli monitor > ${SW_DIR}redis_monitor.log 2>&1 & + echo "Services started and environment variables set." +} + +# Function to stop services (optional, example purpose) +stop_services() { + echo "Stopping services..." + pkill -f redis-server + pkill -f redis-cli + pkill -f remote_server + unset DoRemote + unset useKVCache + unset PYTHONPATH + unset LD_LIBRARY_PATH + echo "Services stopped." +} + +# Parse command line options +while [[ "$1" != "" ]]; do + case "$1" in + -b | --build ) + build_cache + ;; + -c | --start ) + start_services + ;; + -s | --stop ) + stop_services + ;; + -h | --help ) + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " -b, --build Build the software with cache enabled" + echo " -c, --start Start redis server and set environment variables" + echo " -s, --stop Stop the services" + echo " -h, --help Display this help message" + ;; + * ) + echo "Invalid option: $1" + echo "Use -h or --help for usage information." + ;; + esac + shift +done diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index e09ba37093..2f2590d700 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -247,7 +247,20 @@ if (ADIOS2_HAVE_SST) target_link_libraries(adios2_core PRIVATE adios2::thirdparty::EVPath) add_subdirectory(toolkit/remote) endif() - + +if (ADIOS2_HAVE_KVCACHE) + target_sources(adios2_core PRIVATE toolkit/cache/KVCacheCommon.h + toolkit/cache/KVCacheCommon.tcc) + target_link_libraries(adios2_core PRIVATE hiredis::hiredis) + + # Ensure the rpath is set correctly + # message(STATUS "PACKAGE_PREFIX_DIR: ${PACKAGE_PREFIX_DIR}") + set_target_properties(adios2_core PROPERTIES + BUILD_RPATH "$ORIGIN:${PACKAGE_PREFIX_DIR}/lib" + INSTALL_RPATH "$ORIGIN:${PACKAGE_PREFIX_DIR}/lib" + ) +endif () + if(ADIOS2_HAVE_Campaign) target_sources(adios2_core PRIVATE engine/campaign/CampaignReader.cpp diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 339cb16586..f927b2a8d3 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -349,16 +349,182 @@ void BP5Reader::PerformRemoteGets() // TP startGenerate = NOW(); auto GetRequests = m_BP5Deserializer->PendingGetRequests; std::vector handles; + + #ifdef ADIOS2_HAVE_KVCACHE // open kv cache connection + struct RequestInfo + { + size_t ReqSeq; + DataType varType; + size_t ReqCount; + std::string CacheKey; + size_t TypeSize; + Dims Count; + Dims Start; + void *Data; + }; + std::vector getRequestsInfo; + + if (getenv("useKVCache")) + { + m_KVCacheCommon.openConnection(); + + std::vector getRequestsInfo; + } + #endif + + int req_seq = -1; + std::vector handles; for (auto &Req : GetRequests) { - auto handle = - m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, Req.Count, Req.Start, Req.Data); + req_seq++; + #ifdef ADIOS2_HAVE_KVCACHE // get data from cache + if (getenv("useKVCache")) + { + const DataType varType = m_IO.InquireVariableType(Req.VarName); + QueryBox targetBox(Req.Start, Req.Count); + size_t numOfElements = targetBox.size(); + std::string keyPrefix = m_KVCacheCommon.keyPrefix(Req.VarName, Req.RelStep, Req.BlockID); + std::string targetKey = m_KVCacheCommon.keyComposition(keyPrefix, Req.Start, Req.Count); + + // Exact Match: check if targetKey exists + if (m_KVCacheCommon.exists(targetKey)) + { +#define declare_type_get(T) \ + if (varType == helper::GetDataType()) \ + { \ + std::vector reqData; \ + reqData.resize(numOfElements); \ + m_KVCacheCommon.get(targetKey, reqData); \ + std::memcpy(Req.Data, reqData.data(), numOfElements * sizeof(T)); \ + } + ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type_get) +#undef declare_type_get + + } else { + int max_depth = 999; + std::set samePrefixKeys; + m_KVCacheCommon.keyPrefixExistence(keyPrefix, samePrefixKeys); + std::vector regularBoxes; + std::vector cachedBoxes; + std::vector cachedKeys; + + if (getenv("maxDepth")) + { + max_depth = std::stoi(getenv("maxDepth")); + } + + /* + std::cout << "Setting max_depth is: " << max_depth << std::endl; + + // print the size of samePrefixKeys + std::cout << "Same prefix keys size: " << samePrefixKeys.size() << std::endl; + // print out samePrefixKeys + for (auto &key : samePrefixKeys) + { + std::cout << "Same prefix keys: " << key << std::endl; + } + */ + + if (samePrefixKeys.size() > 0) + { + targetBox.getMaxInteractBox(samePrefixKeys, max_depth, 0, regularBoxes, cachedBoxes, cachedKeys); + } else { + regularBoxes.push_back(targetBox); + } + + /* + // print out regularBoxes and cachedBoxes size + std::cout << "Going to retrieve regular boxes size: " << regularBoxes.size() << std::endl; + std::cout << "Already cached boxes size: " << cachedBoxes.size() << std::endl; + + // print out regularBoxes and cachedBoxes by toString + for (int i = 0; i < regularBoxes.size(); i++) + { + std::cout << "Regular box " << i << " : " << regularBoxes[i].toString() << " size: " << regularBoxes[i].size() << std::endl; + } + + for (int i = 0; i < cachedBoxes.size(); i++) + { + std::cout << "Cached box " << i << " : " << cachedBoxes[i].toString() << " size: " << cachedBoxes[i].size() << std::endl; + } + */ + + std::cout << "Going to retrieve "<< regularBoxes.size() " boxes from remote server, and " << cachedBoxes.size() << " boxes from cache" << std::endl; + + +#define declare_type_full_contain(T) \ + if (varType == helper::GetDataType()) \ + { \ + const int typeSize = sizeof(T); \ + for (auto &box : regularBoxes){ \ + RequestInfo ReqInfo; \ + ReqInfo.ReqSeq = req_seq; \ + ReqInfo.varType = varType; \ + ReqInfo.ReqCount = box.size(); \ + ReqInfo.CacheKey = m_KVCacheCommon.keyComposition(keyPrefix, box.start, box.count); \ + ReqInfo.TypeSize = typeSize; \ + ReqInfo.Count = box.count; \ + ReqInfo.Start = box.start; \ + ReqInfo.Data = malloc(box.size() * sizeof(T)); \ + auto handle = m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, box.count, box.start, ReqInfo.Data); \ + handles.push_back(handle); \ + getRequestsInfo.push_back(ReqInfo); \ + } \ + for (int i = 0; i < cachedBoxes.size(); i++){ \ + std::string boxKey = cachedKeys[i]; \ + QueryBox box(boxKey); \ + std::vector srcData; \ + srcData.resize(box.size()); \ + m_KVCacheCommon.get(boxKey, srcData); \ + helper::NdCopy(reinterpret_cast(srcData.data()), helper::CoreDims(cachedBoxes[i].start), cachedBoxes[i].count, true, false, reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, typeSize); \ + } \ + } +ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type_full_contain) +#undef declare_type_full_contain + } + + continue; + } + #endif + + auto handle = m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, Req.Count, Req.Start, Req.Data); handles.push_back(handle); } + + std::size_t handle_seq = -1; for (auto &handle : handles) { + handle_seq++; m_Remote->WaitForGet(handle); + #ifdef ADIOS2_HAVE_KVCACHE // close cache connection + if (getenv("useKVCache")) + { + auto &ReqInfo = getRequestsInfo[handle_seq]; + auto &Req = GetRequests[ReqInfo.ReqSeq]; + helper::NdCopy(reinterpret_cast(ReqInfo.Data), helper::CoreDims(ReqInfo.Start), ReqInfo.Count, true, false, reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, ReqInfo.TypeSize); + + #define declare_type_set(T) \ + if (ReqInfo.varType == helper::GetDataType()) \ + { \ + std::vector reqData; \ + reqData.resize(ReqInfo.ReqCount); \ + std::memcpy(reqData.data(), Req.Data, ReqInfo.ReqCount * sizeof(T)); \ + m_KVCacheCommon.set(ReqInfo.CacheKey, reqData); \ + } + ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type_set) + #undef declare_type_set + + free(ReqInfo.Data); + } + #endif + } + + #ifdef ADIOS2_HAVE_KVCACHE // close cache connection + if (getenv("useKVCache")) + { + m_KVCacheCommon.closeConnection(); } + #endif } void BP5Reader::PerformLocalGets() diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h index a0418c5244..488c332c22 100644 --- a/source/adios2/engine/bp5/BP5Reader.h +++ b/source/adios2/engine/bp5/BP5Reader.h @@ -20,6 +20,10 @@ #include "adios2/toolkit/remote/Remote.h" #include "adios2/toolkit/transportman/TransportMan.h" +#ifdef ADIOS2_HAVE_KVCACHE +#include "adios2/toolkit/cache/KVCacheCommon.h" +#endif + #include #include #include @@ -98,7 +102,9 @@ class BP5Reader : public BP5Engine, public Engine std::unique_ptr m_Remote; bool m_WriterIsActive = true; adios2::profiling::JSONProfiler m_JSONProfiler; - + #ifdef ADIOS2_HAVE_KVCACHE + KVCacheCommon m_KVCacheCommon; + #endif /** used for per-step reads, TODO: to be moved to BP5Deserializer */ size_t m_CurrentStep = 0; size_t m_StepsCount = 0; diff --git a/source/adios2/toolkit/cache/KVCacheCommon.h b/source/adios2/toolkit/cache/KVCacheCommon.h new file mode 100644 index 0000000000..d21b9fa178 --- /dev/null +++ b/source/adios2/toolkit/cache/KVCacheCommon.h @@ -0,0 +1,64 @@ +// +// Created by cguo51 on 12/30/23. +// + +#ifndef ADIOS2_KVCACHECOMMON_H +#define ADIOS2_KVCACHECOMMON_H +#include +#include "adios2/toolkit/cache/QueryBox.h" +#include +#include // For memcpy +#include +#include + +// namespace adios2::KVCache + +namespace adios2 +{ + +class KVCacheCommon +{ +public: + std::string m_host; + int m_port; + redisContext *m_redisContext; + redisReply *m_redisReply; + std::string m_key; + std::string m_value; + std::string m_command; + + KVCacheCommon(std::string host="localhost", int port=6379): m_host(host), m_port(port){}; + + inline void openConnection(); + + inline void closeConnection(); + + template + void set(std::string key, const std::vector& vec); + + template + void get(std::string key, std::vector& vec); + + inline void del(std::string key); + + inline bool exists(std::string key); + + inline std::string keyPrefix(char *VarName, size_t AbsStep, size_t BlockID); + + inline std::string keyComposition(const std::string &key_prefix, Dims Start, Dims Count); + + inline void keyPrefixExistence(const std::string &key_prefix, std::set &keys); + + template + void encodeVector(const std::vector& vec, std::string& encodedString); + + template + void decodeVector(const std::string& str, std::vector& vec); +}; + + +}; // adios2 + +#include "KVCacheCommon.tcc" + +#endif // ADIOS2_KVCACHECOMMON_H diff --git a/source/adios2/toolkit/cache/KVCacheCommon.tcc b/source/adios2/toolkit/cache/KVCacheCommon.tcc new file mode 100644 index 0000000000..ad584f0610 --- /dev/null +++ b/source/adios2/toolkit/cache/KVCacheCommon.tcc @@ -0,0 +1,176 @@ +// +// Created by cguo51 on 12/30/23. +// +#ifndef KVCACHECOMMON_TCC +#define KVCACHECOMMON_TCC + +namespace adios2 +{ + +void KVCacheCommon::openConnection() +{ + m_redisContext = redisConnect(m_host.c_str(), m_port); + if (m_redisContext == NULL || m_redisContext->err) + { + std::cout << "Error to connect to kvcache server: " << m_redisContext->errstr << std::endl; + if (m_redisContext) + { + redisFree(m_redisContext); + } + } + else + { + std::cout << "------------------------------------------------------------" << std::endl; + std::cout << "Connected to kvcache server. KV Cache Version Control: V1.0" << std::endl; + } +} + +void KVCacheCommon::closeConnection() +{ + redisFree(m_redisContext); + std::cout << "KVCache connection closed" << std::endl; +} + +template +void KVCacheCommon::set(std::string key, const std::vector& vec) +{ + encodeVector(vec, m_value); + m_command = "SET " + key + " " + m_value; + m_redisReply = (redisReply *)redisCommand(m_redisContext, m_command.c_str()); + if (m_redisReply == NULL) + { + std::cout << "Error to set key: " << key << std::endl; + } + else + { + std::cout << "SET Key: " << key << " Value size: " << vec.size() << std::endl; + freeReplyObject(m_redisReply); + } +} + +template +void KVCacheCommon::get(std::string key, std::vector& vec) +{ + m_command = "GET " + key; + m_redisReply = (redisReply *)redisCommand(m_redisContext, m_command.c_str()); + if (m_redisReply == NULL) + { + std::cout << "Error to get key: " << key << std::endl; + } + else + { + decodeVector(m_redisReply->str, vec); + freeReplyObject(m_redisReply); + } +} + +void KVCacheCommon::del(std::string key) +{ + m_command = "DEL " + key; + m_redisReply = (redisReply *)redisCommand(m_redisContext, m_command.c_str()); + if (m_redisReply == NULL) + { + std::cout << "Error to delete key: " << key << std::endl; + } + else + { + freeReplyObject(m_redisReply); + } +} + +bool KVCacheCommon::exists(std::string key) +{ + m_command = "EXISTS " + key; + m_redisReply = (redisReply *)redisCommand(m_redisContext, m_command.c_str()); + if (m_redisReply == NULL) + { + std::cout << "The Key: " << key << " does not exist" << std::endl; + return false; + } + else + { + if (!m_redisReply->integer) + { + std::cout << "The Key: " << key << " does not exist" << std::endl; + return false; + } + freeReplyObject(m_redisReply); + return true; + } +} + +std::string KVCacheCommon::keyPrefix(char *VarName, size_t AbsStep, size_t BlockID) +{ + return VarName + std::to_string(AbsStep) + std::to_string(BlockID); +} + +std::string KVCacheCommon::keyComposition(const std::string &key_prefix, Dims Start, Dims Count) +{ + std::string box = QueryBox::serializeQueryBox(QueryBox{Start, Count}); + std::string cacheKey = key_prefix + box; + // replace special characters + std::replace(cacheKey.begin(), cacheKey.end(), '"', '_'); + std::replace(cacheKey.begin(), cacheKey.end(), ',', '_'); + std::replace(cacheKey.begin(), cacheKey.end(), '(', '_'); + std::replace(cacheKey.begin(), cacheKey.end(), ')', '_'); + std::replace(cacheKey.begin(), cacheKey.end(), '[', '_'); + std::replace(cacheKey.begin(), cacheKey.end(), ']', '_'); + std::replace(cacheKey.begin(), cacheKey.end(), '{', '_'); + std::replace(cacheKey.begin(), cacheKey.end(), '}', '_'); + return cacheKey; +} + +void KVCacheCommon::keyPrefixExistence(const std::string &key_prefix, std::set &keys) +{ + std::string keyPattern = key_prefix + "*"; + m_command = "KEYS " + keyPattern; + m_redisReply = (redisReply *)redisCommand(m_redisContext, m_command.c_str()); + if (m_redisReply == NULL) + { + std::cout << "Error to get keys with prefix: " << key_prefix << std::endl; + } + else + { + for (int i = 0; i < m_redisReply->elements; i++) + { + keys.insert(m_redisReply->element[i]->str); + } + freeReplyObject(m_redisReply); + } +} + +template +void KVCacheCommon::encodeVector(const std::vector& vec, std::string& encodedString) { + size_t vecSize = vec.size() * sizeof(T); + const unsigned char* vecBytes = reinterpret_cast(vec.data()); + + size_t encodedSize = vecSize * 3 / 2; + std::vector encodedBytes(encodedSize); + + size_t sizeAfterEncoded = adios2sysBase64_Encode(vecBytes, vecSize, encodedBytes.data(), 0); + + // Resize the vector to the actual size + encodedBytes.resize(sizeAfterEncoded); + + // Convert the encoded bytes to a string + encodedString.assign(encodedBytes.begin(), encodedBytes.end()); +} + +template +void KVCacheCommon::decodeVector(const std::string& str, std::vector& vec) { + size_t decodedSize = str.size() * 2; + std::vector decodedBytes(decodedSize); + + size_t sizeAfterDecoded = adios2sysBase64_Decode(reinterpret_cast(str.data()), str.size(), decodedBytes.data(), decodedSize); + + // Resize the vector to the actual size + decodedBytes.resize(sizeAfterDecoded); + + // Copy the decoded bytes to the vector + vec.resize(sizeAfterDecoded / sizeof(T)); + memcpy(vec.data(), decodedBytes.data(), sizeAfterDecoded); + +} + +}; // namespace adios2 +#endif // KVCACHECOMMON_TCC \ No newline at end of file diff --git a/source/adios2/toolkit/cache/QueryBox.h b/source/adios2/toolkit/cache/QueryBox.h new file mode 100644 index 0000000000..b2b1008200 --- /dev/null +++ b/source/adios2/toolkit/cache/QueryBox.h @@ -0,0 +1,261 @@ +// +// Created by cguo51 on 7/27/23. +// + +#ifndef ADIOS2_KVCACHE_QUERYBOX_H +#define ADIOS2_KVCACHE_QUERYBOX_H + +#include +#include +#include +#include + +namespace adios2 +{ +// QueryBox is a class to represent a query box in a multi-dimensional space +class QueryBox +{ +public: + adios2::Dims start{}; + adios2::Dims count{}; + + // constructor + QueryBox() = default; + QueryBox(const adios2::Dims &start, const adios2::Dims &count) : start(start), count(count){}; + QueryBox(const std::string &key){ + // sample key: "U3218446744073709551615__count_:_64_64_64___start_:_0_0_0__", count [64, 64, 64], start [0, 0, 0] + // using Dims = std::vector; + auto lf_ExtractDimensions = [](const std::string &key, const std::string &delimiter) -> Dims { + size_t const pos = key.find(delimiter); + size_t const end = key.find("__", pos + delimiter.length()); + std::string dimStr = key.substr(pos + delimiter.length(), end - pos - delimiter.length()); + Dims dimensions; + std::istringstream dimStream(dimStr); + std::string token; + while (std::getline(dimStream, token, '_')) { + dimensions.push_back(std::stoul(token)); + } + return dimensions; + }; + + this->start = lf_ExtractDimensions(key, "__start_:_"); + this->count = lf_ExtractDimensions(key, "__count_:_"); + } + + // size + size_t size() const + { + size_t s = 1; + for (auto &d : count) + { + s *= d; + } + return s; + } + + // Serialize QueryBox to a string, like __count_:_64_64_64___start_:_0_0_0__ + static std::string serializeQueryBox(const QueryBox &box) + { + nlohmann::json jsonBox; + jsonBox["start"] = box.start; + jsonBox["count"] = box.count; + return jsonBox.dump(); + } + + // determine if a query box is equal to another query box + bool operator==(const QueryBox &box) const + { + return start == box.start && count == box.count; + } + + // determine if a query box is interacted in another query box, return intersection part as a new query box + bool isInteracted (const QueryBox &box, QueryBox &intersection) const + { + if (start.size() != box.start.size() || start.size() != count.size() || + start.size() != box.count.size()) + { + return false; + } + for (size_t i = 0; i < start.size(); ++i) + { + if (start[i] > box.start[i] + box.count[i] || box.start[i] > start[i] + count[i]) + { + return false; + } + } + intersection.start.resize(start.size()); + intersection.count.resize(count.size()); + for (size_t i = 0; i < start.size(); ++i) + { + intersection.start[i] = std::max(start[i], box.start[i]); + intersection.count[i] = + std::min(start[i] + count[i], box.start[i] + box.count[i]) - intersection.start[i]; + } + return true; + } + + // determine if a query box is fully contained in another query box + bool isFullContainedBy(const QueryBox &box) + { + if (start.size() != box.start.size() || start.size() != count.size() || + start.size() != box.count.size()) + { + return false; + } + for (size_t i = 0; i < start.size(); ++i) + { + if (start[i] < box.start[i] || start[i] + count[i] > box.start[i] + box.count[i]) + { + return false; + } + } + return true; + } + + // cut a query box from another interaction box, return a list of regular box + // remainingBox is the big one, this is small one + void interactionCut(const QueryBox &remainingBox, std::vector ®ularBoxes) + { + if (remainingBox == *this) + { + return; + } + + // find the max cut dimension + size_t maxCutDimSize = 0; + QueryBox maxCutDimBox; + for (size_t i = 0; i < start.size(); ++i) + { + if (start[i] == remainingBox.start[i] && count[i] == remainingBox.count[i]) + { + continue; + } + else { + if (start[i] != remainingBox.start[i]){ + size_t cutDimDiff = start[i] - remainingBox.start[i]; + size_t cutDimSize = remainingBox.size() / remainingBox.count[i] * cutDimDiff; + if (cutDimSize > maxCutDimSize) + { + maxCutDimSize = cutDimSize; + maxCutDimBox = QueryBox(remainingBox.start, remainingBox.count); + maxCutDimBox.count[i] = cutDimDiff; + } + } + + if (start[i] + count[i] != remainingBox.start[i] + remainingBox.count[i]){ + size_t cutDimDiff = remainingBox.start[i] + remainingBox.count[i] - start[i] - count[i]; + size_t cutDimSize = remainingBox.size() / count[i] * cutDimDiff; + if (cutDimSize > maxCutDimSize) + { + maxCutDimSize = cutDimSize; + maxCutDimBox = QueryBox(remainingBox.start, remainingBox.count); + maxCutDimBox.start[i] = start[i] + count[i]; + maxCutDimBox.count[i] = cutDimDiff; + } + } + } + } + + // cut the max cut dimension + if (maxCutDimSize > 0) + { + regularBoxes.push_back(maxCutDimBox); + QueryBox remainingBox1 = QueryBox(remainingBox.start, remainingBox.count); + for (size_t i = 0; i < remainingBox.start.size(); ++i) + { + if (maxCutDimBox.start[i] == remainingBox.start[i] && maxCutDimBox.count[i] == remainingBox.count[i]) + { + continue; + } + else { + if (maxCutDimBox.start[i] != remainingBox.start[i]) + { + remainingBox1.count[i] = maxCutDimBox.start[i] - remainingBox.start[i]; + } else { + remainingBox1.start[i] = maxCutDimBox.start[i] + maxCutDimBox.count[i]; + remainingBox1.count[i] = remainingBox.start[i] + remainingBox.count[i] - remainingBox1.start[i]; + } + + } + } + interactionCut(remainingBox1, regularBoxes); + } + } + + void getMaxInteractBox(const std::set &samePrefixKeys, const size_t &max_depth, size_t current_depth, std::vector ®ularBoxes, std::vector &cachedBox, std::vector &cachedKeys) + { + if (current_depth > max_depth) + { + return; + } + current_depth++; + QueryBox maxInteractBox; + std::string maxInteractKey; + for (auto &key : samePrefixKeys) + { + // std::cout << "Same Prefix Keys: " << key << " Current Depth: " << current_depth << std::endl; + QueryBox const box(key); + QueryBox intersection; + if (this->isInteracted(box, intersection)) + { + if (maxInteractBox.size() < intersection.size()) + { + maxInteractBox = intersection; + maxInteractKey = key; + } + } + } + + if (maxInteractBox.count.size() == 0) + { + return; + } + + // std::cout << "===============================================================================" << std::endl; + // std::cout << "Current Depth: " << current_depth << " Target box: " << this->toString() << " Target box size: " << this->size() << std::endl; + // std::cout << "Pushing maxInteractBox: " << maxInteractBox.toString() << " key: " << maxInteractKey << ", Interacted size: " << maxInteractBox.size() << std::endl; + + cachedBox.push_back(maxInteractBox); + cachedKeys.push_back(maxInteractKey); + + if (current_depth == max_depth) + { + maxInteractBox.interactionCut(*this, regularBoxes); + } else { + std::vector nextBoxes; + maxInteractBox.interactionCut(*this, nextBoxes); + for (auto &box : nextBoxes) + { + box.getMaxInteractBox(samePrefixKeys, max_depth, current_depth, regularBoxes, cachedBox, cachedKeys); + } + } + } + + // rewrite toString + std::string toString() const + { + std::string str = "Box start: ["; + for (size_t i = 0; i < start.size(); ++i) + { + str += std::to_string(start[i]); + if (i != start.size() - 1) + { + str += ", "; + } + } + str += "], count: ["; + for (size_t i = 0; i < count.size(); ++i) + { + str += std::to_string(count[i]); + if (i != count.size() - 1) + { + str += ", "; + } + } + str += "]"; + return str; + } + +}; +}; +#endif // UNITTEST_QUERYBOX_H diff --git a/thirdparty/KWSys/CMakeLists.txt b/thirdparty/KWSys/CMakeLists.txt index 2714fa2bb2..1e7ffbbc27 100644 --- a/thirdparty/KWSys/CMakeLists.txt +++ b/thirdparty/KWSys/CMakeLists.txt @@ -5,6 +5,7 @@ set(KWSYS_USE_CommandLineArguments ON) set(KWSYS_USE_DynamicLoader ON) set(KWSYS_USE_RegularExpression ON) set(KWSYS_USE_SystemTools ON) +set(KWSYS_USE_Base64 ON) if(WIN32) set(KWSYS_BUILD_SHARED OFF) else() From 1e6e1299b6bf578175800277448919252c25d057 Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Mon, 24 Jun 2024 13:52:45 -0400 Subject: [PATCH 02/22] adjust some details for KVCache 1.0 --- .gitignore | 3 +- .../build_scripts/build-adios2-kvcache.sh | 10 +++--- source/adios2/engine/bp5/BP5Reader.cpp | 34 +------------------ source/adios2/toolkit/cache/QueryBox.h | 4 --- 4 files changed, 8 insertions(+), 43 deletions(-) rename examples/cache/build_cache.sh => scripts/build_scripts/build-adios2-kvcache.sh (88%) diff --git a/.gitignore b/.gitignore index e2f2ed3121..10d33af858 100644 --- a/.gitignore +++ b/.gitignore @@ -46,7 +46,8 @@ CMakeSettings.json *.egg-info/ # CMake generated files -build-*/ +build/ +build-cache*/ cmake-build-*/ .idea/ .vscode/ diff --git a/examples/cache/build_cache.sh b/scripts/build_scripts/build-adios2-kvcache.sh similarity index 88% rename from examples/cache/build_cache.sh rename to scripts/build_scripts/build-adios2-kvcache.sh index 00edb612e5..db5ecab5c8 100644 --- a/examples/cache/build_cache.sh +++ b/scripts/build_scripts/build-adios2-kvcache.sh @@ -5,9 +5,9 @@ # We need to build it from source code. You can also use the following scripts to install hiredis (v1.2.0). # sample usage: in project home directory: -# source examples/cache/build_cache.sh --build -# source examples/cache/build_cache.sh --start -# source examples/cache/build_cache.sh --stop +# source scripts/build_scripts/build-adios2-kvcache.sh --build +# source scripts/build_scripts/build-adios2-kvcache.sh --start +# source scripts/build_scripts/build-adios2-kvcache.sh --stop if [ -z ${BUILD_DIR} ] then @@ -60,9 +60,9 @@ start_services() { export DoRemote=1 export useKVCache=1 export PYTHONPATH=${SW_DIR}/adios2/local/lib/python3.10/dist-packages/ - export LD_LIBRARY_PATH=${SW_DIR}/adios2/lib:${SW_DIR}/hiredis/lib:$LD_LIBRARY_PATH + # export LD_LIBRARY_PATH=${SW_DIR}/adios2/lib:${SW_DIR}/hiredis/lib:$LD_LIBRARY_PATH nohup ${SW_DIR}/redis/src/redis-server > ${SW_DIR}redis_server.log 2>&1 & - nohup ${SW_DIR}/adios2/bin/adios2_remote_server > ${SW_DIR}remote_server.log 2>&1 & + nohup ${SW_DIR}/adios2/bin/adios2_remote_server -v > ${SW_DIR}remote_server.log 2>&1 & sleep 5 nohup ${SW_DIR}/redis/src/redis-cli monitor > ${SW_DIR}redis_monitor.log 2>&1 & echo "Services started and environment variables set." diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index f927b2a8d3..6b6a789241 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -367,13 +367,10 @@ void BP5Reader::PerformRemoteGets() if (getenv("useKVCache")) { m_KVCacheCommon.openConnection(); - - std::vector getRequestsInfo; } #endif int req_seq = -1; - std::vector handles; for (auto &Req : GetRequests) { req_seq++; @@ -413,18 +410,6 @@ void BP5Reader::PerformRemoteGets() max_depth = std::stoi(getenv("maxDepth")); } - /* - std::cout << "Setting max_depth is: " << max_depth << std::endl; - - // print the size of samePrefixKeys - std::cout << "Same prefix keys size: " << samePrefixKeys.size() << std::endl; - // print out samePrefixKeys - for (auto &key : samePrefixKeys) - { - std::cout << "Same prefix keys: " << key << std::endl; - } - */ - if (samePrefixKeys.size() > 0) { targetBox.getMaxInteractBox(samePrefixKeys, max_depth, 0, regularBoxes, cachedBoxes, cachedKeys); @@ -432,24 +417,7 @@ void BP5Reader::PerformRemoteGets() regularBoxes.push_back(targetBox); } - /* - // print out regularBoxes and cachedBoxes size - std::cout << "Going to retrieve regular boxes size: " << regularBoxes.size() << std::endl; - std::cout << "Already cached boxes size: " << cachedBoxes.size() << std::endl; - - // print out regularBoxes and cachedBoxes by toString - for (int i = 0; i < regularBoxes.size(); i++) - { - std::cout << "Regular box " << i << " : " << regularBoxes[i].toString() << " size: " << regularBoxes[i].size() << std::endl; - } - - for (int i = 0; i < cachedBoxes.size(); i++) - { - std::cout << "Cached box " << i << " : " << cachedBoxes[i].toString() << " size: " << cachedBoxes[i].size() << std::endl; - } - */ - - std::cout << "Going to retrieve "<< regularBoxes.size() " boxes from remote server, and " << cachedBoxes.size() << " boxes from cache" << std::endl; + std::cout << "Going to retrieve "<< regularBoxes.size() << " boxes from remote server, and " << cachedBoxes.size() << " boxes from cache" << std::endl; #define declare_type_full_contain(T) \ diff --git a/source/adios2/toolkit/cache/QueryBox.h b/source/adios2/toolkit/cache/QueryBox.h index b2b1008200..3f3f33458a 100644 --- a/source/adios2/toolkit/cache/QueryBox.h +++ b/source/adios2/toolkit/cache/QueryBox.h @@ -211,10 +211,6 @@ class QueryBox return; } - // std::cout << "===============================================================================" << std::endl; - // std::cout << "Current Depth: " << current_depth << " Target box: " << this->toString() << " Target box size: " << this->size() << std::endl; - // std::cout << "Pushing maxInteractBox: " << maxInteractBox.toString() << " key: " << maxInteractKey << ", Interacted size: " << maxInteractBox.size() << std::endl; - cachedBox.push_back(maxInteractBox); cachedKeys.push_back(maxInteractKey); From d7f5e279db60d5547fc479fbc69973334d18b1c2 Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Mon, 24 Jun 2024 14:47:57 -0400 Subject: [PATCH 03/22] formatting and remove REQUIRED in AUTO setting --- cmake/DetectOptions.cmake | 2 +- source/adios2/engine/bp5/BP5Reader.cpp | 103 ++++++++++-------- source/adios2/engine/bp5/BP5Reader.h | 4 +- source/adios2/toolkit/cache/KVCacheCommon.h | 17 ++- source/adios2/toolkit/cache/KVCacheCommon.tcc | 19 ++-- source/adios2/toolkit/cache/QueryBox.h | 73 ++++++++----- 6 files changed, 125 insertions(+), 93 deletions(-) diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 1561752642..9e1958ce9d 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -608,7 +608,7 @@ endif() # KVCache if(ADIOS2_USE_Cache STREQUAL AUTO) - find_package(hiredis REQUIRED) + find_package(hiredis) if (hiredis_FOUND) message(STATUS "hiredis found. Turn on KVCache") set(ADIOS2_HAVE_KVCACHE TRUE) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 6b6a789241..982fa4ca38 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -350,7 +350,7 @@ void BP5Reader::PerformRemoteGets() auto GetRequests = m_BP5Deserializer->PendingGetRequests; std::vector handles; - #ifdef ADIOS2_HAVE_KVCACHE // open kv cache connection +#ifdef ADIOS2_HAVE_KVCACHE // open kv cache connection struct RequestInfo { size_t ReqSeq; @@ -363,24 +363,25 @@ void BP5Reader::PerformRemoteGets() void *Data; }; std::vector getRequestsInfo; - - if (getenv("useKVCache")) + + if (getenv("useKVCache")) { m_KVCacheCommon.openConnection(); } - #endif +#endif int req_seq = -1; for (auto &Req : GetRequests) { req_seq++; - #ifdef ADIOS2_HAVE_KVCACHE // get data from cache +#ifdef ADIOS2_HAVE_KVCACHE // get data from cache if (getenv("useKVCache")) { const DataType varType = m_IO.InquireVariableType(Req.VarName); QueryBox targetBox(Req.Start, Req.Count); size_t numOfElements = targetBox.size(); - std::string keyPrefix = m_KVCacheCommon.keyPrefix(Req.VarName, Req.RelStep, Req.BlockID); + std::string keyPrefix = + m_KVCacheCommon.keyPrefix(Req.VarName, Req.RelStep, Req.BlockID); std::string targetKey = m_KVCacheCommon.keyComposition(keyPrefix, Req.Start, Req.Count); // Exact Match: check if targetKey exists @@ -394,10 +395,11 @@ void BP5Reader::PerformRemoteGets() m_KVCacheCommon.get(targetKey, reqData); \ std::memcpy(Req.Data, reqData.data(), numOfElements * sizeof(T)); \ } - ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type_get) + ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type_get) #undef declare_type_get - - } else { + } + else + { int max_depth = 999; std::set samePrefixKeys; m_KVCacheCommon.keyPrefixExistence(keyPrefix, samePrefixKeys); @@ -411,51 +413,62 @@ void BP5Reader::PerformRemoteGets() } if (samePrefixKeys.size() > 0) - { - targetBox.getMaxInteractBox(samePrefixKeys, max_depth, 0, regularBoxes, cachedBoxes, cachedKeys); - } else { + { + targetBox.getMaxInteractBox(samePrefixKeys, max_depth, 0, regularBoxes, + cachedBoxes, cachedKeys); + } + else + { regularBoxes.push_back(targetBox); } - std::cout << "Going to retrieve "<< regularBoxes.size() << " boxes from remote server, and " << cachedBoxes.size() << " boxes from cache" << std::endl; - + std::cout << "Going to retrieve " << regularBoxes.size() + << " boxes from remote server, and " << cachedBoxes.size() + << " boxes from cache" << std::endl; #define declare_type_full_contain(T) \ if (varType == helper::GetDataType()) \ { \ const int typeSize = sizeof(T); \ - for (auto &box : regularBoxes){ \ + for (auto &box : regularBoxes) \ + { \ RequestInfo ReqInfo; \ ReqInfo.ReqSeq = req_seq; \ ReqInfo.varType = varType; \ ReqInfo.ReqCount = box.size(); \ ReqInfo.CacheKey = m_KVCacheCommon.keyComposition(keyPrefix, box.start, box.count); \ - ReqInfo.TypeSize = typeSize; \ - ReqInfo.Count = box.count; \ - ReqInfo.Start = box.start; \ - ReqInfo.Data = malloc(box.size() * sizeof(T)); \ - auto handle = m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, box.count, box.start, ReqInfo.Data); \ - handles.push_back(handle); \ - getRequestsInfo.push_back(ReqInfo); \ + ReqInfo.TypeSize = typeSize; \ + ReqInfo.Count = box.count; \ + ReqInfo.Start = box.start; \ + ReqInfo.Data = malloc(box.size() * sizeof(T)); \ + auto handle = m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, box.count, \ + box.start, ReqInfo.Data); \ + handles.push_back(handle); \ + getRequestsInfo.push_back(ReqInfo); \ } \ - for (int i = 0; i < cachedBoxes.size(); i++){ \ + for (int i = 0; i < cachedBoxes.size(); i++) \ + { \ std::string boxKey = cachedKeys[i]; \ QueryBox box(boxKey); \ std::vector srcData; \ srcData.resize(box.size()); \ m_KVCacheCommon.get(boxKey, srcData); \ - helper::NdCopy(reinterpret_cast(srcData.data()), helper::CoreDims(cachedBoxes[i].start), cachedBoxes[i].count, true, false, reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, typeSize); \ + helper::NdCopy(reinterpret_cast(srcData.data()), \ + helper::CoreDims(cachedBoxes[i].start), cachedBoxes[i].count, true, \ + false, reinterpret_cast(Req.Data), Req.Start, Req.Count, true, \ + false, typeSize); \ } \ } -ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type_full_contain) + ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type_full_contain) #undef declare_type_full_contain } continue; } - #endif +#endif - auto handle = m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, Req.Count, Req.Start, Req.Data); + auto handle = + m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, Req.Count, Req.Start, Req.Data); handles.push_back(handle); } @@ -464,35 +477,37 @@ ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type_full_contain) { handle_seq++; m_Remote->WaitForGet(handle); - #ifdef ADIOS2_HAVE_KVCACHE // close cache connection +#ifdef ADIOS2_HAVE_KVCACHE // close cache connection if (getenv("useKVCache")) { auto &ReqInfo = getRequestsInfo[handle_seq]; auto &Req = GetRequests[ReqInfo.ReqSeq]; - helper::NdCopy(reinterpret_cast(ReqInfo.Data), helper::CoreDims(ReqInfo.Start), ReqInfo.Count, true, false, reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, ReqInfo.TypeSize); - - #define declare_type_set(T) \ - if (ReqInfo.varType == helper::GetDataType()) \ - { \ - std::vector reqData; \ - reqData.resize(ReqInfo.ReqCount); \ - std::memcpy(reqData.data(), Req.Data, ReqInfo.ReqCount * sizeof(T)); \ - m_KVCacheCommon.set(ReqInfo.CacheKey, reqData); \ - } - ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type_set) - #undef declare_type_set - + helper::NdCopy(reinterpret_cast(ReqInfo.Data), helper::CoreDims(ReqInfo.Start), + ReqInfo.Count, true, false, reinterpret_cast(Req.Data), + Req.Start, Req.Count, true, false, ReqInfo.TypeSize); + +#define declare_type_set(T) \ + if (ReqInfo.varType == helper::GetDataType()) \ + { \ + std::vector reqData; \ + reqData.resize(ReqInfo.ReqCount); \ + std::memcpy(reqData.data(), Req.Data, ReqInfo.ReqCount * sizeof(T)); \ + m_KVCacheCommon.set(ReqInfo.CacheKey, reqData); \ + } + ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type_set) +#undef declare_type_set + free(ReqInfo.Data); } - #endif +#endif } - #ifdef ADIOS2_HAVE_KVCACHE // close cache connection +#ifdef ADIOS2_HAVE_KVCACHE // close cache connection if (getenv("useKVCache")) { m_KVCacheCommon.closeConnection(); } - #endif +#endif } void BP5Reader::PerformLocalGets() diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h index 488c332c22..fc9fa68e7b 100644 --- a/source/adios2/engine/bp5/BP5Reader.h +++ b/source/adios2/engine/bp5/BP5Reader.h @@ -102,9 +102,9 @@ class BP5Reader : public BP5Engine, public Engine std::unique_ptr m_Remote; bool m_WriterIsActive = true; adios2::profiling::JSONProfiler m_JSONProfiler; - #ifdef ADIOS2_HAVE_KVCACHE +#ifdef ADIOS2_HAVE_KVCACHE KVCacheCommon m_KVCacheCommon; - #endif +#endif /** used for per-step reads, TODO: to be moved to BP5Deserializer */ size_t m_CurrentStep = 0; size_t m_StepsCount = 0; diff --git a/source/adios2/toolkit/cache/KVCacheCommon.h b/source/adios2/toolkit/cache/KVCacheCommon.h index d21b9fa178..8ee35f57e9 100644 --- a/source/adios2/toolkit/cache/KVCacheCommon.h +++ b/source/adios2/toolkit/cache/KVCacheCommon.h @@ -4,12 +4,12 @@ #ifndef ADIOS2_KVCACHECOMMON_H #define ADIOS2_KVCACHECOMMON_H -#include #include "adios2/toolkit/cache/QueryBox.h" #include -#include // For memcpy -#include +#include // For memcpy +#include #include +#include // namespace adios2::KVCache @@ -27,17 +27,17 @@ class KVCacheCommon std::string m_value; std::string m_command; - KVCacheCommon(std::string host="localhost", int port=6379): m_host(host), m_port(port){}; + KVCacheCommon(std::string host = "localhost", int port = 6379) : m_host(host), m_port(port){}; inline void openConnection(); inline void closeConnection(); template - void set(std::string key, const std::vector& vec); + void set(std::string key, const std::vector &vec); template - void get(std::string key, std::vector& vec); + void get(std::string key, std::vector &vec); inline void del(std::string key); @@ -50,13 +50,12 @@ class KVCacheCommon inline void keyPrefixExistence(const std::string &key_prefix, std::set &keys); template - void encodeVector(const std::vector& vec, std::string& encodedString); + void encodeVector(const std::vector &vec, std::string &encodedString); template - void decodeVector(const std::string& str, std::vector& vec); + void decodeVector(const std::string &str, std::vector &vec); }; - }; // adios2 #include "KVCacheCommon.tcc" diff --git a/source/adios2/toolkit/cache/KVCacheCommon.tcc b/source/adios2/toolkit/cache/KVCacheCommon.tcc index ad584f0610..b9bf3b80ec 100644 --- a/source/adios2/toolkit/cache/KVCacheCommon.tcc +++ b/source/adios2/toolkit/cache/KVCacheCommon.tcc @@ -32,7 +32,7 @@ void KVCacheCommon::closeConnection() } template -void KVCacheCommon::set(std::string key, const std::vector& vec) +void KVCacheCommon::set(std::string key, const std::vector &vec) { encodeVector(vec, m_value); m_command = "SET " + key + " " + m_value; @@ -49,7 +49,7 @@ void KVCacheCommon::set(std::string key, const std::vector& vec) } template -void KVCacheCommon::get(std::string key, std::vector& vec) +void KVCacheCommon::get(std::string key, std::vector &vec) { m_command = "GET " + key; m_redisReply = (redisReply *)redisCommand(m_redisContext, m_command.c_str()); @@ -140,9 +140,10 @@ void KVCacheCommon::keyPrefixExistence(const std::string &key_prefix, std::set -void KVCacheCommon::encodeVector(const std::vector& vec, std::string& encodedString) { +void KVCacheCommon::encodeVector(const std::vector &vec, std::string &encodedString) +{ size_t vecSize = vec.size() * sizeof(T); - const unsigned char* vecBytes = reinterpret_cast(vec.data()); + const unsigned char *vecBytes = reinterpret_cast(vec.data()); size_t encodedSize = vecSize * 3 / 2; std::vector encodedBytes(encodedSize); @@ -157,11 +158,14 @@ void KVCacheCommon::encodeVector(const std::vector& vec, std::string& encoded } template -void KVCacheCommon::decodeVector(const std::string& str, std::vector& vec) { +void KVCacheCommon::decodeVector(const std::string &str, std::vector &vec) +{ size_t decodedSize = str.size() * 2; std::vector decodedBytes(decodedSize); - size_t sizeAfterDecoded = adios2sysBase64_Decode(reinterpret_cast(str.data()), str.size(), decodedBytes.data(), decodedSize); + size_t sizeAfterDecoded = + adios2sysBase64_Decode(reinterpret_cast(str.data()), str.size(), + decodedBytes.data(), decodedSize); // Resize the vector to the actual size decodedBytes.resize(sizeAfterDecoded); @@ -169,8 +173,7 @@ void KVCacheCommon::decodeVector(const std::string& str, std::vector& vec) { // Copy the decoded bytes to the vector vec.resize(sizeAfterDecoded / sizeof(T)); memcpy(vec.data(), decodedBytes.data(), sizeAfterDecoded); - } -}; // namespace adios2 +}; // namespace adios2 #endif // KVCACHECOMMON_TCC \ No newline at end of file diff --git a/source/adios2/toolkit/cache/QueryBox.h b/source/adios2/toolkit/cache/QueryBox.h index 3f3f33458a..cd63590219 100644 --- a/source/adios2/toolkit/cache/QueryBox.h +++ b/source/adios2/toolkit/cache/QueryBox.h @@ -5,10 +5,10 @@ #ifndef ADIOS2_KVCACHE_QUERYBOX_H #define ADIOS2_KVCACHE_QUERYBOX_H -#include -#include #include +#include #include +#include namespace adios2 { @@ -22,17 +22,21 @@ class QueryBox // constructor QueryBox() = default; QueryBox(const adios2::Dims &start, const adios2::Dims &count) : start(start), count(count){}; - QueryBox(const std::string &key){ - // sample key: "U3218446744073709551615__count_:_64_64_64___start_:_0_0_0__", count [64, 64, 64], start [0, 0, 0] - // using Dims = std::vector; - auto lf_ExtractDimensions = [](const std::string &key, const std::string &delimiter) -> Dims { + QueryBox(const std::string &key) + { + // sample key: "U3218446744073709551615__count_:_64_64_64___start_:_0_0_0__", count [64, 64, + // 64], start [0, 0, 0] using Dims = std::vector; + auto lf_ExtractDimensions = [](const std::string &key, + const std::string &delimiter) -> Dims { size_t const pos = key.find(delimiter); size_t const end = key.find("__", pos + delimiter.length()); - std::string dimStr = key.substr(pos + delimiter.length(), end - pos - delimiter.length()); + std::string dimStr = + key.substr(pos + delimiter.length(), end - pos - delimiter.length()); Dims dimensions; std::istringstream dimStream(dimStr); std::string token; - while (std::getline(dimStream, token, '_')) { + while (std::getline(dimStream, token, '_')) + { dimensions.push_back(std::stoul(token)); } return dimensions; @@ -63,13 +67,11 @@ class QueryBox } // determine if a query box is equal to another query box - bool operator==(const QueryBox &box) const - { - return start == box.start && count == box.count; - } + bool operator==(const QueryBox &box) const { return start == box.start && count == box.count; } - // determine if a query box is interacted in another query box, return intersection part as a new query box - bool isInteracted (const QueryBox &box, QueryBox &intersection) const + // determine if a query box is interacted in another query box, return intersection part as a + // new query box + bool isInteracted(const QueryBox &box, QueryBox &intersection) const { if (start.size() != box.start.size() || start.size() != count.size() || start.size() != box.count.size()) @@ -130,8 +132,10 @@ class QueryBox { continue; } - else { - if (start[i] != remainingBox.start[i]){ + else + { + if (start[i] != remainingBox.start[i]) + { size_t cutDimDiff = start[i] - remainingBox.start[i]; size_t cutDimSize = remainingBox.size() / remainingBox.count[i] * cutDimDiff; if (cutDimSize > maxCutDimSize) @@ -142,8 +146,10 @@ class QueryBox } } - if (start[i] + count[i] != remainingBox.start[i] + remainingBox.count[i]){ - size_t cutDimDiff = remainingBox.start[i] + remainingBox.count[i] - start[i] - count[i]; + if (start[i] + count[i] != remainingBox.start[i] + remainingBox.count[i]) + { + size_t cutDimDiff = + remainingBox.start[i] + remainingBox.count[i] - start[i] - count[i]; size_t cutDimSize = remainingBox.size() / count[i] * cutDimDiff; if (cutDimSize > maxCutDimSize) { @@ -163,26 +169,32 @@ class QueryBox QueryBox remainingBox1 = QueryBox(remainingBox.start, remainingBox.count); for (size_t i = 0; i < remainingBox.start.size(); ++i) { - if (maxCutDimBox.start[i] == remainingBox.start[i] && maxCutDimBox.count[i] == remainingBox.count[i]) + if (maxCutDimBox.start[i] == remainingBox.start[i] && + maxCutDimBox.count[i] == remainingBox.count[i]) { continue; } - else { + else + { if (maxCutDimBox.start[i] != remainingBox.start[i]) { remainingBox1.count[i] = maxCutDimBox.start[i] - remainingBox.start[i]; - } else { + } + else + { remainingBox1.start[i] = maxCutDimBox.start[i] + maxCutDimBox.count[i]; - remainingBox1.count[i] = remainingBox.start[i] + remainingBox.count[i] - remainingBox1.start[i]; + remainingBox1.count[i] = + remainingBox.start[i] + remainingBox.count[i] - remainingBox1.start[i]; } - } } interactionCut(remainingBox1, regularBoxes); } } - void getMaxInteractBox(const std::set &samePrefixKeys, const size_t &max_depth, size_t current_depth, std::vector ®ularBoxes, std::vector &cachedBox, std::vector &cachedKeys) + void getMaxInteractBox(const std::set &samePrefixKeys, const size_t &max_depth, + size_t current_depth, std::vector ®ularBoxes, + std::vector &cachedBox, std::vector &cachedKeys) { if (current_depth > max_depth) { @@ -193,7 +205,8 @@ class QueryBox std::string maxInteractKey; for (auto &key : samePrefixKeys) { - // std::cout << "Same Prefix Keys: " << key << " Current Depth: " << current_depth << std::endl; + // std::cout << "Same Prefix Keys: " << key << " Current Depth: " << current_depth << + // std::endl; QueryBox const box(key); QueryBox intersection; if (this->isInteracted(box, intersection)) @@ -217,14 +230,17 @@ class QueryBox if (current_depth == max_depth) { maxInteractBox.interactionCut(*this, regularBoxes); - } else { + } + else + { std::vector nextBoxes; maxInteractBox.interactionCut(*this, nextBoxes); for (auto &box : nextBoxes) { - box.getMaxInteractBox(samePrefixKeys, max_depth, current_depth, regularBoxes, cachedBox, cachedKeys); + box.getMaxInteractBox(samePrefixKeys, max_depth, current_depth, regularBoxes, + cachedBox, cachedKeys); } - } + } } // rewrite toString @@ -251,7 +267,6 @@ class QueryBox str += "]"; return str; } - }; }; #endif // UNITTEST_QUERYBOX_H From 9812eefe82a3eee1d4588ae2c91f71d576af144a Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Mon, 24 Jun 2024 15:45:36 -0400 Subject: [PATCH 04/22] update build-adios2-kvcache.sh for formatting and replace git clone with curl, also fix .gitignore and CMakeLists.txt --- .gitignore | 2 - CMakeLists.txt | 1 + scripts/build_scripts/build-adios2-kvcache.sh | 50 ++++++++++--------- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 10d33af858..ab5d48025c 100644 --- a/.gitignore +++ b/.gitignore @@ -47,8 +47,6 @@ CMakeSettings.json # CMake generated files build/ -build-cache*/ -cmake-build-*/ .idea/ .vscode/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c54c69d57..aaf9a18af7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -189,6 +189,7 @@ adios_option(AWSSDK "Enable support for S3 compatible storage using AWS SDK' adios_option(Derived_Variable "Enable support for derived variables" OFF) adios_option(PIP "Enable support for pip packaging" OFF) adios_option(XRootD "Enable support for XRootD" AUTO) +adios_option(KVCACHE "Enable support for KVCache" AUTO) option(ADIOS2_LIBADIOS_MODE "Install only C/C++ library components" OFF) mark_as_advanced(ADIOS2_LIBADIOS_MODE) diff --git a/scripts/build_scripts/build-adios2-kvcache.sh b/scripts/build_scripts/build-adios2-kvcache.sh index db5ecab5c8..a88212f19f 100644 --- a/scripts/build_scripts/build-adios2-kvcache.sh +++ b/scripts/build_scripts/build-adios2-kvcache.sh @@ -9,50 +9,54 @@ # source scripts/build_scripts/build-adios2-kvcache.sh --start # source scripts/build_scripts/build-adios2-kvcache.sh --stop -if [ -z ${BUILD_DIR} ] +if [ -z "${BUILD_DIR}" ] then - BUILD_DIR=${PWD}/build-cache-test + BUILD_DIR="${PWD}"/build fi -if [ ! -d ${BUILD_DIR} ] +if [ ! -d "${BUILD_DIR}" ] then - mkdir -p ${BUILD_DIR} + mkdir -p "${BUILD_DIR}" fi -SW_DIR=${BUILD_DIR}/sw -if [ ! -d ${SW_DIR} ] +SW_DIR="${BUILD_DIR}"/sw +if [ ! -d "${SW_DIR}" ] then - mkdir -p ${SW_DIR} + mkdir -p "${SW_DIR}" fi build_cache() { # redis - in-memory data structure store - redis_dir=${SW_DIR}/redis - if [ ! -d ${redis_dir} ] + redis_dir="${SW_DIR}"/redis-7.2.3 + if [ ! -d "${redis_dir}" ] then - git clone https://github.com/redis/redis.git ${redis_dir} - cd ${redis_dir} - git checkout tags/7.2.3 + cd "${SW_DIR}" || exit + curl -L -o redis-7.2.3.tar.gz https://github.com/redis/redis/archive/refs/tags/7.2.3.tar.gz + tar -xzf redis-7.2.3.tar.gz + rm redis-7.2.3.tar.gz + cd "${redis_dir}" || exit # cannot accleerate by 'make -j8'. It will cause error. make # hiredis - C client library to connect Redis server - cd ${redis_dir}/deps/hiredis - mkdir build && cd build - cmake .. -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hiredis + cd "${redis_dir}"/deps/hiredis || exit + mkdir build && cd build || exit + cmake .. -DCMAKE_INSTALL_PREFIX="${SW_DIR}"/hiredis make -j32 make install fi - cd ${BUILD_DIR} + cd "${BUILD_DIR}" || exit cmake .. -DADIOS2_USE_Cache=ON \ -DADIOS2_USE_Python=ON \ - -DCMAKE_PREFIX_PATH=${SW_DIR} \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2 + -DCMAKE_PREFIX_PATH="${SW_DIR}" \ + -DCMAKE_INSTALL_PREFIX="${SW_DIR}"/adios2 make -j32 make install - cd ${BUILD_DIR}/../ + cd "${BUILD_DIR}"/../ || exit + echo "Build completed." + unset BUILD_DIR } start_services() { @@ -60,11 +64,10 @@ start_services() { export DoRemote=1 export useKVCache=1 export PYTHONPATH=${SW_DIR}/adios2/local/lib/python3.10/dist-packages/ - # export LD_LIBRARY_PATH=${SW_DIR}/adios2/lib:${SW_DIR}/hiredis/lib:$LD_LIBRARY_PATH - nohup ${SW_DIR}/redis/src/redis-server > ${SW_DIR}redis_server.log 2>&1 & - nohup ${SW_DIR}/adios2/bin/adios2_remote_server -v > ${SW_DIR}remote_server.log 2>&1 & + nohup "${SW_DIR}"/redis-7.2.3/src/redis-server > "${SW_DIR}"/redis_server.log 2>&1 & + nohup "${SW_DIR}"/adios2/bin/adios2_remote_server -v > "${SW_DIR}"/remote_server.log 2>&1 & sleep 5 - nohup ${SW_DIR}/redis/src/redis-cli monitor > ${SW_DIR}redis_monitor.log 2>&1 & + nohup "${SW_DIR}"/redis-7.2.3/src/redis-cli monitor > "${SW_DIR}"/redis_monitor.log 2>&1 & echo "Services started and environment variables set." } @@ -77,7 +80,6 @@ stop_services() { unset DoRemote unset useKVCache unset PYTHONPATH - unset LD_LIBRARY_PATH echo "Services stopped." } From 6d1bf17c69e22077c3917f9506e009a684d53bfb Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Mon, 24 Jun 2024 16:03:27 -0400 Subject: [PATCH 05/22] fix link issue and remove rpath setting in CMakeLists.txt --- scripts/build_scripts/build-adios2-kvcache.sh | 2 +- source/adios2/CMakeLists.txt | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/scripts/build_scripts/build-adios2-kvcache.sh b/scripts/build_scripts/build-adios2-kvcache.sh index a88212f19f..b658196f49 100644 --- a/scripts/build_scripts/build-adios2-kvcache.sh +++ b/scripts/build_scripts/build-adios2-kvcache.sh @@ -49,7 +49,7 @@ build_cache() { cd "${BUILD_DIR}" || exit cmake .. -DADIOS2_USE_Cache=ON \ -DADIOS2_USE_Python=ON \ - -DCMAKE_PREFIX_PATH="${SW_DIR}" \ + -DCMAKE_PREFIX_PATH="${SW_DIR}/hiredis" \ -DCMAKE_INSTALL_PREFIX="${SW_DIR}"/adios2 make -j32 diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 2f2590d700..f262d3b138 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -252,13 +252,6 @@ if (ADIOS2_HAVE_KVCACHE) target_sources(adios2_core PRIVATE toolkit/cache/KVCacheCommon.h toolkit/cache/KVCacheCommon.tcc) target_link_libraries(adios2_core PRIVATE hiredis::hiredis) - - # Ensure the rpath is set correctly - # message(STATUS "PACKAGE_PREFIX_DIR: ${PACKAGE_PREFIX_DIR}") - set_target_properties(adios2_core PROPERTIES - BUILD_RPATH "$ORIGIN:${PACKAGE_PREFIX_DIR}/lib" - INSTALL_RPATH "$ORIGIN:${PACKAGE_PREFIX_DIR}/lib" - ) endif () if(ADIOS2_HAVE_Campaign) From e07c30b6d0a728e32ef6ec7d5f7fff40a8b5d112 Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Tue, 25 Jun 2024 09:03:03 -0400 Subject: [PATCH 06/22] solve unused warnings --- source/adios2/engine/bp5/BP5Reader.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 982fa4ca38..6226989e1c 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -370,10 +370,9 @@ void BP5Reader::PerformRemoteGets() } #endif - int req_seq = -1; - for (auto &Req : GetRequests) + for(size_t req_seq = 0; req_seq < GetRequests.size(); req_seq++) { - req_seq++; + auto &Req = GetRequests[req_seq]; #ifdef ADIOS2_HAVE_KVCACHE // get data from cache if (getenv("useKVCache")) { @@ -472,10 +471,9 @@ void BP5Reader::PerformRemoteGets() handles.push_back(handle); } - std::size_t handle_seq = -1; - for (auto &handle : handles) + for (size_t handle_seq = 0; handle_seq < handles.size(); handle_seq++) { - handle_seq++; + auto handle = handles[handle_seq]; m_Remote->WaitForGet(handle); #ifdef ADIOS2_HAVE_KVCACHE // close cache connection if (getenv("useKVCache")) From 398a0fb3ae17485cf62db5d5895004c229db89f3 Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Tue, 25 Jun 2024 09:09:22 -0400 Subject: [PATCH 07/22] formatting --- source/adios2/engine/bp5/BP5Reader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 6226989e1c..c509ae9340 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -370,7 +370,7 @@ void BP5Reader::PerformRemoteGets() } #endif - for(size_t req_seq = 0; req_seq < GetRequests.size(); req_seq++) + for (size_t req_seq = 0; req_seq < GetRequests.size(); req_seq++) { auto &Req = GetRequests[req_seq]; #ifdef ADIOS2_HAVE_KVCACHE // get data from cache From 2afdca6eab7fb6e632c837064be110c80920152c Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Tue, 25 Jun 2024 09:17:25 -0400 Subject: [PATCH 08/22] move DoRemote and useKVCache declaration --- scripts/build_scripts/build-adios2-kvcache.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/build_scripts/build-adios2-kvcache.sh b/scripts/build_scripts/build-adios2-kvcache.sh index b658196f49..4d99867412 100644 --- a/scripts/build_scripts/build-adios2-kvcache.sh +++ b/scripts/build_scripts/build-adios2-kvcache.sh @@ -57,12 +57,13 @@ build_cache() { cd "${BUILD_DIR}"/../ || exit echo "Build completed." unset BUILD_DIR + + export DoRemote=1 + export useKVCache=1 } start_services() { echo "Starting redis server and setting environment variables..." - export DoRemote=1 - export useKVCache=1 export PYTHONPATH=${SW_DIR}/adios2/local/lib/python3.10/dist-packages/ nohup "${SW_DIR}"/redis-7.2.3/src/redis-server > "${SW_DIR}"/redis_server.log 2>&1 & nohup "${SW_DIR}"/adios2/bin/adios2_remote_server -v > "${SW_DIR}"/remote_server.log 2>&1 & From 5790fcd2d8c079de0ad6dca01975a93e22d55697 Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Wed, 26 Jun 2024 11:20:08 -0400 Subject: [PATCH 09/22] remove all template in cache part & remove using of KWSYS Base64 --- source/adios2/CMakeLists.txt | 4 +- source/adios2/engine/bp5/BP5Reader.cpp | 102 +++++++----------- source/adios2/engine/bp5/BP5Reader.h | 2 +- source/adios2/toolkit/cache/KVCacheCommon.h | 63 ----------- .../KVCacheCommon.cpp} | 61 ++--------- source/adios2/toolkit/kvcache/KVCacheCommon.h | 52 +++++++++ .../toolkit/{cache => kvcache}/QueryBox.h | 10 +- thirdparty/KWSys/CMakeLists.txt | 1 - 8 files changed, 109 insertions(+), 186 deletions(-) delete mode 100644 source/adios2/toolkit/cache/KVCacheCommon.h rename source/adios2/toolkit/{cache/KVCacheCommon.tcc => kvcache/KVCacheCommon.cpp} (64%) create mode 100644 source/adios2/toolkit/kvcache/KVCacheCommon.h rename source/adios2/toolkit/{cache => kvcache}/QueryBox.h (96%) diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index f262d3b138..dc66a9975b 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -249,8 +249,8 @@ if (ADIOS2_HAVE_SST) endif() if (ADIOS2_HAVE_KVCACHE) - target_sources(adios2_core PRIVATE toolkit/cache/KVCacheCommon.h - toolkit/cache/KVCacheCommon.tcc) + target_sources(adios2_core PRIVATE toolkit/kvcache/KVCacheCommon.h + toolkit/kvcache/KVCacheCommon.cpp) target_link_libraries(adios2_core PRIVATE hiredis::hiredis) endif () diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index c509ae9340..6eb3a54811 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -382,20 +382,12 @@ void BP5Reader::PerformRemoteGets() std::string keyPrefix = m_KVCacheCommon.keyPrefix(Req.VarName, Req.RelStep, Req.BlockID); std::string targetKey = m_KVCacheCommon.keyComposition(keyPrefix, Req.Start, Req.Count); + size_t varSize = helper::GetDataTypeSize(varType); // Exact Match: check if targetKey exists if (m_KVCacheCommon.exists(targetKey)) { -#define declare_type_get(T) \ - if (varType == helper::GetDataType()) \ - { \ - std::vector reqData; \ - reqData.resize(numOfElements); \ - m_KVCacheCommon.get(targetKey, reqData); \ - std::memcpy(Req.Data, reqData.data(), numOfElements * sizeof(T)); \ - } - ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type_get) -#undef declare_type_get + m_KVCacheCommon.get(targetKey.c_str(), numOfElements * varSize, Req.Data); } else { @@ -403,7 +395,6 @@ void BP5Reader::PerformRemoteGets() std::set samePrefixKeys; m_KVCacheCommon.keyPrefixExistence(keyPrefix, samePrefixKeys); std::vector regularBoxes; - std::vector cachedBoxes; std::vector cachedKeys; if (getenv("maxDepth")) @@ -414,7 +405,7 @@ void BP5Reader::PerformRemoteGets() if (samePrefixKeys.size() > 0) { targetBox.getMaxInteractBox(samePrefixKeys, max_depth, 0, regularBoxes, - cachedBoxes, cachedKeys); + cachedKeys); } else { @@ -422,44 +413,39 @@ void BP5Reader::PerformRemoteGets() } std::cout << "Going to retrieve " << regularBoxes.size() - << " boxes from remote server, and " << cachedBoxes.size() + << " boxes from remote server, and " << cachedKeys.size() << " boxes from cache" << std::endl; -#define declare_type_full_contain(T) \ - if (varType == helper::GetDataType()) \ - { \ - const int typeSize = sizeof(T); \ - for (auto &box : regularBoxes) \ - { \ - RequestInfo ReqInfo; \ - ReqInfo.ReqSeq = req_seq; \ - ReqInfo.varType = varType; \ - ReqInfo.ReqCount = box.size(); \ - ReqInfo.CacheKey = m_KVCacheCommon.keyComposition(keyPrefix, box.start, box.count); \ - ReqInfo.TypeSize = typeSize; \ - ReqInfo.Count = box.count; \ - ReqInfo.Start = box.start; \ - ReqInfo.Data = malloc(box.size() * sizeof(T)); \ - auto handle = m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, box.count, \ - box.start, ReqInfo.Data); \ - handles.push_back(handle); \ - getRequestsInfo.push_back(ReqInfo); \ - } \ - for (int i = 0; i < cachedBoxes.size(); i++) \ - { \ - std::string boxKey = cachedKeys[i]; \ - QueryBox box(boxKey); \ - std::vector srcData; \ - srcData.resize(box.size()); \ - m_KVCacheCommon.get(boxKey, srcData); \ - helper::NdCopy(reinterpret_cast(srcData.data()), \ - helper::CoreDims(cachedBoxes[i].start), cachedBoxes[i].count, true, \ - false, reinterpret_cast(Req.Data), Req.Start, Req.Count, true, \ - false, typeSize); \ - } \ - } - ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type_full_contain) -#undef declare_type_full_contain + // Get data from remote server + for (auto &box : regularBoxes) + { + RequestInfo ReqInfo; + ReqInfo.ReqSeq = req_seq; + ReqInfo.varType = varType; + ReqInfo.ReqCount = box.size(); + ReqInfo.CacheKey = + m_KVCacheCommon.keyComposition(keyPrefix, box.start, box.count); + ReqInfo.TypeSize = varSize; + ReqInfo.Count = box.count; + ReqInfo.Start = box.start; + ReqInfo.Data = malloc(box.size() * varSize); + auto handle = m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, box.count, + box.start, ReqInfo.Data); + handles.push_back(handle); + getRequestsInfo.push_back(ReqInfo); + } + + // Get data from cache + for (auto &boxKey : cachedKeys) + { + QueryBox box(boxKey); + void *data = malloc(box.size() * varSize); + m_KVCacheCommon.get(boxKey.c_str(), box.size() * varSize, data); + helper::NdCopy(reinterpret_cast(data), box.start, box.count, true, + false, reinterpret_cast(Req.Data), Req.Start, Req.Count, + true, false, varSize); + free(data); + } } continue; @@ -480,21 +466,11 @@ void BP5Reader::PerformRemoteGets() { auto &ReqInfo = getRequestsInfo[handle_seq]; auto &Req = GetRequests[ReqInfo.ReqSeq]; - helper::NdCopy(reinterpret_cast(ReqInfo.Data), helper::CoreDims(ReqInfo.Start), - ReqInfo.Count, true, false, reinterpret_cast(Req.Data), - Req.Start, Req.Count, true, false, ReqInfo.TypeSize); - -#define declare_type_set(T) \ - if (ReqInfo.varType == helper::GetDataType()) \ - { \ - std::vector reqData; \ - reqData.resize(ReqInfo.ReqCount); \ - std::memcpy(reqData.data(), Req.Data, ReqInfo.ReqCount * sizeof(T)); \ - m_KVCacheCommon.set(ReqInfo.CacheKey, reqData); \ - } - ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type_set) -#undef declare_type_set - + helper::NdCopy(reinterpret_cast(ReqInfo.Data), ReqInfo.Start, ReqInfo.Count, + true, false, reinterpret_cast(Req.Data), Req.Start, Req.Count, + true, false, ReqInfo.TypeSize); + m_KVCacheCommon.set(ReqInfo.CacheKey.c_str(), ReqInfo.ReqCount * ReqInfo.TypeSize, + ReqInfo.Data); free(ReqInfo.Data); } #endif diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h index fc9fa68e7b..9eba7b70fc 100644 --- a/source/adios2/engine/bp5/BP5Reader.h +++ b/source/adios2/engine/bp5/BP5Reader.h @@ -21,7 +21,7 @@ #include "adios2/toolkit/transportman/TransportMan.h" #ifdef ADIOS2_HAVE_KVCACHE -#include "adios2/toolkit/cache/KVCacheCommon.h" +#include "adios2/toolkit/kvcache/KVCacheCommon.h" #endif #include diff --git a/source/adios2/toolkit/cache/KVCacheCommon.h b/source/adios2/toolkit/cache/KVCacheCommon.h deleted file mode 100644 index 8ee35f57e9..0000000000 --- a/source/adios2/toolkit/cache/KVCacheCommon.h +++ /dev/null @@ -1,63 +0,0 @@ -// -// Created by cguo51 on 12/30/23. -// - -#ifndef ADIOS2_KVCACHECOMMON_H -#define ADIOS2_KVCACHECOMMON_H -#include "adios2/toolkit/cache/QueryBox.h" -#include -#include // For memcpy -#include -#include -#include - -// namespace adios2::KVCache - -namespace adios2 -{ - -class KVCacheCommon -{ -public: - std::string m_host; - int m_port; - redisContext *m_redisContext; - redisReply *m_redisReply; - std::string m_key; - std::string m_value; - std::string m_command; - - KVCacheCommon(std::string host = "localhost", int port = 6379) : m_host(host), m_port(port){}; - - inline void openConnection(); - - inline void closeConnection(); - - template - void set(std::string key, const std::vector &vec); - - template - void get(std::string key, std::vector &vec); - - inline void del(std::string key); - - inline bool exists(std::string key); - - inline std::string keyPrefix(char *VarName, size_t AbsStep, size_t BlockID); - - inline std::string keyComposition(const std::string &key_prefix, Dims Start, Dims Count); - - inline void keyPrefixExistence(const std::string &key_prefix, std::set &keys); - - template - void encodeVector(const std::vector &vec, std::string &encodedString); - - template - void decodeVector(const std::string &str, std::vector &vec); -}; - -}; // adios2 - -#include "KVCacheCommon.tcc" - -#endif // ADIOS2_KVCACHECOMMON_H diff --git a/source/adios2/toolkit/cache/KVCacheCommon.tcc b/source/adios2/toolkit/kvcache/KVCacheCommon.cpp similarity index 64% rename from source/adios2/toolkit/cache/KVCacheCommon.tcc rename to source/adios2/toolkit/kvcache/KVCacheCommon.cpp index b9bf3b80ec..b845158198 100644 --- a/source/adios2/toolkit/cache/KVCacheCommon.tcc +++ b/source/adios2/toolkit/kvcache/KVCacheCommon.cpp @@ -1,8 +1,10 @@ // // Created by cguo51 on 12/30/23. // -#ifndef KVCACHECOMMON_TCC -#define KVCACHECOMMON_TCC +#ifndef KVCACHECOMMON_CPP +#define KVCACHECOMMON_CPP + +#include "KVCacheCommon.h" namespace adios2 { @@ -31,35 +33,30 @@ void KVCacheCommon::closeConnection() std::cout << "KVCache connection closed" << std::endl; } -template -void KVCacheCommon::set(std::string key, const std::vector &vec) +void KVCacheCommon::set(const char *key, size_t size, void *data) { - encodeVector(vec, m_value); - m_command = "SET " + key + " " + m_value; - m_redisReply = (redisReply *)redisCommand(m_redisContext, m_command.c_str()); + m_redisReply = (redisReply *)redisCommand(m_redisContext, "SET %s %b", key, data, size); if (m_redisReply == NULL) { std::cout << "Error to set key: " << key << std::endl; } else { - std::cout << "SET Key: " << key << " Value size: " << vec.size() << std::endl; + std::cout << "SET Key: " << key << " Value size: " << size << std::endl; freeReplyObject(m_redisReply); } } -template -void KVCacheCommon::get(std::string key, std::vector &vec) +void KVCacheCommon::get(const char *key, size_t size, void *data) { - m_command = "GET " + key; - m_redisReply = (redisReply *)redisCommand(m_redisContext, m_command.c_str()); + m_redisReply = (redisReply *)redisCommand(m_redisContext, "GET %s", key); if (m_redisReply == NULL) { std::cout << "Error to get key: " << key << std::endl; } else { - decodeVector(m_redisReply->str, vec); + memcpy(data, m_redisReply->str, size); freeReplyObject(m_redisReply); } } @@ -139,41 +136,5 @@ void KVCacheCommon::keyPrefixExistence(const std::string &key_prefix, std::set -void KVCacheCommon::encodeVector(const std::vector &vec, std::string &encodedString) -{ - size_t vecSize = vec.size() * sizeof(T); - const unsigned char *vecBytes = reinterpret_cast(vec.data()); - - size_t encodedSize = vecSize * 3 / 2; - std::vector encodedBytes(encodedSize); - - size_t sizeAfterEncoded = adios2sysBase64_Encode(vecBytes, vecSize, encodedBytes.data(), 0); - - // Resize the vector to the actual size - encodedBytes.resize(sizeAfterEncoded); - - // Convert the encoded bytes to a string - encodedString.assign(encodedBytes.begin(), encodedBytes.end()); -} - -template -void KVCacheCommon::decodeVector(const std::string &str, std::vector &vec) -{ - size_t decodedSize = str.size() * 2; - std::vector decodedBytes(decodedSize); - - size_t sizeAfterDecoded = - adios2sysBase64_Decode(reinterpret_cast(str.data()), str.size(), - decodedBytes.data(), decodedSize); - - // Resize the vector to the actual size - decodedBytes.resize(sizeAfterDecoded); - - // Copy the decoded bytes to the vector - vec.resize(sizeAfterDecoded / sizeof(T)); - memcpy(vec.data(), decodedBytes.data(), sizeAfterDecoded); -} - }; // namespace adios2 -#endif // KVCACHECOMMON_TCC \ No newline at end of file +#endif // KVCACHECOMMON_CPP \ No newline at end of file diff --git a/source/adios2/toolkit/kvcache/KVCacheCommon.h b/source/adios2/toolkit/kvcache/KVCacheCommon.h new file mode 100644 index 0000000000..6efbf28add --- /dev/null +++ b/source/adios2/toolkit/kvcache/KVCacheCommon.h @@ -0,0 +1,52 @@ +// +// Created by cguo51 on 12/30/23. +// + +#ifndef ADIOS2_KVCACHECOMMON_H +#define ADIOS2_KVCACHECOMMON_H +#include "QueryBox.h" +#include // For memcpy +#include +#include +#include + +// namespace adios2::KVCache + +namespace adios2 +{ + +class KVCacheCommon +{ +public: + std::string m_host; + int m_port; + redisContext *m_redisContext; + redisReply *m_redisReply; + std::string m_key; + std::string m_value; + std::string m_command; + + KVCacheCommon(std::string host = "localhost", int port = 6379) : m_host(host), m_port(port){}; + + void openConnection(); + + void closeConnection(); + + void set(const char *key, size_t size, void *data); + + void get(const char *key, size_t size, void *data); + + void del(std::string key); + + bool exists(std::string key); + + std::string keyPrefix(char *VarName, size_t AbsStep, size_t BlockID); + + std::string keyComposition(const std::string &key_prefix, Dims Start, Dims Count); + + void keyPrefixExistence(const std::string &key_prefix, std::set &keys); +}; + +}; // adios2 + +#endif // ADIOS2_KVCACHECOMMON_H diff --git a/source/adios2/toolkit/cache/QueryBox.h b/source/adios2/toolkit/kvcache/QueryBox.h similarity index 96% rename from source/adios2/toolkit/cache/QueryBox.h rename to source/adios2/toolkit/kvcache/QueryBox.h index cd63590219..aff0493144 100644 --- a/source/adios2/toolkit/cache/QueryBox.h +++ b/source/adios2/toolkit/kvcache/QueryBox.h @@ -194,7 +194,7 @@ class QueryBox void getMaxInteractBox(const std::set &samePrefixKeys, const size_t &max_depth, size_t current_depth, std::vector ®ularBoxes, - std::vector &cachedBox, std::vector &cachedKeys) + std::vector &cachedKeys) { if (current_depth > max_depth) { @@ -205,8 +205,6 @@ class QueryBox std::string maxInteractKey; for (auto &key : samePrefixKeys) { - // std::cout << "Same Prefix Keys: " << key << " Current Depth: " << current_depth << - // std::endl; QueryBox const box(key); QueryBox intersection; if (this->isInteracted(box, intersection)) @@ -221,10 +219,10 @@ class QueryBox if (maxInteractBox.count.size() == 0) { + regularBoxes.push_back(*this); return; } - cachedBox.push_back(maxInteractBox); cachedKeys.push_back(maxInteractKey); if (current_depth == max_depth) @@ -238,7 +236,7 @@ class QueryBox for (auto &box : nextBoxes) { box.getMaxInteractBox(samePrefixKeys, max_depth, current_depth, regularBoxes, - cachedBox, cachedKeys); + cachedKeys); } } } @@ -269,4 +267,4 @@ class QueryBox } }; }; -#endif // UNITTEST_QUERYBOX_H +#endif // ADIOS2_KVCACHE_QUERYBOX_H diff --git a/thirdparty/KWSys/CMakeLists.txt b/thirdparty/KWSys/CMakeLists.txt index 1e7ffbbc27..2714fa2bb2 100644 --- a/thirdparty/KWSys/CMakeLists.txt +++ b/thirdparty/KWSys/CMakeLists.txt @@ -5,7 +5,6 @@ set(KWSYS_USE_CommandLineArguments ON) set(KWSYS_USE_DynamicLoader ON) set(KWSYS_USE_RegularExpression ON) set(KWSYS_USE_SystemTools ON) -set(KWSYS_USE_Base64 ON) if(WIN32) set(KWSYS_BUILD_SHARED OFF) else() From 22267e9c3b38966a01d07644cdd4e86f3bee6e27 Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Wed, 26 Jun 2024 12:03:32 -0400 Subject: [PATCH 10/22] store all partial cache requests and execuate together while waiting for remote data, which can be easily optimized by using multi-threads later --- source/adios2/engine/bp5/BP5Reader.cpp | 41 ++++++++++++++++++-------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 6eb3a54811..ca481d4998 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -363,6 +363,7 @@ void BP5Reader::PerformRemoteGets() void *Data; }; std::vector getRequestsInfo; + std::vector cachedRequestsInfo; if (getenv("useKVCache")) { @@ -416,16 +417,18 @@ void BP5Reader::PerformRemoteGets() << " boxes from remote server, and " << cachedKeys.size() << " boxes from cache" << std::endl; + RequestInfo ReqInfo; + ReqInfo.ReqSeq = req_seq; + ReqInfo.varType = varType; + ReqInfo.TypeSize = varSize; + // Get data from remote server for (auto &box : regularBoxes) { - RequestInfo ReqInfo; - ReqInfo.ReqSeq = req_seq; - ReqInfo.varType = varType; + ReqInfo.ReqCount = box.size(); ReqInfo.CacheKey = m_KVCacheCommon.keyComposition(keyPrefix, box.start, box.count); - ReqInfo.TypeSize = varSize; ReqInfo.Count = box.count; ReqInfo.Start = box.start; ReqInfo.Data = malloc(box.size() * varSize); @@ -438,13 +441,8 @@ void BP5Reader::PerformRemoteGets() // Get data from cache for (auto &boxKey : cachedKeys) { - QueryBox box(boxKey); - void *data = malloc(box.size() * varSize); - m_KVCacheCommon.get(boxKey.c_str(), box.size() * varSize, data); - helper::NdCopy(reinterpret_cast(data), box.start, box.count, true, - false, reinterpret_cast(Req.Data), Req.Start, Req.Count, - true, false, varSize); - free(data); + ReqInfo.CacheKey = boxKey; + cachedRequestsInfo.push_back(ReqInfo); } } @@ -457,11 +455,30 @@ void BP5Reader::PerformRemoteGets() handles.push_back(handle); } +#ifdef ADIOS2_HAVE_KVCACHE // get data from cache together while waiting for remote data, and can + // easily be optimized by using multi-threads later + if (getenv("useKVCache")) + { + // Get data from cache server + for (auto &ReqInfo : cachedRequestsInfo) + { + QueryBox box(ReqInfo.CacheKey); + auto &Req = GetRequests[ReqInfo.ReqSeq]; + void *data = malloc(box.size() * ReqInfo.TypeSize); + m_KVCacheCommon.get(ReqInfo.CacheKey.c_str(), box.size() * ReqInfo.TypeSize, data); + helper::NdCopy(reinterpret_cast(data), box.start, box.count, true, false, + reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, + ReqInfo.TypeSize); + free(data); + } + } +#endif + for (size_t handle_seq = 0; handle_seq < handles.size(); handle_seq++) { auto handle = handles[handle_seq]; m_Remote->WaitForGet(handle); -#ifdef ADIOS2_HAVE_KVCACHE // close cache connection +#ifdef ADIOS2_HAVE_KVCACHE // set data to cache if (getenv("useKVCache")) { auto &ReqInfo = getRequestsInfo[handle_seq]; From 8079eac0002e5a212a5eec1b614f9da5957438b8 Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Wed, 26 Jun 2024 16:08:25 -0400 Subject: [PATCH 11/22] use Redis Pipeline to accelerate the get and set operations --- source/adios2/engine/bp5/BP5Reader.cpp | 59 +++++++++++++------ .../adios2/toolkit/kvcache/KVCacheCommon.cpp | 46 +++++++++++---- source/adios2/toolkit/kvcache/KVCacheCommon.h | 8 ++- 3 files changed, 80 insertions(+), 33 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index ca481d4998..fa703eee3d 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -358,6 +358,7 @@ void BP5Reader::PerformRemoteGets() size_t ReqCount; std::string CacheKey; size_t TypeSize; + bool DirectCopy; Dims Count; Dims Start; void *Data; @@ -385,10 +386,18 @@ void BP5Reader::PerformRemoteGets() std::string targetKey = m_KVCacheCommon.keyComposition(keyPrefix, Req.Start, Req.Count); size_t varSize = helper::GetDataTypeSize(varType); + RequestInfo ReqInfo; + ReqInfo.ReqSeq = req_seq; + ReqInfo.varType = varType; + ReqInfo.TypeSize = varSize; + // Exact Match: check if targetKey exists if (m_KVCacheCommon.exists(targetKey)) { - m_KVCacheCommon.get(targetKey.c_str(), numOfElements * varSize, Req.Data); + ReqInfo.CacheKey = targetKey; + ReqInfo.ReqCount = numOfElements; + ReqInfo.DirectCopy = true; + cachedRequestsInfo.push_back(ReqInfo); } else { @@ -417,11 +426,6 @@ void BP5Reader::PerformRemoteGets() << " boxes from remote server, and " << cachedKeys.size() << " boxes from cache" << std::endl; - RequestInfo ReqInfo; - ReqInfo.ReqSeq = req_seq; - ReqInfo.varType = varType; - ReqInfo.TypeSize = varSize; - // Get data from remote server for (auto &box : regularBoxes) { @@ -442,6 +446,7 @@ void BP5Reader::PerformRemoteGets() for (auto &boxKey : cachedKeys) { ReqInfo.CacheKey = boxKey; + ReqInfo.DirectCopy = false; cachedRequestsInfo.push_back(ReqInfo); } } @@ -455,21 +460,34 @@ void BP5Reader::PerformRemoteGets() handles.push_back(handle); } -#ifdef ADIOS2_HAVE_KVCACHE // get data from cache together while waiting for remote data, and can - // easily be optimized by using multi-threads later +#ifdef ADIOS2_HAVE_KVCACHE // get data in redis pipeline if (getenv("useKVCache")) { // Get data from cache server for (auto &ReqInfo : cachedRequestsInfo) { - QueryBox box(ReqInfo.CacheKey); + m_KVCacheCommon.AppendCommandInBatch(ReqInfo.CacheKey.c_str(), 1, 0, nullptr); + } + + for (auto &ReqInfo : cachedRequestsInfo) + { auto &Req = GetRequests[ReqInfo.ReqSeq]; - void *data = malloc(box.size() * ReqInfo.TypeSize); - m_KVCacheCommon.get(ReqInfo.CacheKey.c_str(), box.size() * ReqInfo.TypeSize, data); - helper::NdCopy(reinterpret_cast(data), box.start, box.count, true, false, - reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, - ReqInfo.TypeSize); - free(data); + if (ReqInfo.DirectCopy) + { + m_KVCacheCommon.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, + ReqInfo.ReqCount * ReqInfo.TypeSize, Req.Data); + } + else + { + QueryBox box(ReqInfo.CacheKey); + void *data = malloc(box.size() * ReqInfo.TypeSize); + m_KVCacheCommon.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, + box.size() * ReqInfo.TypeSize, data); + helper::NdCopy(reinterpret_cast(data), box.start, box.count, true, false, + reinterpret_cast(Req.Data), Req.Start, Req.Count, true, + false, ReqInfo.TypeSize); + free(data); + } } } #endif @@ -486,8 +504,9 @@ void BP5Reader::PerformRemoteGets() helper::NdCopy(reinterpret_cast(ReqInfo.Data), ReqInfo.Start, ReqInfo.Count, true, false, reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, ReqInfo.TypeSize); - m_KVCacheCommon.set(ReqInfo.CacheKey.c_str(), ReqInfo.ReqCount * ReqInfo.TypeSize, - ReqInfo.Data); + + m_KVCacheCommon.AppendCommandInBatch(ReqInfo.CacheKey.c_str(), 0, + ReqInfo.ReqCount * ReqInfo.TypeSize, ReqInfo.Data); free(ReqInfo.Data); } #endif @@ -496,6 +515,12 @@ void BP5Reader::PerformRemoteGets() #ifdef ADIOS2_HAVE_KVCACHE // close cache connection if (getenv("useKVCache")) { + // Execute batch commands of Set + for (size_t handle_seq = 0; handle_seq < handles.size(); handle_seq++) + { + auto &ReqInfo = getRequestsInfo[handle_seq]; + m_KVCacheCommon.ExecuteBatch(ReqInfo.CacheKey.c_str(), 0, 0, nullptr); + } m_KVCacheCommon.closeConnection(); } #endif diff --git a/source/adios2/toolkit/kvcache/KVCacheCommon.cpp b/source/adios2/toolkit/kvcache/KVCacheCommon.cpp index b845158198..75c38f3e85 100644 --- a/source/adios2/toolkit/kvcache/KVCacheCommon.cpp +++ b/source/adios2/toolkit/kvcache/KVCacheCommon.cpp @@ -61,10 +61,37 @@ void KVCacheCommon::get(const char *key, size_t size, void *data) } } +void KVCacheCommon::AppendCommandInBatch(const char *key, size_t mode, size_t size, void *data) +{ + if (mode == 0) + { + redisAppendCommand(m_redisContext, "SET %s %b", key, data, size); + } + else if (mode == 1) + { + redisAppendCommand(m_redisContext, "GET %s", key); + } +} + +void KVCacheCommon::ExecuteBatch(const char *key, size_t mode, size_t size, void *data) +{ + if (redisGetReply(m_redisContext, (void **)&m_redisReply) == REDIS_OK) + { + if (mode == 1) + { + memcpy(data, m_redisReply->str, size); + } + freeReplyObject(m_redisReply); + } + else + { + std::cout << "Error to execute batch command: " << key << std::endl; + } +} + void KVCacheCommon::del(std::string key) { - m_command = "DEL " + key; - m_redisReply = (redisReply *)redisCommand(m_redisContext, m_command.c_str()); + m_redisReply = (redisReply *)redisCommand(m_redisContext, "DEL %s", key.c_str()); if (m_redisReply == NULL) { std::cout << "Error to delete key: " << key << std::endl; @@ -77,14 +104,8 @@ void KVCacheCommon::del(std::string key) bool KVCacheCommon::exists(std::string key) { - m_command = "EXISTS " + key; - m_redisReply = (redisReply *)redisCommand(m_redisContext, m_command.c_str()); - if (m_redisReply == NULL) - { - std::cout << "The Key: " << key << " does not exist" << std::endl; - return false; - } - else + m_redisReply = (redisReply *)redisCommand(m_redisContext, "EXISTS %s", key.c_str()); + if (m_redisReply != NULL) { if (!m_redisReply->integer) { @@ -94,6 +115,7 @@ bool KVCacheCommon::exists(std::string key) freeReplyObject(m_redisReply); return true; } + return false; } std::string KVCacheCommon::keyPrefix(char *VarName, size_t AbsStep, size_t BlockID) @@ -119,9 +141,7 @@ std::string KVCacheCommon::keyComposition(const std::string &key_prefix, Dims St void KVCacheCommon::keyPrefixExistence(const std::string &key_prefix, std::set &keys) { - std::string keyPattern = key_prefix + "*"; - m_command = "KEYS " + keyPattern; - m_redisReply = (redisReply *)redisCommand(m_redisContext, m_command.c_str()); + m_redisReply = (redisReply *)redisCommand(m_redisContext, "KEYS %s*", key_prefix.c_str()); if (m_redisReply == NULL) { std::cout << "Error to get keys with prefix: " << key_prefix << std::endl; diff --git a/source/adios2/toolkit/kvcache/KVCacheCommon.h b/source/adios2/toolkit/kvcache/KVCacheCommon.h index 6efbf28add..92bdefdd61 100644 --- a/source/adios2/toolkit/kvcache/KVCacheCommon.h +++ b/source/adios2/toolkit/kvcache/KVCacheCommon.h @@ -22,9 +22,6 @@ class KVCacheCommon int m_port; redisContext *m_redisContext; redisReply *m_redisReply; - std::string m_key; - std::string m_value; - std::string m_command; KVCacheCommon(std::string host = "localhost", int port = 6379) : m_host(host), m_port(port){}; @@ -36,6 +33,11 @@ class KVCacheCommon void get(const char *key, size_t size, void *data); + // Batch operations in pipeline, mode 0 for SET, 1 for GET + void AppendCommandInBatch(const char *key, size_t mode, size_t size, void *data); + + void ExecuteBatch(const char *key, size_t mode, size_t size, void *data); + void del(std::string key); bool exists(std::string key); From ced568011c4483df286d28114d7fc4e07f7ef9cd Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Wed, 26 Jun 2024 16:29:30 -0400 Subject: [PATCH 12/22] rename all functions follow Upper Camel Case --- source/adios2/engine/bp5/BP5Reader.cpp | 24 +++++++++---------- .../adios2/toolkit/kvcache/KVCacheCommon.cpp | 20 ++++++++-------- source/adios2/toolkit/kvcache/KVCacheCommon.h | 18 +++++++------- source/adios2/toolkit/kvcache/QueryBox.h | 20 ++++++++-------- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index fa703eee3d..0317ca3df4 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -363,12 +363,12 @@ void BP5Reader::PerformRemoteGets() Dims Start; void *Data; }; - std::vector getRequestsInfo; + std::vector remoteRequestsInfo; std::vector cachedRequestsInfo; if (getenv("useKVCache")) { - m_KVCacheCommon.openConnection(); + m_KVCacheCommon.OpenConnection(); } #endif @@ -382,8 +382,8 @@ void BP5Reader::PerformRemoteGets() QueryBox targetBox(Req.Start, Req.Count); size_t numOfElements = targetBox.size(); std::string keyPrefix = - m_KVCacheCommon.keyPrefix(Req.VarName, Req.RelStep, Req.BlockID); - std::string targetKey = m_KVCacheCommon.keyComposition(keyPrefix, Req.Start, Req.Count); + m_KVCacheCommon.KeyPrefix(Req.VarName, Req.RelStep, Req.BlockID); + std::string targetKey = m_KVCacheCommon.KeyComposition(keyPrefix, Req.Start, Req.Count); size_t varSize = helper::GetDataTypeSize(varType); RequestInfo ReqInfo; @@ -392,7 +392,7 @@ void BP5Reader::PerformRemoteGets() ReqInfo.TypeSize = varSize; // Exact Match: check if targetKey exists - if (m_KVCacheCommon.exists(targetKey)) + if (m_KVCacheCommon.Exists(targetKey)) { ReqInfo.CacheKey = targetKey; ReqInfo.ReqCount = numOfElements; @@ -403,7 +403,7 @@ void BP5Reader::PerformRemoteGets() { int max_depth = 999; std::set samePrefixKeys; - m_KVCacheCommon.keyPrefixExistence(keyPrefix, samePrefixKeys); + m_KVCacheCommon.KeyPrefixExistence(keyPrefix, samePrefixKeys); std::vector regularBoxes; std::vector cachedKeys; @@ -414,7 +414,7 @@ void BP5Reader::PerformRemoteGets() if (samePrefixKeys.size() > 0) { - targetBox.getMaxInteractBox(samePrefixKeys, max_depth, 0, regularBoxes, + targetBox.GetMaxInteractBox(samePrefixKeys, max_depth, 0, regularBoxes, cachedKeys); } else @@ -432,14 +432,14 @@ void BP5Reader::PerformRemoteGets() ReqInfo.ReqCount = box.size(); ReqInfo.CacheKey = - m_KVCacheCommon.keyComposition(keyPrefix, box.start, box.count); + m_KVCacheCommon.KeyComposition(keyPrefix, box.start, box.count); ReqInfo.Count = box.count; ReqInfo.Start = box.start; ReqInfo.Data = malloc(box.size() * varSize); auto handle = m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, box.count, box.start, ReqInfo.Data); handles.push_back(handle); - getRequestsInfo.push_back(ReqInfo); + remoteRequestsInfo.push_back(ReqInfo); } // Get data from cache @@ -499,7 +499,7 @@ void BP5Reader::PerformRemoteGets() #ifdef ADIOS2_HAVE_KVCACHE // set data to cache if (getenv("useKVCache")) { - auto &ReqInfo = getRequestsInfo[handle_seq]; + auto &ReqInfo = remoteRequestsInfo[handle_seq]; auto &Req = GetRequests[ReqInfo.ReqSeq]; helper::NdCopy(reinterpret_cast(ReqInfo.Data), ReqInfo.Start, ReqInfo.Count, true, false, reinterpret_cast(Req.Data), Req.Start, Req.Count, @@ -518,10 +518,10 @@ void BP5Reader::PerformRemoteGets() // Execute batch commands of Set for (size_t handle_seq = 0; handle_seq < handles.size(); handle_seq++) { - auto &ReqInfo = getRequestsInfo[handle_seq]; + auto &ReqInfo = remoteRequestsInfo[handle_seq]; m_KVCacheCommon.ExecuteBatch(ReqInfo.CacheKey.c_str(), 0, 0, nullptr); } - m_KVCacheCommon.closeConnection(); + m_KVCacheCommon.CloseConnection(); } #endif } diff --git a/source/adios2/toolkit/kvcache/KVCacheCommon.cpp b/source/adios2/toolkit/kvcache/KVCacheCommon.cpp index 75c38f3e85..4487a901a6 100644 --- a/source/adios2/toolkit/kvcache/KVCacheCommon.cpp +++ b/source/adios2/toolkit/kvcache/KVCacheCommon.cpp @@ -9,7 +9,7 @@ namespace adios2 { -void KVCacheCommon::openConnection() +void KVCacheCommon::OpenConnection() { m_redisContext = redisConnect(m_host.c_str(), m_port); if (m_redisContext == NULL || m_redisContext->err) @@ -27,13 +27,13 @@ void KVCacheCommon::openConnection() } } -void KVCacheCommon::closeConnection() +void KVCacheCommon::CloseConnection() { redisFree(m_redisContext); std::cout << "KVCache connection closed" << std::endl; } -void KVCacheCommon::set(const char *key, size_t size, void *data) +void KVCacheCommon::Set(const char *key, size_t size, void *data) { m_redisReply = (redisReply *)redisCommand(m_redisContext, "SET %s %b", key, data, size); if (m_redisReply == NULL) @@ -47,7 +47,7 @@ void KVCacheCommon::set(const char *key, size_t size, void *data) } } -void KVCacheCommon::get(const char *key, size_t size, void *data) +void KVCacheCommon::Get(const char *key, size_t size, void *data) { m_redisReply = (redisReply *)redisCommand(m_redisContext, "GET %s", key); if (m_redisReply == NULL) @@ -89,7 +89,7 @@ void KVCacheCommon::ExecuteBatch(const char *key, size_t mode, size_t size, void } } -void KVCacheCommon::del(std::string key) +void KVCacheCommon::Del(std::string key) { m_redisReply = (redisReply *)redisCommand(m_redisContext, "DEL %s", key.c_str()); if (m_redisReply == NULL) @@ -102,7 +102,7 @@ void KVCacheCommon::del(std::string key) } } -bool KVCacheCommon::exists(std::string key) +bool KVCacheCommon::Exists(std::string key) { m_redisReply = (redisReply *)redisCommand(m_redisContext, "EXISTS %s", key.c_str()); if (m_redisReply != NULL) @@ -118,14 +118,14 @@ bool KVCacheCommon::exists(std::string key) return false; } -std::string KVCacheCommon::keyPrefix(char *VarName, size_t AbsStep, size_t BlockID) +std::string KVCacheCommon::KeyPrefix(char *VarName, size_t AbsStep, size_t BlockID) { return VarName + std::to_string(AbsStep) + std::to_string(BlockID); } -std::string KVCacheCommon::keyComposition(const std::string &key_prefix, Dims Start, Dims Count) +std::string KVCacheCommon::KeyComposition(const std::string &key_prefix, Dims Start, Dims Count) { - std::string box = QueryBox::serializeQueryBox(QueryBox{Start, Count}); + std::string box = QueryBox::SerializeQueryBox(QueryBox{Start, Count}); std::string cacheKey = key_prefix + box; // replace special characters std::replace(cacheKey.begin(), cacheKey.end(), '"', '_'); @@ -139,7 +139,7 @@ std::string KVCacheCommon::keyComposition(const std::string &key_prefix, Dims St return cacheKey; } -void KVCacheCommon::keyPrefixExistence(const std::string &key_prefix, std::set &keys) +void KVCacheCommon::KeyPrefixExistence(const std::string &key_prefix, std::set &keys) { m_redisReply = (redisReply *)redisCommand(m_redisContext, "KEYS %s*", key_prefix.c_str()); if (m_redisReply == NULL) diff --git a/source/adios2/toolkit/kvcache/KVCacheCommon.h b/source/adios2/toolkit/kvcache/KVCacheCommon.h index 92bdefdd61..cf6c81c53a 100644 --- a/source/adios2/toolkit/kvcache/KVCacheCommon.h +++ b/source/adios2/toolkit/kvcache/KVCacheCommon.h @@ -25,28 +25,28 @@ class KVCacheCommon KVCacheCommon(std::string host = "localhost", int port = 6379) : m_host(host), m_port(port){}; - void openConnection(); + void OpenConnection(); - void closeConnection(); + void CloseConnection(); - void set(const char *key, size_t size, void *data); + void Set(const char *key, size_t size, void *data); - void get(const char *key, size_t size, void *data); + void Get(const char *key, size_t size, void *data); // Batch operations in pipeline, mode 0 for SET, 1 for GET void AppendCommandInBatch(const char *key, size_t mode, size_t size, void *data); void ExecuteBatch(const char *key, size_t mode, size_t size, void *data); - void del(std::string key); + void Del(std::string key); - bool exists(std::string key); + bool Exists(std::string key); - std::string keyPrefix(char *VarName, size_t AbsStep, size_t BlockID); + std::string KeyPrefix(char *VarName, size_t AbsStep, size_t BlockID); - std::string keyComposition(const std::string &key_prefix, Dims Start, Dims Count); + std::string KeyComposition(const std::string &key_prefix, Dims Start, Dims Count); - void keyPrefixExistence(const std::string &key_prefix, std::set &keys); + void KeyPrefixExistence(const std::string &key_prefix, std::set &keys); }; }; // adios2 diff --git a/source/adios2/toolkit/kvcache/QueryBox.h b/source/adios2/toolkit/kvcache/QueryBox.h index aff0493144..56dc5f8f95 100644 --- a/source/adios2/toolkit/kvcache/QueryBox.h +++ b/source/adios2/toolkit/kvcache/QueryBox.h @@ -58,7 +58,7 @@ class QueryBox } // Serialize QueryBox to a string, like __count_:_64_64_64___start_:_0_0_0__ - static std::string serializeQueryBox(const QueryBox &box) + static std::string SerializeQueryBox(const QueryBox &box) { nlohmann::json jsonBox; jsonBox["start"] = box.start; @@ -71,7 +71,7 @@ class QueryBox // determine if a query box is interacted in another query box, return intersection part as a // new query box - bool isInteracted(const QueryBox &box, QueryBox &intersection) const + bool IsInteracted(const QueryBox &box, QueryBox &intersection) const { if (start.size() != box.start.size() || start.size() != count.size() || start.size() != box.count.size()) @@ -97,7 +97,7 @@ class QueryBox } // determine if a query box is fully contained in another query box - bool isFullContainedBy(const QueryBox &box) + bool IsFullContainedBy(const QueryBox &box) { if (start.size() != box.start.size() || start.size() != count.size() || start.size() != box.count.size()) @@ -116,7 +116,7 @@ class QueryBox // cut a query box from another interaction box, return a list of regular box // remainingBox is the big one, this is small one - void interactionCut(const QueryBox &remainingBox, std::vector ®ularBoxes) + void InteractionCut(const QueryBox &remainingBox, std::vector ®ularBoxes) { if (remainingBox == *this) { @@ -188,11 +188,11 @@ class QueryBox } } } - interactionCut(remainingBox1, regularBoxes); + InteractionCut(remainingBox1, regularBoxes); } } - void getMaxInteractBox(const std::set &samePrefixKeys, const size_t &max_depth, + void GetMaxInteractBox(const std::set &samePrefixKeys, const size_t &max_depth, size_t current_depth, std::vector ®ularBoxes, std::vector &cachedKeys) { @@ -207,7 +207,7 @@ class QueryBox { QueryBox const box(key); QueryBox intersection; - if (this->isInteracted(box, intersection)) + if (this->IsInteracted(box, intersection)) { if (maxInteractBox.size() < intersection.size()) { @@ -227,15 +227,15 @@ class QueryBox if (current_depth == max_depth) { - maxInteractBox.interactionCut(*this, regularBoxes); + maxInteractBox.InteractionCut(*this, regularBoxes); } else { std::vector nextBoxes; - maxInteractBox.interactionCut(*this, nextBoxes); + maxInteractBox.InteractionCut(*this, nextBoxes); for (auto &box : nextBoxes) { - box.getMaxInteractBox(samePrefixKeys, max_depth, current_depth, regularBoxes, + box.GetMaxInteractBox(samePrefixKeys, max_depth, current_depth, regularBoxes, cachedKeys); } } From 5f5c249237c8e17398f750c9a0abb20e29f01cc7 Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Wed, 26 Jun 2024 17:08:21 -0400 Subject: [PATCH 13/22] open redis connection during the m_Remote initialization --- source/adios2/engine/bp5/BP5Reader.cpp | 40 +++++++++---------- source/adios2/engine/bp5/BP5Reader.h | 2 +- source/adios2/toolkit/kvcache/KVCacheCommon.h | 2 + 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 0317ca3df4..ace2f8ea4d 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -311,6 +311,12 @@ void BP5Reader::PerformGets() m_Remote->Open("localhost", EVPathRemoteCommon::ServerPort, RemoteName, m_OpenMode, RowMajorOrdering); } +#endif +#ifdef ADIOS2_HAVE_KVCACHE + if (getenv("useKVCache")) + { + m_KVCache.OpenConnection(); + } #endif if (m_Remote == nullptr) { @@ -365,11 +371,6 @@ void BP5Reader::PerformRemoteGets() }; std::vector remoteRequestsInfo; std::vector cachedRequestsInfo; - - if (getenv("useKVCache")) - { - m_KVCacheCommon.OpenConnection(); - } #endif for (size_t req_seq = 0; req_seq < GetRequests.size(); req_seq++) @@ -381,9 +382,8 @@ void BP5Reader::PerformRemoteGets() const DataType varType = m_IO.InquireVariableType(Req.VarName); QueryBox targetBox(Req.Start, Req.Count); size_t numOfElements = targetBox.size(); - std::string keyPrefix = - m_KVCacheCommon.KeyPrefix(Req.VarName, Req.RelStep, Req.BlockID); - std::string targetKey = m_KVCacheCommon.KeyComposition(keyPrefix, Req.Start, Req.Count); + std::string keyPrefix = m_KVCache.KeyPrefix(Req.VarName, Req.RelStep, Req.BlockID); + std::string targetKey = m_KVCache.KeyComposition(keyPrefix, Req.Start, Req.Count); size_t varSize = helper::GetDataTypeSize(varType); RequestInfo ReqInfo; @@ -392,7 +392,7 @@ void BP5Reader::PerformRemoteGets() ReqInfo.TypeSize = varSize; // Exact Match: check if targetKey exists - if (m_KVCacheCommon.Exists(targetKey)) + if (m_KVCache.Exists(targetKey)) { ReqInfo.CacheKey = targetKey; ReqInfo.ReqCount = numOfElements; @@ -403,7 +403,7 @@ void BP5Reader::PerformRemoteGets() { int max_depth = 999; std::set samePrefixKeys; - m_KVCacheCommon.KeyPrefixExistence(keyPrefix, samePrefixKeys); + m_KVCache.KeyPrefixExistence(keyPrefix, samePrefixKeys); std::vector regularBoxes; std::vector cachedKeys; @@ -431,8 +431,7 @@ void BP5Reader::PerformRemoteGets() { ReqInfo.ReqCount = box.size(); - ReqInfo.CacheKey = - m_KVCacheCommon.KeyComposition(keyPrefix, box.start, box.count); + ReqInfo.CacheKey = m_KVCache.KeyComposition(keyPrefix, box.start, box.count); ReqInfo.Count = box.count; ReqInfo.Start = box.start; ReqInfo.Data = malloc(box.size() * varSize); @@ -466,7 +465,7 @@ void BP5Reader::PerformRemoteGets() // Get data from cache server for (auto &ReqInfo : cachedRequestsInfo) { - m_KVCacheCommon.AppendCommandInBatch(ReqInfo.CacheKey.c_str(), 1, 0, nullptr); + m_KVCache.AppendCommandInBatch(ReqInfo.CacheKey.c_str(), 1, 0, nullptr); } for (auto &ReqInfo : cachedRequestsInfo) @@ -474,15 +473,15 @@ void BP5Reader::PerformRemoteGets() auto &Req = GetRequests[ReqInfo.ReqSeq]; if (ReqInfo.DirectCopy) { - m_KVCacheCommon.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, - ReqInfo.ReqCount * ReqInfo.TypeSize, Req.Data); + m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, + ReqInfo.ReqCount * ReqInfo.TypeSize, Req.Data); } else { QueryBox box(ReqInfo.CacheKey); void *data = malloc(box.size() * ReqInfo.TypeSize); - m_KVCacheCommon.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, - box.size() * ReqInfo.TypeSize, data); + m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, box.size() * ReqInfo.TypeSize, + data); helper::NdCopy(reinterpret_cast(data), box.start, box.count, true, false, reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, ReqInfo.TypeSize); @@ -505,8 +504,8 @@ void BP5Reader::PerformRemoteGets() true, false, reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, ReqInfo.TypeSize); - m_KVCacheCommon.AppendCommandInBatch(ReqInfo.CacheKey.c_str(), 0, - ReqInfo.ReqCount * ReqInfo.TypeSize, ReqInfo.Data); + m_KVCache.AppendCommandInBatch(ReqInfo.CacheKey.c_str(), 0, + ReqInfo.ReqCount * ReqInfo.TypeSize, ReqInfo.Data); free(ReqInfo.Data); } #endif @@ -519,9 +518,8 @@ void BP5Reader::PerformRemoteGets() for (size_t handle_seq = 0; handle_seq < handles.size(); handle_seq++) { auto &ReqInfo = remoteRequestsInfo[handle_seq]; - m_KVCacheCommon.ExecuteBatch(ReqInfo.CacheKey.c_str(), 0, 0, nullptr); + m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 0, 0, nullptr); } - m_KVCacheCommon.CloseConnection(); } #endif } diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h index 9eba7b70fc..701587cd8e 100644 --- a/source/adios2/engine/bp5/BP5Reader.h +++ b/source/adios2/engine/bp5/BP5Reader.h @@ -103,7 +103,7 @@ class BP5Reader : public BP5Engine, public Engine bool m_WriterIsActive = true; adios2::profiling::JSONProfiler m_JSONProfiler; #ifdef ADIOS2_HAVE_KVCACHE - KVCacheCommon m_KVCacheCommon; + KVCacheCommon m_KVCache; #endif /** used for per-step reads, TODO: to be moved to BP5Deserializer */ size_t m_CurrentStep = 0; diff --git a/source/adios2/toolkit/kvcache/KVCacheCommon.h b/source/adios2/toolkit/kvcache/KVCacheCommon.h index cf6c81c53a..ceab61b8ea 100644 --- a/source/adios2/toolkit/kvcache/KVCacheCommon.h +++ b/source/adios2/toolkit/kvcache/KVCacheCommon.h @@ -25,6 +25,8 @@ class KVCacheCommon KVCacheCommon(std::string host = "localhost", int port = 6379) : m_host(host), m_port(port){}; + ~KVCacheCommon() { CloseConnection(); } + void OpenConnection(); void CloseConnection(); From a4f572adfc99fe8be900e71d887143e0cf0d870b Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Wed, 26 Jun 2024 18:03:51 -0400 Subject: [PATCH 14/22] fix free invalid pointer --- source/adios2/toolkit/kvcache/KVCacheCommon.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/adios2/toolkit/kvcache/KVCacheCommon.cpp b/source/adios2/toolkit/kvcache/KVCacheCommon.cpp index 4487a901a6..df79f04ff9 100644 --- a/source/adios2/toolkit/kvcache/KVCacheCommon.cpp +++ b/source/adios2/toolkit/kvcache/KVCacheCommon.cpp @@ -29,8 +29,11 @@ void KVCacheCommon::OpenConnection() void KVCacheCommon::CloseConnection() { - redisFree(m_redisContext); - std::cout << "KVCache connection closed" << std::endl; + if (m_redisContext != nullptr) + { + m_redisContext = nullptr; + std::cout << "KVCache connection closed" << std::endl; + } } void KVCacheCommon::Set(const char *key, size_t size, void *data) From 9414a33f66ac229f5d60f2916881b4f75f354e76 Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Thu, 27 Jun 2024 13:03:51 -0400 Subject: [PATCH 15/22] separate kvcache into PerformRemoteGetsWithKVCache --- source/adios2/engine/bp5/BP5Reader.cpp | 241 ++++++++++++------------- source/adios2/engine/bp5/BP5Reader.h | 2 + 2 files changed, 122 insertions(+), 121 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index ace2f8ea4d..1ac9c1c7d0 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -336,7 +336,18 @@ void BP5Reader::PerformGets() if (m_Remote) { +#ifdef ADIOS2_HAVE_KVCACHE + if (getenv("useKVCache")) + { + PerformRemoteGetsWithKVCache(); + } + else + { + PerformRemoteGets(); + } +#else PerformRemoteGets(); +#endif } else { @@ -350,13 +361,11 @@ void BP5Reader::PerformGets() } } -void BP5Reader::PerformRemoteGets() +void BP5Reader::PerformRemoteGetsWithKVCache() { - // TP startGenerate = NOW(); auto GetRequests = m_BP5Deserializer->PendingGetRequests; std::vector handles; -#ifdef ADIOS2_HAVE_KVCACHE // open kv cache connection struct RequestInfo { size_t ReqSeq; @@ -371,157 +380,147 @@ void BP5Reader::PerformRemoteGets() }; std::vector remoteRequestsInfo; std::vector cachedRequestsInfo; -#endif for (size_t req_seq = 0; req_seq < GetRequests.size(); req_seq++) { auto &Req = GetRequests[req_seq]; -#ifdef ADIOS2_HAVE_KVCACHE // get data from cache - if (getenv("useKVCache")) + const DataType varType = m_IO.InquireVariableType(Req.VarName); + + QueryBox targetBox(Req.Start, Req.Count); + size_t numOfElements = targetBox.size(); + std::string keyPrefix = m_KVCache.KeyPrefix(Req.VarName, Req.RelStep, Req.BlockID); + std::string targetKey = m_KVCache.KeyComposition(keyPrefix, Req.Start, Req.Count); + size_t varSize = helper::GetDataTypeSize(varType); + + RequestInfo ReqInfo; + ReqInfo.ReqSeq = req_seq; + ReqInfo.varType = varType; + ReqInfo.TypeSize = varSize; + + // Exact Match: check if targetKey exists + if (m_KVCache.Exists(targetKey)) { - const DataType varType = m_IO.InquireVariableType(Req.VarName); - QueryBox targetBox(Req.Start, Req.Count); - size_t numOfElements = targetBox.size(); - std::string keyPrefix = m_KVCache.KeyPrefix(Req.VarName, Req.RelStep, Req.BlockID); - std::string targetKey = m_KVCache.KeyComposition(keyPrefix, Req.Start, Req.Count); - size_t varSize = helper::GetDataTypeSize(varType); - - RequestInfo ReqInfo; - ReqInfo.ReqSeq = req_seq; - ReqInfo.varType = varType; - ReqInfo.TypeSize = varSize; - - // Exact Match: check if targetKey exists - if (m_KVCache.Exists(targetKey)) + ReqInfo.CacheKey = targetKey; + ReqInfo.ReqCount = numOfElements; + ReqInfo.DirectCopy = true; + cachedRequestsInfo.push_back(ReqInfo); + } + else + { + int max_depth = 999; + std::set samePrefixKeys; + m_KVCache.KeyPrefixExistence(keyPrefix, samePrefixKeys); + std::vector regularBoxes; + std::vector cachedKeys; + + if (getenv("maxDepth")) { - ReqInfo.CacheKey = targetKey; - ReqInfo.ReqCount = numOfElements; - ReqInfo.DirectCopy = true; - cachedRequestsInfo.push_back(ReqInfo); + max_depth = std::stoi(getenv("maxDepth")); + } + + if (samePrefixKeys.size() > 0) + { + targetBox.GetMaxInteractBox(samePrefixKeys, max_depth, 0, regularBoxes, cachedKeys); } else { - int max_depth = 999; - std::set samePrefixKeys; - m_KVCache.KeyPrefixExistence(keyPrefix, samePrefixKeys); - std::vector regularBoxes; - std::vector cachedKeys; - - if (getenv("maxDepth")) - { - max_depth = std::stoi(getenv("maxDepth")); - } - - if (samePrefixKeys.size() > 0) - { - targetBox.GetMaxInteractBox(samePrefixKeys, max_depth, 0, regularBoxes, - cachedKeys); - } - else - { - regularBoxes.push_back(targetBox); - } - - std::cout << "Going to retrieve " << regularBoxes.size() - << " boxes from remote server, and " << cachedKeys.size() - << " boxes from cache" << std::endl; + regularBoxes.push_back(targetBox); + } - // Get data from remote server - for (auto &box : regularBoxes) - { + std::cout << "Going to retrieve " << regularBoxes.size() + << " boxes from remote server, and " << cachedKeys.size() + << " boxes from cache" << std::endl; - ReqInfo.ReqCount = box.size(); - ReqInfo.CacheKey = m_KVCache.KeyComposition(keyPrefix, box.start, box.count); - ReqInfo.Count = box.count; - ReqInfo.Start = box.start; - ReqInfo.Data = malloc(box.size() * varSize); - auto handle = m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, box.count, - box.start, ReqInfo.Data); - handles.push_back(handle); - remoteRequestsInfo.push_back(ReqInfo); - } + // Get data from remote server + for (auto &box : regularBoxes) + { - // Get data from cache - for (auto &boxKey : cachedKeys) - { - ReqInfo.CacheKey = boxKey; - ReqInfo.DirectCopy = false; - cachedRequestsInfo.push_back(ReqInfo); - } + ReqInfo.ReqCount = box.size(); + ReqInfo.CacheKey = m_KVCache.KeyComposition(keyPrefix, box.start, box.count); + ReqInfo.Count = box.count; + ReqInfo.Start = box.start; + ReqInfo.Data = malloc(box.size() * varSize); + auto handle = m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, box.count, + box.start, ReqInfo.Data); + handles.push_back(handle); + remoteRequestsInfo.push_back(ReqInfo); } - continue; + // Get data from cache + for (auto &boxKey : cachedKeys) + { + ReqInfo.CacheKey = boxKey; + ReqInfo.DirectCopy = false; + cachedRequestsInfo.push_back(ReqInfo); + } } -#endif + } - auto handle = - m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, Req.Count, Req.Start, Req.Data); - handles.push_back(handle); + // Get data from cache server + for (auto &ReqInfo : cachedRequestsInfo) + { + m_KVCache.AppendCommandInBatch(ReqInfo.CacheKey.c_str(), 1, 0, nullptr); } -#ifdef ADIOS2_HAVE_KVCACHE // get data in redis pipeline - if (getenv("useKVCache")) + for (auto &ReqInfo : cachedRequestsInfo) { - // Get data from cache server - for (auto &ReqInfo : cachedRequestsInfo) + auto &Req = GetRequests[ReqInfo.ReqSeq]; + if (ReqInfo.DirectCopy) { - m_KVCache.AppendCommandInBatch(ReqInfo.CacheKey.c_str(), 1, 0, nullptr); + m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, ReqInfo.ReqCount * ReqInfo.TypeSize, + Req.Data); } - - for (auto &ReqInfo : cachedRequestsInfo) + else { - auto &Req = GetRequests[ReqInfo.ReqSeq]; - if (ReqInfo.DirectCopy) - { - m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, - ReqInfo.ReqCount * ReqInfo.TypeSize, Req.Data); - } - else - { - QueryBox box(ReqInfo.CacheKey); - void *data = malloc(box.size() * ReqInfo.TypeSize); - m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, box.size() * ReqInfo.TypeSize, - data); - helper::NdCopy(reinterpret_cast(data), box.start, box.count, true, false, - reinterpret_cast(Req.Data), Req.Start, Req.Count, true, - false, ReqInfo.TypeSize); - free(data); - } + QueryBox box(ReqInfo.CacheKey); + void *data = malloc(box.size() * ReqInfo.TypeSize); + m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, box.size() * ReqInfo.TypeSize, + data); + helper::NdCopy(reinterpret_cast(data), box.start, box.count, true, false, + reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, + ReqInfo.TypeSize); + free(data); } } -#endif for (size_t handle_seq = 0; handle_seq < handles.size(); handle_seq++) { auto handle = handles[handle_seq]; m_Remote->WaitForGet(handle); -#ifdef ADIOS2_HAVE_KVCACHE // set data to cache - if (getenv("useKVCache")) - { - auto &ReqInfo = remoteRequestsInfo[handle_seq]; - auto &Req = GetRequests[ReqInfo.ReqSeq]; - helper::NdCopy(reinterpret_cast(ReqInfo.Data), ReqInfo.Start, ReqInfo.Count, - true, false, reinterpret_cast(Req.Data), Req.Start, Req.Count, - true, false, ReqInfo.TypeSize); - - m_KVCache.AppendCommandInBatch(ReqInfo.CacheKey.c_str(), 0, - ReqInfo.ReqCount * ReqInfo.TypeSize, ReqInfo.Data); - free(ReqInfo.Data); - } -#endif + auto &ReqInfo = remoteRequestsInfo[handle_seq]; + auto &Req = GetRequests[ReqInfo.ReqSeq]; + helper::NdCopy(reinterpret_cast(ReqInfo.Data), ReqInfo.Start, ReqInfo.Count, true, + false, reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, + ReqInfo.TypeSize); + + m_KVCache.AppendCommandInBatch(ReqInfo.CacheKey.c_str(), 0, + ReqInfo.ReqCount * ReqInfo.TypeSize, ReqInfo.Data); + free(ReqInfo.Data); } -#ifdef ADIOS2_HAVE_KVCACHE // close cache connection - if (getenv("useKVCache")) + // Execute batch commands of Set + for (size_t handle_seq = 0; handle_seq < handles.size(); handle_seq++) { - // Execute batch commands of Set - for (size_t handle_seq = 0; handle_seq < handles.size(); handle_seq++) - { - auto &ReqInfo = remoteRequestsInfo[handle_seq]; - m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 0, 0, nullptr); - } + auto &ReqInfo = remoteRequestsInfo[handle_seq]; + m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 0, 0, nullptr); + } +} + +void BP5Reader::PerformRemoteGets() +{ + // TP startGenerate = NOW(); + auto GetRequests = m_BP5Deserializer->PendingGetRequests; + std::vector handles; + for (auto &Req : GetRequests) + { + auto handle = + m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, Req.Count, Req.Start, Req.Data); + handles.push_back(handle); + } + for (auto &handle : handles) + { + m_Remote->WaitForGet(handle); } -#endif } void BP5Reader::PerformLocalGets() diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h index 701587cd8e..7b0288f7d6 100644 --- a/source/adios2/engine/bp5/BP5Reader.h +++ b/source/adios2/engine/bp5/BP5Reader.h @@ -260,6 +260,8 @@ class BP5Reader : public BP5Engine, public Engine void PerformRemoteGets(); + void PerformRemoteGetsWithKVCache(); + void DestructorClose(bool Verbose) noexcept; /* Communicator connecting ranks on each Compute Node. From 8e3de8c1f123d1947b7c4819c56b83b9420153cc Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Thu, 27 Jun 2024 14:20:39 -0400 Subject: [PATCH 16/22] fix building issues without kvcache --- source/adios2/CMakeLists.txt | 3 +-- source/adios2/engine/bp5/BP5Reader.h | 10 +++---- source/adios2/toolkit/kvcache/KVCacheCommon.h | 26 ++++++++++++++++--- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index dc66a9975b..bbe5c08e06 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -249,8 +249,7 @@ if (ADIOS2_HAVE_SST) endif() if (ADIOS2_HAVE_KVCACHE) - target_sources(adios2_core PRIVATE toolkit/kvcache/KVCacheCommon.h - toolkit/kvcache/KVCacheCommon.cpp) + target_sources(adios2_core PRIVATE toolkit/kvcache/KVCacheCommon.cpp) target_link_libraries(adios2_core PRIVATE hiredis::hiredis) endif () diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h index 7b0288f7d6..6bce35b8f1 100644 --- a/source/adios2/engine/bp5/BP5Reader.h +++ b/source/adios2/engine/bp5/BP5Reader.h @@ -17,13 +17,10 @@ #include "adios2/helper/adiosRangeFilter.h" #include "adios2/toolkit/format/bp5/BP5Deserializer.h" #include "adios2/toolkit/format/buffer/heap/BufferMalloc.h" +#include "adios2/toolkit/kvcache/KVCacheCommon.h" #include "adios2/toolkit/remote/Remote.h" #include "adios2/toolkit/transportman/TransportMan.h" -#ifdef ADIOS2_HAVE_KVCACHE -#include "adios2/toolkit/kvcache/KVCacheCommon.h" -#endif - #include #include #include @@ -102,9 +99,10 @@ class BP5Reader : public BP5Engine, public Engine std::unique_ptr m_Remote; bool m_WriterIsActive = true; adios2::profiling::JSONProfiler m_JSONProfiler; -#ifdef ADIOS2_HAVE_KVCACHE + + /* KVCache for remote data */ KVCacheCommon m_KVCache; -#endif + /** used for per-step reads, TODO: to be moved to BP5Deserializer */ size_t m_CurrentStep = 0; size_t m_StepsCount = 0; diff --git a/source/adios2/toolkit/kvcache/KVCacheCommon.h b/source/adios2/toolkit/kvcache/KVCacheCommon.h index ceab61b8ea..0cf8700b9a 100644 --- a/source/adios2/toolkit/kvcache/KVCacheCommon.h +++ b/source/adios2/toolkit/kvcache/KVCacheCommon.h @@ -6,23 +6,26 @@ #define ADIOS2_KVCACHECOMMON_H #include "QueryBox.h" #include // For memcpy -#include #include #include -// namespace adios2::KVCache +#ifdef ADIOS2_HAVE_KVCACHE +#include +#endif namespace adios2 { class KVCacheCommon { -public: +#ifdef ADIOS2_HAVE_KVCACHE +private: std::string m_host; int m_port; redisContext *m_redisContext; redisReply *m_redisReply; +public: KVCacheCommon(std::string host = "localhost", int port = 6379) : m_host(host), m_port(port){}; ~KVCacheCommon() { CloseConnection(); } @@ -49,6 +52,23 @@ class KVCacheCommon std::string KeyComposition(const std::string &key_prefix, Dims Start, Dims Count); void KeyPrefixExistence(const std::string &key_prefix, std::set &keys); +#else +public: + KVCacheCommon() = default; + ~KVCacheCommon() = default; + void OpenConnection(){}; + void CloseConnection(){}; + void AppendCommandInBatch(const char *key, size_t mode, size_t size, void *data){}; + void ExecuteBatch(const char *key, size_t mode, size_t size, void *data){}; + bool Exists(std::string key) { return false; }; + std::string KeyPrefix(char *VarName, size_t AbsStep, size_t BlockID) { return ""; }; + std::string KeyComposition(const std::string &key_prefix, Dims Start, Dims Count) + { + return ""; + }; + void KeyPrefixExistence(const std::string &key_prefix, std::set &keys){}; + +#endif /* ADIOS2_HAVE_KVCACHE */ }; }; // adios2 From b336025485d0d401ca02bf9c4ea3e6dd6ff3ca52 Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Mon, 1 Jul 2024 17:28:18 -0400 Subject: [PATCH 17/22] convert Dims in QueryBox.h to DimsArray --- source/adios2/engine/bp5/BP5Reader.cpp | 36 +- source/adios2/engine/bp5/BP5Reader.h | 2 +- .../adios2/toolkit/kvcache/KVCacheCommon.cpp | 25 +- source/adios2/toolkit/kvcache/KVCacheCommon.h | 17 +- source/adios2/toolkit/kvcache/QueryBox.h | 313 +++++++++++------- 5 files changed, 227 insertions(+), 166 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 1ac9c1c7d0..bdd9ba6845 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -374,8 +374,7 @@ void BP5Reader::PerformRemoteGetsWithKVCache() std::string CacheKey; size_t TypeSize; bool DirectCopy; - Dims Count; - Dims Start; + kvcache::QueryBox ReqBox; void *Data; }; std::vector remoteRequestsInfo; @@ -386,10 +385,10 @@ void BP5Reader::PerformRemoteGetsWithKVCache() auto &Req = GetRequests[req_seq]; const DataType varType = m_IO.InquireVariableType(Req.VarName); - QueryBox targetBox(Req.Start, Req.Count); + kvcache::QueryBox targetBox(Req.Start, Req.Count); size_t numOfElements = targetBox.size(); std::string keyPrefix = m_KVCache.KeyPrefix(Req.VarName, Req.RelStep, Req.BlockID); - std::string targetKey = m_KVCache.KeyComposition(keyPrefix, Req.Start, Req.Count); + std::string targetKey = keyPrefix + targetBox.toString(); size_t varSize = helper::GetDataTypeSize(varType); RequestInfo ReqInfo; @@ -410,7 +409,7 @@ void BP5Reader::PerformRemoteGetsWithKVCache() int max_depth = 999; std::set samePrefixKeys; m_KVCache.KeyPrefixExistence(keyPrefix, samePrefixKeys); - std::vector regularBoxes; + std::vector regularBoxes; std::vector cachedKeys; if (getenv("maxDepth")) @@ -436,12 +435,15 @@ void BP5Reader::PerformRemoteGetsWithKVCache() { ReqInfo.ReqCount = box.size(); - ReqInfo.CacheKey = m_KVCache.KeyComposition(keyPrefix, box.start, box.count); - ReqInfo.Count = box.count; - ReqInfo.Start = box.start; + ReqInfo.CacheKey = keyPrefix + box.toString(); + ReqInfo.ReqBox = box; ReqInfo.Data = malloc(box.size() * varSize); - auto handle = m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, box.count, - box.start, ReqInfo.Data); + std::vector start; + std::vector count; + box.StartToVector(start); + box.CountToVector(count); + auto handle = m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, count, start, + ReqInfo.Data); handles.push_back(handle); remoteRequestsInfo.push_back(ReqInfo); } @@ -472,11 +474,11 @@ void BP5Reader::PerformRemoteGetsWithKVCache() } else { - QueryBox box(ReqInfo.CacheKey); + kvcache::QueryBox box(ReqInfo.CacheKey); void *data = malloc(box.size() * ReqInfo.TypeSize); m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, box.size() * ReqInfo.TypeSize, data); - helper::NdCopy(reinterpret_cast(data), box.start, box.count, true, false, + helper::NdCopy(reinterpret_cast(data), box.Start, box.Count, true, false, reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, ReqInfo.TypeSize); free(data); @@ -489,9 +491,13 @@ void BP5Reader::PerformRemoteGetsWithKVCache() m_Remote->WaitForGet(handle); auto &ReqInfo = remoteRequestsInfo[handle_seq]; auto &Req = GetRequests[ReqInfo.ReqSeq]; - helper::NdCopy(reinterpret_cast(ReqInfo.Data), ReqInfo.Start, ReqInfo.Count, true, - false, reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, - ReqInfo.TypeSize); + std::vector start; + std::vector count; + ReqInfo.ReqBox.StartToVector(start); + ReqInfo.ReqBox.CountToVector(count); + + helper::NdCopy(reinterpret_cast(ReqInfo.Data), start, count, true, false, reinterpret_cast(Req.Data), + Req.Start, Req.Count, true, false, ReqInfo.TypeSize); m_KVCache.AppendCommandInBatch(ReqInfo.CacheKey.c_str(), 0, ReqInfo.ReqCount * ReqInfo.TypeSize, ReqInfo.Data); diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h index 6bce35b8f1..11357a875b 100644 --- a/source/adios2/engine/bp5/BP5Reader.h +++ b/source/adios2/engine/bp5/BP5Reader.h @@ -101,7 +101,7 @@ class BP5Reader : public BP5Engine, public Engine adios2::profiling::JSONProfiler m_JSONProfiler; /* KVCache for remote data */ - KVCacheCommon m_KVCache; + kvcache::KVCacheCommon m_KVCache; /** used for per-step reads, TODO: to be moved to BP5Deserializer */ size_t m_CurrentStep = 0; diff --git a/source/adios2/toolkit/kvcache/KVCacheCommon.cpp b/source/adios2/toolkit/kvcache/KVCacheCommon.cpp index df79f04ff9..fc6bbe89b0 100644 --- a/source/adios2/toolkit/kvcache/KVCacheCommon.cpp +++ b/source/adios2/toolkit/kvcache/KVCacheCommon.cpp @@ -8,10 +8,11 @@ namespace adios2 { - -void KVCacheCommon::OpenConnection() +namespace kvcache +{ +void KVCacheCommon::OpenConnection(std::string host, int port) { - m_redisContext = redisConnect(m_host.c_str(), m_port); + m_redisContext = redisConnect(host.c_str(), port); if (m_redisContext == NULL || m_redisContext->err) { std::cout << "Error to connect to kvcache server: " << m_redisContext->errstr << std::endl; @@ -126,22 +127,6 @@ std::string KVCacheCommon::KeyPrefix(char *VarName, size_t AbsStep, size_t Block return VarName + std::to_string(AbsStep) + std::to_string(BlockID); } -std::string KVCacheCommon::KeyComposition(const std::string &key_prefix, Dims Start, Dims Count) -{ - std::string box = QueryBox::SerializeQueryBox(QueryBox{Start, Count}); - std::string cacheKey = key_prefix + box; - // replace special characters - std::replace(cacheKey.begin(), cacheKey.end(), '"', '_'); - std::replace(cacheKey.begin(), cacheKey.end(), ',', '_'); - std::replace(cacheKey.begin(), cacheKey.end(), '(', '_'); - std::replace(cacheKey.begin(), cacheKey.end(), ')', '_'); - std::replace(cacheKey.begin(), cacheKey.end(), '[', '_'); - std::replace(cacheKey.begin(), cacheKey.end(), ']', '_'); - std::replace(cacheKey.begin(), cacheKey.end(), '{', '_'); - std::replace(cacheKey.begin(), cacheKey.end(), '}', '_'); - return cacheKey; -} - void KVCacheCommon::KeyPrefixExistence(const std::string &key_prefix, std::set &keys) { m_redisReply = (redisReply *)redisCommand(m_redisContext, "KEYS %s*", key_prefix.c_str()); @@ -158,6 +143,6 @@ void KVCacheCommon::KeyPrefixExistence(const std::string &key_prefix, std::set &keys); #else public: @@ -62,15 +59,11 @@ class KVCacheCommon void ExecuteBatch(const char *key, size_t mode, size_t size, void *data){}; bool Exists(std::string key) { return false; }; std::string KeyPrefix(char *VarName, size_t AbsStep, size_t BlockID) { return ""; }; - std::string KeyComposition(const std::string &key_prefix, Dims Start, Dims Count) - { - return ""; - }; void KeyPrefixExistence(const std::string &key_prefix, std::set &keys){}; #endif /* ADIOS2_HAVE_KVCACHE */ }; - +}; // namespace kvcache }; // adios2 #endif // ADIOS2_KVCACHECOMMON_H diff --git a/source/adios2/toolkit/kvcache/QueryBox.h b/source/adios2/toolkit/kvcache/QueryBox.h index 56dc5f8f95..0ad473935e 100644 --- a/source/adios2/toolkit/kvcache/QueryBox.h +++ b/source/adios2/toolkit/kvcache/QueryBox.h @@ -5,158 +5,248 @@ #ifndef ADIOS2_KVCACHE_QUERYBOX_H #define ADIOS2_KVCACHE_QUERYBOX_H -#include +#include +#include #include -#include #include +#include namespace adios2 { -// QueryBox is a class to represent a query box in a multi-dimensional space + +namespace kvcache +{ + +// QueryBox is a class to represent a box in a multi-dimensional space class QueryBox { public: - adios2::Dims start{}; - adios2::Dims count{}; + helper::DimsArray Start; + helper::DimsArray Count; + size_t DimCount; - // constructor - QueryBox() = default; - QueryBox(const adios2::Dims &start, const adios2::Dims &count) : start(start), count(count){}; - QueryBox(const std::string &key) + QueryBox() : Start(helper::MAX_DIMS), Count(helper::MAX_DIMS), DimCount(helper::MAX_DIMS) {} + + explicit QueryBox(size_t dimCount) : Start(dimCount), Count(dimCount) { + DimCount = dimCount; + for (size_t i = 0; i < DimCount; ++i) + { + Start[i] = 0; + Count[i] = 0; + } + } + + QueryBox(const helper::DimsArray &start, const helper::DimsArray &count) + : Start(start.size()), Count(count.size()) { - // sample key: "U3218446744073709551615__count_:_64_64_64___start_:_0_0_0__", count [64, 64, - // 64], start [0, 0, 0] using Dims = std::vector; - auto lf_ExtractDimensions = [](const std::string &key, - const std::string &delimiter) -> Dims { - size_t const pos = key.find(delimiter); - size_t const end = key.find("__", pos + delimiter.length()); - std::string dimStr = - key.substr(pos + delimiter.length(), end - pos - delimiter.length()); - Dims dimensions; + DimCount = start.size(); + for (size_t i = 0; i < DimCount; ++i) + { + Start[i] = start[i]; + Count[i] = count[i]; + } + } + + QueryBox(const QueryBox &box) : Start(box.Start.size()), Count(box.Count.size()) + { + DimCount = box.DimCount; + for (size_t i = 0; i < DimCount; ++i) + { + Start[i] = box.Start[i]; + Count[i] = box.Count[i]; + } + } + + // Constructor with key string + explicit QueryBox(const std::string &key) + : Start(helper::MAX_DIMS), Count(helper::MAX_DIMS), DimCount(helper::MAX_DIMS) + { + // Lambda function to extract dimensions from key string + auto lf_ExtractDimensions = [](const std::string &key, const std::string &delimiter, + std::string &dimStr) { + size_t pos = key.find(delimiter); + if (pos == std::string::npos) + { + throw std::invalid_argument("Delimiter not found in key"); + } + size_t end = key.find("|", pos + delimiter.length()); + if (end == std::string::npos) + { + throw std::invalid_argument("End delimiter not found in key"); + } + dimStr = key.substr(pos + delimiter.length(), end - pos - delimiter.length()); + }; + + auto lf_ExtractBox = [](const std::string &dimStr, helper::DimsArray &data) { std::istringstream dimStream(dimStr); std::string token; + size_t i = 0; while (std::getline(dimStream, token, '_')) { - dimensions.push_back(std::stoul(token)); + data[i] = std::stoul(token); + i++; } - return dimensions; }; - this->start = lf_ExtractDimensions(key, "__start_:_"); - this->count = lf_ExtractDimensions(key, "__count_:_"); + std::string startDimStr; + std::string countDimStr; + + lf_ExtractDimensions(key, "|Start_", startDimStr); + lf_ExtractDimensions(key, "|Count_", countDimStr); + + DimCount = std::count(startDimStr.begin(), startDimStr.end(), '_') + 1; + + lf_ExtractBox(startDimStr, Start); + lf_ExtractBox(countDimStr, Count); } // size size_t size() const { size_t s = 1; - for (auto &d : count) + for (size_t i = 0; i < DimCount; ++i) { - s *= d; + s *= Count[i]; } return s; } - // Serialize QueryBox to a string, like __count_:_64_64_64___start_:_0_0_0__ - static std::string SerializeQueryBox(const QueryBox &box) + // ToString + std::string toString() const { - nlohmann::json jsonBox; - jsonBox["start"] = box.start; - jsonBox["count"] = box.count; - return jsonBox.dump(); + std::string str = "|Start_"; + for (size_t i = 0; i < DimCount; ++i) + { + str += std::to_string(Start[i]); + if (i != DimCount - 1) + { + str += "_"; + } + } + str += "|Count_"; + for (size_t i = 0; i < DimCount; ++i) + { + str += std::to_string(Count[i]); + if (i != DimCount - 1) + { + str += "_"; + } + } + str += "|"; + return str; } - // determine if a query box is equal to another query box - bool operator==(const QueryBox &box) const { return start == box.start && count == box.count; } + // determine if a box is equal to another box + bool operator==(const QueryBox &box) const + { + return std::strcmp(toString().c_str(), box.toString().c_str()) == 0; + } - // determine if a query box is interacted in another query box, return intersection part as a - // new query box - bool IsInteracted(const QueryBox &box, QueryBox &intersection) const + // Assignment operator + QueryBox &operator=(const QueryBox &box) { - if (start.size() != box.start.size() || start.size() != count.size() || - start.size() != box.count.size()) + if (this == &box) { - return false; + return *this; // handle self-assignment } - for (size_t i = 0; i < start.size(); ++i) + + // Copy the size + DimCount = box.DimCount; + + // Copy elements + for (size_t i = 0; i < DimCount; ++i) { - if (start[i] > box.start[i] + box.count[i] || box.start[i] > start[i] + count[i]) - { - return false; - } + Start[i] = box.Start[i]; + Count[i] = box.Count[i]; } - intersection.start.resize(start.size()); - intersection.count.resize(count.size()); - for (size_t i = 0; i < start.size(); ++i) + + return *this; + } + + // convert helper::DimsArray to std::vector + void StartToVector(std::vector &vec) const + { + for (size_t i = 0; i < DimCount; ++i) { - intersection.start[i] = std::max(start[i], box.start[i]); - intersection.count[i] = - std::min(start[i] + count[i], box.start[i] + box.count[i]) - intersection.start[i]; + vec.push_back(Start[i]); } - return true; } - // determine if a query box is fully contained in another query box - bool IsFullContainedBy(const QueryBox &box) + void CountToVector(std::vector &vec) const { - if (start.size() != box.start.size() || start.size() != count.size() || - start.size() != box.count.size()) + for (size_t i = 0; i < DimCount; ++i) + { + vec.push_back(Count[i]); + } + } + + // check if *this is interacted in another box, return the new intersection box pointer + bool IsInteracted(const QueryBox &box, QueryBox &intersection) const + { + if (DimCount != box.DimCount) { return false; } - for (size_t i = 0; i < start.size(); ++i) + + for (size_t i = 0; i < DimCount; ++i) { - if (start[i] < box.start[i] || start[i] + count[i] > box.start[i] + box.count[i]) + if (Start[i] > box.Start[i] + box.Count[i] || box.Start[i] > Start[i] + Count[i]) { return false; } } + + for (size_t i = 0; i < DimCount; ++i) + { + intersection.Start[i] = std::max(Start[i], box.Start[i]); + intersection.Count[i] = + std::min(Start[i] + Count[i], box.Start[i] + box.Count[i]) - intersection.Start[i]; + } return true; } - // cut a query box from another interaction box, return a list of regular box - // remainingBox is the big one, this is small one - void InteractionCut(const QueryBox &remainingBox, std::vector ®ularBoxes) + // cut *this from big box, return a list of regular boxes + // Note: *this must be fully contained by big box + void NdCut(const QueryBox &bigBox, std::vector ®ularBoxes) { - if (remainingBox == *this) + if (bigBox == *this) { return; } - // find the max cut dimension + // find the cut dimension with the biggest size size_t maxCutDimSize = 0; - QueryBox maxCutDimBox; - for (size_t i = 0; i < start.size(); ++i) + QueryBox maxCutDimBox(DimCount); + for (size_t i = 0; i < DimCount; ++i) { - if (start[i] == remainingBox.start[i] && count[i] == remainingBox.count[i]) + // if the start and count are the same, means no cut in this dimension, skip + if (Start[i] == bigBox.Start[i] && Count[i] == bigBox.Count[i]) { continue; } else { - if (start[i] != remainingBox.start[i]) + if (Start[i] != bigBox.Start[i]) { - size_t cutDimDiff = start[i] - remainingBox.start[i]; - size_t cutDimSize = remainingBox.size() / remainingBox.count[i] * cutDimDiff; + size_t cutDimDiff = Start[i] - bigBox.Start[i]; + size_t cutDimSize = bigBox.size() / bigBox.Count[i] * cutDimDiff; if (cutDimSize > maxCutDimSize) { maxCutDimSize = cutDimSize; - maxCutDimBox = QueryBox(remainingBox.start, remainingBox.count); - maxCutDimBox.count[i] = cutDimDiff; + maxCutDimBox = bigBox; + maxCutDimBox.Count[i] = cutDimDiff; } } - - if (start[i] + count[i] != remainingBox.start[i] + remainingBox.count[i]) + if (Start[i] + Count[i] != bigBox.Start[i] + bigBox.Count[i]) { - size_t cutDimDiff = - remainingBox.start[i] + remainingBox.count[i] - start[i] - count[i]; - size_t cutDimSize = remainingBox.size() / count[i] * cutDimDiff; + size_t cutDimDiff = bigBox.Start[i] + bigBox.Count[i] - Start[i] - Count[i]; + size_t cutDimSize = bigBox.size() / bigBox.Count[i] * cutDimDiff; if (cutDimSize > maxCutDimSize) { maxCutDimSize = cutDimSize; - maxCutDimBox = QueryBox(remainingBox.start, remainingBox.count); - maxCutDimBox.start[i] = start[i] + count[i]; - maxCutDimBox.count[i] = cutDimDiff; + maxCutDimBox = bigBox; + maxCutDimBox.Start[i] = Start[i] + Count[i]; + maxCutDimBox.Count[i] = cutDimDiff; } } } @@ -166,29 +256,30 @@ class QueryBox if (maxCutDimSize > 0) { regularBoxes.push_back(maxCutDimBox); - QueryBox remainingBox1 = QueryBox(remainingBox.start, remainingBox.count); - for (size_t i = 0; i < remainingBox.start.size(); ++i) + QueryBox bigBoxRemained(bigBox); + for (size_t i = 0; i < DimCount; ++i) { - if (maxCutDimBox.start[i] == remainingBox.start[i] && - maxCutDimBox.count[i] == remainingBox.count[i]) + if (maxCutDimBox.Start[i] == bigBox.Start[i] && + maxCutDimBox.Count[i] == bigBox.Count[i]) { continue; } else { - if (maxCutDimBox.start[i] != remainingBox.start[i]) + if (maxCutDimBox.Start[i] == bigBox.Start[i]) { - remainingBox1.count[i] = maxCutDimBox.start[i] - remainingBox.start[i]; + bigBoxRemained.Start[i] = maxCutDimBox.Start[i] + maxCutDimBox.Count[i]; + bigBoxRemained.Count[i] = + bigBox.Start[i] + bigBox.Count[i] - bigBoxRemained.Start[i]; } else { - remainingBox1.start[i] = maxCutDimBox.start[i] + maxCutDimBox.count[i]; - remainingBox1.count[i] = - remainingBox.start[i] + remainingBox.count[i] - remainingBox1.start[i]; + bigBoxRemained.Count[i] = maxCutDimBox.Start[i] - bigBox.Start[i]; } } } - InteractionCut(remainingBox1, regularBoxes); + + NdCut(bigBoxRemained, regularBoxes); } } @@ -201,23 +292,25 @@ class QueryBox return; } current_depth++; - QueryBox maxInteractBox; + QueryBox maxInteractBox(this->DimCount); std::string maxInteractKey; + bool foundMaxInteract = false; for (auto &key : samePrefixKeys) { QueryBox const box(key); - QueryBox intersection; + QueryBox intersection(this->DimCount); if (this->IsInteracted(box, intersection)) { if (maxInteractBox.size() < intersection.size()) { maxInteractBox = intersection; maxInteractKey = key; + foundMaxInteract = true; } } } - if (maxInteractBox.count.size() == 0) + if (!foundMaxInteract) { regularBoxes.push_back(*this); return; @@ -225,14 +318,20 @@ class QueryBox cachedKeys.push_back(maxInteractKey); + // If the current box is the max interact box, return, avoid cutting + if (this->size() == maxInteractBox.size()) + { + return; + } + if (current_depth == max_depth) { - maxInteractBox.InteractionCut(*this, regularBoxes); + maxInteractBox.NdCut(*this, regularBoxes); } else { std::vector nextBoxes; - maxInteractBox.InteractionCut(*this, nextBoxes); + maxInteractBox.NdCut(*this, nextBoxes); for (auto &box : nextBoxes) { box.GetMaxInteractBox(samePrefixKeys, max_depth, current_depth, regularBoxes, @@ -240,31 +339,9 @@ class QueryBox } } } - - // rewrite toString - std::string toString() const - { - std::string str = "Box start: ["; - for (size_t i = 0; i < start.size(); ++i) - { - str += std::to_string(start[i]); - if (i != start.size() - 1) - { - str += ", "; - } - } - str += "], count: ["; - for (size_t i = 0; i < count.size(); ++i) - { - str += std::to_string(count[i]); - if (i != count.size() - 1) - { - str += ", "; - } - } - str += "]"; - return str; - } -}; }; + +} // namespace kvcache +} // namespace adios2 + #endif // ADIOS2_KVCACHE_QUERYBOX_H From fbf1cf2e428419d1b8d9c63b9a8d77a0259e8fe9 Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Mon, 1 Jul 2024 19:05:40 -0400 Subject: [PATCH 18/22] fix copy issues in DimArrays --- source/adios2/engine/bp5/BP5Reader.cpp | 18 +++++++++++------- source/adios2/helper/adiosType.h | 10 ++++++++++ source/adios2/toolkit/kvcache/QueryBox.h | 3 ++- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index bdd9ba6845..9a3b3c122b 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -478,7 +478,11 @@ void BP5Reader::PerformRemoteGetsWithKVCache() void *data = malloc(box.size() * ReqInfo.TypeSize); m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, box.size() * ReqInfo.TypeSize, data); - helper::NdCopy(reinterpret_cast(data), box.Start, box.Count, true, false, + + helper::DimsArray startArray(box.DimCount, box.Start); + helper::DimsArray countArray(box.DimCount, box.Count); + + helper::NdCopy(reinterpret_cast(data), startArray, countArray, true, false, reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, ReqInfo.TypeSize); free(data); @@ -491,13 +495,13 @@ void BP5Reader::PerformRemoteGetsWithKVCache() m_Remote->WaitForGet(handle); auto &ReqInfo = remoteRequestsInfo[handle_seq]; auto &Req = GetRequests[ReqInfo.ReqSeq]; - std::vector start; - std::vector count; - ReqInfo.ReqBox.StartToVector(start); - ReqInfo.ReqBox.CountToVector(count); - helper::NdCopy(reinterpret_cast(ReqInfo.Data), start, count, true, false, reinterpret_cast(Req.Data), - Req.Start, Req.Count, true, false, ReqInfo.TypeSize); + helper::DimsArray startArray(ReqInfo.ReqBox.DimCount, ReqInfo.ReqBox.Start); + helper::DimsArray countArray(ReqInfo.ReqBox.DimCount, ReqInfo.ReqBox.Count); + + helper::NdCopy(reinterpret_cast(ReqInfo.Data), startArray, countArray, true, false, + reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, + ReqInfo.TypeSize); m_KVCache.AppendCommandInBatch(ReqInfo.CacheKey.c_str(), 0, ReqInfo.ReqCount * ReqInfo.TypeSize, ReqInfo.Data); diff --git a/source/adios2/helper/adiosType.h b/source/adios2/helper/adiosType.h index 24f360d524..319c1a1d0f 100644 --- a/source/adios2/helper/adiosType.h +++ b/source/adios2/helper/adiosType.h @@ -177,6 +177,16 @@ class DimsArray : public CoreDims { std::copy(d1.begin(), d1.end(), &Dimensions[0]); } + + DimsArray(const DimsArray &d1) : CoreDims(d1.size(), &Dimensions[0]) + { + std::copy(d1.begin(), d1.end(), &Dimensions[0]); + } + + DimsArray(const size_t count, const DimsArray &d1) : CoreDims(count, &Dimensions[0]) + { + std::copy(d1.begin(), d1.end(), &Dimensions[0]); + } }; /** diff --git a/source/adios2/toolkit/kvcache/QueryBox.h b/source/adios2/toolkit/kvcache/QueryBox.h index 0ad473935e..f9e2e60d44 100644 --- a/source/adios2/toolkit/kvcache/QueryBox.h +++ b/source/adios2/toolkit/kvcache/QueryBox.h @@ -27,7 +27,8 @@ class QueryBox QueryBox() : Start(helper::MAX_DIMS), Count(helper::MAX_DIMS), DimCount(helper::MAX_DIMS) {} - explicit QueryBox(size_t dimCount) : Start(dimCount), Count(dimCount) { + explicit QueryBox(size_t dimCount) : Start(dimCount), Count(dimCount) + { DimCount = dimCount; for (size_t i = 0; i < DimCount; ++i) { From 4218e961a17b720236976b74a20d24283a86c505 Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Wed, 3 Jul 2024 19:46:15 -0400 Subject: [PATCH 19/22] update QueryBox to be compatible with PR 4221 --- source/adios2/engine/bp5/BP5Reader.cpp | 79 ++++---- .../adios2/toolkit/kvcache/KVCacheCommon.cpp | 21 +-- source/adios2/toolkit/kvcache/KVCacheCommon.h | 9 +- source/adios2/toolkit/kvcache/QueryBox.h | 168 +++++++----------- 4 files changed, 107 insertions(+), 170 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 9a3b3c122b..b50c2d18ce 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -369,57 +369,58 @@ void BP5Reader::PerformRemoteGetsWithKVCache() struct RequestInfo { size_t ReqSeq; - DataType varType; - size_t ReqCount; - std::string CacheKey; size_t TypeSize; + size_t ReqSize; + std::string CacheKey; bool DirectCopy; kvcache::QueryBox ReqBox; void *Data; + + // Constructor to initialize Start and Count with DimCount + RequestInfo(size_t dimCount) : ReqBox(dimCount) {} }; std::vector remoteRequestsInfo; std::vector cachedRequestsInfo; for (size_t req_seq = 0; req_seq < GetRequests.size(); req_seq++) { - auto &Req = GetRequests[req_seq]; + const auto &Req = GetRequests[req_seq]; const DataType varType = m_IO.InquireVariableType(Req.VarName); + RequestInfo ReqInfo(Req.Count.size()); + ReqInfo.ReqSeq = req_seq; + ReqInfo.TypeSize = helper::GetDataTypeSize(varType); + kvcache::QueryBox targetBox(Req.Start, Req.Count); - size_t numOfElements = targetBox.size(); - std::string keyPrefix = m_KVCache.KeyPrefix(Req.VarName, Req.RelStep, Req.BlockID); + std::string keyPrefix = + Req.VarName + std::to_string(Req.RelStep) + std::to_string(Req.BlockID); std::string targetKey = keyPrefix + targetBox.toString(); - size_t varSize = helper::GetDataTypeSize(varType); - - RequestInfo ReqInfo; - ReqInfo.ReqSeq = req_seq; - ReqInfo.varType = varType; - ReqInfo.TypeSize = varSize; // Exact Match: check if targetKey exists if (m_KVCache.Exists(targetKey)) { ReqInfo.CacheKey = targetKey; - ReqInfo.ReqCount = numOfElements; ReqInfo.DirectCopy = true; + ReqInfo.ReqSize = targetBox.size(); cachedRequestsInfo.push_back(ReqInfo); } else { int max_depth = 999; - std::set samePrefixKeys; - m_KVCache.KeyPrefixExistence(keyPrefix, samePrefixKeys); - std::vector regularBoxes; - std::vector cachedKeys; - if (getenv("maxDepth")) { max_depth = std::stoi(getenv("maxDepth")); } + std::unordered_set samePrefixKeys; + std::vector regularBoxes; + std::vector cachedBoxes; + m_KVCache.KeyPrefixExistence(keyPrefix, samePrefixKeys); + if (samePrefixKeys.size() > 0) { - targetBox.GetMaxInteractBox(samePrefixKeys, max_depth, 0, regularBoxes, cachedKeys); + targetBox.GetMaxInteractBox(samePrefixKeys, max_depth, 0, regularBoxes, + cachedBoxes); } else { @@ -427,17 +428,17 @@ void BP5Reader::PerformRemoteGetsWithKVCache() } std::cout << "Going to retrieve " << regularBoxes.size() - << " boxes from remote server, and " << cachedKeys.size() + << " boxes from remote server, and " << cachedBoxes.size() << " boxes from cache" << std::endl; // Get data from remote server for (auto &box : regularBoxes) { - ReqInfo.ReqCount = box.size(); + ReqInfo.ReqSize = box.size(); ReqInfo.CacheKey = keyPrefix + box.toString(); ReqInfo.ReqBox = box; - ReqInfo.Data = malloc(box.size() * varSize); + ReqInfo.Data = malloc(ReqInfo.ReqSize * ReqInfo.TypeSize); std::vector start; std::vector count; box.StartToVector(start); @@ -449,9 +450,10 @@ void BP5Reader::PerformRemoteGetsWithKVCache() } // Get data from cache - for (auto &boxKey : cachedKeys) + for (auto &box : cachedBoxes) { - ReqInfo.CacheKey = boxKey; + ReqInfo.CacheKey = keyPrefix + box.toString(); + ReqInfo.ReqBox = box; ReqInfo.DirectCopy = false; cachedRequestsInfo.push_back(ReqInfo); } @@ -469,22 +471,17 @@ void BP5Reader::PerformRemoteGetsWithKVCache() auto &Req = GetRequests[ReqInfo.ReqSeq]; if (ReqInfo.DirectCopy) { - m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, ReqInfo.ReqCount * ReqInfo.TypeSize, + m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, ReqInfo.ReqSize * ReqInfo.TypeSize, Req.Data); } else { - kvcache::QueryBox box(ReqInfo.CacheKey); - void *data = malloc(box.size() * ReqInfo.TypeSize); - m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, box.size() * ReqInfo.TypeSize, + void *data = malloc(ReqInfo.ReqBox.size() * ReqInfo.TypeSize); + m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, ReqInfo.ReqBox.size() * ReqInfo.TypeSize, data); - - helper::DimsArray startArray(box.DimCount, box.Start); - helper::DimsArray countArray(box.DimCount, box.Count); - - helper::NdCopy(reinterpret_cast(data), startArray, countArray, true, false, - reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, - ReqInfo.TypeSize); + helper::NdCopy(reinterpret_cast(data), ReqInfo.ReqBox.Start, + ReqInfo.ReqBox.Count, true, false, reinterpret_cast(Req.Data), + Req.Start, Req.Count, true, false, ReqInfo.TypeSize); free(data); } } @@ -495,16 +492,12 @@ void BP5Reader::PerformRemoteGetsWithKVCache() m_Remote->WaitForGet(handle); auto &ReqInfo = remoteRequestsInfo[handle_seq]; auto &Req = GetRequests[ReqInfo.ReqSeq]; - - helper::DimsArray startArray(ReqInfo.ReqBox.DimCount, ReqInfo.ReqBox.Start); - helper::DimsArray countArray(ReqInfo.ReqBox.DimCount, ReqInfo.ReqBox.Count); - - helper::NdCopy(reinterpret_cast(ReqInfo.Data), startArray, countArray, true, false, - reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, - ReqInfo.TypeSize); + helper::NdCopy(reinterpret_cast(ReqInfo.Data), ReqInfo.ReqBox.Start, + ReqInfo.ReqBox.Count, true, false, reinterpret_cast(Req.Data), + Req.Start, Req.Count, true, false, ReqInfo.TypeSize); m_KVCache.AppendCommandInBatch(ReqInfo.CacheKey.c_str(), 0, - ReqInfo.ReqCount * ReqInfo.TypeSize, ReqInfo.Data); + ReqInfo.ReqSize * ReqInfo.TypeSize, ReqInfo.Data); free(ReqInfo.Data); } diff --git a/source/adios2/toolkit/kvcache/KVCacheCommon.cpp b/source/adios2/toolkit/kvcache/KVCacheCommon.cpp index fc6bbe89b0..2dc094ce4f 100644 --- a/source/adios2/toolkit/kvcache/KVCacheCommon.cpp +++ b/source/adios2/toolkit/kvcache/KVCacheCommon.cpp @@ -93,19 +93,6 @@ void KVCacheCommon::ExecuteBatch(const char *key, size_t mode, size_t size, void } } -void KVCacheCommon::Del(std::string key) -{ - m_redisReply = (redisReply *)redisCommand(m_redisContext, "DEL %s", key.c_str()); - if (m_redisReply == NULL) - { - std::cout << "Error to delete key: " << key << std::endl; - } - else - { - freeReplyObject(m_redisReply); - } -} - bool KVCacheCommon::Exists(std::string key) { m_redisReply = (redisReply *)redisCommand(m_redisContext, "EXISTS %s", key.c_str()); @@ -122,12 +109,8 @@ bool KVCacheCommon::Exists(std::string key) return false; } -std::string KVCacheCommon::KeyPrefix(char *VarName, size_t AbsStep, size_t BlockID) -{ - return VarName + std::to_string(AbsStep) + std::to_string(BlockID); -} - -void KVCacheCommon::KeyPrefixExistence(const std::string &key_prefix, std::set &keys) +void KVCacheCommon::KeyPrefixExistence(const std::string &key_prefix, + std::unordered_set &keys) { m_redisReply = (redisReply *)redisCommand(m_redisContext, "KEYS %s*", key_prefix.c_str()); if (m_redisReply == NULL) diff --git a/source/adios2/toolkit/kvcache/KVCacheCommon.h b/source/adios2/toolkit/kvcache/KVCacheCommon.h index a6007cc3dd..f17f25117a 100644 --- a/source/adios2/toolkit/kvcache/KVCacheCommon.h +++ b/source/adios2/toolkit/kvcache/KVCacheCommon.h @@ -42,13 +42,9 @@ class KVCacheCommon void ExecuteBatch(const char *key, size_t mode, size_t size, void *data); - void Del(std::string key); - bool Exists(std::string key); - std::string KeyPrefix(char *VarName, size_t AbsStep, size_t BlockID); - - void KeyPrefixExistence(const std::string &key_prefix, std::set &keys); + void KeyPrefixExistence(const std::string &key_prefix, std::unordered_set &keys); #else public: KVCacheCommon() = default; @@ -58,8 +54,7 @@ class KVCacheCommon void AppendCommandInBatch(const char *key, size_t mode, size_t size, void *data){}; void ExecuteBatch(const char *key, size_t mode, size_t size, void *data){}; bool Exists(std::string key) { return false; }; - std::string KeyPrefix(char *VarName, size_t AbsStep, size_t BlockID) { return ""; }; - void KeyPrefixExistence(const std::string &key_prefix, std::set &keys){}; + void KeyPrefixExistence(const std::string &key_prefix, std::unordered_set &keys){}; #endif /* ADIOS2_HAVE_KVCACHE */ }; diff --git a/source/adios2/toolkit/kvcache/QueryBox.h b/source/adios2/toolkit/kvcache/QueryBox.h index f9e2e60d44..860a391fdb 100644 --- a/source/adios2/toolkit/kvcache/QueryBox.h +++ b/source/adios2/toolkit/kvcache/QueryBox.h @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include namespace adios2 @@ -23,14 +23,10 @@ class QueryBox public: helper::DimsArray Start; helper::DimsArray Count; - size_t DimCount; - - QueryBox() : Start(helper::MAX_DIMS), Count(helper::MAX_DIMS), DimCount(helper::MAX_DIMS) {} explicit QueryBox(size_t dimCount) : Start(dimCount), Count(dimCount) { - DimCount = dimCount; - for (size_t i = 0; i < DimCount; ++i) + for (size_t i = 0; i < dimCount; ++i) { Start[i] = 0; Count[i] = 0; @@ -38,74 +34,17 @@ class QueryBox } QueryBox(const helper::DimsArray &start, const helper::DimsArray &count) - : Start(start.size()), Count(count.size()) - { - DimCount = start.size(); - for (size_t i = 0; i < DimCount; ++i) - { - Start[i] = start[i]; - Count[i] = count[i]; - } - } - - QueryBox(const QueryBox &box) : Start(box.Start.size()), Count(box.Count.size()) + : Start(start), Count(count) { - DimCount = box.DimCount; - for (size_t i = 0; i < DimCount; ++i) - { - Start[i] = box.Start[i]; - Count[i] = box.Count[i]; - } } - // Constructor with key string - explicit QueryBox(const std::string &key) - : Start(helper::MAX_DIMS), Count(helper::MAX_DIMS), DimCount(helper::MAX_DIMS) - { - // Lambda function to extract dimensions from key string - auto lf_ExtractDimensions = [](const std::string &key, const std::string &delimiter, - std::string &dimStr) { - size_t pos = key.find(delimiter); - if (pos == std::string::npos) - { - throw std::invalid_argument("Delimiter not found in key"); - } - size_t end = key.find("|", pos + delimiter.length()); - if (end == std::string::npos) - { - throw std::invalid_argument("End delimiter not found in key"); - } - dimStr = key.substr(pos + delimiter.length(), end - pos - delimiter.length()); - }; - - auto lf_ExtractBox = [](const std::string &dimStr, helper::DimsArray &data) { - std::istringstream dimStream(dimStr); - std::string token; - size_t i = 0; - while (std::getline(dimStream, token, '_')) - { - data[i] = std::stoul(token); - i++; - } - }; - - std::string startDimStr; - std::string countDimStr; - - lf_ExtractDimensions(key, "|Start_", startDimStr); - lf_ExtractDimensions(key, "|Count_", countDimStr); - - DimCount = std::count(startDimStr.begin(), startDimStr.end(), '_') + 1; - - lf_ExtractBox(startDimStr, Start); - lf_ExtractBox(countDimStr, Count); - } + QueryBox(const QueryBox &box) : Start(box.Start), Count(box.Count) {} // size size_t size() const { size_t s = 1; - for (size_t i = 0; i < DimCount; ++i) + for (size_t i = 0; i < Count.size(); ++i) { s *= Count[i]; } @@ -116,19 +55,19 @@ class QueryBox std::string toString() const { std::string str = "|Start_"; - for (size_t i = 0; i < DimCount; ++i) + for (size_t i = 0; i < Start.size(); ++i) { str += std::to_string(Start[i]); - if (i != DimCount - 1) + if (i != Start.size() - 1) { str += "_"; } } str += "|Count_"; - for (size_t i = 0; i < DimCount; ++i) + for (size_t i = 0; i < Start.size(); ++i) { str += std::to_string(Count[i]); - if (i != DimCount - 1) + if (i != Start.size() - 1) { str += "_"; } @@ -151,11 +90,8 @@ class QueryBox return *this; // handle self-assignment } - // Copy the size - DimCount = box.DimCount; - // Copy elements - for (size_t i = 0; i < DimCount; ++i) + for (size_t i = 0; i < box.Start.size(); ++i) { Start[i] = box.Start[i]; Count[i] = box.Count[i]; @@ -167,7 +103,7 @@ class QueryBox // convert helper::DimsArray to std::vector void StartToVector(std::vector &vec) const { - for (size_t i = 0; i < DimCount; ++i) + for (size_t i = 0; i < Start.size(); ++i) { vec.push_back(Start[i]); } @@ -175,7 +111,7 @@ class QueryBox void CountToVector(std::vector &vec) const { - for (size_t i = 0; i < DimCount; ++i) + for (size_t i = 0; i < Count.size(); ++i) { vec.push_back(Count[i]); } @@ -184,12 +120,12 @@ class QueryBox // check if *this is interacted in another box, return the new intersection box pointer bool IsInteracted(const QueryBox &box, QueryBox &intersection) const { - if (DimCount != box.DimCount) + if (Start.size() != box.Start.size()) { return false; } - for (size_t i = 0; i < DimCount; ++i) + for (size_t i = 0; i < Start.size(); ++i) { if (Start[i] > box.Start[i] + box.Count[i] || box.Start[i] > Start[i] + Count[i]) { @@ -197,7 +133,7 @@ class QueryBox } } - for (size_t i = 0; i < DimCount; ++i) + for (size_t i = 0; i < Start.size(); ++i) { intersection.Start[i] = std::max(Start[i], box.Start[i]); intersection.Count[i] = @@ -217,8 +153,8 @@ class QueryBox // find the cut dimension with the biggest size size_t maxCutDimSize = 0; - QueryBox maxCutDimBox(DimCount); - for (size_t i = 0; i < DimCount; ++i) + QueryBox maxCutDimBox(Start.size()); + for (size_t i = 0; i < Start.size(); ++i) { // if the start and count are the same, means no cut in this dimension, skip if (Start[i] == bigBox.Start[i] && Count[i] == bigBox.Count[i]) @@ -258,7 +194,7 @@ class QueryBox { regularBoxes.push_back(maxCutDimBox); QueryBox bigBoxRemained(bigBox); - for (size_t i = 0; i < DimCount; ++i) + for (size_t i = 0; i < Start.size(); ++i) { if (maxCutDimBox.Start[i] == bigBox.Start[i] && maxCutDimBox.Count[i] == bigBox.Count[i]) @@ -284,59 +220,89 @@ class QueryBox } } - void GetMaxInteractBox(const std::set &samePrefixKeys, const size_t &max_depth, - size_t current_depth, std::vector ®ularBoxes, - std::vector &cachedKeys) + void GetMaxInteractBox(const std::unordered_set &samePrefixKeys, + const size_t &max_depth, size_t current_depth, + std::vector ®ularBoxes, std::vector &cachedBoxes) { + // Lambda function to extract dimensions from key string + auto lf_ExtractDimensions = [](const std::string &key, const std::string &delimiter, + helper::DimsArray &data) { + size_t pos = key.find(delimiter); + if (pos == std::string::npos) + { + throw std::invalid_argument("Delimiter not found in key"); + } + size_t end = key.find("|", pos + delimiter.length()); + if (end == std::string::npos) + { + throw std::invalid_argument("End delimiter not found in key"); + } + std::string dimStr = + key.substr(pos + delimiter.length(), end - pos - delimiter.length()); + std::istringstream dimStream(dimStr); + std::string token; + size_t i = 0; + while (std::getline(dimStream, token, '_')) + { + data[i] = std::stoul(token); + i++; + } + }; + if (current_depth > max_depth) { return; } current_depth++; - QueryBox maxInteractBox(this->DimCount); - std::string maxInteractKey; - bool foundMaxInteract = false; + + QueryBox maxSourceBox(this->Start.size()); + QueryBox maxInteract(this->Start.size()); for (auto &key : samePrefixKeys) { - QueryBox const box(key); - QueryBox intersection(this->DimCount); + // Initialize the box from the key + size_t DimCount = std::count(key.begin(), key.end(), '_') / 2; + QueryBox box(DimCount); + lf_ExtractDimensions(key, "|Start_", box.Start); + lf_ExtractDimensions(key, "|Count_", box.Count); + + QueryBox intersection(this->Start.size()); if (this->IsInteracted(box, intersection)) { - if (maxInteractBox.size() < intersection.size()) + if (maxInteract.size() < intersection.size()) { - maxInteractBox = intersection; - maxInteractKey = key; - foundMaxInteract = true; + maxInteract = intersection; + maxSourceBox = box; } } } - if (!foundMaxInteract) + if (maxInteract.size() == 0) { regularBoxes.push_back(*this); + // existing cache has no intersection with new request, return return; } - cachedKeys.push_back(maxInteractKey); + cachedBoxes.push_back(maxSourceBox); - // If the current box is the max interact box, return, avoid cutting - if (this->size() == maxInteractBox.size()) + // If the interaction of current box is equal to the new request, return, avoid cutting + if (this->size() == maxInteract.size()) { return; } if (current_depth == max_depth) { - maxInteractBox.NdCut(*this, regularBoxes); + maxInteract.NdCut(*this, regularBoxes); } else { std::vector nextBoxes; - maxInteractBox.NdCut(*this, nextBoxes); + maxInteract.NdCut(*this, nextBoxes); for (auto &box : nextBoxes) { box.GetMaxInteractBox(samePrefixKeys, max_depth, current_depth, regularBoxes, - cachedKeys); + cachedBoxes); } } } From 738e47eb67a96f663c99dcaa367df4e2b3b6b0f4 Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Wed, 3 Jul 2024 20:06:05 -0400 Subject: [PATCH 20/22] revert changes in source/adios2/helper/adiosType.h --- source/adios2/helper/adiosType.h | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/source/adios2/helper/adiosType.h b/source/adios2/helper/adiosType.h index 319c1a1d0f..24f360d524 100644 --- a/source/adios2/helper/adiosType.h +++ b/source/adios2/helper/adiosType.h @@ -177,16 +177,6 @@ class DimsArray : public CoreDims { std::copy(d1.begin(), d1.end(), &Dimensions[0]); } - - DimsArray(const DimsArray &d1) : CoreDims(d1.size(), &Dimensions[0]) - { - std::copy(d1.begin(), d1.end(), &Dimensions[0]); - } - - DimsArray(const size_t count, const DimsArray &d1) : CoreDims(count, &Dimensions[0]) - { - std::copy(d1.begin(), d1.end(), &Dimensions[0]); - } }; /** From ac40ef69c60fedbcdde063e6aa28c2bbf13eaacb Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Fri, 5 Jul 2024 10:37:09 -0400 Subject: [PATCH 21/22] update format --- source/adios2/engine/bp5/BP5Reader.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index b50c2d18ce..02e4e09998 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -477,8 +477,8 @@ void BP5Reader::PerformRemoteGetsWithKVCache() else { void *data = malloc(ReqInfo.ReqBox.size() * ReqInfo.TypeSize); - m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, ReqInfo.ReqBox.size() * ReqInfo.TypeSize, - data); + m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1, + ReqInfo.ReqBox.size() * ReqInfo.TypeSize, data); helper::NdCopy(reinterpret_cast(data), ReqInfo.ReqBox.Start, ReqInfo.ReqBox.Count, true, false, reinterpret_cast(Req.Data), Req.Start, Req.Count, true, false, ReqInfo.TypeSize); From 1b67484d7e28ea1c2679424940d8f1666db0a5a6 Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Fri, 5 Jul 2024 11:34:13 -0400 Subject: [PATCH 22/22] add static_cast for typeSize from size_t to int --- source/adios2/engine/bp5/BP5Reader.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 02e4e09998..5ee9dff327 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -481,7 +481,7 @@ void BP5Reader::PerformRemoteGetsWithKVCache() ReqInfo.ReqBox.size() * ReqInfo.TypeSize, data); helper::NdCopy(reinterpret_cast(data), ReqInfo.ReqBox.Start, ReqInfo.ReqBox.Count, true, false, reinterpret_cast(Req.Data), - Req.Start, Req.Count, true, false, ReqInfo.TypeSize); + Req.Start, Req.Count, true, false, static_cast(ReqInfo.TypeSize)); free(data); } } @@ -494,7 +494,7 @@ void BP5Reader::PerformRemoteGetsWithKVCache() auto &Req = GetRequests[ReqInfo.ReqSeq]; helper::NdCopy(reinterpret_cast(ReqInfo.Data), ReqInfo.ReqBox.Start, ReqInfo.ReqBox.Count, true, false, reinterpret_cast(Req.Data), - Req.Start, Req.Count, true, false, ReqInfo.TypeSize); + Req.Start, Req.Count, true, false, static_cast(ReqInfo.TypeSize)); m_KVCache.AppendCommandInBatch(ReqInfo.CacheKey.c_str(), 0, ReqInfo.ReqSize * ReqInfo.TypeSize, ReqInfo.Data);