diff --git a/CMakeLists.txt b/CMakeLists.txt index ffafb926bc9..2aae5268c18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -304,8 +304,8 @@ macro(nebula_link_libraries target) ${OPENSSL_CRYPTO_LIBRARY} ${KRB5_LIBRARIES} ${COMPRESSION_LIBRARIES} - ${JEMALLOC_LIB} ${LIBUNWIND_LIBRARIES} + ${JEMALLOC_LIB} dl ${GETTIME_LIB} -pthread diff --git a/cmake/FindNCURSES.cmake b/cmake/FindNCURSES.cmake index 0ed4a4eb414..eb4a0b6f1e9 100644 --- a/cmake/FindNCURSES.cmake +++ b/cmake/FindNCURSES.cmake @@ -24,7 +24,7 @@ find_path( find_path( NCURSES_INCLUDE_DIR - NAMES ncurses/curses.h + NAMES ncurses.h curses.h ncurses/curses.h HINTS ${NCURSES_ROOT_DIR}/include ) diff --git a/src/common/base/EitherOr.h b/src/common/base/EitherOr.h index c8e06fdb6c2..cb916b3b52b 100644 --- a/src/common/base/EitherOr.h +++ b/src/common/base/EitherOr.h @@ -12,8 +12,6 @@ namespace nebula { -namespace { // NOLINT - using LeftType = std::true_type; using RightType = std::false_type; @@ -23,8 +21,6 @@ enum class State : int16_t { RIGHT_TYPE = 2, }; -} // Anonymous namespace - static constexpr LeftType* kConstructLeft = nullptr; static constexpr RightType* kConstructRight = nullptr; diff --git a/src/common/network/NetworkUtils.cpp b/src/common/network/NetworkUtils.cpp index 3147e88568e..d513b4ebfed 100644 --- a/src/common/network/NetworkUtils.cpp +++ b/src/common/network/NetworkUtils.cpp @@ -83,6 +83,7 @@ StatusOr> NetworkUtils::listDeviceA return dev2ipv4s; } + bool NetworkUtils::getDynamicPortRange(uint16_t& low, uint16_t& high) { FILE* pipe = popen("cat /proc/sys/net/ipv4/ip_local_port_range", "r"); if (!pipe) { diff --git a/src/common/test/ServerContext.h b/src/common/test/ServerContext.h index 515d512ae8c..19ebf58f68c 100644 --- a/src/common/test/ServerContext.h +++ b/src/common/test/ServerContext.h @@ -23,39 +23,39 @@ struct ServerContext { if (thread_ != nullptr) { thread_->join(); } - KVStore_ = nullptr; + kvStore_ = nullptr; server_ = nullptr; thread_ = nullptr; VLOG(3) << "~ServerContext"; } + + void mockCommon(const std::string& name, + uint16_t port, + std::shared_ptr handler) { + server_ = std::make_unique(); + server_->setInterface(std::move(handler)); + server_->setPort(port); + thread_ = std::make_unique(name, [this, name] { + server_->serve(); + LOG(INFO) << "The " << name << " server has stopped"; + }); + + while (!server_->getServeEventBase() || + !server_->getServeEventBase()->isRunning()) { + usleep(10000); + } + port_ = server_->getAddress().getPort(); + } + + std::unique_ptr server_{nullptr}; std::unique_ptr thread_{nullptr}; // To keep meta and storage's KVStore - std::unique_ptr KVStore_{nullptr}; + std::unique_ptr kvStore_{nullptr}; uint16_t port_{0}; }; -static void mockCommon(test::ServerContext *sc, - const std::string &name, - uint16_t port, - std::shared_ptr handler) { - if (nullptr == sc) { - LOG(ERROR) << "ServerContext is nullptr"; - return; - } - sc->server_ = std::make_unique(); - sc->server_->setInterface(std::move(handler)); - sc->server_->setPort(port); - sc->thread_ = std::make_unique(name, [&]() { - sc->server_->serve(); - LOG(INFO) << "Stop the server..."; - }); - while (!sc->server_->getServeEventBase() || - !sc->server_->getServeEventBase()->isRunning()) { - } - sc->port_ = sc->server_->getAddress().getPort(); -} } // namespace test } // namespace nebula diff --git a/src/common/test/TestServerContext.cpp b/src/common/test/TestServerContext.cpp index 8c716193d8a..6292601a4d6 100644 --- a/src/common/test/TestServerContext.cpp +++ b/src/common/test/TestServerContext.cpp @@ -24,7 +24,7 @@ class TestServer final : public apache::thrift::ServerInterface { TEST(ServerContext, mockCommon) { auto sc = std::make_unique(); auto handler = std::make_shared(); - test::mockCommon(sc.get(), "test", 0, handler); + sc->mockCommon("test", 0, handler); } } // namespace test diff --git a/src/daemons/CMakeLists.txt b/src/daemons/CMakeLists.txt index b601b86ab53..299c20efcf1 100644 --- a/src/daemons/CMakeLists.txt +++ b/src/daemons/CMakeLists.txt @@ -45,10 +45,13 @@ add_executable( $ $ $ - $ + $ + $ + $ $ $ $ + $ $ $ $ @@ -77,9 +80,11 @@ add_executable( $ $ $ - $ - $ + $ + $ + $ $ + $ $ $ $ diff --git a/src/daemons/MetaDaemon.cpp b/src/daemons/MetaDaemon.cpp index 32bc0de1911..2d6e86c570e 100644 --- a/src/daemons/MetaDaemon.cpp +++ b/src/daemons/MetaDaemon.cpp @@ -11,7 +11,9 @@ #include "webservice/WebService.h" #include "network/NetworkUtils.h" #include "process/ProcessUtils.h" +#include "thread/GenericThreadPool.h" #include "kvstore/PartManager.h" +#include "kvstore/NebulaStore.h" using nebula::ProcessUtils; using nebula::Status; @@ -23,6 +25,8 @@ DEFINE_string(peers, "", "It is a list of IPs split by comma," "the ips number equals replica number." "If empty, it means replica is 1"); DEFINE_string(local_ip, "", "Local ip speicified for NetworkUtils::getLocalIP"); +DEFINE_int32(num_workers, 4, "Number of worker threads"); +DEFINE_int32(num_io_threads, 16, "Number of IO threads"); DECLARE_string(part_man_type); DEFINE_string(pid_file, "pids/nebula-metad.pid", "File to hold the process id"); @@ -89,7 +93,7 @@ int main(int argc, char *argv[]) { LOG(ERROR) << "Bad local host addr, status:" << hostAddrRet.status(); return EXIT_FAILURE; } - auto& localHost = hostAddrRet.value(); + auto& localhost = hostAddrRet.value(); auto peersRet = nebula::network::NetworkUtils::toHosts(FLAGS_peers); if (!peersRet.ok()) { @@ -108,22 +112,32 @@ int main(int argc, char *argv[]) { // The meta server has only one space, one part. partMan->addPart(0, 0, std::move(peersRet.value())); + // Generic thread pool + auto workers = std::make_shared(); + workers->start(FLAGS_num_workers); + + // folly IOThreadPoolExecutor + auto ioPool = std::make_shared(FLAGS_num_io_threads); + nebula::kvstore::KVOptions options; - options.local_ = localHost; options.dataPaths_ = {FLAGS_data_path}; options.partMan_ = std::move(partMan); - std::unique_ptr kvstore( - nebula::kvstore::KVStore::instance(std::move(options))); + std::unique_ptr kvstore = + std::make_unique(std::move(options), + ioPool, + workers, + localhost); auto handler = std::make_shared(kvstore.get()); - nebula::operator<<(operator<<(LOG(INFO), "The meta deamon start on "), localHost); + nebula::operator<<(operator<<(LOG(INFO), "The meta deamon start on "), localhost); try { gServer = std::make_unique(); gServer->setInterface(std::move(handler)); gServer->setPort(FLAGS_port); gServer->setReusePort(FLAGS_reuse_port); gServer->setIdleTimeout(std::chrono::seconds(0)); // No idle timeout on client connection + gServer->setIOThreadPool(ioPool); gServer->serve(); // Will wait until the server shuts down } catch (const std::exception &e) { LOG(ERROR) << "Exception thrown: " << e.what(); diff --git a/src/daemons/StorageDaemon.cpp b/src/daemons/StorageDaemon.cpp index b355d866b37..9780d115808 100644 --- a/src/daemons/StorageDaemon.cpp +++ b/src/daemons/StorageDaemon.cpp @@ -7,9 +7,10 @@ #include "base/Base.h" #include #include "network/NetworkUtils.h" +#include "thread/GenericThreadPool.h" #include "storage/StorageServiceHandler.h" #include "storage/StorageHttpHandler.h" -#include "kvstore/KVStore.h" +#include "kvstore/NebulaStore.h" #include "kvstore/PartManager.h" #include "process/ProcessUtils.h" #include "storage/test/TestUtils.h" @@ -21,13 +22,16 @@ DEFINE_int32(port, 44500, "Storage daemon listening port"); DEFINE_bool(reuse_port, true, "Whether to turn on the SO_REUSEPORT option"); DEFINE_string(data_path, "", "Root data path, multi paths should be split by comma." "For rocksdb engine, one path one instance."); -DEFINE_string(local_ip, "", "Local ip speicified for NetworkUtils::getLocalIP"); DEFINE_bool(mock_server, true, "start mock server"); DEFINE_bool(daemonize, true, "Whether to run the process as a daemon"); DEFINE_string(pid_file, "pids/nebula-storaged.pid", "File to hold the process id"); DEFINE_string(meta_server_addrs, "", "list of meta server addresses," "the format looks like ip1:port1, ip2:port2, ip3:port3"); -DEFINE_int32(io_handlers, 10, "io handlers"); +DEFINE_string(store_type, "nebula", + "Which type of KVStore to be used by the storage daemon." + " Options can be \"nebula\", \"hbase\", etc."); +DEFINE_int32(num_workers, 4, "Number of worker threads"); +DEFINE_int32(num_io_threads, 16, "Number of IO threads"); using nebula::Status; using nebula::HostAddr; @@ -44,6 +48,33 @@ static void signalHandler(int sig); static Status setupSignalHandler(); +std::unique_ptr getStoreInstance( + HostAddr localhost, + std::vector paths, + std::shared_ptr ioPool, + std::shared_ptr workers, + nebula::meta::MetaClient* metaClient) { + nebula::kvstore::KVOptions options; + options.dataPaths_ = std::move(paths); + options.partMan_ = std::make_unique( + localhost, + metaClient); + + if (FLAGS_store_type == "nebula") { + return std::make_unique(std::move(options), + ioPool, + workers, + localhost); + } else if (FLAGS_store_type == "hbase") { + LOG(FATAL) << "HBase store has not been implemented"; + } else { + LOG(FATAL) << "Unknown store type \"" << FLAGS_store_type << "\""; + } + + return nullptr; +} + + int main(int argc, char *argv[]) { google::SetVersionString(nebula::versionString()); folly::init(&argc, &argv, true); @@ -81,10 +112,9 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } - auto result = nebula::network::NetworkUtils::getLocalIP(FLAGS_local_ip); + auto result = nebula::network::NetworkUtils::getLocalIP(); if (!result.ok()) { - LOG(ERROR) << "Get localIp failed, ip " << FLAGS_local_ip - << ", status:" << result.status(); + LOG(ERROR) << "Get localIp failed, status:" << result.status(); return EXIT_FAILURE; } auto hostRet = nebula::network::NetworkUtils::toHostAddr(result.value(), FLAGS_port); @@ -92,7 +122,7 @@ int main(int argc, char *argv[]) { LOG(ERROR) << "Bad local host addr, status:" << hostRet.status(); return EXIT_FAILURE; } - auto& localHost = hostRet.value(); + auto& localhost = hostRet.value(); auto metaAddrsRet = nebula::network::NetworkUtils::toHosts(FLAGS_meta_server_addrs); if (!metaAddrsRet.ok() || metaAddrsRet.value().empty()) { LOG(ERROR) << "Can't get metaServer address, status:" << metaAddrsRet.status() @@ -110,19 +140,25 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } - auto ioThreadPool = std::make_shared(FLAGS_io_handlers); + // Generic thread pool + auto workers = std::make_shared(); + workers->start(FLAGS_num_workers); + + // folly IOThreadPoolExecutor + auto ioThreadPool = std::make_shared(FLAGS_num_io_threads); + + // Meta client auto metaClient = std::make_unique(ioThreadPool, std::move(metaAddrsRet.value()), true); metaClient->init(); - nebula::kvstore::KVOptions options; - options.local_ = localHost; - options.dataPaths_ = std::move(paths); - options.partMan_ = std::make_unique( - options.local_, metaClient.get()); - std::unique_ptr kvstore( - nebula::kvstore::KVStore::instance(std::move(options))); + std::unique_ptr kvstore = getStoreInstance(localhost, + std::move(paths), + ioThreadPool, + workers, + metaClient.get()); + auto schemaMan = nebula::meta::SchemaManager::create(); schemaMan->init(metaClient.get()); @@ -136,6 +172,7 @@ int main(int argc, char *argv[]) { LOG(ERROR) << "Failed to start web service: " << status; return EXIT_FAILURE; } + // Setup the signal handlers status = setupSignalHandler(); if (!status.ok()) { @@ -143,9 +180,10 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } + // Thrift server auto handler = std::make_shared(kvstore.get(), std::move(schemaMan)); try { - nebula::operator<<(operator<<(LOG(INFO), "The storage deamon start on "), localHost); + nebula::operator<<(operator<<(LOG(INFO), "The storage deamon start on "), localhost); gServer = std::make_unique(); gServer->setInterface(std::move(handler)); gServer->setPort(FLAGS_port); diff --git a/src/graph/test/CMakeLists.txt b/src/graph/test/CMakeLists.txt index 046d833d1ca..d4078191641 100644 --- a/src/graph/test/CMakeLists.txt +++ b/src/graph/test/CMakeLists.txt @@ -1,29 +1,35 @@ -add_executable( - session_manager_test - SessionManagerTest.cpp - $ - $ +set(GRAPH_TEST_LIBS $ $ $ + $ $ $ - $ - $ - $ - $ - $ - $ $ $ $ $ $ $ + $ $ $ + $ + $ + $ + $ + $ + $ + $ + $ ) + +add_executable( + session_manager_test + SessionManagerTest.cpp + ${GRAPH_TEST_LIBS} +) nebula_link_libraries( session_manager_test ${THRIFT_LIBRARIES} @@ -32,7 +38,6 @@ nebula_link_libraries( gtest gtest_main ) - nebula_add_test(session_manager_test) @@ -43,32 +48,10 @@ add_executable( TestBase.cpp YieldTest.cpp SchemaTest.cpp - $ - $ - $ $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ $ + ${GRAPH_TEST_LIBS} ) - nebula_link_libraries( query_engine_test ${THRIFT_LIBRARIES} @@ -85,28 +68,9 @@ add_executable( TestEnv.cpp TestBase.cpp GoTest.cpp - $ $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ $ + ${GRAPH_TEST_LIBS} ) nebula_link_libraries( go_test @@ -124,33 +88,13 @@ add_executable( TestEnv.cpp TestBase.cpp GraphHttpHandlerTest.cpp - $ $ $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ $ $ $ - $ $ + ${GRAPH_TEST_LIBS} ) nebula_link_libraries( graph_http_test diff --git a/src/graph/test/TestUtils.h b/src/graph/test/TestUtils.h index e6da5b3f231..5c8b339d314 100644 --- a/src/graph/test/TestUtils.h +++ b/src/graph/test/TestUtils.h @@ -19,7 +19,7 @@ class TestUtils { auto sc = std::make_unique(); auto threadPool = std::make_shared(1); auto interface = std::make_shared(threadPool); - test::mockCommon(sc.get(), "graph", port, interface); + sc->mockCommon("graph", port, interface); LOG(INFO) << "Starting the graph Daemon on port " << sc->port_; return sc; } diff --git a/src/kvstore/CMakeLists.txt b/src/kvstore/CMakeLists.txt index ce2cfcfa602..ed29f0c7166 100644 --- a/src/kvstore/CMakeLists.txt +++ b/src/kvstore/CMakeLists.txt @@ -5,6 +5,7 @@ add_library( PartManager.cpp NebulaStore.cpp RocksEngineConfig.cpp + LogEncoder.cpp ) add_dependencies(kvstore_obj base_obj meta_client) diff --git a/src/kvstore/Common.h b/src/kvstore/Common.h index df363a4301a..04df253f386 100644 --- a/src/kvstore/Common.h +++ b/src/kvstore/Common.h @@ -8,31 +8,30 @@ #define KVSTORE_COMMON_H_ #include "base/Base.h" +#include "rocksdb/slice.h" namespace nebula { namespace kvstore { enum ResultCode { - SUCCEEDED = 0, - ERR_UNKNOWN = -1, - ERR_PART_NOT_FOUND = -2, - ERR_KEY_NOT_FOUND = -3, - ERR_SPACE_NOT_FOUND = -4, - ERR_LEADER_CHANAGED = -5, - ERR_INVALID_ARGUMENT = -6, - ERR_IO_ERROR = -7, + SUCCEEDED = 0, + ERR_SPACE_NOT_FOUND = -1, + ERR_PART_NOT_FOUND = -2, + ERR_KEY_NOT_FOUND = -3, + ERR_CONSENSUS_ERROR = -4, + ERR_LEADER_CHANGED = -5, + ERR_INVALID_ARGUMENT = -6, + ERR_IO_ERROR = -7, + ERR_UNKNOWN = -100, }; -#define KV_DATA_PATH_FORMAT(path, spaceId) \ - folly::stringPrintf("%s/nebula/%d/data", path, spaceId) - -#define KV_WAL_PATH_FORMAT(path, spaceId, partId) \ - folly::stringPrintf("%s/nebula/%d/wals/%d", \ - path, spaceId, partId) -using KVCallback = std::function; - +using KVCallback = std::function; using KV = std::pair; +inline rocksdb::Slice toSlice(const folly::StringPiece& str) { + return rocksdb::Slice(str.begin(), str.size()); +} + } // namespace kvstore } // namespace nebula #endif // KVSTORE_COMMON_H_ diff --git a/src/kvstore/KVEngine.h b/src/kvstore/KVEngine.h index c64e4941e8a..ec57f670a6c 100644 --- a/src/kvstore/KVEngine.h +++ b/src/kvstore/KVEngine.h @@ -14,6 +14,22 @@ namespace nebula { namespace kvstore { +class WriteBatch { +public: + virtual ~WriteBatch() = default; + + virtual ResultCode put(folly::StringPiece key, folly::StringPiece value) = 0; + + virtual ResultCode remove(folly::StringPiece key) = 0; + + virtual ResultCode removePrefix(folly::StringPiece prefix) = 0; + + // Remove all keys in the range [start, end) + virtual ResultCode removeRange(folly::StringPiece start, + folly::StringPiece end) = 0; +}; + + class KVEngine { public: explicit KVEngine(GraphSpaceID spaceId) @@ -21,73 +37,69 @@ class KVEngine { virtual ~KVEngine() = default; - virtual ResultCode get(const std::string& key, - std::string* value) = 0; + // Retrieve the root path for the data + // If the store is persistent, a valid path will be returned + // Otherwise, nullptr will be returned + virtual const char* getDataRoot() const = 0; - virtual ResultCode multiGet(const std::vector& keys, - std::vector* values) = 0; + virtual std::unique_ptr startBatchWrite() = 0; + virtual ResultCode commitBatchWrite(std::unique_ptr batch) = 0; - virtual ResultCode put(std::string key, - std::string value) = 0; + // Read a single key + virtual ResultCode get(const std::string& key, std::string* value) = 0; - virtual ResultCode multiPut(std::vector keyValues) = 0; + // Read a list of keys + virtual ResultCode multiGet(const std::vector& keys, + std::vector* values) = 0; - /** - * Get all results in range [start, end) - * */ + // Get all results in range [start, end) virtual ResultCode range(const std::string& start, const std::string& end, std::unique_ptr* iter) = 0; - /** - * Get all results with 'prefix' str as prefix. - * */ + // Get all results with 'prefix' str as prefix. virtual ResultCode prefix(const std::string& prefix, std::unique_ptr* iter) = 0; + // Get all results in range [start, end) + virtual ResultCode put(std::string key, std::string value) = 0; + + // Get all results with 'prefix' str as prefix. + virtual ResultCode multiPut(std::vector keyValues) = 0; + + // Remove a single key virtual ResultCode remove(const std::string& key) = 0; + // Remove a batch of keys virtual ResultCode multiRemove(std::vector keys) = 0; - /** - * Remove range [start, end) - * */ + // Remove range [start, end) virtual ResultCode removeRange(const std::string& start, const std::string& end) = 0; - /** - * Add partId into current storage engine. - * */ + + // Remove rows with the given prefix + virtual ResultCode removePrefix(const std::string& prefix) = 0; + + // Add partId into current storage engine. virtual void addPart(PartitionID partId) = 0; - /** - * Remove partId from current storage engine. - * */ + // Remove partId from current storage engine. virtual void removePart(PartitionID partId) = 0; - /** - * Return all partIds current storage engine holded. - * */ + // Return all partIds current storage engine holded. virtual std::vector allParts() = 0; - /** - * Return total parts num - * */ + // Return total parts num virtual int32_t totalPartsNum() = 0; - /** - * Ingest sst files - */ + // Ingest sst files virtual ResultCode ingest(const std::vector& files) = 0; - /** - * Set Config Option - */ + // Set Config Option virtual ResultCode setOption(const std::string& configKey, const std::string& configValue) = 0; - /** - * Set DB Config Option - */ + // Set DB Config Option virtual ResultCode setDBOption(const std::string& configKey, const std::string& configValue) = 0; diff --git a/src/kvstore/KVStore.h b/src/kvstore/KVStore.h index 465da1dbd27..9223bdb7a8d 100644 --- a/src/kvstore/KVStore.h +++ b/src/kvstore/KVStore.h @@ -18,27 +18,18 @@ namespace nebula { namespace kvstore { struct KVOptions { - /** - * Local address, it would be used for search related meta information on meta server. - * */ - HostAddr local_; - /** - * Paths for data. It would be used by rocksdb engine. - * Be careful! We should ensure each "paths" has only one instance, otherwise - * it would mix up the data on disk. - * */ + // Paths for data. It would be used by rocksdb engine. + // Be careful! We should ensure each "paths" has only one instance, + // otherwise it would mix up the data on disk. std::vector dataPaths_; - /** - * PartManager instance for kvstore. - * */ + + // PartManager instance for kvstore. std::unique_ptr partMan_{nullptr}; - /** - * Custom MergeOperator used in rocksdb.merge method. - * */ + + // Custom MergeOperator used in rocksdb.merge method. std::shared_ptr mergeOp_{nullptr}; - /** - * Custom CompactionFilter used in compaction. - * */ + + // Custom CompactionFilter used in compaction. std::shared_ptr cfFactory_{nullptr}; }; @@ -47,67 +38,68 @@ struct StoreCapability { static const uint32_t SC_FILTERING = 1; static const uint32_t SC_ASYNC = 2; }; +#define SUPPORT_FILTERING(store) (store.capability() & StoreCapability::SC_FILTERING) /** * Interface for all kv-stores - */ + **/ class KVStore { public: - /** - * Create one new instance each time. - * */ - static KVStore* instance(KVOptions options); - virtual ~KVStore() = default; // Return bit-OR of StoreCapability values; virtual uint32_t capability() const = 0; + // Retrieve the current leader for the given partition. This + // is usually called when ERR_LEADER_CHANGED result code is + // returnde + virtual HostAddr partLeader(GraphSpaceID spaceId, PartitionID partID) = 0; + + virtual PartManager* partManager() const { + return nullptr; + } + + // Read a single key virtual ResultCode get(GraphSpaceID spaceId, PartitionID partId, const std::string& key, std::string* value) = 0; + // Read multiple keys virtual ResultCode multiGet(GraphSpaceID spaceId, PartitionID partId, const std::vector& keys, std::vector* values) = 0; - /** - * Get all results in range [start, end) - * */ + // Get all results in range [start, end) virtual ResultCode range(GraphSpaceID spaceId, PartitionID partId, const std::string& start, const std::string& end, std::unique_ptr* iter) = 0; - /** - * Since the `range' interface will hold references to its 3rd & 4th parameter, in `iter', - * thus the arguments must outlive `iter'. - * Here we forbid one to invoke `range' with rvalues, which is the common mistake. - */ + + // Since the `range' interface will hold references to its 3rd & 4th parameter, in `iter', + // thus the arguments must outlive `iter'. + // Here we forbid one to invoke `range' with rvalues, which is the common mistake. virtual ResultCode range(GraphSpaceID spaceId, PartitionID partId, std::string&& start, std::string&& end, std::unique_ptr* iter) = delete; - /** - * Get all results with prefix. - * */ + // Get all results with prefix. virtual ResultCode prefix(GraphSpaceID spaceId, PartitionID partId, const std::string& prefix, std::unique_ptr* iter) = 0; - /** - * To forbid to pass rvalue via the `prefix' parameter. - */ + + // To forbid to pass rvalue via the `prefix' parameter. virtual ResultCode prefix(GraphSpaceID spaceId, PartitionID partId, std::string&& prefix, std::unique_ptr* iter) = delete; - + // Asynchrous version of remove methods virtual void asyncMultiPut(GraphSpaceID spaceId, PartitionID partId, std::vector keyValues, @@ -129,6 +121,11 @@ class KVStore { const std::string& end, KVCallback cb) = 0; + virtual void asyncRemovePrefix(GraphSpaceID spaceId, + PartitionID partId, + const std::string& prefix, + KVCallback cb) = 0; + protected: KVStore() = default; }; diff --git a/src/kvstore/LogEncoder.cpp b/src/kvstore/LogEncoder.cpp new file mode 100644 index 00000000000..f5636f7a5ea --- /dev/null +++ b/src/kvstore/LogEncoder.cpp @@ -0,0 +1,149 @@ +/* Copyright (c) 2019 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "base/Base.h" +#include "time/TimeUtils.h" +#include "kvstore/LogEncoder.h" + +namespace nebula { +namespace kvstore { + +constexpr auto kHeadLen = sizeof(int64_t) + 1 + sizeof(uint32_t); + + +std::string encodeSingleValue(LogType type, folly::StringPiece val) { + std::string encoded; + encoded.reserve(val.size() + kHeadLen); + // Timstamp (8 bytes) + int64_t ts = time::TimeUtils::nowInMSeconds(); + encoded.append(reinterpret_cast(&ts), sizeof(int64_t)); + // Log type + encoded.append(reinterpret_cast(&type), 1); + // Value length + uint32_t len = static_cast(val.size()); + encoded.append(reinterpret_cast(&len), sizeof(len)); + // Value + encoded.append(val.begin(), val.size()); + + return encoded; +} + + +folly::StringPiece decodeSingleValue(folly::StringPiece encoded) { + // Skip the timestamp and the first type byte + auto* p = encoded.begin() + sizeof(int64_t) + 1; + uint32_t len = *(reinterpret_cast(p)); + DCHECK_EQ(len + kHeadLen, encoded.size()); + return folly::StringPiece(p + sizeof(uint32_t), len); +} + + +std::string encodeMultiValues(LogType type, const std::vector& values) { + size_t totalLen = 0; + for (auto& v : values) { + totalLen += (sizeof(uint32_t) + v.size()); + } + + std::string encoded; + encoded.reserve(totalLen + kHeadLen); + + // Timstamp (8 bytes) + int64_t ts = time::TimeUtils::nowInMSeconds(); + encoded.append(reinterpret_cast(&ts), sizeof(int64_t)); + // Log type + encoded.append(reinterpret_cast(&type), 1); + // Number of values + uint32_t num = values.size(); + encoded.append(reinterpret_cast(&num), sizeof(uint32_t)); + // Values + for (auto& v : values) { + uint32_t len = v.size(); + encoded.append(reinterpret_cast(&len), sizeof(uint32_t)); + encoded.append(v.data(), len); + } + + return encoded; +} + + +std::string encodeMultiValues(LogType type, const std::vector& kvs) { + size_t totalLen = 0; + for (auto& kv : kvs) { + totalLen += (2 * sizeof(uint32_t) + kv.first.size() + kv.second.size()); + } + + std::string encoded; + encoded.reserve(totalLen + kHeadLen); + + // Timstamp (8 bytes) + int64_t ts = time::TimeUtils::nowInMSeconds(); + encoded.append(reinterpret_cast(&ts), sizeof(int64_t)); + // Log type + encoded.append(reinterpret_cast(&type), 1); + // Number of total strings: #keys + #values + uint32_t num = 2 * kvs.size(); + encoded.append(reinterpret_cast(&num), sizeof(uint32_t)); + // Key/value pairs + for (auto& kv : kvs) { + uint32_t len = kv.first.size(); + encoded.append(reinterpret_cast(&len), sizeof(uint32_t)); + encoded.append(kv.first.data(), len); + len = kv.second.size(); + encoded.append(reinterpret_cast(&len), sizeof(uint32_t)); + encoded.append(kv.second.data(), len); + } + + return encoded; +} + + +std::string encodeMultiValues(LogType type, + folly::StringPiece v1, + folly::StringPiece v2) { + std::string encoded; + encoded.reserve(kHeadLen + 2 * sizeof(uint32_t) + v1.size() + v2.size()); + + // Timstamp (8 bytes) + int64_t ts = time::TimeUtils::nowInMSeconds(); + encoded.append(reinterpret_cast(&ts), sizeof(int64_t)); + // Log type + encoded.append(reinterpret_cast(&type), 1); + // Number of values + uint32_t num = 2; + encoded.append(reinterpret_cast(&num), sizeof(uint32_t)); + // Values + uint32_t len = v1.size(); + encoded.append(reinterpret_cast(&len), sizeof(uint32_t)); + encoded.append(v1.begin(), len); + len = v2.size(); + encoded.append(reinterpret_cast(&len), sizeof(uint32_t)); + encoded.append(v2.begin(), len); + + return encoded; +} + + +std::vector decodeMultiValues(folly::StringPiece encoded) { + // Skip the timestamp and the first type byte + auto* p = encoded.begin() + sizeof(int64_t) + 1; + uint32_t numValues = *(reinterpret_cast(p)); + + std::vector values; + p += sizeof(uint32_t); + for (auto i = 0U; i < numValues; i++) { + uint32_t len = *(reinterpret_cast(p)); + DCHECK_LE(p + sizeof(uint32_t) + len, encoded.begin() + encoded.size()); + values.emplace_back(p + sizeof(uint32_t), len); + p += (sizeof(uint32_t) + len); + } + DCHECK_EQ(p, encoded.begin() + encoded.size()); + + return values; +} + +} // namespace kvstore +} // namespace nebula + diff --git a/src/kvstore/LogEncoder.h b/src/kvstore/LogEncoder.h new file mode 100644 index 00000000000..d96a621042b --- /dev/null +++ b/src/kvstore/LogEncoder.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2019 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#ifndef KVSTORE_LOGENCODER_H_ +#define KVSTORE_LOGENCODER_H_ + +#include "kvstore/Common.h" + +namespace nebula { +namespace kvstore { + +enum LogType : char { + OP_PUT = 0x1, + OP_MULTI_PUT = 0x2, + OP_REMOVE = 0x3, + OP_MULTI_REMOVE = 0x4, + OP_REMOVE_PREFIX = 0x5, + OP_REMOVE_RANGE = 0x6, +}; + + +std::string encodeSingleValue(LogType type, folly::StringPiece val); +folly::StringPiece decodeSingleValue(folly::StringPiece encoded); + +std::string encodeMultiValues(LogType type, const std::vector& values); +std::string encodeMultiValues(LogType type, const std::vector& kvs); +std::string encodeMultiValues(LogType type, + folly::StringPiece v1, + folly::StringPiece v2); +std::vector decodeMultiValues(folly::StringPiece encoded); + +} // namespace kvstore +} // namespace nebula +#endif // KVSTORE_LOGENCODER_H_ diff --git a/src/kvstore/NebulaStore.cpp b/src/kvstore/NebulaStore.cpp index 1cef4b24a74..cdfa19ff53d 100644 --- a/src/kvstore/NebulaStore.cpp +++ b/src/kvstore/NebulaStore.cpp @@ -14,22 +14,21 @@ #include "kvstore/RocksEngine.h" DEFINE_string(engine_type, "rocksdb", "rocksdb, memory..."); -DEFINE_string(part_type, "simple", "simple, consensus..."); /** * Check spaceId, partId exists or not. * */ #define CHECK_FOR_WRITE(spaceId, partId, cb) \ folly::RWSpinLock::ReadHolder rh(&lock_); \ - auto it = kvs_.find(spaceId); \ - if (UNLIKELY(it == kvs_.end())) { \ - cb(ResultCode::ERR_SPACE_NOT_FOUND, HostAddr(0, 0)); \ + auto it = spaces_.find(spaceId); \ + if (UNLIKELY(it == spaces_.end())) { \ + cb(ResultCode::ERR_SPACE_NOT_FOUND); \ return; \ } \ auto& parts = it->second->parts_; \ auto partIt = parts.find(partId); \ if (UNLIKELY(partIt == parts.end())) { \ - cb(ResultCode::ERR_PART_NOT_FOUND, HostAddr(0, 0)); \ + cb(ResultCode::ERR_PART_NOT_FOUND); \ return; \ } @@ -40,8 +39,8 @@ DEFINE_string(part_type, "simple", "simple, consensus..."); folly::RWSpinLock::ReadHolder rh(&lock_); \ KVEngine* engine = nullptr; \ do { \ - auto it = kvs_.find(spaceId); \ - if (UNLIKELY(it == kvs_.end())) { \ + auto it = spaces_.find(spaceId); \ + if (UNLIKELY(it == spaces_.end())) { \ return ResultCode::ERR_SPACE_NOT_FOUND; \ } \ auto& parts = it->second->parts_; \ @@ -57,9 +56,9 @@ DEFINE_string(part_type, "simple", "simple, consensus..."); * Check spaceId is exist and return related partitions. */ #define RETURN_IF_SPACE_NOT_FOUND(spaceId, it) \ - it = kvs_.find(spaceId); \ + it = spaces_.find(spaceId); \ do { \ - if (UNLIKELY(it == kvs_.end())) { \ + if (UNLIKELY(it == spaces_.end())) { \ return ResultCode::ERR_SPACE_NOT_FOUND; \ } \ } while (false) @@ -72,93 +71,59 @@ DEFINE_string(part_type, "simple", "simple, consensus..."); return code; \ } + namespace nebula { namespace kvstore { -// static -KVStore* KVStore::instance(KVOptions options) { - auto* instance = new NebulaStore(std::move(options)); - static_cast(instance)->init(); - return instance; -} - - -Engine NebulaStore::newEngine(GraphSpaceID spaceId, std::string rootPath) { - if (FLAGS_engine_type == "rocksdb") { - auto dataPath = KV_DATA_PATH_FORMAT(rootPath.c_str(), spaceId); - auto engine = std::make_pair( - std::unique_ptr( - new RocksEngine(spaceId, - std::move(dataPath), - options_.mergeOp_, - options_.cfFactory_)), - std::move(rootPath)); - return engine; - } else { - LOG(FATAL) << "Unknown Engine type " << FLAGS_engine_type; - } -} - - -std::unique_ptr NebulaStore::newPart(GraphSpaceID spaceId, - PartitionID partId, - const Engine& engine) { - if (FLAGS_part_type == "simple") { - return std::unique_ptr(new SimplePart( - spaceId, - partId, - KV_WAL_PATH_FORMAT(engine.second.c_str(), spaceId, partId), - engine.first.get())); - } else { - LOG(FATAL) << "Unknown Part type " << FLAGS_part_type; - } -} - - void NebulaStore::init() { CHECK(!!partMan_); - LOG(INFO) << "Scan the local path, and init the kvs_"; + LOG(INFO) << "Scan the local path, and init the spaces_"; { folly::RWSpinLock::WriteHolder wh(&lock_); for (auto& path : options_.dataPaths_) { auto rootPath = folly::stringPrintf("%s/nebula", path.c_str()); auto dirs = fs::FileUtils::listAllDirsInDir(rootPath.c_str()); for (auto& dir : dirs) { - LOG(INFO) << "Scan path " << path << "/" << dir; + LOG(INFO) << "Scan path \"" << path << "/" << dir << "\""; try { auto spaceId = folly::to(dir); - if (!partMan_->spaceExist(options_.local_, spaceId)) { - LOG(INFO) << "Space " << spaceId << " not exist any more, remove the data!"; - auto dataPath = folly::stringPrintf("%s/%s", rootPath.c_str(), dir.c_str()); + if (!partMan_->spaceExist(storeSvcAddr_, spaceId)) { + // TODO We might want to have a second thought here. + // Removing the data directly feels a little strong + LOG(INFO) << "Space " << spaceId + << " does not exist any more, remove the data!"; + auto dataPath = folly::stringPrintf("%s/%s", + rootPath.c_str(), + dir.c_str()); CHECK(fs::FileUtils::remove(dataPath.c_str(), true)); continue; } auto engine = newEngine(spaceId, path); - auto spaceIt = this->kvs_.find(spaceId); - if (spaceIt == this->kvs_.end()) { + auto spaceIt = this->spaces_.find(spaceId); + if (spaceIt == this->spaces_.end()) { LOG(INFO) << "Load space " << spaceId << " from disk"; - this->kvs_.emplace(spaceId, std::make_unique()); + spaceIt = this->spaces_.emplace( + spaceId, + std::make_unique()).first; } - auto& spaceKV = this->kvs_[spaceId]; - for (auto& partId : engine.first->allParts()) { - if (!partMan_->partExist(options_.local_, spaceId, partId)) { - LOG(INFO) << "Part " << partId << " not exist any more, remove it!"; - engine.first->removePart(partId); + for (auto& partId : engine->allParts()) { + if (!partMan_->partExist(storeSvcAddr_, spaceId, partId)) { + LOG(INFO) << "Part " << partId + << " does not exist any more, remove it!"; + engine->removePart(partId); continue; } - spaceKV->parts_.emplace(partId, newPart(spaceId, partId, engine)); - LOG(INFO) << "Space " << spaceId - << ", part " << partId << " has been loaded!"; } - spaceKV->engines_.emplace_back(std::move(engine)); + spaceIt->second->engines_.emplace_back(std::move(engine)); } catch (std::exception& e) { - LOG(FATAL) << "Can't convert " << dir; + LOG(FATAL) << "Invalid data directory \"" << dir << "\""; } } } } - LOG(INFO) << "Init data from partManager for " << options_.local_; - auto partsMap = partMan_->parts(options_.local_); + + LOG(INFO) << "Init data from partManager for " << storeSvcAddr_; + auto partsMap = partMan_->parts(storeSvcAddr_); for (auto& entry : partsMap) { auto spaceId = entry.first; addSpace(spaceId); @@ -166,21 +131,36 @@ void NebulaStore::init() { addPart(spaceId, partEntry.first); } } + LOG(INFO) << "Register handler..."; partMan_->registerHandler(this); } +std::unique_ptr NebulaStore::newEngine(GraphSpaceID spaceId, + const std::string& path) { + if (FLAGS_engine_type == "rocksdb") { + return std::make_unique(spaceId, + path, + options_.mergeOp_, + options_.cfFactory_); + } else { + LOG(FATAL) << "Unknown engine type " << FLAGS_engine_type; + return nullptr; + } +} + + void NebulaStore::addSpace(GraphSpaceID spaceId) { folly::RWSpinLock::WriteHolder wh(&lock_); - if (this->kvs_.find(spaceId) != this->kvs_.end()) { + if (this->spaces_.find(spaceId) != this->spaces_.end()) { LOG(INFO) << "Space " << spaceId << " has existed!"; return; } LOG(INFO) << "Create space " << spaceId; - this->kvs_[spaceId] = std::make_unique(); + this->spaces_[spaceId] = std::make_unique(); for (auto& path : options_.dataPaths_) { - this->kvs_[spaceId]->engines_.emplace_back(newEngine(spaceId, path)); + this->spaces_[spaceId]->engines_.emplace_back(newEngine(spaceId, path)); } return; } @@ -188,28 +168,42 @@ void NebulaStore::addSpace(GraphSpaceID spaceId) { void NebulaStore::addPart(GraphSpaceID spaceId, PartitionID partId) { folly::RWSpinLock::WriteHolder wh(&lock_); - auto spaceIt = this->kvs_.find(spaceId); - CHECK(spaceIt != this->kvs_.end()) << "Space should exist!"; + auto spaceIt = this->spaces_.find(spaceId); + CHECK(spaceIt != this->spaces_.end()) << "Space should exist!"; if (spaceIt->second->parts_.find(partId) != spaceIt->second->parts_.end()) { LOG(INFO) << "[" << spaceId << "," << partId << "] has existed!"; return; } - int32_t minIndex = -1, index = 0; + + int32_t minIndex = -1; + int32_t index = 0; int32_t minPartsNum = 0x7FFFFFFF; auto& engines = spaceIt->second->engines_; for (auto& engine : engines) { - if (engine.first->totalPartsNum() < minPartsNum) { - minPartsNum = engine.first->totalPartsNum(); + if (engine->totalPartsNum() < minPartsNum) { + minPartsNum = engine->totalPartsNum(); minIndex = index; } index++; } CHECK_GE(minIndex, 0) << "engines number:" << engines.size(); const auto& targetEngine = engines[minIndex]; + // Write the information into related engine. - targetEngine.first->addPart(partId); - spaceIt->second->parts_.emplace(partId, - newPart(spaceId, partId, targetEngine)); + targetEngine->addPart(partId); + spaceIt->second->parts_.emplace( + partId, + std::make_shared(spaceId, + partId, + raftAddr_, + folly::stringPrintf("%s/wal/%d", + targetEngine->getDataRoot(), + partId), + targetEngine.get(), + ioPool_, + workers_)); + // TODO: Need to pass in the peers + spaceIt->second->parts_[partId]->start({}); LOG(INFO) << "Space " << spaceId << ", part " << partId << " has been added!"; return; } @@ -217,16 +211,16 @@ void NebulaStore::addPart(GraphSpaceID spaceId, PartitionID partId) { void NebulaStore::removeSpace(GraphSpaceID spaceId) { folly::RWSpinLock::WriteHolder wh(&lock_); - auto spaceIt = this->kvs_.find(spaceId); + auto spaceIt = this->spaces_.find(spaceId); auto& engines = spaceIt->second->engines_; for (auto& engine : engines) { - auto parts = engine.first->allParts(); + auto parts = engine->allParts(); for (auto& partId : parts) { - engine.first->removePart(partId); + engine->removePart(partId); } - CHECK_EQ(0, engine.first->totalPartsNum()); + CHECK_EQ(0, engine->totalPartsNum()); } - this->kvs_.erase(spaceIt); + this->spaces_.erase(spaceIt); // TODO(dangleptr): Should we delete the data? LOG(INFO) << "Space " << spaceId << " has been removed!"; } @@ -234,12 +228,15 @@ void NebulaStore::removeSpace(GraphSpaceID spaceId) { void NebulaStore::removePart(GraphSpaceID spaceId, PartitionID partId) { folly::RWSpinLock::WriteHolder wh(&lock_); - auto spaceIt = this->kvs_.find(spaceId); - if (spaceIt != this->kvs_.end()) { + auto spaceIt = this->spaces_.find(spaceId); + if (spaceIt != this->spaces_.end()) { auto partIt = spaceIt->second->parts_.find(partId); if (partIt != spaceIt->second->parts_.end()) { auto* e = partIt->second->engine(); CHECK_NOTNULL(e); + // Stop the raft + partIt->second->stop(); + spaceIt->second->parts_.erase(partId); e->removePart(partId); } } @@ -247,21 +244,26 @@ void NebulaStore::removePart(GraphSpaceID spaceId, PartitionID partId) { } -ResultCode NebulaStore::get(GraphSpaceID spaceId, PartitionID partId, +ResultCode NebulaStore::get(GraphSpaceID spaceId, + PartitionID partId, const std::string& key, std::string* value) { CHECK_AND_RETURN_ENGINE(spaceId, partId); return engine->get(key, value); } -ResultCode NebulaStore::multiGet(GraphSpaceID spaceId, PartitionID partId, + +ResultCode NebulaStore::multiGet(GraphSpaceID spaceId, + PartitionID partId, const std::vector& keys, std::vector* values) { CHECK_AND_RETURN_ENGINE(spaceId, partId); return engine->multiGet(keys, values); } -ResultCode NebulaStore::range(GraphSpaceID spaceId, PartitionID partId, + +ResultCode NebulaStore::range(GraphSpaceID spaceId, + PartitionID partId, const std::string& start, const std::string& end, std::unique_ptr* iter) { @@ -270,7 +272,8 @@ ResultCode NebulaStore::range(GraphSpaceID spaceId, PartitionID partId, } -ResultCode NebulaStore::prefix(GraphSpaceID spaceId, PartitionID partId, +ResultCode NebulaStore::prefix(GraphSpaceID spaceId, + PartitionID partId, const std::string& prefix, std::unique_ptr* iter) { CHECK_AND_RETURN_ENGINE(spaceId, partId); @@ -278,7 +281,8 @@ ResultCode NebulaStore::prefix(GraphSpaceID spaceId, PartitionID partId, } -void NebulaStore::asyncMultiPut(GraphSpaceID spaceId, PartitionID partId, +void NebulaStore::asyncMultiPut(GraphSpaceID spaceId, + PartitionID partId, std::vector keyValues, KVCallback cb) { CHECK_FOR_WRITE(spaceId, partId, cb); @@ -294,6 +298,7 @@ void NebulaStore::asyncRemove(GraphSpaceID spaceId, return partIt->second->asyncRemove(key, std::move(cb)); } + void NebulaStore::asyncMultiRemove(GraphSpaceID spaceId, PartitionID partId, std::vector keys, @@ -302,6 +307,7 @@ void NebulaStore::asyncMultiRemove(GraphSpaceID spaceId, return partIt->second->asyncMultiRemove(std::move(keys), cb); } + void NebulaStore::asyncRemoveRange(GraphSpaceID spaceId, PartitionID partId, const std::string& start, @@ -312,13 +318,22 @@ void NebulaStore::asyncRemoveRange(GraphSpaceID spaceId, } +void NebulaStore::asyncRemovePrefix(GraphSpaceID spaceId, + PartitionID partId, + const std::string& prefix, + KVCallback cb) { + CHECK_FOR_WRITE(spaceId, partId, cb); + return partIt->second->asyncRemovePrefix(prefix, std::move(cb)); +} + + ResultCode NebulaStore::ingest(GraphSpaceID spaceId, const std::string& extra, const std::vector& files) { - std::unordered_map>::iterator it; + decltype(spaces_)::iterator it; RETURN_IF_SPACE_NOT_FOUND(spaceId, it); for (auto& engine : it->second->engines_) { - auto parts = engine.first->allParts(); + auto parts = engine->allParts(); std::vector extras; for (auto part : parts) { for (auto file : files) { @@ -331,7 +346,7 @@ ResultCode NebulaStore::ingest(GraphSpaceID spaceId, extras.emplace_back(std::move(extraPath)); } } - auto code = engine.first->ingest(std::move(extras)); + auto code = engine->ingest(std::move(extras)); RETURN_ON_FAILURE(code); } return ResultCode::SUCCEEDED; @@ -341,10 +356,10 @@ ResultCode NebulaStore::ingest(GraphSpaceID spaceId, ResultCode NebulaStore::setOption(GraphSpaceID spaceId, const std::string& configKey, const std::string& configValue) { - std::unordered_map>::iterator it; + decltype(spaces_)::iterator it; RETURN_IF_SPACE_NOT_FOUND(spaceId, it); for (auto& engine : it->second->engines_) { - auto code = engine.first->setOption(configKey, configValue); + auto code = engine->setOption(configKey, configValue); RETURN_ON_FAILURE(code); } return ResultCode::SUCCEEDED; @@ -354,10 +369,10 @@ ResultCode NebulaStore::setOption(GraphSpaceID spaceId, ResultCode NebulaStore::setDBOption(GraphSpaceID spaceId, const std::string& configKey, const std::string& configValue) { - std::unordered_map>::iterator it; + decltype(spaces_)::iterator it; RETURN_IF_SPACE_NOT_FOUND(spaceId, it); for (auto& engine : it->second->engines_) { - auto code = engine.first->setDBOption(configKey, configValue); + auto code = engine->setDBOption(configKey, configValue); RETURN_ON_FAILURE(code); } return ResultCode::SUCCEEDED; @@ -365,10 +380,10 @@ ResultCode NebulaStore::setDBOption(GraphSpaceID spaceId, ResultCode NebulaStore::compactAll(GraphSpaceID spaceId) { - std::unordered_map>::iterator it; + decltype(spaces_)::iterator it; RETURN_IF_SPACE_NOT_FOUND(spaceId, it); for (auto& engine : it->second->engines_) { - auto code = engine.first->compactAll(); + auto code = engine->compactAll(); RETURN_ON_FAILURE(code); } return ResultCode::SUCCEEDED; diff --git a/src/kvstore/NebulaStore.h b/src/kvstore/NebulaStore.h index 1c864449cd2..0b46e699a31 100644 --- a/src/kvstore/NebulaStore.h +++ b/src/kvstore/NebulaStore.h @@ -10,6 +10,7 @@ #include "base/Base.h" #include #include +#include "raftex/RaftexService.h" #include "kvstore/KVStore.h" #include "kvstore/PartManager.h" #include "kvstore/Part.h" @@ -18,12 +19,9 @@ namespace nebula { namespace kvstore { -// -using Engine = std::pair, std::string>; - -struct GraphSpaceKV { - std::unordered_map> parts_; - std::vector engines_; +struct SpacePartInfo { + std::unordered_map> parts_; + std::vector> engines_; }; @@ -32,22 +30,53 @@ class NebulaStore : public KVStore, public Handler { FRIEND_TEST(NebulaStoreTest, PartsTest); public: - explicit NebulaStore(KVOptions options) - : options_(std::move(options)) { + NebulaStore(KVOptions options, + std::shared_ptr ioPool, + std::shared_ptr workers, + HostAddr serviceAddr) + : ioPool_(ioPool) + , workers_(workers) + , storeSvcAddr_(serviceAddr) + , raftAddr_(getRaftAddr(serviceAddr)) + , options_(std::move(options)) { partMan_ = std::move(options_.partMan_); + init(); } ~NebulaStore() = default; - /** - * Pull meta information from PartManager and init current instance. - * */ + // Calculate the raft service address based on the storage service address + static HostAddr getRaftAddr(HostAddr srvcAddr) { + return HostAddr(srvcAddr.first, srvcAddr.second + 1); + } + + // Pull meta information from the PartManager and initiate + // the current store instance void init(); uint32_t capability() const override { return 0; } + std::shared_ptr getIoPool() const { + return ioPool_; + } + + std::shared_ptr getWorkers() const { + return workers_; + } + + // Return the current leader + HostAddr partLeader(GraphSpaceID spaceId, PartitionID partId) override { + UNUSED(spaceId); + UNUSED(partId); + return {0, 0}; + } + + PartManager* partManager() const override { + return partMan_.get(); + } + ResultCode get(GraphSpaceID spaceId, PartitionID partId, const std::string& key, @@ -58,26 +87,20 @@ class NebulaStore : public KVStore, public Handler { const std::vector& keys, std::vector* values) override; - /** - * Get all results in range [start, end) - * */ + // Get all results in range [start, end) ResultCode range(GraphSpaceID spaceId, PartitionID partId, const std::string& start, const std::string& end, std::unique_ptr* iter) override; - /** - * Get all results with prefix. - * */ + // Get all results with prefix. ResultCode prefix(GraphSpaceID spaceId, PartitionID partId, const std::string& prefix, std::unique_ptr* iter) override; - /** - * async batch put. - * */ + // async batch put. void asyncMultiPut(GraphSpaceID spaceId, PartitionID partId, std::vector keyValues, @@ -99,6 +122,11 @@ class NebulaStore : public KVStore, public Handler { const std::string& end, KVCallback cb) override; + void asyncRemovePrefix(GraphSpaceID spaceId, + PartitionID partId, + const std::string& prefix, + KVCallback cb) override; + ResultCode ingest(GraphSpaceID spaceId, const std::string& extra, const std::vector& files); @@ -126,17 +154,22 @@ class NebulaStore : public KVStore, public Handler { void removePart(GraphSpaceID spaceId, PartitionID partId) override; private: - Engine newEngine(GraphSpaceID spaceId, std::string rootPath); - - std::unique_ptr newPart(GraphSpaceID spaceId, - PartitionID partId, - const Engine& engine); + std::unique_ptr newEngine(GraphSpaceID spaceId, const std::string& path); private: - std::unordered_map> kvs_; + // The lock used to protect spaces_ folly::RWSpinLock lock_; - std::unique_ptr partMan_{nullptr}; + std::unordered_map> spaces_; + + std::shared_ptr ioPool_; + std::shared_ptr workers_; + HostAddr storeSvcAddr_; + HostAddr raftAddr_; KVOptions options_; + + std::unique_ptr partMan_{nullptr}; + + std::shared_ptr raftService_; }; } // namespace kvstore diff --git a/src/kvstore/Part.cpp b/src/kvstore/Part.cpp index 5f2d05172a8..50876b54f77 100644 --- a/src/kvstore/Part.cpp +++ b/src/kvstore/Part.cpp @@ -5,34 +5,235 @@ */ #include "kvstore/Part.h" +#include "wal/BufferFlusher.h" +#include "kvstore/LogEncoder.h" + +DEFINE_int32(cluster_id, 0, "A unique id for each cluster"); namespace nebula { namespace kvstore { -void SimplePart::asyncMultiPut(std::vector keyValues, KVCallback cb) { - CHECK_NOTNULL(engine_); - auto ret = engine_->multiPut(std::move(keyValues)); - cb(ret, HostAddr(0, 0)); +using raftex::AppendLogResult; + +const char kLastCommittedIdKey[] = "_last_committed_log_id"; + +namespace { + +ResultCode toResultCode(AppendLogResult res) { + switch (res) { + case AppendLogResult::SUCCEEDED: + return ResultCode::SUCCEEDED; + case AppendLogResult::E_NOT_A_LEADER: + return ResultCode::ERR_LEADER_CHANGED; + default: + return ResultCode::ERR_CONSENSUS_ERROR; + } } -void SimplePart::asyncRemove(const std::string& key, KVCallback cb) { - CHECK_NOTNULL(engine_); - auto ret = engine_->remove(key); - cb(ret, HostAddr(0, 0)); + +wal::BufferFlusher* getBufferFlusher() { + static wal::BufferFlusher flusher; + return &flusher; } -void SimplePart::asyncMultiRemove(std::vector keys, KVCallback cb) { - CHECK_NOTNULL(engine_); - auto ret = engine_->multiRemove(std::move(keys)); - cb(ret, HostAddr(0, 0)); +} // Anonymous namespace + + +Part::Part(GraphSpaceID spaceId, + PartitionID partId, + HostAddr localAddr, + const std::string& walPath, + KVEngine* engine, + std::shared_ptr ioPool, + std::shared_ptr workers) + : RaftPart(FLAGS_cluster_id, + spaceId, + partId, + localAddr, + walPath, + getBufferFlusher(), + ioPool, + workers) + , spaceId_(spaceId) + , partId_(partId) + , walPath_(walPath) + , engine_(engine) { } -void SimplePart::asyncRemoveRange(const std::string& start, - const std::string& end, - KVCallback cb) { - CHECK_NOTNULL(engine_); - auto ret = engine_->removeRange(start, end); - cb(ret, HostAddr(0, 0)); + +LogID Part::lastCommittedLogId() { + std::string val; + ResultCode res = engine_->get(kLastCommittedIdKey, &val); + if (res != ResultCode::SUCCEEDED) { + LOG(ERROR) << "Cannot fetch the last committed log id from the storage engine"; + return 0; + } + CHECK_EQ(val.size(), sizeof(LogID)); + + LogID lastId; + memcpy(reinterpret_cast(&lastId), val.data(), sizeof(LogID)); + + return lastId; +} + + +void Part::asyncPut(folly::StringPiece key, folly::StringPiece value, KVCallback cb) { + std::string log = encodeMultiValues(OP_PUT, key, value);; + + appendAsync(FLAGS_cluster_id, std::move(log)) + .then([callback = std::move(cb)] (AppendLogResult res) { + callback(toResultCode(res)); + }); +} + + +void Part::asyncMultiPut(const std::vector& keyValues, KVCallback cb) { + std::string log = encodeMultiValues(OP_MULTI_PUT, keyValues);; + + appendAsync(FLAGS_cluster_id, std::move(log)) + .then([callback = std::move(cb)] (AppendLogResult res) { + callback(toResultCode(res)); + }); +} + + +void Part::asyncRemove(folly::StringPiece key, KVCallback cb) { + std::string log = encodeSingleValue(OP_REMOVE, key);; + + appendAsync(FLAGS_cluster_id, std::move(log)) + .then([callback = std::move(cb)] (AppendLogResult res) { + callback(toResultCode(res)); + }); +} + + +void Part::asyncMultiRemove(const std::vector& keys, KVCallback cb) { + std::string log = encodeMultiValues(OP_MULTI_REMOVE, keys);; + + appendAsync(FLAGS_cluster_id, std::move(log)) + .then([callback = std::move(cb)] (AppendLogResult res) { + callback(toResultCode(res)); + }); +} + + +void Part::asyncRemovePrefix(folly::StringPiece prefix, KVCallback cb) { + std::string log = encodeSingleValue(OP_REMOVE_PREFIX, prefix);; + + appendAsync(FLAGS_cluster_id, std::move(log)) + .then([callback = std::move(cb)] (AppendLogResult res) { + callback(toResultCode(res)); + }); +} + + +void Part::asyncRemoveRange(folly::StringPiece start, + folly::StringPiece end, + KVCallback cb) { + std::string log = encodeMultiValues(OP_REMOVE_RANGE, start, end);; + + appendAsync(FLAGS_cluster_id, std::move(log)) + .then([callback = std::move(cb)] (AppendLogResult res) { + callback(toResultCode(res)); + }); +} + + +void Part::onLostLeadership(TermID term) { + VLOG(1) << "Lost the leadership for the term " << term; +} + + +void Part::onElected(TermID term) { + VLOG(1) << "Being elected as the leader for the term " << term; +} + + +std::string Part::compareAndSet(const std::string& log) { + UNUSED(log); + LOG(FATAL) << "To be implemented"; +} + + +bool Part::commitLogs(std::unique_ptr iter) { + auto batch = engine_->startBatchWrite(); + LogID lastId = -1; + while (iter->valid()) { + lastId = iter->logId(); + auto log = iter->logMsg(); + DCHECK_GE(log.size(), sizeof(int64_t) + 1 + sizeof(uint32_t)); + // Skip the timestamp (type of int64_t) + switch (log[sizeof(int64_t)]) { + case OP_PUT: { + auto pieces = decodeMultiValues(log); + DCHECK_EQ(2, pieces.size()); + if (batch->put(pieces[0], pieces[1]) != ResultCode::SUCCEEDED) { + LOG(ERROR) << "Failed to call WriteBatch::put()"; + return false; + } + break; + } + case OP_MULTI_PUT: { + auto kvs = decodeMultiValues(log); + // Make the number of values are an even number + DCHECK_EQ((kvs.size() + 1) / 2, kvs.size() / 2); + for (size_t i = 0; i < kvs.size(); i += 2) { + if (batch->put(kvs[i], kvs[i + 1]) != ResultCode::SUCCEEDED) { + LOG(ERROR) << "Failed to call WriteBatch::put()"; + return false; + } + } + break; + } + case OP_REMOVE: { + auto key = decodeSingleValue(log); + if (batch->remove(key) != ResultCode::SUCCEEDED) { + LOG(ERROR) << "Failed to call WriteBatch::remove()"; + return false; + } + break; + } + case OP_MULTI_REMOVE: { + auto keys = decodeMultiValues(log); + for (auto k : keys) { + if (batch->remove(k) != ResultCode::SUCCEEDED) { + LOG(ERROR) << "Failed to call WriteBatch::remove()"; + return false; + } + } + break; + } + case OP_REMOVE_PREFIX: { + auto prefix = decodeSingleValue(log); + if (batch->removePrefix(prefix) != ResultCode::SUCCEEDED) { + LOG(ERROR) << "Failed to call WriteBatch::removePrefix()"; + return false; + } + break; + } + case OP_REMOVE_RANGE: { + auto range = decodeMultiValues(log); + DCHECK_EQ(2, range.size()); + if (batch->removeRange(range[0], range[1]) != ResultCode::SUCCEEDED) { + LOG(ERROR) << "Failed to call WriteBatch::removeRange()"; + return false; + } + break; + } + default: { + LOG(FATAL) << "Unknown operation: " << static_cast(log[0]); + } + } + + ++(*iter); + } + + if (lastId >= 0) { + batch->put(kLastCommittedIdKey, + folly::StringPiece(reinterpret_cast(&lastId), sizeof(LogID))); + } + + return engine_->commitBatchWrite(std::move(batch)) == ResultCode::SUCCEEDED; } } // namespace kvstore diff --git a/src/kvstore/Part.h b/src/kvstore/Part.h index 93d367c4a05..3f8aef1b320 100644 --- a/src/kvstore/Part.h +++ b/src/kvstore/Part.h @@ -8,22 +8,22 @@ #define KVSTORE_PART_H_ #include "base/Base.h" +#include "raftex/RaftPart.h" #include "kvstore/Common.h" #include "kvstore/KVEngine.h" namespace nebula { namespace kvstore { -class Part { +class Part : public raftex::RaftPart { public: Part(GraphSpaceID spaceId, PartitionID partId, + HostAddr localAddr, const std::string& walPath, - KVEngine* engine) - : spaceId_(spaceId) - , partId_(partId) - , walPath_(walPath) - , engine_(engine) {} + KVEngine* engine, + std::shared_ptr pool, + std::shared_ptr workers); virtual ~Part() = default; @@ -31,15 +31,28 @@ class Part { return engine_; } - virtual void asyncMultiPut(std::vector keyValues, KVCallback cb) = 0; + void asyncPut(folly::StringPiece key, folly::StringPiece value, KVCallback cb); + void asyncMultiPut(const std::vector& keyValues, KVCallback cb); - virtual void asyncRemove(const std::string& key, KVCallback cb) = 0; + void asyncRemove(folly::StringPiece key, KVCallback cb); + void asyncMultiRemove(const std::vector& keys, KVCallback cb); + void asyncRemovePrefix(folly::StringPiece prefix, KVCallback cb); + void asyncRemoveRange(folly::StringPiece start, + folly::StringPiece end, + KVCallback cb); - virtual void asyncMultiRemove(std::vector keys, KVCallback cb) = 0; + /** + * Methods inherited from RaftPart + */ + LogID lastCommittedLogId() override; - virtual void asyncRemoveRange(const std::string& start, - const std::string& end, - KVCallback cb) = 0; + void onLostLeadership(TermID term) override; + + void onElected(TermID term) override; + + std::string compareAndSet(const std::string& log) override; + + bool commitLogs(std::unique_ptr iter) override; protected: GraphSpaceID spaceId_; @@ -48,28 +61,6 @@ class Part { KVEngine* engine_ = nullptr; }; -/** - * Bypass raft, just write into storage when asyncMultiPut invoked. - * */ -class SimplePart final : public Part { -public: - SimplePart(GraphSpaceID spaceId, PartitionID partId, - const std::string& walPath, KVEngine* engine) - : Part(spaceId, partId, walPath, engine) {} - - void asyncMultiPut(std::vector keyValues, KVCallback cb) override; - - void asyncRemove(const std::string& key, KVCallback cb) override; - - void asyncMultiRemove(std::vector keys, KVCallback cb) override; - - void asyncRemoveRange(const std::string& start, - const std::string& end, - KVCallback cb) override; -}; - - - } // namespace kvstore } // namespace nebula #endif // KVSTORE_PART_H_ diff --git a/src/kvstore/PartManager.h b/src/kvstore/PartManager.h index 1fdcdbc1f3f..9dda54343e4 100644 --- a/src/kvstore/PartManager.h +++ b/src/kvstore/PartManager.h @@ -23,6 +23,7 @@ class Handler { virtual void removePart(GraphSpaceID spaceId, PartitionID partId) = 0; }; + /** * This class manages all meta information one storage host needed. * */ @@ -63,6 +64,7 @@ class PartManager { Handler* handler_ = nullptr; }; + /** : * Memory based PartManager, it is used in UTs now. * */ @@ -127,6 +129,7 @@ class MemPartManager final : public PartManager { PartsMap partsMap_; }; + class MetaServerBasedPartManager : public PartManager, public meta::MetaChangedListener { public: explicit MetaServerBasedPartManager(HostAddr host, meta::MetaClient *client = nullptr); @@ -159,7 +162,7 @@ class MetaServerBasedPartManager : public PartManager, public meta::MetaChangedL } /** - * for UTS, because the port is choosed by system, + * for UTs, because the port is choosed by system, * we should update port after thrift setup * */ void setLocalHost(HostAddr localHost) { diff --git a/src/kvstore/RocksEngine.cpp b/src/kvstore/RocksEngine.cpp index ffd77d12d0d..522751da190 100644 --- a/src/kvstore/RocksEngine.cpp +++ b/src/kvstore/RocksEngine.cpp @@ -19,16 +19,91 @@ using fs::FileType; const char* kSystemParts = "__system__parts__"; +namespace { + +/*************************************** + * + * Implementation of WriteBatch + * + **************************************/ +class RocksWriteBatch : public WriteBatch { +private: + rocksdb::WriteBatch batch_; + rocksdb::DB* db_{nullptr}; + +public: + explicit RocksWriteBatch(rocksdb::DB* db) : db_(db) {} + + virtual ~RocksWriteBatch() = default; + + ResultCode put(folly::StringPiece key, folly::StringPiece value) override { + if (batch_.Put(toSlice(key), toSlice(value)).ok()) { + return ResultCode::SUCCEEDED; + } else { + return ResultCode::ERR_UNKNOWN; + } + } + + ResultCode remove(folly::StringPiece key) override { + if (batch_.Delete(toSlice(key)).ok()) { + return ResultCode::SUCCEEDED; + } else { + return ResultCode::ERR_UNKNOWN; + } + } + + ResultCode removePrefix(folly::StringPiece prefix) override { + rocksdb::Slice pre(prefix.begin(), prefix.size()); + rocksdb::ReadOptions options; + std::unique_ptr iter(db_->NewIterator(options)); + iter->Seek(pre); + while (iter->Valid()) { + if (iter->key().starts_with(pre)) { + if (!batch_.Delete(iter->key()).ok()) { + return ResultCode::ERR_UNKNOWN; + } + } else { + // Done + break; + } + iter->Next(); + } + return ResultCode::SUCCEEDED; + } + + // Remove all keys in the range [start, end) + ResultCode removeRange(folly::StringPiece start, folly::StringPiece end) override { + if (batch_.DeleteRange(toSlice(start), toSlice(end)).ok()) { + return ResultCode::SUCCEEDED; + } else { + return ResultCode::ERR_UNKNOWN; + } + } + + rocksdb::WriteBatch* data() { + return &batch_; + } +}; + +} // Anonymous namespace + + +/*************************************** + * + * Implementation of WriteBatch + * + **************************************/ RocksEngine::RocksEngine(GraphSpaceID spaceId, const std::string& dataPath, std::shared_ptr mergeOp, std::shared_ptr cfFactory) : KVEngine(spaceId) - , dataPath_(dataPath) { - LOG(INFO) << "open rocksdb on " << dataPath; - if (FileUtils::fileType(dataPath.c_str()) == FileType::NOTEXIST) { - FileUtils::makeDir(dataPath); + , dataPath_(folly::stringPrintf("%s/nebula/%d", dataPath.c_str(), spaceId)) { + auto path = folly::stringPrintf("%s/data", dataPath_.c_str()); + if (FileUtils::fileType(path.c_str()) == FileType::NOTEXIST) { + FileUtils::makeDir(path); } + LOG(INFO) << "open rocksdb on " << path; rocksdb::Options options; rocksdb::DB* db = nullptr; @@ -40,16 +115,30 @@ RocksEngine::RocksEngine(GraphSpaceID spaceId, if (cfFactory != nullptr) { options.compaction_filter_factory = cfFactory; } - status = rocksdb::DB::Open(options, dataPath_, &db); + status = rocksdb::DB::Open(options, path, &db); CHECK(status.ok()); db_.reset(db); partsNum_ = allParts().size(); } -RocksEngine::~RocksEngine() { +std::unique_ptr RocksEngine::startBatchWrite() { + return std::make_unique(db_.get()); +} + + +ResultCode RocksEngine::commitBatchWrite(std::unique_ptr batch) { + rocksdb::WriteOptions options; + options.disableWAL = FLAGS_rocksdb_disable_wal; + auto* b = static_cast(batch.get()); + rocksdb::Status status = db_->Write(options, b->data()); + if (status.ok()) { + return ResultCode::SUCCEEDED; + } + return ResultCode::ERR_UNKNOWN; } + ResultCode RocksEngine::get(const std::string& key, std::string* value) { rocksdb::ReadOptions options; rocksdb::Status status = db_->Get(options, rocksdb::Slice(key), value); @@ -64,6 +153,7 @@ ResultCode RocksEngine::get(const std::string& key, std::string* value) { } } + ResultCode RocksEngine::multiGet(const std::vector& keys, std::vector* values) { rocksdb::ReadOptions options; @@ -84,6 +174,32 @@ ResultCode RocksEngine::multiGet(const std::vector& keys, } } + +ResultCode RocksEngine::range(const std::string& start, + const std::string& end, + std::unique_ptr* storageIter) { + rocksdb::ReadOptions options; + rocksdb::Iterator* iter = db_->NewIterator(options); + if (iter) { + iter->Seek(rocksdb::Slice(start)); + } + storageIter->reset(new RocksRangeIter(iter, start, end)); + return ResultCode::SUCCEEDED; +} + + +ResultCode RocksEngine::prefix(const std::string& prefix, + std::unique_ptr* storageIter) { + rocksdb::ReadOptions options; + rocksdb::Iterator* iter = db_->NewIterator(options); + if (iter) { + iter->Seek(rocksdb::Slice(prefix)); + } + storageIter->reset(new RocksPrefixIter(iter, prefix)); + return ResultCode::SUCCEEDED; +} + + ResultCode RocksEngine::put(std::string key, std::string value) { rocksdb::WriteOptions options; options.disableWAL = FLAGS_rocksdb_disable_wal; @@ -114,31 +230,6 @@ ResultCode RocksEngine::multiPut(std::vector keyValues) { } -ResultCode RocksEngine::range(const std::string& start, - const std::string& end, - std::unique_ptr* storageIter) { - rocksdb::ReadOptions options; - rocksdb::Iterator* iter = db_->NewIterator(options); - if (iter) { - iter->Seek(rocksdb::Slice(start)); - } - storageIter->reset(new RocksRangeIter(iter, start, end)); - return ResultCode::SUCCEEDED; -} - - -ResultCode RocksEngine::prefix(const std::string& prefix, - std::unique_ptr* storageIter) { - rocksdb::ReadOptions options; - rocksdb::Iterator* iter = db_->NewIterator(options); - if (iter) { - iter->Seek(rocksdb::Slice(prefix)); - } - storageIter->reset(new RocksPrefixIter(iter, prefix)); - return ResultCode::SUCCEEDED; -} - - ResultCode RocksEngine::remove(const std::string& key) { rocksdb::WriteOptions options; options.disableWAL = FLAGS_rocksdb_disable_wal; @@ -151,6 +242,7 @@ ResultCode RocksEngine::remove(const std::string& key) { } } + ResultCode RocksEngine::multiRemove(std::vector keys) { rocksdb::WriteBatch deletes(FLAGS_batch_reserved_bytes); for (size_t i = 0; i < keys.size(); i++) { @@ -172,6 +264,8 @@ ResultCode RocksEngine::removeRange(const std::string& start, const std::string& end) { rocksdb::WriteOptions options; options.disableWAL = FLAGS_rocksdb_disable_wal; + // TODO(sye) Given the RocksDB version we are using, + // we should avoud using DeleteRange auto status = db_->DeleteRange(options, db_->DefaultColumnFamily(), start, end); if (status.ok()) { return ResultCode::SUCCEEDED; @@ -182,6 +276,35 @@ ResultCode RocksEngine::removeRange(const std::string& start, } +ResultCode RocksEngine::removePrefix(const std::string& prefix) { + rocksdb::Slice pre(prefix.data(), prefix.size()); + rocksdb::ReadOptions readOptions; + rocksdb::WriteBatch batch; + std::unique_ptr iter(db_->NewIterator(readOptions)); + iter->Seek(pre); + while (iter->Valid()) { + if (iter->key().starts_with(pre)) { + auto status = batch.Delete(iter->key()); + if (!status.ok()) { + return ResultCode::ERR_UNKNOWN; + } + } else { + // Done + break; + } + iter->Next(); + } + + rocksdb::WriteOptions writeOptions; + writeOptions.disableWAL = FLAGS_rocksdb_disable_wal; + if (db_->Write(writeOptions, &batch).ok()) { + return ResultCode::SUCCEEDED; + } else { + return ResultCode::ERR_UNKNOWN; + } +} + + std::string RocksEngine::partKey(PartitionID partId) { std::string key; static const size_t prefixLen = ::strlen(kSystemParts); @@ -217,6 +340,7 @@ std::vector RocksEngine::allParts() { static const size_t prefixLen = ::strlen(kSystemParts); static const std::string prefixStr(kSystemParts, prefixLen); CHECK_EQ(ResultCode::SUCCEEDED, this->prefix(prefixStr, &iter)); + std::vector parts; while (iter->valid()) { auto key = iter->key(); @@ -277,6 +401,7 @@ ResultCode RocksEngine::setDBOption(const std::string& configKey, } } + ResultCode RocksEngine::compactAll() { rocksdb::CompactRangeOptions options; rocksdb::Status status = db_->CompactRange(options, nullptr, nullptr); @@ -290,4 +415,3 @@ ResultCode RocksEngine::compactAll() { } // namespace kvstore } // namespace nebula - diff --git a/src/kvstore/RocksEngine.h b/src/kvstore/RocksEngine.h index f0d44d7ca19..ec854fc386c 100644 --- a/src/kvstore/RocksEngine.h +++ b/src/kvstore/RocksEngine.h @@ -86,6 +86,11 @@ class RocksPrefixIter : public KVIterator { }; +/************************************************************************** + * + * An implementation of KVEngine based on Rocksdb + * + *************************************************************************/ class RocksEngine : public KVEngine { FRIEND_TEST(RocksEngineTest, SimpleTest); @@ -95,18 +100,22 @@ class RocksEngine : public KVEngine { std::shared_ptr mergeOp = nullptr, std::shared_ptr cfFactory = nullptr); - ~RocksEngine(); + ~RocksEngine() = default; - ResultCode get(const std::string& key, - std::string* value) override; + const char* getDataRoot() const override { + return dataPath_.c_str(); + } - ResultCode multiGet(const std::vector& keys, - std::vector* values) override; + std::unique_ptr startBatchWrite() override; + ResultCode commitBatchWrite(std::unique_ptr batch) override; - ResultCode put(std::string key, - std::string value) override; + /********************* + * Data retrieval + ********************/ + ResultCode get(const std::string& key, std::string* value) override; - ResultCode multiPut(std::vector keyValues) override; + ResultCode multiGet(const std::vector& keys, + std::vector* values) override; ResultCode range(const std::string& start, const std::string& end, @@ -115,6 +124,13 @@ class RocksEngine : public KVEngine { ResultCode prefix(const std::string& prefix, std::unique_ptr* iter) override; + /********************* + * Data modification + ********************/ + ResultCode put(std::string key, std::string value) override; + + ResultCode multiPut(std::vector keyValues) override; + ResultCode remove(const std::string& key) override; ResultCode multiRemove(std::vector keys) override; @@ -122,6 +138,11 @@ class RocksEngine : public KVEngine { ResultCode removeRange(const std::string& start, const std::string& end) override; + ResultCode removePrefix(const std::string& prefix) override; + + /********************* + * Non-data operation + ********************/ void addPart(PartitionID partId) override; void removePart(PartitionID partId) override; @@ -145,7 +166,6 @@ class RocksEngine : public KVEngine { private: std::string dataPath_; - std::string extraPath_; std::unique_ptr db_{nullptr}; int32_t partsNum_ = -1; }; diff --git a/src/kvstore/test/CMakeLists.txt b/src/kvstore/test/CMakeLists.txt index ed3471aabb7..f06451f9fd3 100644 --- a/src/kvstore/test/CMakeLists.txt +++ b/src/kvstore/test/CMakeLists.txt @@ -5,17 +5,21 @@ add_executable( $ $ $ - $ - $ + $ + $ + $ $ $ $ $ + $ + $ + $ ) nebula_link_libraries( part_test - ${ROCKSDB_LIBRARIES} ${THRIFT_LIBRARIES} + ${ROCKSDB_LIBRARIES} wangle gtest ) @@ -28,17 +32,21 @@ add_executable( $ $ $ - $ + $ + $ + $ $ $ $ $ $ + $ + $ ) nebula_link_libraries( rocks_engine_test - ${ROCKSDB_LIBRARIES} ${THRIFT_LIBRARIES} + ${ROCKSDB_LIBRARIES} wangle gtest ) @@ -51,17 +59,21 @@ add_executable( $ $ $ - $ + $ + $ + $ $ $ $ $ $ + $ + $ ) nebula_link_libraries( nebula_store_test - ${ROCKSDB_LIBRARIES} ${THRIFT_LIBRARIES} + ${ROCKSDB_LIBRARIES} wangle gtest ) @@ -74,17 +86,21 @@ add_executable( $ $ $ - $ - $ + $ + $ + $ $ $ $ $ + $ + $ + $ ) nebula_link_libraries( load_test - ${ROCKSDB_LIBRARIES} ${THRIFT_LIBRARIES} + ${ROCKSDB_LIBRARIES} wangle gtest ) @@ -95,13 +111,50 @@ add_executable( RocksEngineConfigTest.cpp ../RocksEngine.cpp ../RocksEngineConfig.cpp + $ + $ + $ $ + $ $ + $ + $ + $ ) nebula_link_libraries( rocks_engine_config_test + ${THRIFT_LIBRARIES} ${ROCKSDB_LIBRARIES} + wangle gtest ) nebula_add_test(rocks_engine_config_test) +add_executable( + log_encoder_test + LogEncoderTest.cpp + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ +) +nebula_link_libraries( + log_encoder_test + ${THRIFT_LIBRARIES} + ${ROCKSDB_LIBRARIES} + wangle + gtest +) +nebula_add_test(log_encoder_test) + + diff --git a/src/kvstore/test/LoadTest.cpp b/src/kvstore/test/LoadTest.cpp index 27a5d9db4c8..6534ab5ca99 100644 --- a/src/kvstore/test/LoadTest.cpp +++ b/src/kvstore/test/LoadTest.cpp @@ -33,7 +33,7 @@ TEST(Load, SSTLoad) { std::string result; EXPECT_EQ(ResultCode::SUCCEEDED, engine->get("key", &result)); - EXPECT_EQ(result, "value"); + EXPECT_EQ("value", result); } } // namespace kvstore diff --git a/src/kvstore/test/LogEncoderTest.cpp b/src/kvstore/test/LogEncoderTest.cpp new file mode 100644 index 00000000000..4469e23e964 --- /dev/null +++ b/src/kvstore/test/LogEncoderTest.cpp @@ -0,0 +1,129 @@ +/* Copyright (c) 2019 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "base/Base.h" +#include +#include "kvstore/LogEncoder.h" + + +namespace nebula { +namespace kvstore { + +TEST(LogEncoderTest, SingleValueTest) { + // Normal value + { + std::string val("Hello World"); + auto encoded = encodeSingleValue(OP_REMOVE, val); + ASSERT_EQ(sizeof(int64_t) + sizeof(uint32_t) + val.size() + 1, encoded.size()); + + auto decoded = decodeSingleValue(encoded); + ASSERT_EQ(val, decoded.toString()); + } + // Empty value + { + std::string val; + auto encoded = encodeSingleValue(OP_REMOVE, val); + ASSERT_EQ(sizeof(int64_t) + sizeof(uint32_t) + 1, encoded.size()); + + auto decoded = decodeSingleValue(encoded); + ASSERT_TRUE(decoded.empty()); + } +} + + +TEST(LogEncoderTest, MultiValuesTest) { + // Empty values + { + std::vector values; + auto encoded = encodeMultiValues(OP_MULTI_REMOVE, values); + ASSERT_EQ(1 + sizeof(uint32_t) + sizeof(int64_t), encoded.size()); + + auto decoded = decodeMultiValues(encoded); + ASSERT_TRUE(decoded.empty()); + } + + // Multi values + { + std::vector values; + // 3 values + for (int i = 0; i < 3; i++) { + values.emplace_back(folly::stringPrintf("Value%03d", i)); + } + // 2 empty values + for (int i = 0; i < 2; i++) { + values.emplace_back(); + } + auto encoded = encodeMultiValues(OP_MULTI_REMOVE, values); + auto decoded = decodeMultiValues(encoded); + ASSERT_EQ(5, decoded.size()); + for (int i = 0; i < 3; i++) { + ASSERT_EQ(folly::stringPrintf("Value%03d", i), decoded[i].toString()); + } + for (int i = 3; i < 5; i++) { + ASSERT_TRUE(decoded[i].empty()); + } + } + + // Two values + { + std::string v1("Hello"); + std::string v2("World"); + std::string v3; + + auto encoded = encodeMultiValues(OP_PUT, v1, v2); + auto decoded = decodeMultiValues(encoded); + EXPECT_EQ(2, decoded.size()); + EXPECT_EQ(v1, decoded[0].toString()); + EXPECT_EQ(v2, decoded[1].toString()); + + encoded = encodeMultiValues(OP_PUT, v3, v2); + decoded = decodeMultiValues(encoded); + EXPECT_EQ(2, decoded.size()); + EXPECT_TRUE(decoded[0].empty()); + EXPECT_EQ(v2, decoded[1].toString()); + } + + // Multi pairs + { + std::vector kvs; + // 2 pairs + for (int i = 0; i < 2; i++) { + kvs.emplace_back(folly::stringPrintf("Key%03d", i), + folly::stringPrintf("Value%03d", i)); + } + // 1 empty value + kvs.emplace_back("Key002", ""); + auto encoded = encodeMultiValues(OP_MULTI_PUT, kvs); + + auto decoded = decodeMultiValues(encoded); + // Total 3 pairs = 6 strings + ASSERT_EQ(6, decoded.size()); + // Check keys + for (int i = 0; i < 3; i++) { + ASSERT_EQ(folly::stringPrintf("Key%03d", i), decoded[i * 2].toString()); + } + // Check values + for (int i = 0; i < 2; i++) { + ASSERT_EQ(folly::stringPrintf("Value%03d", i), decoded[i * 2 + 1].toString()); + } + // Check the third empty value + ASSERT_TRUE(decoded[5].empty()); + } +} + +} // namespace kvstore +} // namespace nebula + + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + folly::init(&argc, &argv, true); + google::SetStderrLogging(google::INFO); + + return RUN_ALL_TESTS(); +} + + diff --git a/src/kvstore/test/NebulaStoreTest.cpp b/src/kvstore/test/NebulaStoreTest.cpp index f6b3be12b47..d4872a99ecf 100644 --- a/src/kvstore/test/NebulaStoreTest.cpp +++ b/src/kvstore/test/NebulaStoreTest.cpp @@ -13,22 +13,26 @@ #include "kvstore/PartManager.h" #include "kvstore/RocksEngine.h" + namespace nebula { namespace kvstore { +auto workers = std::make_shared(); +auto ioThreadPool = std::make_shared(4); + template void dump(const std::vector& v) { std::stringstream ss; for (auto& e : v) { ss << e << ", "; } - LOG(INFO) << ss.str(); + VLOG(1) << ss.str(); } TEST(NebulaStoreTest, SimpleTest) { - fs::TempDir rootPath("/tmp/kvstore_test.XXXXXX"); auto partMan = std::make_unique(); + // GraphSpaceID => {PartitionIDs} // 1 => {0, 1, 2, 3, 4, 5} // 2 => {0, 1, 2, 3, 4, 5} @@ -38,58 +42,63 @@ TEST(NebulaStoreTest, SimpleTest) { } } - LOG(INFO) << "Total space num " << partMan->partsMap_.size() - << ", " << partMan->parts(HostAddr(0, 0)).size(); + VLOG(1) << "Total space num is " << partMan->partsMap_.size() + << ", total local partitions num is " + << partMan->parts(HostAddr(0, 0)).size(); + fs::TempDir rootPath("/tmp/nebula_store_test.XXXXXX"); std::vector paths; paths.push_back(folly::stringPrintf("%s/disk1", rootPath.path())); paths.push_back(folly::stringPrintf("%s/disk2", rootPath.path())); - std::unique_ptr store; KVOptions options; - options.local_ = HostAddr(0, 0); options.dataPaths_ = std::move(paths); options.partMan_ = std::move(partMan); - store.reset(static_cast(KVStore::instance(std::move(options)))); - EXPECT_EQ(2, store->kvs_.size()); - - EXPECT_EQ(6, store->kvs_[1]->parts_.size()); - EXPECT_EQ(2, store->kvs_[1]->engines_.size()); - EXPECT_EQ(folly::stringPrintf("%s/disk1", rootPath.path()), - store->kvs_[1]->engines_[0].second); - EXPECT_EQ(folly::stringPrintf("%s/disk2", rootPath.path()), - store->kvs_[1]->engines_[1].second); - - EXPECT_EQ(6, store->kvs_[2]->parts_.size()); - EXPECT_EQ(2, store->kvs_[2]->engines_.size()); - EXPECT_EQ(folly::stringPrintf("%s/disk1", rootPath.path()), - store->kvs_[2]->engines_[0].second); - EXPECT_EQ(folly::stringPrintf("%s/disk2", rootPath.path()), - store->kvs_[2]->engines_[1].second); - - store->asyncMultiPut(0, 0, {{"key", "val"}}, [](ResultCode code, HostAddr addr) { - UNUSED(addr); + HostAddr local = {0, 0}; + auto store = std::make_unique(std::move(options), + ioThreadPool, + workers, + local); + sleep(1); + EXPECT_EQ(2, store->spaces_.size()); + + EXPECT_EQ(6, store->spaces_[1]->parts_.size()); + EXPECT_EQ(2, store->spaces_[1]->engines_.size()); + EXPECT_EQ(folly::stringPrintf("%s/disk1/nebula/1", rootPath.path()), + store->spaces_[1]->engines_[0]->getDataRoot()); + EXPECT_EQ(folly::stringPrintf("%s/disk2/nebula/1", rootPath.path()), + store->spaces_[1]->engines_[1]->getDataRoot()); + + EXPECT_EQ(6, store->spaces_[2]->parts_.size()); + EXPECT_EQ(2, store->spaces_[2]->engines_.size()); + EXPECT_EQ(folly::stringPrintf("%s/disk1/nebula/2", rootPath.path()), + store->spaces_[2]->engines_[0]->getDataRoot()); + EXPECT_EQ(folly::stringPrintf("%s/disk2/nebula/2", rootPath.path()), + store->spaces_[2]->engines_[1]->getDataRoot()); + + store->asyncMultiPut(0, 0, {{"key", "val"}}, [](ResultCode code) { EXPECT_EQ(ResultCode::ERR_SPACE_NOT_FOUND, code); }); - store->asyncMultiPut(1, 6, {{"key", "val"}}, [](ResultCode code, HostAddr addr) { - UNUSED(addr); + store->asyncMultiPut(1, 6, {{"key", "val"}}, [](ResultCode code) { EXPECT_EQ(ResultCode::ERR_PART_NOT_FOUND, code); }); - LOG(INFO) << "Put some data then read them..."; + VLOG(1) << "Put some data then read them..."; + std::string prefix = "prefix"; std::vector data; for (auto i = 0; i < 100; i++) { - data.emplace_back( - prefix + std::string(reinterpret_cast(&i), sizeof(int32_t)), - folly::stringPrintf("val_%d", i)); + data.emplace_back(prefix + std::string(reinterpret_cast(&i), + sizeof(int32_t)), + folly::stringPrintf("val_%d", i)); } - store->asyncMultiPut(1, 1, std::move(data), [](ResultCode code, HostAddr addr){ - UNUSED(addr); + store->asyncMultiPut(1, 1, std::move(data), [](ResultCode code){ EXPECT_EQ(ResultCode::SUCCEEDED, code); }); - int32_t start = 0, end = 100; + + int32_t start = 0; + int32_t end = 100; std::string s(reinterpret_cast(&start), sizeof(int32_t)); std::string e(reinterpret_cast(&end), sizeof(int32_t)); s = prefix + s; @@ -97,8 +106,9 @@ TEST(NebulaStoreTest, SimpleTest) { std::unique_ptr iter; EXPECT_EQ(ResultCode::SUCCEEDED, store->range(1, 1, s, e, &iter)); int num = 0; + auto prefixLen = prefix.size(); while (iter->valid()) { - auto key = *reinterpret_cast(iter->key().data() + prefix.size()); + auto key = *reinterpret_cast(iter->key().data() + prefixLen); auto val = iter->val(); EXPECT_EQ(num, key); EXPECT_EQ(folly::stringPrintf("val_%d", num), val); @@ -110,8 +120,9 @@ TEST(NebulaStoreTest, SimpleTest) { TEST(NebulaStoreTest, PartsTest) { - fs::TempDir rootPath("/tmp/kvstore_test.XXXXXX"); + fs::TempDir rootPath("/tmp/nebula_store_test.XXXXXX"); auto partMan = std::make_unique(); + // GraphSpaceID => {PartitionIDs} // 0 => {0, 1, 2, 3...9} // The parts on PartMan is 0...9 @@ -124,8 +135,9 @@ TEST(NebulaStoreTest, PartsTest) { paths.push_back(folly::stringPrintf("%s/disk2", rootPath.path())); for (size_t i = 0; i < paths.size(); i++) { - auto db = std::make_unique(0, - folly::stringPrintf("%s/nebula/%d/data", paths[i].c_str(), 0)); + auto db = std::make_unique( + 0, /* spaceId */ + folly::stringPrintf("%s/nebula/%d/data", paths[i].c_str(), 0)); for (auto partId = 0; partId < 3; partId++) { db->addPart(5 * i + partId); } @@ -137,12 +149,14 @@ TEST(NebulaStoreTest, PartsTest) { // disk1: 0, 1, 2, 10 // disk2: 5, 6, 7, 15 - std::unique_ptr store; KVOptions options; - options.local_ = HostAddr(0, 0); options.dataPaths_ = std::move(paths); options.partMan_ = std::move(partMan); - store.reset(static_cast(KVStore::instance(std::move(options)))); + HostAddr local = {0, 0}; + auto store = std::make_unique(std::move(options), + ioThreadPool, + workers, + local); auto check = [&](GraphSpaceID spaceId) { // After init, the parts should be 0-9, and the distribution should be @@ -150,9 +164,12 @@ TEST(NebulaStoreTest, PartsTest) { // disk2: 5, 6, 7, x1, y1 // x, y, x1, y1 in {3, 4, 8, 9} for (auto i = 0; i < 2; i++) { - ASSERT_EQ(folly::stringPrintf("%s/disk%d", rootPath.path(), i + 1), - store->kvs_[spaceId]->engines_[i].second); - auto parts = store->kvs_[spaceId]->engines_[i].first->allParts(); + ASSERT_EQ(folly::stringPrintf("%s/disk%d/nebula/%d", + rootPath.path(), + i + 1, + spaceId), + store->spaces_[spaceId]->engines_[i]->getDataRoot()); + auto parts = store->spaces_[spaceId]->engines_[i]->allParts(); dump(parts); ASSERT_EQ(5, parts.size()); } @@ -170,11 +187,12 @@ TEST(NebulaStoreTest, PartsTest) { for (auto partId = 0; partId < 5; partId++) { pm->removePart(0, partId); } + int32_t totalParts = 0; for (auto i = 0; i < 2; i++) { - ASSERT_EQ(folly::stringPrintf("%s/disk%d", rootPath.path(), i + 1), - store->kvs_[0]->engines_[i].second); - auto parts = store->kvs_[0]->engines_[i].first->allParts(); + ASSERT_EQ(folly::stringPrintf("%s/disk%d/nebula/0", rootPath.path(), i + 1), + store->spaces_[0]->engines_[i]->getDataRoot()); + auto parts = store->spaces_[0]->engines_[i]->allParts(); dump(parts); totalParts += parts.size(); } @@ -183,7 +201,7 @@ TEST(NebulaStoreTest, PartsTest) { for (auto partId = 5; partId < 10; partId++) { pm->removePart(0, partId); } - ASSERT_TRUE(store->kvs_.find(0) == store->kvs_.end()); + ASSERT_TRUE(store->spaces_.find(0) == store->spaces_.end()); } } // namespace kvstore @@ -195,6 +213,8 @@ int main(int argc, char** argv) { folly::init(&argc, &argv, true); google::SetStderrLogging(google::INFO); + nebula::kvstore::workers->start(4); + return RUN_ALL_TESTS(); } diff --git a/src/kvstore/test/PartTest.cpp b/src/kvstore/test/PartTest.cpp index 97c2defbb0a..9f7381bc915 100644 --- a/src/kvstore/test/PartTest.cpp +++ b/src/kvstore/test/PartTest.cpp @@ -13,10 +13,6 @@ namespace nebula { namespace kvstore { -TEST(PartTest, SimpleTest) { - LOG(INFO) << "Simple test for shard class..."; -} - TEST(PartTest, RocksTest) { fs::TempDir dataPath("/tmp/rocksdb_test.XXXXXX"); rocksdb::Options options; @@ -28,7 +24,8 @@ TEST(PartTest, RocksTest) { rocksdb::WriteBatch updates; std::vector kvs; for (uint32_t i = 0; i < 1000; i++) { - kvs.emplace_back(folly::stringPrintf("key%d", i), folly::stringPrintf("val%d", i)); + kvs.emplace_back(folly::stringPrintf("key%d", i), + folly::stringPrintf("val%d", i)); } for (auto& kv : kvs) { updates.Put(rocksdb::Slice(kv.first), rocksdb::Slice(kv.second)); @@ -52,6 +49,7 @@ TEST(PartTest, RocksTest) { } // namespace kvstore } // namespace nebula + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); folly::init(&argc, &argv, true); diff --git a/src/kvstore/test/RocksEngineConfigTest.cpp b/src/kvstore/test/RocksEngineConfigTest.cpp index 1dcc180d6f7..6727cb853bb 100644 --- a/src/kvstore/test/RocksEngineConfigTest.cpp +++ b/src/kvstore/test/RocksEngineConfigTest.cpp @@ -11,15 +11,18 @@ #include "rocksdb/convenience.h" #include "rocksdb/utilities/options_util.h" #include "rocksdb/slice_transform.h" -#include #include "fs/TempDir.h" +#include "kvstore/RocksEngine.h" #include "kvstore/RocksEngineConfig.h" +#define KV_DATA_PATH_FORMAT(path, spaceId) \ + folly::stringPrintf("%s/nebula/%d/data", path, spaceId) + namespace nebula { namespace kvstore { -TEST(RocksEngineConfigTest, simpleOptionTest) { - fs::TempDir rootPath("/tmp/kvstore_test.XXXXXX"); +TEST(RocksEngineConfigTest, SimpleOptionTest) { + fs::TempDir rootPath("/tmp/SimpleOptionTest.XXXXXX"); FLAGS_rocksdb_db_options = "stats_dump_period_sec=200;" "enable_write_thread_adaptive_yield=false;" @@ -30,7 +33,7 @@ TEST(RocksEngineConfigTest, simpleOptionTest) { FLAGS_rocksdb_block_based_table_options = "block_restart_interval=2"; // Create the RocksEngine instance - auto engine = std::make_unique(0, KV_DATA_PATH_FORMAT(rootPath.path(), 0)); + auto engine = std::make_unique(0, rootPath.path()); engine.reset(); ASSERT_NE(nebula::fs::FileType::NOTEXIST, @@ -45,7 +48,10 @@ TEST(RocksEngineConfigTest, simpleOptionTest) { rocksdb::Env::Default(), &loadedDbOpt, &loadedCfDescs); - ASSERT_TRUE(s.ok()); + ASSERT_TRUE(s.ok()) + << "Unexpected error happens when loading the option file from \"" + << KV_DATA_PATH_FORMAT(rootPath.path(), 0) + << "\": " << s.ToString(); EXPECT_EQ(200, loadedDbOpt.stats_dump_period_sec); EXPECT_EQ(false, loadedDbOpt.enable_write_thread_adaptive_yield); diff --git a/src/kvstore/test/RocksEngineTest.cpp b/src/kvstore/test/RocksEngineTest.cpp index 11945ecd299..8845fe96e99 100644 --- a/src/kvstore/test/RocksEngineTest.cpp +++ b/src/kvstore/test/RocksEngineTest.cpp @@ -15,17 +15,17 @@ namespace nebula { namespace kvstore { TEST(RocksEngineTest, SimpleTest) { - fs::TempDir rootPath("/tmp/rocksdb_engine_test.XXXXXX"); + fs::TempDir rootPath("/tmp/rocksdb_engine_SimpleTest.XXXXXX"); auto engine = std::make_unique(0, rootPath.path()); EXPECT_EQ(ResultCode::SUCCEEDED, engine->put("key", "val")); std::string val; EXPECT_EQ(ResultCode::SUCCEEDED, engine->get("key", &val)); - EXPECT_EQ(val, "val"); + EXPECT_EQ("val", val); } TEST(RocksEngineTest, RangeTest) { - fs::TempDir rootPath("/tmp/rocksdb_engine_test.XXXXXX"); + fs::TempDir rootPath("/tmp/rocksdb_engine_RangeTest.XXXXXX"); auto engine = std::make_unique(0, rootPath.path()); std::vector data; for (int32_t i = 10; i < 20; i++) { @@ -33,10 +33,15 @@ TEST(RocksEngineTest, RangeTest) { folly::stringPrintf("val_%d", i)); } EXPECT_EQ(ResultCode::SUCCEEDED, engine->multiPut(std::move(data))); - auto checkRange = [&](int32_t start, int32_t end, - int32_t expectedFrom, int32_t expectedTotal) { - LOG(INFO) << "start " << start << ", end " << end - << ", expectedFrom " << expectedFrom << ", expectedTotal " << expectedTotal; + + auto checkRange = [&](int32_t start, + int32_t end, + int32_t expectedFrom, + int32_t expectedTotal) { + VLOG(1) << "start " << start + << ", end " << end + << ", expectedFrom " << expectedFrom + << ", expectedTotal " << expectedTotal; std::string s(reinterpret_cast(&start), sizeof(int32_t)); std::string e(reinterpret_cast(&end), sizeof(int32_t)); std::unique_ptr iter; @@ -63,7 +68,7 @@ TEST(RocksEngineTest, RangeTest) { TEST(RocksEngineTest, PrefixTest) { - fs::TempDir rootPath("/tmp/rocksdb_engine_test.XXXXXX"); + fs::TempDir rootPath("/tmp/rocksdb_engine_PrefixTest.XXXXXX"); auto engine = std::make_unique(0, rootPath.path()); LOG(INFO) << "Write data in batch and scan them..."; std::vector data; @@ -80,10 +85,14 @@ TEST(RocksEngineTest, PrefixTest) { folly::stringPrintf("val_%d", i)); } EXPECT_EQ(ResultCode::SUCCEEDED, engine->multiPut(std::move(data))); + auto checkPrefix = [&](const std::string& prefix, - int32_t expectedFrom, int32_t expectedTotal) { - LOG(INFO) << "prefix " << prefix - << ", expectedFrom " << expectedFrom << ", expectedTotal " << expectedTotal; + int32_t expectedFrom, + int32_t expectedTotal) { + VLOG(1) << "prefix " << prefix + << ", expectedFrom " << expectedFrom + << ", expectedTotal " << expectedTotal; + std::unique_ptr iter; EXPECT_EQ(ResultCode::SUCCEEDED, engine->prefix(prefix, &iter)); int num = 0; @@ -105,35 +114,35 @@ TEST(RocksEngineTest, PrefixTest) { TEST(RocksEngineTest, RemoveTest) { - fs::TempDir rootPath("/tmp/rocksdb_engine_test.XXXXXX"); + fs::TempDir rootPath("/tmp/rocksdb_engine_RemoveTest.XXXXXX"); auto engine = std::make_unique(0, rootPath.path()); EXPECT_EQ(ResultCode::SUCCEEDED, engine->put("key", "val")); std::string val; EXPECT_EQ(ResultCode::SUCCEEDED, engine->get("key", &val)); - EXPECT_EQ(val, "val"); + EXPECT_EQ("val", val); EXPECT_EQ(ResultCode::SUCCEEDED, engine->remove("key")); EXPECT_EQ(ResultCode::ERR_KEY_NOT_FOUND, engine->get("key", &val)); } TEST(RocksEngineTest, RemoveRangeTest) { - fs::TempDir rootPath("/tmp/rocksdb_remove_range_test.XXXXXX"); + fs::TempDir rootPath("/tmp/rocksdb_engine_RemoveRangeTest.XXXXXX"); auto engine = std::make_unique(0, rootPath.path()); for (int32_t i = 0; i < 100; i++) { - EXPECT_EQ(ResultCode::SUCCEEDED, engine->put( - std::string(reinterpret_cast(&i), sizeof(int32_t)), - folly::stringPrintf("%d_val", i))); + std::string key(reinterpret_cast(&i), sizeof(int32_t)); + std::string value(folly::stringPrintf("%d_val", i)); + EXPECT_EQ(ResultCode::SUCCEEDED, engine->put(key, value)); std::string val; - EXPECT_EQ(ResultCode::SUCCEEDED, engine->get( - std::string(reinterpret_cast(&i), sizeof(int32_t)), - &val)); - EXPECT_EQ(val, folly::stringPrintf("%d_val", i)); + EXPECT_EQ(ResultCode::SUCCEEDED, engine->get(key, &val)); + EXPECT_EQ(value, val); } { int32_t s = 0, e = 50; - EXPECT_EQ(ResultCode::SUCCEEDED, engine->removeRange( - std::string(reinterpret_cast(&s), sizeof(int32_t)), - std::string(reinterpret_cast(&e), sizeof(int32_t)))); + EXPECT_EQ( + ResultCode::SUCCEEDED, + engine->removeRange( + std::string(reinterpret_cast(&s), sizeof(int32_t)), + std::string(reinterpret_cast(&e), sizeof(int32_t)))); } { int32_t s = 0, e = 100; @@ -158,7 +167,7 @@ TEST(RocksEngineTest, RemoveRangeTest) { TEST(RocksEngineTest, OptionTest) { - fs::TempDir rootPath("/tmp/rocksdb_option_test.XXXXXX"); + fs::TempDir rootPath("/tmp/rocksdb_engine_OptionTest.XXXXXX"); auto engine = std::make_unique(0, rootPath.path()); EXPECT_EQ(ResultCode::SUCCEEDED, engine->setOption("disable_auto_compactions", "true")); @@ -173,11 +182,12 @@ TEST(RocksEngineTest, OptionTest) { EXPECT_EQ(ResultCode::SUCCEEDED, engine->setDBOption("max_background_compactions", "2_")); EXPECT_EQ(ResultCode::ERR_INVALID_ARGUMENT, - engine->setDBOption("max_background_compactions", "bad_value")); + engine->setDBOption("max_background_compactions", "bad_value")); } + TEST(RocksEngineTest, CompactTest) { - fs::TempDir rootPath("/tmp/rocksdb_compact_test.XXXXXX"); + fs::TempDir rootPath("/tmp/rocksdb_engine_CompactTest.XXXXXX"); auto engine = std::make_unique(0, rootPath.path()); std::vector data; for (int32_t i = 2; i < 8; i++) { diff --git a/src/meta/client/MetaClient.cpp b/src/meta/client/MetaClient.cpp index 6f668eaf585..05a1fce6f35 100644 --- a/src/meta/client/MetaClient.cpp +++ b/src/meta/client/MetaClient.cpp @@ -21,19 +21,24 @@ MetaClient::MetaClient(std::shared_ptr ioThreadPool : ioThreadPool_(ioThreadPool) , addrs_(std::move(addrs)) , sendHeartBeat_(sendHeartBeat) { - CHECK(ioThreadPool_ != nullptr && !addrs_.empty()); - clientsMan_ = std::make_shared>(); + CHECK(ioThreadPool_ != nullptr) << "IOThreadPool is required"; + CHECK(!addrs_.empty()) + << "No meta server address is specified. Meta server is required"; + clientsMan_ = std::make_shared< + thrift::ThriftClientManager + >(); updateHost(); LOG(INFO) << "Create meta client to " << active_; } + MetaClient::~MetaClient() { bgThread_.stop(); bgThread_.wait(); VLOG(3) << "~MetaClient"; } + void MetaClient::init() { loadDataThreadFunc(); CHECK(bgThread_.start()); @@ -53,6 +58,7 @@ void MetaClient::init() { } } + void MetaClient::heartBeatThreadFunc() { if (listener_ == nullptr) { VLOG(1) << "Can't send heartbeat due to listener_ is nullptr!"; @@ -65,6 +71,7 @@ void MetaClient::heartBeatThreadFunc() { } } + void MetaClient::loadDataThreadFunc() { if (ioThreadPool_->numThreads() <= 0) { LOG(ERROR) << "The threads number in ioThreadPool should be greater than 0"; @@ -96,7 +103,8 @@ void MetaClient::loadDataThreadFunc() { spaceCache->spaceName = space.second; spaceCache->partsOnHost_ = reverse(partsAlloc); spaceCache->partsAlloc_ = std::move(partsAlloc); - VLOG(3) << "Load space " << spaceId << ", parts num:" << spaceCache->partsAlloc_.size(); + VLOG(2) << "Load space " << spaceId + << ", parts num:" << spaceCache->partsAlloc_.size(); // loadSchemas if (!loadSchemas(spaceId, @@ -124,6 +132,7 @@ void MetaClient::loadDataThreadFunc() { LOG(INFO) << "Load data completed!"; } + bool MetaClient::loadSchemas(GraphSpaceID spaceId, std::shared_ptr spaceInfoCache, SpaceTagNameIdMap &tagNameIdMap, @@ -192,6 +201,7 @@ bool MetaClient::loadSchemas(GraphSpaceID spaceId, return true; } + std::unordered_map> MetaClient::reverse(const PartsAlloc& parts) { std::unordered_map> hosts; @@ -203,6 +213,7 @@ MetaClient::reverse(const PartsAlloc& parts) { return hosts; } + template> MetaClient::getResponse( return f; } + std::vector MetaClient::to(const std::vector& tHosts) { std::vector hosts; hosts.resize(tHosts.size()); @@ -252,6 +264,7 @@ std::vector MetaClient::to(const std::vector& return hosts; } + std::vector MetaClient::toSpaceIdName(const std::vector& tIdNames) { std::vector idNames; idNames.resize(tIdNames.size()); @@ -261,6 +274,7 @@ std::vector MetaClient::toSpaceIdName(const std::vector Status MetaClient::handleResponse(const RESP& resp) { switch (resp.get_code()) { @@ -283,6 +297,7 @@ Status MetaClient::handleResponse(const RESP& resp) { } } + PartsMap MetaClient::doGetPartsMap(const HostAddr& host, const std::unordered_map< GraphSpaceID, @@ -306,6 +321,7 @@ PartsMap MetaClient::doGetPartsMap(const HostAddr& host, return partMap; } + void MetaClient::diff(const std::unordered_map>& newCache) { if (listener_ == nullptr) { @@ -389,6 +405,7 @@ MetaClient::createSpace(std::string name, int32_t partsNum, int32_t replicaFacto }, true); } + folly::Future>> MetaClient::listSpaces() { cpp2::ListSpacesReq req; return getResponse(std::move(req), [] (auto client, auto request) { @@ -438,6 +455,7 @@ folly::Future> MetaClient::addHosts(const std::vector& }, true); } + folly::Future>> MetaClient::listHosts() { cpp2::ListHostsReq req; return getResponse(std::move(req), [] (auto client, auto request) { @@ -447,6 +465,7 @@ folly::Future>> MetaClient::listHosts() { }); } + folly::Future> MetaClient::removeHosts(const std::vector& hosts) { std::vector thriftHosts; thriftHosts.resize(hosts.size()); @@ -465,6 +484,7 @@ folly::Future> MetaClient::removeHosts(const std::vector>>> MetaClient::getPartsAlloc(GraphSpaceID spaceId) { cpp2::GetPartsAllocReq req; @@ -480,6 +500,7 @@ MetaClient::getPartsAlloc(GraphSpaceID spaceId) { }); } + StatusOr MetaClient::getSpaceIdByNameFromCache(const std::string& name) { folly::RWSpinLock::ReadHolder holder(localCacheLock_); @@ -490,6 +511,7 @@ MetaClient::getSpaceIdByNameFromCache(const std::string& name) { return Status::SpaceNotFound(); } + StatusOr MetaClient::getTagIDByNameFromCache(const GraphSpaceID& space, const std::string& name) { folly::RWSpinLock::ReadHolder holder(localCacheLock_); @@ -500,6 +522,7 @@ StatusOr MetaClient::getTagIDByNameFromCache(const GraphSpaceID& space, return it->second; } + StatusOr MetaClient::getEdgeTypeByNameFromCache(const GraphSpaceID& space, const std::string& name) { folly::RWSpinLock::ReadHolder holder(localCacheLock_); @@ -510,6 +533,7 @@ StatusOr MetaClient::getEdgeTypeByNameFromCache(const GraphSpaceID& sp return it->second; } + folly::Future> MetaClient::multiPut(std::string segment, std::vector> pairs) { @@ -532,6 +556,7 @@ MetaClient::multiPut(std::string segment, }, true); } + folly::Future> MetaClient::get(std::string segment, std::string key) { if (!nebula::meta::MetaCommon::checkSegment(segment) @@ -548,6 +573,7 @@ MetaClient::get(std::string segment, std::string key) { }); } + folly::Future>> MetaClient::multiGet(std::string segment, std::vector keys) { if (!nebula::meta::MetaCommon::checkSegment(segment) @@ -564,6 +590,7 @@ MetaClient::multiGet(std::string segment, std::vector keys) { }); } + folly::Future>> MetaClient::scan(std::string segment, std::string start, std::string end) { if (!nebula::meta::MetaCommon::checkSegment(segment) @@ -581,6 +608,7 @@ MetaClient::scan(std::string segment, std::string start, std::string end) { }); } + folly::Future> MetaClient::remove(std::string segment, std::string key) { if (!nebula::meta::MetaCommon::checkSegment(segment) @@ -597,6 +625,7 @@ MetaClient::remove(std::string segment, std::string key) { }, true); } + folly::Future> MetaClient::removeRange(std::string segment, std::string start, std::string end) { if (!nebula::meta::MetaCommon::checkSegment(segment) @@ -614,11 +643,13 @@ MetaClient::removeRange(std::string segment, std::string start, std::string end) }, true); } + PartsMap MetaClient::getPartsMapFromCache(const HostAddr& host) { folly::RWSpinLock::ReadHolder holder(localCacheLock_); return doGetPartsMap(host, localCache_); } + PartMeta MetaClient::getPartMetaFromCache(GraphSpaceID spaceId, PartitionID partId) { folly::RWSpinLock::ReadHolder holder(localCacheLock_); auto it = localCache_.find(spaceId); @@ -633,6 +664,7 @@ PartMeta MetaClient::getPartMetaFromCache(GraphSpaceID spaceId, PartitionID part return pm; } + bool MetaClient::checkPartExistInCache(const HostAddr& host, GraphSpaceID spaceId, PartitionID partId) { @@ -651,6 +683,7 @@ bool MetaClient::checkPartExistInCache(const HostAddr& host, return false; } + bool MetaClient::checkSpaceExistInCache(const HostAddr& host, GraphSpaceID spaceId) { folly::RWSpinLock::ReadHolder holder(localCacheLock_); @@ -664,6 +697,7 @@ bool MetaClient::checkSpaceExistInCache(const HostAddr& host, return false; } + int32_t MetaClient::partsNum(GraphSpaceID spaceId) { folly::RWSpinLock::ReadHolder holder(localCacheLock_); auto it = localCache_.find(spaceId); @@ -671,6 +705,7 @@ int32_t MetaClient::partsNum(GraphSpaceID spaceId) { return it->second->partsAlloc_.size(); } + folly::Future> MetaClient::createTagSchema(GraphSpaceID spaceId, std::string name, nebula::cpp2::Schema schema) { cpp2::CreateTagReq req; @@ -685,6 +720,7 @@ MetaClient::createTagSchema(GraphSpaceID spaceId, std::string name, nebula::cpp2 }, true); } + folly::Future> MetaClient::alterTagSchema(GraphSpaceID spaceId, std::string name, @@ -701,6 +737,7 @@ MetaClient::alterTagSchema(GraphSpaceID spaceId, }, true); } + folly::Future>> MetaClient::listTagSchemas(GraphSpaceID spaceId) { cpp2::ListTagsReq req; @@ -712,6 +749,7 @@ MetaClient::listTagSchemas(GraphSpaceID spaceId) { }); } + folly::Future> MetaClient::dropTagSchema(int32_t spaceId, std::string tagName) { cpp2::DropTagReq req; @@ -724,6 +762,7 @@ MetaClient::dropTagSchema(int32_t spaceId, std::string tagName) { }, true); } + folly::Future> MetaClient::getTagSchema(int32_t spaceId, int32_t tagId, int64_t version) { cpp2::GetTagReq req; @@ -737,6 +776,7 @@ MetaClient::getTagSchema(int32_t spaceId, int32_t tagId, int64_t version) { }); } + folly::Future> MetaClient::createEdgeSchema(GraphSpaceID spaceId, std::string name, nebula::cpp2::Schema schema) { cpp2::CreateEdgeReq req; @@ -750,6 +790,7 @@ MetaClient::createEdgeSchema(GraphSpaceID spaceId, std::string name, nebula::cpp }, true); } + folly::Future> MetaClient::alterEdgeSchema(GraphSpaceID spaceId, std::string name, @@ -765,6 +806,7 @@ MetaClient::alterEdgeSchema(GraphSpaceID spaceId, }, true); } + folly::Future>> MetaClient::listEdgeSchemas(GraphSpaceID spaceId) { cpp2::ListEdgesReq req; @@ -776,6 +818,7 @@ MetaClient::listEdgeSchemas(GraphSpaceID spaceId) { }); } + folly::Future> MetaClient::getEdgeSchema(GraphSpaceID spaceId, int32_t edgeType, SchemaVer version) { cpp2::GetEdgeReq req; @@ -789,6 +832,7 @@ MetaClient::getEdgeSchema(GraphSpaceID spaceId, int32_t edgeType, SchemaVer vers }); } + folly::Future> MetaClient::dropEdgeSchema(GraphSpaceID spaceId, std::string name) { cpp2::DropEdgeReq req; @@ -801,6 +845,7 @@ MetaClient::dropEdgeSchema(GraphSpaceID spaceId, std::string name) { }, true); } + StatusOr> MetaClient::getTagSchemaFromCache(GraphSpaceID spaceId, TagID tagID, SchemaVer ver) { folly::RWSpinLock::ReadHolder holder(localCacheLock_); @@ -818,6 +863,7 @@ MetaClient::getTagSchemaFromCache(GraphSpaceID spaceId, TagID tagID, SchemaVer v } } + StatusOr> MetaClient::getEdgeSchemaFromCache( GraphSpaceID spaceId, EdgeType edgeType, SchemaVer ver) { folly::RWSpinLock::ReadHolder holder(localCacheLock_); @@ -838,6 +884,7 @@ StatusOr> MetaClient::getEdgeSchemaFromC } } + SchemaVer MetaClient::getNewestTagVerFromCache(const GraphSpaceID& space, const TagID& tagId) { folly::RWSpinLock::ReadHolder holder(localCacheLock_); auto it = spaceNewestTagVerMap_.find(std::make_pair(space, tagId)); @@ -847,6 +894,7 @@ SchemaVer MetaClient::getNewestTagVerFromCache(const GraphSpaceID& space, const return it->second; } + SchemaVer MetaClient::getNewestEdgeVerFromCache(const GraphSpaceID& space, const EdgeType& edgeType) { folly::RWSpinLock::ReadHolder holder(localCacheLock_); @@ -857,6 +905,7 @@ SchemaVer MetaClient::getNewestEdgeVerFromCache(const GraphSpaceID& space, return it->second; } + folly::Future> MetaClient::heartbeat() { CHECK_NOTNULL(listener_); auto localHost = listener_->getLocalHost(); diff --git a/src/meta/processors/BaseProcessor.inl b/src/meta/processors/BaseProcessor.inl index a40c2606fd2..66099e59e05 100644 --- a/src/meta/processors/BaseProcessor.inl +++ b/src/meta/processors/BaseProcessor.inl @@ -12,14 +12,16 @@ namespace meta { template void BaseProcessor::doPut(std::vector data) { - kvstore_->asyncMultiPut(kDefaultSpaceId_, kDefaultPartId_, std::move(data), - [this] (kvstore::ResultCode code, HostAddr leader) { - UNUSED(leader); + kvstore_->asyncMultiPut(kDefaultSpaceId_, + kDefaultPartId_, + std::move(data), + [this] (kvstore::ResultCode code) { this->resp_.set_code(to(code)); this->onFinished(); }); } + template StatusOr> BaseProcessor::doPrefix(const std::string& key) { @@ -31,11 +33,11 @@ BaseProcessor::doPrefix(const std::string& key) { return iter; } + template StatusOr BaseProcessor::doGet(const std::string& key) { std::string value; - auto code = kvstore_->get(kDefaultSpaceId_, kDefaultPartId_, - key, &value); + auto code = kvstore_->get(kDefaultSpaceId_, kDefaultPartId_, key, &value); switch (code) { case kvstore::ResultCode::SUCCEEDED: return value; @@ -46,55 +48,62 @@ StatusOr BaseProcessor::doGet(const std::string& key) { } } + template StatusOr> BaseProcessor::doMultiGet(const std::vector& keys) { std::vector values; - auto code = kvstore_->multiGet(kDefaultSpaceId_, kDefaultPartId_, - keys, &values); + auto code = kvstore_->multiGet(kDefaultSpaceId_, kDefaultPartId_, keys, &values); if (code != kvstore::ResultCode::SUCCEEDED) { return Status::Error("MultiGet Failed"); } return values; } + template void BaseProcessor::doRemove(const std::string& key) { - kvstore_->asyncRemove(kDefaultSpaceId_, kDefaultPartId_, key, - [this] (kvstore::ResultCode code, HostAddr leader) { - UNUSED(leader); + kvstore_->asyncRemove(kDefaultSpaceId_, + kDefaultPartId_, + key, + [this] (kvstore::ResultCode code) { this->resp_.set_code(to(code)); this->onFinished(); }); } + template void BaseProcessor::doMultiRemove(std::vector keys) { - kvstore_->asyncMultiRemove(kDefaultSpaceId_, kDefaultPartId_, std::move(keys), - [this] (kvstore::ResultCode code, HostAddr leader) { - UNUSED(leader); + kvstore_->asyncMultiRemove(kDefaultSpaceId_, + kDefaultPartId_, + std::move(keys), + [this] (kvstore::ResultCode code) { this->resp_.set_code(to(code)); this->onFinished(); }); } + template void BaseProcessor::doRemoveRange(const std::string& start, const std::string& end) { - kvstore_->asyncRemoveRange(kDefaultSpaceId_, kDefaultPartId_, start, end, - [this] (kvstore::ResultCode code, HostAddr leader) { - UNUSED(leader); + kvstore_->asyncRemoveRange(kDefaultSpaceId_, + kDefaultPartId_, + start, + end, + [this] (kvstore::ResultCode code) { this->resp_.set_code(to(code)); this->onFinished(); }); } + template StatusOr> BaseProcessor::doScan(const std::string& start, const std::string& end) { std::unique_ptr iter; - auto code = kvstore_->range(kDefaultSpaceId_, kDefaultPartId_, - start, end, &iter); + auto code = kvstore_->range(kDefaultSpaceId_, kDefaultPartId_, start, end, &iter); if (code != kvstore::ResultCode::SUCCEEDED) { return Status::Error("Scan Failed"); } @@ -107,6 +116,7 @@ StatusOr> BaseProcessor::doScan(const std::string return values; } + template StatusOr> BaseProcessor::allHosts() { std::vector hosts; @@ -116,6 +126,7 @@ StatusOr> BaseProcessor::allHosts() { if (ret != kvstore::ResultCode::SUCCEEDED) { return Status::Error("Can't find any hosts"); } + while (iter->valid()) { nebula::cpp2::HostAddr h; auto hostAddrPiece = iter->key().subpiece(prefix.size()); @@ -126,6 +137,7 @@ StatusOr> BaseProcessor::allHosts() { return hosts; } + template int32_t BaseProcessor::autoIncrementId() { folly::SharedMutex::WriteHolder holder(LockUtils::idLock()); @@ -139,17 +151,20 @@ int32_t BaseProcessor::autoIncrementId() { } else { id = *reinterpret_cast(val.c_str()) + 1; } + std::vector data; data.emplace_back(kIdKey, std::string(reinterpret_cast(&id), sizeof(id))); - kvstore_->asyncMultiPut(kDefaultSpaceId_, kDefaultPartId_, std::move(data), - [] (kvstore::ResultCode code, HostAddr leader) { - UNUSED(leader); + kvstore_->asyncMultiPut(kDefaultSpaceId_, + kDefaultPartId_, + std::move(data), + [this] (kvstore::ResultCode code) { CHECK_EQ(code, kvstore::ResultCode::SUCCEEDED); }); return id; } + template Status BaseProcessor::spaceExist(GraphSpaceID spaceId) { folly::SharedMutex::ReadHolder rHolder(LockUtils::spaceLock()); @@ -162,6 +177,7 @@ Status BaseProcessor::spaceExist(GraphSpaceID spaceId) { return Status::SpaceNotFound(); } + template Status BaseProcessor::hostExist(const std::string& hostKey) { std::string val; @@ -172,6 +188,7 @@ Status BaseProcessor::hostExist(const std::string& hostKey) { return Status::HostNotFound(); } + template StatusOr BaseProcessor::getSpaceId(const std::string& name) { auto indexKey = MetaServiceUtils::indexSpaceKey(name); @@ -183,6 +200,7 @@ StatusOr BaseProcessor::getSpaceId(const std::string& name) return Status::SpaceNotFound(folly::stringPrintf("Space %s not found", name.c_str())); } + template StatusOr BaseProcessor::getTagId(GraphSpaceID spaceId, const std::string& name) { auto indexKey = MetaServiceUtils::indexTagKey(spaceId, name); @@ -194,8 +212,10 @@ StatusOr BaseProcessor::getTagId(GraphSpaceID spaceId, const std::s return Status::TagNotFound(folly::stringPrintf("Tag %s not found", name.c_str())); } + template -StatusOr BaseProcessor::getEdgeType(GraphSpaceID spaceId, const std::string& name) { +StatusOr BaseProcessor::getEdgeType(GraphSpaceID spaceId, + const std::string& name) { auto indexKey = MetaServiceUtils::indexEdgeKey(spaceId, name); auto ret = doGet(indexKey); if (ret.ok()) { diff --git a/src/meta/processors/partsMan/CreateSpaceProcessor.cpp b/src/meta/processors/partsMan/CreateSpaceProcessor.cpp index 9e2694b7c58..639f224581b 100644 --- a/src/meta/processors/partsMan/CreateSpaceProcessor.cpp +++ b/src/meta/processors/partsMan/CreateSpaceProcessor.cpp @@ -30,6 +30,7 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { onFinished(); return; } + auto spaceId = autoIncrementId(); auto spaceName = properties.get_space_name(); auto partitionNum = properties.get_partition_num(); @@ -57,6 +58,7 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { doPut(std::move(data)); } + std::vector CreateSpaceProcessor::pickHosts(PartitionID partId, const std::vector& hosts, diff --git a/src/meta/processors/schemaMan/AlterEdgeProcessor.cpp b/src/meta/processors/schemaMan/AlterEdgeProcessor.cpp index c8b271acbda..44dc1a40948 100644 --- a/src/meta/processors/schemaMan/AlterEdgeProcessor.cpp +++ b/src/meta/processors/schemaMan/AlterEdgeProcessor.cpp @@ -27,7 +27,8 @@ void AlterEdgeProcessor::process(const cpp2::AlterEdgeReq& req) { auto code = kvstore_->prefix(kDefaultSpaceId_, kDefaultPartId_, edgePrefix, &iter); if (code != kvstore::ResultCode::SUCCEEDED || !iter->valid()) { LOG(WARNING) << "Edge could not be found " << req.get_edge_name() - << ", spaceId " << req.get_space_id() << ", edgeType " << edgeType; + << ", spaceId " << req.get_space_id() + << ", edgeType " << edgeType; resp_.set_code(cpp2::ErrorCode::E_NOT_FOUND); onFinished(); return; diff --git a/src/meta/processors/schemaMan/AlterTagProcessor.cpp b/src/meta/processors/schemaMan/AlterTagProcessor.cpp index 3a8efdb8e6d..055f23e65e1 100644 --- a/src/meta/processors/schemaMan/AlterTagProcessor.cpp +++ b/src/meta/processors/schemaMan/AlterTagProcessor.cpp @@ -27,13 +27,14 @@ void AlterTagProcessor::process(const cpp2::AlterTagReq& req) { auto code = kvstore_->prefix(kDefaultSpaceId_, kDefaultPartId_, tagPrefix, &iter); if (code != kvstore::ResultCode::SUCCEEDED || !iter->valid()) { LOG(WARNING) << "Tag could not be found " << req.get_tag_name() - << ", spaceId " << req.get_space_id() << ", tagId " << tagId; + << ", spaceId " << req.get_space_id() + << ", tagId " << tagId; resp_.set_code(cpp2::ErrorCode::E_NOT_FOUND); onFinished(); return; } - // Get lasted version of tag + // Get the last version of the tag auto version = MetaServiceUtils::parseTagVersion(iter->key()) + 1; auto schema = MetaServiceUtils::parseSchema(iter->val()); auto columns = schema.get_columns(); diff --git a/src/meta/processors/schemaMan/GetTagProcessor.cpp b/src/meta/processors/schemaMan/GetTagProcessor.cpp index e09b75d6e81..743d040e022 100644 --- a/src/meta/processors/schemaMan/GetTagProcessor.cpp +++ b/src/meta/processors/schemaMan/GetTagProcessor.cpp @@ -17,14 +17,17 @@ void GetTagProcessor::process(const cpp2::GetTagReq& req) { req.get_version()); auto ret = doGet(std::move(tagKey)); if (!ret.ok()) { - LOG(ERROR) << "Get Tag SpaceID: " << req.get_space_id() << ", tagID: " << req.get_tag_id() - << ", version " << req.get_version() << " not found"; + LOG(ERROR) << "Get Tag SpaceID: " << req.get_space_id() + << ", tagID: " << req.get_tag_id() + << ", version " << req.get_version() << " Not Found"; resp_.set_code(cpp2::ErrorCode::E_NOT_FOUND); onFinished(); return; } - VLOG(3) << "Get Tag SpaceID: " << req.get_space_id() << ", tagID: " << req.get_tag_id() - << ", version " << req.get_version(); + + LOG(INFO) << "Get Tag SpaceID: " << req.get_space_id() + << ", tagID: " << req.get_tag_id() + << ", version " << req.get_version(); resp_.set_code(cpp2::ErrorCode::SUCCEEDED); resp_.set_schema(MetaServiceUtils::parseSchema(ret.value())); onFinished(); diff --git a/src/meta/test/CMakeLists.txt b/src/meta/test/CMakeLists.txt index 6cf3965f648..7359c31e9a8 100644 --- a/src/meta/test/CMakeLists.txt +++ b/src/meta/test/CMakeLists.txt @@ -26,8 +26,13 @@ add_executable( $ $ $ - $ + $ + $ + $ $ + $ + $ + $ $ $ $ @@ -52,9 +57,13 @@ add_executable( $ $ $ + $ + $ + $ $ $ $ + $ $ $ $ @@ -68,6 +77,7 @@ nebula_link_libraries( ) nebula_add_test(hb_processor_test) + add_executable( meta_client_test MetaClientTest.cpp @@ -76,8 +86,11 @@ add_executable( $ $ $ - $ + $ + $ + $ $ + $ $ $ $ @@ -121,10 +134,14 @@ add_executable( $ $ $ + $ + $ + $ $ $ $ $ + $ $ $ $ diff --git a/src/meta/test/HBProcessorTest.cpp b/src/meta/test/HBProcessorTest.cpp index f9e66956522..75f47640f86 100644 --- a/src/meta/test/HBProcessorTest.cpp +++ b/src/meta/test/HBProcessorTest.cpp @@ -24,7 +24,8 @@ using apache::thrift::FragileConstructor::FRAGILE; TEST(HBProcessorTest, HBTest) { fs::TempDir rootPath("/tmp/HBTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + auto kv = TestUtils::initKV(rootPath.path()); + { std::vector thriftHosts; for (auto i = 0; i < 10; i++) { @@ -83,6 +84,7 @@ TEST(HBProcessorTest, HBTest) { } // namespace meta } // namespace nebula + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); folly::init(&argc, &argv, true); diff --git a/src/meta/test/ProcessorTest.cpp b/src/meta/test/ProcessorTest.cpp index 10f8ae2c279..9e35d04a1c7 100644 --- a/src/meta/test/ProcessorTest.cpp +++ b/src/meta/test/ProcessorTest.cpp @@ -42,7 +42,8 @@ using apache::thrift::FragileConstructor::FRAGILE; TEST(ProcessorTest, AddHostsTest) { fs::TempDir rootPath("/tmp/AddHostsTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + auto kv = TestUtils::initKV(rootPath.path()); + { std::vector thriftHosts; for (auto i = 0; i < 10; i++) { @@ -116,10 +117,12 @@ TEST(ProcessorTest, AddHostsTest) { } } + TEST(ProcessorTest, CreateSpaceTest) { fs::TempDir rootPath("/tmp/CreateSpaceTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + auto kv = TestUtils::initKV(rootPath.path()); auto hostsNum = TestUtils::createSomeHosts(kv.get()); + { cpp2::SpaceProperties properties; properties.set_space_name("default_space"); @@ -186,10 +189,12 @@ TEST(ProcessorTest, CreateSpaceTest) { } } + TEST(ProcessorTest, CreateTagTest) { fs::TempDir rootPath("/tmp/CreateTagTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + auto kv = TestUtils::initKV(rootPath.path()); TestUtils::createSomeHosts(kv.get()); + { cpp2::SpaceProperties properties; properties.set_space_name("default_space"); @@ -237,10 +242,12 @@ TEST(ProcessorTest, CreateTagTest) { } } + TEST(ProcessorTest, CreateEdgeTest) { fs::TempDir rootPath("/tmp/CreateEdgeTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + auto kv = TestUtils::initKV(rootPath.path()); TestUtils::createSomeHosts(kv.get()); + { cpp2::SpaceProperties properties; properties.set_space_name("default_space"); @@ -330,9 +337,10 @@ TEST(ProcessorTest, CreateEdgeTest) { } } + TEST(ProcessorTest, KVOperationTest) { fs::TempDir rootPath("/tmp/KVOperationTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + auto kv = TestUtils::initKV(rootPath.path()); auto hostsNum = TestUtils::createSomeHosts(kv.get()); UNUSED(hostsNum); @@ -469,9 +477,10 @@ TEST(ProcessorTest, KVOperationTest) { } } + TEST(ProcessorTest, ListOrGetTagsTest) { - fs::TempDir rootPath("/tmp/ListTagsTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + fs::TempDir rootPath("/tmp/ListOrGetTagsTest.XXXXXX"); + auto kv = TestUtils::initKV(rootPath.path()); ASSERT_TRUE(TestUtils::assembleSpace(kv.get(), 1)); TestUtils::mockTag(kv.get(), 10); @@ -520,9 +529,10 @@ TEST(ProcessorTest, ListOrGetTagsTest) { } } + TEST(ProcessorTest, ListOrGetEdgesTest) { - fs::TempDir rootPath("/tmp/ListEdgesTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + fs::TempDir rootPath("/tmp/ListOrGetEdgesTest.XXXXXX"); + auto kv = TestUtils::initKV(rootPath.path()); ASSERT_TRUE(TestUtils::assembleSpace(kv.get(), 1)); TestUtils::mockEdge(kv.get(), 10); @@ -573,42 +583,47 @@ TEST(ProcessorTest, ListOrGetEdgesTest) { } } + TEST(ProcessorTest, DropTagTest) { - fs::TempDir rootPath("/tmp/DropTagTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); - ASSERT_TRUE(TestUtils::assembleSpace(kv.get(), 1)); - TestUtils::mockTag(kv.get(), 1); - { - // remove tag processor test - cpp2::DropTagReq req; - req.set_space_id(1); - req.set_tag_name("tag_0"); - auto* processor = DropTagProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(cpp2::ErrorCode::SUCCEEDED, resp.get_code()); - } - { - // check tag data has been deleted. - std::string tagVal; - kvstore::ResultCode ret; - std::unique_ptr iter; - ret = kv.get()->get(0, 0, std::move(MetaServiceUtils::indexTagKey(1, "tag_1")), - &tagVal); - ASSERT_EQ(kvstore::ResultCode::ERR_KEY_NOT_FOUND, ret); - std::string tagPrefix = "__tags__"; - ret = kv.get()->prefix(0, 0, tagPrefix, &iter); - ASSERT_EQ(kvstore::ResultCode::SUCCEEDED, ret); - ASSERT_FALSE(iter->valid()); - } + fs::TempDir rootPath("/tmp/DropTagTest.XXXXXX"); + auto kv = TestUtils::initKV(rootPath.path()); + ASSERT_TRUE(TestUtils::assembleSpace(kv.get(), 1)); + TestUtils::mockTag(kv.get(), 1); + + // remove tag processor test + { + cpp2::DropTagReq req; + req.set_space_id(1); + req.set_tag_name("tag_0"); + auto* processor = DropTagProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } + + // check tag data has been deleted. + { + std::string tagVal; + kvstore::ResultCode ret; + std::unique_ptr iter; + ret = kv.get()->get(0, 0, std::move(MetaServiceUtils::indexTagKey(1, "tag_1")), + &tagVal); + ASSERT_EQ(kvstore::ResultCode::ERR_KEY_NOT_FOUND, ret); + std::string tagPrefix = "__tags__"; + ret = kv.get()->prefix(0, 0, tagPrefix, &iter); + ASSERT_EQ(kvstore::ResultCode::SUCCEEDED, ret); + ASSERT_FALSE(iter->valid()); + } } + TEST(ProcessorTest, DropEdgeTest) { fs::TempDir rootPath("/tmp/DropEdgeTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + auto kv = TestUtils::initKV(rootPath.path()); ASSERT_TRUE(TestUtils::assembleSpace(kv.get(), 1)); TestUtils::mockEdge(kv.get(), 1); + // Space not exist { cpp2::DropEdgeReq req; @@ -658,11 +673,13 @@ TEST(ProcessorTest, DropEdgeTest) { } } + TEST(ProcessorTest, AlterTagTest) { fs::TempDir rootPath("/tmp/AlterTagTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + auto kv = TestUtils::initKV(rootPath.path()); ASSERT_TRUE(TestUtils::assembleSpace(kv.get(), 1)); TestUtils::mockTag(kv.get(), 1); + // Alter tag processor test { cpp2::AlterTagReq req; @@ -798,11 +815,13 @@ TEST(ProcessorTest, AlterTagTest) { } } + TEST(ProcessorTest, AlterEdgeTest) { fs::TempDir rootPath("/tmp/AlterEdgeTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + auto kv = TestUtils::initKV(rootPath.path()); ASSERT_TRUE(TestUtils::assembleSpace(kv.get(), 1)); TestUtils::mockEdge(kv.get(), 1); + // Drop all, then add { cpp2::AlterEdgeReq req; @@ -1010,10 +1029,12 @@ TEST(ProcessorTest, AlterEdgeTest) { } } + TEST(ProcessorTest, SameNameTagsTest) { - fs::TempDir rootPath("/tmp/CreateSpaceTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + fs::TempDir rootPath("/tmp/SameNameTagsTest.XXXXXX"); + auto kv = TestUtils::initKV(rootPath.path()); TestUtils::createSomeHosts(kv.get()); + { cpp2::SpaceProperties properties; properties.set_space_name("default_space"); @@ -1116,9 +1137,11 @@ TEST(ProcessorTest, SameNameTagsTest) { ASSERT_EQ("default_tag", tags[0].get_tag_name()); } } + } // namespace meta } // namespace nebula + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); folly::init(&argc, &argv, true); diff --git a/src/meta/test/TestUtils.h b/src/meta/test/TestUtils.h index 47ec58437f3..8dd3a7b5ed8 100644 --- a/src/meta/test/TestUtils.h +++ b/src/meta/test/TestUtils.h @@ -29,8 +29,12 @@ using nebula::cpp2::SupportedType; class TestUtils { public: - static kvstore::KVStore* initKV(const char* rootPath) { + static std::unique_ptr initKV(const char* rootPath) { + auto workers = std::make_shared(); + workers->start(4); + auto ioPool = std::make_shared(4); auto partMan = std::make_unique(); + // GraphSpaceID => {PartitionIDs} // 0 => {0} auto& partsMap = partMan->partsMap(); @@ -40,13 +44,16 @@ class TestUtils { paths.push_back(folly::stringPrintf("%s/disk1", rootPath)); kvstore::KVOptions options; - options.local_ = HostAddr(0, 0); options.dataPaths_ = std::move(paths); options.partMan_ = std::move(partMan); - - kvstore::NebulaStore* kv = static_cast( - kvstore::KVStore::instance(std::move(options))); - return kv; + HostAddr localhost = HostAddr(0, 0); + + auto store = std::make_unique(std::move(options), + ioPool, + workers, + localhost); + sleep(1); + return std::move(store); } static nebula::cpp2::ColumnDef columnDef(int32_t index, nebula::cpp2::SupportedType st) { @@ -107,10 +114,9 @@ class TestUtils { std::vector data; data.emplace_back(MetaServiceUtils::spaceKey(id), "test_space"); kv->asyncMultiPut(0, 0, std::move(data), - [&] (kvstore::ResultCode code, HostAddr leader) { - ret = (code == kvstore::ResultCode::SUCCEEDED); - UNUSED(leader); - }); + [&] (kvstore::ResultCode code) { + ret = (code == kvstore::ResultCode::SUCCEEDED); + }); return ret; } @@ -134,10 +140,9 @@ class TestUtils { } kv->asyncMultiPut(0, 0, std::move(tags), - [] (kvstore::ResultCode code, HostAddr leader) { - ASSERT_EQ(kvstore::ResultCode::SUCCEEDED, code); - UNUSED(leader); - }); + [] (kvstore::ResultCode code) { + ASSERT_EQ(kvstore::ResultCode::SUCCEEDED, code); + }); } static void mockEdge(kvstore::KVStore* kv, int32_t edgeNum, SchemaVer version = 0) { @@ -160,21 +165,23 @@ class TestUtils { MetaServiceUtils::schemaEdgeVal(edgeName, srcsch)); } - kv->asyncMultiPut(0, 0, std::move(edges), - [] (kvstore::ResultCode code, HostAddr leader) { + kv->asyncMultiPut(0, 0, std::move(edges), [] (kvstore::ResultCode code) { ASSERT_EQ(kvstore::ResultCode::SUCCEEDED, code); - UNUSED(leader); }); } static std::unique_ptr mockMetaServer(uint16_t port, const char* dataPath) { + LOG(INFO) << "Initializing KVStore at \"" << dataPath << "\""; + auto sc = std::make_unique(); - sc->KVStore_ = std::unique_ptr(TestUtils::initKV(dataPath)); - auto handler = std::make_shared(sc->KVStore_.get()); - test::mockCommon(sc.get(), "meta", port, handler); - LOG(INFO) << "Starting the Meta Daemon on port " << sc->port_ - << ", path " << dataPath; + sc->kvStore_ = TestUtils::initKV(dataPath); + + auto handler = std::make_shared(sc->kvStore_.get()); + sc->mockCommon("meta", port, handler); + LOG(INFO) << "The Meta Daemon started on port " << sc->port_ + << ", data path is at \"" << dataPath << "\""; + return sc; } }; diff --git a/src/raftex/RaftPart.cpp b/src/raftex/RaftPart.cpp index 9ef1d090604..c95463b281c 100644 --- a/src/raftex/RaftPart.cpp +++ b/src/raftex/RaftPart.cpp @@ -284,7 +284,7 @@ void RaftPart::stop() { } -typename RaftPart::AppendLogResult RaftPart::canAppendLogs( +AppendLogResult RaftPart::canAppendLogs( std::lock_guard& lck) { UNUSED(lck); if (status_ == Status::STARTING) { @@ -304,7 +304,7 @@ typename RaftPart::AppendLogResult RaftPart::canAppendLogs( } -folly::Future RaftPart::appendAsync(ClusterID source, +folly::Future RaftPart::appendAsync(ClusterID source, std::string log) { if (source < 0) { source = clusterId_; @@ -313,17 +313,17 @@ folly::Future RaftPart::appendAsync(ClusterID source, } -folly::Future RaftPart::casAsync(std::string log) { +folly::Future RaftPart::casAsync(std::string log) { return appendLogAsync(clusterId_, true, std::move(log)); } -folly::Future RaftPart::appendLogAsync(ClusterID source, +folly::Future RaftPart::appendLogAsync(ClusterID source, bool isCAS, std::string log) { LogCache swappedOutLogs; LogID firstId; - auto retFuture = folly::Future::makeEmpty(); + auto retFuture = folly::Future::makeEmpty(); { std::lock_guard lck(raftLock_); @@ -599,6 +599,7 @@ void RaftPart::processAppendLogResponses( if (iter.leadByCAS()) { sendingPromise_.setOneSingleValue(AppendLogResult::SUCCEEDED); } + VLOG(2) << idStr_ << "Succeeded in committing the logs"; // Step 5: Check whether need to continue // the log replication @@ -625,6 +626,7 @@ void RaftPart::processAppendLogResponses( logs_.clear(); } else { replicatingLogs_ = false; + VLOG(2) << idStr_ << "No more log to be replicated"; } } } else { @@ -1175,7 +1177,7 @@ cpp2::ErrorCode RaftPart::verifyLeader( } -folly::Future RaftPart::sendHeartbeat() { +folly::Future RaftPart::sendHeartbeat() { using namespace folly; // NOLINT since the fancy overload of | operator VLOG(2) << idStr_ << "Sending heartbeat to all other hosts"; diff --git a/src/raftex/RaftPart.h b/src/raftex/RaftPart.h index 2fe5f0e755c..7def7474e18 100644 --- a/src/raftex/RaftPart.h +++ b/src/raftex/RaftPart.h @@ -29,23 +29,23 @@ class BufferFlusher; namespace raftex { +enum class AppendLogResult { + SUCCEEDED = 0, + E_CAS_FAILURE = -1, + E_NOT_A_LEADER = -2, + E_STOPPED = -3, + E_NOT_READY = -4, + E_BUFFER_OVERFLOW = -5, + E_WAL_FAILURE = -6, +}; + class Host; class AppendLogsIterator; + class RaftPart : public std::enable_shared_from_this { friend class AppendLogsIterator; public: - enum class AppendLogResult { - SUCCEEDED = 0, - E_CAS_FAILURE = -1, - E_NOT_A_LEADER = -2, - E_STOPPED = -3, - E_NOT_READY = -4, - E_BUFFER_OVERFLOW = -5, - E_WAL_FAILURE = -6, - }; - - virtual ~RaftPart(); bool isRunning() const { @@ -156,11 +156,9 @@ class RaftPart : public std::enable_shared_from_this { // The method will be invoked by start() // - // Inherited classes can implement this method to provide the last + // Inherited classes should implement this method to provide the last // committed log id - virtual LogID lastCommittedLogId() { - return 0; - } + virtual LogID lastCommittedLogId() = 0; // This method is called when this partition's leader term // is finished, either by receiving a new leader election diff --git a/src/raftex/test/LogAppendTest.cpp b/src/raftex/test/LogAppendTest.cpp index d7514414c7d..247b7303ba0 100644 --- a/src/raftex/test/LogAppendTest.cpp +++ b/src/raftex/test/LogAppendTest.cpp @@ -43,8 +43,7 @@ TEST(LogAppend, SimpleAppendWithOneCopy) { msgs.emplace_back( folly::stringPrintf("Test Log Message %03d", i)); auto fut = leader->appendAsync(0, msgs.back()); - ASSERT_EQ(RaftPart::AppendLogResult::SUCCEEDED, - std::move(fut).get()); + ASSERT_EQ(AppendLogResult::SUCCEEDED, std::move(fut).get()); } LOG(INFO) << "<===== Finish appending logs"; @@ -91,8 +90,7 @@ TEST(LogAppend, SimpleAppendWithThreeCopies) { msgs.emplace_back( folly::stringPrintf("Test Log Message %03d", i)); auto fut = leader->appendAsync(0, msgs.back()); - ASSERT_EQ(RaftPart::AppendLogResult::SUCCEEDED, - std::move(fut).get()); + ASSERT_EQ(AppendLogResult::SUCCEEDED, std::move(fut).get()); } LOG(INFO) << "<===== Finish appending logs"; @@ -144,14 +142,13 @@ TEST(LogAppend, MultiThreadAppend) { auto fut = leader->appendAsync( 0, folly::stringPrintf("Log %03d for t%d", j, i)); if (fut.isReady() && - fut.value() == RaftPart::AppendLogResult::E_BUFFER_OVERFLOW) { + fut.value() == AppendLogResult::E_BUFFER_OVERFLOW) { // Buffer overflow, while a little usleep(5000); continue; } else if (j == numLogs) { // Only wait on the last log messaage - ASSERT_EQ(RaftPart::AppendLogResult::SUCCEEDED, - std::move(fut).get()); + ASSERT_EQ(AppendLogResult::SUCCEEDED, std::move(fut).get()); } break; } while (true); diff --git a/src/raftex/test/TestShard.h b/src/raftex/test/TestShard.h index 4f7bf622581..821442d9f26 100644 --- a/src/raftex/test/TestShard.h +++ b/src/raftex/test/TestShard.h @@ -33,6 +33,10 @@ class TestShard : public RaftPart { std::function becomeLeaderCB); + LogID lastCommittedLogId() override { + return 0; + } + std::shared_ptr getService() const { return service_; } diff --git a/src/storage/BaseProcessor.inl b/src/storage/BaseProcessor.inl index e07bb818408..75cf9abbbbb 100644 --- a/src/storage/BaseProcessor.inl +++ b/src/storage/BaseProcessor.inl @@ -15,7 +15,7 @@ cpp2::ErrorCode BaseProcessor::to(kvstore::ResultCode code) { switch (code) { case kvstore::ResultCode::SUCCEEDED: return cpp2::ErrorCode::SUCCEEDED; - case kvstore::ResultCode::ERR_LEADER_CHANAGED: + case kvstore::ResultCode::ERR_LEADER_CHANGED: return cpp2::ErrorCode::E_LEADER_CHANGED; case kvstore::ResultCode::ERR_SPACE_NOT_FOUND: return cpp2::ErrorCode::E_SPACE_NOT_FOUND; @@ -26,17 +26,22 @@ cpp2::ErrorCode BaseProcessor::to(kvstore::ResultCode code) { } } + template void BaseProcessor::doPut(GraphSpaceID spaceId, PartitionID partId, std::vector data) { - this->kvstore_->asyncMultiPut(spaceId, partId, std::move(data), - [partId, this](kvstore::ResultCode code, HostAddr addr) { + this->kvstore_->asyncMultiPut(spaceId, + partId, + std::move(data), + [spaceId, partId, this](kvstore::ResultCode code) { VLOG(3) << "partId:" << partId << ", code:" << static_cast(code); + cpp2::ResultCode thriftResult; thriftResult.set_code(to(code)); thriftResult.set_part_id(partId); - if (code == kvstore::ResultCode::ERR_LEADER_CHANAGED) { + if (code == kvstore::ResultCode::ERR_LEADER_CHANGED) { + auto addr = kvstore_->partLeader(spaceId, partId); thriftResult.get_leader()->set_ip(addr.first); thriftResult.get_leader()->set_port(addr.second); } @@ -58,6 +63,5 @@ void BaseProcessor::doPut(GraphSpaceID spaceId, }); } - } // namespace storage } // namespace nebula diff --git a/src/storage/QueryStatsProcessor.cpp b/src/storage/QueryStatsProcessor.cpp index 02fb50a51d8..fffda80fcc1 100644 --- a/src/storage/QueryStatsProcessor.cpp +++ b/src/storage/QueryStatsProcessor.cpp @@ -65,6 +65,7 @@ void QueryStatsProcessor::calcResult(std::vector&& props) { resp_.set_data(writer.encode()); } + kvstore::ResultCode QueryStatsProcessor::processVertex(PartitionID partId, VertexID vId, std::vector& tagContexts, diff --git a/src/storage/test/AddEdgesTest.cpp b/src/storage/test/AddEdgesTest.cpp index 49160bc5242..ad11a22cfa1 100644 --- a/src/storage/test/AddEdgesTest.cpp +++ b/src/storage/test/AddEdgesTest.cpp @@ -17,8 +17,9 @@ namespace storage { TEST(AddEdgesTest, SimpleTest) { fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + std::unique_ptr kv = TestUtils::initKV(rootPath.path()); auto* processor = AddEdgesProcessor::instance(kv.get(), nullptr); + LOG(INFO) << "Build AddEdgesRequest..."; cpp2::AddEdgesRequest req; req.space_id = 0; diff --git a/src/storage/test/AddVerticesTest.cpp b/src/storage/test/AddVerticesTest.cpp index a9c8b7f1c89..f0aa517f730 100644 --- a/src/storage/test/AddVerticesTest.cpp +++ b/src/storage/test/AddVerticesTest.cpp @@ -17,8 +17,9 @@ namespace storage { TEST(AddVerticesTest, SimpleTest) { fs::TempDir rootPath("/tmp/AddVerticesTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + std::unique_ptr kv = TestUtils::initKV(rootPath.path()); auto* processor = AddVerticesProcessor::instance(kv.get(), nullptr); + LOG(INFO) << "Build AddVerticesRequest..."; cpp2::AddVerticesRequest req; req.space_id = 0; diff --git a/src/storage/test/CMakeLists.txt b/src/storage/test/CMakeLists.txt index 483cd61b314..7ece1969088 100644 --- a/src/storage/test/CMakeLists.txt +++ b/src/storage/test/CMakeLists.txt @@ -1,26 +1,39 @@ -add_library( - adHocSchema_obj OBJECT - AdHocSchemaManager.cpp -) -add_dependencies(adHocSchema_obj schema_obj base_obj) - -add_executable( - add_vertices_test - AddVerticesTest.cpp +set(storage_test_deps $ + $ $ $ $ - $ $ - $ + $ + $ + $ $ $ - $ $ + $ $ $ $ + $ +) + + +add_library( + adHocSchema_obj OBJECT + AdHocSchemaManager.cpp +) +add_dependencies( + adHocSchema_obj + schema_obj + base_obj +) + + +add_executable( + add_vertices_test + AddVerticesTest.cpp + ${storage_test_deps} ) nebula_link_libraries( add_vertices_test @@ -35,20 +48,7 @@ nebula_add_test(add_vertices_test) add_executable( storage_service_handler_test StorageServiceHandlerTest.cpp - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ + ${storage_test_deps} ) nebula_link_libraries( storage_service_handler_test @@ -63,20 +63,7 @@ nebula_add_test(storage_service_handler_test) add_executable( add_edges_test AddEdgesTest.cpp - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ + ${storage_test_deps} ) nebula_link_libraries( add_edges_test @@ -91,21 +78,8 @@ nebula_add_test(add_edges_test) add_executable( query_bound_test QueryBoundTest.cpp - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ $ + ${storage_test_deps} ) nebula_link_libraries( query_bound_test @@ -120,21 +94,8 @@ nebula_add_test(query_bound_test) add_executable( vertex_props_test QueryVertexPropsTest.cpp - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ $ + ${storage_test_deps} ) nebula_link_libraries( vertex_props_test @@ -149,21 +110,8 @@ nebula_add_test(vertex_props_test) add_executable( edge_props_test QueryEdgePropsTest.cpp - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ $ + ${storage_test_deps} ) nebula_link_libraries( edge_props_test @@ -178,21 +126,8 @@ nebula_add_test(edge_props_test) add_executable( query_stats_test QueryStatsTest.cpp - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ $ + ${storage_test_deps} ) nebula_link_libraries( query_stats_test @@ -208,22 +143,9 @@ add_executable( storage_client_test StorageClientTest.cpp $ - $ - $ - $ $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ $ + ${storage_test_deps} ) nebula_link_libraries( storage_client_test @@ -240,16 +162,13 @@ add_executable( StorageHttpHandlerTest.cpp $ $ - $ - $ - $ $ - $ - $ $ + $ + $ + ${storage_test_deps} ) - - nebula_link_libraries( +nebula_link_libraries( storage_http_test proxygenhttpserver proxygenlib diff --git a/src/storage/test/QueryBoundTest.cpp b/src/storage/test/QueryBoundTest.cpp index 007c34f099d..ac02a410dba 100644 --- a/src/storage/test/QueryBoundTest.cpp +++ b/src/storage/test/QueryBoundTest.cpp @@ -66,13 +66,13 @@ void mockData(kvstore::KVStore* kv) { } kv->asyncMultiPut( 0, partId, std::move(data), - [&](kvstore::ResultCode code, HostAddr addr) { + [&](kvstore::ResultCode code) { EXPECT_EQ(code, kvstore::ResultCode::SUCCEEDED); - UNUSED(addr); }); } } + void buildRequest(cpp2::GetNeighborsRequest& req, bool outBound = true) { req.set_space_id(0); decltype(req.parts) tmpIds; @@ -104,6 +104,7 @@ void buildRequest(cpp2::GetNeighborsRequest& req, bool outBound = true) { req.set_return_columns(std::move(tmpColumns)); } + void checkResponse(cpp2::QueryResponse& resp, bool outBound = true) { int32_t edgeFields = outBound ? 12 : 2; int32_t dstIdFrom = outBound ? 10001 : 20001; @@ -171,11 +172,13 @@ void checkResponse(cpp2::QueryResponse& resp, bool outBound = true) { } } + TEST(QueryBoundTest, OutBoundSimpleTest) { fs::TempDir rootPath("/tmp/QueryBoundTest.XXXXXX"); + std::unique_ptr kv = TestUtils::initKV(rootPath.path()); + LOG(INFO) << "Prepare meta..."; - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); - auto schemaMan = TestUtils::mockSchemaMan();; + auto schemaMan = TestUtils::mockSchemaMan(); mockData(kv.get()); cpp2::GetNeighborsRequest req; @@ -191,10 +194,12 @@ TEST(QueryBoundTest, OutBoundSimpleTest) { checkResponse(resp); } + TEST(QueryBoundTest, inBoundSimpleTest) { fs::TempDir rootPath("/tmp/QueryBoundTest.XXXXXX"); LOG(INFO) << "Prepare meta..."; - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + std::unique_ptr kv = TestUtils::initKV(rootPath.path()); + auto schemaMan = TestUtils::mockSchemaMan(); mockData(kv.get()); diff --git a/src/storage/test/QueryEdgePropsTest.cpp b/src/storage/test/QueryEdgePropsTest.cpp index 5db1e561a92..40050a44fc0 100644 --- a/src/storage/test/QueryEdgePropsTest.cpp +++ b/src/storage/test/QueryEdgePropsTest.cpp @@ -38,13 +38,13 @@ void mockData(kvstore::KVStore* kv) { } kv->asyncMultiPut( 0, partId, std::move(data), - [&](kvstore::ResultCode code, HostAddr addr) { + [&](kvstore::ResultCode code) { EXPECT_EQ(code, kvstore::ResultCode::SUCCEEDED); - UNUSED(addr); }); } } + void buildRequest(cpp2::EdgePropRequest& req) { req.set_space_id(0); decltype(req.parts) tmpEdges; @@ -67,6 +67,7 @@ void buildRequest(cpp2::EdgePropRequest& req) { req.set_return_columns(std::move(tmpColumns)); } + void checkResponse(cpp2::EdgePropResponse& resp) { EXPECT_EQ(0, resp.result.failed_codes.size()); EXPECT_EQ(13, resp.schema.columns.size()); @@ -116,9 +117,11 @@ void checkResponse(cpp2::EdgePropResponse& resp) { EXPECT_EQ(210, rowNum); } + TEST(QueryEdgePropsTest, SimpleTest) { fs::TempDir rootPath("/tmp/QueryEdgePropsTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + std::unique_ptr kv = TestUtils::initKV(rootPath.path()); + LOG(INFO) << "Prepare meta..."; auto schemaMan = TestUtils::mockSchemaMan(); LOG(INFO) << "Prepare data..."; diff --git a/src/storage/test/QueryStatsTest.cpp b/src/storage/test/QueryStatsTest.cpp index ab92195a1cd..2ff9d8e7be5 100644 --- a/src/storage/test/QueryStatsTest.cpp +++ b/src/storage/test/QueryStatsTest.cpp @@ -48,14 +48,13 @@ void mockData(kvstore::KVStore* kv) { data.emplace_back(std::move(key), std::move(val)); } } - kv->asyncMultiPut(0, partId, std::move(data), - [&](kvstore::ResultCode code, HostAddr addr) { + kv->asyncMultiPut(0, partId, std::move(data), [&](kvstore::ResultCode code) { EXPECT_EQ(code, kvstore::ResultCode::SUCCEEDED); - UNUSED(addr); }); } } + void buildRequest(cpp2::GetNeighborsRequest& req) { req.set_space_id(0); decltype(req.parts) tmpIds; @@ -84,6 +83,7 @@ void buildRequest(cpp2::GetNeighborsRequest& req) { req.set_return_columns(std::move(tmpColumns)); } + void checkResponse(const cpp2::QueryStatsResponse& resp) { EXPECT_EQ(0, resp.result.failed_codes.size()); @@ -131,9 +131,11 @@ void checkResponse(const cpp2::QueryStatsResponse& resp) { } } + TEST(QueryStatsTest, StatsSimpleTest) { fs::TempDir rootPath("/tmp/QueryStatsTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + std::unique_ptr kv = TestUtils::initKV(rootPath.path()); + LOG(INFO) << "Prepare meta..."; auto schemaMan = TestUtils::mockSchemaMan(); mockData(kv.get()); @@ -157,6 +159,7 @@ int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); folly::init(&argc, &argv, true); google::SetStderrLogging(google::INFO); + return RUN_ALL_TESTS(); } diff --git a/src/storage/test/QueryVertexPropsTest.cpp b/src/storage/test/QueryVertexPropsTest.cpp index 1a784236ef1..4c8cc0b5796 100644 --- a/src/storage/test/QueryVertexPropsTest.cpp +++ b/src/storage/test/QueryVertexPropsTest.cpp @@ -19,7 +19,8 @@ namespace storage { TEST(QueryVertexPropsTest, SimpleTest) { fs::TempDir rootPath("/tmp/QueryVertexPropsTest.XXXXXX"); - std::unique_ptr kv(TestUtils::initKV(rootPath.path())); + std::unique_ptr kv = TestUtils::initKV(rootPath.path()); + LOG(INFO) << "Prepare meta..."; auto schemaMan = TestUtils::mockSchemaMan(); @@ -44,9 +45,8 @@ TEST(QueryVertexPropsTest, SimpleTest) { 0, partId, std::move(data), - [&](kvstore::ResultCode code, HostAddr addr) { + [&](kvstore::ResultCode code) { EXPECT_EQ(code, kvstore::ResultCode::SUCCEEDED); - UNUSED(addr); }); } @@ -65,10 +65,10 @@ TEST(QueryVertexPropsTest, SimpleTest) { // Return tag props col_0, col_2, col_4 decltype(req.return_columns) tmpColumns; for (int i = 0; i < 3; i++) { - tmpColumns.emplace_back(TestUtils::propDef(cpp2::PropOwner::SOURCE, - folly::stringPrintf("tag_%d_col_%d", - 3001 + i*2, i*2), - 3001 + i*2)); + tmpColumns.emplace_back(TestUtils::propDef(cpp2::PropOwner::SOURCE, + folly::stringPrintf("tag_%d_col_%d", + 3001 + i*2, i*2), + 3001 + i*2)); } req.set_return_columns(std::move(tmpColumns)); diff --git a/src/storage/test/StorageClientTest.cpp b/src/storage/test/StorageClientTest.cpp index 4899aca43e1..3daad55f5dd 100644 --- a/src/storage/test/StorageClientTest.cpp +++ b/src/storage/test/StorageClientTest.cpp @@ -17,12 +17,15 @@ DECLARE_string(meta_server_addrs); DECLARE_int32(load_data_interval_secs); +DECLARE_int32(heartbeat_interval_secs); namespace nebula { namespace storage { TEST(StorageClientTest, VerticesInterfacesTest) { FLAGS_load_data_interval_secs = 1; + FLAGS_heartbeat_interval_secs = 1; + fs::TempDir rootPath("/tmp/StorageClientTest.XXXXXX"); GraphSpaceID spaceId = 0; uint32_t localIp; @@ -49,7 +52,14 @@ TEST(StorageClientTest, VerticesInterfacesTest) { // for mockStorageServer MetaServerBasedPartManager, use ephemeral port uint32_t localDataPort = 0; std::string dataPath = folly::stringPrintf("%s/data", rootPath.path()); - auto sc = TestUtils::mockStorageServer(mClient.get(), dataPath.c_str(), localIp, localDataPort); + auto sc = TestUtils::mockStorageServer(mClient.get(), + dataPath.c_str(), + localIp, + localDataPort, + // TODO We are using the memory version of + // SchemaMan We need to switch to Meta Server + // based version + false); localDataPort = sc->port_; LOG(INFO) << "Add hosts and create space...."; @@ -58,11 +68,16 @@ TEST(StorageClientTest, VerticesInterfacesTest) { while (meta::ActiveHostsManHolder::hostsMan()->getActiveHosts().size() == 0) { usleep(1000); } + VLOG(1) << "The storage server has been added to the meta service"; + auto ret = mClient->createSpace("default", 10, 1).get(); + ASSERT_TRUE(ret.ok()) << ret.status(); spaceId = ret.value(); + LOG(INFO) << "Created space \"default\", its id is " << spaceId; sleep(2 * FLAGS_load_data_interval_secs + 1); auto client = std::make_unique(threadPool, mClient.get()); + // VerticesInterfacesTest(addVertices and getVertexProps) { LOG(INFO) << "Prepare vertices data..."; @@ -91,8 +106,15 @@ TEST(StorageClientTest, VerticesInterfacesTest) { auto f = client->addVertices(spaceId, std::move(vertices), true); LOG(INFO) << "Waiting for the response..."; auto resp = std::move(f).get(); - ASSERT_TRUE(resp.succeeded()); + if (!resp.succeeded()) { + for (auto& err : resp.failedParts()) { + LOG(ERROR) << "Partition " << err.first + << " failed: " << static_cast(err.second); + } + ASSERT_TRUE(resp.succeeded()); + } } + { std::vector vIds; std::vector retCols; @@ -107,6 +129,17 @@ TEST(StorageClientTest, VerticesInterfacesTest) { } auto f = client->getVertexProps(spaceId, std::move(vIds), std::move(retCols)); auto resp = std::move(f).get(); + if (VLOG_IS_ON(2)) { + if (!resp.succeeded()) { + std::stringstream ss; + for (auto& p : resp.failedParts()) { + ss << "Part " << p.first + << ": " << static_cast(p.second) + << "; "; + } + VLOG(2) << "Failed partitions:: " << ss.str(); + } + } ASSERT_TRUE(resp.succeeded()); auto& results = resp.responses(); @@ -162,6 +195,7 @@ TEST(StorageClientTest, VerticesInterfacesTest) { auto resp = std::move(f).get(); ASSERT_TRUE(resp.succeeded()); } + { std::vector edgeKeys; std::vector retCols; @@ -225,7 +259,6 @@ TEST(StorageClientTest, VerticesInterfacesTest) { threadPool.reset(); } - } // namespace storage } // namespace nebula diff --git a/src/storage/test/StorageServiceHandlerTest.cpp b/src/storage/test/StorageServiceHandlerTest.cpp index e419bf852d7..06ac6483ba4 100644 --- a/src/storage/test/StorageServiceHandlerTest.cpp +++ b/src/storage/test/StorageServiceHandlerTest.cpp @@ -25,9 +25,9 @@ TEST(StorageServiceHandlerTest, FutureAddVerticesTest) { req.parts.emplace(0, TestUtils::setupVertices(0, 10, 10)); req.parts.emplace(1, TestUtils::setupVertices(1, 20, 30)); LOG(INFO) << "Test FutureAddVerticesTest..."; - std::unique_ptr kvstore(TestUtils::initKV(rootPath.path())); - auto storageServiceHandler = std::make_unique(kvstore.get(), nullptr); + std::unique_ptr kvstore = TestUtils::initKV(rootPath.path()); + auto storageServiceHandler = std::make_unique(kvstore.get(), nullptr); auto resp = storageServiceHandler->future_addVertices(req).get(); EXPECT_EQ(typeid(cpp2::ExecResponse).name() , typeid(resp).name()); diff --git a/src/storage/test/TestUtils.h b/src/storage/test/TestUtils.h index e78ac217fc8..c94300bc167 100644 --- a/src/storage/test/TestUtils.h +++ b/src/storage/test/TestUtils.h @@ -19,33 +19,49 @@ #include -DECLARE_string(part_man_type); - namespace nebula { namespace storage { class TestUtils { public: - static kvstore::KVStore* initKV(const char* rootPath) { - auto partMan = std::make_unique(); - // GraphSpaceID => {PartitionIDs} - // 0 => {0, 1, 2, 3, 4, 5} - auto& partsMap = partMan->partsMap(); - for (auto partId = 0; partId < 6; partId++) { - partsMap[0][partId] = PartMeta(); + static std::unique_ptr initKV( + const char* rootPath, + HostAddr localhost = {0, 0}, + meta::MetaClient* mClient = nullptr, + bool useMetaServer = false) { + auto workers = std::make_shared(); + workers->start(4); + auto ioPool = std::make_shared(4); + + kvstore::KVOptions options; + if (useMetaServer) { + options.partMan_ = std::make_unique( + localhost, + mClient); + } else { + auto memPartMan = std::make_unique(); + // GraphSpaceID => {PartitionIDs} + // 0 => {0, 1, 2, 3, 4, 5} + auto& partsMap = memPartMan->partsMap(); + for (auto partId = 0; partId < 6; partId++) { + partsMap[0][partId] = PartMeta(); + } + + options.partMan_ = std::move(memPartMan); } + std::vector paths; paths.push_back(folly::stringPrintf("%s/disk1", rootPath)); paths.push_back(folly::stringPrintf("%s/disk2", rootPath)); - kvstore::KVOptions options; - options.local_ = HostAddr(0, 0); + // Prepare KVStore options.dataPaths_ = std::move(paths); - options.partMan_ = std::move(partMan); - - kvstore::NebulaStore* kv = static_cast( - kvstore::KVStore::instance(std::move(options))); - return kv; + auto store = std::make_unique(std::move(options), + ioPool, + workers, + localhost); + sleep(1); + return store; } static std::unique_ptr mockSchemaMan(GraphSpaceID spaceId = 0) { @@ -162,39 +178,36 @@ class TestUtils { const char* dataPath, uint32_t ip, uint32_t port = 0, - bool realMetaServer = false) { + bool useMetaServer = false) { auto sc = std::make_unique(); - std::vector paths; - paths.push_back(folly::stringPrintf("%s/disk1", dataPath)); - paths.push_back(folly::stringPrintf("%s/disk2", dataPath)); - kvstore::KVOptions options; - options.local_ = HostAddr(ip, port); - options.dataPaths_ = std::move(paths); - options.partMan_ = std::make_unique( - options.local_, mClient); - // Save MetaServerBasedPartManager to update port when ThriftServer setup - auto partManagerPtr = static_cast( - options.partMan_.get()); - kvstore::NebulaStore* kvPtr = static_cast( - kvstore::KVStore::instance(std::move(options))); - // Keep KVStore can't free out of this function - sc->KVStore_ = std::unique_ptr(kvPtr); + // Always use the Meta Service in this case + sc->kvStore_ = TestUtils::initKV(dataPath, {ip, port}, mClient, true); + std::unique_ptr schemaMan; - if (!realMetaServer) { + if (!useMetaServer) { schemaMan = TestUtils::mockSchemaMan(1); } else { LOG(INFO) << "Create real schemaManager"; schemaMan = meta::SchemaManager::create(); schemaMan->init(mClient); } + auto handler = std::make_shared( - sc->KVStore_.get(), std::move(schemaMan)); - test::mockCommon(sc.get(), "storage", port, handler); - if (nullptr != partManagerPtr) { - partManagerPtr->setLocalHost(HostAddr(ip, sc->port_)); + sc->kvStore_.get(), std::move(schemaMan)); + sc->mockCommon("storage", port, handler); + auto ptr = dynamic_cast( + sc->kvStore_->partManager()); + if (ptr) { + ptr->setLocalHost(HostAddr(ip, sc->port_)); + } else { + VLOG(1) << "Not using a MetaServerBasedPartManager"; } - LOG(INFO) << "Starting the storage Daemon on port " << sc->port_ - << ", path " << dataPath; + + // Sleep one second to wait for the leader election + sleep(1); + + LOG(INFO) << "The storage daemon started on port " << sc->port_ + << ", data path is at \"" << dataPath << "\""; return sc; } }; diff --git a/src/tools/storage-perf/CMakeLists.txt b/src/tools/storage-perf/CMakeLists.txt index 778ab65de6b..b90475c36f1 100644 --- a/src/tools/storage-perf/CMakeLists.txt +++ b/src/tools/storage-perf/CMakeLists.txt @@ -8,6 +8,9 @@ add_executable( $ $ $ + $ + $ + $ $ $ $ diff --git a/src/wal/FileBasedWal.cpp b/src/wal/FileBasedWal.cpp index 579a85ced12..cf3d5d409da 100644 --- a/src/wal/FileBasedWal.cpp +++ b/src/wal/FileBasedWal.cpp @@ -38,6 +38,11 @@ FileBasedWal::FileBasedWal(const folly::StringPiece dir, , policy_(std::move(policy)) , maxFileSize_(policy_.fileSize * 1024L * 1024L) , maxBufferSize_(policy_.bufferSize * 1024L * 1024L) { + // Make sure WAL directory exist + if (FileUtils::fileType(dir_.c_str()) == fs::FileType::NOTEXIST) { + FileUtils::makeDir(dir_); + } + scanAllWalFiles(); if (!walFiles_.empty()) { auto& info = walFiles_.rbegin()->second; diff --git a/src/webservice/test/TestUtils.h b/src/webservice/test/TestUtils.h index 85f601a3f6a..d92e1c11b8c 100644 --- a/src/webservice/test/TestUtils.h +++ b/src/webservice/test/TestUtils.h @@ -5,6 +5,7 @@ */ #include "base/Base.h" +#include "webservice/WebService.h" #include "process/ProcessUtils.h" namespace nebula {