diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index d1b0ca10021..90de6129c5a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,5 +1,5 @@ @@ -8,13 +8,13 @@ In order to review PR more efficiently, please add information according to the - [ ] feature - [ ] enhancement -#### What problem does this PR slove? +#### What problem(s) does this PR solve? Issue(s) number: Description: -#### How do you slove it? +#### How do you solve it? @@ -26,7 +26,7 @@ Description: Tests: - [ ] Unit test(positive and negative cases) - [ ] Function test -- [ ] Perfomance test +- [ ] Performance test - [ ] N/A Affects: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 19004ffab4c..23e5d29fe19 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -82,7 +82,8 @@ jobs: with: context: . file: ./docker/Dockerfile.${{ matrix.service }} - platforms: linux/amd64,linux/arm64 + # platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 tags: | vesoft/nebula-${{ matrix.service }}:nightly push: true diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 80da0b19ae5..2891dd7a0ce 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -40,7 +40,7 @@ jobs: - uses: ./.github/actions/tagname-action id: tag - name: package - run: ./package/package.sh -b ${{ steps.tag.outputs.tag }} -t RelWithDebInfo -r OFF -p ON -s TRUE + run: ./package/package.sh -v ${{ steps.tag.outputs.tag }} -t RelWithDebInfo -r OFF -p ON -s TRUE - name: output some vars run: | tar zcf ${{ env.CPACK_DIR }}/nebula-${{ steps.tag.outputs.tagnum }}.tar.gz --exclude=${{ env.BUILD_DIR }} ./* @@ -82,7 +82,7 @@ jobs: id: tagname - id: docker run: | - majorver=$(git tag -l --sort=v:refname | tail -n1 | cut -f1 -d'.') + majorver=$(git tag -l --sort=v:refname | tail -n1 | cut -f1 -d".") tag="" if [[ $majorver == ${{ steps.tagname.outputs.majorver }} ]]; then tag="${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:latest" @@ -99,7 +99,8 @@ jobs: with: context: . file: ./docker/Dockerfile.${{ matrix.service }} - platforms: linux/amd64,linux/arm64 + # platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 tags: | ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.tag }} ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.majorver }} diff --git a/package/package.sh b/package/package.sh index 23abc71482f..299bdbc2ece 100755 --- a/package/package.sh +++ b/package/package.sh @@ -6,14 +6,13 @@ # -v: The version of package, the version should be match tag name, default value is null # -n: Package to one or multi-packages, `ON` means one package, `OFF` means multi packages, default value is `ON` # -s: Whether to strip the package, default value is `FALSE` -# -b: Branch, default master # -d: Whether to enable sanitizer, default OFF # -t: Build type, default Release # -j: Number of threads, default $(nproc) # -r: Whether enable compressed debug info, default ON # -p: Whether dump the symbols from binary by dump_syms # -# usage: ./package.sh -v -n -s -b +# usage: ./package.sh -v -n -s # set -e @@ -21,13 +20,13 @@ set -e version="" package_one=ON strip_enable="FALSE" -usage="Usage: ${0} -v -n -s -b -g -j -t " +usage="Usage: ${0} -v -n -s -g -j -t " project_dir="$(cd "$(dirname "$0")" && pwd)/.." build_dir=${project_dir}/pkg-build enablesanitizer="OFF" static_sanitizer="OFF" build_type="Release" -branch="master" +branch=$(git rev-parse --abbrev-ref HEAD) jobs=$(nproc) enable_compressed_debug_info=ON dump_symbols=OFF @@ -47,9 +46,6 @@ do s) strip_enable=$OPTARG ;; - b) - branch=$OPTARG - ;; d) enablesanitizer="ON" if [ "$OPTARG" == "static" ]; then @@ -135,9 +131,8 @@ function build { san=$2 ssan=$3 build_type=$4 - branch=$5 - package_tar=$6 - install_prefix=$7 + package_tar=$5 + install_prefix=$6 mkdir -p ${build_dir} @@ -210,7 +205,7 @@ function dump_syms { } # The main -build $version $enablesanitizer $static_sanitizer $build_type $branch "OFF" "/usr/local/nebula" +build $version $enablesanitizer $static_sanitizer $build_type "OFF" "/usr/local/nebula" package $strip_enable if [[ $dump_symbols == ON ]]; then echo ">>> start dump symbols <<<" @@ -218,5 +213,5 @@ if [[ $dump_symbols == ON ]]; then fi # tar package -build $version $enablesanitizer $static_sanitizer $build_type $branch "ON" "/" +build $version $enablesanitizer $static_sanitizer $build_type "ON" "/" package $strip_enable diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index 48413c7fd21..db75a884c8f 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -64,14 +64,15 @@ DEFINE_validator(failed_login_attempts, &ValidateFailedLoginAttempts); namespace nebula { namespace meta { +Indexes buildIndexes(std::vector indexItemVec); + MetaClient::MetaClient(std::shared_ptr ioThreadPool, std::vector addrs, const MetaClientOptions& options) : ioThreadPool_(ioThreadPool), addrs_(std::move(addrs)), options_(options), - sessionMap_(new SessionMap{}), - killedPlans_(new folly::F14FastSet>{}) { + metadata_(new MetaData()) { CHECK(ioThreadPool_ != nullptr) << "IOThreadPool is required"; CHECK(!addrs_.empty()) << "No meta server address is specified or can be solved. Meta server is required"; @@ -86,9 +87,9 @@ MetaClient::MetaClient(std::shared_ptr ioThreadPool } MetaClient::~MetaClient() { + notifyStop(); stop(); - delete sessionMap_.load(); - delete killedPlans_.load(); + delete metadata_.load(); VLOG(3) << "~MetaClient"; } @@ -142,13 +143,18 @@ bool MetaClient::waitForMetadReady(int count, int retryIntervalSecs) { return ready_; } -void MetaClient::stop() { +void MetaClient::notifyStop() { if (bgThread_ != nullptr) { bgThread_->stop(); + } + isRunning_ = false; +} + +void MetaClient::stop() { + if (bgThread_ != nullptr) { bgThread_->wait(); bgThread_.reset(); } - isRunning_ = false; } void MetaClient::heartBeatThreadFunc() { @@ -175,8 +181,6 @@ bool MetaClient::loadUsersAndRoles() { } decltype(userRolesMap_) userRolesMap; decltype(userPasswordMap_) userPasswordMap; - decltype(userPasswordAttemptsRemain_) userPasswordAttemptsRemain; - decltype(userLoginLockTime_) userLoginLockTime; // List of username std::unordered_set userNameList; @@ -188,40 +192,37 @@ bool MetaClient::loadUsersAndRoles() { } userRolesMap[user.first] = rolesRet.value(); userPasswordMap[user.first] = user.second; - userPasswordAttemptsRemain[user.first] = FLAGS_failed_login_attempts; - userLoginLockTime[user.first] = 0; userNameList.emplace(user.first); } - { - folly::RWSpinLock::WriteHolder holder(localCacheLock_); - userRolesMap_ = std::move(userRolesMap); - userPasswordMap_ = std::move(userPasswordMap); - - // This method is called periodically by the heartbeat thread, but we don't want to reset the - // failed login attempts every time. Remove expired users from cache - for (auto& ele : userPasswordAttemptsRemain) { - if (userNameList.count(ele.first) == 0) { - userPasswordAttemptsRemain.erase(ele.first); - } - } - for (auto& ele : userLoginLockTime) { - if (userNameList.count(ele.first) == 0) { - userLoginLockTime.erase(ele.first); + + userRolesMap_ = std::move(userRolesMap); + userPasswordMap_ = std::move(userPasswordMap); + + // Remove expired users from cache + auto removeExpiredUser = [&](folly::ConcurrentHashMap& userMap, + const std::unordered_set& userList) { + for (auto iter = userMap.begin(); iter != userMap.end();) { + if (!userList.count(iter->first)) { + iter = userMap.erase(iter); + } else { + ++iter; } } + }; + removeExpiredUser(userPasswordAttemptsRemain_, userNameList); + removeExpiredUser(userLoginLockTime_, userNameList); + // This method is called periodically by the heartbeat thread, but we don't want to reset the + // failed login attempts every time. + for (const auto& user : userNameList) { // If the user is not in the map, insert value with the default value - for (const auto& user : userNameList) { - if (userPasswordAttemptsRemain.count(user) == 0) { - userPasswordAttemptsRemain[user] = FLAGS_failed_login_attempts; - } - if (userLoginLockTime.count(user) == 0) { - userLoginLockTime[user] = 0; - } + // Do nothing if the account is already in the map + if (userPasswordAttemptsRemain_.find(user) == userPasswordAttemptsRemain_.end()) { + userPasswordAttemptsRemain_.insert(user, FLAGS_failed_login_attempts); + } + if (userLoginLockTime_.find(user) == userLoginLockTime_.end()) { + userLoginLockTime_.insert(user, 0); } - - userPasswordAttemptsRemain_ = std::move(userPasswordAttemptsRemain); - userLoginLockTime_ = std::move(userLoginLockTime); } return true; } @@ -343,7 +344,6 @@ bool MetaClient::loadData() { decltype(localCache_) oldCache; { - folly::RWSpinLock::WriteHolder holder(localCacheLock_); oldCache = std::move(localCache_); localCache_ = std::move(cache); spaceIndexByName_ = std::move(spaceIndexByName); @@ -358,7 +358,37 @@ bool MetaClient::loadData() { } localDataLastUpdateTime_.store(metadLastUpdateTime_.load()); - + auto newMetaData = new MetaData(); + + for (auto& spaceInfo : localCache_) { + GraphSpaceID spaceId = spaceInfo.first; + std::shared_ptr info = spaceInfo.second; + std::shared_ptr infoDeepCopy = std::make_shared(*info); + infoDeepCopy->tagSchemas_ = buildTagSchemas(infoDeepCopy->tagItemVec_, &infoDeepCopy->pool_); + infoDeepCopy->edgeSchemas_ = buildEdgeSchemas(infoDeepCopy->edgeItemVec_, &infoDeepCopy->pool_); + infoDeepCopy->tagIndexes_ = buildIndexes(infoDeepCopy->tagIndexItemVec_); + infoDeepCopy->edgeIndexes_ = buildIndexes(infoDeepCopy->edgeIndexItemVec_); + newMetaData->localCache_[spaceId] = infoDeepCopy; + } + newMetaData->spaceIndexByName_ = spaceIndexByName_; + newMetaData->spaceTagIndexByName_ = spaceTagIndexByName_; + newMetaData->spaceEdgeIndexByName_ = spaceEdgeIndexByName_; + newMetaData->spaceEdgeIndexByType_ = spaceEdgeIndexByType_; + newMetaData->spaceNewestTagVerMap_ = spaceNewestTagVerMap_; + newMetaData->spaceNewestEdgeVerMap_ = spaceNewestEdgeVerMap_; + newMetaData->spaceTagIndexById_ = spaceTagIndexById_; + newMetaData->spaceAllEdgeMap_ = spaceAllEdgeMap_; + + newMetaData->userRolesMap_ = userRolesMap_; + newMetaData->storageHosts_ = storageHosts_; + newMetaData->fulltextIndexMap_ = fulltextIndexMap_; + newMetaData->userPasswordMap_ = userPasswordMap_; + newMetaData->sessionMap_ = std::move(sessionMap_); + newMetaData->killedPlans_ = std::move(killedPlans_); + newMetaData->serviceClientList_ = std::move(serviceClientList_); + auto oldMetaData = metadata_.load(); + metadata_.store(newMetaData); + folly::rcu_retire(oldMetaData); diff(oldCache, localCache_); listenerDiff(oldCache, localCache_); loadRemoteListeners(); @@ -512,7 +542,7 @@ bool MetaClient::loadSchemas(GraphSpaceID spaceId, return true; } -static Indexes buildIndexes(std::vector indexItemVec) { +Indexes buildIndexes(std::vector indexItemVec) { Indexes indexes; for (auto index : indexItemVec) { auto indexName = index.get_index_name(); @@ -581,10 +611,7 @@ bool MetaClient::loadGlobalServiceClients() { LOG(ERROR) << "List services failed, status:" << ret.status(); return false; } - { - folly::RWSpinLock::WriteHolder holder(localCacheLock_); - serviceClientList_ = std::move(ret).value(); - } + serviceClientList_ = std::move(ret).value(); return true; } @@ -594,53 +621,15 @@ bool MetaClient::loadFulltextIndexes() { LOG(ERROR) << "List fulltext indexes failed, status:" << ftRet.status(); return false; } - { - folly::RWSpinLock::WriteHolder holder(localCacheLock_); - fulltextIndexMap_ = std::move(ftRet).value(); - } + fulltextIndexMap_ = std::move(ftRet).value(); return true; } -const MetaClient::ThreadLocalInfo& MetaClient::getThreadLocalInfo() { - ThreadLocalInfo& threadLocalInfo = folly::SingletonThreadLocal::get(); - - if (threadLocalInfo.localLastUpdateTime_ < localDataLastUpdateTime_) { - threadLocalInfo.localLastUpdateTime_ = localDataLastUpdateTime_; - - folly::RWSpinLock::ReadHolder holder(localCacheLock_); - for (auto& spaceInfo : localCache_) { - GraphSpaceID spaceId = spaceInfo.first; - std::shared_ptr info = spaceInfo.second; - std::shared_ptr infoDeepCopy = std::make_shared(*info); - infoDeepCopy->tagSchemas_ = buildTagSchemas(infoDeepCopy->tagItemVec_, &infoDeepCopy->pool_); - infoDeepCopy->edgeSchemas_ = - buildEdgeSchemas(infoDeepCopy->edgeItemVec_, &infoDeepCopy->pool_); - infoDeepCopy->tagIndexes_ = buildIndexes(infoDeepCopy->tagIndexItemVec_); - infoDeepCopy->edgeIndexes_ = buildIndexes(infoDeepCopy->edgeIndexItemVec_); - threadLocalInfo.localCache_[spaceId] = infoDeepCopy; - } - threadLocalInfo.spaceIndexByName_ = spaceIndexByName_; - threadLocalInfo.spaceTagIndexByName_ = spaceTagIndexByName_; - threadLocalInfo.spaceEdgeIndexByName_ = spaceEdgeIndexByName_; - threadLocalInfo.spaceEdgeIndexByType_ = spaceEdgeIndexByType_; - threadLocalInfo.spaceNewestTagVerMap_ = spaceNewestTagVerMap_; - threadLocalInfo.spaceNewestEdgeVerMap_ = spaceNewestEdgeVerMap_; - threadLocalInfo.spaceTagIndexById_ = spaceTagIndexById_; - threadLocalInfo.spaceAllEdgeMap_ = spaceAllEdgeMap_; - - threadLocalInfo.userRolesMap_ = userRolesMap_; - threadLocalInfo.storageHosts_ = storageHosts_; - threadLocalInfo.fulltextIndexMap_ = fulltextIndexMap_; - threadLocalInfo.userPasswordMap_ = userPasswordMap_; - } - - return threadLocalInfo; -} - Status MetaClient::checkTagIndexed(GraphSpaceID spaceId, IndexID indexID) { - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto it = threadLocalInfo.localCache_.find(spaceId); - if (it != threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.localCache_.find(spaceId); + if (it != metadata.localCache_.end()) { auto indexIt = it->second->tagIndexes_.find(indexID); if (indexIt != it->second->tagIndexes_.end()) { return Status::OK(); @@ -652,9 +641,10 @@ Status MetaClient::checkTagIndexed(GraphSpaceID spaceId, IndexID indexID) { } Status MetaClient::checkEdgeIndexed(GraphSpaceID space, IndexID indexID) { - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto it = threadLocalInfo.localCache_.find(space); - if (it != threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.localCache_.find(space); + if (it != metadata.localCache_.end()) { auto indexIt = it->second->edgeIndexes_.find(indexID); if (indexIt != it->second->edgeIndexes_.end()) { return Status::OK(); @@ -1334,9 +1324,10 @@ StatusOr MetaClient::getSpaceIdByNameFromCache(const std::string& if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto it = threadLocalInfo.spaceIndexByName_.find(name); - if (it != threadLocalInfo.spaceIndexByName_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceIndexByName_.find(name); + if (it != metadata.spaceIndexByName_.end()) { return it->second; } return Status::SpaceNotFound(); @@ -1346,9 +1337,10 @@ StatusOr MetaClient::getSpaceNameByIdFromCache(GraphSpaceID spaceId if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto spaceIt = threadLocalInfo.localCache_.find(spaceId); - if (spaceIt == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { LOG(ERROR) << "Space " << spaceId << " not found!"; return Status::Error("Space %d not found", spaceId); } @@ -1360,9 +1352,10 @@ StatusOr MetaClient::getTagIDByNameFromCache(const GraphSpaceID& space, if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto it = threadLocalInfo.spaceTagIndexByName_.find(std::make_pair(space, name)); - if (it == threadLocalInfo.spaceTagIndexByName_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceTagIndexByName_.find(std::make_pair(space, name)); + if (it == metadata.spaceTagIndexByName_.end()) { return Status::Error("TagName `%s' is nonexistent", name.c_str()); } return it->second; @@ -1373,9 +1366,10 @@ StatusOr MetaClient::getTagNameByIdFromCache(const GraphSpaceID& sp if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto it = threadLocalInfo.spaceTagIndexById_.find(std::make_pair(space, tagId)); - if (it == threadLocalInfo.spaceTagIndexById_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceTagIndexById_.find(std::make_pair(space, tagId)); + if (it == metadata.spaceTagIndexById_.end()) { return Status::Error("TagID `%d' is nonexistent", tagId); } return it->second; @@ -1386,9 +1380,10 @@ StatusOr MetaClient::getEdgeTypeByNameFromCache(const GraphSpaceID& sp if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto it = threadLocalInfo.spaceEdgeIndexByName_.find(std::make_pair(space, name)); - if (it == threadLocalInfo.spaceEdgeIndexByName_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceEdgeIndexByName_.find(std::make_pair(space, name)); + if (it == metadata.spaceEdgeIndexByName_.end()) { return Status::Error("EdgeName `%s' is nonexistent", name.c_str()); } return it->second; @@ -1399,9 +1394,10 @@ StatusOr MetaClient::getEdgeNameByTypeFromCache(const GraphSpaceID& if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto it = threadLocalInfo.spaceEdgeIndexByType_.find(std::make_pair(space, edgeType)); - if (it == threadLocalInfo.spaceEdgeIndexByType_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceEdgeIndexByType_.find(std::make_pair(space, edgeType)); + if (it == metadata.spaceEdgeIndexByType_.end()) { return Status::Error("EdgeType `%d' is nonexistent", edgeType); } return it->second; @@ -1411,9 +1407,10 @@ StatusOr> MetaClient::getAllEdgeFromCache(const GraphSp if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto it = threadLocalInfo.spaceAllEdgeMap_.find(space); - if (it == threadLocalInfo.spaceAllEdgeMap_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceAllEdgeMap_.find(space); + if (it == metadata.spaceAllEdgeMap_.end()) { return Status::Error("SpaceId `%d' is nonexistent", space); } return it->second; @@ -1548,14 +1545,16 @@ folly::Future> MetaClient::removeRange(std::string segment, } PartsMap MetaClient::getPartsMapFromCache(const HostAddr& host) { - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - return doGetPartsMap(host, threadLocalInfo.localCache_); + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + return doGetPartsMap(host, metadata.localCache_); } StatusOr MetaClient::getPartHostsFromCache(GraphSpaceID spaceId, PartitionID partId) { - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto it = threadLocalInfo.localCache_.find(spaceId); - if (it == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.localCache_.find(spaceId); + if (it == metadata.localCache_.end()) { return Status::Error("Space not found, spaceid: %d", spaceId); } auto& cache = it->second; @@ -1573,9 +1572,10 @@ StatusOr MetaClient::getPartHostsFromCache(GraphSpaceID spaceId, Part Status MetaClient::checkPartExistInCache(const HostAddr& host, GraphSpaceID spaceId, PartitionID partId) { - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto it = threadLocalInfo.localCache_.find(spaceId); - if (it != threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.localCache_.find(spaceId); + if (it != metadata.localCache_.end()) { auto partsIt = it->second->partsOnHost_.find(host); if (partsIt != it->second->partsOnHost_.end()) { for (auto& pId : partsIt->second) { @@ -1592,9 +1592,10 @@ Status MetaClient::checkPartExistInCache(const HostAddr& host, } Status MetaClient::checkSpaceExistInCache(const HostAddr& host, GraphSpaceID spaceId) { - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto it = threadLocalInfo.localCache_.find(spaceId); - if (it != threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.localCache_.find(spaceId); + if (it != metadata.localCache_.end()) { auto partsIt = it->second->partsOnHost_.find(host); if (partsIt != it->second->partsOnHost_.end() && !partsIt->second.empty()) { return Status::OK(); @@ -1606,9 +1607,10 @@ Status MetaClient::checkSpaceExistInCache(const HostAddr& host, GraphSpaceID spa } StatusOr MetaClient::partsNum(GraphSpaceID spaceId) { - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto it = threadLocalInfo.localCache_.find(spaceId); - if (it == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.localCache_.find(spaceId); + if (it == metadata.localCache_.end()) { return Status::Error("Space not found, spaceid: %d", spaceId); } return it->second->partsAlloc_.size(); @@ -1999,9 +2001,10 @@ StatusOr MetaClient::getSpaceVidLen(const GraphSpaceID& spaceId) { if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto spaceIt = threadLocalInfo.localCache_.find(spaceId); - if (spaceIt == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { LOG(ERROR) << "Space " << spaceId << " not found!"; return Status::Error("Space %d not found", spaceId); } @@ -2017,9 +2020,10 @@ StatusOr MetaClient::getSpaceVidType(const GraphSpac if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto spaceIt = threadLocalInfo.localCache_.find(spaceId); - if (spaceIt == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { LOG(ERROR) << "Space " << spaceId << " not found!"; return Status::Error("Space %d not found", spaceId); } @@ -2038,9 +2042,10 @@ StatusOr MetaClient::getSpaceDesc(const GraphSpaceID& space) { if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto spaceIt = threadLocalInfo.localCache_.find(space); - if (spaceIt == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(space); + if (spaceIt == metadata.localCache_.end()) { LOG(ERROR) << "Space " << space << " not found!"; return Status::Error("Space %d not found", space); } @@ -2061,9 +2066,10 @@ StatusOr> MetaClient::getTagSchemaFr if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto spaceIt = threadLocalInfo.localCache_.find(spaceId); - if (spaceIt != threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt != metadata.localCache_.end()) { auto tagIt = spaceIt->second->tagSchemas_.find(tagID); if (tagIt != spaceIt->second->tagSchemas_.end() && !tagIt->second.empty()) { size_t vNum = tagIt->second.size(); @@ -2081,9 +2087,10 @@ StatusOr> MetaClient::getEdgeSchemaF if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto spaceIt = threadLocalInfo.localCache_.find(spaceId); - if (spaceIt != threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt != metadata.localCache_.end()) { auto edgeIt = spaceIt->second->edgeSchemas_.find(edgeType); if (edgeIt != spaceIt->second->edgeSchemas_.end() && !edgeIt->second.empty()) { size_t vNum = edgeIt->second.size(); @@ -2100,9 +2107,10 @@ StatusOr MetaClient::getAllVerTagSchema(GraphSpaceID spaceId) { if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto iter = threadLocalInfo.localCache_.find(spaceId); - if (iter == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto iter = metadata.localCache_.find(spaceId); + if (iter == metadata.localCache_.end()) { return Status::Error("Space %d not found", spaceId); } return iter->second->tagSchemas_; @@ -2112,9 +2120,10 @@ StatusOr MetaClient::getAllLatestVerTagSchema(const GraphSpaceID& spa if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto iter = threadLocalInfo.localCache_.find(spaceId); - if (iter == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto iter = metadata.localCache_.find(spaceId); + if (iter == metadata.localCache_.end()) { return Status::Error("Space %d not found", spaceId); } TagSchema tagsSchema; @@ -2130,9 +2139,10 @@ StatusOr MetaClient::getAllVerEdgeSchema(GraphSpaceID spaceId) { if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto iter = threadLocalInfo.localCache_.find(spaceId); - if (iter == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto iter = metadata.localCache_.find(spaceId); + if (iter == metadata.localCache_.end()) { return Status::Error("Space %d not found", spaceId); } return iter->second->edgeSchemas_; @@ -2142,9 +2152,10 @@ StatusOr MetaClient::getAllLatestVerEdgeSchemaFromCache(const GraphS if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto iter = threadLocalInfo.localCache_.find(spaceId); - if (iter == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto iter = metadata.localCache_.find(spaceId); + if (iter == metadata.localCache_.end()) { return Status::Error("Space %d not found", spaceId); } EdgeSchema edgesSchema; @@ -2232,9 +2243,10 @@ StatusOr> MetaClient::getTagIndexFromCache(Grap return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto spaceIt = threadLocalInfo.localCache_.find(spaceId); - if (spaceIt == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { VLOG(3) << "Space " << spaceId << " not found!"; return Status::SpaceNotFound(); } else { @@ -2269,9 +2281,10 @@ StatusOr> MetaClient::getEdgeIndexFromCache(Gra return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto spaceIt = threadLocalInfo.localCache_.find(spaceId); - if (spaceIt == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { VLOG(3) << "Space " << spaceId << " not found!"; return Status::SpaceNotFound(); } else { @@ -2306,9 +2319,10 @@ StatusOr>> MetaClient::getTagIndexe return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto spaceIt = threadLocalInfo.localCache_.find(spaceId); - if (spaceIt == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { VLOG(3) << "Space " << spaceId << " not found!"; return Status::SpaceNotFound(); } else { @@ -2329,9 +2343,10 @@ StatusOr>> MetaClient::getEdgeIndex return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto spaceIt = threadLocalInfo.localCache_.find(spaceId); - if (spaceIt == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { VLOG(3) << "Space " << spaceId << " not found!"; return Status::SpaceNotFound(); } else { @@ -2405,9 +2420,10 @@ std::vector MetaClient::getRolesByUserFromCache(const std::strin if (!ready_) { return std::vector(0); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto iter = threadLocalInfo.userRolesMap_.find(user); - if (iter == threadLocalInfo.userRolesMap_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto iter = metadata.userRolesMap_.find(user); + if (iter == metadata.userRolesMap_.end()) { return std::vector(0); } return iter->second; @@ -2418,21 +2434,16 @@ Status MetaClient::authCheckFromCache(const std::string& account, const std::str if (!ready_) { return Status::Error("Meta Service not ready"); } - - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - // Check user existence - auto iter = threadLocalInfo.userPasswordMap_.find(account); - if (iter == threadLocalInfo.userPasswordMap_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto iter = metadata.userPasswordMap_.find(account); + if (iter == metadata.userPasswordMap_.end()) { return Status::Error("User not exist"); } + auto lockedSince = userLoginLockTime_[account]; + auto passwordAttemtRemain = userPasswordAttemptsRemain_[account]; - folly::RWSpinLock::WriteHolder holder(localCacheLock_); - - auto& lockedSince = userLoginLockTime_[account]; - auto& passwordAttemtRemain = userPasswordAttemptsRemain_[account]; - LOG(INFO) << "Thread id: " << std::this_thread::get_id() - << " ,passwordAttemtRemain: " << passwordAttemtRemain; - // lockedSince is non-zero means the account has been locked + // If lockedSince is non-zero, it means the account has been locked if (lockedSince != 0) { auto remainingLockTime = (lockedSince + FLAGS_password_lock_time_in_secs) - time::WallClock::fastNowInSec(); @@ -2446,8 +2457,9 @@ Status MetaClient::authCheckFromCache(const std::string& account, const std::str remainingLockTime); } // Clear lock state and reset attempts - lockedSince = 0; - passwordAttemtRemain = FLAGS_failed_login_attempts; + userLoginLockTime_.assign_if_equal(account, lockedSince, 0); + userPasswordAttemptsRemain_.assign_if_equal( + account, passwordAttemtRemain, FLAGS_failed_login_attempts); } if (iter->second != password) { @@ -2458,12 +2470,14 @@ Status MetaClient::authCheckFromCache(const std::string& account, const std::str // If the password is not correct and passwordAttemtRemain > 0, // Allow another attemp + passwordAttemtRemain = userPasswordAttemptsRemain_[account]; if (passwordAttemtRemain > 0) { - --passwordAttemtRemain; - if (passwordAttemtRemain == 0) { + auto newAttemtRemain = passwordAttemtRemain - 1; + userPasswordAttemptsRemain_.assign_if_equal(account, passwordAttemtRemain, newAttemtRemain); + if (newAttemtRemain == 0) { // If the remaining attemps is 0, failed to authenticate // Block user login - lockedSince = time::WallClock::fastNowInSec(); + userLoginLockTime_.assign_if_equal(account, 0, time::WallClock::fastNowInSec()); return Status::Error( "%d times consecutive incorrect passwords has been input, user name: %s has been " "locked, try again in %d seconds", @@ -2471,14 +2485,14 @@ Status MetaClient::authCheckFromCache(const std::string& account, const std::str account.c_str(), FLAGS_password_lock_time_in_secs); } - LOG(ERROR) << "Invalid password, remaining attempts: " << passwordAttemtRemain; - return Status::Error("Invalid password, remaining attempts: %d", passwordAttemtRemain); + LOG(ERROR) << "Invalid password, remaining attempts: " << newAttemtRemain; + return Status::Error("Invalid password, remaining attempts: %d", newAttemtRemain); } } - // Reset password attempts - passwordAttemtRemain = FLAGS_failed_login_attempts; - lockedSince = 0; + // Authentication succeed, reset password attempts + userPasswordAttemptsRemain_.assign(account, FLAGS_failed_login_attempts); + userLoginLockTime_.assign(account, 0); return Status::OK(); } @@ -2486,18 +2500,20 @@ bool MetaClient::checkShadowAccountFromCache(const std::string& account) { if (!ready_) { return false; } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto iter = threadLocalInfo.userPasswordMap_.find(account); - if (iter != threadLocalInfo.userPasswordMap_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto iter = metadata.userPasswordMap_.find(account); + if (iter != metadata.userPasswordMap_.end()) { return true; } return false; } StatusOr MetaClient::getTermFromCache(GraphSpaceID spaceId, PartitionID partId) { - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto spaceInfo = threadLocalInfo.localCache_.find(spaceId); - if (spaceInfo == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceInfo = metadata.localCache_.find(spaceId); + if (spaceInfo == metadata.localCache_.end()) { return Status::Error("Term not found!"); } @@ -2514,8 +2530,9 @@ StatusOr> MetaClient::getStorageHosts() { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - return threadLocalInfo.storageHosts_; + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + return metadata.storageHosts_; } StatusOr MetaClient::getLatestTagVersionFromCache(const GraphSpaceID& space, @@ -2523,9 +2540,10 @@ StatusOr MetaClient::getLatestTagVersionFromCache(const GraphSpaceID& if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto it = threadLocalInfo.spaceNewestTagVerMap_.find(std::make_pair(space, tagId)); - if (it == threadLocalInfo.spaceNewestTagVerMap_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceNewestTagVerMap_.find(std::make_pair(space, tagId)); + if (it == metadata.spaceNewestTagVerMap_.end()) { return Status::TagNotFound(); } return it->second; @@ -2536,9 +2554,10 @@ StatusOr MetaClient::getLatestEdgeVersionFromCache(const GraphSpaceID if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto it = threadLocalInfo.spaceNewestEdgeVerMap_.find(std::make_pair(space, edgeType)); - if (it == threadLocalInfo.spaceNewestEdgeVerMap_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceNewestEdgeVerMap_.find(std::make_pair(space, edgeType)); + if (it == metadata.spaceNewestEdgeVerMap_.end()) { return Status::EdgeNotFound(); } return it->second; @@ -2992,9 +3011,10 @@ MetaClient::getListenersBySpaceHostFromCache(GraphSpaceID spaceId, const HostAdd if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto spaceIt = threadLocalInfo.localCache_.find(spaceId); - if (spaceIt == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { VLOG(3) << "Space " << spaceId << " not found!"; return Status::SpaceNotFound(); } @@ -3011,8 +3031,9 @@ StatusOr MetaClient::getListenersByHostFromCache(const HostAddr& h if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - return doGetListenersMap(host, threadLocalInfo.localCache_); + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + return doGetListenersMap(host, metadata.localCache_); } ListenersMap MetaClient::doGetListenersMap(const HostAddr& host, const LocalCache& localCache) { @@ -3048,9 +3069,10 @@ StatusOr MetaClient::getListenerHostsBySpacePartType(GraphSpaceID spac if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto spaceIt = threadLocalInfo.localCache_.find(spaceId); - if (spaceIt == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { VLOG(3) << "Space " << spaceId << " not found!"; return Status::SpaceNotFound(); } @@ -3069,9 +3091,10 @@ StatusOr> MetaClient::getListenerHostTypeBySpace if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - auto spaceIt = threadLocalInfo.localCache_.find(spaceId); - if (spaceIt == threadLocalInfo.localCache_.end()) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { VLOG(3) << "Space " << spaceId << " not found!"; return Status::SpaceNotFound(); } @@ -3160,8 +3183,9 @@ void MetaClient::updateNestedGflags(const std::unordered_map optionMap.emplace(value.first, value.second.toString()); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - for (const auto& spaceEntry : threadLocalInfo.localCache_) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + for (const auto& spaceEntry : metadata.localCache_) { listener_->onSpaceOptionUpdated(spaceEntry.first, optionMap); } } @@ -3448,11 +3472,11 @@ StatusOr> MetaClient::getServiceClientsFromCach if (!ready_) { return Status::Error("Not ready!"); } - - folly::RWSpinLock::ReadHolder holder(localCacheLock_); + folly::rcu_reader guard; + auto& metadata = *metadata_.load(); if (type == cpp2::ExternalServiceType::ELASTICSEARCH) { - auto sIter = serviceClientList_.find(type); - if (sIter != serviceClientList_.end()) { + auto sIter = metadata.serviceClientList_.find(type); + if (sIter != metadata.serviceClientList_.end()) { return sIter->second; } } @@ -3514,8 +3538,9 @@ StatusOr> MetaClient::getFTIndexe if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - return threadLocalInfo.fulltextIndexMap_; + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + return metadata.fulltextIndexMap_; } StatusOr> MetaClient::getFTIndexBySpaceFromCache( @@ -3523,9 +3548,10 @@ StatusOr> MetaClient::getFTIndexB if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); std::unordered_map indexes; - for (const auto& it : threadLocalInfo.fulltextIndexMap_) { + for (const auto& it : metadata.fulltextIndexMap_) { if (it.second.get_space_id() == spaceId) { indexes[it.first] = it.second; } @@ -3538,8 +3564,9 @@ StatusOr> MetaClient::getFTIndexBySpaceSch if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - for (auto& it : threadLocalInfo.fulltextIndexMap_) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + for (auto& it : metadata.fulltextIndexMap_) { auto id = it.second.get_depend_schema().getType() == nebula::cpp2::SchemaID::Type::edge_type ? it.second.get_depend_schema().get_edge_type() : it.second.get_depend_schema().get_tag_id(); @@ -3555,12 +3582,13 @@ StatusOr MetaClient::getFTIndexByNameFromCache(GraphSpaceID space if (!ready_) { return Status::Error("Not ready!"); } - const ThreadLocalInfo& threadLocalInfo = getThreadLocalInfo(); - if (threadLocalInfo.fulltextIndexMap_.find(name) != fulltextIndexMap_.end() && - threadLocalInfo.fulltextIndexMap_.at(name).get_space_id() != spaceId) { + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + if (metadata.fulltextIndexMap_.find(name) != fulltextIndexMap_.end() && + metadata.fulltextIndexMap_.at(name).get_space_id() != spaceId) { return Status::IndexNotFound(); } - return threadLocalInfo.fulltextIndexMap_.at(name); + return metadata.fulltextIndexMap_.at(name); } folly::Future> MetaClient::createSession( @@ -3712,22 +3740,16 @@ bool MetaClient::loadSessions() { LOG(ERROR) << "List sessions failed, status:" << session_list.status(); return false; } - SessionMap* oldSessionMap = sessionMap_.load(); - SessionMap* newSessionMap = new SessionMap(*oldSessionMap); - auto oldKilledPlan = killedPlans_.load(); - auto newKilledPlan = new folly::F14FastSet>(*oldKilledPlan); + sessionMap_.clear(); + killedPlans_.clear(); for (auto& session : session_list.value().get_sessions()) { - (*newSessionMap)[session.get_session_id()] = session; + sessionMap_[session.get_session_id()] = session; for (auto& query : session.get_queries()) { if (query.second.get_status() == cpp2::QueryStatus::KILLING) { - newKilledPlan->insert({session.get_session_id(), query.first}); + killedPlans_.insert({session.get_session_id(), query.first}); } } } - sessionMap_.store(newSessionMap); - killedPlans_.store(newKilledPlan); - folly::rcu_retire(oldKilledPlan); - folly::rcu_retire(oldSessionMap); return true; } @@ -3736,9 +3758,9 @@ StatusOr MetaClient::getSessionFromCache(const nebula::SessionID& return Status::Error("Not ready!"); } folly::rcu_reader guard; - auto session_map = sessionMap_.load(); - auto it = session_map->find(session_id); - if (it != session_map->end()) { + auto& sessionMap = metadata_.load()->sessionMap_; + auto it = sessionMap.find(session_id); + if (it != sessionMap.end()) { return it->second; } return Status::SessionNotFound(); @@ -3752,7 +3774,7 @@ bool MetaClient::checkIsPlanKilled(SessionID sessionId, ExecutionPlanID planId) return false; } folly::rcu_reader guard; - return killedPlans_.load()->count({sessionId, planId}); + return metadata_.load()->killedPlans_.count({sessionId, planId}); } Status MetaClient::verifyVersion() { diff --git a/src/clients/meta/MetaClient.h b/src/clients/meta/MetaClient.h index 7103c88e6c7..8ac7e42ff02 100644 --- a/src/clients/meta/MetaClient.h +++ b/src/clients/meta/MetaClient.h @@ -7,6 +7,7 @@ #define CLIENTS_META_METACLIENT_H_ #include +#include #include #include #include @@ -145,9 +146,9 @@ using UserRolesMap = std::unordered_map // get user password by account using UserPasswordMap = std::unordered_map; // Mapping of user name and remaining wrong password attempts -using UserPasswordAttemptsRemain = std::unordered_map; +using UserPasswordAttemptsRemain = folly::ConcurrentHashMap; // Mapping of user name and the timestamp when the account is locked -using UserLoginLockTime = std::unordered_map; +using UserLoginLockTime = folly::ConcurrentHashMap; // config cache, get config via module and name using MetaConfigMap = @@ -239,6 +240,8 @@ class MetaClient { bool waitForMetadReady(int count = -1, int retryIntervalSecs = FLAGS_heartbeat_interval_secs); + void notifyStop(); + void stop(); void registerListener(MetaChangedListener* listener) { @@ -773,7 +776,7 @@ class MetaClient { std::atomic metadLastUpdateTime_{0}; int64_t metaServerVersion_{-1}; - static constexpr int64_t EXPECT_META_VERSION = 2; + static constexpr int64_t EXPECT_META_VERSION = 3; // leadersLock_ is used to protect leadersInfo folly::RWSpinLock leadersLock_; @@ -789,7 +792,8 @@ class MetaClient { // Only report dir info once when started bool dirInfoReported_ = false; - struct ThreadLocalInfo { + + struct MetaData { int64_t localLastUpdateTime_{-2}; LocalCache localCache_; SpaceNameIdMap spaceIndexByName_; @@ -805,9 +809,12 @@ class MetaClient { std::vector storageHosts_; FTIndexMap fulltextIndexMap_; UserPasswordMap userPasswordMap_; - }; - const ThreadLocalInfo& getThreadLocalInfo(); + SessionMap sessionMap_; + folly::F14FastSet> killedPlans_; + + ServiceClientsList serviceClientList_; + }; void addSchemaField(NebulaSchemaProvider* schema, const cpp2::ColumnDef& col, ObjectPool* pool); @@ -836,7 +843,6 @@ class MetaClient { ServiceClientsList serviceClientList_; FTIndexMap fulltextIndexMap_; - mutable folly::RWSpinLock localCacheLock_; // The listener_ is the NebulaStore MetaChangedListener* listener_{nullptr}; // The lock used to protect listener_ @@ -854,8 +860,9 @@ class MetaClient { MetaClientOptions options_; std::vector storageHosts_; int64_t heartbeatTime_; - std::atomic sessionMap_; - std::atomic>*> killedPlans_; + SessionMap sessionMap_; + folly::F14FastSet> killedPlans_; + std::atomic metadata_; }; } // namespace meta diff --git a/src/common/expression/test/CMakeLists.txt b/src/common/expression/test/CMakeLists.txt index b71eb7aac88..6c00bd7031f 100644 --- a/src/common/expression/test/CMakeLists.txt +++ b/src/common/expression/test/CMakeLists.txt @@ -48,6 +48,14 @@ set(expression_test_common_libs $ ) + +if(ENABLE_STANDALONE_VERSION) +set(expression_test_common_libs + ${expression_test_common_libs} + $ +) +endif() + nebula_add_library( expr_ctx_mock_obj OBJECT ExpressionContextMock.cpp diff --git a/src/common/function/FunctionManager.cpp b/src/common/function/FunctionManager.cpp index 2735b9b1d41..7262b7fea9c 100644 --- a/src/common/function/FunctionManager.cpp +++ b/src/common/function/FunctionManager.cpp @@ -38,6 +38,7 @@ std::unordered_map FunctionManager::variadicFunReturnT {"concat", Value::Type::STRING}, {"concat_ws", Value::Type::STRING}, {"cos_similarity", Value::Type::FLOAT}, + {"coalesce", Value::Type::__EMPTY__}, }; std::unordered_map> FunctionManager::typeSignature_ = { @@ -283,10 +284,6 @@ std::unordered_map> FunctionManager::typ { TypeSignature({Value::Type::LIST}, Value::Type::__EMPTY__), }}, - {"coalesce", - { - TypeSignature({Value::Type::LIST}, Value::Type::__EMPTY__), - }}, {"range", {TypeSignature({Value::Type::INT, Value::Type::INT}, Value::Type::LIST), TypeSignature({Value::Type::INT, Value::Type::INT, Value::Type::INT}, Value::Type::LIST)}}, @@ -1757,7 +1754,7 @@ FunctionManager::FunctionManager() { attr.maxArity_ = 1; attr.isPure_ = false; attr.body_ = [](const auto &args) -> Value { - if (args.size() == 0) { + if (args.empty()) { return time::WallClock::fastNowInSec(); } auto status = time::TimeUtils::toTimestamp(args[0].get()); @@ -2092,29 +2089,15 @@ FunctionManager::FunctionManager() { { auto &attr = functions_["coalesce"]; attr.minArity_ = 1; - attr.maxArity_ = 1; + attr.maxArity_ = INT64_MAX; attr.isPure_ = true; attr.body_ = [](const auto &args) -> Value { - switch (args[0].get().type()) { - case Value::Type::NULLVALUE: { - return Value::kNullValue; - } - case Value::Type::LIST: { - auto &list = args[0].get().getList(); - if (list.values.empty()) { - return Value::kNullValue; - } - for (auto &i : list.values) { - if (i != Value::kNullValue) { - return i; - } - } - return Value::kNullValue; - } - default: { - return Value::kNullBadType; + for (size_t i = 0; i < args.size(); ++i) { + if (args[i].get().type() != Value::Type::NULLVALUE) { + return args[i].get(); } } + return Value::kNullValue; }; } { diff --git a/src/common/function/test/FunctionManagerTest.cpp b/src/common/function/test/FunctionManagerTest.cpp index 0bcc1f133d7..7c0dc1a423e 100644 --- a/src/common/function/test/FunctionManagerTest.cpp +++ b/src/common/function/test/FunctionManagerTest.cpp @@ -1648,26 +1648,29 @@ TEST_F(FunctionManagerTest, ScalarFunctionTest) { TEST_FUNCTION(last, args_["nullvalue"], Value::kNullValue); TEST_FUNCTION(coalesce, args_["nullvalue"], Value::kNullValue); + TEST_FUNCTION(coalesce, args_["range1"], 1); + TEST_FUNCTION(coalesce, args_["float"], 1.1); + std::vector args; List list; list.values.emplace_back(Value::kNullValue); - args.push_back(list); + args.emplace_back(list); TEST_FUNCTION(head, args, Value::kNullValue); TEST_FUNCTION(last, args, Value::kNullValue); - TEST_FUNCTION(coalesce, args, Value::kNullValue); + TEST_FUNCTION(coalesce, args, list); list.values.insert(list.values.begin(), "head"); args[0] = list; TEST_FUNCTION(head, args, "head"); TEST_FUNCTION(last, args, Value::kNullValue); - TEST_FUNCTION(coalesce, args, "head"); + TEST_FUNCTION(coalesce, args, list); list.values.emplace_back("last"); args[0] = list; TEST_FUNCTION(head, args, "head") TEST_FUNCTION(last, args, "last"); - TEST_FUNCTION(coalesce, args, "head"); + TEST_FUNCTION(coalesce, args, list); } { // length(null) return null diff --git a/src/daemons/CMakeLists.txt b/src/daemons/CMakeLists.txt index 1d476baf543..026ca7512b9 100644 --- a/src/daemons/CMakeLists.txt +++ b/src/daemons/CMakeLists.txt @@ -99,6 +99,7 @@ nebula_add_executable( $ $ $ + $ $ ${common_deps} ${storage_meta_deps} diff --git a/src/daemons/GraphDaemon.cpp b/src/daemons/GraphDaemon.cpp index 0b76f558120..217ce7aa1f7 100644 --- a/src/daemons/GraphDaemon.cpp +++ b/src/daemons/GraphDaemon.cpp @@ -30,8 +30,8 @@ using nebula::fs::FileUtils; using nebula::graph::GraphService; using nebula::network::NetworkUtils; -static void signalHandler(int sig); -static Status setupSignalHandler(); +static void signalHandler(nebula::graph::GraphServer *graphServer, int sig); +static Status setupSignalHandler(nebula::graph::GraphServer *graphServer); static void printHelp(const char *prog); #if defined(__x86_64__) extern Status setupBreakpad(); @@ -40,8 +40,6 @@ extern Status setupBreakpad(); DECLARE_string(flagfile); DECLARE_bool(containerized); -std::unique_ptr gServer; - int main(int argc, char *argv[]) { google::SetVersionString(nebula::versionString()); if (argc == 1) { @@ -153,37 +151,37 @@ int main(int argc, char *argv[]) { } LOG(INFO) << "Number of worker threads: " << FLAGS_num_worker_threads; + auto graphServer = std::make_unique(localhost); // Setup the signal handlers - status = setupSignalHandler(); + status = setupSignalHandler(graphServer.get()); if (!status.ok()) { LOG(ERROR) << status; return EXIT_FAILURE; } - gServer = std::make_unique(localhost); - - if (!gServer->start()) { + if (!graphServer->start()) { LOG(ERROR) << "The graph server start failed"; return EXIT_FAILURE; } - gServer->waitUntilStop(); + graphServer->waitUntilStop(); LOG(INFO) << "The graph Daemon stopped"; return EXIT_SUCCESS; } -Status setupSignalHandler() { +Status setupSignalHandler(nebula::graph::GraphServer *graphServer) { return nebula::SignalHandler::install( - {SIGINT, SIGTERM}, - [](nebula::SignalHandler::GeneralSignalInfo *info) { signalHandler(info->sig()); }); + {SIGINT, SIGTERM}, [graphServer](nebula::SignalHandler::GeneralSignalInfo *info) { + signalHandler(graphServer, info->sig()); + }); } -void signalHandler(int sig) { +void signalHandler(nebula::graph::GraphServer *graphServer, int sig) { switch (sig) { case SIGINT: case SIGTERM: FLOG_INFO("Signal %d(%s) received, stopping this server", sig, ::strsignal(sig)); - gServer->notifyStop(); + graphServer->notifyStop(); break; default: FLOG_ERROR("Signal %d(%s) received but ignored", sig, ::strsignal(sig)); diff --git a/src/daemons/MetaDaemon.cpp b/src/daemons/MetaDaemon.cpp index 211a88d17d0..8c8eb306611 100644 --- a/src/daemons/MetaDaemon.cpp +++ b/src/daemons/MetaDaemon.cpp @@ -53,12 +53,11 @@ DEFINE_int32(meta_http_thread_num, 3, "Number of meta daemon's http thread"); DEFINE_string(pid_file, "pids/nebula-metad.pid", "File to hold the process id"); DEFINE_bool(daemonize, true, "Whether run as a daemon process"); -static std::unique_ptr gServer; static std::unique_ptr gKVStore; -static void signalHandler(int sig); +static void signalHandler(apache::thrift::ThriftServer* metaServer, int sig); static void waitForStop(); -static Status setupSignalHandler(); +static Status setupSignalHandler(apache::thrift::ThriftServer* metaServer); #if defined(__x86_64__) extern Status setupBreakpad(); #endif @@ -189,8 +188,9 @@ int main(int argc, char* argv[]) { } } + auto metaServer = std::make_unique(); // Setup the signal handlers - status = setupSignalHandler(); + status = setupSignalHandler(metaServer.get()); if (!status.ok()) { LOG(ERROR) << status; return EXIT_FAILURE; @@ -215,14 +215,13 @@ int main(int argc, char* argv[]) { std::make_shared(gKVStore.get(), metaClusterId()); LOG(INFO) << "The meta daemon start on " << localhost; try { - gServer = std::make_unique(); - gServer->setPort(FLAGS_port); - gServer->setIdleTimeout(std::chrono::seconds(0)); // No idle timeout on client connection - gServer->setInterface(std::move(handler)); + metaServer->setPort(FLAGS_port); + metaServer->setIdleTimeout(std::chrono::seconds(0)); // No idle timeout on client connection + metaServer->setInterface(std::move(handler)); if (FLAGS_enable_ssl || FLAGS_enable_meta_ssl) { - gServer->setSSLConfig(nebula::sslContextConfig()); + metaServer->setSSLConfig(nebula::sslContextConfig()); } - gServer->serve(); // Will wait until the server shuts down + metaServer->serve(); // Will wait until the server shuts down waitForStop(); } catch (const std::exception& e) { LOG(ERROR) << "Exception thrown: " << e.what(); @@ -233,19 +232,20 @@ int main(int argc, char* argv[]) { return EXIT_SUCCESS; } -Status setupSignalHandler() { +Status setupSignalHandler(apache::thrift::ThriftServer* metaServer) { return nebula::SignalHandler::install( - {SIGINT, SIGTERM}, - [](nebula::SignalHandler::GeneralSignalInfo* info) { signalHandler(info->sig()); }); + {SIGINT, SIGTERM}, [metaServer](nebula::SignalHandler::GeneralSignalInfo* info) { + signalHandler(metaServer, info->sig()); + }); } -void signalHandler(int sig) { +void signalHandler(apache::thrift::ThriftServer* metaServer, int sig) { switch (sig) { case SIGINT: case SIGTERM: FLOG_INFO("Signal %d(%s) received, stopping this server", sig, ::strsignal(sig)); - if (gServer) { - gServer->stop(); + if (metaServer) { + metaServer->stop(); } break; default: diff --git a/src/daemons/MetaDaemonInit.cpp b/src/daemons/MetaDaemonInit.cpp index c41b1e02c06..09640d5dc58 100644 --- a/src/daemons/MetaDaemonInit.cpp +++ b/src/daemons/MetaDaemonInit.cpp @@ -138,26 +138,22 @@ std::unique_ptr initKV(std::vector p LOG(ERROR) << "Meta version is invalid"; return nullptr; } else if (version == nebula::meta::MetaVersion::V1) { - if (leader == localhost) { - LOG(INFO) << "I am leader, begin upgrade meta data"; - // need to upgrade the v1.0 meta data format to v2.0 meta data format - auto ret = nebula::meta::MetaVersionMan::updateMetaV1ToV2(kvstore.get()); - if (!ret.ok()) { - LOG(ERROR) << ret; - return nullptr; - } - } else { - LOG(INFO) << "I am follower, wait for leader to sync upgrade"; - while (version != nebula::meta::MetaVersion::V2) { - VLOG(1) << "Waiting for leader to upgrade"; - sleep(1); - version = nebula::meta::MetaVersionMan::getMetaVersionFromKV(kvstore.get()); - } + auto ret = nebula::meta::MetaVersionMan::updateMetaV1ToV2(kvstore.get()); + if (!ret.ok()) { + LOG(ERROR) << ret; + return nullptr; + } + + nebula::meta::MetaVersionMan::setMetaVersionToKV(kvstore.get(), nebula::meta::MetaVersion::V2); + } else if (version == nebula::meta::MetaVersion::V2) { + LOG(INFO) << "version 3"; + auto ret = nebula::meta::MetaVersionMan::updateMetaV2ToV3(kvstore.get()); + if (!ret.ok()) { + LOG(ERROR) << ret; + return nullptr; } - } - if (leader == localhost) { - nebula::meta::MetaVersionMan::setMetaVersionToKV(kvstore.get()); + nebula::meta::MetaVersionMan::setMetaVersionToKV(kvstore.get(), nebula::meta::MetaVersion::V3); } LOG(INFO) << "Nebula store init succeeded, clusterId " << gClusterId; diff --git a/src/daemons/StorageDaemon.cpp b/src/daemons/StorageDaemon.cpp index e40625b531d..5b22349b83d 100644 --- a/src/daemons/StorageDaemon.cpp +++ b/src/daemons/StorageDaemon.cpp @@ -45,14 +45,12 @@ using nebula::Status; using nebula::StatusOr; using nebula::network::NetworkUtils; -static void signalHandler(int sig); -static Status setupSignalHandler(); +static void signalHandler(nebula::storage::StorageServer *storageServer, int sig); +static Status setupSignalHandler(nebula::storage::StorageServer *storageServer); #if defined(__x86_64__) extern Status setupBreakpad(); #endif -std::unique_ptr gStorageServer; - int main(int argc, char *argv[]) { google::SetVersionString(nebula::versionString()); // Detect if the server has already been started @@ -150,8 +148,10 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } + auto storageServer = std::make_unique( + localhost, metaAddrsRet.value(), paths, FLAGS_wal_path, FLAGS_listener_path); // Setup the signal handlers - status = setupSignalHandler(); + status = setupSignalHandler(storageServer.get()); if (!status.ok()) { LOG(ERROR) << status; return EXIT_FAILURE; @@ -172,32 +172,31 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } - gStorageServer = std::make_unique( - localhost, metaAddrsRet.value(), paths, FLAGS_wal_path, FLAGS_listener_path); - if (!gStorageServer->start()) { + if (!storageServer->start()) { LOG(ERROR) << "Storage server start failed"; - gStorageServer->stop(); + storageServer->stop(); return EXIT_FAILURE; } - gStorageServer->waitUntilStop(); + storageServer->waitUntilStop(); LOG(INFO) << "The storage Daemon stopped"; return EXIT_SUCCESS; } -Status setupSignalHandler() { +Status setupSignalHandler(nebula::storage::StorageServer *storageServer) { return nebula::SignalHandler::install( - {SIGINT, SIGTERM}, - [](nebula::SignalHandler::GeneralSignalInfo *info) { signalHandler(info->sig()); }); + {SIGINT, SIGTERM}, [storageServer](nebula::SignalHandler::GeneralSignalInfo *info) { + signalHandler(storageServer, info->sig()); + }); } -void signalHandler(int sig) { +void signalHandler(nebula::storage::StorageServer *storageServer, int sig) { switch (sig) { case SIGINT: case SIGTERM: FLOG_INFO("Signal %d(%s) received, stopping this server", sig, ::strsignal(sig)); - if (gStorageServer) { - gStorageServer->notifyStop(); + if (storageServer) { + storageServer->notifyStop(); } break; default: diff --git a/src/graph/context/test/CMakeLists.txt b/src/graph/context/test/CMakeLists.txt index 779335bc5a0..450f869d4b9 100644 --- a/src/graph/context/test/CMakeLists.txt +++ b/src/graph/context/test/CMakeLists.txt @@ -48,6 +48,13 @@ SET(CONTEXT_TEST_LIBS $ ) +if(ENABLE_STANDALONE_VERSION) +set(CONTEXT_TEST_LIBS + ${CONTEXT_TEST_LIBS} + $ +) +endif() + nebula_add_test( NAME context_test SOURCES diff --git a/src/graph/executor/StorageAccessExecutor.h b/src/graph/executor/StorageAccessExecutor.h index 01eccc22f67..37bb0454213 100644 --- a/src/graph/executor/StorageAccessExecutor.h +++ b/src/graph/executor/StorageAccessExecutor.h @@ -79,7 +79,8 @@ class StorageAccessExecutor : public Executor { return Status::Error(std::move(error)); } case nebula::cpp2::ErrorCode::E_LEADER_CHANGED: - return Status::Error("Storage Error: The leader has changed. Try again later"); + return Status::Error( + folly::sformat("Storage Error: Not the leader of {}. Please retry later.", partId)); case nebula::cpp2::ErrorCode::E_INVALID_FILTER: return Status::Error("Storage Error: Invalid filter."); case nebula::cpp2::ErrorCode::E_INVALID_UPDATER: @@ -88,6 +89,8 @@ class StorageAccessExecutor : public Executor { return Status::Error("Storage Error: Invalid space vid len."); case nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND: return Status::Error("Storage Error: Space not found."); + case nebula::cpp2::ErrorCode::E_PART_NOT_FOUND: + return Status::Error(folly::sformat("Storage Error: Part {} not found.", partId)); case nebula::cpp2::ErrorCode::E_TAG_NOT_FOUND: return Status::Error("Storage Error: Tag not found."); case nebula::cpp2::ErrorCode::E_TAG_PROP_NOT_FOUND: @@ -108,14 +111,25 @@ class StorageAccessExecutor : public Executor { "The not null field doesn't have a default value."); case nebula::cpp2::ErrorCode::E_OUT_OF_RANGE: return Status::Error("Storage Error: Out of range value."); - case nebula::cpp2::ErrorCode::E_ATOMIC_OP_FAILED: - return Status::Error("Storage Error: Atomic operation failed."); case nebula::cpp2::ErrorCode::E_DATA_CONFLICT_ERROR: return Status::Error( "Storage Error: More than one request trying to " "add/update/delete one edge/vertex at the same time."); case nebula::cpp2::ErrorCode::E_FILTER_OUT: return Status::OK(); + case nebula::cpp2::ErrorCode::E_RAFT_TERM_OUT_OF_DATE: + return Status::Error(folly::sformat( + "Storage Error: Term of part {} is out of date. Please retry later.", partId)); + case nebula::cpp2::ErrorCode::E_RAFT_WAL_FAIL: + return Status::Error("Storage Error: Write wal failed. Probably disk is almost full."); + case nebula::cpp2::ErrorCode::E_RAFT_WRITE_BLOCKED: + return Status::Error( + "Storage Error: Write is blocked when creating snapshot. Please retry later."); + case nebula::cpp2::ErrorCode::E_RAFT_BUFFER_OVERFLOW: + return Status::Error(folly::sformat( + "Storage Error: Part {} raft buffer is full. Please retry later.", partId)); + case nebula::cpp2::ErrorCode::E_RAFT_ATOMIC_OP_FAILED: + return Status::Error("Storage Error: Atomic operation failed."); default: auto status = Status::Error("Storage Error: part: %d, error: %s(%d).", partId, diff --git a/src/graph/executor/query/LeftJoinExecutor.cpp b/src/graph/executor/query/LeftJoinExecutor.cpp index 9dddd2ee0d2..6a8725ad076 100644 --- a/src/graph/executor/query/LeftJoinExecutor.cpp +++ b/src/graph/executor/query/LeftJoinExecutor.cpp @@ -33,14 +33,14 @@ folly::Future LeftJoinExecutor::join(const std::vector& has DataSet result; if (hashKeys.size() == 1 && probeKeys.size() == 1) { std::unordered_map> hashTable; - hashTable.reserve(rhsIter_->size() == 0 ? 1 : rhsIter_->size()); + hashTable.reserve(rhsIter_->empty() ? 1 : rhsIter_->size()); if (!lhsIter_->empty()) { buildSingleKeyHashTable(probeKeys.front(), rhsIter_.get(), hashTable); result = singleKeyProbe(hashKeys.front(), lhsIter_.get(), hashTable); } } else { std::unordered_map> hashTable; - hashTable.reserve(rhsIter_->size() == 0 ? 1 : rhsIter_->size()); + hashTable.reserve(rhsIter_->empty() ? 1 : rhsIter_->size()); if (!lhsIter_->empty()) { buildHashTable(probeKeys, rhsIter_.get(), hashTable); result = probe(hashKeys, lhsIter_.get(), hashTable); diff --git a/src/graph/executor/test/CMakeLists.txt b/src/graph/executor/test/CMakeLists.txt index 5f8aba7d7fb..4e567905da9 100644 --- a/src/graph/executor/test/CMakeLists.txt +++ b/src/graph/executor/test/CMakeLists.txt @@ -58,6 +58,20 @@ SET(EXEC_QUERY_TEST_OBJS $ ) +if(ENABLE_STANDALONE_VERSION) + +nebula_add_library( + storage_server_stub_obj OBJECT + StorageServerStub.cpp +) + +set(EXEC_QUERY_TEST_OBJS + ${EXEC_QUERY_TEST_OBJS} + $ + $ +) +endif() + SET(EXEC_QUERY_TEST_LIBS ${THRIFT_LIBRARIES} gtest diff --git a/src/graph/executor/test/StorageServerStub.cpp b/src/graph/executor/test/StorageServerStub.cpp new file mode 100644 index 00000000000..87b2a22528d --- /dev/null +++ b/src/graph/executor/test/StorageServerStub.cpp @@ -0,0 +1,154 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/Base.h" +#include "storage/GraphStorageLocalServer.h" + +#define LOCAL_RETURN_FUTURE(threadManager, respType, callFunc) \ + UNUSED(request); \ + auto promise = std::make_shared>(); \ + respType dummyResp; \ + auto f = promise->getFuture(); \ + promise->setValue(std::move(dummyResp)); \ + return f; + +namespace nebula::storage { + +std::mutex mutex_; +std::shared_ptr instance_ = nullptr; + +void GraphStorageLocalServer::setThreadManager( + std::shared_ptr threadManager) { + // lock? + threadManager_ = threadManager; +} + +void GraphStorageLocalServer::setInterface( + std::shared_ptr handler) { + handler_ = handler; +} + +void GraphStorageLocalServer::serve() { + if (serving_) { + LOG(WARNING) << "Server already serving"; + return; + } + // do nothing, wait stop + serving_ = true; + sem_.wait(); +} + +void GraphStorageLocalServer::stop() { + if (!serving_) { + LOG(WARNING) << "Can't stop server not serving"; + return; + } + sem_.signal(); + serving_ = false; +} + +folly::Future GraphStorageLocalServer::future_getNeighbors( + const cpp2::GetNeighborsRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::GetNeighborsResponse, future_getNeighbors); +} + +folly::Future GraphStorageLocalServer::future_addVertices( + const cpp2::AddVerticesRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::ExecResponse, future_addVertices); +} + +folly::Future GraphStorageLocalServer::future_chainAddEdges( + const cpp2::AddEdgesRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::ExecResponse, future_chainAddEdges); +} + +folly::Future GraphStorageLocalServer::future_addEdges( + const cpp2::AddEdgesRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::ExecResponse, future_addEdges); +} + +folly::Future GraphStorageLocalServer::future_getProps( + const cpp2::GetPropRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::GetPropResponse, future_getProps); +} + +folly::Future GraphStorageLocalServer::future_deleteEdges( + const cpp2::DeleteEdgesRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::ExecResponse, future_deleteEdges); +} + +folly::Future GraphStorageLocalServer::future_chainDeleteEdges( + const cpp2::DeleteEdgesRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::ExecResponse, future_chainDeleteEdges); +} + +folly::Future GraphStorageLocalServer::future_deleteVertices( + const cpp2::DeleteVerticesRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::ExecResponse, future_deleteVertices); +} + +folly::Future GraphStorageLocalServer::future_deleteTags( + const cpp2::DeleteTagsRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::ExecResponse, future_deleteTags); +} + +folly::Future GraphStorageLocalServer::future_updateVertex( + const cpp2::UpdateVertexRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::UpdateResponse, future_updateVertex); +} + +folly::Future GraphStorageLocalServer::future_chainUpdateEdge( + const cpp2::UpdateEdgeRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::UpdateResponse, future_chainUpdateEdge); +} + +folly::Future GraphStorageLocalServer::future_updateEdge( + const cpp2::UpdateEdgeRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::UpdateResponse, future_updateEdge); +} + +folly::Future GraphStorageLocalServer::future_getUUID( + const cpp2::GetUUIDReq& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::GetUUIDResp, future_getUUID); +} + +folly::Future GraphStorageLocalServer::future_lookupIndex( + const cpp2::LookupIndexRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::LookupIndexResp, future_lookupIndex); +} + +folly::Future GraphStorageLocalServer::future_lookupAndTraverse( + const cpp2::LookupAndTraverseRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::GetNeighborsResponse, future_lookupAndTraverse); +} + +folly::Future GraphStorageLocalServer::future_scanVertex( + const cpp2::ScanVertexRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::ScanResponse, future_scanVertex); +} + +folly::Future GraphStorageLocalServer::future_scanEdge( + const cpp2::ScanEdgeRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::ScanResponse, future_scanEdge); +} + +folly::Future GraphStorageLocalServer::future_get( + const cpp2::KVGetRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::KVGetResponse, future_get); +} + +folly::Future GraphStorageLocalServer::future_put( + const cpp2::KVPutRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::ExecResponse, future_put); +} + +folly::Future GraphStorageLocalServer::future_remove( + const cpp2::KVRemoveRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::ExecResponse, future_remove); +} + +} // namespace nebula::storage diff --git a/src/graph/optimizer/test/CMakeLists.txt b/src/graph/optimizer/test/CMakeLists.txt index ae0f1357bb3..96f3937b2d3 100644 --- a/src/graph/optimizer/test/CMakeLists.txt +++ b/src/graph/optimizer/test/CMakeLists.txt @@ -54,6 +54,13 @@ set(OPTIMIZER_TEST_LIB $ ) +if(ENABLE_STANDALONE_VERSION) +set(OPTIMIZER_TEST_LIB + ${OPTIMIZER_TEST_LIB} + $ +) +endif() + nebula_add_test( NAME index_scan_rule_test diff --git a/src/graph/planner/match/ScanSeek.cpp b/src/graph/planner/match/ScanSeek.cpp index 8b08a157858..659efeb6be1 100644 --- a/src/graph/planner/match/ScanSeek.cpp +++ b/src/graph/planner/match/ScanSeek.cpp @@ -28,7 +28,7 @@ StatusOr ScanSeek::transformEdge(EdgeContext *edgeCtx) { bool ScanSeek::matchNode(NodeContext *nodeCtx) { auto &node = *nodeCtx->info; // only require the tag - if (node.tids.size() == 0) { + if (node.tids.empty()) { // empty labels means all labels const auto *qctx = nodeCtx->matchClauseCtx->qctx; auto allLabels = qctx->schemaMng()->getAllTags(nodeCtx->matchClauseCtx->space.id); diff --git a/src/graph/planner/test/CMakeLists.txt b/src/graph/planner/test/CMakeLists.txt index ac07fbb708f..ab1f8c50f4a 100644 --- a/src/graph/planner/test/CMakeLists.txt +++ b/src/graph/planner/test/CMakeLists.txt @@ -2,6 +2,18 @@ # # This source code is licensed under Apache 2.0 License. +set(EXEC_PLAN_TEST_FLAG_DEPS + $ +) + +if(ENABLE_STANDALONE_VERSION) +set(EXEC_PLAN_TEST_FLAG_DEPS + ${EXEC_PLAN_TEST_FLAG_DEPS} + $ + $ +) +endif() + nebula_add_test( NAME execution_plan_test SOURCES @@ -39,7 +51,7 @@ nebula_add_test( $ $ $ - $ + ${EXEC_PLAN_TEST_FLAG_DEPS} $ $ $ diff --git a/src/graph/service/CMakeLists.txt b/src/graph/service/CMakeLists.txt index 2ff0c6406dc..7e55616b2b8 100644 --- a/src/graph/service/CMakeLists.txt +++ b/src/graph/service/CMakeLists.txt @@ -27,3 +27,4 @@ nebula_add_library( CloudAuthenticator.cpp ) +nebula_add_subdirectory(test) diff --git a/src/graph/service/test/CMakeLists.txt b/src/graph/service/test/CMakeLists.txt new file mode 100644 index 00000000000..c1678473db8 --- /dev/null +++ b/src/graph/service/test/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +# note: These flags is used for standalone version ut. + +nebula_add_library( + sa_test_graph_flags_obj OBJECT + StandAloneTestGraphFlags.cpp +) diff --git a/src/graph/service/test/StandAloneTestGraphFlags.cpp b/src/graph/service/test/StandAloneTestGraphFlags.cpp new file mode 100644 index 00000000000..de8de4c59d5 --- /dev/null +++ b/src/graph/service/test/StandAloneTestGraphFlags.cpp @@ -0,0 +1,13 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "graph/service/GraphFlags.h" +#include "version/Version.h" + +DEFINE_uint32(ft_request_retry_times, 3, "Retry times if fulltext request failed"); +DEFINE_bool(enable_client_white_list, true, "Turn on/off the client white list."); +DEFINE_string(client_white_list, + nebula::getOriginVersion() + ":2.5.0:2.5.1:2.6.0", + "A white list for different client versions, separate with colon."); diff --git a/src/graph/session/GraphSessionManager.cpp b/src/graph/session/GraphSessionManager.cpp index a7ebdccbbc6..f7fda2cd93a 100644 --- a/src/graph/session/GraphSessionManager.cpp +++ b/src/graph/session/GraphSessionManager.cpp @@ -148,6 +148,7 @@ void GraphSessionManager::removeSession(SessionID id) { if (!resp.ok()) { // it will delete by reclaim LOG(ERROR) << "Remove session `" << id << "' failed: " << resp.status(); + return; } activeSessions_.erase(iter); } diff --git a/src/graph/util/ExpressionUtils.cpp b/src/graph/util/ExpressionUtils.cpp index 22e7d80c247..77cb5de29a3 100644 --- a/src/graph/util/ExpressionUtils.cpp +++ b/src/graph/util/ExpressionUtils.cpp @@ -6,8 +6,10 @@ #include "graph/util/ExpressionUtils.h" #include +#include #include "common/base/ObjectPool.h" +#include "common/expression/Expression.h" #include "common/expression/PropertyExpression.h" #include "common/function/AggFunctionManager.h" #include "graph/context/QueryContext.h" @@ -984,5 +986,183 @@ bool ExpressionUtils::isGeoIndexAcceleratedPredicate(const Expression *expr) { return false; } +bool ExpressionUtils::checkExprDepth(const Expression *expr) { + std::queue exprQueue; + exprQueue.emplace(expr); + int depth = 0; + while (!exprQueue.empty()) { + int size = exprQueue.size(); + while (size > 0) { + const Expression *cur = exprQueue.front(); + exprQueue.pop(); + switch (cur->kind()) { + case Expression::Kind::kConstant: + case Expression::Kind::kVar: { + break; + } + case Expression::Kind::kAdd: + case Expression::Kind::kMinus: + case Expression::Kind::kMultiply: + case Expression::Kind::kDivision: + case Expression::Kind::kMod: { + auto *ariExpr = static_cast(cur); + exprQueue.emplace(ariExpr->left()); + exprQueue.emplace(ariExpr->right()); + break; + } + case Expression::Kind::kIsNull: + case Expression::Kind::kIsNotNull: + case Expression::Kind::kIsEmpty: + case Expression::Kind::kIsNotEmpty: + case Expression::Kind::kUnaryPlus: + case Expression::Kind::kUnaryNegate: + case Expression::Kind::kUnaryNot: + case Expression::Kind::kUnaryIncr: + case Expression::Kind::kUnaryDecr: { + auto *unaExpr = static_cast(cur); + exprQueue.emplace(unaExpr->operand()); + break; + } + case Expression::Kind::kRelEQ: + case Expression::Kind::kRelNE: + case Expression::Kind::kRelLT: + case Expression::Kind::kRelLE: + case Expression::Kind::kRelGT: + case Expression::Kind::kRelGE: + case Expression::Kind::kRelREG: + case Expression::Kind::kContains: + case Expression::Kind::kNotContains: + case Expression::Kind::kStartsWith: + case Expression::Kind::kNotStartsWith: + case Expression::Kind::kEndsWith: + case Expression::Kind::kNotEndsWith: + case Expression::Kind::kRelNotIn: + case Expression::Kind::kRelIn: { + auto *relExpr = static_cast(cur); + exprQueue.emplace(relExpr->left()); + exprQueue.emplace(relExpr->right()); + break; + } + case Expression::Kind::kList: { + auto *listExpr = static_cast(cur); + for (auto &item : listExpr->items()) { + exprQueue.emplace(item); + } + break; + } + case Expression::Kind::kSet: { + auto *setExpr = static_cast(cur); + for (auto &item : setExpr->items()) { + exprQueue.emplace(item); + } + break; + } + case Expression::Kind::kMap: { + auto *mapExpr = static_cast(cur); + for (auto &item : mapExpr->items()) { + exprQueue.emplace(item.second); + } + break; + } + case Expression::Kind::kCase: { + auto *caseExpr = static_cast(cur); + if (caseExpr->hasCondition()) { + exprQueue.emplace(caseExpr->condition()); + } + if (caseExpr->hasDefault()) { + exprQueue.emplace(caseExpr->defaultResult()); + } + for (auto &whenThen : caseExpr->cases()) { + exprQueue.emplace(whenThen.when); + exprQueue.emplace(whenThen.then); + } + break; + } + case Expression::Kind::kListComprehension: { + auto *lcExpr = static_cast(cur); + exprQueue.emplace(lcExpr->collection()); + if (lcExpr->hasFilter()) { + exprQueue.emplace(lcExpr->filter()); + } + if (lcExpr->hasMapping()) { + exprQueue.emplace(lcExpr->mapping()); + } + break; + } + case Expression::Kind::kPredicate: { + auto *predExpr = static_cast(cur); + exprQueue.emplace(predExpr->collection()); + if (predExpr->hasFilter()) { + exprQueue.emplace(predExpr->filter()); + } + break; + } + case Expression::Kind::kReduce: { + auto *reduceExpr = static_cast(cur); + exprQueue.emplace(reduceExpr->collection()); + exprQueue.emplace(reduceExpr->mapping()); + break; + } + case Expression::Kind::kLogicalAnd: + case Expression::Kind::kLogicalOr: + case Expression::Kind::kLogicalXor: { + auto *logExpr = static_cast(cur); + for (auto &op : logExpr->operands()) { + exprQueue.emplace(op); + } + break; + } + case Expression::Kind::kTypeCasting: { + auto *typExpr = static_cast(cur); + exprQueue.emplace(typExpr->operand()); + break; + } + case Expression::Kind::kFunctionCall: { + auto *funExpr = static_cast(cur); + auto &args = funExpr->args()->args(); + for (auto iter = args.begin(); iter < args.end(); ++iter) { + exprQueue.emplace(*iter); + } + break; + } + case Expression::Kind::kTagProperty: + case Expression::Kind::kSrcProperty: + case Expression::Kind::kEdgeRank: + case Expression::Kind::kEdgeDst: + case Expression::Kind::kEdgeSrc: + case Expression::Kind::kEdgeType: + case Expression::Kind::kEdgeProperty: + case Expression::Kind::kInputProperty: + case Expression::Kind::kSubscript: + case Expression::Kind::kAttribute: + case Expression::Kind::kLabelAttribute: + case Expression::Kind::kVertex: + case Expression::Kind::kEdge: + case Expression::Kind::kLabel: + case Expression::Kind::kVarProperty: + case Expression::Kind::kDstProperty: + case Expression::Kind::kUUID: + case Expression::Kind::kPathBuild: + case Expression::Kind::kColumn: + case Expression::Kind::kTSPrefix: + case Expression::Kind::kTSWildcard: + case Expression::Kind::kTSRegexp: + case Expression::Kind::kTSFuzzy: + case Expression::Kind::kAggregate: + case Expression::Kind::kSubscriptRange: + case Expression::Kind::kVersionedVar: + default: + break; + } + size -= 1; + } + depth += 1; + if (depth > ExpressionUtils::kMaxDepth) { + return false; + } + } + return true; +} // namespace graph + } // namespace graph } // namespace nebula diff --git a/src/graph/util/ExpressionUtils.h b/src/graph/util/ExpressionUtils.h index 19193cb7a86..f42241f348b 100644 --- a/src/graph/util/ExpressionUtils.h +++ b/src/graph/util/ExpressionUtils.h @@ -164,6 +164,10 @@ class ExpressionUtils { // TODO(jie) Move it to a better place static bool isGeoIndexAcceleratedPredicate(const Expression* expr); + + static bool checkExprDepth(const Expression* expr); + + static constexpr int32_t kMaxDepth = 512; }; } // namespace graph diff --git a/src/graph/util/test/CMakeLists.txt b/src/graph/util/test/CMakeLists.txt index 05d1b74119b..64084ffda7d 100644 --- a/src/graph/util/test/CMakeLists.txt +++ b/src/graph/util/test/CMakeLists.txt @@ -1,3 +1,14 @@ +set(UTIL_TEST_FLAG_DEPS + $ +) + +if(ENABLE_STANDALONE_VERSION) +set(UTIL_TEST_FLAG_DEPS + ${UTIL_TEST_FLAG_DEPS} + $ +) +endif() + nebula_add_test( NAME utils_test SOURCES @@ -38,7 +49,7 @@ nebula_add_test( $ $ $ - $ + ${UTIL_TEST_FLAG_DEPS} $ $ $ diff --git a/src/graph/validator/test/CMakeLists.txt b/src/graph/validator/test/CMakeLists.txt index ae4362aee0d..b8b4e279a56 100644 --- a/src/graph/validator/test/CMakeLists.txt +++ b/src/graph/validator/test/CMakeLists.txt @@ -61,6 +61,13 @@ set(VALIDATOR_TEST_LIBS $ ) +if(ENABLE_STANDALONE_VERSION) +set(VALIDATOR_TEST_LIBS + ${VALIDATOR_TEST_LIBS} + $ +) +endif() + nebula_add_test( NAME validator_test diff --git a/src/graph/visitor/test/CMakeLists.txt b/src/graph/visitor/test/CMakeLists.txt index d3e9d5e2638..8544b361b67 100644 --- a/src/graph/visitor/test/CMakeLists.txt +++ b/src/graph/visitor/test/CMakeLists.txt @@ -2,6 +2,17 @@ # # This source code is licensed under Apache 2.0 License. +set(EXPR_FLAG_DEPS + $ +) + +if(ENABLE_STANDALONE_VERSION) +set(EXPR_FLAG_DEPS + ${EXPR_FLAG_DEPS} + $ +) +endif() + nebula_add_test( NAME expr_visitor_test @@ -21,7 +32,7 @@ nebula_add_test( $ $ $ - $ + ${EXPR_FLAG_DEPS} $ $ $ diff --git a/src/interface/common.thrift b/src/interface/common.thrift index a81ceaa91bf..d21f704027a 100644 --- a/src/interface/common.thrift +++ b/src/interface/common.thrift @@ -419,9 +419,7 @@ enum ErrorCode { E_FIELD_UNSET = -3007, // Value exceeds the range of type E_OUT_OF_RANGE = -3008, - // Atomic operation failed - E_ATOMIC_OP_FAILED = -3009, - E_DATA_CONFLICT_ERROR = -3010, // data conflict, for index write without toss. + E_DATA_CONFLICT_ERROR = -3010, // data conflict, for index write without toss. E_WRITE_STALLED = -3011, @@ -473,5 +471,30 @@ enum ErrorCode { // get worker id E_WORKER_ID_FAILED = -3062, + // 35xx for storaged raft + E_RAFT_UNKNOWN_PART = -3500, + // Raft consensus errors + E_RAFT_LOG_GAP = -3501, + E_RAFT_LOG_STALE = -3502, + E_RAFT_TERM_OUT_OF_DATE = -3503, + // Raft state errors + E_RAFT_WAITING_SNAPSHOT = -3511, + E_RAFT_SENDING_SNAPSHOT = -3512, + E_RAFT_INVALID_PEER = -3513, + E_RAFT_NOT_READY = -3514, + E_RAFT_STOPPED = -3515, + E_RAFT_BAD_ROLE = -3516, + // Local errors + E_RAFT_WAL_FAIL = -3521, + E_RAFT_HOST_STOPPED = -3522, + E_RAFT_TOO_MANY_REQUESTS = -3523, + E_RAFT_PERSIST_SNAPSHOT_FAILED = -3524, + E_RAFT_RPC_EXCEPTION = -3525, + E_RAFT_NO_WAL_FOUND = -3526, + E_RAFT_HOST_PAUSED = -3527, + E_RAFT_WRITE_BLOCKED = -3528, + E_RAFT_BUFFER_OVERFLOW = -3529, + E_RAFT_ATOMIC_OP_FAILED = -3530, + E_UNKNOWN = -8000, } (cpp.enum_strict) diff --git a/src/interface/raftex.thrift b/src/interface/raftex.thrift index 7d002dd0f49..27d9405c641 100644 --- a/src/interface/raftex.thrift +++ b/src/interface/raftex.thrift @@ -22,33 +22,6 @@ enum Status { WAITING_SNAPSHOT = 3; // Waiting for the snapshot. } (cpp.enum_strict) -enum ErrorCode { - SUCCEEDED = 0; - - E_UNKNOWN_PART = -1; - - // Raft consensus errors - E_LOG_GAP = -2; - E_LOG_STALE = -3; - E_TERM_OUT_OF_DATE = -4; - - // Raft state errors - E_WAITING_SNAPSHOT = -5; // The follower is waiting a snapshot - E_BAD_STATE = -6; - E_WRONG_LEADER = -7; - E_NOT_READY = -8; - E_BAD_ROLE = -9, - - // Local errors - E_WAL_FAIL = -10; - E_HOST_STOPPED = -11; - E_TOO_MANY_REQUESTS = -12; - E_PERSIST_SNAPSHOT_FAILED = -13; - E_RPC_EXCEPTION = -14; // An thrift internal exception was thrown - E_NO_WAL_FOUND = -15; - E_APPLY_FAIL = -16; - E_HOST_PAUSED = -17; -} typedef i64 (cpp.type = "nebula::ClusterID") ClusterID typedef i32 (cpp.type = "nebula::GraphSpaceID") GraphSpaceID @@ -73,8 +46,8 @@ struct AskForVoteRequest { // Response message for the vote call struct AskForVoteResponse { - 1: ErrorCode error_code; - 2: TermID current_term; + 1: common.ErrorCode error_code; + 2: TermID current_term; } // Log entries being sent to follower, logId is not included, it could be calculated by @@ -98,13 +71,13 @@ struct AppendLogRequest { } struct AppendLogResponse { - 1: ErrorCode error_code; - 2: TermID current_term; - 3: string leader_addr; - 4: Port leader_port; - 5: LogID committed_log_id; - 6: LogID last_matched_log_id; - 7: TermID last_matched_log_term; + 1: common.ErrorCode error_code; + 2: TermID current_term; + 3: string leader_addr; + 4: Port leader_port; + 5: LogID committed_log_id; + 6: LogID last_matched_log_id; + 7: TermID last_matched_log_term; } struct SendSnapshotRequest { @@ -133,17 +106,17 @@ struct HeartbeatRequest { } struct HeartbeatResponse { - 1: ErrorCode error_code; - 2: TermID current_term; - 3: string leader_addr; - 4: Port leader_port; - 5: LogID committed_log_id; - 6: LogID last_log_id; - 7: TermID last_log_term; + 1: common.ErrorCode error_code; + 2: TermID current_term; + 3: string leader_addr; + 4: Port leader_port; + 5: LogID committed_log_id; + 6: LogID last_log_id; + 7: TermID last_log_term; } struct SendSnapshotResponse { - 1: ErrorCode error_code; + 1: common.ErrorCode error_code; } struct GetStateRequest { @@ -152,14 +125,14 @@ struct GetStateRequest { } struct GetStateResponse { - 1: ErrorCode error_code; - 2: Role role; - 3: TermID term; - 4: bool is_leader; - 5: LogID committed_log_id; - 6: LogID last_log_id; - 7: TermID last_log_term; - 8: Status status; + 1: common.ErrorCode error_code; + 2: Role role; + 3: TermID term; + 4: bool is_leader; + 5: LogID committed_log_id; + 6: LogID last_log_id; + 7: TermID last_log_term; + 8: Status status; } service RaftexService { diff --git a/src/kvstore/Listener.h b/src/kvstore/Listener.h index 403af197b14..8ebdb707870 100644 --- a/src/kvstore/Listener.h +++ b/src/kvstore/Listener.h @@ -149,13 +149,13 @@ class Listener : public raftex::RaftPart { LOG(INFO) << idStr_ << "Find the new leader " << nLeader; } - raftex::cpp2::ErrorCode checkPeer(const HostAddr& candidate) override { + nebula::cpp2::ErrorCode checkPeer(const HostAddr& candidate) override { CHECK(!raftLock_.try_lock()); if (peers_.find(candidate) == peers_.end()) { LOG(WARNING) << idStr_ << "The candidate " << candidate << " is not in my peers"; - return raftex::cpp2::ErrorCode::E_WRONG_LEADER; + return nebula::cpp2::ErrorCode::E_RAFT_INVALID_PEER; } - return raftex::cpp2::ErrorCode::SUCCEEDED; + return nebula::cpp2::ErrorCode::SUCCEEDED; } // For listener, we just return true directly. Another background thread trigger the actual diff --git a/src/kvstore/Part.cpp b/src/kvstore/Part.cpp index 87f64f96bf5..a5f1bd190ff 100644 --- a/src/kvstore/Part.cpp +++ b/src/kvstore/Part.cpp @@ -17,8 +17,6 @@ DEFINE_int32(cluster_id, 0, "A unique id for each cluster"); namespace nebula { namespace kvstore { -using nebula::raftex::AppendLogResult; - Part::Part(GraphSpaceID spaceId, PartitionID partId, HostAddr localAddr, @@ -69,104 +67,96 @@ void Part::asyncPut(folly::StringPiece key, folly::StringPiece value, KVCallback std::string log = encodeMultiValues(OP_PUT, key, value); appendAsync(FLAGS_cluster_id, std::move(log)) - .thenValue([this, callback = std::move(cb)](AppendLogResult res) mutable { - callback(this->toResultCode(res)); - }); + .thenValue( + [callback = std::move(cb)](nebula::cpp2::ErrorCode code) mutable { callback(code); }); } void Part::asyncAppendBatch(std::string&& batch, KVCallback cb) { appendAsync(FLAGS_cluster_id, std::move(batch)) - .thenValue([this, callback = std::move(cb)](AppendLogResult res) mutable { - callback(this->toResultCode(res)); - }); + .thenValue( + [callback = std::move(cb)](nebula::cpp2::ErrorCode code) mutable { callback(code); }); } void Part::asyncMultiPut(const std::vector& keyValues, KVCallback cb) { std::string log = encodeMultiValues(OP_MULTI_PUT, keyValues); appendAsync(FLAGS_cluster_id, std::move(log)) - .thenValue([this, callback = std::move(cb)](AppendLogResult res) mutable { - callback(this->toResultCode(res)); - }); + .thenValue( + [callback = std::move(cb)](nebula::cpp2::ErrorCode code) mutable { callback(code); }); } void Part::asyncRemove(folly::StringPiece key, KVCallback cb) { std::string log = encodeSingleValue(OP_REMOVE, key); appendAsync(FLAGS_cluster_id, std::move(log)) - .thenValue([this, callback = std::move(cb)](AppendLogResult res) mutable { - callback(this->toResultCode(res)); - }); + .thenValue( + [callback = std::move(cb)](nebula::cpp2::ErrorCode code) mutable { callback(code); }); } void Part::asyncMultiRemove(const std::vector& keys, KVCallback cb) { std::string log = encodeMultiValues(OP_MULTI_REMOVE, keys); appendAsync(FLAGS_cluster_id, std::move(log)) - .thenValue([this, callback = std::move(cb)](AppendLogResult res) mutable { - callback(this->toResultCode(res)); - }); + .thenValue( + [callback = std::move(cb)](nebula::cpp2::ErrorCode code) mutable { callback(code); }); } void Part::asyncRemoveRange(folly::StringPiece start, folly::StringPiece end, KVCallback cb) { std::string log = encodeMultiValues(OP_REMOVE_RANGE, start, end); appendAsync(FLAGS_cluster_id, std::move(log)) - .thenValue([this, callback = std::move(cb)](AppendLogResult res) mutable { - callback(this->toResultCode(res)); - }); + .thenValue( + [callback = std::move(cb)](nebula::cpp2::ErrorCode code) mutable { callback(code); }); } void Part::sync(KVCallback cb) { - sendCommandAsync("").thenValue([this, callback = std::move(cb)](AppendLogResult res) mutable { - callback(this->toResultCode(res)); - }); + sendCommandAsync("").thenValue( + [callback = std::move(cb)](nebula::cpp2::ErrorCode code) mutable { callback(code); }); } void Part::asyncAtomicOp(raftex::AtomicOp op, KVCallback cb) { atomicOpAsync(std::move(op)) - .thenValue([this, callback = std::move(cb)](AppendLogResult res) mutable { - callback(this->toResultCode(res)); - }); + .thenValue( + [callback = std::move(cb)](nebula::cpp2::ErrorCode code) mutable { callback(code); }); } void Part::asyncAddLearner(const HostAddr& learner, KVCallback cb) { std::string log = encodeHost(OP_ADD_LEARNER, learner); sendCommandAsync(std::move(log)) - .thenValue([callback = std::move(cb), learner, this](AppendLogResult res) mutable { + .thenValue([callback = std::move(cb), learner, this](nebula::cpp2::ErrorCode code) mutable { LOG(INFO) << idStr_ << "add learner " << learner - << ", result: " << static_cast(this->toResultCode(res)); - callback(this->toResultCode(res)); + << ", result: " << apache::thrift::util::enumNameSafe(code); + callback(code); }); } void Part::asyncTransferLeader(const HostAddr& target, KVCallback cb) { std::string log = encodeHost(OP_TRANS_LEADER, target); sendCommandAsync(std::move(log)) - .thenValue([callback = std::move(cb), target, this](AppendLogResult res) mutable { + .thenValue([callback = std::move(cb), target, this](nebula::cpp2::ErrorCode code) mutable { LOG(INFO) << idStr_ << "transfer leader to " << target - << ", result: " << static_cast(this->toResultCode(res)); - callback(this->toResultCode(res)); + << ", result: " << apache::thrift::util::enumNameSafe(code); + callback(code); }); } void Part::asyncAddPeer(const HostAddr& peer, KVCallback cb) { std::string log = encodeHost(OP_ADD_PEER, peer); sendCommandAsync(std::move(log)) - .thenValue([callback = std::move(cb), peer, this](AppendLogResult res) mutable { + .thenValue([callback = std::move(cb), peer, this](nebula::cpp2::ErrorCode code) mutable { LOG(INFO) << idStr_ << "add peer " << peer - << ", result: " << static_cast(this->toResultCode(res)); - callback(this->toResultCode(res)); + << ", result: " << apache::thrift::util::enumNameSafe(code); + callback(code); }); } void Part::asyncRemovePeer(const HostAddr& peer, KVCallback cb) { std::string log = encodeHost(OP_REMOVE_PEER, peer); sendCommandAsync(std::move(log)) - .thenValue([callback = std::move(cb), peer, this](AppendLogResult res) mutable { + .thenValue([callback = std::move(cb), peer, this](nebula::cpp2::ErrorCode code) mutable { LOG(INFO) << idStr_ << "remove peer " << peer - << ", result: " << static_cast(this->toResultCode(res)); - callback(this->toResultCode(res)); + << ", result: " << apache::thrift::util::enumNameSafe(code); + callback(code); }); } @@ -519,24 +509,5 @@ nebula::cpp2::ErrorCode Part::cleanup() { std::move(batch), FLAGS_rocksdb_disable_wal, FLAGS_rocksdb_wal_sync, true); } -// TODO(pandasheep) unify raft errorcode -nebula::cpp2::ErrorCode Part::toResultCode(raftex::AppendLogResult res) { - switch (res) { - case raftex::AppendLogResult::SUCCEEDED: - return nebula::cpp2::ErrorCode::SUCCEEDED; - case raftex::AppendLogResult::E_NOT_A_LEADER: - return nebula::cpp2::ErrorCode::E_LEADER_CHANGED; - case raftex::AppendLogResult::E_WRITE_BLOCKING: - return nebula::cpp2::ErrorCode::E_CHECKPOINT_BLOCKED; - case raftex::AppendLogResult::E_ATOMIC_OP_FAILURE: - return nebula::cpp2::ErrorCode::E_ATOMIC_OP_FAILED; - case raftex::AppendLogResult::E_BUFFER_OVERFLOW: - return nebula::cpp2::ErrorCode::E_CONSENSUS_ERROR; - default: - LOG(ERROR) << idStr_ << "Consensus error " << static_cast(res); - return nebula::cpp2::ErrorCode::E_CONSENSUS_ERROR; - } -} - } // namespace kvstore } // namespace nebula diff --git a/src/kvstore/Part.h b/src/kvstore/Part.h index 346776b3a13..94b2f676a08 100644 --- a/src/kvstore/Part.h +++ b/src/kvstore/Part.h @@ -115,8 +115,6 @@ class Part : public raftex::RaftPart { nebula::cpp2::ErrorCode cleanup() override; - nebula::cpp2::ErrorCode toResultCode(raftex::AppendLogResult res); - public: struct CallbackOptions { GraphSpaceID spaceId; diff --git a/src/kvstore/plugins/hbase/HBaseClient.cpp b/src/kvstore/plugins/hbase/HBaseClient.cpp index bbdd1178842..6e870bead63 100644 --- a/src/kvstore/plugins/hbase/HBaseClient.cpp +++ b/src/kvstore/plugins/hbase/HBaseClient.cpp @@ -188,7 +188,7 @@ ResultCode HBaseClient::range(const std::string& tableName, while (true) { std::vector tResultList; client_->sync_getScannerRows(tResultList, scannerId, kScanRowNum); - if (tResultList.size() == 0) break; + if (tResultList.empty()) break; for (auto& tResult : tResultList) { std::vector tColumnValueList = tResult.columnValues; if (tColumnValueList.size() > 0) { diff --git a/src/kvstore/raftex/Host.cpp b/src/kvstore/raftex/Host.cpp index faf610aa922..33d2d3f7708 100644 --- a/src/kvstore/raftex/Host.cpp +++ b/src/kvstore/raftex/Host.cpp @@ -43,18 +43,18 @@ void Host::waitForStop() { LOG(INFO) << idStr_ << "The host has been stopped!"; } -cpp2::ErrorCode Host::canAppendLog() const { +nebula::cpp2::ErrorCode Host::canAppendLog() const { CHECK(!lock_.try_lock()); if (stopped_) { VLOG(2) << idStr_ << "The host is stopped, just return"; - return cpp2::ErrorCode::E_HOST_STOPPED; + return nebula::cpp2::ErrorCode::E_RAFT_HOST_STOPPED; } if (paused_) { VLOG(2) << idStr_ << "The host is paused, due to losing leadership"; - return cpp2::ErrorCode::E_HOST_PAUSED; + return nebula::cpp2::ErrorCode::E_RAFT_HOST_PAUSED; } - return cpp2::ErrorCode::SUCCEEDED; + return nebula::cpp2::ErrorCode::SUCCEEDED; } folly::Future Host::askForVote(const cpp2::AskForVoteRequest& req, @@ -64,7 +64,7 @@ folly::Future Host::askForVote(const cpp2::AskForVoteR if (stopped_) { VLOG(2) << idStr_ << "The Host is not in a proper status, do not send"; cpp2::AskForVoteResponse resp; - resp.error_code_ref() = cpp2::ErrorCode::E_HOST_STOPPED; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_HOST_STOPPED; return resp; } } @@ -89,7 +89,7 @@ folly::Future Host::appendLogs(folly::EventBase* eb, if (UNLIKELY(sendingSnapshot_)) { LOG_EVERY_N(INFO, 500) << idStr_ << "The target host is waiting for a snapshot"; - res = cpp2::ErrorCode::E_WAITING_SNAPSHOT; + res = nebula::cpp2::ErrorCode::E_RAFT_WAITING_SNAPSHOT; } else if (requestOnGoing_) { // buffer incoming request to pendingReq_ if (cachingPromise_.size() <= FLAGS_max_outstanding_requests) { @@ -97,11 +97,11 @@ folly::Future Host::appendLogs(folly::EventBase* eb, return cachingPromise_.getFuture(); } else { LOG_EVERY_N(INFO, 200) << idStr_ << "Too many requests are waiting, return error"; - res = cpp2::ErrorCode::E_TOO_MANY_REQUESTS; + res = nebula::cpp2::ErrorCode::E_RAFT_TOO_MANY_REQUESTS; } } - if (res != cpp2::ErrorCode::SUCCEEDED) { + if (res != nebula::cpp2::ErrorCode::SUCCEEDED) { VLOG(2) << idStr_ << "The host is not in a proper status, just return"; cpp2::AppendLogResponse r; r.error_code_ref() = res; @@ -168,16 +168,16 @@ void Host::appendLogsInternal(folly::EventBase* eb, std::shared_ptrlastLogIdSent_ << ", lastLogTermSent_ " << self->lastLogTermSent_; switch (resp.get_error_code()) { - case cpp2::ErrorCode::SUCCEEDED: - case cpp2::ErrorCode::E_LOG_GAP: - case cpp2::ErrorCode::E_LOG_STALE: { + case nebula::cpp2::ErrorCode::SUCCEEDED: + case nebula::cpp2::ErrorCode::E_RAFT_LOG_GAP: + case nebula::cpp2::ErrorCode::E_RAFT_LOG_STALE: { VLOG(2) << self->idStr_ << "AppendLog request sent successfully"; std::shared_ptr newReq; { std::lock_guard g(self->lock_); auto res = self->canAppendLog(); - if (res != cpp2::ErrorCode::SUCCEEDED) { + if (res != nebula::cpp2::ErrorCode::SUCCEEDED) { cpp2::AppendLogResponse r; r.error_code_ref() = res; self->setResponse(r); @@ -214,7 +214,7 @@ void Host::appendLogsInternal(folly::EventBase* eb, std::shared_ptridStr_ << ex.what(); cpp2::AppendLogResponse r; - r.error_code_ref() = cpp2::ErrorCode::E_RPC_EXCEPTION; + r.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_RPC_EXCEPTION; { std::lock_guard g(self->lock_); if (ex.getType() == TransportException::TIMED_OUT) { @@ -254,7 +254,7 @@ void Host::appendLogsInternal(folly::EventBase* eb, std::shared_ptr{}, [self = shared_from_this()](std::exception&& ex) { VLOG(2) << self->idStr_ << ex.what(); cpp2::AppendLogResponse r; - r.error_code_ref() = cpp2::ErrorCode::E_RPC_EXCEPTION; + r.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_RPC_EXCEPTION; { std::lock_guard g(self->lock_); self->setResponse(r); @@ -264,7 +264,8 @@ void Host::appendLogsInternal(folly::EventBase* eb, std::shared_ptr> Host::prepareAppendLogRequest() { +ErrorOr> +Host::prepareAppendLogRequest() { CHECK(!lock_.try_lock()); VLOG(2) << idStr_ << "Prepare AppendLogs request from Log " << lastLogIdSent_ + 1 << " to " << logIdToSend_; @@ -279,7 +280,7 @@ ErrorOr> Host::prepareA << idStr_ << "My lastLogId in wal is " << part_->wal()->lastLogId() << ", but you are seeking " << lastLogIdSent_ + 1 << ", so i have nothing to send, logIdToSend_ = " << logIdToSend_; - return cpp2::ErrorCode::E_NO_WAL_FOUND; + return nebula::cpp2::ErrorCode::E_RAFT_NO_WAL_FOUND; } auto it = part_->wal()->iterator(lastLogIdSent_ + 1, logIdToSend_); @@ -307,16 +308,16 @@ ErrorOr> Host::prepareA if (!it->valid() && (lastLogIdSent_ + static_cast(logs.size()) != logIdToSend_)) { LOG_IF(INFO, FLAGS_trace_raft) << idStr_ << "Can't find log in wal, logIdToSend_ = " << logIdToSend_; - return cpp2::ErrorCode::E_NO_WAL_FOUND; + return nebula::cpp2::ErrorCode::E_RAFT_NO_WAL_FOUND; } req->log_str_list_ref() = std::move(logs); return req; } else { - return cpp2::ErrorCode::E_NO_WAL_FOUND; + return nebula::cpp2::ErrorCode::E_RAFT_NO_WAL_FOUND; } } -cpp2::ErrorCode Host::startSendSnapshot() { +nebula::cpp2::ErrorCode Host::startSendSnapshot() { CHECK(!lock_.try_lock()); if (!sendingSnapshot_) { LOG(INFO) << idStr_ << "Can't find log " << lastLogIdSent_ + 1 << " in wal, send the snapshot" @@ -345,7 +346,7 @@ cpp2::ErrorCode Host::startSendSnapshot() { } else { LOG_EVERY_N(INFO, 100) << idStr_ << "The snapshot req is in queue, please wait for a moment"; } - return cpp2::ErrorCode::E_WAITING_SNAPSHOT; + return nebula::cpp2::ErrorCode::E_RAFT_WAITING_SNAPSHOT; } folly::Future Host::sendAppendLogRequest( @@ -355,7 +356,7 @@ folly::Future Host::sendAppendLogRequest( { std::lock_guard g(lock_); auto res = canAppendLog(); - if (res != cpp2::ErrorCode::SUCCEEDED) { + if (res != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(WARNING) << idStr_ << "The Host is not in a proper status, do not send"; cpp2::AppendLogResponse resp; resp.error_code_ref() = res; @@ -395,7 +396,7 @@ folly::Future Host::sendHeartbeat( VLOG(3) << self->idStr_ << "heartbeat call got response"; if (t.hasException()) { cpp2::HeartbeatResponse resp; - resp.error_code_ref() = cpp2::ErrorCode::E_RPC_EXCEPTION; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_RPC_EXCEPTION; pro.setValue(std::move(resp)); return; } else { @@ -412,7 +413,7 @@ folly::Future Host::sendHeartbeatRequest( { std::lock_guard g(lock_); auto res = canAppendLog(); - if (res != cpp2::ErrorCode::SUCCEEDED) { + if (res != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(WARNING) << idStr_ << "The Host is not in a proper status, do not send"; cpp2::HeartbeatResponse resp; resp.error_code_ref() = res; diff --git a/src/kvstore/raftex/Host.h b/src/kvstore/raftex/Host.h index 8e1c3c14609..41781c9e540 100644 --- a/src/kvstore/raftex/Host.h +++ b/src/kvstore/raftex/Host.h @@ -97,7 +97,7 @@ class Host final : public std::enable_shared_from_this { } private: - cpp2::ErrorCode canAppendLog() const; + nebula::cpp2::ErrorCode canAppendLog() const; folly::Future sendAppendLogRequest( folly::EventBase* eb, std::shared_ptr req); @@ -107,9 +107,10 @@ class Host final : public std::enable_shared_from_this { folly::Future sendHeartbeatRequest( folly::EventBase* eb, std::shared_ptr req); - ErrorOr> prepareAppendLogRequest(); + ErrorOr> + prepareAppendLogRequest(); - cpp2::ErrorCode startSendSnapshot(); + nebula::cpp2::ErrorCode startSendSnapshot(); bool noRequest() const; diff --git a/src/kvstore/raftex/RaftPart.cpp b/src/kvstore/raftex/RaftPart.cpp index 1b845a89f40..79bd7af85d0 100644 --- a/src/kvstore/raftex/RaftPart.cpp +++ b/src/kvstore/raftex/RaftPart.cpp @@ -351,24 +351,24 @@ void RaftPart::stop() { LOG(INFO) << idStr_ << "Partition has been stopped"; } -AppendLogResult RaftPart::canAppendLogs() { +nebula::cpp2::ErrorCode RaftPart::canAppendLogs() { DCHECK(!raftLock_.try_lock()); if (UNLIKELY(status_ != Status::RUNNING)) { LOG(ERROR) << idStr_ << "The partition is not running"; - return AppendLogResult::E_STOPPED; + return nebula::cpp2::ErrorCode::E_RAFT_STOPPED; } if (UNLIKELY(role_ != Role::LEADER)) { LOG_EVERY_N(WARNING, 1000) << idStr_ << "The partition is not a leader"; - return AppendLogResult::E_NOT_A_LEADER; + return nebula::cpp2::ErrorCode::E_LEADER_CHANGED; } - return AppendLogResult::SUCCEEDED; + return nebula::cpp2::ErrorCode::SUCCEEDED; } -AppendLogResult RaftPart::canAppendLogs(TermID termId) { +nebula::cpp2::ErrorCode RaftPart::canAppendLogs(TermID termId) { DCHECK(!raftLock_.try_lock()); if (UNLIKELY(term_ != termId)) { VLOG(2) << idStr_ << "Term has been updated, origin " << termId << ", new " << term_; - return AppendLogResult::E_TERM_OUT_OF_DATE; + return nebula::cpp2::ErrorCode::E_RAFT_TERM_OUT_OF_DATE; } return canAppendLogs(); } @@ -519,7 +519,7 @@ void RaftPart::removePeer(const HostAddr& peer) { } } -cpp2::ErrorCode RaftPart::checkPeer(const HostAddr& candidate) { +nebula::cpp2::ErrorCode RaftPart::checkPeer(const HostAddr& candidate) { CHECK(!raftLock_.try_lock()); auto hosts = followers(); auto it = std::find_if(hosts.begin(), hosts.end(), [&candidate](const auto& h) { @@ -527,9 +527,9 @@ cpp2::ErrorCode RaftPart::checkPeer(const HostAddr& candidate) { }); if (it == hosts.end()) { LOG(INFO) << idStr_ << "The candidate " << candidate << " is not in my peers"; - return cpp2::ErrorCode::E_WRONG_LEADER; + return nebula::cpp2::ErrorCode::E_RAFT_INVALID_PEER; } - return cpp2::ErrorCode::SUCCEEDED; + return nebula::cpp2::ErrorCode::SUCCEEDED; } void RaftPart::addListenerPeer(const HostAddr& listener) { @@ -593,41 +593,41 @@ void RaftPart::commitRemovePeer(const HostAddr& peer) { removePeer(peer); } -folly::Future RaftPart::appendAsync(ClusterID source, std::string log) { +folly::Future RaftPart::appendAsync(ClusterID source, std::string log) { if (source < 0) { source = clusterId_; } return appendLogAsync(source, LogType::NORMAL, std::move(log)); } -folly::Future RaftPart::atomicOpAsync(AtomicOp op) { +folly::Future RaftPart::atomicOpAsync(AtomicOp op) { return appendLogAsync(clusterId_, LogType::ATOMIC_OP, "", std::move(op)); } -folly::Future RaftPart::sendCommandAsync(std::string log) { +folly::Future RaftPart::sendCommandAsync(std::string log) { return appendLogAsync(clusterId_, LogType::COMMAND, std::move(log)); } -folly::Future RaftPart::appendLogAsync(ClusterID source, - LogType logType, - std::string log, - AtomicOp op) { +folly::Future RaftPart::appendLogAsync(ClusterID source, + LogType logType, + std::string log, + AtomicOp op) { if (blocking_) { // No need to block heartbeats and empty log. if ((logType == LogType::NORMAL && !log.empty()) || logType == LogType::ATOMIC_OP) { - return AppendLogResult::E_WRITE_BLOCKING; + return nebula::cpp2::ErrorCode::E_RAFT_WRITE_BLOCKED; } } LogCache swappedOutLogs; - auto retFuture = folly::Future::makeEmpty(); + auto retFuture = folly::Future::makeEmpty(); if (bufferOverFlow_) { LOG_EVERY_N(WARNING, 100) << idStr_ << "The appendLog buffer is full." " Please slow down the log appending rate." << "replicatingLogs_ :" << replicatingLogs_; - return AppendLogResult::E_BUFFER_OVERFLOW; + return nebula::cpp2::ErrorCode::E_RAFT_BUFFER_OVERFLOW; } { std::lock_guard lck(logsLock_); @@ -641,7 +641,7 @@ folly::Future RaftPart::appendLogAsync(ClusterID source, " Please slow down the log appending rate." << "replicatingLogs_ :" << replicatingLogs_; bufferOverFlow_ = true; - return AppendLogResult::E_BUFFER_OVERFLOW; + return nebula::cpp2::ErrorCode::E_RAFT_BUFFER_OVERFLOW; } VLOG(2) << idStr_ << "Appending logs to the buffer"; @@ -677,11 +677,11 @@ folly::Future RaftPart::appendLogAsync(ClusterID source, LogID firstId = 0; TermID termId = 0; - AppendLogResult res; + nebula::cpp2::ErrorCode res; { std::lock_guard g(raftLock_); res = canAppendLogs(); - if (res == AppendLogResult::SUCCEEDED) { + if (res == nebula::cpp2::ErrorCode::SUCCEEDED) { firstId = lastLogId_ + 1; termId = term_; } @@ -705,7 +705,8 @@ folly::Future RaftPart::appendLogAsync(ClusterID source, auto opRet = opCB(); if (!opRet.hasValue()) { // Failed - sendingPromise_.setOneSingleValue(AppendLogResult::E_ATOMIC_OP_FAILURE); + sendingPromise_.setOneSingleValue( + nebula::cpp2::ErrorCode::E_RAFT_ATOMIC_OP_FAILED); } return opRet; }); @@ -728,11 +729,11 @@ void RaftPart::appendLogsInternal(AppendLogsIterator iter, TermID termId) { replicatingLogs_ = false; return; } - AppendLogResult res = AppendLogResult::SUCCEEDED; + nebula::cpp2::ErrorCode res = nebula::cpp2::ErrorCode::SUCCEEDED; do { std::lock_guard g(raftLock_); res = canAppendLogs(termId); - if (res != AppendLogResult::SUCCEEDED) { + if (res != nebula::cpp2::ErrorCode::SUCCEEDED) { break; } currTerm = term_; @@ -743,7 +744,7 @@ void RaftPart::appendLogsInternal(AppendLogsIterator iter, TermID termId) { SlowOpTracker tracker; if (!wal_->appendLogs(iter)) { LOG_EVERY_N(WARNING, 100) << idStr_ << "Failed to write into WAL"; - res = AppendLogResult::E_WAL_FAILURE; + res = nebula::cpp2::ErrorCode::E_RAFT_WAL_FAIL; lastLogId_ = wal_->lastLogId(); lastLogTerm_ = wal_->lastLogTerm(); break; @@ -776,11 +777,11 @@ void RaftPart::replicateLogs(folly::EventBase* eb, using namespace folly; // NOLINT since the fancy overload of | operator decltype(hosts_) hosts; - AppendLogResult res = AppendLogResult::SUCCEEDED; + nebula::cpp2::ErrorCode res = nebula::cpp2::ErrorCode::SUCCEEDED; do { std::lock_guard g(raftLock_); res = canAppendLogs(currTerm); - if (res != AppendLogResult::SUCCEEDED) { + if (res != nebula::cpp2::ErrorCode::SUCCEEDED) { lastLogId_ = wal_->lastLogId(); lastLogTerm_ = wal_->lastLogTerm(); break; @@ -817,7 +818,7 @@ void RaftPart::replicateLogs(folly::EventBase* eb, quorum_, // Result evaluator [hosts](size_t index, cpp2::AppendLogResponse& resp) { - return resp.get_error_code() == cpp2::ErrorCode::SUCCEEDED && + return resp.get_error_code() == nebula::cpp2::ErrorCode::SUCCEEDED && !hosts[index]->isLearner(); }) .via(executor_.get()) @@ -865,13 +866,13 @@ void RaftPart::processAppendLogResponses(const AppendLogResponses& resps, TermID highestTerm = currTerm; for (auto& res : resps) { if (!hosts[res.first]->isLearner() && - res.second.get_error_code() == cpp2::ErrorCode::SUCCEEDED) { + res.second.get_error_code() == nebula::cpp2::ErrorCode::SUCCEEDED) { ++numSucceeded; } highestTerm = std::max(highestTerm, res.second.get_current_term()); } - AppendLogResult res = AppendLogResult::SUCCEEDED; + nebula::cpp2::ErrorCode res = nebula::cpp2::ErrorCode::SUCCEEDED; { std::lock_guard g(raftLock_); if (highestTerm > term_) { @@ -880,7 +881,7 @@ void RaftPart::processAppendLogResponses(const AppendLogResponses& resps, leader_ = HostAddr("", 0); lastLogId_ = wal_->lastLogId(); lastLogTerm_ = wal_->lastLogTerm(); - res = AppendLogResult::E_TERM_OUT_OF_DATE; + res = nebula::cpp2::ErrorCode::E_RAFT_TERM_OUT_OF_DATE; } } if (!checkAppendLogResult(res)) { @@ -895,7 +896,7 @@ void RaftPart::processAppendLogResponses(const AppendLogResponses& resps, do { std::lock_guard g(raftLock_); res = canAppendLogs(currTerm); - if (res != AppendLogResult::SUCCEEDED) { + if (res != nebula::cpp2::ErrorCode::SUCCEEDED) { lastLogId_ = wal_->lastLogId(); lastLogTerm_ = wal_->lastLogTerm(); break; @@ -943,10 +944,10 @@ void RaftPart::processAppendLogResponses(const AppendLogResponses& resps, // Step 4: Fulfill the promise if (iter.hasNonAtomicOpLogs()) { - sendingPromise_.setOneSharedValue(AppendLogResult::SUCCEEDED); + sendingPromise_.setOneSharedValue(nebula::cpp2::ErrorCode::SUCCEEDED); } if (iter.leadByAtomicOp()) { - sendingPromise_.setOneSingleValue(AppendLogResult::SUCCEEDED); + sendingPromise_.setOneSingleValue(nebula::cpp2::ErrorCode::SUCCEEDED); } // Step 5: Check whether need to continue // the log replication @@ -971,7 +972,8 @@ void RaftPart::processAppendLogResponses(const AppendLogResponses& resps, auto opRet = op(); if (!opRet.hasValue()) { // Failed - sendingPromise_.setOneSingleValue(AppendLogResult::E_ATOMIC_OP_FAILURE); + sendingPromise_.setOneSingleValue( + nebula::cpp2::ErrorCode::E_RAFT_ATOMIC_OP_FAILED); } return opRet; }); @@ -1064,7 +1066,7 @@ void RaftPart::getState(cpp2::GetStateResponse& resp) { resp.term_ref() = term_; resp.role_ref() = role_; resp.is_leader_ref() = role_ == Role::LEADER; - resp.error_code_ref() = cpp2::ErrorCode::SUCCEEDED; + resp.error_code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED; resp.committed_log_id_ref() = committedLogId_; resp.last_log_id_ref() = lastLogId_; resp.last_log_term_ref() = lastLogTerm_; @@ -1111,7 +1113,7 @@ bool RaftPart::processElectionResponses(const RaftPart::ElectionResponses& resul size_t numSucceeded = 0; TermID highestTerm = isPreVote ? proposedTerm - 1 : proposedTerm; for (auto& r : results) { - if (r.second.get_error_code() == cpp2::ErrorCode::SUCCEEDED) { + if (r.second.get_error_code() == nebula::cpp2::ErrorCode::SUCCEEDED) { ++numSucceeded; } else { LOG(WARNING) << idStr_ << "Receive response about askForVote from " @@ -1207,7 +1209,8 @@ folly::Future RaftPart::leaderElection(bool isPreVote) { quorum_, // Result evaluator [hosts](size_t idx, cpp2::AskForVoteResponse& resp) { - return resp.get_error_code() == cpp2::ErrorCode::SUCCEEDED && !hosts[idx]->isLearner(); + return resp.get_error_code() == nebula::cpp2::ErrorCode::SUCCEEDED && + !hosts[idx]->isLearner(); }) .via(executor_.get()) .then([self = shared_from_this(), pro = std::move(promise), hosts, proposedTerm, isPreVote]( @@ -1330,19 +1333,19 @@ void RaftPart::processAskForVoteRequest(const cpp2::AskForVoteRequest& req, // Make sure the partition is running if (UNLIKELY(status_ == Status::STOPPED)) { LOG(INFO) << idStr_ << "The part has been stopped, skip the request"; - resp.error_code_ref() = cpp2::ErrorCode::E_BAD_STATE; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_STOPPED; return; } if (UNLIKELY(status_ == Status::STARTING)) { LOG(INFO) << idStr_ << "The partition is still starting"; - resp.error_code_ref() = cpp2::ErrorCode::E_NOT_READY; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_NOT_READY; return; } if (UNLIKELY(status_ == Status::WAITING_SNAPSHOT)) { LOG(INFO) << idStr_ << "The partition is still waiting snapshot"; - resp.error_code_ref() = cpp2::ErrorCode::E_WAITING_SNAPSHOT; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_WAITING_SNAPSHOT; return; } @@ -1350,13 +1353,13 @@ void RaftPart::processAskForVoteRequest(const cpp2::AskForVoteRequest& req, << lastLogId_ << ", lastLogTerm " << lastLogTerm_ << ", committedLogId " << committedLogId_ << ", term " << term_; if (role_ == Role::LEARNER) { - resp.error_code_ref() = cpp2::ErrorCode::E_BAD_ROLE; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_BAD_ROLE; return; } auto candidate = HostAddr(req.get_candidate_addr(), req.get_candidate_port()); auto code = checkPeer(candidate); - if (code != cpp2::ErrorCode::SUCCEEDED) { + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { resp.error_code_ref() = code; return; } @@ -1366,7 +1369,7 @@ void RaftPart::processAskForVoteRequest(const cpp2::AskForVoteRequest& req, LOG(INFO) << idStr_ << "The partition currently is on term " << term_ << ", the term proposed by the candidate is " << req.get_term() << ", so it will be rejected"; - resp.error_code_ref() = cpp2::ErrorCode::E_TERM_OUT_OF_DATE; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_TERM_OUT_OF_DATE; return; } @@ -1392,7 +1395,7 @@ void RaftPart::processAskForVoteRequest(const cpp2::AskForVoteRequest& req, LOG(INFO) << idStr_ << "The partition's last term to receive a log is " << lastLogTerm_ << ", which is newer than the candidate's log " << req.get_last_log_term() << ". So the candidate will be rejected"; - resp.error_code_ref() = cpp2::ErrorCode::E_TERM_OUT_OF_DATE; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_TERM_OUT_OF_DATE; return; } @@ -1402,7 +1405,7 @@ void RaftPart::processAskForVoteRequest(const cpp2::AskForVoteRequest& req, LOG(INFO) << idStr_ << "The partition's last log id is " << lastLogId_ << ". The candidate's last log id " << req.get_last_log_id() << " is smaller, so it will be rejected, candidate is " << candidate; - resp.error_code_ref() = cpp2::ErrorCode::E_LOG_STALE; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_LOG_STALE; return; } } @@ -1420,7 +1423,7 @@ void RaftPart::processAskForVoteRequest(const cpp2::AskForVoteRequest& req, LOG(INFO) << idStr_ << "We have voted " << votedAddr_ << " on term " << votedTerm_ << ", so we should reject the candidate " << candidate << " request on term " << req.get_term(); - resp.error_code_ref() = cpp2::ErrorCode::E_TERM_OUT_OF_DATE; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_TERM_OUT_OF_DATE; return; } @@ -1430,7 +1433,7 @@ void RaftPart::processAskForVoteRequest(const cpp2::AskForVoteRequest& req, if (req.get_is_pre_vote()) { // return succeed if it is prevote, do not change any state - resp.error_code_ref() = cpp2::ErrorCode::SUCCEEDED; + resp.error_code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED; return; } @@ -1456,7 +1459,7 @@ void RaftPart::processAskForVoteRequest(const cpp2::AskForVoteRequest& req, leader_ = HostAddr("", 0); votedAddr_ = candidate; votedTerm_ = req.get_term(); - resp.error_code_ref() = cpp2::ErrorCode::SUCCEEDED; + resp.error_code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED; resp.current_term_ref() = term_; // Reset the last message time @@ -1496,24 +1499,24 @@ void RaftPart::processAppendLogRequest(const cpp2::AppendLogRequest& req, // Check status if (UNLIKELY(status_ == Status::STOPPED)) { VLOG(2) << idStr_ << "The part has been stopped, skip the request"; - resp.error_code_ref() = cpp2::ErrorCode::E_BAD_STATE; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_STOPPED; return; } if (UNLIKELY(status_ == Status::STARTING)) { VLOG(2) << idStr_ << "The partition is still starting"; - resp.error_code_ref() = cpp2::ErrorCode::E_NOT_READY; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_NOT_READY; return; } if (UNLIKELY(status_ == Status::WAITING_SNAPSHOT)) { VLOG(2) << idStr_ << "The partition is waiting for snapshot"; - resp.error_code_ref() = cpp2::ErrorCode::E_WAITING_SNAPSHOT; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_WAITING_SNAPSHOT; return; } // Check leadership - cpp2::ErrorCode err = verifyLeader(req); + nebula::cpp2::ErrorCode err = verifyLeader(req); // Set term_ again because it may be modified in verifyLeader resp.current_term_ref() = term_; - if (err != cpp2::ErrorCode::SUCCEEDED) { + if (err != nebula::cpp2::ErrorCode::SUCCEEDED) { // Wrong leadership VLOG(2) << idStr_ << "Will not follow the leader"; resp.error_code_ref() = err; @@ -1549,7 +1552,7 @@ void RaftPart::processAppendLogRequest(const cpp2::AppendLogRequest& req, wal_->getLogTerm(req.get_last_log_id_sent()) != req.get_last_log_term_sent()) { resp.last_matched_log_id_ref() = committedLogId_; resp.last_matched_log_term() = committedLogTerm_; - resp.error_code() = cpp2::ErrorCode::E_LOG_GAP; + resp.error_code() = nebula::cpp2::ErrorCode::E_RAFT_LOG_GAP; // lastMatchedLogId is committedLogId_ return; } @@ -1593,7 +1596,7 @@ void RaftPart::processAppendLogRequest(const cpp2::AppendLogRequest& req, resp.last_matched_log_id_ref() = lastLogId_; resp.last_matched_log_term_ref() = lastLogTerm_; } else { - resp.error_code_ref() = cpp2::ErrorCode::E_WAL_FAIL; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_WAL_FAIL; return; } } while (false); @@ -1613,7 +1616,7 @@ void RaftPart::processAppendLogRequest(const cpp2::AppendLogRequest& req, committedLogId_ = lastCommitId; committedLogTerm_ = lastCommitTerm; resp.committed_log_id_ref() = lastLogIdCanCommit; - resp.error_code_ref() = cpp2::ErrorCode::SUCCEEDED; + resp.error_code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED; } else if (code == nebula::cpp2::ErrorCode::E_WRITE_STALLED) { VLOG(1) << idStr_ << "Follower delay committing log " << committedLogId_ + 1 << " to " << lastLogIdCanCommit; @@ -1621,15 +1624,15 @@ void RaftPart::processAppendLogRequest(const cpp2::AppendLogRequest& req, // 1. As a follower, upcoming request will try to commit them // 2. If it is elected as leader later, it will try to commit them as well resp.committed_log_id_ref() = committedLogId_; - resp.error_code_ref() = cpp2::ErrorCode::SUCCEEDED; + resp.error_code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED; } else { LOG(ERROR) << idStr_ << "Failed to commit log " << committedLogId_ + 1 << " to " << req.get_committed_log_id(); resp.committed_log_id_ref() = committedLogId_; - resp.error_code_ref() = cpp2::ErrorCode::E_WAL_FAIL; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_WAL_FAIL; } } else { - resp.error_code_ref() = cpp2::ErrorCode::SUCCEEDED; + resp.error_code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED; } // Reset the timeout timer again in case wal and commit takes longer time than @@ -1638,11 +1641,11 @@ void RaftPart::processAppendLogRequest(const cpp2::AppendLogRequest& req, } template -cpp2::ErrorCode RaftPart::verifyLeader(const REQ& req) { +nebula::cpp2::ErrorCode RaftPart::verifyLeader(const REQ& req) { DCHECK(!raftLock_.try_lock()); auto peer = HostAddr(req.get_leader_addr(), req.get_leader_port()); auto code = checkPeer(peer); - if (code != cpp2::ErrorCode::SUCCEEDED) { + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { return code; } @@ -1651,7 +1654,7 @@ cpp2::ErrorCode RaftPart::verifyLeader(const REQ& req) { if (req.get_current_term() < term_) { LOG_EVERY_N(INFO, 100) << idStr_ << "The current role is " << roleStr(role_) << ". The local term is " << term_ << ". The remote term is not newer"; - return cpp2::ErrorCode::E_TERM_OUT_OF_DATE; + return nebula::cpp2::ErrorCode::E_RAFT_TERM_OUT_OF_DATE; } else if (req.get_current_term() > term_) { // found new leader with higher term } else { @@ -1662,7 +1665,7 @@ cpp2::ErrorCode RaftPart::verifyLeader(const REQ& req) { // I know who is leader if (LIKELY(leader_ == peer)) { // Same leader - return cpp2::ErrorCode::SUCCEEDED; + return nebula::cpp2::ErrorCode::SUCCEEDED; } else { LOG(ERROR) << idStr_ << "Split brain happens, will follow the new leader " << peer << " on term " << req.get_current_term(); @@ -1700,7 +1703,7 @@ cpp2::ErrorCode RaftPart::verifyLeader(const REQ& req) { bgWorkers_->addTask([self = shared_from_this(), oldTerm] { self->onLostLeadership(oldTerm); }); } bgWorkers_->addTask([self = shared_from_this()] { self->onDiscoverNewLeader(self->leader_); }); - return cpp2::ErrorCode::SUCCEEDED; + return nebula::cpp2::ErrorCode::SUCCEEDED; } void RaftPart::processHeartbeatRequest(const cpp2::HeartbeatRequest& req, @@ -1735,19 +1738,19 @@ void RaftPart::processHeartbeatRequest(const cpp2::HeartbeatRequest& req, // Check status if (UNLIKELY(status_ == Status::STOPPED)) { VLOG(2) << idStr_ << "The part has been stopped, skip the request"; - resp.error_code_ref() = cpp2::ErrorCode::E_BAD_STATE; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_STOPPED; return; } if (UNLIKELY(status_ == Status::STARTING)) { VLOG(2) << idStr_ << "The partition is still starting"; - resp.error_code_ref() = cpp2::ErrorCode::E_NOT_READY; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_NOT_READY; return; } // Check leadership - cpp2::ErrorCode err = verifyLeader(req); + nebula::cpp2::ErrorCode err = verifyLeader(req); // Set term_ again because it may be modified in verifyLeader resp.current_term_ref() = term_; - if (err != cpp2::ErrorCode::SUCCEEDED) { + if (err != nebula::cpp2::ErrorCode::SUCCEEDED) { // Wrong leadership VLOG(2) << idStr_ << "Will not follow the leader"; resp.error_code_ref() = err; @@ -1758,7 +1761,7 @@ void RaftPart::processHeartbeatRequest(const cpp2::HeartbeatRequest& req, lastMsgRecvDur_.reset(); // As for heartbeat, return ok after verifyLeader - resp.error_code_ref() = cpp2::ErrorCode::SUCCEEDED; + resp.error_code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED; return; } @@ -1771,24 +1774,24 @@ void RaftPart::processSendSnapshotRequest(const cpp2::SendSnapshotRequest& req, // Check status if (UNLIKELY(status_ == Status::STOPPED)) { LOG(ERROR) << idStr_ << "The part has been stopped, skip the request"; - resp.error_code_ref() = cpp2::ErrorCode::E_BAD_STATE; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_STOPPED; return; } if (UNLIKELY(status_ == Status::STARTING)) { LOG(ERROR) << idStr_ << "The partition is still starting"; - resp.error_code_ref() = cpp2::ErrorCode::E_NOT_READY; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_NOT_READY; return; } if (UNLIKELY(role_ != Role::FOLLOWER && role_ != Role::LEARNER)) { LOG(ERROR) << idStr_ << "Bad role " << roleStr(role_); - resp.error_code_ref() = cpp2::ErrorCode::E_BAD_STATE; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_STOPPED; return; } if (UNLIKELY(leader_ != HostAddr(req.get_leader_addr(), req.get_leader_port()) || term_ != req.get_term())) { LOG(ERROR) << idStr_ << "Term out of date, current term " << term_ << ", received term " << req.get_term(); - resp.error_code_ref() = cpp2::ErrorCode::E_TERM_OUT_OF_DATE; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_TERM_OUT_OF_DATE; return; } if (status_ != Status::WAITING_SNAPSHOT) { @@ -1807,7 +1810,7 @@ void RaftPart::processSendSnapshotRequest(const cpp2::SendSnapshotRequest& req, LOG(ERROR) << idStr_ << "Bad snapshot, total rows received " << lastTotalCount_ << ", total rows sended " << req.get_total_count() << ", total size received " << lastTotalSize_ << ", total size sended " << req.get_total_size(); - resp.error_code_ref() = cpp2::ErrorCode::E_PERSIST_SNAPSHOT_FAILED; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_PERSIST_SNAPSHOT_FAILED; return; } if (req.get_done()) { @@ -1824,7 +1827,7 @@ void RaftPart::processSendSnapshotRequest(const cpp2::SendSnapshotRequest& req, << ", committedLogTerm_ " << committedLogTerm_ << ", lastLodId " << lastLogId_ << ", lastLogTermId " << lastLogTerm_; } - resp.error_code_ref() = cpp2::ErrorCode::SUCCEEDED; + resp.error_code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED; return; } @@ -1871,7 +1874,8 @@ void RaftPart::sendHeartbeat() { hosts.size(), // Result evaluator [hosts](size_t index, cpp2::HeartbeatResponse& resp) { - return resp.get_error_code() == cpp2::ErrorCode::SUCCEEDED && !hosts[index]->isLearner(); + return resp.get_error_code() == nebula::cpp2::ErrorCode::SUCCEEDED && + !hosts[index]->isLearner(); }) .then([replica, hosts = std::move(hosts), startMs, currTerm, this]( folly::Try&& resps) { @@ -1880,7 +1884,7 @@ void RaftPart::sendHeartbeat() { TermID highestTerm = currTerm; for (auto& resp : *resps) { if (!hosts[resp.first]->isLearner() && - resp.second.get_error_code() == cpp2::ErrorCode::SUCCEEDED) { + resp.second.get_error_code() == nebula::cpp2::ErrorCode::SUCCEEDED) { ++numSucceeded; } highestTerm = std::max(highestTerm, resp.second.get_current_term()); @@ -1933,8 +1937,8 @@ std::pair RaftPart::lastLogInfo() const { return std::make_pair(wal_->lastLogId(), wal_->lastLogTerm()); } -bool RaftPart::checkAppendLogResult(AppendLogResult res) { - if (res != AppendLogResult::SUCCEEDED) { +bool RaftPart::checkAppendLogResult(nebula::cpp2::ErrorCode res) { + if (res != nebula::cpp2::ErrorCode::SUCCEEDED) { { std::lock_guard lck(logsLock_); logs_.clear(); @@ -1959,16 +1963,16 @@ void RaftPart::reset() { lastTotalSize_ = 0; } -AppendLogResult RaftPart::isCatchedUp(const HostAddr& peer) { +nebula::cpp2::ErrorCode RaftPart::isCatchedUp(const HostAddr& peer) { std::lock_guard lck(raftLock_); LOG(INFO) << idStr_ << "Check whether I catch up"; if (role_ != Role::LEADER) { LOG(INFO) << idStr_ << "I am not the leader"; - return AppendLogResult::E_NOT_A_LEADER; + return nebula::cpp2::ErrorCode::E_LEADER_CHANGED; } if (peer == addr_) { LOG(INFO) << idStr_ << "I am the leader"; - return AppendLogResult::SUCCEEDED; + return nebula::cpp2::ErrorCode::SUCCEEDED; } for (auto& host : hosts_) { if (host->addr_ == peer) { @@ -1976,13 +1980,13 @@ AppendLogResult RaftPart::isCatchedUp(const HostAddr& peer) { host->followerCommittedLogId_ < wal_->firstLogId()) { LOG(INFO) << idStr_ << "The committed log id of peer is " << host->followerCommittedLogId_ << ", which is invalid or less than my first wal log id"; - return AppendLogResult::E_SENDING_SNAPSHOT; + return nebula::cpp2::ErrorCode::E_RAFT_SENDING_SNAPSHOT; } - return host->sendingSnapshot_ ? AppendLogResult::E_SENDING_SNAPSHOT - : AppendLogResult::SUCCEEDED; + return host->sendingSnapshot_ ? nebula::cpp2::ErrorCode::E_RAFT_SENDING_SNAPSHOT + : nebula::cpp2::ErrorCode::SUCCEEDED; } } - return AppendLogResult::E_INVALID_PEER; + return nebula::cpp2::ErrorCode::E_RAFT_INVALID_PEER; } bool RaftPart::linkCurrentWAL(const char* newPath) { diff --git a/src/kvstore/raftex/RaftPart.h b/src/kvstore/raftex/RaftPart.h index b5f99de4219..8cfaa55d81f 100644 --- a/src/kvstore/raftex/RaftPart.h +++ b/src/kvstore/raftex/RaftPart.h @@ -15,6 +15,7 @@ #include "common/time/Duration.h" #include "common/utils/LogIterator.h" #include "interface/gen-cpp2/RaftexServiceAsyncClient.h" +#include "interface/gen-cpp2/common_types.h" #include "interface/gen-cpp2/raftex_types.h" #include "kvstore/Common.h" #include "kvstore/DiskManager.h" @@ -33,21 +34,6 @@ class FileBasedWal; namespace raftex { -enum class AppendLogResult { - SUCCEEDED = 0, - E_ATOMIC_OP_FAILURE = -1, - E_NOT_A_LEADER = -2, - E_STOPPED = -3, - E_NOT_READY = -4, - E_BUFFER_OVERFLOW = -5, - E_WAL_FAILURE = -6, - E_TERM_OUT_OF_DATE = -7, - E_SENDING_SNAPSHOT = -8, - E_INVALID_PEER = -9, - E_NOT_ENOUGH_ACKS = -10, - E_WRITE_BLOCKING = -11, -}; - enum class LogType { NORMAL = 0x00, ATOMIC_OP = 0x01, @@ -173,23 +159,23 @@ class RaftPart : public std::enable_shared_from_this { * * If the source == -1, the current clusterId will be used ****************************************************************/ - folly::Future appendAsync(ClusterID source, std::string log); + folly::Future appendAsync(ClusterID source, std::string log); /**************************************************************** * Run the op atomically. ***************************************************************/ - folly::Future atomicOpAsync(AtomicOp op); + folly::Future atomicOpAsync(AtomicOp op); /** * Asynchronously send one command. * */ - folly::Future sendCommandAsync(std::string log); + folly::Future sendCommandAsync(std::string log); /** * Check if the peer has catched up data from leader. If leader is sending the * snapshot, the method will return false. * */ - AppendLogResult isCatchedUp(const HostAddr& peer); + nebula::cpp2::ErrorCode isCatchedUp(const HostAddr& peer); bool linkCurrentWAL(const char* newPath); @@ -284,7 +270,7 @@ class RaftPart : public std::enable_shared_from_this { virtual void onDiscoverNewLeader(HostAddr nLeader) = 0; // Check if we can accept candidate's message - virtual cpp2::ErrorCode checkPeer(const HostAddr& candidate); + virtual nebula::cpp2::ErrorCode checkPeer(const HostAddr& candidate); // The inherited classes need to implement this method to commit a batch of log messages. // Return {error code, last commit log id, last commit log term}. @@ -330,7 +316,7 @@ class RaftPart : public std::enable_shared_from_this { const char* roleStr(Role role) const; template - cpp2::ErrorCode verifyLeader(const REQ& req); + nebula::cpp2::ErrorCode verifyLeader(const REQ& req); /***************************************************************** * @@ -379,16 +365,16 @@ class RaftPart : public std::enable_shared_from_this { // Check whether new logs can be appended // Pre-condition: The caller needs to hold the raftLock_ - AppendLogResult canAppendLogs(); + nebula::cpp2::ErrorCode canAppendLogs(); // Also check if term has changed // Pre-condition: The caller needs to hold the raftLock_ - AppendLogResult canAppendLogs(TermID currTerm); + nebula::cpp2::ErrorCode canAppendLogs(TermID currTerm); - folly::Future appendLogAsync(ClusterID source, - LogType logType, - std::string log, - AtomicOp cb = nullptr); + folly::Future appendLogAsync(ClusterID source, + LogType logType, + std::string log, + AtomicOp cb = nullptr); void appendLogsInternal(AppendLogsIterator iter, TermID termId); @@ -414,7 +400,7 @@ class RaftPart : public std::enable_shared_from_this { // counted in std::vector> followers() const; - bool checkAppendLogResult(AppendLogResult res); + bool checkAppendLogResult(nebula::cpp2::ErrorCode res); void updateQuorum(); @@ -511,13 +497,13 @@ class RaftPart : public std::enable_shared_from_this { mutable std::mutex logsLock_; std::atomic_bool replicatingLogs_{false}; std::atomic_bool bufferOverFlow_{false}; - PromiseSet cachingPromise_; + PromiseSet cachingPromise_; LogCache logs_; // Partition level lock to synchronize the access of the partition mutable std::mutex raftLock_; - PromiseSet sendingPromise_; + PromiseSet sendingPromise_; Status status_; Role role_; diff --git a/src/kvstore/raftex/RaftexService.cpp b/src/kvstore/raftex/RaftexService.cpp index 622d61edd32..ba7d926584f 100644 --- a/src/kvstore/raftex/RaftexService.cpp +++ b/src/kvstore/raftex/RaftexService.cpp @@ -181,7 +181,7 @@ void RaftexService::getState(cpp2::GetStateResponse& resp, const cpp2::GetStateR part->getState(resp); } else { resp.term_ref() = -1; - resp.error_code_ref() = cpp2::ErrorCode::E_UNKNOWN_PART; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_UNKNOWN_PART; } } @@ -189,7 +189,7 @@ void RaftexService::askForVote(cpp2::AskForVoteResponse& resp, const cpp2::AskFo auto part = findPart(req.get_space(), req.get_part()); if (!part) { // Not found - resp.error_code_ref() = cpp2::ErrorCode::E_UNKNOWN_PART; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_UNKNOWN_PART; return; } @@ -200,7 +200,7 @@ void RaftexService::appendLog(cpp2::AppendLogResponse& resp, const cpp2::AppendL auto part = findPart(req.get_space(), req.get_part()); if (!part) { // Not found - resp.error_code_ref() = cpp2::ErrorCode::E_UNKNOWN_PART; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_UNKNOWN_PART; return; } @@ -212,7 +212,7 @@ void RaftexService::sendSnapshot(cpp2::SendSnapshotResponse& resp, auto part = findPart(req.get_space(), req.get_part()); if (!part) { // Not found - resp.error_code_ref() = cpp2::ErrorCode::E_UNKNOWN_PART; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_UNKNOWN_PART; return; } @@ -226,7 +226,7 @@ void RaftexService::async_eb_heartbeat( auto part = findPart(req.get_space(), req.get_part()); if (!part) { // Not found - resp.error_code_ref() = cpp2::ErrorCode::E_UNKNOWN_PART; + resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_UNKNOWN_PART; callback->result(resp); return; } diff --git a/src/kvstore/raftex/SnapshotManager.cpp b/src/kvstore/raftex/SnapshotManager.cpp index 559f58377a1..f6e07a90a99 100644 --- a/src/kvstore/raftex/SnapshotManager.cpp +++ b/src/kvstore/raftex/SnapshotManager.cpp @@ -72,7 +72,7 @@ folly::Future>> SnapshotManager::sendSnapshot( // occupied. try { auto resp = std::move(f).get(); - if (resp.get_error_code() == cpp2::ErrorCode::SUCCEEDED) { + if (resp.get_error_code() == nebula::cpp2::ErrorCode::SUCCEEDED) { VLOG(1) << part->idStr_ << "has sended count " << totalCount; if (status == SnapshotStatus::DONE) { LOG(INFO) << part->idStr_ << "Finished, totalCount " << totalCount diff --git a/src/kvstore/raftex/test/LogAppendTest.cpp b/src/kvstore/raftex/test/LogAppendTest.cpp index 4e3fc92d791..d6e8b1852ea 100644 --- a/src/kvstore/raftex/test/LogAppendTest.cpp +++ b/src/kvstore/raftex/test/LogAppendTest.cpp @@ -88,11 +88,11 @@ TEST(LogAppend, MultiThreadAppend) { for (int j = 1; j <= numLogs; ++j) { do { auto fut = leader->appendAsync(0, folly::stringPrintf("Log %03d for t%d", j, i)); - if (fut.isReady() && fut.value() == AppendLogResult::E_BUFFER_OVERFLOW) { + if (fut.isReady() && fut.value() == nebula::cpp2::ErrorCode::E_RAFT_BUFFER_OVERFLOW) { LOG(FATAL) << "Should not reach here"; } else if (j == numLogs) { // Only wait on the last log message - ASSERT_EQ(AppendLogResult::SUCCEEDED, std::move(fut).get()); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, std::move(fut).get()); } break; } while (true); diff --git a/src/kvstore/raftex/test/LogCASTest.cpp b/src/kvstore/raftex/test/LogCASTest.cpp index fab2b040b29..afd95ec8c47 100644 --- a/src/kvstore/raftex/test/LogCASTest.cpp +++ b/src/kvstore/raftex/test/LogCASTest.cpp @@ -210,8 +210,8 @@ TEST_F(LogCASTest, EmptyTest) { LOG(INFO) << "return empty string for atomic operation!"; folly::Baton<> baton; leader_->atomicOpAsync([log = std::move(log)]() mutable { return std::string(""); }) - .thenValue([&baton](AppendLogResult res) { - ASSERT_EQ(AppendLogResult::SUCCEEDED, res); + .thenValue([&baton](nebula::cpp2::ErrorCode res) { + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, res); baton.post(); }); baton.wait(); @@ -220,8 +220,8 @@ TEST_F(LogCASTest, EmptyTest) { LOG(INFO) << "return none string for atomic operation!"; folly::Baton<> baton; leader_->atomicOpAsync([log = std::move(log)]() mutable { return folly::none; }) - .thenValue([&baton](AppendLogResult res) { - ASSERT_EQ(AppendLogResult::E_ATOMIC_OP_FAILURE, res); + .thenValue([&baton](nebula::cpp2::ErrorCode res) { + ASSERT_EQ(nebula::cpp2::ErrorCode::E_RAFT_ATOMIC_OP_FAILED, res); baton.post(); }); baton.wait(); diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index ee0025d35aa..2ff62fc1fe5 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -114,12 +114,14 @@ nebula_add_library( add_dependencies( meta_version_man_obj meta_v1_thrift_obj + meta_v2_thrift_obj ) set(meta_test_deps $ $ $ + $ $ $ $ @@ -175,6 +177,13 @@ set(meta_test_deps $ ) +if(ENABLE_STANDALONE_VERSION) +set(meta_test_deps + ${meta_test_deps} + $ +) +endif() + nebula_add_subdirectory(http) nebula_add_subdirectory(test) nebula_add_subdirectory(upgrade) diff --git a/src/meta/MetaVersionMan.cpp b/src/meta/MetaVersionMan.cpp index dd61af74c2e..68f139a2a31 100644 --- a/src/meta/MetaVersionMan.cpp +++ b/src/meta/MetaVersionMan.cpp @@ -5,10 +5,12 @@ #include "meta/MetaVersionMan.h" +#include "meta/ActiveHostsMan.h" #include "meta/processors/job/JobDescription.h" #include "meta/processors/job/JobUtils.h" #include "meta/upgrade/MetaDataUpgrade.h" #include "meta/upgrade/v1/MetaServiceUtilsV1.h" +#include "meta/upgrade/v2/MetaServiceUtilsV2.h" DEFINE_bool(null_type, true, "set schema to support null type"); DEFINE_bool(print_info, false, "enable to print the rewrite data"); @@ -26,7 +28,7 @@ MetaVersion MetaVersionMan::getMetaVersionFromKV(kvstore::KVStore* kv) { auto code = kv->get(kDefaultSpaceId, kDefaultPartId, kMetaVersionKey, &value, true); if (code == nebula::cpp2::ErrorCode::SUCCEEDED) { auto version = *reinterpret_cast(value.data()); - return (version == MetaVersion::V2) ? MetaVersion::V2 : MetaVersion::UNKNOWN; + return version; } else { return getVersionByHost(kv); } @@ -42,19 +44,18 @@ MetaVersion MetaVersionMan::getVersionByHost(kvstore::KVStore* kv) { } if (iter->valid()) { auto v1KeySize = hostPrefix.size() + sizeof(int64_t); - return (iter->key().size() == v1KeySize) ? MetaVersion::V1 : MetaVersion::V2; + return (iter->key().size() == v1KeySize) ? MetaVersion::V1 : MetaVersion::V3; } - // No hosts exists, regard as regard as version 2 - return MetaVersion::V2; + // No hosts exists, regard as version 3 + return MetaVersion::V3; } // static -bool MetaVersionMan::setMetaVersionToKV(kvstore::KVStore* kv) { +bool MetaVersionMan::setMetaVersionToKV(kvstore::KVStore* kv, MetaVersion version) { CHECK_NOTNULL(kv); - auto v2 = MetaVersion::V2; std::vector data; data.emplace_back(kMetaVersionKey, - std::string(reinterpret_cast(&v2), sizeof(MetaVersion))); + std::string(reinterpret_cast(&version), sizeof(MetaVersion))); bool ret = true; folly::Baton baton; kv->asyncMultiPut( @@ -63,7 +64,7 @@ bool MetaVersionMan::setMetaVersionToKV(kvstore::KVStore* kv) { LOG(ERROR) << "Put failed, error: " << static_cast(code); ret = false; } else { - LOG(INFO) << "Write meta version 2 succeeds"; + LOG(INFO) << "Write meta version 3 succeeds"; } baton.post(); }); @@ -80,7 +81,28 @@ Status MetaVersionMan::updateMetaV1ToV2(kvstore::KVStore* kv) { LOG(ERROR) << "Create snapshot failed: " << snapshot; return Status::Error("Create snapshot failed"); } - auto status = doUpgrade(kv); + auto status = doUpgradeV1ToV2(kv); + if (!status.ok()) { + // rollback by snapshot + return status; + } + // delete snapshot file + auto dmRet = kv->dropCheckpoint(kDefaultSpaceId, snapshot); + if (dmRet != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Delete snapshot: " << snapshot << " failed, You need to delete it manually"; + } + return Status::OK(); +} + +Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVStore* kv) { + CHECK_NOTNULL(kv); + auto snapshot = folly::format("META_UPGRADE_SNAPSHOT_{}", MetaKeyUtils::genTimestampStr()).str(); + auto meteRet = kv->createCheckpoint(kDefaultSpaceId, snapshot); + if (meteRet.isLeftType()) { + LOG(ERROR) << "Create snapshot failed: " << snapshot; + return Status::Error("Create snapshot failed"); + } + auto status = doUpgradeV2ToV3(kv); if (!status.ok()) { // rollback by snapshot return status; @@ -94,7 +116,7 @@ Status MetaVersionMan::updateMetaV1ToV2(kvstore::KVStore* kv) { } // static -Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { +Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVStore* kv) { MetaDataUpgrade upgrader(kv); { // kSpacesTable @@ -105,7 +127,7 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { Status status = Status::OK(); while (iter->valid()) { if (FLAGS_print_info) { - upgrader.printSpaces(iter->val()); + upgrader.printSpacesV1(iter->val()); } status = upgrader.rewriteSpaces(iter->key(), iter->val()); if (!status.ok()) { @@ -158,6 +180,7 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { } } } + { // kLeadersTable auto prefix = nebula::meta::v1::kLeadersTable; @@ -178,6 +201,7 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { } } } + { // kTagsTable auto prefix = nebula::meta::v1::kTagsTable; @@ -327,7 +351,95 @@ Status MetaVersionMan::doUpgrade(kvstore::KVStore* kv) { } } } - if (!setMetaVersionToKV(kv)) { + if (!setMetaVersionToKV(kv, MetaVersion::V2)) { + return Status::Error("Persist meta version failed"); + } else { + return Status::OK(); + } +} + +Status MetaVersionMan::doUpgradeV2ToV3(kvstore::KVStore* kv) { + MetaDataUpgrade upgrader(kv); + // Step 1: Upgrade HeartBeat into machine list + { + // collect all hosts association with zone + std::vector zoneHosts; + const auto& zonePrefix = MetaKeyUtils::zonePrefix(); + std::unique_ptr zoneIter; + auto code = kv->prefix(kDefaultSpaceId, kDefaultPartId, zonePrefix, &zoneIter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Get active hosts failed"; + return Status::Error("Get hosts failed"); + } + + while (zoneIter->valid()) { + auto hosts = MetaKeyUtils::parseZoneHosts(zoneIter->val()); + if (!hosts.empty()) { + zoneHosts.insert(zoneHosts.end(), hosts.begin(), hosts.end()); + } + zoneIter->next(); + } + + const auto& prefix = MetaKeyUtils::hostPrefix(); + std::unique_ptr iter; + code = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Get active hosts failed"; + return Status::Error("Get hosts failed"); + } + + std::vector data; + while (iter->valid()) { + auto info = HostInfo::decode(iter->val()); + + if (info.role_ == meta::cpp2::HostRole::STORAGE) { + // Save the machine information + auto host = MetaKeyUtils::parseHostKey(iter->key()); + auto machineKey = MetaKeyUtils::machineKey(host.host, host.port); + data.emplace_back(std::move(machineKey), ""); + + auto hostIt = std::find(zoneHosts.begin(), zoneHosts.end(), host); + if (hostIt == zoneHosts.end()) { + // Save the zone information + auto zoneName = folly::stringPrintf("default_zone_%s_%d", host.host.c_str(), host.port); + auto zoneKey = MetaKeyUtils::zoneKey(std::move(zoneName)); + auto zoneVal = MetaKeyUtils::zoneVal({host}); + data.emplace_back(std::move(zoneKey), std::move(zoneVal)); + } + } + iter->next(); + } + auto status = upgrader.saveMachineAndZone(std::move(data)); + if (!status.ok()) { + LOG(ERROR) << status; + return status; + } + } + + // Step 2: Update Create space properties about Group + { + const auto& prefix = MetaKeyUtils::spacePrefix(); + std::unique_ptr iter; + auto code = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Get spaces failed"; + return Status::Error("Get spaces failed"); + } + + while (iter->valid()) { + if (FLAGS_print_info) { + upgrader.printSpacesV2(iter->val()); + } + auto spaceProperties = meta::v2::MetaServiceUtilsV2::parseSpace(iter->val()); + auto status = upgrader.rewriteSpacesV2ToV3(iter->key(), iter->val()); + if (!status.ok()) { + LOG(ERROR) << status; + return status; + } + iter->next(); + } + } + if (!setMetaVersionToKV(kv, MetaVersion::V3)) { return Status::Error("Persist meta version failed"); } else { return Status::OK(); diff --git a/src/meta/MetaVersionMan.h b/src/meta/MetaVersionMan.h index 59737175430..c6e31b8d01d 100644 --- a/src/meta/MetaVersionMan.h +++ b/src/meta/MetaVersionMan.h @@ -17,6 +17,7 @@ enum class MetaVersion { UNKNOWN = 0, V1 = 1, V2 = 2, + V3 = 3, }; /** @@ -28,17 +29,21 @@ class MetaVersionMan final { static MetaVersion getMetaVersionFromKV(kvstore::KVStore* kv); - static bool setMetaVersionToKV(kvstore::KVStore* kv); + static bool setMetaVersionToKV(kvstore::KVStore* kv, MetaVersion version); static Status updateMetaV1ToV2(kvstore::KVStore* kv); + static Status updateMetaV2ToV3(kvstore::KVStore* kv); + private: static MetaVersion getVersionByHost(kvstore::KVStore* kv); - static Status doUpgrade(kvstore::KVStore* kv); + static Status doUpgradeV1ToV2(kvstore::KVStore* kv); + + static Status doUpgradeV2ToV3(kvstore::KVStore* kv); }; } // namespace meta } // namespace nebula -#endif // META_ROOTUSERMAN_H_ +#endif // META_METAVERSIONMAN_H_ diff --git a/src/meta/processors/BaseProcessor-inl.h b/src/meta/processors/BaseProcessor-inl.h index bfb203d89d6..85cd44ceafd 100644 --- a/src/meta/processors/BaseProcessor-inl.h +++ b/src/meta/processors/BaseProcessor-inl.h @@ -262,6 +262,15 @@ nebula::cpp2::ErrorCode BaseProcessor::machineExist(const std::string& mac return nebula::error(ret); } +template +nebula::cpp2::ErrorCode BaseProcessor::hostExist(const std::string& hostKey) { + auto ret = doGet(hostKey); + if (nebula::ok(ret)) { + return nebula::cpp2::ErrorCode::SUCCEEDED; + } + return nebula::error(ret); +} + template nebula::cpp2::ErrorCode BaseProcessor::includeByZone(const std::vector& hosts) { const auto& prefix = MetaKeyUtils::zonePrefix(); diff --git a/src/meta/processors/BaseProcessor.h b/src/meta/processors/BaseProcessor.h index f9def2c631e..76430930cf4 100644 --- a/src/meta/processors/BaseProcessor.h +++ b/src/meta/processors/BaseProcessor.h @@ -193,6 +193,8 @@ class BaseProcessor { * */ nebula::cpp2::ErrorCode machineExist(const std::string& machineKey); + nebula::cpp2::ErrorCode hostExist(const std::string& hostKey); + /** * Check hosts has been include by zone or not. * */ diff --git a/src/meta/processors/admin/HBProcessor.cpp b/src/meta/processors/admin/HBProcessor.cpp index 7e78aba9732..83c2984f513 100644 --- a/src/meta/processors/admin/HBProcessor.cpp +++ b/src/meta/processors/admin/HBProcessor.cpp @@ -58,7 +58,7 @@ void HBProcessor::process(const cpp2::HBReq& req) { for (const auto& [spaceId, partDiskMap] : *req.get_disk_parts()) { for (const auto& [path, partList] : partDiskMap) { auto partListVal = MetaKeyUtils::diskPartsVal(partList); - std::string key = MetaKeyUtils::diskPartsKey(host, spaceId, path); + auto key = MetaKeyUtils::diskPartsKey(host, spaceId, path); std::vector data; data.emplace_back(key, partListVal); // doPut() not work, will trigger the asan: use heap memory which is free diff --git a/src/meta/processors/admin/ListClusterInfoProcessor.cpp b/src/meta/processors/admin/ListClusterInfoProcessor.cpp index 3ebd993d35f..3c291fa9ca8 100644 --- a/src/meta/processors/admin/ListClusterInfoProcessor.cpp +++ b/src/meta/processors/admin/ListClusterInfoProcessor.cpp @@ -37,8 +37,8 @@ void ListClusterInfoProcessor::process(const cpp2::ListClusterInfoReq& req) { } auto iter = nebula::value(iterRet).get(); for (; iter->valid(); iter->next()) { - HostAddr addr = MetaKeyUtils::parseHostKey(iter->key()); - HostInfo info = HostInfo::decode(iter->val()); + auto addr = MetaKeyUtils::parseHostKey(iter->key()); + auto info = HostInfo::decode(iter->val()); cpp2::ServiceInfo service; service.role_ref() = info.role_; diff --git a/src/meta/processors/job/DataBalanceJobExecutor.cpp b/src/meta/processors/job/DataBalanceJobExecutor.cpp index d125bb2ca25..e2abaca40fc 100644 --- a/src/meta/processors/job/DataBalanceJobExecutor.cpp +++ b/src/meta/processors/job/DataBalanceJobExecutor.cpp @@ -99,7 +99,7 @@ Status DataBalanceJobExecutor::buildBalancePlan() { for (Host* h : hostVec) { totalPartNum += h->parts_.size(); } - if (hostVec.size() == 0) { + if (hostVec.empty()) { LOG(ERROR) << "rebalance error: zone has no host"; return {}; } diff --git a/src/meta/processors/job/ZoneBalanceJobExecutor.cpp b/src/meta/processors/job/ZoneBalanceJobExecutor.cpp index 0d1833247ce..a152bd02d75 100644 --- a/src/meta/processors/job/ZoneBalanceJobExecutor.cpp +++ b/src/meta/processors/job/ZoneBalanceJobExecutor.cpp @@ -121,7 +121,7 @@ nebula::cpp2::ErrorCode ZoneBalanceJobExecutor::rebalanceActiveZones( for (auto& z : sortedActiveZonesRef) { totalPartNum += z->partNum_; } - if (sortedActiveZonesRef.size() == 0) { + if (sortedActiveZonesRef.empty()) { LOG(ERROR) << "rebalance error: no active zones"; return nebula::cpp2::ErrorCode::E_NO_HOSTS; } diff --git a/src/meta/processors/parts/CreateSpaceProcessor.cpp b/src/meta/processors/parts/CreateSpaceProcessor.cpp index 8da74f9e0a3..a2b3b0cf396 100644 --- a/src/meta/processors/parts/CreateSpaceProcessor.cpp +++ b/src/meta/processors/parts/CreateSpaceProcessor.cpp @@ -9,6 +9,7 @@ DEFINE_int32(default_parts_num, 100, "The default number of parts when a space is created"); DEFINE_int32(default_replica_factor, 1, "The default replica factor when a space is created"); +DECLARE_uint32(expired_time_factor); namespace nebula { namespace meta { @@ -129,19 +130,27 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { zoneIter->next(); } - int32_t zoneNum = zones.size(); - if (replicaFactor > zoneNum) { - LOG(ERROR) << "Replication number should less than or equal to zone number."; - handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); - onFinished(); - return; - } - properties.zone_names_ref() = zones; } else { zones = properties.get_zone_names(); } + auto it = std::unique(zones.begin(), zones.end()); + if (it != zones.end()) { + LOG(ERROR) << "Zones have duplicated."; + handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); + onFinished(); + return; + } + + int32_t zoneNum = zones.size(); + if (replicaFactor > zoneNum) { + LOG(ERROR) << "Replication number should less than or equal to zone number."; + handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); + onFinished(); + return; + } + data.emplace_back(MetaKeyUtils::indexSpaceKey(spaceName), std::string(reinterpret_cast(&spaceId), sizeof(spaceId))); data.emplace_back(MetaKeyUtils::spaceKey(spaceId), MetaKeyUtils::spaceVal(properties)); @@ -165,28 +174,6 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { return; } - int32_t zoneNum = zones.size(); - if (replicaFactor > zoneNum) { - LOG(ERROR) << "Replication number should less than or equal to zone number."; - LOG(ERROR) << "Replication number: " << replicaFactor << ", Zones size: " << zones.size(); - handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM); - onFinished(); - return; - } - - auto hostLoadingRet = getHostLoading(); - if (!nebula::ok(hostLoadingRet)) { - LOG(ERROR) << "Get host loading failed."; - auto retCode = nebula::error(hostLoadingRet); - if (retCode != nebula::cpp2::ErrorCode::E_LEADER_CHANGED) { - retCode = nebula::cpp2::ErrorCode::E_INVALID_PARM; - } - handleErrorCode(retCode); - onFinished(); - return; - } - - hostLoading_ = std::move(nebula::value(hostLoadingRet)); std::unordered_map zoneHosts; for (auto& zone : zones) { auto zoneKey = MetaKeyUtils::zoneKey(zone); @@ -200,16 +187,33 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { break; } + auto now = time::WallClock::fastNowInMilliSec(); auto hosts = MetaKeyUtils::parseZoneHosts(std::move(nebula::value(zoneValueRet))); for (auto& host : hosts) { - auto hostIter = hostLoading_.find(host); - if (hostIter == hostLoading_.end()) { - hostLoading_[host] = 0; - zoneLoading_[zone] += 0; + auto key = MetaKeyUtils::hostKey(host.host, host.port); + auto ret = doGet(key); + if (!nebula::ok(ret)) { + code = nebula::error(ret); + LOG(ERROR) << "Get host " << host << " failed."; + break; + } + + HostInfo info = HostInfo::decode(nebula::value(ret)); + if (now - info.lastHBTimeInMilliSec_ < + FLAGS_heartbeat_interval_secs * FLAGS_expired_time_factor * 1000) { + auto hostIter = hostLoading_.find(host); + if (hostIter == hostLoading_.end()) { + hostLoading_[host] = 0; + zoneLoading_[zone] += 0; + } else { + zoneLoading_[zone] += hostIter->second; + } } else { - zoneLoading_[zone] += hostIter->second; + LOG(WARNING) << "Host " << host << " expired"; } } + + CHECK_CODE_AND_BREAK(); zoneHosts[zone] = std::move(hosts); } @@ -248,7 +252,7 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { ss << host << ", "; } - VLOG(3) << "Space " << spaceId << " part " << partId << " hosts " << ss.str(); + LOG(INFO) << "Space " << spaceId << " part " << partId << " hosts " << ss.str(); data.emplace_back(MetaKeyUtils::partKey(spaceId, partId), MetaKeyUtils::partVal(partHosts)); } @@ -264,28 +268,6 @@ void CreateSpaceProcessor::process(const cpp2::CreateSpaceReq& req) { LOG(INFO) << "Create space " << spaceName; } -ErrorOr> -CreateSpaceProcessor::getHostLoading() { - const auto& prefix = MetaKeyUtils::partPrefix(); - auto iterRet = doPrefix(prefix); - - if (!nebula::ok(iterRet)) { - LOG(ERROR) << "Prefix Parts Failed"; - return nebula::error(iterRet); - } - - std::unordered_map result; - auto iter = nebula::value(iterRet).get(); - while (iter->valid()) { - auto hosts = MetaKeyUtils::parsePartVal(iter->val()); - for (auto& host : hosts) { - result[host]++; - } - iter->next(); - } - return result; -} - StatusOr CreateSpaceProcessor::pickHostsWithZone( const std::vector& zones, const std::unordered_map& zoneHosts) { diff --git a/src/meta/processors/parts/CreateSpaceProcessor.h b/src/meta/processors/parts/CreateSpaceProcessor.h index 570ab11cd74..67768039dbb 100644 --- a/src/meta/processors/parts/CreateSpaceProcessor.h +++ b/src/meta/processors/parts/CreateSpaceProcessor.h @@ -29,9 +29,6 @@ class CreateSpaceProcessor : public BaseProcessor { StatusOr pickHostsWithZone(const std::vector& zones, const std::unordered_map& zoneHosts); - // Get all host's part loading - ErrorOr> getHostLoading(); - // Get the zones with the least load StatusOr> pickLightLoadZones(int32_t replicaFactor); diff --git a/src/meta/processors/parts/ListHostsProcessor.cpp b/src/meta/processors/parts/ListHostsProcessor.cpp index 3495ee62628..f472d75c9e7 100644 --- a/src/meta/processors/parts/ListHostsProcessor.cpp +++ b/src/meta/processors/parts/ListHostsProcessor.cpp @@ -115,6 +115,7 @@ nebula::cpp2::ErrorCode ListHostsProcessor::allHostsWithStatus(cpp2::HostRole ro auto now = time::WallClock::fastNowInMilliSec(); std::vector removeHostsKey; + std::vector heartbeatHosts; for (auto iter = nebula::value(ret).get(); iter->valid(); iter->next()) { HostInfo info = HostInfo::decode(iter->val()); if (info.role_ != role) { @@ -123,6 +124,7 @@ nebula::cpp2::ErrorCode ListHostsProcessor::allHostsWithStatus(cpp2::HostRole ro cpp2::HostItem item; auto host = MetaKeyUtils::parseHostKey(iter->key()); + heartbeatHosts.emplace_back(host); item.hostAddr_ref() = std::move(host); item.role_ref() = info.role_; @@ -154,6 +156,28 @@ nebula::cpp2::ErrorCode ListHostsProcessor::allHostsWithStatus(cpp2::HostRole ro } } + if (role == cpp2::HostRole::STORAGE) { + const auto& machinePrefix = MetaKeyUtils::machinePrefix(); + auto machineRet = doPrefix(machinePrefix); + if (!nebula::ok(machineRet)) { + auto retCode = nebula::error(machineRet); + LOG(ERROR) << "List Machines Failed, error: " << apache::thrift::util::enumNameSafe(retCode); + return retCode; + } + + for (auto iter = nebula::value(machineRet).get(); iter->valid(); iter->next()) { + auto host = MetaKeyUtils::parseMachineKey(iter->key()); + auto it = std::find(heartbeatHosts.begin(), heartbeatHosts.end(), host); + if (it == heartbeatHosts.end()) { + cpp2::HostItem item; + item.hostAddr_ref() = std::move(host); + item.role_ref() = cpp2::HostRole::STORAGE; + item.status_ref() = cpp2::HostStatus::OFFLINE; + hostItems_.emplace_back(std::move(item)); + } + } + } + removeExpiredHosts(std::move(removeHostsKey)); return nebula::cpp2::ErrorCode::SUCCEEDED; } diff --git a/src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp b/src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp index 5ad66ab7f30..93480117cad 100644 --- a/src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp +++ b/src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp @@ -5,8 +5,6 @@ #include "meta/processors/zone/AddHostsIntoZoneProcessor.h" -DECLARE_int32(heartbeat_interval_secs); - namespace nebula { namespace meta { @@ -98,11 +96,6 @@ void AddHostsIntoZoneProcessor::process(const cpp2::AddHostsIntoZoneReq& req) { zoneHosts.insert(zoneHosts.end(), hosts.begin(), hosts.end()); data.emplace_back(std::move(zoneKey), MetaKeyUtils::zoneVal(std::move(zoneHosts))); - HostInfo info(0, cpp2::HostRole::STORAGE, ""); - for (auto& host : hosts) { - data.emplace_back(MetaKeyUtils::hostKey(host.host, host.port), HostInfo::encodeV2(info)); - } - LOG(INFO) << "Add Hosts Into Zone " << zoneName; doSyncPutAndUpdate(std::move(data)); } diff --git a/src/meta/processors/zone/AddHostsProcessor.cpp b/src/meta/processors/zone/AddHostsProcessor.cpp index a4010ce0362..dc4096c7322 100644 --- a/src/meta/processors/zone/AddHostsProcessor.cpp +++ b/src/meta/processors/zone/AddHostsProcessor.cpp @@ -7,6 +7,9 @@ #include "version/Version.h" +DECLARE_uint32(expired_time_factor); +DECLARE_int32(removed_threshold_sec); + namespace nebula { namespace meta { @@ -62,10 +65,6 @@ void AddHostsProcessor::process(const cpp2::AddHostsReq& req) { return; } - HostInfo info(0, cpp2::HostRole::STORAGE, ""); - for (auto& host : hosts) { - data.emplace_back(MetaKeyUtils::hostKey(host.host, host.port), HostInfo::encodeV2(info)); - } doPut(std::move(data)); } diff --git a/src/meta/processors/zone/DropHostsProcessor.cpp b/src/meta/processors/zone/DropHostsProcessor.cpp index 81e74c9f70b..f9138b7061e 100644 --- a/src/meta/processors/zone/DropHostsProcessor.cpp +++ b/src/meta/processors/zone/DropHostsProcessor.cpp @@ -127,11 +127,20 @@ void DropHostsProcessor::process(const cpp2::DropHostsReq& req) { auto machineKey = MetaKeyUtils::machineKey(host.host, host.port); auto ret = machineExist(machineKey); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "The host " << host << " not existed!"; + LOG(ERROR) << "The machine " << host << " not existed!"; code = nebula::cpp2::ErrorCode::E_NO_HOSTS; break; } holder->remove(std::move(machineKey)); + + auto hostKey = MetaKeyUtils::hostKey(host.host, host.port); + ret = hostExist(hostKey); + if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "The host " << host << " not existed!"; + code = nebula::cpp2::ErrorCode::E_NO_HOSTS; + break; + } + holder->remove(std::move(hostKey)); } if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { diff --git a/src/meta/processors/zone/GetZoneProcessor.cpp b/src/meta/processors/zone/GetZoneProcessor.cpp index 882424055de..f5445b456b4 100644 --- a/src/meta/processors/zone/GetZoneProcessor.cpp +++ b/src/meta/processors/zone/GetZoneProcessor.cpp @@ -11,19 +11,6 @@ namespace meta { void GetZoneProcessor::process(const cpp2::GetZoneReq& req) { folly::SharedMutex::ReadHolder rHolder(LockUtils::zoneLock()); auto zoneName = req.get_zone_name(); - auto zoneIdRet = getZoneId(zoneName); - if (!nebula::ok(zoneIdRet)) { - auto retCode = nebula::error(zoneIdRet); - if (retCode == nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND) { - LOG(ERROR) << "Get Zone Failed, Zone " << zoneName << " not found."; - } else { - LOG(ERROR) << "Get Zone Failed, error: " << apache::thrift::util::enumNameSafe(retCode); - } - handleErrorCode(retCode); - onFinished(); - return; - } - auto zoneKey = MetaKeyUtils::zoneKey(zoneName); auto zoneValueRet = doGet(std::move(zoneKey)); if (!nebula::ok(zoneValueRet)) { diff --git a/src/meta/processors/zone/ListZonesProcessor.cpp b/src/meta/processors/zone/ListZonesProcessor.cpp index 61a1b07d2ff..282468ecf13 100644 --- a/src/meta/processors/zone/ListZonesProcessor.cpp +++ b/src/meta/processors/zone/ListZonesProcessor.cpp @@ -27,7 +27,7 @@ void ListZonesProcessor::process(const cpp2::ListZonesReq&) { auto hosts = MetaKeyUtils::parseZoneHosts(iter->val()); cpp2::Zone zone; zone.zone_name_ref() = std::move(zoneName); - if (hosts.size() != 0) { + if (!hosts.empty()) { zone.nodes_ref() = std::move(hosts); } else { zone.nodes_ref() = {HostAddr("", 0)}; diff --git a/src/meta/processors/zone/MergeZoneProcessor.cpp b/src/meta/processors/zone/MergeZoneProcessor.cpp index c390211f50e..fe8f1f227bd 100644 --- a/src/meta/processors/zone/MergeZoneProcessor.cpp +++ b/src/meta/processors/zone/MergeZoneProcessor.cpp @@ -147,7 +147,6 @@ void MergeZoneProcessor::process(const cpp2::MergeZoneReq& req) { auto id = MetaKeyUtils::spaceId(iter->key()); auto properties = MetaKeyUtils::parseSpace(iter->val()); auto spaceZones = properties.get_zone_names(); - bool replacement = false; for (auto& zone : zones) { auto it = std::find(spaceZones.begin(), spaceZones.end(), zone); diff --git a/src/meta/processors/zone/RenameZoneProcessor.cpp b/src/meta/processors/zone/RenameZoneProcessor.cpp index 02bfa3ce0d9..11176c2fc6f 100644 --- a/src/meta/processors/zone/RenameZoneProcessor.cpp +++ b/src/meta/processors/zone/RenameZoneProcessor.cpp @@ -44,7 +44,6 @@ void RenameZoneProcessor::process(const cpp2::RenameZoneReq& req) { return; } - // std::vector data; auto batchHolder = std::make_unique(); auto iter = nebula::value(ret).get(); while (iter->valid()) { @@ -62,7 +61,7 @@ void RenameZoneProcessor::process(const cpp2::RenameZoneReq& req) { iter->next(); } - batchHolder->remove(MetaKeyUtils::zoneKey(originalZoneKey)); + batchHolder->remove(MetaKeyUtils::zoneKey(originalZoneName)); batchHolder->put(std::move(zoneKey), std::move(originalZoneValue)); auto batch = encodeBatchValue(std::move(batchHolder)->getBatch()); doBatchOperation(std::move(batch)); diff --git a/src/meta/test/AuthProcessorTest.cpp b/src/meta/test/AuthProcessorTest.cpp index b80fdd35e5d..ef5f1993492 100644 --- a/src/meta/test/AuthProcessorTest.cpp +++ b/src/meta/test/AuthProcessorTest.cpp @@ -172,6 +172,10 @@ TEST(AuthProcessorTest, GrantRevokeTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + TestUtils::registerHB(kv.get(), hosts); + } GraphSpaceID space1, space2; // create space1 { diff --git a/src/meta/test/IndexProcessorTest.cpp b/src/meta/test/IndexProcessorTest.cpp index 379e563e15a..a6fdd518372 100644 --- a/src/meta/test/IndexProcessorTest.cpp +++ b/src/meta/test/IndexProcessorTest.cpp @@ -2047,6 +2047,10 @@ TEST(ProcessorTest, IndexIdInSpaceRangeTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + TestUtils::registerHB(kv.get(), hosts); + } // mock one space and ten tag, ten edge { // space Id is 1 diff --git a/src/meta/test/MetaClientTest.cpp b/src/meta/test/MetaClientTest.cpp index dc7315aa395..b5675e2d110 100644 --- a/src/meta/test/MetaClientTest.cpp +++ b/src/meta/test/MetaClientTest.cpp @@ -45,7 +45,8 @@ TEST(MetaClientTest, InterfacesTest) { GraphSpaceID spaceId = 0; { std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; - auto result = client->addHosts(std::move(hosts)).get(); + auto result = client->addHosts(hosts).get(); + TestUtils::registerHB(cluster.metaKV_.get(), hosts); EXPECT_TRUE(result.ok()); } { @@ -356,7 +357,8 @@ TEST(MetaClientTest, TagTest) { auto* client = cluster.metaClient_.get(); { std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; - auto result = client->addHosts(std::move(hosts)).get(); + auto result = client->addHosts(hosts).get(); + TestUtils::registerHB(cluster.metaKV_.get(), hosts); EXPECT_TRUE(result.ok()); } meta::cpp2::SpaceDesc spaceDesc; @@ -592,7 +594,8 @@ TEST(MetaClientTest, EdgeTest) { auto* client = cluster.metaClient_.get(); { std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; - auto result = client->addHosts(std::move(hosts)).get(); + auto result = client->addHosts(hosts).get(); + TestUtils::registerHB(cluster.metaKV_.get(), hosts); EXPECT_TRUE(result.ok()); } meta::cpp2::SpaceDesc spaceDesc; @@ -702,7 +705,8 @@ TEST(MetaClientTest, TagIndexTest) { auto* client = cluster.metaClient_.get(); { std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; - auto result = client->addHosts(std::move(hosts)).get(); + auto result = client->addHosts(hosts).get(); + TestUtils::registerHB(cluster.metaKV_.get(), hosts); EXPECT_TRUE(result.ok()); } @@ -882,7 +886,8 @@ TEST(MetaClientTest, EdgeIndexTest) { auto* client = cluster.metaClient_.get(); { std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; - auto result = client->addHosts(std::move(hosts)).get(); + auto result = client->addHosts(hosts).get(); + TestUtils::registerHB(cluster.metaKV_.get(), hosts); EXPECT_TRUE(result.ok()); } @@ -1203,6 +1208,7 @@ TEST(MetaClientTest, DiffTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { TestUtils::registerHB(kv, {{"0", 0}}); } meta::MetaClientOptions options; options.localHost_ = {"0", 0}; @@ -1281,7 +1287,8 @@ TEST(MetaClientTest, ListenerDiffTest) { auto client = std::make_unique(threadPool, metaAddrs, options); { std::vector hosts = {{"0", 0}}; - auto result = client->addHosts(std::move(hosts)).get(); + auto result = client->addHosts(hosts).get(); + TestUtils::registerHB(cluster.metaKV_.get(), hosts); EXPECT_TRUE(result.ok()); } client->waitForMetadReady(); @@ -1364,19 +1371,14 @@ TEST(MetaClientTest, ListenerDiffTest) { TEST(MetaClientTest, HeartbeatTest) { FLAGS_heartbeat_interval_secs = 1; - // const nebula::ClusterID kClusterId = 10; fs::TempDir rootPath("/tmp/HeartbeatTest.XXXXXX"); mock::MockCluster cluster; cluster.startMeta(rootPath.path()); auto* kv = cluster.metaKV_.get(); - // meta::MetaClientOptions options; TestUtils::createSomeHosts(kv, {{"0", 0}}); HostAddr localHost("0", 0); - // options.localHost_ = localHost; - // options.clusterId_ = kClusterId; - // options.role_ = meta::cpp2::HostRole::STORAGE; cluster.initMetaClient(); auto* client = cluster.metaClient_.get(); @@ -1700,7 +1702,8 @@ TEST(MetaClientTest, ListenerTest) { auto client = std::make_shared(threadPool, localhosts); { std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; - auto result = client->addHosts(std::move(hosts)).get(); + auto result = client->addHosts(hosts).get(); + TestUtils::registerHB(cluster.metaKV_.get(), hosts); EXPECT_TRUE(result.ok()); } client->waitForMetadReady(); @@ -1899,6 +1902,7 @@ TEST(MetaClientTest, AddHostsIntoZoneTest) { cluster.startMeta(rootPath.path()); cluster.initMetaClient(); auto* client = cluster.metaClient_.get(); + auto* kv = cluster.metaKV_.get(); { // Add host into zone with duplicate hosts std::vector hosts = {{"127.0.0.1", 8988}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; @@ -1951,6 +1955,7 @@ TEST(MetaClientTest, AddHostsIntoZoneTest) { auto result = client->addHostsIntoZone(std::move(hosts), "zone_1", false).get(); EXPECT_FALSE(result.ok()); } + { TestUtils::registerHB(kv, {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}); } { // Drop hosts which is empty. std::vector hosts = {}; diff --git a/src/meta/test/ProcessorTest.cpp b/src/meta/test/ProcessorTest.cpp index b942e98afdd..7e7f5c3edad 100644 --- a/src/meta/test/ProcessorTest.cpp +++ b/src/meta/test/ProcessorTest.cpp @@ -324,6 +324,10 @@ TEST(ProcessorTest, SpaceTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + TestUtils::registerHB(kv.get(), hosts); + } int32_t hostsNum = 4; { cpp2::SpaceDesc properties; @@ -490,6 +494,10 @@ TEST(ProcessorTest, CreateTagTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + TestUtils::registerHB(kv.get(), hosts); + } { cpp2::SpaceDesc properties; properties.space_name_ref() = "first_space"; @@ -693,6 +701,10 @@ TEST(ProcessorTest, CreateEdgeTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + TestUtils::registerHB(kv.get(), hosts); + } { cpp2::SpaceDesc properties; properties.space_name_ref() = "default_space"; @@ -885,6 +897,10 @@ TEST(ProcessorTest, KVOperationTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + TestUtils::registerHB(kv.get(), hosts); + } { cpp2::SpaceDesc properties; properties.space_name_ref() = "default_space"; @@ -2286,6 +2302,10 @@ TEST(ProcessorTest, SameNameTagsTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + TestUtils::registerHB(kv.get(), hosts); + } { cpp2::SpaceDesc properties; properties.space_name_ref() = "default_space"; @@ -2534,6 +2554,10 @@ TEST(ProcessorTest, TagIdAndEdgeTypeInSpaceRangeTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { + std::vector hosts = {{"0", 0}, {"1", 1}, {"2", 2}, {"3", 3}}; + TestUtils::registerHB(kv.get(), hosts); + } // mock one space and ten tag, ten edge { // space Id is 1 @@ -3052,6 +3076,10 @@ TEST(ProcessorTest, AddHostsIntoZoneTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { + std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + TestUtils::registerHB(kv.get(), hosts); + } { // Add host into zone with zone name conflict. cpp2::AddHostsIntoZoneReq req; @@ -3077,6 +3105,10 @@ TEST(ProcessorTest, AddHostsIntoZoneTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { + std::vector hosts = {{"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}; + TestUtils::registerHB(kv.get(), hosts); + } { // Add existed hosts. cpp2::AddHostsReq req; @@ -3182,6 +3214,10 @@ TEST(ProcessorTest, DropHostsTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { + std::vector hosts = {{"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + TestUtils::registerHB(kv.get(), hosts); + } { // Attempt to register heartbeat const ClusterID kClusterId = 10; @@ -3297,6 +3333,11 @@ TEST(ProcessorTest, DropHostsTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { + std::vector hosts = { + {"127.0.0.1", 8976}, {"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}; + TestUtils::registerHB(kv.get(), hosts); + } { // Show the zones created by add hosts cpp2::ListZonesReq req; @@ -3338,6 +3379,24 @@ TEST(ProcessorTest, DropHostsTest) { ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); ASSERT_EQ(4, resp.get_id().get_space_id()); } + { + // Create Space on cluster, the replica number same with the zone size + cpp2::SpaceDesc properties; + properties.space_name_ref() = "default_space_on_zone_duplicate"; + properties.partition_num_ref() = 9; + properties.replica_factor_ref() = 3; + properties.charset_name_ref() = "utf8"; + properties.collate_name_ref() = "utf8_bin"; + std::vector zones = {"zone_0", "zone_1", "zone_1"}; + properties.zone_names_ref() = std::move(zones); + cpp2::CreateSpaceReq req; + req.properties_ref() = std::move(properties); + auto* processor = CreateSpaceProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_INVALID_PARM, resp.get_code()); + } { // Create Space on cluster, the replica number less than the zone size cpp2::SpaceDesc properties; @@ -3355,7 +3414,7 @@ TEST(ProcessorTest, DropHostsTest) { processor->process(req); auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); - ASSERT_EQ(5, resp.get_id().get_space_id()); + ASSERT_EQ(6, resp.get_id().get_space_id()); } { // Create Space on cluster, the replica number greater than the zone size @@ -3888,7 +3947,6 @@ TEST(ProcessorTest, MergeZoneTest) { ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } { - LOG(INFO) << "========================"; cpp2::MergeZoneReq req; req.zones_ref() = {"default_zone_127.0.0.1_8978", "z_1"}; req.zone_name_ref() = "z_1"; @@ -3945,6 +4003,11 @@ TEST(ProcessorTest, DivideZoneTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { + std::vector hosts = { + {"127.0.0.1", 8986}, {"127.0.0.1", 8987}, {"127.0.0.1", 8988}, {"127.0.0.1", 8989}}; + TestUtils::registerHB(kv.get(), hosts); + } { cpp2::ListZonesReq req; auto* processor = ListZonesProcessor::instance(kv.get()); @@ -4102,6 +4165,11 @@ TEST(ProcessorTest, DivideZoneTest) { auto resp = std::move(f).get(); ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); } + { + std::vector hosts = { + {"127.0.0.1", 8976}, {"127.0.0.1", 8977}, {"127.0.0.1", 8978}, {"127.0.0.1", 8979}}; + TestUtils::registerHB(kv.get(), hosts); + } { cpp2::ListZonesReq req; auto* processor = ListZonesProcessor::instance(kv.get()); diff --git a/src/meta/upgrade/CMakeLists.txt b/src/meta/upgrade/CMakeLists.txt index 7b895c31e36..b0700952b86 100644 --- a/src/meta/upgrade/CMakeLists.txt +++ b/src/meta/upgrade/CMakeLists.txt @@ -6,11 +6,14 @@ nebula_add_library( meta_data_upgrade_obj OBJECT MetaDataUpgrade.cpp v1/MetaServiceUtilsV1.cpp + v2/MetaServiceUtilsV2.cpp ) add_dependencies( meta_data_upgrade_obj meta_v1_thrift_obj + meta_v2_thrift_obj ) nebula_add_subdirectory(v1) +nebula_add_subdirectory(v2) diff --git a/src/meta/upgrade/MetaDataUpgrade.cpp b/src/meta/upgrade/MetaDataUpgrade.cpp index 8bd89806738..e057e8d6f85 100644 --- a/src/meta/upgrade/MetaDataUpgrade.cpp +++ b/src/meta/upgrade/MetaDataUpgrade.cpp @@ -19,6 +19,7 @@ #include "meta/ActiveHostsMan.h" #include "meta/MetaServiceUtils.h" #include "meta/upgrade/v1/MetaServiceUtilsV1.h" +#include "meta/upgrade/v2/MetaServiceUtilsV2.h" DECLARE_bool(null_type); DECLARE_uint32(string_index_limit); @@ -62,6 +63,68 @@ Status MetaDataUpgrade::rewriteSpaces(const folly::StringPiece &key, return Status::OK(); } +Status MetaDataUpgrade::rewriteSpacesV2ToV3(const folly::StringPiece &key, + const folly::StringPiece &val) { + auto oldProps = meta::v2::MetaServiceUtilsV2::parseSpace(val); + cpp2::SpaceDesc spaceDesc; + spaceDesc.space_name_ref() = oldProps.get_space_name(); + spaceDesc.partition_num_ref() = oldProps.get_partition_num(); + spaceDesc.replica_factor_ref() = oldProps.get_replica_factor(); + spaceDesc.charset_name_ref() = oldProps.get_charset_name(); + spaceDesc.collate_name_ref() = oldProps.get_collate_name(); + cpp2::ColumnTypeDef def; + auto &type = oldProps.get_vid_type(); + def.type_length_ref() = *type.get_type_length(); + def.type_ref() = convertToPropertyType(type.get_type()); + + if (type.geo_shape_ref().has_value()) { + def.geo_shape_ref() = convertToGeoShape(*type.get_geo_shape()); + } + spaceDesc.vid_type_ref() = std::move(def); + if (oldProps.isolation_level_ref().has_value()) { + if (*oldProps.isolation_level_ref() == nebula::meta::v2::cpp2::IsolationLevel::DEFAULT) { + spaceDesc.isolation_level_ref() = nebula::meta::cpp2::IsolationLevel::DEFAULT; + } else { + spaceDesc.isolation_level_ref() = nebula::meta::cpp2::IsolationLevel::TOSS; + } + } + + if (oldProps.comment_ref().has_value()) { + spaceDesc.comment_ref() = *oldProps.comment_ref(); + } + + if (oldProps.group_name_ref().has_value()) { + auto groupName = *oldProps.group_name_ref(); + auto groupKey = meta::v2::MetaServiceUtilsV2::groupKey(groupName); + std::string zoneValue; + auto code = kv_->get(kDefaultSpaceId, kDefaultPartId, std::move(groupKey), &zoneValue); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + return Status::Error("Get Group Failed"); + } + + auto zones = meta::v2::MetaServiceUtilsV2::parseZoneNames(std::move(zoneValue)); + spaceDesc.zone_names_ref() = std::move(zones); + } else { + const auto &zonePrefix = MetaKeyUtils::zonePrefix(); + std::unique_ptr iter; + auto code = kv_->prefix(kDefaultSpaceId, kDefaultPartId, zonePrefix, &iter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + return Status::Error("Get Zones Failed"); + } + + std::vector<::std::string> zones; + while (iter->valid()) { + auto zoneName = MetaKeyUtils::parseZoneName(iter->key()); + zones.emplace_back(std::move(zoneName)); + iter->next(); + } + spaceDesc.zone_names_ref() = std::move(zones); + } + + NG_LOG_AND_RETURN_IF_ERROR(put(key, MetaKeyUtils::spaceVal(spaceDesc))); + return Status::OK(); +} + Status MetaDataUpgrade::rewriteParts(const folly::StringPiece &key, const folly::StringPiece &val) { auto oldHosts = meta::v1::MetaServiceUtilsV1::parsePartVal(val); std::vector newHosts; @@ -278,6 +341,60 @@ Status MetaDataUpgrade::convertToNewIndexColumns( return Status::OK(); } +nebula::cpp2::PropertyType MetaDataUpgrade::convertToPropertyType( + nebula::meta::v2::cpp2::PropertyType type) { + switch (type) { + case nebula::meta::v2::cpp2::PropertyType::BOOL: + return nebula::cpp2::PropertyType::BOOL; + case nebula::meta::v2::cpp2::PropertyType::INT64: + return nebula::cpp2::PropertyType::INT64; + case nebula::meta::v2::cpp2::PropertyType::VID: + return nebula::cpp2::PropertyType::VID; + case nebula::meta::v2::cpp2::PropertyType::FLOAT: + return nebula::cpp2::PropertyType::FLOAT; + case nebula::meta::v2::cpp2::PropertyType::DOUBLE: + return nebula::cpp2::PropertyType::DOUBLE; + case nebula::meta::v2::cpp2::PropertyType::STRING: + return nebula::cpp2::PropertyType::STRING; + case nebula::meta::v2::cpp2::PropertyType::FIXED_STRING: + return nebula::cpp2::PropertyType::FIXED_STRING; + case nebula::meta::v2::cpp2::PropertyType::INT8: + return nebula::cpp2::PropertyType::INT8; + case nebula::meta::v2::cpp2::PropertyType::INT16: + return nebula::cpp2::PropertyType::INT16; + case nebula::meta::v2::cpp2::PropertyType::INT32: + return nebula::cpp2::PropertyType::INT32; + case nebula::meta::v2::cpp2::PropertyType::TIMESTAMP: + return nebula::cpp2::PropertyType::TIMESTAMP; + case nebula::meta::v2::cpp2::PropertyType::DATE: + return nebula::cpp2::PropertyType::DATE; + case nebula::meta::v2::cpp2::PropertyType::DATETIME: + return nebula::cpp2::PropertyType::DATETIME; + case nebula::meta::v2::cpp2::PropertyType::TIME: + return nebula::cpp2::PropertyType::TIME; + case nebula::meta::v2::cpp2::PropertyType::GEOGRAPHY: + return nebula::cpp2::PropertyType::GEOGRAPHY; + default: + return nebula::cpp2::PropertyType::UNKNOWN; + } +} + +nebula::meta::cpp2::GeoShape MetaDataUpgrade::convertToGeoShape( + nebula::meta::v2::cpp2::GeoShape shape) { + switch (shape) { + case nebula::meta::v2::cpp2::GeoShape::ANY: + return nebula::meta::cpp2::GeoShape::ANY; + case nebula::meta::v2::cpp2::GeoShape::POINT: + return nebula::meta::cpp2::GeoShape::POINT; + case nebula::meta::v2::cpp2::GeoShape::LINESTRING: + return nebula::meta::cpp2::GeoShape::LINESTRING; + case nebula::meta::v2::cpp2::GeoShape::POLYGON: + return nebula::meta::cpp2::GeoShape::POLYGON; + default: + LOG(FATAL) << "Unimplemented"; + } +} + void MetaDataUpgrade::printHost(const folly::StringPiece &key, const folly::StringPiece &val) { auto host = meta::v1::MetaServiceUtilsV1::parseHostKey(key); auto info = HostInfo::decodeV1(val); @@ -288,7 +405,7 @@ void MetaDataUpgrade::printHost(const folly::StringPiece &key, const folly::Stri LOG(INFO) << "Host info: gitInfoSha_: " << info.gitInfoSha_; } -void MetaDataUpgrade::printSpaces(const folly::StringPiece &val) { +void MetaDataUpgrade::printSpacesV1(const folly::StringPiece &val) { auto oldProps = meta::v1::MetaServiceUtilsV1::parseSpace(val); LOG(INFO) << "Space name: " << oldProps.get_space_name(); LOG(INFO) << "Partition num: " << oldProps.get_partition_num(); @@ -297,6 +414,18 @@ void MetaDataUpgrade::printSpaces(const folly::StringPiece &val) { LOG(INFO) << "Collate name: " << oldProps.get_collate_name(); } +void MetaDataUpgrade::printSpacesV2(const folly::StringPiece &val) { + auto oldProps = meta::v2::MetaServiceUtilsV2::parseSpace(val); + LOG(INFO) << "Space name: " << oldProps.get_space_name(); + LOG(INFO) << "Partition num: " << oldProps.get_partition_num(); + LOG(INFO) << "Replica factor: " << oldProps.get_replica_factor(); + LOG(INFO) << "Charset name: " << oldProps.get_charset_name(); + LOG(INFO) << "Collate name: " << oldProps.get_collate_name(); + if (oldProps.group_name_ref().has_value()) { + LOG(INFO) << "Group name: " << *oldProps.group_name_ref(); + } +} + void MetaDataUpgrade::printParts(const folly::StringPiece &key, const folly::StringPiece &val) { auto spaceId = meta::v1::MetaServiceUtilsV1::parsePartKeySpaceId(key); auto partId = meta::v1::MetaServiceUtilsV1::parsePartKeyPartId(key); @@ -434,5 +563,10 @@ void MetaDataUpgrade::printJobDesc(const folly::StringPiece &key, const folly::S LOG(INFO) << "JobDesc stopTime: " << stopTime; } +Status MetaDataUpgrade::saveMachineAndZone(std::vector data) { + NG_LOG_AND_RETURN_IF_ERROR(put(data)); + return Status::OK(); +} + } // namespace meta } // namespace nebula diff --git a/src/meta/upgrade/MetaDataUpgrade.h b/src/meta/upgrade/MetaDataUpgrade.h index dbea1afcbfc..bd4ffea08a1 100644 --- a/src/meta/upgrade/MetaDataUpgrade.h +++ b/src/meta/upgrade/MetaDataUpgrade.h @@ -14,6 +14,7 @@ #include "kvstore/KVStore.h" #include "meta/processors/Common.h" #include "meta/upgrade/v1/gen-cpp2/meta_types.h" +#include "meta/upgrade/v2/gen-cpp2/meta_types.h" namespace nebula { namespace meta { @@ -33,10 +34,15 @@ class MetaDataUpgrade final { Status rewriteConfigs(const folly::StringPiece &key, const folly::StringPiece &val); Status rewriteJobDesc(const folly::StringPiece &key, const folly::StringPiece &val); + Status rewriteSpacesV2ToV3(const folly::StringPiece &key, const folly::StringPiece &val); + Status deleteKeyVal(const folly::StringPiece &key); + Status saveMachineAndZone(std::vector data); + void printHost(const folly::StringPiece &key, const folly::StringPiece &val); - void printSpaces(const folly::StringPiece &val); + void printSpacesV1(const folly::StringPiece &val); + void printSpacesV2(const folly::StringPiece &val); void printParts(const folly::StringPiece &key, const folly::StringPiece &val); void printLeaders(const folly::StringPiece &key); void printSchemas(const folly::StringPiece &val); @@ -67,6 +73,26 @@ class MetaDataUpgrade final { return Status::OK(); } + Status put(std::vector data) { + folly::Baton baton; + auto ret = nebula::cpp2::ErrorCode::SUCCEEDED; + kv_->asyncMultiPut(kDefaultSpaceId, + kDefaultPartId, + std::move(data), + [&ret, &baton](nebula::cpp2::ErrorCode code) { + if (nebula::cpp2::ErrorCode::SUCCEEDED != code) { + ret = code; + LOG(INFO) << "Put data error on meta server"; + } + baton.post(); + }); + baton.wait(); + if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { + return Status::Error("Put data failed"); + } + return Status::OK(); + } + Status remove(const folly::StringPiece &key) { std::vector keys{key.str()}; folly::Baton baton; @@ -94,6 +120,10 @@ class MetaDataUpgrade final { Status convertToNewIndexColumns(const std::vector &oldCols, std::vector &newCols); + nebula::cpp2::PropertyType convertToPropertyType(nebula::meta::v2::cpp2::PropertyType type); + + nebula::meta::cpp2::GeoShape convertToGeoShape(nebula::meta::v2::cpp2::GeoShape shape); + private: kvstore::KVStore *kv_ = nullptr; }; diff --git a/src/meta/upgrade/v2/CMakeLists.txt b/src/meta/upgrade/v2/CMakeLists.txt new file mode 100644 index 00000000000..df5d9d2549f --- /dev/null +++ b/src/meta/upgrade/v2/CMakeLists.txt @@ -0,0 +1,24 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +set(THRIFT1 ${Fbthrift_BIN}) + +set(meta_v2_sources + gen-cpp2/meta_constants.cpp + gen-cpp2/meta_data.cpp + gen-cpp2/meta_metadata.cpp + gen-cpp2/meta_types.cpp + ) + +add_custom_command( + OUTPUT ${meta_v2_sources} + COMMAND "${THRIFT1}" "--strict" "--allow-neg-enum-vals" "--gen" "mstch_cpp2:include_prefix=\"meta/upgrade/v2\",process_in_event_base,stack_arguments" "-o" "." "${CMAKE_CURRENT_SOURCE_DIR}/meta.thrift" + DEPENDS meta.thrift + ) + +nebula_add_library(meta_v2_thrift_obj OBJECT ${meta_v2_sources}) + +target_compile_options(meta_v2_thrift_obj PRIVATE "-Wno-pedantic") +target_compile_options(meta_v2_thrift_obj PRIVATE "-Wno-extra") +target_compile_options(meta_v2_thrift_obj PRIVATE "-Wno-deprecated-declarations") diff --git a/src/meta/upgrade/v2/MetaServiceUtilsV2.cpp b/src/meta/upgrade/v2/MetaServiceUtilsV2.cpp new file mode 100644 index 00000000000..bb23cd29c74 --- /dev/null +++ b/src/meta/upgrade/v2/MetaServiceUtilsV2.cpp @@ -0,0 +1,32 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "meta/upgrade/v2/MetaServiceUtilsV2.h" + +#include +#include + +namespace nebula::meta::v2 { + +std::string MetaServiceUtilsV2::groupKey(const std::string& group) { + std::string key; + key.reserve(kGroupsTable.size() + group.size()); + key.append(kGroupsTable.data(), kGroupsTable.size()).append(group); + return key; +} + +std::vector MetaServiceUtilsV2::parseZoneNames(folly::StringPiece rawData) { + std::vector zones; + folly::split(',', rawData.str(), zones); + return zones; +} + +cpp2::SpaceDesc MetaServiceUtilsV2::parseSpace(folly::StringPiece rawData) { + cpp2::SpaceDesc spaceDesc; + apache::thrift::CompactSerializer::deserialize(rawData, spaceDesc); + return spaceDesc; +} + +} // namespace nebula::meta::v2 diff --git a/src/meta/upgrade/v2/MetaServiceUtilsV2.h b/src/meta/upgrade/v2/MetaServiceUtilsV2.h new file mode 100644 index 00000000000..47bf010ce98 --- /dev/null +++ b/src/meta/upgrade/v2/MetaServiceUtilsV2.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef TOOLS_METADATAUPDATETOOL_OLDTHRIFT_METADATAUPDATE_V2_H_ +#define TOOLS_METADATAUPDATETOOL_OLDTHRIFT_METADATAUPDATE_V2_H_ + +#include "common/base/Base.h" +#include "common/base/Status.h" +#include "common/thrift/ThriftTypes.h" +#include "interface/gen-cpp2/meta_types.h" +#include "kvstore/Common.h" +#include "meta/upgrade/v2/gen-cpp2/meta_types.h" + +namespace nebula::meta::v2 { + +const std::string kGroupsTable = "__groups__"; // NOLINT + +class MetaServiceUtilsV2 final { + public: + MetaServiceUtilsV2() = delete; + + static std::string groupKey(const std::string& group); + + static std::vector parseZoneNames(folly::StringPiece rawData); + + static cpp2::SpaceDesc parseSpace(folly::StringPiece rawData); +}; + +} // namespace nebula::meta::v2 + +#endif // TOOLS_METADATAUPDATETOOL_OLDTHRIFT_METADATAUPDATE_V2_H_ diff --git a/src/meta/upgrade/v2/meta.thrift b/src/meta/upgrade/v2/meta.thrift new file mode 100644 index 00000000000..d46922478b8 --- /dev/null +++ b/src/meta/upgrade/v2/meta.thrift @@ -0,0 +1,81 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +namespace cpp nebula.meta.v2 + +cpp_include "common/thrift/ThriftTypes.h" + +typedef i32 (cpp.type = "nebula::GraphSpaceID") GraphSpaceID +typedef i32 (cpp.type = "nebula::PartitionID") PartitionID +typedef i32 (cpp.type = "nebula::TagID") TagID +typedef i32 (cpp.type = "nebula::EdgeType") EdgeType +typedef i64 (cpp.type = "nebula::EdgeRanking") EdgeRanking +typedef i64 (cpp.type = "nebula::VertexID") VertexID +typedef i32 (cpp.type = "nebula::IndexID") IndexID + +typedef i32 IPv4 +typedef i32 (cpp.type = "nebula::Port") Port + +typedef i64 (cpp.type = "nebula::SchemaVer") SchemaVer + +struct SpaceDesc { + 1: binary space_name, + 2: i32 partition_num = 0, + 3: i32 replica_factor = 0, + 4: binary charset_name, + 5: binary collate_name, + 6: ColumnTypeDef vid_type = {"type": PropertyType.FIXED_STRING, "type_length": 8}, + 7: optional binary group_name, + 8: optional IsolationLevel isolation_level, + 9: optional binary comment, +} + +enum PropertyType { + UNKNOWN = 0, + + // Simple types + BOOL = 1, + INT64 = 2, // This is the same as INT in v1 + VID = 3, // Deprecated, only supported by v1 + FLOAT = 4, + DOUBLE = 5, + STRING = 6, + // String with fixed length. If the string content is shorteri + // than the given length, '\0' will be padded to the end + FIXED_STRING = 7, // New in v2 + INT8 = 8, // New in v2 + INT16 = 9, // New in v2 + INT32 = 10, // New in v2 + + // Date time + TIMESTAMP = 21, + DATE = 24, + DATETIME = 25, + TIME = 26, + + // Geo spatial + GEOGRAPHY = 31, +} (cpp.enum_strict) + +// Geo shape type +enum GeoShape { + ANY = 0, + POINT = 1, + LINESTRING = 2, + POLYGON = 3, +} (cpp.enum_strict) + +struct ColumnTypeDef { + 1: required PropertyType type, + // type_length is valid for fixed_string type + 2: optional i16 type_length = 0, + // geo_shape is valid for geography type + 3: optional GeoShape geo_shape, +} + +enum IsolationLevel { + DEFAULT = 0x00, // allow add half edge(either in or out edge succeeded) + TOSS = 0x01, // add in and out edge atomic +} (cpp.enum_strict) \ No newline at end of file diff --git a/src/mock/LocalServer.h b/src/mock/LocalServer.h new file mode 100644 index 00000000000..1edd2464914 --- /dev/null +++ b/src/mock/LocalServer.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#pragma once + +#include + +#include "common/base/Base.h" +#include "common/thread/NamedThread.h" +#include "storage/GraphStorageLocalServer.h" + +namespace nebula { +namespace mock { + +struct LocalServer { + ~LocalServer() { + if (server_ != nullptr) { + server_->stop(); + } + if (thread_ != nullptr) { + thread_->join(); + } + VLOG(3) << "~LocalServer"; + } + + void start(const std::string& name, + uint16_t port, + std::shared_ptr handler) { + UNUSED(port); + std::shared_ptr workers = + apache::thrift::concurrency::PriorityThreadManager::newPriorityThreadManager(1); + workers->setNamePrefix("executor"); + workers->start(); + + server_ = storage::GraphStorageLocalServer::getInstance(); + server_->setThreadManager(workers); + server_->setInterface(std::move(handler)); + thread_ = std::make_unique(name, [this, name] { + server_->serve(); + LOG(INFO) << "The " << name << " server has been stopped"; + }); + usleep(10000); + } + + std::shared_ptr server_{nullptr}; + std::unique_ptr thread_{nullptr}; + uint16_t port_{0}; +}; + +} // namespace mock +} // namespace nebula diff --git a/src/mock/MockCluster.cpp b/src/mock/MockCluster.cpp index e2c2732e9ba..0eb5407fa6f 100644 --- a/src/mock/MockCluster.cpp +++ b/src/mock/MockCluster.cpp @@ -226,10 +226,17 @@ void MockCluster::startStorage(HostAddr addr, storageAdminServer_->start("admin-storage", addr.port - 1, adminHandler); LOG(INFO) << "The admin storage daemon started on port " << storageAdminServer_->port_; +#ifndef BUILD_STANDALONE graphStorageServer_ = std::make_unique(); auto graphHandler = std::make_shared(env); graphStorageServer_->start("graph-storage", addr.port, graphHandler); LOG(INFO) << "The graph storage daemon started on port " << graphStorageServer_->port_; +#else + graphStorageServer_ = std::make_unique(); + auto graphHandler = std::make_shared(env); + graphStorageServer_->start("graph-storage", addr.port, graphHandler); + LOG(INFO) << "The graph storage daemon started on Local server."; +#endif } std::unique_ptr MockCluster::memSchemaMan(SchemaVer schemaVerCount, diff --git a/src/mock/MockCluster.h b/src/mock/MockCluster.h index 1e898df72a3..45493ce4aec 100644 --- a/src/mock/MockCluster.h +++ b/src/mock/MockCluster.h @@ -18,6 +18,7 @@ #include "kvstore/KVStore.h" #include "kvstore/NebulaStore.h" #include "kvstore/PartManager.h" +#include "mock/LocalServer.h" #include "mock/RpcServer.h" #include "storage/BaseProcessor.h" #include "storage/GraphStorageServiceHandler.h" @@ -94,9 +95,11 @@ class MockCluster { void stop() { if (metaClient_) { + metaClient_->notifyStop(); metaClient_->stop(); } if (lMetaClient_) { + metaClient_->notifyStop(); lMetaClient_->stop(); } if (metaKV_) { @@ -117,7 +120,11 @@ class MockCluster { std::unique_ptr metaKV_{nullptr}; std::unique_ptr storageAdminServer_{nullptr}; +#ifndef BUILD_STANDALONE std::unique_ptr graphStorageServer_{nullptr}; +#else + std::unique_ptr graphStorageServer_{nullptr}; +#endif std::unique_ptr storageKV_{nullptr}; std::unique_ptr storageEnv_{nullptr}; diff --git a/src/parser/TraverseSentences.cpp b/src/parser/TraverseSentences.cpp index 24a613f215f..9114bed6124 100644 --- a/src/parser/TraverseSentences.cpp +++ b/src/parser/TraverseSentences.cpp @@ -162,6 +162,10 @@ std::string FetchEdgesSentence::toString() const { } else { buf += edgeKeys_->toString(); } + if (yieldClause_ != nullptr) { + buf += " "; + buf += yieldClause_->toString(); + } return buf; } diff --git a/src/parser/parser.yy b/src/parser/parser.yy index 8c61c1f0e27..1cdf988d921 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -223,7 +223,7 @@ static constexpr size_t kCommentLengthLimit = 256; %token STRING VARIABLE LABEL IPV4 %type name_label unreserved_keyword predicate_name -%type expression +%type expression expression_internal %type property_expression %type vertex_prop_expression %type edge_prop_expression @@ -556,6 +556,18 @@ unreserved_keyword ; expression + : expression_internal { + if(!graph::ExpressionUtils::checkExprDepth($1)){ + // delete $1; + std::ostringstream errStr; + errStr << "The above expression's depth exceeds the maximum depth:" << graph::ExpressionUtils::kMaxDepth; + throw nebula::GraphParser::syntax_error(@1, errStr.str()); + } + $$ = $1; + } + ; + +expression_internal : constant_expression { $$ = $1; } @@ -572,7 +584,7 @@ expression } | MINUS { scanner.setUnaryMinus(true); - } expression %prec UNARY_MINUS { + } expression_internal %prec UNARY_MINUS { if (scanner.isIntMin()) { $$ = $3; scanner.setIsIntMin(false); @@ -581,98 +593,98 @@ expression } scanner.setUnaryMinus(false); } - | PLUS expression %prec UNARY_PLUS { + | PLUS expression_internal %prec UNARY_PLUS { $$ = UnaryExpression::makePlus(qctx->objPool(), $2); } - | NOT expression { + | NOT expression_internal { $$ = UnaryExpression::makeNot(qctx->objPool(), $2); } - | KW_NOT expression { + | KW_NOT expression_internal { $$ = UnaryExpression::makeNot(qctx->objPool(), $2); } - | L_PAREN type_spec R_PAREN expression %prec CASTING { + | L_PAREN type_spec R_PAREN expression_internal %prec CASTING { $$ = TypeCastingExpression::make(qctx->objPool(), graph::SchemaUtil::propTypeToValueType($2->type), $4); delete $2; } - | expression STAR expression { + | expression_internal STAR expression_internal { $$ = ArithmeticExpression::makeMultiply(qctx->objPool(), $1, $3); } - | expression DIV expression { + | expression_internal DIV expression_internal { $$ = ArithmeticExpression::makeDivision(qctx->objPool(), $1, $3); } - | expression MOD expression { + | expression_internal MOD expression_internal { $$ = ArithmeticExpression::makeMod(qctx->objPool(), $1, $3); } - | expression PLUS expression { + | expression_internal PLUS expression_internal { $$ = ArithmeticExpression::makeAdd(qctx->objPool(), $1, $3); } - | expression MINUS expression { + | expression_internal MINUS expression_internal { $$ = ArithmeticExpression::makeMinus(qctx->objPool(), $1, $3); } - | expression LT expression { + | expression_internal LT expression_internal { $$ = RelationalExpression::makeLT(qctx->objPool(), $1, $3); } - | expression GT expression { + | expression_internal GT expression_internal { $$ = RelationalExpression::makeGT(qctx->objPool(), $1, $3); } - | expression LE expression { + | expression_internal LE expression_internal { $$ = RelationalExpression::makeLE(qctx->objPool(), $1, $3); } - | expression GE expression { + | expression_internal GE expression_internal { $$ = RelationalExpression::makeGE(qctx->objPool(), $1, $3); } - | expression REG expression { + | expression_internal REG expression_internal { $$ = RelationalExpression::makeREG(qctx->objPool(), $1, $3); } - | expression KW_IN expression { + | expression_internal KW_IN expression_internal { $$ = RelationalExpression::makeIn(qctx->objPool(), $1, $3); } - | expression KW_NOT_IN expression { + | expression_internal KW_NOT_IN expression_internal { $$ = RelationalExpression::makeNotIn(qctx->objPool(), $1, $3); } - | expression KW_CONTAINS expression { + | expression_internal KW_CONTAINS expression_internal { $$ = RelationalExpression::makeContains(qctx->objPool(), $1, $3); } - | expression KW_NOT_CONTAINS expression { + | expression_internal KW_NOT_CONTAINS expression_internal { $$ = RelationalExpression::makeNotContains(qctx->objPool(), $1, $3); } - | expression KW_STARTS_WITH expression { + | expression_internal KW_STARTS_WITH expression_internal { $$ = RelationalExpression::makeStartsWith(qctx->objPool(), $1, $3); } - | expression KW_NOT_STARTS_WITH expression { + | expression_internal KW_NOT_STARTS_WITH expression_internal { $$ = RelationalExpression::makeNotStartsWith(qctx->objPool(), $1, $3); } - | expression KW_ENDS_WITH expression { + | expression_internal KW_ENDS_WITH expression_internal { $$ = RelationalExpression::makeEndsWith(qctx->objPool(), $1, $3); } - | expression KW_NOT_ENDS_WITH expression { + | expression_internal KW_NOT_ENDS_WITH expression_internal { $$ = RelationalExpression::makeNotEndsWith(qctx->objPool(), $1, $3); } - | expression KW_IS_NULL { + | expression_internal KW_IS_NULL { $$ = UnaryExpression::makeIsNull(qctx->objPool(), $1); } - | expression KW_IS_NOT_NULL { + | expression_internal KW_IS_NOT_NULL { $$ = UnaryExpression::makeIsNotNull(qctx->objPool(), $1); } - | expression KW_IS_EMPTY { + | expression_internal KW_IS_EMPTY { $$ = UnaryExpression::makeIsEmpty(qctx->objPool(), $1); } - | expression KW_IS_NOT_EMPTY { + | expression_internal KW_IS_NOT_EMPTY { $$ = UnaryExpression::makeIsNotEmpty(qctx->objPool(), $1); } - | expression EQ expression { + | expression_internal EQ expression_internal { $$ = RelationalExpression::makeEQ(qctx->objPool(), $1, $3); } - | expression NE expression { + | expression_internal NE expression_internal { $$ = RelationalExpression::makeNE(qctx->objPool(), $1, $3); } - | expression KW_AND expression { + | expression_internal KW_AND expression_internal { $$ = LogicalExpression::makeAnd(qctx->objPool(), $1, $3); } - | expression KW_OR expression { + | expression_internal KW_OR expression_internal { $$ = LogicalExpression::makeOr(qctx->objPool(), $1, $3); } - | expression KW_XOR expression { + | expression_internal KW_XOR expression_internal { $$ = LogicalExpression::makeXor(qctx->objPool(), $1, $3); } | case_expression { @@ -712,7 +724,7 @@ constant_expression ; compound_expression - : L_PAREN expression R_PAREN { + : L_PAREN expression_internal R_PAREN { $$ = $2; } | property_expression { @@ -751,51 +763,51 @@ property_expression ; subscript_expression - : name_label L_BRACKET expression R_BRACKET { + : name_label L_BRACKET expression_internal R_BRACKET { $$ = SubscriptExpression::make(qctx->objPool(), LabelExpression::make(qctx->objPool(), *$1), $3); delete $1; } - | VARIABLE L_BRACKET expression R_BRACKET { + | VARIABLE L_BRACKET expression_internal R_BRACKET { $$ = SubscriptExpression::make(qctx->objPool(), VariableExpression::make(qctx->objPool(), *$1), $3); delete $1; } - | compound_expression L_BRACKET expression R_BRACKET { + | compound_expression L_BRACKET expression_internal R_BRACKET { $$ = SubscriptExpression::make(qctx->objPool(), $1, $3); } ; subscript_range_expression - : name_label L_BRACKET expression DOT_DOT expression R_BRACKET { + : name_label L_BRACKET expression_internal DOT_DOT expression_internal R_BRACKET { $$ = SubscriptRangeExpression::make(qctx->objPool(), LabelExpression::make(qctx->objPool(), *$1), $3, $5); delete($1); } - | name_label L_BRACKET DOT_DOT expression R_BRACKET { + | name_label L_BRACKET DOT_DOT expression_internal R_BRACKET { $$ = SubscriptRangeExpression::make(qctx->objPool(), LabelExpression::make(qctx->objPool(), *$1), nullptr, $4); delete($1); } - | name_label L_BRACKET expression DOT_DOT R_BRACKET { + | name_label L_BRACKET expression_internal DOT_DOT R_BRACKET { $$ = SubscriptRangeExpression::make(qctx->objPool(), LabelExpression::make(qctx->objPool(), *$1), $3, nullptr); delete($1); } - | VARIABLE L_BRACKET expression DOT_DOT expression R_BRACKET { + | VARIABLE L_BRACKET expression_internal DOT_DOT expression_internal R_BRACKET { $$ = SubscriptRangeExpression::make(qctx->objPool(), VariableExpression::make(qctx->objPool(), *$1), $3, $5); delete($1); } - | VARIABLE L_BRACKET DOT_DOT expression R_BRACKET { + | VARIABLE L_BRACKET DOT_DOT expression_internal R_BRACKET { $$ = SubscriptRangeExpression::make(qctx->objPool(), VariableExpression::make(qctx->objPool(), *$1), nullptr, $4); delete($1); } - | VARIABLE L_BRACKET expression DOT_DOT R_BRACKET { + | VARIABLE L_BRACKET expression_internal DOT_DOT R_BRACKET { $$ = SubscriptRangeExpression::make(qctx->objPool(), VariableExpression::make(qctx->objPool(), *$1), $3, nullptr); delete($1); } - | compound_expression L_BRACKET expression DOT_DOT expression R_BRACKET { + | compound_expression L_BRACKET expression_internal DOT_DOT expression_internal R_BRACKET { $$ = SubscriptRangeExpression::make(qctx->objPool(), $1, $3, $5); } - | compound_expression L_BRACKET DOT_DOT expression R_BRACKET { + | compound_expression L_BRACKET DOT_DOT expression_internal R_BRACKET { $$ = SubscriptRangeExpression::make(qctx->objPool(), $1, nullptr, $4); } - | compound_expression L_BRACKET expression DOT_DOT R_BRACKET { + | compound_expression L_BRACKET expression_internal DOT_DOT R_BRACKET { $$ = SubscriptRangeExpression::make(qctx->objPool(), $1, $3, nullptr); } ; @@ -832,7 +844,7 @@ generic_case_expression ; conditional_expression - : expression QM expression COLON expression { + : expression_internal QM expression_internal COLON expression_internal { auto cases = CaseList::make(qctx->objPool()); cases->add($1, $3); auto expr = CaseExpression::make(qctx->objPool(), cases, false); @@ -845,7 +857,7 @@ case_condition : %empty { $$ = nullptr; } - | expression { + | expression_internal { $$ = $1; } ; @@ -854,17 +866,17 @@ case_default : %empty { $$ = nullptr; } - | KW_ELSE expression { + | KW_ELSE expression_internal { $$ = $2; } ; when_then_list - : KW_WHEN expression KW_THEN expression { + : KW_WHEN expression_internal KW_THEN expression_internal { $$ = CaseList::make(qctx->objPool()); $$->add($2, $4); } - | when_then_list KW_WHEN expression KW_THEN expression { + | when_then_list KW_WHEN expression_internal KW_THEN expression_internal { $1->add($3, $5); $$ = $1; } @@ -878,7 +890,7 @@ predicate_name ; predicate_expression - : predicate_name L_PAREN expression KW_IN expression KW_WHERE expression R_PAREN { + : predicate_name L_PAREN expression_internal KW_IN expression_internal KW_WHERE expression_internal R_PAREN { if ($3->kind() != Expression::Kind::kLabel) { delete $1; throw nebula::GraphParser::syntax_error(@3, "The loop variable must be a label in predicate functions"); @@ -889,7 +901,7 @@ predicate_expression $$ = expr; delete $1; } - | KW_EXISTS L_PAREN expression R_PAREN { + | KW_EXISTS L_PAREN expression_internal R_PAREN { if ($3->kind() != Expression::Kind::kLabelAttribute && $3->kind() != Expression::Kind::kAttribute && $3->kind() != Expression::Kind::kSubscript) { throw nebula::GraphParser::syntax_error(@3, "The exists only accept LabelAttribute, Attribute and Subscript"); @@ -899,7 +911,7 @@ predicate_expression ; list_comprehension_expression - : L_BRACKET expression KW_IN expression KW_WHERE expression R_BRACKET { + : L_BRACKET expression_internal KW_IN expression_internal KW_WHERE expression_internal R_BRACKET { if ($2->kind() != Expression::Kind::kLabel) { throw nebula::GraphParser::syntax_error(@2, "The loop variable must be a label in list comprehension"); } @@ -908,7 +920,7 @@ list_comprehension_expression nebula::graph::ParserUtil::rewriteLC(qctx, expr, innerVar); $$ = expr; } - | L_BRACKET expression KW_IN expression PIPE expression R_BRACKET { + | L_BRACKET expression_internal KW_IN expression_internal PIPE expression_internal R_BRACKET { if ($2->kind() != Expression::Kind::kLabel) { throw nebula::GraphParser::syntax_error(@2, "The loop variable must be a label in list comprehension"); } @@ -917,7 +929,7 @@ list_comprehension_expression nebula::graph::ParserUtil::rewriteLC(qctx, expr, innerVar); $$ = expr; } - | L_BRACKET expression KW_IN expression KW_WHERE expression PIPE expression R_BRACKET { + | L_BRACKET expression_internal KW_IN expression_internal KW_WHERE expression_internal PIPE expression_internal R_BRACKET { if ($2->kind() != Expression::Kind::kLabel) { throw nebula::GraphParser::syntax_error(@2, "The loop variable must be a label in list comprehension"); } @@ -929,7 +941,7 @@ list_comprehension_expression ; reduce_expression - : KW_REDUCE L_PAREN name_label ASSIGN expression COMMA name_label KW_IN expression PIPE expression R_PAREN { + : KW_REDUCE L_PAREN name_label ASSIGN expression_internal COMMA name_label KW_IN expression_internal PIPE expression_internal R_PAREN { auto *expr = ReduceExpression::make(qctx->objPool(), *$3, $5, *$7, $9, $11); nebula::graph::ParserUtil::rewriteReduce(qctx, expr, *$3, *$7); $$ = expr; @@ -1016,7 +1028,7 @@ function_call_expression throw nebula::GraphParser::syntax_error(@1, "Unknown function "); } } - | LABEL L_PAREN KW_DISTINCT expression R_PAREN { + | LABEL L_PAREN KW_DISTINCT expression_internal R_PAREN { if (AggFunctionManager::find(*$1).ok()) { $$ = AggregateExpression::make(qctx->objPool(), *$1, $4, true); delete($1); @@ -1136,13 +1148,13 @@ argument_list Expression *arg = EdgeExpression::make(qctx->objPool()); $$->addArgument(arg); } - | expression { + | expression_internal { $$ = ArgumentList::make(qctx->objPool()); Expression* arg = nullptr; arg = $1; $$->addArgument(arg); } - | argument_list COMMA expression { + | argument_list COMMA expression_internal { $$ = $1; Expression* arg = nullptr; arg = $3; @@ -1280,11 +1292,11 @@ opt_expression_list ; expression_list - : expression { + : expression_internal { $$ = ExpressionList::make(qctx->objPool()); $$->add($1); } - | expression_list COMMA expression { + | expression_list COMMA expression_internal { $$ = $1; $$->add($3); } @@ -1309,12 +1321,12 @@ opt_map_item_list ; map_item_list - : name_label COLON expression { + : name_label COLON expression_internal { $$ = MapItemList::make(qctx->objPool()); $$->add(*$1, $3); delete $1; } - | map_item_list COMMA name_label COLON expression { + | map_item_list COMMA name_label COLON expression_internal { $$ = $1; $$->add(*$3, $5); delete $3; @@ -1623,47 +1635,19 @@ unwind_clause with_clause : KW_WITH match_return_items match_order_by match_skip match_limit where_clause { - if ($6 && graph::ExpressionUtils::findAny($6->filter(),{Expression::Kind::kAggregate})) { - delete($2); - delete($3); - delete($4); - delete($5); - delete($6); - throw nebula::GraphParser::syntax_error(@6, "Invalid use of aggregating function in this context."); - } $$ = new WithClause($2, $3, $4, $5, $6, false/*distinct*/); } | KW_WITH KW_DISTINCT match_return_items match_order_by match_skip match_limit where_clause { - if ($7 && graph::ExpressionUtils::findAny($7->filter(),{Expression::Kind::kAggregate})) { - delete($3); - delete($4); - delete($5); - delete($6); - delete($7); - throw nebula::GraphParser::syntax_error(@7, "Invalid use of aggregating function in this context."); - } $$ = new WithClause($3, $4, $5, $6, $7, true); } ; match_clause : KW_MATCH match_path_list where_clause { - if ($3 && graph::ExpressionUtils::findAny($3->filter(),{Expression::Kind::kAggregate})) { - delete($2); - delete($3); - throw nebula::GraphParser::syntax_error(@3, "Invalid use of aggregating function in this context."); - } else { - $$ = new MatchClause($2, $3, false/*optional*/); - } + $$ = new MatchClause($2, $3, false/*optional*/); } | KW_OPTIONAL KW_MATCH match_path_list where_clause { - if ($4 && graph::ExpressionUtils::findAny($4->filter(),{Expression::Kind::kAggregate})) { - delete($3); - delete($4); - throw nebula::GraphParser::syntax_error(@4, "Invalid use of aggregating function in this context."); - } else { - $$ = new MatchClause($3, $4, true); - } + $$ = new MatchClause($3, $4, true); } ; diff --git a/src/parser/test/CMakeLists.txt b/src/parser/test/CMakeLists.txt index 755d2c7a350..0ad0b9ce2a2 100644 --- a/src/parser/test/CMakeLists.txt +++ b/src/parser/test/CMakeLists.txt @@ -49,6 +49,13 @@ set(PARSER_TEST_LIBS $ ) +if(ENABLE_STANDALONE_VERSION) +set(PARSER_TEST_LIBS + ${PARSER_TEST_LIBS} + $ +) +endif() + nebula_add_test( NAME parser_test SOURCES ParserTest.cpp diff --git a/src/parser/test/ParserBenchmark.cpp b/src/parser/test/ParserBenchmark.cpp index b1af6f3dc42..d420f1547d9 100644 --- a/src/parser/test/ParserBenchmark.cpp +++ b/src/parser/test/ParserBenchmark.cpp @@ -14,9 +14,7 @@ auto simpleQuery = "USE myspace"; auto complexQuery = "GO 2 STEPS FROM 123456789 OVER myedge " "WHERE alias.prop1 + alias.prop2 * alias.prop3 > alias.prop4 AND " - "alias.prop5 == alias.prop6 YIELD 1 AS first, 2 AS second"; - -auto qctx = std::make_unique(); + "alias.prop5 == alias.prop6 YIELD 1+1+1+1+1+1+1+1 AS first, 2 AS second"; size_t SimpleQuery(size_t iters, size_t nrThreads) { constexpr size_t ops = 500000UL; @@ -24,6 +22,7 @@ size_t SimpleQuery(size_t iters, size_t nrThreads) { auto parse = [&]() { auto n = iters * ops; for (auto i = 0UL; i < n; i++) { + auto qctx = std::make_unique(); // static thread_local GQLParser parser; GQLParser parser(qctx.get()); auto result = parser.parse(simpleQuery); @@ -48,6 +47,7 @@ size_t ComplexQuery(size_t iters, size_t nrThreads) { auto parse = [&]() { auto n = iters * ops; for (auto i = 0UL; i < n; i++) { + auto qctx = std::make_unique(); // static thread_local GQLParser parser; GQLParser parser(qctx.get()); auto result = parser.parse(complexQuery); @@ -87,11 +87,13 @@ BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(ComplexQuery, 48_thread, 48) int main(int argc, char **argv) { gflags::ParseCommandLineFlags(&argc, &argv, true); { + auto qctx = std::make_unique(); GQLParser parser(qctx.get()); auto result = parser.parse(simpleQuery); CHECK(result.ok()) << result.status(); } { + auto qctx = std::make_unique(); GQLParser parser(qctx.get()); auto result = parser.parse(complexQuery); CHECK(result.ok()) << result.status(); diff --git a/src/storage/GraphStorageLocalServer.cpp b/src/storage/GraphStorageLocalServer.cpp index 9627d1c4588..9feb7bfcf55 100644 --- a/src/storage/GraphStorageLocalServer.cpp +++ b/src/storage/GraphStorageLocalServer.cpp @@ -8,14 +8,15 @@ #include #include "common/base/Base.h" - -#define LOCAL_RETURN_FUTURE(threadManager, respType, callFunc) \ - auto promise = std::make_shared>(); \ - auto f = promise->getFuture(); \ - threadManager->add([&, promise] { \ - handler_->callFunc(request).thenValue( \ - [promise](respType&& resp) { promise->setValue(std::move(resp)); }); \ - }); \ +#include "storage/GraphStorageServiceHandler.h" + +#define LOCAL_RETURN_FUTURE(threadManager, respType, callFunc) \ + auto promise = std::make_shared>(); \ + auto f = promise->getFuture(); \ + threadManager->add([&, promise] { \ + std::dynamic_pointer_cast(handler_)->callFunc(request).thenValue( \ + [promise](respType&& resp) { promise->setValue(std::move(resp)); }); \ + }); \ return f; namespace nebula::storage { @@ -29,7 +30,8 @@ void GraphStorageLocalServer::setThreadManager( threadManager_ = threadManager; } -void GraphStorageLocalServer::setInterface(std::shared_ptr handler) { +void GraphStorageLocalServer::setInterface( + std::shared_ptr handler) { handler_ = handler; } @@ -82,6 +84,11 @@ folly::Future GraphStorageLocalServer::future_deleteEdges( LOCAL_RETURN_FUTURE(threadManager_, cpp2::ExecResponse, future_deleteEdges); } +folly::Future GraphStorageLocalServer::future_chainDeleteEdges( + const cpp2::DeleteEdgesRequest& request) { + LOCAL_RETURN_FUTURE(threadManager_, cpp2::ExecResponse, future_chainDeleteEdges); +} + folly::Future GraphStorageLocalServer::future_deleteVertices( const cpp2::DeleteVerticesRequest& request) { LOCAL_RETURN_FUTURE(threadManager_, cpp2::ExecResponse, future_deleteVertices); diff --git a/src/storage/GraphStorageLocalServer.h b/src/storage/GraphStorageLocalServer.h index 10e27463f7c..7c8e4df0b33 100644 --- a/src/storage/GraphStorageLocalServer.h +++ b/src/storage/GraphStorageLocalServer.h @@ -11,10 +11,9 @@ #include #include "common/base/Base.h" +#include "common/cpp/helpers.h" #include "folly/fibers/Semaphore.h" #include "interface/gen-cpp2/GraphStorageServiceAsyncClient.h" -#include "storage/GraphStorageServiceHandler.h" - namespace nebula::storage { class GraphStorageLocalServer final : public nebula::cpp::NonCopyable, public nebula::cpp::NonMovable { @@ -24,7 +23,7 @@ class GraphStorageLocalServer final : public nebula::cpp::NonCopyable, return instance; } void setThreadManager(std::shared_ptr threadManager); - void setInterface(std::shared_ptr iface); + void setInterface(std::shared_ptr handler); void stop(); void serve(); @@ -36,6 +35,8 @@ class GraphStorageLocalServer final : public nebula::cpp::NonCopyable, folly::Future future_addEdges(const cpp2::AddEdgesRequest& request); folly::Future future_getProps(const cpp2::GetPropRequest& request); folly::Future future_deleteEdges(const cpp2::DeleteEdgesRequest& request); + folly::Future future_chainDeleteEdges( + const cpp2::DeleteEdgesRequest& request); folly::Future future_deleteVertices( const cpp2::DeleteVerticesRequest& request); folly::Future future_deleteTags(const cpp2::DeleteTagsRequest& request); @@ -58,7 +59,7 @@ class GraphStorageLocalServer final : public nebula::cpp::NonCopyable, private: std::shared_ptr threadManager_; - std::shared_ptr handler_; + std::shared_ptr handler_; folly::fibers::Semaphore sem_{0}; static std::mutex mutex_; bool serving_ = {false}; diff --git a/src/storage/StorageServer.cpp b/src/storage/StorageServer.cpp index 66b697031c3..e0a449cf734 100644 --- a/src/storage/StorageServer.cpp +++ b/src/storage/StorageServer.cpp @@ -363,6 +363,9 @@ void StorageServer::notifyStop() { serverStatus_ = STATUS_STOPPED; cvStop_.notify_one(); } + if (metaClient_) { + metaClient_->notifyStop(); + } } void StorageServer::stop() { @@ -396,7 +399,7 @@ void StorageServer::stop() { taskMgr_->shutdown(); } if (metaClient_) { - metaClient_->stop(); + metaClient_->notifyStop(); } if (kvstore_) { kvstore_.reset(); diff --git a/src/storage/admin/AdminProcessor.h b/src/storage/admin/AdminProcessor.h index bf352a6e925..16d69f57ac0 100644 --- a/src/storage/admin/AdminProcessor.h +++ b/src/storage/admin/AdminProcessor.h @@ -274,19 +274,19 @@ class WaitingForCatchUpDataProcessor : public BaseProcessor << ", part " << partId << ", remaining " << retry << " retry times" << ", result " << static_cast(res); switch (res) { - case raftex::AppendLogResult::SUCCEEDED: + case nebula::cpp2::ErrorCode::SUCCEEDED: onFinished(); return; - case raftex::AppendLogResult::E_INVALID_PEER: + case nebula::cpp2::ErrorCode::E_RAFT_INVALID_PEER: this->pushResultCode(nebula::cpp2::ErrorCode::E_INVALID_PEER, partId); onFinished(); return; - case raftex::AppendLogResult::E_NOT_A_LEADER: { + case nebula::cpp2::ErrorCode::E_LEADER_CHANGED: { handleLeaderChanged(spaceId, partId); onFinished(); return; } - case raftex::AppendLogResult::E_SENDING_SNAPSHOT: + case nebula::cpp2::ErrorCode::E_RAFT_SENDING_SNAPSHOT: LOG(INFO) << "Space " << spaceId << ", partId " << partId << " is still sending snapshot, please wait..."; break; diff --git a/src/storage/http/StorageHttpDownloadHandler.cpp b/src/storage/http/StorageHttpDownloadHandler.cpp index 30b2cdfa169..ed15bc77f50 100644 --- a/src/storage/http/StorageHttpDownloadHandler.cpp +++ b/src/storage/http/StorageHttpDownloadHandler.cpp @@ -94,7 +94,7 @@ void StorageHttpDownloadHandler::onEOM() noexcept { if (helper_->checkHadoopPath()) { std::vector parts; folly::split(",", partitions_, parts, true); - if (parts.size() == 0) { + if (parts.empty()) { ResponseBuilder(downstream_) .status(400, "SSTFile download failed") .body("Partitions should be not empty") diff --git a/src/storage/mutate/AddEdgesProcessor.cpp b/src/storage/mutate/AddEdgesProcessor.cpp index 0c060509fcb..8a68b25ac18 100644 --- a/src/storage/mutate/AddEdgesProcessor.cpp +++ b/src/storage/mutate/AddEdgesProcessor.cpp @@ -161,8 +161,7 @@ void AddEdgesProcessor::doProcessWithIndex(const cpp2::AddEdgesRequest& req) { dummyLock.reserve(newEdges.size()); auto code = nebula::cpp2::ErrorCode::SUCCEEDED; - std::unordered_set visited; - visited.reserve(newEdges.size()); + deleteDupEdge(const_cast&>(newEdges)); for (auto& newEdge : newEdges) { auto edgeKey = *newEdge.key_ref(); auto l = std::make_tuple(spaceId_, @@ -203,9 +202,6 @@ void AddEdgesProcessor::doProcessWithIndex(const cpp2::AddEdgesRequest& req) { *edgeKey.edge_type_ref(), *edgeKey.ranking_ref(), edgeKey.dst_ref()->getStr()); - if (ifNotExists_ && !visited.emplace(key).second) { - continue; - } auto schema = env_->schemaMan_->getEdgeSchema(spaceId_, std::abs(*edgeKey.edge_type_ref())); if (!schema) { LOG(ERROR) << "Space " << spaceId_ << ", Edge " << *edgeKey.edge_type_ref() << " invalid"; @@ -439,7 +435,7 @@ ErrorOr AddEdgesProcessor::addEdges( } } /* - * step 3 , Insert new vertex data + * step 3 , Insert new edge data */ auto key = e.first; auto prop = e.second; @@ -488,5 +484,48 @@ std::vector AddEdgesProcessor::indexKeys( std::move(values).value()); } +/* + * Batch insert + * ifNotExist_ is true. Only keep the first one when edgeKey is same + * ifNotExist_ is false. Only keep the last one when edgeKey is same + */ +void AddEdgesProcessor::deleteDupEdge(std::vector& edges) { + std::unordered_set visited; + visited.reserve(edges.size()); + if (ifNotExists_) { + auto iter = edges.begin(); + while (iter != edges.end()) { + auto edgeKeyRef = iter->key_ref(); + auto key = NebulaKeyUtils::edgeKey(spaceVidLen_, + 0, // it's ok, just distinguish between different edgekey + edgeKeyRef->src_ref()->getStr(), + edgeKeyRef->get_edge_type(), + edgeKeyRef->get_ranking(), + edgeKeyRef->dst_ref()->getStr()); + if (!visited.emplace(key).second) { + iter = edges.erase(iter); + } else { + ++iter; + } + } + } else { + auto iter = edges.rbegin(); + while (iter != edges.rend()) { + auto edgeKeyRef = iter->key_ref(); + auto key = NebulaKeyUtils::edgeKey(spaceVidLen_, + 0, // it's ok, just distinguish between different edgekey + edgeKeyRef->src_ref()->getStr(), + edgeKeyRef->get_edge_type(), + edgeKeyRef->get_ranking(), + edgeKeyRef->dst_ref()->getStr()); + if (!visited.emplace(key).second) { + iter = decltype(iter)(edges.erase(std::next(iter).base())); + } else { + ++iter; + } + } + } +} + } // namespace storage } // namespace nebula diff --git a/src/storage/mutate/AddEdgesProcessor.h b/src/storage/mutate/AddEdgesProcessor.h index cec28b69e3c..7c3ee3d0c7f 100644 --- a/src/storage/mutate/AddEdgesProcessor.h +++ b/src/storage/mutate/AddEdgesProcessor.h @@ -49,6 +49,8 @@ class AddEdgesProcessor : public BaseProcessor { std::shared_ptr index, const meta::SchemaProviderIf* latestSchema); + void deleteDupEdge(std::vector& edges); + private: GraphSpaceID spaceId_; std::vector> indexes_; diff --git a/src/storage/mutate/AddVerticesProcessor.cpp b/src/storage/mutate/AddVerticesProcessor.cpp index 86a8ca9530e..83fcab649a3 100644 --- a/src/storage/mutate/AddVerticesProcessor.cpp +++ b/src/storage/mutate/AddVerticesProcessor.cpp @@ -147,8 +147,7 @@ void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& re auto code = nebula::cpp2::ErrorCode::SUCCEEDED; // cache tagKey - std::unordered_set visited; - visited.reserve(vertices.size()); + deleteDupVid(const_cast&>(vertices)); for (auto& vertex : vertices) { auto vid = vertex.get_id().getStr(); const auto& newTags = vertex.get_tags(); @@ -181,9 +180,6 @@ void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& re } auto key = NebulaKeyUtils::tagKey(spaceVidLen_, partId, vid, tagId); - if (ifNotExists_ && !visited.emplace(key).second) { - continue; - } auto props = newTag.get_props(); auto iter = propNamesMap.find(tagId); std::vector propNames; @@ -232,8 +228,7 @@ void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& re if (oReader != nullptr) { auto ois = indexKeys(partId, vid, oReader.get(), index, schema.get()); if (!ois.empty()) { - // Check the index is building for the specified partition or - // not. + // Check the index is building for the specified partition or not auto indexState = env_->getIndexState(spaceId_, partId); if (env_->checkRebuilding(indexState)) { auto delOpKey = OperationKeyUtils::deleteOperationKey(partId); @@ -345,5 +340,36 @@ std::vector AddVerticesProcessor::indexKeys( spaceVidLen_, partId, index->get_index_id(), vId, std::move(values).value()); } +/* + * Batch insert + * ifNotExist_ is true. Only keep the first one when vid is same + * ifNotExist_ is false. Only keep the last one when vid is same + */ +void AddVerticesProcessor::deleteDupVid(std::vector& vertices) { + std::unordered_set visited; + visited.reserve(vertices.size()); + if (ifNotExists_) { + auto iter = vertices.begin(); + while (iter != vertices.end()) { + const auto& vid = iter->get_id().getStr(); + if (!visited.emplace(vid).second) { + iter = vertices.erase(iter); + } else { + ++iter; + } + } + } else { + auto iter = vertices.rbegin(); + while (iter != vertices.rend()) { + const auto& vid = iter->get_id().getStr(); + if (!visited.emplace(vid).second) { + iter = decltype(iter)(vertices.erase(std::next(iter).base())); + } else { + ++iter; + } + } + } +} + } // namespace storage } // namespace nebula diff --git a/src/storage/mutate/AddVerticesProcessor.h b/src/storage/mutate/AddVerticesProcessor.h index fecca60516b..71725c9b5be 100644 --- a/src/storage/mutate/AddVerticesProcessor.h +++ b/src/storage/mutate/AddVerticesProcessor.h @@ -44,6 +44,8 @@ class AddVerticesProcessor : public BaseProcessor { std::shared_ptr index, const meta::SchemaProviderIf* latestSchema); + void deleteDupVid(std::vector& vertices); + private: GraphSpaceID spaceId_; std::vector> indexes_; diff --git a/src/storage/test/CMakeLists.txt b/src/storage/test/CMakeLists.txt index 44604cfbac2..38d59a24f93 100644 --- a/src/storage/test/CMakeLists.txt +++ b/src/storage/test/CMakeLists.txt @@ -2,6 +2,7 @@ set(storage_test_deps $ $ $ + $ $ $ $ @@ -58,6 +59,13 @@ set(storage_test_deps $ ) +if(ENABLE_STANDALONE_VERSION) +set(storage_test_deps + ${storage_test_deps} + $ +) +endif() + nebula_add_test( NAME task_manager_test diff --git a/src/storage/test/ChainTestUtils.h b/src/storage/test/ChainTestUtils.h index d94f30b2a74..0fd04ca00ee 100644 --- a/src/storage/test/ChainTestUtils.h +++ b/src/storage/test/ChainTestUtils.h @@ -226,14 +226,14 @@ class MetaClientTestUpdater { static void addLocalCache(meta::MetaClient& mClient, GraphSpaceID spaceId, std::shared_ptr spInfoCache) { - mClient.localCache_[spaceId] = spInfoCache; + mClient.metadata_.load()->localCache_[spaceId] = spInfoCache; } static meta::SpaceInfoCache* getLocalCache(meta::MetaClient* mClient, GraphSpaceID spaceId) { - if (mClient->localCache_.count(spaceId) == 0) { + if (mClient->metadata_.load()->localCache_.count(spaceId) == 0) { return nullptr; } - return mClient->localCache_[spaceId].get(); + return mClient->metadata_.load()->localCache_[spaceId].get(); } static void addPartTerm(meta::MetaClient* mClient, diff --git a/src/storage/test/KVClientTest.cpp b/src/storage/test/KVClientTest.cpp index e52c9451e95..26b9c2cc47b 100644 --- a/src/storage/test/KVClientTest.cpp +++ b/src/storage/test/KVClientTest.cpp @@ -51,6 +51,10 @@ TEST(KVClientTest, SimpleTest) { auto result = metaClient->addHosts(std::move(hosts)).get(); EXPECT_TRUE(result.ok()); } + { + std::vector hosts = {storageAddr}; + nebula::meta::TestUtils::registerHB(cluster.metaKV_.get(), hosts); + } cluster.startStorage(storageAddr, storagePath.path()); auto client = cluster.initGraphStorageClient(); diff --git a/src/storage/test/KillQueryTest.cpp b/src/storage/test/KillQueryTest.cpp index 815ddad74a5..8516244c5bb 100644 --- a/src/storage/test/KillQueryTest.cpp +++ b/src/storage/test/KillQueryTest.cpp @@ -19,7 +19,7 @@ class KillQueryMetaWrapper { public: explicit KillQueryMetaWrapper(MetaClient* client) : client_(client) {} void killQuery(SessionID session_id, ExecutionPlanID plan_id) { - client_->killedPlans_.load()->emplace(session_id, plan_id); + client_->metadata_.load()->killedPlans_.emplace(session_id, plan_id); } private: diff --git a/src/tools/db-dump/CMakeLists.txt b/src/tools/db-dump/CMakeLists.txt index 80919a420a6..c489c751a6f 100644 --- a/src/tools/db-dump/CMakeLists.txt +++ b/src/tools/db-dump/CMakeLists.txt @@ -1,6 +1,7 @@ set(tools_test_deps $ $ + $ $ $ $ diff --git a/src/tools/db-upgrade/CMakeLists.txt b/src/tools/db-upgrade/CMakeLists.txt index 22c951d3ad9..674b8973910 100644 --- a/src/tools/db-upgrade/CMakeLists.txt +++ b/src/tools/db-upgrade/CMakeLists.txt @@ -10,6 +10,7 @@ nebula_add_executable( OBJECTS $ $ + $ $ $ $ diff --git a/src/tools/meta-dump/CMakeLists.txt b/src/tools/meta-dump/CMakeLists.txt index 466b91ef62d..1e827f5dfce 100644 --- a/src/tools/meta-dump/CMakeLists.txt +++ b/src/tools/meta-dump/CMakeLists.txt @@ -6,6 +6,7 @@ nebula_add_executable( OBJECTS $ $ + $ $ $ $ diff --git a/src/tools/simple-kv-verify/CMakeLists.txt b/src/tools/simple-kv-verify/CMakeLists.txt index 81de926c583..01e01040542 100644 --- a/src/tools/simple-kv-verify/CMakeLists.txt +++ b/src/tools/simple-kv-verify/CMakeLists.txt @@ -6,6 +6,7 @@ nebula_add_executable( OBJECTS $ $ + $ $ $ $ diff --git a/src/tools/storage-perf/CMakeLists.txt b/src/tools/storage-perf/CMakeLists.txt index 41f9b0a18a5..bdde2f7bcdb 100644 --- a/src/tools/storage-perf/CMakeLists.txt +++ b/src/tools/storage-perf/CMakeLists.txt @@ -1,6 +1,7 @@ set(perf_test_deps $ $ + $ $ $ $ diff --git a/src/tools/storage-perf/StoragePerfTool.cpp b/src/tools/storage-perf/StoragePerfTool.cpp index ae95292ee85..f507c06237c 100644 --- a/src/tools/storage-perf/StoragePerfTool.cpp +++ b/src/tools/storage-perf/StoragePerfTool.cpp @@ -142,6 +142,7 @@ class Perf { t.join(); } + mClient_->notifyStop(); mClient_->stop(); threadPool_->stop(); LOG(INFO) << "Total time cost " << duration.elapsedInMSec() << "ms, " diff --git a/tests/tck/features/expression/Depth.feature b/tests/tck/features/expression/Depth.feature new file mode 100644 index 00000000000..71bf5e3498e --- /dev/null +++ b/tests/tck/features/expression/Depth.feature @@ -0,0 +1,159 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +Feature: Check Expression Depth + + Background: + Given a graph with space named "nba" + + Scenario: yield exceeds expression + When executing query: + """ + YIELD 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 AS result + """ + Then the result should be, in any order: + | result | + | 488 | + When executing query: + """ + YIELD 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 AS result + """ + Then a SyntaxError should be raised at runtime: The above expression's depth exceeds the maximum depth + When executing query: + """ + YIELD 1 IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS + NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL IS NULL AS result + """ + Then a SyntaxError should be raised at runtime: The above expression's depth exceeds the maximum depth diff --git a/tests/tck/features/function/coalesce.feature b/tests/tck/features/function/coalesce.feature new file mode 100644 index 00000000000..c06ae41ec50 --- /dev/null +++ b/tests/tck/features/function/coalesce.feature @@ -0,0 +1,41 @@ +Feature: Fetch Int Vid Edges + + Background: + Test coalesce function + + Scenario: test normal case + When executing query: + """ + RETURN coalesce(null,1) as result; + """ + Then the result should be, in any order: + | result | + | 1 | + When executing query: + """ + RETURN coalesce(1,2,3) as result; + """ + Then the result should be, in any order: + | result | + | 1 | + When executing query: + """ + RETURN coalesce(null) as result; + """ + Then the result should be, in any order: + | result | + | NULL | + When executing query: + """ + RETURN coalesce(null,[1,2,3]) as result; + """ + Then the result should be, in any order: + | result | + | [1, 2, 3] | + When executing query: + """ + RETURN coalesce(null,1.234) as result; + """ + Then the result should be, in any order: + | result | + | 1.234 | diff --git a/tests/tck/features/insert/InsertIfNotExists.feature b/tests/tck/features/insert/InsertIfNotExists.feature index 38eb9786d26..c766c6ddabe 100644 --- a/tests/tck/features/insert/InsertIfNotExists.feature +++ b/tests/tck/features/insert/InsertIfNotExists.feature @@ -276,3 +276,267 @@ Feature: Insert vertex and edge with if not exists | like.likeness | | 200 | And drop the used space + + Scenario: vertices index and data consistency check + Given an empty graph + And create a space with following options: + | partition_num | 9 | + | replica_factor | 1 | + | vid_type | FIXED_STRING(20) | + And having executed: + """ + CREATE TAG IF NOT EXISTS student(name string, age int); + CREATE TAG INDEX index_s_age on student(age); + """ + And wait 6 seconds + When try to execute query: + """ + INSERT VERTEX + student(name, age) + VALUES + "zhang":("zhang", 19), + "zhang":("zhang", 29), + "zhang":("zhang", 39), + "wang":("wang", 18), + "li":("li", 16), + "wang":("wang", 38); + """ + Then the execution should be successful + When executing query: + """ + LOOKUP ON student WHERE student.age < 30 YIELD student.age AS age + """ + Then the result should be, in any order, with relax comparison: + | age | + | 16 | + When executing query: + """ + LOOKUP ON student WHERE student.age > 30 YIELD student.age AS age + """ + Then the result should be, in any order, with relax comparison: + | age | + | 39 | + | 38 | + When executing query: + """ + LOOKUP ON student WHERE student.age < 30 YIELD student.name AS name, student.age AS age + """ + Then the result should be, in any order, with relax comparison: + | name | age | + | "li" | 16 | + When executing query: + """ + LOOKUP ON student WHERE student.age > 30 YIELD student.name as name, student.age AS age + """ + Then the result should be, in any order, with relax comparison: + | name | age | + | "zhang" | 39 | + | "wang" | 38 | + When executing query: + """ + FETCH PROP ON student "zhang", "wang", "li" YIELD student.name as name, student.age as age + """ + Then the result should be, in any order, with relax comparison: + | name | age | + | "zhang" | 39 | + | "wang" | 38 | + | "li" | 16 | + When try to execute query: + """ + DELETE TAG student FROM "zhang", "wang", "li"; + """ + Then the execution should be successful + When try to execute query: + """ + INSERT VERTEX IF NOT EXISTS + student(name, age) + VALUES + "zhao":("zhao", 19), + "zhao":("zhao", 29), + "zhao":("zhao", 39), + "qian":("qian", 18), + "sun":("sun", 16), + "qian":("qian", 38), + "chen":("chen", 40), + "chen":("chen", 35); + """ + Then the execution should be successful + When executing query: + """ + LOOKUP ON student WHERE student.age < 30 YIELD student.age AS age + """ + Then the result should be, in any order, with relax comparison: + | age | + | 19 | + | 18 | + | 16 | + When executing query: + """ + LOOKUP ON student WHERE student.age > 30 YIELD student.age AS age + """ + Then the result should be, in any order, with relax comparison: + | age | + | 40 | + When executing query: + """ + LOOKUP ON student WHERE student.age < 30 YIELD student.name AS name, student.age AS age + """ + Then the result should be, in any order, with relax comparison: + | name | age | + | "zhao" | 19 | + | "qian" | 18 | + | "sun" | 16 | + When executing query: + """ + LOOKUP ON student WHERE student.age > 30 YIELD student.name as name, student.age AS age + """ + Then the result should be, in any order, with relax comparison: + | name | age | + | "chen" | 40 | + When executing query: + """ + FETCH PROP ON student "zhao", "qian", "sun", "chen" YIELD student.name as name, student.age as age + """ + Then the result should be, in any order, with relax comparison: + | name | age | + | "zhao" | 19 | + | "qian" | 18 | + | "sun" | 16 | + | "chen" | 40 | + And drop the used space + + Scenario: edge index and data consistency check + Given an empty graph + And create a space with following options: + | partition_num | 9 | + | replica_factor | 1 | + | vid_type | FIXED_STRING(20) | + And having executed: + """ + CREATE TAG IF NOT EXISTS student(name string, age int); + CREATE EDGE IF NOT EXISTS like(likeness int, t1 int); + CREATE EDGE INDEX index_l_likeness on like(likeness); + """ + And wait 6 seconds + When try to execute query: + """ + INSERT VERTEX + student(name, age) + VALUES + "zhang":("zhang", 19), + "wang":("wang", 18), + "li":("li", 16); + INSERT EDGE + like(likeness, t1) + VALUES + "zhang"->"wang":(19, 19), + "zhang"->"li":(42, 42), + "zhang"->"li":(20, 20), + "zhang"->"wang":(39, 39), + "wang"->"li":(18, 18), + "wang"->"zhang":(41, 41); + """ + Then the execution should be successful + When executing query: + """ + LOOKUP ON like WHERE like.likeness < 30 YIELD src(edge) as src, dst(edge) as dst, like.likeness as likeness + """ + Then the result should be, in any order, with relax comparison: + | src | dst | likeness | + | "zhang" | "li" | 20 | + | "wang" | "li" | 18 | + When executing query: + """ + LOOKUP ON like WHERE like.likeness > 30 YIELD src(edge) as src, dst(edge) as dst, like.likeness as likeness + """ + Then the result should be, in any order, with relax comparison: + | src | dst | likeness | + | "zhang" | "wang" | 39 | + | "wang" | "zhang" | 41 | + When executing query: + """ + LOOKUP ON like WHERE like.likeness < 30 YIELD src(edge) as src, dst(edge) as dst, like.likeness as likeness, like.t1 as t1 + """ + Then the result should be, in any order, with relax comparison: + | src | dst | likeness | t1 | + | "zhang" | "li" | 20 | 20 | + | "wang" | "li" | 18 | 18 | + When executing query: + """ + LOOKUP ON like WHERE like.likeness > 30 YIELD src(edge) as src, dst(edge) as dst, like.likeness as likeness, like.t1 as t1 + """ + Then the result should be, in any order, with relax comparison: + | src | dst | likeness | t1 | + | "zhang" | "wang" | 39 | 39 | + | "wang" | "zhang" | 41 | 41 | + When executing query: + """ + FETCH PROP ON like "zhang"->"wang", "zhang"->"li", "wang"->"li", "wang"->"zhang" YIELD src(edge) as src, dst(edge) as dst, like.likeness as likeness + """ + Then the result should be, in any order, with relax comparison: + | src | dst | likeness | + | "zhang" | "wang" | 39 | + | "zhang" | "li" | 20 | + | "wang" | "li" | 18 | + | "wang" | "zhang" | 41 | + When try to execute query: + """ + DELETE EDGE like "zhang"->"wang", "zhang"->"li", "wang"->"li", "wang"->"zhang"; + """ + Then the execution should be successful + When try to execute query: + """ + INSERT EDGE IF NOT EXISTS + like(likeness, t1) + VALUES + "zhang"->"wang":(19, 19), + "zhang"->"li":(42, 42), + "zhang"->"li":(20, 20), + "zhang"->"wang":(39, 39), + "wang"->"li":(18, 18), + "wang"->"zhang":(41, 41); + """ + Then the execution should be successful + When executing query: + """ + LOOKUP ON like WHERE like.likeness < 30 YIELD src(edge) as src, dst(edge) as dst, like.likeness as likeness + """ + Then the result should be, in any order, with relax comparison: + | src | dst | likeness | + | "zhang" | "wang" | 19 | + | "wang" | "li" | 18 | + When executing query: + """ + LOOKUP ON like WHERE like.likeness > 30 YIELD src(edge) as src, dst(edge) as dst, like.likeness as likeness + """ + Then the result should be, in any order, with relax comparison: + | src | dst | likeness | + | "zhang" | "li" | 42 | + | "wang" | "zhang" | 41 | + When executing query: + """ + LOOKUP ON like WHERE like.likeness < 30 YIELD src(edge) as src, dst(edge) as dst, like.likeness as likeness, like.t1 as t1 + """ + Then the result should be, in any order, with relax comparison: + | src | dst | likeness | t1 | + | "zhang" | "wang" | 19 | 19 | + | "wang" | "li" | 18 | 18 | + When executing query: + """ + LOOKUP ON like WHERE like.likeness > 30 YIELD src(edge) as src, dst(edge) as dst, like.likeness as likeness, like.t1 as t1 + """ + Then the result should be, in any order, with relax comparison: + | src | dst | likeness | t1 | + | "zhang" | "li" | 42 | 42 | + | "wang" | "zhang" | 41 | 41 | + When executing query: + """ + FETCH PROP ON like "zhang"->"wang", "zhang"->"li", "wang"->"li", "wang"->"zhang" YIELD src(edge) as src, dst(edge) as dst, like.likeness as likeness + """ + Then the result should be, in any order, with relax comparison: + | src | dst | likeness | + | "zhang" | "wang" | 19 | + | "zhang" | "li" | 42 | + | "wang" | "li" | 18 | + | "wang" | "zhang" | 41 | + And drop the used space diff --git a/tests/tck/features/lookup/LookUp.feature b/tests/tck/features/lookup/LookUp.feature index bac70bd930f..2c54e6fe452 100644 --- a/tests/tck/features/lookup/LookUp.feature +++ b/tests/tck/features/lookup/LookUp.feature @@ -842,15 +842,16 @@ Feature: LookUpTest_Vid_String "104":("yyy", 28), "105":("zzz", 21), "106":("kkk", 21), + "121":("Useless", 60), "121":("Useless", 20); - INSERT VERTEX - team(name) - VALUES - "200":("Warriors"), - "201":("Nuggets"), - "202":("oopp"), - "203":("iiiooo"), - "204":("opl"); + INSERT VERTEX + team(name) + VALUES + "200":("Warriors"), + "201":("Nuggets"), + "202":("oopp"), + "203":("iiiooo"), + "204":("opl"); """ When executing query: """