diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index 4432797c3be..0122008e52d 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -2227,7 +2227,6 @@ bool MetaClient::checkShadowAccountFromCache(const std::string& account) const { return false; } - StatusOr MetaClient::getTermFromCache(GraphSpaceID spaceId, PartitionID partId) const { folly::RWSpinLock::ReadHolder holder(localCacheLock_); auto spaceInfo = localCache_.find(spaceId); diff --git a/src/storage/test/ChainAddEdgesTest.cpp b/src/storage/test/ChainAddEdgesTest.cpp index 80ab36d8509..30b56f087bb 100644 --- a/src/storage/test/ChainAddEdgesTest.cpp +++ b/src/storage/test/ChainAddEdgesTest.cpp @@ -1,30 +1,26 @@ - /* Copyright (c) 2021 vesoft inc. All rights reserved. +/* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ -#include #include #include +#include #include #include -#include +#include #include #include "common/fs/TempDir.h" - -#include "storage/CommonUtils.h" -#include "storage/transaction/ConsistUtil.h" -#include "storage/transaction/ChainAddEdgesProcessorLocal.h" -#include "storage/transaction/ChainAddEdgesGroupProcessor.h" - #include "mock/MockCluster.h" #include "mock/MockData.h" -#include "storage/test/TestUtils.h" +#include "storage/CommonUtils.h" #include "storage/test/ChainTestUtils.h" - - +#include "storage/test/TestUtils.h" +#include "storage/transaction/ChainAddEdgesGroupProcessor.h" +#include "storage/transaction/ChainAddEdgesProcessorLocal.h" +#include "storage/transaction/ConsistUtil.h" namespace nebula { namespace storage { @@ -35,179 +31,176 @@ constexpr int32_t fackTerm = 1; // make sure test class works well TEST(ChainAddEdgesTest, TestUtilsTest) { - fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - env->metaClient_ = mClient.get(); - MetaClientTestUpdater::addPartTerm(env->metaClient_, mockSpaceId, mockPartNum, fackTerm); - - auto* processor = new FakeChainAddEdgesProcessorLocal(env); - - processor->rcPrepareLocal = nebula::cpp2::ErrorCode::SUCCEEDED; - processor->rcProcessRemote = nebula::cpp2::ErrorCode::SUCCEEDED; - processor->rcProcessLocal = nebula::cpp2::ErrorCode::SUCCEEDED; - - LOG(INFO) << "Build AddEdgesRequest..."; - cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); - - LOG(INFO) << "Test AddEdgesProcessor..."; - auto fut = processor->getFuture(); - processor->process(req); - auto resp = std::move(fut).get(); - EXPECT_EQ(0, resp.result.failed_parts.size()); - - LOG(INFO) << "Check data in kv store..."; - // The number of data in serve is 334 - checkAddEdgesData(req, env, 0, 0); + fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + env->metaClient_ = mClient.get(); + MetaClientTestUpdater::addPartTerm(env->metaClient_, mockSpaceId, mockPartNum, fackTerm); + + auto* processor = new FakeChainAddEdgesProcessorLocal(env); + + processor->rcPrepareLocal = nebula::cpp2::ErrorCode::SUCCEEDED; + processor->rcProcessRemote = nebula::cpp2::ErrorCode::SUCCEEDED; + processor->rcProcessLocal = nebula::cpp2::ErrorCode::SUCCEEDED; + + LOG(INFO) << "Build AddEdgesRequest..."; + cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); + + LOG(INFO) << "Test AddEdgesProcessor..."; + auto fut = processor->getFuture(); + processor->process(req); + auto resp = std::move(fut).get(); + EXPECT_EQ(0, resp.result.failed_parts.size()); + + LOG(INFO) << "Check data in kv store..."; + // The number of data in serve is 334 + checkAddEdgesData(req, env, 0, 0); } TEST(ChainAddEdgesTest, prepareLocalSucceedTest) { - fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - env->metaClient_ = mClient.get(); - MetaClientTestUpdater::addPartTerm(env->metaClient_, mockSpaceId, mockPartNum, fackTerm); - auto* proc = new FakeChainAddEdgesProcessorLocal(env); - - proc->rcProcessRemote = nebula::cpp2::ErrorCode::E_RPC_FAILURE; - - LOG(INFO) << "Build AddEdgesRequest..."; - cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); - - LOG(INFO) << "Test AddEdgesProcessor..."; - auto fut = proc->getFuture(); - proc->process(req); - auto resp = std::move(fut).get(); - EXPECT_EQ(0, resp.result.failed_parts.size()); - - ChainTestUtils util; - // none of really edge key should be inserted - EXPECT_EQ(334, numOfKey(req, util.genKey, env)); - EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); - EXPECT_EQ(334, numOfKey(req, util.genDoublePrime, env)); + fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + env->metaClient_ = mClient.get(); + MetaClientTestUpdater::addPartTerm(env->metaClient_, mockSpaceId, mockPartNum, fackTerm); + auto* proc = new FakeChainAddEdgesProcessorLocal(env); + + proc->rcProcessRemote = nebula::cpp2::ErrorCode::E_RPC_FAILURE; + + LOG(INFO) << "Build AddEdgesRequest..."; + cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); + + LOG(INFO) << "Test AddEdgesProcessor..."; + auto fut = proc->getFuture(); + proc->process(req); + auto resp = std::move(fut).get(); + EXPECT_EQ(0, resp.result.failed_parts.size()); + + ChainTestUtils util; + // none of really edge key should be inserted + EXPECT_EQ(334, numOfKey(req, util.genKey, env)); + EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); + EXPECT_EQ(334, numOfKey(req, util.genDoublePrime, env)); } - TEST(ChainAddEdgesTest, processRemoteSucceededTest) { - fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - - env->metaClient_ = mClient.get(); - auto* proc = new FakeChainAddEdgesProcessorLocal(env); - MetaClientTestUpdater::addPartTerm(env->metaClient_, mockSpaceId, mockPartNum, fackTerm); - - proc->rcProcessRemote = nebula::cpp2::ErrorCode::SUCCEEDED; - - LOG(INFO) << "Build AddEdgesRequest..."; - cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); - - LOG(INFO) << "Test AddEdgesProcessor..."; - auto fut = proc->getFuture(); - proc->process(req); - auto resp = std::move(fut).get(); - EXPECT_EQ(0, resp.result.failed_parts.size()); - - ChainTestUtils util; - // none of really edge key should be inserted - EXPECT_EQ(334, numOfKey(req, util.genKey, env)); - // prime key should be deleted - EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); - EXPECT_EQ(0, numOfKey(req, util.genDoublePrime, env)); + fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + + env->metaClient_ = mClient.get(); + auto* proc = new FakeChainAddEdgesProcessorLocal(env); + MetaClientTestUpdater::addPartTerm(env->metaClient_, mockSpaceId, mockPartNum, fackTerm); + + proc->rcProcessRemote = nebula::cpp2::ErrorCode::SUCCEEDED; + + LOG(INFO) << "Build AddEdgesRequest..."; + cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); + + LOG(INFO) << "Test AddEdgesProcessor..."; + auto fut = proc->getFuture(); + proc->process(req); + auto resp = std::move(fut).get(); + EXPECT_EQ(0, resp.result.failed_parts.size()); + + ChainTestUtils util; + // none of really edge key should be inserted + EXPECT_EQ(334, numOfKey(req, util.genKey, env)); + // prime key should be deleted + EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); + EXPECT_EQ(0, numOfKey(req, util.genDoublePrime, env)); } - // check prepareLocal() will set prime key properly TEST(ChainAddEdgesTest, processRemoteFailedTest) { - fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - env->metaClient_ = mClient.get(); - MetaClientTestUpdater::addPartTerm(env->metaClient_, mockSpaceId, mockPartNum, fackTerm); - - auto* proc = new FakeChainAddEdgesProcessorLocal(env); - proc->rcProcessRemote = nebula::cpp2::ErrorCode::E_OUTDATED_TERM; - - LOG(INFO) << "Build AddEdgesRequest..."; - cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); - - LOG(INFO) << "Test AddEdgesProcessor..."; - auto fut = proc->getFuture(); - proc->process(req); - auto resp = std::move(fut).get(); - EXPECT_EQ(0, resp.result.failed_parts.size()); - - ChainTestUtils util; - // none of really edge key should be inserted - EXPECT_EQ(0, numOfKey(req, util.genKey, env)); - // prime key should be deleted - EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); - EXPECT_EQ(0, numOfKey(req, util.genDoublePrime, env)); + fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + env->metaClient_ = mClient.get(); + MetaClientTestUpdater::addPartTerm(env->metaClient_, mockSpaceId, mockPartNum, fackTerm); + + auto* proc = new FakeChainAddEdgesProcessorLocal(env); + proc->rcProcessRemote = nebula::cpp2::ErrorCode::E_OUTDATED_TERM; + + LOG(INFO) << "Build AddEdgesRequest..."; + cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); + + LOG(INFO) << "Test AddEdgesProcessor..."; + auto fut = proc->getFuture(); + proc->process(req); + auto resp = std::move(fut).get(); + EXPECT_EQ(0, resp.result.failed_parts.size()); + + ChainTestUtils util; + // none of really edge key should be inserted + EXPECT_EQ(0, numOfKey(req, util.genKey, env)); + // prime key should be deleted + EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); + EXPECT_EQ(0, numOfKey(req, util.genDoublePrime, env)); } TEST(ChainAddEdgesTest, processRemoteUnknownTest) { - fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - env->metaClient_ = mClient.get(); - MetaClientTestUpdater::addPartTerm(env->metaClient_, mockSpaceId, mockPartNum, fackTerm); - - auto* proc = new FakeChainAddEdgesProcessorLocal(env); - - proc->rcProcessRemote = nebula::cpp2::ErrorCode::E_RPC_FAILURE; - - LOG(INFO) << "Build AddEdgesRequest..."; - cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); - - LOG(INFO) << "Test AddEdgesProcessor..."; - auto fut = proc->getFuture(); - proc->process(req); - auto resp = std::move(fut).get(); - EXPECT_EQ(0, resp.result.failed_parts.size()); - - ChainTestUtils util; - // none of really edge key should be inserted - EXPECT_EQ(334, numOfKey(req, util.genKey, env)); - // prime key should be deleted - EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); - EXPECT_EQ(334, numOfKey(req, util.genDoublePrime, env)); + fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + env->metaClient_ = mClient.get(); + MetaClientTestUpdater::addPartTerm(env->metaClient_, mockSpaceId, mockPartNum, fackTerm); + + auto* proc = new FakeChainAddEdgesProcessorLocal(env); + + proc->rcProcessRemote = nebula::cpp2::ErrorCode::E_RPC_FAILURE; + + LOG(INFO) << "Build AddEdgesRequest..."; + cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); + + LOG(INFO) << "Test AddEdgesProcessor..."; + auto fut = proc->getFuture(); + proc->process(req); + auto resp = std::move(fut).get(); + EXPECT_EQ(0, resp.result.failed_parts.size()); + + ChainTestUtils util; + // none of really edge key should be inserted + EXPECT_EQ(334, numOfKey(req, util.genKey, env)); + // prime key should be deleted + EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); + EXPECT_EQ(334, numOfKey(req, util.genDoublePrime, env)); } - // make a reversed request, make sure it can be added successfully TEST(ChainAddEdgesTest, processRemoteTest) { - fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - env->metaClient_ = mClient.get(); - MetaClientTestUpdater::addPartTerm(env->metaClient_, mockSpaceId, mockPartNum, fackTerm); + env->metaClient_ = mClient.get(); + MetaClientTestUpdater::addPartTerm(env->metaClient_, mockSpaceId, mockPartNum, fackTerm); - auto* proc = new FakeChainAddEdgesProcessorLocal(env); - LOG(INFO) << "Build AddEdgesRequest..."; - cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); + auto* proc = new FakeChainAddEdgesProcessorLocal(env); + LOG(INFO) << "Build AddEdgesRequest..."; + cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); - auto reversedRequest = proc->reverseRequestForward(req); + auto reversedRequest = proc->reverseRequestForward(req); } } // namespace storage } // namespace nebula int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - folly::init(&argc, &argv, false); - google::SetStderrLogging(google::INFO); - return RUN_ALL_TESTS(); + testing::InitGoogleTest(&argc, argv); + folly::init(&argc, &argv, false); + google::SetStderrLogging(google::INFO); + return RUN_ALL_TESTS(); } diff --git a/src/storage/test/ChainResumeEdgeTest.cpp b/src/storage/test/ChainResumeEdgeTest.cpp index 47e463d32dc..e67b67067ce 100644 --- a/src/storage/test/ChainResumeEdgeTest.cpp +++ b/src/storage/test/ChainResumeEdgeTest.cpp @@ -1,30 +1,28 @@ - /* Copyright (c) 2021 vesoft inc. All rights reserved. +/* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ -#include #include #include +#include #include #include -#include +#include #include #include "common/fs/TempDir.h" - -#include "storage/CommonUtils.h" -#include "storage/transaction/ConsistUtil.h" -#include "storage/transaction/ChainAddEdgesProcessorLocal.h" -#include "storage/transaction/ChainAddEdgesGroupProcessor.h" -#include "storage/transaction/ChainResumeProcessor.h" - #include "mock/MockCluster.h" #include "mock/MockData.h" -#include "storage/test/TestUtils.h" +#include "storage/CommonUtils.h" #include "storage/test/ChainTestUtils.h" #include "storage/test/QueryTestUtils.h" +#include "storage/test/TestUtils.h" +#include "storage/transaction/ChainAddEdgesGroupProcessor.h" +#include "storage/transaction/ChainAddEdgesProcessorLocal.h" +#include "storage/transaction/ChainResumeProcessor.h" +#include "storage/transaction/ConsistUtil.h" namespace nebula { namespace storage { @@ -46,37 +44,37 @@ ChainUpdateEdgeTestHelper helper; * keyOfRequest: false */ TEST(ChainResumeEdgesTest, resumeTest1) { - fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - - env->metaClient_ = mClient.get(); - auto* proc = new FakeChainAddEdgesProcessorLocal(env); - - proc->rcProcessRemote = nebula::cpp2::ErrorCode::SUCCEEDED; - proc->rcProcessLocal = nebula::cpp2::ErrorCode::SUCCEEDED; - - LOG(INFO) << "Build AddEdgesRequest..."; - cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); - - auto fut = proc->getFuture(); - proc->process(req); - auto resp = std::move(fut).get(); - EXPECT_EQ(0, resp.result.failed_parts.size()); - EXPECT_EQ(0, numOfKey(req, gTestUtil.genKey, env)); - EXPECT_EQ(334, numOfKey(req, gTestUtil.genPrime, env)); - EXPECT_EQ(0, numOfKey(req, gTestUtil.genDoublePrime, env)); - - auto* iClient = FakeInternalStorageClient::instance(env); - FakeInternalStorageClient::hookInternalStorageClient(env, iClient); - ChainResumeProcessor resumeProc(env); - resumeProc.process(); - - EXPECT_EQ(334, numOfKey(req, gTestUtil.genKey, env)); - EXPECT_EQ(0, numOfKey(req, gTestUtil.genPrime, env)); - EXPECT_EQ(0, numOfKey(req, gTestUtil.genDoublePrime, env)); + fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + + env->metaClient_ = mClient.get(); + auto* proc = new FakeChainAddEdgesProcessorLocal(env); + + proc->rcProcessRemote = nebula::cpp2::ErrorCode::SUCCEEDED; + proc->rcProcessLocal = nebula::cpp2::ErrorCode::SUCCEEDED; + + LOG(INFO) << "Build AddEdgesRequest..."; + cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); + + auto fut = proc->getFuture(); + proc->process(req); + auto resp = std::move(fut).get(); + EXPECT_EQ(0, resp.result.failed_parts.size()); + EXPECT_EQ(0, numOfKey(req, gTestUtil.genKey, env)); + EXPECT_EQ(334, numOfKey(req, gTestUtil.genPrime, env)); + EXPECT_EQ(0, numOfKey(req, gTestUtil.genDoublePrime, env)); + + auto* iClient = FakeInternalStorageClient::instance(env); + FakeInternalStorageClient::hookInternalStorageClient(env, iClient); + ChainResumeProcessor resumeProc(env); + resumeProc.process(); + + EXPECT_EQ(334, numOfKey(req, gTestUtil.genKey, env)); + EXPECT_EQ(0, numOfKey(req, gTestUtil.genPrime, env)); + EXPECT_EQ(0, numOfKey(req, gTestUtil.genDoublePrime, env)); } /** @@ -89,82 +87,82 @@ TEST(ChainResumeEdgesTest, resumeTest1) { * keyOfRequest: false */ TEST(ChainResumeEdgesTest, resumeTest2) { - fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - - env->metaClient_ = mClient.get(); - auto* proc = new FakeChainAddEdgesProcessorLocal(env); - - proc->rcProcessRemote = nebula::cpp2::ErrorCode::SUCCEEDED; - proc->rcProcessLocal = nebula::cpp2::ErrorCode::SUCCEEDED; - - LOG(INFO) << "Build AddEdgesRequest..."; - cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); - - LOG(INFO) << "Test AddEdgesProcessor..."; - auto fut = proc->getFuture(); - proc->process(req); - auto resp = std::move(fut).get(); - EXPECT_EQ(0, resp.result.failed_parts.size()); - - ChainTestUtils util; - EXPECT_EQ(0, numOfKey(req, util.genKey, env)); - EXPECT_EQ(334, numOfKey(req, util.genPrime, env)); - EXPECT_EQ(0, numOfKey(req, util.genDoublePrime, env)); - - auto* iClient = FakeInternalStorageClient::instance(env, nebula::cpp2::ErrorCode::E_UNKNOWN); - FakeInternalStorageClient::hookInternalStorageClient(env, iClient); - ChainResumeProcessor resumeProc(env); - resumeProc.process(); - - EXPECT_EQ(0, numOfKey(req, util.genKey, env)); - EXPECT_EQ(334, numOfKey(req, util.genPrime, env)); - EXPECT_EQ(0, numOfKey(req, util.genDoublePrime, env)); + fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + + env->metaClient_ = mClient.get(); + auto* proc = new FakeChainAddEdgesProcessorLocal(env); + + proc->rcProcessRemote = nebula::cpp2::ErrorCode::SUCCEEDED; + proc->rcProcessLocal = nebula::cpp2::ErrorCode::SUCCEEDED; + + LOG(INFO) << "Build AddEdgesRequest..."; + cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); + + LOG(INFO) << "Test AddEdgesProcessor..."; + auto fut = proc->getFuture(); + proc->process(req); + auto resp = std::move(fut).get(); + EXPECT_EQ(0, resp.result.failed_parts.size()); + + ChainTestUtils util; + EXPECT_EQ(0, numOfKey(req, util.genKey, env)); + EXPECT_EQ(334, numOfKey(req, util.genPrime, env)); + EXPECT_EQ(0, numOfKey(req, util.genDoublePrime, env)); + + auto* iClient = FakeInternalStorageClient::instance(env, nebula::cpp2::ErrorCode::E_UNKNOWN); + FakeInternalStorageClient::hookInternalStorageClient(env, iClient); + ChainResumeProcessor resumeProc(env); + resumeProc.process(); + + EXPECT_EQ(0, numOfKey(req, util.genKey, env)); + EXPECT_EQ(334, numOfKey(req, util.genPrime, env)); + EXPECT_EQ(0, numOfKey(req, util.genDoublePrime, env)); } /** * @brief resumePrimeTest3 (resume insert prime outdated) */ TEST(ChainResumeEdgesTest, resumePrimeTest3) { - fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - - env->metaClient_ = mClient.get(); - auto* proc = new FakeChainAddEdgesProcessorLocal(env); - - proc->rcProcessRemote = nebula::cpp2::ErrorCode::SUCCEEDED; - proc->rcProcessLocal = nebula::cpp2::ErrorCode::SUCCEEDED; - - LOG(INFO) << "Build AddEdgesRequest..."; - cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); - - LOG(INFO) << "Test AddEdgesProcessor..."; - auto fut = proc->getFuture(); - proc->process(req); - auto resp = std::move(fut).get(); - EXPECT_EQ(0, resp.result.failed_parts.size()); - - ChainTestUtils util; - EXPECT_EQ(0, numOfKey(req, util.genKey, env)); - EXPECT_EQ(334, numOfKey(req, util.genPrime, env)); - EXPECT_EQ(0, numOfKey(req, util.genDoublePrime, env)); - - auto error = nebula::cpp2::ErrorCode::E_RPC_FAILURE; - auto* iClient = FakeInternalStorageClient::instance(env, error); - FakeInternalStorageClient::hookInternalStorageClient(env, iClient); - ChainResumeProcessor resumeProc(env); - resumeProc.process(); - - // none of really edge key should be inserted - EXPECT_EQ(334, numOfKey(req, util.genKey, env)); - EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); - EXPECT_EQ(334, numOfKey(req, util.genDoublePrime, env)); + fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + + env->metaClient_ = mClient.get(); + auto* proc = new FakeChainAddEdgesProcessorLocal(env); + + proc->rcProcessRemote = nebula::cpp2::ErrorCode::SUCCEEDED; + proc->rcProcessLocal = nebula::cpp2::ErrorCode::SUCCEEDED; + + LOG(INFO) << "Build AddEdgesRequest..."; + cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); + + LOG(INFO) << "Test AddEdgesProcessor..."; + auto fut = proc->getFuture(); + proc->process(req); + auto resp = std::move(fut).get(); + EXPECT_EQ(0, resp.result.failed_parts.size()); + + ChainTestUtils util; + EXPECT_EQ(0, numOfKey(req, util.genKey, env)); + EXPECT_EQ(334, numOfKey(req, util.genPrime, env)); + EXPECT_EQ(0, numOfKey(req, util.genDoublePrime, env)); + + auto error = nebula::cpp2::ErrorCode::E_RPC_FAILURE; + auto* iClient = FakeInternalStorageClient::instance(env, error); + FakeInternalStorageClient::hookInternalStorageClient(env, iClient); + ChainResumeProcessor resumeProc(env); + resumeProc.process(); + + // none of really edge key should be inserted + EXPECT_EQ(334, numOfKey(req, util.genKey, env)); + EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); + EXPECT_EQ(334, numOfKey(req, util.genDoublePrime, env)); } /** @@ -177,367 +175,366 @@ TEST(ChainResumeEdgesTest, resumePrimeTest3) { * keyOfRequest: true */ TEST(ChainResumeEdgesTest, resumeTest4) { - fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - - env->metaClient_ = mClient.get(); - auto* proc = new FakeChainAddEdgesProcessorLocal(env); - - proc->rcProcessRemote = nebula::cpp2::ErrorCode::E_RPC_FAILURE; - - LOG(INFO) << "Build AddEdgesRequest..."; - int partNum = 1; - cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, partNum); - - LOG(INFO) << "Test AddEdgesProcessor..."; - auto fut = proc->getFuture(); - proc->process(req); - auto resp = std::move(fut).get(); - EXPECT_EQ(0, resp.result.failed_parts.size()); - - // ChainTestUtils util; - EXPECT_EQ(334, numOfKey(req, gTestUtil.genKey, env)); - EXPECT_EQ(0, numOfKey(req, gTestUtil.genPrime, env)); - EXPECT_EQ(334, numOfKey(req, gTestUtil.genDoublePrime, env)); - - auto error = nebula::cpp2::ErrorCode::E_UNKNOWN; - auto* iClient = FakeInternalStorageClient::instance(env, error); - FakeInternalStorageClient::hookInternalStorageClient(env, iClient); - ChainResumeProcessor resumeProc(env); - resumeProc.process(); - - EXPECT_EQ(334, numOfKey(req, gTestUtil.genKey, env)); - EXPECT_EQ(0, numOfKey(req, gTestUtil.genPrime, env)); - EXPECT_EQ(334, numOfKey(req, gTestUtil.genDoublePrime, env)); + fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + + env->metaClient_ = mClient.get(); + auto* proc = new FakeChainAddEdgesProcessorLocal(env); + + proc->rcProcessRemote = nebula::cpp2::ErrorCode::E_RPC_FAILURE; + + LOG(INFO) << "Build AddEdgesRequest..."; + int partNum = 1; + cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, partNum); + + LOG(INFO) << "Test AddEdgesProcessor..."; + auto fut = proc->getFuture(); + proc->process(req); + auto resp = std::move(fut).get(); + EXPECT_EQ(0, resp.result.failed_parts.size()); + + // ChainTestUtils util; + EXPECT_EQ(334, numOfKey(req, gTestUtil.genKey, env)); + EXPECT_EQ(0, numOfKey(req, gTestUtil.genPrime, env)); + EXPECT_EQ(334, numOfKey(req, gTestUtil.genDoublePrime, env)); + + auto error = nebula::cpp2::ErrorCode::E_UNKNOWN; + auto* iClient = FakeInternalStorageClient::instance(env, error); + FakeInternalStorageClient::hookInternalStorageClient(env, iClient); + ChainResumeProcessor resumeProc(env); + resumeProc.process(); + + EXPECT_EQ(334, numOfKey(req, gTestUtil.genKey, env)); + EXPECT_EQ(0, numOfKey(req, gTestUtil.genPrime, env)); + EXPECT_EQ(334, numOfKey(req, gTestUtil.genDoublePrime, env)); } /** * @brief resumeTest5 (resume double prime, but outdated) */ TEST(ChainResumeEdgesTest, resumeTest5) { - fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - - env->metaClient_ = mClient.get(); - auto* proc = new FakeChainAddEdgesProcessorLocal(env); - - proc->rcProcessRemote = nebula::cpp2::ErrorCode::E_RPC_FAILURE; - - LOG(INFO) << "Build AddEdgesRequest..."; - cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); - - LOG(INFO) << "Test AddEdgesProcessor..."; - auto fut = proc->getFuture(); - proc->process(req); - auto resp = std::move(fut).get(); - EXPECT_EQ(0, resp.result.failed_parts.size()); - - ChainTestUtils util; - EXPECT_EQ(334, numOfKey(req, util.genKey, env)); - EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); - EXPECT_EQ(334, numOfKey(req, util.genDoublePrime, env)); - - auto error = nebula::cpp2::ErrorCode::E_RPC_FAILURE; - auto* iClient = FakeInternalStorageClient::instance(env, error); - FakeInternalStorageClient::hookInternalStorageClient(env, iClient); - ChainResumeProcessor resumeProc(env); - resumeProc.process(); - - EXPECT_EQ(334, numOfKey(req, util.genKey, env)); - EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); - EXPECT_EQ(334, numOfKey(req, util.genDoublePrime, env)); + fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + + env->metaClient_ = mClient.get(); + auto* proc = new FakeChainAddEdgesProcessorLocal(env); + + proc->rcProcessRemote = nebula::cpp2::ErrorCode::E_RPC_FAILURE; + + LOG(INFO) << "Build AddEdgesRequest..."; + cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); + + LOG(INFO) << "Test AddEdgesProcessor..."; + auto fut = proc->getFuture(); + proc->process(req); + auto resp = std::move(fut).get(); + EXPECT_EQ(0, resp.result.failed_parts.size()); + + ChainTestUtils util; + EXPECT_EQ(334, numOfKey(req, util.genKey, env)); + EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); + EXPECT_EQ(334, numOfKey(req, util.genDoublePrime, env)); + + auto error = nebula::cpp2::ErrorCode::E_RPC_FAILURE; + auto* iClient = FakeInternalStorageClient::instance(env, error); + FakeInternalStorageClient::hookInternalStorageClient(env, iClient); + ChainResumeProcessor resumeProc(env); + resumeProc.process(); + + EXPECT_EQ(334, numOfKey(req, util.genKey, env)); + EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); + EXPECT_EQ(334, numOfKey(req, util.genDoublePrime, env)); } /** * @brief resumeTest6 (resume add edge double prime, succeeded) */ TEST(ChainResumeEdgesTest, resumeTest6) { - fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - - env->metaClient_ = mClient.get(); - auto* proc = new FakeChainAddEdgesProcessorLocal(env); - - proc->rcProcessRemote = nebula::cpp2::ErrorCode::E_RPC_FAILURE; - - LOG(INFO) << "Build AddEdgesRequest..."; - cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); - - LOG(INFO) << "Test AddEdgesProcessor..."; - auto fut = proc->getFuture(); - proc->process(req); - auto resp = std::move(fut).get(); - EXPECT_EQ(0, resp.result.failed_parts.size()); - - ChainTestUtils util; - EXPECT_EQ(334, numOfKey(req, util.genKey, env)); - EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); - EXPECT_EQ(334, numOfKey(req, util.genDoublePrime, env)); - - auto* iClient = FakeInternalStorageClient::instance(env); - FakeInternalStorageClient::hookInternalStorageClient(env, iClient); - ChainResumeProcessor resumeProc(env); - resumeProc.process(); - - EXPECT_EQ(334, numOfKey(req, util.genKey, env)); - EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); - EXPECT_EQ(0, numOfKey(req, util.genDoublePrime, env)); + fs::TempDir rootPath("/tmp/AddEdgesTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + + env->metaClient_ = mClient.get(); + auto* proc = new FakeChainAddEdgesProcessorLocal(env); + + proc->rcProcessRemote = nebula::cpp2::ErrorCode::E_RPC_FAILURE; + + LOG(INFO) << "Build AddEdgesRequest..."; + cpp2::AddEdgesRequest req = mock::MockData::mockAddEdgesReq(false, 1); + + LOG(INFO) << "Test AddEdgesProcessor..."; + auto fut = proc->getFuture(); + proc->process(req); + auto resp = std::move(fut).get(); + EXPECT_EQ(0, resp.result.failed_parts.size()); + + ChainTestUtils util; + EXPECT_EQ(334, numOfKey(req, util.genKey, env)); + EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); + EXPECT_EQ(334, numOfKey(req, util.genDoublePrime, env)); + + auto* iClient = FakeInternalStorageClient::instance(env); + FakeInternalStorageClient::hookInternalStorageClient(env, iClient); + ChainResumeProcessor resumeProc(env); + resumeProc.process(); + + EXPECT_EQ(334, numOfKey(req, util.genKey, env)); + EXPECT_EQ(0, numOfKey(req, util.genPrime, env)); + EXPECT_EQ(0, numOfKey(req, util.genDoublePrime, env)); } // resume an update left prime, check resume succeeded TEST(ChainUpdateEdgeTest, resumeTest7) { - fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - env->metaClient_ = mClient.get(); - - auto parts = cluster.getTotalParts(); - EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); - - LOG(INFO) << "Test UpdateEdgeRequest..."; - auto req = helper.makeDefaultRequest(); - - LOG(INFO) << "Fake Prime..."; - auto* proc = new FakeChainUpdateProcessor(env); - auto f = proc->getFuture(); - proc->rcProcessRemote = Code::SUCCEEDED; - proc->rcProcessLocal = Code::SUCCEEDED; - proc->process(req); - auto resp = std::move(f).get(); - - // EXPECT_TRUE(helper.checkResp(req, resp)); - EXPECT_FALSE(helper.checkRequestUpdated(env, req)); - EXPECT_TRUE(helper.edgeExist(env, req)); - EXPECT_TRUE(helper.primeExist(env, req)); - EXPECT_FALSE(helper.doublePrimeExist(env, req)); - - auto* iClient = FakeInternalStorageClient::instance(env); - FakeInternalStorageClient::hookInternalStorageClient(env, iClient); - ChainResumeProcessor resumeProc(env); - resumeProc.process(); - - EXPECT_TRUE(helper.edgeExist(env, req)); - EXPECT_FALSE(helper.primeExist(env, req)); - EXPECT_FALSE(helper.doublePrimeExist(env, req)); + fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + env->metaClient_ = mClient.get(); + + auto parts = cluster.getTotalParts(); + EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); + + LOG(INFO) << "Test UpdateEdgeRequest..."; + auto req = helper.makeDefaultRequest(); + + LOG(INFO) << "Fake Prime..."; + auto* proc = new FakeChainUpdateProcessor(env); + auto f = proc->getFuture(); + proc->rcProcessRemote = Code::SUCCEEDED; + proc->rcProcessLocal = Code::SUCCEEDED; + proc->process(req); + auto resp = std::move(f).get(); + + // EXPECT_TRUE(helper.checkResp(req, resp)); + EXPECT_FALSE(helper.checkRequestUpdated(env, req)); + EXPECT_TRUE(helper.edgeExist(env, req)); + EXPECT_TRUE(helper.primeExist(env, req)); + EXPECT_FALSE(helper.doublePrimeExist(env, req)); + + auto* iClient = FakeInternalStorageClient::instance(env); + FakeInternalStorageClient::hookInternalStorageClient(env, iClient); + ChainResumeProcessor resumeProc(env); + resumeProc.process(); + + EXPECT_TRUE(helper.edgeExist(env, req)); + EXPECT_FALSE(helper.primeExist(env, req)); + EXPECT_FALSE(helper.doublePrimeExist(env, req)); } // resume an update left prime, resume failed TEST(ChainUpdateEdgeTest, resumeTest8) { - fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - env->metaClient_ = mClient.get(); - - auto parts = cluster.getTotalParts(); - EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); - - LOG(INFO) << "Test UpdateEdgeRequest..."; - auto req = helper.makeDefaultRequest(); - - LOG(INFO) << "Fake Prime..."; - auto* proc = new FakeChainUpdateProcessor(env); - auto f = proc->getFuture(); - proc->rcProcessRemote = Code::SUCCEEDED; - proc->rcProcessLocal = Code::SUCCEEDED; - proc->process(req); - auto resp = std::move(f).get(); - - // EXPECT_TRUE(helper.checkResp(req, resp)); - EXPECT_FALSE(helper.checkRequestUpdated(env, req)); - EXPECT_TRUE(helper.edgeExist(env, req)); - EXPECT_TRUE(helper.primeExist(env, req)); - EXPECT_FALSE(helper.doublePrimeExist(env, req)); - - auto* iClient = FakeInternalStorageClient::instance(env); - iClient->setErrorCode(Code::E_UNKNOWN); - FakeInternalStorageClient::hookInternalStorageClient(env, iClient); - ChainResumeProcessor resumeProc(env); - resumeProc.process(); - - EXPECT_TRUE(helper.edgeExist(env, req)); - EXPECT_TRUE(helper.primeExist(env, req)); - EXPECT_FALSE(helper.doublePrimeExist(env, req)); + fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + env->metaClient_ = mClient.get(); + + auto parts = cluster.getTotalParts(); + EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); + + LOG(INFO) << "Test UpdateEdgeRequest..."; + auto req = helper.makeDefaultRequest(); + + LOG(INFO) << "Fake Prime..."; + auto* proc = new FakeChainUpdateProcessor(env); + auto f = proc->getFuture(); + proc->rcProcessRemote = Code::SUCCEEDED; + proc->rcProcessLocal = Code::SUCCEEDED; + proc->process(req); + auto resp = std::move(f).get(); + + // EXPECT_TRUE(helper.checkResp(req, resp)); + EXPECT_FALSE(helper.checkRequestUpdated(env, req)); + EXPECT_TRUE(helper.edgeExist(env, req)); + EXPECT_TRUE(helper.primeExist(env, req)); + EXPECT_FALSE(helper.doublePrimeExist(env, req)); + + auto* iClient = FakeInternalStorageClient::instance(env); + iClient->setErrorCode(Code::E_UNKNOWN); + FakeInternalStorageClient::hookInternalStorageClient(env, iClient); + ChainResumeProcessor resumeProc(env); + resumeProc.process(); + + EXPECT_TRUE(helper.edgeExist(env, req)); + EXPECT_TRUE(helper.primeExist(env, req)); + EXPECT_FALSE(helper.doublePrimeExist(env, req)); } // resume an update left prime, resume outdated TEST(ChainUpdateEdgeTest, resumeTest9) { - fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - env->metaClient_ = mClient.get(); - - auto parts = cluster.getTotalParts(); - EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); - - LOG(INFO) << "Test UpdateEdgeRequest..."; - auto req = helper.makeDefaultRequest(); - - LOG(INFO) << "Fake Prime..."; - auto* proc = new FakeChainUpdateProcessor(env); - auto f = proc->getFuture(); - proc->rcProcessRemote = Code::SUCCEEDED; - proc->rcProcessLocal = Code::SUCCEEDED; - proc->process(req); - auto resp = std::move(f).get(); - - // EXPECT_TRUE(helper.checkResp(req, resp)); - EXPECT_FALSE(helper.checkRequestUpdated(env, req)); - EXPECT_TRUE(helper.edgeExist(env, req)); - EXPECT_TRUE(helper.primeExist(env, req)); - EXPECT_FALSE(helper.doublePrimeExist(env, req)); - - auto* iClient = FakeInternalStorageClient::instance(env); - iClient->setErrorCode(Code::E_RPC_FAILURE); - FakeInternalStorageClient::hookInternalStorageClient(env, iClient); - ChainResumeProcessor resumeProc(env); - resumeProc.process(); - - EXPECT_TRUE(helper.edgeExist(env, req)); - EXPECT_FALSE(helper.primeExist(env, req)); - EXPECT_TRUE(helper.doublePrimeExist(env, req)); + fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + env->metaClient_ = mClient.get(); + + auto parts = cluster.getTotalParts(); + EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); + + LOG(INFO) << "Test UpdateEdgeRequest..."; + auto req = helper.makeDefaultRequest(); + + LOG(INFO) << "Fake Prime..."; + auto* proc = new FakeChainUpdateProcessor(env); + auto f = proc->getFuture(); + proc->rcProcessRemote = Code::SUCCEEDED; + proc->rcProcessLocal = Code::SUCCEEDED; + proc->process(req); + auto resp = std::move(f).get(); + + // EXPECT_TRUE(helper.checkResp(req, resp)); + EXPECT_FALSE(helper.checkRequestUpdated(env, req)); + EXPECT_TRUE(helper.edgeExist(env, req)); + EXPECT_TRUE(helper.primeExist(env, req)); + EXPECT_FALSE(helper.doublePrimeExist(env, req)); + + auto* iClient = FakeInternalStorageClient::instance(env); + iClient->setErrorCode(Code::E_RPC_FAILURE); + FakeInternalStorageClient::hookInternalStorageClient(env, iClient); + ChainResumeProcessor resumeProc(env); + resumeProc.process(); + + EXPECT_TRUE(helper.edgeExist(env, req)); + EXPECT_FALSE(helper.primeExist(env, req)); + EXPECT_TRUE(helper.doublePrimeExist(env, req)); } // resume an update left prime, check resume succeeded TEST(ChainUpdateEdgeTest, resumeTest10) { - fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - env->metaClient_ = mClient.get(); - - auto parts = cluster.getTotalParts(); - EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); - - LOG(INFO) << "Test UpdateEdgeRequest..."; - auto req = helper.makeDefaultRequest(); - - LOG(INFO) << "Fake Prime..."; - auto* proc = new FakeChainUpdateProcessor(env); - auto f = proc->getFuture(); - proc->rcProcessRemote = Code::E_RPC_FAILURE; - // proc->rcProcessLocal = Code::SUCCEEDED; - proc->process(req); - auto resp = std::move(f).get(); - - EXPECT_TRUE(helper.checkRequestUpdated(env, req)); - EXPECT_TRUE(helper.edgeExist(env, req)); - EXPECT_FALSE(helper.primeExist(env, req)); - EXPECT_TRUE(helper.doublePrimeExist(env, req)); - - auto* iClient = FakeInternalStorageClient::instance(env); - FakeInternalStorageClient::hookInternalStorageClient(env, iClient); - ChainResumeProcessor resumeProc(env); - resumeProc.process(); - - EXPECT_TRUE(helper.edgeExist(env, req)); - EXPECT_FALSE(helper.primeExist(env, req)); - EXPECT_FALSE(helper.doublePrimeExist(env, req)); + fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + env->metaClient_ = mClient.get(); + + auto parts = cluster.getTotalParts(); + EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); + + LOG(INFO) << "Test UpdateEdgeRequest..."; + auto req = helper.makeDefaultRequest(); + + LOG(INFO) << "Fake Prime..."; + auto* proc = new FakeChainUpdateProcessor(env); + auto f = proc->getFuture(); + proc->rcProcessRemote = Code::E_RPC_FAILURE; + // proc->rcProcessLocal = Code::SUCCEEDED; + proc->process(req); + auto resp = std::move(f).get(); + + EXPECT_TRUE(helper.checkRequestUpdated(env, req)); + EXPECT_TRUE(helper.edgeExist(env, req)); + EXPECT_FALSE(helper.primeExist(env, req)); + EXPECT_TRUE(helper.doublePrimeExist(env, req)); + + auto* iClient = FakeInternalStorageClient::instance(env); + FakeInternalStorageClient::hookInternalStorageClient(env, iClient); + ChainResumeProcessor resumeProc(env); + resumeProc.process(); + + EXPECT_TRUE(helper.edgeExist(env, req)); + EXPECT_FALSE(helper.primeExist(env, req)); + EXPECT_FALSE(helper.doublePrimeExist(env, req)); } // resume an update left prime, resume failed TEST(ChainUpdateEdgeTest, resumeTest11) { - fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - env->metaClient_ = mClient.get(); - - auto parts = cluster.getTotalParts(); - EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); - - LOG(INFO) << "Test UpdateEdgeRequest..."; - auto req = helper.makeDefaultRequest(); - - LOG(INFO) << "Fake Prime..."; - auto* proc = new FakeChainUpdateProcessor(env); - auto f = proc->getFuture(); - proc->rcProcessRemote = Code::E_RPC_FAILURE; - // proc->rcProcessLocal = Code::SUCCEEDED; - proc->process(req); - auto resp = std::move(f).get(); - - EXPECT_TRUE(helper.checkRequestUpdated(env, req)); - EXPECT_TRUE(helper.edgeExist(env, req)); - EXPECT_FALSE(helper.primeExist(env, req)); - EXPECT_TRUE(helper.doublePrimeExist(env, req)); - - auto* iClient = FakeInternalStorageClient::instance(env); - iClient->setErrorCode(Code::E_UNKNOWN); - FakeInternalStorageClient::hookInternalStorageClient(env, iClient); - ChainResumeProcessor resumeProc(env); - resumeProc.process(); - - EXPECT_TRUE(helper.edgeExist(env, req)); - EXPECT_FALSE(helper.primeExist(env, req)); - EXPECT_TRUE(helper.doublePrimeExist(env, req)); + fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + env->metaClient_ = mClient.get(); + + auto parts = cluster.getTotalParts(); + EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); + + LOG(INFO) << "Test UpdateEdgeRequest..."; + auto req = helper.makeDefaultRequest(); + + LOG(INFO) << "Fake Prime..."; + auto* proc = new FakeChainUpdateProcessor(env); + auto f = proc->getFuture(); + proc->rcProcessRemote = Code::E_RPC_FAILURE; + // proc->rcProcessLocal = Code::SUCCEEDED; + proc->process(req); + auto resp = std::move(f).get(); + + EXPECT_TRUE(helper.checkRequestUpdated(env, req)); + EXPECT_TRUE(helper.edgeExist(env, req)); + EXPECT_FALSE(helper.primeExist(env, req)); + EXPECT_TRUE(helper.doublePrimeExist(env, req)); + + auto* iClient = FakeInternalStorageClient::instance(env); + iClient->setErrorCode(Code::E_UNKNOWN); + FakeInternalStorageClient::hookInternalStorageClient(env, iClient); + ChainResumeProcessor resumeProc(env); + resumeProc.process(); + + EXPECT_TRUE(helper.edgeExist(env, req)); + EXPECT_FALSE(helper.primeExist(env, req)); + EXPECT_TRUE(helper.doublePrimeExist(env, req)); } // resume an update left prime, resume outdated TEST(ChainUpdateEdgeTest, resumeTest12) { - fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - env->metaClient_ = mClient.get(); - - auto parts = cluster.getTotalParts(); - EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); - - LOG(INFO) << "Test UpdateEdgeRequest..."; - auto req = helper.makeDefaultRequest(); - - LOG(INFO) << "Fake Prime..."; - auto* proc = new FakeChainUpdateProcessor(env); - auto f = proc->getFuture(); - proc->rcProcessRemote = Code::E_RPC_FAILURE; - // proc->rcProcessLocal = Code::SUCCEEDED; - proc->process(req); - auto resp = std::move(f).get(); - - EXPECT_TRUE(helper.checkRequestUpdated(env, req)); - EXPECT_TRUE(helper.edgeExist(env, req)); - EXPECT_FALSE(helper.primeExist(env, req)); - EXPECT_TRUE(helper.doublePrimeExist(env, req)); - - auto* iClient = FakeInternalStorageClient::instance(env); - iClient->setErrorCode(Code::E_RPC_FAILURE); - FakeInternalStorageClient::hookInternalStorageClient(env, iClient); - ChainResumeProcessor resumeProc(env); - resumeProc.process(); - - EXPECT_TRUE(helper.edgeExist(env, req)); - EXPECT_FALSE(helper.primeExist(env, req)); - EXPECT_TRUE(helper.doublePrimeExist(env, req)); + fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + env->metaClient_ = mClient.get(); + + auto parts = cluster.getTotalParts(); + EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); + + LOG(INFO) << "Test UpdateEdgeRequest..."; + auto req = helper.makeDefaultRequest(); + + LOG(INFO) << "Fake Prime..."; + auto* proc = new FakeChainUpdateProcessor(env); + auto f = proc->getFuture(); + proc->rcProcessRemote = Code::E_RPC_FAILURE; + // proc->rcProcessLocal = Code::SUCCEEDED; + proc->process(req); + auto resp = std::move(f).get(); + + EXPECT_TRUE(helper.checkRequestUpdated(env, req)); + EXPECT_TRUE(helper.edgeExist(env, req)); + EXPECT_FALSE(helper.primeExist(env, req)); + EXPECT_TRUE(helper.doublePrimeExist(env, req)); + + auto* iClient = FakeInternalStorageClient::instance(env); + iClient->setErrorCode(Code::E_RPC_FAILURE); + FakeInternalStorageClient::hookInternalStorageClient(env, iClient); + ChainResumeProcessor resumeProc(env); + resumeProc.process(); + + EXPECT_TRUE(helper.edgeExist(env, req)); + EXPECT_FALSE(helper.primeExist(env, req)); + EXPECT_TRUE(helper.doublePrimeExist(env, req)); } } // namespace storage } // namespace nebula int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - folly::init(&argc, &argv, false); - google::SetStderrLogging(google::INFO); - return RUN_ALL_TESTS(); + testing::InitGoogleTest(&argc, argv); + folly::init(&argc, &argv, false); + google::SetStderrLogging(google::INFO); + return RUN_ALL_TESTS(); } - /** * @brief resumeTest1 (prime add edge can be resumed) * insert resume diff --git a/src/storage/test/ChainTestUtils.h b/src/storage/test/ChainTestUtils.h index d5aa7a44333..0c262f6d746 100644 --- a/src/storage/test/ChainTestUtils.h +++ b/src/storage/test/ChainTestUtils.h @@ -21,459 +21,453 @@ extern const int32_t mockSpaceVidLen; using KeyGenerator = std::function; class ChainTestUtils { -public: - ChainTestUtils() { - genKey = [&](PartitionID partId, const cpp2::NewEdge& edge) { - auto key = ConsistUtil::edgeKey(spaceVidLen_, partId, edge.get_key()); - return key; - }; - genPrime = [&](PartitionID partId, const cpp2::NewEdge& edge) { - auto key = ConsistUtil::primeKey(spaceVidLen_, partId, edge.get_key()); - return key; - }; - genDoublePrime = [&](PartitionID partId, const cpp2::NewEdge& edge) { - auto key = ConsistUtil::doublePrime(spaceVidLen_, partId, edge.get_key()); - return key; - }; - } -public: - int32_t spaceVidLen_{32}; - KeyGenerator genKey; - KeyGenerator genPrime; - KeyGenerator genDoublePrime; + public: + ChainTestUtils() { + genKey = [&](PartitionID partId, const cpp2::NewEdge& edge) { + auto key = ConsistUtil::edgeKey(spaceVidLen_, partId, edge.get_key()); + return key; + }; + genPrime = [&](PartitionID partId, const cpp2::NewEdge& edge) { + auto key = ConsistUtil::primeKey(spaceVidLen_, partId, edge.get_key()); + return key; + }; + genDoublePrime = [&](PartitionID partId, const cpp2::NewEdge& edge) { + auto key = ConsistUtil::doublePrime(spaceVidLen_, partId, edge.get_key()); + return key; + }; + } + + public: + int32_t spaceVidLen_{32}; + KeyGenerator genKey; + KeyGenerator genPrime; + KeyGenerator genDoublePrime; }; // , StorageEnv* env int numOfKey(const cpp2::AddEdgesRequest& req, KeyGenerator gen, StorageEnv* env) { - int numOfEdges = 0; - int totalEdge = 0; - auto spaceId = req.get_space_id(); - for (auto& edgesOfPart : *req.parts_ref()) { - auto partId = edgesOfPart.first; - auto& edgeVec = edgesOfPart.second; - for (auto& edge : edgeVec) { - ++totalEdge; - auto key = gen(partId, edge); - std::unique_ptr iter; - EXPECT_EQ(Code::SUCCEEDED, env->kvstore_->prefix(spaceId, partId, key, &iter)); - if (iter && iter->valid()) { - ++numOfEdges; - } else { - // LOG(INFO) << "key: " << key << " not exist"; - } - } - } - LOG(INFO) << "numOfEdges = " << numOfEdges; - LOG(INFO) << "totalEdge = " << totalEdge; - return numOfEdges; + int numOfEdges = 0; + int totalEdge = 0; + auto spaceId = req.get_space_id(); + for (auto& edgesOfPart : *req.parts_ref()) { + auto partId = edgesOfPart.first; + auto& edgeVec = edgesOfPart.second; + for (auto& edge : edgeVec) { + ++totalEdge; + auto key = gen(partId, edge); + std::unique_ptr iter; + EXPECT_EQ(Code::SUCCEEDED, env->kvstore_->prefix(spaceId, partId, key, &iter)); + if (iter && iter->valid()) { + ++numOfEdges; + } else { + // LOG(INFO) << "key: " << key << " not exist"; + } + } + } + LOG(INFO) << "numOfEdges = " << numOfEdges; + LOG(INFO) << "totalEdge = " << totalEdge; + return numOfEdges; } std::pair extractSpaceAndPart(const cpp2::AddEdgesRequest& req) { - auto spaceId = req.get_space_id(); - CHECK_EQ(req.get_parts().size(), 1); - auto partId = req.get_parts().begin()->first; - return std::make_pair(spaceId, partId); + auto spaceId = req.get_space_id(); + CHECK_EQ(req.get_parts().size(), 1); + auto partId = req.get_parts().begin()->first; + return std::make_pair(spaceId, partId); } bool keyExist(StorageEnv* env, GraphSpaceID spaceId, PartitionID partId, std::string key) { - // std::unique_ptr iter; - std::string ignoreVal; - auto rc = env->kvstore_->get(spaceId, partId, key, &ignoreVal); - return rc == Code::SUCCEEDED; + // std::unique_ptr iter; + std::string ignoreVal; + auto rc = env->kvstore_->get(spaceId, partId, key, &ignoreVal); + return rc == Code::SUCCEEDED; } class FakeChainAddEdgesProcessorLocal : public ChainAddEdgesProcessorLocal { - FRIEND_TEST(ChainAddEdgesTest, prepareLocalSucceededTest); - FRIEND_TEST(ChainAddEdgesTest, processRemoteSucceededTest); - FRIEND_TEST(ChainAddEdgesTest, processRemoteFailedTest); - FRIEND_TEST(ChainAddEdgesTest, processRemoteOutdatedTest); - // all the above will test succeeded path of process local - // the failed path of process local will be tested in resume test -public: - explicit FakeChainAddEdgesProcessorLocal(StorageEnv* env) : ChainAddEdgesProcessorLocal(env) { - spaceVidLen_ = 32; - } - - folly::SemiFuture prepareLocal() override { - LOG(INFO) << "FakeChainAddEdgesProcessorLocal::" << __func__ << "()"; - if (rcPrepareLocal) { - LOG(INFO) << "Fake return " << apache::thrift::util::enumNameSafe(*rcPrepareLocal); - return *rcPrepareLocal; - } - LOG(INFO) << "forward to ChainAddEdgesProcessorLocal::prepareLocal()"; - return ChainAddEdgesProcessorLocal::prepareLocal(); - } - - folly::SemiFuture processRemote(Code code) override { - LOG(INFO) << "FakeChainAddEdgesProcessorLocal::" << __func__ << "()"; - if (rcProcessRemote) { - LOG(INFO) << "processRemote() fake return " - << apache::thrift::util::enumNameSafe(*rcProcessRemote); - LOG_IF(FATAL, code != Code::SUCCEEDED) << "cheat must base on truth"; - return *rcProcessRemote; - } - LOG(INFO) << "forward to ChainAddEdgesProcessorLocal::processRemote()"; - return ChainAddEdgesProcessorLocal::processRemote(code); - } - - folly::SemiFuture processLocal(Code code) override { - LOG(INFO) << "FakeChainAddEdgesProcessorLocal::" << __func__ << "()"; - if (rcProcessLocal) { - LOG(INFO) << "Fake return " << apache::thrift::util::enumNameSafe(*rcProcessLocal); - return *rcProcessLocal; - } - LOG(INFO) << "forward to ChainAddEdgesProcessorLocal::processLocal()"; - return ChainAddEdgesProcessorLocal::processLocal(code); - } - - cpp2::AddEdgesRequest reverseRequestForward(const cpp2::AddEdgesRequest& req) { - return ChainAddEdgesProcessorLocal::reverseRequest(req); - } - - folly::Optional rcPrepareLocal; - - folly::Optional rcProcessRemote; - - folly::Optional rcProcessLocal; + FRIEND_TEST(ChainAddEdgesTest, prepareLocalSucceededTest); + FRIEND_TEST(ChainAddEdgesTest, processRemoteSucceededTest); + FRIEND_TEST(ChainAddEdgesTest, processRemoteFailedTest); + FRIEND_TEST(ChainAddEdgesTest, processRemoteOutdatedTest); + // all the above will test succeeded path of process local + // the failed path of process local will be tested in resume test + public: + explicit FakeChainAddEdgesProcessorLocal(StorageEnv* env) : ChainAddEdgesProcessorLocal(env) { + spaceVidLen_ = 32; + } + + folly::SemiFuture prepareLocal() override { + LOG(INFO) << "FakeChainAddEdgesProcessorLocal::" << __func__ << "()"; + if (rcPrepareLocal) { + LOG(INFO) << "Fake return " << apache::thrift::util::enumNameSafe(*rcPrepareLocal); + return *rcPrepareLocal; + } + LOG(INFO) << "forward to ChainAddEdgesProcessorLocal::prepareLocal()"; + return ChainAddEdgesProcessorLocal::prepareLocal(); + } + + folly::SemiFuture processRemote(Code code) override { + LOG(INFO) << "FakeChainAddEdgesProcessorLocal::" << __func__ << "()"; + if (rcProcessRemote) { + LOG(INFO) << "processRemote() fake return " + << apache::thrift::util::enumNameSafe(*rcProcessRemote); + LOG_IF(FATAL, code != Code::SUCCEEDED) << "cheat must base on truth"; + return *rcProcessRemote; + } + LOG(INFO) << "forward to ChainAddEdgesProcessorLocal::processRemote()"; + return ChainAddEdgesProcessorLocal::processRemote(code); + } + + folly::SemiFuture processLocal(Code code) override { + LOG(INFO) << "FakeChainAddEdgesProcessorLocal::" << __func__ << "()"; + if (rcProcessLocal) { + LOG(INFO) << "Fake return " << apache::thrift::util::enumNameSafe(*rcProcessLocal); + return *rcProcessLocal; + } + LOG(INFO) << "forward to ChainAddEdgesProcessorLocal::processLocal()"; + return ChainAddEdgesProcessorLocal::processLocal(code); + } + + cpp2::AddEdgesRequest reverseRequestForward(const cpp2::AddEdgesRequest& req) { + return ChainAddEdgesProcessorLocal::reverseRequest(req); + } + + folly::Optional rcPrepareLocal; + + folly::Optional rcProcessRemote; + + folly::Optional rcProcessLocal; }; class FakeChainUpdateProcessor : public ChainUpdateEdgeProcessorLocal { -public: - explicit FakeChainUpdateProcessor(StorageEnv* env) : ChainUpdateEdgeProcessorLocal(env) { - spaceVidLen_ = 32; - } - - folly::SemiFuture prepareLocal() override { - LOG(INFO) << "FakeChainUpdateEdgeProcessorA::" << __func__ << "()"; - if (rcPrepareLocal) { - LOG(INFO) << "Fake return " << apache::thrift::util::enumNameSafe(*rcPrepareLocal); - return *rcPrepareLocal; - } - LOG(INFO) << "forward to ChainUpdateEdgeProcessorLocal::prepareLocal()"; - return ChainUpdateEdgeProcessorLocal::prepareLocal(); - } - - folly::SemiFuture processRemote(Code code) override { - LOG(INFO) << "FakeChainUpdateEdgeProcessorA::" << __func__ << "()"; - if (rcProcessRemote) { - LOG(INFO) << "processRemote() fake return " - << apache::thrift::util::enumNameSafe(*rcProcessRemote); - LOG_IF(FATAL, code != Code::SUCCEEDED) << "cheat must base on truth"; - return *rcProcessRemote; - } - LOG(INFO) << "forward to ChainUpdateEdgeProcessorLocal::processRemote()"; - return ChainUpdateEdgeProcessorLocal::processRemote(code); - } - - folly::SemiFuture processLocal(Code code) override { - LOG(INFO) << "FakeChainUpdateEdgeProcessorA::" << __func__ << "()"; - if (rcProcessLocal) { - LOG(INFO) << "processLocal() fake return " - << apache::thrift::util::enumNameSafe(*rcProcessLocal); - return *rcProcessLocal; - } - LOG(INFO) << "forward to ChainUpdateEdgeProcessorLocal::processLocal()"; - return ChainUpdateEdgeProcessorLocal::processLocal(code); - } - -public: - folly::Optional rcPrepareLocal; - folly::Optional rcProcessRemote; - folly::Optional rcProcessLocal; + public: + explicit FakeChainUpdateProcessor(StorageEnv* env) : ChainUpdateEdgeProcessorLocal(env) { + spaceVidLen_ = 32; + } + + folly::SemiFuture prepareLocal() override { + LOG(INFO) << "FakeChainUpdateEdgeProcessorA::" << __func__ << "()"; + if (rcPrepareLocal) { + LOG(INFO) << "Fake return " << apache::thrift::util::enumNameSafe(*rcPrepareLocal); + return *rcPrepareLocal; + } + LOG(INFO) << "forward to ChainUpdateEdgeProcessorLocal::prepareLocal()"; + return ChainUpdateEdgeProcessorLocal::prepareLocal(); + } + + folly::SemiFuture processRemote(Code code) override { + LOG(INFO) << "FakeChainUpdateEdgeProcessorA::" << __func__ << "()"; + if (rcProcessRemote) { + LOG(INFO) << "processRemote() fake return " + << apache::thrift::util::enumNameSafe(*rcProcessRemote); + LOG_IF(FATAL, code != Code::SUCCEEDED) << "cheat must base on truth"; + return *rcProcessRemote; + } + LOG(INFO) << "forward to ChainUpdateEdgeProcessorLocal::processRemote()"; + return ChainUpdateEdgeProcessorLocal::processRemote(code); + } + + folly::SemiFuture processLocal(Code code) override { + LOG(INFO) << "FakeChainUpdateEdgeProcessorA::" << __func__ << "()"; + if (rcProcessLocal) { + LOG(INFO) << "processLocal() fake return " + << apache::thrift::util::enumNameSafe(*rcProcessLocal); + return *rcProcessLocal; + } + LOG(INFO) << "forward to ChainUpdateEdgeProcessorLocal::processLocal()"; + return ChainUpdateEdgeProcessorLocal::processLocal(code); + } + + public: + folly::Optional rcPrepareLocal; + folly::Optional rcProcessRemote; + folly::Optional rcProcessLocal; }; class MetaClientTestUpdater { -public: - MetaClientTestUpdater() = default; - - static void addLocalCache(meta::MetaClient& mClient, - GraphSpaceID spaceId, - std::shared_ptr spInfoCache) { - mClient.localCache_[spaceId] = spInfoCache; - } - - static meta::SpaceInfoCache* getLocalCache(meta::MetaClient* mClient, GraphSpaceID spaceId) { - if (mClient->localCache_.count(spaceId) == 0) { - return nullptr; - } - return mClient->localCache_[spaceId].get(); - } + public: + MetaClientTestUpdater() = default; - static void addPartTerm(meta::MetaClient* mClient, + static void addLocalCache(meta::MetaClient& mClient, GraphSpaceID spaceId, - PartitionID partId, - TermID termId) { - auto* pCache = getLocalCache(mClient, spaceId); - if (pCache == nullptr) { - auto spCache = std::make_shared(); - addLocalCache(*mClient, spaceId, spCache); - pCache = getLocalCache(mClient, spaceId); - } - pCache->termOfPartition_[partId] = termId; - } - - static std::unique_ptr makeDefaultMetaClient() { - auto exec = std::make_shared(3); - std::vector addrs(1); - meta::MetaClientOptions options; - - auto mClient = std::make_unique(exec, addrs, options); - mClient->localCache_[mockSpaceId] = std::make_shared(); - for (int i = 0; i != mockPartNum; ++i) { - mClient->localCache_[mockSpaceId]->termOfPartition_[i] = i; - auto ignoreItem = mClient->localCache_[mockSpaceId]->partsAlloc_[i]; - UNUSED(ignoreItem); - } - meta::cpp2::ColumnTypeDef type; - type.set_type(meta::cpp2::PropertyType::FIXED_STRING); - type.set_type_length(32); - - mClient->localCache_[mockSpaceId]->spaceDesc_.set_vid_type(std::move(type)); - mClient->ready_ = true; - return mClient; - } + std::shared_ptr spInfoCache) { + mClient.localCache_[spaceId] = spInfoCache; + } + + static meta::SpaceInfoCache* getLocalCache(meta::MetaClient* mClient, GraphSpaceID spaceId) { + if (mClient->localCache_.count(spaceId) == 0) { + return nullptr; + } + return mClient->localCache_[spaceId].get(); + } + + static void addPartTerm(meta::MetaClient* mClient, + GraphSpaceID spaceId, + PartitionID partId, + TermID termId) { + auto* pCache = getLocalCache(mClient, spaceId); + if (pCache == nullptr) { + auto spCache = std::make_shared(); + addLocalCache(*mClient, spaceId, spCache); + pCache = getLocalCache(mClient, spaceId); + } + pCache->termOfPartition_[partId] = termId; + } + + static std::unique_ptr makeDefaultMetaClient() { + auto exec = std::make_shared(3); + std::vector addrs(1); + meta::MetaClientOptions options; + + auto mClient = std::make_unique(exec, addrs, options); + mClient->localCache_[mockSpaceId] = std::make_shared(); + for (int i = 0; i != mockPartNum; ++i) { + mClient->localCache_[mockSpaceId]->termOfPartition_[i] = i; + auto ignoreItem = mClient->localCache_[mockSpaceId]->partsAlloc_[i]; + UNUSED(ignoreItem); + } + meta::cpp2::ColumnTypeDef type; + type.set_type(meta::cpp2::PropertyType::FIXED_STRING); + type.set_type_length(32); + + mClient->localCache_[mockSpaceId]->spaceDesc_.set_vid_type(std::move(type)); + mClient->ready_ = true; + return mClient; + } }; class FakeInternalStorageClient : public InternalStorageClient { -public: - explicit FakeInternalStorageClient(StorageEnv* env, - std::shared_ptr pool, - Code code) - : InternalStorageClient(pool, env->metaClient_), env_(env), code_(code) {} - - void chainUpdateEdge(cpp2::UpdateEdgeRequest& req, - TermID termOfSrc, - folly::Optional optVersion, - folly::Promise&& p, - folly::EventBase* evb = nullptr) override { - cpp2::ChainUpdateEdgeRequest chainReq; - chainReq.set_update_edge_request(req); - chainReq.set_term(termOfSrc); - - auto* proc = ChainUpdateEdgeProcessorRemote::instance(env_); - auto f = proc->getFuture(); - proc->process(chainReq); - auto resp = std::move(f).get(); - - p.setValue(code_); - UNUSED(optVersion); - UNUSED(evb); - } - - void setErrorCode(Code code) { - code_ = code; - } - - void chainAddEdges(cpp2::AddEdgesRequest& req, - TermID termId, + public: + explicit FakeInternalStorageClient(StorageEnv* env, + std::shared_ptr pool, + Code code) + : InternalStorageClient(pool, env->metaClient_), env_(env), code_(code) {} + + void chainUpdateEdge(cpp2::UpdateEdgeRequest& req, + TermID termOfSrc, folly::Optional optVersion, - folly::Promise<::nebula::cpp2::ErrorCode>&& p, + folly::Promise&& p, folly::EventBase* evb = nullptr) override { - UNUSED(req); - UNUSED(termId); - UNUSED(optVersion); - p.setValue(code_); - UNUSED(evb); - } - - static FakeInternalStorageClient* instance(StorageEnv* env, Code fakeCode = Code::SUCCEEDED) { - auto pool = std::make_shared(3); - return new FakeInternalStorageClient(env, pool, fakeCode); - } - - static void hookInternalStorageClient(StorageEnv* env, InternalStorageClient* client) { - env->txnMan_->iClient_ = client; - } - -private: - StorageEnv* env_{nullptr}; - Code code_{Code::SUCCEEDED}; + cpp2::ChainUpdateEdgeRequest chainReq; + chainReq.set_update_edge_request(req); + chainReq.set_term(termOfSrc); + + auto* proc = ChainUpdateEdgeProcessorRemote::instance(env_); + auto f = proc->getFuture(); + proc->process(chainReq); + auto resp = std::move(f).get(); + + p.setValue(code_); + UNUSED(optVersion); + UNUSED(evb); + } + + void setErrorCode(Code code) { code_ = code; } + + void chainAddEdges(cpp2::AddEdgesRequest& req, + TermID termId, + folly::Optional optVersion, + folly::Promise<::nebula::cpp2::ErrorCode>&& p, + folly::EventBase* evb = nullptr) override { + UNUSED(req); + UNUSED(termId); + UNUSED(optVersion); + p.setValue(code_); + UNUSED(evb); + } + + static FakeInternalStorageClient* instance(StorageEnv* env, Code fakeCode = Code::SUCCEEDED) { + auto pool = std::make_shared(3); + return new FakeInternalStorageClient(env, pool, fakeCode); + } + + static void hookInternalStorageClient(StorageEnv* env, InternalStorageClient* client) { + env->txnMan_->iClient_ = client; + } + + private: + StorageEnv* env_{nullptr}; + Code code_{Code::SUCCEEDED}; }; struct ChainUpdateEdgeTestHelper { - ChainUpdateEdgeTestHelper() { - sEdgeType = std::to_string(std::abs(edgeType_)); - } - - cpp2::EdgeKey defaultEdgeKey() { - cpp2::EdgeKey ret; - ret.set_src(srcId_); - ret.set_edge_type(edgeType_); - ret.set_ranking(rank_); - ret.set_dst(dstId_); - return ret; - } - - std::vector defaultUpdateProps() { - ObjectPool objPool; - LOG(INFO) << "Build updated props..."; - std::vector props; - // int: 101.teamCareer = 20 - cpp2::UpdatedProp uProp1; - uProp1.set_name("teamCareer"); - // ConstantExpression val1(20); - const auto& val1 = *ConstantExpression::make(&objPool, 20); - uProp1.set_value(Expression::encode(val1)); - props.emplace_back(uProp1); - - // bool: 101.type = trade - cpp2::UpdatedProp uProp2; - uProp2.set_name("type"); - std::string colnew("trade"); - // ConstantExpression val2(colnew); - const auto& val2 = *ConstantExpression::make(&objPool, colnew); - uProp2.set_value(Expression::encode(val2)); - props.emplace_back(uProp2); - return props; - } - - std::vector defaultRetProps() { - ObjectPool objPool; - std::vector props; - std::vector cols{ - "playerName", "teamName", "teamCareer", "type", kSrc, kType, kRank, kDst}; - for (auto& colName : cols) { - const auto& exp = *EdgePropertyExpression::make(&objPool, sEdgeType, colName); - props.emplace_back(Expression::encode(exp)); - } - return props; - } - - cpp2::UpdateEdgeRequest makeDefaultRequest() { - auto edgeKey = defaultEdgeKey(); - auto updateProps = defaultUpdateProps(); - auto retProps = defaultRetProps(); - return makeRequest(edgeKey, updateProps, retProps); - } - - cpp2::UpdateEdgeRequest reverseRequest(StorageEnv* env, const cpp2::UpdateEdgeRequest& req) { - ChainUpdateEdgeProcessorLocal proc(env); - return proc.reverseRequest(req); - } - - cpp2::UpdateEdgeRequest makeInvalidRequest() { - VertexID srcInvalid{"Spurssssssss"}; - auto edgeKey = defaultEdgeKey(); - edgeKey.set_src(srcInvalid); - auto updateProps = defaultUpdateProps(); - auto retProps = defaultRetProps(); - return makeRequest(edgeKey, updateProps, retProps); - } - - cpp2::UpdateEdgeRequest makeRequest(const cpp2::EdgeKey& edgeKey, - const std::vector& updateProps, - const std::vector& retCols) { - cpp2::UpdateEdgeRequest req; - auto partId = std::hash()(edgeKey.get_src().getStr()) % mockPartNum + 1; - req.set_space_id(mockSpaceId); - req.set_part_id(partId); - req.set_edge_key(edgeKey); - req.set_updated_props(updateProps); - req.set_return_props(retCols); - req.set_insertable(false); - return req; - } - - bool checkResp2(cpp2::UpdateResponse& resp) { - LOG(INFO) << "checkResp2(cpp2::UpdateResponse& resp)"; - if (!resp.props_ref()) { - LOG(INFO) << "!resp.props_ref()"; - return false; - } else { - LOG(INFO) << "resp.props_ref()"; - EXPECT_EQ(9, (*resp.props_ref()).colNames.size()); - EXPECT_EQ("_inserted", (*resp.props_ref()).colNames[0]); - EXPECT_EQ("101.playerName", (*resp.props_ref()).colNames[1]); - EXPECT_EQ("101.teamName", (*resp.props_ref()).colNames[2]); - EXPECT_EQ("101.teamCareer", (*resp.props_ref()).colNames[3]); - EXPECT_EQ("101.type", (*resp.props_ref()).colNames[4]); - EXPECT_EQ(std::string("101.").append(kSrc), (*resp.props_ref()).colNames[5]); - EXPECT_EQ(std::string("101.").append(kType), (*resp.props_ref()).colNames[6]); - EXPECT_EQ(std::string("101.").append(kRank), (*resp.props_ref()).colNames[7]); - EXPECT_EQ(std::string("101.").append(kDst), (*resp.props_ref()).colNames[8]); - EXPECT_EQ(1, (*resp.props_ref()).rows.size()); - EXPECT_EQ(9, (*resp.props_ref()).rows[0].values.size()); - EXPECT_EQ(false, (*resp.props_ref()).rows[0].values[0].getBool()); - EXPECT_EQ("Tim Duncan", (*resp.props_ref()).rows[0].values[1].getStr()); - EXPECT_EQ("Spurs", (*resp.props_ref()).rows[0].values[2].getStr()); - EXPECT_EQ(20, (*resp.props_ref()).rows[0].values[3].getInt()); - EXPECT_EQ("trade", (*resp.props_ref()).rows[0].values[4].getStr()); - EXPECT_EQ("Spurs", (*resp.props_ref()).rows[0].values[5].getStr()); - EXPECT_EQ(-101, (*resp.props_ref()).rows[0].values[6].getInt()); - EXPECT_EQ(1997, (*resp.props_ref()).rows[0].values[7].getInt()); - EXPECT_EQ("Tim Duncan", (*resp.props_ref()).rows[0].values[8].getStr()); - } - return true; - } - - bool edgeExist(StorageEnv* env, cpp2::UpdateEdgeRequest& req) { - auto partId = req.get_part_id(); - auto key = ConsistUtil::edgeKey(mockSpaceVidLen, partId, req.get_edge_key()); - return keyExist(env, mockSpaceId, partId, key); - } - - bool primeExist(StorageEnv* env, cpp2::UpdateEdgeRequest& req) { - auto partId = req.get_part_id(); - auto key = ConsistUtil::primeKey(mockSpaceVidLen, partId, req.get_edge_key()); - return keyExist(env, mockSpaceId, partId, key); - } - - bool doublePrimeExist(StorageEnv* env, cpp2::UpdateEdgeRequest& req) { - auto partId = req.get_part_id(); - auto key = ConsistUtil::doublePrime(mockSpaceVidLen, partId, req.get_edge_key()); - return keyExist(env, mockSpaceId, partId, key); - } - - bool keyExist(StorageEnv* env, - GraphSpaceID spaceId, - PartitionID partId, - const std::string& key) { - std::string val; - auto rc = env->kvstore_->get(spaceId, partId, key, &val); - return rc == Code::SUCCEEDED; - } - - bool checkRequestUpdated(StorageEnv* env, cpp2::UpdateEdgeRequest& req) { - // get serve from kvstore directly - bool ret = true; - auto& key = req.get_edge_key(); - auto partId = req.get_part_id(); - auto prefix = ConsistUtil::edgeKey(mockSpaceVidLen, partId, req.get_edge_key()); - std::unique_ptr iter; - auto rc = env->kvstore_->prefix(mockSpaceId, partId, prefix, &iter); - EXPECT_EQ(Code::SUCCEEDED, rc); - EXPECT_TRUE(iter && iter->valid()); - - auto edgeType = key.get_edge_type(); - auto edgeReader = RowReaderWrapper::getEdgePropReader( - env->schemaMan_, mockSpaceId, std::abs(edgeType), iter->val()); - - LOG(INFO) << "req.get_updated_props().size() = " << req.get_updated_props().size(); - ObjectPool objPool; - for (auto& prop : req.get_updated_props()) { - LOG(INFO) << "prop name = " << prop.get_name(); - auto enVal = prop.get_value(); - auto expression = Expression::decode(&objPool, enVal); - ConstantExpression* cexpr = static_cast(expression); - auto val1 = cexpr->value(); - auto val2 = edgeReader->getValueByName(prop.get_name()); - - // EXPECT_EQ(val1, val2); - if (val1 != val2) { - ret = false; - } - } - - return ret; - } - -public: - VertexID srcId_{"Spurs"}; - VertexID dstId_{"Tim Duncan"}; - EdgeRanking rank_{1997}; - EdgeType edgeType_{-101}; - storage::cpp2::EdgeKey updateKey_; - std::string sEdgeType; + ChainUpdateEdgeTestHelper() { sEdgeType = std::to_string(std::abs(edgeType_)); } + + cpp2::EdgeKey defaultEdgeKey() { + cpp2::EdgeKey ret; + ret.set_src(srcId_); + ret.set_edge_type(edgeType_); + ret.set_ranking(rank_); + ret.set_dst(dstId_); + return ret; + } + + std::vector defaultUpdateProps() { + ObjectPool objPool; + LOG(INFO) << "Build updated props..."; + std::vector props; + // int: 101.teamCareer = 20 + cpp2::UpdatedProp uProp1; + uProp1.set_name("teamCareer"); + // ConstantExpression val1(20); + const auto& val1 = *ConstantExpression::make(&objPool, 20); + uProp1.set_value(Expression::encode(val1)); + props.emplace_back(uProp1); + + // bool: 101.type = trade + cpp2::UpdatedProp uProp2; + uProp2.set_name("type"); + std::string colnew("trade"); + // ConstantExpression val2(colnew); + const auto& val2 = *ConstantExpression::make(&objPool, colnew); + uProp2.set_value(Expression::encode(val2)); + props.emplace_back(uProp2); + return props; + } + + std::vector defaultRetProps() { + ObjectPool objPool; + std::vector props; + std::vector cols{ + "playerName", "teamName", "teamCareer", "type", kSrc, kType, kRank, kDst}; + for (auto& colName : cols) { + const auto& exp = *EdgePropertyExpression::make(&objPool, sEdgeType, colName); + props.emplace_back(Expression::encode(exp)); + } + return props; + } + + cpp2::UpdateEdgeRequest makeDefaultRequest() { + auto edgeKey = defaultEdgeKey(); + auto updateProps = defaultUpdateProps(); + auto retProps = defaultRetProps(); + return makeRequest(edgeKey, updateProps, retProps); + } + + cpp2::UpdateEdgeRequest reverseRequest(StorageEnv* env, const cpp2::UpdateEdgeRequest& req) { + ChainUpdateEdgeProcessorLocal proc(env); + return proc.reverseRequest(req); + } + + cpp2::UpdateEdgeRequest makeInvalidRequest() { + VertexID srcInvalid{"Spurssssssss"}; + auto edgeKey = defaultEdgeKey(); + edgeKey.set_src(srcInvalid); + auto updateProps = defaultUpdateProps(); + auto retProps = defaultRetProps(); + return makeRequest(edgeKey, updateProps, retProps); + } + + cpp2::UpdateEdgeRequest makeRequest(const cpp2::EdgeKey& edgeKey, + const std::vector& updateProps, + const std::vector& retCols) { + cpp2::UpdateEdgeRequest req; + auto partId = std::hash()(edgeKey.get_src().getStr()) % mockPartNum + 1; + req.set_space_id(mockSpaceId); + req.set_part_id(partId); + req.set_edge_key(edgeKey); + req.set_updated_props(updateProps); + req.set_return_props(retCols); + req.set_insertable(false); + return req; + } + + bool checkResp2(cpp2::UpdateResponse& resp) { + LOG(INFO) << "checkResp2(cpp2::UpdateResponse& resp)"; + if (!resp.props_ref()) { + LOG(INFO) << "!resp.props_ref()"; + return false; + } else { + LOG(INFO) << "resp.props_ref()"; + EXPECT_EQ(9, (*resp.props_ref()).colNames.size()); + EXPECT_EQ("_inserted", (*resp.props_ref()).colNames[0]); + EXPECT_EQ("101.playerName", (*resp.props_ref()).colNames[1]); + EXPECT_EQ("101.teamName", (*resp.props_ref()).colNames[2]); + EXPECT_EQ("101.teamCareer", (*resp.props_ref()).colNames[3]); + EXPECT_EQ("101.type", (*resp.props_ref()).colNames[4]); + EXPECT_EQ(std::string("101.").append(kSrc), (*resp.props_ref()).colNames[5]); + EXPECT_EQ(std::string("101.").append(kType), (*resp.props_ref()).colNames[6]); + EXPECT_EQ(std::string("101.").append(kRank), (*resp.props_ref()).colNames[7]); + EXPECT_EQ(std::string("101.").append(kDst), (*resp.props_ref()).colNames[8]); + EXPECT_EQ(1, (*resp.props_ref()).rows.size()); + EXPECT_EQ(9, (*resp.props_ref()).rows[0].values.size()); + EXPECT_EQ(false, (*resp.props_ref()).rows[0].values[0].getBool()); + EXPECT_EQ("Tim Duncan", (*resp.props_ref()).rows[0].values[1].getStr()); + EXPECT_EQ("Spurs", (*resp.props_ref()).rows[0].values[2].getStr()); + EXPECT_EQ(20, (*resp.props_ref()).rows[0].values[3].getInt()); + EXPECT_EQ("trade", (*resp.props_ref()).rows[0].values[4].getStr()); + EXPECT_EQ("Spurs", (*resp.props_ref()).rows[0].values[5].getStr()); + EXPECT_EQ(-101, (*resp.props_ref()).rows[0].values[6].getInt()); + EXPECT_EQ(1997, (*resp.props_ref()).rows[0].values[7].getInt()); + EXPECT_EQ("Tim Duncan", (*resp.props_ref()).rows[0].values[8].getStr()); + } + return true; + } + + bool edgeExist(StorageEnv* env, cpp2::UpdateEdgeRequest& req) { + auto partId = req.get_part_id(); + auto key = ConsistUtil::edgeKey(mockSpaceVidLen, partId, req.get_edge_key()); + return keyExist(env, mockSpaceId, partId, key); + } + + bool primeExist(StorageEnv* env, cpp2::UpdateEdgeRequest& req) { + auto partId = req.get_part_id(); + auto key = ConsistUtil::primeKey(mockSpaceVidLen, partId, req.get_edge_key()); + return keyExist(env, mockSpaceId, partId, key); + } + + bool doublePrimeExist(StorageEnv* env, cpp2::UpdateEdgeRequest& req) { + auto partId = req.get_part_id(); + auto key = ConsistUtil::doublePrime(mockSpaceVidLen, partId, req.get_edge_key()); + return keyExist(env, mockSpaceId, partId, key); + } + + bool keyExist(StorageEnv* env, GraphSpaceID spaceId, PartitionID partId, const std::string& key) { + std::string val; + auto rc = env->kvstore_->get(spaceId, partId, key, &val); + return rc == Code::SUCCEEDED; + } + + bool checkRequestUpdated(StorageEnv* env, cpp2::UpdateEdgeRequest& req) { + // get serve from kvstore directly + bool ret = true; + auto& key = req.get_edge_key(); + auto partId = req.get_part_id(); + auto prefix = ConsistUtil::edgeKey(mockSpaceVidLen, partId, req.get_edge_key()); + std::unique_ptr iter; + auto rc = env->kvstore_->prefix(mockSpaceId, partId, prefix, &iter); + EXPECT_EQ(Code::SUCCEEDED, rc); + EXPECT_TRUE(iter && iter->valid()); + + auto edgeType = key.get_edge_type(); + auto edgeReader = RowReaderWrapper::getEdgePropReader( + env->schemaMan_, mockSpaceId, std::abs(edgeType), iter->val()); + + LOG(INFO) << "req.get_updated_props().size() = " << req.get_updated_props().size(); + ObjectPool objPool; + for (auto& prop : req.get_updated_props()) { + LOG(INFO) << "prop name = " << prop.get_name(); + auto enVal = prop.get_value(); + auto expression = Expression::decode(&objPool, enVal); + ConstantExpression* cexpr = static_cast(expression); + auto val1 = cexpr->value(); + auto val2 = edgeReader->getValueByName(prop.get_name()); + + // EXPECT_EQ(val1, val2); + if (val1 != val2) { + ret = false; + } + } + + return ret; + } + + public: + VertexID srcId_{"Spurs"}; + VertexID dstId_{"Tim Duncan"}; + EdgeRanking rank_{1997}; + EdgeType edgeType_{-101}; + storage::cpp2::EdgeKey updateKey_; + std::string sEdgeType; }; // class ChainResumeProcessorTestHelper { @@ -496,6 +490,5 @@ struct ChainUpdateEdgeTestHelper { // ChainResumeProcessor* proc_{nullptr}; // }; - } // namespace storage } // namespace nebula diff --git a/src/storage/test/ChainUpdateEdgeTest.cpp b/src/storage/test/ChainUpdateEdgeTest.cpp index 37f011e3c71..0b5c96f4e3f 100644 --- a/src/storage/test/ChainUpdateEdgeTest.cpp +++ b/src/storage/test/ChainUpdateEdgeTest.cpp @@ -1,32 +1,30 @@ - /* Copyright (c) 2021 vesoft inc. All rights reserved. +/* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ -#include #include #include +#include #include #include -#include +#include #include #include "common/fs/TempDir.h" - -#include "storage/CommonUtils.h" -#include "storage/mutate/UpdateEdgeProcessor.h" -#include "storage/transaction/ConsistUtil.h" -#include "storage/transaction/ChainUpdateEdgeProcessorRemote.h" -#include "storage/transaction/ChainAddEdgesProcessorLocal.h" -#include "storage/transaction/ChainAddEdgesGroupProcessor.h" -#include "storage/transaction/ChainResumeProcessor.h" - #include "mock/MockCluster.h" #include "mock/MockData.h" +#include "storage/CommonUtils.h" +#include "storage/mutate/UpdateEdgeProcessor.h" +#include "storage/test/ChainTestUtils.h" #include "storage/test/QueryTestUtils.h" #include "storage/test/TestUtils.h" -#include "storage/test/ChainTestUtils.h" +#include "storage/transaction/ChainAddEdgesGroupProcessor.h" +#include "storage/transaction/ChainAddEdgesProcessorLocal.h" +#include "storage/transaction/ChainResumeProcessor.h" +#include "storage/transaction/ChainUpdateEdgeProcessorRemote.h" +#include "storage/transaction/ConsistUtil.h" namespace nebula { namespace storage { @@ -41,139 +39,135 @@ ChainTestUtils gTestUtil; ChainUpdateEdgeTestHelper helper; TEST(ChainUpdateEdgeTest, updateTest1) { - fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - env->metaClient_ = mClient.get(); - - auto parts = cluster.getTotalParts(); - EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); - - LOG(INFO) << "Test updateTest1..."; - auto req = helper.makeDefaultRequest(); - - env->txnMan_->iClient_ = FakeInternalStorageClient::instance(env); - auto reversedRequest = helper.reverseRequest(env, req); - - auto* proc = new FakeChainUpdateProcessor(env); - LOG(INFO) << "proc: " << proc; - auto f = proc->getFuture(); - proc->process(req); - auto resp = std::move(f).get(); - - EXPECT_TRUE(helper.checkResp2(resp)); - EXPECT_TRUE(helper.checkRequestUpdated(env, req)); - EXPECT_TRUE(helper.checkRequestUpdated(env, reversedRequest)); - EXPECT_TRUE(helper.edgeExist(env, req)); - EXPECT_FALSE(helper.primeExist(env, req)); - EXPECT_FALSE(helper.doublePrimeExist(env, req)); + fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + env->metaClient_ = mClient.get(); + + auto parts = cluster.getTotalParts(); + EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); + + LOG(INFO) << "Test updateTest1..."; + auto req = helper.makeDefaultRequest(); + + env->txnMan_->iClient_ = FakeInternalStorageClient::instance(env); + auto reversedRequest = helper.reverseRequest(env, req); + + auto* proc = new FakeChainUpdateProcessor(env); + LOG(INFO) << "proc: " << proc; + auto f = proc->getFuture(); + proc->process(req); + auto resp = std::move(f).get(); + + EXPECT_TRUE(helper.checkResp2(resp)); + EXPECT_TRUE(helper.checkRequestUpdated(env, req)); + EXPECT_TRUE(helper.checkRequestUpdated(env, reversedRequest)); + EXPECT_TRUE(helper.edgeExist(env, req)); + EXPECT_FALSE(helper.primeExist(env, req)); + EXPECT_FALSE(helper.doublePrimeExist(env, req)); } TEST(ChainUpdateEdgeTest, updateTest2) { - fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - env->metaClient_ = mClient.get(); - - auto parts = cluster.getTotalParts(); - EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); - - LOG(INFO) << "Test UpdateEdgeRequest..."; - auto goodRequest = helper.makeDefaultRequest(); - EXPECT_TRUE(helper.edgeExist(env, goodRequest)); - EXPECT_FALSE(helper.primeExist(env, goodRequest)); - EXPECT_FALSE(helper.doublePrimeExist(env, goodRequest)); - - auto badRequest = helper.makeInvalidRequest(); - - auto* proc = new FakeChainUpdateProcessor(env); - auto f = proc->getFuture(); - proc->rcProcessRemote = Code::E_KEY_NOT_FOUND; - proc->process(badRequest); - auto resp = std::move(f).get(); - - EXPECT_EQ(1, (*resp.result_ref()).failed_parts.size()); - EXPECT_FALSE(helper.checkResp2(resp)); - EXPECT_FALSE(helper.edgeExist(env, badRequest)); - EXPECT_FALSE(helper.primeExist(env, badRequest)); - EXPECT_FALSE(helper.doublePrimeExist(env, badRequest)); + fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + env->metaClient_ = mClient.get(); + + auto parts = cluster.getTotalParts(); + EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); + + LOG(INFO) << "Test UpdateEdgeRequest..."; + auto goodRequest = helper.makeDefaultRequest(); + EXPECT_TRUE(helper.edgeExist(env, goodRequest)); + EXPECT_FALSE(helper.primeExist(env, goodRequest)); + EXPECT_FALSE(helper.doublePrimeExist(env, goodRequest)); + + auto badRequest = helper.makeInvalidRequest(); + + auto* proc = new FakeChainUpdateProcessor(env); + auto f = proc->getFuture(); + proc->rcProcessRemote = Code::E_KEY_NOT_FOUND; + proc->process(badRequest); + auto resp = std::move(f).get(); + + EXPECT_EQ(1, (*resp.result_ref()).failed_parts.size()); + EXPECT_FALSE(helper.checkResp2(resp)); + EXPECT_FALSE(helper.edgeExist(env, badRequest)); + EXPECT_FALSE(helper.primeExist(env, badRequest)); + EXPECT_FALSE(helper.doublePrimeExist(env, badRequest)); } TEST(ChainUpdateEdgeTest, updateTest3) { - fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - env->metaClient_ = mClient.get(); - - auto parts = cluster.getTotalParts(); - EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); - - LOG(INFO) << "Test UpdateEdgeRequest..."; - auto goodRequest = helper.makeDefaultRequest(); - EXPECT_TRUE(helper.edgeExist(env, goodRequest)); - EXPECT_FALSE(helper.primeExist(env, goodRequest)); - EXPECT_FALSE(helper.doublePrimeExist(env, goodRequest)); - - auto* proc = new FakeChainUpdateProcessor(env); - auto f = proc->getFuture(); - proc->rcProcessRemote = Code::SUCCEEDED; - proc->rcProcessLocal = Code::SUCCEEDED; - proc->process(goodRequest); - auto resp = std::move(f).get(); - - EXPECT_TRUE(helper.edgeExist(env, goodRequest)); - EXPECT_TRUE(helper.primeExist(env, goodRequest)); - EXPECT_FALSE(helper.doublePrimeExist(env, goodRequest)); + fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + env->metaClient_ = mClient.get(); + + auto parts = cluster.getTotalParts(); + EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); + + LOG(INFO) << "Test UpdateEdgeRequest..."; + auto goodRequest = helper.makeDefaultRequest(); + EXPECT_TRUE(helper.edgeExist(env, goodRequest)); + EXPECT_FALSE(helper.primeExist(env, goodRequest)); + EXPECT_FALSE(helper.doublePrimeExist(env, goodRequest)); + + auto* proc = new FakeChainUpdateProcessor(env); + auto f = proc->getFuture(); + proc->rcProcessRemote = Code::SUCCEEDED; + proc->rcProcessLocal = Code::SUCCEEDED; + proc->process(goodRequest); + auto resp = std::move(f).get(); + + EXPECT_TRUE(helper.edgeExist(env, goodRequest)); + EXPECT_TRUE(helper.primeExist(env, goodRequest)); + EXPECT_FALSE(helper.doublePrimeExist(env, goodRequest)); } TEST(ChainUpdateEdgeTest, updateTest4) { - fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); - mock::MockCluster cluster; - cluster.initStorageKV(rootPath.path()); - auto* env = cluster.storageEnv_.get(); - auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); - env->metaClient_ = mClient.get(); - - auto parts = cluster.getTotalParts(); - EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); - - LOG(INFO) << "Test UpdateEdgeRequest..."; - auto goodRequest = helper.makeDefaultRequest(); - EXPECT_TRUE(helper.edgeExist(env, goodRequest)); - EXPECT_FALSE(helper.primeExist(env, goodRequest)); - EXPECT_FALSE(helper.doublePrimeExist(env, goodRequest)); - - auto* proc = new FakeChainUpdateProcessor(env); - auto f = proc->getFuture(); - proc->rcProcessRemote = Code::E_RPC_FAILURE; - proc->process(goodRequest); - auto resp = std::move(f).get(); - - EXPECT_TRUE(helper.edgeExist(env, goodRequest)); - EXPECT_FALSE(helper.primeExist(env, goodRequest)); - EXPECT_TRUE(helper.doublePrimeExist(env, goodRequest)); + fs::TempDir rootPath("/tmp/UpdateEdgeTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto mClient = MetaClientTestUpdater::makeDefaultMetaClient(); + env->metaClient_ = mClient.get(); + + auto parts = cluster.getTotalParts(); + EXPECT_TRUE(QueryTestUtils::mockEdgeData(env, parts, mockSpaceVidLen)); + + LOG(INFO) << "Test UpdateEdgeRequest..."; + auto goodRequest = helper.makeDefaultRequest(); + EXPECT_TRUE(helper.edgeExist(env, goodRequest)); + EXPECT_FALSE(helper.primeExist(env, goodRequest)); + EXPECT_FALSE(helper.doublePrimeExist(env, goodRequest)); + + auto* proc = new FakeChainUpdateProcessor(env); + auto f = proc->getFuture(); + proc->rcProcessRemote = Code::E_RPC_FAILURE; + proc->process(goodRequest); + auto resp = std::move(f).get(); + + EXPECT_TRUE(helper.edgeExist(env, goodRequest)); + EXPECT_FALSE(helper.primeExist(env, goodRequest)); + EXPECT_TRUE(helper.doublePrimeExist(env, goodRequest)); } - - } // namespace storage } // namespace nebula int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - folly::init(&argc, &argv, false); - google::SetStderrLogging(google::INFO); - return RUN_ALL_TESTS(); + testing::InitGoogleTest(&argc, argv); + folly::init(&argc, &argv, false); + google::SetStderrLogging(google::INFO); + return RUN_ALL_TESTS(); } - - // ***** Test Plan ***** /** * @brief updateTest1 (update a normal edge will succeed) diff --git a/src/storage/test/ConsistUtilTest.cpp b/src/storage/test/ConsistUtilTest.cpp index 8361323b570..ea9544e9ada 100644 --- a/src/storage/test/ConsistUtilTest.cpp +++ b/src/storage/test/ConsistUtilTest.cpp @@ -1,34 +1,31 @@ - /* Copyright (c) 2021 vesoft inc. All rights reserved. +/* Copyright (c) 2021 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License, * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ -#include #include #include #include #include -#include "folly/String.h" +#include +#include "folly/String.h" #include "storage/transaction/ConsistUtil.h" - #define LOG_FMT(...) LOG(INFO) << folly::sformat(__VA_ARGS__) namespace nebula { namespace storage { -TEST(ConsistUtilTest, primeTable) { -} - +TEST(ConsistUtilTest, primeTable) {} } // namespace storage } // namespace nebula int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - folly::init(&argc, &argv, false); - google::SetStderrLogging(google::INFO); - return RUN_ALL_TESTS(); + testing::InitGoogleTest(&argc, argv); + folly::init(&argc, &argv, false); + google::SetStderrLogging(google::INFO); + return RUN_ALL_TESTS(); } diff --git a/src/storage/test/TossTestExecutor.h b/src/storage/test/TossTestExecutor.h index b06234db4d5..3bea706c0d7 100644 --- a/src/storage/test/TossTestExecutor.h +++ b/src/storage/test/TossTestExecutor.h @@ -15,377 +15,365 @@ using StorageClient = storage::GraphStorageClient; template class StorageResponseReader { -public: - explicit StorageResponseReader(StorageRpcResponse& resp) : resp_(&resp) {} - - bool isLeaderChange() { - auto& c = resp_->failedParts(); - return std::any_of(c.begin(), c.end(), [](auto it){ - return it.second == cpp2::ErrorCode::E_LEADER_CHANGED; - }); + public: + explicit StorageResponseReader(StorageRpcResponse& resp) : resp_(&resp) {} + + bool isLeaderChange() { + auto& c = resp_->failedParts(); + return std::any_of( + c.begin(), c.end(), [](auto it) { return it.second == cpp2::ErrorCode::E_LEADER_CHANGED; }); + } + + std::vector& data() { return resp_->responses(); } + + cpp2::ErrorCode firstErrCode() { + for (auto& p : resp_->failedParts()) { + if (p.second != cpp2::ErrorCode::SUCCEEDED) { + return p.second; + } } + return cpp2::ErrorCode::SUCCEEDED; + } - std::vector& data() { - return resp_->responses(); - } - - cpp2::ErrorCode firstErrCode() { - for (auto& p : resp_->failedParts()) { - if (p.second != cpp2::ErrorCode::SUCCEEDED) { - return p.second; - } - } - return cpp2::ErrorCode::SUCCEEDED; - } - -private: - StorageRpcResponse* resp_; + private: + StorageRpcResponse* resp_; }; class GetNeighborsExecutor { - TestSpace* s_; - std::vector data_; - -public: - GetNeighborsExecutor(TestSpace* s, cpp2::NewEdge edge) - : GetNeighborsExecutor(s, std::vector{edge}) { - } - - GetNeighborsExecutor(TestSpace* s, - std::vector edges, - int64_t limit = std::numeric_limits::max()) - : s_(s), edges_(std::move(edges)), limit_(limit) { - env_ = TossEnvironment::getInstance(); - auto uniEdges = uniqueEdges(edges_); - data_ = run(uniEdges); - } - - std::vector data() { - return data_; - } - - std::vector uniqueEdges(const std::vector& __edges) { - std::vector edges(__edges); - std::sort(edges.begin(), edges.end(), [](const auto& a, const auto& b) { - if (a.key.src == b.key.src) { - return a.key.dst < b.key.dst; - } - return a.key.src < b.key.src; - }); - auto last = std::unique(edges.begin(), edges.end(), [](const auto& a, const auto& b) { - return a.key.src == b.key.src && a.key.dst == b.key.dst; - }); - edges.erase(last, edges.end()); - return edges; - } - - /** - * @brief Get the Nei Props object, - * will unique same src & dst input edges. - */ - std::vector run(const std::vector& edges) { - bool retLeaderChange = false; - LOG(INFO) << "edges.size()=" << edges.size() << ", edges.size()=" << edges.size(); - do { - auto f = rpc(edges); - f.wait(); - if (!f.valid()) { - LOG(ERROR) << "!f.valid()"; - break; - } - if (f.value().succeeded()) { - return readStorageResp(f.value()); - } else { - LOG(ERROR) << "!f.value().succeeded()"; - } - auto parts = f.value().failedParts(); - for (auto& part : parts) { - if (part.second == cpp2::ErrorCode::E_LEADER_CHANGED) { - retLeaderChange = true; - break; - } - } - } while (retLeaderChange); - - LOG(ERROR) << "getOutNeighborsProps failed"; - std::vector ret; - return ret; - } - - folly::SemiFuture> rpc( - const std::vector& edges) { - // para3 - std::vector vertices; - std::set vids; - for (auto& e : edges) { - // vids.insert(e.key.src); - vids.insert(e.key.dst); - } - for (auto& vid : vids) { - Row row; - row.emplace_back(vid); - vertices.emplace_back(row); + TestSpace* s_; + std::vector data_; + + public: + GetNeighborsExecutor(TestSpace* s, cpp2::NewEdge edge) + : GetNeighborsExecutor(s, std::vector{edge}) {} + + GetNeighborsExecutor(TestSpace* s, + std::vector edges, + int64_t limit = std::numeric_limits::max()) + : s_(s), edges_(std::move(edges)), limit_(limit) { + env_ = TossEnvironment::getInstance(); + auto uniEdges = uniqueEdges(edges_); + data_ = run(uniEdges); + } + + std::vector data() { return data_; } + + std::vector uniqueEdges(const std::vector& __edges) { + std::vector edges(__edges); + std::sort(edges.begin(), edges.end(), [](const auto& a, const auto& b) { + if (a.key.src == b.key.src) { + return a.key.dst < b.key.dst; + } + return a.key.src < b.key.src; + }); + auto last = std::unique(edges.begin(), edges.end(), [](const auto& a, const auto& b) { + return a.key.src == b.key.src && a.key.dst == b.key.dst; + }); + edges.erase(last, edges.end()); + return edges; + } + + /** + * @brief Get the Nei Props object, + * will unique same src & dst input edges. + */ + std::vector run(const std::vector& edges) { + bool retLeaderChange = false; + LOG(INFO) << "edges.size()=" << edges.size() << ", edges.size()=" << edges.size(); + do { + auto f = rpc(edges); + f.wait(); + if (!f.valid()) { + LOG(ERROR) << "!f.valid()"; + break; + } + if (f.value().succeeded()) { + return readStorageResp(f.value()); + } else { + LOG(ERROR) << "!f.value().succeeded()"; + } + auto parts = f.value().failedParts(); + for (auto& part : parts) { + if (part.second == cpp2::ErrorCode::E_LEADER_CHANGED) { + retLeaderChange = true; + break; } - // para 4 - std::vector edgeTypes; - // para 5 - cpp2::EdgeDirection edgeDirection = cpp2::EdgeDirection::BOTH; - // para 6 - std::vector* statProps = nullptr; - // para 7 - std::vector* vertexProps = nullptr; - // para 8 - // const std::vector edgeProps(1); - std::vector edgeProps(1); - edgeProps.back().type = 0 - edges[0].key.edge_type; - // para 9 - const std::vector* expressions = nullptr; - // para 10 - bool dedup = false; - // para 11 - bool random = false; - // para 12 - const std::vector orderBy = std::vector(); - - auto colNames = TossTestUtils::makeColNames(edges.back().props.size()); - - return s_->sClient_->getNeighbors(s_->getSpaceId(), - colNames, - vertices, - edgeTypes, - edgeDirection, - statProps, - vertexProps, - &edgeProps, - expressions, - dedup, - random, - orderBy, - limit_); + } + } while (retLeaderChange); + + LOG(ERROR) << "getOutNeighborsProps failed"; + std::vector ret; + return ret; + } + + folly::SemiFuture> rpc( + const std::vector& edges) { + // para3 + std::vector vertices; + std::set vids; + for (auto& e : edges) { + // vids.insert(e.key.src); + vids.insert(e.key.dst); } - - std::vector readStorageResp(StorageRpcResponse& rpc) { - std::vector ret; - LOG(INFO) << "rpc.responses().size()=" << rpc.responses().size(); - for (auto& resp : rpc.responses()) { - auto sub = readGetNeighborsResp(resp); - ret.insert(ret.end(), sub.begin(), sub.end()); - } - return ret; + for (auto& vid : vids) { + Row row; + row.emplace_back(vid); + vertices.emplace_back(row); } - - std::vector readGetNeighborsResp(cpp2::GetNeighborsResponse& resp) { - std::vector ret; - auto& ds = resp.vertices; - LOG(INFO) << "ds.rows.size()=" << ds.rows.size(); - for (auto& row : ds.rows) { - LOG(INFO) << "row.values.size()=" << row.values.size(); - for (auto& val : row.values) { - LOG(INFO) << "row.val = " << val.toString(); - } - ret.emplace_back(row.values[3].toString()); - } - return ret; + // para 4 + std::vector edgeTypes; + // para 5 + cpp2::EdgeDirection edgeDirection = cpp2::EdgeDirection::BOTH; + // para 6 + std::vector* statProps = nullptr; + // para 7 + std::vector* vertexProps = nullptr; + // para 8 + // const std::vector edgeProps(1); + std::vector edgeProps(1); + edgeProps.back().type = 0 - edges[0].key.edge_type; + // para 9 + const std::vector* expressions = nullptr; + // para 10 + bool dedup = false; + // para 11 + bool random = false; + // para 12 + const std::vector orderBy = std::vector(); + + auto colNames = TossTestUtils::makeColNames(edges.back().props.size()); + + return s_->sClient_->getNeighbors(s_->getSpaceId(), + colNames, + vertices, + edgeTypes, + edgeDirection, + statProps, + vertexProps, + &edgeProps, + expressions, + dedup, + random, + orderBy, + limit_); + } + + std::vector readStorageResp(StorageRpcResponse& rpc) { + std::vector ret; + LOG(INFO) << "rpc.responses().size()=" << rpc.responses().size(); + for (auto& resp : rpc.responses()) { + auto sub = readGetNeighborsResp(resp); + ret.insert(ret.end(), sub.begin(), sub.end()); } + return ret; + } + + std::vector readGetNeighborsResp(cpp2::GetNeighborsResponse& resp) { + std::vector ret; + auto& ds = resp.vertices; + LOG(INFO) << "ds.rows.size()=" << ds.rows.size(); + for (auto& row : ds.rows) { + LOG(INFO) << "row.values.size()=" << row.values.size(); + for (auto& val : row.values) { + LOG(INFO) << "row.val = " << val.toString(); + } + ret.emplace_back(row.values[3].toString()); + } + return ret; + } -private: - std::vector edges_; - int64_t limit_{0}; - TossEnvironment* env_; + private: + std::vector edges_; + int64_t limit_{0}; + TossEnvironment* env_; }; struct AddEdgeExecutor { - TestSpace* s_; - std::vector edges_; - cpp2::ErrorCode code_; - -public: - AddEdgeExecutor(TestSpace* s, cpp2::NewEdge edge) : s_(s) { - appendEdge(std::move(edge)); - rpc(); - } - - AddEdgeExecutor(TestSpace* s, std::vector edges) : s_(s) { - for (auto& eg : edges) { - appendEdge(std::move(eg)); - } - rpc(); - } - - void appendEdge(cpp2::NewEdge&& edge) { - CHECK_GT(edge.key.edge_type, 0); - edges_.emplace_back(std::move(edge)); - edges_.back().key.edge_type = 0 - edges_.back().key.edge_type; - std::swap(edges_.back().key.src, edges_.back().key.dst); - } - - void rpc() { - // LOG(INFO) << "AddEdgeExecutor::rpc(), spaceId=" << s_->getSpaceId(); - auto propNames = TossTestUtils::makeColNames(edges_.back().props.size()); - bool overwritable = true; - folly::EventBase* evb = nullptr; - bool useToss = s_->useToss_; - int retry = 10; - do { - // LOG(INFO) << "AddEdgeExecutor::rpc(), do retry = " << retry; - auto f = s_->sClient_->addEdges( - s_->getSpaceId(), edges_, propNames, overwritable, evb, useToss); - f.wait(); - CHECK(f.hasValue()); - StorageResponseReader reader(f.value()); - if (reader.isLeaderChange()) { - continue; - } - code_ = reader.firstErrCode(); - break; - } while (retry-- > 0); - } - - bool ok() { - return code_ == cpp2::ErrorCode::SUCCEEDED; - } - - cpp2::ErrorCode code() { - return code_; + TestSpace* s_; + std::vector edges_; + cpp2::ErrorCode code_; + + public: + AddEdgeExecutor(TestSpace* s, cpp2::NewEdge edge) : s_(s) { + appendEdge(std::move(edge)); + rpc(); + } + + AddEdgeExecutor(TestSpace* s, std::vector edges) : s_(s) { + for (auto& eg : edges) { + appendEdge(std::move(eg)); } + rpc(); + } + + void appendEdge(cpp2::NewEdge&& edge) { + CHECK_GT(edge.key.edge_type, 0); + edges_.emplace_back(std::move(edge)); + edges_.back().key.edge_type = 0 - edges_.back().key.edge_type; + std::swap(edges_.back().key.src, edges_.back().key.dst); + } + + void rpc() { + // LOG(INFO) << "AddEdgeExecutor::rpc(), spaceId=" << s_->getSpaceId(); + auto propNames = TossTestUtils::makeColNames(edges_.back().props.size()); + bool overwritable = true; + folly::EventBase* evb = nullptr; + bool useToss = s_->useToss_; + int retry = 10; + do { + // LOG(INFO) << "AddEdgeExecutor::rpc(), do retry = " << retry; + auto f = + s_->sClient_->addEdges(s_->getSpaceId(), edges_, propNames, overwritable, evb, useToss); + f.wait(); + CHECK(f.hasValue()); + StorageResponseReader reader(f.value()); + if (reader.isLeaderChange()) { + continue; + } + code_ = reader.firstErrCode(); + break; + } while (retry-- > 0); + } + + bool ok() { return code_ == cpp2::ErrorCode::SUCCEEDED; } + + cpp2::ErrorCode code() { return code_; } }; struct GetPropsExecutor { - TossEnvironment* env_; - TestSpace* s_; - cpp2::NewEdge edge_; - std::vector result_; - -public: - GetPropsExecutor(TestSpace* s, cpp2::NewEdge edge) : s_(s), edge_(std::move(edge)) { - env_ = TossEnvironment::getInstance(); - CHECK_GT(edge_.key.edge_type, 0); - std::swap(edge_.key.src, edge_.key.dst); - edge_.key.edge_type = 0 - edge_.key.edge_type; - rpc(); - } - - GetPropsExecutor(TestSpace* s, std::vector edges) - : GetPropsExecutor(s, edges[0]) {} - - std::vector data() { - return result_; - } - - void rpc() { - nebula::Row row; - row.values.emplace_back(edge_.key.src); - row.values.emplace_back(edge_.key.edge_type); - row.values.emplace_back(edge_.key.ranking); - row.values.emplace_back(edge_.key.dst); - - nebula::DataSet ds; - ds.rows.emplace_back(std::move(row)); - - std::vector props(1); - props.back().type = edge_.key.edge_type; - - int retry = 10; - - do { - LOG(INFO) << "enter do{} while"; - auto frpc = s_->sClient_ - ->getProps(s_->getSpaceId(), - ds, /*DataSet*/ - nullptr, /*vector*/ - &props, /*vector*/ - nullptr /*expressions*/) - .via(env_->executor_.get()); - frpc.wait(); - CHECK(frpc.hasValue()); - StorageResponseReader reader(frpc.value()); - if (reader.isLeaderChange()) { - continue; - } - - auto resps = reader.data(); - CHECK_EQ(resps.size(), 1); - cpp2::GetPropResponse& propResp = resps.front(); - cpp2::ResponseCommon result = propResp.get_result(); - nebula::DataSet& dataSet = propResp.props; - std::vector& rows = dataSet.rows; - if (rows.empty()) { - LOG(FATAL) << "getProps() dataSet.rows.empty())"; - } - LOG(INFO) << "rows.size() = " << rows.size(); - for (auto& r : rows) { - LOG(INFO) << "values.size() = " << r.values.size(); - for (auto& val : r.values) { - LOG(INFO) << "val: " << val.toString(); - } - } - result_ = rows[0].values; - if (result_.empty()) { - LOG(FATAL) << "getProps() ret.empty())"; - } - break; - } while (retry-- > 0); - } + TossEnvironment* env_; + TestSpace* s_; + cpp2::NewEdge edge_; + std::vector result_; + + public: + GetPropsExecutor(TestSpace* s, cpp2::NewEdge edge) : s_(s), edge_(std::move(edge)) { + env_ = TossEnvironment::getInstance(); + CHECK_GT(edge_.key.edge_type, 0); + std::swap(edge_.key.src, edge_.key.dst); + edge_.key.edge_type = 0 - edge_.key.edge_type; + rpc(); + } + + GetPropsExecutor(TestSpace* s, std::vector edges) + : GetPropsExecutor(s, edges[0]) {} + + std::vector data() { return result_; } + + void rpc() { + nebula::Row row; + row.values.emplace_back(edge_.key.src); + row.values.emplace_back(edge_.key.edge_type); + row.values.emplace_back(edge_.key.ranking); + row.values.emplace_back(edge_.key.dst); + + nebula::DataSet ds; + ds.rows.emplace_back(std::move(row)); + + std::vector props(1); + props.back().type = edge_.key.edge_type; + + int retry = 10; + + do { + LOG(INFO) << "enter do{} while"; + auto frpc = s_->sClient_ + ->getProps(s_->getSpaceId(), + ds, /*DataSet*/ + nullptr, /*vector*/ + &props, /*vector*/ + nullptr /*expressions*/) + .via(env_->executor_.get()); + frpc.wait(); + CHECK(frpc.hasValue()); + StorageResponseReader reader(frpc.value()); + if (reader.isLeaderChange()) { + continue; + } + + auto resps = reader.data(); + CHECK_EQ(resps.size(), 1); + cpp2::GetPropResponse& propResp = resps.front(); + cpp2::ResponseCommon result = propResp.get_result(); + nebula::DataSet& dataSet = propResp.props; + std::vector& rows = dataSet.rows; + if (rows.empty()) { + LOG(FATAL) << "getProps() dataSet.rows.empty())"; + } + LOG(INFO) << "rows.size() = " << rows.size(); + for (auto& r : rows) { + LOG(INFO) << "values.size() = " << r.values.size(); + for (auto& val : r.values) { + LOG(INFO) << "val: " << val.toString(); + } + } + result_ = rows[0].values; + if (result_.empty()) { + LOG(FATAL) << "getProps() ret.empty())"; + } + break; + } while (retry-- > 0); + } }; struct UpdateExecutor { - TossEnvironment* env_; - TestSpace* s_; - cpp2::NewEdge edge_; - std::vector updatedProps_; - bool insertable_{false}; - std::vector returnProps_; - std::string condition_; - -public: - UpdateExecutor(TestSpace* s, cpp2::NewEdge edge) : s_(s), edge_(std::move(edge)) { - env_ = TossEnvironment::getInstance(); - CHECK_GT(edge_.key.edge_type, 0); - std::swap(edge_.key.src, edge_.key.dst); - edge_.key.edge_type = 0 - edge_.key.edge_type; - prepareParameters(); - rpc(); - } - - void prepareParameters() { - cpp2::UpdatedProp uProp1; - uProp1.set_name("c1"); - ConstantExpression val1(edge_.props[0].getInt()); - uProp1.set_value(Expression::encode(val1)); - updatedProps_.emplace_back(uProp1); - - cpp2::UpdatedProp uProp2; - uProp2.set_name("c2"); - ConstantExpression val2(edge_.props[1].getStr()); - uProp2.set_value(Expression::encode(val2)); - - updatedProps_.emplace_back(uProp2); - } - - void rpc() { - for (;;) { - auto f = s_->sClient_->updateEdge( - s_->getSpaceId(), edge_.key, updatedProps_, insertable_, returnProps_, condition_); - f.wait(); - CHECK(f.hasValue()); - if (!f.value().ok()) { - LOG(FATAL) << f.value().status().toString(); - } - storage::cpp2::UpdateResponse resp = f.value().value(); - if (resp.result.get_failed_parts().empty()) { - break; - } - - CHECK_EQ(resp.result.get_failed_parts().size(), 1U); - cpp2::PartitionResult result = resp.result.get_failed_parts().back(); - if (result.code == cpp2::ErrorCode::E_LEADER_CHANGED) { - LOG(INFO) << "update edge leader changed, retry"; - continue; - } - LOG(ERROR) << "update edge err: " << cpp2::_ErrorCode_VALUES_TO_NAMES.at(result.code); - break; - } + TossEnvironment* env_; + TestSpace* s_; + cpp2::NewEdge edge_; + std::vector updatedProps_; + bool insertable_{false}; + std::vector returnProps_; + std::string condition_; + + public: + UpdateExecutor(TestSpace* s, cpp2::NewEdge edge) : s_(s), edge_(std::move(edge)) { + env_ = TossEnvironment::getInstance(); + CHECK_GT(edge_.key.edge_type, 0); + std::swap(edge_.key.src, edge_.key.dst); + edge_.key.edge_type = 0 - edge_.key.edge_type; + prepareParameters(); + rpc(); + } + + void prepareParameters() { + cpp2::UpdatedProp uProp1; + uProp1.set_name("c1"); + ConstantExpression val1(edge_.props[0].getInt()); + uProp1.set_value(Expression::encode(val1)); + updatedProps_.emplace_back(uProp1); + + cpp2::UpdatedProp uProp2; + uProp2.set_name("c2"); + ConstantExpression val2(edge_.props[1].getStr()); + uProp2.set_value(Expression::encode(val2)); + + updatedProps_.emplace_back(uProp2); + } + + void rpc() { + for (;;) { + auto f = s_->sClient_->updateEdge( + s_->getSpaceId(), edge_.key, updatedProps_, insertable_, returnProps_, condition_); + f.wait(); + CHECK(f.hasValue()); + if (!f.value().ok()) { + LOG(FATAL) << f.value().status().toString(); + } + storage::cpp2::UpdateResponse resp = f.value().value(); + if (resp.result.get_failed_parts().empty()) { + break; + } + + CHECK_EQ(resp.result.get_failed_parts().size(), 1U); + cpp2::PartitionResult result = resp.result.get_failed_parts().back(); + if (result.code == cpp2::ErrorCode::E_LEADER_CHANGED) { + LOG(INFO) << "update edge leader changed, retry"; + continue; + } + LOG(ERROR) << "update edge err: " << cpp2::_ErrorCode_VALUES_TO_NAMES.at(result.code); + break; } + } }; } // namespace storage diff --git a/src/storage/transaction/ChainAddEdgesGroupProcessor.cpp b/src/storage/transaction/ChainAddEdgesGroupProcessor.cpp index d813461ac73..75bc48286dc 100644 --- a/src/storage/transaction/ChainAddEdgesGroupProcessor.cpp +++ b/src/storage/transaction/ChainAddEdgesGroupProcessor.cpp @@ -5,76 +5,73 @@ */ #include "storage/transaction/ChainAddEdgesGroupProcessor.h" + +#include "storage/StorageFlags.h" +#include "storage/mutate/AddEdgesProcessor.h" #include "storage/transaction/ChainAddEdgesProcessorLocal.h" #include "storage/transaction/ConsistUtil.h" #include "storage/transaction/TransactionManager.h" -#include "storage/mutate/AddEdgesProcessor.h" -#include "storage/StorageFlags.h" namespace nebula { namespace storage { - void ChainAddEdgesGroupProcessor::process(const cpp2::AddEdgesRequest& req) { - auto space = req.get_space_id(); - ShuffledReq shuffledReq; - shuffleRequest(req, shuffledReq); + auto space = req.get_space_id(); + ShuffledReq shuffledReq; + shuffleRequest(req, shuffledReq); - auto delegateProcess = [&](auto& item) { - ++callingNum_; - auto localPartId = item.first.first; - auto* proc = ChainAddEdgesProcessorLocal::instance(env_); - proc->setRemotePartId(item.first.second); - proc->process(item.second); - proc->getFuture().thenValue([=](auto&& resp) { - auto code = resp.get_result().get_failed_parts().empty() - ? nebula::cpp2::ErrorCode::SUCCEEDED - : resp.get_result().get_failed_parts().begin()->get_code(); - handleAsync(space, localPartId, code); - }); - }; + auto delegateProcess = [&](auto& item) { + ++callingNum_; + auto localPartId = item.first.first; + auto* proc = ChainAddEdgesProcessorLocal::instance(env_); + proc->setRemotePartId(item.first.second); + proc->process(item.second); + proc->getFuture().thenValue([=](auto&& resp) { + auto code = resp.get_result().get_failed_parts().empty() + ? nebula::cpp2::ErrorCode::SUCCEEDED + : resp.get_result().get_failed_parts().begin()->get_code(); + handleAsync(space, localPartId, code); + }); + }; - std::for_each(shuffledReq.begin(), shuffledReq.end(), delegateProcess); + std::for_each(shuffledReq.begin(), shuffledReq.end(), delegateProcess); } void ChainAddEdgesGroupProcessor::shuffleRequest(const cpp2::AddEdgesRequest& req, ShuffledReq& shuffledReq) { - auto numOfPart = env_->metaClient_->partsNum(req.get_space_id()); - if (!numOfPart.ok()) { - return; - } - auto getPart = [&](auto& vid) { - return env_->metaClient_->partId(numOfPart.value(), vid); - }; - - auto genNewReq = [&](auto& reqIn) { - cpp2::AddEdgesRequest ret; - ret.set_space_id(reqIn.get_space_id()); - ret.set_prop_names(reqIn.get_prop_names()); - ret.set_if_not_exists(reqIn.get_if_not_exists()); - return ret; - }; + auto numOfPart = env_->metaClient_->partsNum(req.get_space_id()); + if (!numOfPart.ok()) { + return; + } + auto getPart = [&](auto& vid) { return env_->metaClient_->partId(numOfPart.value(), vid); }; - auto shufflePart = [&](auto&& part) { - auto& localPart = part.first; + auto genNewReq = [&](auto& reqIn) { + cpp2::AddEdgesRequest ret; + ret.set_space_id(reqIn.get_space_id()); + ret.set_prop_names(reqIn.get_prop_names()); + ret.set_if_not_exists(reqIn.get_if_not_exists()); + return ret; + }; - auto shuffleEdge = [&](auto&& edge) { - auto remotePart = getPart(edge.get_key().get_dst().getStr()); - auto key = std::make_pair(localPart, remotePart); - auto it = shuffledReq.find(key); - if (it == shuffledReq.end()) { - shuffledReq[key] = genNewReq(req); - } - (*shuffledReq[key].parts_ref())[localPart].emplace_back(edge); - }; + auto shufflePart = [&](auto&& part) { + auto& localPart = part.first; - std::for_each(part.second.begin(), part.second.end(), shuffleEdge); + auto shuffleEdge = [&](auto&& edge) { + auto remotePart = getPart(edge.get_key().get_dst().getStr()); + auto key = std::make_pair(localPart, remotePart); + auto it = shuffledReq.find(key); + if (it == shuffledReq.end()) { + shuffledReq[key] = genNewReq(req); + } + (*shuffledReq[key].parts_ref())[localPart].emplace_back(edge); }; - auto& parts = req.get_parts(); - std::for_each(parts.begin(), parts.end(), shufflePart); -} + std::for_each(part.second.begin(), part.second.end(), shuffleEdge); + }; + auto& parts = req.get_parts(); + std::for_each(parts.begin(), parts.end(), shufflePart); +} } // namespace storage } // namespace nebula diff --git a/src/storage/transaction/ChainAddEdgesGroupProcessor.h b/src/storage/transaction/ChainAddEdgesGroupProcessor.h index 4fac012983e..3f4fbdb8a57 100644 --- a/src/storage/transaction/ChainAddEdgesGroupProcessor.h +++ b/src/storage/transaction/ChainAddEdgesGroupProcessor.h @@ -7,28 +7,27 @@ #pragma once #include "storage/BaseProcessor.h" -#include "storage/transaction/ConsistUtil.h" #include "storage/transaction/ChainBaseProcessor.h" +#include "storage/transaction/ConsistUtil.h" #include "storage/transaction/TransactionManager.h" namespace nebula { namespace storage { class ChainAddEdgesGroupProcessor : public BaseProcessor { -public: - static ChainAddEdgesGroupProcessor* instance(StorageEnv* env) { - return new ChainAddEdgesGroupProcessor(env); - } + public: + static ChainAddEdgesGroupProcessor* instance(StorageEnv* env) { + return new ChainAddEdgesGroupProcessor(env); + } - void process(const cpp2::AddEdgesRequest& req); + void process(const cpp2::AddEdgesRequest& req); -protected: - explicit ChainAddEdgesGroupProcessor(StorageEnv* env) - : BaseProcessor(env) {} + protected: + explicit ChainAddEdgesGroupProcessor(StorageEnv* env) : BaseProcessor(env) {} - using Chain = std::pair; - using ShuffledReq = std::unordered_map; - void shuffleRequest(const cpp2::AddEdgesRequest& src, ShuffledReq& dst); + using Chain = std::pair; + using ShuffledReq = std::unordered_map; + void shuffleRequest(const cpp2::AddEdgesRequest& src, ShuffledReq& dst); }; } // namespace storage diff --git a/src/storage/transaction/ChainAddEdgesProcessorLocal.cpp b/src/storage/transaction/ChainAddEdgesProcessorLocal.cpp index ae128d714f3..1f762d29718 100644 --- a/src/storage/transaction/ChainAddEdgesProcessorLocal.cpp +++ b/src/storage/transaction/ChainAddEdgesProcessorLocal.cpp @@ -4,23 +4,24 @@ * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ +#include "storage/transaction/ChainAddEdgesProcessorLocal.h" #include -#include "storage/transaction/ChainAddEdgesProcessorLocal.h" + +#include "storage/StorageFlags.h" +#include "storage/mutate/AddEdgesProcessor.h" #include "storage/transaction/ConsistUtil.h" #include "storage/transaction/TransactionManager.h" -#include "storage/mutate/AddEdgesProcessor.h" -#include "storage/StorageFlags.h" namespace nebula { namespace storage { void ChainAddEdgesProcessorLocal::process(const cpp2::AddEdgesRequest& req) { - if (!prepareRequest(req)) { - finish(); - return; - } - env_->txnMan_->addChainTask(this); + if (!prepareRequest(req)) { + finish(); + return; + } + env_->txnMan_->addChainTask(this); } /** @@ -30,296 +31,294 @@ void ChainAddEdgesProcessorLocal::process(const cpp2::AddEdgesRequest& req) { * 3. write edge prime(key = edge prime, value = transaction id) */ folly::SemiFuture ChainAddEdgesProcessorLocal::prepareLocal() { - if (!lockEdges(req_)) { - return Code::E_WRITE_WRITE_CONFLICT; - } - - auto [pro, fut] = folly::makePromiseContract(); - env_->kvstore_->asyncMultiPut( - spaceId_, localPartId_, makePrime(), [p = std::move(pro)](auto rc) mutable { - LOG_IF(WARNING, rc != nebula::cpp2::ErrorCode::SUCCEEDED) - << "kvstore err: " << static_cast(rc); - p.setValue(rc); - }); - return std::move(fut); + if (!lockEdges(req_)) { + return Code::E_WRITE_WRITE_CONFLICT; + } + + auto [pro, fut] = folly::makePromiseContract(); + env_->kvstore_->asyncMultiPut( + spaceId_, localPartId_, makePrime(), [p = std::move(pro)](auto rc) mutable { + LOG_IF(WARNING, rc != nebula::cpp2::ErrorCode::SUCCEEDED) + << "kvstore err: " << static_cast(rc); + p.setValue(rc); + }); + return std::move(fut); } folly::SemiFuture ChainAddEdgesProcessorLocal::processRemote(Code code) { - if (code != Code::SUCCEEDED) { - return code; - } - CHECK_EQ(req_.get_parts().size(), 1); - auto reversedRequest = reverseRequest(req_); - CHECK_EQ(reversedRequest.get_parts().size(), 1); - auto [pro, fut] = folly::makePromiseContract(); - doRpc(std::move(pro), std::move(reversedRequest)); - return std::move(fut); + if (code != Code::SUCCEEDED) { + return code; + } + CHECK_EQ(req_.get_parts().size(), 1); + auto reversedRequest = reverseRequest(req_); + CHECK_EQ(reversedRequest.get_parts().size(), 1); + auto [pro, fut] = folly::makePromiseContract(); + doRpc(std::move(pro), std::move(reversedRequest)); + return std::move(fut); } folly::SemiFuture ChainAddEdgesProcessorLocal::processLocal(Code code) { - LOG(INFO) << __func__ << "(), code = " << apache::thrift::util::enumNameSafe(code); - if (!checkTerm(req_)) { - LOG(WARNING) << "E_OUTDATED_TERM"; - return Code::E_OUTDATED_TERM; - } - - if (code == Code::E_RPC_FAILURE) { - kvAppend_ = makeDoublePrime(); - lk_->forceUnlock(); - markDanglingEdge(); - } - - erasePrime(); - if (code == Code::SUCCEEDED || code == Code::E_RPC_FAILURE) { - return forwardToDelegateProcessor(); - } else { - abort(); - } - return code; + LOG(INFO) << __func__ << "(), code = " << apache::thrift::util::enumNameSafe(code); + if (!checkTerm(req_)) { + LOG(WARNING) << "E_OUTDATED_TERM"; + return Code::E_OUTDATED_TERM; + } + + if (code == Code::E_RPC_FAILURE) { + kvAppend_ = makeDoublePrime(); + lk_->forceUnlock(); + markDanglingEdge(); + } + + erasePrime(); + if (code == Code::SUCCEEDED || code == Code::E_RPC_FAILURE) { + return forwardToDelegateProcessor(); + } else { + abort(); + } + return code; } void ChainAddEdgesProcessorLocal::markDanglingEdge() { - auto keys = sEdgeKey(req_); - for (auto& key : keys) { - env_->txnMan_->markDanglingEdge(spaceId_, key); - } + auto keys = sEdgeKey(req_); + for (auto& key : keys) { + env_->txnMan_->markDanglingEdge(spaceId_, key); + } } bool ChainAddEdgesProcessorLocal::prepareRequest(const cpp2::AddEdgesRequest& req) { - CHECK_EQ(req.get_parts().size(), 1); - req_ = req; - spaceId_ = req_.get_space_id(); - localPartId_ = req.get_parts().begin()->first; - auto localTerm = env_->txnMan_->getTerm(spaceId_, localPartId_); - if (localTerm.ok()) { - localTerm_ = localTerm.value(); - } else { - LOG(WARNING) << "getTerm failed: space = " << spaceId_ << ", part = " << localPartId_; - pushResultCode(nebula::cpp2::ErrorCode::E_NO_TERM, localPartId_); - return false; - } - - auto vidLen = env_->schemaMan_->getSpaceVidLen(spaceId_); - if (!vidLen.ok()) { - LOG(ERROR) << "getSpaceVidLen failed, spaceId_: " << spaceId_ - << ", status: " << vidLen.status(); - setErrorCode(Code::E_INVALID_SPACEVIDLEN); - return false; - } - spaceVidLen_ = vidLen.value(); - return true; + CHECK_EQ(req.get_parts().size(), 1); + req_ = req; + spaceId_ = req_.get_space_id(); + localPartId_ = req.get_parts().begin()->first; + auto localTerm = env_->txnMan_->getTerm(spaceId_, localPartId_); + if (localTerm.ok()) { + localTerm_ = localTerm.value(); + } else { + LOG(WARNING) << "getTerm failed: space = " << spaceId_ << ", part = " << localPartId_; + pushResultCode(nebula::cpp2::ErrorCode::E_NO_TERM, localPartId_); + return false; + } + + auto vidLen = env_->schemaMan_->getSpaceVidLen(spaceId_); + if (!vidLen.ok()) { + LOG(ERROR) << "getSpaceVidLen failed, spaceId_: " << spaceId_ + << ", status: " << vidLen.status(); + setErrorCode(Code::E_INVALID_SPACEVIDLEN); + return false; + } + spaceVidLen_ = vidLen.value(); + return true; } folly::SemiFuture ChainAddEdgesProcessorLocal::forwardToDelegateProcessor() { - auto* proc = AddEdgesProcessor::instance(env_, nullptr); - proc->consistOp_ = [&](kvstore::BatchHolder& a, std::vector* b) { - callbackOfChainOp(a, b); - }; - proc->process(req_); - // LOG(ERROR) << "what if proc->process() failed"; - auto futProc = proc->getFuture(); - auto [pro, fut] = folly::makePromiseContract(); - std::move(futProc).thenValue( - [&, p = std::move(pro)](auto&& resp) mutable { p.setValue(extractRpcError(resp)); }); - return std::move(fut); + auto* proc = AddEdgesProcessor::instance(env_, nullptr); + proc->consistOp_ = [&](kvstore::BatchHolder& a, std::vector* b) { + callbackOfChainOp(a, b); + }; + proc->process(req_); + // LOG(ERROR) << "what if proc->process() failed"; + auto futProc = proc->getFuture(); + auto [pro, fut] = folly::makePromiseContract(); + std::move(futProc).thenValue( + [&, p = std::move(pro)](auto&& resp) mutable { p.setValue(extractRpcError(resp)); }); + return std::move(fut); } Code ChainAddEdgesProcessorLocal::extractRpcError(const cpp2::ExecResponse& resp) { - Code ret = Code::SUCCEEDED; - auto& respComn = resp.get_result(); - for (auto& part : respComn.get_failed_parts()) { - ret = part.code; - } - return ret; + Code ret = Code::SUCCEEDED; + auto& respComn = resp.get_result(); + for (auto& part : respComn.get_failed_parts()) { + ret = part.code; + } + return ret; } void ChainAddEdgesProcessorLocal::doRpc(folly::Promise&& promise, cpp2::AddEdgesRequest&& req, int retry) noexcept { - if (retry > retryLimit_) { - promise.setValue(Code::E_LEADER_CHANGED); - return; + if (retry > retryLimit_) { + promise.setValue(Code::E_LEADER_CHANGED); + return; + } + // CHECK_NE(req.get_parts().size(), 1); + auto* iClient = env_->txnMan_->getInternalClient(); + folly::Promise p; + auto f = p.getFuture(); + iClient->chainAddEdges(req, localTerm_, edgeVer_, std::move(p)); + + std::move(f).thenTry([=, p = std::move(promise)](auto&& t) mutable { + auto code = t.hasValue() ? t.value() : Code::E_RPC_FAILURE; + switch (code) { + case Code::E_LEADER_CHANGED: + doRpc(std::move(p), std::move(req), ++retry); + break; + default: + p.setValue(code); + break; } - // CHECK_NE(req.get_parts().size(), 1); - auto* iClient = env_->txnMan_->getInternalClient(); - folly::Promise p; - auto f = p.getFuture(); - iClient->chainAddEdges(req, localTerm_, edgeVer_, std::move(p)); - - std::move(f).thenTry([=, p = std::move(promise)](auto&& t) mutable { - auto code = t.hasValue() ? t.value() : Code::E_RPC_FAILURE; - switch (code) { - case Code::E_LEADER_CHANGED: - doRpc(std::move(p), std::move(req), ++retry); - break; - default: - p.setValue(code); - break; - } - return code; - }); + return code; + }); } void ChainAddEdgesProcessorLocal::callbackOfChainOp(kvstore::BatchHolder& batch, std::vector* pData) { - if (pData != nullptr) { - for (auto& kv : *pData) { - batch.put(std::string(kv.first), std::string(kv.second)); - } + if (pData != nullptr) { + for (auto& kv : *pData) { + batch.put(std::string(kv.first), std::string(kv.second)); } - for (auto& key : kvErased_) { - batch.remove(std::string(key)); - } - for (auto& kv : kvAppend_) { - batch.put(std::string(kv.first), std::string(kv.second)); - } - // LOG(WARNING) << "will do perf optimize later"; + } + for (auto& key : kvErased_) { + batch.remove(std::string(key)); + } + for (auto& kv : kvAppend_) { + batch.put(std::string(kv.first), std::string(kv.second)); + } + // LOG(WARNING) << "will do perf optimize later"; } folly::SemiFuture ChainAddEdgesProcessorLocal::abort() { - if (kvErased_.empty()) { - return Code::SUCCEEDED; - } - auto [pro, fut] = folly::makePromiseContract(); - env_->kvstore_->asyncMultiRemove(req_.get_space_id(), - localPartId_, - std::move(kvErased_), - [p = std::move(pro)](auto rc) mutable { - LOG_IF(WARNING, rc != nebula::cpp2::ErrorCode::SUCCEEDED) - << "error: " << static_cast(rc); - p.setValue(rc); - }); - return std::move(fut); + if (kvErased_.empty()) { + return Code::SUCCEEDED; + } + auto [pro, fut] = folly::makePromiseContract(); + env_->kvstore_->asyncMultiRemove(req_.get_space_id(), + localPartId_, + std::move(kvErased_), + [p = std::move(pro)](auto rc) mutable { + LOG_IF(WARNING, rc != nebula::cpp2::ErrorCode::SUCCEEDED) + << "error: " << static_cast(rc); + p.setValue(rc); + }); + return std::move(fut); } std::vector ChainAddEdgesProcessorLocal::makePrime() { - std::vector ret; - for (auto& edge : req_.get_parts().begin()->second) { - auto key = ConsistUtil::primeKey(spaceVidLen_, localPartId_, edge.get_key()); - - auto req = makeSingleEdgeRequest(localPartId_, edge); - std::string val; - apache::thrift::CompactSerializer::serialize(req, &val); - val.append(ConsistUtil::insertIdentifier()); - - ret.emplace_back(std::make_pair(std::move(key), std::move(val))); - } - return ret; + std::vector ret; + for (auto& edge : req_.get_parts().begin()->second) { + auto key = ConsistUtil::primeKey(spaceVidLen_, localPartId_, edge.get_key()); + + auto req = makeSingleEdgeRequest(localPartId_, edge); + std::string val; + apache::thrift::CompactSerializer::serialize(req, &val); + val.append(ConsistUtil::insertIdentifier()); + + ret.emplace_back(std::make_pair(std::move(key), std::move(val))); + } + return ret; } std::vector ChainAddEdgesProcessorLocal::makeDoublePrime() { - std::vector ret; - for (auto& edge : req_.get_parts().begin()->second) { - auto key = ConsistUtil::doublePrime(spaceVidLen_, localPartId_, edge.get_key()); - - auto req = makeSingleEdgeRequest(localPartId_, edge); - std::string val; - apache::thrift::CompactSerializer::serialize(req, &val); - val.append(ConsistUtil::insertIdentifier()); - - ret.emplace_back(std::make_pair(std::move(key), std::move(val))); - } - return ret; + std::vector ret; + for (auto& edge : req_.get_parts().begin()->second) { + auto key = ConsistUtil::doublePrime(spaceVidLen_, localPartId_, edge.get_key()); + + auto req = makeSingleEdgeRequest(localPartId_, edge); + std::string val; + apache::thrift::CompactSerializer::serialize(req, &val); + val.append(ConsistUtil::insertIdentifier()); + + ret.emplace_back(std::make_pair(std::move(key), std::move(val))); + } + return ret; } void ChainAddEdgesProcessorLocal::erasePrime() { - auto fn = [&](const cpp2::NewEdge& edge) { - return ConsistUtil::primeKey(spaceVidLen_, localPartId_, edge.get_key()); - }; - for (auto& edge : req_.get_parts().begin()->second) { - kvErased_.push_back(fn(edge)); - } + auto fn = [&](const cpp2::NewEdge& edge) { + return ConsistUtil::primeKey(spaceVidLen_, localPartId_, edge.get_key()); + }; + for (auto& edge : req_.get_parts().begin()->second) { + kvErased_.push_back(fn(edge)); + } } void ChainAddEdgesProcessorLocal::eraseDoublePrime() { - auto fn = [&](const cpp2::NewEdge& edge) { - return ConsistUtil::doublePrime(spaceVidLen_, localPartId_, edge.get_key()); - }; - for (auto& edge : req_.get_parts().begin()->second) { - kvErased_.push_back(fn(edge)); - } + auto fn = [&](const cpp2::NewEdge& edge) { + return ConsistUtil::doublePrime(spaceVidLen_, localPartId_, edge.get_key()); + }; + for (auto& edge : req_.get_parts().begin()->second) { + kvErased_.push_back(fn(edge)); + } } bool ChainAddEdgesProcessorLocal::lockEdges(const cpp2::AddEdgesRequest& req) { - std::vector keys; - auto partId = req.get_parts().begin()->first; - for (auto& edge : req.get_parts().begin()->second) { - keys.emplace_back(ConsistUtil::edgeKey(spaceVidLen_, partId, edge.get_key())); - } - auto* lockCore = env_->txnMan_->getLockCore(req.get_space_id()); - lk_ = std::make_unique(lockCore, keys); - return lk_->isLocked(); + std::vector keys; + auto partId = req.get_parts().begin()->first; + for (auto& edge : req.get_parts().begin()->second) { + keys.emplace_back(ConsistUtil::edgeKey(spaceVidLen_, partId, edge.get_key())); + } + auto* lockCore = env_->txnMan_->getLockCore(req.get_space_id()); + lk_ = std::make_unique(lockCore, keys); + return lk_->isLocked(); } // we need to check term at both remote phase and local commit bool ChainAddEdgesProcessorLocal::checkTerm(const cpp2::AddEdgesRequest& req) { - auto space = req.get_space_id(); - auto partId = req.get_parts().begin()->first; - auto ret = env_->txnMan_->checkTerm(space, partId, localTerm_); - LOG_IF(WARNING, !ret) << "check term failed, localTerm_ = " << localTerm_; - return ret; + auto space = req.get_space_id(); + auto partId = req.get_parts().begin()->first; + auto ret = env_->txnMan_->checkTerm(space, partId, localTerm_); + LOG_IF(WARNING, !ret) << "check term failed, localTerm_ = " << localTerm_; + return ret; } // check if current edge is not newer than the one trying to resume. // this function only take effect in resume mode bool ChainAddEdgesProcessorLocal::checkVersion(const cpp2::AddEdgesRequest& req) { - auto part = req.get_parts().begin()->first; - auto sKeys = sEdgeKey(req); - auto currVer = ConsistUtil::getMultiEdgeVers(env_->kvstore_, spaceId_, part, sKeys); - for (auto i = 0U; i != currVer.size(); ++i) { - if (currVer[i] < resumedEdgeVer_) { - return false; - } + auto part = req.get_parts().begin()->first; + auto sKeys = sEdgeKey(req); + auto currVer = ConsistUtil::getMultiEdgeVers(env_->kvstore_, spaceId_, part, sKeys); + for (auto i = 0U; i != currVer.size(); ++i) { + if (currVer[i] < resumedEdgeVer_) { + return false; } - return true; + } + return true; } -std::vector -ChainAddEdgesProcessorLocal::sEdgeKey(const cpp2::AddEdgesRequest& req) { - std::vector ret; - for (auto& edgesOfPart : req.get_parts()) { - auto partId = edgesOfPart.first; - for (auto& edge : edgesOfPart.second) { - ret.emplace_back(ConsistUtil::edgeKey(spaceVidLen_, partId, edge.get_key())); - } +std::vector ChainAddEdgesProcessorLocal::sEdgeKey(const cpp2::AddEdgesRequest& req) { + std::vector ret; + for (auto& edgesOfPart : req.get_parts()) { + auto partId = edgesOfPart.first; + for (auto& edge : edgesOfPart.second) { + ret.emplace_back(ConsistUtil::edgeKey(spaceVidLen_, partId, edge.get_key())); } - return ret; + } + return ret; } -cpp2::AddEdgesRequest -ChainAddEdgesProcessorLocal::reverseRequest(const cpp2::AddEdgesRequest& req) { - cpp2::AddEdgesRequest reversedRequest; - for (auto& edgesOfPart : *req.parts_ref()) { - for (auto& newEdge : edgesOfPart.second) { - (*reversedRequest.parts_ref())[remotePartId_].emplace_back(newEdge); - auto& newEdgeRef = (*reversedRequest.parts_ref())[remotePartId_].back(); - ConsistUtil::reverseEdgeKeyInplace(*newEdgeRef.key_ref()); - } +cpp2::AddEdgesRequest ChainAddEdgesProcessorLocal::reverseRequest( + const cpp2::AddEdgesRequest& req) { + cpp2::AddEdgesRequest reversedRequest; + for (auto& edgesOfPart : *req.parts_ref()) { + for (auto& newEdge : edgesOfPart.second) { + (*reversedRequest.parts_ref())[remotePartId_].emplace_back(newEdge); + auto& newEdgeRef = (*reversedRequest.parts_ref())[remotePartId_].back(); + ConsistUtil::reverseEdgeKeyInplace(*newEdgeRef.key_ref()); } - reversedRequest.set_space_id(req.get_space_id()); - reversedRequest.set_prop_names(req.get_prop_names()); - reversedRequest.set_if_not_exists(req.get_if_not_exists()); - return reversedRequest; + } + reversedRequest.set_space_id(req.get_space_id()); + reversedRequest.set_prop_names(req.get_prop_names()); + reversedRequest.set_if_not_exists(req.get_if_not_exists()); + return reversedRequest; } void ChainAddEdgesProcessorLocal::finish() { - pushResultCode(code_, localPartId_); - onFinished(); + pushResultCode(code_, localPartId_); + onFinished(); } cpp2::AddEdgesRequest ChainAddEdgesProcessorLocal::makeSingleEdgeRequest( - PartitionID partId, - const cpp2::NewEdge& edge) { - cpp2::AddEdgesRequest req; - req.set_space_id(req_.get_space_id()); - req.set_prop_names(req_.get_prop_names()); - req.set_if_not_exists(req_.get_if_not_exists()); - - std::unordered_map> newParts; - newParts[partId].emplace_back(edge); - - req.set_parts(newParts); - return req; + PartitionID partId, const cpp2::NewEdge& edge) { + cpp2::AddEdgesRequest req; + req.set_space_id(req_.get_space_id()); + req.set_prop_names(req_.get_prop_names()); + req.set_if_not_exists(req_.get_if_not_exists()); + + std::unordered_map> newParts; + newParts[partId].emplace_back(edge); + + req.set_parts(newParts); + return req; } } // namespace storage diff --git a/src/storage/transaction/ChainAddEdgesProcessorLocal.h b/src/storage/transaction/ChainAddEdgesProcessorLocal.h index d85326739e0..aab887ccac7 100644 --- a/src/storage/transaction/ChainAddEdgesProcessorLocal.h +++ b/src/storage/transaction/ChainAddEdgesProcessorLocal.h @@ -7,8 +7,8 @@ #pragma once #include "interface/gen-cpp2/storage_types.h" -#include "storage/BaseProcessor.h" #include "kvstore/LogEncoder.h" +#include "storage/BaseProcessor.h" #include "storage/transaction/ConsistUtil.h" #include "storage/transaction/TransactionManager.h" @@ -17,111 +17,107 @@ namespace storage { class ChainAddEdgesProcessorLocal : public BaseProcessor, public ChainBaseProcessor { - friend class ChainResumeProcessorTestHelper; // for test friendly -public: - static ChainAddEdgesProcessorLocal* instance(StorageEnv* env) { - return new ChainAddEdgesProcessorLocal(env); - } + friend class ChainResumeProcessorTestHelper; // for test friendly + public: + static ChainAddEdgesProcessorLocal* instance(StorageEnv* env) { + return new ChainAddEdgesProcessorLocal(env); + } - virtual ~ChainAddEdgesProcessorLocal() = default; + virtual ~ChainAddEdgesProcessorLocal() = default; - virtual void process(const cpp2::AddEdgesRequest& req); + virtual void process(const cpp2::AddEdgesRequest& req); - folly::SemiFuture prepareLocal() override; + folly::SemiFuture prepareLocal() override; - folly::SemiFuture processRemote(Code code) override; + folly::SemiFuture processRemote(Code code) override; - folly::SemiFuture processLocal(Code code) override; + folly::SemiFuture processLocal(Code code) override; - void setRemotePartId(PartitionID remotePartId) { remotePartId_ = remotePartId; } + void setRemotePartId(PartitionID remotePartId) { remotePartId_ = remotePartId; } - void finish() override; + void finish() override; -protected: - explicit ChainAddEdgesProcessorLocal(StorageEnv* env) - : BaseProcessor(env) {} + protected: + explicit ChainAddEdgesProcessorLocal(StorageEnv* env) : BaseProcessor(env) {} - bool prepareRequest(const cpp2::AddEdgesRequest& req); + bool prepareRequest(const cpp2::AddEdgesRequest& req); - /** - * @brief resume and set req_ & txnId_ from the val of (double)prime - * @return true if resume succeeded - */ - bool deserializeRequest(GraphSpaceID spaceId, PartitionID partId, folly::StringPiece val); + /** + * @brief resume and set req_ & txnId_ from the val of (double)prime + * @return true if resume succeeded + */ + bool deserializeRequest(GraphSpaceID spaceId, PartitionID partId, folly::StringPiece val); - void doRpc(folly::Promise&& pro, - cpp2::AddEdgesRequest&& req, - int retry = 0) noexcept; + void doRpc(folly::Promise&& pro, cpp2::AddEdgesRequest&& req, int retry = 0) noexcept; - bool lockEdges(const cpp2::AddEdgesRequest& req); + bool lockEdges(const cpp2::AddEdgesRequest& req); - bool checkTerm(const cpp2::AddEdgesRequest& req); + bool checkTerm(const cpp2::AddEdgesRequest& req); - bool checkVersion(const cpp2::AddEdgesRequest& req); + bool checkVersion(const cpp2::AddEdgesRequest& req); - /** - * @brief This is a call back function, to let AddEdgesProcessor so some - * addition thing for chain operation - * @param batch if target edge has index - * @param pData if target edge has no index. - */ - void callbackOfChainOp(kvstore::BatchHolder& batch, std::vector* pData); + /** + * @brief This is a call back function, to let AddEdgesProcessor so some + * addition thing for chain operation + * @param batch if target edge has index + * @param pData if target edge has no index. + */ + void callbackOfChainOp(kvstore::BatchHolder& batch, std::vector* pData); - /** - * @brief helper function to generate string form of keys of request - */ - std::vector sEdgeKey(const cpp2::AddEdgesRequest& req); + /** + * @brief helper function to generate string form of keys of request + */ + std::vector sEdgeKey(const cpp2::AddEdgesRequest& req); - /** - * @brief normally, the prime/double prime keys will be deleted at AddEdgeProcessor - * (in a batch of finial edges), but sometimes ChainProcessor will stop early - * and we need this do the clean thing. - */ - folly::SemiFuture abort(); + /** + * @brief normally, the prime/double prime keys will be deleted at AddEdgeProcessor + * (in a batch of finial edges), but sometimes ChainProcessor will stop early + * and we need this do the clean thing. + */ + folly::SemiFuture abort(); - /** - * @brief helper function to get the reversed request of the normal incoming req. - * (to use that for reversed edge) - */ - cpp2::AddEdgesRequest reverseRequest(const cpp2::AddEdgesRequest& req); + /** + * @brief helper function to get the reversed request of the normal incoming req. + * (to use that for reversed edge) + */ + cpp2::AddEdgesRequest reverseRequest(const cpp2::AddEdgesRequest& req); - Code extractRpcError(const cpp2::ExecResponse& resp); + Code extractRpcError(const cpp2::ExecResponse& resp); - /** - * @brief a normal AddEdgeRequest may contain multi edges - * even though they will fail or succeed as a batch in this time - * some of them may by overwrite by othere request - * so when resume each edge - */ - cpp2::AddEdgesRequest makeSingleEdgeRequest(PartitionID partId, const cpp2::NewEdge& edge); + /** + * @brief a normal AddEdgeRequest may contain multi edges + * even though they will fail or succeed as a batch in this time + * some of them may by overwrite by othere request + * so when resume each edge + */ + cpp2::AddEdgesRequest makeSingleEdgeRequest(PartitionID partId, const cpp2::NewEdge& edge); - std::vector makePrime(); + std::vector makePrime(); - std::vector makeDoublePrime(); + std::vector makeDoublePrime(); - void erasePrime(); + void erasePrime(); - void eraseDoublePrime(); + void eraseDoublePrime(); - folly::SemiFuture forwardToDelegateProcessor(); + folly::SemiFuture forwardToDelegateProcessor(); - void markDanglingEdge(); + void markDanglingEdge(); -protected: - GraphSpaceID spaceId_; - PartitionID localPartId_; - PartitionID remotePartId_; - cpp2::AddEdgesRequest req_; - std::unique_ptr lk_; - int retryLimit_{10}; - TermID localTerm_{-1}; + protected: + GraphSpaceID spaceId_; + PartitionID localPartId_; + PartitionID remotePartId_; + cpp2::AddEdgesRequest req_; + std::unique_ptr lk_; + int retryLimit_{10}; + TermID localTerm_{-1}; - std::vector kvErased_; - std::vector kvAppend_; - folly::Optional edgeVer_{folly::none}; - int64_t resumedEdgeVer_{-1}; + std::vector kvErased_; + std::vector kvAppend_; + folly::Optional edgeVer_{folly::none}; + int64_t resumedEdgeVer_{-1}; }; - } // namespace storage } // namespace nebula diff --git a/src/storage/transaction/ChainAddEdgesProcessorRemote.cpp b/src/storage/transaction/ChainAddEdgesProcessorRemote.cpp index 8d9887838df..fa7e3d8a7ff 100644 --- a/src/storage/transaction/ChainAddEdgesProcessorRemote.cpp +++ b/src/storage/transaction/ChainAddEdgesProcessorRemote.cpp @@ -4,8 +4,9 @@ * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ -#include "storage/mutate/AddEdgesProcessor.h" #include "storage/transaction/ChainAddEdgesProcessorRemote.h" + +#include "storage/mutate/AddEdgesProcessor.h" #include "storage/transaction/ConsistUtil.h" #include "storage/transaction/TransactionManager.h" @@ -13,70 +14,70 @@ namespace nebula { namespace storage { void ChainAddEdgesProcessorRemote::process(const cpp2::ChainAddEdgesRequest& req) { - auto partId = req.get_parts().begin()->first; - auto code = nebula::cpp2::ErrorCode::SUCCEEDED; - do { - if (!checkTerm(req)) { - LOG(WARNING) << "invalid term"; - code = nebula::cpp2::ErrorCode::E_OUTDATED_TERM; - break; - } - - if (!checkVersion(req)) { - code = nebula::cpp2::ErrorCode::E_OUTDATED_EDGE; - break; - } - } while (0); + auto partId = req.get_parts().begin()->first; + auto code = nebula::cpp2::ErrorCode::SUCCEEDED; + do { + if (!checkTerm(req)) { + LOG(WARNING) << "invalid term"; + code = nebula::cpp2::ErrorCode::E_OUTDATED_TERM; + break; + } - if (code == nebula::cpp2::ErrorCode::SUCCEEDED) { - forwardRequest(req); - } else { - pushResultCode(code, partId); - onFinished(); + if (!checkVersion(req)) { + code = nebula::cpp2::ErrorCode::E_OUTDATED_EDGE; + break; } + } while (0); + + if (code == nebula::cpp2::ErrorCode::SUCCEEDED) { + forwardRequest(req); + } else { + pushResultCode(code, partId); + onFinished(); + } } bool ChainAddEdgesProcessorRemote::checkTerm(const cpp2::ChainAddEdgesRequest& req) { - auto partId = req.get_parts().begin()->first; - return env_->txnMan_->checkTerm(req.get_space_id(), partId, req.get_term()); + auto partId = req.get_parts().begin()->first; + return env_->txnMan_->checkTerm(req.get_space_id(), partId, req.get_term()); } bool ChainAddEdgesProcessorRemote::checkVersion(const cpp2::ChainAddEdgesRequest& req) { - if (!req.edge_version_ref()) { - return true; - } - auto spaceId = req.get_space_id(); - auto partId = req.get_parts().begin()->first; - auto strEdgeKeys = getStrEdgeKeys(req); - auto currVer = ConsistUtil::getMultiEdgeVers(env_->kvstore_, spaceId, partId, strEdgeKeys); - auto edgeVer = *req.edge_version_ref(); - for (auto i = 0U; i != currVer.size(); ++i) { - if (currVer[i] > edgeVer) { - return false; - } - } + if (!req.edge_version_ref()) { return true; + } + auto spaceId = req.get_space_id(); + auto partId = req.get_parts().begin()->first; + auto strEdgeKeys = getStrEdgeKeys(req); + auto currVer = ConsistUtil::getMultiEdgeVers(env_->kvstore_, spaceId, partId, strEdgeKeys); + auto edgeVer = *req.edge_version_ref(); + for (auto i = 0U; i != currVer.size(); ++i) { + if (currVer[i] > edgeVer) { + return false; + } + } + return true; } void ChainAddEdgesProcessorRemote::forwardRequest(const cpp2::ChainAddEdgesRequest& req) { - auto* proc = AddEdgesProcessor::instance(env_); - proc->getFuture().thenValue([&](auto&& resp){ - this->result_ = resp.get_result(); - this->onFinished(); - }); - proc->process(ConsistUtil::makeDirectAddReq(req)); + auto* proc = AddEdgesProcessor::instance(env_); + proc->getFuture().thenValue([&](auto&& resp) { + this->result_ = resp.get_result(); + this->onFinished(); + }); + proc->process(ConsistUtil::makeDirectAddReq(req)); } std::vector ChainAddEdgesProcessorRemote::getStrEdgeKeys( const cpp2::ChainAddEdgesRequest& req) { - std::vector ret; - for (auto& edgesOfPart : req.get_parts()) { - auto partId = edgesOfPart.first; - for (auto& edge : edgesOfPart.second) { - ret.emplace_back(ConsistUtil::edgeKey(spaceVidLen_, partId, edge.get_key())); - } + std::vector ret; + for (auto& edgesOfPart : req.get_parts()) { + auto partId = edgesOfPart.first; + for (auto& edge : edgesOfPart.second) { + ret.emplace_back(ConsistUtil::edgeKey(spaceVidLen_, partId, edge.get_key())); } - return ret; + } + return ret; } } // namespace storage diff --git a/src/storage/transaction/ChainBaseProcessor.h b/src/storage/transaction/ChainBaseProcessor.h index 9717c050e89..93fd3a9bd3b 100644 --- a/src/storage/transaction/ChainBaseProcessor.h +++ b/src/storage/transaction/ChainBaseProcessor.h @@ -6,8 +6,8 @@ #pragma once -#include "storage/CommonUtils.h" #include "common/utils/MemoryLockWrapper.h" +#include "storage/CommonUtils.h" namespace nebula { namespace storage { @@ -21,31 +21,30 @@ using Code = ::nebula::cpp2::ErrorCode; * */ class ChainBaseProcessor { -public: - virtual ~ChainBaseProcessor() = default; + public: + virtual ~ChainBaseProcessor() = default; - virtual folly::SemiFuture prepareLocal() { return Code::SUCCEEDED; } + virtual folly::SemiFuture prepareLocal() { return Code::SUCCEEDED; } - virtual folly::SemiFuture processRemote(Code code) { return code; } + virtual folly::SemiFuture processRemote(Code code) { return code; } - virtual folly::SemiFuture processLocal(Code code) { return code; } + virtual folly::SemiFuture processLocal(Code code) { return code; } - virtual folly::Future getFinished() { return finished_.getFuture(); } + virtual folly::Future getFinished() { return finished_.getFuture(); } - virtual void finish() = 0; + virtual void finish() = 0; -protected: - void setErrorCode(Code code) { - if (code_ == Code::SUCCEEDED) { - code_ = code; - } + protected: + void setErrorCode(Code code) { + if (code_ == Code::SUCCEEDED) { + code_ = code; } + } -protected: - Code code_ = Code::SUCCEEDED; - folly::Promise finished_; + protected: + Code code_ = Code::SUCCEEDED; + folly::Promise finished_; }; - } // namespace storage } // namespace nebula diff --git a/src/storage/transaction/ChainProcessorFactory.cpp b/src/storage/transaction/ChainProcessorFactory.cpp index cd9bfeb49a3..23b81d9d4fa 100644 --- a/src/storage/transaction/ChainProcessorFactory.cpp +++ b/src/storage/transaction/ChainProcessorFactory.cpp @@ -5,6 +5,7 @@ */ #include "storage/transaction/ChainProcessorFactory.h" + #include "storage/transaction/ConsistUtil.h" #include "storage/transaction/ResumeAddEdgeProcessor.h" #include "storage/transaction/ResumeAddEdgeRemoteProcessor.h" @@ -16,48 +17,47 @@ namespace storage { ChainBaseProcessor* ChainProcessorFactory::makeProcessor(StorageEnv* env, const ResumeOptions& options) { - ChainBaseProcessor* ret = nullptr; - auto requestType = ConsistUtil::parseType(options.primeValue); - switch (requestType) { - case RequestType::INSERT: { - switch (options.resumeType) { - case ResumeType::RESUME_CHAIN: { - ret = ResumeAddEdgeProcessor::instance(env, options.primeValue); - break; - } - case ResumeType::RESUME_REMOTE: { - ret = ResumeAddEdgeRemoteProcessor::instance(env, options.primeValue); - break; - } - case ResumeType::UNKNOWN: { - LOG(FATAL) << "ResumeType::UNKNOWN: not supposed run here"; - } - } - break; + ChainBaseProcessor* ret = nullptr; + auto requestType = ConsistUtil::parseType(options.primeValue); + switch (requestType) { + case RequestType::INSERT: { + switch (options.resumeType) { + case ResumeType::RESUME_CHAIN: { + ret = ResumeAddEdgeProcessor::instance(env, options.primeValue); + break; } - case RequestType::UPDATE: { - switch (options.resumeType) { - case ResumeType::RESUME_CHAIN: { - ret = ResumeUpdateProcessor::instance(env, options.primeValue); - break; - } - case ResumeType::RESUME_REMOTE: { - ret = ResumeUpdateRemoteProcessor::instance(env, options.primeValue); - break; - } - case ResumeType::UNKNOWN: { - LOG(FATAL) << "ResumeType::UNKNOWN: not supposed run here"; - } - } - break; + case ResumeType::RESUME_REMOTE: { + ret = ResumeAddEdgeRemoteProcessor::instance(env, options.primeValue); + break; } - case RequestType::UNKNOWN: { - LOG(FATAL) << "RequestType::UNKNOWN: not supposed run here"; + case ResumeType::UNKNOWN: { + LOG(FATAL) << "ResumeType::UNKNOWN: not supposed run here"; } + } + break; } - return ret; + case RequestType::UPDATE: { + switch (options.resumeType) { + case ResumeType::RESUME_CHAIN: { + ret = ResumeUpdateProcessor::instance(env, options.primeValue); + break; + } + case ResumeType::RESUME_REMOTE: { + ret = ResumeUpdateRemoteProcessor::instance(env, options.primeValue); + break; + } + case ResumeType::UNKNOWN: { + LOG(FATAL) << "ResumeType::UNKNOWN: not supposed run here"; + } + } + break; + } + case RequestType::UNKNOWN: { + LOG(FATAL) << "RequestType::UNKNOWN: not supposed run here"; + } + } + return ret; } - } // namespace storage } // namespace nebula diff --git a/src/storage/transaction/ChainProcessorFactory.h b/src/storage/transaction/ChainProcessorFactory.h index 3deeaa23c68..857b4b8a56b 100644 --- a/src/storage/transaction/ChainProcessorFactory.h +++ b/src/storage/transaction/ChainProcessorFactory.h @@ -13,20 +13,20 @@ namespace nebula { namespace storage { enum class ResumeType { - UNKNOWN = 0, - RESUME_CHAIN, - RESUME_REMOTE, + UNKNOWN = 0, + RESUME_CHAIN, + RESUME_REMOTE, }; struct ResumeOptions { - ResumeOptions(ResumeType tp, std::string val) : resumeType(tp), primeValue(std::move(val)) {} - ResumeType resumeType; - std::string primeValue; + ResumeOptions(ResumeType tp, std::string val) : resumeType(tp), primeValue(std::move(val)) {} + ResumeType resumeType; + std::string primeValue; }; class ChainProcessorFactory { -public: - static ChainBaseProcessor* makeProcessor(StorageEnv* env, const ResumeOptions& options); + public: + static ChainBaseProcessor* makeProcessor(StorageEnv* env, const ResumeOptions& options); }; } // namespace storage diff --git a/src/storage/transaction/ChainResumeProcessor.cpp b/src/storage/transaction/ChainResumeProcessor.cpp index 85cf3d6e677..709b4bd30db 100644 --- a/src/storage/transaction/ChainResumeProcessor.cpp +++ b/src/storage/transaction/ChainResumeProcessor.cpp @@ -5,53 +5,54 @@ */ #include "storage/transaction/ChainResumeProcessor.h" -#include "storage/transaction/ChainUpdateEdgeProcessorLocal.h" + #include "storage/transaction/ChainAddEdgesProcessorLocal.h" +#include "storage/transaction/ChainProcessorFactory.h" +#include "storage/transaction/ChainUpdateEdgeProcessorLocal.h" #include "storage/transaction/ConsistUtil.h" #include "storage/transaction/TransactionManager.h" -#include "storage/transaction/ChainProcessorFactory.h" namespace nebula { namespace storage { void ChainResumeProcessor::process() { - std::unordered_map> leaders; - if (env_->kvstore_->allLeader(leaders) == 0) { - LOG(INFO) << "no leader found, skip any resume process"; - return; - } - std::unique_ptr iter; - for (auto& leader : leaders) { - auto spaceId = leader.first; - LOG(INFO) << "leader.second.size() = " << leader.second.size(); - for (auto& partInfo : leader.second) { - auto partId = partInfo.get_part_id(); - auto prefix = ConsistUtil::primePrefix(partId); - auto rc = env_->kvstore_->prefix(spaceId, partId, prefix, &iter); - if (rc != nebula::cpp2::ErrorCode::SUCCEEDED) { - break; - } - for (; iter->valid(); iter->next()) { - ResumeOptions opt(ResumeType::RESUME_CHAIN, iter->val().str()); - auto* proc = ChainProcessorFactory::makeProcessor(env_, opt); - futs.emplace_back(proc->getFinished()); - env_->txnMan_->addChainTask(proc); - } + std::unordered_map> leaders; + if (env_->kvstore_->allLeader(leaders) == 0) { + LOG(INFO) << "no leader found, skip any resume process"; + return; + } + std::unique_ptr iter; + for (auto& leader : leaders) { + auto spaceId = leader.first; + LOG(INFO) << "leader.second.size() = " << leader.second.size(); + for (auto& partInfo : leader.second) { + auto partId = partInfo.get_part_id(); + auto prefix = ConsistUtil::primePrefix(partId); + auto rc = env_->kvstore_->prefix(spaceId, partId, prefix, &iter); + if (rc != nebula::cpp2::ErrorCode::SUCCEEDED) { + break; + } + for (; iter->valid(); iter->next()) { + ResumeOptions opt(ResumeType::RESUME_CHAIN, iter->val().str()); + auto* proc = ChainProcessorFactory::makeProcessor(env_, opt); + futs.emplace_back(proc->getFinished()); + env_->txnMan_->addChainTask(proc); + } - prefix = ConsistUtil::doublePrimePrefix(partId); - rc = env_->kvstore_->prefix(spaceId, partId, prefix, &iter); - if (rc != nebula::cpp2::ErrorCode::SUCCEEDED) { - break; - } - for (; iter->valid(); iter->next()) { - ResumeOptions opt(ResumeType::RESUME_REMOTE, iter->val().str()); - auto* proc = ChainProcessorFactory::makeProcessor(env_, opt); - futs.emplace_back(proc->getFinished()); - env_->txnMan_->addChainTask(proc); - } - } + prefix = ConsistUtil::doublePrimePrefix(partId); + rc = env_->kvstore_->prefix(spaceId, partId, prefix, &iter); + if (rc != nebula::cpp2::ErrorCode::SUCCEEDED) { + break; + } + for (; iter->valid(); iter->next()) { + ResumeOptions opt(ResumeType::RESUME_REMOTE, iter->val().str()); + auto* proc = ChainProcessorFactory::makeProcessor(env_, opt); + futs.emplace_back(proc->getFinished()); + env_->txnMan_->addChainTask(proc); + } } - folly::collectAll(futs).get(); + } + folly::collectAll(futs).get(); } } // namespace storage diff --git a/src/storage/transaction/ChainResumeProcessor.h b/src/storage/transaction/ChainResumeProcessor.h index 2d2b0b693da..0611c29d44e 100644 --- a/src/storage/transaction/ChainResumeProcessor.h +++ b/src/storage/transaction/ChainResumeProcessor.h @@ -6,26 +6,27 @@ #pragma once +#include "clients/storage/InternalStorageClient.h" #include "common/utils/NebulaKeyUtils.h" -#include "storage/transaction/TransactionManager.h" -#include "storage/transaction/ChainBaseProcessor.h" #include "storage/transaction/ChainAddEdgesProcessorLocal.h" +#include "storage/transaction/ChainBaseProcessor.h" #include "storage/transaction/ChainUpdateEdgeProcessorLocal.h" -#include "clients/storage/InternalStorageClient.h" +#include "storage/transaction/TransactionManager.h" namespace nebula { namespace storage { class ChainResumeProcessor { - friend class ChainResumeProcessorTestHelper; -public: - explicit ChainResumeProcessor(StorageEnv* env) : env_(env) {} + friend class ChainResumeProcessorTestHelper; + + public: + explicit ChainResumeProcessor(StorageEnv* env) : env_(env) {} - void process(); + void process(); -private: - StorageEnv* env_{nullptr}; - std::vector> futs; + private: + StorageEnv* env_{nullptr}; + std::vector> futs; }; } // namespace storage diff --git a/src/storage/transaction/ChainUpdateEdgeProcessorLocal.cpp b/src/storage/transaction/ChainUpdateEdgeProcessorLocal.cpp index dea9da755a5..b56609bdf77 100644 --- a/src/storage/transaction/ChainUpdateEdgeProcessorLocal.cpp +++ b/src/storage/transaction/ChainUpdateEdgeProcessorLocal.cpp @@ -4,43 +4,45 @@ * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ -#include #include "storage/transaction/ChainUpdateEdgeProcessorLocal.h" + +#include + +#include "storage/StorageFlags.h" +#include "storage/mutate/UpdateEdgeProcessor.h" #include "storage/transaction/ConsistUtil.h" #include "storage/transaction/TransactionManager.h" -#include "storage/mutate/UpdateEdgeProcessor.h" -#include "storage/StorageFlags.h" namespace nebula { namespace storage { void ChainUpdateEdgeProcessorLocal::process(const cpp2::UpdateEdgeRequest& req) { - if (!prepareRequest(req)) { - onFinished(); - } + if (!prepareRequest(req)) { + onFinished(); + } - env_->txnMan_->addChainTask(this); + env_->txnMan_->addChainTask(this); } bool ChainUpdateEdgeProcessorLocal::prepareRequest(const cpp2::UpdateEdgeRequest& req) { - req_ = req; - spaceId_ = req.get_space_id(); - partId_ = req_.get_part_id(); - - auto rc = getSpaceVidLen(spaceId_); - if (rc != nebula::cpp2::ErrorCode::SUCCEEDED) { - pushResultCode(rc, partId_); - return false; - } - - auto __term = env_->txnMan_->getTerm(req_.get_space_id(), partId_); - if (__term.ok()) { - termOfPrepare_ = __term.value(); - } else { - pushResultCode(Code::E_PART_NOT_FOUND, partId_); - return false; - } - return true; + req_ = req; + spaceId_ = req.get_space_id(); + partId_ = req_.get_part_id(); + + auto rc = getSpaceVidLen(spaceId_); + if (rc != nebula::cpp2::ErrorCode::SUCCEEDED) { + pushResultCode(rc, partId_); + return false; + } + + auto __term = env_->txnMan_->getTerm(req_.get_space_id(), partId_); + if (__term.ok()) { + termOfPrepare_ = __term.value(); + } else { + pushResultCode(Code::E_PART_NOT_FOUND, partId_); + return false; + } + return true; } /** @@ -48,224 +50,220 @@ bool ChainUpdateEdgeProcessorLocal::prepareRequest(const cpp2::UpdateEdgeRequest * 2. set edge prime * */ folly::SemiFuture ChainUpdateEdgeProcessorLocal::prepareLocal() { - if (!setLock()) { - LOG(INFO) << "set lock failed, return E_DATA_CONFLICT_ERROR"; - return Code::E_DATA_CONFLICT_ERROR; - } - - auto key = ConsistUtil::primeKey(spaceVidLen_, partId_, req_.get_edge_key()); - - std::string val; - apache::thrift::CompactSerializer::serialize(req_, &val); - val.append(ConsistUtil::updateIdentifier()); - - std::vector data{{key, val}}; - auto c = folly::makePromiseContract(); - env_->kvstore_->asyncMultiPut(spaceId_, - partId_, - std::move(data), - [p = std::move(c.first)](auto rc) mutable { - p.setValue(rc); - }); - return std::move(c.second); + if (!setLock()) { + LOG(INFO) << "set lock failed, return E_DATA_CONFLICT_ERROR"; + return Code::E_DATA_CONFLICT_ERROR; + } + + auto key = ConsistUtil::primeKey(spaceVidLen_, partId_, req_.get_edge_key()); + + std::string val; + apache::thrift::CompactSerializer::serialize(req_, &val); + val.append(ConsistUtil::updateIdentifier()); + + std::vector data{{key, val}}; + auto c = folly::makePromiseContract(); + env_->kvstore_->asyncMultiPut( + spaceId_, partId_, std::move(data), [p = std::move(c.first)](auto rc) mutable { + p.setValue(rc); + }); + return std::move(c.second); } folly::SemiFuture ChainUpdateEdgeProcessorLocal::processRemote(Code code) { - LOG(INFO) << __func__ << "(), code = " << apache::thrift::util::enumNameSafe(code); - if (code != Code::SUCCEEDED) { - return code; - } - auto [pro, fut] = folly::makePromiseContract(); - doRpc(std::move(pro)); - return std::move(fut); + LOG(INFO) << __func__ << "(), code = " << apache::thrift::util::enumNameSafe(code); + if (code != Code::SUCCEEDED) { + return code; + } + auto [pro, fut] = folly::makePromiseContract(); + doRpc(std::move(pro)); + return std::move(fut); } folly::SemiFuture ChainUpdateEdgeProcessorLocal::processLocal(Code code) { - LOG(INFO) << __func__ << "(), code = " << apache::thrift::util::enumNameSafe(code); - if (code != Code::SUCCEEDED && code_ == Code::SUCCEEDED) { - code_ = code; - } - - if (!checkTerm()) { - LOG(WARNING) << "checkTerm() failed"; - return Code::E_OUTDATED_TERM; - } - - if (code == Code::E_RPC_FAILURE) { - appendDoublePrime(); - } - - if (code == Code::SUCCEEDED || code == Code::E_RPC_FAILURE) { - erasePrime(); - forwardToDelegateProcessor(); - } else { - abort(); - } - - // LOG(INFO) << "~processNormalLocal(), code_: " << apache::thrift::util::enumNameSafe(code); - return code_; + LOG(INFO) << __func__ << "(), code = " << apache::thrift::util::enumNameSafe(code); + if (code != Code::SUCCEEDED && code_ == Code::SUCCEEDED) { + code_ = code; + } + + if (!checkTerm()) { + LOG(WARNING) << "checkTerm() failed"; + return Code::E_OUTDATED_TERM; + } + + if (code == Code::E_RPC_FAILURE) { + appendDoublePrime(); + } + + if (code == Code::SUCCEEDED || code == Code::E_RPC_FAILURE) { + erasePrime(); + forwardToDelegateProcessor(); + } else { + abort(); + } + + // LOG(INFO) << "~processNormalLocal(), code_: " << apache::thrift::util::enumNameSafe(code); + return code_; } void ChainUpdateEdgeProcessorLocal::doRpc(folly::Promise&& promise, int retry) noexcept { - if (retry > retryLimit_) { - promise.setValue(Code::E_LEADER_CHANGED); - return; + if (retry > retryLimit_) { + promise.setValue(Code::E_LEADER_CHANGED); + return; + } + auto* iClient = env_->txnMan_->getInternalClient(); + folly::Promise p; + auto reversedReq = reverseRequest(req_); + iClient->chainUpdateEdge(reversedReq, termOfPrepare_, ver_, std::move(p)); + + auto f = p.getFuture(); + std::move(f).thenTry([=, p = std::move(promise)](auto&& t) mutable { + auto code = t.hasValue() ? t.value() : Code::E_RPC_FAILURE; + LOG(INFO) << "code = " << apache::thrift::util::enumNameSafe(code); + switch (code) { + case Code::E_LEADER_CHANGED: + doRpc(std::move(p), ++retry); + break; + default: + p.setValue(code); + break; } - auto* iClient = env_->txnMan_->getInternalClient(); - folly::Promise p; - auto reversedReq = reverseRequest(req_); - iClient->chainUpdateEdge(reversedReq, termOfPrepare_, ver_, std::move(p)); - - auto f = p.getFuture(); - std::move(f).thenTry([=, p = std::move(promise)](auto&& t) mutable { - auto code = t.hasValue() ? t.value() : Code::E_RPC_FAILURE; - LOG(INFO) << "code = " << apache::thrift::util::enumNameSafe(code); - switch (code) { - case Code::E_LEADER_CHANGED: - doRpc(std::move(p), ++retry); - break; - default: - p.setValue(code); - break; - } - return code; - }); + return code; + }); } -folly::SemiFuture -ChainUpdateEdgeProcessorLocal::processResumeRemoteLocal(Code code) { - LOG(INFO) << __func__ << "(), code = " << apache::thrift::util::enumNameSafe(code); - if (code == Code::SUCCEEDED) { - auto partId = req_.get_part_id(); - auto key = ConsistUtil::doublePrime(spaceVidLen_, partId, req_.get_edge_key()); - kvErased_.emplace_back(key); - abort(); - } - return code; +folly::SemiFuture ChainUpdateEdgeProcessorLocal::processResumeRemoteLocal(Code code) { + LOG(INFO) << __func__ << "(), code = " << apache::thrift::util::enumNameSafe(code); + if (code == Code::SUCCEEDED) { + auto partId = req_.get_part_id(); + auto key = ConsistUtil::doublePrime(spaceVidLen_, partId, req_.get_edge_key()); + kvErased_.emplace_back(key); + abort(); + } + return code; } void ChainUpdateEdgeProcessorLocal::erasePrime() { - auto key = ConsistUtil::primeKey(spaceVidLen_, partId_, req_.get_edge_key()); - kvErased_.emplace_back(std::move(key)); + auto key = ConsistUtil::primeKey(spaceVidLen_, partId_, req_.get_edge_key()); + kvErased_.emplace_back(std::move(key)); } void ChainUpdateEdgeProcessorLocal::appendDoublePrime() { - auto key = ConsistUtil::doublePrime(spaceVidLen_, partId_, req_.get_edge_key()); - std::string val; - apache::thrift::CompactSerializer::serialize(req_, &val); - val += ConsistUtil::updateIdentifier(); - kvAppend_.emplace_back(std::make_pair(std::move(key), std::move(val))); + auto key = ConsistUtil::doublePrime(spaceVidLen_, partId_, req_.get_edge_key()); + std::string val; + apache::thrift::CompactSerializer::serialize(req_, &val); + val += ConsistUtil::updateIdentifier(); + kvAppend_.emplace_back(std::make_pair(std::move(key), std::move(val))); } void ChainUpdateEdgeProcessorLocal::forwardToDelegateProcessor() { - kUpdateEdgeCounters.init("update_edge"); - UpdateEdgeProcessor::ContextAdjuster fn = [=](EdgeContext& ctx) { - ctx.kvAppend = std::move(kvAppend_); - ctx.kvErased = std::move(kvErased_); - }; - - auto* proc = UpdateEdgeProcessor::instance(env_); - proc->adjustContext(std::move(fn)); - auto f = proc->getFuture(); - proc->process(req_); - auto resp = std::move(f).get(); - code_ = getErrorCode(resp); - std::swap(resp_, resp); + kUpdateEdgeCounters.init("update_edge"); + UpdateEdgeProcessor::ContextAdjuster fn = [=](EdgeContext& ctx) { + ctx.kvAppend = std::move(kvAppend_); + ctx.kvErased = std::move(kvErased_); + }; + + auto* proc = UpdateEdgeProcessor::instance(env_); + proc->adjustContext(std::move(fn)); + auto f = proc->getFuture(); + proc->process(req_); + auto resp = std::move(f).get(); + code_ = getErrorCode(resp); + std::swap(resp_, resp); } Code ChainUpdateEdgeProcessorLocal::checkAndBuildContexts(const cpp2::UpdateEdgeRequest&) { - return Code::SUCCEEDED; + return Code::SUCCEEDED; } std::string ChainUpdateEdgeProcessorLocal::sEdgeKey(const cpp2::UpdateEdgeRequest& req) { - return ConsistUtil::edgeKey(spaceVidLen_, req.get_part_id(), req.get_edge_key()); + return ConsistUtil::edgeKey(spaceVidLen_, req.get_part_id(), req.get_edge_key()); } void ChainUpdateEdgeProcessorLocal::finish() { - LOG(INFO) << "ChainUpdateEdgeProcessorLocal::finish()"; - pushResultCode(code_, req_.get_part_id()); - onFinished(); + LOG(INFO) << "ChainUpdateEdgeProcessorLocal::finish()"; + pushResultCode(code_, req_.get_part_id()); + onFinished(); } bool ChainUpdateEdgeProcessorLocal::checkTerm() { - return env_->txnMan_->checkTerm(req_.get_space_id(), req_.get_part_id(), termOfPrepare_); + return env_->txnMan_->checkTerm(req_.get_space_id(), req_.get_part_id(), termOfPrepare_); } bool ChainUpdateEdgeProcessorLocal::checkVersion() { - if (!ver_) { - return true; - } - auto [ver, rc] = ConsistUtil::versionOfUpdateReq(env_, req_); - if (rc != Code::SUCCEEDED) { - return false; - } - return *ver_ == ver; + if (!ver_) { + return true; + } + auto [ver, rc] = ConsistUtil::versionOfUpdateReq(env_, req_); + if (rc != Code::SUCCEEDED) { + return false; + } + return *ver_ == ver; } void ChainUpdateEdgeProcessorLocal::abort() { - auto key = ConsistUtil::primeKey(spaceVidLen_, partId_, req_.get_edge_key()); - kvErased_.emplace_back(std::move(key)); - - folly::Baton baton; - env_->kvstore_->asyncMultiRemove( - req_.get_space_id(), req_.get_part_id(), std::move(kvErased_), [&](auto rc) mutable { - LOG_IF(WARNING, rc != Code::SUCCEEDED) << "error: " << static_cast(rc); - baton.post(); - }); - baton.wait(); + auto key = ConsistUtil::primeKey(spaceVidLen_, partId_, req_.get_edge_key()); + kvErased_.emplace_back(std::move(key)); + + folly::Baton baton; + env_->kvstore_->asyncMultiRemove( + req_.get_space_id(), req_.get_part_id(), std::move(kvErased_), [&](auto rc) mutable { + LOG_IF(WARNING, rc != Code::SUCCEEDED) << "error: " << static_cast(rc); + baton.post(); + }); + baton.wait(); } cpp2::UpdateEdgeRequest ChainUpdateEdgeProcessorLocal::reverseRequest( const cpp2::UpdateEdgeRequest& req) { - cpp2::UpdateEdgeRequest reversedRequest(req); - auto reversedEdgeKey = ConsistUtil::reverseEdgeKey(req.get_edge_key()); - reversedRequest.set_edge_key(reversedEdgeKey); - - auto partsNum = env_->metaClient_->partsNum(req.get_space_id()); - CHECK(partsNum.ok()); - auto srcVid = reversedRequest.get_edge_key().get_src().getStr(); - auto partId = env_->metaClient_->partId(partsNum.value(), srcVid); - // CHECK(partId.ok()); - // reversedRequest.set_part_id(partId.value()); - reversedRequest.set_part_id(partId); - - return reversedRequest; + cpp2::UpdateEdgeRequest reversedRequest(req); + auto reversedEdgeKey = ConsistUtil::reverseEdgeKey(req.get_edge_key()); + reversedRequest.set_edge_key(reversedEdgeKey); + + auto partsNum = env_->metaClient_->partsNum(req.get_space_id()); + CHECK(partsNum.ok()); + auto srcVid = reversedRequest.get_edge_key().get_src().getStr(); + auto partId = env_->metaClient_->partId(partsNum.value(), srcVid); + // CHECK(partId.ok()); + // reversedRequest.set_part_id(partId.value()); + reversedRequest.set_part_id(partId); + + return reversedRequest; } bool ChainUpdateEdgeProcessorLocal::setLock() { - auto spaceId = req_.get_space_id(); - auto* lockCore = env_->txnMan_->getLockCore(spaceId); - if (lockCore == nullptr) { - return false; - } - auto key = ConsistUtil::edgeKey(spaceVidLen_, req_.get_part_id(), req_.get_edge_key()); - lk_ = std::make_unique>(lockCore, key); - return lk_->isLocked(); + auto spaceId = req_.get_space_id(); + auto* lockCore = env_->txnMan_->getLockCore(spaceId); + if (lockCore == nullptr) { + return false; + } + auto key = ConsistUtil::edgeKey(spaceVidLen_, req_.get_part_id(), req_.get_edge_key()); + lk_ = std::make_unique>(lockCore, key); + return lk_->isLocked(); } int64_t ChainUpdateEdgeProcessorLocal::getVersion(const cpp2::UpdateEdgeRequest& req) { - int64_t invalidVer = -1; - auto spaceId = req.get_space_id(); - auto vIdLen = env_->metaClient_->getSpaceVidLen(spaceId); - if (!vIdLen.ok()) { - LOG(WARNING) << vIdLen.status().toString(); - return invalidVer; - } - auto partId = req.get_part_id(); - auto key = ConsistUtil::edgeKey(vIdLen.value(), partId, req.get_edge_key()); - return ConsistUtil::getSingleEdgeVer(env_->kvstore_, spaceId, partId, key);; + int64_t invalidVer = -1; + auto spaceId = req.get_space_id(); + auto vIdLen = env_->metaClient_->getSpaceVidLen(spaceId); + if (!vIdLen.ok()) { + LOG(WARNING) << vIdLen.status().toString(); + return invalidVer; + } + auto partId = req.get_part_id(); + auto key = ConsistUtil::edgeKey(vIdLen.value(), partId, req.get_edge_key()); + return ConsistUtil::getSingleEdgeVer(env_->kvstore_, spaceId, partId, key); } nebula::cpp2::ErrorCode ChainUpdateEdgeProcessorLocal::getErrorCode( const cpp2::UpdateResponse& resp) { - auto& respCommon = resp.get_result(); - auto& parts = respCommon.get_failed_parts(); - if (parts.empty()) { - return nebula::cpp2::ErrorCode::SUCCEEDED; - } - return parts.front().get_code(); + auto& respCommon = resp.get_result(); + auto& parts = respCommon.get_failed_parts(); + if (parts.empty()) { + return nebula::cpp2::ErrorCode::SUCCEEDED; + } + return parts.front().get_code(); } } // namespace storage } // namespace nebula - diff --git a/src/storage/transaction/ChainUpdateEdgeProcessorLocal.h b/src/storage/transaction/ChainUpdateEdgeProcessorLocal.h index b9b9702fdc9..6980449c273 100644 --- a/src/storage/transaction/ChainUpdateEdgeProcessorLocal.h +++ b/src/storage/transaction/ChainUpdateEdgeProcessorLocal.h @@ -6,12 +6,13 @@ #pragma once +#include + +#include "common/utils/MemoryLockWrapper.h" #include "storage/query/QueryBaseProcessor.h" -#include "storage/transaction/ConsistUtil.h" #include "storage/transaction/ChainBaseProcessor.h" +#include "storage/transaction/ConsistUtil.h" #include "storage/transaction/TransactionManager.h" -#include "common/utils/MemoryLockWrapper.h" -#include namespace nebula { namespace storage { @@ -19,74 +20,75 @@ namespace storage { class ChainUpdateEdgeProcessorLocal : public QueryBaseProcessor, public ChainBaseProcessor { - friend struct ChainUpdateEdgeTestHelper; -public: - using Code = ::nebula::cpp2::ErrorCode; - static ChainUpdateEdgeProcessorLocal* instance(StorageEnv* env) { - return new ChainUpdateEdgeProcessorLocal(env); - } + friend struct ChainUpdateEdgeTestHelper; + + public: + using Code = ::nebula::cpp2::ErrorCode; + static ChainUpdateEdgeProcessorLocal* instance(StorageEnv* env) { + return new ChainUpdateEdgeProcessorLocal(env); + } - void process(const cpp2::UpdateEdgeRequest& req) override; + void process(const cpp2::UpdateEdgeRequest& req) override; - folly::SemiFuture prepareLocal() override; + folly::SemiFuture prepareLocal() override; - folly::SemiFuture processRemote(Code code) override; + folly::SemiFuture processRemote(Code code) override; - folly::SemiFuture processLocal(Code code) override; + folly::SemiFuture processLocal(Code code) override; - void onProcessFinished() override {} + void onProcessFinished() override {} - void finish() override; + void finish() override; - virtual ~ChainUpdateEdgeProcessorLocal() = default; + virtual ~ChainUpdateEdgeProcessorLocal() = default; -protected: - explicit ChainUpdateEdgeProcessorLocal(StorageEnv* env) - : QueryBaseProcessor(env, nullptr) {} + protected: + explicit ChainUpdateEdgeProcessorLocal(StorageEnv* env) + : QueryBaseProcessor(env, nullptr) {} - std::string edgeKey(const cpp2::UpdateEdgeRequest& req); + std::string edgeKey(const cpp2::UpdateEdgeRequest& req); - void doRpc(folly::Promise&& promise, int retry = 0) noexcept; + void doRpc(folly::Promise&& promise, int retry = 0) noexcept; - bool checkTerm(); + bool checkTerm(); - bool checkVersion(); + bool checkVersion(); - folly::SemiFuture processResumeRemoteLocal(Code code); + folly::SemiFuture processResumeRemoteLocal(Code code); - folly::SemiFuture processNormalLocal(Code code); + folly::SemiFuture processNormalLocal(Code code); - void abort(); + void abort(); - bool prepareRequest(const cpp2::UpdateEdgeRequest& req); + bool prepareRequest(const cpp2::UpdateEdgeRequest& req); - void erasePrime(); + void erasePrime(); - void appendDoublePrime(); + void appendDoublePrime(); - void forwardToDelegateProcessor(); + void forwardToDelegateProcessor(); - std::string sEdgeKey(const cpp2::UpdateEdgeRequest& req); + std::string sEdgeKey(const cpp2::UpdateEdgeRequest& req); - cpp2::UpdateEdgeRequest reverseRequest(const cpp2::UpdateEdgeRequest& req); + cpp2::UpdateEdgeRequest reverseRequest(const cpp2::UpdateEdgeRequest& req); - bool setLock(); + bool setLock(); - int64_t getVersion(const cpp2::UpdateEdgeRequest& req); + int64_t getVersion(const cpp2::UpdateEdgeRequest& req); - nebula::cpp2::ErrorCode getErrorCode(const cpp2::UpdateResponse& resp); + nebula::cpp2::ErrorCode getErrorCode(const cpp2::UpdateResponse& resp); - Code checkAndBuildContexts(const cpp2::UpdateEdgeRequest& req) override; + Code checkAndBuildContexts(const cpp2::UpdateEdgeRequest& req) override; -protected: - cpp2::UpdateEdgeRequest req_; - std::unique_ptr lk_; - PartitionID partId_; - int retryLimit_{10}; - TermID termOfPrepare_{-1}; - std::vector kvErased_; - std::vector kvAppend_; - folly::Optional ver_{folly::none}; + protected: + cpp2::UpdateEdgeRequest req_; + std::unique_ptr lk_; + PartitionID partId_; + int retryLimit_{10}; + TermID termOfPrepare_{-1}; + std::vector kvErased_; + std::vector kvAppend_; + folly::Optional ver_{folly::none}; }; } // namespace storage diff --git a/src/storage/transaction/ChainUpdateEdgeProcessorRemote.cpp b/src/storage/transaction/ChainUpdateEdgeProcessorRemote.cpp index 27bbf11376a..7c194f45c6a 100644 --- a/src/storage/transaction/ChainUpdateEdgeProcessorRemote.cpp +++ b/src/storage/transaction/ChainUpdateEdgeProcessorRemote.cpp @@ -4,8 +4,9 @@ * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ -#include "storage/mutate/UpdateEdgeProcessor.h" #include "storage/transaction/ChainUpdateEdgeProcessorRemote.h" + +#include "storage/mutate/UpdateEdgeProcessor.h" #include "storage/transaction/ConsistUtil.h" #include "storage/transaction/TransactionManager.h" @@ -15,51 +16,51 @@ namespace storage { using Code = ::nebula::cpp2::ErrorCode; void ChainUpdateEdgeProcessorRemote::process(const cpp2::ChainUpdateEdgeRequest& req) { - auto rc = Code::SUCCEEDED; - if (!checkTerm(req)) { - LOG(WARNING) << "invalid term"; - rc = Code::E_OUTDATED_TERM; - } + auto rc = Code::SUCCEEDED; + if (!checkTerm(req)) { + LOG(WARNING) << "invalid term"; + rc = Code::E_OUTDATED_TERM; + } - if (!checkVersion(req)) { - LOG(WARNING) << "invalid term"; - rc = Code::E_OUTDATED_EDGE; - } + if (!checkVersion(req)) { + LOG(WARNING) << "invalid term"; + rc = Code::E_OUTDATED_EDGE; + } - auto& updateRequest = req.get_update_edge_request(); - if (rc != Code::SUCCEEDED) { - pushResultCode(rc, updateRequest.get_part_id()); - } else { - updateEdge(req); - } - onFinished(); + auto& updateRequest = req.get_update_edge_request(); + if (rc != Code::SUCCEEDED) { + pushResultCode(rc, updateRequest.get_part_id()); + } else { + updateEdge(req); + } + onFinished(); } bool ChainUpdateEdgeProcessorRemote::checkTerm(const cpp2::ChainUpdateEdgeRequest& req) { - auto partId = req.get_update_edge_request().get_part_id(); - return env_->txnMan_->checkTerm(req.get_space_id(), partId, req.get_term()); + auto partId = req.get_update_edge_request().get_part_id(); + return env_->txnMan_->checkTerm(req.get_space_id(), partId, req.get_term()); } bool ChainUpdateEdgeProcessorRemote::checkVersion(const cpp2::ChainUpdateEdgeRequest& req) { - if (!req.edge_version_ref()) { - return true; - } - auto verExpected = *req.edge_version_ref(); - auto& updateRequest = req.get_update_edge_request(); - auto [verActually, rc] = ConsistUtil::versionOfUpdateReq(env_, updateRequest); - if (rc != Code::SUCCEEDED) { - return false; - } - return verExpected >= verActually; + if (!req.edge_version_ref()) { + return true; + } + auto verExpected = *req.edge_version_ref(); + auto& updateRequest = req.get_update_edge_request(); + auto [verActually, rc] = ConsistUtil::versionOfUpdateReq(env_, updateRequest); + if (rc != Code::SUCCEEDED) { + return false; + } + return verExpected >= verActually; } // forward to UpdateEdgeProcessor void ChainUpdateEdgeProcessorRemote::updateEdge(const cpp2::ChainUpdateEdgeRequest& req) { - auto* proc = UpdateEdgeProcessor::instance(env_, counters_); - auto f = proc->getFuture(); - proc->process(req.get_update_edge_request()); - auto resp = std::move(f).get(); - std::swap(resp_, resp); + auto* proc = UpdateEdgeProcessor::instance(env_, counters_); + auto f = proc->getFuture(); + proc->process(req.get_update_edge_request()); + auto resp = std::move(f).get(); + std::swap(resp_, resp); } } // namespace storage diff --git a/src/storage/transaction/ChainUpdateEdgeProcessorRemote.h b/src/storage/transaction/ChainUpdateEdgeProcessorRemote.h index 7456514d189..3001700d686 100644 --- a/src/storage/transaction/ChainUpdateEdgeProcessorRemote.h +++ b/src/storage/transaction/ChainUpdateEdgeProcessorRemote.h @@ -6,33 +6,33 @@ #pragma once +#include "common/utils/MemoryLockWrapper.h" #include "storage/BaseProcessor.h" #include "storage/transaction/TransactionManager.h" -#include "common/utils/MemoryLockWrapper.h" namespace nebula { namespace storage { class ChainUpdateEdgeProcessorRemote : public BaseProcessor { -public: - static ChainUpdateEdgeProcessorRemote* instance(StorageEnv* env) { - return new ChainUpdateEdgeProcessorRemote(env); - } + public: + static ChainUpdateEdgeProcessorRemote* instance(StorageEnv* env) { + return new ChainUpdateEdgeProcessorRemote(env); + } - void process(const cpp2::ChainUpdateEdgeRequest& req); + void process(const cpp2::ChainUpdateEdgeRequest& req); -private: - explicit ChainUpdateEdgeProcessorRemote(StorageEnv* env) - : BaseProcessor(env) {} + private: + explicit ChainUpdateEdgeProcessorRemote(StorageEnv* env) + : BaseProcessor(env) {} - bool checkTerm(const cpp2::ChainUpdateEdgeRequest& req); + bool checkTerm(const cpp2::ChainUpdateEdgeRequest& req); - bool checkVersion(const cpp2::ChainUpdateEdgeRequest& req); + bool checkVersion(const cpp2::ChainUpdateEdgeRequest& req); - void updateEdge(const cpp2::ChainUpdateEdgeRequest& req); + void updateEdge(const cpp2::ChainUpdateEdgeRequest& req); -private: - std::unique_ptr lk_; + private: + std::unique_ptr lk_; }; } // namespace storage diff --git a/src/storage/transaction/ConsistUtil.cpp b/src/storage/transaction/ConsistUtil.cpp index b953414a7fb..c5b340611b4 100644 --- a/src/storage/transaction/ConsistUtil.cpp +++ b/src/storage/transaction/ConsistUtil.cpp @@ -4,179 +4,173 @@ * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ +#include "storage/transaction/ConsistUtil.h" + #include -#include #include +#include -#include "storage/transaction/ConsistUtil.h" #include "common/utils/NebulaKeyUtils.h" namespace nebula { namespace storage { -static const std::string kPrimeTable{"__prime__"}; // NOLINT -static const std::string kDoublePrimeTable{"__prime_prime__"}; // NOLINT +static const std::string kPrimeTable{"__prime__"}; // NOLINT +static const std::string kDoublePrimeTable{"__prime_prime__"}; // NOLINT static const std::string kTempRequestTable{"__temp_request__"}; // NOLINT -std::string ConsistUtil::primeTable() { - return kPrimeTable; -} +std::string ConsistUtil::primeTable() { return kPrimeTable; } -std::string ConsistUtil::doublePrimeTable() { - return kDoublePrimeTable; -} +std::string ConsistUtil::doublePrimeTable() { return kDoublePrimeTable; } // std::string ConsistUtil::tempRequestTable() { // return kTempRequestTable; // } std::string ConsistUtil::primePrefix(PartitionID partId) { - return kPrimeTable + NebulaKeyUtils::edgePrefix(partId); + return kPrimeTable + NebulaKeyUtils::edgePrefix(partId); } std::string ConsistUtil::doublePrimePrefix(PartitionID partId) { - return kDoublePrimeTable + NebulaKeyUtils::edgePrefix(partId); + return kDoublePrimeTable + NebulaKeyUtils::edgePrefix(partId); } std::string ConsistUtil::primeKey(size_t vIdLen, PartitionID partId, const cpp2::EdgeKey& edgeKey) { - return kPrimeTable + NebulaKeyUtils::edgeKey(vIdLen, - partId, - edgeKey.get_src().getStr(), - edgeKey.get_edge_type(), - edgeKey.get_ranking(), - edgeKey.get_dst().getStr()); + return kPrimeTable + NebulaKeyUtils::edgeKey(vIdLen, + partId, + edgeKey.get_src().getStr(), + edgeKey.get_edge_type(), + edgeKey.get_ranking(), + edgeKey.get_dst().getStr()); } std::string ConsistUtil::doublePrime(size_t vIdLen, PartitionID partId, const cpp2::EdgeKey& key) { - return kDoublePrimeTable + NebulaKeyUtils::edgeKey(vIdLen, - partId, - key.get_src().getStr(), - key.get_edge_type(), - key.get_ranking(), - key.get_dst().getStr()); + return kDoublePrimeTable + NebulaKeyUtils::edgeKey(vIdLen, + partId, + key.get_src().getStr(), + key.get_edge_type(), + key.get_ranking(), + key.get_dst().getStr()); } RequestType ConsistUtil::parseType(folly::StringPiece val) { - char identifier = val.str().back(); - switch (identifier) { - case 'u': - return RequestType::UPDATE; - case 'a': - return RequestType::INSERT; - default: - LOG(FATAL) << "shoule not happend, identifier is " << identifier; - // return RequestType::UNKNOWN; - } + char identifier = val.str().back(); + switch (identifier) { + case 'u': + return RequestType::UPDATE; + case 'a': + return RequestType::INSERT; + default: + LOG(FATAL) << "shoule not happend, identifier is " << identifier; + // return RequestType::UNKNOWN; + } } cpp2::UpdateEdgeRequest ConsistUtil::parseUpdateRequest(folly::StringPiece val) { - cpp2::UpdateEdgeRequest req; - apache::thrift::CompactSerializer::deserialize(val, req); - return req; + cpp2::UpdateEdgeRequest req; + apache::thrift::CompactSerializer::deserialize(val, req); + return req; } cpp2::AddEdgesRequest ConsistUtil::parseAddRequest(folly::StringPiece val) { - cpp2::AddEdgesRequest req; - apache::thrift::CompactSerializer::deserialize(val, req); - return req; + cpp2::AddEdgesRequest req; + apache::thrift::CompactSerializer::deserialize(val, req); + return req; } std::string ConsistUtil::strUUID() { - static boost::uuids::random_generator gen; - return boost::uuids::to_string(gen()); + static boost::uuids::random_generator gen; + return boost::uuids::to_string(gen()); } std::string ConsistUtil::ConsistUtil::edgeKey(size_t vIdLen, PartitionID partId, const cpp2::EdgeKey& key) { - return NebulaKeyUtils::edgeKey(vIdLen, - partId, - key.get_src().getStr(), - *key.edge_type_ref(), - *key.ranking_ref(), - (*key.dst_ref()).getStr()); + return NebulaKeyUtils::edgeKey(vIdLen, + partId, + key.get_src().getStr(), + *key.edge_type_ref(), + *key.ranking_ref(), + (*key.dst_ref()).getStr()); } std::vector ConsistUtil::getMultiEdgeVers(kvstore::KVStore* store, - GraphSpaceID spaceId, - PartitionID partId, - const std::vector& keys) { - std::vector ret(keys.size()); - std::vector _keys(keys); - auto rc = nebula::cpp2::ErrorCode::SUCCEEDED; - std::vector status; - std::vector vals; - std::tie(rc, status) = - store->multiGet(spaceId, partId, std::move(_keys), &vals); - if (rc != nebula::cpp2::ErrorCode::SUCCEEDED && - rc != nebula::cpp2::ErrorCode::E_PARTIAL_RESULT) { - return ret; - } - for (auto i = 0U; i != ret.size(); ++i) { - ret[i] = getTimestamp(vals[i]); - } + GraphSpaceID spaceId, + PartitionID partId, + const std::vector& keys) { + std::vector ret(keys.size()); + std::vector _keys(keys); + auto rc = nebula::cpp2::ErrorCode::SUCCEEDED; + std::vector status; + std::vector vals; + std::tie(rc, status) = store->multiGet(spaceId, partId, std::move(_keys), &vals); + if (rc != nebula::cpp2::ErrorCode::SUCCEEDED && rc != nebula::cpp2::ErrorCode::E_PARTIAL_RESULT) { return ret; + } + for (auto i = 0U; i != ret.size(); ++i) { + ret[i] = getTimestamp(vals[i]); + } + return ret; } // return -1 if edge version not exist int64_t ConsistUtil::getSingleEdgeVer(kvstore::KVStore* store, - GraphSpaceID spaceId, - PartitionID partId, - const std::string& key) { - static int64_t invalidEdgeVer = -1; - std::string val; - auto rc = store->get(spaceId, partId, key, &val); - if (rc != nebula::cpp2::ErrorCode::SUCCEEDED) { - return invalidEdgeVer; - } - return getTimestamp(val); + GraphSpaceID spaceId, + PartitionID partId, + const std::string& key) { + static int64_t invalidEdgeVer = -1; + std::string val; + auto rc = store->get(spaceId, partId, key, &val); + if (rc != nebula::cpp2::ErrorCode::SUCCEEDED) { + return invalidEdgeVer; + } + return getTimestamp(val); } int64_t ConsistUtil::getTimestamp(const std::string& val) noexcept { - return *reinterpret_cast(val.data() + (val.size() - sizeof(int64_t))); + return *reinterpret_cast(val.data() + (val.size() - sizeof(int64_t))); } cpp2::AddEdgesRequest ConsistUtil::makeDirectAddReq(const cpp2::ChainAddEdgesRequest& req) { - cpp2::AddEdgesRequest ret; - ret.set_space_id(req.get_space_id()); - ret.set_parts(req.get_parts()); - ret.set_prop_names(req.get_prop_names()); - ret.set_if_not_exists(req.get_if_not_exists()); - return ret; + cpp2::AddEdgesRequest ret; + ret.set_space_id(req.get_space_id()); + ret.set_parts(req.get_parts()); + ret.set_prop_names(req.get_prop_names()); + ret.set_if_not_exists(req.get_if_not_exists()); + return ret; } cpp2::EdgeKey ConsistUtil::reverseEdgeKey(const cpp2::EdgeKey& edgeKey) { - cpp2::EdgeKey reversedKey(edgeKey); - std::swap(*reversedKey.src_ref(), *reversedKey.dst_ref()); - *reversedKey.edge_type_ref() = 0 - edgeKey.get_edge_type(); - return reversedKey; + cpp2::EdgeKey reversedKey(edgeKey); + std::swap(*reversedKey.src_ref(), *reversedKey.dst_ref()); + *reversedKey.edge_type_ref() = 0 - edgeKey.get_edge_type(); + return reversedKey; } void ConsistUtil::reverseEdgeKeyInplace(cpp2::EdgeKey& edgeKey) { - cpp2::EdgeKey reversedKey(edgeKey); - std::swap(*edgeKey.src_ref(), *edgeKey.dst_ref()); - *edgeKey.edge_type_ref() = 0 - edgeKey.get_edge_type(); + cpp2::EdgeKey reversedKey(edgeKey); + std::swap(*edgeKey.src_ref(), *edgeKey.dst_ref()); + *edgeKey.edge_type_ref() = 0 - edgeKey.get_edge_type(); } std::pair ConsistUtil::versionOfUpdateReq( - StorageEnv* env, - const cpp2::UpdateEdgeRequest& req) { - int64_t ver = -1; - auto rc = nebula::cpp2::ErrorCode::SUCCEEDED; - - do { - auto spaceId = req.get_space_id(); - auto stVidLen = env->metaClient_->getSpaceVidLen(spaceId); - if (!stVidLen.ok()) { - rc = nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND; - break; - } - auto vIdLen = stVidLen.value(); - auto partId = req.get_part_id(); - auto key = ConsistUtil::edgeKey(vIdLen, partId, req.get_edge_key()); - ver = ConsistUtil::getSingleEdgeVer(env->kvstore_, spaceId, partId, key); - } while (0); - - return std::make_pair(ver, rc); + StorageEnv* env, const cpp2::UpdateEdgeRequest& req) { + int64_t ver = -1; + auto rc = nebula::cpp2::ErrorCode::SUCCEEDED; + + do { + auto spaceId = req.get_space_id(); + auto stVidLen = env->metaClient_->getSpaceVidLen(spaceId); + if (!stVidLen.ok()) { + rc = nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND; + break; + } + auto vIdLen = stVidLen.value(); + auto partId = req.get_part_id(); + auto key = ConsistUtil::edgeKey(vIdLen, partId, req.get_edge_key()); + ver = ConsistUtil::getSingleEdgeVer(env->kvstore_, spaceId, partId, key); + } while (0); + + return std::make_pair(ver, rc); } // std::string ConsistUtil::primeKey(size_t vIdLen, diff --git a/src/storage/transaction/ConsistUtil.h b/src/storage/transaction/ConsistUtil.h index e919ffc46c6..d84585bac73 100644 --- a/src/storage/transaction/ConsistUtil.h +++ b/src/storage/transaction/ConsistUtil.h @@ -6,97 +6,93 @@ #pragma once -#include "storage/CommonUtils.h" -#include "interface/gen-cpp2/storage_types.h" #include "common/time/WallClock.h" #include "common/utils/MemoryLockWrapper.h" #include "common/utils/NebulaKeyUtils.h" +#include "interface/gen-cpp2/storage_types.h" #include "kvstore/KVStore.h" +#include "storage/CommonUtils.h" namespace nebula { namespace storage { enum class RequestType { - UNKNOWN, - INSERT, - UPDATE, + UNKNOWN, + INSERT, + UPDATE, }; enum class ChainProcessType { - NORMAL = 0, - RESUME_CHAIN = 1, - RESUME_REMOTE = 2, + NORMAL = 0, + RESUME_CHAIN = 1, + RESUME_REMOTE = 2, }; - class ConsistUtil final { -public: -static std::string primeTable(); + public: + static std::string primeTable(); -static std::string doublePrimeTable(); + static std::string doublePrimeTable(); -static std::string edgeKey(size_t vIdLen, PartitionID partId, const cpp2::EdgeKey& key); + static std::string edgeKey(size_t vIdLen, PartitionID partId, const cpp2::EdgeKey& key); -static std::string primeKey(size_t vIdLen, - PartitionID partId, - const cpp2::EdgeKey& edgeKey); + static std::string primeKey(size_t vIdLen, PartitionID partId, const cpp2::EdgeKey& edgeKey); -static std::string doublePrime(size_t vIdLen, PartitionID partId, const cpp2::EdgeKey& edgeKey); + static std::string doublePrime(size_t vIdLen, PartitionID partId, const cpp2::EdgeKey& edgeKey); -static std::string primePrefix(PartitionID partId); + static std::string primePrefix(PartitionID partId); -static std::string doublePrimePrefix(PartitionID partId); + static std::string doublePrimePrefix(PartitionID partId); -static std::string primeKey(size_t vIdLen, - PartitionID partId, - const VertexID& srcId, - EdgeType type, - EdgeRanking rank, - const VertexID& dstId); + static std::string primeKey(size_t vIdLen, + PartitionID partId, + const VertexID& srcId, + EdgeType type, + EdgeRanking rank, + const VertexID& dstId); -static std::string doublePrime(size_t vIdLen, - PartitionID partId, - const VertexID& srcId, - EdgeType type, - EdgeRanking rank, - const VertexID& dstId); + static std::string doublePrime(size_t vIdLen, + PartitionID partId, + const VertexID& srcId, + EdgeType type, + EdgeRanking rank, + const VertexID& dstId); -static RequestType parseType(folly::StringPiece val); + static RequestType parseType(folly::StringPiece val); -static cpp2::UpdateEdgeRequest parseUpdateRequest(folly::StringPiece val); + static cpp2::UpdateEdgeRequest parseUpdateRequest(folly::StringPiece val); -static cpp2::AddEdgesRequest parseAddRequest(folly::StringPiece val); + static cpp2::AddEdgesRequest parseAddRequest(folly::StringPiece val); -static std::string strUUID(); + static std::string strUUID(); -static std::string tempRequestTable(); + static std::string tempRequestTable(); -static std::vector getMultiEdgeVers(kvstore::KVStore* store, - GraphSpaceID spaceId, - PartitionID partId, - const std::vector& keys); + static std::vector getMultiEdgeVers(kvstore::KVStore* store, + GraphSpaceID spaceId, + PartitionID partId, + const std::vector& keys); -// return -1 if edge version not exist -static int64_t getSingleEdgeVer(kvstore::KVStore* store, - GraphSpaceID spaceId, - PartitionID partId, - const std::string& key); + // return -1 if edge version not exist + static int64_t getSingleEdgeVer(kvstore::KVStore* store, + GraphSpaceID spaceId, + PartitionID partId, + const std::string& key); -static int64_t getTimestamp(const std::string& val) noexcept; + static int64_t getTimestamp(const std::string& val) noexcept; -static cpp2::AddEdgesRequest makeDirectAddReq(const cpp2::ChainAddEdgesRequest& req); + static cpp2::AddEdgesRequest makeDirectAddReq(const cpp2::ChainAddEdgesRequest& req); -static cpp2::EdgeKey reverseEdgeKey(const cpp2::EdgeKey& edgeKey); + static cpp2::EdgeKey reverseEdgeKey(const cpp2::EdgeKey& edgeKey); -static void reverseEdgeKeyInplace(cpp2::EdgeKey& edgeKey); + static void reverseEdgeKeyInplace(cpp2::EdgeKey& edgeKey); -static std::string insertIdentifier() noexcept { return "a"; } + static std::string insertIdentifier() noexcept { return "a"; } -static std::string updateIdentifier() noexcept { return "u"; } + static std::string updateIdentifier() noexcept { return "u"; } -static std::pair versionOfUpdateReq( - StorageEnv* env, - const cpp2::UpdateEdgeRequest& req); + static std::pair versionOfUpdateReq( + StorageEnv* env, const cpp2::UpdateEdgeRequest& req); }; } // namespace storage diff --git a/src/storage/transaction/ResumeAddEdgeProcessor.cpp b/src/storage/transaction/ResumeAddEdgeProcessor.cpp index 043a79db150..5f8a1007c2a 100644 --- a/src/storage/transaction/ResumeAddEdgeProcessor.cpp +++ b/src/storage/transaction/ResumeAddEdgeProcessor.cpp @@ -11,51 +11,51 @@ namespace storage { ResumeAddEdgeProcessor::ResumeAddEdgeProcessor(StorageEnv* env, const std::string& val) : ChainAddEdgesProcessorLocal(env) { - req_ = ConsistUtil::parseAddRequest(val); - ChainAddEdgesProcessorLocal::prepareRequest(req_); + req_ = ConsistUtil::parseAddRequest(val); + ChainAddEdgesProcessorLocal::prepareRequest(req_); } folly::SemiFuture ResumeAddEdgeProcessor::prepareLocal() { - if (code_ != Code::SUCCEEDED) { - return code_; - } - auto spaceId = req_.get_space_id(); - std::vector keys = sEdgeKey(req_); - auto vers = ConsistUtil::getMultiEdgeVers(env_->kvstore_, spaceId, localPartId_, keys); - edgeVer_ = vers.front(); - - return Code::SUCCEEDED; + if (code_ != Code::SUCCEEDED) { + return code_; + } + auto spaceId = req_.get_space_id(); + std::vector keys = sEdgeKey(req_); + auto vers = ConsistUtil::getMultiEdgeVers(env_->kvstore_, spaceId, localPartId_, keys); + edgeVer_ = vers.front(); + + return Code::SUCCEEDED; } folly::SemiFuture ResumeAddEdgeProcessor::processRemote(Code code) { - return ChainAddEdgesProcessorLocal::processRemote(code); + return ChainAddEdgesProcessorLocal::processRemote(code); } folly::SemiFuture ResumeAddEdgeProcessor::processLocal(Code code) { - setErrorCode(code); - - if (!checkTerm(req_)) { - LOG(WARNING) << "E_OUTDATED_TERM"; - return Code::E_OUTDATED_TERM; - } - - if (!checkVersion(req_)) { - LOG(WARNING) << "E_OUTDATED_EDGE"; - return Code::E_OUTDATED_EDGE; - } - - if (code == Code::E_RPC_FAILURE) { - kvAppend_ = ChainAddEdgesProcessorLocal::makeDoublePrime(); - } - - if (code == Code::E_RPC_FAILURE || code == Code::SUCCEEDED) { - // if there are something wrong other than rpc failure - // we need to keep the resume retry(by not remove those prime key) - erasePrime(); - return ChainAddEdgesProcessorLocal::forwardToDelegateProcessor(); - } - - return code; + setErrorCode(code); + + if (!checkTerm(req_)) { + LOG(WARNING) << "E_OUTDATED_TERM"; + return Code::E_OUTDATED_TERM; + } + + if (!checkVersion(req_)) { + LOG(WARNING) << "E_OUTDATED_EDGE"; + return Code::E_OUTDATED_EDGE; + } + + if (code == Code::E_RPC_FAILURE) { + kvAppend_ = ChainAddEdgesProcessorLocal::makeDoublePrime(); + } + + if (code == Code::E_RPC_FAILURE || code == Code::SUCCEEDED) { + // if there are something wrong other than rpc failure + // we need to keep the resume retry(by not remove those prime key) + erasePrime(); + return ChainAddEdgesProcessorLocal::forwardToDelegateProcessor(); + } + + return code; } } // namespace storage diff --git a/src/storage/transaction/ResumeAddEdgeProcessor.h b/src/storage/transaction/ResumeAddEdgeProcessor.h index 9ffcd5b5227..8ce3cf3e377 100644 --- a/src/storage/transaction/ResumeAddEdgeProcessor.h +++ b/src/storage/transaction/ResumeAddEdgeProcessor.h @@ -12,21 +12,21 @@ namespace nebula { namespace storage { class ResumeAddEdgeProcessor : public ChainAddEdgesProcessorLocal { -public: - static ResumeAddEdgeProcessor* instance(StorageEnv* env, const std::string& val) { - return new ResumeAddEdgeProcessor(env, val); - } + public: + static ResumeAddEdgeProcessor* instance(StorageEnv* env, const std::string& val) { + return new ResumeAddEdgeProcessor(env, val); + } - folly::SemiFuture prepareLocal() override; + folly::SemiFuture prepareLocal() override; - folly::SemiFuture processRemote(nebula::cpp2::ErrorCode code) override; + folly::SemiFuture processRemote(nebula::cpp2::ErrorCode code) override; - folly::SemiFuture processLocal(nebula::cpp2::ErrorCode code) override; + folly::SemiFuture processLocal(nebula::cpp2::ErrorCode code) override; - virtual ~ResumeAddEdgeProcessor() = default; + virtual ~ResumeAddEdgeProcessor() = default; -protected: - ResumeAddEdgeProcessor(StorageEnv* env, const std::string& val); + protected: + ResumeAddEdgeProcessor(StorageEnv* env, const std::string& val); }; } // namespace storage diff --git a/src/storage/transaction/ResumeAddEdgeRemoteProcessor.cpp b/src/storage/transaction/ResumeAddEdgeRemoteProcessor.cpp index 9958c02fa7f..61d5fdf799f 100644 --- a/src/storage/transaction/ResumeAddEdgeRemoteProcessor.cpp +++ b/src/storage/transaction/ResumeAddEdgeRemoteProcessor.cpp @@ -11,81 +11,80 @@ namespace storage { ResumeAddEdgeRemoteProcessor::ResumeAddEdgeRemoteProcessor(StorageEnv* env, const std::string& val) : ChainAddEdgesProcessorLocal(env) { - req_ = ConsistUtil::parseAddRequest(val); - ChainAddEdgesProcessorLocal::prepareRequest(req_); + req_ = ConsistUtil::parseAddRequest(val); + ChainAddEdgesProcessorLocal::prepareRequest(req_); } folly::SemiFuture ResumeAddEdgeRemoteProcessor::prepareLocal() { - if (!lockEdges()) { - return Code::E_WRITE_WRITE_CONFLICT; - } - - if (!checkTerm(req_)) { - LOG(WARNING) << "E_OUTDATED_TERM"; - return Code::E_OUTDATED_TERM; - } - - if (!checkVersion(req_)) { - LOG(WARNING) << "E_OUTDATED_EDGE"; - return Code::E_OUTDATED_EDGE; - } - - auto spaceId = req_.get_space_id(); - std::vector keys = sEdgeKey(req_); - auto vers = ConsistUtil::getMultiEdgeVers(env_->kvstore_, spaceId, localPartId_, keys); - edgeVer_ = vers.front(); - - return Code::SUCCEEDED; + if (!lockEdges()) { + return Code::E_WRITE_WRITE_CONFLICT; + } + + if (!checkTerm(req_)) { + LOG(WARNING) << "E_OUTDATED_TERM"; + return Code::E_OUTDATED_TERM; + } + + if (!checkVersion(req_)) { + LOG(WARNING) << "E_OUTDATED_EDGE"; + return Code::E_OUTDATED_EDGE; + } + + auto spaceId = req_.get_space_id(); + std::vector keys = sEdgeKey(req_); + auto vers = ConsistUtil::getMultiEdgeVers(env_->kvstore_, spaceId, localPartId_, keys); + edgeVer_ = vers.front(); + + return Code::SUCCEEDED; } folly::SemiFuture ResumeAddEdgeRemoteProcessor::processRemote(Code code) { - return ChainAddEdgesProcessorLocal::processRemote(code); + return ChainAddEdgesProcessorLocal::processRemote(code); } folly::SemiFuture ResumeAddEdgeRemoteProcessor::processLocal(Code code) { - if (!checkTerm(req_)) { - LOG(WARNING) << "E_OUTDATED_TERM"; - return Code::E_OUTDATED_TERM; - } - - if (!checkVersion(req_)) { - LOG(WARNING) << "E_OUTDATED_EDGE"; - return Code::E_OUTDATED_EDGE; - } - - if (code == Code::E_OUTDATED_TERM) { - // E_OUTDATED_TERM indicate this host is no longer the leader of curr part - // any following kv operation will fail - // due to not allowed to write from follower - return code; - } - - if (code == Code::E_RPC_FAILURE) { - // nothing to do, as we are already an rpc failure - } - - if (code == Code::SUCCEEDED) { - // if there are something wrong other than rpc failure - // we need to keep the resume retry(by not remove those prime key) - ChainAddEdgesProcessorLocal::eraseDoublePrime(); - return forwardToDelegateProcessor(); - } - + if (!checkTerm(req_)) { + LOG(WARNING) << "E_OUTDATED_TERM"; + return Code::E_OUTDATED_TERM; + } + + if (!checkVersion(req_)) { + LOG(WARNING) << "E_OUTDATED_EDGE"; + return Code::E_OUTDATED_EDGE; + } + + if (code == Code::E_OUTDATED_TERM) { + // E_OUTDATED_TERM indicate this host is no longer the leader of curr part + // any following kv operation will fail + // due to not allowed to write from follower return code; + } + + if (code == Code::E_RPC_FAILURE) { + // nothing to do, as we are already an rpc failure + } + + if (code == Code::SUCCEEDED) { + // if there are something wrong other than rpc failure + // we need to keep the resume retry(by not remove those prime key) + ChainAddEdgesProcessorLocal::eraseDoublePrime(); + return forwardToDelegateProcessor(); + } + + return code; } bool ResumeAddEdgeRemoteProcessor::lockEdges() { - std::vector keys; - auto spaceId = req_.get_space_id(); - auto partId = req_.get_parts().begin()->first; - for (auto& edge : req_.get_parts().begin()->second) { - keys.emplace_back(ConsistUtil::edgeKey(spaceVidLen_, partId, edge.get_key())); - } - resumeLock_ = std::make_unique(env_->txnMan_, spaceId, keys[0]); - - return resumeLock_->isLocked(); + std::vector keys; + auto spaceId = req_.get_space_id(); + auto partId = req_.get_parts().begin()->first; + for (auto& edge : req_.get_parts().begin()->second) { + keys.emplace_back(ConsistUtil::edgeKey(spaceVidLen_, partId, edge.get_key())); + } + resumeLock_ = std::make_unique(env_->txnMan_, spaceId, keys[0]); + + return resumeLock_->isLocked(); } } // namespace storage } // namespace nebula - diff --git a/src/storage/transaction/ResumeAddEdgeRemoteProcessor.h b/src/storage/transaction/ResumeAddEdgeRemoteProcessor.h index 91d26641da9..4ab508a9e92 100644 --- a/src/storage/transaction/ResumeAddEdgeRemoteProcessor.h +++ b/src/storage/transaction/ResumeAddEdgeRemoteProcessor.h @@ -13,25 +13,25 @@ namespace nebula { namespace storage { class ResumeAddEdgeRemoteProcessor : public ChainAddEdgesProcessorLocal { -public: - static ResumeAddEdgeRemoteProcessor* instance(StorageEnv* env, const std::string& val) { - return new ResumeAddEdgeRemoteProcessor(env, val); - } + public: + static ResumeAddEdgeRemoteProcessor* instance(StorageEnv* env, const std::string& val) { + return new ResumeAddEdgeRemoteProcessor(env, val); + } - folly::SemiFuture prepareLocal() override; + folly::SemiFuture prepareLocal() override; - folly::SemiFuture processRemote(nebula::cpp2::ErrorCode code) override; + folly::SemiFuture processRemote(nebula::cpp2::ErrorCode code) override; - folly::SemiFuture processLocal(nebula::cpp2::ErrorCode code) override; + folly::SemiFuture processLocal(nebula::cpp2::ErrorCode code) override; - virtual ~ResumeAddEdgeRemoteProcessor() = default; + virtual ~ResumeAddEdgeRemoteProcessor() = default; -protected: - ResumeAddEdgeRemoteProcessor(StorageEnv* env, const std::string& val); + protected: + ResumeAddEdgeRemoteProcessor(StorageEnv* env, const std::string& val); - bool lockEdges(); + bool lockEdges(); - std::unique_ptr resumeLock_; + std::unique_ptr resumeLock_; }; } // namespace storage diff --git a/src/storage/transaction/ResumeLockGuard.h b/src/storage/transaction/ResumeLockGuard.h index 081d9cdcaf1..12f94fba972 100644 --- a/src/storage/transaction/ResumeLockGuard.h +++ b/src/storage/transaction/ResumeLockGuard.h @@ -6,8 +6,8 @@ #pragma once - #include + #include "common/utils/MemoryLockCore.h" namespace nebula { @@ -15,51 +15,47 @@ namespace storage { // RAII style to easily control the lock acquire / release class ResumeLockGuard { -public: - ResumeLockGuard(TransactionManager* txnMgr, GraphSpaceID spaceId, const std::string& key) - : key_(key) { - lockCore_ = txnMgr->getLockCore(spaceId); - if (txnMgr->takeDanglingEdge(spaceId, key)) { - locked_ = true; - } else { - locked_ = lockCore_->try_lock(key_); - } + public: + ResumeLockGuard(TransactionManager* txnMgr, GraphSpaceID spaceId, const std::string& key) + : key_(key) { + lockCore_ = txnMgr->getLockCore(spaceId); + if (txnMgr->takeDanglingEdge(spaceId, key)) { + locked_ = true; + } else { + locked_ = lockCore_->try_lock(key_); } + } - ResumeLockGuard(const ResumeLockGuard&) = delete; + ResumeLockGuard(const ResumeLockGuard&) = delete; - ResumeLockGuard(ResumeLockGuard&& lg) noexcept - : key_(std::move(lg.key_)), lockCore_(lg.lockCore_), locked_(lg.locked_) {} + ResumeLockGuard(ResumeLockGuard&& lg) noexcept + : key_(std::move(lg.key_)), lockCore_(lg.lockCore_), locked_(lg.locked_) {} - ResumeLockGuard& operator=(const ResumeLockGuard&) = delete; + ResumeLockGuard& operator=(const ResumeLockGuard&) = delete; - ResumeLockGuard& operator=(ResumeLockGuard&& lg) noexcept { - if (this != &lg) { - lockCore_ = lg.lockCore_; - key_ = std::move(lg.key_); - locked_ = lg.locked_; - } - return *this; + ResumeLockGuard& operator=(ResumeLockGuard&& lg) noexcept { + if (this != &lg) { + lockCore_ = lg.lockCore_; + key_ = std::move(lg.key_); + locked_ = lg.locked_; } + return *this; + } - ~ResumeLockGuard() { - if (locked_) { - lockCore_->unlock(key_); - } + ~ResumeLockGuard() { + if (locked_) { + lockCore_->unlock(key_); } + } - bool isLocked() const noexcept { - return locked_; - } + bool isLocked() const noexcept { return locked_; } - operator bool() const noexcept { - return isLocked(); - } + operator bool() const noexcept { return isLocked(); } -protected: - std::string key_; - MemoryLockCore* lockCore_; - bool locked_{false}; + protected: + std::string key_; + MemoryLockCore* lockCore_; + bool locked_{false}; }; } // namespace storage diff --git a/src/storage/transaction/ResumeUpdateProcessor.cpp b/src/storage/transaction/ResumeUpdateProcessor.cpp index b5bce58d8fb..76f8eb81582 100644 --- a/src/storage/transaction/ResumeUpdateProcessor.cpp +++ b/src/storage/transaction/ResumeUpdateProcessor.cpp @@ -4,64 +4,63 @@ * attached with Common Clause Condition 1.0, found in the LICENSES directory. */ -#include #include "storage/transaction/ResumeUpdateProcessor.h" +#include + namespace nebula { namespace storage { ResumeUpdateProcessor::ResumeUpdateProcessor(StorageEnv* env, const std::string& val) : ChainUpdateEdgeProcessorLocal(env) { - req_ = ConsistUtil::parseUpdateRequest(val); - ChainUpdateEdgeProcessorLocal::prepareRequest(req_); + req_ = ConsistUtil::parseUpdateRequest(val); + ChainUpdateEdgeProcessorLocal::prepareRequest(req_); } folly::SemiFuture ResumeUpdateProcessor::prepareLocal() { - if (!setLock()) { - LOG(INFO) << "set lock failed, return E_DATA_CONFLICT_ERROR"; - return Code::E_DATA_CONFLICT_ERROR; - } - ver_ = getVersion(req_); + if (!setLock()) { + LOG(INFO) << "set lock failed, return E_DATA_CONFLICT_ERROR"; + return Code::E_DATA_CONFLICT_ERROR; + } + ver_ = getVersion(req_); - return Code::SUCCEEDED; + return Code::SUCCEEDED; } folly::SemiFuture ResumeUpdateProcessor::processRemote(Code code) { - return ChainUpdateEdgeProcessorLocal::processRemote(code); + return ChainUpdateEdgeProcessorLocal::processRemote(code); } folly::SemiFuture ResumeUpdateProcessor::processLocal(Code code) { - setErrorCode(code); + setErrorCode(code); - if (!checkTerm()) { - LOG(WARNING) << "E_OUTDATED_TERM"; - return Code::E_OUTDATED_TERM; - } + if (!checkTerm()) { + LOG(WARNING) << "E_OUTDATED_TERM"; + return Code::E_OUTDATED_TERM; + } - if (!checkVersion()) { - LOG(WARNING) << "E_OUTDATED_EDGE"; - return Code::E_OUTDATED_EDGE; - } + if (!checkVersion()) { + LOG(WARNING) << "E_OUTDATED_EDGE"; + return Code::E_OUTDATED_EDGE; + } - if (code == Code::E_RPC_FAILURE) { - appendDoublePrime(); - } + if (code == Code::E_RPC_FAILURE) { + appendDoublePrime(); + } - if (code == Code::E_RPC_FAILURE || code == Code::SUCCEEDED) { - // if there are something wrong other than rpc failure - // we need to keep the resume retry(by not remove those prime key) - auto key = ConsistUtil::primeKey(spaceVidLen_, partId_, req_.get_edge_key()); - kvErased_.emplace_back(std::move(key)); - forwardToDelegateProcessor(); - return code_; - } + if (code == Code::E_RPC_FAILURE || code == Code::SUCCEEDED) { + // if there are something wrong other than rpc failure + // we need to keep the resume retry(by not remove those prime key) + auto key = ConsistUtil::primeKey(spaceVidLen_, partId_, req_.get_edge_key()); + kvErased_.emplace_back(std::move(key)); + forwardToDelegateProcessor(); + return code_; + } - return code; + return code; } -void ResumeUpdateProcessor::finish() { - onFinished(); -} +void ResumeUpdateProcessor::finish() { onFinished(); } } // namespace storage } // namespace nebula diff --git a/src/storage/transaction/ResumeUpdateProcessor.h b/src/storage/transaction/ResumeUpdateProcessor.h index 32ee96d0d79..ed9e8e0ff33 100644 --- a/src/storage/transaction/ResumeUpdateProcessor.h +++ b/src/storage/transaction/ResumeUpdateProcessor.h @@ -18,27 +18,27 @@ namespace storage { * it will create this processor to resume the complete update process */ class ResumeUpdateProcessor : public ChainUpdateEdgeProcessorLocal { -public: - static ResumeUpdateProcessor* instance(StorageEnv* env, const std::string& val) { - return new ResumeUpdateProcessor(env, val); - } + public: + static ResumeUpdateProcessor* instance(StorageEnv* env, const std::string& val) { + return new ResumeUpdateProcessor(env, val); + } - folly::SemiFuture prepareLocal() override; + folly::SemiFuture prepareLocal() override; - folly::SemiFuture processRemote(nebula::cpp2::ErrorCode code) override; + folly::SemiFuture processRemote(nebula::cpp2::ErrorCode code) override; - folly::SemiFuture processLocal(nebula::cpp2::ErrorCode code) override; + folly::SemiFuture processLocal(nebula::cpp2::ErrorCode code) override; - void finish() override; + void finish() override; - virtual ~ResumeUpdateProcessor() = default; + virtual ~ResumeUpdateProcessor() = default; -protected: - ResumeUpdateProcessor(StorageEnv* env, const std::string& val); + protected: + ResumeUpdateProcessor(StorageEnv* env, const std::string& val); - bool lockEdge(); + bool lockEdge(); - std::unique_ptr resumeLock_; + std::unique_ptr resumeLock_; }; } // namespace storage diff --git a/src/storage/transaction/ResumeUpdateRemoteProcessor.cpp b/src/storage/transaction/ResumeUpdateRemoteProcessor.cpp index 2defcd3696c..8618d0a68c6 100644 --- a/src/storage/transaction/ResumeUpdateRemoteProcessor.cpp +++ b/src/storage/transaction/ResumeUpdateRemoteProcessor.cpp @@ -11,61 +11,59 @@ namespace storage { ResumeUpdateRemoteProcessor::ResumeUpdateRemoteProcessor(StorageEnv* env, const std::string& val) : ChainUpdateEdgeProcessorLocal(env) { - req_ = ConsistUtil::parseUpdateRequest(val); - ChainUpdateEdgeProcessorLocal::prepareRequest(req_); + req_ = ConsistUtil::parseUpdateRequest(val); + ChainUpdateEdgeProcessorLocal::prepareRequest(req_); } folly::SemiFuture ResumeUpdateRemoteProcessor::prepareLocal() { - if (!lockEdge(req_)) { - LOG(INFO) << "set lock failed, return E_DATA_CONFLICT_ERROR"; - return Code::E_DATA_CONFLICT_ERROR; - } - ver_ = getVersion(req_); + if (!lockEdge(req_)) { + LOG(INFO) << "set lock failed, return E_DATA_CONFLICT_ERROR"; + return Code::E_DATA_CONFLICT_ERROR; + } + ver_ = getVersion(req_); - return Code::SUCCEEDED; + return Code::SUCCEEDED; } folly::SemiFuture ResumeUpdateRemoteProcessor::processRemote(Code code) { - return ChainUpdateEdgeProcessorLocal::processRemote(code); + return ChainUpdateEdgeProcessorLocal::processRemote(code); } folly::SemiFuture ResumeUpdateRemoteProcessor::processLocal(Code code) { - setErrorCode(code); + setErrorCode(code); - if (!checkTerm()) { - LOG(WARNING) << "E_OUTDATED_TERM"; - return Code::E_OUTDATED_TERM; - } + if (!checkTerm()) { + LOG(WARNING) << "E_OUTDATED_TERM"; + return Code::E_OUTDATED_TERM; + } - if (!checkVersion()) { - LOG(WARNING) << "E_OUTDATED_EDGE"; - return Code::E_OUTDATED_EDGE; - } - - if (code == Code::SUCCEEDED) { - // if there are something wrong other than rpc failure - // we need to keep the resume retry(by not remove those prime key) - auto key = ConsistUtil::doublePrime(spaceVidLen_, partId_, req_.get_edge_key()); - kvErased_.emplace_back(std::move(key)); - forwardToDelegateProcessor(); - return code; - } else { - // we can't decide if the double prime shoule be deleted. - // so do nothing - } + if (!checkVersion()) { + LOG(WARNING) << "E_OUTDATED_EDGE"; + return Code::E_OUTDATED_EDGE; + } + if (code == Code::SUCCEEDED) { + // if there are something wrong other than rpc failure + // we need to keep the resume retry(by not remove those prime key) + auto key = ConsistUtil::doublePrime(spaceVidLen_, partId_, req_.get_edge_key()); + kvErased_.emplace_back(std::move(key)); + forwardToDelegateProcessor(); return code; + } else { + // we can't decide if the double prime shoule be deleted. + // so do nothing + } + + return code; } bool ResumeUpdateRemoteProcessor::lockEdge(const cpp2::UpdateEdgeRequest& req) { - auto key = sEdgeKey(req); - resumeLock_ = std::make_unique(env_->txnMan_, req.get_space_id(), key); - return resumeLock_->isLocked(); + auto key = sEdgeKey(req); + resumeLock_ = std::make_unique(env_->txnMan_, req.get_space_id(), key); + return resumeLock_->isLocked(); } -void ResumeUpdateRemoteProcessor::finish() { - onFinished(); -} +void ResumeUpdateRemoteProcessor::finish() { onFinished(); } } // namespace storage } // namespace nebula diff --git a/src/storage/transaction/ResumeUpdateRemoteProcessor.h b/src/storage/transaction/ResumeUpdateRemoteProcessor.h index 72a35161b9d..b1cf9515ff0 100644 --- a/src/storage/transaction/ResumeUpdateRemoteProcessor.h +++ b/src/storage/transaction/ResumeUpdateRemoteProcessor.h @@ -18,27 +18,27 @@ namespace storage { * it will create this processor to resume the complete update process */ class ResumeUpdateRemoteProcessor : public ChainUpdateEdgeProcessorLocal { -public: - static ResumeUpdateRemoteProcessor* instance(StorageEnv* env, const std::string& val) { - return new ResumeUpdateRemoteProcessor(env, val); - } + public: + static ResumeUpdateRemoteProcessor* instance(StorageEnv* env, const std::string& val) { + return new ResumeUpdateRemoteProcessor(env, val); + } - folly::SemiFuture prepareLocal() override; + folly::SemiFuture prepareLocal() override; - folly::SemiFuture processRemote(nebula::cpp2::ErrorCode code) override; + folly::SemiFuture processRemote(nebula::cpp2::ErrorCode code) override; - folly::SemiFuture processLocal(nebula::cpp2::ErrorCode code) override; + folly::SemiFuture processLocal(nebula::cpp2::ErrorCode code) override; - void finish() override; + void finish() override; - virtual ~ResumeUpdateRemoteProcessor() = default; + virtual ~ResumeUpdateRemoteProcessor() = default; -protected: - ResumeUpdateRemoteProcessor(StorageEnv* env, const std::string& val); + protected: + ResumeUpdateRemoteProcessor(StorageEnv* env, const std::string& val); - bool lockEdge(const cpp2::UpdateEdgeRequest& req); + bool lockEdge(const cpp2::UpdateEdgeRequest& req); - std::unique_ptr resumeLock_; + std::unique_ptr resumeLock_; }; } // namespace storage