From 9255df29f1e948358d72f65553ad04d0f3bc0b42 Mon Sep 17 00:00:00 2001 From: Sophie <84560950+Sophie-Xie@users.noreply.github.com> Date: Mon, 28 Mar 2022 16:25:31 +0800 Subject: [PATCH 01/24] =?UTF-8?q?fix=20docker=20image=20version=20?= =?UTF-8?q?=EF=BC=88typo=EF=BC=89=20(#4087)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix docker image version * fix package command Co-authored-by: HarrisChu <1726587+HarrisChu@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- docker/Dockerfile | 5 +---- docker/Dockerfile.graphd | 4 +--- docker/Dockerfile.metad | 4 +--- docker/Dockerfile.storaged | 4 +--- docker/Dockerfile.tools | 4 +--- 6 files changed, 6 insertions(+), 17 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 070dabdc7e9..e15a2406e80 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -110,7 +110,7 @@ jobs: push: true build-args: | BRANCH=${{ steps.tagname.outputs.tag }} - VERSON=${{ steps.tagname.outputs.tagnum }} + VERSION=${{ steps.tagname.outputs.tagnum }} test: name: test diff --git a/docker/Dockerfile b/docker/Dockerfile index 0a7a8bfcda4..87eb94cc7fd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,10 +6,7 @@ ARG VERSION= COPY . /home/nebula/BUILD RUN cd /home/nebula/BUILD/package \ - && [[ -z "${VERSION}" ]] \ - && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \ - || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON - + && ./package.sh -n OFF -v "${VERSION}" -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON FROM centos:7 as graphd COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-common.rpm /usr/local/nebula/nebula-common.rpm diff --git a/docker/Dockerfile.graphd b/docker/Dockerfile.graphd index 75bf3e3984e..39cc470075a 100644 --- a/docker/Dockerfile.graphd +++ b/docker/Dockerfile.graphd @@ -6,9 +6,7 @@ ARG VERSION= COPY . /home/nebula/BUILD RUN cd /home/nebula/BUILD/package \ - && [[ -z "${VERSION}" ]] \ - && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \ - || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON + && ./package.sh -n OFF -v "${VERSION}" -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON FROM centos:7 diff --git a/docker/Dockerfile.metad b/docker/Dockerfile.metad index b4ab34f8409..d82f67b46cb 100644 --- a/docker/Dockerfile.metad +++ b/docker/Dockerfile.metad @@ -6,9 +6,7 @@ ARG VERSION= COPY . /home/nebula/BUILD RUN cd /home/nebula/BUILD/package \ - && [[ -z "${VERSION}" ]] \ - && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \ - || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON + && ./package.sh -n OFF -v "${VERSION}" -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON FROM centos:7 diff --git a/docker/Dockerfile.storaged b/docker/Dockerfile.storaged index 6ca29687e37..d131dd05cb7 100644 --- a/docker/Dockerfile.storaged +++ b/docker/Dockerfile.storaged @@ -6,9 +6,7 @@ ARG VERSION= COPY . /home/nebula/BUILD RUN cd /home/nebula/BUILD/package \ - && [[ -z "${VERSION}" ]] \ - && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \ - || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON + && ./package.sh -n OFF -v "${VERSION}" -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON FROM centos:7 diff --git a/docker/Dockerfile.tools b/docker/Dockerfile.tools index fd696809109..5bb004fb378 100644 --- a/docker/Dockerfile.tools +++ b/docker/Dockerfile.tools @@ -6,9 +6,7 @@ ARG VERSION= COPY . /home/nebula/BUILD RUN cd /home/nebula/BUILD/package \ - && [[ -z "${VERSION}" ]] \ - && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \ - || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON + && ./package.sh -n OFF -v "${VERSION}" -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON FROM centos:7 From 02b8b3976fbe7f35400f4d0477506962efef299a Mon Sep 17 00:00:00 2001 From: shylock <33566796+Shylock-Hg@users.noreply.github.com> Date: Tue, 29 Mar 2022 10:55:04 +0800 Subject: [PATCH 02/24] Fix breakpad config. (#4086) * Fix breakpad config. * Fix breakpad. --- CMakeLists.txt | 1 - cmake/nebula/BreakPadConfig.cmake | 12 ------------ cmake/nebula/ThirdPartyConfig.cmake | 13 +++++++++++++ 3 files changed, 13 insertions(+), 13 deletions(-) delete mode 100644 cmake/nebula/BreakPadConfig.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 877974b7b48..d5c55196392 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,6 @@ endif() list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/nebula) -include(BreakPadConfig) include(PlatformCheck) include(NebulaCMakeMacros) include(GeneralCMakeOptions) diff --git a/cmake/nebula/BreakPadConfig.cmake b/cmake/nebula/BreakPadConfig.cmake deleted file mode 100644 index e038a11cb00..00000000000 --- a/cmake/nebula/BreakPadConfig.cmake +++ /dev/null @@ -1,12 +0,0 @@ -# Breakpad -if (ENABLE_BREAKPAD) - if (NOT ${CMAKE_BUILD_TYPE} STREQUAL "Debug" AND NOT ${CMAKE_BUILD_TYPE} STREQUAL "RelWithDebInfo") - MESSAGE(FATAL_ERROR "Breakpad need debug info.") - endif() -endif() -if (NOT ${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "x86_64") - set(ENABLE_BREAKPAD OFF) -endif() -if (ENABLE_BREAKPAD) - add_compile_options(-DENABLE_BREAKPAD=1) -endif() diff --git a/cmake/nebula/ThirdPartyConfig.cmake b/cmake/nebula/ThirdPartyConfig.cmake index 3b7ef413eff..da60815594d 100644 --- a/cmake/nebula/ThirdPartyConfig.cmake +++ b/cmake/nebula/ThirdPartyConfig.cmake @@ -76,6 +76,19 @@ else() set(GETTIME_LIB) endif() +# Breakpad +if (ENABLE_BREAKPAD) + if (NOT ${CMAKE_BUILD_TYPE} STREQUAL "Debug" AND NOT ${CMAKE_BUILD_TYPE} STREQUAL "RelWithDebInfo") + MESSAGE(FATAL_ERROR "Breakpad need debug info.") + endif() +endif() +if (NOT ${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "x86_64") + set(ENABLE_BREAKPAD OFF) +endif() +if (ENABLE_BREAKPAD) + add_compile_options(-DENABLE_BREAKPAD=1) +endif() + message("") find_package(Bzip2 REQUIRED) From 0649da8933f17c621b9e7bbd734b955ebf1f1668 Mon Sep 17 00:00:00 2001 From: yaphet <4414314+darionyaphet@users.noreply.github.com> Date: Tue, 29 Mar 2022 15:39:20 +0800 Subject: [PATCH 03/24] Migrate download and ingest into job manager (#3994) * download and ingest job * fix comment * remove some useless flag --- src/clients/meta/MetaClient.cpp | 44 ---- src/clients/meta/MetaClient.h | 7 - src/daemons/MetaDaemon.cpp | 4 +- src/daemons/MetaDaemonInit.cpp | 19 +- src/daemons/MetaDaemonInit.h | 5 +- src/daemons/StandAloneDaemon.cpp | 7 +- src/graph/executor/CMakeLists.txt | 2 - src/graph/executor/Executor.cpp | 8 - src/graph/executor/admin/DownloadExecutor.cpp | 33 --- src/graph/executor/admin/DownloadExecutor.h | 25 --- src/graph/executor/admin/IngestExecutor.cpp | 28 --- src/graph/executor/admin/IngestExecutor.h | 25 --- src/graph/planner/plan/Admin.h | 49 ----- src/graph/planner/plan/PlanNode.cpp | 4 - src/graph/planner/plan/PlanNode.h | 2 - src/graph/service/PermissionCheck.cpp | 4 +- src/graph/validator/AdminJobValidator.h | 6 +- src/graph/validator/CMakeLists.txt | 2 - src/graph/validator/DownloadValidator.cpp | 30 --- src/graph/validator/DownloadValidator.h | 30 --- src/graph/validator/IngestValidator.cpp | 22 -- src/graph/validator/IngestValidator.h | 29 --- src/graph/validator/Validator.cpp | 6 - src/kvstore/NebulaStore.cpp | 48 ++-- src/meta/CMakeLists.txt | 7 +- src/meta/http/CMakeLists.txt | 5 +- src/meta/http/MetaHttpDownloadHandler.cpp | 205 ------------------ src/meta/http/MetaHttpDownloadHandler.h | 67 ------ src/meta/http/MetaHttpIngestHandler.cpp | 158 -------------- src/meta/http/MetaHttpIngestHandler.h | 61 ------ src/meta/http/test/CMakeLists.txt | 31 --- .../http/test/MetaHttpDownloadHandlerTest.cpp | 123 ----------- .../http/test/MetaHttpIngestHandlerTest.cpp | 120 ---------- .../http/test/MetaHttpReplaceHandlerTest.cpp | 2 - src/meta/processors/job/CompactJobExecutor.h | 2 +- .../processors/job/DataBalanceJobExecutor.h | 2 +- .../processors/job/DownloadJobExecutor.cpp | 119 ++++++++++ src/meta/processors/job/DownloadJobExecutor.h | 50 +++++ src/meta/processors/job/FlushJobExecutor.h | 2 +- src/meta/processors/job/GetStatsProcessor.h | 2 +- src/meta/processors/job/IngestJobExecutor.cpp | 54 +++++ src/meta/processors/job/IngestJobExecutor.h | 40 ++++ src/meta/processors/job/JobDescription.h | 6 +- src/meta/processors/job/JobExecutor.cpp | 15 +- src/meta/processors/job/JobManager.cpp | 15 +- src/meta/processors/job/JobManager.h | 2 + src/meta/test/JobManagerTest.cpp | 84 +++++-- src/meta/test/MockHdfsHelper.h | 34 +-- src/parser/AdminSentences.cpp | 5 +- src/parser/AdminSentences.h | 9 - src/parser/MutateSentences.cpp | 9 - src/parser/MutateSentences.h | 69 ------ src/parser/Sentence.h | 2 - src/parser/parser.yy | 32 ++- src/parser/test/ParserTest.cpp | 6 + src/storage/CMakeLists.txt | 4 +- src/storage/StorageServer.cpp | 12 - src/storage/admin/AdminTask.cpp | 8 + src/storage/admin/AdminTask.h | 19 +- src/storage/admin/AdminTaskProcessor.h | 1 - src/storage/admin/CompactTask.cpp | 4 + src/storage/admin/CompactTask.h | 3 +- src/storage/admin/DownloadTask.cpp | 59 +++++ src/storage/admin/DownloadTask.h | 42 ++++ src/storage/admin/FlushTask.cpp | 8 +- src/storage/admin/FlushTask.h | 4 +- src/storage/admin/IngestTask.cpp | 51 +++++ src/storage/admin/IngestTask.h | 30 +++ src/storage/admin/RebuildEdgeIndexTask.cpp | 4 +- src/storage/admin/RebuildFTIndexTask.cpp | 7 +- src/storage/admin/RebuildFTIndexTask.h | 4 +- src/storage/admin/RebuildIndexTask.cpp | 8 +- src/storage/admin/RebuildIndexTask.h | 2 + src/storage/admin/StatsTask.cpp | 9 +- src/storage/admin/StatsTask.h | 3 +- src/storage/admin/StopAdminTaskProcessor.h | 1 - .../http/StorageHttpDownloadHandler.cpp | 197 ----------------- src/storage/http/StorageHttpDownloadHandler.h | 69 ------ src/storage/http/StorageHttpIngestHandler.cpp | 103 --------- src/storage/http/StorageHttpIngestHandler.h | 53 ----- src/storage/test/AdminTaskManagerTest.cpp | 7 +- src/storage/test/CMakeLists.txt | 36 +-- .../test/StorageHttpDownloadHandlerTest.cpp | 116 ---------- .../test/StorageHttpIngestHandlerTest.cpp | 104 --------- src/tools/CMakeLists.txt | 59 +++++ src/tools/db-dump/CMakeLists.txt | 58 ----- src/tools/db-upgrade/CMakeLists.txt | 55 +---- src/tools/meta-dump/CMakeLists.txt | 56 +---- src/tools/simple-kv-verify/CMakeLists.txt | 54 +---- src/tools/storage-perf/CMakeLists.txt | 61 +----- src/webservice/Common.cpp | 3 - src/webservice/WebService.cpp | 1 + 92 files changed, 756 insertions(+), 2336 deletions(-) delete mode 100644 src/graph/executor/admin/DownloadExecutor.cpp delete mode 100644 src/graph/executor/admin/DownloadExecutor.h delete mode 100644 src/graph/executor/admin/IngestExecutor.cpp delete mode 100644 src/graph/executor/admin/IngestExecutor.h delete mode 100644 src/graph/validator/DownloadValidator.cpp delete mode 100644 src/graph/validator/DownloadValidator.h delete mode 100644 src/graph/validator/IngestValidator.cpp delete mode 100644 src/graph/validator/IngestValidator.h delete mode 100644 src/meta/http/MetaHttpDownloadHandler.cpp delete mode 100644 src/meta/http/MetaHttpDownloadHandler.h delete mode 100644 src/meta/http/MetaHttpIngestHandler.cpp delete mode 100644 src/meta/http/MetaHttpIngestHandler.h delete mode 100644 src/meta/http/test/MetaHttpDownloadHandlerTest.cpp delete mode 100644 src/meta/http/test/MetaHttpIngestHandlerTest.cpp create mode 100644 src/meta/processors/job/DownloadJobExecutor.cpp create mode 100644 src/meta/processors/job/DownloadJobExecutor.h create mode 100644 src/meta/processors/job/IngestJobExecutor.cpp create mode 100644 src/meta/processors/job/IngestJobExecutor.h create mode 100644 src/storage/admin/DownloadTask.cpp create mode 100644 src/storage/admin/DownloadTask.h create mode 100644 src/storage/admin/IngestTask.cpp create mode 100644 src/storage/admin/IngestTask.h delete mode 100644 src/storage/http/StorageHttpDownloadHandler.cpp delete mode 100644 src/storage/http/StorageHttpDownloadHandler.h delete mode 100644 src/storage/http/StorageHttpIngestHandler.cpp delete mode 100644 src/storage/http/StorageHttpIngestHandler.h delete mode 100644 src/storage/test/StorageHttpDownloadHandlerTest.cpp delete mode 100644 src/storage/test/StorageHttpIngestHandlerTest.cpp diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index eb0806989a0..0371034b818 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -28,8 +28,6 @@ #include "version/Version.h" #include "webservice/Common.h" -DECLARE_int32(ws_meta_http_port); - DEFINE_uint32(expired_time_factor, 5, "The factor of expired time based on heart beat interval"); DEFINE_int32(heartbeat_interval_secs, 10, "Heartbeat interval in seconds"); DEFINE_int32(meta_client_retry_times, 3, "meta client retry times, 0 means no retry"); @@ -3530,48 +3528,6 @@ folly::Future> MetaClient::getWorkerId(std::string ipAddr) { return future; } -folly::Future> MetaClient::download(const std::string& hdfsHost, - int32_t hdfsPort, - const std::string& hdfsPath, - GraphSpaceID spaceId) { - auto url = folly::stringPrintf("http://%s:%d/download-dispatch?host=%s&port=%d&path=%s&space=%d", - leader_.host.c_str(), - FLAGS_ws_meta_http_port, - hdfsHost.c_str(), - hdfsPort, - hdfsPath.c_str(), - spaceId); - auto func = [url] { - auto result = http::HttpClient::get(url); - if (result.ok() && result.value() == "SSTFile dispatch successfully") { - LOG(INFO) << "Download Successfully"; - return true; - } else { - LOG(ERROR) << "Download Failed: " << result.value(); - return false; - } - }; - return folly::async(func); -} - -folly::Future> MetaClient::ingest(GraphSpaceID spaceId) { - auto url = folly::stringPrintf("http://%s:%d/ingest-dispatch?space=%d", - leader_.host.c_str(), - FLAGS_ws_meta_http_port, - spaceId); - auto func = [url] { - auto result = http::HttpClient::get(url); - if (result.ok() && result.value() == "SSTFile ingest successfully") { - LOG(INFO) << "Ingest Successfully"; - return true; - } else { - LOG(ERROR) << "Ingest Failed"; - return false; - } - }; - return folly::async(func); -} - folly::Future> MetaClient::getSegmentId(int64_t length) { auto req = cpp2::GetSegmentIdReq(); req.length_ref() = length; diff --git a/src/clients/meta/MetaClient.h b/src/clients/meta/MetaClient.h index 43c281b3629..7de27e61f7a 100644 --- a/src/clients/meta/MetaClient.h +++ b/src/clients/meta/MetaClient.h @@ -633,13 +633,6 @@ class MetaClient : public BaseMetaClient { nebula::cpp2::ErrorCode taskErrCode, cpp2::StatsItem* statisticItem); - folly::Future> download(const std::string& hdfsHost, - int32_t hdfsPort, - const std::string& hdfsPath, - GraphSpaceID spaceId); - - folly::Future> ingest(GraphSpaceID spaceId); - folly::Future> getWorkerId(std::string ipAddr); folly::Future> getSegmentId(int64_t length) override; diff --git a/src/daemons/MetaDaemon.cpp b/src/daemons/MetaDaemon.cpp index c6a88cf6a7b..735a8a548db 100644 --- a/src/daemons/MetaDaemon.cpp +++ b/src/daemons/MetaDaemon.cpp @@ -26,8 +26,6 @@ #include "meta/MetaServiceHandler.h" #include "meta/MetaVersionMan.h" #include "meta/RootUserMan.h" -#include "meta/http/MetaHttpDownloadHandler.h" -#include "meta/http/MetaHttpIngestHandler.h" #include "meta/http/MetaHttpReplaceHostHandler.h" #include "meta/processors/job/JobManager.h" #include "meta/stats/MetaStats.h" @@ -152,7 +150,7 @@ int main(int argc, char* argv[]) { pool->start(FLAGS_meta_http_thread_num, "http thread pool"); auto webSvc = std::make_unique(); - status = initWebService(webSvc.get(), gKVStore.get(), helper.get(), pool.get()); + status = initWebService(webSvc.get(), gKVStore.get()); if (!status.ok()) { LOG(ERROR) << "Init web service failed: " << status; return EXIT_FAILURE; diff --git a/src/daemons/MetaDaemonInit.cpp b/src/daemons/MetaDaemonInit.cpp index 4ed4d3e3f25..25261014255 100644 --- a/src/daemons/MetaDaemonInit.cpp +++ b/src/daemons/MetaDaemonInit.cpp @@ -23,8 +23,6 @@ #include "meta/KVBasedClusterIdMan.h" #include "meta/MetaServiceHandler.h" #include "meta/MetaVersionMan.h" -#include "meta/http/MetaHttpDownloadHandler.h" -#include "meta/http/MetaHttpIngestHandler.h" #include "meta/http/MetaHttpReplaceHostHandler.h" #include "meta/processors/job/JobManager.h" #include "meta/stats/MetaStats.h" @@ -45,7 +43,7 @@ DEFINE_int32(meta_num_io_threads, 16, "Number of IO threads"); DEFINE_int32(meta_num_worker_threads, 32, "Number of workers"); DEFINE_string(meta_data_path, "", "Root data path"); DECLARE_string(meta_server_addrs); // use define from grap flags. -DECLARE_int32(ws_meta_http_port); +DEFINE_int32(ws_meta_http_port, 11000, "Port to listen on Meta with HTTP protocol"); #endif using nebula::web::PathParams; @@ -160,22 +158,9 @@ std::unique_ptr initKV(std::vector p return kvstore; } -nebula::Status initWebService(nebula::WebService* svc, - nebula::kvstore::KVStore* kvstore, - nebula::hdfs::HdfsCommandHelper* helper, - nebula::thread::GenericThreadPool* pool) { +nebula::Status initWebService(nebula::WebService* svc, nebula::kvstore::KVStore* kvstore) { LOG(INFO) << "Starting Meta HTTP Service"; auto& router = svc->router(); - router.get("/download-dispatch").handler([kvstore, helper, pool](PathParams&&) { - auto handler = new nebula::meta::MetaHttpDownloadHandler(); - handler->init(kvstore, helper, pool); - return handler; - }); - router.get("/ingest-dispatch").handler([kvstore, pool](PathParams&&) { - auto handler = new nebula::meta::MetaHttpIngestHandler(); - handler->init(kvstore, pool); - return handler; - }); router.get("/replace").handler([kvstore](PathParams&&) { auto handler = new nebula::meta::MetaHttpReplaceHostHandler(); handler->init(kvstore); diff --git a/src/daemons/MetaDaemonInit.h b/src/daemons/MetaDaemonInit.h index 3d40ded967e..0a94ae4bbd4 100644 --- a/src/daemons/MetaDaemonInit.h +++ b/src/daemons/MetaDaemonInit.h @@ -17,8 +17,5 @@ nebula::ClusterID& metaClusterId(); std::unique_ptr initKV(std::vector peers, nebula::HostAddr localhost); -nebula::Status initWebService(nebula::WebService* svc, - nebula::kvstore::KVStore* kvstore, - nebula::hdfs::HdfsCommandHelper* helper, - nebula::thread::GenericThreadPool* pool); +nebula::Status initWebService(nebula::WebService* svc, nebula::kvstore::KVStore* kvstore); #endif diff --git a/src/daemons/StandAloneDaemon.cpp b/src/daemons/StandAloneDaemon.cpp index 1e44e8a47bc..2f36a4f98cd 100644 --- a/src/daemons/StandAloneDaemon.cpp +++ b/src/daemons/StandAloneDaemon.cpp @@ -29,8 +29,6 @@ #include "meta/MetaServiceHandler.h" #include "meta/MetaVersionMan.h" #include "meta/RootUserMan.h" -#include "meta/http/MetaHttpDownloadHandler.h" -#include "meta/http/MetaHttpIngestHandler.h" #include "meta/http/MetaHttpReplaceHostHandler.h" #include "meta/processors/job/JobManager.h" #include "meta/stats/MetaStats.h" @@ -209,11 +207,8 @@ int main(int argc, char *argv[]) { return; } LOG(INFO) << "Start http service"; - auto helper = std::make_unique(); - auto pool = std::make_unique(); - pool->start(FLAGS_meta_http_thread_num, "http thread pool"); auto webSvc = std::make_unique(); - status = initWebService(webSvc.get(), gMetaKVStore.get(), helper.get(), pool.get()); + status = initWebService(webSvc.get(), gMetaKVStore.get()); if (!status.ok()) { LOG(ERROR) << "Init web service failed: " << status; return; diff --git a/src/graph/executor/CMakeLists.txt b/src/graph/executor/CMakeLists.txt index 2bb009d79bb..40f815b6379 100644 --- a/src/graph/executor/CMakeLists.txt +++ b/src/graph/executor/CMakeLists.txt @@ -68,8 +68,6 @@ nebula_add_library( admin/PartExecutor.cpp admin/CharsetExecutor.cpp admin/ShowStatsExecutor.cpp - admin/DownloadExecutor.cpp - admin/IngestExecutor.cpp admin/ConfigExecutor.cpp admin/ZoneExecutor.cpp admin/ShowServiceClientsExecutor.cpp diff --git a/src/graph/executor/Executor.cpp b/src/graph/executor/Executor.cpp index 5125de41377..27e24966d83 100644 --- a/src/graph/executor/Executor.cpp +++ b/src/graph/executor/Executor.cpp @@ -23,11 +23,9 @@ #include "graph/executor/admin/ConfigExecutor.h" #include "graph/executor/admin/CreateUserExecutor.h" #include "graph/executor/admin/DescribeUserExecutor.h" -#include "graph/executor/admin/DownloadExecutor.h" #include "graph/executor/admin/DropHostsExecutor.h" #include "graph/executor/admin/DropUserExecutor.h" #include "graph/executor/admin/GrantRoleExecutor.h" -#include "graph/executor/admin/IngestExecutor.h" #include "graph/executor/admin/KillQueryExecutor.h" #include "graph/executor/admin/ListRolesExecutor.h" #include "graph/executor/admin/ListUserRolesExecutor.h" @@ -518,12 +516,6 @@ Executor *Executor::makeExecutor(QueryContext *qctx, const PlanNode *node) { case PlanNode::Kind::kSignOutService: { return pool->add(new SignOutServiceExecutor(node, qctx)); } - case PlanNode::Kind::kDownload: { - return pool->add(new DownloadExecutor(node, qctx)); - } - case PlanNode::Kind::kIngest: { - return pool->add(new IngestExecutor(node, qctx)); - } case PlanNode::Kind::kShowSessions: { return pool->add(new ShowSessionsExecutor(node, qctx)); } diff --git a/src/graph/executor/admin/DownloadExecutor.cpp b/src/graph/executor/admin/DownloadExecutor.cpp deleted file mode 100644 index 94e44b1d897..00000000000 --- a/src/graph/executor/admin/DownloadExecutor.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "graph/executor/admin/DownloadExecutor.h" - -#include "graph/context/QueryContext.h" -#include "graph/planner/plan/Admin.h" - -namespace nebula { -namespace graph { - -folly::Future DownloadExecutor::execute() { - SCOPED_TIMER(&execTime_); - auto *dNode = asNode(node()); - auto spaceId = qctx()->rctx()->session()->space().id; - return qctx() - ->getMetaClient() - ->download(dNode->getHdfsHost(), dNode->getHdfsPort(), dNode->getHdfsPath(), spaceId) - .via(runner()) - .thenValue([this](StatusOr resp) { - SCOPED_TIMER(&execTime_); - NG_RETURN_IF_ERROR(resp); - if (!resp.value()) { - return Status::Error("Download failed!"); - } - return Status::OK(); - }); -} - -} // namespace graph -} // namespace nebula diff --git a/src/graph/executor/admin/DownloadExecutor.h b/src/graph/executor/admin/DownloadExecutor.h deleted file mode 100644 index c0912c9aedf..00000000000 --- a/src/graph/executor/admin/DownloadExecutor.h +++ /dev/null @@ -1,25 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef GRAPH_EXECUTOR_ADMIN_DOWNLOADEXECUTOR_H_ -#define GRAPH_EXECUTOR_ADMIN_DOWNLOADEXECUTOR_H_ - -#include "graph/executor/Executor.h" - -namespace nebula { -namespace graph { - -class DownloadExecutor final : public Executor { - public: - DownloadExecutor(const PlanNode *node, QueryContext *qctx) - : Executor("DownloadExecutor", node, qctx) {} - - folly::Future execute() override; -}; - -} // namespace graph -} // namespace nebula - -#endif // GRAPH_EXECUTOR_ADMIN_DOWNLOADEXECUTOR_H_ diff --git a/src/graph/executor/admin/IngestExecutor.cpp b/src/graph/executor/admin/IngestExecutor.cpp deleted file mode 100644 index 9cb397e3006..00000000000 --- a/src/graph/executor/admin/IngestExecutor.cpp +++ /dev/null @@ -1,28 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "graph/executor/admin/IngestExecutor.h" - -#include "graph/context/QueryContext.h" -#include "graph/planner/plan/Admin.h" - -namespace nebula { -namespace graph { - -folly::Future IngestExecutor::execute() { - auto spaceId = qctx()->rctx()->session()->space().id; - return qctx()->getMetaClient()->ingest(spaceId).via(runner()).thenValue( - [this](StatusOr resp) { - SCOPED_TIMER(&execTime_); - NG_RETURN_IF_ERROR(resp); - if (!resp.value()) { - return Status::Error("Ingest failed!"); - } - return Status::OK(); - }); -} - -} // namespace graph -} // namespace nebula diff --git a/src/graph/executor/admin/IngestExecutor.h b/src/graph/executor/admin/IngestExecutor.h deleted file mode 100644 index 8a84c1dbf67..00000000000 --- a/src/graph/executor/admin/IngestExecutor.h +++ /dev/null @@ -1,25 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef GRAPH_EXECUTOR_ADMIN_INGESTEXECUTOR_H_ -#define GRAPH_EXECUTOR_ADMIN_INGESTEXECUTOR_H_ - -#include "graph/executor/Executor.h" - -namespace nebula { -namespace graph { - -class IngestExecutor final : public Executor { - public: - IngestExecutor(const PlanNode *node, QueryContext *qctx) - : Executor("IngestExecutor", node, qctx) {} - - folly::Future execute() override; -}; - -} // namespace graph -} // namespace nebula - -#endif // GRAPH_EXECUTOR_ADMIN_INGESTEXECUTOR_H_ diff --git a/src/graph/planner/plan/Admin.h b/src/graph/planner/plan/Admin.h index cc5973286d9..3b390a47f16 100644 --- a/src/graph/planner/plan/Admin.h +++ b/src/graph/planner/plan/Admin.h @@ -537,55 +537,6 @@ class ShowListener final : public SingleDependencyNode { : SingleDependencyNode(qctx, Kind::kShowListener, input) {} }; -class Download final : public SingleDependencyNode { - public: - static Download* make(QueryContext* qctx, - PlanNode* input, - std::string hdfsHost, - int32_t hdfsPort, - std::string hdfsPath) { - return qctx->objPool()->add(new Download(qctx, input, hdfsHost, hdfsPort, hdfsPath)); - } - - const std::string& getHdfsHost() const { - return hdfsHost_; - } - - int32_t getHdfsPort() const { - return hdfsPort_; - } - - const std::string& getHdfsPath() const { - return hdfsPath_; - } - - private: - Download(QueryContext* qctx, - PlanNode* dep, - std::string hdfsHost, - int32_t hdfsPort, - std::string hdfsPath) - : SingleDependencyNode(qctx, Kind::kDownload, dep), - hdfsHost_(hdfsHost), - hdfsPort_(hdfsPort), - hdfsPath_(hdfsPath) {} - - private: - std::string hdfsHost_; - int32_t hdfsPort_; - std::string hdfsPath_; -}; - -class Ingest final : public SingleDependencyNode { - public: - static Ingest* make(QueryContext* qctx, PlanNode* dep) { - return qctx->objPool()->add(new Ingest(qctx, dep)); - } - - private: - Ingest(QueryContext* qctx, PlanNode* dep) : SingleDependencyNode(qctx, Kind::kIngest, dep) {} -}; - // User related Node class CreateUser final : public CreateNode { public: diff --git a/src/graph/planner/plan/PlanNode.cpp b/src/graph/planner/plan/PlanNode.cpp index dfb4978efd0..8ceade20b92 100644 --- a/src/graph/planner/plan/PlanNode.cpp +++ b/src/graph/planner/plan/PlanNode.cpp @@ -277,10 +277,6 @@ const char* PlanNode::toString(PlanNode::Kind kind) { return "SignInService"; case Kind::kSignOutService: return "SignOutService"; - case Kind::kDownload: - return "Download"; - case Kind::kIngest: - return "Ingest"; case Kind::kShowSessions: return "ShowSessions"; case Kind::kUpdateSession: diff --git a/src/graph/planner/plan/PlanNode.h b/src/graph/planner/plan/PlanNode.h index beff38e0aa9..9c1e49b7cd9 100644 --- a/src/graph/planner/plan/PlanNode.h +++ b/src/graph/planner/plan/PlanNode.h @@ -175,8 +175,6 @@ class PlanNode { kShowFTIndexes, kSignInService, kSignOutService, - kDownload, - kIngest, kShowSessions, kUpdateSession, diff --git a/src/graph/service/PermissionCheck.cpp b/src/graph/service/PermissionCheck.cpp index 2ff0475b27a..8939a249ae5 100644 --- a/src/graph/service/PermissionCheck.cpp +++ b/src/graph/service/PermissionCheck.cpp @@ -11,7 +11,7 @@ namespace graph { /** * Read space : kUse, kDescribeSpace * Write space : kCreateSpace, kDropSpace, kClearSpace, kCreateSnapshot, - * kDropSnapshot, kBalance, kAdmin, kConfig, kIngest, kDownload + * kDropSnapshot, kBalance, kAdmin, kConfig * Read schema : kDescribeTag, kDescribeEdge, * kDescribeTagIndex, kDescribeEdgeIndex * Write schema : kCreateTag, kAlterTag, kCreateEdge, @@ -68,8 +68,6 @@ namespace graph { case Sentence::Kind::kShowConfigs: case Sentence::Kind::kSetConfig: case Sentence::Kind::kGetConfig: - case Sentence::Kind::kIngest: - case Sentence::Kind::kDownload: case Sentence::Kind::kSignOutService: case Sentence::Kind::kSignInService: { return PermissionManager::canWriteSpace(session); diff --git a/src/graph/validator/AdminJobValidator.h b/src/graph/validator/AdminJobValidator.h index e6930fa9ff6..65d9df8b869 100644 --- a/src/graph/validator/AdminJobValidator.h +++ b/src/graph/validator/AdminJobValidator.h @@ -37,14 +37,12 @@ class AdminJobValidator final : public Validator { case meta::cpp2::JobType::STATS: case meta::cpp2::JobType::COMPACT: case meta::cpp2::JobType::FLUSH: + case meta::cpp2::JobType::DOWNLOAD: + case meta::cpp2::JobType::INGEST: case meta::cpp2::JobType::DATA_BALANCE: case meta::cpp2::JobType::LEADER_BALANCE: case meta::cpp2::JobType::ZONE_BALANCE: return true; - // TODO: download and ingest need to be refactored to use the rpc protocol. - // Currently they are using their own validator - case meta::cpp2::JobType::DOWNLOAD: - case meta::cpp2::JobType::INGEST: case meta::cpp2::JobType::UNKNOWN: return false; } diff --git a/src/graph/validator/CMakeLists.txt b/src/graph/validator/CMakeLists.txt index 722f3725a36..ac7dd18981e 100644 --- a/src/graph/validator/CMakeLists.txt +++ b/src/graph/validator/CMakeLists.txt @@ -27,8 +27,6 @@ nebula_add_library( FindPathValidator.cpp LookupValidator.cpp MatchValidator.cpp - DownloadValidator.cpp - IngestValidator.cpp ) nebula_add_subdirectory(test) diff --git a/src/graph/validator/DownloadValidator.cpp b/src/graph/validator/DownloadValidator.cpp deleted file mode 100644 index 73b4797ec5a..00000000000 --- a/src/graph/validator/DownloadValidator.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "graph/validator/DownloadValidator.h" - -#include "graph/planner/plan/Admin.h" -#include "parser/MutateSentences.h" - -namespace nebula { -namespace graph { - -// Plan to download SST file from HDFS -Status DownloadValidator::toPlan() { - auto sentence = static_cast(sentence_); - if (sentence->host() == nullptr || sentence->port() == 0 || sentence->path() == nullptr) { - return Status::SemanticError( - "HDFS path illegal." - "Should be HDFS://${HDFS_HOST}:${HDFS_PORT}/${HDFS_PATH}"); - } - auto *doNode = - Download::make(qctx_, nullptr, *sentence->host(), sentence->port(), *sentence->path()); - root_ = doNode; - tail_ = root_; - return Status::OK(); -} - -} // namespace graph -} // namespace nebula diff --git a/src/graph/validator/DownloadValidator.h b/src/graph/validator/DownloadValidator.h deleted file mode 100644 index 27ff4bab89a..00000000000 --- a/src/graph/validator/DownloadValidator.h +++ /dev/null @@ -1,30 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef GRAPH_VALIDATOR_DOWNLOADVALIDATOR_H_ -#define GRAPH_VALIDATOR_DOWNLOADVALIDATOR_H_ - -#include "graph/validator/Validator.h" -#include "parser/AdminSentences.h" - -namespace nebula { -namespace graph { - -class DownloadValidator final : public Validator { - public: - DownloadValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) {} - - private: - Status validateImpl() override { - return Status::OK(); - } - - Status toPlan() override; -}; - -} // namespace graph -} // namespace nebula - -#endif // GRAPH_VALIDATOR_DOWNLOADVALIDATOR_H_ diff --git a/src/graph/validator/IngestValidator.cpp b/src/graph/validator/IngestValidator.cpp deleted file mode 100644 index 6798e4271f0..00000000000 --- a/src/graph/validator/IngestValidator.cpp +++ /dev/null @@ -1,22 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "graph/validator/IngestValidator.h" - -#include "graph/planner/plan/Admin.h" - -namespace nebula { -namespace graph { - -// Plan to ingest SST file in server side -Status IngestValidator::toPlan() { - auto *doNode = Ingest::make(qctx_, nullptr); - root_ = doNode; - tail_ = root_; - return Status::OK(); -} - -} // namespace graph -} // namespace nebula diff --git a/src/graph/validator/IngestValidator.h b/src/graph/validator/IngestValidator.h deleted file mode 100644 index eb560626963..00000000000 --- a/src/graph/validator/IngestValidator.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef GRAPH_VALIDATOR_INGESTVALIDATOR_H_ -#define GRAPH_VALIDATOR_INGESTVALIDATOR_H_ - -#include "graph/validator/Validator.h" - -namespace nebula { -namespace graph { - -class IngestValidator final : public Validator { - public: - IngestValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) {} - - private: - Status validateImpl() override { - return Status::OK(); - } - - Status toPlan() override; -}; - -} // namespace graph -} // namespace nebula - -#endif // GRAPH_VALIDATOR_INGESTVALIDATOR_H_ diff --git a/src/graph/validator/Validator.cpp b/src/graph/validator/Validator.cpp index d87ea35bec7..58330d75132 100644 --- a/src/graph/validator/Validator.cpp +++ b/src/graph/validator/Validator.cpp @@ -16,7 +16,6 @@ #include "graph/validator/AdminJobValidator.h" #include "graph/validator/AdminValidator.h" #include "graph/validator/AssignmentValidator.h" -#include "graph/validator/DownloadValidator.h" #include "graph/validator/ExplainValidator.h" #include "graph/validator/FetchEdgesValidator.h" #include "graph/validator/FetchVerticesValidator.h" @@ -24,7 +23,6 @@ #include "graph/validator/GetSubgraphValidator.h" #include "graph/validator/GoValidator.h" #include "graph/validator/GroupByValidator.h" -#include "graph/validator/IngestValidator.h" #include "graph/validator/LimitValidator.h" #include "graph/validator/LookupValidator.h" #include "graph/validator/MaintainValidator.h" @@ -238,10 +236,6 @@ std::unique_ptr Validator::makeValidator(Sentence* sentence, QueryCon return std::make_unique(sentence, context); case Sentence::Kind::kSignOutService: return std::make_unique(sentence, context); - case Sentence::Kind::kDownload: - return std::make_unique(sentence, context); - case Sentence::Kind::kIngest: - return std::make_unique(sentence, context); case Sentence::Kind::kCreateFTIndex: return std::make_unique(sentence, context); case Sentence::Kind::kDropFTIndex: diff --git a/src/kvstore/NebulaStore.cpp b/src/kvstore/NebulaStore.cpp index e087beae4a4..2c551b497c8 100644 --- a/src/kvstore/NebulaStore.cpp +++ b/src/kvstore/NebulaStore.cpp @@ -883,33 +883,41 @@ nebula::cpp2::ErrorCode NebulaStore::ingest(GraphSpaceID spaceId) { if (!ok(spaceRet)) { return error(spaceRet); } + LOG(INFO) << "Ingesting space " << spaceId; auto space = nebula::value(spaceRet); + std::vector threads; + nebula::cpp2::ErrorCode code = nebula::cpp2::ErrorCode::SUCCEEDED; for (auto& engine : space->engines_) { - auto parts = engine->allParts(); - for (auto part : parts) { - auto ret = this->engine(spaceId, part); - if (!ok(ret)) { - return error(ret); - } - - auto path = folly::stringPrintf("%s/download/%d", value(ret)->getDataRoot(), part); - if (!fs::FileUtils::exist(path)) { - VLOG(1) << path << " not existed while ingesting"; - continue; - } + threads.emplace_back(std::thread([&engine, &code, this, spaceId] { + auto parts = engine->allParts(); + for (auto part : parts) { + auto ret = this->engine(spaceId, part); + if (!ok(ret)) { + code = error(ret); + } else { + auto path = folly::stringPrintf("%s/download/%d", value(ret)->getDataRoot(), part); + if (!fs::FileUtils::exist(path)) { + LOG(INFO) << path << " not existed"; + continue; + } - auto files = nebula::fs::FileUtils::listAllFilesInDir(path.c_str(), true, "*.sst"); - for (auto file : files) { - VLOG(1) << "Ingesting extra file: " << file; - auto code = engine->ingest(std::vector({file})); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - return code; + auto files = nebula::fs::FileUtils::listAllFilesInDir(path.c_str(), true, "*.sst"); + auto result = engine->ingest(std::vector(files)); + if (result != nebula::cpp2::ErrorCode::SUCCEEDED) { + code = result; + } } } - } + })); } - return nebula::cpp2::ErrorCode::SUCCEEDED; + + // Wait for all threads to finish + for (auto& t : threads) { + t.join(); + } + LOG(INFO) << "Space " << spaceId << " ingest done."; + return code; } nebula::cpp2::ErrorCode NebulaStore::setOption(GraphSpaceID spaceId, diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index 1a16db69c6e..d805bdbd075 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -45,8 +45,6 @@ nebula_add_library( processors/admin/CreateSnapshotProcessor.cpp processors/admin/DropSnapshotProcessor.cpp processors/admin/ListSnapshotsProcessor.cpp - processors/job/BalancePlan.cpp - processors/job/BalanceTask.cpp processors/admin/AdminClient.cpp processors/admin/SnapShot.cpp processors/admin/CreateBackupProcessor.cpp @@ -70,6 +68,10 @@ nebula_add_library( processors/job/SimpleConcurrentJobExecutor.cpp processors/job/CompactJobExecutor.cpp processors/job/FlushJobExecutor.cpp + processors/job/DownloadJobExecutor.cpp + processors/job/IngestJobExecutor.cpp + processors/job/BalancePlan.cpp + processors/job/BalanceTask.cpp processors/job/BalanceJobExecutor.cpp processors/job/ZoneBalanceJobExecutor.cpp processors/job/DataBalanceJobExecutor.cpp @@ -112,6 +114,7 @@ add_dependencies( ) set(meta_test_deps + $ $ $ $ diff --git a/src/meta/http/CMakeLists.txt b/src/meta/http/CMakeLists.txt index 34d2e95b4fe..c346be8d85d 100644 --- a/src/meta/http/CMakeLists.txt +++ b/src/meta/http/CMakeLists.txt @@ -2,12 +2,9 @@ # # This source code is licensed under Apache 2.0 License. - nebula_add_library( meta_http_handler OBJECT - MetaHttpIngestHandler.cpp - MetaHttpDownloadHandler.cpp MetaHttpReplaceHostHandler.cpp - ) +) nebula_add_subdirectory(test) diff --git a/src/meta/http/MetaHttpDownloadHandler.cpp b/src/meta/http/MetaHttpDownloadHandler.cpp deleted file mode 100644 index 6c88e6927db..00000000000 --- a/src/meta/http/MetaHttpDownloadHandler.cpp +++ /dev/null @@ -1,205 +0,0 @@ -/* Copyright (c) 2019 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "meta/http/MetaHttpDownloadHandler.h" - -#include -#include -#include -#include -#include - -#include "common/hdfs/HdfsHelper.h" -#include "common/http/HttpClient.h" -#include "common/network/NetworkUtils.h" -#include "common/process/ProcessUtils.h" -#include "common/thread/GenericThreadPool.h" -#include "common/utils/MetaKeyUtils.h" -#include "webservice/Common.h" -#include "webservice/WebService.h" - -DECLARE_int32(ws_storage_http_port); - -namespace nebula { -namespace meta { - -using proxygen::HTTPMessage; -using proxygen::HTTPMethod; -using proxygen::ProxygenError; -using proxygen::ResponseBuilder; -using proxygen::UpgradeProtocol; - -void MetaHttpDownloadHandler::init(nebula::kvstore::KVStore *kvstore, - nebula::hdfs::HdfsHelper *helper, - nebula::thread::GenericThreadPool *pool) { - kvstore_ = kvstore; - helper_ = helper; - pool_ = pool; - CHECK_NOTNULL(kvstore_); - CHECK_NOTNULL(helper_); - CHECK_NOTNULL(pool_); -} - -void MetaHttpDownloadHandler::onRequest(std::unique_ptr headers) noexcept { - if (headers->getMethod().value() != HTTPMethod::GET) { - // Unsupported method - err_ = HttpCode::E_UNSUPPORTED_METHOD; - return; - } - - if (!headers->hasQueryParam("host") || !headers->hasQueryParam("port") || - !headers->hasQueryParam("path") || !headers->hasQueryParam("space")) { - LOG(INFO) << "Illegal Argument"; - err_ = HttpCode::E_ILLEGAL_ARGUMENT; - return; - } - - hdfsHost_ = headers->getQueryParam("host"); - hdfsPort_ = headers->getIntQueryParam("port"); - hdfsPath_ = headers->getQueryParam("path"); - spaceID_ = headers->getIntQueryParam("space"); -} - -void MetaHttpDownloadHandler::onBody(std::unique_ptr) noexcept { - // Do nothing, we only support GET -} - -void MetaHttpDownloadHandler::onEOM() noexcept { - switch (err_) { - case HttpCode::E_UNSUPPORTED_METHOD: - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::METHOD_NOT_ALLOWED), - WebServiceUtils::toString(HttpStatusCode::METHOD_NOT_ALLOWED)) - .sendWithEOM(); - return; - case HttpCode::E_ILLEGAL_ARGUMENT: - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::BAD_REQUEST), - WebServiceUtils::toString(HttpStatusCode::BAD_REQUEST)) - .sendWithEOM(); - return; - default: - break; - } - - if (helper_->checkHadoopPath()) { - if (dispatchSSTFiles(hdfsHost_, hdfsPort_, hdfsPath_)) { - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::OK), - WebServiceUtils::toString(HttpStatusCode::OK)) - .body("SSTFile dispatch successfully") - .sendWithEOM(); - } else { - LOG(INFO) << "SSTFile dispatch failed"; - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::FORBIDDEN), - WebServiceUtils::toString(HttpStatusCode::FORBIDDEN)) - .body("SSTFile dispatch failed") - .sendWithEOM(); - } - } else { - LOG(INFO) << "Hadoop Home not exist"; - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::NOT_FOUND), - WebServiceUtils::toString(HttpStatusCode::NOT_FOUND)) - .sendWithEOM(); - } -} - -void MetaHttpDownloadHandler::onUpgrade(UpgradeProtocol) noexcept { - // Do nothing -} - -void MetaHttpDownloadHandler::requestComplete() noexcept { - delete this; -} - -void MetaHttpDownloadHandler::onError(ProxygenError error) noexcept { - LOG(INFO) << "Web Service MetaHttpDownloadHandler got error : " - << proxygen::getErrorString(error); -} - -bool MetaHttpDownloadHandler::dispatchSSTFiles(const std::string &hdfsHost, - int hdfsPort, - const std::string &hdfsPath) { - auto result = helper_->ls(hdfsHost, hdfsPort, hdfsPath); - if (!result.ok()) { - LOG(INFO) << "Dispatch SSTFile Failed"; - return false; - } - std::vector files; - folly::split("\n", result.value(), files, true); - - std::unique_ptr iter; - auto prefix = MetaKeyUtils::partPrefix(spaceID_); - auto ret = kvstore_->prefix(0, 0, prefix, &iter); - if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(INFO) << "Fetch Parts Failed"; - return false; - } - - int32_t partSize{0}; - std::unordered_map> hostPartition; - while (iter->valid()) { - auto key = iter->key(); - PartitionID partId; - memcpy(&partId, key.data() + prefix.size(), sizeof(PartitionID)); - for (auto host : MetaKeyUtils::parsePartVal(iter->val())) { - auto address = HostAddr(host.host, host.port); - auto addressIter = hostPartition.find(address); - if (addressIter == hostPartition.end()) { - std::vector partitions; - hostPartition.emplace(address, partitions); - } - hostPartition[address].emplace_back(partId); - } - partSize++; - iter->next(); - } - - std::vector> futures; - for (auto &pair : hostPartition) { - std::string partsStr; - folly::join(",", pair.second, partsStr); - - auto hostaddr = pair.first.host; - auto dispatcher = [hostaddr, hdfsHost, hdfsPort, hdfsPath, partsStr, this]() { - static const char *tmp = "http://%s:%d/%s?host=%s&port=%d&path=%s&parts=%s&space=%d"; - std::string url = folly::stringPrintf(tmp, - hostaddr.c_str(), - FLAGS_ws_storage_http_port, - "download", - hdfsHost.c_str(), - hdfsPort, - hdfsPath.c_str(), - partsStr.c_str(), - spaceID_); - auto downloadResult = nebula::http::HttpClient::get(url); - return downloadResult.ok() && downloadResult.value() == "SSTFile download successfully"; - }; - auto future = pool_->addTask(dispatcher); - futures.push_back(std::move(future)); - } - - bool successfully{true}; - auto tries = folly::collectAll(std::move(futures)).get(); - for (const auto &t : tries) { - if (t.hasException()) { - LOG(INFO) << "Download Failed: " << t.exception(); - successfully = false; - break; - } - if (!t.value()) { - successfully = false; - break; - } - } - - LOG(INFO) << "Download tasks have finished"; - return successfully; -} - -} // namespace meta -} // namespace nebula diff --git a/src/meta/http/MetaHttpDownloadHandler.h b/src/meta/http/MetaHttpDownloadHandler.h deleted file mode 100644 index 77ee273c686..00000000000 --- a/src/meta/http/MetaHttpDownloadHandler.h +++ /dev/null @@ -1,67 +0,0 @@ -/* Copyright (c) 2019 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef META_HTTP_METAHTTPDOWNLOADHANDLER_H_ -#define META_HTTP_METAHTTPDOWNLOADHANDLER_H_ - -#include - -#include "common/base/Base.h" -#include "common/hdfs/HdfsHelper.h" -#include "common/thread/GenericThreadPool.h" -#include "kvstore/KVStore.h" -#include "webservice/Common.h" - -namespace nebula { -namespace meta { - -using nebula::HttpCode; - -/** - * @brief Download sst files from hdfs to every storaged download folder. - * It will send download http request to every storaged, letting them - * download the corressponding sst files. - * Functions such as onRequest, onBody... and requestComplete are inherited - * from RequestHandler, we will check request parameters in onRequest and - * call main logic in onEOM. - */ -class MetaHttpDownloadHandler : public proxygen::RequestHandler { - public: - MetaHttpDownloadHandler() = default; - - void init(nebula::kvstore::KVStore *kvstore, - nebula::hdfs::HdfsHelper *helper, - nebula::thread::GenericThreadPool *pool); - - void onRequest(std::unique_ptr headers) noexcept override; - - void onBody(std::unique_ptr body) noexcept override; - - void onEOM() noexcept override; - - void onUpgrade(proxygen::UpgradeProtocol protocol) noexcept override; - - void requestComplete() noexcept override; - - void onError(proxygen::ProxygenError error) noexcept override; - - private: - bool dispatchSSTFiles(const std::string &host, int32_t port, const std::string &path); - - private: - HttpCode err_{HttpCode::SUCCEEDED}; - std::string hdfsHost_; - int32_t hdfsPort_; - std::string hdfsPath_; - GraphSpaceID spaceID_; - nebula::kvstore::KVStore *kvstore_; - nebula::hdfs::HdfsHelper *helper_; - nebula::thread::GenericThreadPool *pool_; -}; - -} // namespace meta -} // namespace nebula - -#endif // META_HTTP_METAHTTPDOWNLOADHANDLER_H_ diff --git a/src/meta/http/MetaHttpIngestHandler.cpp b/src/meta/http/MetaHttpIngestHandler.cpp deleted file mode 100644 index 27f76a890c3..00000000000 --- a/src/meta/http/MetaHttpIngestHandler.cpp +++ /dev/null @@ -1,158 +0,0 @@ -/* Copyright (c) 2019 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "meta/http/MetaHttpIngestHandler.h" - -#include -#include -#include - -#include "common/http/HttpClient.h" -#include "common/network/NetworkUtils.h" -#include "common/process/ProcessUtils.h" -#include "common/thread/GenericThreadPool.h" -#include "common/utils/MetaKeyUtils.h" -#include "webservice/Common.h" -#include "webservice/WebService.h" - -DECLARE_int32(ws_storage_http_port); - -namespace nebula { -namespace meta { - -using proxygen::HTTPMessage; -using proxygen::HTTPMethod; -using proxygen::ProxygenError; -using proxygen::ResponseBuilder; -using proxygen::UpgradeProtocol; - -void MetaHttpIngestHandler::init(nebula::kvstore::KVStore *kvstore, - nebula::thread::GenericThreadPool *pool) { - kvstore_ = kvstore; - pool_ = pool; - CHECK_NOTNULL(kvstore_); - CHECK_NOTNULL(pool_); -} - -void MetaHttpIngestHandler::onRequest(std::unique_ptr headers) noexcept { - if (headers->getMethod().value() != HTTPMethod::GET) { - // Unsupported method - err_ = HttpCode::E_UNSUPPORTED_METHOD; - return; - } - - if (!headers->hasQueryParam("space")) { - err_ = HttpCode::E_ILLEGAL_ARGUMENT; - return; - } - - space_ = headers->getIntQueryParam("space"); -} - -void MetaHttpIngestHandler::onBody(std::unique_ptr) noexcept { - // Do nothing, we only support GET -} - -void MetaHttpIngestHandler::onEOM() noexcept { - switch (err_) { - case HttpCode::E_UNSUPPORTED_METHOD: - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::METHOD_NOT_ALLOWED), - WebServiceUtils::toString(HttpStatusCode::METHOD_NOT_ALLOWED)) - .sendWithEOM(); - return; - case HttpCode::E_ILLEGAL_ARGUMENT: - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::BAD_REQUEST), - WebServiceUtils::toString(HttpStatusCode::BAD_REQUEST)) - .sendWithEOM(); - return; - default: - break; - } - - if (ingestSSTFiles(space_)) { - LOG(INFO) << "SSTFile ingest successfully "; - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::OK), - WebServiceUtils::toString(HttpStatusCode::OK)) - .body("SSTFile ingest successfully") - .sendWithEOM(); - } else { - LOG(INFO) << "SSTFile ingest failed"; - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::FORBIDDEN), - WebServiceUtils::toString(HttpStatusCode::FORBIDDEN)) - .body("SSTFile ingest failed") - .sendWithEOM(); - } -} - -void MetaHttpIngestHandler::onUpgrade(UpgradeProtocol) noexcept { - // Do nothing -} - -void MetaHttpIngestHandler::requestComplete() noexcept { - delete this; -} - -void MetaHttpIngestHandler::onError(ProxygenError error) noexcept { - LOG(INFO) << "Web Service MetaHttpIngestHandler got error : " << proxygen::getErrorString(error); -} - -bool MetaHttpIngestHandler::ingestSSTFiles(GraphSpaceID space) { - std::unique_ptr iter; - auto prefix = MetaKeyUtils::partPrefix(space); - - static const GraphSpaceID metaSpaceId = 0; - static const PartitionID metaPartId = 0; - auto ret = kvstore_->prefix(metaSpaceId, metaPartId, prefix, &iter); - if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(INFO) << "Fetch Parts Failed"; - return false; - } - - std::set storageIPs; - while (iter->valid()) { - for (auto &host : MetaKeyUtils::parsePartVal(iter->val())) { - if (storageIPs.count(host.host) == 0) { - storageIPs.insert(std::move(host.host)); - } - } - iter->next(); - } - - std::vector> futures; - - for (auto &storageIP : storageIPs) { - auto dispatcher = [storageIP, space]() { - static const char *tmp = "http://%s:%d/ingest?space=%d"; - auto url = folly::stringPrintf(tmp, storageIP.c_str(), FLAGS_ws_storage_http_port, space); - auto ingestResult = nebula::http::HttpClient::get(url); - return ingestResult.ok() && ingestResult.value() == "SSTFile ingest successfully"; - }; - auto future = pool_->addTask(dispatcher); - futures.push_back(std::move(future)); - } - - bool successfully{true}; - auto tries = folly::collectAll(std::move(futures)).get(); - for (const auto &t : tries) { - if (t.hasException()) { - LOG(INFO) << "Ingest Failed: " << t.exception(); - successfully = false; - break; - } - if (!t.value()) { - successfully = false; - break; - } - } - LOG(INFO) << "Ingest tasks have finished"; - return successfully; -} - -} // namespace meta -} // namespace nebula diff --git a/src/meta/http/MetaHttpIngestHandler.h b/src/meta/http/MetaHttpIngestHandler.h deleted file mode 100644 index acddf4affc8..00000000000 --- a/src/meta/http/MetaHttpIngestHandler.h +++ /dev/null @@ -1,61 +0,0 @@ -/* Copyright (c) 2019 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef META_HTTP_METAHTTPINGESTHANDLER_H -#define META_HTTP_METAHTTPINGESTHANDLER_H - -#include - -#include "common/base/Base.h" -#include "common/thread/GenericThreadPool.h" -#include "kvstore/KVStore.h" -#include "webservice/Common.h" - -namespace nebula { -namespace meta { - -using nebula::HttpCode; - -/** - * @brief Ingest should be called after download successfully. - * It will instruct relative storaged to ingest sst files - * from local download folder by sending http request. - * It will handle one space each time. - * Functions such as onRequest, onBody... and requestComplete are inherited - * from RequestHandler, we will check request parameters in onRequest and - * call main logic in onEOM. - * - */ -class MetaHttpIngestHandler : public proxygen::RequestHandler { - public: - MetaHttpIngestHandler() = default; - - void init(nebula::kvstore::KVStore *kvstore, nebula::thread::GenericThreadPool *pool); - - void onRequest(std::unique_ptr headers) noexcept override; - - void onBody(std::unique_ptr body) noexcept override; - - void onEOM() noexcept override; - - void onUpgrade(proxygen::UpgradeProtocol protocol) noexcept override; - - void requestComplete() noexcept override; - - void onError(proxygen::ProxygenError error) noexcept override; - - bool ingestSSTFiles(GraphSpaceID space); - - private: - HttpCode err_{HttpCode::SUCCEEDED}; - GraphSpaceID space_; - nebula::kvstore::KVStore *kvstore_; - nebula::thread::GenericThreadPool *pool_; -}; - -} // namespace meta -} // namespace nebula - -#endif // META_HTTP_METAHTTPINGESTHANDLER_H diff --git a/src/meta/http/test/CMakeLists.txt b/src/meta/http/test/CMakeLists.txt index 9f116aefe5e..c4d059fb41b 100644 --- a/src/meta/http/test/CMakeLists.txt +++ b/src/meta/http/test/CMakeLists.txt @@ -2,37 +2,6 @@ # # This source code is licensed under Apache 2.0 License. -nebula_add_test( - NAME meta_http_download_test - SOURCES MetaHttpDownloadHandlerTest.cpp - OBJECTS - $ - $ - ${meta_test_deps} - LIBRARIES - ${ROCKSDB_LIBRARIES} - ${THRIFT_LIBRARIES} - ${PROXYGEN_LIBRARIES} - wangle - gtest - ) - -nebula_add_test( - NAME meta_http_ingest_test - SOURCES MetaHttpIngestHandlerTest.cpp - OBJECTS - $ - $ - ${meta_test_deps} - LIBRARIES - ${ROCKSDB_LIBRARIES} - ${THRIFT_LIBRARIES} - ${PROXYGEN_LIBRARIES} - wangle - gtest - ) - - nebula_add_test( NAME meta_http_replace_test diff --git a/src/meta/http/test/MetaHttpDownloadHandlerTest.cpp b/src/meta/http/test/MetaHttpDownloadHandlerTest.cpp deleted file mode 100644 index 5661cbc10c5..00000000000 --- a/src/meta/http/test/MetaHttpDownloadHandlerTest.cpp +++ /dev/null @@ -1,123 +0,0 @@ -/* Copyright (c) 2019 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include - -#include "common/base/Base.h" -#include "common/fs/TempDir.h" -#include "common/http/HttpClient.h" -#include "meta/http/MetaHttpDownloadHandler.h" -#include "meta/test/MockHdfsHelper.h" -#include "meta/test/TestUtils.h" -#include "storage/http/StorageHttpDownloadHandler.h" -#include "webservice/Router.h" -#include "webservice/WebService.h" - -DECLARE_string(pid_file); -DECLARE_int32(ws_storage_http_port); - -namespace nebula { -namespace meta { - -std::unique_ptr helper = std::make_unique(); - -class MetaHttpDownloadHandlerTestEnv : public ::testing::Environment { - public: - void SetUp() override { - FLAGS_ws_ip = "127.0.0.1"; - FLAGS_ws_http_port = 0; - VLOG(1) << "Starting web service..."; - - rootPath_ = std::make_unique("/tmp/MetaHttpDownloadHandler.XXXXXX"); - kv_ = MockCluster::initMetaKV(rootPath_->path()); - TestUtils::createSomeHosts(kv_.get()); - TestUtils::assembleSpace(kv_.get(), 1, 2); - - // Because we reuse the kvstore for storage handler, let's add part - // manually. - auto* partMan = static_cast(kv_->partManager()); - partMan->addPart(1, 1); - partMan->addPart(1, 2); - - // wait for the leader election - sleep(3); - - pool_ = std::make_unique(); - pool_->start(3); - - webSvc_ = std::make_unique(); - auto& router = webSvc_->router(); - - router.get("/download-dispatch").handler([this](nebula::web::PathParams&&) { - auto handler = new meta::MetaHttpDownloadHandler(); - handler->init(kv_.get(), helper.get(), pool_.get()); - return handler; - }); - router.get("/download").handler([this](nebula::web::PathParams&&) { - auto handler = new storage::StorageHttpDownloadHandler(); - std::vector paths{rootPath_->path()}; - handler->init(helper.get(), pool_.get(), kv_.get(), paths); - return handler; - }); - auto status = webSvc_->start(); - FLAGS_ws_storage_http_port = FLAGS_ws_http_port; - ASSERT_TRUE(status.ok()) << status; - } - - void TearDown() override { - kv_.reset(); - rootPath_.reset(); - webSvc_.reset(); - pool_->stop(); - VLOG(1) << "Web service stopped"; - } - - private: - std::unique_ptr webSvc_; - std::unique_ptr rootPath_; - std::unique_ptr kv_; - std::unique_ptr pool_; -}; - -TEST(MetaHttpDownloadHandlerTest, MetaDownloadTest) { - { - auto url = "/download-dispatch"; - auto request = - folly::stringPrintf("http://%s:%d%s", FLAGS_ws_ip.c_str(), FLAGS_ws_http_port, url); - auto resp = http::HttpClient::get(request); - ASSERT_TRUE(resp.ok()); - ASSERT_TRUE(resp.value().empty()); - } - { - auto url = "/download-dispatch?host=127.0.0.1&port=9000&path=/data&space=1"; - auto request = - folly::stringPrintf("http://%s:%d%s", FLAGS_ws_ip.c_str(), FLAGS_ws_http_port, url); - auto resp = http::HttpClient::get(request); - ASSERT_TRUE(resp.ok()); - ASSERT_EQ("SSTFile dispatch successfully", resp.value()); - } - { - helper = std::make_unique(); - auto url = "/download-dispatch?host=127.0.0.1&port=9000&path=/data&space=1"; - auto request = - folly::stringPrintf("http://%s:%d%s", FLAGS_ws_ip.c_str(), FLAGS_ws_http_port, url); - auto resp = http::HttpClient::get(request); - ASSERT_TRUE(resp.ok()); - ASSERT_EQ("SSTFile dispatch failed", resp.value()); - } -} - -} // namespace meta -} // namespace nebula - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - folly::init(&argc, &argv, true); - google::SetStderrLogging(google::INFO); - - ::testing::AddGlobalTestEnvironment(new nebula::meta::MetaHttpDownloadHandlerTestEnv()); - - return RUN_ALL_TESTS(); -} diff --git a/src/meta/http/test/MetaHttpIngestHandlerTest.cpp b/src/meta/http/test/MetaHttpIngestHandlerTest.cpp deleted file mode 100644 index 083cc4c1c69..00000000000 --- a/src/meta/http/test/MetaHttpIngestHandlerTest.cpp +++ /dev/null @@ -1,120 +0,0 @@ -/* Copyright (c) 2019 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include -#include - -#include "common/base/Base.h" -#include "common/fs/TempDir.h" -#include "common/http/HttpClient.h" -#include "common/thread/GenericThreadPool.h" -#include "meta/http/MetaHttpIngestHandler.h" -#include "meta/test/TestUtils.h" -#include "storage/http/StorageHttpIngestHandler.h" -#include "webservice/Router.h" -#include "webservice/WebService.h" - -DECLARE_int32(ws_storage_http_port); - -namespace nebula { -namespace meta { - -class MetaHttpIngestHandlerTestEnv : public ::testing::Environment { - public: - void SetUp() override { - FLAGS_ws_ip = "127.0.0.1"; - FLAGS_ws_http_port = 0; - VLOG(1) << "Starting web service..."; - - rootPath_ = std::make_unique("/tmp/MetaHttpIngestHandler.XXXXXX"); - kv_ = MockCluster::initMetaKV(rootPath_->path()); - TestUtils::createSomeHosts(kv_.get()); - TestUtils::assembleSpace(kv_.get(), 1, 1); - pool_ = std::make_unique(); - pool_->start(1); - - webSvc_ = std::make_unique(); - - auto& router = webSvc_->router(); - router.get("/ingest-dispatch").handler([this](nebula::web::PathParams&&) { - auto handler = new meta::MetaHttpIngestHandler(); - handler->init(kv_.get(), pool_.get()); - return handler; - }); - router.get("/ingest").handler([this](nebula::web::PathParams&&) { - auto handler = new storage::StorageHttpIngestHandler(); - handler->init(kv_.get()); - return handler; - }); - auto status = webSvc_->start(); - FLAGS_ws_storage_http_port = FLAGS_ws_http_port; - ASSERT_TRUE(status.ok()) << status; - } - - void TearDown() override { - kv_.reset(); - rootPath_.reset(); - webSvc_.reset(); - pool_->stop(); - VLOG(1) << "Web service stopped"; - } - - private: - std::unique_ptr webSvc_; - std::unique_ptr rootPath_; - std::unique_ptr kv_; - std::unique_ptr pool_; -}; - -TEST(MetaHttpIngestHandlerTest, MetaIngestTest) { - auto path = "/tmp/MetaHttpIngestData.XXXXXX"; - std::unique_ptr externalPath = std::make_unique(path); - auto partPath = folly::stringPrintf("%s/nebula/1/download/1", externalPath->path()); - ASSERT_TRUE(nebula::fs::FileUtils::makeDir(partPath)); - - auto options = rocksdb::Options(); - auto env = rocksdb::EnvOptions(); - rocksdb::SstFileWriter writer{env, options}; - auto sstPath = folly::stringPrintf("%s/data.sst", partPath.c_str()); - auto status = writer.Open(sstPath); - ASSERT_EQ(rocksdb::Status::OK(), status); - - for (auto i = 0; i < 10; i++) { - status = writer.Put(folly::stringPrintf("key_%d", i), folly::stringPrintf("val_%d", i)); - ASSERT_EQ(rocksdb::Status::OK(), status); - } - status = writer.Finish(); - ASSERT_EQ(rocksdb::Status::OK(), status); - - { - auto url = "/ingest-dispatch"; - auto request = - folly::stringPrintf("http://%s:%d%s", FLAGS_ws_ip.c_str(), FLAGS_ws_http_port, url); - auto resp = http::HttpClient::get(request); - ASSERT_TRUE(resp.ok()); - ASSERT_TRUE(resp.value().empty()); - } - { - auto url = "/ingest-dispatch?space=0"; - auto request = - folly::stringPrintf("http://%s:%d%s", FLAGS_ws_ip.c_str(), FLAGS_ws_http_port, url); - auto resp = http::HttpClient::get(request); - ASSERT_TRUE(resp.ok()); - ASSERT_EQ("SSTFile ingest successfully", resp.value()); - } -} - -} // namespace meta -} // namespace nebula - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - folly::init(&argc, &argv, true); - google::SetStderrLogging(google::INFO); - - ::testing::AddGlobalTestEnvironment(new nebula::meta::MetaHttpIngestHandlerTestEnv()); - - return RUN_ALL_TESTS(); -} diff --git a/src/meta/http/test/MetaHttpReplaceHandlerTest.cpp b/src/meta/http/test/MetaHttpReplaceHandlerTest.cpp index 1cd1b6782cd..c0c001f342d 100644 --- a/src/meta/http/test/MetaHttpReplaceHandlerTest.cpp +++ b/src/meta/http/test/MetaHttpReplaceHandlerTest.cpp @@ -16,8 +16,6 @@ #include "webservice/Router.h" #include "webservice/WebService.h" -DECLARE_int32(ws_storage_http_port); - namespace nebula { namespace meta { diff --git a/src/meta/processors/job/CompactJobExecutor.h b/src/meta/processors/job/CompactJobExecutor.h index b9bd56d36a2..11b32382984 100644 --- a/src/meta/processors/job/CompactJobExecutor.h +++ b/src/meta/processors/job/CompactJobExecutor.h @@ -12,7 +12,7 @@ namespace nebula { namespace meta { /** - * @brief Executor for compact job, always called by job manager + * @brief Executor for compact job, always called by job manager. */ class CompactJobExecutor : public SimpleConcurrentJobExecutor { public: diff --git a/src/meta/processors/job/DataBalanceJobExecutor.h b/src/meta/processors/job/DataBalanceJobExecutor.h index 0ba45a01728..c2dbad2ed6e 100644 --- a/src/meta/processors/job/DataBalanceJobExecutor.h +++ b/src/meta/processors/job/DataBalanceJobExecutor.h @@ -12,7 +12,7 @@ namespace nebula { namespace meta { /** - * @brief Executor for balance in zone, always called by job manager + * @brief Executor for balance in zone, always called by job manager. */ class DataBalanceJobExecutor : public BalanceJobExecutor { FRIEND_TEST(BalanceTest, BalanceDataPlanTest); diff --git a/src/meta/processors/job/DownloadJobExecutor.cpp b/src/meta/processors/job/DownloadJobExecutor.cpp new file mode 100644 index 00000000000..a396a00af02 --- /dev/null +++ b/src/meta/processors/job/DownloadJobExecutor.cpp @@ -0,0 +1,119 @@ +/* Copyright (c) 2022 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "meta/processors/job/DownloadJobExecutor.h" + +#include "common/hdfs/HdfsHelper.h" +#include "common/utils/MetaKeyUtils.h" +#include "meta/MetaServiceUtils.h" + +namespace nebula { +namespace meta { + +DownloadJobExecutor::DownloadJobExecutor(GraphSpaceID space, + JobID jobId, + kvstore::KVStore* kvstore, + AdminClient* adminClient, + const std::vector& paras) + : SimpleConcurrentJobExecutor(space, jobId, kvstore, adminClient, paras) { + helper_ = std::make_unique(); +} + +bool DownloadJobExecutor::check() { + if (paras_.size() != 1) { + return false; + } + + auto& url = paras_[0]; + std::string hdfsPrefix = "hdfs://"; + if (url.find(hdfsPrefix) != 0) { + LOG(ERROR) << "URL should start with " << hdfsPrefix; + return false; + } + + auto u = url.substr(hdfsPrefix.size(), url.size()); + std::vector tokens; + folly::split(":", u, tokens); + if (tokens.size() == 2) { + host_ = std::make_unique(tokens[0]); + int32_t position = tokens[1].find_first_of("/"); + if (position != -1) { + try { + port_ = folly::to(tokens[1].toString().substr(0, position).c_str()); + } catch (const std::exception& ex) { + LOG(ERROR) << "URL's port parse failed: " << url; + return false; + } + path_ = + std::make_unique(tokens[1].toString().substr(position, tokens[1].size())); + } else { + LOG(ERROR) << "URL Parse Failed: " << url; + return false; + } + } else { + LOG(ERROR) << "URL Parse Failed: " << url; + return false; + } + + return true; +} + +nebula::cpp2::ErrorCode DownloadJobExecutor::prepare() { + auto errOrHost = getTargetHost(space_); + if (!nebula::ok(errOrHost)) { + LOG(ERROR) << "Can't get any host according to space"; + return nebula::error(errOrHost); + } + + LOG(INFO) << "HDFS host: " << *host_.get() << " port: " << port_ << " path: " << *path_.get(); + + auto listResult = helper_->ls(*host_.get(), port_, *path_.get()); + if (!listResult.ok()) { + LOG(ERROR) << "Dispatch SSTFile Failed"; + return nebula::cpp2::ErrorCode::E_INVALID_JOB; + } + + taskParameters_.emplace_back(*host_.get()); + taskParameters_.emplace_back(folly::to(port_)); + taskParameters_.emplace_back(*path_.get()); + std::unique_ptr iter; + auto prefix = MetaKeyUtils::partPrefix(space_); + auto result = kvstore_->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + if (result != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Fetch Parts Failed"; + } + return result; +} + +folly::Future DownloadJobExecutor::executeInternal(HostAddr&& address, + std::vector&& parts) { + folly::Promise pro; + auto f = pro.getFuture(); + adminClient_ + ->addTask(cpp2::JobType::DOWNLOAD, + jobId_, + taskId_++, + space_, + std::move(address), + taskParameters_, + std::move(parts)) + .then([pro = std::move(pro)](auto&& t) mutable { + CHECK(!t.hasException()); + auto status = std::move(t).value(); + if (status.ok()) { + pro.setValue(Status::OK()); + } else { + pro.setValue(status.status()); + } + }); + return f; +} + +nebula::cpp2::ErrorCode DownloadJobExecutor::stop() { + return nebula::cpp2::ErrorCode::SUCCEEDED; +} + +} // namespace meta +} // namespace nebula diff --git a/src/meta/processors/job/DownloadJobExecutor.h b/src/meta/processors/job/DownloadJobExecutor.h new file mode 100644 index 00000000000..4c4c0c12cc5 --- /dev/null +++ b/src/meta/processors/job/DownloadJobExecutor.h @@ -0,0 +1,50 @@ +/* Copyright (c) 2022 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef META_DOWNLOADJOBEXECUTOR_H_ +#define META_DOWNLOADJOBEXECUTOR_H_ + +#include "common/hdfs/HdfsCommandHelper.h" +#include "meta/processors/job/SimpleConcurrentJobExecutor.h" + +namespace nebula { +namespace meta { + +/** + * @brief Executor for download job, always called by job manager. + */ +class DownloadJobExecutor : public SimpleConcurrentJobExecutor { + FRIEND_TEST(JobManagerTest, DownloadJob); + FRIEND_TEST(JobManagerTest, IngestJob); + + public: + DownloadJobExecutor(GraphSpaceID space, + JobID jobId, + kvstore::KVStore* kvstore, + AdminClient* adminClient, + const std::vector& params); + + bool check() override; + + nebula::cpp2::ErrorCode prepare() override; + + nebula::cpp2::ErrorCode stop() override; + + protected: + folly::Future executeInternal(HostAddr&& address, + std::vector&& parts) override; + + private: + std::unique_ptr host_; + int32_t port_; + std::unique_ptr path_; + std::unique_ptr helper_; + std::vector taskParameters_; +}; + +} // namespace meta +} // namespace nebula + +#endif // META_DOWNLOADJOBEXECUTOR_H_ diff --git a/src/meta/processors/job/FlushJobExecutor.h b/src/meta/processors/job/FlushJobExecutor.h index f0b2dd0d953..d4d007b8eae 100644 --- a/src/meta/processors/job/FlushJobExecutor.h +++ b/src/meta/processors/job/FlushJobExecutor.h @@ -12,7 +12,7 @@ namespace nebula { namespace meta { /** - * @brief Executor for flush job, always called by job manager + * @brief Executor for flush job, always called by job manager. */ class FlushJobExecutor : public SimpleConcurrentJobExecutor { public: diff --git a/src/meta/processors/job/GetStatsProcessor.h b/src/meta/processors/job/GetStatsProcessor.h index 3d7d6214dbe..d62e8898d3e 100644 --- a/src/meta/processors/job/GetStatsProcessor.h +++ b/src/meta/processors/job/GetStatsProcessor.h @@ -12,7 +12,7 @@ namespace nebula { namespace meta { /** - * @brief Get stats of a job, return error if the job is running or failed + * @brief Get stats of a job, return error if the job is running or failed. */ class GetStatsProcessor : public BaseProcessor { public: diff --git a/src/meta/processors/job/IngestJobExecutor.cpp b/src/meta/processors/job/IngestJobExecutor.cpp new file mode 100644 index 00000000000..81e8c212003 --- /dev/null +++ b/src/meta/processors/job/IngestJobExecutor.cpp @@ -0,0 +1,54 @@ +/* Copyright (c) 2022 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "meta/processors/job/IngestJobExecutor.h" + +#include "common/utils/MetaKeyUtils.h" +#include "meta/MetaServiceUtils.h" + +namespace nebula { +namespace meta { + +IngestJobExecutor::IngestJobExecutor(GraphSpaceID space, + JobID jobId, + kvstore::KVStore* kvstore, + AdminClient* adminClient, + const std::vector& paras) + : SimpleConcurrentJobExecutor(space, jobId, kvstore, adminClient, paras) {} + +bool IngestJobExecutor::check() { + return paras_.empty(); +} + +nebula::cpp2::ErrorCode IngestJobExecutor::prepare() { + return nebula::cpp2::ErrorCode::SUCCEEDED; +} + +folly::Future IngestJobExecutor::executeInternal(HostAddr&& address, + std::vector&& parts) { + folly::Promise pro; + auto f = pro.getFuture(); + adminClient_ + ->addTask(cpp2::JobType::INGEST, + jobId_, + taskId_++, + space_, + std::move(address), + taskParameters_, + std::move(parts)) + .then([pro = std::move(pro)](auto&& t) mutable { + CHECK(!t.hasException()); + auto status = std::move(t).value(); + if (status.ok()) { + pro.setValue(Status::OK()); + } else { + pro.setValue(status.status()); + } + }); + return f; +} + +} // namespace meta +} // namespace nebula diff --git a/src/meta/processors/job/IngestJobExecutor.h b/src/meta/processors/job/IngestJobExecutor.h new file mode 100644 index 00000000000..d3d91600c74 --- /dev/null +++ b/src/meta/processors/job/IngestJobExecutor.h @@ -0,0 +1,40 @@ +/* Copyright (c) 2022 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef META_INGESTJOBEXECUTOR_H_ +#define META_INGESTJOBEXECUTOR_H_ + +#include "meta/processors/job/MetaJobExecutor.h" +#include "meta/processors/job/SimpleConcurrentJobExecutor.h" + +namespace nebula { +namespace meta { + +/** + * @brief Executor for ingest job, always called by job manager. + */ +class IngestJobExecutor : public SimpleConcurrentJobExecutor { + public: + IngestJobExecutor(GraphSpaceID space, + JobID jobId, + kvstore::KVStore* kvstore, + AdminClient* adminClient, + const std::vector& params); + + bool check() override; + + nebula::cpp2::ErrorCode prepare() override; + + folly::Future executeInternal(HostAddr&& address, + std::vector&& parts) override; + + private: + std::vector taskParameters_; +}; + +} // namespace meta +} // namespace nebula + +#endif // META_INGESTJOBEXECUTOR_H_ diff --git a/src/meta/processors/job/JobDescription.h b/src/meta/processors/job/JobDescription.h index 3ae3ffa786c..a1328aa8d9a 100644 --- a/src/meta/processors/job/JobDescription.h +++ b/src/meta/processors/job/JobDescription.h @@ -3,8 +3,8 @@ * This source code is licensed under Apache 2.0 License. */ -#ifndef META_KVJOBDESCRIPTION_H_ -#define META_KVJOBDESCRIPTION_H_ +#ifndef META_JOBDESCRIPTION_H_ +#define META_JOBDESCRIPTION_H_ #include @@ -182,4 +182,4 @@ class JobDescription { } // namespace meta } // namespace nebula -#endif // META_KVJOBDESCRIPTION_H_ +#endif // META_JOBDESCRIPTION_H_ diff --git a/src/meta/processors/job/JobExecutor.cpp b/src/meta/processors/job/JobExecutor.cpp index 41c5cfca5c4..1a11a221e36 100644 --- a/src/meta/processors/job/JobExecutor.cpp +++ b/src/meta/processors/job/JobExecutor.cpp @@ -14,7 +14,9 @@ #include "meta/processors/admin/AdminClient.h" #include "meta/processors/job/CompactJobExecutor.h" #include "meta/processors/job/DataBalanceJobExecutor.h" +#include "meta/processors/job/DownloadJobExecutor.h" #include "meta/processors/job/FlushJobExecutor.h" +#include "meta/processors/job/IngestJobExecutor.h" #include "meta/processors/job/LeaderBalanceJobExecutor.h" #include "meta/processors/job/RebuildEdgeJobExecutor.h" #include "meta/processors/job/RebuildFTJobExecutor.h" @@ -48,6 +50,16 @@ std::unique_ptr JobExecutorFactory::createJobExecutor(const JobDesc case cpp2::JobType::COMPACT: ret.reset(new CompactJobExecutor(jd.getSpace(), jd.getJobId(), store, client, jd.getParas())); break; + case cpp2::JobType::FLUSH: + ret.reset(new FlushJobExecutor(jd.getSpace(), jd.getJobId(), store, client, jd.getParas())); + break; + case cpp2::JobType::DOWNLOAD: + ret.reset( + new DownloadJobExecutor(jd.getSpace(), jd.getJobId(), store, client, jd.getParas())); + break; + case cpp2::JobType::INGEST: + ret.reset(new IngestJobExecutor(jd.getSpace(), jd.getJobId(), store, client, jd.getParas())); + break; case cpp2::JobType::DATA_BALANCE: ret.reset(new DataBalanceJobExecutor(jd, store, client, jd.getParas())); break; @@ -58,9 +70,6 @@ std::unique_ptr JobExecutorFactory::createJobExecutor(const JobDesc ret.reset( new LeaderBalanceJobExecutor(jd.getSpace(), jd.getJobId(), store, client, jd.getParas())); break; - case cpp2::JobType::FLUSH: - ret.reset(new FlushJobExecutor(jd.getSpace(), jd.getJobId(), store, client, jd.getParas())); - break; case cpp2::JobType::REBUILD_TAG_INDEX: ret.reset( new RebuildTagJobExecutor(jd.getSpace(), jd.getJobId(), store, client, jd.getParas())); diff --git a/src/meta/processors/job/JobManager.cpp b/src/meta/processors/job/JobManager.cpp index a1d7f928285..d6967693e98 100644 --- a/src/meta/processors/job/JobManager.cpp +++ b/src/meta/processors/job/JobManager.cpp @@ -121,7 +121,7 @@ void JobManager::scheduleThread() { std::tuple opJobId; while (!tryDequeue(opJobId)) { if (status_.load(std::memory_order_acquire) == JbmgrStatus::STOPPED) { - LOG(INFO) << "[JobManager] detect shutdown called, exit"; + LOG(INFO) << "Detect shutdown called, exit"; break; } usleep(FLAGS_job_check_intervals); @@ -133,12 +133,12 @@ void JobManager::scheduleThread() { std::lock_guard lk(muJobFinished_[spaceId]); auto jobDescRet = JobDescription::loadJobDescription(spaceId, jodId, kvStore_); if (!nebula::ok(jobDescRet)) { - LOG(INFO) << "[JobManager] load an invalid job from space " << spaceId << " jodId " << jodId; + LOG(INFO) << "Load an invalid job from space " << spaceId << " jodId " << jodId; continue; // leader change or archive happened } auto jobDesc = nebula::value(jobDescRet); if (!jobDesc.setStatus(cpp2::JobStatus::RUNNING, jobOp == JbOp::RECOVER)) { - LOG(INFO) << "[JobManager] skip job space " << spaceId << " jodId " << jodId; + LOG(INFO) << "Skip job space " << spaceId << " jodId " << jodId; continue; } @@ -158,8 +158,7 @@ void JobManager::scheduleThread() { } bool JobManager::runJobInternal(const JobDescription& jobDesc, JbOp op) { - std::unique_ptr je = - JobExecutorFactory::createJobExecutor(jobDesc, kvStore_, adminClient_); + auto je = JobExecutorFactory::createJobExecutor(jobDesc, kvStore_, adminClient_); JobExecutor* jobExec = je.get(); runningJobs_.emplace(jobDesc.getJobId(), std::move(je)); @@ -370,7 +369,7 @@ nebula::cpp2::ErrorCode JobManager::reportTaskFinish(const cpp2::ReportTaskReq& }); if (task == tasks.end()) { LOG(INFO) << folly::sformat( - "report an invalid or outdate task, will ignore this report, job={}, " + "Report an invalid or outdate task, will ignore this report, job={}, " "task={}", jobId, taskId); @@ -517,7 +516,7 @@ ErrorOr> JobManager::showJob // skip expired job, default 1 week if (isExpiredJob(optJob)) { lastExpiredJobId = optJob.getJobId(); - LOG(INFO) << "remove expired job " << lastExpiredJobId; + LOG(INFO) << "Remove expired job " << lastExpiredJobId; expiredJobKeys.emplace_back(jobKey); continue; } @@ -701,7 +700,7 @@ ErrorOr JobManager::recoverJob( nebula::cpp2::ErrorCode JobManager::save(const std::string& k, const std::string& v) { std::vector data{std::make_pair(k, v)}; folly::Baton baton; - auto rc = nebula::cpp2::ErrorCode::SUCCEEDED; + nebula::cpp2::ErrorCode rc = nebula::cpp2::ErrorCode::SUCCEEDED; kvStore_->asyncMultiPut( kDefaultSpaceId, kDefaultPartId, std::move(data), [&](nebula::cpp2::ErrorCode code) { rc = code; diff --git a/src/meta/processors/job/JobManager.h b/src/meta/processors/job/JobManager.h index 50cf1243e73..22371b4df8b 100644 --- a/src/meta/processors/job/JobManager.h +++ b/src/meta/processors/job/JobManager.h @@ -41,6 +41,8 @@ class JobManager : public boost::noncopyable, public nebula::cpp::NonMovable { FRIEND_TEST(JobManagerTest, RecoverJob); FRIEND_TEST(JobManagerTest, AddRebuildTagIndexJob); FRIEND_TEST(JobManagerTest, AddRebuildEdgeIndexJob); + FRIEND_TEST(JobManagerTest, DownloadJob); + FRIEND_TEST(JobManagerTest, IngestJob); FRIEND_TEST(GetStatsTest, StatsJob); FRIEND_TEST(GetStatsTest, MockSingleMachineTest); FRIEND_TEST(GetStatsTest, MockMultiMachineTest); diff --git a/src/meta/test/JobManagerTest.cpp b/src/meta/test/JobManagerTest.cpp index b9cc94d1bf3..e0e677739fe 100644 --- a/src/meta/test/JobManagerTest.cpp +++ b/src/meta/test/JobManagerTest.cpp @@ -4,25 +4,34 @@ */ #include +#include #include #include "common/base/Base.h" #include "common/fs/TempDir.h" #include "kvstore/Common.h" #include "meta/ActiveHostsMan.h" +#include "meta/processors/job/DownloadJobExecutor.h" +#include "meta/processors/job/IngestJobExecutor.h" #include "meta/processors/job/JobManager.h" #include "meta/processors/job/TaskDescription.h" #include "meta/test/MockAdminClient.h" +#include "meta/test/MockHdfsHelper.h" #include "meta/test/TestUtils.h" #include "webservice/WebService.h" -DECLARE_int32(ws_storage_http_port); - namespace nebula { namespace meta { +using ::testing::_; +using ::testing::AtLeast; +using ::testing::ByMove; using ::testing::DefaultValue; +using ::testing::NaggyMock; using ::testing::NiceMock; +using ::testing::Return; +using ::testing::SetArgPointee; +using ::testing::StrictMock; class JobManagerTest : public ::testing::Test { protected: @@ -113,6 +122,56 @@ TEST_F(JobManagerTest, AddRebuildEdgeIndexJob) { ASSERT_TRUE(result); } +TEST_F(JobManagerTest, DownloadJob) { + auto rootPath = std::make_unique("/tmp/JobManagerTest.XXXXXX"); + mock::MockCluster cluster; + std::unique_ptr kv = cluster.initMetaKV(rootPath->path()); + ASSERT_TRUE(TestUtils::createSomeHosts(kv.get())); + TestUtils::assembleSpace(kv.get(), 1, 1); + std::vector paras{"hdfs://127.0.0.1:9000/test_space"}; + GraphSpaceID space = 1; + JobID jobId = 11; + JobDescription job(space, jobId, cpp2::JobType::DOWNLOAD, paras); + + MockAdminClient adminClient; + EXPECT_CALL(adminClient, addTask(_, _, _, _, _, _, _)) + .WillOnce(Return(ByMove(folly::makeFuture(Status::OK())))); + + auto executor = + new DownloadJobExecutor(space, job.getJobId(), kv.get(), &adminClient, job.getParas()); + executor->helper_ = std::make_unique(); + + ASSERT_TRUE(executor->check()); + auto code = executor->prepare(); + ASSERT_EQ(code, nebula::cpp2::ErrorCode::SUCCEEDED); + code = executor->execute(); + ASSERT_EQ(code, nebula::cpp2::ErrorCode::SUCCEEDED); +} + +TEST_F(JobManagerTest, IngestJob) { + auto rootPath = std::make_unique("/tmp/DownloadAndIngestTest.XXXXXX"); + mock::MockCluster cluster; + std::unique_ptr kv = cluster.initMetaKV(rootPath->path()); + ASSERT_TRUE(TestUtils::createSomeHosts(kv.get())); + TestUtils::assembleSpace(kv.get(), 1, 1); + std::vector paras{}; + GraphSpaceID space = 1; + JobID jobId = 11; + JobDescription job(space, jobId, cpp2::JobType::INGEST, paras); + + MockAdminClient adminClient; + EXPECT_CALL(adminClient, addTask(_, _, _, _, _, _, _)) + .WillOnce(Return(ByMove(folly::makeFuture(Status::OK())))); + auto executor = + new IngestJobExecutor(space, job.getJobId(), kv.get(), &adminClient, job.getParas()); + + ASSERT_TRUE(executor->check()); + auto code = executor->prepare(); + ASSERT_EQ(code, nebula::cpp2::ErrorCode::SUCCEEDED); + code = executor->execute(); + ASSERT_EQ(code, nebula::cpp2::ErrorCode::SUCCEEDED); +} + TEST_F(JobManagerTest, StatsJob) { std::unique_ptr> jobMgr = getJobManager(); // For preventing job schedule in JobManager @@ -368,7 +427,7 @@ TEST_F(JobManagerTest, ShowJob) { jd.setStatus(cpp2::JobStatus::FINISHED); jobMgr->addJob(jd, adminClient_.get()); - int32_t jobId2 = jd.getJobId(); + JobID jobId2 = jd.getJobId(); int32_t task1 = 0; auto host1 = toHost("127.0.0.1"); @@ -434,7 +493,7 @@ TEST_F(JobManagerTest, ShowJobInOtherSpace) { jd.setStatus(cpp2::JobStatus::FINISHED); jobMgr->addJob(jd, adminClient_.get()); - int32_t jobId2 = jd.getJobId(); + JobID jobId2 = jd.getJobId(); int32_t task1 = 0; auto host1 = toHost("127.0.0.1"); @@ -470,7 +529,6 @@ TEST_F(JobManagerTest, RecoverJob) { jobMgr->status_ = JobManager::JbmgrStatus::STOPPED; jobMgr->bgThread_.join(); GraphSpaceID spaceId = 1; - int32_t nJob = 3; for (auto jobId = 0; jobId < nJob; ++jobId) { JobDescription jd(spaceId, jobId, cpp2::JobType::FLUSH); @@ -518,7 +576,7 @@ TEST(JobDescriptionTest, Ctor2) { TEST(JobDescriptionTest, ParseKey) { GraphSpaceID spaceId = 1; - int32_t jobId = std::pow(2, 16); + JobID jobId = std::pow(2, 16); JobDescription jd(spaceId, jobId, cpp2::JobType::COMPACT); ASSERT_EQ(jobId, jd.getJobId()); ASSERT_EQ(cpp2::JobType::COMPACT, jd.getJobType()); @@ -531,7 +589,7 @@ TEST(JobDescriptionTest, ParseKey) { TEST(JobDescriptionTest, ParseVal) { GraphSpaceID spaceId = 1; - int32_t jobId = std::pow(2, 15); + JobID jobId = std::pow(2, 15); JobDescription jd(spaceId, jobId, cpp2::JobType::FLUSH); auto status = cpp2::JobStatus::FINISHED; jd.setStatus(cpp2::JobStatus::RUNNING); @@ -558,8 +616,8 @@ TEST(JobDescriptionTest, ParseVal) { TEST(TaskDescriptionTest, Ctor) { GraphSpaceID spaceId = 1; - int32_t jobId = std::pow(2, 4); - int32_t taskId = 2; + JobID jobId = std::pow(2, 4); + TaskID taskId = 2; auto dest = toHost(""); TaskDescription td(spaceId, jobId, taskId, dest); auto status = cpp2::JobStatus::RUNNING; @@ -574,8 +632,8 @@ TEST(TaskDescriptionTest, Ctor) { TEST(TaskDescriptionTest, ParseKey) { GraphSpaceID spaceId = 1; - int32_t jobId = std::pow(2, 5); - int32_t taskId = 2; + JobID jobId = std::pow(2, 5); + TaskID taskId = 2; std::string dest{"127.0.0.1"}; TaskDescription td(spaceId, jobId, taskId, toHost(dest)); @@ -589,8 +647,8 @@ TEST(TaskDescriptionTest, ParseKey) { TEST(TaskDescriptionTest, ParseVal) { GraphSpaceID spaceId = 1; - int32_t jobId = std::pow(2, 5); - int32_t taskId = 3; + JobID jobId = std::pow(2, 5); + TaskID taskId = 3; std::string dest{"127.0.0.1"}; TaskDescription td(spaceId, jobId, taskId, toHost(dest)); diff --git a/src/meta/test/MockHdfsHelper.h b/src/meta/test/MockHdfsHelper.h index 9d911466efc..295bfbad92d 100644 --- a/src/meta/test/MockHdfsHelper.h +++ b/src/meta/test/MockHdfsHelper.h @@ -13,24 +13,15 @@ namespace meta { class MockHdfsOKHelper : public nebula::hdfs::HdfsHelper { public: - StatusOr ls(const std::string& hdfsHost, - int32_t hdfsPort, - const std::string& hdfsPath) override { - UNUSED(hdfsHost); - UNUSED(hdfsPort); - UNUSED(hdfsPath); + StatusOr ls(const std::string&, int32_t, const std::string&) override { sleep(1); return "total 2\n0000.sst\n000.sst"; } - StatusOr copyToLocal(const std::string& hdfsHost, - int32_t hdfsPort, - const std::string& hdfsPath, - const std::string& localPath) override { - UNUSED(hdfsHost); - UNUSED(hdfsPort); - UNUSED(hdfsPath); - UNUSED(localPath); + StatusOr copyToLocal(const std::string&, + int32_t, + const std::string&, + const std::string&) override { sleep(1); return ""; } @@ -42,22 +33,15 @@ class MockHdfsOKHelper : public nebula::hdfs::HdfsHelper { class MockHdfsNotExistHelper : public nebula::hdfs::HdfsHelper { public: - StatusOr ls(const std::string& hdfsHost, - int32_t hdfsPort, - const std::string& hdfsPath) override { - UNUSED(hdfsHost); - UNUSED(hdfsPort); + StatusOr ls(const std::string&, int32_t, const std::string& hdfsPath) override { sleep(1); return Status::Error(folly::stringPrintf("HDFS Path %s Not Exist", hdfsPath.c_str())); } - StatusOr copyToLocal(const std::string& hdfsHost, - int32_t hdfsPort, + StatusOr copyToLocal(const std::string&, + int32_t, const std::string& hdfsPath, - const std::string& localPath) override { - UNUSED(hdfsHost); - UNUSED(hdfsPort); - UNUSED(localPath); + const std::string&) override { sleep(1); return Status::Error(folly::stringPrintf("HDFS Path %s Not Exist", hdfsPath.c_str())); } diff --git a/src/parser/AdminSentences.cpp b/src/parser/AdminSentences.cpp index c9a1e2ff13c..b72735bb18a 100644 --- a/src/parser/AdminSentences.cpp +++ b/src/parser/AdminSentences.cpp @@ -258,10 +258,9 @@ std::string AdminJobSentence::toString() const { case meta::cpp2::JobType::STATS: return "SUBMIT JOB STATS"; case meta::cpp2::JobType::DOWNLOAD: - return paras_.empty() ? "DOWNLOAD HDFS " - : folly::stringPrintf("DOWNLOAD HDFS %s", paras_[0].c_str()); + return folly::stringPrintf("SUBMIT JOB DOWNLOAD HDFS \"%s\"", paras_[0].c_str()); case meta::cpp2::JobType::INGEST: - return "INGEST"; + return "SUBMIT JOB INGEST"; case meta::cpp2::JobType::DATA_BALANCE: if (paras_.empty()) { return "SUBMIT JOB BALANCE IN ZONE"; diff --git a/src/parser/AdminSentences.h b/src/parser/AdminSentences.h index 2ece4c54405..23b7a8e0230 100644 --- a/src/parser/AdminSentences.h +++ b/src/parser/AdminSentences.h @@ -259,15 +259,6 @@ class SpaceOptItem final { } } - std::string getGroupName() const { - if (isString()) { - return asString(); - } else { - LOG(ERROR) << "group name value illegal."; - return ""; - } - } - OptionType getOptType() const { return optType_; } diff --git a/src/parser/MutateSentences.cpp b/src/parser/MutateSentences.cpp index 077d58d1b11..09d4596b339 100644 --- a/src/parser/MutateSentences.cpp +++ b/src/parser/MutateSentences.cpp @@ -283,13 +283,4 @@ std::string DeleteEdgesSentence::toString() const { return buf; } -std::string DownloadSentence::toString() const { - return folly::stringPrintf( - "DOWNLOAD HDFS \"hdfs://%s:%d%s\"", host_.get()->c_str(), port_, path_.get()->c_str()); -} - -std::string IngestSentence::toString() const { - return "INGEST"; -} - } // namespace nebula diff --git a/src/parser/MutateSentences.h b/src/parser/MutateSentences.h index 2fcc12ed31b..42347cb0b78 100644 --- a/src/parser/MutateSentences.h +++ b/src/parser/MutateSentences.h @@ -610,74 +610,5 @@ class DeleteEdgesSentence final : public Sentence { std::unique_ptr edgeKeyRef_; }; -class DownloadSentence final : public Sentence { - public: - DownloadSentence() { - kind_ = Kind::kDownload; - } - - const std::string *host() const { - return host_.get(); - } - - int32_t port() const { - return port_; - } - - void setPort(int32_t port) { - port_ = port; - } - - const std::string *path() const { - return path_.get(); - } - - void setUrl(std::string &url) { - static std::string hdfsPrefix = "hdfs://"; - if (url.find(hdfsPrefix) != 0) { - LOG(ERROR) << "URL should start with " << hdfsPrefix; - return; - } - - std::string u = url.substr(hdfsPrefix.size(), url.size()); - std::vector tokens; - folly::split(":", u, tokens); - if (tokens.size() == 2) { - host_ = std::make_unique(tokens[0]); - int32_t position = tokens[1].find_first_of("/"); - if (position != -1) { - try { - port_ = folly::to(tokens[1].toString().substr(0, position).c_str()); - } catch (const std::exception &ex) { - LOG(ERROR) << "URL's port parse failed: " << url; - return; - } - path_ = - std::make_unique(tokens[1].toString().substr(position, tokens[1].size())); - } else { - LOG(ERROR) << "URL Parse Failed: " << url; - } - } else { - LOG(ERROR) << "URL Parse Failed: " << url; - } - } - - std::string toString() const override; - - private: - std::unique_ptr host_; - int32_t port_; - std::unique_ptr path_; -}; - -class IngestSentence final : public Sentence { - public: - IngestSentence() { - kind_ = Kind::kIngest; - } - - std::string toString() const override; -}; - } // namespace nebula #endif // PARSER_MUTATESENTENCES_H_ diff --git a/src/parser/Sentence.h b/src/parser/Sentence.h index 9a10e5e2daa..18b998a4d65 100644 --- a/src/parser/Sentence.h +++ b/src/parser/Sentence.h @@ -97,8 +97,6 @@ class Sentence { kGrant, kRevoke, kChangePassword, - kDownload, - kIngest, kOrderBy, kShowConfigs, kSetConfig, diff --git a/src/parser/parser.yy b/src/parser/parser.yy index 5abb2a1e95f..9147a668c30 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -3237,10 +3237,18 @@ delete_tag_sentence download_sentence : KW_DOWNLOAD KW_HDFS STRING { - auto sentence = new DownloadSentence(); - sentence->setUrl(*$3); + auto sentence = new AdminJobSentence(meta::cpp2::JobOp::ADD, + meta::cpp2::JobType::DOWNLOAD); + sentence->addPara(*$3); + $$ = sentence; + } + ; + +ingest_sentence + : KW_INGEST { + auto sentence = new AdminJobSentence(meta::cpp2::JobOp::ADD, + meta::cpp2::JobType::INGEST); $$ = sentence; - delete $3; } ; @@ -3256,13 +3264,6 @@ delete_edge_sentence } ; -ingest_sentence - : KW_INGEST { - auto sentence = new IngestSentence(); - $$ = sentence; - } - ; - admin_job_sentence : KW_SUBMIT KW_JOB KW_COMPACT { auto sentence = new AdminJobSentence(meta::cpp2::JobOp::ADD, @@ -3274,6 +3275,17 @@ admin_job_sentence meta::cpp2::JobType::FLUSH); $$ = sentence; } + | KW_SUBMIT KW_JOB KW_DOWNLOAD KW_HDFS STRING { + auto sentence = new AdminJobSentence(meta::cpp2::JobOp::ADD, + meta::cpp2::JobType::DOWNLOAD); + sentence->addPara(*$5); + $$ = sentence; + } + | KW_SUBMIT KW_JOB KW_INGEST { + auto sentence = new AdminJobSentence(meta::cpp2::JobOp::ADD, + meta::cpp2::JobType::INGEST); + $$ = sentence; + } | KW_SUBMIT KW_JOB KW_STATS { auto sentence = new AdminJobSentence(meta::cpp2::JobOp::ADD, meta::cpp2::JobType::STATS); diff --git a/src/parser/test/ParserTest.cpp b/src/parser/test/ParserTest.cpp index 71112a058fb..4517c67761b 100644 --- a/src/parser/test/ParserTest.cpp +++ b/src/parser/test/ParserTest.cpp @@ -3220,6 +3220,12 @@ TEST_F(ParserTest, JobTest) { }; checkTest("SUBMIT JOB COMPACT", "SUBMIT JOB COMPACT"); checkTest("SUBMIT JOB FLUSH", "SUBMIT JOB FLUSH"); + + checkTest("SUBMIT JOB DOWNLOAD HDFS \"hdfs://127.0.0.1:9090/data\"", + "SUBMIT JOB DOWNLOAD HDFS \"hdfs://127.0.0.1:9090/data\""); + + checkTest("SUBMIT JOB INGEST", "SUBMIT JOB INGEST"); + checkTest("SUBMIT JOB STATS", "SUBMIT JOB STATS"); checkTest("SUBMIT JOB BALANCE IN ZONE", "SUBMIT JOB BALANCE IN ZONE"); checkTest( diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 55874b5cc30..86d4ab2758e 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -16,6 +16,8 @@ nebula_add_library( admin/AdminTask.cpp admin/CompactTask.cpp admin/FlushTask.cpp + admin/DownloadTask.cpp + admin/IngestTask.cpp admin/RebuildIndexTask.cpp admin/RebuildTagIndexTask.cpp admin/RebuildEdgeIndexTask.cpp @@ -64,8 +66,6 @@ nebula_add_library( nebula_add_library( storage_http_handler OBJECT - http/StorageHttpIngestHandler.cpp - http/StorageHttpDownloadHandler.cpp http/StorageHttpAdminHandler.cpp http/StorageHttpStatsHandler.cpp http/StorageHttpPropertyHandler.cpp diff --git a/src/storage/StorageServer.cpp b/src/storage/StorageServer.cpp index 90b4305877f..f2ed6487d49 100644 --- a/src/storage/StorageServer.cpp +++ b/src/storage/StorageServer.cpp @@ -27,8 +27,6 @@ #include "storage/StorageAdminServiceHandler.h" #include "storage/StorageFlags.h" #include "storage/http/StorageHttpAdminHandler.h" -#include "storage/http/StorageHttpDownloadHandler.h" -#include "storage/http/StorageHttpIngestHandler.h" #include "storage/http/StorageHttpPropertyHandler.h" #include "storage/http/StorageHttpStatsHandler.h" #include "storage/transaction/TransactionManager.h" @@ -104,16 +102,6 @@ bool StorageServer::initWebService() { webSvc_ = std::make_unique(); auto& router = webSvc_->router(); - router.get("/download").handler([this](web::PathParams&&) { - auto* handler = new storage::StorageHttpDownloadHandler(); - handler->init(hdfsHelper_.get(), webWorkers_.get(), kvstore_.get(), dataPaths_); - return handler; - }); - router.get("/ingest").handler([this](web::PathParams&&) { - auto handler = new nebula::storage::StorageHttpIngestHandler(); - handler->init(kvstore_.get()); - return handler; - }); router.get("/admin").handler([this](web::PathParams&&) { return new storage::StorageHttpAdminHandler(schemaMan_.get(), kvstore_.get()); }); diff --git a/src/storage/admin/AdminTask.cpp b/src/storage/admin/AdminTask.cpp index 90608c18f26..6f44d9595ac 100644 --- a/src/storage/admin/AdminTask.cpp +++ b/src/storage/admin/AdminTask.cpp @@ -6,7 +6,9 @@ #include "storage/admin/AdminTask.h" #include "storage/admin/CompactTask.h" +#include "storage/admin/DownloadTask.h" #include "storage/admin/FlushTask.h" +#include "storage/admin/IngestTask.h" #include "storage/admin/RebuildEdgeIndexTask.h" #include "storage/admin/RebuildFTIndexTask.h" #include "storage/admin/RebuildTagIndexTask.h" @@ -37,6 +39,12 @@ std::shared_ptr AdminTaskFactory::createAdminTask(StorageEnv* env, Ta case meta::cpp2::JobType::STATS: ret = std::make_shared(env, std::move(ctx)); break; + case meta::cpp2::JobType::DOWNLOAD: + ret = std::make_shared(env, std::move(ctx)); + break; + case meta::cpp2::JobType::INGEST: + ret = std::make_shared(env, std::move(ctx)); + break; default: break; } diff --git a/src/storage/admin/AdminTask.h b/src/storage/admin/AdminTask.h index 8e2ebbcb46f..a2d47fc357e 100644 --- a/src/storage/admin/AdminTask.h +++ b/src/storage/admin/AdminTask.h @@ -9,9 +9,6 @@ #include #include -#include "common/thrift/ThriftTypes.h" -#include "interface/gen-cpp2/meta_types.h" -#include "interface/gen-cpp2/storage_types.h" #include "kvstore/Common.h" #include "kvstore/NebulaStore.h" #include "storage/CommonUtils.h" @@ -19,6 +16,8 @@ namespace nebula { namespace storage { +using TaskFunction = std::function; + /** * @brief Subtask class for admin tasks. An admin task comprises a sequence of subtasks. * @@ -27,7 +26,7 @@ class AdminSubTask { public: AdminSubTask() = default; - explicit AdminSubTask(std::function f) : run_(f) {} + explicit AdminSubTask(TaskFunction f) : run_(f) {} /** * @brief Entry point to invoke sub tasks function. @@ -39,7 +38,7 @@ class AdminSubTask { } private: - std::function run_; + TaskFunction run_; }; enum class TaskPriority : int8_t { LO, MID, HI }; @@ -89,6 +88,12 @@ class AdminTask { virtual ~AdminTask() {} + /** + * @brief Check the argument + * + */ + virtual bool check() = 0; + /** * @brief Set the Callback object * @@ -172,7 +177,7 @@ class AdminTask { * @param rc Errorcode. */ virtual void subTaskFinish(nebula::cpp2::ErrorCode rc) { - auto suc = nebula::cpp2::ErrorCode::SUCCEEDED; + nebula::cpp2::ErrorCode suc = nebula::cpp2::ErrorCode::SUCCEEDED; rc_.compare_exchange_strong(suc, rc); } @@ -183,7 +188,7 @@ class AdminTask { virtual void cancel() { FLOG_INFO("task(%d, %d) cancelled", ctx_.jobId_, ctx_.taskId_); canceled_ = true; - auto suc = nebula::cpp2::ErrorCode::SUCCEEDED; + nebula::cpp2::ErrorCode suc = nebula::cpp2::ErrorCode::SUCCEEDED; rc_.compare_exchange_strong(suc, nebula::cpp2::ErrorCode::E_USER_CANCEL); } diff --git a/src/storage/admin/AdminTaskProcessor.h b/src/storage/admin/AdminTaskProcessor.h index de3f3cdb7d5..76c0beda78e 100644 --- a/src/storage/admin/AdminTaskProcessor.h +++ b/src/storage/admin/AdminTaskProcessor.h @@ -7,7 +7,6 @@ #define STORAGE_ADMIN_ADMINTASKPROCESSOR_H_ #include "common/base/Base.h" -#include "common/thrift/ThriftTypes.h" #include "interface/gen-cpp2/meta_types.h" #include "kvstore/NebulaStore.h" #include "storage/BaseProcessor.h" diff --git a/src/storage/admin/CompactTask.cpp b/src/storage/admin/CompactTask.cpp index 913a6bc6924..a746d71c427 100644 --- a/src/storage/admin/CompactTask.cpp +++ b/src/storage/admin/CompactTask.cpp @@ -10,6 +10,10 @@ namespace nebula { namespace storage { +bool CompactTask::check() { + return env_->kvstore_ != nullptr; +} + ErrorOr> CompactTask::genSubTasks() { std::vector ret; if (!env_->kvstore_) { diff --git a/src/storage/admin/CompactTask.h b/src/storage/admin/CompactTask.h index e01bc2d0a8d..e9479ca2dc4 100644 --- a/src/storage/admin/CompactTask.h +++ b/src/storage/admin/CompactTask.h @@ -6,7 +6,6 @@ #ifndef STORAGE_ADMIN_COMPACTTASK_H_ #define STORAGE_ADMIN_COMPACTTASK_H_ -#include "common/thrift/ThriftTypes.h" #include "kvstore/KVEngine.h" #include "kvstore/NebulaStore.h" #include "storage/admin/AdminTask.h" @@ -22,6 +21,8 @@ class CompactTask : public AdminTask { public: CompactTask(StorageEnv* env, TaskContext&& ctx) : AdminTask(env, std::move(ctx)) {} + bool check() override; + /** * @brief Generate subtasks for compact. * diff --git a/src/storage/admin/DownloadTask.cpp b/src/storage/admin/DownloadTask.cpp new file mode 100644 index 00000000000..cca84d1a759 --- /dev/null +++ b/src/storage/admin/DownloadTask.cpp @@ -0,0 +1,59 @@ +/* Copyright (c) 2022 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "storage/admin/DownloadTask.h" + +#include "common/fs/FileUtils.h" + +namespace nebula { +namespace storage { + +bool DownloadTask::check() { + return env_->kvstore_ != nullptr; +} + +ErrorOr> DownloadTask::genSubTasks() { + auto space = *ctx_.parameters_.space_id_ref(); + auto parts = *ctx_.parameters_.parts_ref(); + auto paras = ctx_.parameters_.task_specific_paras_ref(); + if (!paras.has_value() || paras->size() != 3) { + LOG(ERROR) << "Download Task should be three parameters"; + return nebula::cpp2::ErrorCode::E_INVALID_PARM; + } + + hdfsHost_ = (*paras)[0]; + hdfsPort_ = folly::to((*paras)[1]); + hdfsPath_ = (*paras)[2]; + std::vector tasks; + for (const auto& part : parts) { + TaskFunction task = std::bind(&DownloadTask::subTask, this, space, part); + tasks.emplace_back(std::move(task)); + } + return tasks; +} + +nebula::cpp2::ErrorCode DownloadTask::subTask(GraphSpaceID space, PartitionID part) { + LOG(INFO) << "Space: " << space << " Part: " << part; + auto hdfsPartPath = folly::stringPrintf("%s/%d", hdfsPath_.c_str(), part); + auto partResult = env_->kvstore_->part(space, part); + if (!ok(partResult)) { + LOG(ERROR) << "Can't found space: " << space << ", part: " << part; + return nebula::cpp2::ErrorCode::E_PART_NOT_FOUND; + } + + auto localPath = folly::stringPrintf("%s/download/", value(partResult)->engine()->getDataRoot()); + if (fs::FileUtils::fileType(localPath.c_str()) == fs::FileType::NOTEXIST) { + if (!fs::FileUtils::makeDir(localPath)) { + return nebula::cpp2::ErrorCode::E_TASK_EXECUTION_FAILED; + } + } + + auto result = helper_->copyToLocal(hdfsHost_, hdfsPort_, hdfsPartPath, localPath); + return result.ok() ? nebula::cpp2::ErrorCode::SUCCEEDED + : nebula::cpp2::ErrorCode::E_TASK_EXECUTION_FAILED; +} + +} // namespace storage +} // namespace nebula diff --git a/src/storage/admin/DownloadTask.h b/src/storage/admin/DownloadTask.h new file mode 100644 index 00000000000..2b3f8785931 --- /dev/null +++ b/src/storage/admin/DownloadTask.h @@ -0,0 +1,42 @@ +/* Copyright (c) 2022 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef STORAGE_ADMIN_DOWNLOADTASK_H_ +#define STORAGE_ADMIN_DOWNLOADTASK_H_ + +#include "common/hdfs/HdfsCommandHelper.h" +#include "storage/admin/AdminTask.h" + +namespace nebula { +namespace storage { + +/** + * @brief Task class to handle storage download task. + * + */ +class DownloadTask : public AdminTask { + public: + DownloadTask(StorageEnv* env, TaskContext&& ctx) : AdminTask(env, std::move(ctx)) { + helper_ = std::make_unique(); + } + + bool check() override; + + ErrorOr> genSubTasks() override; + + private: + nebula::cpp2::ErrorCode subTask(GraphSpaceID space, PartitionID part); + + private: + std::string hdfsPath_; + std::string hdfsHost_; + int32_t hdfsPort_; + std::unique_ptr helper_; +}; + +} // namespace storage +} // namespace nebula + +#endif // STORAGE_ADMIN_DOWNLOADTASK_H_ diff --git a/src/storage/admin/FlushTask.cpp b/src/storage/admin/FlushTask.cpp index 6d9f4be9bac..6f6ff86e85e 100644 --- a/src/storage/admin/FlushTask.cpp +++ b/src/storage/admin/FlushTask.cpp @@ -10,12 +10,12 @@ namespace nebula { namespace storage { +bool FlushTask::check() { + return env_->kvstore_ != nullptr; +} + ErrorOr> FlushTask::genSubTasks() { std::vector ret; - if (!env_->kvstore_) { - return ret; - } - auto* store = dynamic_cast(env_->kvstore_); auto errOrSpace = store->space(*ctx_.parameters_.space_id_ref()); if (!ok(errOrSpace)) { diff --git a/src/storage/admin/FlushTask.h b/src/storage/admin/FlushTask.h index 243dda1e8c4..ea83a6fecf9 100644 --- a/src/storage/admin/FlushTask.h +++ b/src/storage/admin/FlushTask.h @@ -6,7 +6,6 @@ #ifndef STORAGE_ADMIN_FLUSHTASK_H_ #define STORAGE_ADMIN_FLUSHTASK_H_ -#include "common/thrift/ThriftTypes.h" #include "kvstore/KVEngine.h" #include "kvstore/NebulaStore.h" #include "storage/admin/AdminTask.h" @@ -21,6 +20,9 @@ namespace storage { class FlushTask : public AdminTask { public: FlushTask(StorageEnv* env, TaskContext&& ctx) : AdminTask(env, std::move(ctx)) {} + + bool check() override; + /** * @brief Generage subtasks for flushing. * diff --git a/src/storage/admin/IngestTask.cpp b/src/storage/admin/IngestTask.cpp new file mode 100644 index 00000000000..9a85ae10139 --- /dev/null +++ b/src/storage/admin/IngestTask.cpp @@ -0,0 +1,51 @@ +/* Copyright (c) 2022 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "storage/admin/IngestTask.h" + +#include "common/fs/FileUtils.h" + +namespace nebula { +namespace storage { + +bool IngestTask::check() { + return env_->kvstore_ != nullptr; +} + +ErrorOr> IngestTask::genSubTasks() { + std::vector results; + auto* store = dynamic_cast(env_->kvstore_); + auto errOrSpace = store->space(*ctx_.parameters_.space_id_ref()); + if (!ok(errOrSpace)) { + LOG(ERROR) << "Space not found"; + return error(errOrSpace); + } + + auto space = nebula::value(errOrSpace); + results.emplace_back([space = space]() { + for (auto& engine : space->engines_) { + auto parts = engine->allParts(); + for (auto part : parts) { + auto path = folly::stringPrintf("%s/download/%d", engine->getDataRoot(), part); + if (!fs::FileUtils::exist(path)) { + LOG(INFO) << path << " not existed"; + continue; + } + + auto files = nebula::fs::FileUtils::listAllFilesInDir(path.c_str(), true, "*.sst"); + LOG(INFO) << "Ingest files: " << files.size(); + auto code = engine->ingest(std::vector(files)); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + return code; + } + } + } + return nebula::cpp2::ErrorCode::SUCCEEDED; + }); + return results; +} + +} // namespace storage +} // namespace nebula diff --git a/src/storage/admin/IngestTask.h b/src/storage/admin/IngestTask.h new file mode 100644 index 00000000000..e46754ac3cb --- /dev/null +++ b/src/storage/admin/IngestTask.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2022 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef STORAGE_ADMIN_INGESTTASK_H_ +#define STORAGE_ADMIN_INGESTTASK_H_ + +#include "storage/admin/AdminTask.h" + +namespace nebula { +namespace storage { + +/** + * @brief Task class to handle storage ingest task. + * + */ +class IngestTask : public AdminTask { + public: + IngestTask(StorageEnv* env, TaskContext&& ctx) : AdminTask(env, std::move(ctx)) {} + + bool check() override; + + ErrorOr> genSubTasks() override; +}; + +} // namespace storage +} // namespace nebula + +#endif // STORAGE_ADMIN_INGESTTASK_H_ diff --git a/src/storage/admin/RebuildEdgeIndexTask.cpp b/src/storage/admin/RebuildEdgeIndexTask.cpp index 0e14e2f1068..78fa54d0286 100644 --- a/src/storage/admin/RebuildEdgeIndexTask.cpp +++ b/src/storage/admin/RebuildEdgeIndexTask.cpp @@ -48,8 +48,8 @@ nebula::cpp2::ErrorCode RebuildEdgeIndexTask::buildIndexGlobal(GraphSpaceID spac LOG(INFO) << "Get space edge schema failed"; return nebula::cpp2::ErrorCode::E_EDGE_NOT_FOUND; } - auto schemas = schemasRet.value(); + auto schemas = schemasRet.value(); auto vidSize = vidSizeRet.value(); std::unique_ptr iter; const auto& prefix = NebulaKeyUtils::edgePrefix(part); @@ -114,8 +114,8 @@ nebula::cpp2::ErrorCode RebuildEdgeIndexTask::buildIndexGlobal(GraphSpaceID spac iter->next(); continue; } - auto* schema = schemaIter->second.get(); + auto* schema = schemaIter->second.get(); auto ttlProp = CommonUtils::ttlProps(schema); if (ttlProp.first && CommonUtils::checkDataExpiredForTTL( schema, reader.get(), ttlProp.second.second, ttlProp.second.first)) { diff --git a/src/storage/admin/RebuildFTIndexTask.cpp b/src/storage/admin/RebuildFTIndexTask.cpp index bffb59b5d2b..fc8af8989ea 100644 --- a/src/storage/admin/RebuildFTIndexTask.cpp +++ b/src/storage/admin/RebuildFTIndexTask.cpp @@ -12,6 +12,10 @@ DECLARE_uint32(raft_heartbeat_interval_secs); namespace nebula { namespace storage { +bool RebuildFTIndexTask::check() { + return env_->kvstore_ != nullptr; +} + ErrorOr> RebuildFTIndexTask::genSubTasks() { std::vector tasks; VLOG(1) << "Begin rebuild fulltext indexes, space : " << *ctx_.parameters_.space_id_ref(); @@ -47,8 +51,7 @@ ErrorOr> RebuildFTIndexTask:: VLOG(1) << folly::sformat("Processing fulltext rebuild subtask, space={}, part={}", *ctx_.parameters_.space_id_ref(), part); - std::function task = - std::bind(&RebuildFTIndexTask::taskByPart, this, listener); + TaskFunction task = std::bind(&RebuildFTIndexTask::taskByPart, this, listener); tasks.emplace_back(std::move(task)); } return tasks; diff --git a/src/storage/admin/RebuildFTIndexTask.h b/src/storage/admin/RebuildFTIndexTask.h index 528e508dfaf..72835b8fd7f 100644 --- a/src/storage/admin/RebuildFTIndexTask.h +++ b/src/storage/admin/RebuildFTIndexTask.h @@ -6,7 +6,6 @@ #ifndef STORAGE_ADMIN_REBUILDFTINDEXTASK_H_ #define STORAGE_ADMIN_REBUILDFTINDEXTASK_H_ -#include "common/thrift/ThriftTypes.h" #include "kvstore/KVEngine.h" #include "kvstore/NebulaStore.h" #include "storage/admin/AdminTask.h" @@ -21,6 +20,9 @@ namespace storage { class RebuildFTIndexTask : public AdminTask { public: RebuildFTIndexTask(StorageEnv* env, TaskContext&& ctx) : AdminTask(env, std::move(ctx)) {} + + bool check() override; + /** * @brief Generate subtasks for rebuilding FT index. * diff --git a/src/storage/admin/RebuildIndexTask.cpp b/src/storage/admin/RebuildIndexTask.cpp index ca1988873b4..d3e883e27ba 100644 --- a/src/storage/admin/RebuildIndexTask.cpp +++ b/src/storage/admin/RebuildIndexTask.cpp @@ -14,6 +14,10 @@ namespace storage { const int32_t kReserveNum = 1024 * 4; +bool RebuildIndexTask::check() { + return env_->kvstore_ != nullptr; +} + RebuildIndexTask::RebuildIndexTask(StorageEnv* env, TaskContext&& ctx) : AdminTask(env, std::move(ctx)) { // Rebuild index rate is limited to FLAGS_rebuild_index_part_rate_limit * SubTaskConcurrency. As @@ -26,7 +30,6 @@ RebuildIndexTask::RebuildIndexTask(StorageEnv* env, TaskContext&& ctx) } ErrorOr> RebuildIndexTask::genSubTasks() { - CHECK_NOTNULL(env_->kvstore_); space_ = *ctx_.parameters_.space_id_ref(); auto parts = *ctx_.parameters_.parts_ref(); @@ -67,8 +70,7 @@ ErrorOr> RebuildIndexTask::ge for (const auto& part : parts) { env_->rebuildIndexGuard_->insert_or_assign(std::make_tuple(space_, part), IndexState::STARTING); - std::function task = - std::bind(&RebuildIndexTask::invoke, this, space_, part, items); + TaskFunction task = std::bind(&RebuildIndexTask::invoke, this, space_, part, items); tasks.emplace_back(std::move(task)); } return tasks; diff --git a/src/storage/admin/RebuildIndexTask.h b/src/storage/admin/RebuildIndexTask.h index 6c889855cb6..2f3eac072b9 100644 --- a/src/storage/admin/RebuildIndexTask.h +++ b/src/storage/admin/RebuildIndexTask.h @@ -29,6 +29,8 @@ class RebuildIndexTask : public AdminTask { LOG(INFO) << "Release Rebuild Task"; } + bool check() override; + /** * @brief Generate subtasks for rebuilding index. * diff --git a/src/storage/admin/StatsTask.cpp b/src/storage/admin/StatsTask.cpp index acb2ae89681..8008df3173d 100644 --- a/src/storage/admin/StatsTask.cpp +++ b/src/storage/admin/StatsTask.cpp @@ -14,8 +14,11 @@ namespace nebula { namespace storage { +bool StatsTask::check() { + return env_->kvstore_ != nullptr && env_->schemaMan_ != nullptr; +} + nebula::cpp2::ErrorCode StatsTask::getSchemas(GraphSpaceID spaceId) { - CHECK_NOTNULL(env_->schemaMan_); auto tags = env_->schemaMan_->getAllVerTagSchema(spaceId); if (!tags.ok()) { return nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND; @@ -61,8 +64,7 @@ ErrorOr> StatsTask::genSubTas std::vector tasks; for (const auto& part : parts) { - std::function task = - std::bind(&StatsTask::genSubTask, this, spaceId_, part, tags_, edges_); + TaskFunction task = std::bind(&StatsTask::genSubTask, this, spaceId_, part, tags_, edges_); tasks.emplace_back(std::move(task)); } return tasks; @@ -100,7 +102,6 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId, auto partitionNum = partitionNumRet.value(); LOG(INFO) << "Start stats task"; - CHECK_NOTNULL(env_->kvstore_); auto tagPrefix = NebulaKeyUtils::tagPrefix(part); std::unique_ptr tagIter; auto edgePrefix = NebulaKeyUtils::edgePrefix(part); diff --git a/src/storage/admin/StatsTask.h b/src/storage/admin/StatsTask.h index 9de607e151e..5b618fc0d63 100644 --- a/src/storage/admin/StatsTask.h +++ b/src/storage/admin/StatsTask.h @@ -6,7 +6,6 @@ #ifndef STORAGE_ADMIN_STATSTASK_H_ #define STORAGE_ADMIN_STATSTASK_H_ -#include "common/thrift/ThriftTypes.h" #include "interface/gen-cpp2/meta_types.h" #include "kvstore/KVEngine.h" #include "kvstore/NebulaStore.h" @@ -28,6 +27,8 @@ class StatsTask : public AdminTask { LOG(INFO) << "Release Stats Task"; } + bool check() override; + /** * @brief Generate sub tasks for StatsTask. * diff --git a/src/storage/admin/StopAdminTaskProcessor.h b/src/storage/admin/StopAdminTaskProcessor.h index b8363283916..b3418e50041 100644 --- a/src/storage/admin/StopAdminTaskProcessor.h +++ b/src/storage/admin/StopAdminTaskProcessor.h @@ -7,7 +7,6 @@ #define STORAGE_ADMIN_STOPADMINTASKPROCESSOR_H_ #include "common/base/Base.h" -#include "common/thrift/ThriftTypes.h" #include "kvstore/NebulaStore.h" #include "storage/BaseProcessor.h" #include "storage/StorageFlags.h" diff --git a/src/storage/http/StorageHttpDownloadHandler.cpp b/src/storage/http/StorageHttpDownloadHandler.cpp deleted file mode 100644 index ed15bc77f50..00000000000 --- a/src/storage/http/StorageHttpDownloadHandler.cpp +++ /dev/null @@ -1,197 +0,0 @@ -/* Copyright (c) 2019 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "storage/http/StorageHttpDownloadHandler.h" - -#include -#include -#include - -#include "common/fs/FileUtils.h" -#include "common/hdfs/HdfsHelper.h" -#include "common/process/ProcessUtils.h" -#include "kvstore/Part.h" -#include "webservice/Common.h" - -DEFINE_int32(download_thread_num, 3, "download thread number"); - -namespace nebula { -namespace storage { - -using proxygen::HTTPMessage; -using proxygen::HTTPMethod; -using proxygen::ProxygenError; -using proxygen::ResponseBuilder; -using proxygen::UpgradeProtocol; - -void StorageHttpDownloadHandler::init(nebula::hdfs::HdfsHelper* helper, - nebula::thread::GenericThreadPool* pool, - nebula::kvstore::KVStore* kvstore, - std::vector paths) { - helper_ = helper; - pool_ = pool; - kvstore_ = kvstore; - paths_ = paths; - CHECK_NOTNULL(helper_); - CHECK_NOTNULL(pool_); - CHECK_NOTNULL(kvstore_); - CHECK(!paths_.empty()); -} - -void StorageHttpDownloadHandler::onRequest(std::unique_ptr headers) noexcept { - if (headers->getMethod().value() != HTTPMethod::GET) { - // Unsupported method - err_ = HttpCode::E_UNSUPPORTED_METHOD; - return; - } - - if (!headers->hasQueryParam("host") || !headers->hasQueryParam("port") || - !headers->hasQueryParam("path") || !headers->hasQueryParam("parts") || - !headers->hasQueryParam("space")) { - LOG(ERROR) << "Illegal Argument"; - err_ = HttpCode::E_ILLEGAL_ARGUMENT; - return; - } - - hdfsHost_ = headers->getQueryParam("host"); - hdfsPort_ = headers->getIntQueryParam("port"); - hdfsPath_ = headers->getQueryParam("path"); - partitions_ = headers->getQueryParam("parts"); - spaceID_ = headers->getIntQueryParam("space"); - - for (auto& path : paths_) { - auto downloadPath = folly::stringPrintf("%s/nebula/%d/download", path.c_str(), spaceID_); - if (fs::FileUtils::fileType(downloadPath.c_str()) == fs::FileType::NOTEXIST) { - fs::FileUtils::makeDir(downloadPath); - } - } -} - -void StorageHttpDownloadHandler::onBody(std::unique_ptr) noexcept { - // Do nothing, we only support GET -} - -void StorageHttpDownloadHandler::onEOM() noexcept { - switch (err_) { - case HttpCode::E_UNSUPPORTED_METHOD: - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::METHOD_NOT_ALLOWED), - WebServiceUtils::toString(HttpStatusCode::METHOD_NOT_ALLOWED)) - .sendWithEOM(); - return; - case HttpCode::E_ILLEGAL_ARGUMENT: - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::BAD_REQUEST), - WebServiceUtils::toString(HttpStatusCode::BAD_REQUEST)) - .sendWithEOM(); - return; - default: - break; - } - - if (helper_->checkHadoopPath()) { - std::vector parts; - folly::split(",", partitions_, parts, true); - if (parts.empty()) { - ResponseBuilder(downstream_) - .status(400, "SSTFile download failed") - .body("Partitions should be not empty") - .sendWithEOM(); - } - - if (downloadSSTFiles(hdfsHost_, hdfsPort_, hdfsPath_, parts)) { - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::OK), - WebServiceUtils::toString(HttpStatusCode::OK)) - .body("SSTFile download successfully") - .sendWithEOM(); - } else { - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::FORBIDDEN), - WebServiceUtils::toString(HttpStatusCode::FORBIDDEN)) - .body("SSTFile download failed") - .sendWithEOM(); - } - } else { - LOG(ERROR) << "HADOOP_HOME not exist"; - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::NOT_FOUND), - WebServiceUtils::toString(HttpStatusCode::NOT_FOUND)) - .sendWithEOM(); - } -} - -void StorageHttpDownloadHandler::onUpgrade(UpgradeProtocol) noexcept { - // Do nothing -} - -void StorageHttpDownloadHandler::requestComplete() noexcept { - delete this; -} - -void StorageHttpDownloadHandler::onError(ProxygenError error) noexcept { - LOG(ERROR) << "Web Service StorageHttpDownloadHandler got error: " - << proxygen::getErrorString(error); -} - -bool StorageHttpDownloadHandler::downloadSSTFiles(const std::string& hdfsHost, - int32_t hdfsPort, - const std::string& hdfsPath, - const std::vector& parts) { - static std::atomic_flag isRunning = ATOMIC_FLAG_INIT; - if (isRunning.test_and_set()) { - LOG(ERROR) << "Download is not completed"; - return false; - } - - std::vector> futures; - - for (auto& part : parts) { - PartitionID partId; - try { - partId = folly::to(part); - } catch (const std::exception& ex) { - isRunning.clear(); - LOG(ERROR) << "Invalid part: \"" << part << "\""; - return false; - } - - auto downloader = [hdfsHost, hdfsPort, hdfsPath, partId, this]() { - auto hdfsPartPath = folly::stringPrintf("%s/%d", hdfsPath.c_str(), partId); - auto partResult = kvstore_->part(spaceID_, partId); - if (!ok(partResult)) { - LOG(ERROR) << "Can't found space: " << spaceID_ << ", part: " << partId; - return false; - } - - auto localPath = - folly::stringPrintf("%s/download/", value(partResult)->engine()->getDataRoot()); - auto result = this->helper_->copyToLocal(hdfsHost, hdfsPort, hdfsPartPath, localPath); - return result.ok() && result.value().empty(); - }; - auto future = pool_->addTask(downloader); - futures.push_back(std::move(future)); - } - - bool successfully{true}; - auto tries = folly::collectAll(futures).get(); - for (const auto& t : tries) { - if (t.hasException()) { - LOG(ERROR) << "Download Failed: " << t.exception(); - successfully = false; - break; - } - if (!t.value()) { - successfully = false; - break; - } - } - LOG(INFO) << "Download tasks have finished"; - isRunning.clear(); - return successfully; -} - -} // namespace storage -} // namespace nebula diff --git a/src/storage/http/StorageHttpDownloadHandler.h b/src/storage/http/StorageHttpDownloadHandler.h deleted file mode 100644 index 1a4d8656d22..00000000000 --- a/src/storage/http/StorageHttpDownloadHandler.h +++ /dev/null @@ -1,69 +0,0 @@ -/* Copyright (c) 2019 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef STORAGE_HTTP_STORAGEHTTPDOWNLOADHANDLER_H_ -#define STORAGE_HTTP_STORAGEHTTPDOWNLOADHANDLER_H_ - -#include - -#include "common/base/Base.h" -#include "common/hdfs/HdfsHelper.h" -#include "common/thread/GenericThreadPool.h" -#include "kvstore/KVStore.h" -#include "webservice/Common.h" - -namespace nebula { -namespace storage { - -using nebula::HttpCode; - -/** - * @brief down load files from hdfs - * - */ -class StorageHttpDownloadHandler : public proxygen::RequestHandler { - public: - StorageHttpDownloadHandler() = default; - - void init(nebula::hdfs::HdfsHelper *helper, - nebula::thread::GenericThreadPool *pool, - nebula::kvstore::KVStore *kvstore, - std::vector paths); - - void onRequest(std::unique_ptr headers) noexcept override; - - void onBody(std::unique_ptr body) noexcept override; - - void onEOM() noexcept override; - - void onUpgrade(proxygen::UpgradeProtocol protocol) noexcept override; - - void requestComplete() noexcept override; - - void onError(proxygen::ProxygenError error) noexcept override; - - private: - bool downloadSSTFiles(const std::string &url, - int port, - const std::string &path, - const std::vector &parts); - - private: - HttpCode err_{HttpCode::SUCCEEDED}; - GraphSpaceID spaceID_; - std::string hdfsHost_; - int32_t hdfsPort_; - std::string hdfsPath_; - std::string partitions_; - nebula::hdfs::HdfsHelper *helper_; - nebula::thread::GenericThreadPool *pool_; - nebula::kvstore::KVStore *kvstore_; - std::vector paths_; -}; - -} // namespace storage -} // namespace nebula - -#endif // STORAGE_HTTP_STORAGEHTTPDOWNLOADHANDLER_H_ diff --git a/src/storage/http/StorageHttpIngestHandler.cpp b/src/storage/http/StorageHttpIngestHandler.cpp deleted file mode 100644 index 8c44cf4bf72..00000000000 --- a/src/storage/http/StorageHttpIngestHandler.cpp +++ /dev/null @@ -1,103 +0,0 @@ -/* Copyright (c) 2019 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "storage/http/StorageHttpIngestHandler.h" - -#include -#include -#include - -namespace nebula { -namespace storage { - -using proxygen::HTTPMessage; -using proxygen::HTTPMethod; -using proxygen::ProxygenError; -using proxygen::ResponseBuilder; -using proxygen::UpgradeProtocol; - -void StorageHttpIngestHandler::init(nebula::kvstore::KVStore *kvstore) { - kvstore_ = kvstore; - CHECK_NOTNULL(kvstore_); -} - -void StorageHttpIngestHandler::onRequest(std::unique_ptr headers) noexcept { - if (headers->getMethod().value() != HTTPMethod::GET) { - // Unsupported method - err_ = HttpCode::E_UNSUPPORTED_METHOD; - return; - } - - if (!headers->hasQueryParam("space")) { - err_ = HttpCode::E_ILLEGAL_ARGUMENT; - return; - } - - space_ = headers->getIntQueryParam("space"); -} - -void StorageHttpIngestHandler::onBody(std::unique_ptr) noexcept { - // Do nothing, we only support GET -} - -void StorageHttpIngestHandler::onEOM() noexcept { - switch (err_) { - case HttpCode::E_UNSUPPORTED_METHOD: - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::METHOD_NOT_ALLOWED), - WebServiceUtils::toString(HttpStatusCode::METHOD_NOT_ALLOWED)) - .sendWithEOM(); - return; - case HttpCode::E_ILLEGAL_ARGUMENT: - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::BAD_REQUEST), - WebServiceUtils::toString(HttpStatusCode::BAD_REQUEST)) - .sendWithEOM(); - return; - default: - break; - } - - if (ingestSSTFiles(space_)) { - LOG(ERROR) << "SSTFile ingest successfully "; - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::OK), - WebServiceUtils::toString(HttpStatusCode::OK)) - .body("SSTFile ingest successfully") - .sendWithEOM(); - } else { - LOG(ERROR) << "SSTFile ingest failed"; - ResponseBuilder(downstream_) - .status(WebServiceUtils::to(HttpStatusCode::FORBIDDEN), - WebServiceUtils::toString(HttpStatusCode::FORBIDDEN)) - .body("SSTFile ingest failed") - .sendWithEOM(); - } -} - -void StorageHttpIngestHandler::onUpgrade(UpgradeProtocol) noexcept { - // Do nothing -} - -void StorageHttpIngestHandler::requestComplete() noexcept { - delete this; -} - -void StorageHttpIngestHandler::onError(ProxygenError error) noexcept { - LOG(ERROR) << "Web Service MetaHttpIngestHandler Failed: " << proxygen::getErrorString(error); -} - -bool StorageHttpIngestHandler::ingestSSTFiles(GraphSpaceID space) { - auto code = kvstore_->ingest(space); - if (code == nebula::cpp2::ErrorCode::SUCCEEDED) { - return true; - } else { - LOG(ERROR) << "SSTFile Ingest Failed: " << apache::thrift::util::enumNameSafe(code); - return false; - } -} - -} // namespace storage -} // namespace nebula diff --git a/src/storage/http/StorageHttpIngestHandler.h b/src/storage/http/StorageHttpIngestHandler.h deleted file mode 100644 index b3beebbb846..00000000000 --- a/src/storage/http/StorageHttpIngestHandler.h +++ /dev/null @@ -1,53 +0,0 @@ -/* Copyright (c) 2019 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef STORAGE_HTTP_STORAGEHTTPINGESTHANDLER_H_ -#define STORAGE_HTTP_STORAGEHTTPINGESTHANDLER_H_ - -#include - -#include "common/base/Base.h" -#include "kvstore/KVStore.h" -#include "webservice/Common.h" - -namespace nebula { -namespace storage { - -using nebula::HttpCode; - -/** - * @brief ingest from from downloaded file. - * - */ -class StorageHttpIngestHandler : public proxygen::RequestHandler { - public: - StorageHttpIngestHandler() = default; - - void init(nebula::kvstore::KVStore *kvstore); - - void onRequest(std::unique_ptr headers) noexcept override; - - void onBody(std::unique_ptr body) noexcept override; - - void onEOM() noexcept override; - - void onUpgrade(proxygen::UpgradeProtocol protocol) noexcept override; - - void requestComplete() noexcept override; - - void onError(proxygen::ProxygenError error) noexcept override; - - bool ingestSSTFiles(GraphSpaceID space); - - private: - HttpCode err_{HttpCode::SUCCEEDED}; - nebula::kvstore::KVStore *kvstore_; - GraphSpaceID space_; -}; - -} // namespace storage -} // namespace nebula - -#endif // STORAGE_HTTP_STORAGEHTTPINGESTHANDLER_H_ diff --git a/src/storage/test/AdminTaskManagerTest.cpp b/src/storage/test/AdminTaskManagerTest.cpp index 7173eaee892..a55590ea8ed 100644 --- a/src/storage/test/AdminTaskManagerTest.cpp +++ b/src/storage/test/AdminTaskManagerTest.cpp @@ -42,12 +42,17 @@ struct HookableTask : public AdminTask { HookableTask() { fGenSubTasks = [&]() { return subTasks; }; } + ErrOrSubTasks genSubTasks() override { LOG(INFO) << "HookableTask::genSubTasks() subTasks.size()=" << subTasks.size(); return fGenSubTasks(); } - void addSubTask(std::function subTask) { + bool check() override { + return true; + } + + void addSubTask(TaskFunction subTask) { subTasks.emplace_back(subTask); } diff --git a/src/storage/test/CMakeLists.txt b/src/storage/test/CMakeLists.txt index 5810bf61560..c6923bf1fcb 100644 --- a/src/storage/test/CMakeLists.txt +++ b/src/storage/test/CMakeLists.txt @@ -1,4 +1,5 @@ set(storage_test_deps + $ $ $ $ @@ -461,40 +462,6 @@ nebula_add_test( gtest ) -nebula_add_test( - NAME - storage_http_download_test - SOURCES - StorageHttpDownloadHandlerTest.cpp - OBJECTS - $ - $ - ${storage_test_deps} - LIBRARIES - ${ROCKSDB_LIBRARIES} - ${THRIFT_LIBRARIES} - ${PROXYGEN_LIBRARIES} - wangle - gtest -) - -nebula_add_test( - NAME - storage_http_ingest_test - SOURCES - StorageHttpIngestHandlerTest.cpp - OBJECTS - $ - $ - ${storage_test_deps} - LIBRARIES - ${ROCKSDB_LIBRARIES} - ${THRIFT_LIBRARIES} - ${PROXYGEN_LIBRARIES} - wangle - gtest -) - nebula_add_test( NAME storage_http_property_test @@ -769,4 +736,3 @@ nebula_add_executable( wangle gtest ) - diff --git a/src/storage/test/StorageHttpDownloadHandlerTest.cpp b/src/storage/test/StorageHttpDownloadHandlerTest.cpp deleted file mode 100644 index ec88273affc..00000000000 --- a/src/storage/test/StorageHttpDownloadHandlerTest.cpp +++ /dev/null @@ -1,116 +0,0 @@ -/* Copyright (c) 2019 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include - -#include "common/base/Base.h" -#include "common/fs/TempDir.h" -#include "common/http/HttpClient.h" -#include "mock/MockCluster.h" -#include "mock/MockData.h" -#include "storage/http/StorageHttpDownloadHandler.h" -#include "storage/test/MockHdfsHelper.h" -#include "storage/test/TestUtils.h" -#include "webservice/Router.h" -#include "webservice/WebService.h" - -DECLARE_string(meta_server_addrs); - -namespace nebula { -namespace storage { - -std::unique_ptr helper = std::make_unique(); - -class StorageHttpDownloadHandlerTestEnv : public ::testing::Environment { - public: - void SetUp() override { - FLAGS_ws_ip = "127.0.0.1"; - FLAGS_ws_http_port = 0; - - rootPath_ = std::make_unique("/tmp/StorageHttpDownloadHandler.XXXXXX"); - cluster_ = std::make_unique(); - cluster_->initStorageKV(rootPath_->path()); - - pool_ = std::make_unique(); - pool_->start(1); - - VLOG(1) << "Starting web service..."; - webSvc_ = std::make_unique(); - auto& router = webSvc_->router(); - router.get("/download").handler([this](nebula::web::PathParams&&) { - auto handler = new storage::StorageHttpDownloadHandler(); - std::vector paths{rootPath_->path()}; - handler->init(helper.get(), pool_.get(), cluster_->storageEnv_->kvstore_, paths); - return handler; - }); - auto status = webSvc_->start(); - ASSERT_TRUE(status.ok()) << status; - } - - void TearDown() override { - cluster_.reset(); - rootPath_.reset(); - webSvc_.reset(); - pool_->stop(); - VLOG(1) << "Web service stopped"; - } - - private: - std::unique_ptr cluster_; - std::unique_ptr webSvc_; - std::unique_ptr rootPath_; - std::unique_ptr pool_; -}; - -TEST(StorageHttpDownloadHandlerTest, StorageDownloadTest) { - { - auto url = "/download"; - auto request = - folly::stringPrintf("http://%s:%d%s", FLAGS_ws_ip.c_str(), FLAGS_ws_http_port, url); - auto resp = http::HttpClient::get(request); - ASSERT_TRUE(resp.ok()); - ASSERT_TRUE(resp.value().empty()); - } - { - auto url = "/download?host=127.0.0.1&port=9000&path=/data&parts=1&space=1"; - auto request = - folly::stringPrintf("http://%s:%d%s", FLAGS_ws_ip.c_str(), FLAGS_ws_http_port, url); - auto resp = http::HttpClient::get(request); - ASSERT_TRUE(resp.ok()); - ASSERT_EQ("SSTFile download successfully", resp.value()); - } - { - auto url = - "/download?host=127.0.0.1&port=9000&path=/" - "data&parts=illegal-part&space=1"; - auto request = - folly::stringPrintf("http://%s:%d%s", FLAGS_ws_ip.c_str(), FLAGS_ws_http_port, url); - auto resp = http::HttpClient::get(request); - ASSERT_TRUE(resp.ok()); - ASSERT_EQ("SSTFile download failed", resp.value()); - } - { - helper = std::make_unique(); - auto url = "/download?host=127.0.0.1&port=9000&path=/data&parts=1&space=1"; - auto request = - folly::stringPrintf("http://%s:%d%s", FLAGS_ws_ip.c_str(), FLAGS_ws_http_port, url); - auto resp = http::HttpClient::get(request); - ASSERT_TRUE(resp.ok()); - ASSERT_EQ("SSTFile download failed", resp.value()); - } -} - -} // namespace storage -} // namespace nebula - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - folly::init(&argc, &argv, true); - google::SetStderrLogging(google::INFO); - - ::testing::AddGlobalTestEnvironment(new nebula::storage::StorageHttpDownloadHandlerTestEnv()); - - return RUN_ALL_TESTS(); -} diff --git a/src/storage/test/StorageHttpIngestHandlerTest.cpp b/src/storage/test/StorageHttpIngestHandlerTest.cpp deleted file mode 100644 index 0c18c19741e..00000000000 --- a/src/storage/test/StorageHttpIngestHandlerTest.cpp +++ /dev/null @@ -1,104 +0,0 @@ -/* Copyright (c) 2019 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include -#include - -#include "common/base/Base.h" -#include "common/fs/TempDir.h" -#include "common/http/HttpClient.h" -#include "mock/MockCluster.h" -#include "mock/MockData.h" -#include "storage/http/StorageHttpIngestHandler.h" -#include "webservice/Router.h" -#include "webservice/WebService.h" - -namespace nebula { -namespace storage { - -class StorageHttpIngestHandlerTestEnv : public ::testing::Environment { - public: - void SetUp() override { - FLAGS_ws_ip = "127.0.0.1"; - FLAGS_ws_http_port = 0; - VLOG(1) << "Starting web service..."; - - rootPath_ = std::make_unique("/tmp/StorageHttpIngestHandler.XXXXXX"); - cluster_ = std::make_unique(); - cluster_->initStorageKV(rootPath_->path()); - - auto partPath = folly::stringPrintf("%s/disk1/nebula/0/download/0", rootPath_->path()); - ASSERT_TRUE(nebula::fs::FileUtils::makeDir(partPath)); - - auto options = rocksdb::Options(); - auto env = rocksdb::EnvOptions(); - rocksdb::SstFileWriter writer{env, options}; - - auto sstPath = folly::stringPrintf("%s/data.sst", partPath.c_str()); - auto status = writer.Open(sstPath); - ASSERT_EQ(rocksdb::Status::OK(), status); - - for (auto i = 0; i < 10; i++) { - status = writer.Put(folly::stringPrintf("key_%d", i), folly::stringPrintf("val_%d", i)); - ASSERT_EQ(rocksdb::Status::OK(), status); - } - status = writer.Finish(); - ASSERT_EQ(rocksdb::Status::OK(), status); - - webSvc_ = std::make_unique(); - auto& router = webSvc_->router(); - router.get("/ingest").handler([this](nebula::web::PathParams&&) { - auto handler = new storage::StorageHttpIngestHandler(); - handler->init(cluster_->storageEnv_->kvstore_); - return handler; - }); - auto webStatus = webSvc_->start(); - ASSERT_TRUE(webStatus.ok()) << webStatus; - } - - void TearDown() override { - cluster_.reset(); - rootPath_.reset(); - webSvc_.reset(); - VLOG(1) << "Web service stopped"; - } - - private: - std::unique_ptr rootPath_; - std::unique_ptr cluster_; - std::unique_ptr webSvc_; -}; - -TEST(StorageHttpIngestHandlerTest, StorageIngestTest) { - { - auto url = "/ingest?space=1"; - auto request = - folly::stringPrintf("http://%s:%d%s", FLAGS_ws_ip.c_str(), FLAGS_ws_http_port, url); - auto resp = http::HttpClient::get(request); - ASSERT_TRUE(resp.ok()); - ASSERT_EQ("SSTFile ingest successfully", resp.value()); - } - { - auto url = "/ingest?space=0"; - auto request = - folly::stringPrintf("http://%s:%d%s", FLAGS_ws_ip.c_str(), FLAGS_ws_http_port, url); - auto resp = http::HttpClient::get(request); - ASSERT_TRUE(resp.ok()); - ASSERT_EQ("SSTFile ingest failed", resp.value()); - } -} - -} // namespace storage -} // namespace nebula - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - folly::init(&argc, &argv, true); - google::SetStderrLogging(google::INFO); - - ::testing::AddGlobalTestEnvironment(new nebula::storage::StorageHttpIngestHandlerTestEnv()); - - return RUN_ALL_TESTS(); -} diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index efc6bdb4d52..27d5728574a 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -1,5 +1,64 @@ # note: standalone version don't have dependent test tools for now. +set(tools_test_deps + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ +) + if(NOT ENABLE_STANDALONE_VERSION) nebula_add_subdirectory(storage-perf) nebula_add_subdirectory(simple-kv-verify) diff --git a/src/tools/db-dump/CMakeLists.txt b/src/tools/db-dump/CMakeLists.txt index 66084aa6ef3..62f9432c1fa 100644 --- a/src/tools/db-dump/CMakeLists.txt +++ b/src/tools/db-dump/CMakeLists.txt @@ -1,61 +1,3 @@ -set(tools_test_deps - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ -) - nebula_add_executable( NAME db_dump diff --git a/src/tools/db-upgrade/CMakeLists.txt b/src/tools/db-upgrade/CMakeLists.txt index 29ea63b0755..af0b1efbd26 100644 --- a/src/tools/db-upgrade/CMakeLists.txt +++ b/src/tools/db-upgrade/CMakeLists.txt @@ -8,60 +8,7 @@ nebula_add_executable( NebulaKeyUtilsV3.cpp DbUpgrader.cpp OBJECTS - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ + ${tools_test_deps} LIBRARIES ${ROCKSDB_LIBRARIES} ${THRIFT_LIBRARIES} diff --git a/src/tools/meta-dump/CMakeLists.txt b/src/tools/meta-dump/CMakeLists.txt index fbbe2c4b30f..47fbcc1be56 100644 --- a/src/tools/meta-dump/CMakeLists.txt +++ b/src/tools/meta-dump/CMakeLists.txt @@ -4,61 +4,7 @@ nebula_add_executable( SOURCES MetaDumpTool.cpp OBJECTS - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ + ${tools_test_deps} LIBRARIES ${ROCKSDB_LIBRARIES} ${THRIFT_LIBRARIES} diff --git a/src/tools/simple-kv-verify/CMakeLists.txt b/src/tools/simple-kv-verify/CMakeLists.txt index 1d752a3f671..5b5d3bdf220 100644 --- a/src/tools/simple-kv-verify/CMakeLists.txt +++ b/src/tools/simple-kv-verify/CMakeLists.txt @@ -4,59 +4,7 @@ nebula_add_executable( SOURCES SimpleKVVerifyTool.cpp OBJECTS - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ + ${tools_test_deps} LIBRARIES ${ROCKSDB_LIBRARIES} ${THRIFT_LIBRARIES} diff --git a/src/tools/storage-perf/CMakeLists.txt b/src/tools/storage-perf/CMakeLists.txt index 2e85f342958..cccddaaadb2 100644 --- a/src/tools/storage-perf/CMakeLists.txt +++ b/src/tools/storage-perf/CMakeLists.txt @@ -1,67 +1,10 @@ -set(perf_test_deps - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ -) - nebula_add_executable( NAME storage_perf SOURCES StoragePerfTool.cpp OBJECTS - ${perf_test_deps} + ${tools_test_deps} LIBRARIES ${ROCKSDB_LIBRARIES} ${THRIFT_LIBRARIES} @@ -84,7 +27,7 @@ nebula_add_executable( SOURCES StorageIntegrityTool.cpp OBJECTS - ${perf_test_deps} + ${tools_test_deps} LIBRARIES ${ROCKSDB_LIBRARIES} ${THRIFT_LIBRARIES} diff --git a/src/webservice/Common.cpp b/src/webservice/Common.cpp index f73f8907bff..9fdf8a8cffe 100644 --- a/src/webservice/Common.cpp +++ b/src/webservice/Common.cpp @@ -7,9 +7,6 @@ #include -DEFINE_int32(ws_meta_http_port, 11000, "Port to listen on Meta with HTTP protocol"); -DEFINE_int32(ws_storage_http_port, 12000, "Port to listen on Storage with HTTP protocol"); - namespace nebula { std::unordered_map WebServiceUtils::kStatusStringMap_ = { diff --git a/src/webservice/WebService.cpp b/src/webservice/WebService.cpp index 93331967ea3..b7532a5ad66 100644 --- a/src/webservice/WebService.cpp +++ b/src/webservice/WebService.cpp @@ -22,6 +22,7 @@ DEFINE_string(ws_ip, "0.0.0.0", "IP/Hostname to bind to"); DEFINE_int32(ws_threads, 4, "Number of threads for the web service."); #ifdef BUILD_STANDALONE +DEFINE_int32(ws_storage_http_port, 12000, "Port to listen on Storage with HTTP protocol"); DEFINE_int32(ws_storage_threads, 4, "Number of threads for the web service."); #endif From 5edd02ce6bb81114a1a1ec926c318fbd766c2d6b Mon Sep 17 00:00:00 2001 From: "lionel.liu@vesoft.com" <52276794+liuyu85cn@users.noreply.github.com> Date: Tue, 29 Mar 2022 17:35:35 +0800 Subject: [PATCH 04/24] chagne some log to dlog (#4094) Co-authored-by: panda-sheep <59197347+panda-sheep@users.noreply.github.com> Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/storage/transaction/TransactionManager.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/storage/transaction/TransactionManager.cpp b/src/storage/transaction/TransactionManager.cpp index b35961a73f4..bcc2204c163 100644 --- a/src/storage/transaction/TransactionManager.cpp +++ b/src/storage/transaction/TransactionManager.cpp @@ -180,7 +180,7 @@ void TransactionManager::addPrime(GraphSpaceID spaceId, } void TransactionManager::onNewPartAdded(std::shared_ptr& part) { - LOG(INFO) << folly::sformat("space={}, part={} added", part->spaceId(), part->partitionId()); + DLOG(INFO) << folly::sformat("space={}, part={} added", part->spaceId(), part->partitionId()); auto fnLeaderReady = std::bind(&TransactionManager::onLeaderElectedWrapper, this, std::placeholders::_1); auto fnLeaderLost = @@ -190,10 +190,10 @@ void TransactionManager::onNewPartAdded(std::shared_ptr& part) { } void TransactionManager::onLeaderLostWrapper(const ::nebula::kvstore::Part::CallbackOptions& opt) { - LOG(INFO) << folly::sformat("leader lost, del space={}, part={}, term={} from white list", - opt.spaceId, - opt.partId, - opt.term); + DLOG(INFO) << folly::sformat("leader lost, del space={}, part={}, term={} from white list", + opt.spaceId, + opt.partId, + opt.term); // clean some out-dated item in memory lock for (auto cit = memLocks_.cbegin(); cit != memLocks_.cend();) { auto& [spaceId, partId, termId] = cit->first; @@ -210,13 +210,13 @@ void TransactionManager::onLeaderLostWrapper(const ::nebula::kvstore::Part::Call void TransactionManager::onLeaderElectedWrapper( const ::nebula::kvstore::Part::CallbackOptions& opt) { - LOG(INFO) << folly::sformat( + DLOG(INFO) << folly::sformat( "leader get do scanPrimes space={}, part={}, term={}", opt.spaceId, opt.partId, opt.term); scanPrimes(opt.spaceId, opt.partId, opt.term); } void TransactionManager::scanPrimes(GraphSpaceID spaceId, PartitionID partId, TermID termId) { - LOG(INFO) << folly::sformat( + DLOG(INFO) << folly::sformat( "{}(), space={}, part={}, term={}", __func__, spaceId, partId, termId); std::vector prefixVec{ConsistUtil::primePrefix(partId), ConsistUtil::doublePrimePrefix(partId)}; @@ -264,8 +264,8 @@ void TransactionManager::scanPrimes(GraphSpaceID spaceId, PartitionID partId, Te currTerm_.insert_or_assign(termKey, termId); prevTerms_.insert_or_assign(termKey, prevTerm); - LOG(INFO) << "set curr term spaceId = " << spaceId << ", partId = " << partId - << ", termId = " << termId; + DLOG(INFO) << "set curr term spaceId = " << spaceId << ", partId = " << partId + << ", termId = " << termId; } folly::EventBase* TransactionManager::getEventBase() { From 808f75a197b77f7b3248480bcf89706af4a6b302 Mon Sep 17 00:00:00 2001 From: panda-sheep <59197347+panda-sheep@users.noreply.github.com> Date: Wed, 30 Mar 2022 14:14:10 +0800 Subject: [PATCH 05/24] job manager test (#4085) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/graph/executor/admin/SubmitJobExecutor.cpp | 3 ++- src/meta/processors/job/BalanceJobExecutor.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/graph/executor/admin/SubmitJobExecutor.cpp b/src/graph/executor/admin/SubmitJobExecutor.cpp index 046c1d7096a..de4e353ad9d 100644 --- a/src/graph/executor/admin/SubmitJobExecutor.cpp +++ b/src/graph/executor/admin/SubmitJobExecutor.cpp @@ -156,7 +156,8 @@ nebula::DataSet SubmitJobExecutor::buildShowResultData( folly::sformat("Succeeded:{}", succeeded), folly::sformat("Failed:{}", failed), folly::sformat("In Progress:{}", inProgress), - folly::sformat("Invalid:{}", invalid)})); + folly::sformat("Invalid:{}", invalid), + ("")})); return v; } else { nebula::DataSet v( diff --git a/src/meta/processors/job/BalanceJobExecutor.cpp b/src/meta/processors/job/BalanceJobExecutor.cpp index 8244c816bea..331541a01bb 100644 --- a/src/meta/processors/job/BalanceJobExecutor.cpp +++ b/src/meta/processors/job/BalanceJobExecutor.cpp @@ -20,7 +20,7 @@ BalanceJobExecutor::BalanceJobExecutor(GraphSpaceID space, : MetaJobExecutor(space, jobId, kvstore, adminClient, paras) {} bool BalanceJobExecutor::check() { - return !paras_.empty(); + return true; } nebula::cpp2::ErrorCode BalanceJobExecutor::prepare() { From 90a2c0241650835a9df4f521d2eb99acfb9b7f34 Mon Sep 17 00:00:00 2001 From: shylock <33566796+Shylock-Hg@users.noreply.github.com> Date: Wed, 30 Mar 2022 17:12:42 +0800 Subject: [PATCH 06/24] Fix build path. (#4091) Co-authored-by: jie.wang <38901892+jievince@users.noreply.github.com> Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/common/expression/PathBuildExpression.cpp | 25 ++++++++++++++++--- .../features/bugfix/BuildPathMistake.feature | 18 +++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 tests/tck/features/bugfix/BuildPathMistake.feature diff --git a/src/common/expression/PathBuildExpression.cpp b/src/common/expression/PathBuildExpression.cpp index 6bf1f2b733d..a82095d211a 100644 --- a/src/common/expression/PathBuildExpression.cpp +++ b/src/common/expression/PathBuildExpression.cpp @@ -39,10 +39,14 @@ bool PathBuildExpression::buildPath(const Value& value, Path& path) const { switch (value.type()) { case Value::Type::EDGE: { Step step; + bool result = true; if (!path.steps.empty()) { - getEdge(value, path.steps.back().dst.vid, step); + result = getEdge(value, path.steps.back().dst.vid, step); } else { - getEdge(value, path.src.vid, step); + result = getEdge(value, path.src.vid, step); + } + if (!result) { + return result; } path.steps.emplace_back(std::move(step)); break; @@ -68,9 +72,20 @@ bool PathBuildExpression::buildPath(const Value& value, Path& path) const { break; } case Value::Type::LIST: { + bool result = true; for (const auto& val : value.getList().values) { if (!buildPath(val, path)) { - return false; + result = false; + break; + } + } + if (!result) { + for (std::vector::const_reverse_iterator it = value.getList().values.rbegin(); + it != value.getList().values.rend(); + ++it) { + if (!buildPath(*it, path)) { + return false; + } } } break; @@ -110,12 +125,14 @@ bool PathBuildExpression::getEdge(const Value& value, const Value& lastStepVid, step.ranking = edge.ranking; step.props = edge.props; step.dst.vid = edge.dst; - } else { + } else if (lastStepVid == edge.dst) { step.type = -edge.type; step.name = edge.name; step.ranking = edge.ranking; step.props = edge.props; step.dst.vid = edge.src; + } else { + return false; } return true; } diff --git a/tests/tck/features/bugfix/BuildPathMistake.feature b/tests/tck/features/bugfix/BuildPathMistake.feature new file mode 100644 index 00000000000..26861695b8e --- /dev/null +++ b/tests/tck/features/bugfix/BuildPathMistake.feature @@ -0,0 +1,18 @@ +# Copyright (c) 2022 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +Feature: Test crash when null path expression + + Background: + Given a graph with space named "nba" + + Scenario: Null path expression in multiple patterns + When executing query: + """ + MATCH p = ()-[:like*2]->(v:player) WHERE id(v) == 'Grant Hill' RETURN p + """ + Then the result should be, in any order, with relax comparison: + | p | + | <("Yao Ming" :player{age: 38, name: "Yao Ming"})-[:like@0 {likeness: 90}]->("Tracy McGrady" :player{age: 39, name: "Tracy McGrady"})-[:like@0 {likeness: 90}]->("Grant Hill" :player{age: 46, name: "Grant Hill"})> | + | <("Grant Hill" :player{age: 46, name: "Grant Hill"})<-[:like@0 {likeness: 90}]-("Tracy McGrady" :player{age: 39, name: "Tracy McGrady"})<-[:like@0 {likeness: 90}]-("Grant Hill" :player{age: 46, name: "Grant Hill"})> | + | <("Vince Carter" :player{age: 42, name: "Vince Carter"})-[:like@0 {likeness: 90}]->("Tracy McGrady" :player{age: 39, name: "Tracy McGrady"})-[:like@0 {likeness: 90}]->("Grant Hill" :player{age: 46, name: "Grant Hill"})> | From d2da606731f6bf6bddfca1abbf653ff212a4ef35 Mon Sep 17 00:00:00 2001 From: Doodle <13706157+critical27@users.noreply.github.com> Date: Wed, 30 Mar 2022 18:06:27 +0800 Subject: [PATCH 07/24] Refactor StorageServer stop (#4034) * use ThrfitServer::steup/cleanUp when setup-up/tear-down, rebased * fix standalone * disable thread test may timeout --- .../thread/test/GenericThreadPoolTest.cpp | 4 +- src/common/thread/test/GenericWorkerTest.cpp | 6 +- src/graph/executor/test/StorageServerStub.cpp | 19 -- src/kvstore/NebulaStore.cpp | 6 +- src/kvstore/NebulaStore.h | 2 +- src/kvstore/raftex/RaftPart.cpp | 2 + src/kvstore/raftex/RaftexService.cpp | 123 ++-------- src/kvstore/raftex/RaftexService.h | 49 +--- src/kvstore/raftex/test/RaftexTestBase.cpp | 3 +- src/mock/LocalServer.h | 12 +- src/mock/MockCluster.h | 18 +- src/storage/GraphStorageLocalServer.cpp | 19 -- src/storage/GraphStorageLocalServer.h | 5 - src/storage/StorageServer.cpp | 229 ++++++++---------- src/storage/StorageServer.h | 33 ++- 15 files changed, 175 insertions(+), 355 deletions(-) diff --git a/src/common/thread/test/GenericThreadPoolTest.cpp b/src/common/thread/test/GenericThreadPoolTest.cpp index d68db379966..6007336c187 100644 --- a/src/common/thread/test/GenericThreadPoolTest.cpp +++ b/src/common/thread/test/GenericThreadPoolTest.cpp @@ -93,7 +93,7 @@ static testing::AssertionResult msAboutEqual(size_t expected, size_t actual) { return testing::AssertionFailure() << "actual: " << actual << ", expected: " << expected; } -TEST(GenericThreadPool, addDelayTask) { +TEST(GenericThreadPool, DISABLED_addDelayTask) { GenericThreadPool pool; ASSERT_TRUE(pool.start(1)); { @@ -108,7 +108,7 @@ TEST(GenericThreadPool, addDelayTask) { } } -TEST(GenericThreadPool, addRepeatTask) { +TEST(GenericThreadPool, DISABLED_addRepeatTask) { GenericThreadPool pool; ASSERT_TRUE(pool.start(1)); { diff --git a/src/common/thread/test/GenericWorkerTest.cpp b/src/common/thread/test/GenericWorkerTest.cpp index 0080f39fdb2..8338600738c 100644 --- a/src/common/thread/test/GenericWorkerTest.cpp +++ b/src/common/thread/test/GenericWorkerTest.cpp @@ -93,7 +93,7 @@ static testing::AssertionResult msAboutEqual(size_t expected, size_t actual) { return testing::AssertionFailure() << "actual: " << actual << ", expected: " << expected; } -TEST(GenericWorker, addDelayTask) { +TEST(GenericWorker, DISABLED_addDelayTask) { GenericWorker worker; ASSERT_TRUE(worker.start()); { @@ -108,7 +108,7 @@ TEST(GenericWorker, addDelayTask) { } } -TEST(GenericWorker, addRepeatTask) { +TEST(GenericWorker, DISABLED_addRepeatTask) { GenericWorker worker; ASSERT_TRUE(worker.start()); { @@ -120,7 +120,7 @@ TEST(GenericWorker, addRepeatTask) { } } -TEST(GenericWorker, DISABLE_purgeRepeatTask) { +TEST(GenericWorker, DISABLED_purgeRepeatTask) { GenericWorker worker; ASSERT_TRUE(worker.start()); { diff --git a/src/graph/executor/test/StorageServerStub.cpp b/src/graph/executor/test/StorageServerStub.cpp index 9ace772300d..0ad57cb12af 100644 --- a/src/graph/executor/test/StorageServerStub.cpp +++ b/src/graph/executor/test/StorageServerStub.cpp @@ -31,25 +31,6 @@ void GraphStorageLocalServer::setInterface( handler_ = handler; } -void GraphStorageLocalServer::serve() { - if (serving_) { - LOG(WARNING) << "Server already serving"; - return; - } - // do nothing, wait stop - serving_ = true; - sem_.wait(); -} - -void GraphStorageLocalServer::stop() { - if (!serving_) { - LOG(WARNING) << "Can't stop server not serving"; - return; - } - sem_.signal(); - serving_ = false; -} - folly::Future GraphStorageLocalServer::future_getNeighbors( const cpp2::GetNeighborsRequest& request) { LOCAL_RETURN_FUTURE(threadManager_, cpp2::GetNeighborsResponse, future_getNeighbors); diff --git a/src/kvstore/NebulaStore.cpp b/src/kvstore/NebulaStore.cpp index 2c551b497c8..72800398209 100644 --- a/src/kvstore/NebulaStore.cpp +++ b/src/kvstore/NebulaStore.cpp @@ -36,9 +36,6 @@ NebulaStore::~NebulaStore() { stop(); LOG(INFO) << "Cut off the relationship with meta client"; options_.partMan_.reset(); - raftService_->stop(); - LOG(INFO) << "Waiting for the raft service stop..."; - raftService_->waitUntilStop(); bgWorkers_->stop(); bgWorkers_->wait(); storeWorker_->stop(); @@ -56,7 +53,7 @@ bool NebulaStore::init() { CHECK(storeWorker_->start()); snapshot_.reset(new NebulaSnapshotManager(this)); raftService_ = raftex::RaftexService::createService(ioPool_, workers_, raftAddr_.port); - if (!raftService_->start()) { + if (raftService_ == nullptr) { LOG(ERROR) << "Start the raft service failed"; return false; } @@ -273,6 +270,7 @@ void NebulaStore::stop() { LOG(INFO) << "Stop the raft service..."; raftService_->stop(); + LOG(INFO) << "Stop kv engine..."; for (const auto& space : spaces_) { for (const auto& engine : space.second->engines_) { engine->stop(); diff --git a/src/kvstore/NebulaStore.h b/src/kvstore/NebulaStore.h index de9345f0810..1fadcb3e827 100644 --- a/src/kvstore/NebulaStore.h +++ b/src/kvstore/NebulaStore.h @@ -111,7 +111,7 @@ class NebulaStore : public KVStore, public Handler { bool init(); /** - * @brief Stop the raft service and kv engine + * @brief Stop the engine */ void stop() override; diff --git a/src/kvstore/raftex/RaftPart.cpp b/src/kvstore/raftex/RaftPart.cpp index 02d2b707e1e..45a94408fc9 100644 --- a/src/kvstore/raftex/RaftPart.cpp +++ b/src/kvstore/raftex/RaftPart.cpp @@ -341,6 +341,8 @@ void RaftPart::stop() { VLOG(1) << idStr_ << "Invoked stop() on all peer hosts"; + // Host::waitForStop will wait a callback executed in ioThreadPool, so make sure the + // RaftPart::stop SHOULD NOT be executed in the same ioThreadPool for (auto& h : hosts) { VLOG(1) << idStr_ << "Waiting " << h->idStr() << " to stop"; h->waitForStop(); diff --git a/src/kvstore/raftex/RaftexService.cpp b/src/kvstore/raftex/RaftexService.cpp index 314f7bd8b84..a39ac5a78d7 100644 --- a/src/kvstore/raftex/RaftexService.cpp +++ b/src/kvstore/raftex/RaftexService.cpp @@ -21,93 +21,37 @@ namespace raftex { * ******************************************************/ std::shared_ptr RaftexService::createService( - std::shared_ptr pool, + std::shared_ptr ioPool, std::shared_ptr workers, uint16_t port) { - auto svc = std::shared_ptr(new RaftexService()); - CHECK(svc != nullptr) << "Failed to create a raft service"; - - svc->server_ = std::make_unique(); - CHECK(svc->server_ != nullptr) << "Failed to create a thrift server"; - svc->server_->setInterface(svc); - - svc->initThriftServer(pool, workers, port); - return svc; -} - -RaftexService::~RaftexService() {} - -bool RaftexService::start() { - serverThread_.reset(new std::thread([&] { serve(); })); - - waitUntilReady(); - - // start failed, reclaim resource - if (status_.load() != STATUS_RUNNING) { - waitUntilStop(); - return false; - } - - return true; -} - -void RaftexService::waitUntilReady() { - while (status_.load() == STATUS_NOT_RUNNING) { - std::this_thread::sleep_for(std::chrono::microseconds(100)); - } -} - -void RaftexService::initThriftServer(std::shared_ptr pool, - std::shared_ptr workers, - uint16_t port) { - LOG(INFO) << "Init thrift server for raft service, port: " << port; - if (FLAGS_enable_ssl) { - server_->setSSLConfig(nebula::sslContextConfig()); - } - server_->setPort(port); - server_->setIdleTimeout(std::chrono::seconds(0)); - if (pool != nullptr) { - server_->setIOThreadPool(pool); - } - if (workers != nullptr) { - server_->setThreadManager( - std::dynamic_pointer_cast(workers)); - } - server_->setStopWorkersOnStopListening(false); -} - -bool RaftexService::setup() { try { - server_->setup(); - serverPort_ = server_->getAddress().getPort(); - - LOG(INFO) << "Starting the Raftex Service on " << serverPort_; + auto svc = std::shared_ptr(new RaftexService()); + auto server = std::make_unique(); + server->setPort(port); + server->setIdleTimeout(std::chrono::seconds(0)); + if (ioPool != nullptr) { + server->setIOThreadPool(ioPool); + } + if (workers != nullptr) { + server->setThreadManager( + std::dynamic_pointer_cast(workers)); + } + if (FLAGS_enable_ssl) { + server->setSSLConfig(nebula::sslContextConfig()); + } + server->setInterface(svc); + server->setup(); + svc->server_ = std::move(server); + svc->serverPort_ = svc->server_->getAddress().getPort(); + LOG(INFO) << "Start raft service on " << svc->serverPort_; + return svc; } catch (const std::exception& e) { - LOG(ERROR) << "Setup the Raftex Service failed, error: " << e.what(); - return false; + LOG(ERROR) << "Start raft service failed: " << e.what(); + return nullptr; + } catch (...) { + LOG(ERROR) << "Start raft service failed"; + return nullptr; } - - return true; -} - -void RaftexService::serve() { - LOG(INFO) << "Starting the Raftex Service"; - - if (!setup()) { - status_.store(STATUS_SETUP_FAILED); - return; - } - - SCOPE_EXIT { - server_->cleanUp(); - }; - - status_.store(STATUS_RUNNING); - LOG(INFO) << "Start the Raftex Service successfully"; - server_->getEventBaseManager()->getEventBase()->loopForever(); - - status_.store(STATUS_NOT_RUNNING); - LOG(INFO) << "The Raftex Service stopped"; } std::shared_ptr RaftexService::getIOThreadPool() const { @@ -119,11 +63,6 @@ std::shared_ptr RaftexService::getThreadManager() { } void RaftexService::stop() { - int expected = STATUS_RUNNING; - if (!status_.compare_exchange_strong(expected, STATUS_NOT_RUNNING)) { - return; - } - // stop service LOG(INFO) << "Stopping the raftex service on port " << serverPort_; std::unordered_map, std::shared_ptr> parts; @@ -139,16 +78,6 @@ void RaftexService::stop() { server_->stop(); } -void RaftexService::waitUntilStop() { - if (serverThread_) { - serverThread_->join(); - serverThread_.reset(); - server_.reset(); - LOG(INFO) << "Server thread has stopped. Service on port " << serverPort_ - << " is ready to be destroyed"; - } -} - void RaftexService::addPartition(std::shared_ptr part) { // todo(doodle): If we need to start both listener and normal replica on same // hosts, this class need to be aware of type. diff --git a/src/kvstore/raftex/RaftexService.h b/src/kvstore/raftex/RaftexService.h index 74d9245b7bf..e6fb496ac24 100644 --- a/src/kvstore/raftex/RaftexService.h +++ b/src/kvstore/raftex/RaftexService.h @@ -27,20 +27,20 @@ class RaftexService : public cpp2::RaftexServiceSvIf { /** * @brief Create a raft service * - * @param pool IOThreadPool to use + * @param ioPool IOThreadPool to use * @param workers Worker thread pool to use * @param port Listen port of thrift server * @return std::shared_ptr */ static std::shared_ptr createService( - std::shared_ptr pool, + std::shared_ptr ioPool, std::shared_ptr workers, uint16_t port = 0); /** * @brief Destroy the Raftex Service */ - virtual ~RaftexService(); + virtual ~RaftexService() = default; /** * @brief Return the raft thrift server port @@ -64,22 +64,10 @@ class RaftexService : public cpp2::RaftexServiceSvIf { std::shared_ptr getThreadManager(); /** - * @brief Start the raft thrift server - * - * @return Whether start succeed - */ - bool start(); - - /** - * @brief Set the state to stopped + * @brief Stop all partitions and wait thrift server stops */ void stop(); - /** - * @brief Wait until the thrift server has been stopped - */ - void waitUntilStop(); - /** * @brief Handle leader election request in worker thread * @@ -143,40 +131,11 @@ class RaftexService : public cpp2::RaftexServiceSvIf { std::shared_ptr findPart(GraphSpaceID spaceId, PartitionID partId); private: - /** - * @brief Start the thrift server - * - * @param pool IO thread pool - * @param workers Worker thread pool - * @param port Thrift port to listener - */ - void initThriftServer(std::shared_ptr pool, - std::shared_ptr workers, - uint16_t port = 0); - - /** - * @brief Prepare the setup of thrift server - * - * @return Return whether succeed - */ - bool setup(); - void serve(); - - /** - * @brief Wait until the service is ready to serve - */ - void waitUntilReady(); - RaftexService() = default; - private: std::unique_ptr server_; - std::unique_ptr serverThread_; uint32_t serverPort_; - enum RaftServiceStatus { STATUS_NOT_RUNNING = 0, STATUS_SETUP_FAILED = 1, STATUS_RUNNING = 2 }; - std::atomic_int status_{STATUS_NOT_RUNNING}; - folly::RWSpinLock partsLock_; std::unordered_map, std::shared_ptr> parts_; }; diff --git a/src/kvstore/raftex/test/RaftexTestBase.cpp b/src/kvstore/raftex/test/RaftexTestBase.cpp index 951dea666b9..ca59ccaeefb 100644 --- a/src/kvstore/raftex/test/RaftexTestBase.cpp +++ b/src/kvstore/raftex/test/RaftexTestBase.cpp @@ -161,7 +161,6 @@ void setupRaft(int32_t numCopies, // Set up services for (int i = 0; i < numCopies; ++i) { services.emplace_back(RaftexService::createService(nullptr, nullptr)); - if (!services.back()->start()) return; uint16_t port = services.back()->getServerPort(); allHosts.emplace_back(ipStr, port); } @@ -217,7 +216,7 @@ void finishRaft(std::vector>& services, workers->wait(); LOG(INFO) << "Waiting for all service stopped"; for (auto& svc : services) { - svc->waitUntilStop(); + svc->stop(); } } diff --git a/src/mock/LocalServer.h b/src/mock/LocalServer.h index dcff604f557..e065e3ab1ab 100644 --- a/src/mock/LocalServer.h +++ b/src/mock/LocalServer.h @@ -17,18 +17,13 @@ namespace mock { struct LocalServer { ~LocalServer() { - if (server_ != nullptr) { - server_->stop(); - } - if (thread_ != nullptr) { - thread_->join(); - } VLOG(3) << "~LocalServer"; } void start(const std::string& name, uint16_t port, std::shared_ptr handler) { + UNUSED(name); UNUSED(port); std::shared_ptr workers = apache::thrift::concurrency::PriorityThreadManager::newPriorityThreadManager(1); @@ -38,15 +33,10 @@ struct LocalServer { server_ = storage::GraphStorageLocalServer::getInstance(); server_->setThreadManager(workers); server_->setInterface(std::move(handler)); - thread_ = std::make_unique(name, [this, name] { - server_->serve(); - LOG(INFO) << "The " << name << " server has been stopped"; - }); usleep(10000); } std::shared_ptr server_{nullptr}; - std::unique_ptr thread_{nullptr}; uint16_t port_{0}; }; diff --git a/src/mock/MockCluster.h b/src/mock/MockCluster.h index 45493ce4aec..28b380e793d 100644 --- a/src/mock/MockCluster.h +++ b/src/mock/MockCluster.h @@ -34,8 +34,6 @@ class MockCluster { ~MockCluster() { stop(); - storageAdminServer_.reset(); - graphStorageServer_.reset(); } void startAll(); @@ -94,6 +92,16 @@ class MockCluster { } void stop() { + if (storageKV_) { + storageKV_->stop(); + } + if (metaKV_) { + metaKV_->stop(); + } + + storageAdminServer_.reset(); + graphStorageServer_.reset(); + if (metaClient_) { metaClient_->notifyStop(); metaClient_->stop(); @@ -102,12 +110,6 @@ class MockCluster { metaClient_->notifyStop(); lMetaClient_->stop(); } - if (metaKV_) { - metaKV_->stop(); - } - if (storageKV_) { - storageKV_->stop(); - } if (esListener_) { esListener_->stop(); } diff --git a/src/storage/GraphStorageLocalServer.cpp b/src/storage/GraphStorageLocalServer.cpp index 697af47f436..5e318285fc5 100644 --- a/src/storage/GraphStorageLocalServer.cpp +++ b/src/storage/GraphStorageLocalServer.cpp @@ -39,25 +39,6 @@ void GraphStorageLocalServer::setInterface( handler_ = handler; } -void GraphStorageLocalServer::serve() { - if (serving_) { - LOG(WARNING) << "Server already serving"; - return; - } - // do nothing, wait stop - serving_ = true; - sem_.wait(); -} - -void GraphStorageLocalServer::stop() { - if (!serving_) { - LOG(WARNING) << "Can't stop server not serving"; - return; - } - sem_.signal(); - serving_ = false; -} - folly::Future GraphStorageLocalServer::future_getNeighbors( const cpp2::GetNeighborsRequest& request) { LOCAL_RETURN_FUTURE(cpp2::GetNeighborsResponse, future_getNeighbors); diff --git a/src/storage/GraphStorageLocalServer.h b/src/storage/GraphStorageLocalServer.h index 39d93929058..3ab374b84bd 100644 --- a/src/storage/GraphStorageLocalServer.h +++ b/src/storage/GraphStorageLocalServer.h @@ -25,8 +25,6 @@ class GraphStorageLocalServer final : public boost::noncopyable, public nebula:: } void setThreadManager(std::shared_ptr threadManager); void setInterface(std::shared_ptr handler); - void stop(); - void serve(); public: folly::Future future_getNeighbors( @@ -61,9 +59,6 @@ class GraphStorageLocalServer final : public boost::noncopyable, public nebula:: private: std::shared_ptr threadManager_; std::shared_ptr handler_; - folly::fibers::Semaphore sem_{0}; - static std::mutex mutex_; - bool serving_ = {false}; }; } // namespace nebula::storage #endif diff --git a/src/storage/StorageServer.cpp b/src/storage/StorageServer.cpp index f2ed6487d49..03067fb9a14 100644 --- a/src/storage/StorageServer.cpp +++ b/src/storage/StorageServer.cpp @@ -61,10 +61,6 @@ StorageServer::StorageServer(HostAddr localHost, walPath_(std::move(walPath)), listenerPath_(std::move(listenerPath)) {} -StorageServer::~StorageServer() { - stop(); -} - std::unique_ptr StorageServer::getStoreInstance() { kvstore::KVOptions options; options.dataPaths_ = dataPaths_; @@ -226,102 +222,13 @@ bool StorageServer::start() { return false; } - storageThread_.reset(new std::thread([this] { - try { - auto handler = std::make_shared(env_.get()); -#ifndef BUILD_STANDALONE - storageServer_ = std::make_unique(); - storageServer_->setPort(FLAGS_port); - storageServer_->setIdleTimeout(std::chrono::seconds(0)); - storageServer_->setIOThreadPool(ioThreadPool_); - storageServer_->setStopWorkersOnStopListening(false); - if (FLAGS_enable_ssl) { - storageServer_->setSSLConfig(nebula::sslContextConfig()); - } -#else - storageServer_ = GraphStorageLocalServer::getInstance(); -#endif - - storageServer_->setThreadManager(workers_); - storageServer_->setInterface(std::move(handler)); - ServiceStatus expected = STATUS_UNINITIALIZED; - if (!storageSvcStatus_.compare_exchange_strong(expected, STATUS_RUNNING)) { - LOG(ERROR) << "Impossible! How could it happen!"; - return; - } - LOG(INFO) << "The storage service start on " << localHost_; - storageServer_->serve(); // Will wait until the server shuts down - } catch (const std::exception& e) { - LOG(ERROR) << "Start storage service failed, error:" << e.what(); - } - storageSvcStatus_.store(STATUS_STOPPED); - LOG(INFO) << "The storage service stopped"; - })); - - adminThread_.reset(new std::thread([this] { - try { - auto handler = std::make_shared(env_.get()); - auto adminAddr = Utils::getAdminAddrFromStoreAddr(localHost_); - adminServer_ = std::make_unique(); - adminServer_->setPort(adminAddr.port); - adminServer_->setIdleTimeout(std::chrono::seconds(0)); - adminServer_->setIOThreadPool(ioThreadPool_); - adminServer_->setThreadManager(workers_); - adminServer_->setStopWorkersOnStopListening(false); - adminServer_->setInterface(std::move(handler)); - if (FLAGS_enable_ssl) { - adminServer_->setSSLConfig(nebula::sslContextConfig()); - } - - ServiceStatus expected = STATUS_UNINITIALIZED; - if (!adminSvcStatus_.compare_exchange_strong(expected, STATUS_RUNNING)) { - LOG(ERROR) << "Impossible! How could it happen!"; - return; - } - LOG(INFO) << "The admin service start on " << adminAddr; - adminServer_->serve(); // Will wait until the server shuts down - } catch (const std::exception& e) { - LOG(ERROR) << "Start admin service failed, error:" << e.what(); - } - adminSvcStatus_.store(STATUS_STOPPED); - LOG(INFO) << "The admin service stopped"; - })); - - internalStorageThread_.reset(new std::thread([this] { - try { - auto handler = std::make_shared(env_.get()); - auto internalAddr = Utils::getInternalAddrFromStoreAddr(localHost_); - internalStorageServer_ = std::make_unique(); - internalStorageServer_->setPort(internalAddr.port); - internalStorageServer_->setIdleTimeout(std::chrono::seconds(0)); - internalStorageServer_->setIOThreadPool(ioThreadPool_); - internalStorageServer_->setThreadManager(workers_); - internalStorageServer_->setStopWorkersOnStopListening(false); - internalStorageServer_->setInterface(std::move(handler)); - if (FLAGS_enable_ssl) { - internalStorageServer_->setSSLConfig(nebula::sslContextConfig()); - } - - internalStorageSvcStatus_.store(STATUS_RUNNING); - LOG(INFO) << "The internal storage service start(same with admin) on " << internalAddr; - internalStorageServer_->serve(); // Will wait until the server shuts down - } catch (const std::exception& e) { - LOG(ERROR) << "Start internal storage service failed, error:" << e.what(); - } - internalStorageSvcStatus_.store(STATUS_STOPPED); - LOG(INFO) << "The internal storage service stopped"; - })); - - while (storageSvcStatus_.load() == STATUS_UNINITIALIZED || - adminSvcStatus_.load() == STATUS_UNINITIALIZED || - internalStorageSvcStatus_.load() == STATUS_UNINITIALIZED) { - std::this_thread::sleep_for(std::chrono::microseconds(100)); - } - - if (storageSvcStatus_.load() != STATUS_RUNNING || adminSvcStatus_.load() != STATUS_RUNNING || - internalStorageSvcStatus_.load() != STATUS_RUNNING) { + storageServer_ = getStorageServer(); + adminServer_ = getAdminServer(); + internalStorageServer_ = getInternalServer(); + if (!storageServer_ || !adminServer_ || !internalStorageServer_) { return false; } + { std::lock_guard lkStop(muStop_); if (serverStatus_ != STATUS_UNINITIALIZED) { @@ -342,10 +249,6 @@ void StorageServer::waitUntilStop() { } this->stop(); - - adminThread_->join(); - storageThread_->join(); - internalStorageThread_->join(); } void StorageServer::notifyStop() { @@ -360,29 +263,27 @@ void StorageServer::notifyStop() { } void StorageServer::stop() { - if (adminSvcStatus_.load() == ServiceStatus::STATUS_STOPPED && - storageSvcStatus_.load() == ServiceStatus::STATUS_STOPPED && - internalStorageSvcStatus_.load() == ServiceStatus::STATUS_STOPPED) { - LOG(INFO) << "All services has been stopped"; - return; - } - - ServiceStatus adminExpected = ServiceStatus::STATUS_RUNNING; - adminSvcStatus_.compare_exchange_strong(adminExpected, STATUS_STOPPED); - - ServiceStatus storageExpected = ServiceStatus::STATUS_RUNNING; - storageSvcStatus_.compare_exchange_strong(storageExpected, STATUS_STOPPED); - - ServiceStatus interStorageExpected = ServiceStatus::STATUS_RUNNING; - internalStorageSvcStatus_.compare_exchange_strong(interStorageExpected, STATUS_STOPPED); + // Stop http service + webSvc_.reset(); - // kvstore need to stop back ground job before http server dctor + // Stop all thrift server: raft/storage/admin/internal if (kvstore_) { + // stop kvstore background job and raft services kvstore_->stop(); } + if (adminServer_) { + adminServer_->cleanUp(); + } + if (internalStorageServer_) { + internalStorageServer_->cleanUp(); + } + if (storageServer_) { +#ifndef BUILD_STANDALONE + storageServer_->cleanUp(); +#endif + } - webSvc_.reset(); - + // Stop all interface related to kvstore if (txnMan_) { txnMan_->stop(); txnMan_->join(); @@ -391,19 +292,93 @@ void StorageServer::stop() { taskMgr_->shutdown(); } if (metaClient_) { - metaClient_->notifyStop(); + metaClient_->stop(); } + + // Stop kvstore if (kvstore_) { kvstore_.reset(); } - if (adminServer_) { - adminServer_->stop(); +} + +#ifndef BUILD_STANDALONE +std::unique_ptr StorageServer::getStorageServer() { + try { + auto handler = std::make_shared(env_.get()); + auto server = std::make_unique(); + server->setPort(FLAGS_port); + server->setIdleTimeout(std::chrono::seconds(0)); + server->setIOThreadPool(ioThreadPool_); + server->setThreadManager(workers_); + if (FLAGS_enable_ssl) { + server->setSSLConfig(nebula::sslContextConfig()); + } + server->setInterface(std::move(handler)); + server->setup(); + return server; + } catch (const std::exception& e) { + LOG(ERROR) << "Start storage server failed: " << e.what(); + return nullptr; + } catch (...) { + LOG(ERROR) << "Start storage server failed"; + return nullptr; } - if (internalStorageServer_) { - internalStorageServer_->stop(); +} +#else +std::shared_ptr StorageServer::getStorageServer() { + auto handler = std::make_shared(env_.get()); + auto server = GraphStorageLocalServer::getInstance(); + server->setThreadManager(workers_); + server->setInterface(std::move(handler)); + return server; +} +#endif + +std::unique_ptr StorageServer::getAdminServer() { + try { + auto handler = std::make_shared(env_.get()); + auto adminAddr = Utils::getAdminAddrFromStoreAddr(localHost_); + auto server = std::make_unique(); + server->setPort(adminAddr.port); + server->setIdleTimeout(std::chrono::seconds(0)); + server->setIOThreadPool(ioThreadPool_); + server->setThreadManager(workers_); + if (FLAGS_enable_ssl) { + server->setSSLConfig(nebula::sslContextConfig()); + } + server->setInterface(std::move(handler)); + server->setup(); + return server; + } catch (const std::exception& e) { + LOG(ERROR) << "Start amdin server failed: " << e.what(); + return nullptr; + } catch (...) { + LOG(ERROR) << "Start amdin server failed"; + return nullptr; } - if (storageServer_) { - storageServer_->stop(); +} + +std::unique_ptr StorageServer::getInternalServer() { + try { + auto handler = std::make_shared(env_.get()); + auto internalAddr = Utils::getInternalAddrFromStoreAddr(localHost_); + auto server = std::make_unique(); + server->setPort(internalAddr.port); + server->setIdleTimeout(std::chrono::seconds(0)); + server->setIOThreadPool(ioThreadPool_); + server->setThreadManager(workers_); + if (FLAGS_enable_ssl) { + server->setSSLConfig(nebula::sslContextConfig()); + } + server->setInterface(std::move(handler)); + server->setup(); + return server; + } catch (const std::exception& e) { + LOG(ERROR) << "Start internal storage server failed: " << e.what(); + return nullptr; + } catch (...) { + LOG(ERROR) << "Start internal storage server failed"; + return nullptr; } } diff --git a/src/storage/StorageServer.h b/src/storage/StorageServer.h index 1cd7ab61019..94d52569d6a 100644 --- a/src/storage/StorageServer.h +++ b/src/storage/StorageServer.h @@ -19,11 +19,10 @@ #include "storage/GraphStorageLocalServer.h" #include "storage/admin/AdminTaskManager.h" #include "storage/transaction/TransactionManager.h" +#include "webservice/WebService.h" namespace nebula { -class WebService; - namespace storage { class StorageServer final { @@ -34,8 +33,6 @@ class StorageServer final { std::string walPath = "", std::string listenerPath = ""); - ~StorageServer(); - // Return false if failed. bool start(); @@ -68,6 +65,26 @@ class StorageServer final { int32_t getAdminStoreSeqId(); bool initWebService(); + + /** + * @brief storage thrift server, mainly for graph query + */ +#ifndef BUILD_STANDALONE + std::unique_ptr getStorageServer(); +#else + std::shared_ptr getStorageServer(); +#endif + + /** + * @brief admin thrift server, mainly for meta to control storage + */ + std::unique_ptr getAdminServer(); + + /** + * @brief internal thrift server, mainly for toss now + */ + std::unique_ptr getInternalServer(); + /** * @brief used by all thrift client, and kvstore. * default num is 16 @@ -75,20 +92,12 @@ class StorageServer final { std::shared_ptr ioThreadPool_; std::shared_ptr workers_; - std::unique_ptr storageThread_; - std::unique_ptr adminThread_; - std::atomic storageSvcStatus_{STATUS_UNINITIALIZED}; - std::atomic adminSvcStatus_{STATUS_UNINITIALIZED}; - #ifndef BUILD_STANDALONE std::unique_ptr storageServer_; #else std::shared_ptr storageServer_; #endif std::unique_ptr adminServer_; - - std::unique_ptr internalStorageThread_; - std::atomic internalStorageSvcStatus_{STATUS_UNINITIALIZED}; std::unique_ptr internalStorageServer_; std::unique_ptr webSvc_; From 28c24fe28c30ad18d799570992423886158808c0 Mon Sep 17 00:00:00 2001 From: Doodle <13706157+critical27@users.noreply.github.com> Date: Wed, 30 Mar 2022 20:12:28 +0800 Subject: [PATCH 08/24] fix crash raftLock, enable balance by experimental (#4088) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/graph/validator/AdminJobValidator.cpp | 4 +- src/kvstore/Listener.cpp | 2 +- src/kvstore/Listener.h | 1 + src/kvstore/NebulaStore.cpp | 2 +- src/kvstore/Part.cpp | 8 +- src/kvstore/Part.h | 6 +- src/kvstore/raftex/RaftPart.cpp | 138 ++++++++++++---------- src/kvstore/raftex/RaftPart.h | 14 ++- src/kvstore/raftex/test/TestShard.cpp | 7 +- src/kvstore/raftex/test/TestShard.h | 5 +- 10 files changed, 108 insertions(+), 79 deletions(-) diff --git a/src/graph/validator/AdminJobValidator.cpp b/src/graph/validator/AdminJobValidator.cpp index 5338a8caff4..c1b8ae2fad6 100644 --- a/src/graph/validator/AdminJobValidator.cpp +++ b/src/graph/validator/AdminJobValidator.cpp @@ -13,7 +13,9 @@ namespace graph { Status AdminJobValidator::validateImpl() { if (sentence_->getJobType() == meta::cpp2::JobType::DATA_BALANCE || sentence_->getJobType() == meta::cpp2::JobType::ZONE_BALANCE) { - return Status::SemanticError("Data balance not support"); + if (!FLAGS_enable_experimental_feature) { + return Status::SemanticError("Data balance not support"); + } } // Note: The last parameter of paras is no longer spacename diff --git a/src/kvstore/Listener.cpp b/src/kvstore/Listener.cpp index a535a268d0b..3121031e043 100644 --- a/src/kvstore/Listener.cpp +++ b/src/kvstore/Listener.cpp @@ -129,7 +129,7 @@ void Listener::cleanWal() { } std::tuple Listener::commitLogs( - std::unique_ptr iter, bool) { + std::unique_ptr iter, bool, bool) { LogID lastId = kNoCommitLogId; TermID lastTerm = kNoCommitLogTerm; while (iter->valid()) { diff --git a/src/kvstore/Listener.h b/src/kvstore/Listener.h index 05f6e07742e..05523c2b248 100644 --- a/src/kvstore/Listener.h +++ b/src/kvstore/Listener.h @@ -254,6 +254,7 @@ class Listener : public raftex::RaftPart { * @return std::tuple */ std::tuple commitLogs(std::unique_ptr, + bool, bool) override; /** diff --git a/src/kvstore/NebulaStore.cpp b/src/kvstore/NebulaStore.cpp index 72800398209..ee4c08b9151 100644 --- a/src/kvstore/NebulaStore.cpp +++ b/src/kvstore/NebulaStore.cpp @@ -189,7 +189,7 @@ void NebulaStore::loadPartFromDataPath() { } if (raftPeer.status == Peer::Status::kLearner) { - part->addLearner(addr); + part->addLearner(addr, true); } } } diff --git a/src/kvstore/Part.cpp b/src/kvstore/Part.cpp index d08b060c192..6c3a10cab85 100644 --- a/src/kvstore/Part.cpp +++ b/src/kvstore/Part.cpp @@ -211,7 +211,7 @@ void Part::onDiscoverNewLeader(HostAddr nLeader) { } std::tuple Part::commitLogs( - std::unique_ptr iter, bool wait) { + std::unique_ptr iter, bool wait, bool needLock) { SCOPED_TIMER(&execTime_); auto batch = engine_->startBatchWrite(); LogID lastId = kNoCommitLogId; @@ -309,12 +309,12 @@ std::tuple Part::commitLogs( } case OP_TRANS_LEADER: { auto newLeader = decodeHost(OP_TRANS_LEADER, log); - commitTransLeader(newLeader); + commitTransLeader(newLeader, needLock); break; } case OP_REMOVE_PEER: { auto peer = decodeHost(OP_REMOVE_PEER, log); - commitRemovePeer(peer); + commitRemovePeer(peer, needLock); break; } default: { @@ -395,7 +395,7 @@ bool Part::preProcessLog(LogID logId, TermID termId, ClusterID clusterId, const case OP_ADD_LEARNER: { auto learner = decodeHost(OP_ADD_LEARNER, log); LOG(INFO) << idStr_ << "preprocess add learner " << learner; - addLearner(learner); + addLearner(learner, false); // persist the part learner info in case of storaged restarting engine_->updatePart(partId_, Peer(learner, Peer::Status::kLearner)); break; diff --git a/src/kvstore/Part.h b/src/kvstore/Part.h index 6098d6e2000..d19138ce189 100644 --- a/src/kvstore/Part.h +++ b/src/kvstore/Part.h @@ -252,11 +252,15 @@ class Part : public raftex::RaftPart { * * @param iter Wal log iterator * @param wait Whether we should until all data applied to state machine + * @param needLock Whether need to acquire raftLock_ before operations. When the raftLock_ has + * been acquired before commitLogs is invoked, needLock is false (e.g. commitLogs by follower). If + * the lock has not been acquired, needLock is true (e.g. commitLogs by leader). * @return std::tuple * */ std::tuple commitLogs(std::unique_ptr iter, - bool wait) override; + bool wait, + bool needLock) override; /** * @brief Some special log need to be pre-processed when appending to wal diff --git a/src/kvstore/raftex/RaftPart.cpp b/src/kvstore/raftex/RaftPart.cpp index 45a94408fc9..e2605971c0e 100644 --- a/src/kvstore/raftex/RaftPart.cpp +++ b/src/kvstore/raftex/RaftPart.cpp @@ -383,28 +383,27 @@ nebula::cpp2::ErrorCode RaftPart::canAppendLogs(TermID termId) { return nebula::cpp2::ErrorCode::SUCCEEDED; } -void RaftPart::addLearner(const HostAddr& addr) { - bool acquiredHere = raftLock_.try_lock(); // because addLearner may be called in the - // NebulaStore, in which we could not access raftLock_ - if (addr == addr_) { - if (acquiredHere) { - raftLock_.unlock(); +void RaftPart::addLearner(const HostAddr& addr, bool needLock) { + auto addLearner = [&] { + if (addr == addr_) { + VLOG(1) << idStr_ << "I am learner!"; + return; } - VLOG(1) << idStr_ << "I am learner!"; - return; - } - auto it = std::find_if( - hosts_.begin(), hosts_.end(), [&addr](const auto& h) { return h->address() == addr; }); - if (it == hosts_.end()) { - hosts_.emplace_back(std::make_shared(addr, shared_from_this(), true)); - VLOG(1) << idStr_ << "Add learner " << addr; + auto it = std::find_if( + hosts_.begin(), hosts_.end(), [&addr](const auto& h) { return h->address() == addr; }); + if (it == hosts_.end()) { + hosts_.emplace_back(std::make_shared(addr, shared_from_this(), true)); + VLOG(1) << idStr_ << "Add learner " << addr; + } else { + VLOG(1) << idStr_ << "The host " << addr << " has been existed as " + << ((*it)->isLearner() ? " learner " : " group member"); + } + }; + if (needLock) { + std::lock_guard guard(raftLock_); + addLearner(); } else { - VLOG(1) << idStr_ << "The host " << addr << " has been existed as " - << ((*it)->isLearner() ? " learner " : " group member"); - } - - if (acquiredHere) { - raftLock_.unlock(); + addLearner(); } } @@ -437,40 +436,44 @@ void RaftPart::preProcessTransLeader(const HostAddr& target) { } } -void RaftPart::commitTransLeader(const HostAddr& target) { - bool needToUnlock = raftLock_.try_lock(); - VLOG(1) << idStr_ << "Commit transfer leader to " << target; - switch (role_) { - case Role::LEADER: { - if (target != addr_ && !hosts_.empty()) { - auto iter = std::find_if( - hosts_.begin(), hosts_.end(), [](const auto& h) { return !h->isLearner(); }); - if (iter != hosts_.end()) { - lastMsgRecvDur_.reset(); - role_ = Role::FOLLOWER; - leader_ = HostAddr("", 0); - for (auto& host : hosts_) { - host->pause(); +void RaftPart::commitTransLeader(const HostAddr& target, bool needLock) { + auto transfer = [&] { + VLOG(1) << idStr_ << "Commit transfer leader to " << target; + switch (role_) { + case Role::LEADER: { + if (target != addr_ && !hosts_.empty()) { + auto iter = std::find_if( + hosts_.begin(), hosts_.end(), [](const auto& h) { return !h->isLearner(); }); + if (iter != hosts_.end()) { + lastMsgRecvDur_.reset(); + role_ = Role::FOLLOWER; + leader_ = HostAddr("", 0); + for (auto& host : hosts_) { + host->pause(); + } + VLOG(1) << idStr_ << "Give up my leadership!"; } - VLOG(1) << idStr_ << "Give up my leadership!"; + } else { + VLOG(1) << idStr_ << "I am already the leader!"; } - } else { - VLOG(1) << idStr_ << "I am already the leader!"; + break; + } + case Role::FOLLOWER: + case Role::CANDIDATE: { + VLOG(1) << idStr_ << "I am " << roleStr(role_) << ", just wait for the new leader!"; + break; + } + case Role::LEARNER: { + VLOG(1) << idStr_ << "I am learner, not in the raft group, skip the log"; + break; } - break; - } - case Role::FOLLOWER: - case Role::CANDIDATE: { - VLOG(1) << idStr_ << "I am " << roleStr(role_) << ", just wait for the new leader!"; - break; - } - case Role::LEARNER: { - VLOG(1) << idStr_ << "I am learner, not in the raft group, skip the log"; - break; } - } - if (needToUnlock) { - raftLock_.unlock(); + }; + if (needLock) { + std::lock_guard guard(raftLock_); + transfer(); + } else { + transfer(); } } @@ -596,19 +599,21 @@ void RaftPart::preProcessRemovePeer(const HostAddr& peer) { removePeer(peer); } -void RaftPart::commitRemovePeer(const HostAddr& peer) { - bool needToUnlock = raftLock_.try_lock(); - SCOPE_EXIT { - if (needToUnlock) { - raftLock_.unlock(); +void RaftPart::commitRemovePeer(const HostAddr& peer, bool needLock) { + auto remove = [&] { + if (role_ == Role::FOLLOWER || role_ == Role::LEARNER) { + VLOG(1) << idStr_ << "I am " << roleStr(role_) << ", skip remove peer in commit"; + return; } + CHECK(Role::LEADER == role_); + removePeer(peer); }; - if (role_ == Role::FOLLOWER || role_ == Role::LEARNER) { - VLOG(1) << idStr_ << "I am " << roleStr(role_) << ", skip remove peer in commit"; - return; + if (needLock) { + std::lock_guard guard(raftLock_); + remove(); + } else { + remove(); } - CHECK(Role::LEADER == role_); - removePeer(peer); } folly::Future RaftPart::appendAsync(ClusterID source, std::string log) { @@ -932,7 +937,14 @@ void RaftPart::processAppendLogResponses(const AppendLogResponses& resps, { auto walIt = wal_->iterator(committedId + 1, lastLogId); // Step 3: Commit the batch - auto [code, lastCommitId, lastCommitTerm] = commitLogs(std::move(walIt), true); + /* + As for leader, we did't acquire raftLock because it would block heartbeat. Instead, we + protect the partition by the logsLock_, there won't be another out-going logs. So the third + parameters need to be true, we would grab the lock for some special operations. Besides, + leader neet to wait all logs applied to state machine, so the second parameters need to be + true so the second parameters need to be true. + */ + auto [code, lastCommitId, lastCommitTerm] = commitLogs(std::move(walIt), true, true); if (code == nebula::cpp2::ErrorCode::SUCCEEDED) { stats::StatsManager::addValue(kCommitLogLatencyUs, execTime_); std::lock_guard g(raftLock_); @@ -1664,7 +1676,9 @@ void RaftPart::processAppendLogRequest(const cpp2::AppendLogRequest& req, CHECK_LE(lastLogIdCanCommit, wal_->lastLogId()); if (lastLogIdCanCommit > committedLogId_) { auto walIt = wal_->iterator(committedLogId_ + 1, lastLogIdCanCommit); - auto [code, lastCommitId, lastCommitTerm] = commitLogs(std::move(walIt), false); + // follower do not wait all logs applied to state machine, so second parameter is false. And the + // raftLock_ has been acquired, so the third parameter is false as well. + auto [code, lastCommitId, lastCommitTerm] = commitLogs(std::move(walIt), false, false); if (code == nebula::cpp2::ErrorCode::SUCCEEDED) { stats::StatsManager::addValue(kCommitLogLatencyUs, execTime_); VLOG(4) << idStr_ << "Follower succeeded committing log " << committedLogId_ + 1 << " to " diff --git a/src/kvstore/raftex/RaftPart.h b/src/kvstore/raftex/RaftPart.h index 18cc0ec3bbc..96be8fb58a6 100644 --- a/src/kvstore/raftex/RaftPart.h +++ b/src/kvstore/raftex/RaftPart.h @@ -176,15 +176,17 @@ class RaftPart : public std::enable_shared_from_this { * @brief Add a raft learner to its peers * * @param learner Learner address + * @param needLock Whether need to acquire lock in the function */ - void addLearner(const HostAddr& learner); + void addLearner(const HostAddr& learner, bool needLock); /** * @brief When commit to state machine, old leader will step down as follower * * @param target Target new leader + * @param needLock Whether need to acquire lock in the function */ - void commitTransLeader(const HostAddr& target); + void commitTransLeader(const HostAddr& target, bool needLock); /** * @brief Pre-process of transfer leader, target new leader will start election task to background @@ -207,8 +209,9 @@ class RaftPart : public std::enable_shared_from_this { * preProcessRemovePeer, leader will remove in commitRemovePeer * * @param peer Target peer to remove + * @param needLock Whether need to acquire lock in the function */ - void commitRemovePeer(const HostAddr& peer); + void commitRemovePeer(const HostAddr& peer, bool needLock); // All learner and listener are raft learner. The difference between listener and learner is that // learner could be promoted to follower, but listener could not. (learner are added to hosts_, @@ -491,13 +494,16 @@ class RaftPart : public std::enable_shared_from_this { * * @param iter Log iterator of all logs to commit * @param wait Whether wait until all logs has been applied to state machine + * @param needLock Whether need to acquire raftLock_ before operations. When the raftLock_ has + * been acquired before commitLogs is invoked, needLock is false (e.g. commitLogs by follower). If + * the lock has not been acquired, needLock is true (e.g. commitLogs by leader). * @return std::tuple * Return {error code, last commit log id, last commit log term}. When no logs applied to state * machine or error occurs when calling commitLogs, kNoCommitLogId and kNoCommitLogTerm are * returned. */ virtual std::tuple commitLogs( - std::unique_ptr iter, bool wait) = 0; + std::unique_ptr iter, bool wait, bool needLock) = 0; /** * @brief A interface to pre-process wal, mainly for membership change diff --git a/src/kvstore/raftex/test/TestShard.cpp b/src/kvstore/raftex/test/TestShard.cpp index d1c2a19e1b7..b7009423271 100644 --- a/src/kvstore/raftex/test/TestShard.cpp +++ b/src/kvstore/raftex/test/TestShard.cpp @@ -166,7 +166,8 @@ void TestShard::onLeaderReady(TermID term) { } std::tuple TestShard::commitLogs( - std::unique_ptr iter, bool) { + std::unique_ptr iter, bool wait, bool needLock) { + UNUSED(wait); LogID lastId = kNoCommitLogId; TermID lastTerm = kNoCommitLogTerm; int32_t commitLogsNum = 0; @@ -178,12 +179,12 @@ std::tuple TestShard::commitLogs( switch (static_cast(log[0])) { case CommandType::TRANSFER_LEADER: { auto nLeader = decodeTransferLeader(log); - commitTransLeader(nLeader); + commitTransLeader(nLeader, needLock); break; } case CommandType::REMOVE_PEER: { auto peer = decodeRemovePeer(log); - commitRemovePeer(peer); + commitRemovePeer(peer, needLock); break; } case CommandType::ADD_PEER: diff --git a/src/kvstore/raftex/test/TestShard.h b/src/kvstore/raftex/test/TestShard.h index 9e0b3fbd3d9..ddb5bc6f5b8 100644 --- a/src/kvstore/raftex/test/TestShard.h +++ b/src/kvstore/raftex/test/TestShard.h @@ -79,14 +79,15 @@ class TestShard : public RaftPart { void onDiscoverNewLeader(HostAddr) override {} std::tuple commitLogs(std::unique_ptr iter, - bool wait) override; + bool wait, + bool needLock) override; bool preProcessLog(LogID, TermID, ClusterID, const std::string& log) override { if (!log.empty()) { switch (static_cast(log[0])) { case CommandType::ADD_LEARNER: { auto learner = decodeLearner(log); - addLearner(learner); + addLearner(learner, false); LOG(INFO) << idStr_ << "Add learner " << learner; break; } From 1e68162c6e46b601575cd708e296538b7db884d8 Mon Sep 17 00:00:00 2001 From: jimingquan Date: Wed, 30 Mar 2022 21:02:42 +0800 Subject: [PATCH 09/24] add executor comment (#4033) * add comment * add traverseExecutor's comment * add argument comment * add datacollect comment * delete vlog * fix log * fix comment Co-authored-by: yaphet <4414314+darionyaphet@users.noreply.github.com> Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/graph/executor/ExecutionError.h | 9 ++-- src/graph/executor/Executor.cpp | 9 ++-- src/graph/executor/Executor.h | 18 ++------ src/graph/executor/StorageAccessExecutor.cpp | 8 ++-- src/graph/executor/StorageAccessExecutor.h | 8 ++-- src/graph/executor/admin/AddHostsExecutor.cpp | 7 ++-- src/graph/executor/admin/AddHostsExecutor.h | 10 +++-- .../executor/admin/ChangePasswordExecutor.cpp | 9 ++-- .../executor/admin/ChangePasswordExecutor.h | 7 ++-- src/graph/executor/admin/CharsetExecutor.cpp | 10 ++--- src/graph/executor/admin/CharsetExecutor.h | 7 ++-- src/graph/executor/admin/ConfigExecutor.cpp | 16 +++----- src/graph/executor/admin/ConfigExecutor.h | 7 ++-- .../executor/admin/CreateUserExecutor.cpp | 9 ++-- src/graph/executor/admin/CreateUserExecutor.h | 7 ++-- .../executor/admin/DescribeUserExecutor.cpp | 12 +++--- .../executor/admin/DescribeUserExecutor.h | 7 ++-- .../executor/admin/DropHostsExecutor.cpp | 7 ++-- src/graph/executor/admin/DropHostsExecutor.h | 7 ++-- src/graph/executor/admin/DropUserExecutor.cpp | 9 ++-- src/graph/executor/admin/DropUserExecutor.h | 7 ++-- .../executor/admin/GrantRoleExecutor.cpp | 8 ++-- src/graph/executor/admin/GrantRoleExecutor.h | 10 ++--- .../executor/admin/KillQueryExecutor.cpp | 11 ++--- src/graph/executor/admin/KillQueryExecutor.h | 7 ++-- .../executor/admin/ListRolesExecutor.cpp | 9 ++-- src/graph/executor/admin/ListRolesExecutor.h | 7 ++-- .../executor/admin/ListUserRolesExecutor.cpp | 9 ++-- .../executor/admin/ListUserRolesExecutor.h | 7 ++-- .../executor/admin/ListUsersExecutor.cpp | 8 ++-- src/graph/executor/admin/ListUsersExecutor.h | 7 ++-- src/graph/executor/admin/ListenerExecutor.cpp | 13 +++--- src/graph/executor/admin/ListenerExecutor.h | 7 ++-- src/graph/executor/admin/PartExecutor.cpp | 17 ++++---- src/graph/executor/admin/PartExecutor.h | 7 ++-- .../executor/admin/RevokeRoleExecutor.cpp | 9 ++-- src/graph/executor/admin/RevokeRoleExecutor.h | 7 ++-- src/graph/executor/admin/SessionExecutor.cpp | 11 ++--- src/graph/executor/admin/SessionExecutor.h | 8 ++-- .../executor/admin/ShowHostsExecutor.cpp | 12 ++---- src/graph/executor/admin/ShowHostsExecutor.h | 7 ++-- .../executor/admin/ShowMetaLeaderExecutor.cpp | 10 ++--- .../executor/admin/ShowMetaLeaderExecutor.h | 7 ++-- .../executor/admin/ShowQueriesExecutor.cpp | 9 ++-- .../executor/admin/ShowQueriesExecutor.h | 7 ++-- .../admin/ShowServiceClientsExecutor.cpp | 11 ++--- .../admin/ShowServiceClientsExecutor.h | 7 ++-- .../executor/admin/ShowStatsExecutor.cpp | 12 ++---- src/graph/executor/admin/ShowStatsExecutor.h | 7 ++-- .../executor/admin/SignInServiceExecutor.cpp | 8 ++-- .../executor/admin/SignInServiceExecutor.h | 7 ++-- .../executor/admin/SignOutServiceExecutor.cpp | 7 ++-- .../executor/admin/SignOutServiceExecutor.h | 7 ++-- src/graph/executor/admin/SnapshotExecutor.cpp | 17 +++----- src/graph/executor/admin/SnapshotExecutor.h | 7 ++-- src/graph/executor/admin/SpaceExecutor.cpp | 41 +++++++++---------- src/graph/executor/admin/SpaceExecutor.h | 7 ++-- .../executor/admin/SubmitJobExecutor.cpp | 13 +++--- src/graph/executor/admin/SubmitJobExecutor.h | 7 ++-- .../executor/admin/SwitchSpaceExecutor.cpp | 11 ++--- .../executor/admin/SwitchSpaceExecutor.h | 7 ++-- .../executor/admin/UpdateUserExecutor.cpp | 9 ++-- src/graph/executor/admin/UpdateUserExecutor.h | 7 ++-- src/graph/executor/admin/ZoneExecutor.cpp | 22 +++++----- src/graph/executor/admin/ZoneExecutor.h | 7 ++-- .../algo/CartesianProductExecutor.cpp | 9 ++-- .../executor/algo/CartesianProductExecutor.h | 7 ++-- src/graph/executor/algo/SubgraphExecutor.cpp | 9 ++-- src/graph/executor/algo/SubgraphExecutor.h | 7 ++-- src/graph/executor/logic/ArgumentExecutor.cpp | 8 ++-- src/graph/executor/logic/ArgumentExecutor.h | 12 +++--- src/graph/executor/logic/LoopExecutor.cpp | 16 ++------ src/graph/executor/logic/LoopExecutor.h | 10 ++--- .../executor/logic/PassThroughExecutor.cpp | 11 ++--- .../executor/logic/PassThroughExecutor.h | 9 ++-- src/graph/executor/logic/SelectExecutor.cpp | 11 ++--- src/graph/executor/logic/SelectExecutor.h | 10 ++--- src/graph/executor/logic/StartExecutor.cpp | 10 ++--- src/graph/executor/logic/StartExecutor.h | 9 ++-- src/graph/executor/maintain/EdgeExecutor.cpp | 38 ++++++++--------- src/graph/executor/maintain/EdgeExecutor.h | 7 ++-- .../executor/maintain/EdgeIndexExecutor.cpp | 36 ++++++++-------- .../executor/maintain/EdgeIndexExecutor.h | 7 ++-- .../executor/maintain/FTIndexExecutor.cpp | 25 ++++++----- src/graph/executor/maintain/FTIndexExecutor.h | 10 ++--- src/graph/executor/maintain/TagExecutor.cpp | 38 ++++++++--------- src/graph/executor/maintain/TagExecutor.h | 7 ++-- .../executor/maintain/TagIndexExecutor.cpp | 35 ++++++++-------- .../executor/maintain/TagIndexExecutor.h | 7 ++-- src/graph/executor/mutate/DeleteExecutor.cpp | 16 ++------ src/graph/executor/mutate/DeleteExecutor.h | 8 ++-- src/graph/executor/mutate/InsertExecutor.cpp | 10 ++--- src/graph/executor/mutate/InsertExecutor.h | 7 ++-- src/graph/executor/mutate/UpdateExecutor.cpp | 20 ++++----- src/graph/executor/mutate/UpdateExecutor.h | 8 ++-- .../executor/query/AggregateExecutor.cpp | 14 ++----- src/graph/executor/query/AggregateExecutor.h | 10 ++--- .../executor/query/AppendVerticesExecutor.cpp | 7 ++-- .../executor/query/AppendVerticesExecutor.h | 12 +++--- src/graph/executor/query/AssignExecutor.cpp | 9 ++-- src/graph/executor/query/AssignExecutor.h | 9 ++-- .../executor/query/DataCollectExecutor.cpp | 10 ++--- .../executor/query/DataCollectExecutor.h | 28 ++++++++++--- src/graph/executor/query/DedupExecutor.cpp | 14 ++----- src/graph/executor/query/DedupExecutor.h | 10 ++--- src/graph/executor/query/FilterExecutor.cpp | 14 ++----- src/graph/executor/query/FilterExecutor.h | 9 ++-- src/graph/executor/query/GetEdgesExecutor.cpp | 13 ++---- src/graph/executor/query/GetEdgesExecutor.h | 8 ++-- .../executor/query/GetNeighborsExecutor.cpp | 17 ++------ .../executor/query/GetNeighborsExecutor.h | 11 +++-- src/graph/executor/query/GetPropExecutor.h | 2 +- .../executor/query/GetVerticesExecutor.cpp | 12 ++---- .../executor/query/GetVerticesExecutor.h | 9 ++-- .../executor/query/IndexScanExecutor.cpp | 12 ++---- src/graph/executor/query/IndexScanExecutor.h | 12 +++--- .../executor/query/InnerJoinExecutor.cpp | 11 ++--- src/graph/executor/query/InnerJoinExecutor.h | 14 +++---- .../executor/query/IntersectExecutor.cpp | 12 ++---- src/graph/executor/query/IntersectExecutor.h | 9 ++-- src/graph/executor/query/JoinExecutor.cpp | 14 ++----- src/graph/executor/query/JoinExecutor.h | 7 ++-- src/graph/executor/query/LeftJoinExecutor.cpp | 12 ++---- src/graph/executor/query/LeftJoinExecutor.h | 13 +++--- src/graph/executor/query/LimitExecutor.cpp | 9 ++-- src/graph/executor/query/LimitExecutor.h | 10 ++--- src/graph/executor/query/MinusExecutor.cpp | 11 ++--- src/graph/executor/query/MinusExecutor.h | 9 ++-- src/graph/executor/query/ProjectExecutor.cpp | 13 ++---- src/graph/executor/query/ProjectExecutor.h | 9 ++-- .../executor/query/RollUpApplyExecutor.cpp | 5 +-- .../executor/query/RollUpApplyExecutor.h | 2 +- src/graph/executor/query/SampleExecutor.cpp | 9 ++-- src/graph/executor/query/SampleExecutor.h | 9 ++-- .../executor/query/ScanEdgesExecutor.cpp | 10 ++--- src/graph/executor/query/ScanEdgesExecutor.h | 9 ++-- .../executor/query/ScanVerticesExecutor.cpp | 10 ++--- .../executor/query/ScanVerticesExecutor.h | 10 ++--- src/graph/executor/query/SetExecutor.cpp | 12 ++---- src/graph/executor/query/SetExecutor.h | 14 ++----- src/graph/executor/query/SortExecutor.cpp | 10 ++--- src/graph/executor/query/SortExecutor.h | 8 ++-- src/graph/executor/query/TopNExecutor.cpp | 10 ++--- src/graph/executor/query/TopNExecutor.h | 8 ++-- src/graph/executor/query/TraverseExecutor.cpp | 16 ++------ src/graph/executor/query/TraverseExecutor.h | 35 +++++++++++----- .../query/UnionAllVersionVarExecutor.cpp | 9 ++-- .../query/UnionAllVersionVarExecutor.h | 9 ++-- src/graph/executor/query/UnionExecutor.cpp | 10 ++--- src/graph/executor/query/UnionExecutor.h | 10 ++--- src/graph/executor/query/UnwindExecutor.cpp | 12 ++---- src/graph/executor/query/UnwindExecutor.h | 9 ++-- 152 files changed, 668 insertions(+), 1005 deletions(-) diff --git a/src/graph/executor/ExecutionError.h b/src/graph/executor/ExecutionError.h index fb986e3732a..5d2a08cb7a3 100644 --- a/src/graph/executor/ExecutionError.h +++ b/src/graph/executor/ExecutionError.h @@ -1,13 +1,10 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_EXECUTIONERROR_H_ #define GRAPH_EXECUTOR_EXECUTIONERROR_H_ -#include - #include "common/base/Status.h" namespace nebula { diff --git a/src/graph/executor/Executor.cpp b/src/graph/executor/Executor.cpp index 27e24966d83..33ed32d44c0 100644 --- a/src/graph/executor/Executor.cpp +++ b/src/graph/executor/Executor.cpp @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/Executor.h" @@ -13,9 +12,7 @@ #include "common/base/ObjectPool.h" #include "common/memory/MemoryUtils.h" #include "common/stats/StatsManager.h" -#include "common/time/ScopedTimer.h" #include "graph/context/ExecutionContext.h" -#include "graph/context/QueryContext.h" #include "graph/executor/ExecutionError.h" #include "graph/executor/admin/AddHostsExecutor.h" #include "graph/executor/admin/ChangePasswordExecutor.h" diff --git a/src/graph/executor/Executor.h b/src/graph/executor/Executor.h index e71ac701a65..6e9b932f86b 100644 --- a/src/graph/executor/Executor.h +++ b/src/graph/executor/Executor.h @@ -1,31 +1,21 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_EXECUTOR_H_ #define GRAPH_EXECUTOR_EXECUTOR_H_ - #include #include -#include -#include -#include -#include "common/base/Status.h" #include "common/cpp/helpers.h" -#include "common/datatypes/Value.h" #include "common/time/Duration.h" #include "common/time/ScopedTimer.h" #include "graph/context/ExecutionContext.h" - +#include "graph/context/QueryContext.h" namespace nebula { namespace graph { - class PlanNode; class QueryContext; - class Executor : private boost::noncopyable, private cpp::NonMovable { public: // Create executor according to plan node diff --git a/src/graph/executor/StorageAccessExecutor.cpp b/src/graph/executor/StorageAccessExecutor.cpp index 201902c8ded..c315cdde008 100644 --- a/src/graph/executor/StorageAccessExecutor.cpp +++ b/src/graph/executor/StorageAccessExecutor.cpp @@ -1,12 +1,10 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/StorageAccessExecutor.h" #include - #include "graph/context/Iterator.h" #include "graph/context/QueryExpressionContext.h" #include "graph/util/SchemaUtil.h" diff --git a/src/graph/executor/StorageAccessExecutor.h b/src/graph/executor/StorageAccessExecutor.h index 37bb0454213..0978ca5bba3 100644 --- a/src/graph/executor/StorageAccessExecutor.h +++ b/src/graph/executor/StorageAccessExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_STORAGEACCESSEXECUTOR_H_ #define GRAPH_EXECUTOR_STORAGEACCESSEXECUTOR_H_ @@ -9,7 +8,6 @@ #include #include "clients/storage/StorageClientBase.h" -#include "graph/context/QueryContext.h" #include "graph/executor/Executor.h" namespace nebula { diff --git a/src/graph/executor/admin/AddHostsExecutor.cpp b/src/graph/executor/admin/AddHostsExecutor.cpp index d24706b114f..462e4043fc6 100644 --- a/src/graph/executor/admin/AddHostsExecutor.cpp +++ b/src/graph/executor/admin/AddHostsExecutor.cpp @@ -1,7 +1,6 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/AddHostsExecutor.h" diff --git a/src/graph/executor/admin/AddHostsExecutor.h b/src/graph/executor/admin/AddHostsExecutor.h index 833f430e816..ae8e93f9593 100644 --- a/src/graph/executor/admin/AddHostsExecutor.h +++ b/src/graph/executor/admin/AddHostsExecutor.h @@ -1,13 +1,15 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_ADD_HOST_EXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_ADD_HOST_EXECUTOR_H_ #include "graph/executor/Executor.h" +// setting Storage hosts in the configuration files only registers the hosts on the Meta side, +// but does not add them into the cluster. You must run the ADD HOSTS statement to add +// the Storage hosts. namespace nebula { namespace graph { diff --git a/src/graph/executor/admin/ChangePasswordExecutor.cpp b/src/graph/executor/admin/ChangePasswordExecutor.cpp index 48df995fd7c..8e7c4f8d86a 100644 --- a/src/graph/executor/admin/ChangePasswordExecutor.cpp +++ b/src/graph/executor/admin/ChangePasswordExecutor.cpp @@ -1,13 +1,10 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/ChangePasswordExecutor.h" #include - -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/ChangePasswordExecutor.h b/src/graph/executor/admin/ChangePasswordExecutor.h index 2f20da3607b..bb23da34c53 100644 --- a/src/graph/executor/admin/ChangePasswordExecutor.h +++ b/src/graph/executor/admin/ChangePasswordExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_CHANGEPASSWORDEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_CHANGEPASSWORDEXECUTOR_H_ diff --git a/src/graph/executor/admin/CharsetExecutor.cpp b/src/graph/executor/admin/CharsetExecutor.cpp index 1ac91bae195..38b0b360632 100644 --- a/src/graph/executor/admin/CharsetExecutor.cpp +++ b/src/graph/executor/admin/CharsetExecutor.cpp @@ -1,13 +1,9 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/CharsetExecutor.h" -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" - namespace nebula { namespace graph { folly::Future ShowCharsetExecutor::execute() { diff --git a/src/graph/executor/admin/CharsetExecutor.h b/src/graph/executor/admin/CharsetExecutor.h index 5c910d48c97..0b869774814 100644 --- a/src/graph/executor/admin/CharsetExecutor.h +++ b/src/graph/executor/admin/CharsetExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_CHARSETEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_CHARSETEXECUTOR_H_ diff --git a/src/graph/executor/admin/ConfigExecutor.cpp b/src/graph/executor/admin/ConfigExecutor.cpp index e5f49b6261b..3d0525baddf 100644 --- a/src/graph/executor/admin/ConfigExecutor.cpp +++ b/src/graph/executor/admin/ConfigExecutor.cpp @@ -1,15 +1,11 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/ConfigExecutor.h" #include - #include "common/conf/Configuration.h" -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" #include "graph/util/SchemaUtil.h" @@ -49,7 +45,7 @@ folly::Future ShowConfigsExecutor::execute() { .thenValue([this, scNode](StatusOr> resp) { if (!resp.ok()) { auto module = apache::thrift::util::enumNameSafe(scNode->getModule()); - LOG(ERROR) << "Show configs `" << module << "' failed: " << resp.status(); + LOG(WARNING) << "Show configs `" << module << "' failed: " << resp.status(); return Status::Error( "Show config `%s' failed: %s", module.c_str(), resp.status().toString().c_str()); } @@ -90,7 +86,7 @@ folly::Future SetConfigExecutor::execute() { .via(runner()) .thenValue([scNode](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "Set config `" << scNode->getName() << "' failed: " << resp.status(); + LOG(WARNING) << "Set config `" << scNode->getName() << "' failed: " << resp.status(); return Status::Error("Set config `%s' failed: %s", scNode->getName().c_str(), resp.status().toString().c_str()); @@ -109,7 +105,7 @@ folly::Future GetConfigExecutor::execute() { .via(runner()) .thenValue([this, gcNode](StatusOr> resp) { if (!resp.ok()) { - LOG(ERROR) << "Get config `" << gcNode->getName() << "' failed: " << resp.status(); + LOG(WARNING) << "Get config `" << gcNode->getName() << "' failed: " << resp.status(); return Status::Error("Get config `%s' failed: %s", gcNode->getName().c_str(), resp.status().toString().c_str()); diff --git a/src/graph/executor/admin/ConfigExecutor.h b/src/graph/executor/admin/ConfigExecutor.h index 3bc6e5b40d2..99b8b716532 100644 --- a/src/graph/executor/admin/ConfigExecutor.h +++ b/src/graph/executor/admin/ConfigExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_CONFIGEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_CONFIGEXECUTOR_H_ diff --git a/src/graph/executor/admin/CreateUserExecutor.cpp b/src/graph/executor/admin/CreateUserExecutor.cpp index 83ed5b00d8c..b5e9c29c2ca 100644 --- a/src/graph/executor/admin/CreateUserExecutor.cpp +++ b/src/graph/executor/admin/CreateUserExecutor.cpp @@ -1,13 +1,10 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/CreateUserExecutor.h" #include - -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/CreateUserExecutor.h b/src/graph/executor/admin/CreateUserExecutor.h index 19f2c68e27d..1171ed9103f 100644 --- a/src/graph/executor/admin/CreateUserExecutor.h +++ b/src/graph/executor/admin/CreateUserExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_CREATEUSEREXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_CREATEUSEREXECUTOR_H_ diff --git a/src/graph/executor/admin/DescribeUserExecutor.cpp b/src/graph/executor/admin/DescribeUserExecutor.cpp index 9b2d893b7f8..6353287f985 100644 --- a/src/graph/executor/admin/DescribeUserExecutor.cpp +++ b/src/graph/executor/admin/DescribeUserExecutor.cpp @@ -1,13 +1,10 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/DescribeUserExecutor.h" #include - -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" #include "interface/gen-cpp2/meta_types.h" @@ -43,7 +40,8 @@ folly::Future DescribeUserExecutor::describeUser() { v.emplace_back(nebula::Row({apache::thrift::util::enumNameSafe(item.get_role_type()), spaceNameResult.value()})); } else { - LOG(ERROR) << " Space name of " << item.get_space_id() << " no found"; + LOG(WARNING) << " Space name of " << item.get_space_id() + << " no found: " << spaceNameResult.status(); return Status::Error("Space not found"); } } diff --git a/src/graph/executor/admin/DescribeUserExecutor.h b/src/graph/executor/admin/DescribeUserExecutor.h index 629e1cb7854..6badd084555 100644 --- a/src/graph/executor/admin/DescribeUserExecutor.h +++ b/src/graph/executor/admin/DescribeUserExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_DESCRIBEUSEREXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_DESCRIBEUSEREXECUTOR_H_ diff --git a/src/graph/executor/admin/DropHostsExecutor.cpp b/src/graph/executor/admin/DropHostsExecutor.cpp index db51555efd5..01e6f1d1ec2 100644 --- a/src/graph/executor/admin/DropHostsExecutor.cpp +++ b/src/graph/executor/admin/DropHostsExecutor.cpp @@ -1,7 +1,6 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/DropHostsExecutor.h" diff --git a/src/graph/executor/admin/DropHostsExecutor.h b/src/graph/executor/admin/DropHostsExecutor.h index f2fc896c0fb..b89b3265980 100644 --- a/src/graph/executor/admin/DropHostsExecutor.h +++ b/src/graph/executor/admin/DropHostsExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_DROP_HOST_EXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_DROP_HOST_EXECUTOR_H_ diff --git a/src/graph/executor/admin/DropUserExecutor.cpp b/src/graph/executor/admin/DropUserExecutor.cpp index 7f07c3f86f4..21051b6e125 100644 --- a/src/graph/executor/admin/DropUserExecutor.cpp +++ b/src/graph/executor/admin/DropUserExecutor.cpp @@ -1,11 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/DropUserExecutor.h" - -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/DropUserExecutor.h b/src/graph/executor/admin/DropUserExecutor.h index bfd32c807c7..63f76cb8ecc 100644 --- a/src/graph/executor/admin/DropUserExecutor.h +++ b/src/graph/executor/admin/DropUserExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_DROPUSEREXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_DROPUSEREXECUTOR_H_ diff --git a/src/graph/executor/admin/GrantRoleExecutor.cpp b/src/graph/executor/admin/GrantRoleExecutor.cpp index 5cb7813936b..d332b97a279 100644 --- a/src/graph/executor/admin/GrantRoleExecutor.cpp +++ b/src/graph/executor/admin/GrantRoleExecutor.cpp @@ -1,11 +1,9 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/GrantRoleExecutor.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" #include "graph/service/PermissionManager.h" #include "interface/gen-cpp2/meta_types.h" diff --git a/src/graph/executor/admin/GrantRoleExecutor.h b/src/graph/executor/admin/GrantRoleExecutor.h index c7f03616115..2befe404cdb 100644 --- a/src/graph/executor/admin/GrantRoleExecutor.h +++ b/src/graph/executor/admin/GrantRoleExecutor.h @@ -1,13 +1,13 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_GRANTROLEEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_GRANTROLEEXECUTOR_H_ #include "graph/executor/Executor.h" - +// users with the GOD role or the ADMIN role can run GRANT ROLE to assign a built-in role +// in a graph space to a user. namespace nebula { namespace graph { diff --git a/src/graph/executor/admin/KillQueryExecutor.cpp b/src/graph/executor/admin/KillQueryExecutor.cpp index f9b7044f791..47628803d55 100644 --- a/src/graph/executor/admin/KillQueryExecutor.cpp +++ b/src/graph/executor/admin/KillQueryExecutor.cpp @@ -1,11 +1,8 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/KillQueryExecutor.h" - -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" namespace nebula { @@ -28,7 +25,7 @@ folly::Future KillQueryExecutor::execute() { if (listResp.ok()) { sessionsInMeta = std::move(listResp.value()).get_sessions(); } else { - LOG(ERROR) << listResp.status(); + LOG(WARNING) << "List session fail: " << listResp.status(); } auto status = verifyTheQueriesByMetaInfo(toBeVerifiedQueries, sessionsInMeta); diff --git a/src/graph/executor/admin/KillQueryExecutor.h b/src/graph/executor/admin/KillQueryExecutor.h index 5eea04c647c..92456c701fe 100644 --- a/src/graph/executor/admin/KillQueryExecutor.h +++ b/src/graph/executor/admin/KillQueryExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_KILLQUERYEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_KILLQUERYEXECUTOR_H_ diff --git a/src/graph/executor/admin/ListRolesExecutor.cpp b/src/graph/executor/admin/ListRolesExecutor.cpp index 198714e0954..17d6b22f85b 100644 --- a/src/graph/executor/admin/ListRolesExecutor.cpp +++ b/src/graph/executor/admin/ListRolesExecutor.cpp @@ -1,13 +1,10 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/ListRolesExecutor.h" #include - -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" #include "graph/service/PermissionManager.h" diff --git a/src/graph/executor/admin/ListRolesExecutor.h b/src/graph/executor/admin/ListRolesExecutor.h index 735f4f4dd01..04516bebdeb 100644 --- a/src/graph/executor/admin/ListRolesExecutor.h +++ b/src/graph/executor/admin/ListRolesExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_LISTROLESEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_LISTROLESEXECUTOR_H_ diff --git a/src/graph/executor/admin/ListUserRolesExecutor.cpp b/src/graph/executor/admin/ListUserRolesExecutor.cpp index 4637721f392..201a3b8fcf4 100644 --- a/src/graph/executor/admin/ListUserRolesExecutor.cpp +++ b/src/graph/executor/admin/ListUserRolesExecutor.cpp @@ -1,13 +1,10 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/ListUserRolesExecutor.h" #include - -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/ListUserRolesExecutor.h b/src/graph/executor/admin/ListUserRolesExecutor.h index 27f757accd2..7d10ac39706 100644 --- a/src/graph/executor/admin/ListUserRolesExecutor.h +++ b/src/graph/executor/admin/ListUserRolesExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_LISTUSERROLESEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_LISTUSERROLESEXECUTOR_H_ diff --git a/src/graph/executor/admin/ListUsersExecutor.cpp b/src/graph/executor/admin/ListUsersExecutor.cpp index b196d660ad3..062ae32cbea 100644 --- a/src/graph/executor/admin/ListUsersExecutor.cpp +++ b/src/graph/executor/admin/ListUsersExecutor.cpp @@ -1,11 +1,9 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/ListUsersExecutor.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/ListUsersExecutor.h b/src/graph/executor/admin/ListUsersExecutor.h index 02fb737a112..b806eca754e 100644 --- a/src/graph/executor/admin/ListUsersExecutor.h +++ b/src/graph/executor/admin/ListUsersExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_LISTUSERSEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_LISTUSERSEXECUTOR_H_ diff --git a/src/graph/executor/admin/ListenerExecutor.cpp b/src/graph/executor/admin/ListenerExecutor.cpp index 11ebf97e8a6..8e65009b484 100644 --- a/src/graph/executor/admin/ListenerExecutor.cpp +++ b/src/graph/executor/admin/ListenerExecutor.cpp @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/ListenerExecutor.h" @@ -23,7 +22,7 @@ folly::Future AddListenerExecutor::execute() { .thenValue([this](StatusOr resp) { SCOPED_TIMER(&execTime_); if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "Add listener fail: " << resp.status(); return resp.status(); } return Status::OK(); @@ -41,7 +40,7 @@ folly::Future RemoveListenerExecutor::execute() { .thenValue([this](StatusOr resp) { SCOPED_TIMER(&execTime_); if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "Remove listener fail: " << resp.status(); return resp.status(); } return Status::OK(); @@ -55,7 +54,7 @@ folly::Future ShowListenerExecutor::execute() { [this](StatusOr> resp) { SCOPED_TIMER(&execTime_); if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "Show listener fail: " << resp.status(); return resp.status(); } diff --git a/src/graph/executor/admin/ListenerExecutor.h b/src/graph/executor/admin/ListenerExecutor.h index 63907a8e625..8b11a05b6c7 100644 --- a/src/graph/executor/admin/ListenerExecutor.h +++ b/src/graph/executor/admin/ListenerExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_LISTENEREXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_LISTENEREXECUTOR_H_ diff --git a/src/graph/executor/admin/PartExecutor.cpp b/src/graph/executor/admin/PartExecutor.cpp index 786f236d33c..dfd88907206 100644 --- a/src/graph/executor/admin/PartExecutor.cpp +++ b/src/graph/executor/admin/PartExecutor.cpp @@ -1,12 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/PartExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" using nebula::network::NetworkUtils; @@ -17,13 +13,14 @@ folly::Future ShowPartsExecutor::execute() { SCOPED_TIMER(&execTime_); auto* spNode = asNode(node()); + const auto& spaceId = spNode->getSpaceId(); return qctx() ->getMetaClient() - ->listParts(spNode->getSpaceId(), spNode->getPartIds()) + ->listParts(spaceId, spNode->getPartIds()) .via(runner()) - .thenValue([this](StatusOr> resp) { + .thenValue([this, spaceId](StatusOr> resp) { if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Show Parts fail: " << resp.status(); return resp.status(); } auto partItems = std::move(resp).value(); diff --git a/src/graph/executor/admin/PartExecutor.h b/src/graph/executor/admin/PartExecutor.h index fde97f6cc23..b7d25859dbd 100644 --- a/src/graph/executor/admin/PartExecutor.h +++ b/src/graph/executor/admin/PartExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_PARTEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_PARTEXECUTOR_H_ diff --git a/src/graph/executor/admin/RevokeRoleExecutor.cpp b/src/graph/executor/admin/RevokeRoleExecutor.cpp index 37ae972ceb5..be59f42166a 100644 --- a/src/graph/executor/admin/RevokeRoleExecutor.cpp +++ b/src/graph/executor/admin/RevokeRoleExecutor.cpp @@ -1,11 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/RevokeRoleExecutor.h" - -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" #include "graph/service/PermissionManager.h" #include "interface/gen-cpp2/meta_types.h" diff --git a/src/graph/executor/admin/RevokeRoleExecutor.h b/src/graph/executor/admin/RevokeRoleExecutor.h index 69fbd41d423..122c0ff9414 100644 --- a/src/graph/executor/admin/RevokeRoleExecutor.h +++ b/src/graph/executor/admin/RevokeRoleExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_REVOKEROLEEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_REVOKEROLEEXECUTOR_H_ diff --git a/src/graph/executor/admin/SessionExecutor.cpp b/src/graph/executor/admin/SessionExecutor.cpp index b4e4a852288..578625232f3 100644 --- a/src/graph/executor/admin/SessionExecutor.cpp +++ b/src/graph/executor/admin/SessionExecutor.cpp @@ -1,11 +1,9 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/SessionExecutor.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" #include "interface/gen-cpp2/common_types.h" #include "interface/gen-cpp2/meta_types.h" @@ -100,7 +98,6 @@ void ShowSessionsExecutor::addSessions(const meta::cpp2::Session &session, DataS } folly::Future UpdateSessionExecutor::execute() { - VLOG(1) << "Update sessions to metad"; SCOPED_TIMER(&execTime_); auto *updateNode = asNode(node()); std::vector sessions; @@ -109,7 +106,7 @@ folly::Future UpdateSessionExecutor::execute() { [this](auto &&resp) { SCOPED_TIMER(&execTime_); if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "Update session fail " << resp.status(); return resp.status(); } return Status::OK(); diff --git a/src/graph/executor/admin/SessionExecutor.h b/src/graph/executor/admin/SessionExecutor.h index 110fbf687c3..46f493660bd 100644 --- a/src/graph/executor/admin/SessionExecutor.h +++ b/src/graph/executor/admin/SessionExecutor.h @@ -1,14 +1,12 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_SESSIONEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_SESSIONEXECUTOR_H_ #include "common/time/TimeUtils.h" #include "graph/executor/Executor.h" -#include "graph/service/RequestContext.h" namespace nebula { namespace graph { diff --git a/src/graph/executor/admin/ShowHostsExecutor.cpp b/src/graph/executor/admin/ShowHostsExecutor.cpp index 1b317949447..23860c5fa3d 100644 --- a/src/graph/executor/admin/ShowHostsExecutor.cpp +++ b/src/graph/executor/admin/ShowHostsExecutor.cpp @@ -1,14 +1,10 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/ShowHostsExecutor.h" #include - -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" namespace nebula { @@ -149,7 +145,7 @@ folly::Future ShowHostsExecutor::showHosts() { .via(runner()) .thenValue([=, type = shNode->getType()](auto &&resp) { if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "Show host fail: " << resp.status(); return resp.status(); } auto value = std::forward(resp).value(); diff --git a/src/graph/executor/admin/ShowHostsExecutor.h b/src/graph/executor/admin/ShowHostsExecutor.h index 5ef27542f61..a5e02ea6661 100644 --- a/src/graph/executor/admin/ShowHostsExecutor.h +++ b/src/graph/executor/admin/ShowHostsExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_SHOW_HOSTS_EXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_SHOW_HOSTS_EXECUTOR_H_ diff --git a/src/graph/executor/admin/ShowMetaLeaderExecutor.cpp b/src/graph/executor/admin/ShowMetaLeaderExecutor.cpp index 8d10f523150..21a43387a22 100644 --- a/src/graph/executor/admin/ShowMetaLeaderExecutor.cpp +++ b/src/graph/executor/admin/ShowMetaLeaderExecutor.cpp @@ -1,15 +1,11 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/ShowMetaLeaderExecutor.h" #include - -#include "common/time/ScopedTimer.h" #include "common/time/TimeUtils.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/ShowMetaLeaderExecutor.h b/src/graph/executor/admin/ShowMetaLeaderExecutor.h index fe3a2f2b074..d59f5dd70a5 100644 --- a/src/graph/executor/admin/ShowMetaLeaderExecutor.h +++ b/src/graph/executor/admin/ShowMetaLeaderExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_SHOWMETALEADEREXECUTOR_H #define GRAPH_EXECUTOR_ADMIN_SHOWMETALEADEREXECUTOR_H diff --git a/src/graph/executor/admin/ShowQueriesExecutor.cpp b/src/graph/executor/admin/ShowQueriesExecutor.cpp index b2166e5a567..ec77ec13f72 100644 --- a/src/graph/executor/admin/ShowQueriesExecutor.cpp +++ b/src/graph/executor/admin/ShowQueriesExecutor.cpp @@ -1,15 +1,12 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/ShowQueriesExecutor.h" #include -#include "common/time/ScopedTimer.h" #include "common/time/TimeUtils.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/ShowQueriesExecutor.h b/src/graph/executor/admin/ShowQueriesExecutor.h index 267d15ddcca..d6361c67573 100644 --- a/src/graph/executor/admin/ShowQueriesExecutor.h +++ b/src/graph/executor/admin/ShowQueriesExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_SHOWQUERIESEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_SHOWQUERIESEXECUTOR_H_ diff --git a/src/graph/executor/admin/ShowServiceClientsExecutor.cpp b/src/graph/executor/admin/ShowServiceClientsExecutor.cpp index af9748e60d1..25f7425de20 100644 --- a/src/graph/executor/admin/ShowServiceClientsExecutor.cpp +++ b/src/graph/executor/admin/ShowServiceClientsExecutor.cpp @@ -1,13 +1,10 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/ShowServiceClientsExecutor.h" #include - -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" #include "graph/service/PermissionManager.h" #include "interface/gen-cpp2/meta_types.h" @@ -27,7 +24,7 @@ folly::Future ShowServiceClientsExecutor::showServiceClients() { return qctx()->getMetaClient()->listServiceClients(type).via(runner()).thenValue( [this](auto &&resp) { if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "Show service client fail: " << resp.status(); return resp.status(); } auto values = std::move(resp).value(); diff --git a/src/graph/executor/admin/ShowServiceClientsExecutor.h b/src/graph/executor/admin/ShowServiceClientsExecutor.h index e524954c9d3..ea4d77066cb 100644 --- a/src/graph/executor/admin/ShowServiceClientsExecutor.h +++ b/src/graph/executor/admin/ShowServiceClientsExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_SHOW_SERVICE_CLIENTS_EXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_SHOW_SERVICE_CLIENTS_EXECUTOR_H_ diff --git a/src/graph/executor/admin/ShowStatsExecutor.cpp b/src/graph/executor/admin/ShowStatsExecutor.cpp index aaaccbc2799..d9c0715c8eb 100644 --- a/src/graph/executor/admin/ShowStatsExecutor.cpp +++ b/src/graph/executor/admin/ShowStatsExecutor.cpp @@ -1,12 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/ShowStatsExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" #include "graph/service/PermissionManager.h" #include "graph/util/SchemaUtil.h" @@ -21,7 +17,7 @@ folly::Future ShowStatsExecutor::execute() { return qctx()->getMetaClient()->getStats(spaceId).via(runner()).thenValue( [this, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Show status failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Show status failed: " << resp.status(); return resp.status(); } auto statsItem = std::move(resp).value(); diff --git a/src/graph/executor/admin/ShowStatsExecutor.h b/src/graph/executor/admin/ShowStatsExecutor.h index 1e6e3bcd1d6..0b76f922f79 100644 --- a/src/graph/executor/admin/ShowStatsExecutor.h +++ b/src/graph/executor/admin/ShowStatsExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_SHOWSTATUSEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_SHOWSTATUSEXECUTOR_H_ diff --git a/src/graph/executor/admin/SignInServiceExecutor.cpp b/src/graph/executor/admin/SignInServiceExecutor.cpp index f83bb068314..34dfbd86d45 100644 --- a/src/graph/executor/admin/SignInServiceExecutor.cpp +++ b/src/graph/executor/admin/SignInServiceExecutor.cpp @@ -1,10 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/SignInServiceExecutor.h" - #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/SignInServiceExecutor.h b/src/graph/executor/admin/SignInServiceExecutor.h index 83d4fd22ed9..c02ffc4306b 100644 --- a/src/graph/executor/admin/SignInServiceExecutor.h +++ b/src/graph/executor/admin/SignInServiceExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_SIGNINSERVICEEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_SIGNINSERVICEEXECUTOR_H_ diff --git a/src/graph/executor/admin/SignOutServiceExecutor.cpp b/src/graph/executor/admin/SignOutServiceExecutor.cpp index 3c6f81c8969..8ecefd7cbcf 100644 --- a/src/graph/executor/admin/SignOutServiceExecutor.cpp +++ b/src/graph/executor/admin/SignOutServiceExecutor.cpp @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/SignOutServiceExecutor.h" diff --git a/src/graph/executor/admin/SignOutServiceExecutor.h b/src/graph/executor/admin/SignOutServiceExecutor.h index 1e1ded6de35..e1521d6b6d4 100644 --- a/src/graph/executor/admin/SignOutServiceExecutor.h +++ b/src/graph/executor/admin/SignOutServiceExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_SIGNOUTTSSERVICEEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_SIGNOUTTSSERVICEEXECUTOR_H_ diff --git a/src/graph/executor/admin/SnapshotExecutor.cpp b/src/graph/executor/admin/SnapshotExecutor.cpp index 43f0d6869b3..2e7f81e4466 100644 --- a/src/graph/executor/admin/SnapshotExecutor.cpp +++ b/src/graph/executor/admin/SnapshotExecutor.cpp @@ -1,14 +1,9 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/SnapshotExecutor.h" - #include - -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" namespace nebula { @@ -19,7 +14,7 @@ folly::Future CreateSnapshotExecutor::execute() { return qctx()->getMetaClient()->createSnapshot().via(runner()).thenValue([](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "Create snapshot fail: " << resp.status(); return resp.status(); } return Status::OK(); @@ -36,7 +31,7 @@ folly::Future DropSnapshotExecutor::execute() { .via(runner()) .thenValue([](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "Drop snapshot fail: " << resp.status(); return resp.status(); } return Status::OK(); @@ -49,7 +44,7 @@ folly::Future ShowSnapshotsExecutor::execute() { return qctx()->getMetaClient()->listSnapshots().via(runner()).thenValue( [this](StatusOr> resp) { if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "Show snapshot fail: " << resp.status(); return resp.status(); } diff --git a/src/graph/executor/admin/SnapshotExecutor.h b/src/graph/executor/admin/SnapshotExecutor.h index c3ef848c52b..b78188f472d 100644 --- a/src/graph/executor/admin/SnapshotExecutor.h +++ b/src/graph/executor/admin/SnapshotExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_SNAPSHOTEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_SNAPSHOTEXECUTOR_H_ diff --git a/src/graph/executor/admin/SpaceExecutor.cpp b/src/graph/executor/admin/SpaceExecutor.cpp index d08fca3b7e8..6028d6e899b 100644 --- a/src/graph/executor/admin/SpaceExecutor.cpp +++ b/src/graph/executor/admin/SpaceExecutor.cpp @@ -1,13 +1,10 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/SpaceExecutor.h" #include "common/stats/StatsManager.h" -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" #include "graph/service/PermissionManager.h" #include "graph/stats/GraphStats.h" @@ -26,7 +23,7 @@ folly::Future CreateSpaceExecutor::execute() { .via(runner()) .thenValue([](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "Create space fail: " << resp.status(); return resp.status(); } return Status::OK(); @@ -45,7 +42,7 @@ folly::Future CreateSpaceAsExecutor::execute() { .via(runner()) .thenValue([](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "Create space as node fail: " << resp.status(); return resp.status(); } return Status::OK(); @@ -56,13 +53,11 @@ folly::Future DescSpaceExecutor::execute() { SCOPED_TIMER(&execTime_); auto *dsNode = asNode(node()); - return qctx() - ->getMetaClient() - ->getSpace(dsNode->getSpaceName()) - .via(runner()) - .thenValue([this](StatusOr resp) { + const auto &spaceName = dsNode->getSpaceName(); + return qctx()->getMetaClient()->getSpace(spaceName).via(runner()).thenValue( + [this, spaceName](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "Desc space: " << spaceName << " fail: " << resp.status(); return resp.status(); } auto &spaceItem = resp.value(); @@ -139,7 +134,7 @@ folly::Future DropSpaceExecutor::execute() { .via(runner()) .thenValue([this, dsNode, spaceIdRet, ftIndexes](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "Drop space `" << dsNode->getSpaceName() << "' failed: " << resp.status(); + LOG(WARNING) << "Drop space `" << dsNode->getSpaceName() << "' failed: " << resp.status(); return resp.status(); } unRegisterSpaceLevelMetrics(dsNode->getSpaceName()); @@ -152,7 +147,7 @@ folly::Future DropSpaceExecutor::execute() { if (!ftIndexes.empty()) { auto tsRet = FTIndexUtils::getTSClients(qctx()->getMetaClient()); if (!tsRet.ok()) { - LOG(WARNING) << "Get text search clients failed"; + LOG(WARNING) << "Get text search clients failed: " << tsRet.status(); return Status::OK(); } for (const auto &ftindex : ftIndexes) { @@ -200,7 +195,8 @@ folly::Future ClearSpaceExecutor::execute() { .via(runner()) .thenValue([this, csNode, spaceIdRet, ftIndexes = std::move(ftIndexes)](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "Clear space `" << csNode->getSpaceName() << "' failed: " << resp.status(); + LOG(WARNING) << "Clear space `" << csNode->getSpaceName() + << "' failed: " << resp.status(); return resp.status(); } if (!ftIndexes.empty()) { @@ -226,7 +222,7 @@ folly::Future ShowSpacesExecutor::execute() { return qctx()->getMetaClient()->listSpaces().via(runner()).thenValue( [this](StatusOr> resp) { if (!resp.ok()) { - LOG(ERROR) << "Show spaces failed: " << resp.status(); + LOG(WARNING) << "Show spaces failed: " << resp.status(); return resp.status(); } auto spaceItems = std::move(resp).value(); @@ -261,8 +257,8 @@ folly::Future ShowCreateSpaceExecutor::execute() { .via(runner()) .thenValue([this, scsNode](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "Show create space `" << scsNode->getSpaceName() - << "' failed: " << resp.status(); + LOG(WARNING) << "Show create space `" << scsNode->getSpaceName() + << "' failed: " << resp.status(); return resp.status(); } auto properties = resp.value().get_properties(); @@ -318,14 +314,15 @@ folly::Future ShowCreateSpaceExecutor::execute() { folly::Future AlterSpaceExecutor::execute() { SCOPED_TIMER(&execTime_); auto *asnode = asNode(node()); + const auto &spaceName = asnode->getSpaceName(); return qctx() ->getMetaClient() ->alterSpace(asnode->getSpaceName(), asnode->getAlterSpaceOp(), asnode->getParas()) .via(runner()) - .thenValue([this](StatusOr &&resp) { + .thenValue([this, spaceName](StatusOr &&resp) { SCOPED_TIMER(&execTime_); if (!resp.ok()) { - LOG(ERROR) << resp.status().toString(); + LOG(WARNING) << "Alter space : " << spaceName << " fail: " << resp.status(); return std::move(resp).status(); } return Status::OK(); diff --git a/src/graph/executor/admin/SpaceExecutor.h b/src/graph/executor/admin/SpaceExecutor.h index 98552765b76..ad7efc2a457 100644 --- a/src/graph/executor/admin/SpaceExecutor.h +++ b/src/graph/executor/admin/SpaceExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_SPACESEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_SPACESEXECUTOR_H_ diff --git a/src/graph/executor/admin/SubmitJobExecutor.cpp b/src/graph/executor/admin/SubmitJobExecutor.cpp index de4e353ad9d..b3df72ee3cf 100644 --- a/src/graph/executor/admin/SubmitJobExecutor.cpp +++ b/src/graph/executor/admin/SubmitJobExecutor.cpp @@ -1,15 +1,12 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/SubmitJobExecutor.h" #include -#include "common/time/ScopedTimer.h" #include "common/time/TimeUtils.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" namespace nebula { @@ -28,11 +25,11 @@ folly::Future SubmitJobExecutor::execute() { ->getMetaClient() ->submitJob(spaceId, jobOp, jobType, params) .via(runner()) - .thenValue([jobOp, this](StatusOr &&resp) { + .thenValue([jobOp, spaceId, this](StatusOr &&resp) { SCOPED_TIMER(&execTime_); if (!resp.ok()) { - LOG(ERROR) << resp.status().toString(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Submit job fail: " << resp.status(); return std::move(resp).status(); } auto status = buildResult(jobOp, std::move(resp).value()); diff --git a/src/graph/executor/admin/SubmitJobExecutor.h b/src/graph/executor/admin/SubmitJobExecutor.h index d299c9a7b0a..34c9a9704bf 100644 --- a/src/graph/executor/admin/SubmitJobExecutor.h +++ b/src/graph/executor/admin/SubmitJobExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_SUBMIT_JOB_EXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_SUBMIT_JOB_EXECUTOR_H_ diff --git a/src/graph/executor/admin/SwitchSpaceExecutor.cpp b/src/graph/executor/admin/SwitchSpaceExecutor.cpp index df7217d8f60..d777742ab7e 100644 --- a/src/graph/executor/admin/SwitchSpaceExecutor.cpp +++ b/src/graph/executor/admin/SwitchSpaceExecutor.cpp @@ -1,13 +1,10 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/SwitchSpaceExecutor.h" #include "clients/meta/MetaClient.h" -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Query.h" #include "graph/service/PermissionManager.h" #include "interface/gen-cpp2/meta_types.h" @@ -23,7 +20,7 @@ folly::Future SwitchSpaceExecutor::execute() { return qctx()->getMetaClient()->getSpace(spaceName).via(runner()).thenValue( [spaceName, this](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "Switch space :`" << spaceName << "' fail: " << resp.status(); return resp.status(); } diff --git a/src/graph/executor/admin/SwitchSpaceExecutor.h b/src/graph/executor/admin/SwitchSpaceExecutor.h index 633d03e9127..9d14c78d2bf 100644 --- a/src/graph/executor/admin/SwitchSpaceExecutor.h +++ b/src/graph/executor/admin/SwitchSpaceExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_SWITCHSPACEEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_SWITCHSPACEEXECUTOR_H_ diff --git a/src/graph/executor/admin/UpdateUserExecutor.cpp b/src/graph/executor/admin/UpdateUserExecutor.cpp index 6d1027c4c19..499fb2b98fc 100644 --- a/src/graph/executor/admin/UpdateUserExecutor.cpp +++ b/src/graph/executor/admin/UpdateUserExecutor.cpp @@ -1,12 +1,11 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. + #include "graph/executor/admin/UpdateUserExecutor.h" #include -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/UpdateUserExecutor.h b/src/graph/executor/admin/UpdateUserExecutor.h index a7e85bfef5b..7dfb98f3fbc 100644 --- a/src/graph/executor/admin/UpdateUserExecutor.h +++ b/src/graph/executor/admin/UpdateUserExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_UPDATEUSEREXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_UPDATEUSEREXECUTOR_H_ diff --git a/src/graph/executor/admin/ZoneExecutor.cpp b/src/graph/executor/admin/ZoneExecutor.cpp index f33909a7efd..cf5933a1211 100644 --- a/src/graph/executor/admin/ZoneExecutor.cpp +++ b/src/graph/executor/admin/ZoneExecutor.cpp @@ -1,11 +1,9 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/ZoneExecutor.h" -#include "common/time/ScopedTimer.h" #include "graph/planner/plan/Admin.h" namespace nebula { @@ -20,7 +18,7 @@ folly::Future MergeZoneExecutor::execute() { .via(runner()) .thenValue([](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "Merge Zone Failed :" << resp.status(); + LOG(WARNING) << "Merge Zone Failed :" << resp.status(); return resp.status(); } return Status::OK(); @@ -36,7 +34,7 @@ folly::Future RenameZoneExecutor::execute() { .via(runner()) .thenValue([](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "Rename Zone Failed :" << resp.status(); + LOG(WARNING) << "Rename Zone Failed :" << resp.status(); return resp.status(); } return Status::OK(); @@ -52,7 +50,7 @@ folly::Future DropZoneExecutor::execute() { .via(runner()) .thenValue([](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "Drop Zone Failed :" << resp.status(); + LOG(WARNING) << "Drop Zone Failed :" << resp.status(); return resp.status(); } return Status::OK(); @@ -68,7 +66,7 @@ folly::Future DivideZoneExecutor::execute() { .via(runner()) .thenValue([](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "Split Zone Failed :" << resp.status(); + LOG(WARNING) << "Split Zone Failed :" << resp.status(); return resp.status(); } return Status::OK(); @@ -84,7 +82,7 @@ folly::Future DescribeZoneExecutor::execute() { .via(runner()) .thenValue([this](StatusOr> resp) { if (!resp.ok()) { - LOG(ERROR) << "Describe Zone Failed: " << resp.status(); + LOG(WARNING) << "Describe Zone Failed: " << resp.status(); return resp.status(); } @@ -111,7 +109,7 @@ folly::Future AddHostsIntoZoneExecutor::execute() { .via(runner()) .thenValue([](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "Add Host Into Zone Failed: " << resp.status(); + LOG(WARNING) << "Add Host Into Zone Failed: " << resp.status(); return resp.status(); } return Status::OK(); @@ -123,7 +121,7 @@ folly::Future ListZonesExecutor::execute() { return qctx()->getMetaClient()->listZones().via(runner()).thenValue( [this](StatusOr> resp) { if (!resp.ok()) { - LOG(ERROR) << "List Zones Failed: " << resp.status(); + LOG(WARNING) << "List Zones Failed: " << resp.status(); return resp.status(); } diff --git a/src/graph/executor/admin/ZoneExecutor.h b/src/graph/executor/admin/ZoneExecutor.h index d742ab2ba03..4fe722bc455 100644 --- a/src/graph/executor/admin/ZoneExecutor.h +++ b/src/graph/executor/admin/ZoneExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_ZONEEXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_ZONEEXECUTOR_H_ diff --git a/src/graph/executor/algo/CartesianProductExecutor.cpp b/src/graph/executor/algo/CartesianProductExecutor.cpp index fe2aa79fbf2..125653dc6b0 100644 --- a/src/graph/executor/algo/CartesianProductExecutor.cpp +++ b/src/graph/executor/algo/CartesianProductExecutor.cpp @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/algo/CartesianProductExecutor.h" @@ -33,7 +32,6 @@ folly::Future CartesianProductExecutor::execute() { std::make_move_iterator(cols.begin()), std::make_move_iterator(cols.end())); } - VLOG(1) << "Cartesian Product is : " << result; return finish(ResultBuilder().value(Value(std::move(result))).build()); } @@ -66,7 +64,6 @@ folly::Future BiCartesianProductExecutor::execute() { DataSet result; doCartesianProduct(lds, rds, result); result.colNames = BiCP->colNames(); - VLOG(1) << "Cartesian Product is : " << result; return finish(ResultBuilder().value(Value(std::move(result))).build()); } } // namespace graph diff --git a/src/graph/executor/algo/CartesianProductExecutor.h b/src/graph/executor/algo/CartesianProductExecutor.h index afd3c7ece08..6042642d214 100644 --- a/src/graph/executor/algo/CartesianProductExecutor.h +++ b/src/graph/executor/algo/CartesianProductExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ALGO_CARTESIANPRODUCTEXECUTOR_H_ #define GRAPH_EXECUTOR_ALGO_CARTESIANPRODUCTEXECUTOR_H_ diff --git a/src/graph/executor/algo/SubgraphExecutor.cpp b/src/graph/executor/algo/SubgraphExecutor.cpp index 8b98b6f9202..29db544272e 100644 --- a/src/graph/executor/algo/SubgraphExecutor.cpp +++ b/src/graph/executor/algo/SubgraphExecutor.cpp @@ -1,7 +1,6 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/algo/SubgraphExecutor.h" @@ -20,10 +19,8 @@ folly::Future SubgraphExecutor::execute() { const auto& currentStepVal = ectx_->getValue(subgraph->currentStepVar()); DCHECK(currentStepVal.isInt()); auto currentStep = currentStepVal.getInt(); - VLOG(1) << "Current Step is: " << currentStep << " Total Steps is: " << steps; auto resultVar = subgraph->resultVar(); - VLOG(1) << "input: " << subgraph->inputVar() << " output: " << node()->outputVar(); auto iter = ectx_->getResult(subgraph->inputVar()).iter(); auto gnSize = iter->size(); diff --git a/src/graph/executor/algo/SubgraphExecutor.h b/src/graph/executor/algo/SubgraphExecutor.h index 0bd388c76f2..25353bddfaa 100644 --- a/src/graph/executor/algo/SubgraphExecutor.h +++ b/src/graph/executor/algo/SubgraphExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ALGO_SUBGRAPHEXECUTOR_H_ #define GRAPH_EXECUTOR_ALGO_SUBGRAPHEXECUTOR_H_ diff --git a/src/graph/executor/logic/ArgumentExecutor.cpp b/src/graph/executor/logic/ArgumentExecutor.cpp index 5aa86ff8cba..0c1cd69c533 100644 --- a/src/graph/executor/logic/ArgumentExecutor.cpp +++ b/src/graph/executor/logic/ArgumentExecutor.cpp @@ -1,10 +1,8 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/logic/ArgumentExecutor.h" - #include "graph/planner/plan/Logic.h" namespace nebula { diff --git a/src/graph/executor/logic/ArgumentExecutor.h b/src/graph/executor/logic/ArgumentExecutor.h index 7c6e7eea828..89edbb8431b 100644 --- a/src/graph/executor/logic/ArgumentExecutor.h +++ b/src/graph/executor/logic/ArgumentExecutor.h @@ -1,13 +1,15 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_LOGIC_ARGUMENTEXECUTOR_H #define GRAPH_EXECUTOR_LOGIC_ARGUMENTEXECUTOR_H #include "graph/executor/Executor.h" - +// only used in match scenarios +// indicates the variable to be used as an argument to the right-hand side of an Apply operator. +// E.g MATCH (n)-[]-(l), (l)-[]-(m) return n,l,m +// MATCH (n)-[]-(l) MATCH (l)-[]-(m) return n,l,m namespace nebula { namespace graph { class ArgumentExecutor final : public Executor { diff --git a/src/graph/executor/logic/LoopExecutor.cpp b/src/graph/executor/logic/LoopExecutor.cpp index c61977c2e94..b07687d3b41 100644 --- a/src/graph/executor/logic/LoopExecutor.cpp +++ b/src/graph/executor/logic/LoopExecutor.cpp @@ -1,19 +1,10 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/logic/LoopExecutor.h" - -#include - -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryExpressionContext.h" #include "graph/planner/plan/Logic.h" -#include "graph/planner/plan/Query.h" -#include "interface/gen-cpp2/common_types.h" -using folly::stringPrintf; namespace nebula { namespace graph { @@ -29,7 +20,6 @@ folly::Future LoopExecutor::execute() { QueryExpressionContext ctx(ectx_); auto value = expr->eval(ctx); - VLOG(1) << "Loop condition: " << expr->toString() << " val: " << value; DCHECK(value.isBool()); finally_ = !(value.isBool() && value.getBool()); return finish(ResultBuilder().value(std::move(value)).iter(Iterator::Kind::kDefault).build()); diff --git a/src/graph/executor/logic/LoopExecutor.h b/src/graph/executor/logic/LoopExecutor.h index 7345e2dd6ec..3dfa88a2f6a 100644 --- a/src/graph/executor/logic/LoopExecutor.h +++ b/src/graph/executor/logic/LoopExecutor.h @@ -1,13 +1,13 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_LOGIC_LOOPEXECUTOR_H_ #define GRAPH_EXECUTOR_LOGIC_LOOPEXECUTOR_H_ #include "graph/executor/Executor.h" - +// like for logic in c language +// when the finally_ is true, execute the loopBody operator, otherwise terminate the loop namespace nebula { namespace graph { diff --git a/src/graph/executor/logic/PassThroughExecutor.cpp b/src/graph/executor/logic/PassThroughExecutor.cpp index e380fbe977a..28101a54770 100644 --- a/src/graph/executor/logic/PassThroughExecutor.cpp +++ b/src/graph/executor/logic/PassThroughExecutor.cpp @@ -1,16 +1,13 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/logic/PassThroughExecutor.h" -#include "common/time/ScopedTimer.h" -#include "graph/planner/plan/Query.h" +#include "graph/planner/plan/PlanNode.h" namespace nebula { namespace graph { - folly::Future PassThroughExecutor::execute() { SCOPED_TIMER(&execTime_); diff --git a/src/graph/executor/logic/PassThroughExecutor.h b/src/graph/executor/logic/PassThroughExecutor.h index 32b90a0ae33..ebc236dc4c3 100644 --- a/src/graph/executor/logic/PassThroughExecutor.h +++ b/src/graph/executor/logic/PassThroughExecutor.h @@ -1,13 +1,12 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_LOGIC_PASSTHROUGHEXECUTOR_H_ #define GRAPH_EXECUTOR_LOGIC_PASSTHROUGHEXECUTOR_H_ #include "graph/executor/Executor.h" - +// merge two execution streams into one. do nothing namespace nebula { namespace graph { diff --git a/src/graph/executor/logic/SelectExecutor.cpp b/src/graph/executor/logic/SelectExecutor.cpp index 110d0e80825..5fd0810311d 100644 --- a/src/graph/executor/logic/SelectExecutor.cpp +++ b/src/graph/executor/logic/SelectExecutor.cpp @@ -1,14 +1,9 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/logic/SelectExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryExpressionContext.h" #include "graph/planner/plan/Logic.h" -#include "graph/planner/plan/Query.h" namespace nebula { namespace graph { diff --git a/src/graph/executor/logic/SelectExecutor.h b/src/graph/executor/logic/SelectExecutor.h index 0707646b27c..422965a0419 100644 --- a/src/graph/executor/logic/SelectExecutor.h +++ b/src/graph/executor/logic/SelectExecutor.h @@ -1,13 +1,13 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_LOGIC_SELECTEXECUTOR_H_ #define GRAPH_EXECUTOR_LOGIC_SELECTEXECUTOR_H_ #include "graph/executor/Executor.h" - +// like IF ELSE logic in c language +// when the condition_ is true, execute the then operator, otherwise execute the else operator namespace nebula { namespace graph { diff --git a/src/graph/executor/logic/StartExecutor.cpp b/src/graph/executor/logic/StartExecutor.cpp index 62effcf90c4..f668b6737f1 100644 --- a/src/graph/executor/logic/StartExecutor.cpp +++ b/src/graph/executor/logic/StartExecutor.cpp @@ -1,13 +1,9 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/logic/StartExecutor.h" -#include "common/time/ScopedTimer.h" -#include "graph/planner/plan/PlanNode.h" - namespace nebula { namespace graph { diff --git a/src/graph/executor/logic/StartExecutor.h b/src/graph/executor/logic/StartExecutor.h index 0c81e00b1aa..e1685a06b85 100644 --- a/src/graph/executor/logic/StartExecutor.h +++ b/src/graph/executor/logic/StartExecutor.h @@ -1,13 +1,12 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_LOGIC_STARTEXECUTOR_H_ #define GRAPH_EXECUTOR_LOGIC_STARTEXECUTOR_H_ #include "graph/executor/Executor.h" - +// the leaf node of the execution plan does not depend on other operators, do nothing namespace nebula { namespace graph { diff --git a/src/graph/executor/maintain/EdgeExecutor.cpp b/src/graph/executor/maintain/EdgeExecutor.cpp index 4ddd35dcd41..81ed6c1d8aa 100644 --- a/src/graph/executor/maintain/EdgeExecutor.cpp +++ b/src/graph/executor/maintain/EdgeExecutor.cpp @@ -1,12 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/maintain/EdgeExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Maintain.h" #include "graph/util/SchemaUtil.h" @@ -24,8 +20,8 @@ folly::Future CreateEdgeExecutor::execute() { .via(runner()) .thenValue([ceNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Create edge `" << ceNode->getName() - << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Create edge `" << ceNode->getName() + << "' failed: " << resp.status(); return resp.status(); } return Status::OK(); @@ -43,13 +39,14 @@ folly::Future DescEdgeExecutor::execute() { .via(runner()) .thenValue([this, deNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Desc edge `" << deNode->getName() + << "' failed: " << resp.status(); return resp.status(); } auto ret = SchemaUtil::toDescSchema(resp.value()); if (!ret.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Desc edge `" << deNode->getName() - << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Desc edge `" << deNode->getName() + << "' failed: " << resp.status(); return ret.status(); } return finish(ResultBuilder() @@ -70,8 +67,8 @@ folly::Future DropEdgeExecutor::execute() { .via(runner()) .thenValue([deNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Drop edge `" << deNode->getName() - << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Drop edge `" << deNode->getName() + << "' failed: " << resp.status(); return resp.status(); } return Status::OK(); @@ -85,7 +82,7 @@ folly::Future ShowEdgesExecutor::execute() { return qctx()->getMetaClient()->listEdgeSchemas(spaceId).via(runner()).thenValue( [this, spaceId](StatusOr> resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Show edges failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Show edges failed: " << resp.status(); return resp.status(); } auto edgeItems = std::move(resp).value(); @@ -119,13 +116,14 @@ folly::Future ShowCreateEdgeExecutor::execute() { .via(runner()) .thenValue([this, sceNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", ShowCreate edge `" << sceNode->getName() - << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", ShowCreate edge `" << sceNode->getName() + << "' failed: " << resp.status(); return resp.status(); } auto ret = SchemaUtil::toShowCreateSchema(false, sceNode->getName(), resp.value()); if (!ret.ok()) { - LOG(ERROR) << ret.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", ShowCreate edge `" << sceNode->getName() + << "' failed: " << resp.status(); return ret.status(); } return finish( @@ -144,8 +142,8 @@ folly::Future AlterEdgeExecutor::execute() { .via(runner()) .thenValue([this, aeNode](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << aeNode->space() << ", Alter edge `" << aeNode->getName() - << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << aeNode->space() << ", Alter edge `" << aeNode->getName() + << "' failed: " << resp.status(); return resp.status(); } return finish(ResultBuilder().value(Value()).iter(Iterator::Kind::kDefault).build()); diff --git a/src/graph/executor/maintain/EdgeExecutor.h b/src/graph/executor/maintain/EdgeExecutor.h index bc6032d108d..4f5ebfab699 100644 --- a/src/graph/executor/maintain/EdgeExecutor.h +++ b/src/graph/executor/maintain/EdgeExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_MAINTAIN_EDGEEXECUTOR_H_ #define GRAPH_EXECUTOR_MAINTAIN_EDGEEXECUTOR_H_ diff --git a/src/graph/executor/maintain/EdgeIndexExecutor.cpp b/src/graph/executor/maintain/EdgeIndexExecutor.cpp index cfe1b85ba7d..818e7ed5f08 100644 --- a/src/graph/executor/maintain/EdgeIndexExecutor.cpp +++ b/src/graph/executor/maintain/EdgeIndexExecutor.cpp @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/maintain/EdgeIndexExecutor.h" @@ -28,8 +27,9 @@ folly::Future CreateEdgeIndexExecutor::execute() { .via(runner()) .thenValue([ceiNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Create index `" << ceiNode->getIndexName() - << "' at edge: `" << ceiNode->getSchemaName() << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Create index `" << ceiNode->getIndexName() + << "' at edge: `" << ceiNode->getSchemaName() + << "' failed: " << resp.status(); return resp.status(); } return Status::OK(); @@ -47,8 +47,8 @@ folly::Future DropEdgeIndexExecutor::execute() { .via(runner()) .thenValue([deiNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Drop edge index`" << deiNode->getIndexName() - << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Drop edge index`" << deiNode->getIndexName() + << "' failed: " << resp.status(); return resp.status(); } return Status::OK(); @@ -66,14 +66,15 @@ folly::Future DescEdgeIndexExecutor::execute() { .via(runner()) .thenValue([this, deiNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Desc edge index`" << deiNode->getIndexName() - << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Desc edge index`" << deiNode->getIndexName() + << "' failed: " << resp.status(); return resp.status(); } auto ret = IndexUtil::toDescIndex(resp.value()); if (!ret.ok()) { - LOG(ERROR) << ret.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Desc edge index`" << deiNode->getIndexName() + << "' failed: " << resp.status(); return ret.status(); } return finish( @@ -92,13 +93,14 @@ folly::Future ShowCreateEdgeIndexExecutor::execute() { .via(runner()) .thenValue([this, sceiNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Show create edge index `" - << sceiNode->getIndexName() << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Show create edge index `" + << sceiNode->getIndexName() << "' failed: " << resp.status(); return resp.status(); } auto ret = IndexUtil::toShowCreateIndex(false, sceiNode->getIndexName(), resp.value()); if (!ret.ok()) { - LOG(ERROR) << ret.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Show create edge index `" + << sceiNode->getIndexName() << "' failed: " << resp.status(); return ret.status(); } return finish( @@ -114,7 +116,7 @@ folly::Future ShowEdgeIndexesExecutor::execute() { return qctx()->getMetaClient()->listEdgeIndexes(spaceId).via(runner()).thenValue( [this, spaceId, bySchema](StatusOr> resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Show edge indexes failed" << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Show edge indexes failed" << resp.status(); return resp.status(); } @@ -166,8 +168,8 @@ folly::Future ShowEdgeIndexStatusExecutor::execute() { return qctx()->getMetaClient()->listEdgeIndexStatus(spaceId).via(runner()).thenValue( [this, spaceId](StatusOr> resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Show edge index status failed" - << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Show edge index status failed" + << resp.status(); return resp.status(); } diff --git a/src/graph/executor/maintain/EdgeIndexExecutor.h b/src/graph/executor/maintain/EdgeIndexExecutor.h index f477e0b6136..ad3c0f076c0 100644 --- a/src/graph/executor/maintain/EdgeIndexExecutor.h +++ b/src/graph/executor/maintain/EdgeIndexExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_MAINTAIN_EDGEINDEXEXECUTOR_H_ #define GRAPH_EXECUTOR_MAINTAIN_EDGEINDEXEXECUTOR_H_ diff --git a/src/graph/executor/maintain/FTIndexExecutor.cpp b/src/graph/executor/maintain/FTIndexExecutor.cpp index 27d4af5c900..78178142d14 100644 --- a/src/graph/executor/maintain/FTIndexExecutor.cpp +++ b/src/graph/executor/maintain/FTIndexExecutor.cpp @@ -1,11 +1,8 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/maintain/FTIndexExecutor.h" - -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Maintain.h" #include "graph/util/FTIndexUtils.h" #include "interface/gen-cpp2/meta_types.h" @@ -22,8 +19,8 @@ folly::Future CreateFTIndexExecutor::execute() { .via(runner()) .thenValue([inode](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "Create fulltext index `" << inode->getIndexName() << "' " - << "failed: " << resp.status(); + LOG(WARNING) << "Create fulltext index `" << inode->getIndexName() + << "' failed: " << resp.status(); return resp.status(); } return Status::OK(); @@ -39,13 +36,13 @@ folly::Future DropFTIndexExecutor::execute() { .via(runner()) .thenValue([this, inode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Drop fulltext index `" << inode->getName() - << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Drop fulltext index `" << inode->getName() + << "' failed: " << resp.status(); return resp.status(); } auto tsRet = FTIndexUtils::getTSClients(qctx()->getMetaClient()); if (!tsRet.ok()) { - LOG(WARNING) << "Get text search clients failed"; + LOG(WARNING) << "Get text search clients failed: " << tsRet.status(); } auto ftRet = FTIndexUtils::dropTSIndex(std::move(tsRet).value(), inode->getName()); if (!ftRet.ok()) { @@ -62,7 +59,8 @@ folly::Future ShowFTIndexesExecutor::execute() { return qctx()->getMetaClient()->listFTIndexes().via(runner()).thenValue( [this, spaceId](StatusOr> resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Show fulltext indexes failed" << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Show fulltext indexes failed" + << resp.status(); return resp.status(); } @@ -79,7 +77,8 @@ folly::Future ShowFTIndexesExecutor::execute() { isEdge ? this->qctx_->schemaMng()->toEdgeName(spaceId, shmId.get_edge_type()) : this->qctx_->schemaMng()->toTagName(spaceId, shmId.get_tag_id()); if (!shmNameRet.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Get schema name failed"; + LOG(WARNING) << "SpaceId: " << spaceId + << ", Get schema name failed: " << shmNameRet.status(); return shmNameRet.status(); } std::string fields; diff --git a/src/graph/executor/maintain/FTIndexExecutor.h b/src/graph/executor/maintain/FTIndexExecutor.h index 5d402f42690..cbe65faddf3 100644 --- a/src/graph/executor/maintain/FTIndexExecutor.h +++ b/src/graph/executor/maintain/FTIndexExecutor.h @@ -1,13 +1,13 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_ADMIN_SHOW_FT_INDEXES_EXECUTOR_H_ #define GRAPH_EXECUTOR_ADMIN_SHOW_FT_INDEXES_EXECUTOR_H_ #include "graph/executor/Executor.h" - +// full-text indexes are used to do prefix, wildcard, regexp, and fuzzy search on a string property. +// you can use the WHERE clause to specify the search strings in LOOKUP statements. namespace nebula { namespace graph { diff --git a/src/graph/executor/maintain/TagExecutor.cpp b/src/graph/executor/maintain/TagExecutor.cpp index fcb6485ad0a..2b3c842b227 100644 --- a/src/graph/executor/maintain/TagExecutor.cpp +++ b/src/graph/executor/maintain/TagExecutor.cpp @@ -1,12 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/maintain/TagExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Maintain.h" #include "graph/util/SchemaUtil.h" @@ -24,8 +20,8 @@ folly::Future CreateTagExecutor::execute() { .via(runner()) .thenValue([ctNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Create tag `" << ctNode->getName() - << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Create tag `" << ctNode->getName() + << "' failed: " << resp.status(); return resp.status(); } return Status::OK(); @@ -43,13 +39,14 @@ folly::Future DescTagExecutor::execute() { .via(runner()) .thenValue([this, dtNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Desc tag `" << dtNode->getName() - << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Desc tag `" << dtNode->getName() + << "' failed: " << resp.status(); return resp.status(); } auto ret = SchemaUtil::toDescSchema(resp.value()); if (!ret.ok()) { - LOG(ERROR) << ret.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Desc tag `" << dtNode->getName() + << "' failed: " << resp.status(); return ret.status(); } return finish( @@ -68,8 +65,8 @@ folly::Future DropTagExecutor::execute() { .via(runner()) .thenValue([dtNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Drop tag `" << dtNode->getName() - << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Drop tag `" << dtNode->getName() + << "' failed: " << resp.status(); return resp.status(); } return Status::OK(); @@ -83,7 +80,7 @@ folly::Future ShowTagsExecutor::execute() { return qctx()->getMetaClient()->listTagSchemas(spaceId).via(runner()).thenValue( [this, spaceId](StatusOr> resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Show tags failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Show tags failed: " << resp.status(); return resp.status(); } auto tagItems = std::move(resp).value(); @@ -117,13 +114,14 @@ folly::Future ShowCreateTagExecutor::execute() { .via(runner()) .thenValue([this, sctNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Show create tag `" << sctNode->getName() - << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Show create tag `" << sctNode->getName() + << "' failed: " << resp.status(); return resp.status(); } auto ret = SchemaUtil::toShowCreateSchema(true, sctNode->getName(), resp.value()); if (!ret.ok()) { - LOG(ERROR) << ret.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Show create tag `" << sctNode->getName() + << "' failed: " << resp.status(); return ret.status(); } return finish( @@ -142,8 +140,8 @@ folly::Future AlterTagExecutor::execute() { .via(runner()) .thenValue([aeNode](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << aeNode->space() << ", Alter tag `" << aeNode->getName() - << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << aeNode->space() << ", Alter tag `" << aeNode->getName() + << "' failed: " << resp.status(); return resp.status(); } return Status::OK(); diff --git a/src/graph/executor/maintain/TagExecutor.h b/src/graph/executor/maintain/TagExecutor.h index 7be41c8aed9..d1851a2a3a9 100644 --- a/src/graph/executor/maintain/TagExecutor.h +++ b/src/graph/executor/maintain/TagExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_MAINTAIN_TAGEXECUTOR_H_ #define GRAPH_EXECUTOR_MAINTAIN_TAGEXECUTOR_H_ diff --git a/src/graph/executor/maintain/TagIndexExecutor.cpp b/src/graph/executor/maintain/TagIndexExecutor.cpp index 86c53301193..b9687a895cb 100644 --- a/src/graph/executor/maintain/TagIndexExecutor.cpp +++ b/src/graph/executor/maintain/TagIndexExecutor.cpp @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/maintain/TagIndexExecutor.h" @@ -28,8 +27,9 @@ folly::Future CreateTagIndexExecutor::execute() { .via(runner()) .thenValue([ctiNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Create index `" << ctiNode->getIndexName() - << "' at tag: `" << ctiNode->getSchemaName() << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Create index `" << ctiNode->getIndexName() + << "' at tag: `" << ctiNode->getSchemaName() + << "' failed: " << resp.status(); return resp.status(); } return Status::OK(); @@ -47,8 +47,8 @@ folly::Future DropTagIndexExecutor::execute() { .via(runner()) .thenValue([dtiNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Drop tag index `" << dtiNode->getIndexName() - << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Drop tag index `" << dtiNode->getIndexName() + << "' failed: " << resp.status(); return resp.status(); } return Status::OK(); @@ -66,14 +66,15 @@ folly::Future DescTagIndexExecutor::execute() { .via(runner()) .thenValue([this, dtiNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Desc tag index `" << dtiNode->getIndexName() - << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Desc tag index `" << dtiNode->getIndexName() + << "' failed: " << resp.status(); return resp.status(); } auto ret = IndexUtil::toDescIndex(resp.value()); if (!ret.ok()) { - LOG(ERROR) << ret.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Desc tag index `" << dtiNode->getIndexName() + << "' failed: " << resp.status(); return ret.status(); } return finish( @@ -92,13 +93,14 @@ folly::Future ShowCreateTagIndexExecutor::execute() { .via(runner()) .thenValue([this, sctiNode, spaceId](StatusOr resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Show create tag index `" - << sctiNode->getIndexName() << "' failed: " << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Show create tag index `" + << sctiNode->getIndexName() << "' failed: " << resp.status(); return resp.status(); } auto ret = IndexUtil::toShowCreateIndex(true, sctiNode->getIndexName(), resp.value()); if (!ret.ok()) { - LOG(ERROR) << ret.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Show create tag index `" + << sctiNode->getIndexName() << "' failed: " << resp.status(); return ret.status(); } return finish( @@ -114,7 +116,7 @@ folly::Future ShowTagIndexesExecutor::execute() { return qctx()->getMetaClient()->listTagIndexes(spaceId).via(runner()).thenValue( [this, spaceId, bySchema](StatusOr> resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Show tag indexes failed" << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId << ", Show tag indexes failed" << resp.status(); return resp.status(); } @@ -167,7 +169,8 @@ folly::Future ShowTagIndexStatusExecutor::execute() { return qctx()->getMetaClient()->listTagIndexStatus(spaceId).via(runner()).thenValue( [this, spaceId](StatusOr> resp) { if (!resp.ok()) { - LOG(ERROR) << "SpaceId: " << spaceId << ", Show tag index status failed" << resp.status(); + LOG(WARNING) << "SpaceId: " << spaceId + << ", Show tag index status failed: " << resp.status(); return resp.status(); } diff --git a/src/graph/executor/maintain/TagIndexExecutor.h b/src/graph/executor/maintain/TagIndexExecutor.h index 5dc7e5c0565..c9456984f30 100644 --- a/src/graph/executor/maintain/TagIndexExecutor.h +++ b/src/graph/executor/maintain/TagIndexExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_MAINTAIN_TAGINDEXEXECUTOR_H_ #define GRAPH_EXECUTOR_MAINTAIN_TAGINDEXEXECUTOR_H_ diff --git a/src/graph/executor/mutate/DeleteExecutor.cpp b/src/graph/executor/mutate/DeleteExecutor.cpp index bcee155e38e..33341fd352d 100644 --- a/src/graph/executor/mutate/DeleteExecutor.cpp +++ b/src/graph/executor/mutate/DeleteExecutor.cpp @@ -1,12 +1,7 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. -#include "DeleteExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/executor/mutate/DeleteExecutor.h" #include "graph/planner/plan/Mutate.h" #include "graph/service/GraphFlags.h" @@ -38,7 +33,6 @@ folly::Future DeleteVerticesExecutor::deleteVertices() { inputVar = static_cast(gn)->inputVar(); } DCHECK(!inputVar.empty()); - VLOG(2) << "inputVar: " << inputVar; auto& inputResult = ectx_->getResult(inputVar); auto iter = inputResult.iter(); vertices.reserve(iter->size()); @@ -46,7 +40,6 @@ folly::Future DeleteVerticesExecutor::deleteVertices() { for (; iter->valid(); iter->next()) { auto val = Expression::eval(vidRef, ctx(iter.get())); if (val.isNull() || val.empty()) { - VLOG(3) << "NULL or EMPTY vid"; continue; } if (!SchemaUtil::isValidVid(val, *spaceInfo.spaceDesc.vid_type_ref())) { @@ -104,7 +97,6 @@ folly::Future DeleteTagsExecutor::deleteTags() { DCHECK(!iter->row()->empty()); auto val = Expression::eval(vidRef, ctx(iter.get())); if (val.isNull() || val.empty()) { - VLOG(3) << "NULL or EMPTY vid"; continue; } if (!SchemaUtil::isValidVid(val, *spaceInfo.spaceDesc.vid_type_ref())) { @@ -157,7 +149,6 @@ folly::Future DeleteEdgesExecutor::deleteEdges() { storage::cpp2::EdgeKey edgeKey; auto srcId = Expression::eval(edgeKeyRef->srcid(), ctx(iter.get())); if (srcId.isNull() || srcId.empty()) { - VLOG(3) << "NULL or EMPTY vid"; continue; } if (!SchemaUtil::isValidVid(srcId, *spaceInfo.spaceDesc.vid_type_ref())) { @@ -200,7 +191,6 @@ folly::Future DeleteEdgesExecutor::deleteEdges() { } if (edgeKeys.empty()) { - VLOG(2) << "Empty edgeKeys"; return Status::OK(); } diff --git a/src/graph/executor/mutate/DeleteExecutor.h b/src/graph/executor/mutate/DeleteExecutor.h index 36007e30402..891778d9c2c 100644 --- a/src/graph/executor/mutate/DeleteExecutor.h +++ b/src/graph/executor/mutate/DeleteExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_MUTATE_DELETEVERTICESEXECUTOR_H_ #define GRAPH_EXECUTOR_MUTATE_DELETEVERTICESEXECUTOR_H_ @@ -22,6 +21,7 @@ class DeleteVerticesExecutor final : public StorageAccessExecutor { folly::Future deleteVertices(); }; +// delete the specified tag at the specified vertex class DeleteTagsExecutor final : public StorageAccessExecutor { public: DeleteTagsExecutor(const PlanNode *node, QueryContext *qctx) diff --git a/src/graph/executor/mutate/InsertExecutor.cpp b/src/graph/executor/mutate/InsertExecutor.cpp index e926d497fe0..14b75e8aff3 100644 --- a/src/graph/executor/mutate/InsertExecutor.cpp +++ b/src/graph/executor/mutate/InsertExecutor.cpp @@ -1,12 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/mutate/InsertExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Mutate.h" #include "graph/service/GraphFlags.h" diff --git a/src/graph/executor/mutate/InsertExecutor.h b/src/graph/executor/mutate/InsertExecutor.h index 184cdeac1de..2694b81cb85 100644 --- a/src/graph/executor/mutate/InsertExecutor.h +++ b/src/graph/executor/mutate/InsertExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_MUTATE_INSERTVERTICESEXECUTOR_H_ #define GRAPH_EXECUTOR_MUTATE_INSERTVERTICESEXECUTOR_H_ diff --git a/src/graph/executor/mutate/UpdateExecutor.cpp b/src/graph/executor/mutate/UpdateExecutor.cpp index d151cc63ef7..63e774b22d0 100644 --- a/src/graph/executor/mutate/UpdateExecutor.cpp +++ b/src/graph/executor/mutate/UpdateExecutor.cpp @@ -1,12 +1,9 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. -#include "UpdateExecutor.h" +#include "graph/executor/mutate/UpdateExecutor.h" -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Mutate.h" #include "graph/service/GraphFlags.h" #include "graph/util/SchemaUtil.h" @@ -21,13 +18,12 @@ StatusOr UpdateBaseExecutor::handleResult(DataSet &&data) { if (yieldNames_.empty()) { return Status::OK(); } - LOG(ERROR) << "Empty return props"; return Status::Error("Empty return props"); } if (yieldNames_.size() != data.colNames.size() - 1) { - LOG(ERROR) << "Expect colName size is " << yieldNames_.size() << ", return colName size is " - << data.colNames.size() - 1; + LOG(WARNING) << "Expect colName size is " << yieldNames_.size() << ", return colName size is " + << data.colNames.size() - 1; return Status::Error("Wrong return prop size"); } DataSet result; @@ -67,7 +63,7 @@ folly::Future UpdateVertexExecutor::execute() { .thenValue([this](StatusOr resp) { SCOPED_TIMER(&execTime_); if (!resp.ok()) { - LOG(ERROR) << resp.status(); + LOG(WARNING) << "Update vertices fail: " << resp.status(); return resp.status(); } auto value = std::move(resp).value(); @@ -118,7 +114,7 @@ folly::Future UpdateEdgeExecutor::execute() { .thenValue([this](StatusOr resp) { SCOPED_TIMER(&execTime_); if (!resp.ok()) { - LOG(ERROR) << "Update edge failed: " << resp.status(); + LOG(WARNING) << "Update edge failed: " << resp.status(); return resp.status(); } auto value = std::move(resp).value(); diff --git a/src/graph/executor/mutate/UpdateExecutor.h b/src/graph/executor/mutate/UpdateExecutor.h index 728a85f83cc..9cb13e8d86b 100644 --- a/src/graph/executor/mutate/UpdateExecutor.h +++ b/src/graph/executor/mutate/UpdateExecutor.h @@ -1,12 +1,10 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_MUTATE_UPDATEEXECUTOR_H_ #define GRAPH_EXECUTOR_MUTATE_UPDATEEXECUTOR_H_ -#include "common/base/StatusOr.h" #include "graph/executor/StorageAccessExecutor.h" namespace nebula { diff --git a/src/graph/executor/query/AggregateExecutor.cpp b/src/graph/executor/query/AggregateExecutor.cpp index 42358baf28a..5b36b8a9f66 100644 --- a/src/graph/executor/query/AggregateExecutor.cpp +++ b/src/graph/executor/query/AggregateExecutor.cpp @@ -1,16 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/AggregateExecutor.h" - -#include "common/datatypes/List.h" -#include "common/expression/AggregateExpression.h" -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryExpressionContext.h" -#include "graph/context/Result.h" -#include "graph/planner/plan/PlanNode.h" #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/AggregateExecutor.h b/src/graph/executor/query/AggregateExecutor.h index df82eabbbea..b256d1fce06 100644 --- a/src/graph/executor/query/AggregateExecutor.h +++ b/src/graph/executor/query/AggregateExecutor.h @@ -1,13 +1,13 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_AGGREGATEEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_AGGREGATEEXECUTOR_H_ #include "graph/executor/Executor.h" - +// calculate a set of data uniformly. use values ​​from multiple records as input +// and convert those values ​​into one value to aggregate all records namespace nebula { namespace graph { diff --git a/src/graph/executor/query/AppendVerticesExecutor.cpp b/src/graph/executor/query/AppendVerticesExecutor.cpp index a092182a8c8..a7aa369ded4 100644 --- a/src/graph/executor/query/AppendVerticesExecutor.cpp +++ b/src/graph/executor/query/AppendVerticesExecutor.cpp @@ -1,7 +1,6 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2021 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/AppendVerticesExecutor.h" diff --git a/src/graph/executor/query/AppendVerticesExecutor.h b/src/graph/executor/query/AppendVerticesExecutor.h index 109b7a08fc2..b95cbd7671f 100644 --- a/src/graph/executor/query/AppendVerticesExecutor.h +++ b/src/graph/executor/query/AppendVerticesExecutor.h @@ -1,14 +1,16 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_APPENDVERTICESEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_APPENDVERTICESEXECUTOR_H_ #include "graph/executor/query/GetPropExecutor.h" #include "graph/planner/plan/Query.h" - +// only used in match scenarios +// due to the architecture design, the attributes of the destination point and the edge are +// not stored together, so the last step in the match statement needs to call this operator +// to obtain the attribute of the destination point namespace nebula { namespace graph { diff --git a/src/graph/executor/query/AssignExecutor.cpp b/src/graph/executor/query/AssignExecutor.cpp index 173f6c125d7..016fbbb59fe 100644 --- a/src/graph/executor/query/AssignExecutor.cpp +++ b/src/graph/executor/query/AssignExecutor.cpp @@ -1,10 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/AssignExecutor.h" - #include "graph/planner/plan/Query.h" namespace nebula { @@ -20,7 +18,6 @@ folly::Future AssignExecutor::execute() { auto varName = item.first; auto& valueExpr = item.second; auto value = valueExpr->eval(ctx); - VLOG(1) << "VarName is: " << varName << " value is : " << value; if (value.isDataSet()) { ectx_->setResult(varName, ResultBuilder().value(std::move(value)).build()); } else { diff --git a/src/graph/executor/query/AssignExecutor.h b/src/graph/executor/query/AssignExecutor.h index b21ac28a53a..9aae1996aad 100644 --- a/src/graph/executor/query/AssignExecutor.h +++ b/src/graph/executor/query/AssignExecutor.h @@ -1,13 +1,12 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_ASSIGNEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_ASSIGNEXECUTOR_H_ #include "graph/executor/Executor.h" - +// assign value to variable namespace nebula { namespace graph { diff --git a/src/graph/executor/query/DataCollectExecutor.cpp b/src/graph/executor/query/DataCollectExecutor.cpp index 7fefd9012a4..4edef430b23 100644 --- a/src/graph/executor/query/DataCollectExecutor.cpp +++ b/src/graph/executor/query/DataCollectExecutor.cpp @@ -1,11 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/DataCollectExecutor.h" - -#include "common/time/ScopedTimer.h" #include "graph/planner/plan/Query.h" namespace nebula { @@ -342,7 +339,6 @@ Status DataCollectExecutor::collectPathProp(const std::vector& vars } ds.rows.emplace_back(Row({std::move(path)})); } - VLOG(2) << "Path with props : \n" << ds; result_.setDataSet(std::move(ds)); return Status::OK(); } diff --git a/src/graph/executor/query/DataCollectExecutor.h b/src/graph/executor/query/DataCollectExecutor.h index fcc45ef5beb..c229721dec6 100644 --- a/src/graph/executor/query/DataCollectExecutor.h +++ b/src/graph/executor/query/DataCollectExecutor.h @@ -1,13 +1,31 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_DATACOLLECTEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_DATACOLLECTEXECUTOR_H_ #include "graph/executor/Executor.h" - +// DataCollect used to collect multiple versions of results(LOOP operator exist in execution plan) +// OR used after filter operator (the result of the filter operator has no dataset but an iterator) +// +// Member: +// `colNames_` : save the column name of the result of the DataCollect +// Funcitons: +// `collectSubgraph` : receive result from GetNeighbors, collect vertices & edges by calling +// GetNeighborIter's getVertices & getEdges interface +// +// `rowBaseMove` : Collect the latest version of the results +// +// `collectMToN` : Only used in GO MToN scenarios. Collect the results generated by MTON +// steps (multi-version) +// +// `collectAllPaths` : Only used in FindPath scenarios. Collect the paths generated by the FindPath +// operator (multi-version) +// +// `collectPathProp` : Only used in FindPath scenarios, the input is a collection of vertices and +// edges with attributes and paths without attributes. then populate the attributes of vertices +// and edges into paths namespace nebula { namespace graph { class DataCollectExecutor final : public Executor { diff --git a/src/graph/executor/query/DedupExecutor.cpp b/src/graph/executor/query/DedupExecutor.cpp index cb52c38935f..4a5f4ed9e7c 100644 --- a/src/graph/executor/query/DedupExecutor.cpp +++ b/src/graph/executor/query/DedupExecutor.cpp @@ -1,14 +1,10 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/DedupExecutor.h" -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryExpressionContext.h" #include "graph/planner/plan/Query.h" - namespace nebula { namespace graph { folly::Future DedupExecutor::execute() { @@ -23,9 +19,7 @@ folly::Future DedupExecutor::execute() { } if (UNLIKELY(iter->isGetNeighborsIter() || iter->isDefaultIter())) { - auto e = Status::Error("Invalid iterator kind, %d", static_cast(iter->kind())); - LOG(ERROR) << e; - return e; + return Status::Error("Invalid iterator kind, %d", static_cast(iter->kind())); } std::unordered_set unique; unique.reserve(iter->size()); diff --git a/src/graph/executor/query/DedupExecutor.h b/src/graph/executor/query/DedupExecutor.h index 9ebb3fb9248..ca2c477f0f6 100644 --- a/src/graph/executor/query/DedupExecutor.h +++ b/src/graph/executor/query/DedupExecutor.h @@ -1,13 +1,13 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_DEDUPEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_DEDUPEXECUTOR_H_ #include "graph/executor/Executor.h" - +// delete the corresponding iterator, when there are duplicate rows in the dataset. +// and then save the filtered iterator to the result namespace nebula { namespace graph { diff --git a/src/graph/executor/query/FilterExecutor.cpp b/src/graph/executor/query/FilterExecutor.cpp index 3d67bcb5c6d..87d9775c9bb 100644 --- a/src/graph/executor/query/FilterExecutor.cpp +++ b/src/graph/executor/query/FilterExecutor.cpp @@ -1,12 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/FilterExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryExpressionContext.h" #include "graph/planner/plan/Query.h" namespace nebula { @@ -23,16 +19,12 @@ folly::Future FilterExecutor::execute() { return status; } - VLOG(2) << "Get input var: " << filter->inputVar() - << ", iterator type: " << static_cast(iter->kind()); - ResultBuilder builder; builder.value(result.valuePtr()); QueryExpressionContext ctx(ectx_); auto condition = filter->condition(); while (iter->valid()) { auto val = condition->eval(ctx(iter)); - DLOG(ERROR) << "DEBUG POINT: filter val: " << val; if (val.isBadNull() || (!val.empty() && !val.isImplicitBool() && !val.isNull())) { return Status::Error("Wrong type result, the type should be NULL, EMPTY, BOOL"); } diff --git a/src/graph/executor/query/FilterExecutor.h b/src/graph/executor/query/FilterExecutor.h index cfa9b963864..9772e07d5aa 100644 --- a/src/graph/executor/query/FilterExecutor.h +++ b/src/graph/executor/query/FilterExecutor.h @@ -1,13 +1,14 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_FILTEREXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_FILTEREXECUTOR_H_ #include "graph/executor/Executor.h" +// delete the corresponding iterator when the row in the dataset does not meet the conditions +// and save the filtered iterator to the result namespace nebula { namespace graph { diff --git a/src/graph/executor/query/GetEdgesExecutor.cpp b/src/graph/executor/query/GetEdgesExecutor.cpp index 157a56ab92a..1972d0f1fd4 100644 --- a/src/graph/executor/query/GetEdgesExecutor.cpp +++ b/src/graph/executor/query/GetEdgesExecutor.cpp @@ -1,14 +1,9 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/GetEdgesExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Query.h" -#include "graph/util/SchemaUtil.h" using nebula::storage::StorageClient; using nebula::storage::StorageRpcResponse; @@ -23,7 +18,6 @@ folly::Future GetEdgesExecutor::execute() { DataSet GetEdgesExecutor::buildRequestDataSet(const GetEdges *ge) { auto valueIter = ectx_->getResult(ge->inputVar()).iter(); - VLOG(1) << "GE input var:" << ge->inputVar() << " iter kind: " << valueIter->kind(); QueryExpressionContext exprCtx(qctx()->ectx()); nebula::DataSet edges({kSrc, kType, kRank, kDst}); @@ -41,7 +35,6 @@ DataSet GetEdgesExecutor::buildRequestDataSet(const GetEdges *ge) { continue; } if (!rank.isInt()) { - LOG(WARNING) << "wrong rank type"; continue; } edges.emplace_back(Row({std::move(src), type, rank, std::move(dst)})); diff --git a/src/graph/executor/query/GetEdgesExecutor.h b/src/graph/executor/query/GetEdgesExecutor.h index 39b57f03167..39802c21f43 100644 --- a/src/graph/executor/query/GetEdgesExecutor.h +++ b/src/graph/executor/query/GetEdgesExecutor.h @@ -1,13 +1,13 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_GETEDGESEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_GETEDGESEXECUTOR_H_ #include "graph/executor/query/GetPropExecutor.h" +// get user-specified edge attributes namespace nebula { namespace graph { class GetEdges; diff --git a/src/graph/executor/query/GetNeighborsExecutor.cpp b/src/graph/executor/query/GetNeighborsExecutor.cpp index c315ce53910..d2aff232875 100644 --- a/src/graph/executor/query/GetNeighborsExecutor.cpp +++ b/src/graph/executor/query/GetNeighborsExecutor.cpp @@ -1,17 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/GetNeighborsExecutor.h" - -#include - -#include "clients/storage/StorageClient.h" -#include "common/datatypes/List.h" -#include "common/datatypes/Vertex.h" -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/service/GraphFlags.h" using nebula::storage::StorageClient; @@ -24,7 +15,6 @@ namespace graph { DataSet GetNeighborsExecutor::buildRequestDataSet() { SCOPED_TIMER(&execTime_); auto inputVar = gn_->inputVar(); - VLOG(1) << node()->outputVar() << " : " << inputVar; auto iter = ectx_->getResult(inputVar).iter(); return buildRequestDataSetByVidType(iter.get(), gn_->src(), gn_->dedup()); } @@ -99,7 +89,6 @@ Status GetNeighborsExecutor::handleResponse(RpcResponse& resps) { for (auto& resp : responses) { auto dataset = resp.get_vertices(); if (dataset == nullptr) { - LOG(INFO) << "Empty dataset in response"; continue; } diff --git a/src/graph/executor/query/GetNeighborsExecutor.h b/src/graph/executor/query/GetNeighborsExecutor.h index bc1818df645..e7d79786e59 100644 --- a/src/graph/executor/query/GetNeighborsExecutor.h +++ b/src/graph/executor/query/GetNeighborsExecutor.h @@ -1,18 +1,17 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_GETNEIGHBORSEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_GETNEIGHBORSEXECUTOR_H_ #include "graph/executor/StorageAccessExecutor.h" #include "graph/planner/plan/Query.h" -#include "interface/gen-cpp2/storage_types.h" +// get the attributes of the start point and edges specified by the user from the storage layer +// the returned result is a list, the format of each dataset refers to the storage.thrift namespace nebula { namespace graph { - class GetNeighborsExecutor final : public StorageAccessExecutor { public: GetNeighborsExecutor(const PlanNode* node, QueryContext* qctx) diff --git a/src/graph/executor/query/GetPropExecutor.h b/src/graph/executor/query/GetPropExecutor.h index 313e3f2bef2..c266539c891 100644 --- a/src/graph/executor/query/GetPropExecutor.h +++ b/src/graph/executor/query/GetPropExecutor.h @@ -30,7 +30,7 @@ class GetPropExecutor : public StorageAccessExecutor { if (resp.props_ref().has_value()) { if (UNLIKELY(!v.append(std::move(*resp.props_ref())))) { // it's impossible according to the interface - LOG(WARNING) << "Heterogeneous props dataset"; + LOG(ERROR) << "Heterogeneous props dataset"; state = Result::State::kPartialSuccess; } } else { diff --git a/src/graph/executor/query/GetVerticesExecutor.cpp b/src/graph/executor/query/GetVerticesExecutor.cpp index 0b519945632..6a71c737e67 100644 --- a/src/graph/executor/query/GetVerticesExecutor.cpp +++ b/src/graph/executor/query/GetVerticesExecutor.cpp @@ -1,14 +1,9 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/GetVerticesExecutor.h" -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" -#include "graph/util/SchemaUtil.h" - using nebula::storage::StorageClient; using nebula::storage::StorageRpcResponse; using nebula::storage::cpp2::GetPropResponse; @@ -67,7 +62,6 @@ DataSet GetVerticesExecutor::buildRequestDataSet(const GetVertices *gv) { // Accept Table such as | $a | $b | $c |... as input which one column indicate // src auto valueIter = ectx_->getResult(gv->inputVar()).iter(); - VLOG(3) << "GV input var: " << gv->inputVar() << " iter kind: " << valueIter->kind(); return buildRequestDataSetByVidType(valueIter.get(), gv->src(), gv->dedup()); } diff --git a/src/graph/executor/query/GetVerticesExecutor.h b/src/graph/executor/query/GetVerticesExecutor.h index 2bd1f08410b..24d2c249039 100644 --- a/src/graph/executor/query/GetVerticesExecutor.h +++ b/src/graph/executor/query/GetVerticesExecutor.h @@ -1,14 +1,13 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_GETVERTICESEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_GETVERTICESEXECUTOR_H_ #include "graph/executor/query/GetPropExecutor.h" #include "graph/planner/plan/Query.h" - +// get user-specified vertices attributes namespace nebula { namespace graph { diff --git a/src/graph/executor/query/IndexScanExecutor.cpp b/src/graph/executor/query/IndexScanExecutor.cpp index 70c5bd00831..39631767b83 100644 --- a/src/graph/executor/query/IndexScanExecutor.cpp +++ b/src/graph/executor/query/IndexScanExecutor.cpp @@ -1,14 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/IndexScanExecutor.h" - -#include - -#include "graph/context/QueryContext.h" -#include "graph/planner/plan/PlanNode.h" #include "graph/service/GraphFlags.h" using nebula::storage::StorageClient; diff --git a/src/graph/executor/query/IndexScanExecutor.h b/src/graph/executor/query/IndexScanExecutor.h index 4d5e489ec20..3a095c8a599 100644 --- a/src/graph/executor/query/IndexScanExecutor.h +++ b/src/graph/executor/query/IndexScanExecutor.h @@ -1,16 +1,14 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_INDEXSCANEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_INDEXSCANEXECUTOR_H_ -#include "clients/storage/StorageClient.h" #include "graph/executor/StorageAccessExecutor.h" #include "graph/planner/plan/Query.h" -#include "interface/gen-cpp2/storage_types.h" - +// used in lookup and match scenarios. +// fetch data from storage layer, according to the index selected by the optimizer. namespace nebula { namespace graph { diff --git a/src/graph/executor/query/InnerJoinExecutor.cpp b/src/graph/executor/query/InnerJoinExecutor.cpp index 928525eef7b..acebeabb3ce 100644 --- a/src/graph/executor/query/InnerJoinExecutor.cpp +++ b/src/graph/executor/query/InnerJoinExecutor.cpp @@ -1,13 +1,8 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/InnerJoinExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/context/Iterator.h" -#include "graph/context/QueryExpressionContext.h" #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/InnerJoinExecutor.h b/src/graph/executor/query/InnerJoinExecutor.h index 7d914bfd807..c205c83be06 100644 --- a/src/graph/executor/query/InnerJoinExecutor.h +++ b/src/graph/executor/query/InnerJoinExecutor.h @@ -1,13 +1,11 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_INNERJOINEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_INNERJOINEXECUTOR_H_ #include "graph/executor/query/JoinExecutor.h" - namespace nebula { namespace graph { @@ -43,10 +41,8 @@ class InnerJoinExecutor : public JoinExecutor { bool exchange_{false}; }; -/** - * No diffrence with inner join in processing data, but the dependencies would be executed in - * paralell. - */ +// No diffrence with inner join in processing data, but the dependencies would be executed in +// paralell. class BiInnerJoinExecutor final : public InnerJoinExecutor { public: BiInnerJoinExecutor(const PlanNode* node, QueryContext* qctx); diff --git a/src/graph/executor/query/IntersectExecutor.cpp b/src/graph/executor/query/IntersectExecutor.cpp index 7d006ff84ac..5e2ae7fbe66 100644 --- a/src/graph/executor/query/IntersectExecutor.cpp +++ b/src/graph/executor/query/IntersectExecutor.cpp @@ -1,14 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/IntersectExecutor.h" - -#include - -#include "common/time/ScopedTimer.h" -#include "graph/planner/plan/PlanNode.h" #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/IntersectExecutor.h b/src/graph/executor/query/IntersectExecutor.h index d15d36457d8..56dbee184ef 100644 --- a/src/graph/executor/query/IntersectExecutor.h +++ b/src/graph/executor/query/IntersectExecutor.h @@ -1,13 +1,12 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_INTERSECTEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_INTERSECTEXECUTOR_H_ #include "graph/executor/query/SetExecutor.h" - +// get the intersection of left & right sets namespace nebula { namespace graph { diff --git a/src/graph/executor/query/JoinExecutor.cpp b/src/graph/executor/query/JoinExecutor.cpp index a5bad010ead..c3d73cc487e 100644 --- a/src/graph/executor/query/JoinExecutor.cpp +++ b/src/graph/executor/query/JoinExecutor.cpp @@ -1,12 +1,8 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/JoinExecutor.h" - -#include "graph/context/Iterator.h" -#include "graph/context/QueryExpressionContext.h" #include "graph/planner/plan/Query.h" namespace nebula { @@ -16,7 +12,6 @@ Status JoinExecutor::checkInputDataSets() { auto* join = asNode(node()); lhsIter_ = ectx_->getVersionedResult(join->leftVar().first, join->leftVar().second).iter(); DCHECK(!!lhsIter_); - VLOG(1) << "lhs: " << join->leftVar().first << " " << lhsIter_->size(); if (lhsIter_->isGetNeighborsIter() || lhsIter_->isDefaultIter()) { std::stringstream ss; ss << "Join executor does not support " << lhsIter_->kind(); @@ -24,7 +19,6 @@ Status JoinExecutor::checkInputDataSets() { } rhsIter_ = ectx_->getVersionedResult(join->rightVar().first, join->rightVar().second).iter(); DCHECK(!!rhsIter_); - VLOG(1) << "rhs: " << join->rightVar().first << " " << rhsIter_->size(); if (rhsIter_->isGetNeighborsIter() || rhsIter_->isDefaultIter()) { std::stringstream ss; ss << "Join executor does not support " << rhsIter_->kind(); @@ -38,7 +32,6 @@ Status JoinExecutor::checkBiInputDataSets() { auto* join = asNode(node()); lhsIter_ = ectx_->getResult(join->leftInputVar()).iter(); DCHECK(!!lhsIter_); - VLOG(1) << "lhs: " << join->leftInputVar() << " " << lhsIter_->size(); if (lhsIter_->isGetNeighborsIter() || lhsIter_->isDefaultIter()) { std::stringstream ss; ss << "Join executor does not support " << lhsIter_->kind(); @@ -46,7 +39,6 @@ Status JoinExecutor::checkBiInputDataSets() { } rhsIter_ = ectx_->getResult(join->rightInputVar()).iter(); DCHECK(!!rhsIter_); - VLOG(1) << "rhs: " << join->rightInputVar() << " " << rhsIter_->size(); if (rhsIter_->isGetNeighborsIter() || rhsIter_->isDefaultIter()) { std::stringstream ss; ss << "Join executor does not support " << rhsIter_->kind(); diff --git a/src/graph/executor/query/JoinExecutor.h b/src/graph/executor/query/JoinExecutor.h index d0c1e1bbe90..1df26db54d9 100644 --- a/src/graph/executor/query/JoinExecutor.h +++ b/src/graph/executor/query/JoinExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_JOINEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_JOINEXECUTOR_H_ diff --git a/src/graph/executor/query/LeftJoinExecutor.cpp b/src/graph/executor/query/LeftJoinExecutor.cpp index 6a8725ad076..06725b00a62 100644 --- a/src/graph/executor/query/LeftJoinExecutor.cpp +++ b/src/graph/executor/query/LeftJoinExecutor.cpp @@ -1,13 +1,8 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/LeftJoinExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/context/Iterator.h" -#include "graph/context/QueryExpressionContext.h" #include "graph/planner/plan/Query.h" namespace nebula { @@ -48,7 +43,6 @@ folly::Future LeftJoinExecutor::join(const std::vector& has } result.colNames = colNames; - VLOG(2) << node_->toString() << ", result: " << result; return finish(ResultBuilder().value(Value(std::move(result))).build()); } diff --git a/src/graph/executor/query/LeftJoinExecutor.h b/src/graph/executor/query/LeftJoinExecutor.h index 224b7cca6c9..e444bd77a10 100644 --- a/src/graph/executor/query/LeftJoinExecutor.h +++ b/src/graph/executor/query/LeftJoinExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_LEFTJOINEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_LEFTJOINEXECUTOR_H_ @@ -42,10 +41,8 @@ class LeftJoinExecutor : public JoinExecutor { size_t rightColSize_{0}; }; -/** - * No diffrence with left join in processing data, but the dependencies would be executed in - * paralell. - */ +// No diffrence with left join in processing data, but the dependencies would be executed in +// paralell. class BiLeftJoinExecutor final : public LeftJoinExecutor { public: BiLeftJoinExecutor(const PlanNode* node, QueryContext* qctx); diff --git a/src/graph/executor/query/LimitExecutor.cpp b/src/graph/executor/query/LimitExecutor.cpp index 61a13c42e43..83937194400 100644 --- a/src/graph/executor/query/LimitExecutor.cpp +++ b/src/graph/executor/query/LimitExecutor.cpp @@ -1,11 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/LimitExecutor.h" - -#include "common/time/ScopedTimer.h" #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/LimitExecutor.h b/src/graph/executor/query/LimitExecutor.h index b605e1bae8c..b7495a9faf7 100644 --- a/src/graph/executor/query/LimitExecutor.h +++ b/src/graph/executor/query/LimitExecutor.h @@ -1,13 +1,11 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_LIMITEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_LIMITEXECUTOR_H_ #include "graph/executor/Executor.h" - +// takes iterators of data with user-specified limits and save them to the result namespace nebula { namespace graph { diff --git a/src/graph/executor/query/MinusExecutor.cpp b/src/graph/executor/query/MinusExecutor.cpp index b362a8a98a3..d2b4e600f80 100644 --- a/src/graph/executor/query/MinusExecutor.cpp +++ b/src/graph/executor/query/MinusExecutor.cpp @@ -1,13 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/MinusExecutor.h" - -#include - -#include "common/time/ScopedTimer.h" #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/MinusExecutor.h b/src/graph/executor/query/MinusExecutor.h index 669a4d00794..db04b77a51e 100644 --- a/src/graph/executor/query/MinusExecutor.h +++ b/src/graph/executor/query/MinusExecutor.h @@ -1,13 +1,12 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_MINUSEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_MINUSEXECUTOR_H_ #include "graph/executor/query/SetExecutor.h" - +// leftSet - rightSet namespace nebula { namespace graph { diff --git a/src/graph/executor/query/ProjectExecutor.cpp b/src/graph/executor/query/ProjectExecutor.cpp index 5734f1b1efe..bae734c8d49 100644 --- a/src/graph/executor/query/ProjectExecutor.cpp +++ b/src/graph/executor/query/ProjectExecutor.cpp @@ -1,14 +1,9 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/ProjectExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryExpressionContext.h" #include "graph/planner/plan/Query.h" -#include "parser/Clauses.h" namespace nebula { namespace graph { @@ -21,7 +16,6 @@ folly::Future ProjectExecutor::execute() { DCHECK(!!iter); QueryExpressionContext ctx(ectx_); - VLOG(1) << "input: " << project->inputVar(); DataSet ds; ds.colNames = project->colNames(); ds.rows.reserve(!iter->isGetNeighborsIter() ? iter->size() : 0); @@ -33,7 +27,6 @@ folly::Future ProjectExecutor::execute() { } ds.rows.emplace_back(std::move(row)); } - VLOG(1) << node()->outputVar() << ":" << ds; return finish(ResultBuilder().value(Value(std::move(ds))).build()); } diff --git a/src/graph/executor/query/ProjectExecutor.h b/src/graph/executor/query/ProjectExecutor.h index 28ddf6d4353..08b36070aa2 100644 --- a/src/graph/executor/query/ProjectExecutor.h +++ b/src/graph/executor/query/ProjectExecutor.h @@ -1,13 +1,12 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_PROJECTEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_PROJECTEXECUTOR_H_ #include "graph/executor/Executor.h" - +// select user-specified columns from a table namespace nebula { namespace graph { diff --git a/src/graph/executor/query/RollUpApplyExecutor.cpp b/src/graph/executor/query/RollUpApplyExecutor.cpp index 98802e252ec..2d651ef9039 100644 --- a/src/graph/executor/query/RollUpApplyExecutor.cpp +++ b/src/graph/executor/query/RollUpApplyExecutor.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2022 vesoft inc. All rights reserved. +/* Copyright (c) 2020 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License. */ @@ -21,7 +21,6 @@ Status RollUpApplyExecutor::checkBiInputDataSets() { auto* rollUpApply = asNode(node()); lhsIter_ = ectx_->getResult(rollUpApply->leftInputVar()).iter(); DCHECK(!!lhsIter_); - VLOG(1) << "lhs: " << rollUpApply->leftInputVar() << " " << lhsIter_->size(); if (lhsIter_->isGetNeighborsIter() || lhsIter_->isDefaultIter()) { std::stringstream ss; ss << "RollUpApply executor does not support " << lhsIter_->kind(); @@ -29,7 +28,6 @@ Status RollUpApplyExecutor::checkBiInputDataSets() { } rhsIter_ = ectx_->getResult(rollUpApply->rightInputVar()).iter(); DCHECK(!!rhsIter_); - VLOG(1) << "rhs: " << rollUpApply->rightInputVar() << " " << rhsIter_->size(); if (rhsIter_->isGetNeighborsIter() || rhsIter_->isDefaultIter()) { std::stringstream ss; ss << "RollUpApply executor does not support " << rhsIter_->kind(); @@ -161,7 +159,6 @@ folly::Future RollUpApplyExecutor::rollUpApply() { result = probe(rollUpApplyNode->compareCols(), lhsIter_.get(), hashTable); } result.colNames = rollUpApplyNode->colNames(); - DLOG(ERROR) << "DEBUG POINT result of roll up apply: " << result; return finish(ResultBuilder().value(Value(std::move(result))).build()); } diff --git a/src/graph/executor/query/RollUpApplyExecutor.h b/src/graph/executor/query/RollUpApplyExecutor.h index a9175270b4d..470746b90cd 100644 --- a/src/graph/executor/query/RollUpApplyExecutor.h +++ b/src/graph/executor/query/RollUpApplyExecutor.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2022 vesoft inc. All rights reserved. +/* Copyright (c) 2020 vesoft inc. All rights reserved. * * This source code is licensed under Apache 2.0 License. */ diff --git a/src/graph/executor/query/SampleExecutor.cpp b/src/graph/executor/query/SampleExecutor.cpp index 6625107bdaa..f955f2d4371 100644 --- a/src/graph/executor/query/SampleExecutor.cpp +++ b/src/graph/executor/query/SampleExecutor.cpp @@ -1,11 +1,8 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/SampleExecutor.h" - -#include "common/time/ScopedTimer.h" #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/SampleExecutor.h b/src/graph/executor/query/SampleExecutor.h index 3eef2569d3c..20e84a0e820 100644 --- a/src/graph/executor/query/SampleExecutor.h +++ b/src/graph/executor/query/SampleExecutor.h @@ -1,13 +1,12 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_SAMPLEEXECUTOR_H #define GRAPH_EXECUTOR_QUERY_SAMPLEEXECUTOR_H #include "graph/executor/Executor.h" - +// invoke the sampling algorithm to select the specified amount of data namespace nebula { namespace graph { diff --git a/src/graph/executor/query/ScanEdgesExecutor.cpp b/src/graph/executor/query/ScanEdgesExecutor.cpp index 7f07759a549..aa4a50201b3 100644 --- a/src/graph/executor/query/ScanEdgesExecutor.cpp +++ b/src/graph/executor/query/ScanEdgesExecutor.cpp @@ -1,12 +1,8 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/ScanEdgesExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/planner/plan/Query.h" #include "graph/util/SchemaUtil.h" diff --git a/src/graph/executor/query/ScanEdgesExecutor.h b/src/graph/executor/query/ScanEdgesExecutor.h index c2385182e29..fbae33dce39 100644 --- a/src/graph/executor/query/ScanEdgesExecutor.h +++ b/src/graph/executor/query/ScanEdgesExecutor.h @@ -1,10 +1,9 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/GetPropExecutor.h" - +// full table scan edge, but must have limit parameter namespace nebula { namespace graph { class ScanEdgesExecutor final : public GetPropExecutor { diff --git a/src/graph/executor/query/ScanVerticesExecutor.cpp b/src/graph/executor/query/ScanVerticesExecutor.cpp index 490990c29a2..ab66625ba8c 100644 --- a/src/graph/executor/query/ScanVerticesExecutor.cpp +++ b/src/graph/executor/query/ScanVerticesExecutor.cpp @@ -1,12 +1,10 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/ScanVerticesExecutor.h" -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" +#include "graph/planner/plan/Query.h" #include "graph/util/SchemaUtil.h" using nebula::storage::StorageClient; diff --git a/src/graph/executor/query/ScanVerticesExecutor.h b/src/graph/executor/query/ScanVerticesExecutor.h index bbe6df30cd3..2e4c54411b3 100644 --- a/src/graph/executor/query/ScanVerticesExecutor.h +++ b/src/graph/executor/query/ScanVerticesExecutor.h @@ -1,14 +1,12 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_SCANVERTICESEXECUTOR_H #define GRAPH_EXECUTOR_QUERY_SCANVERTICESEXECUTOR_H #include "graph/executor/query/GetPropExecutor.h" -#include "graph/planner/plan/Query.h" - +// full table scan vertices, but must have limit parameter namespace nebula { namespace graph { diff --git a/src/graph/executor/query/SetExecutor.cpp b/src/graph/executor/query/SetExecutor.cpp index 5b5eb716c73..e0d7276087a 100644 --- a/src/graph/executor/query/SetExecutor.cpp +++ b/src/graph/executor/query/SetExecutor.cpp @@ -1,14 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/SetExecutor.h" - -#include - -#include "common/datatypes/DataSet.h" -#include "graph/context/ExecutionContext.h" #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/SetExecutor.h b/src/graph/executor/query/SetExecutor.h index 2f6ae21ba53..b0648f9b1cc 100644 --- a/src/graph/executor/query/SetExecutor.h +++ b/src/graph/executor/query/SetExecutor.h @@ -1,21 +1,13 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_SETEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_SETEXECUTOR_H_ - -#include - #include "graph/executor/Executor.h" - namespace nebula { - class Iterator; - namespace graph { - class SetExecutor : public Executor { public: Status checkInputDataSets(); diff --git a/src/graph/executor/query/SortExecutor.cpp b/src/graph/executor/query/SortExecutor.cpp index a8cc53bfd35..0e9086d8027 100644 --- a/src/graph/executor/query/SortExecutor.cpp +++ b/src/graph/executor/query/SortExecutor.cpp @@ -1,11 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/SortExecutor.h" - -#include "common/time/ScopedTimer.h" #include "graph/planner/plan/Query.h" namespace nebula { @@ -23,7 +20,6 @@ folly::Future SortExecutor::execute() { if (UNLIKELY(!iter->isSequentialIter())) { std::stringstream ss; ss << "Internal error: Sort executor does not supported " << iter->kind(); - LOG(ERROR) << ss.str(); return Status::Error(ss.str()); } diff --git a/src/graph/executor/query/SortExecutor.h b/src/graph/executor/query/SortExecutor.h index f0c3767e248..29b96fbad33 100644 --- a/src/graph/executor/query/SortExecutor.h +++ b/src/graph/executor/query/SortExecutor.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_SORTEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_SORTEXECUTOR_H_ @@ -10,7 +9,6 @@ namespace nebula { namespace graph { - class SortExecutor final : public Executor { public: SortExecutor(const PlanNode *node, QueryContext *qctx) : Executor("SortExecutor", node, qctx) {} diff --git a/src/graph/executor/query/TopNExecutor.cpp b/src/graph/executor/query/TopNExecutor.cpp index b2cc15d422d..150765d49f9 100644 --- a/src/graph/executor/query/TopNExecutor.cpp +++ b/src/graph/executor/query/TopNExecutor.cpp @@ -1,11 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/TopNExecutor.h" - -#include "common/time/ScopedTimer.h" #include "graph/planner/plan/Query.h" namespace nebula { @@ -22,7 +19,6 @@ folly::Future TopNExecutor::execute() { if (UNLIKELY(!result.iter()->isSequentialIter())) { std::stringstream ss; ss << "Internal error: Sort executor does not supported " << iter->kind(); - LOG(ERROR) << ss.str(); return Status::Error(ss.str()); } diff --git a/src/graph/executor/query/TopNExecutor.h b/src/graph/executor/query/TopNExecutor.h index 22024846617..64b604535e4 100644 --- a/src/graph/executor/query/TopNExecutor.h +++ b/src/graph/executor/query/TopNExecutor.h @@ -1,13 +1,11 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_TOPNEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_TOPNEXECUTOR_H_ #include "graph/executor/Executor.h" - namespace nebula { namespace graph { diff --git a/src/graph/executor/query/TraverseExecutor.cpp b/src/graph/executor/query/TraverseExecutor.cpp index 2a1ba9e3f35..2460012dbf0 100644 --- a/src/graph/executor/query/TraverseExecutor.cpp +++ b/src/graph/executor/query/TraverseExecutor.cpp @@ -1,17 +1,9 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/TraverseExecutor.h" - -#include - #include "clients/storage/StorageClient.h" -#include "common/datatypes/List.h" -#include "common/datatypes/Vertex.h" -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryContext.h" #include "graph/service/GraphFlags.h" #include "graph/util/SchemaUtil.h" @@ -76,7 +68,6 @@ Status TraverseExecutor::buildRequestDataSet() { folly::Future TraverseExecutor::traverse() { if (reqDs_.rows.empty()) { - VLOG(1) << "Empty input."; DataSet emptyResult; return finish(ResultBuilder().value(Value(std::move(emptyResult))).build()); } @@ -150,7 +141,6 @@ folly::Future TraverseExecutor::handleResponse(RpcResponse&& resps) { for (auto& resp : responses) { auto dataset = resp.get_vertices(); if (dataset == nullptr) { - LOG(INFO) << "Empty dataset in response"; continue; } list.values.emplace_back(std::move(*dataset)); diff --git a/src/graph/executor/query/TraverseExecutor.h b/src/graph/executor/query/TraverseExecutor.h index 25ec9704ea9..49e59fe7eb5 100644 --- a/src/graph/executor/query/TraverseExecutor.h +++ b/src/graph/executor/query/TraverseExecutor.h @@ -1,21 +1,34 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef EXECUTOR_QUERY_TRAVERSEEXECUTOR_H_ #define EXECUTOR_QUERY_TRAVERSEEXECUTOR_H_ -#include - -#include "clients/storage/StorageClient.h" -#include "common/base/StatusOr.h" -#include "common/datatypes/Value.h" -#include "common/datatypes/Vertex.h" #include "graph/executor/StorageAccessExecutor.h" #include "graph/planner/plan/Query.h" #include "interface/gen-cpp2/storage_types.h" - +// only used in match scenarios +// invoke the getNeighbors interface, according to the number of times specified by the user, +// and assemble the result into paths +// +// The definition of path is : array of vertex and edges +// Eg a->b->c. path is [Vertex(a), [Edge(a->b), Vertex(b), Edge(b->c), Vertex(c)]] +// the purpose is to extract the path by pathBuildExpression +// `resDs_` : keep result dataSet +// +// Member: +// `paths_` : hash table array, paths_[i] means that the length that paths in the i-th array +// element is i +// KEY in the hash table : the vid of the destination Vertex +// VALUE in the hash table : collection of paths that destionation vid is `KEY` +// +// Functions: +// `buildRequestDataSet` : constructs the input DataSet for getNeightbors +// `buildInterimPath` : construct collection of paths after expanded and put it into the paths_ +// `getNeighbors` : invoke the getNeightbors interface +// `releasePrevPaths` : deleted The path whose length does not meet the user-defined length +// `hasSameEdge` : check if there are duplicate edges in path namespace nebula { namespace graph { diff --git a/src/graph/executor/query/UnionAllVersionVarExecutor.cpp b/src/graph/executor/query/UnionAllVersionVarExecutor.cpp index 856ba363bd0..92557f31803 100644 --- a/src/graph/executor/query/UnionAllVersionVarExecutor.cpp +++ b/src/graph/executor/query/UnionAllVersionVarExecutor.cpp @@ -1,11 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/UnionAllVersionVarExecutor.h" - -#include "common/time/ScopedTimer.h" #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/UnionAllVersionVarExecutor.h b/src/graph/executor/query/UnionAllVersionVarExecutor.h index f8f69955e66..671ed85371f 100644 --- a/src/graph/executor/query/UnionAllVersionVarExecutor.h +++ b/src/graph/executor/query/UnionAllVersionVarExecutor.h @@ -1,13 +1,12 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_UNIONALLVERSIONVAREXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_UNIONALLVERSIONVAREXECUTOR_H_ #include "graph/executor/Executor.h" - +// leftSet + rightSet. column names must be the same namespace nebula { namespace graph { diff --git a/src/graph/executor/query/UnionExecutor.cpp b/src/graph/executor/query/UnionExecutor.cpp index ceb139d7b3c..b1731964c11 100644 --- a/src/graph/executor/query/UnionExecutor.cpp +++ b/src/graph/executor/query/UnionExecutor.cpp @@ -1,12 +1,8 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/UnionExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/context/ExecutionContext.h" #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/UnionExecutor.h b/src/graph/executor/query/UnionExecutor.h index d1c13673803..e586ea4a850 100644 --- a/src/graph/executor/query/UnionExecutor.h +++ b/src/graph/executor/query/UnionExecutor.h @@ -1,13 +1,13 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_UNIONEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_UNIONEXECUTOR_H_ #include "graph/executor/query/SetExecutor.h" - +// leftSet + rightSet. column names must be the same +// the result set is not duplicated namespace nebula { namespace graph { diff --git a/src/graph/executor/query/UnwindExecutor.cpp b/src/graph/executor/query/UnwindExecutor.cpp index dcb11c9507d..28fb8aa5718 100644 --- a/src/graph/executor/query/UnwindExecutor.cpp +++ b/src/graph/executor/query/UnwindExecutor.cpp @@ -1,14 +1,9 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #include "graph/executor/query/UnwindExecutor.h" - -#include "common/time/ScopedTimer.h" -#include "graph/context/QueryExpressionContext.h" #include "graph/planner/plan/Query.h" -#include "parser/Clauses.h" namespace nebula { namespace graph { @@ -37,7 +32,6 @@ folly::Future UnwindExecutor::execute() { ds.rows.emplace_back(std::move(row)); } } - VLOG(1) << "Unwind result is: " << ds; return finish(ResultBuilder().value(Value(std::move(ds))).build()); } diff --git a/src/graph/executor/query/UnwindExecutor.h b/src/graph/executor/query/UnwindExecutor.h index cdfb9ff669f..81b16f2a18b 100644 --- a/src/graph/executor/query/UnwindExecutor.h +++ b/src/graph/executor/query/UnwindExecutor.h @@ -1,13 +1,12 @@ -/* Copyright (c) 2020 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ +// Copyright (c) 2020 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. #ifndef GRAPH_EXECUTOR_QUERY_UNWINDEXECUTOR_H_ #define GRAPH_EXECUTOR_QUERY_UNWINDEXECUTOR_H_ #include "graph/executor/Executor.h" - +// expand multiple columns of data into one column namespace nebula { namespace graph { From 400deb3b0339597ec4e5e8b9d841e409de6bb10f Mon Sep 17 00:00:00 2001 From: Alex Xing <90179377+SuperYoko@users.noreply.github.com> Date: Thu, 31 Mar 2022 17:46:14 +0800 Subject: [PATCH 10/24] Add notice for nebula status (#4100) * Add notice for nebula status * Highlight and Uppercase Add hosts, add docs link for add host. * we found a compromise Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- scripts/nebula.service | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/nebula.service b/scripts/nebula.service index 7fea4a2f977..98c011b3c59 100755 --- a/scripts/nebula.service +++ b/scripts/nebula.service @@ -234,6 +234,10 @@ function status_daemon { port=${GREEN}${port}${NC} else port=${BLINK}${RED}${port}${NC} + if [[$daemon_name == nebula-storaged]]; then + WARN "${daemon_name} after v3.0.0 will not start service until it is added to cluster." + WARN "See Manage Storage hosts:${RED}ADD HOSTS${NC} in https://docs.nebula-graph.io/" + fi fi INFO "${daemon_name}(${daemon_version}): Running as $(cat ${pid_file}), Listening on ${port}" else From 9e7cd0644c3c8db18118ac1a0719cdbb72bfc532 Mon Sep 17 00:00:00 2001 From: Yee <2520865+yixinglu@users.noreply.github.com> Date: Thu, 31 Mar 2022 18:28:40 +0800 Subject: [PATCH 11/24] Trigger release workflow manually (#4105) * Cleanup nightly workflow * Fix pull request workflow * Fix tagname action * Fix gcc version * Semicolon * workflow dispatch * ignore secrets * Restore * format * Remove default value of version * Restore fastcov Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- .github/actions/tagname-action/action.yml | 14 +++++----- .github/workflows/nightly.yml | 33 +++++++++++------------ .github/workflows/pull_request.yml | 7 +++-- .github/workflows/rc.yml | 19 ++++++++----- .github/workflows/release.yml | 2 +- .gitignore | 1 + 6 files changed, 40 insertions(+), 36 deletions(-) diff --git a/.github/actions/tagname-action/action.yml b/.github/actions/tagname-action/action.yml index 27e74404236..4118adf3741 100644 --- a/.github/actions/tagname-action/action.yml +++ b/.github/actions/tagname-action/action.yml @@ -1,14 +1,14 @@ -name: 'Extract tag information' -description: 'Extract tag information' +name: "Extract tag information" +description: "Extract tag information" outputs: tag: - description: 'tag name' + description: "tag name" value: ${{ steps.tag.outputs.tag }} tagnum: - description: 'tag number' + description: "tag number" value: ${{ steps.tag.outputs.tagnum }} majorver: - description: 'major version' + description: "major version" value: ${{ steps.tag.outputs.majorver }} runs: using: "composite" @@ -16,8 +16,8 @@ runs: - id: tag run: | tag=$(echo ${{ github.ref }} | rev | cut -d/ -f1 | rev) - tagnum=$(echo $tag | sed 's/^v//') - majorver=$(echo $tag | cut -d '.' -f 1) + tagnum=$(echo $tag | sed "s/^v//") + majorver=$(echo $tag | cut -d "." -f 1) echo "::set-output name=tag::$tag" echo "::set-output name=tagnum::$tagnum" echo "::set-output name=majorver::$majorver" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 775b04c20a7..ab12140cb83 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -46,10 +46,10 @@ jobs: find pkg-build/cpack_output -type f \( -iname \*.deb -o -iname \*.rpm -o -iname \*.tar.gz \) -exec bash -c "sha256sum {} > {}.sha256sum.txt" \; subdir=$(date -u +%Y.%m.%d) echo "::set-output name=subdir::$subdir" - - uses: actions/upload-artifact@v1 - with: - name: ${{ matrix.os }}-nightly - path: pkg-build/cpack_output + # - uses: actions/upload-artifact@v1 + # with: + # name: ${{ matrix.os }}-nightly + # path: pkg-build/cpack_output - uses: ./.github/actions/upload-to-oss-action with: key-id: ${{ secrets.OSS_ID }} @@ -75,12 +75,12 @@ jobs: with: run: sh -c "find . -mindepth 1 -delete" - uses: actions/checkout@v2 - - uses: docker/setup-qemu-action@v1 - - uses: docker/setup-buildx-action@v1 - uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - uses: docker/setup-qemu-action@v1 + - uses: docker/setup-buildx-action@v1 - uses: docker/build-push-action@v2 with: context: . @@ -97,9 +97,9 @@ jobs: fail-fast: false matrix: os: - - ubuntu2004 + - centos7 compiler: - - gcc-9.2 + - gcc-9.3 container: image: vesoft/nebula-dev:${{ matrix.os }} env: @@ -114,26 +114,25 @@ jobs: run: sh -c "find . -mindepth 1 -delete" - uses: actions/checkout@v2 - name: Prepare environment - id: prepare run: | [ -d build/ ] && rm -rf build/* || mkdir -p build make init -C tests - name: CMake - id: cmake run: | cmake \ - -DCMAKE_CXX_COMPILER=$TOOLSET_CLANG_DIR/bin/g++ \ - -DCMAKE_C_COMPILER=$TOOLSET_CLANG_DIR/bin/gcc \ + -DCMAKE_CXX_COMPILER=$TOOLSET_GCC_DIR/bin/g++ \ + -DCMAKE_C_COMPILER=$TOOLSET_GCC_DIR/bin/gcc \ -DCMAKE_BUILD_TYPE=Debug \ -DENABLE_TESTING=on \ -DENABLE_COVERAGE=on \ + -GNinja -B build - echo "::set-output name=j::8" - name: Make run: | ccache -z - cmake --build build/ -j $(nproc) + ninja -j $(nproc) ccache -s + working-directory: build/ - name: CTest env: ASAN_OPTIONS: fast_unwind_on_malloc=1 @@ -147,12 +146,12 @@ jobs: timeout-minutes: 2 - name: Pytest run: | - make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} test + make RM_DIR=false DEBUG=false J=8 test working-directory: tests/ timeout-minutes: 15 - name: TCK run: | - make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} tck + make RM_DIR=false DEBUG=false J=8 tck working-directory: tests/ timeout-minutes: 60 - name: Down cluster @@ -162,7 +161,7 @@ jobs: timeout-minutes: 2 - name: coverage run: | - ~/.local/bin/fastcov -d build -l -o fastcov.info -p --exclude /usr/include --exclude=/opt/vesoft --exclude scanner.lex + ~/.local/bin/fastcov -d build -l -o fastcov.info -p --exclude /usr/include --exclude=/opt/vesoft --exclude scanner.lex - uses: codecov/codecov-action@v2 with: files: fastcov.info diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 16a41382348..448a46b5738 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -5,7 +5,7 @@ on: types: [synchronize, reopened, labeled] branches: - master - - 'v[0-9]+.*' + - 'release-**' concurrency: group: ${{ github.head_ref }} @@ -183,11 +183,11 @@ jobs: working-directory: tests/ timeout-minutes: 2 - name: coverage - if: ${{ matrix.compiler == 'gcc-9.2' && matrix.os == 'ubuntu2004' }} + if: ${{ matrix.compiler == 'gcc-9.3' && matrix.os == 'ubuntu2004' }} run: | ~/.local/bin/fastcov -d build -l -o fastcov.info -p --exclude /usr/include --exclude=/opt/vesoft --exclude scanner.lex - uses: codecov/codecov-action@v2 - if: ${{ matrix.compiler == 'gcc-9.2' && matrix.os == 'ubuntu2004' }} + if: ${{ matrix.compiler == 'gcc-9.3' && matrix.os == 'ubuntu2004' }} with: files: fastcov.info fail_ci_if_error: false @@ -227,7 +227,6 @@ jobs: run: sh -c "find . -mindepth 1 -delete" - uses: actions/checkout@v2 - name: Prepare environment - id: prepare run: | [ -d build/ ] && rm -rf build/* || mkdir -p build make init -C tests diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index e15a2406e80..f79a7a48798 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -1,9 +1,11 @@ name: rc on: - push: - branches: - - 'v[0-9]+.*' + workflow_dispatch: + inputs: + version: + description: "version such as '3.1.0'" + required: true concurrency: group: rc @@ -40,8 +42,8 @@ jobs: - uses: actions/setup-go@v2 with: go-version: '^1.16.7' - - uses: ./.github/actions/tagname-action - id: tag + - id: tag + run: echo ::set-output name=tagnum::${{ github.event.inputs.version }} - name: package run: ./package/package.sh -v ${{ steps.tag.outputs.tagnum }} -t RelWithDebInfo -r OFF -p ON -s TRUE -k ON - name: output some vars @@ -81,8 +83,11 @@ jobs: with: run: sh -c "find . -mindepth 1 -delete" - uses: actions/checkout@v2 - - uses: ./.github/actions/tagname-action - id: tagname + - id: tagname + run: | + echo ::set-output name=tag::"v${{ github.event.inputs.version }}" + echo ::set-output name=tagnum::"${{ github.event.inputs.version }}" + echo ::set-output name=majorver::"v$(echo ${{ github.event.inputs.version }} | cut -f1 -d'.')" - id: docker run: | majorver=$(git tag -l --sort=v:refname | tail -n1 | cut -f1 -d".") diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ff1cc187ec1..7bd28c4857b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,7 +51,7 @@ jobs: id: tagname - id: docker run: | - majorver=$(git tag -l --sort=v:refname | tail -n1 | cut -f1 -d'.') + majorver=$(git tag -l --sort=v:refname | tail -n1 | cut -f1 -d".") tag="" if [[ $majorver == ${{ steps.tagname.outputs.majorver }} ]]; then tag="latest" diff --git a/.gitignore b/.gitignore index b13cd6c5215..8890b286dbc 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ modules/ # tests !tests/Makefile +tests/secrets reformat-gherkin nebula-python From 65dd2f3e14a3b9297da112cb646672e550618a34 Mon Sep 17 00:00:00 2001 From: yaphet <4414314+darionyaphet@users.noreply.github.com> Date: Fri, 1 Apr 2022 10:56:04 +0800 Subject: [PATCH 12/24] fix memory leak (#4103) --- src/meta/test/JobManagerTest.cpp | 8 ++++---- src/parser/parser.yy | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/meta/test/JobManagerTest.cpp b/src/meta/test/JobManagerTest.cpp index e0e677739fe..4587e1758ed 100644 --- a/src/meta/test/JobManagerTest.cpp +++ b/src/meta/test/JobManagerTest.cpp @@ -137,8 +137,8 @@ TEST_F(JobManagerTest, DownloadJob) { EXPECT_CALL(adminClient, addTask(_, _, _, _, _, _, _)) .WillOnce(Return(ByMove(folly::makeFuture(Status::OK())))); - auto executor = - new DownloadJobExecutor(space, job.getJobId(), kv.get(), &adminClient, job.getParas()); + auto executor = std::make_unique( + space, job.getJobId(), kv.get(), &adminClient, job.getParas()); executor->helper_ = std::make_unique(); ASSERT_TRUE(executor->check()); @@ -162,8 +162,8 @@ TEST_F(JobManagerTest, IngestJob) { MockAdminClient adminClient; EXPECT_CALL(adminClient, addTask(_, _, _, _, _, _, _)) .WillOnce(Return(ByMove(folly::makeFuture(Status::OK())))); - auto executor = - new IngestJobExecutor(space, job.getJobId(), kv.get(), &adminClient, job.getParas()); + auto executor = std::make_unique( + space, job.getJobId(), kv.get(), &adminClient, job.getParas()); ASSERT_TRUE(executor->check()); auto code = executor->prepare(); diff --git a/src/parser/parser.yy b/src/parser/parser.yy index 9147a668c30..7c5d7b7aa90 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -3241,6 +3241,7 @@ download_sentence meta::cpp2::JobType::DOWNLOAD); sentence->addPara(*$3); $$ = sentence; + delete $3; } ; @@ -3280,6 +3281,7 @@ admin_job_sentence meta::cpp2::JobType::DOWNLOAD); sentence->addPara(*$5); $$ = sentence; + delete($5); } | KW_SUBMIT KW_JOB KW_INGEST { auto sentence = new AdminJobSentence(meta::cpp2::JobOp::ADD, From 809af73c70c77967b78be65396e98a9c8073262e Mon Sep 17 00:00:00 2001 From: Wey Gu Date: Fri, 1 Apr 2022 15:32:35 +0800 Subject: [PATCH 13/24] Param MATCH vid seek fix (#4024) * Add qctx to isEvaluableExpr for kRelIn Fixing MATCH (n) WHERE id(n) IN [$t] RETURN n * Fix eq kVal parameter vidseek * Fix id(n) IN [$var] case * lint add explicit and remove tailing semicolon * clang-format-10 * double check type of rightListValue Addressing Kyle's review comment * ut added in features/yield/parameter.feature * support kAttribute match (n) where n in $list_var return n * Add kSubscript for eq case MATCH (v) WHERE id(v) == $p7.a.b.d[4] RETURN v Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> Co-authored-by: kyle.cao Co-authored-by: cpw <13495049+CPWstatic@users.noreply.github.com> --- src/graph/planner/match/VertexIdSeek.cpp | 2 +- src/graph/visitor/VidExtractVisitor.cpp | 51 ++++++++++++++-------- src/graph/visitor/VidExtractVisitor.h | 4 +- tests/tck/features/yield/parameter.feature | 35 ++++++++++++++- 4 files changed, 72 insertions(+), 20 deletions(-) diff --git a/src/graph/planner/match/VertexIdSeek.cpp b/src/graph/planner/match/VertexIdSeek.cpp index 84fcce02e13..8d112ae493f 100644 --- a/src/graph/planner/match/VertexIdSeek.cpp +++ b/src/graph/planner/match/VertexIdSeek.cpp @@ -36,7 +36,7 @@ bool VertexIdSeek::matchNode(NodeContext *nodeCtx) { return false; } - VidExtractVisitor vidExtractVisitor; + VidExtractVisitor vidExtractVisitor(matchClauseCtx->qctx); matchClauseCtx->where->filter->accept(&vidExtractVisitor); auto vidResult = vidExtractVisitor.moveVidPattern(); if (vidResult.spec != VidExtractVisitor::VidPattern::Special::kInUsed) { diff --git a/src/graph/visitor/VidExtractVisitor.cpp b/src/graph/visitor/VidExtractVisitor.cpp index 179abcf8824..18893ec15d6 100644 --- a/src/graph/visitor/VidExtractVisitor.cpp +++ b/src/graph/visitor/VidExtractVisitor.cpp @@ -2,9 +2,9 @@ * * This source code is licensed under Apache 2.0 License. */ - #include "graph/visitor/VidExtractVisitor.h" +#include "graph/context/QueryContext.h" #include "graph/util/ExpressionUtils.h" namespace nebula { @@ -152,8 +152,9 @@ void VidExtractVisitor::visit(RelationalExpression *expr) { return; } if (expr->left()->kind() != Expression::Kind::kFunctionCall || - expr->right()->kind() != Expression::Kind::kList || - !ExpressionUtils::isEvaluableExpr(expr->right())) { + !(expr->right()->kind() == Expression::Kind::kList || + expr->right()->kind() == Expression::Kind::kAttribute) || + !ExpressionUtils::isEvaluableExpr(expr->right(), qctx_)) { vidPattern_ = VidPattern{}; return; } @@ -165,12 +166,11 @@ void VidExtractVisitor::visit(RelationalExpression *expr) { return; } - auto *listExpr = static_cast(expr->right()); - QueryExpressionContext ctx; - vidPattern_ = - VidPattern{VidPattern::Special::kInUsed, - {{fCallExpr->args()->args().front()->toString(), - {VidPattern::Vids::Kind::kIn, listExpr->eval(ctx(nullptr)).getList()}}}}; + auto rightListValue = expr->right()->eval(graph::QueryExpressionContext(qctx_->ectx())()); + DCHECK(rightListValue.isList()); + vidPattern_ = VidPattern{VidPattern::Special::kInUsed, + {{fCallExpr->args()->args().front()->toString(), + {VidPattern::Vids::Kind::kIn, rightListValue.getList()}}}}; } else if (expr->kind() == Expression::Kind::kRelEQ) { // id(V) == vid if (expr->left()->kind() == Expression::Kind::kLabelAttribute) { @@ -188,7 +188,9 @@ void VidExtractVisitor::visit(RelationalExpression *expr) { return; } if (expr->left()->kind() != Expression::Kind::kFunctionCall || - expr->right()->kind() != Expression::Kind::kConstant) { + (expr->right()->kind() != Expression::Kind::kConstant && + expr->right()->kind() != Expression::Kind::kVar && + expr->right()->kind() != Expression::Kind::kSubscript)) { vidPattern_ = VidPattern{}; return; } @@ -198,14 +200,29 @@ void VidExtractVisitor::visit(RelationalExpression *expr) { vidPattern_ = VidPattern{}; return; } - const auto *constExpr = static_cast(expr->right()); - if (!SchemaUtil::isValidVid(constExpr->value())) { - vidPattern_ = VidPattern{}; - return; + if (expr->right()->kind() == Expression::Kind::kConstant) { + const auto *constExpr = static_cast(expr->right()); + if (!SchemaUtil::isValidVid(constExpr->value())) { + vidPattern_ = VidPattern{}; + return; + } + vidPattern_ = VidPattern{VidPattern::Special::kInUsed, + {{fCallExpr->args()->args().front()->toString(), + {VidPattern::Vids::Kind::kIn, List({constExpr->value()})}}}}; + } else if ((expr->right()->kind() == Expression::Kind::kVar || + expr->right()->kind() == Expression::Kind::kSubscript) && + ExpressionUtils::isEvaluableExpr(expr->right(), qctx_)) { + auto rValue = expr->right()->eval(graph::QueryExpressionContext(qctx_->ectx())()); + if (SchemaUtil::isValidVid(rValue)) { + vidPattern_ = VidPattern{VidPattern::Special::kInUsed, + {{fCallExpr->args()->args().front()->toString(), + {VidPattern::Vids::Kind::kIn, List({rValue})}}}}; + return; + } else { + vidPattern_ = VidPattern{}; + return; + } } - vidPattern_ = VidPattern{VidPattern::Special::kInUsed, - {{fCallExpr->args()->args().front()->toString(), - {VidPattern::Vids::Kind::kIn, List({constExpr->value()})}}}}; } else { if (ExpressionUtils::isPropertyExpr(expr->left())) { vidPattern_ = VidPattern{VidPattern::Special::kInUsed, diff --git a/src/graph/visitor/VidExtractVisitor.h b/src/graph/visitor/VidExtractVisitor.h index 3145d45cd8c..6f5f5826f49 100644 --- a/src/graph/visitor/VidExtractVisitor.h +++ b/src/graph/visitor/VidExtractVisitor.h @@ -9,6 +9,7 @@ #include #include "common/expression/ExprVisitor.h" +#include "graph/context/QueryContext.h" #include "graph/util/SchemaUtil.h" namespace nebula { @@ -25,7 +26,7 @@ namespace graph { // other vid make it eval to TRUE! class VidExtractVisitor final : public ExprVisitor { public: - VidExtractVisitor() = default; + explicit VidExtractVisitor(const QueryContext *qctx = nullptr) : qctx_(qctx) {} struct VidPattern { enum class Special { @@ -110,6 +111,7 @@ class VidExtractVisitor final : public ExprVisitor { void visitBinaryExpr(BinaryExpression *expr); VidPattern vidPattern_{}; + const QueryContext *qctx_{nullptr}; }; std::ostream &operator<<(std::ostream &os, const VidExtractVisitor::VidPattern &vp); diff --git a/tests/tck/features/yield/parameter.feature b/tests/tck/features/yield/parameter.feature index 311f562e60d..fd5ea74fdbf 100644 --- a/tests/tck/features/yield/parameter.feature +++ b/tests/tck/features/yield/parameter.feature @@ -5,7 +5,7 @@ Feature: Parameter Background: Given a graph with space named "nba" - Given parameters: {"p1":1,"p2":true,"p3":"Tim Duncan","p4":3.3,"p5":[1,true,3],"p6":{"a":3,"b":false,"c":"Tim Duncan"},"p7":{"a":{"b":{"c":"Tim Duncan","d":[1,2,3,true,"Tim Duncan"]}}}} + Given parameters: {"p1":1,"p2":true,"p3":"Tim Duncan","p4":3.3,"p5":[1,true,3],"p6":{"a":3,"b":false,"c":"Tim Duncan"},"p7":{"a":{"b":{"c":"Tim Duncan","d":[1,2,3,true,"Tim Duncan"]}}},"p8":"Manu Ginobili"} Scenario: return parameters When executing query: @@ -17,6 +17,39 @@ Feature: Parameter | 2 | false | "Tim Duncanef" | 4.1 | [1,true,3] | 3 | true | Scenario: cypher with parameters + When executing query: + """ + MATCH (v) WHERE id(v)==$p3 + RETURN id(v) AS v + """ + Then the result should be, in any order: + | v | + | "Tim Duncan" | + When executing query: + """ + MATCH (v) WHERE id(v) IN [$p3,$p8] + RETURN id(v) AS v + """ + Then the result should be, in any order: + | v | + | "Tim Duncan" | + | "Manu Ginobili" | + When executing query: + """ + MATCH (v) WHERE id(v) == $p7.a.b.d[4] + RETURN id(v) AS v + """ + Then the result should be, in any order: + | v | + | "Tim Duncan" | + When executing query: + """ + MATCH (v) WHERE id(v) IN $p7.a.b.d + RETURN id(v) AS v + """ + Then the result should be, in any order: + | v | + | "Tim Duncan" | When executing query: """ MATCH (v:player)-[:like]->(n) WHERE id(v)==$p3 and n.player.age>$p1+29 From abd7355093443f6a0065794fa198e612476f9630 Mon Sep 17 00:00:00 2001 From: "kyle.cao" Date: Fri, 1 Apr 2022 20:20:02 +0800 Subject: [PATCH 14/24] disable console pkg (#4110) * disable console pkg * fix --- package/package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/package.sh b/package/package.sh index 24791a7251e..4ec2376e617 100755 --- a/package/package.sh +++ b/package/package.sh @@ -20,7 +20,7 @@ set -e version="" -package_one=ON +package_one=OFF strip_enable="FALSE" usage="Usage: ${0} -v -n -s -g -j -t " project_dir="$(cd "$(dirname "$0")" && pwd)/.." From 3bef944dc386bbd757af6007c8d53fccc1309889 Mon Sep 17 00:00:00 2001 From: "kyle.cao" Date: Sat, 2 Apr 2022 10:53:10 +0800 Subject: [PATCH 15/24] disable console pkg (#4112) --- package/package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/package.sh b/package/package.sh index 4ec2376e617..94b8efd7d54 100755 --- a/package/package.sh +++ b/package/package.sh @@ -20,7 +20,7 @@ set -e version="" -package_one=OFF +package_one=ON strip_enable="FALSE" usage="Usage: ${0} -v -n -s -g -j -t " project_dir="$(cd "$(dirname "$0")" && pwd)/.." @@ -28,7 +28,7 @@ build_dir=${project_dir}/pkg-build enablesanitizer="OFF" static_sanitizer="OFF" build_type="Release" -build_console="ON" +build_console="OFF" branch=$(git rev-parse --abbrev-ref HEAD) jobs=$(nproc) enable_compressed_debug_info=ON From 82c70ea5ea76615f70a5c86716e958d6ce9eee65 Mon Sep 17 00:00:00 2001 From: Yichen Wang <18348405+Aiee@users.noreply.github.com> Date: Sat, 2 Apr 2022 13:53:37 +0800 Subject: [PATCH 16/24] Fix service crash caused by using function call as a part of the filter in `LOOKUP` (#4111) * Fix function call purity check * Add UT for purity check * Add TCK cases --- src/common/function/FunctionManager.cpp | 209 +++++++++--------- src/common/function/FunctionManager.h | 21 +- .../function/test/FunctionManagerTest.cpp | 31 +++ src/graph/optimizer/OptimizerUtils.cpp | 11 +- tests/tck/features/bugfix/LookupIn.feature | 39 ++++ 5 files changed, 205 insertions(+), 106 deletions(-) diff --git a/src/common/function/FunctionManager.cpp b/src/common/function/FunctionManager.cpp index c0a60cdb84b..18f63b46831 100644 --- a/src/common/function/FunctionManager.cpp +++ b/src/common/function/FunctionManager.cpp @@ -456,7 +456,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["abs"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -478,21 +478,21 @@ FunctionManager::FunctionManager() { auto &attr = functions_["bit_and"]; attr.minArity_ = 2; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { return args[0].get() & args[1].get(); }; } { auto &attr = functions_["bit_or"]; attr.minArity_ = 2; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { return args[0].get() | args[1].get(); }; } { auto &attr = functions_["bit_xor"]; attr.minArity_ = 2; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { return args[0].get() ^ args[1].get(); }; } { @@ -500,7 +500,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["floor"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -523,7 +523,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["ceil"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -546,7 +546,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["round"]; attr.minArity_ = 1; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -575,7 +575,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["sqrt"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -606,7 +606,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["cbrt"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -629,7 +629,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["hypot"]; attr.minArity_ = 2; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (args[0].get().isNumeric() && args[1].get().isNumeric()) { auto x = args[0].get().isInt() ? args[0].get().getInt() : args[0].get().getFloat(); @@ -644,7 +644,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["pow"]; attr.minArity_ = 2; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (args[0].get().isNumeric() && args[1].get().isNumeric()) { auto base = args[0].get().isInt() ? args[0].get().getInt() : args[0].get().getFloat(); @@ -663,7 +663,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["e"]; attr.minArity_ = 0; attr.maxArity_ = 0; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { UNUSED(args); return M_E; @@ -674,7 +674,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["exp"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -697,7 +697,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["exp2"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -720,7 +720,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["log"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -751,7 +751,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["log2"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -782,7 +782,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["log10"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -813,7 +813,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["pi"]; attr.minArity_ = 0; attr.maxArity_ = 0; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { UNUSED(args); return M_PI; @@ -824,7 +824,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["radians"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -845,7 +845,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["sin"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -867,7 +867,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["asin"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -889,7 +889,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["cos"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -911,7 +911,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["acos"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -933,7 +933,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["tan"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -955,7 +955,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["atan"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -977,7 +977,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["sign"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1001,7 +1001,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["rand"]; attr.minArity_ = 0; attr.maxArity_ = 0; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { UNUSED(args); return folly::Random::randDouble01(); @@ -1012,7 +1012,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["rand32"]; attr.minArity_ = 0; attr.maxArity_ = 2; - attr.isPure_ = false; + setCompleteNonPure(attr); attr.body_ = [](const auto &args) -> Value { if (args.empty()) { auto value = folly::Random::rand32(); @@ -1049,7 +1049,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["rand64"]; attr.minArity_ = 0; attr.maxArity_ = 2; - attr.isPure_ = false; + setCompleteNonPure(attr); attr.body_ = [](const auto &args) -> Value { if (args.empty()) { return static_cast(folly::Random::rand64()); @@ -1080,7 +1080,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["now"]; attr.minArity_ = 0; attr.maxArity_ = 0; - attr.isPure_ = false; + setCompleteNonPure(attr); attr.body_ = [](const auto &args) -> Value { UNUSED(args); return ::time(NULL); @@ -1090,7 +1090,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["strcasecmp"]; attr.minArity_ = 2; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (args[0].get().isStr() && args[1].get().isStr()) { return static_cast( @@ -1103,7 +1103,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["lower"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1125,7 +1125,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["upper"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1149,7 +1149,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["length"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1173,7 +1173,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["trim"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1193,7 +1193,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["ltrim"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1213,7 +1213,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["rtrim"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1233,7 +1233,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["left"]; attr.minArity_ = 2; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1261,7 +1261,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["right"]; attr.minArity_ = 2; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1293,7 +1293,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["replace"]; attr.minArity_ = 3; attr.maxArity_ = 3; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (args[0].get().isNull() || args[1].get().isNull() || args[2].get().isNull()) { return Value::kNullValue; @@ -1311,7 +1311,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["reverse"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1338,7 +1338,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["split"]; attr.minArity_ = 2; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1371,7 +1371,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["tostring"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: @@ -1412,35 +1412,35 @@ FunctionManager::FunctionManager() { auto &attr = functions_["toboolean"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { return Value(args[0].get()).toBool(); }; } { auto &attr = functions_["tofloat"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { return Value(args[0].get()).toFloat(); }; } { auto &attr = functions_["tointeger"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { return Value(args[0].get()).toInt(); }; } { auto &attr = functions_["toset"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { return Value(args[0].get()).toSet(); }; } { auto &attr = functions_["lpad"]; attr.minArity_ = 3; attr.maxArity_ = 3; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (args[0].get().isStr() && args[1].get().isInt() && args[2].get().isStr()) { auto value = args[0].get().getStr(); @@ -1469,7 +1469,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["rpad"]; attr.minArity_ = 3; attr.maxArity_ = 3; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (args[0].get().isStr() && args[1].get().isInt() && args[2].get().isStr()) { auto value = args[0].get().getStr(); @@ -1499,7 +1499,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["substr"]; attr.minArity_ = 2; attr.maxArity_ = 3; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { auto argSize = args.size(); if (args[0].get().isNull()) { @@ -1534,7 +1534,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["hash"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: @@ -1561,7 +1561,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["udf_is_in"]; attr.minArity_ = 2; attr.maxArity_ = INT64_MAX; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { return std::find(args.begin() + 1, args.end(), args[0].get()) != args.end(); }; @@ -1570,7 +1570,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["near"]; attr.minArity_ = 2; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { // auto result = geo::GeoFilter::near(args); // if (!result.ok()) { @@ -1587,7 +1587,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["cos_similarity"]; attr.minArity_ = 2; attr.maxArity_ = INT64_MAX; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (args.size() % 2 != 0) { LOG(ERROR) << "The number of arguments must be even."; @@ -1621,7 +1621,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["size"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: @@ -1658,7 +1658,8 @@ FunctionManager::FunctionManager() { // 1 for string or map attr.minArity_ = 0; attr.maxArity_ = 1; - attr.isPure_ = false; + attr.isPure_[0] = false; + attr.isPure_[1] = true; attr.body_ = [](const auto &args) -> Value { switch (args.size()) { case 0: { @@ -1696,7 +1697,8 @@ FunctionManager::FunctionManager() { // 1 for string or map attr.minArity_ = 0; attr.maxArity_ = 1; - attr.isPure_ = false; + attr.isPure_[0] = false; + attr.isPure_[1] = true; attr.body_ = [](const auto &args) -> Value { switch (args.size()) { case 0: { @@ -1730,7 +1732,8 @@ FunctionManager::FunctionManager() { // 1 for string or map attr.minArity_ = 0; attr.maxArity_ = 1; - attr.isPure_ = false; + attr.isPure_[0] = false; + attr.isPure_[1] = true; attr.body_ = [](const auto &args) -> Value { switch (args.size()) { case 0: { @@ -1762,7 +1765,8 @@ FunctionManager::FunctionManager() { auto &attr = functions_["timestamp"]; attr.minArity_ = 0; attr.maxArity_ = 1; - attr.isPure_ = false; + attr.isPure_[0] = false; + attr.isPure_[1] = true; attr.body_ = [](const auto &args) -> Value { if (args.empty()) { return ::time(NULL); @@ -1778,7 +1782,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["range"]; attr.minArity_ = 2; attr.maxArity_ = 3; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isInt() || !args[1].get().isInt()) { return Value::kNullBadType; @@ -1808,7 +1812,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["id"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1827,7 +1831,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["tags"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1851,7 +1855,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["properties"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1882,7 +1886,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["type"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1901,7 +1905,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["typeid"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1920,7 +1924,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["src"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1940,7 +1944,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["dst"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1960,7 +1964,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["none_direct_dst"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -1995,7 +1999,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["rank"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -2014,7 +2018,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["startnode"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -2036,7 +2040,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["endnode"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -2062,7 +2066,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["head"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -2081,7 +2085,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["last"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -2100,7 +2104,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["coalesce"]; attr.minArity_ = 1; attr.maxArity_ = INT64_MAX; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { for (size_t i = 0; i < args.size(); ++i) { if (args[i].get().type() != Value::Type::NULLVALUE) { @@ -2114,7 +2118,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["keys"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { std::set tmp; switch (args[0].get().type()) { @@ -2154,7 +2158,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["nodes"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -2179,7 +2183,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["tail"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -2202,7 +2206,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["relationships"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { switch (args[0].get().type()) { case Value::Type::NULLVALUE: { @@ -2236,7 +2240,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["hassameedgeinpath"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isPath()) { return Value::kNullBadType; @@ -2249,7 +2253,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["hassamevertexinpath"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isPath()) { return Value::kNullBadType; @@ -2262,7 +2266,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["reversepath"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isPath()) { return Value::kNullBadType; @@ -2276,7 +2280,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["datasetrowcol"]; attr.minArity_ = 3; attr.maxArity_ = 3; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isDataSet() || !args[1].get().isInt() || !(args[2].get().isInt() || args[2].get().isStr())) { @@ -2307,7 +2311,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["concat"]; attr.minArity_ = 1; attr.maxArity_ = INT64_MAX; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { std::stringstream os; for (size_t i = 0; i < args.size(); ++i) { @@ -2355,7 +2359,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["concat_ws"]; attr.minArity_ = 2; attr.maxArity_ = INT64_MAX; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (args[0].get().isNull() || !args[0].get().isStr()) { return Value::kNullValue; @@ -2393,7 +2397,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["st_point"]; attr.minArity_ = 2; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isNumeric() || !args[1].get().isNumeric()) { return Value::kNullBadType; @@ -2409,7 +2413,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["st_geogfromtext"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isStr()) { return Value::kNullBadType; @@ -2447,7 +2451,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["st_astext"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isGeography()) { return Value::kNullBadType; @@ -2474,7 +2478,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["st_centroid"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isGeography()) { return Value::kNullBadType; @@ -2487,7 +2491,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["st_isvalid"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isGeography()) { return Value::kNullBadType; @@ -2500,7 +2504,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["st_intersects"]; attr.minArity_ = 2; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isGeography() || !args[1].get().isGeography()) { return Value::kNullBadType; @@ -2513,7 +2517,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["st_covers"]; attr.minArity_ = 2; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isGeography() || !args[1].get().isGeography()) { return Value::kNullBadType; @@ -2525,7 +2529,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["st_coveredby"]; attr.minArity_ = 2; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isGeography() || !args[1].get().isGeography()) { return Value::kNullBadType; @@ -2538,7 +2542,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["st_dwithin"]; attr.minArity_ = 3; attr.maxArity_ = 4; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isGeography() || !args[1].get().isGeography() || !args[2].get().isNumeric()) { @@ -2563,7 +2567,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["st_distance"]; attr.minArity_ = 2; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isGeography() || !args[1].get().isGeography()) { return Value::kNullBadType; @@ -2576,7 +2580,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["s2_cellidfrompoint"]; attr.minArity_ = 1; attr.maxArity_ = 2; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isGeography()) { return Value::kNullBadType; @@ -2607,7 +2611,7 @@ FunctionManager::FunctionManager() { auto &attr = functions_["s2_coveringcellids"]; attr.minArity_ = 1; attr.maxArity_ = 5; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { if (!args[0].get().isGeography()) { return Value::kNullBadType; @@ -2672,14 +2676,14 @@ FunctionManager::FunctionManager() { auto &attr = functions_["is_edge"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { return args[0].get().isEdge(); }; } { auto &attr = functions_["duration"]; attr.minArity_ = 1; attr.maxArity_ = 1; - attr.isPure_ = true; + attr.isAlwaysPure_ = true; attr.body_ = [](const auto &args) -> Value { const auto &arg = args[0].get(); switch (arg.type()) { @@ -2720,7 +2724,14 @@ Status FunctionManager::find(const std::string &func, const size_t arity) { /*static*/ StatusOr FunctionManager::getIsPure(const std::string &func, size_t arity) { auto result = instance().getInternal(func, arity); NG_RETURN_IF_ERROR(result); - return result.value().isPure_; + auto attr = std::move(result.value()); + + if (attr.isAlwaysPure_) { + return true; + } + + // If the function is not always pure, lookup the map to find purity. + return attr.isPure_.at(arity); } /*static*/ StatusOr FunctionManager::getInternal( diff --git a/src/common/function/FunctionManager.h b/src/common/function/FunctionManager.h index 44509fd79f9..8b4014bf66f 100644 --- a/src/common/function/FunctionManager.h +++ b/src/common/function/FunctionManager.h @@ -64,14 +64,31 @@ class FunctionManager final { const std::vector &argsType); private: + // The attributes of the function call struct FunctionAttributes final { size_t minArity_{0}; size_t maxArity_{0}; - // pure means same input same result - bool isPure_{true}; + // Pure means if the function takes the same input, it returns the same result + // The key here is the number of arity, and the key represents whether the function is pure for + // that number of arity. + // For example, timestamp() is non-pure because it returns the current timestamp, while + // timestamp(1) is pure because the output is always 1. + std::unordered_map isPure_; + // The flag indicates whether the function is always pure. + // Some functions allow using int64_max as the maxArity_, so add isAlwaysPure_ to avoid + // allocating huge amount of memory for these functions. + bool isAlwaysPure_{false}; Function body_; }; + // Sets the function to NON-pure for all numbers of arity it take, which means the function is + // always NON-pure. + void setCompleteNonPure(FunctionAttributes &attr) { + for (auto i = attr.minArity_; i <= attr.maxArity_; ++i) { + attr.isPure_[i] = false; + } + } + /** * FunctionManager functions as a singleton, since the dynamic loading is * process-wide. diff --git a/src/common/function/test/FunctionManagerTest.cpp b/src/common/function/test/FunctionManagerTest.cpp index 7c0dc1a423e..75fbd9e7628 100644 --- a/src/common/function/test/FunctionManagerTest.cpp +++ b/src/common/function/test/FunctionManagerTest.cpp @@ -13,6 +13,7 @@ #include "common/datatypes/Path.h" #include "common/datatypes/Set.h" #include "common/datatypes/Vertex.h" +#include "common/expression/FunctionCallExpression.h" #include "common/function/FunctionManager.h" #include "common/time/TimeUtils.h" #include "common/time/TimezoneInfo.h" @@ -1908,6 +1909,36 @@ TEST_F(FunctionManagerTest, DataSetRowCol) { } } +TEST_F(FunctionManagerTest, PurityTest) { + // Always non-pure + auto result = FunctionManager::getIsPure("now", 0); + ASSERT_TRUE(result.ok() && result.value() == false); + + // Always pure + result = FunctionManager::getIsPure("coalesce", 1); + ASSERT_TRUE(result.ok() && result.value() == true); + result = FunctionManager::getIsPure("coalesce", 100); + ASSERT_TRUE(result.ok() && result.value() == true); + result = FunctionManager::getIsPure("coalesce", INT64_MAX); + ASSERT_TRUE(result.ok() && result.value() == true); + + // Not always pure, purity depends on arity number + result = FunctionManager::getIsPure("time", 0); + ASSERT_TRUE(result.ok() && result.value() == false); + result = FunctionManager::getIsPure("time", 1); + ASSERT_TRUE(result.ok() && result.value() == true); + + result = FunctionManager::getIsPure("timestamp", 0); + ASSERT_TRUE(result.ok() && result.value() == false); + result = FunctionManager::getIsPure("timestamp", 1); + ASSERT_TRUE(result.ok() && result.value() == true); + + result = FunctionManager::getIsPure("date", 0); + ASSERT_TRUE(result.ok() && result.value() == false); + result = FunctionManager::getIsPure("date", 1); + ASSERT_TRUE(result.ok() && result.value() == true); +} + } // namespace nebula int main(int argc, char **argv) { diff --git a/src/graph/optimizer/OptimizerUtils.cpp b/src/graph/optimizer/OptimizerUtils.cpp index c5c8dad6c78..8ecb4fa59ae 100644 --- a/src/graph/optimizer/OptimizerUtils.cpp +++ b/src/graph/optimizer/OptimizerUtils.cpp @@ -105,12 +105,13 @@ StatusOr selectRelExprIndex(const ColumnDef& field, } auto right = expr->right(); - if (expr->kind() == Expression::Kind::kRelIn) { // container expressions - DCHECK(right->isContainerExpr()); - } else { // other expressions - DCHECK(right->kind() == Expression::Kind::kConstant); - } + // At this point all foldable expressions should already be folded. + if (right->kind() != Expression::Kind::kConstant) { + return Status::Error("The expression %s cannot be used to generate a index column hint", + right->toString().c_str()); + } + DCHECK(right->kind() == Expression::Kind::kConstant); const auto& value = static_cast(right)->value(); ScoredColumnHint hint; diff --git a/tests/tck/features/bugfix/LookupIn.feature b/tests/tck/features/bugfix/LookupIn.feature index 32800b83903..d67c361e5ef 100644 --- a/tests/tck/features/bugfix/LookupIn.feature +++ b/tests/tck/features/bugfix/LookupIn.feature @@ -39,3 +39,42 @@ Feature: Lookup_In | "Luka Doncic" | 20 | | "Kobe Bryant" | 40 | Then drop the used space + + # Crash caused by using function call as a part of the filter + # Fix https://github.com/vesoft-inc/nebula/issues/4097 + Scenario: lookup in where filter contains a function call + Given an empty graph + And create a space with following options: + | partition_num | 9 | + | replica_factor | 1 | + | vid_type | FIXED_STRING(30) | + | charset | utf8 | + | collate | utf8_bin | + And having executed: + """ + CREATE TAG tag_1(col1 date, col2 double); + """ + When executing query: + """ + CREATE TAG INDEX single_tag_index ON tag_1(col1); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX single_tag_index2 ON tag_1(col2); + """ + Then the execution should be successful + And wait 6 seconds + When executing query: + """ + LOOKUP ON tag_1 WHERE tag_1.col1 == date('2022-03-25') + AND tag_1.col2 > 10 YIELD tag_1.col1 + """ + Then the execution should be successful + When executing query: + """ + LOOKUP ON tag_1 WHERE tag_1.col1 == date() + AND tag_1.col2 > 10 YIELD tag_1.col1 + """ + Then the execution should be successful + Then drop the used space From daaed0abace28d372a98c383303f34c9c318337b Mon Sep 17 00:00:00 2001 From: Yee <2520865+yixinglu@users.noreply.github.com> Date: Sat, 2 Apr 2022 15:43:07 +0800 Subject: [PATCH 17/24] Fix cmake command error (#4114) --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ab12140cb83..b49e89ddbc7 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -125,7 +125,7 @@ jobs: -DCMAKE_BUILD_TYPE=Debug \ -DENABLE_TESTING=on \ -DENABLE_COVERAGE=on \ - -GNinja + -GNinja \ -B build - name: Make run: | From 6b20961ee13ca1f8acf3499a0d98f516158ba707 Mon Sep 17 00:00:00 2001 From: Doodle <13706157+critical27@users.noreply.github.com> Date: Wed, 6 Apr 2022 12:54:12 +0800 Subject: [PATCH 18/24] fix apply outdate membership change (#4107) * fix apply outdate membership change * fix transfer leader to '':0 cause crash Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/kvstore/LogEncoder.cpp | 4 +- src/kvstore/Part.cpp | 73 ++++++++++++++++++----- src/meta/processors/admin/AdminClient.cpp | 4 ++ 3 files changed, 63 insertions(+), 18 deletions(-) diff --git a/src/kvstore/LogEncoder.cpp b/src/kvstore/LogEncoder.cpp index db5adfa08df..170bcaad02d 100644 --- a/src/kvstore/LogEncoder.cpp +++ b/src/kvstore/LogEncoder.cpp @@ -217,8 +217,6 @@ decodeBatchValue(folly::StringPiece encoded) { std::string encodeHost(LogType type, const HostAddr& host) { std::string encoded; - // 15 refers to "255.255.255.255" - encoded.reserve(sizeof(int64_t) + 1 + 15 + sizeof(int)); int64_t ts = time::WallClock::fastNowInMilliSec(); std::string encodedHost; apache::thrift::CompactSerializer::serialize(host, &encodedHost); @@ -232,7 +230,7 @@ std::string encodeHost(LogType type, const HostAddr& host) { HostAddr decodeHost(LogType type, const folly::StringPiece& encoded) { HostAddr addr; - CHECK_GE(encoded.size(), sizeof(int64_t) + 1 + sizeof(size_t) + sizeof(Port)); + CHECK_GE(encoded.size(), sizeof(int64_t) + 1); CHECK(encoded[sizeof(int64_t)] == type); folly::StringPiece raw = encoded; diff --git a/src/kvstore/Part.cpp b/src/kvstore/Part.cpp index 6c3a10cab85..92fe487b6c8 100644 --- a/src/kvstore/Part.cpp +++ b/src/kvstore/Part.cpp @@ -212,6 +212,8 @@ void Part::onDiscoverNewLeader(HostAddr nLeader) { std::tuple Part::commitLogs( std::unique_ptr iter, bool wait, bool needLock) { + // We should apply any membership change which happens before start time. Because when we start + // up, the peers comes from meta, has already contains all previous changes. SCOPED_TIMER(&execTime_); auto batch = engine_->startBatchWrite(); LogID lastId = kNoCommitLogId; @@ -309,12 +311,26 @@ std::tuple Part::commitLogs( } case OP_TRANS_LEADER: { auto newLeader = decodeHost(OP_TRANS_LEADER, log); - commitTransLeader(newLeader, needLock); + auto ts = getTimestamp(log); + if (ts > startTimeMs_) { + commitTransLeader(newLeader, needLock); + } else { + VLOG(2) << idStr_ << "Skip commit stale transfer leader " << newLeader + << ", the part is opened at " << startTimeMs_ << ", but the log timestamp is " + << ts; + } break; } case OP_REMOVE_PEER: { auto peer = decodeHost(OP_REMOVE_PEER, log); - commitRemovePeer(peer, needLock); + auto ts = getTimestamp(log); + if (ts > startTimeMs_) { + commitRemovePeer(peer, needLock); + } else { + VLOG(2) << idStr_ << "Skip commit stale remove peer " << peer + << ", the part is opened at " << startTimeMs_ << ", but the log timestamp is " + << ts; + } break; } default: { @@ -389,36 +405,63 @@ nebula::cpp2::ErrorCode Part::putCommitMsg(WriteBatch* batch, } bool Part::preProcessLog(LogID logId, TermID termId, ClusterID clusterId, const std::string& log) { + // We should apply any membership change which happens before start time. Because when we start + // up, the peers comes from meta, has already contains all previous changes. VLOG(4) << idStr_ << "logId " << logId << ", termId " << termId << ", clusterId " << clusterId; if (!log.empty()) { switch (log[sizeof(int64_t)]) { case OP_ADD_LEARNER: { auto learner = decodeHost(OP_ADD_LEARNER, log); - LOG(INFO) << idStr_ << "preprocess add learner " << learner; - addLearner(learner, false); - // persist the part learner info in case of storaged restarting - engine_->updatePart(partId_, Peer(learner, Peer::Status::kLearner)); + auto ts = getTimestamp(log); + if (ts > startTimeMs_) { + VLOG(1) << idStr_ << "preprocess add learner " << learner; + addLearner(learner, false); + // persist the part learner info in case of storaged restarting + engine_->updatePart(partId_, Peer(learner, Peer::Status::kLearner)); + } else { + VLOG(1) << idStr_ << "Skip stale add learner " << learner << ", the part is opened at " + << startTimeMs_ << ", but the log timestamp is " << ts; + } break; } case OP_TRANS_LEADER: { auto newLeader = decodeHost(OP_TRANS_LEADER, log); - LOG(INFO) << idStr_ << "preprocess trans leader " << newLeader; - preProcessTransLeader(newLeader); + auto ts = getTimestamp(log); + if (ts > startTimeMs_) { + VLOG(1) << idStr_ << "preprocess trans leader " << newLeader; + preProcessTransLeader(newLeader); + } else { + VLOG(1) << idStr_ << "Skip stale transfer leader " << newLeader + << ", the part is opened at " << startTimeMs_ << ", but the log timestamp is " + << ts; + } break; } case OP_ADD_PEER: { auto peer = decodeHost(OP_ADD_PEER, log); - LOG(INFO) << idStr_ << "preprocess add peer " << peer; - addPeer(peer); - engine_->updatePart(partId_, Peer(peer, Peer::Status::kPromotedPeer)); + auto ts = getTimestamp(log); + if (ts > startTimeMs_) { + VLOG(1) << idStr_ << "preprocess add peer " << peer; + addPeer(peer); + engine_->updatePart(partId_, Peer(peer, Peer::Status::kPromotedPeer)); + } else { + VLOG(1) << idStr_ << "Skip stale add peer " << peer << ", the part is opened at " + << startTimeMs_ << ", but the log timestamp is " << ts; + } break; } case OP_REMOVE_PEER: { auto peer = decodeHost(OP_REMOVE_PEER, log); - LOG(INFO) << idStr_ << "preprocess remove peer " << peer; - preProcessRemovePeer(peer); - // remove peer in the persist info - engine_->updatePart(partId_, Peer(peer, Peer::Status::kDeleted)); + auto ts = getTimestamp(log); + if (ts > startTimeMs_) { + VLOG(1) << idStr_ << "preprocess remove peer " << peer; + preProcessRemovePeer(peer); + // remove peer in the persist info + engine_->updatePart(partId_, Peer(peer, Peer::Status::kDeleted)); + } else { + VLOG(1) << idStr_ << "Skip stale remove peer " << peer << ", the part is opened at " + << startTimeMs_ << ", but the log timestamp is " << ts; + } break; } default: { diff --git a/src/meta/processors/admin/AdminClient.cpp b/src/meta/processors/admin/AdminClient.cpp index dbcfdd90a2d..d9e493df81f 100644 --- a/src/meta/processors/admin/AdminClient.cpp +++ b/src/meta/processors/admin/AdminClient.cpp @@ -43,6 +43,7 @@ folly::Future AdminClient::transLeader(GraphSpaceID spaceId, } auto target = dst; if (dst == kRandomPeer) { + // pick a alive host as target when dst not specified for (auto& p : peers) { if (p != src) { auto retCode = ActiveHostsMan::isLived(kv_, p); @@ -53,6 +54,9 @@ folly::Future AdminClient::transLeader(GraphSpaceID spaceId, } } } + if (target == kRandomPeer) { + return Status::Error("No active peers found"); + } req.new_leader_ref() = std::move(target); return getResponseFromPart( Utils::getAdminAddrFromStoreAddr(src), From 99d93f5d2e585d2327801735ee9c8ee60b873c19 Mon Sep 17 00:00:00 2001 From: "pengwei.song" <90180021+pengweisong@users.noreply.github.com> Date: Wed, 6 Apr 2022 13:42:40 +0800 Subject: [PATCH 19/24] add code for part peers backward compatible (#4101) * add code for part peers backward compatible * add balance keys * fix bug: pass tests * add gflag && ignore * change int to int64_t * chage to emplace Co-authored-by: panda-sheep <59197347+panda-sheep@users.noreply.github.com> Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/common/utils/NebulaKeyUtils.cpp | 11 ++++ src/common/utils/NebulaKeyUtils.h | 15 +++++ src/common/utils/Types.h | 1 + src/kvstore/Common.h | 77 ++++++++++++++++++------ src/kvstore/KVEngine.h | 9 ++- src/kvstore/NebulaStore.cpp | 88 +++++++++++++++++++++------- src/kvstore/RocksEngine.cpp | 29 ++++++--- src/kvstore/RocksEngine.h | 18 ++++-- src/kvstore/test/NebulaStoreTest.cpp | 5 +- 9 files changed, 194 insertions(+), 59 deletions(-) diff --git a/src/common/utils/NebulaKeyUtils.cpp b/src/common/utils/NebulaKeyUtils.cpp index 31b24fc57ba..d026f3efed0 100644 --- a/src/common/utils/NebulaKeyUtils.cpp +++ b/src/common/utils/NebulaKeyUtils.cpp @@ -119,6 +119,17 @@ std::string NebulaKeyUtils::systemPartKey(PartitionID partId) { return key; } +// static +std::string NebulaKeyUtils::systemBalanceKey(PartitionID partId) { + uint32_t item = (partId << kPartitionOffset) | static_cast(NebulaKeyType::kSystem); + uint32_t type = static_cast(NebulaSystemKeyType::kSystemBalance); + std::string key; + key.reserve(kSystemLen); + key.append(reinterpret_cast(&item), sizeof(PartitionID)) + .append(reinterpret_cast(&type), sizeof(NebulaSystemKeyType)); + return key; +} + // static std::string NebulaKeyUtils::kvKey(PartitionID partId, const folly::StringPiece& name) { std::string key; diff --git a/src/common/utils/NebulaKeyUtils.h b/src/common/utils/NebulaKeyUtils.h index 17cbd88bccc..6608c4d5bdf 100644 --- a/src/common/utils/NebulaKeyUtils.h +++ b/src/common/utils/NebulaKeyUtils.h @@ -77,6 +77,8 @@ class NebulaKeyUtils final { static std::string systemPartKey(PartitionID partId); + static std::string systemBalanceKey(PartitionID partId); + static std::string kvKey(PartitionID partId, const folly::StringPiece& name); static std::string kvPrefix(PartitionID partId); @@ -189,6 +191,19 @@ class NebulaKeyUtils final { return static_cast(type) == NebulaSystemKeyType::kSystemPart; } + static bool isSystemBalance(const folly::StringPiece& rawKey) { + if (rawKey.size() != kSystemLen) { + return false; + } + if (!isSystem(rawKey)) { + return false; + } + auto position = rawKey.data() + sizeof(PartitionID); + auto len = sizeof(NebulaSystemKeyType); + auto type = readInt(position, len); + return static_cast(type) == NebulaSystemKeyType::kSystemBalance; + } + static VertexIDSlice getSrcId(size_t vIdLen, const folly::StringPiece& rawKey) { if (rawKey.size() < kEdgeLen + (vIdLen << 1)) { dumpBadKey(rawKey, kEdgeLen + (vIdLen << 1), vIdLen); diff --git a/src/common/utils/Types.h b/src/common/utils/Types.h index e365fb4270e..c5a812b9a16 100644 --- a/src/common/utils/Types.h +++ b/src/common/utils/Types.h @@ -26,6 +26,7 @@ enum class NebulaKeyType : uint32_t { enum class NebulaSystemKeyType : uint32_t { kSystemCommit = 0x00000001, kSystemPart = 0x00000002, + kSystemBalance = 0x00000003, }; enum class NebulaOperationType : uint32_t { diff --git a/src/kvstore/Common.h b/src/kvstore/Common.h index e2ee85dd030..c97e6c878a7 100644 --- a/src/kvstore/Common.h +++ b/src/kvstore/Common.h @@ -9,12 +9,17 @@ #include #include +#include + #include "common/base/Base.h" #include "common/datatypes/HostAddr.h" #include "common/thrift/ThriftTypes.h" +#include "common/time/WallClock.h" #include "common/utils/Types.h" #include "interface/gen-cpp2/common_types.h" +DECLARE_int64(balance_expired_sesc); + namespace nebula { namespace kvstore { @@ -59,6 +64,7 @@ struct Peer { Status status; Peer() : addr(), status(Status::kNormalPeer) {} + explicit Peer(HostAddr a) : addr(a), status(Status::kNormalPeer) {} Peer(HostAddr a, Status s) : addr(a), status(s) {} std::string toString() const { @@ -106,26 +112,37 @@ inline std::ostream& operator<<(std::ostream& os, const Peer& peer) { struct Peers { private: std::map peers; + int64_t createdTime; public: - Peers() {} + Peers() { + createdTime = time::WallClock::fastNowInSec(); + } explicit Peers(const std::vector& addrs) { // from normal peers for (auto& addr : addrs) { peers[addr] = Peer(addr, Peer::Status::kNormalPeer); } + createdTime = time::WallClock::fastNowInSec(); } explicit Peers(const std::vector& ps) { for (auto& p : ps) { peers[p.addr] = p; } + createdTime = time::WallClock::fastNowInSec(); + } + explicit Peers(std::map ps) : peers(std::move(ps)) { + createdTime = time::WallClock::fastNowInSec(); } - explicit Peers(std::map ps) : peers(std::move(ps)) {} void addOrUpdate(const Peer& peer) { peers[peer.addr] = peer; } - bool get(const HostAddr& addr, Peer* peer) { + bool exist(const HostAddr& addr) const { + return peers.find(addr) != peers.end(); + } + + bool get(const HostAddr& addr, Peer* peer) const { auto it = peers.find(addr); if (it == peers.end()) { return false; @@ -141,7 +158,7 @@ struct Peers { peers.erase(addr); } - size_t size() { + size_t size() const { return peers.size(); } @@ -149,31 +166,53 @@ struct Peers { return peers; } + bool allNormalPeers() const { + for (const auto& [addr, peer] : peers) { + if (peer.status == Peer::Status::kDeleted) { + continue; + } + + if (peer.status != Peer::Status::kNormalPeer) { + return false; + } + } + return true; + } + + bool isExpired() const { + return time::WallClock::fastNowInSec() - createdTime > FLAGS_balance_expired_sesc; + } + + void setCreatedTime(int64_t time) { + createdTime = time; + } + std::string toString() const { std::stringstream os; os << "version:1," - << "count:" << peers.size() << "\n"; + << "count:" << peers.size() << ",ts:" << createdTime << "\n"; for (const auto& [_, p] : peers) { os << p << "\n"; } return os.str(); } - static std::pair extractHeader(const std::string& header) { - auto pos = header.find(":"); - if (pos == std::string::npos) { + static std::tuple extractHeader(const std::string& header) { + std::vector fields; + folly::split(":", header, fields, true); + if (fields.size() != 4) { LOG(INFO) << "Parse part peers header error:" << header; - return {0, 0}; + return {0, 0, 0}; } - int version = std::stoi(header.substr(pos + 1)); - pos = header.find(":", pos + 1); - if (pos == std::string::npos) { - LOG(INFO) << "Parse part peers header error:" << header; - return {0, 0}; - } - int count = std::stoi(header.substr(pos + 1)); - return {version, count}; + int version = std::stoi(fields[1]); + int count = std::stoi(fields[2]); + + int64_t createdTime; + std::istringstream iss(fields[3]); + iss >> createdTime; + + return {version, count, createdTime}; } static Peers fromString(const std::string& str) { @@ -186,7 +225,7 @@ struct Peers { return peers; } - auto [version, count] = extractHeader(lines[0]); + auto [version, count, createdTime] = extractHeader(lines[0]); if (version != 1) { LOG(INFO) << "Wrong peers format version:" << version; return peers; @@ -198,6 +237,8 @@ struct Peers { return peers; } + peers.setCreatedTime(createdTime); + // skip header for (size_t i = 1; i < lines.size(); ++i) { auto& line = lines[i]; diff --git a/src/kvstore/KVEngine.h b/src/kvstore/KVEngine.h index 5d48206104b..d3a98d044cc 100644 --- a/src/kvstore/KVEngine.h +++ b/src/kvstore/KVEngine.h @@ -228,11 +228,10 @@ class KVEngine { virtual void addPart(PartitionID partId, const Peers& raftPeers) = 0; /** - * @brief Update part info. Could only update the peers' persist info now. + * @brief Update part info. Could only update the persist peers info in balancing now. * * @param partId - * @param raftPeer 1. if raftPeer.status is kDeleted, delete this peer. - * 2. if raftPeer.status is others, add or update this peer + * @param raftPeer */ virtual void updatePart(PartitionID partId, const Peer& raftPeer) = 0; @@ -251,11 +250,11 @@ class KVEngine { virtual std::vector allParts() = 0; /** - * @brief Return all partId->raft peers that current storage engine holds. + * @brief Return all balancing partId->raft peers that current storage engine holds. * * @return std::map partId-> raft peers for each part, including learners */ - virtual std::map allPartPeers() = 0; + virtual std::map balancePartPeers() = 0; /** * @brief Return total parts num diff --git a/src/kvstore/NebulaStore.cpp b/src/kvstore/NebulaStore.cpp index ee4c08b9151..c5766d582bb 100644 --- a/src/kvstore/NebulaStore.cpp +++ b/src/kvstore/NebulaStore.cpp @@ -13,6 +13,8 @@ #include "common/fs/FileUtils.h" #include "common/network/NetworkUtils.h" +#include "common/time/WallClock.h" +#include "common/utils/NebulaKeyUtils.h" #include "kvstore/NebulaSnapshotManager.h" #include "kvstore/RocksEngine.h" @@ -81,7 +83,8 @@ bool NebulaStore::init() { void NebulaStore::loadPartFromDataPath() { CHECK(!!options_.partMan_); LOG(INFO) << "Scan the local path, and init the spaces_"; - std::unordered_set> spacePartIdSet; + // avoid duplicate engine created + std::unordered_set> partSet; for (auto& path : options_.dataPaths_) { auto rootPath = folly::stringPrintf("%s/nebula", path.c_str()); auto dirs = fs::FileUtils::listAllDirsInDir(rootPath.c_str()); @@ -101,35 +104,78 @@ void NebulaStore::loadPartFromDataPath() { continue; } - // Load raft peers info which persisted to local engine. - // If the partition was in balancing process before restart, we should keep it - // though the part is not in the meta. auto engine = newEngine(spaceId, path, options_.walPath_); std::map partRaftPeers; - for (auto& [partId, raftPeers] : engine->allPartPeers()) { - bool isNormalPeer = true; - - Peer raftPeer; - bool exist = raftPeers.get(raftAddr_, &raftPeer); - if (exist) { - if (raftPeer.status != Peer::Status::kNormalPeer) { - isNormalPeer = false; + + // load balancing part info which persisted to local engine. + for (auto& [partId, raftPeers] : engine->balancePartPeers()) { + CHECK_NE(raftPeers.size(), 0); + + if (raftPeers.isExpired()) { + LOG(INFO) << "Space: " << spaceId << ", part:" << partId + << " balancing info expired, ignore it."; + continue; + } + + auto spacePart = std::make_pair(spaceId, partId); + if (partSet.find(spacePart) == partSet.end()) { + partSet.emplace(std::make_pair(spaceId, partId)); + + // join the balancing peers with meta peers + auto metaStatus = options_.partMan_->partMeta(spaceId, partId); + if (!metaStatus.ok()) { + LOG(INFO) << "Space: " << spaceId << "; partId: " << partId + << " does not exist in part manager when join balancing."; + } else { + auto partMeta = metaStatus.value(); + for (auto& h : partMeta.hosts_) { + auto raftAddr = getRaftAddr(h); + if (!raftPeers.exist(raftAddr)) { + VLOG(1) << "Add raft peer " << raftAddr; + raftPeers.addOrUpdate(Peer(raftAddr)); + } + } } + + partRaftPeers.emplace(partId, raftPeers); + } + } + + // load normal part ids which persisted to local engine. + for (auto& partId : engine->allParts()) { + // first priority: balancing + bool inBalancing = partRaftPeers.find(partId) != partRaftPeers.end(); + if (inBalancing) { + continue; } - if (!options_.partMan_->partExist(storeSvcAddr_, spaceId, partId).ok() && isNormalPeer) { - LOG(INFO) << "Part " << partId - << " is a normal peer and does not exist in meta any more, will remove it!"; + // second priority: meta + if (!options_.partMan_->partExist(storeSvcAddr_, spaceId, partId).ok()) { + LOG(INFO) + << "Part " << partId + << " is not in balancing and does not exist in meta any more, will remove it!"; engine->removePart(partId); continue; - } else { - auto spacePart = std::make_pair(spaceId, partId); - if (spacePartIdSet.find(spacePart) == spacePartIdSet.end()) { - spacePartIdSet.emplace(spacePart); - partRaftPeers.emplace(partId, raftPeers); + } + + auto spacePart = std::make_pair(spaceId, partId); + if (partSet.find(spacePart) == partSet.end()) { + partSet.emplace(spacePart); + + // fill the peers + auto metaStatus = options_.partMan_->partMeta(spaceId, partId); + CHECK(metaStatus.ok()); + auto partMeta = metaStatus.value(); + Peers peers; + for (auto& h : partMeta.hosts_) { + VLOG(1) << "Add raft peer " << getRaftAddr(h); + peers.addOrUpdate(Peer(getRaftAddr(h))); } + partRaftPeers.emplace(partId, peers); } } + + // there is no valid part in this engine, remove it if (partRaftPeers.empty()) { engine.reset(); // close engine if (!options_.partMan_->spaceExist(storeSvcAddr_, spaceId).ok()) { @@ -141,7 +187,7 @@ void NebulaStore::loadPartFromDataPath() { continue; } - // add to spaces if the part should exist + // add to spaces KVEngine* enginePtr = nullptr; { folly::RWSpinLock::WriteHolder wh(&lock_); diff --git a/src/kvstore/RocksEngine.cpp b/src/kvstore/RocksEngine.cpp index f57ec600b68..7f45f4bb409 100644 --- a/src/kvstore/RocksEngine.cpp +++ b/src/kvstore/RocksEngine.cpp @@ -15,6 +15,9 @@ #include "kvstore/KVStore.h" DEFINE_bool(move_files, false, "Move the SST files instead of copy when ingest into dataset"); +DEFINE_int64(balance_expired_sesc, + 86400, + "The expired time of balancing part info persisted in the storaged"); namespace nebula { namespace kvstore { @@ -320,30 +323,39 @@ std::string RocksEngine::partKey(PartitionID partId) { return NebulaKeyUtils::systemPartKey(partId); } +std::string RocksEngine::balanceKey(PartitionID partId) { + return NebulaKeyUtils::systemBalanceKey(partId); +} + void RocksEngine::addPart(PartitionID partId, const Peers& raftPeers) { - auto ret = put(partKey(partId), raftPeers.toString()); + auto ret = put(partKey(partId), ""); if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { partsNum_++; CHECK_GE(partsNum_, 0); } + + if (!raftPeers.allNormalPeers()) { + put(balanceKey(partId), raftPeers.toString()); + } } void RocksEngine::updatePart(PartitionID partId, const Peer& raftPeer) { std::string val; - auto ret = get(partKey(partId), &val); + auto ret = get(balanceKey(partId), &val); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(INFO) << "Update part failed when get, partId=" << partId; return; } auto peers = Peers::fromString(val); - if (raftPeer.status == Peer::Status::kDeleted) { - peers.remove(raftPeer.addr); + peers.addOrUpdate(raftPeer); + + if (peers.allNormalPeers()) { + ret = remove(balanceKey(partId)); } else { - peers.addOrUpdate(raftPeer); + ret = put(balanceKey(partId), peers.toString()); } - ret = put(partKey(partId), peers.toString()); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(INFO) << "Update part failed when put back, partId=" << partId; } @@ -354,6 +366,7 @@ void RocksEngine::removePart(PartitionID partId) { options.disableWAL = FLAGS_rocksdb_disable_wal; std::vector sysKeysToDelete; sysKeysToDelete.emplace_back(partKey(partId)); + sysKeysToDelete.emplace_back(balanceKey(partId)); sysKeysToDelete.emplace_back(NebulaKeyUtils::systemCommitKey(partId)); auto code = multiRemove(sysKeysToDelete); if (code == nebula::cpp2::ErrorCode::SUCCEEDED) { @@ -386,7 +399,7 @@ std::vector RocksEngine::allParts() { return parts; } -std::map RocksEngine::allPartPeers() { +std::map RocksEngine::balancePartPeers() { std::unique_ptr iter; std::map partRaftPeers; static const std::string prefixStr = NebulaKeyUtils::systemPrefix(); @@ -397,7 +410,7 @@ std::map RocksEngine::allPartPeers() { while (iter->valid()) { auto key = iter->key(); - if (!NebulaKeyUtils::isSystemPart(key)) { + if (!NebulaKeyUtils::isSystemBalance(key)) { iter->next(); continue; } diff --git a/src/kvstore/RocksEngine.h b/src/kvstore/RocksEngine.h index 7384ed218fc..9cc9b057301 100644 --- a/src/kvstore/RocksEngine.h +++ b/src/kvstore/RocksEngine.h @@ -401,11 +401,11 @@ class RocksEngine : public KVEngine { void addPart(PartitionID partId, const Peers& raftPeers = {}) override; /** - * @brief Update part info. Could only update the persist peers info now. + * @brief Update part info. Could only update the persist peers info in balancing now. * * @param partId - * @param raftPeer 1. if raftPeer.status is kDeleted, delete this peer. - * 2. if raftPeer.status is others, add or update this peer + * @param raftPeer + * */ void updatePart(PartitionID partId, const Peer& raftPeer) override; @@ -424,11 +424,11 @@ class RocksEngine : public KVEngine { std::vector allParts() override; /** - * @brief Retrun all the part->raft peers in rocksdb engine by scanning system part key. + * @brief Retrun all the balancing part->raft peers in rocksdb engine by scanning system part key. * * @return std::map */ - std::map allPartPeers() override; + std::map balancePartPeers() override; /** * @brief Return total partition numbers @@ -528,6 +528,14 @@ class RocksEngine : public KVEngine { */ std::string partKey(PartitionID partId); + /** + * @brief System balance key, containing balancing info + * + * @param partId + * @return std::string + */ + std::string balanceKey(PartitionID partId); + /** * @brief Open the rocksdb backup engine, mainly for rocksdb PlainTable mounted on tmpfs/ramfs * diff --git a/src/kvstore/test/NebulaStoreTest.cpp b/src/kvstore/test/NebulaStoreTest.cpp index f603e97bd56..c8abdcb7171 100644 --- a/src/kvstore/test/NebulaStoreTest.cpp +++ b/src/kvstore/test/NebulaStoreTest.cpp @@ -166,7 +166,7 @@ TEST(NebulaStoreTest, PartsTest) { // disk2: 5, 6, 7, 15 KVOptions options; - options.dataPaths_ = std::move(paths); + options.dataPaths_ = paths; options.partMan_ = std::move(partMan); HostAddr local = {"", 0}; auto store = @@ -291,11 +291,12 @@ TEST(NebulaStoreTest, PersistPeersTest) { // disk2: 5, 6, 7, 15 KVOptions options; - options.dataPaths_ = std::move(paths); + options.dataPaths_ = paths; options.partMan_ = std::move(partMan); auto store = std::make_unique(std::move(options), ioThreadPool, local, getHandlers()); store->init(); + auto check = [&](GraphSpaceID spaceId) { for (int i = 0; i < static_cast(paths.size()); i++) { ASSERT_EQ(folly::stringPrintf("%s/disk%d/nebula/%d", rootPath.path(), i + 1, spaceId), From 27066cc7ce3e3fb9203ea44974ec6cc1dcd57eb3 Mon Sep 17 00:00:00 2001 From: "haifei.zhao" <32253291+zhaohaifei@users.noreply.github.com> Date: Wed, 6 Apr 2022 14:07:30 +0800 Subject: [PATCH 20/24] fixed (#4116) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/meta/processors/admin/VerifyClientVersionProcessor.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/meta/processors/admin/VerifyClientVersionProcessor.cpp b/src/meta/processors/admin/VerifyClientVersionProcessor.cpp index df00c1cdbff..ab400bc4cca 100644 --- a/src/meta/processors/admin/VerifyClientVersionProcessor.cpp +++ b/src/meta/processors/admin/VerifyClientVersionProcessor.cpp @@ -31,8 +31,7 @@ void VerifyClientVersionProcessor::process(const cpp2::VerifyClientVersionReq& r auto versionVal = MetaKeyUtils::versionVal(req.get_build_version().c_str()); std::vector versionData; versionData.emplace_back(std::move(versionKey), std::move(versionVal)); - doSyncPut(versionData); - resp_.code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED; + handleErrorCode(doSyncPut(versionData)); } onFinished(); } From 8202b54730c4f7d230c780c777795afffa0cda50 Mon Sep 17 00:00:00 2001 From: "haifei.zhao" <32253291+zhaohaifei@users.noreply.github.com> Date: Thu, 7 Apr 2022 14:44:21 +0800 Subject: [PATCH 21/24] move KW_CLEAR to unresolved keyword (#4118) --- .linters/cpp/checkKeyword.py | 1 - scripts/nebula.service | 2 +- src/parser/parser.yy | 1 + src/parser/scanner.lex | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.linters/cpp/checkKeyword.py b/.linters/cpp/checkKeyword.py index bbe3dec3238..4055255051d 100755 --- a/.linters/cpp/checkKeyword.py +++ b/.linters/cpp/checkKeyword.py @@ -75,7 +75,6 @@ 'KW_ADD', 'KW_CREATE', 'KW_DROP', - 'KW_CLEAR', 'KW_REMOVE', 'KW_IF', 'KW_NOT', diff --git a/scripts/nebula.service b/scripts/nebula.service index 98c011b3c59..a61c38e669e 100755 --- a/scripts/nebula.service +++ b/scripts/nebula.service @@ -234,7 +234,7 @@ function status_daemon { port=${GREEN}${port}${NC} else port=${BLINK}${RED}${port}${NC} - if [[$daemon_name == nebula-storaged]]; then + if [[ $daemon_name == nebula-storaged ]]; then WARN "${daemon_name} after v3.0.0 will not start service until it is added to cluster." WARN "See Manage Storage hosts:${RED}ADD HOSTS${NC} in https://docs.nebula-graph.io/" fi diff --git a/src/parser/parser.yy b/src/parser/parser.yy index 7c5d7b7aa90..434fcc3690e 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -576,6 +576,7 @@ unreserved_keyword | KW_MERGE { $$ = new std::string("merge"); } | KW_DIVIDE { $$ = new std::string("divide"); } | KW_RENAME { $$ = new std::string("rename"); } + | KW_CLEAR { $$ = new std::string("clear"); } ; expression diff --git a/src/parser/scanner.lex b/src/parser/scanner.lex index ecdeb88b19a..7650feab8f5 100644 --- a/src/parser/scanner.lex +++ b/src/parser/scanner.lex @@ -144,7 +144,6 @@ LABEL_FULL_WIDTH {CN_EN_FULL_WIDTH}{CN_EN_NUM_FULL_WIDTH}* "ADD" { return TokenType::KW_ADD; } "CREATE" { return TokenType::KW_CREATE;} "DROP" { return TokenType::KW_DROP; } -"CLEAR" { return TokenType::KW_CLEAR; } "REMOVE" { return TokenType::KW_REMOVE; } "IF" { return TokenType::KW_IF; } "NOT" { return TokenType::KW_NOT; } @@ -308,6 +307,7 @@ LABEL_FULL_WIDTH {CN_EN_FULL_WIDTH}{CN_EN_NUM_FULL_WIDTH}* "MERGE" { return TokenType::KW_MERGE; } "RENAME" { return TokenType::KW_RENAME; } "DIVIDE" { return TokenType::KW_DIVIDE; } +"CLEAR" { return TokenType::KW_CLEAR; } "TRUE" { yylval->boolval = true; return TokenType::BOOL; } "FALSE" { yylval->boolval = false; return TokenType::BOOL; } From 07cbbac4934e1ee2c043d888431ab4cdeeea87f2 Mon Sep 17 00:00:00 2001 From: panda-sheep <59197347+panda-sheep@users.noreply.github.com> Date: Thu, 7 Apr 2022 18:27:07 +0800 Subject: [PATCH 22/24] Limits for add and recover balance data or zone balance jobs (#4104) * If there are failed or stopped data balance or zone balance job, must firstly recover it * address wenhaocs's comments * address comments * add more comments * recover job * add more ut * adjust comment format * adjust comment format * adjust comment format --- src/clients/meta/MetaClient.cpp | 2 + src/common/graph/Response.h | 2 + src/graph/executor/StorageAccessExecutor.cpp | 1 + .../executor/admin/ChangePasswordExecutor.cpp | 1 + src/graph/executor/admin/ConfigExecutor.cpp | 1 + .../executor/admin/CreateUserExecutor.cpp | 1 + .../executor/admin/DescribeUserExecutor.cpp | 1 + src/graph/executor/admin/DropUserExecutor.cpp | 1 + .../executor/admin/KillQueryExecutor.cpp | 1 + .../executor/admin/ListRolesExecutor.cpp | 1 + .../executor/admin/ListUserRolesExecutor.cpp | 1 + src/graph/executor/admin/PartExecutor.cpp | 1 + .../executor/admin/RevokeRoleExecutor.cpp | 1 + .../executor/admin/ShowHostsExecutor.cpp | 1 + .../executor/admin/ShowMetaLeaderExecutor.cpp | 1 + .../admin/ShowServiceClientsExecutor.cpp | 1 + .../executor/admin/ShowStatsExecutor.cpp | 1 + .../executor/admin/SignInServiceExecutor.cpp | 1 + src/graph/executor/admin/SnapshotExecutor.cpp | 2 + src/graph/executor/logic/ArgumentExecutor.cpp | 1 + src/graph/executor/logic/LoopExecutor.cpp | 2 +- src/graph/executor/logic/SelectExecutor.cpp | 1 + src/graph/executor/maintain/EdgeExecutor.cpp | 1 + .../executor/maintain/FTIndexExecutor.cpp | 1 + src/graph/executor/maintain/TagExecutor.cpp | 1 + src/graph/executor/mutate/DeleteExecutor.cpp | 1 + src/graph/executor/mutate/InsertExecutor.cpp | 1 + .../executor/query/AggregateExecutor.cpp | 1 + src/graph/executor/query/AssignExecutor.cpp | 1 + .../executor/query/DataCollectExecutor.cpp | 1 + src/graph/executor/query/FilterExecutor.cpp | 1 + src/graph/executor/query/GetEdgesExecutor.cpp | 1 + .../executor/query/GetNeighborsExecutor.cpp | 1 + .../executor/query/IndexScanExecutor.cpp | 1 + .../executor/query/InnerJoinExecutor.cpp | 1 + .../executor/query/IntersectExecutor.cpp | 1 + src/graph/executor/query/JoinExecutor.cpp | 1 + src/graph/executor/query/LeftJoinExecutor.cpp | 1 + src/graph/executor/query/LimitExecutor.cpp | 1 + src/graph/executor/query/MinusExecutor.cpp | 1 + src/graph/executor/query/ProjectExecutor.cpp | 1 + src/graph/executor/query/SampleExecutor.cpp | 1 + .../executor/query/ScanEdgesExecutor.cpp | 1 + src/graph/executor/query/SetExecutor.cpp | 1 + src/graph/executor/query/SortExecutor.cpp | 1 + src/graph/executor/query/TopNExecutor.cpp | 1 + src/graph/executor/query/TraverseExecutor.cpp | 1 + .../query/UnionAllVersionVarExecutor.cpp | 1 + src/graph/executor/query/UnionExecutor.cpp | 1 + src/graph/executor/query/UnwindExecutor.cpp | 1 + src/interface/common.thrift | 1 + src/meta/processors/job/AdminJobProcessor.cpp | 12 +- src/meta/processors/job/JobDescription.h | 3 +- src/meta/processors/job/JobManager.cpp | 170 +++++++++++++++++- src/meta/processors/job/JobManager.h | 25 ++- src/meta/test/JobManagerTest.cpp | 133 +++++++++++--- 56 files changed, 359 insertions(+), 39 deletions(-) diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index 0371034b818..d57d6012ea3 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -915,6 +915,8 @@ Status MetaClient::handleResponse(const RESP& resp) { return Status::Error("No valid job!"); case nebula::cpp2::ErrorCode::E_JOB_NOT_IN_SPACE: return Status::Error("Job not existed in chosen space!"); + case nebula::cpp2::ErrorCode::E_JOB_NEED_RECOVER: + return Status::Error("Need to recover failed data balance job or zone balance job firstly!"); case nebula::cpp2::ErrorCode::E_BACKUP_EMPTY_TABLE: return Status::Error("Backup empty table!"); case nebula::cpp2::ErrorCode::E_BACKUP_TABLE_FAILED: diff --git a/src/common/graph/Response.h b/src/common/graph/Response.h index 7c4679ef35f..abf151811dc 100644 --- a/src/common/graph/Response.h +++ b/src/common/graph/Response.h @@ -106,6 +106,8 @@ X(E_BALANCER_FAILURE, -2047) \ X(E_JOB_NOT_FINISHED, -2048) \ X(E_TASK_REPORT_OUT_DATE, -2049) \ + X(E_JOB_NOT_IN_SPACE, -2050) \ + X(E_JOB_NEED_RECOVER, -2051) \ X(E_INVALID_JOB, -2065) \ \ /* Backup Failure */ \ diff --git a/src/graph/executor/StorageAccessExecutor.cpp b/src/graph/executor/StorageAccessExecutor.cpp index c315cdde008..00c29e5226e 100644 --- a/src/graph/executor/StorageAccessExecutor.cpp +++ b/src/graph/executor/StorageAccessExecutor.cpp @@ -5,6 +5,7 @@ #include "graph/executor/StorageAccessExecutor.h" #include + #include "graph/context/Iterator.h" #include "graph/context/QueryExpressionContext.h" #include "graph/util/SchemaUtil.h" diff --git a/src/graph/executor/admin/ChangePasswordExecutor.cpp b/src/graph/executor/admin/ChangePasswordExecutor.cpp index 8e7c4f8d86a..da5ac0b1761 100644 --- a/src/graph/executor/admin/ChangePasswordExecutor.cpp +++ b/src/graph/executor/admin/ChangePasswordExecutor.cpp @@ -5,6 +5,7 @@ #include "graph/executor/admin/ChangePasswordExecutor.h" #include + #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/ConfigExecutor.cpp b/src/graph/executor/admin/ConfigExecutor.cpp index 3d0525baddf..1b91e4ff177 100644 --- a/src/graph/executor/admin/ConfigExecutor.cpp +++ b/src/graph/executor/admin/ConfigExecutor.cpp @@ -5,6 +5,7 @@ #include "graph/executor/admin/ConfigExecutor.h" #include + #include "common/conf/Configuration.h" #include "graph/planner/plan/Admin.h" #include "graph/util/SchemaUtil.h" diff --git a/src/graph/executor/admin/CreateUserExecutor.cpp b/src/graph/executor/admin/CreateUserExecutor.cpp index b5e9c29c2ca..34d9984c780 100644 --- a/src/graph/executor/admin/CreateUserExecutor.cpp +++ b/src/graph/executor/admin/CreateUserExecutor.cpp @@ -5,6 +5,7 @@ #include "graph/executor/admin/CreateUserExecutor.h" #include + #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/DescribeUserExecutor.cpp b/src/graph/executor/admin/DescribeUserExecutor.cpp index 6353287f985..e1a3f8e473e 100644 --- a/src/graph/executor/admin/DescribeUserExecutor.cpp +++ b/src/graph/executor/admin/DescribeUserExecutor.cpp @@ -5,6 +5,7 @@ #include "graph/executor/admin/DescribeUserExecutor.h" #include + #include "graph/planner/plan/Admin.h" #include "interface/gen-cpp2/meta_types.h" diff --git a/src/graph/executor/admin/DropUserExecutor.cpp b/src/graph/executor/admin/DropUserExecutor.cpp index 21051b6e125..23015981870 100644 --- a/src/graph/executor/admin/DropUserExecutor.cpp +++ b/src/graph/executor/admin/DropUserExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/DropUserExecutor.h" + #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/KillQueryExecutor.cpp b/src/graph/executor/admin/KillQueryExecutor.cpp index 47628803d55..55d827c6316 100644 --- a/src/graph/executor/admin/KillQueryExecutor.cpp +++ b/src/graph/executor/admin/KillQueryExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/KillQueryExecutor.h" + #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/ListRolesExecutor.cpp b/src/graph/executor/admin/ListRolesExecutor.cpp index 17d6b22f85b..b2e3639ec80 100644 --- a/src/graph/executor/admin/ListRolesExecutor.cpp +++ b/src/graph/executor/admin/ListRolesExecutor.cpp @@ -5,6 +5,7 @@ #include "graph/executor/admin/ListRolesExecutor.h" #include + #include "graph/planner/plan/Admin.h" #include "graph/service/PermissionManager.h" diff --git a/src/graph/executor/admin/ListUserRolesExecutor.cpp b/src/graph/executor/admin/ListUserRolesExecutor.cpp index 201a3b8fcf4..e5eefb02135 100644 --- a/src/graph/executor/admin/ListUserRolesExecutor.cpp +++ b/src/graph/executor/admin/ListUserRolesExecutor.cpp @@ -5,6 +5,7 @@ #include "graph/executor/admin/ListUserRolesExecutor.h" #include + #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/PartExecutor.cpp b/src/graph/executor/admin/PartExecutor.cpp index dfd88907206..5868b0e6d40 100644 --- a/src/graph/executor/admin/PartExecutor.cpp +++ b/src/graph/executor/admin/PartExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/PartExecutor.h" + #include "graph/planner/plan/Admin.h" using nebula::network::NetworkUtils; diff --git a/src/graph/executor/admin/RevokeRoleExecutor.cpp b/src/graph/executor/admin/RevokeRoleExecutor.cpp index be59f42166a..3182d75eb40 100644 --- a/src/graph/executor/admin/RevokeRoleExecutor.cpp +++ b/src/graph/executor/admin/RevokeRoleExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/RevokeRoleExecutor.h" + #include "graph/planner/plan/Admin.h" #include "graph/service/PermissionManager.h" #include "interface/gen-cpp2/meta_types.h" diff --git a/src/graph/executor/admin/ShowHostsExecutor.cpp b/src/graph/executor/admin/ShowHostsExecutor.cpp index 23860c5fa3d..999214c858c 100644 --- a/src/graph/executor/admin/ShowHostsExecutor.cpp +++ b/src/graph/executor/admin/ShowHostsExecutor.cpp @@ -5,6 +5,7 @@ #include "graph/executor/admin/ShowHostsExecutor.h" #include + #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/ShowMetaLeaderExecutor.cpp b/src/graph/executor/admin/ShowMetaLeaderExecutor.cpp index 21a43387a22..86a95241915 100644 --- a/src/graph/executor/admin/ShowMetaLeaderExecutor.cpp +++ b/src/graph/executor/admin/ShowMetaLeaderExecutor.cpp @@ -5,6 +5,7 @@ #include "graph/executor/admin/ShowMetaLeaderExecutor.h" #include + #include "common/time/TimeUtils.h" #include "graph/planner/plan/Admin.h" diff --git a/src/graph/executor/admin/ShowServiceClientsExecutor.cpp b/src/graph/executor/admin/ShowServiceClientsExecutor.cpp index 25f7425de20..651763f2dc3 100644 --- a/src/graph/executor/admin/ShowServiceClientsExecutor.cpp +++ b/src/graph/executor/admin/ShowServiceClientsExecutor.cpp @@ -5,6 +5,7 @@ #include "graph/executor/admin/ShowServiceClientsExecutor.h" #include + #include "graph/planner/plan/Admin.h" #include "graph/service/PermissionManager.h" #include "interface/gen-cpp2/meta_types.h" diff --git a/src/graph/executor/admin/ShowStatsExecutor.cpp b/src/graph/executor/admin/ShowStatsExecutor.cpp index d9c0715c8eb..56f60ba53a2 100644 --- a/src/graph/executor/admin/ShowStatsExecutor.cpp +++ b/src/graph/executor/admin/ShowStatsExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/ShowStatsExecutor.h" + #include "graph/planner/plan/Admin.h" #include "graph/service/PermissionManager.h" #include "graph/util/SchemaUtil.h" diff --git a/src/graph/executor/admin/SignInServiceExecutor.cpp b/src/graph/executor/admin/SignInServiceExecutor.cpp index 34dfbd86d45..10d4b6ee130 100644 --- a/src/graph/executor/admin/SignInServiceExecutor.cpp +++ b/src/graph/executor/admin/SignInServiceExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/SignInServiceExecutor.h" + #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/admin/SnapshotExecutor.cpp b/src/graph/executor/admin/SnapshotExecutor.cpp index 2e7f81e4466..477de45ddc0 100644 --- a/src/graph/executor/admin/SnapshotExecutor.cpp +++ b/src/graph/executor/admin/SnapshotExecutor.cpp @@ -3,7 +3,9 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/admin/SnapshotExecutor.h" + #include + #include "graph/planner/plan/Admin.h" namespace nebula { diff --git a/src/graph/executor/logic/ArgumentExecutor.cpp b/src/graph/executor/logic/ArgumentExecutor.cpp index 0c1cd69c533..941690efab0 100644 --- a/src/graph/executor/logic/ArgumentExecutor.cpp +++ b/src/graph/executor/logic/ArgumentExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/logic/ArgumentExecutor.h" + #include "graph/planner/plan/Logic.h" namespace nebula { diff --git a/src/graph/executor/logic/LoopExecutor.cpp b/src/graph/executor/logic/LoopExecutor.cpp index b07687d3b41..e45faaa7696 100644 --- a/src/graph/executor/logic/LoopExecutor.cpp +++ b/src/graph/executor/logic/LoopExecutor.cpp @@ -3,8 +3,8 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/logic/LoopExecutor.h" -#include "graph/planner/plan/Logic.h" +#include "graph/planner/plan/Logic.h" namespace nebula { namespace graph { diff --git a/src/graph/executor/logic/SelectExecutor.cpp b/src/graph/executor/logic/SelectExecutor.cpp index 5fd0810311d..3df0d48a066 100644 --- a/src/graph/executor/logic/SelectExecutor.cpp +++ b/src/graph/executor/logic/SelectExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/logic/SelectExecutor.h" + #include "graph/planner/plan/Logic.h" namespace nebula { diff --git a/src/graph/executor/maintain/EdgeExecutor.cpp b/src/graph/executor/maintain/EdgeExecutor.cpp index 81ed6c1d8aa..ed4f144141d 100644 --- a/src/graph/executor/maintain/EdgeExecutor.cpp +++ b/src/graph/executor/maintain/EdgeExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/maintain/EdgeExecutor.h" + #include "graph/planner/plan/Maintain.h" #include "graph/util/SchemaUtil.h" diff --git a/src/graph/executor/maintain/FTIndexExecutor.cpp b/src/graph/executor/maintain/FTIndexExecutor.cpp index 78178142d14..4b32d3939c6 100644 --- a/src/graph/executor/maintain/FTIndexExecutor.cpp +++ b/src/graph/executor/maintain/FTIndexExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/maintain/FTIndexExecutor.h" + #include "graph/planner/plan/Maintain.h" #include "graph/util/FTIndexUtils.h" #include "interface/gen-cpp2/meta_types.h" diff --git a/src/graph/executor/maintain/TagExecutor.cpp b/src/graph/executor/maintain/TagExecutor.cpp index 2b3c842b227..c30578388dd 100644 --- a/src/graph/executor/maintain/TagExecutor.cpp +++ b/src/graph/executor/maintain/TagExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/maintain/TagExecutor.h" + #include "graph/planner/plan/Maintain.h" #include "graph/util/SchemaUtil.h" diff --git a/src/graph/executor/mutate/DeleteExecutor.cpp b/src/graph/executor/mutate/DeleteExecutor.cpp index 33341fd352d..b3df2670e68 100644 --- a/src/graph/executor/mutate/DeleteExecutor.cpp +++ b/src/graph/executor/mutate/DeleteExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/mutate/DeleteExecutor.h" + #include "graph/planner/plan/Mutate.h" #include "graph/service/GraphFlags.h" #include "graph/util/SchemaUtil.h" diff --git a/src/graph/executor/mutate/InsertExecutor.cpp b/src/graph/executor/mutate/InsertExecutor.cpp index 14b75e8aff3..2b082c35d97 100644 --- a/src/graph/executor/mutate/InsertExecutor.cpp +++ b/src/graph/executor/mutate/InsertExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/mutate/InsertExecutor.h" + #include "graph/planner/plan/Mutate.h" #include "graph/service/GraphFlags.h" diff --git a/src/graph/executor/query/AggregateExecutor.cpp b/src/graph/executor/query/AggregateExecutor.cpp index 5b36b8a9f66..76c54278359 100644 --- a/src/graph/executor/query/AggregateExecutor.cpp +++ b/src/graph/executor/query/AggregateExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/AggregateExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/AssignExecutor.cpp b/src/graph/executor/query/AssignExecutor.cpp index 016fbbb59fe..646d87d1003 100644 --- a/src/graph/executor/query/AssignExecutor.cpp +++ b/src/graph/executor/query/AssignExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/AssignExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/DataCollectExecutor.cpp b/src/graph/executor/query/DataCollectExecutor.cpp index 4edef430b23..fc14157221e 100644 --- a/src/graph/executor/query/DataCollectExecutor.cpp +++ b/src/graph/executor/query/DataCollectExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/DataCollectExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/FilterExecutor.cpp b/src/graph/executor/query/FilterExecutor.cpp index 87d9775c9bb..4c9482a7061 100644 --- a/src/graph/executor/query/FilterExecutor.cpp +++ b/src/graph/executor/query/FilterExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/FilterExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/GetEdgesExecutor.cpp b/src/graph/executor/query/GetEdgesExecutor.cpp index 1972d0f1fd4..a129df1d3d2 100644 --- a/src/graph/executor/query/GetEdgesExecutor.cpp +++ b/src/graph/executor/query/GetEdgesExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/GetEdgesExecutor.h" + #include "graph/planner/plan/Query.h" using nebula::storage::StorageClient; diff --git a/src/graph/executor/query/GetNeighborsExecutor.cpp b/src/graph/executor/query/GetNeighborsExecutor.cpp index d2aff232875..9c90e66c578 100644 --- a/src/graph/executor/query/GetNeighborsExecutor.cpp +++ b/src/graph/executor/query/GetNeighborsExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/GetNeighborsExecutor.h" + #include "graph/service/GraphFlags.h" using nebula::storage::StorageClient; diff --git a/src/graph/executor/query/IndexScanExecutor.cpp b/src/graph/executor/query/IndexScanExecutor.cpp index 39631767b83..c74f715b037 100644 --- a/src/graph/executor/query/IndexScanExecutor.cpp +++ b/src/graph/executor/query/IndexScanExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/IndexScanExecutor.h" + #include "graph/service/GraphFlags.h" using nebula::storage::StorageClient; diff --git a/src/graph/executor/query/InnerJoinExecutor.cpp b/src/graph/executor/query/InnerJoinExecutor.cpp index acebeabb3ce..e52c40b6cbd 100644 --- a/src/graph/executor/query/InnerJoinExecutor.cpp +++ b/src/graph/executor/query/InnerJoinExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/InnerJoinExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/IntersectExecutor.cpp b/src/graph/executor/query/IntersectExecutor.cpp index 5e2ae7fbe66..dfb1216a257 100644 --- a/src/graph/executor/query/IntersectExecutor.cpp +++ b/src/graph/executor/query/IntersectExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/IntersectExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/JoinExecutor.cpp b/src/graph/executor/query/JoinExecutor.cpp index c3d73cc487e..11fd5a18e71 100644 --- a/src/graph/executor/query/JoinExecutor.cpp +++ b/src/graph/executor/query/JoinExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/JoinExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/LeftJoinExecutor.cpp b/src/graph/executor/query/LeftJoinExecutor.cpp index 06725b00a62..fd43ac30139 100644 --- a/src/graph/executor/query/LeftJoinExecutor.cpp +++ b/src/graph/executor/query/LeftJoinExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/LeftJoinExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/LimitExecutor.cpp b/src/graph/executor/query/LimitExecutor.cpp index 83937194400..38502d2bf3b 100644 --- a/src/graph/executor/query/LimitExecutor.cpp +++ b/src/graph/executor/query/LimitExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/LimitExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/MinusExecutor.cpp b/src/graph/executor/query/MinusExecutor.cpp index d2b4e600f80..0ce5b9d9045 100644 --- a/src/graph/executor/query/MinusExecutor.cpp +++ b/src/graph/executor/query/MinusExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/MinusExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/ProjectExecutor.cpp b/src/graph/executor/query/ProjectExecutor.cpp index bae734c8d49..b4f80b9b484 100644 --- a/src/graph/executor/query/ProjectExecutor.cpp +++ b/src/graph/executor/query/ProjectExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/ProjectExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/SampleExecutor.cpp b/src/graph/executor/query/SampleExecutor.cpp index f955f2d4371..66cfcc9fd94 100644 --- a/src/graph/executor/query/SampleExecutor.cpp +++ b/src/graph/executor/query/SampleExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/SampleExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/ScanEdgesExecutor.cpp b/src/graph/executor/query/ScanEdgesExecutor.cpp index aa4a50201b3..de354a8748f 100644 --- a/src/graph/executor/query/ScanEdgesExecutor.cpp +++ b/src/graph/executor/query/ScanEdgesExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/ScanEdgesExecutor.h" + #include "graph/planner/plan/Query.h" #include "graph/util/SchemaUtil.h" diff --git a/src/graph/executor/query/SetExecutor.cpp b/src/graph/executor/query/SetExecutor.cpp index e0d7276087a..3681536a873 100644 --- a/src/graph/executor/query/SetExecutor.cpp +++ b/src/graph/executor/query/SetExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/SetExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/SortExecutor.cpp b/src/graph/executor/query/SortExecutor.cpp index 0e9086d8027..ebeba5cd854 100644 --- a/src/graph/executor/query/SortExecutor.cpp +++ b/src/graph/executor/query/SortExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/SortExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/TopNExecutor.cpp b/src/graph/executor/query/TopNExecutor.cpp index 150765d49f9..e685a8ff31b 100644 --- a/src/graph/executor/query/TopNExecutor.cpp +++ b/src/graph/executor/query/TopNExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/TopNExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/TraverseExecutor.cpp b/src/graph/executor/query/TraverseExecutor.cpp index 2460012dbf0..0210a5ce6c5 100644 --- a/src/graph/executor/query/TraverseExecutor.cpp +++ b/src/graph/executor/query/TraverseExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/TraverseExecutor.h" + #include "clients/storage/StorageClient.h" #include "graph/service/GraphFlags.h" #include "graph/util/SchemaUtil.h" diff --git a/src/graph/executor/query/UnionAllVersionVarExecutor.cpp b/src/graph/executor/query/UnionAllVersionVarExecutor.cpp index 92557f31803..019b1b6261a 100644 --- a/src/graph/executor/query/UnionAllVersionVarExecutor.cpp +++ b/src/graph/executor/query/UnionAllVersionVarExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/UnionAllVersionVarExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/UnionExecutor.cpp b/src/graph/executor/query/UnionExecutor.cpp index b1731964c11..25e203ab632 100644 --- a/src/graph/executor/query/UnionExecutor.cpp +++ b/src/graph/executor/query/UnionExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/UnionExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/graph/executor/query/UnwindExecutor.cpp b/src/graph/executor/query/UnwindExecutor.cpp index 28fb8aa5718..de1d774e0a7 100644 --- a/src/graph/executor/query/UnwindExecutor.cpp +++ b/src/graph/executor/query/UnwindExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/query/UnwindExecutor.h" + #include "graph/planner/plan/Query.h" namespace nebula { diff --git a/src/interface/common.thrift b/src/interface/common.thrift index 86b262e0910..5214b5b63a9 100644 --- a/src/interface/common.thrift +++ b/src/interface/common.thrift @@ -391,6 +391,7 @@ enum ErrorCode { E_JOB_NOT_FINISHED = -2048, E_TASK_REPORT_OUT_DATE = -2049, E_JOB_NOT_IN_SPACE = -2050, + E_JOB_NEED_RECOVER = -2051, E_INVALID_JOB = -2065, // Backup Failure diff --git a/src/meta/processors/job/AdminJobProcessor.cpp b/src/meta/processors/job/AdminJobProcessor.cpp index b073ffb4ecf..dfd3e96ae33 100644 --- a/src/meta/processors/job/AdminJobProcessor.cpp +++ b/src/meta/processors/job/AdminJobProcessor.cpp @@ -90,7 +90,7 @@ void AdminJobProcessor::process(const cpp2::AdminJobReq& req) { break; } case nebula::meta::cpp2::JobOp::RECOVER: { - // Note that the last parameter is no longer spaceId + // Note that the last parameter is no longer spaceName std::vector jobIds; jobIds.reserve(paras.size()); for (size_t i = 0; i < paras.size(); i++) { @@ -125,13 +125,19 @@ nebula::cpp2::ErrorCode AdminJobProcessor::addJobProcess(const cpp2::AdminJobReq // Check if job not exists JobID jId = 0; - auto jobExist = jobMgr_->checkJobExist(spaceId_, type, paras, jId); - if (jobExist) { + auto runningJobExist = jobMgr_->checkOnRunningJobExist(spaceId_, type, paras, jId); + if (runningJobExist) { LOG(INFO) << "Job has already exists: " << jId; result.job_id_ref() = jId; return nebula::cpp2::ErrorCode::SUCCEEDED; } + auto retCode = jobMgr_->checkNeedRecoverJobExist(spaceId_, type); + if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(INFO) << "There is a failed data balance or zone balance job, need to recover it firstly!"; + return retCode; + } + folly::SharedMutex::WriteHolder holder(LockUtils::lock()); auto jobId = autoIncrementId(); if (!nebula::ok(jobId)) { diff --git a/src/meta/processors/job/JobDescription.h b/src/meta/processors/job/JobDescription.h index a1328aa8d9a..16c505d61a6 100644 --- a/src/meta/processors/job/JobDescription.h +++ b/src/meta/processors/job/JobDescription.h @@ -159,8 +159,7 @@ class JobDescription { cpp2::JobDesc toJobDesc(); bool operator==(const JobDescription& that) const { - return space_ == that.space_ && type_ == that.type_ && paras_ == that.paras_ && - status_ == that.status_; + return space_ == that.space_ && type_ == that.type_ && paras_ == that.paras_; } bool operator!=(const JobDescription& that) const { diff --git a/src/meta/processors/job/JobManager.cpp b/src/meta/processors/job/JobManager.cpp index d6967693e98..d3e32a4b078 100644 --- a/src/meta/processors/job/JobManager.cpp +++ b/src/meta/processors/job/JobManager.cpp @@ -248,7 +248,7 @@ nebula::cpp2::ErrorCode JobManager::jobFinished(GraphSpaceID spaceId, // Set the errorcode of the job nebula::cpp2::ErrorCode jobErrCode = nebula::cpp2::ErrorCode::SUCCEEDED; - if (jobStatus != cpp2::JobStatus::FINISHED) { + if (jobStatus == cpp2::JobStatus::FAILED) { // Traverse the tasks and find the first task errorcode unsuccessful auto jobKey = MetaKeyUtils::jobKey(spaceId, jobId); std::unique_ptr iter; @@ -571,10 +571,10 @@ nebula::cpp2::ErrorCode JobManager::removeExpiredJobs( return ret; } -bool JobManager::checkJobExist(GraphSpaceID spaceId, - const cpp2::JobType& jobType, - const std::vector& paras, - JobID& jobId) { +bool JobManager::checkOnRunningJobExist(GraphSpaceID spaceId, + const cpp2::JobType& jobType, + const std::vector& paras, + JobID& jobId) { JobDescription jobDesc(spaceId, 0, jobType, paras); auto it = inFlightJobs_.begin(); while (it != inFlightJobs_.end()) { @@ -587,6 +587,35 @@ bool JobManager::checkJobExist(GraphSpaceID spaceId, return false; } +nebula::cpp2::ErrorCode JobManager::checkNeedRecoverJobExist(GraphSpaceID spaceId, + const cpp2::JobType& jobType) { + if (jobType == cpp2::JobType::DATA_BALANCE || jobType == cpp2::JobType::ZONE_BALANCE) { + std::unique_ptr iter; + auto jobPre = MetaKeyUtils::jobPrefix(spaceId); + auto retCode = kvStore_->prefix(kDefaultSpaceId, kDefaultPartId, jobPre, &iter); + if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(INFO) << "Fetch jobs failed, error: " << apache::thrift::util::enumNameSafe(retCode); + return retCode; + } + + for (; iter->valid(); iter->next()) { + if (!MetaKeyUtils::isJobKey(iter->key())) { + continue; + } + auto tup = MetaKeyUtils::parseJobVal(iter->val()); + auto type = std::get<0>(tup); + auto status = std::get<2>(tup); + if (type == cpp2::JobType::DATA_BALANCE || type == cpp2::JobType::ZONE_BALANCE) { + // QUEUE: The job has not been executed, the machine restarted + if (status == cpp2::JobStatus::FAILED || status == cpp2::JobStatus::QUEUE) { + return nebula::cpp2::ErrorCode::E_JOB_NEED_RECOVER; + } + } + } + } + return nebula::cpp2::ErrorCode::SUCCEEDED; +} + ErrorOr>> JobManager::showJob(GraphSpaceID spaceId, JobID jobId) { auto jobKey = MetaKeyUtils::jobKey(spaceId, jobId); @@ -661,6 +690,7 @@ ErrorOr JobManager::recoverJob( } else { std::vector jobKeys; jobKeys.reserve(jobIds.size()); + std::vector> totalJobKVs; for (int jobId : jobIds) { jobKeys.emplace_back(MetaKeyUtils::jobKey(spaceId, jobId)); } @@ -671,7 +701,132 @@ ErrorOr JobManager::recoverJob( return retCode.first; } for (size_t i = 0; i < jobKeys.size(); i++) { - jobKVs.emplace_back(std::make_pair(jobKeys[i], jobVals[i])); + totalJobKVs.emplace_back(std::make_pair(jobKeys[i], jobVals[i])); + } + + // For DATA_BALANCE and ZONE_BALANCE job, jobs with STOPPED, FAILED, QUEUE status + // !!! The following situations can be recovered, only for jobs of the same type + // of DATA_BALANCE or ZONE_BALANCE。 + // QUEUE: The job has not been executed, then the machine restarted. + // FAILED: + // The failed job will be recovered. + // FAILED and QUEUE jobs will not exist at the same time. + // STOPPED: + // If only one stopped jobId is specified, No FINISHED job or FAILED job of the + // same type after this job. + // If multiple jobs of the same type are specified, only starttime latest jobId + // will can be recovered, no FINISHED job or FAILED job of the same type after this latest job. + // The same type of STOPPED job exists in the following form, sorted by starttime: + // STOPPED job1, FAILED job2 + // recover job job1 failed + // recover job job2 success + // STOPPED job1, FINISHED job2, STOPPED job3 + // recover job job1 failed + // recover job job3 success + // recover job job1,job3 Only job3 can recover + std::unordered_map> dupResult; + std::unordered_map> dupkeyVal; + + for (auto& jobkv : totalJobKVs) { + auto optJobRet = JobDescription::makeJobDescription(jobkv.first, jobkv.second); + if (nebula::ok(optJobRet)) { + auto optJob = nebula::value(optJobRet); + auto jobStatus = optJob.getStatus(); + auto jobId = optJob.getJobId(); + auto jobType = optJob.getJobType(); + auto jobStartTime = optJob.getStartTime(); + if (jobStatus != cpp2::JobStatus::QUEUE && jobStatus != cpp2::JobStatus::FAILED && + jobStatus != cpp2::JobStatus::STOPPED) { + continue; + } + + // handle DATA_BALANCE and ZONE_BALANCE + if (jobType == cpp2::JobType::DATA_BALANCE || jobType == cpp2::JobType::ZONE_BALANCE) { + // FAILED and QUEUE jobs will not exist at the same time. + if (jobStatus == cpp2::JobStatus::FAILED || jobStatus == cpp2::JobStatus::QUEUE) { + dupResult[jobType] = std::make_tuple(jobId, jobStartTime, jobStatus); + dupkeyVal.emplace(jobId, std::make_pair(jobkv.first, jobkv.second)); + continue; + } + + // current recover job status is stopped + auto findJobIter = dupResult.find(jobType); + if (findJobIter != dupResult.end()) { + auto oldJobInfo = findJobIter->second; + if (std::get<2>(oldJobInfo) != cpp2::JobStatus::STOPPED) { + continue; + } + } + + // For a stopped job, check whether there is the same type of finished or + // failed job after it. + std::unique_ptr iter; + auto jobPre = MetaKeyUtils::jobPrefix(spaceId); + auto code = kvStore_->prefix(kDefaultSpaceId, kDefaultPartId, jobPre, &iter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(INFO) << "Fetch jobs failed, error: " << apache::thrift::util::enumNameSafe(code); + return code; + } + + bool findRest = false; + for (; iter->valid(); iter->next()) { + if (!MetaKeyUtils::isJobKey(iter->key())) { + continue; + } + + // eliminate oneself + auto keyPair = MetaKeyUtils::parseJobKey(iter->key()); + auto destJobId = keyPair.second; + if (destJobId == jobId) { + continue; + } + auto tup = MetaKeyUtils::parseJobVal(iter->val()); + auto destJobType = std::get<0>(tup); + auto destJobStatus = std::get<2>(tup); + auto destJobStartTime = std::get<3>(tup); + if (jobType == destJobType) { + // There is a specific type of failed job that does not allow recovery for the type of + // stopped job + if (destJobStatus == cpp2::JobStatus::FAILED) { + LOG(ERROR) << "There is a specific type of failed job that does not allow recovery " + "for the type of stopped job"; + findRest = true; + break; + } else if (destJobStatus == cpp2::JobStatus::FINISHED) { + // Compare the start time of the job + if (destJobStartTime > jobStartTime) { + findRest = true; + break; + } + } + } + } + if (!findRest) { + auto findStoppedJobIter = dupResult.find(jobType); + if (findStoppedJobIter != dupResult.end()) { + // update stopped job + auto oldJobInfo = findStoppedJobIter->second; + auto oldJobStartTime = std::get<1>(oldJobInfo); + if (jobStartTime > oldJobStartTime) { + auto oldJobId = std::get<0>(oldJobInfo); + dupResult[jobType] = std::make_tuple(jobId, jobStartTime, jobStatus); + dupkeyVal.erase(oldJobId); + dupkeyVal.emplace(jobId, std::make_pair(jobkv.first, jobkv.second)); + } + } else { + // insert + dupResult[jobType] = std::make_tuple(jobId, jobStartTime, jobStatus); + dupkeyVal.emplace(jobId, std::make_pair(jobkv.first, jobkv.second)); + } + } + } else { + jobKVs.emplace_back(std::make_pair(jobkv.first, jobkv.second)); + } + } + } + for (auto& key : dupResult) { + auto jId = std::get<0>(key.second); + jobKVs.emplace_back(dupkeyVal[jId]); } } @@ -684,7 +839,8 @@ ErrorOr JobManager::recoverJob( optJob.getStatus() == cpp2::JobStatus::STOPPED))) { // Check if the job exists JobID jId = 0; - auto jobExist = checkJobExist(spaceId, optJob.getJobType(), optJob.getParas(), jId); + auto jobExist = + checkOnRunningJobExist(spaceId, optJob.getJobType(), optJob.getParas(), jId); if (!jobExist) { auto jobId = optJob.getJobId(); enqueue(spaceId, jobId, JbOp::RECOVER, optJob.getJobType()); diff --git a/src/meta/processors/job/JobManager.h b/src/meta/processors/job/JobManager.h index 22371b4df8b..3fc6b5c965e 100644 --- a/src/meta/processors/job/JobManager.h +++ b/src/meta/processors/job/JobManager.h @@ -92,18 +92,31 @@ class JobManager : public boost::noncopyable, public nebula::cpp::NonMovable { nebula::cpp2::ErrorCode addJob(JobDescription& jobDesc, AdminClient* client); /** - * @brief The same job is in jobMap + * @brief The same job in inFlightJobs_. + * Jobs in inFlightJobs_ have three status: + * QUEUE: when adding a job + * FAILED or STOPPED: when recover job * * @param spaceId * @param type * @param paras * @param jobId If the job exists, jobId is the id of the existing job - * @return + * @return True if job exists. + */ + bool checkOnRunningJobExist(GraphSpaceID spaceId, + const cpp2::JobType& type, + const std::vector& paras, + JobID& jobId); + /** + * @brief In the current space, if there is a failed data balance job or zone balance job, + * need to recover the job first, otherwise cannot add this type of job. + * + * @param spaceId + * @param jobType + * @return nebula::cpp2::ErrorCode */ - bool checkJobExist(GraphSpaceID spaceId, - const cpp2::JobType& type, - const std::vector& paras, - JobID& jobId); + nebula::cpp2::ErrorCode checkNeedRecoverJobExist(GraphSpaceID spaceId, + const cpp2::JobType& jobType); /** * @brief Load all jobs of the space from kvStore and convert to cpp2::JobDesc diff --git a/src/meta/test/JobManagerTest.cpp b/src/meta/test/JobManagerTest.cpp index 4587e1758ed..48761f3223c 100644 --- a/src/meta/test/JobManagerTest.cpp +++ b/src/meta/test/JobManagerTest.cpp @@ -90,6 +90,22 @@ TEST_F(JobManagerTest, AddJob) { JobDescription jobDesc(spaceId, jobId, cpp2::JobType::COMPACT); auto rc = jobMgr->addJob(jobDesc, adminClient_.get()); ASSERT_EQ(rc, nebula::cpp2::ErrorCode::SUCCEEDED); + + // If there is a failed data balance job, a new job cannot be added + JobID jobId3 = 3; + JobDescription jobDesc3(spaceId, jobId3, cpp2::JobType::DATA_BALANCE); + jobDesc3.setStatus(cpp2::JobStatus::FAILED); + auto jobKey = MetaKeyUtils::jobKey(jobDesc3.getSpace(), jobDesc3.getJobId()); + auto jobVal = MetaKeyUtils::jobVal(jobDesc3.getJobType(), + jobDesc3.getParas(), + jobDesc3.getStatus(), + jobDesc3.getStartTime(), + jobDesc3.getStopTime(), + jobDesc3.getErrorCode()); + jobMgr->save(std::move(jobKey), std::move(jobVal)); + + rc = jobMgr->checkNeedRecoverJobExist(spaceId, jobDesc3.getJobType()); + ASSERT_EQ(rc, nebula::cpp2::ErrorCode::E_JOB_NEED_RECOVER); } TEST_F(JobManagerTest, AddRebuildTagIndexJob) { @@ -287,7 +303,8 @@ TEST_F(JobManagerTest, JobDeduplication) { JobID jobId3 = 17; JobDescription jobDesc3(spaceId, jobId3, cpp2::JobType::LEADER_BALANCE); JobID jId3 = 0; - auto jobExist = jobMgr->checkJobExist(spaceId, jobDesc3.getJobType(), jobDesc3.getParas(), jId3); + auto jobExist = + jobMgr->checkOnRunningJobExist(spaceId, jobDesc3.getJobType(), jobDesc3.getParas(), jId3); if (!jobExist) { auto rc3 = jobMgr->addJob(jobDesc3, adminClient_.get()); ASSERT_EQ(rc3, nebula::cpp2::ErrorCode::SUCCEEDED); @@ -296,7 +313,8 @@ TEST_F(JobManagerTest, JobDeduplication) { JobID jobId4 = 18; JobDescription jobDesc4(spaceId, jobId4, cpp2::JobType::COMPACT); JobID jId4 = 0; - jobExist = jobMgr->checkJobExist(spaceId, jobDesc4.getJobType(), jobDesc4.getParas(), jId4); + jobExist = + jobMgr->checkOnRunningJobExist(spaceId, jobDesc4.getJobType(), jobDesc4.getParas(), jId4); if (!jobExist) { auto rc4 = jobMgr->addJob(jobDesc4, adminClient_.get()); ASSERT_NE(rc4, nebula::cpp2::ErrorCode::SUCCEEDED); @@ -524,26 +542,101 @@ TEST_F(JobManagerTest, ShowJobInOtherSpace) { } TEST_F(JobManagerTest, RecoverJob) { - std::unique_ptr> jobMgr = getJobManager(); - // set status to prevent running the job since AdminClient is a injector - jobMgr->status_ = JobManager::JbmgrStatus::STOPPED; - jobMgr->bgThread_.join(); - GraphSpaceID spaceId = 1; - int32_t nJob = 3; - for (auto jobId = 0; jobId < nJob; ++jobId) { - JobDescription jd(spaceId, jobId, cpp2::JobType::FLUSH); - auto jobKey = MetaKeyUtils::jobKey(jd.getSpace(), jd.getJobId()); - auto jobVal = MetaKeyUtils::jobVal(jd.getJobType(), - jd.getParas(), - jd.getStatus(), - jd.getStartTime(), - jd.getStopTime(), - jd.getErrorCode()); - jobMgr->save(jobKey, jobVal); + // case 1,recover Queue status job + { + std::unique_ptr> jobMgr = getJobManager(); + // set status to prevent running the job since AdminClient is a injector + jobMgr->status_ = JobManager::JbmgrStatus::STOPPED; + jobMgr->bgThread_.join(); + GraphSpaceID spaceId = 1; + int32_t nJob = 3; + for (auto jobId = 0; jobId < nJob; ++jobId) { + JobDescription jd(spaceId, jobId, cpp2::JobType::FLUSH); + auto jobKey = MetaKeyUtils::jobKey(jd.getSpace(), jd.getJobId()); + auto jobVal = MetaKeyUtils::jobVal(jd.getJobType(), + jd.getParas(), + jd.getStatus(), + jd.getStartTime(), + jd.getStopTime(), + jd.getErrorCode()); + jobMgr->save(jobKey, jobVal); + } + + auto nJobRecovered = jobMgr->recoverJob(spaceId, nullptr); + ASSERT_EQ(nebula::value(nJobRecovered), 1); + } + + // case 2 + // For the same type of job, if there are stopped jobs and failed jobs in turn + // recover stopped job failed + // recover failed job succeeded + { + std::unique_ptr> jobMgr = getJobManager(); + // set status to prevent running the job since AdminClient is a injector + jobMgr->status_ = JobManager::JbmgrStatus::STOPPED; + jobMgr->bgThread_.join(); + GraphSpaceID spaceId = 1; + int32_t nJob = 3; + for (auto jobId = 0; jobId < nJob; ++jobId) { + cpp2::JobStatus jobStatus; + if (jobId == 2) { + jobStatus = cpp2::JobStatus::FAILED; + } else { + jobStatus = cpp2::JobStatus::STOPPED; + } + JobDescription jd(spaceId, jobId, cpp2::JobType::DATA_BALANCE, {}, jobStatus); + auto jobKey = MetaKeyUtils::jobKey(jd.getSpace(), jd.getJobId()); + auto jobVal = MetaKeyUtils::jobVal(jd.getJobType(), + jd.getParas(), + jd.getStatus(), + jobId + 100000, + jobId + 110000, + jd.getErrorCode()); + jobMgr->save(jobKey, jobVal); + } + + auto nJobRecovered = jobMgr->recoverJob(spaceId, nullptr, {1}); + ASSERT_EQ(nebula::value(nJobRecovered), 0); + + nJobRecovered = jobMgr->recoverJob(spaceId, nullptr, {2}); + ASSERT_EQ(nebula::value(nJobRecovered), 1); } - auto nJobRecovered = jobMgr->recoverJob(spaceId, nullptr); - ASSERT_EQ(nebula::value(nJobRecovered), 1); + // case 3 + // For the same type of job, if there are stopped jobs and finished jobs, stopped in turn + // recover stopped job befor finished job failed + // recover stopped job after finished job succeeded + { + std::unique_ptr> jobMgr = getJobManager(); + // set status to prevent running the job since AdminClient is a injector + jobMgr->status_ = JobManager::JbmgrStatus::STOPPED; + jobMgr->bgThread_.join(); + GraphSpaceID spaceId = 1; + int32_t nJob = 4; + for (auto jobId = 0; jobId < nJob; ++jobId) { + cpp2::JobStatus jobStatus; + if (jobId == 2) { + jobStatus = cpp2::JobStatus::FINISHED; + } else { + jobStatus = cpp2::JobStatus::STOPPED; + } + JobDescription jd(spaceId, jobId, cpp2::JobType::DATA_BALANCE, {}, jobStatus); + auto jobKey = MetaKeyUtils::jobKey(jd.getSpace(), jd.getJobId()); + auto jobVal = MetaKeyUtils::jobVal(jd.getJobType(), + jd.getParas(), + jd.getStatus(), + jobId + 100000, + jobId + 110000, + jd.getErrorCode()); + jobMgr->save(jobKey, jobVal); + } + + auto nJobRecovered = jobMgr->recoverJob(spaceId, nullptr, {1}); + ASSERT_EQ(nebula::value(nJobRecovered), 0); + + nJobRecovered = jobMgr->recoverJob(spaceId, nullptr, {3}); + ASSERT_EQ(nebula::value(nJobRecovered), 1); + } } TEST(JobDescriptionTest, Ctor) { From 10b1b643e89851a6e264f649299923b265b62048 Mon Sep 17 00:00:00 2001 From: cpw <13495049+CPWstatic@users.noreply.github.com> Date: Mon, 11 Apr 2022 11:58:31 +0800 Subject: [PATCH 23/24] Fix group by outputs. (#4128) * Fix group by outputs. * Add test. * Fix format. Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/graph/validator/GroupByValidator.cpp | 6 +++++- tests/tck/features/aggregate/Agg.feature | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/graph/validator/GroupByValidator.cpp b/src/graph/validator/GroupByValidator.cpp index 69723bb03c3..8441b72668b 100644 --- a/src/graph/validator/GroupByValidator.cpp +++ b/src/graph/validator/GroupByValidator.cpp @@ -20,6 +20,11 @@ Status GroupByValidator::validateImpl() { NG_RETURN_IF_ERROR(validateYield(groupBySentence->yieldClause())); NG_RETURN_IF_ERROR(groupClauseSemanticCheck()); + for (auto* col : groupBySentence->yieldClause()->columns()) { + auto type = deduceExprType(col->expr()); + outputs_.emplace_back(col->name(), std::move(type).value()); + } + return Status::OK(); } @@ -155,7 +160,6 @@ Status GroupByValidator::groupClauseSemanticCheck() { for (auto i = 0u; i < groupItems_.size(); ++i) { auto type = deduceExprType(groupItems_[i]); NG_RETURN_IF_ERROR(type); - outputs_.emplace_back(aggOutputColNames_[i], std::move(type).value()); } // check exprProps if (!exprProps_.srcTagProps().empty() || !exprProps_.dstTagProps().empty()) { diff --git a/tests/tck/features/aggregate/Agg.feature b/tests/tck/features/aggregate/Agg.feature index 509f5109400..d678308e132 100644 --- a/tests/tck/features/aggregate/Agg.feature +++ b/tests/tck/features/aggregate/Agg.feature @@ -613,6 +613,21 @@ Feature: Basic Aggregate and GroupBy | sum | | 6 | + Scenario: Reference the output of group by + When executing query: + """ + GO FROM "Tim Duncan" OVER * YIELD dst(edge) as dst, $$.player.age as age + | GROUP BY $-.dst YIELD (sum($-.age)+3) as age + | ORDER BY $-.age + """ + Then the result should be, in order: + | age | + | 3 | + | 34 | + | 36 | + | 75 | + | 85 | + Scenario: Error Check When executing query: """ From d949fafb1c4c2142494c80d2878c77640368f022 Mon Sep 17 00:00:00 2001 From: liwenhui-soul <38217397+liwenhui-soul@users.noreply.github.com> Date: Mon, 11 Apr 2022 12:46:17 +0800 Subject: [PATCH 24/24] remove the conception of zone in all place (#4119) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/clients/meta/MetaClient.cpp | 4 +- src/graph/executor/admin/SpaceExecutor.cpp | 4 -- .../processors/job/ZoneBalanceJobExecutor.cpp | 18 +++--- .../processors/zone/AddHostsProcessor.cpp | 24 ++++++++ .../processors/zone/DropHostsProcessor.cpp | 15 ++++- src/meta/test/BalancerTest.cpp | 1 - src/parser/parser.yy | 61 +++++-------------- src/parser/test/ParserTest.cpp | 33 ---------- tests/tck/features/schema/Comment.feature | 12 ++-- 9 files changed, 67 insertions(+), 105 deletions(-) diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index d57d6012ea3..af2c3ef870e 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -846,9 +846,9 @@ Status MetaClient::handleResponse(const RESP& resp) { case nebula::cpp2::ErrorCode::E_WRONGCLUSTER: return Status::Error("Wrong cluster!"); case nebula::cpp2::ErrorCode::E_ZONE_NOT_ENOUGH: - return Status::Error("Zone not enough!"); + return Status::Error("Host not enough!"); case nebula::cpp2::ErrorCode::E_ZONE_IS_EMPTY: - return Status::Error("Zone is empty!"); + return Status::Error("Host not exist!"); case nebula::cpp2::ErrorCode::E_STORE_FAILURE: return Status::Error("Store failure!"); case nebula::cpp2::ErrorCode::E_BAD_BALANCE_PLAN: diff --git a/src/graph/executor/admin/SpaceExecutor.cpp b/src/graph/executor/admin/SpaceExecutor.cpp index 6028d6e899b..95f0602ba0d 100644 --- a/src/graph/executor/admin/SpaceExecutor.cpp +++ b/src/graph/executor/admin/SpaceExecutor.cpp @@ -77,7 +77,6 @@ folly::Future DescSpaceExecutor::execute() { "Collate", "Vid Type", "Atomic Edge", - "Zones", "Comment"}; Row row; row.values.emplace_back(spaceId); @@ -94,9 +93,6 @@ folly::Future DescSpaceExecutor::execute() { } row.values.emplace_back(sAtomicEdge); - auto zoneNames = folly::join(",", properties.get_zone_names()); - row.values.emplace_back(zoneNames); - if (properties.comment_ref().has_value()) { row.values.emplace_back(*properties.comment_ref()); } else { diff --git a/src/meta/processors/job/ZoneBalanceJobExecutor.cpp b/src/meta/processors/job/ZoneBalanceJobExecutor.cpp index 7d628f6a188..f37971cb6f0 100644 --- a/src/meta/processors/job/ZoneBalanceJobExecutor.cpp +++ b/src/meta/processors/job/ZoneBalanceJobExecutor.cpp @@ -30,6 +30,13 @@ nebula::cpp2::ErrorCode ZoneBalanceJobExecutor::prepare() { for (size_t i = 0; i < paras_.size(); i++) { lostZones_.emplace_back(paras_[i]); } + std::vector newZones; + newZones.reserve(lostZones_.size()); + for (auto& name : lostZones_) { + auto host = HostAddr::fromString(name); + newZones.emplace_back(folly::stringPrintf("default_zone_%s_%d", host.host.c_str(), host.port)); + } + lostZones_.swap(newZones); return nebula::cpp2::ErrorCode::SUCCEEDED; } @@ -49,16 +56,7 @@ folly::Future ZoneBalanceJobExecutor::executeInternal() { return status; } } - plan_->setFinishCallBack([this](meta::cpp2::JobStatus status) { - if (status == meta::cpp2::JobStatus::FINISHED) { - folly::SharedMutex::WriteHolder holder(LockUtils::lock()); - nebula::cpp2::ErrorCode ret = updateMeta(); - if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - status = meta::cpp2::JobStatus::FAILED; - } - } - executorOnFinished_(status); - }); + plan_->setFinishCallBack([this](meta::cpp2::JobStatus status) { executorOnFinished_(status); }); plan_->invoke(); return Status::OK(); } diff --git a/src/meta/processors/zone/AddHostsProcessor.cpp b/src/meta/processors/zone/AddHostsProcessor.cpp index b2af49e7a61..3d91ac194e8 100644 --- a/src/meta/processors/zone/AddHostsProcessor.cpp +++ b/src/meta/processors/zone/AddHostsProcessor.cpp @@ -34,6 +34,20 @@ void AddHostsProcessor::process(const cpp2::AddHostsReq& req) { std::vector data; nebula::cpp2::ErrorCode code = nebula::cpp2::ErrorCode::SUCCEEDED; + std::map spaceMap; + std::string spacePrefix = MetaKeyUtils::spacePrefix(); + std::unique_ptr spaceIter; + auto spaceRet = kvstore_->prefix(kDefaultSpaceId, kDefaultPartId, spacePrefix, &spaceIter); + if (spaceRet != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(spaceRet); + onFinished(); + return; + } + while (spaceIter->valid()) { + spaceMap.emplace(MetaKeyUtils::spaceId(spaceIter->key()), + MetaKeyUtils::parseSpace(spaceIter->val())); + spaceIter->next(); + } for (auto& host : hosts) { // Ensure that the node is not registered. auto machineKey = MetaKeyUtils::machineKey(host.host, host.port); @@ -56,6 +70,16 @@ void AddHostsProcessor::process(const cpp2::AddHostsReq& req) { auto zoneVal = MetaKeyUtils::zoneVal({host}); data.emplace_back(std::move(machineKey), ""); data.emplace_back(std::move(zoneKey), std::move(zoneVal)); + for (auto& [spaceId, properties] : spaceMap) { + const std::vector& curZones = properties.get_zone_names(); + std::set zm(curZones.begin(), curZones.end()); + zm.emplace(zoneName); + std::vector newZones(zm.begin(), zm.end()); + properties.zone_names_ref() = std::move(newZones); + } + } + for (auto& [spaceId, properties] : spaceMap) { + data.emplace_back(MetaKeyUtils::spaceKey(spaceId), MetaKeyUtils::spaceVal(properties)); } if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { diff --git a/src/meta/processors/zone/DropHostsProcessor.cpp b/src/meta/processors/zone/DropHostsProcessor.cpp index 2523f38fe85..c4a26e07112 100644 --- a/src/meta/processors/zone/DropHostsProcessor.cpp +++ b/src/meta/processors/zone/DropHostsProcessor.cpp @@ -34,6 +34,7 @@ void DropHostsProcessor::process(const cpp2::DropHostsReq& req) { auto spaceIterRet = doPrefix(spacePrefix); auto spaceIter = nebula::value(spaceIterRet).get(); nebula::cpp2::ErrorCode code = nebula::cpp2::ErrorCode::SUCCEEDED; + std::map spaceMap; while (spaceIter->valid()) { auto spaceId = MetaKeyUtils::spaceId(spaceIter->key()); auto spaceKey = MetaKeyUtils::spaceKey(spaceId); @@ -44,7 +45,7 @@ void DropHostsProcessor::process(const cpp2::DropHostsReq& req) { << " error: " << apache::thrift::util::enumNameSafe(code); break; } - + spaceMap.emplace(spaceId, MetaKeyUtils::parseSpace(spaceIter->val())); const auto& partPrefix = MetaKeyUtils::partPrefix(spaceId); auto partIterRet = doPrefix(partPrefix); auto partIter = nebula::value(partIterRet).get(); @@ -96,6 +97,14 @@ void DropHostsProcessor::process(const cpp2::DropHostsReq& req) { } holder->remove(MetaKeyUtils::zoneKey(zoneName)); + + for (auto& [spaceId, properties] : spaceMap) { + const std::vector& curZones = properties.get_zone_names(); + std::set zm(curZones.begin(), curZones.end()); + zm.erase(zoneName); + std::vector newZones(zm.begin(), zm.end()); + properties.zone_names_ref() = std::move(newZones); + } } else { // Delete some hosts in the zone for (auto& h : hosts) { @@ -112,7 +121,9 @@ void DropHostsProcessor::process(const cpp2::DropHostsReq& req) { CHECK_CODE_AND_BREAK(); iter->next(); } - + for (auto& [spaceId, properties] : spaceMap) { + holder->put(MetaKeyUtils::spaceKey(spaceId), MetaKeyUtils::spaceVal(properties)); + } if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { handleErrorCode(code); onFinished(); diff --git a/src/meta/test/BalancerTest.cpp b/src/meta/test/BalancerTest.cpp index 362e1f50db3..07b4c08b2e5 100644 --- a/src/meta/test/BalancerTest.cpp +++ b/src/meta/test/BalancerTest.cpp @@ -790,7 +790,6 @@ TEST(BalanceTest, NormalZoneTest) { ret = balancer.executeInternal(); baton.wait(); EXPECT_EQ(Status::OK(), ret.value()); - verifyMetaZone(kv, balancer.spaceInfo_.spaceId_, {"1", "2", "3", "4"}); verifyBalanceTask( kv, balancer.jobId_, BalanceTaskStatus::END, BalanceTaskResult::SUCCEEDED, partCount, 12); } diff --git a/src/parser/parser.yy b/src/parser/parser.yy index 434fcc3690e..a92e4c0f1d3 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -3326,36 +3326,20 @@ admin_job_sentence meta::cpp2::JobType::LEADER_BALANCE); $$ = sentence; } - | KW_SUBMIT KW_JOB KW_BALANCE KW_IN KW_ZONE { - auto sentence = new AdminJobSentence(meta::cpp2::JobOp::ADD, - meta::cpp2::JobType::DATA_BALANCE); - $$ = sentence; - } - | KW_SUBMIT KW_JOB KW_BALANCE KW_IN KW_ZONE KW_REMOVE host_list { - auto sentence = new AdminJobSentence(meta::cpp2::JobOp::ADD, - meta::cpp2::JobType::DATA_BALANCE); - HostList* hl = $7; - std::vector has = hl->hosts(); - for (HostAddr& ha: has) { - sentence->addPara(ha.toString()); - } - delete hl; - $$ = sentence; - } - | KW_SUBMIT KW_JOB KW_BALANCE KW_ACROSS KW_ZONE { + | KW_SUBMIT KW_JOB KW_BALANCE KW_DATA { auto sentence = new AdminJobSentence(meta::cpp2::JobOp::ADD, meta::cpp2::JobType::ZONE_BALANCE); $$ = sentence; } - | KW_SUBMIT KW_JOB KW_BALANCE KW_ACROSS KW_ZONE KW_REMOVE zone_name_list { + | KW_SUBMIT KW_JOB KW_BALANCE KW_DATA KW_REMOVE host_list { auto sentence = new AdminJobSentence(meta::cpp2::JobOp::ADD, meta::cpp2::JobType::ZONE_BALANCE); - ZoneNameList* nl = $7; - std::vector names = nl->zoneNames(); - for (std::string& name: names) { - sentence->addPara(name); + HostList* hl = $6; + std::vector hosts = hl->hosts(); + for (HostAddr& host: hosts) { + sentence->addPara(host.toString()); } - delete nl; + delete hl; $$ = sentence; } ; @@ -3794,37 +3778,20 @@ balance_sentence meta::cpp2::JobType::LEADER_BALANCE); $$ = sentence; } - | - KW_BALANCE KW_IN KW_ZONE { - auto sentence = new AdminJobSentence(meta::cpp2::JobOp::ADD, - meta::cpp2::JobType::DATA_BALANCE); - $$ = sentence; - } - | KW_BALANCE KW_IN KW_ZONE KW_REMOVE host_list { - auto sentence = new AdminJobSentence(meta::cpp2::JobOp::ADD, - meta::cpp2::JobType::DATA_BALANCE); - HostList* hl = $5; - std::vector has = hl->hosts(); - for (HostAddr& ha: has) { - sentence->addPara(ha.toString()); - } - delete hl; - $$ = sentence; - } - | KW_BALANCE KW_ACROSS KW_ZONE { + | KW_BALANCE KW_DATA { auto sentence = new AdminJobSentence(meta::cpp2::JobOp::ADD, meta::cpp2::JobType::ZONE_BALANCE); $$ = sentence; } - | KW_BALANCE KW_ACROSS KW_ZONE KW_REMOVE zone_name_list { + | KW_BALANCE KW_DATA KW_REMOVE host_list { auto sentence = new AdminJobSentence(meta::cpp2::JobOp::ADD, meta::cpp2::JobType::ZONE_BALANCE); - ZoneNameList* nl = $5; - std::vector names = nl->zoneNames(); - for (std::string& name: names) { - sentence->addPara(name); + HostList* hl = $4; + std::vector hosts = hl->hosts(); + for (HostAddr& host: hosts) { + sentence->addPara(host.toString()); } - delete nl; + delete hl; $$ = sentence; } ; diff --git a/src/parser/test/ParserTest.cpp b/src/parser/test/ParserTest.cpp index 4517c67761b..d25cdd76705 100644 --- a/src/parser/test/ParserTest.cpp +++ b/src/parser/test/ParserTest.cpp @@ -2009,34 +2009,6 @@ TEST_F(ParserTest, ConfigOperation) { } } -TEST_F(ParserTest, BalanceOperation) { - { - std::string query = "BALANCE LEADER"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "BALANCE IN ZONE"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "BALANCE ACROSS ZONE"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "BALANCE IN ZONE REMOVE 192.168.0.1:50000,192.168.0.1:50001"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "BALANCE IN ZONE REMOVE 192.168.0.1:50000,\"localhost\":50001"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } -} - TEST_F(ParserTest, CrashByFuzzer) { { std::string query = ";YIELD\nI41( ,1)GEGE.INGEST"; @@ -3227,11 +3199,6 @@ TEST_F(ParserTest, JobTest) { checkTest("SUBMIT JOB INGEST", "SUBMIT JOB INGEST"); checkTest("SUBMIT JOB STATS", "SUBMIT JOB STATS"); - checkTest("SUBMIT JOB BALANCE IN ZONE", "SUBMIT JOB BALANCE IN ZONE"); - checkTest( - "SUBMIT JOB BALANCE IN ZONE REMOVE 192.168.0.1:50000, 192.168.0.1:50001, 192.168.0.1:50002", - "SUBMIT JOB BALANCE IN ZONE REMOVE \"192.168.0.1\":50000, \"192.168.0.1\":50001, " - "\"192.168.0.1\":50002"); checkTest("SUBMIT JOB BALANCE LEADER", "SUBMIT JOB BALANCE LEADER"); checkTest("SHOW JOBS", "SHOW JOBS"); checkTest("SHOW JOB 111", "SHOW JOB 111"); diff --git a/tests/tck/features/schema/Comment.feature b/tests/tck/features/schema/Comment.feature index deb866a161e..d157eb98847 100644 --- a/tests/tck/features/schema/Comment.feature +++ b/tests/tck/features/schema/Comment.feature @@ -22,8 +22,8 @@ Feature: Schema Comment DESC SPACE ; """ Then the result should be, in any order: - | ID | Name | Partition Number | Replica Factor | Charset | Collate | Vid Type | Atomic Edge | Zones | Comment | - | /\d+/ | "" | 100 | 1 | "utf8" | "utf8_bin" | "FIXED_STRING(8)" | false | /^.+?\d$/ | "" | + | ID | Name | Partition Number | Replica Factor | Charset | Collate | Vid Type | Atomic Edge | Comment | + | /\d+/ | "" | 100 | 1 | "utf8" | "utf8_bin" | "FIXED_STRING(8)" | false | "" | When executing query: """ DROP SPACE ; @@ -54,8 +54,8 @@ Feature: Schema Comment DESC SPACE test_comment_not_set; """ Then the result should be, in any order: - | ID | Name | Partition Number | Replica Factor | Charset | Collate | Vid Type | Atomic Edge | Zones | Comment | - | /\d+/ | "test_comment_not_set" | 100 | 1 | "utf8" | "utf8_bin" | "FIXED_STRING(8)" | false | /^.+?\d$/ | EMPTY | + | ID | Name | Partition Number | Replica Factor | Charset | Collate | Vid Type | Atomic Edge | Comment | + | /\d+/ | "test_comment_not_set" | 100 | 1 | "utf8" | "utf8_bin" | "FIXED_STRING(8)" | false | EMPTY | When executing query: """ DROP SPACE test_comment_not_set; @@ -81,8 +81,8 @@ Feature: Schema Comment DESC SPACE test_comment_empty; """ Then the result should be, in any order: - | ID | Name | Partition Number | Replica Factor | Charset | Collate | Vid Type | Atomic Edge | Zones | Comment | - | /\d+/ | "test_comment_empty" | 100 | 1 | "utf8" | "utf8_bin" | "FIXED_STRING(8)" | false | /^.+?\d$/ | "" | + | ID | Name | Partition Number | Replica Factor | Charset | Collate | Vid Type | Atomic Edge | Comment | + | /\d+/ | "test_comment_empty" | 100 | 1 | "utf8" | "utf8_bin" | "FIXED_STRING(8)" | false | "" | When executing query: """ DROP SPACE test_comment_empty;