Merge branch 'master' of https://github.com/vesoft-inc/nebula into fe…

…ature/match-path-pattern-expression

.github/workflows/nightly.yml

@@ -82,8 +82,7 @@ jobs:
  with:
  context: .
  file: ./docker/Dockerfile.${{ matrix.service }}
- # platforms: linux/amd64,linux/arm64
- platforms: linux/amd64
+ platforms: linux/amd64,linux/arm64
  tags: |
  vesoft/nebula-${{ matrix.service }}:nightly
  push: true

.github/workflows/rc.yml

@@ -99,8 +99,7 @@ jobs:
  with:
  context: .
  file: ./docker/Dockerfile.${{ matrix.service }}
- # platforms: linux/amd64,linux/arm64
- platforms: linux/amd64
+ platforms: linux/amd64,linux/arm64
  tags: |
  ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.tag }}
  ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.majorver }}

CMakeLists.txt

@@ -51,6 +51,7 @@ include(GitHooksConfig)
 include(GitInfoConfig)
 include(NebulaCustomTargets)
 include(IncludeWhatYouUse)
+include(TimeTrace)
 
 add_custom_target(
  clang-format

cmake/nebula/TimeTrace.cmake

@@ -0,0 +1,6 @@
+if(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang")
+ option(ENABLE_BUILD_WITH_TIME_TRACE "Enable -ftime-trace to generate time tracing .json files on clang" OFF)
+ if(ENABLE_BUILD_WITH_TIME_TRACE)
+ add_compile_options(-ftime-trace)
+ endif()
+endif()

src/clients/meta/MetaClient.h

@@ -220,8 +220,8 @@ class MetaClient {
  FRIEND_TEST(MetaClientTest, RetryUntilLimitTest);
  FRIEND_TEST(MetaClientTest, RocksdbOptionsTest);
  FRIEND_TEST(MetaClientTest, VerifyClientTest);
- friend class KillQueryMetaWrapper;
  FRIEND_TEST(ChainAddEdgesTest, AddEdgesLocalTest);
+ friend class KillQueryMetaWrapper;
  friend class storage::MetaClientTestUpdater;
 
  public:

src/common/stats/StatsManager.h

@@ -133,6 +133,7 @@ class StatsManager final {
  std::vector<StatsMethod> methods,
  std::vector<std::pair<std::string, double>> percentiles);
 
+ // Dynamically register some stats with label values
  static CounterId counterWithLabels(const CounterId& id, const std::vector<LabelPair>& labels);
  static CounterId histoWithLabels(const CounterId& id, const std::vector<LabelPair>& labels);
 
@@ -166,6 +167,7 @@ class StatsManager final {
  StatsManager(const StatsManager&) = delete;
  StatsManager(StatsManager&&) = delete;
 
+ // Parse percentiles from the string
  static bool strToPct(folly::StringPiece part, double& pct);
  static void parseStats(const folly::StringPiece stats,
  std::vector<StatsMethod>& methods,

src/common/utils/MetaKeyUtils.cpp

@@ -740,16 +740,6 @@ std::string MetaKeyUtils::indexIndexKey(GraphSpaceID spaceID, const std::string&
  return key;
 }
 
-std::string MetaKeyUtils::indexGroupKey(const std::string& name) {
- EntryType type = EntryType::GROUP;
- std::string key;
- key.reserve(128);
- key.append(kIndexTable.data(), kIndexTable.size())
- .append(reinterpret_cast<const char*>(&type), sizeof(EntryType))
- .append(name);
- return key;
-}
-
 std::string MetaKeyUtils::indexZoneKey(const std::string& name) {
  EntryType type = EntryType::ZONE;
  std::string key;

src/common/utils/MetaKeyUtils.h

@@ -57,7 +57,6 @@ enum class EntryType : int8_t {
  EDGE = 0x03,
  INDEX = 0x04,
  CONFIG = 0x05,
- GROUP = 0x06,
  ZONE = 0x07,
 };
 
@@ -239,8 +238,6 @@ class MetaKeyUtils final {
 
  static std::string indexIndexKey(GraphSpaceID spaceId, const std::string& name);
 
- static std::string indexGroupKey(const std::string& name);
-
  static std::string indexZoneKey(const std::string& name);
 
  static std::string userPrefix();

src/graph/executor/admin/KillQueryExecutor.cpp

@@ -13,8 +13,6 @@ namespace graph {
 folly::Future<Status> KillQueryExecutor::execute() {
  SCOPED_TIMER(&execTime_);
 
- // TODO: permission check
-
  QueriesMap toBeVerifiedQueries;
  QueriesMap killQueries;
  NG_RETURN_IF_ERROR(verifyTheQueriesByLocalCache(toBeVerifiedQueries, killQueries));
@@ -92,9 +90,15 @@ Status KillQueryExecutor::verifyTheQueriesByLocalCache(QueriesMap& toBeVerifiedQ
  auto sessionPtr = sessionMgr->findSessionFromCache(sessionId);
  if (sessionPtr == nullptr) {
  toBeVerifiedQueries[sessionId].emplace(epId);
- } else if (!sessionPtr->findQuery(epId)) {
- return Status::Error(
- "ExecutionPlanId[%ld] does not exist in Session[%ld].", epId, sessionId);
+ } else { // Sessions in current host
+ // Only GOD role could kill others' queries.
+ if (sessionPtr->user() != session->user() && !session->isGod()) {
+ return Status::PermissionError("Only GOD role could kill others' queries.");
+ }
+ if (!sessionPtr->findQuery(epId)) {
+ return Status::Error(
+ "ExecutionPlanId[%ld] does not exist in Session[%ld].", epId, sessionId);
+ }
  }
  killQueries[sessionId].emplace(epId);
  }
@@ -125,6 +129,7 @@ void KillQueryExecutor::killCurrentHostQueries(const QueriesMap& killQueries,
 
 Status KillQueryExecutor::verifyTheQueriesByMetaInfo(
  const QueriesMap& toBeVerifiedQueries, const std::vector<meta::cpp2::Session>& sessionsInMeta) {
+ auto* currentSession = qctx()->rctx()->session();
  for (auto& s : toBeVerifiedQueries) {
  auto sessionId = s.first;
  auto found = std::find_if(sessionsInMeta.begin(), sessionsInMeta.end(), [sessionId](auto& val) {
@@ -139,6 +144,10 @@ Status KillQueryExecutor::verifyTheQueriesByMetaInfo(
  "ExecutionPlanId[%ld] does not exist in Session[%ld].", epId, sessionId);
  }
  }
+ // Only GOD role could kill others' queries.
+ if (found->get_user_name() != currentSession->user() && !currentSession->isGod()) {
+ return Status::PermissionError("Only GOD role could kill others' queries.");
+ }
  }
  return Status::OK();
 }

src/graph/service/PermissionCheck.cpp

@@ -212,8 +212,10 @@ Status PermissionCheck::permissionCheck(ClientSession *session,
  // No permission checking for sequential sentence.
  return Status::OK();
  }
- case Sentence::Kind::kShowQueries:
- case Sentence::Kind::kKillQuery: {
+ case Sentence::Kind::kKillQuery:
+ // Only GOD could kill all queries, other roles only could kill own queries.
+ return Status::OK();
+ case Sentence::Kind::kShowQueries: {
  return Status::OK();
  }
  }

src/graph/stats/GraphStats.h

@@ -15,11 +15,13 @@ DECLARE_bool(enable_space_level_metrics);
 namespace nebula {
 
 // Query
+// A sequential sentence is treated as one query
 extern stats::CounterId kNumQueries;
 extern stats::CounterId kNumActiveQueries;
 extern stats::CounterId kNumSlowQueries;
 extern stats::CounterId kNumQueryErrors;
 extern stats::CounterId kNumQueryErrorsLeaderChanges;
+// A sequential sentence is treated as multiple sentences seperated by `;`
 extern stats::CounterId kNumSentences;
 extern stats::CounterId kQueryLatencyUs;
 extern stats::CounterId kSlowQueryLatencyUs;

src/kvstore/stats/KVStats.h

@@ -11,6 +11,7 @@
 
 namespace nebula {
 
+// Raft related stats
 extern stats::CounterId kCommitLogLatencyUs;
 extern stats::CounterId kCommitSnapshotLatencyUs;
 extern stats::CounterId kTransferLeaderLatencyUs;

src/meta/ActiveHostsMan.cpp

@@ -20,9 +20,9 @@ namespace meta {
 nebula::cpp2::ErrorCode ActiveHostsMan::updateHostInfo(kvstore::KVStore* kv,
  const HostAddr& hostAddr,
  const HostInfo& info,
+ std::vector<kvstore::KV>& data,
  const AllLeaders* allLeaders) {
  CHECK_NOTNULL(kv);
- std::vector<kvstore::KV> data;
  std::vector<std::string> leaderKeys;
  std::vector<int64_t> terms;
  if (allLeaders != nullptr) {
@@ -64,22 +64,11 @@ nebula::cpp2::ErrorCode ActiveHostsMan::updateHostInfo(kvstore::KVStore* kv,
  bool hasUpdate = !data.empty();
  data.emplace_back(MetaKeyUtils::hostKey(hostAddr.host, hostAddr.port), HostInfo::encodeV2(info));
 
- folly::SharedMutex::WriteHolder wHolder(LockUtils::spaceLock());
- folly::Baton<true, std::atomic> baton;
- nebula::cpp2::ErrorCode ret;
- kv->asyncMultiPut(
- kDefaultSpaceId, kDefaultPartId, std::move(data), [&](nebula::cpp2::ErrorCode code) {
- ret = code;
- baton.post();
- });
- baton.wait();
- if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
- return ret;
- }
  if (hasUpdate) {
- ret = LastUpdateTimeMan::update(kv, time::WallClock::fastNowInMilliSec());
+ auto timeInMilliSec = time::WallClock::fastNowInMilliSec();
+ LastUpdateTimeMan::update(data, timeInMilliSec);
  }
- return ret;
+ return nebula::cpp2::ErrorCode::SUCCEEDED;
 }
 
 bool ActiveHostsMan::machineRegisted(kvstore::KVStore* kv, const HostAddr& hostAddr) {
@@ -259,36 +248,14 @@ ErrorOr<nebula::cpp2::ErrorCode, HostInfo> ActiveHostsMan::getHostInfo(kvstore::
  return HostInfo::decode(hostValue);
 }
 
-nebula::cpp2::ErrorCode LastUpdateTimeMan::update(kvstore::KVStore* kv,
- const int64_t timeInMilliSec) {
- CHECK_NOTNULL(kv);
- std::vector<kvstore::KV> data;
+void LastUpdateTimeMan::update(std::vector<kvstore::KV>& data, const int64_t timeInMilliSec) {
  data.emplace_back(MetaKeyUtils::lastUpdateTimeKey(),
  MetaKeyUtils::lastUpdateTimeVal(timeInMilliSec));
-
- folly::SharedMutex::WriteHolder wHolder(LockUtils::lastUpdateTimeLock());
- folly::Baton<true, std::atomic> baton;
- nebula::cpp2::ErrorCode ret;
- kv->asyncMultiPut(
- kDefaultSpaceId, kDefaultPartId, std::move(data), [&](nebula::cpp2::ErrorCode code) {
- ret = code;
- baton.post();
- });
- baton.wait();
- return ret;
 }
 
-ErrorOr<nebula::cpp2::ErrorCode, int64_t> LastUpdateTimeMan::get(kvstore::KVStore* kv) {
- CHECK_NOTNULL(kv);
- auto key = MetaKeyUtils::lastUpdateTimeKey();
- std::string val;
- auto retCode = kv->get(kDefaultSpaceId, kDefaultPartId, key, &val);
- if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) {
- LOG(INFO) << "Get last update time failed, error: "
- << apache::thrift::util::enumNameSafe(retCode);
- return retCode;
- }
- return *reinterpret_cast<const int64_t*>(val.data());
+void LastUpdateTimeMan::update(kvstore::BatchHolder* batchHolder, const int64_t timeInMilliSec) {
+ batchHolder->put(MetaKeyUtils::lastUpdateTimeKey(),
+ MetaKeyUtils::lastUpdateTimeVal(timeInMilliSec));
 }
 
 } // namespace meta

src/meta/ActiveHostsMan.h

@@ -9,6 +9,7 @@
 #include "common/base/Base.h"
 #include "common/utils/MetaKeyUtils.h"
 #include "kvstore/KVStore.h"
+#include "kvstore/LogEncoder.h"
 
 namespace nebula {
 namespace meta {
@@ -130,6 +131,7 @@ class ActiveHostsMan final {
  static nebula::cpp2::ErrorCode updateHostInfo(kvstore::KVStore* kv,
  const HostAddr& hostAddr,
  const HostInfo& info,
+ std::vector<kvstore::KV>& data,
  const AllLeaders* leaderParts = nullptr);
 
  /**
@@ -212,9 +214,9 @@ class LastUpdateTimeMan final {
  public:
  ~LastUpdateTimeMan() = default;
 
- static nebula::cpp2::ErrorCode update(kvstore::KVStore* kv, const int64_t timeInMilliSec);
+ static void update(std::vector<kvstore::KV>& data, const int64_t timeInMilliSec);
 
- static ErrorOr<nebula::cpp2::ErrorCode, int64_t> get(kvstore::KVStore* kv);
+ static void update(kvstore::BatchHolder* batchHolder, const int64_t timeInMilliSec);
 
  protected:
  LastUpdateTimeMan() = default;

src/meta/http/MetaHttpReplaceHostHandler.cpp

@@ -106,7 +106,7 @@ void MetaHttpReplaceHostHandler::onError(ProxygenError error) noexcept {
 }
 
 bool MetaHttpReplaceHostHandler::replaceHostInPart(std::string ipv4From, std::string ipv4To) {
- folly::SharedMutex::WriteHolder wHolder(LockUtils::spaceLock());
+ folly::SharedMutex::WriteHolder holder(LockUtils::lock());
  const auto& spacePrefix = MetaKeyUtils::spacePrefix();
  std::unique_ptr<kvstore::KVIterator> iter;
  auto kvRet = kvstore_->prefix(kDefaultSpaceId, kDefaultPartId, spacePrefix, &iter);
@@ -163,7 +163,7 @@ bool MetaHttpReplaceHostHandler::replaceHostInPart(std::string ipv4From, std::st
 }
 
 bool MetaHttpReplaceHostHandler::replaceHostInZone(std::string ipv4From, std::string ipv4To) {
- folly::SharedMutex::WriteHolder wHolder(LockUtils::spaceLock());
+ folly::SharedMutex::WriteHolder holder(LockUtils::lock());
  const auto& zonePrefix = MetaKeyUtils::zonePrefix();
  std::unique_ptr<kvstore::KVIterator> iter;
  auto kvRet = kvstore_->prefix(kDefaultSpaceId, kDefaultPartId, zonePrefix, &iter);