Cherry pick v3.0.0 (0117-0124)

.github/CODEOWNERS

@@ -0,0 +1,6 @@
+#Require an approved review in PRs including files with a designated code owner.
+/conf/ @vesoft-inc/tech-committee-reviewers
+/src/kvstore/raftex/ @critical27 @sherman-the-tank
+/cmake/ @sherman-the-tank @yixinglu
+*.thrift @vesoft-inc/tech-committee-reviewers
+*.yy @CPWstatic

cmake/ThriftGenerate.cmake

@@ -86,7 +86,7 @@ add_custom_command(
  --gen "js:node:"
  --gen "csharp"
  --gen "java:hashcode"
- --gen "go:thrift_import=github.com/facebook/fbthrift/thrift/lib/go/thrift,package_prefix=github.com/vesoft-inc/nebula-go/v2/,use_context"
+ --gen "go:thrift_import=github.com/facebook/fbthrift/thrift/lib/go/thrift,package_prefix=github.com/vesoft-inc/nebula-go/v3/,use_context"
  -o "." "${file_path}/${file_name}.thrift"
  COMMAND
  mkdir -p "./gen-rust/${file_name}"

src/codec/RowReaderV2.h

@@ -26,7 +26,7 @@ class RowReaderV2 : public RowReader {
  FRIEND_TEST(ScanEdgePropBench, ProcessEdgeProps);
 
  public:
- virtual ~RowReaderV2() = default;
+ ~RowReaderV2() override = default;
 
  Value getValueByName(const std::string& prop) const noexcept override;
  Value getValueByIndex(const int64_t index) const noexcept override;

src/common/base/SanitizerOptions.cpp

@@ -21,7 +21,9 @@ const char* __asan_default_options() {
  "fast_unwind_on_malloc=0 \n"
  "detect_stack_use_after_return=1 \n"
  "alloc_dealloc_mismatch=1 \n"
- "new_delete_type_mismatch=1 \n"
+ // todo(doodle): Reopen when https://github.com/vesoft-inc/nebula/issues/3690
+ // addressed throughly
+ "new_delete_type_mismatch=0 \n"
  "strict_init_order=1 \n"
  "intercept_tls_get_addr=1 \n"
  "symbolize_inline_frames=1 \n"

src/common/expression/PropertyExpression.h

@@ -155,6 +155,10 @@ class LabelTagPropertyExpression final : public PropertyExpression {
  return label_;
  }
 
+ void setLabel(Expression* label) {
+ label_ = label;
+ }
+
  private:
  LabelTagPropertyExpression(ObjectPool* pool,
  Expression* label = nullptr,

src/graph/executor/admin/ShowHostsExecutor.cpp

@@ -30,7 +30,8 @@ folly::Future<Status> ShowHostsExecutor::showHosts() {
  "Status",
  "Leader count",
  "Leader distribution",
- "Partition distribution"});
+ "Partition distribution",
+ "Version"});
 
  std::map<std::string, int64_t> leaderPartsCount;
  std::map<std::string, int64_t> allPartsCount;
@@ -82,6 +83,7 @@ folly::Future<Status> ShowHostsExecutor::showHosts() {
  r.emplace_back(leaderCount);
  r.emplace_back(leaders.str());
  r.emplace_back(parts.str());
+ r.emplace_back(host.version_ref().has_value() ? Value(*host.version_ref()) : Value());
  v.emplace_back(std::move(r));
  } // row loop
  {
@@ -148,7 +150,7 @@ folly::Future<Status> ShowHostsExecutor::showHosts() {
  LOG(ERROR) << resp.status();
  return resp.status();
  }
- auto value = std::move(resp).value();
+ auto value = std::forward<decltype(resp)>(resp).value();
  if (type == meta::cpp2::ListHostType::ALLOC) {
  return finish(makeTraditionalResult(value));
  }

src/graph/optimizer/OptGroup.cpp

@@ -45,6 +45,7 @@ void OptGroup::addGroupNode(OptGroupNode *groupNode) {
  DCHECK(groupNode != nullptr);
  DCHECK(groupNode->group() == this);
  groupNodes_.emplace_back(groupNode);
+ groupNode->node()->updateSymbols();
 }
 
 OptGroupNode *OptGroup::makeGroupNode(PlanNode *node) {

src/graph/planner/match/MatchPlanner.h

@@ -14,7 +14,7 @@ namespace graph {
 class MatchPlanner final : public Planner {
  public:
  static std::unique_ptr<MatchPlanner> make() {
- return std::unique_ptr<MatchPlanner>(new MatchPlanner());
+ return std::make_unique<MatchPlanner>();
  }
 
  static bool match(AstContext* astCtx);

src/graph/planner/plan/PlanNode.cpp

@@ -370,6 +370,15 @@ void PlanNode::releaseSymbols() {
  }
 }
 
+void PlanNode::updateSymbols() {
+ auto symTbl = qctx_->symTable();
+ for (auto out : outputVars_) {
+ if (out != nullptr) {
+ symTbl->updateWrittenBy(out->name, out->name, this);
+ }
+ }
+}
+
 std::ostream& operator<<(std::ostream& os, PlanNode::Kind kind) {
  os << PlanNode::toString(kind);
  return os;

src/graph/planner/plan/PlanNode.h

@@ -269,6 +269,8 @@ class PlanNode {
 
  void releaseSymbols();
 
+ void updateSymbols();
+
  static const char* toString(Kind kind);
  std::string toString() const;
 

src/graph/service/GraphFlags.cpp

@@ -56,7 +56,7 @@ DEFINE_bool(enable_optimizer, false, "Whether to enable optimizer");
 DEFINE_uint32(ft_request_retry_times, 3, "Retry times if fulltext request failed");
 DEFINE_bool(enable_client_white_list, true, "Turn on/off the client white list.");
 DEFINE_string(client_white_list,
- nebula::getOriginVersion() + ":2.5.0:2.5.1:2.6.0",
+ nebula::getOriginVersion() + ":3.0.0",
  "A white list for different client versions, separate with colon.");
 
 #endif

src/graph/service/test/StandAloneTestGraphFlags.cpp

@@ -9,5 +9,5 @@
 DEFINE_uint32(ft_request_retry_times, 3, "Retry times if fulltext request failed");
 DEFINE_bool(enable_client_white_list, true, "Turn on/off the client white list.");
 DEFINE_string(client_white_list,
- nebula::getOriginVersion() + ":2.5.0:2.5.1:2.6.0",
+ nebula::getOriginVersion() + ":3.0.0",
  "A white list for different client versions, separate with colon.");

src/graph/util/ExpressionUtils.cpp

@@ -126,18 +126,24 @@ bool ExpressionUtils::isEvaluableExpr(const Expression *expr, const QueryContext
 }
 
 // rewrite Attribute to LabelTagProp
-Expression *ExpressionUtils::rewriteAttr2LabelTagProp(const Expression *expr) {
+Expression *ExpressionUtils::rewriteAttr2LabelTagProp(
+ const Expression *expr, const std::unordered_map<std::string, AliasType> &aliasTypeMap) {
  ObjectPool *pool = expr->getObjPool();
 
- auto matcher = [](const Expression *e) -> bool {
- if (e->kind() == Expression::Kind::kAttribute) {
- auto attrExpr = static_cast<const AttributeExpression *>(e);
- if (attrExpr->left()->kind() == Expression::Kind::kLabelAttribute &&
- attrExpr->right()->kind() == Expression::Kind::kConstant) {
- return true;
- }
+ auto matcher = [&aliasTypeMap](const Expression *e) -> bool {
+ if (e->kind() != Expression::Kind::kAttribute) {
+ return false;
  }
- return false;
+ auto attrExpr = static_cast<const AttributeExpression *>(e);
+ if (attrExpr->left()->kind() != Expression::Kind::kLabelAttribute) {
+ return false;
+ }
+ auto label = static_cast<const LabelAttributeExpression *>(attrExpr->left())->left()->name();
+ auto iter = aliasTypeMap.find(label);
+ if (iter == aliasTypeMap.end() || iter->second != AliasType::kNode) {
+ return false;
+ }
+ return true;
  };
 
  auto rewriter = [pool](const Expression *e) -> Expression * {

src/graph/util/ExpressionUtils.h

@@ -15,14 +15,14 @@
 #include "common/expression/PropertyExpression.h"
 #include "common/expression/TypeCastingExpression.h"
 #include "common/expression/UnaryExpression.h"
+#include "graph/context/ast/CypherAstContext.h"
 #include "graph/visitor/EvaluableExprVisitor.h"
 #include "graph/visitor/FindVisitor.h"
 #include "graph/visitor/RewriteVisitor.h"
 
 namespace nebula {
 class ObjectPool;
 namespace graph {
-
 class ExpressionUtils {
  public:
  explicit ExpressionUtils(...) = delete;
@@ -55,7 +55,8 @@ class ExpressionUtils {
 
  static bool isEvaluableExpr(const Expression* expr, const QueryContext* qctx = nullptr);
 
- static Expression* rewriteAttr2LabelTagProp(const Expression* expr);
+ static Expression* rewriteAttr2LabelTagProp(
+ const Expression* expr, const std::unordered_map<std::string, AliasType>& aliasTypeMap);
 
  static Expression* rewriteLabelAttr2TagProp(const Expression* expr);
 

src/graph/validator/AdminJobValidator.cpp

@@ -11,6 +11,10 @@ namespace nebula {
 namespace graph {
 
 Status AdminJobValidator::validateImpl() {
+ if (sentence_->getCmd() == meta::cpp2::AdminCmd::DATA_BALANCE ||
+ sentence_->getCmd() == meta::cpp2::AdminCmd::ZONE_BALANCE) {
+ return Status::SemanticError("Data balance not support");
+ }
  if (sentence_->getOp() == meta::cpp2::AdminJobOp::ADD) {
  auto cmd = sentence_->getCmd();
  if (requireSpace()) {

src/graph/validator/MatchValidator.cpp

@@ -273,7 +273,8 @@ Status MatchValidator::validateFilter(const Expression *filter,
  auto transformRes = ExpressionUtils::filterTransform(filter);
  NG_RETURN_IF_ERROR(transformRes);
  // rewrite Attribute to LabelTagProperty
- whereClauseCtx.filter = ExpressionUtils::rewriteAttr2LabelTagProp(transformRes.value());
+ whereClauseCtx.filter = ExpressionUtils::rewriteAttr2LabelTagProp(
+ transformRes.value(), whereClauseCtx.aliasesAvailable);
 
  auto typeStatus = deduceExprType(whereClauseCtx.filter);
  NG_RETURN_IF_ERROR(typeStatus);
@@ -383,7 +384,8 @@ Status MatchValidator::validateReturn(MatchReturn *ret,
  ExpressionUtils::hasAny(column->expr(), {Expression::Kind::kAggregate})) {
  retClauseCtx.yield->hasAgg_ = true;
  }
- column->setExpr(ExpressionUtils::rewriteAttr2LabelTagProp(column->expr()));
+ column->setExpr(ExpressionUtils::rewriteAttr2LabelTagProp(
+ column->expr(), retClauseCtx.yield->aliasesAvailable));
  exprs.push_back(column->expr());
  columns->addColumn(column->clone().release());
  }
@@ -459,7 +461,8 @@ Status MatchValidator::validateWith(const WithClause *with,
  }
  if (with->returnItems()->columns()) {
  for (auto *column : with->returnItems()->columns()->columns()) {
- column->setExpr(ExpressionUtils::rewriteAttr2LabelTagProp(column->expr()));
+ column->setExpr(ExpressionUtils::rewriteAttr2LabelTagProp(
+ column->expr(), withClauseCtx.yield->aliasesAvailable));
  columns->addColumn(column->clone().release());
  }
  }
@@ -819,7 +822,7 @@ Status MatchValidator::checkAlias(
  auto name = static_cast<const VariablePropertyExpression *>(labelExpr)->prop();
  auto res = getAliasType(aliasesAvailable, name);
  NG_RETURN_IF_ERROR(res);
- if (res.value() != AliasType::kNode) {
+ if (res.value() == AliasType::kEdge || res.value() == AliasType::kPath) {
  return Status::SemanticError("The type of `%s' should be tag", name.c_str());
  }
  return Status::OK();

src/graph/visitor/RewriteVisitor.cpp

@@ -237,6 +237,18 @@ void RewriteVisitor::visit(PathBuildExpression *expr) {
  }
 }
 
+void RewriteVisitor::visit(LabelTagPropertyExpression *expr) {
+ if (!care(expr->kind())) {
+ return;
+ }
+ auto label = expr->label();
+ if (matcher_(label)) {
+ expr->setLabel(rewriter_(label));
+ } else {
+ label->accept(this);
+ }
+}
+
 void RewriteVisitor::visit(AttributeExpression *expr) {
  if (!care(expr->kind())) {
  return;

src/graph/visitor/RewriteVisitor.h

@@ -67,12 +67,12 @@ class RewriteVisitor final : public ExprVisitorImpl {
  void visit(RelationalExpression*) override;
  void visit(SubscriptExpression*) override;
  void visit(PathBuildExpression*) override;
+ void visit(LabelTagPropertyExpression*) override;
  void visit(SubscriptRangeExpression*) override;
  void visit(ConstantExpression*) override {}
  void visit(LabelExpression*) override {}
  void visit(UUIDExpression*) override {}
  void visit(LabelAttributeExpression*) override {}
- void visit(LabelTagPropertyExpression*) override {}
  void visit(VariableExpression*) override {}
  void visit(VersionedVariableExpression*) override {}
  void visit(TagPropertyExpression*) override {}

src/interface/common.thrift

@@ -10,7 +10,7 @@ namespace java com.vesoft.nebula
 namespace go nebula
 namespace js nebula
 namespace csharp nebula
-namespace py nebula2.common
+namespace py nebula3.common
 
 cpp_include "common/thrift/ThriftTypes.h"
 cpp_include "common/datatypes/DateOps-inl.h"
@@ -34,7 +34,7 @@ cpp_include "common/datatypes/DurationOps-inl.h"
  *
  */
 
-const binary (cpp.type = "char const *") version = "2.6.0"
+const binary (cpp.type = "char const *") version = "3.0.0"
 
 typedef i64 (cpp.type = "nebula::ClusterID") ClusterID
 typedef i32 (cpp.type = "nebula::GraphSpaceID") GraphSpaceID

src/interface/graph.thrift

@@ -9,7 +9,7 @@ namespace java com.vesoft.nebula.graph
 namespace go nebula.graph
 namespace js nebula.graph
 namespace csharp nebula.graph
-namespace py nebula2.graph
+namespace py nebula3.graph
 
 include "common.thrift"
 

src/interface/meta.thrift

@@ -8,7 +8,7 @@ namespace java com.vesoft.nebula.meta
 namespace go nebula.meta
 namespace js nebula.meta
 namespace csharp nebula.meta
-namespace py nebula2.meta
+namespace py nebula3.meta
 
 include "common.thrift"
 

src/interface/storage.thrift

@@ -9,7 +9,7 @@ namespace java com.vesoft.nebula.storage
 namespace go nebula.storage
 namespace csharp nebula.storage
 namespace js nebula.storage
-namespace py nebula2.storage
+namespace py nebula3.storage
 
 include "common.thrift"
 include "meta.thrift"

src/kvstore/KVEngine.h

@@ -49,6 +49,18 @@ class KVEngine {
  bool sync,
  bool wait) = 0;
 
+ /**
+ * @brief Get the Snapshot from kv engine.
+ *
+ * @return const void* snapshot pointer.
+ */
+ virtual const void* GetSnapshot() = 0;
+ /**
+ * @brief Release snapshot from kv engine.
+ *
+ * @param snapshot
+ */
+ virtual void ReleaseSnapshot(const void* snapshot) = 0;
  // Read a single key
  virtual nebula::cpp2::ErrorCode get(const std::string& key, std::string* value) = 0;
 
@@ -62,9 +74,17 @@ class KVEngine {
  const std::string& end,
  std::unique_ptr<KVIterator>* iter) = 0;
 
- // Get all results with 'prefix' str as prefix.
+ /**
+ * @brief Get all results with 'prefix' str as prefix.
+ *
+ * @param prefix Prefix string.
+ * @param snapshot Snapshot from kv engine. nullptr means no snapshot.
+ * @param iter Iterator for this prefix range.
+ * @return nebula::cpp2::ErrorCode
+ */
  virtual nebula::cpp2::ErrorCode prefix(const std::string& prefix,
- std::unique_ptr<KVIterator>* iter) = 0;
+ std::unique_ptr<KVIterator>* iter,
+ const void* snapshot = nullptr) = 0;
 
  // Get all results with 'prefix' str as prefix starting form 'start'
  virtual nebula::cpp2::ErrorCode rangeWithPrefix(const std::string& start,