Refactor storage index

src/common/utils/IndexKeyUtils.h

@@ -6,13 +6,14 @@
 #ifndef COMMON_UTILS_INDEXKEYUTILS_H_
 #define COMMON_UTILS_INDEXKEYUTILS_H_
 
+#include <cmath>
+
 #include "codec/RowReader.h"
 #include "common/base/Base.h"
 #include "common/base/StatusOr.h"
 #include "common/geo/GeoIndex.h"
 #include "common/utils/Types.h"
 #include "interface/gen-cpp2/meta_types.h"
-
 namespace nebula {
 
 using PropertyType = nebula::cpp2::PropertyType;
@@ -197,39 +198,39 @@ class IndexKeyUtils final {
  */
 
  static std::string encodeDouble(double v) {
- if (v < 0) {
- /**
- * TODO : now, the -(std::numeric_limits<double>::min())
- * have a problem of precision overflow. current return value is -nan.
- */
- auto* c1 = reinterpret_cast<const char*>(&v);
- auto i = *reinterpret_cast<const int64_t*>(c1);
- i = -(std::numeric_limits<int64_t>::max() + i);
- auto* c2 = reinterpret_cast<const char*>(&i);
- v = *reinterpret_cast<const double*>(c2);
+ if (std::isnan(v)) {
+ return std::string(sizeof(double), '\xFF');
+ } else if (v >= 0) {
+ auto val = folly::Endian::big(v);
+ auto* c = reinterpret_cast<char*>(&val);
+ c[0] |= 0x80;
+ std::string raw;
+ raw.reserve(sizeof(double));
+ raw.append(c, sizeof(double));
+ return raw;
+ } else {
+ int64_t* x = reinterpret_cast<int64_t*>(&v);
+ *x = ~(*x);
+ auto val = folly::Endian::big(v);
+ auto* c = reinterpret_cast<char*>(&val);
+ std::string raw;
+ raw.reserve(sizeof(double));
+ raw.append(c, sizeof(double));
+ return raw;
  }
- auto val = folly::Endian::big(v);
- auto* c = reinterpret_cast<char*>(&val);
- c[0] ^= 0x80;
- std::string raw;
- raw.reserve(sizeof(double));
- raw.append(c, sizeof(double));
- return raw;
  }
 
  static double decodeDouble(const folly::StringPiece& raw) {
- char* v = const_cast<char*>(raw.data());
- v[0] ^= 0x80;
- auto val = *reinterpret_cast<const double*>(v);
+ int64_t val = *reinterpret_cast<const int64_t*>(raw.data());
  val = folly::Endian::big(val);
  if (val < 0) {
- auto* c1 = reinterpret_cast<const char*>(&val);
- auto i = *reinterpret_cast<const int64_t*>(c1);
- i = -(std::numeric_limits<int64_t>::max() + i);
- auto* c2 = reinterpret_cast<const char*>(&i);
- val = *reinterpret_cast<const double*>(c2);
+ val &= 0x7fffffffffffffff;
+ } else {
+ val = ~val;
  }
- return val;
+ double ret;
+ ::memcpy(&ret, &val, 8);
+ return ret;
  }
 
  static std::string encodeTime(const nebula::Time& t) {

src/common/utils/test/CMakeLists.txt

@@ -11,6 +11,7 @@ nebula_add_test(
  $<TARGET_OBJECTS:wkt_wkb_io_obj>
  $<TARGET_OBJECTS:meta_thrift_obj>
  $<TARGET_OBJECTS:common_thrift_obj>
+ $<TARGET_OBJECTS:storage_thrift_obj>
  $<TARGET_OBJECTS:geo_index_obj>
  LIBRARIES
  gtest
@@ -30,6 +31,7 @@ nebula_add_test(
  $<TARGET_OBJECTS:wkt_wkb_io_obj>
  $<TARGET_OBJECTS:meta_thrift_obj>
  $<TARGET_OBJECTS:common_thrift_obj>
+ $<TARGET_OBJECTS:storage_thrift_obj>
  $<TARGET_OBJECTS:geo_index_obj>
  LIBRARIES
  gtest
@@ -52,6 +54,7 @@ nebula_add_test(
  $<TARGET_OBJECTS:wkt_wkb_io_obj>
  $<TARGET_OBJECTS:meta_thrift_obj>
  $<TARGET_OBJECTS:common_thrift_obj>
+ $<TARGET_OBJECTS:storage_thrift_obj>
  $<TARGET_OBJECTS:geo_index_obj>
  LIBRARIES
  gtest

src/daemons/CMakeLists.txt

@@ -45,6 +45,7 @@ set(storage_meta_deps
  $<TARGET_OBJECTS:raftex_thrift_obj>
  $<TARGET_OBJECTS:hdfs_helper_obj>
  $<TARGET_OBJECTS:ft_es_storage_adapter_obj>
+ $<TARGET_OBJECTS:storage_thrift_obj>
  $<TARGET_OBJECTS:geo_index_obj>
 )
 
@@ -128,6 +129,7 @@ nebula_add_executable(
  $<TARGET_OBJECTS:charset_obj>
  $<TARGET_OBJECTS:graph_obj>
  $<TARGET_OBJECTS:ft_es_graph_adapter_obj>
+ $<TARGET_OBJECTS:storage_thrift_obj>
  $<TARGET_OBJECTS:geo_index_obj>
  ${common_deps}
  LIBRARIES

src/graph/optimizer/OptimizerUtils.cpp

@@ -464,18 +464,7 @@ Value OptimizerUtils::normalizeValue(const meta::cpp2::ColumnDef& col, const Val
  if (!col.type.type_length_ref().has_value()) {
  return Value::kNullBadType;
  }
- if (!v.isStr()) {
- return v;
- }
- auto len = static_cast<size_t>(*col.get_type().get_type_length());
- if (v.getStr().size() > len) {
- return Value(v.getStr().substr(0, len));
- } else {
- std::string s;
- s.reserve(len);
- s.append(v.getStr()).append(len - v.getStr().size(), '\0');
- return Value(std::move(s));
- }
+ return v;
  }
  case Value::Type::__EMPTY__:
  case Value::Type::NULLVALUE:

src/interface/storage.thrift

@@ -506,6 +506,10 @@ struct IndexColumnHint {
  2: ScanType scan_type,
  3: common.Value begin_value,
  4: common.Value end_value,
+ // When `columnhint` means ` >= begin_value`, `include_begin` is true
+ // and include_end is similar
+ 5: bool include_begin = true,
+ 6: bool include_end = false,
 }
 
 struct IndexQueryContext {

src/kvstore/test/CMakeLists.txt

@@ -34,6 +34,7 @@ set(KVSTORE_TEST_LIBS
  $<TARGET_OBJECTS:ft_es_storage_adapter_obj>
  $<TARGET_OBJECTS:version_obj>
  $<TARGET_OBJECTS:ssl_obj>
+ $<TARGET_OBJECTS:storage_thrift_obj>
  $<TARGET_OBJECTS:geo_index_obj>
 )
 

src/storage/CMakeLists.txt

@@ -27,6 +27,7 @@ nebula_add_library(
 nebula_add_library(
  graph_storage_service_handler OBJECT
  GraphStorageServiceHandler.cpp
+ ExprVisitorBase.cpp
  context/StorageExpressionContext.cpp
  mutate/AddVerticesProcessor.cpp
  mutate/DeleteVerticesProcessor.cpp
@@ -40,6 +41,14 @@ nebula_add_library(
  query/ScanVertexProcessor.cpp
  query/ScanEdgeProcessor.cpp
  index/LookupProcessor.cpp
+ exec/IndexNode.cpp
+ exec/IndexDedupNode.cpp
+ exec/IndexEdgeScanNode.cpp
+ exec/IndexLimitNode.cpp
+ exec/IndexProjectionNode.cpp
+ exec/IndexScanNode.cpp
+ exec/IndexSelectionNode.cpp
+ exec/IndexVertexScanNode.cpp
 )
 
 nebula_add_library(

src/storage/CommonUtils.h

@@ -200,6 +200,9 @@ struct RuntimeContext {
  ObjectPool* objPool() { return &planContext_->objPool_; }
 
  bool isPlanKilled() {
+ if (env() == nullptr) {
+ return false;
+ }
  return env()->metaClient_ &&
  env()->metaClient_->checkIsPlanKilled(planContext_->sessionId_, planContext_->planId_);
  }

src/storage/ExprVisitorBase.cpp

@@ -0,0 +1,85 @@
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License.
+ */
+#include "storage/ExprVisitorBase.h"
+namespace nebula {
+namespace storage {
+void ExprVisitorBase::visit(ConstantExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(UnaryExpression *expr) { expr->operand()->accept(this); }
+void ExprVisitorBase::visit(TypeCastingExpression *expr) { expr->operand()->accept(this); }
+void ExprVisitorBase::visit(LabelExpression *expr) { fatal(expr); }
+void ExprVisitorBase::visit(LabelAttributeExpression *expr) { fatal(expr); }
+// binary expression
+void ExprVisitorBase::visit(ArithmeticExpression *expr) {
+ expr->left()->accept(this);
+ expr->right()->accept(this);
+}
+void ExprVisitorBase::visit(RelationalExpression *expr) {
+ expr->left()->accept(this);
+ expr->right()->accept(this);
+}
+void ExprVisitorBase::visit(SubscriptExpression *expr) {
+ expr->left()->accept(this);
+ expr->right()->accept(this);
+}
+void ExprVisitorBase::visit(AttributeExpression *expr) { fatal(expr); }
+void ExprVisitorBase::visit(LogicalExpression *expr) {
+ for (auto operand : expr->operands()) {
+ operand->accept(this);
+ }
+}
+// function call
+void ExprVisitorBase::visit(FunctionCallExpression *expr) {
+ for (auto arg : expr->args()->args()) {
+ arg->accept(this);
+ }
+}
+void ExprVisitorBase::visit(AggregateExpression *expr) { fatal(expr); }
+void ExprVisitorBase::visit(UUIDExpression *expr) { UNUSED(expr); }
+// variable expression
+void ExprVisitorBase::visit(VariableExpression *expr) { fatal(expr); }
+void ExprVisitorBase::visit(VersionedVariableExpression *expr) { fatal(expr); }
+// container expression
+void ExprVisitorBase::visit(ListExpression *expr) {
+ for (auto item : expr->items()) {
+ item->accept(this);
+ }
+}
+void ExprVisitorBase::visit(SetExpression *expr) {
+ for (auto item : expr->items()) {
+ item->accept(this);
+ }
+}
+void ExprVisitorBase::visit(MapExpression *expr) { UNUSED(expr); }
+// property Expression
+void ExprVisitorBase::visit(TagPropertyExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(EdgePropertyExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(InputPropertyExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(VariablePropertyExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(DestPropertyExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(SourcePropertyExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(EdgeSrcIdExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(EdgeTypeExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(EdgeRankExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(EdgeDstIdExpression *expr) { UNUSED(expr); }
+// vertex/edge expression
+void ExprVisitorBase::visit(VertexExpression *expr) { UNUSED(expr); }
+void ExprVisitorBase::visit(EdgeExpression *expr) { UNUSED(expr); }
+// case expression
+void ExprVisitorBase::visit(CaseExpression *expr) { UNUSED(expr); }
+// path build expression
+void ExprVisitorBase::visit(PathBuildExpression *expr) { fatal(expr); }
+// column expression
+void ExprVisitorBase::visit(ColumnExpression *expr) { fatal(expr); }
+// predicate expression
+void ExprVisitorBase::visit(PredicateExpression *expr) { fatal(expr); }
+// list comprehension expression
+void ExprVisitorBase::visit(ListComprehensionExpression *expr) { fatal(expr); }
+// reduce expression
+void ExprVisitorBase::visit(ReduceExpression *expr) { fatal(expr); }
+// subscript range expression
+void ExprVisitorBase::visit(SubscriptRangeExpression *expr) { fatal(expr); }
+
+} // namespace storage
+} // namespace nebula

src/storage/ExprVisitorBase.h

@@ -0,0 +1,71 @@
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License.
+ */
+
+#pragma once
+#include "common/expression/ExprVisitor.h"
+namespace nebula {
+namespace storage {
+
+class ExprVisitorBase : public ::nebula::ExprVisitor {
+ public:
+ void visit(ConstantExpression *expr) override;
+ void visit(UnaryExpression *expr) override;
+ void visit(TypeCastingExpression *expr) override;
+ void visit(LabelExpression *expr) override;
+ void visit(LabelAttributeExpression *expr) override;
+ // binary expression
+ void visit(ArithmeticExpression *expr) override;
+ void visit(RelationalExpression *expr) override;
+ void visit(SubscriptExpression *expr) override;
+ void visit(AttributeExpression *expr) override;
+ void visit(LogicalExpression *expr) override;
+ // function call
+ void visit(FunctionCallExpression *expr) override;
+ void visit(AggregateExpression *expr) override;
+ void visit(UUIDExpression *expr) override;
+ // variable expression
+ void visit(VariableExpression *expr) override;
+ void visit(VersionedVariableExpression *expr) override;
+ // container expression
+ void visit(ListExpression *expr) override;
+ void visit(SetExpression *expr) override;
+ void visit(MapExpression *expr) override;
+ // property Expression
+ void visit(TagPropertyExpression *expr) override;
+ void visit(EdgePropertyExpression *expr) override;
+ void visit(InputPropertyExpression *expr) override;
+ void visit(VariablePropertyExpression *expr) override;
+ void visit(DestPropertyExpression *expr) override;
+ void visit(SourcePropertyExpression *expr) override;
+ void visit(EdgeSrcIdExpression *expr) override;
+ void visit(EdgeTypeExpression *expr) override;
+ void visit(EdgeRankExpression *expr) override;
+ void visit(EdgeDstIdExpression *expr) override;
+ // vertex/edge expression
+ void visit(VertexExpression *expr) override;
+ void visit(EdgeExpression *expr) override;
+ // case expression
+ void visit(CaseExpression *expr) override;
+ // path build expression
+ void visit(PathBuildExpression *expr) override;
+ // column expression
+ void visit(ColumnExpression *expr) override;
+ // predicate expression
+ void visit(PredicateExpression *expr) override;
+ // list comprehension expression
+ void visit(ListComprehensionExpression *expr) override;
+ // reduce expression
+ void visit(ReduceExpression *expr) override;
+ // subscript range expression
+ void visit(SubscriptRangeExpression *expr) override;
+
+ private:
+ using ::nebula::ExprVisitor::visit;
+ inline void fatal(Expression *expr) {
+ LOG(FATAL) << "Unexpect expression kind " << static_cast<int>(expr->kind()) << " at storage";
+ }
+};
+} // namespace storage
+} // namespace nebula