Merge branch 'master' into nebula-breakpad

.dockerignore

@@ -13,6 +13,7 @@ _build.log
 _install/
 install/
 install_manifest.txt
+cmake-build-*
 
 # ccls
 .ccls

.github/workflows/rc.yml

@@ -46,7 +46,7 @@ jobs:
         run: ./package/package.sh -v ${{ steps.tag.outputs.tagnum }} -t RelWithDebInfo -r OFF -p ON -s TRUE -k ON
       - name: output some vars
         run: |
-          tar zcf ${{ env.CPACK_DIR }}/nebula-${{ steps.tag.outputs.tagnum }}.tar.gz --exclude=${{ env.BUILD_DIR }} ./*
+          tar zcf ${{ env.CPACK_DIR }}/nebula-graph-${{ steps.tag.outputs.tagnum }}.tar.gz --exclude=${{ env.BUILD_DIR }} ./*
           find ${{ env.CPACK_DIR }} -type f \( -iname \*.deb -o -iname \*.rpm -o -iname \*.tar.gz \) -exec bash -c "sha256sum {} > {}.sha256sum.txt" \;
       - uses: ./.github/actions/upload-to-oss-action
         with:

docker/Dockerfile

@@ -5,7 +5,7 @@ ARG BRANCH=master
 COPY . /home/nebula/BUILD
 
 RUN cd /home/nebula/BUILD/package \
-  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -r OFF -p ON -s TRUE -k ON
+  && ./package.sh -n OFF -b ${BRANCH} -c OFF -k ON
 
 FROM centos:7 as graphd
 

docker/Dockerfile.graphd

@@ -5,7 +5,7 @@ COPY . /home/nebula/BUILD
 ARG BRANCH=master
 
 RUN cd /home/nebula/BUILD/package \
-  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -k ON
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON
 
 FROM centos:7
 

docker/Dockerfile.metad

@@ -5,7 +5,7 @@ ARG BRANCH=master
 COPY . /home/nebula/BUILD
 
 RUN cd /home/nebula/BUILD/package \
-  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -k ON
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON
 
 FROM centos:7
 

docker/Dockerfile.storaged

@@ -5,7 +5,7 @@ ARG BRANCH=master
 COPY . /home/nebula/BUILD
 
 RUN cd /home/nebula/BUILD/package \
-  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -k ON
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON
 
 FROM centos:7
 

docker/Dockerfile.tools

@@ -5,7 +5,7 @@ ARG BRANCH=master
 COPY . /home/nebula/BUILD
 
 RUN cd /home/nebula/BUILD/package \
-  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -k ON
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON
 
 FROM centos:7
 

package/package.sh

@@ -9,11 +9,12 @@
 #   -d: Whether to enable sanitizer, default OFF
 #   -t: Build type, default Release
 #   -j: Number of threads, default $(nproc)
-#   -r: Whether enable compressed debug info, default ON
-#   -p: Whether dump the symbols from binary by dump_syms
+#   -r: Whether to enable compressed debug info, default ON
+#   -p: Whether to dump the symbols from binary by dump_syms
+#   -c: Whether to enable console building, default ON
 #   -k: Whether to enable breakpad, default OFF
 #
-# usage: ./package.sh -v <version> -n <ON/OFF> -s <TRUE/FALSE>
+# usage: ./package.sh -v <version> -n <ON/OFF> -s <TRUE/FALSE> -c <ON/OFF>
 #
 
 set -e
@@ -27,16 +28,17 @@ build_dir=${project_dir}/pkg-build
 enablesanitizer="OFF"
 static_sanitizer="OFF"
 build_type="Release"
+build_console="ON"
 branch=$(git rev-parse --abbrev-ref HEAD)
 jobs=$(nproc)
 enable_compressed_debug_info=ON
-dump_symbols=OFF
+dump_symbols="OFF"
 dump_syms_tool_dir=
 system_name=
 install_prefix=/usr/local/nebula
 enable_breakpad="OFF"
 
-while getopts v:n:s:b:d:t:r:p:j:k: opt;
+while getopts v:n:s:b:d:t:r:p:j:c:k: opt;
 do
     case $opt in
         v)
@@ -58,6 +60,9 @@ do
         t)
             build_type=$OPTARG
             ;;
+        c)
+            build_console=$OPTARG
+            ;;
         j)
             jobs=$OPTARG
             ;;
@@ -101,6 +106,7 @@ strip_enable: $strip_enable
 enablesanitizer: $enablesanitizer
 static_sanitizer: $static_sanitizer
 build_type: $build_type
+build_console: $build_console
 branch: $branch
 enable_compressed_debug_info: $enable_compressed_debug_info
 dump_symbols: $dump_symbols
@@ -123,7 +129,7 @@ function _build_graph {
           -DENABLE_STATIC_UBSAN=${ssan} \
           -DCMAKE_INSTALL_PREFIX=${install_prefix} \
           -DENABLE_TESTING=OFF \
-          -DENABLE_CONSOLE_COMPILATION=ON \
+          -DENABLE_CONSOLE_COMPILATION=${build_console} \
           -DENABLE_PACK_ONE=${package_one} \
           -DENABLE_COMPRESSED_DEBUG_INFO=${enable_compressed_debug_info} \
           -DENABLE_PACKAGE_TAR=${package_tar} \
@@ -220,7 +226,7 @@ function dump_syms {
 # The main
 build $version $enablesanitizer $static_sanitizer $build_type "OFF" "/usr/local/nebula"
 package $strip_enable
-if [[ $dump_symbols == ON ]]; then
+if [[ "$dump_symbols" == "ON" ]]; then
     echo ">>> start dump symbols <<<"
     dump_syms
 fi

resources/gflags.json

@@ -4,7 +4,6 @@
         "v",
         "heartbeat_interval_secs",
         "meta_client_retry_times",
-        "slow_op_threshold_ms",
         "clean_wal_interval_secs",
         "wal_ttl",
         "clean_wal_interval_secs",

src/clients/meta/MetaClient.cpp

@@ -738,11 +738,12 @@ void MetaClient::getResponse(Request req,
               }
 
               auto&& resp = t.value();
-              if (resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED) {
+              auto code = resp.get_code();
+              if (code == nebula::cpp2::ErrorCode::SUCCEEDED) {
                 // succeeded
                 pro.setValue(respGen(std::move(resp)));
                 return;
-              } else if (resp.get_code() == nebula::cpp2::ErrorCode::E_LEADER_CHANGED) {
+              } else if (code == nebula::cpp2::ErrorCode::E_LEADER_CHANGED) {
                 updateLeader(resp.get_leader());
                 if (retry < retryLimit) {
                   evb->runAfterDelay(
@@ -765,7 +766,7 @@ void MetaClient::getResponse(Request req,
                       FLAGS_meta_client_retry_interval_secs * 1000);
                   return;
                 }
-              } else if (resp.get_code() == nebula::cpp2::ErrorCode::E_CLIENT_SERVER_INCOMPATIBLE) {
+              } else if (code == nebula::cpp2::ErrorCode::E_CLIENT_SERVER_INCOMPATIBLE) {
                 pro.setValue(respGen(std::move(resp)));
                 return;
               } else if (resp.get_code() == nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND) {

src/common/base/CMakeLists.txt

@@ -13,7 +13,6 @@ nebula_add_library(
     Status.cpp
     SanitizerOptions.cpp
     SignalHandler.cpp
-    SlowOpTracker.cpp
     ${gdb_debug_script}
 )
 

src/common/base/test/CMakeLists.txt

@@ -79,13 +79,6 @@ nebula_add_test(
     LIBRARIES gtest gtest_main
 )
 
-nebula_add_test(
-    NAME slow_op_tracker_test
-    SOURCES SlowOpTrackerTest.cpp
-    OBJECTS $<TARGET_OBJECTS:base_obj> $<TARGET_OBJECTS:time_obj>
-    LIBRARIES gtest gtest_main
-)
-
 nebula_add_test(
     NAME lru_test
     SOURCES ConcurrentLRUCacheTest.cpp

src/common/meta/GflagsManager.cpp

@@ -55,7 +55,6 @@ std::unordered_map<std::string, std::pair<cpp2::ConfigMode, bool>> GflagsManager
       {"heartbeat_interval_secs", {cpp2::ConfigMode::MUTABLE, false}},
       {"agent_heartbeat_interval_secs", {cpp2::ConfigMode::MUTABLE, false}},
       {"meta_client_retry_times", {cpp2::ConfigMode::MUTABLE, false}},
-      {"slow_op_threshold_ms", {cpp2::ConfigMode::MUTABLE, false}},
       {"wal_ttl", {cpp2::ConfigMode::MUTABLE, false}},
       {"clean_wal_interval_secs", {cpp2::ConfigMode::MUTABLE, false}},
       {"custom_filter_interval_secs", {cpp2::ConfigMode::MUTABLE, false}},

src/common/utils/MetaKeyUtils.cpp

@@ -524,6 +524,10 @@ std::string MetaKeyUtils::schemaEdgesPrefix(GraphSpaceID spaceId) {
   return key;
 }
 
+const std::string& MetaKeyUtils::schemaEdgesPrefix() {
+  return kEdgesTable;
+}
+
 std::string MetaKeyUtils::schemaEdgeKey(GraphSpaceID spaceId,
                                         EdgeType edgeType,
                                         SchemaVer version) {
@@ -590,6 +594,10 @@ std::string MetaKeyUtils::schemaTagPrefix(GraphSpaceID spaceId, TagID tagId) {
   return key;
 }
 
+const std::string& MetaKeyUtils::schemaTagsPrefix() {
+  return kTagsTable;
+}
+
 std::string MetaKeyUtils::schemaTagsPrefix(GraphSpaceID spaceId) {
   std::string key;
   key.reserve(kTagsTable.size() + sizeof(GraphSpaceID));
@@ -625,6 +633,10 @@ std::string MetaKeyUtils::indexVal(const nebula::meta::cpp2::IndexItem& item) {
   return value;
 }
 
+const std::string& MetaKeyUtils::indexPrefix() {
+  return kIndexesTable;
+}
+
 std::string MetaKeyUtils::indexPrefix(GraphSpaceID spaceId) {
   std::string key;
   key.reserve(kIndexesTable.size() + sizeof(GraphSpaceID));
@@ -1237,7 +1249,10 @@ GraphSpaceID MetaKeyUtils::parseLocalIdSpace(folly::StringPiece rawData) {
 }
 
 /**
- * diskPartsKey = kDiskPartsTable + len(serialized(hostAddr)) + serialized(hostAddr) + path
+ * diskPartsKey = kDiskPartsTable +
+ *                len(serialized(hostAddr)) + serialized(hostAddr) +
+ *                space id +
+ *                disk path
  */
 
 HostAddr MetaKeyUtils::parseDiskPartsHost(const folly::StringPiece& rawData) {

src/common/utils/MetaKeyUtils.h

@@ -182,6 +182,8 @@ class MetaKeyUtils final {
 
   static std::string schemaEdgesPrefix(GraphSpaceID spaceId);
 
+  static const std::string& schemaEdgesPrefix();
+
   static std::string schemaEdgeKey(GraphSpaceID spaceId, EdgeType edgeType, SchemaVer version);
 
   static EdgeType parseEdgeType(folly::StringPiece key);
@@ -198,6 +200,8 @@ class MetaKeyUtils final {
 
   static std::string schemaTagsPrefix(GraphSpaceID spaceId);
 
+  static const std::string& schemaTagsPrefix();
+
   static meta::cpp2::Schema parseSchema(folly::StringPiece rawData);
 
   static std::string indexKey(GraphSpaceID spaceId, IndexID indexID);
@@ -206,6 +210,8 @@ class MetaKeyUtils final {
 
   static std::string indexPrefix(GraphSpaceID spaceId);
 
+  static const std::string& indexPrefix();
+
   static IndexID parseIndexesKeyIndexID(folly::StringPiece key);
 
   static meta::cpp2::IndexItem parseIndex(const folly::StringPiece& rawData);

src/graph/validator/AdminJobValidator.h

@@ -30,6 +30,7 @@ class AdminJobValidator final : public Validator {
     switch (sentence_->getOp()) {
       case meta::cpp2::AdminJobOp::ADD:
         switch (sentence_->getCmd()) {
+          // All jobs are space-level, except for the jobs that need to be refactored.
           case meta::cpp2::AdminCmd::REBUILD_TAG_INDEX:
           case meta::cpp2::AdminCmd::REBUILD_EDGE_INDEX:
           case meta::cpp2::AdminCmd::REBUILD_FULLTEXT_INDEX:
@@ -40,7 +41,8 @@ class AdminJobValidator final : public Validator {
           case meta::cpp2::AdminCmd::LEADER_BALANCE:
           case meta::cpp2::AdminCmd::ZONE_BALANCE:
             return true;
-          // TODO: Also space related, but not available in CreateJobExecutor now.
+          // TODO: download and ingest need to be refactored to use the rpc protocol.
+          // Currently they are using their own validator
           case meta::cpp2::AdminCmd::DOWNLOAD:
           case meta::cpp2::AdminCmd::INGEST:
           case meta::cpp2::AdminCmd::UNKNOWN:

src/interface/storage.thrift

@@ -816,12 +816,11 @@ struct ListClusterInfoReq {
 }
 
 struct AddTaskRequest {
-    // rebuild index / flush / compact / statis
+    // Task distributed to storage to execute, e.g. flush, compact, stats, etc.
     1: meta.AdminCmd                        cmd
     2: i32                                  job_id
     3: i32                                  task_id
     4: TaskPara                             para
-    5: optional i32                         concurrency
 }
 
 struct AddTaskResp {

src/kvstore/KVEngine.h

@@ -117,7 +117,9 @@ class KVEngine {
    * @param value Pointer of value
    * @return nebula::cpp2::ErrorCode
    */
-  virtual nebula::cpp2::ErrorCode get(const std::string& key, std::string* value) = 0;
+  virtual nebula::cpp2::ErrorCode get(const std::string& key,
+                                      std::string* value,
+                                      const void* snapshot = nullptr) = 0;
 
   /**
    * @brief Read a list of keys