Cherry pick v3.0.1 (#4007)

* fix fix 0 file error (#3920)

* Fix crash of null path expression. (#3915)

* Fix crash of null path expression.

* Add test case.

Co-authored-by: jakevin <30525741+jackwener@users.noreply.github.com>

* Change the name of source code zip (#3999)

* Fix service crash caused by connecting using a pre-v2.6.0 client (#3942)

* Reject clients with a version lower than 2.6.0

* Add TTL for clientAddr_

* Fix tests

* Use client_idle_timeout_secs as the clientAddrTimeout

* Change the param of authCheckFromCache()

Co-authored-by: panda-sheep <59197347+panda-sheep@users.noreply.github.com>
Co-authored-by: shylock <33566796+Shylock-Hg@users.noreply.github.com>
Co-authored-by: jakevin <30525741+jackwener@users.noreply.github.com>
Co-authored-by: Yichen Wang <18348405+Aiee@users.noreply.github.com>

.github/workflows/rc.yml

@@ -43,7 +43,7 @@ jobs:
         run: ./package/package.sh -v ${{ steps.tag.outputs.tagnum }} -t RelWithDebInfo -r OFF -p ON -s TRUE
       - name: output some vars
         run: |
-          tar zcf ${{ env.CPACK_DIR }}/nebula-${{ steps.tag.outputs.tagnum }}.tar.gz --exclude=${{ env.BUILD_DIR }} ./*
+          tar zcf ${{ env.CPACK_DIR }}/nebula-graph-${{ steps.tag.outputs.tagnum }}.tar.gz --exclude=${{ env.BUILD_DIR }} ./*
           find ${{ env.CPACK_DIR }} -type f \( -iname \*.deb -o -iname \*.rpm -o -iname \*.tar.gz \) -exec bash -c "sha256sum {} > {}.sha256sum.txt" \;
       - uses: ./.github/actions/upload-to-oss-action
         with:

src/clients/meta/MetaClient.cpp

@@ -2430,6 +2430,7 @@ Status MetaClient::authCheckFromCache(const std::string& account, const std::str
   if (!ready_) {
     return Status::Error("Meta Service not ready");
   }
+
   folly::rcu_reader guard;
   const auto& metadata = *metadata_.load();
   auto iter = metadata.userPasswordMap_.find(account);
@@ -2600,6 +2601,11 @@ folly::Future<StatusOr<bool>> MetaClient::heartbeat() {
     }
   }
 
+  // TTL for clientAddrMap
+  // If multiple connections are created but do not authenticate, the clientAddrMap_ will keep
+  // growing. This is to clear the clientAddrMap_ regularly.
+  clearClientAddrMap();
+
   // info used in the agent, only set once
   // TOOD(spw): if we could add data path(disk) dynamicly in the future, it should be
   // reported every time it changes
@@ -3795,5 +3801,21 @@ Status MetaClient::verifyVersion() {
   }
   return Status::OK();
 }
+
+void MetaClient::clearClientAddrMap() {
+  if (clientAddrMap_.size() == 0) {
+    return;
+  }
+
+  auto curTimestamp = time::WallClock::fastNowInSec();
+  for (auto it = clientAddrMap_.cbegin(); it != clientAddrMap_.cend();) {
+    // The clientAddr is expired
+    if (it->second < curTimestamp) {
+      it = clientAddrMap_.erase(it);
+    } else {
+      ++it;
+    }
+  }
+}
 }  // namespace meta
 }  // namespace nebula

src/clients/meta/MetaClient.h

@@ -155,6 +155,8 @@ using MetaConfigMap =
 using FTIndexMap = std::unordered_map<std::string, cpp2::FTIndex>;
 
 using SessionMap = std::unordered_map<SessionID, cpp2::Session>;
+
+using clientAddrMap = folly::ConcurrentHashMap<HostAddr, int64_t>;
 class MetaChangedListener {
  public:
   virtual ~MetaChangedListener() = default;
@@ -669,6 +671,10 @@ class MetaClient {
     return options_.localHost_.toString();
   }
 
+  clientAddrMap& getClientAddrMap() {
+    return clientAddrMap_;
+  }
+
  protected:
   // Return true if load succeeded.
   bool loadData();
@@ -755,6 +761,9 @@ class MetaClient {
 
   Status verifyVersion();
 
+  // Removes expired keys in the clientAddrMap_
+  void clearClientAddrMap();
+
  private:
   std::shared_ptr<folly::IOThreadPoolExecutor> ioThreadPool_;
   std::shared_ptr<thrift::ThriftClientManager<cpp2::MetaServiceAsyncClient>> clientsMan_;
@@ -835,6 +844,18 @@ class MetaClient {
   NameIndexMap tagNameIndexMap_;
   NameIndexMap edgeNameIndexMap_;
 
+  // TODO(Aiee) This is a walkaround to address the problem that using a lower version(< v2.6.0)
+  // client to connect with higher version(>= v3.0.0) Nebula service will cause a crash.
+  //
+  // The key here is the host of the client that sends the request, and the value indicates the
+  // expiration of the key because we don't want to keep the key forever.
+  //
+  // The assumption here is that there is ONLY ONE VERSION of the client in the host.
+  //
+  // This map will be updated when verifyVersion() is called. Only the clients since v2.6.0 will
+  // call verifyVersion(), thus we could determine whether the client version is lower than v2.6.0
+  clientAddrMap clientAddrMap_;
+
   // Global service client
   ServiceClientsList serviceClientList_;
   FTIndexMap fulltextIndexMap_;

src/graph/planner/match/MatchClausePlanner.cpp

@@ -409,7 +409,7 @@ Status MatchClausePlanner::projectColumnsBySymbols(MatchClauseContext* matchClau
     auto iter = std::find_if(aliases.begin(), aliases.end(), [](const auto& alias) {
       return alias.second == AliasType::kPath;
     });
-    if (iter != aliases.end()) {
+    if (iter != aliases.end() && path.pathBuild != nullptr) {
       auto& alias = iter->first;
       columns->addColumn(buildPathColumn(path.pathBuild, alias));
       colNames.emplace_back(alias);

src/graph/service/GraphService.cpp

@@ -23,6 +23,9 @@
 namespace nebula {
 namespace graph {
 
+// The default value is 28800 seconds
+const int64_t clientAddrTimeout = FLAGS_client_idle_timeout_secs;
+
 Status GraphService::init(std::shared_ptr<folly::IOThreadPoolExecutor> ioExecutor,
                           const HostAddr& hostAddr) {
   auto addrs = network::NetworkUtils::toHosts(FLAGS_meta_server_addrs);
@@ -68,8 +71,10 @@ folly::Future<AuthResponse> GraphService::future_authenticate(const std::string&
 
   auto ctx = std::make_unique<RequestContext<AuthResponse>>();
   auto future = ctx->future();
-  // check username and password failed
-  auto authResult = auth(username, password);
+  // Check username and password failed
+  // Check whether the client has called verifyClientVersion()
+  auto clientAddr = HostAddr(peer->getAddressStr(), peer->getPort());
+  auto authResult = auth(username, password, clientAddr);
   if (!authResult.ok()) {
     ctx->resp().errorCode = ErrorCode::E_BAD_USERNAME_PASSWORD;
     ctx->resp().errorMsg.reset(new std::string(authResult.toString()));
@@ -201,12 +206,29 @@ folly::Future<std::string> GraphService::future_executeJsonWithParameter(
   });
 }
 
-Status GraphService::auth(const std::string& username, const std::string& password) {
+Status GraphService::auth(const std::string& username,
+                          const std::string& password,
+                          const HostAddr& clientIp) {
   if (!FLAGS_enable_authorize) {
     return Status::OK();
   }
 
   if (FLAGS_auth_type == "password") {
+    auto metaClient = queryEngine_->metaClient();
+    // TODO(Aiee) This is a walkaround to address the problem that using a lower version(< v2.6.0)
+    // client to connect with higher version(>= v3.0.0) Nebula service will cause a crash.
+    //
+    // Only the clients since v2.6.0 will call verifyVersion(), thus we could determine whether the
+    // client version is lower than v2.6.0
+    auto clientAddrIt = metaClient->getClientAddrMap().find(clientIp);
+    if (clientAddrIt == metaClient->getClientAddrMap().end()) {
+      return Status::Error(
+          folly::sformat("The version of the client sending request from {} is lower than v2.6.0, "
+                         "please update the client.",
+                         clientIp.toString()));
+    }
+
+    // Auth with PasswordAuthenticator
     auto authenticator = std::make_unique<PasswordAuthenticator>(queryEngine_->metaClient());
     return authenticator->auth(username, encryption::MD5Utils::md5Encode(password));
   } else if (FLAGS_auth_type == "cloud") {
@@ -229,6 +251,7 @@ folly::Future<cpp2::VerifyClientVersionResp> GraphService::future_verifyClientVe
   folly::splitTo<std::string>(
       ":", FLAGS_client_white_list, std::inserter(whiteList, whiteList.begin()));
   cpp2::VerifyClientVersionResp resp;
+
   if (FLAGS_enable_client_white_list && whiteList.find(req.get_version()) == whiteList.end()) {
     resp.error_code_ref() = nebula::cpp2::ErrorCode::E_CLIENT_SERVER_INCOMPATIBLE;
     resp.error_msg_ref() = folly::stringPrintf(
@@ -238,6 +261,14 @@ folly::Future<cpp2::VerifyClientVersionResp> GraphService::future_verifyClientVe
   } else {
     resp.error_code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED;
   }
+
+  // The client sent request has a version >= v2.6.0, mark the address as valid
+  auto* peer = getRequestContext()->getPeerAddress();
+  auto clientAddr = HostAddr(peer->getAddressStr(), peer->getPort());
+
+  auto ttlTimestamp = time::WallClock::fastNowInSec() + clientAddrTimeout;
+  auto clientAddrMap = &metaClient_->getClientAddrMap();
+  clientAddrMap->insert_or_assign(clientAddr, ttlTimestamp);
   return folly::makeFuture<cpp2::VerifyClientVersionResp>(std::move(resp));
 }
 }  // namespace graph

src/graph/service/GraphService.h

@@ -54,7 +54,7 @@ class GraphService final : public cpp2::GraphServiceSvIf {
   std::unique_ptr<meta::MetaClient> metaClient_;
 
  private:
-  Status auth(const std::string& username, const std::string& password);
+  Status auth(const std::string& username, const std::string& password, const HostAddr& clientIp);
 
   std::unique_ptr<GraphSessionManager> sessionManager_;
   std::unique_ptr<QueryEngine> queryEngine_;

src/graph/service/PasswordAuthenticator.h

@@ -16,6 +16,7 @@ class PasswordAuthenticator final : public Authenticator {
  public:
   explicit PasswordAuthenticator(meta::MetaClient* client);
 
+  // Authenticates the user by checking the user/password cache in the meta
   Status auth(const std::string& user, const std::string& password) override;
 
  private:

src/tools/db-upgrade/DbUpgrader.cpp

@@ -991,9 +991,12 @@ void UpgraderSpace::doProcessV3() {
   while (unFinishedPart_ != 0) {
     sleep(10);
   }
-  auto code = readEngine_->ingest(ingest_sst_file_, true);
-  if (code != ::nebula::cpp2::ErrorCode::SUCCEEDED) {
-    LOG(FATAL) << "Faild upgrade 2:3 when ingest sst file:" << static_cast<int>(code);
+
+  if (ingest_sst_file_.size() != 0) {
+    auto code = readEngine_->ingest(ingest_sst_file_, true);
+    if (code != ::nebula::cpp2::ErrorCode::SUCCEEDED) {
+      LOG(FATAL) << "Faild upgrade 2:3 when ingest sst file:" << static_cast<int>(code);
+    }
   }
   readEngine_->put(NebulaKeyUtils::dataVersionKey(), NebulaKeyUtilsV3::dataVersionValue());
 }

tests/tck/features/bugfix/CrashWhenNullPathExpr.feature

@@ -0,0 +1,17 @@
+# Copyright (c) 2022 vesoft inc. All rights reserved.
+#
+# This source code is licensed under Apache 2.0 License.
+Feature: Test crash when null path expression
+
+  Background:
+    Given a graph with space named "nba"
+
+  Scenario: Null path expression in multiple patterns
+    When executing query:
+      """
+      MATCH (p:player {name: 'Yao Ming'} ), (t:team {name: 'Rockets'}), pth = (p)-[:serve*1..4]-(t)
+      RETURN pth
+      """
+    Then the result should be, in any order, with relax comparison:
+      | pth                                                                                                                       |
+      | <("Yao Ming":player{age:38,name:"Yao Ming"})-[:serve@0{end_year:2011,start_year:2002}]->("Rockets":team{name:"Rockets"})> |