Cherry pick v3.0.1 (vesoft-inc#4007)

* fix fix 0 file error (vesoft-inc#3920)

* Fix crash of null path expression. (vesoft-inc#3915)

* Fix crash of null path expression.

* Add test case.

Co-authored-by: jakevin <30525741+jackwener@users.noreply.github.com>

* Change the name of source code zip (vesoft-inc#3999)

* Fix service crash caused by connecting using a pre-v2.6.0 client (vesoft-inc#3942)

* Reject clients with a version lower than 2.6.0

* Add TTL for clientAddr_

* Fix tests

* Use client_idle_timeout_secs as the clientAddrTimeout

* Change the param of authCheckFromCache()

Co-authored-by: panda-sheep <59197347+panda-sheep@users.noreply.github.com>
Co-authored-by: shylock <33566796+Shylock-Hg@users.noreply.github.com>
Co-authored-by: jakevin <30525741+jackwener@users.noreply.github.com>
Co-authored-by: Yichen Wang <18348405+Aiee@users.noreply.github.com>

.github/workflows/rc.yml

@@ -43,7 +43,7 @@ jobs:
  run: ./package/package.sh -v ${{ steps.tag.outputs.tagnum }} -t RelWithDebInfo -r OFF -p ON -s TRUE
  - name: output some vars
  run: |
- tar zcf ${{ env.CPACK_DIR }}/nebula-${{ steps.tag.outputs.tagnum }}.tar.gz --exclude=${{ env.BUILD_DIR }} ./*
+ tar zcf ${{ env.CPACK_DIR }}/nebula-graph-${{ steps.tag.outputs.tagnum }}.tar.gz --exclude=${{ env.BUILD_DIR }} ./*
  find ${{ env.CPACK_DIR }} -type f \( -iname \*.deb -o -iname \*.rpm -o -iname \*.tar.gz \) -exec bash -c "sha256sum {} > {}.sha256sum.txt" \;
  - uses: ./.github/actions/upload-to-oss-action
  with:

src/clients/meta/MetaClient.cpp

@@ -2430,6 +2430,7 @@ Status MetaClient::authCheckFromCache(const std::string& account, const std::str
  if (!ready_) {
  return Status::Error("Meta Service not ready");
  }
+
  folly::rcu_reader guard;
  const auto& metadata = *metadata_.load();
  auto iter = metadata.userPasswordMap_.find(account);
@@ -2600,6 +2601,11 @@ folly::Future<StatusOr<bool>> MetaClient::heartbeat() {
  }
  }
 
+ // TTL for clientAddrMap
+ // If multiple connections are created but do not authenticate, the clientAddrMap_ will keep
+ // growing. This is to clear the clientAddrMap_ regularly.
+ clearClientAddrMap();
+
  // info used in the agent, only set once
  // TOOD(spw): if we could add data path(disk) dynamicly in the future, it should be
  // reported every time it changes
@@ -3795,5 +3801,21 @@ Status MetaClient::verifyVersion() {
  }
  return Status::OK();
 }
+
+void MetaClient::clearClientAddrMap() {
+ if (clientAddrMap_.size() == 0) {
+ return;
+ }
+
+ auto curTimestamp = time::WallClock::fastNowInSec();
+ for (auto it = clientAddrMap_.cbegin(); it != clientAddrMap_.cend();) {
+ // The clientAddr is expired
+ if (it->second < curTimestamp) {
+ it = clientAddrMap_.erase(it);
+ } else {
+ ++it;
+ }
+ }
+}
 } // namespace meta
 } // namespace nebula

src/clients/meta/MetaClient.h

@@ -155,6 +155,8 @@ using MetaConfigMap =
 using FTIndexMap = std::unordered_map<std::string, cpp2::FTIndex>;
 
 using SessionMap = std::unordered_map<SessionID, cpp2::Session>;
+
+using clientAddrMap = folly::ConcurrentHashMap<HostAddr, int64_t>;
 class MetaChangedListener {
  public:
  virtual ~MetaChangedListener() = default;
@@ -669,6 +671,10 @@ class MetaClient {
  return options_.localHost_.toString();
  }
 
+ clientAddrMap& getClientAddrMap() {
+ return clientAddrMap_;
+ }
+
  protected:
  // Return true if load succeeded.
  bool loadData();
@@ -755,6 +761,9 @@ class MetaClient {
 
  Status verifyVersion();
 
+ // Removes expired keys in the clientAddrMap_
+ void clearClientAddrMap();
+
  private:
  std::shared_ptr<folly::IOThreadPoolExecutor> ioThreadPool_;
  std::shared_ptr<thrift::ThriftClientManager<cpp2::MetaServiceAsyncClient>> clientsMan_;
@@ -835,6 +844,18 @@ class MetaClient {
  NameIndexMap tagNameIndexMap_;
  NameIndexMap edgeNameIndexMap_;
 
+ // TODO(Aiee) This is a walkaround to address the problem that using a lower version(< v2.6.0)
+ // client to connect with higher version(>= v3.0.0) Nebula service will cause a crash.
+ //
+ // The key here is the host of the client that sends the request, and the value indicates the
+ // expiration of the key because we don't want to keep the key forever.
+ //
+ // The assumption here is that there is ONLY ONE VERSION of the client in the host.
+ //
+ // This map will be updated when verifyVersion() is called. Only the clients since v2.6.0 will
+ // call verifyVersion(), thus we could determine whether the client version is lower than v2.6.0
+ clientAddrMap clientAddrMap_;
+
  // Global service client
  ServiceClientsList serviceClientList_;
  FTIndexMap fulltextIndexMap_;

src/graph/planner/match/MatchClausePlanner.cpp

@@ -409,7 +409,7 @@ Status MatchClausePlanner::projectColumnsBySymbols(MatchClauseContext* matchClau
  auto iter = std::find_if(aliases.begin(), aliases.end(), [](const auto& alias) {
  return alias.second == AliasType::kPath;
  });
- if (iter != aliases.end()) {
+ if (iter != aliases.end() && path.pathBuild != nullptr) {
  auto& alias = iter->first;
  columns->addColumn(buildPathColumn(path.pathBuild, alias));
  colNames.emplace_back(alias);

src/graph/service/GraphService.cpp

@@ -23,6 +23,9 @@
 namespace nebula {
 namespace graph {
 
+// The default value is 28800 seconds
+const int64_t clientAddrTimeout = FLAGS_client_idle_timeout_secs;
+
 Status GraphService::init(std::shared_ptr<folly::IOThreadPoolExecutor> ioExecutor,
  const HostAddr& hostAddr) {
  auto addrs = network::NetworkUtils::toHosts(FLAGS_meta_server_addrs);
@@ -68,8 +71,10 @@ folly::Future<AuthResponse> GraphService::future_authenticate(const std::string&
 
  auto ctx = std::make_unique<RequestContext<AuthResponse>>();
  auto future = ctx->future();
- // check username and password failed
- auto authResult = auth(username, password);
+ // Check username and password failed
+ // Check whether the client has called verifyClientVersion()
+ auto clientAddr = HostAddr(peer->getAddressStr(), peer->getPort());
+ auto authResult = auth(username, password, clientAddr);
  if (!authResult.ok()) {
  ctx->resp().errorCode = ErrorCode::E_BAD_USERNAME_PASSWORD;
  ctx->resp().errorMsg.reset(new std::string(authResult.toString()));
@@ -201,12 +206,29 @@ folly::Future<std::string> GraphService::future_executeJsonWithParameter(
  });
 }
 
-Status GraphService::auth(const std::string& username, const std::string& password) {
+Status GraphService::auth(const std::string& username,
+ const std::string& password,
+ const HostAddr& clientIp) {
  if (!FLAGS_enable_authorize) {
  return Status::OK();
  }
 
  if (FLAGS_auth_type == "password") {
+ auto metaClient = queryEngine_->metaClient();
+ // TODO(Aiee) This is a walkaround to address the problem that using a lower version(< v2.6.0)
+ // client to connect with higher version(>= v3.0.0) Nebula service will cause a crash.
+ //
+ // Only the clients since v2.6.0 will call verifyVersion(), thus we could determine whether the
+ // client version is lower than v2.6.0
+ auto clientAddrIt = metaClient->getClientAddrMap().find(clientIp);
+ if (clientAddrIt == metaClient->getClientAddrMap().end()) {
+ return Status::Error(
+ folly::sformat("The version of the client sending request from {} is lower than v2.6.0, "
+ "please update the client.",
+ clientIp.toString()));
+ }
+
+ // Auth with PasswordAuthenticator
  auto authenticator = std::make_unique<PasswordAuthenticator>(queryEngine_->metaClient());
  return authenticator->auth(username, encryption::MD5Utils::md5Encode(password));
  } else if (FLAGS_auth_type == "cloud") {
@@ -229,6 +251,7 @@ folly::Future<cpp2::VerifyClientVersionResp> GraphService::future_verifyClientVe
  folly::splitTo<std::string>(
  ":", FLAGS_client_white_list, std::inserter(whiteList, whiteList.begin()));
  cpp2::VerifyClientVersionResp resp;
+
  if (FLAGS_enable_client_white_list && whiteList.find(req.get_version()) == whiteList.end()) {
  resp.error_code_ref() = nebula::cpp2::ErrorCode::E_CLIENT_SERVER_INCOMPATIBLE;
  resp.error_msg_ref() = folly::stringPrintf(
@@ -238,6 +261,14 @@ folly::Future<cpp2::VerifyClientVersionResp> GraphService::future_verifyClientVe
  } else {
  resp.error_code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED;
  }
+
+ // The client sent request has a version >= v2.6.0, mark the address as valid
+ auto* peer = getRequestContext()->getPeerAddress();
+ auto clientAddr = HostAddr(peer->getAddressStr(), peer->getPort());
+
+ auto ttlTimestamp = time::WallClock::fastNowInSec() + clientAddrTimeout;
+ auto clientAddrMap = &metaClient_->getClientAddrMap();
+ clientAddrMap->insert_or_assign(clientAddr, ttlTimestamp);
  return folly::makeFuture<cpp2::VerifyClientVersionResp>(std::move(resp));
 }
 } // namespace graph

src/graph/service/GraphService.h

@@ -54,7 +54,7 @@ class GraphService final : public cpp2::GraphServiceSvIf {
  std::unique_ptr<meta::MetaClient> metaClient_;
 
  private:
- Status auth(const std::string& username, const std::string& password);
+ Status auth(const std::string& username, const std::string& password, const HostAddr& clientIp);
 
  std::unique_ptr<GraphSessionManager> sessionManager_;
  std::unique_ptr<QueryEngine> queryEngine_;

src/graph/service/PasswordAuthenticator.h

@@ -16,6 +16,7 @@ class PasswordAuthenticator final : public Authenticator {
  public:
  explicit PasswordAuthenticator(meta::MetaClient* client);
 
+ // Authenticates the user by checking the user/password cache in the meta
  Status auth(const std::string& user, const std::string& password) override;
 
  private:

src/tools/db-upgrade/DbUpgrader.cpp

@@ -991,9 +991,12 @@ void UpgraderSpace::doProcessV3() {
  while (unFinishedPart_ != 0) {
  sleep(10);
  }
- auto code = readEngine_->ingest(ingest_sst_file_, true);
- if (code != ::nebula::cpp2::ErrorCode::SUCCEEDED) {
- LOG(FATAL) << "Faild upgrade 2:3 when ingest sst file:" << static_cast<int>(code);
+
+ if (ingest_sst_file_.size() != 0) {
+ auto code = readEngine_->ingest(ingest_sst_file_, true);
+ if (code != ::nebula::cpp2::ErrorCode::SUCCEEDED) {
+ LOG(FATAL) << "Faild upgrade 2:3 when ingest sst file:" << static_cast<int>(code);
+ }
  }
  readEngine_->put(NebulaKeyUtils::dataVersionKey(), NebulaKeyUtilsV3::dataVersionValue());
 }

tests/tck/features/bugfix/CrashWhenNullPathExpr.feature

@@ -0,0 +1,17 @@
+# Copyright (c) 2022 vesoft inc. All rights reserved.
+#
+# This source code is licensed under Apache 2.0 License.
+Feature: Test crash when null path expression
+
+ Background:
+ Given a graph with space named "nba"
+
+ Scenario: Null path expression in multiple patterns
+ When executing query:
+ """
+ MATCH (p:player {name: 'Yao Ming'} ), (t:team {name: 'Rockets'}), pth = (p)-[:serve*1..4]-(t)
+ RETURN pth
+ """
+ Then the result should be, in any order, with relax comparison:
+ | pth |
+ | <("Yao Ming":player{age:38,name:"Yao Ming"})-[:serve@0{end_year:2011,start_year:2002}]->("Rockets":team{name:"Rockets"})> |