diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index ab5c56bde50..949e842943b 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -1370,7 +1370,7 @@ Status MetaClient::handleResponse(const RESP& resp) { case nebula::cpp2::ErrorCode::E_HOST_CAN_NOT_BE_ADDED: return Status::Error("Could not add a host, which is not a storage and not expired either"); default: - return Status::Error("Unknown error!"); + return Status::Error("Unknown error %d!", static_cast(resp.get_code())); } } diff --git a/src/common/expression/Expression.cpp b/src/common/expression/Expression.cpp index 478bdbe3c71..88fc42bd602 100644 --- a/src/common/expression/Expression.cpp +++ b/src/common/expression/Expression.cpp @@ -512,10 +512,7 @@ Expression* Expression::decode(ObjectPool* pool, Expression::Decoder& decoder) { exp->resetFrom(decoder); return exp; } - case Expression::Kind::kTSPrefix: - case Expression::Kind::kTSWildcard: - case Expression::Kind::kTSRegexp: - case Expression::Kind::kTSFuzzy: { + case Expression::Kind::kESQUERY: { LOG(FATAL) << "Should not decode text search expression"; return exp; } @@ -721,17 +718,8 @@ std::ostream& operator<<(std::ostream& os, Expression::Kind kind) { case Expression::Kind::kPathBuild: os << "PathBuild"; break; - case Expression::Kind::kTSPrefix: - os << "Prefix"; - break; - case Expression::Kind::kTSWildcard: - os << "Wildcard"; - break; - case Expression::Kind::kTSRegexp: - os << "Regexp"; - break; - case Expression::Kind::kTSFuzzy: - os << "Fuzzy"; + case Expression::Kind::kESQUERY: + os << "ESQuery"; break; case Expression::Kind::kListComprehension: os << "ListComprehension"; diff --git a/src/common/expression/Expression.h b/src/common/expression/Expression.h index 31b4dcfac73..2749305cf6d 100644 --- a/src/common/expression/Expression.h +++ b/src/common/expression/Expression.h @@ -96,10 +96,7 @@ class Expression { kPathBuild, // text or key word search expression - kTSPrefix, - kTSWildcard, - kTSRegexp, - kTSFuzzy, + kESQUERY, kAggregate, kIsNull, diff --git a/src/common/expression/TextSearchExpression.cpp b/src/common/expression/TextSearchExpression.cpp index a8e556def24..f88112a0466 100644 --- a/src/common/expression/TextSearchExpression.cpp +++ b/src/common/expression/TextSearchExpression.cpp @@ -10,29 +10,14 @@ namespace nebula { bool TextSearchArgument::operator==(const TextSearchArgument& rhs) const { - return val_ == rhs.val_ && op_ == rhs.op_ && fuzziness_ == rhs.fuzziness_ && - limit_ == rhs.limit_ && timeout_ == rhs.timeout_; + return index_ == rhs.index_ && query_ == rhs.query_; } std::string TextSearchArgument::toString() const { std::string buf; buf.reserve(64); - buf = from_ + "." + prop_ + ", "; - buf += "\"" + val_ + "\""; - if (fuzziness_ == -1) { - buf += ", AUTO, "; - buf += ((op_ == "or") ? "OR" : "AND"); - } else if (fuzziness_ > -1) { - buf += ", "; - buf += folly::stringPrintf("%d, ", fuzziness_); - buf += ((op_ == "or") ? "OR" : "AND"); - } - if (limit_ != -1) { - buf += folly::stringPrintf(", %d", limit_); - } - if (timeout_ != -1) { - buf += folly::stringPrintf(", %d", timeout_); - } + buf += index_ + ", "; + buf += "\"" + query_ + "\""; return buf; } @@ -49,20 +34,8 @@ std::string TextSearchExpression::toString() const { std::string buf; buf.reserve(64); switch (kind_) { - case Kind::kTSWildcard: { - buf = "WILDCARD("; - break; - } - case Kind::kTSPrefix: { - buf = "PREFIX("; - break; - } - case Kind::kTSFuzzy: { - buf = "FUZZY("; - break; - } - case Kind::kTSRegexp: { - buf = "REGEXP("; + case Kind::kESQUERY: { + buf = "ES_QUERY("; break; } default: { diff --git a/src/common/expression/TextSearchExpression.h b/src/common/expression/TextSearchExpression.h index b4239d91933..08156dd9075 100644 --- a/src/common/expression/TextSearchExpression.h +++ b/src/common/expression/TextSearchExpression.h @@ -14,60 +14,19 @@ namespace nebula { class TextSearchArgument final { public: static TextSearchArgument* make(ObjectPool* pool, - const std::string& from, - const std::string& prop, - const std::string& val) { - return pool->makeAndAdd(from, prop, val); + const std::string& index, + const std::string& query) { + return pool->makeAndAdd(index, query); } ~TextSearchArgument() = default; - void setVal(const std::string& val) { - val_ = val; + std::string& index() { + return index_; } - const std::string& from() { - return from_; - } - - const std::string& prop() { - return prop_; - } - - const std::string& val() const { - return val_; - } - - void setOP(const std::string& op) { - op_ = op; - } - - const std::string& op() const { - return op_; - } - - void setFuzziness(int32_t fuzz) { - fuzziness_ = fuzz; - } - - int32_t fuzziness() { - return fuzziness_; - } - - void setLimit(int32_t limit) { - limit_ = limit; - } - - int32_t limit() { - return limit_; - } - - void setTimeout(int32_t timeout) { - timeout_ = timeout; - } - - int32_t timeout() { - return timeout_; + std::string& query() { + return query_; } bool operator==(const TextSearchArgument& rhs) const; @@ -76,35 +35,18 @@ class TextSearchArgument final { private: friend ObjectPool; - TextSearchArgument(const std::string& from, const std::string& prop, const std::string& val) - : from_(from), prop_(prop), val_(val) {} + TextSearchArgument(const std::string& index, const std::string& query) + : index_(index), query_(query) {} private: - std::string from_; - std::string prop_; - std::string val_; - std::string op_; - int32_t fuzziness_{-2}; - int32_t limit_{10000}; - int32_t timeout_{-1}; + std::string index_; + std::string query_; }; class TextSearchExpression : public Expression { public: - static TextSearchExpression* makePrefix(ObjectPool* pool, TextSearchArgument* arg) { - return pool->makeAndAdd(pool, Kind::kTSPrefix, arg); - } - - static TextSearchExpression* makeWildcard(ObjectPool* pool, TextSearchArgument* arg) { - return pool->makeAndAdd(pool, Kind::kTSWildcard, arg); - } - - static TextSearchExpression* makeRegexp(ObjectPool* pool, TextSearchArgument* arg) { - return pool->makeAndAdd(pool, Kind::kTSRegexp, arg); - } - - static TextSearchExpression* makeFuzzy(ObjectPool* pool, TextSearchArgument* arg) { - return pool->makeAndAdd(pool, Kind::kTSFuzzy, arg); + static TextSearchExpression* makeQuery(ObjectPool* pool, TextSearchArgument* arg) { + return pool->makeAndAdd(pool, Kind::kESQUERY, arg); } static TextSearchExpression* make(ObjectPool* pool, Kind kind, TextSearchArgument* arg) { @@ -125,7 +67,7 @@ class TextSearchExpression : public Expression { std::string toString() const override; Expression* clone() const override { - auto arg = TextSearchArgument::make(pool_, arg_->from(), arg_->prop(), arg_->val()); + auto arg = TextSearchArgument::make(pool_, arg_->index(), arg_->query()); return TextSearchExpression::make(pool_, kind_, arg); } diff --git a/src/common/function/FunctionManager.cpp b/src/common/function/FunctionManager.cpp index a4f77969c1b..b96ba774656 100644 --- a/src/common/function/FunctionManager.cpp +++ b/src/common/function/FunctionManager.cpp @@ -436,6 +436,7 @@ std::unordered_map> FunctionManager::typ {"json_extract", {TypeSignature({Value::Type::STRING}, Value::Type::MAP), TypeSignature({Value::Type::STRING}, Value::Type::NULLVALUE)}}, + {"score", {TypeSignature({}, Value::Type::__EMPTY__)}}, }; // static @@ -2937,6 +2938,17 @@ FunctionManager::FunctionManager() { return Value::kNullValue; }; } + // Score function is used to identify the score of a full-text search + { + auto &attr = functions_["score"]; + attr.minArity_ = 0; + attr.maxArity_ = 0; + attr.isAlwaysPure_ = true; + attr.body_ = [](const auto &) -> Value { + // Only placeholder, will be replaced by actual expression and need not to be evaluated + return Value::kNullValue; + }; + } } // NOLINT // static diff --git a/src/common/plugin/fulltext/elasticsearch/ESAdapter.cpp b/src/common/plugin/fulltext/elasticsearch/ESAdapter.cpp index fc6dceb18c6..d582b43e90f 100644 --- a/src/common/plugin/fulltext/elasticsearch/ESAdapter.cpp +++ b/src/common/plugin/fulltext/elasticsearch/ESAdapter.cpp @@ -13,12 +13,9 @@ namespace nebula::plugin { using namespace fmt::literals; // NOLINT -ESQueryResult::Item::Item(const std::string& v, const std::string& t) : vid(v), text(t) {} -ESQueryResult::Item::Item(const std::string& s, - const std::string& d, - int64_t r, - const std::string& t) - : src(s), dst(d), rank(r), text(t) {} +ESQueryResult::Item::Item(const std::string& v, double sc) : vid(v), score(sc) {} +ESQueryResult::Item::Item(const std::string& s, const std::string& d, int64_t r, double sc) + : src(s), dst(d), rank(r), score(sc) {} void ESBulk::put(const std::string& indexName, const std::string& vid, @@ -68,8 +65,10 @@ void ESAdapter::setClients(std::vector&& clients) { clients_ = std::move(clients); } -Status ESAdapter::createIndex(const std::string& name) { - folly::dynamic mappings = folly::parseJson(R"( +Status ESAdapter::createIndex(const std::string& name, + const std::vector& fields, + const std::string& analyzer) { + folly::dynamic obj = folly::parseJson(R"( { "mappings":{ "properties":{ @@ -84,15 +83,22 @@ Status ESAdapter::createIndex(const std::string& name) { }, "rank": { "type": "long" - }, - "text": { - "type": "keyword" } } } } )"); - auto result = randomClient().createIndex(name, mappings); + auto& mappings = obj["mappings"]; + for (auto& field : fields) { + folly::dynamic f = folly::dynamic::object(); + f["type"] = "text"; + if (!analyzer.empty()) { + f["analyzer"] = analyzer; + } + mappings["properties"][field] = std::move(f); + } + + auto result = randomClient().createIndex(name, obj); if (!result.ok()) { return result.status(); } @@ -181,70 +187,21 @@ Status ESAdapter::bulk(const ESBulk& bulk, bool refresh) { return Status::Error(folly::toJson(resp)); } -StatusOr ESAdapter::prefix(const std::string& index, - const std::string& pattern, - int64_t size, - int64_t timeout) { +StatusOr ESAdapter::queryString(const std::string& index, + const std::string& query, + int64_t from, + int64_t size) { folly::dynamic body = folly::dynamic::object(); body["query"] = folly::dynamic::object(); - body["query"]["prefix"] = folly::dynamic::object(); - body["query"]["prefix"]["text"] = pattern; + body["query"]["query_string"] = folly::dynamic::object(); + body["query"]["query_string"]["query"] = query; if (size > 0) { body["size"] = size; - body["from"] = 0; + body["from"] = from; } - return ESAdapter::query(index, body, timeout); + return ESAdapter::query(index, body, 2000); } -StatusOr ESAdapter::wildcard(const std::string& index, - const std::string& pattern, - int64_t size, - int64_t timeout) { - folly::dynamic body = folly::dynamic::object(); - body["query"] = folly::dynamic::object(); - body["query"]["wildcard"] = folly::dynamic::object("text", pattern); - if (size > 0) { - body["size"] = size; - body["from"] = 0; - } - return ESAdapter::query(index, body, timeout); -} - -StatusOr ESAdapter::regexp(const std::string& index, - const std::string& pattern, - int64_t size, - int64_t timeout) { - folly::dynamic body = folly::dynamic::object(); - body["query"] = folly::dynamic::object(); - body["query"]["regexp"] = folly::dynamic::object("text", pattern); - if (size > 0) { - body["size"] = size; - body["from"] = 0; - } - return ESAdapter::query(index, body, timeout); -} - -StatusOr ESAdapter::fuzzy(const std::string& index, - const std::string& pattern, - const std::string& fuzziness, - int64_t size, - int64_t timeout) { - folly::dynamic body = folly::dynamic::object(); - body["query"] = folly::dynamic::object(); - body["query"]["fuzzy"] = folly::dynamic::object(); - body["query"]["fuzzy"]["text"] = folly::dynamic::object("value", pattern)("fuzziness", fuzziness); - if (size > 0) { - body["size"] = size; - body["from"] = 0; - } - return ESAdapter::query(index, body, timeout); -} - -// StatusOr ESAdapter::term(const std::string& index,const std::vector& -// words){ -// folly::dynamic query = folly::dynamic::object("query",folly::dynam) -// } - StatusOr ESAdapter::match_all(const std::string& index) { folly::dynamic body = folly::dynamic::object(); body["query"] = folly::dynamic::object(); @@ -266,7 +223,11 @@ StatusOr ESAdapter::query(const std::string& index, for (auto& hit : hits) { auto source = hit["_source"]; ESQueryResult::Item item; - item.text = source["text"].getString(); + if (hit["_score"].isInt()) { + item.score = hit["_score"].getInt(); + } else if (hit["_score"].isDouble()) { + item.score = hit["_score"].getDouble(); + } item.src = source["src"].getString(); item.dst = source["dst"].getString(); item.rank = source["rank"].getInt(); diff --git a/src/common/plugin/fulltext/elasticsearch/ESAdapter.h b/src/common/plugin/fulltext/elasticsearch/ESAdapter.h index b1db86d2868..a4488b9060d 100644 --- a/src/common/plugin/fulltext/elasticsearch/ESAdapter.h +++ b/src/common/plugin/fulltext/elasticsearch/ESAdapter.h @@ -17,18 +17,20 @@ namespace nebula::plugin { struct ESQueryResult { struct Item { Item() = default; - Item(const std::string& vid, const std::string& text); - Item(const std::string& src, const std::string& dst, int64_t rank, const std::string& text); + Item(const std::string& vid, double score); + Item(const std::string& src, const std::string& dst, int64_t rank, double score); + std::string vid; // for vertex std::string src, dst; // for edge int64_t rank = 0; // for edge - std::string text; + double score; + bool operator==(const Item& item) const { return vid == item.vid && src == item.src && dst == item.dst && rank == item.rank && - text == item.text; + score == item.score; } std::ostream& operator<<(std::ostream& out) { - out << fmt::format("vid={}, src={}, dst={}, rank={}, text={}", vid, src, dst, rank, text); + out << fmt::format("vid={}, src={}, dst={}, rank={}, score={}", vid, src, dst, rank, score); return out; } }; @@ -67,143 +69,19 @@ class ESAdapter { } virtual ~ESAdapter() = default; virtual void setClients(std::vector&& clients); - virtual Status createIndex(const std::string& name); + virtual Status createIndex(const std::string& name, + const std::vector& fields, + const std::string& analyzer); virtual Status dropIndex(const std::string& name); virtual Status clearIndex(const std::string& name, bool refresh = false); virtual StatusOr isIndexExist(const std::string& name); virtual Status bulk(const ESBulk& bulk, bool refresh = false); - /** - * @brief - * - * Query: - * - * GET /_search - * { - * "query": { - * "prefix": { - * "text": { - * "value": $pattern - * } - * } - * } - * } - * - * @param index - * @param pattern - * @return StatusOr - */ - virtual StatusOr prefix(const std::string& index, - const std::string& pattern, - int64_t size = -1, - int64_t timeout = -1); - - /** - * @brief - * - * - * Query: - * - * GET /_search - * { - * "query": { - * "fuzzy": { - * "text": { - * "value": $pattern - * } - * } - * } - * } - * - * @param index - * @param pattern - * @return StatusOr - */ - virtual StatusOr fuzzy(const std::string& index, - const std::string& pattern, - const std::string& fuzziness, - int64_t size = -1, - int64_t timeout = -1); - - /** - * @brief - * - * Query: - * - * GET /_search - * { - * "query": { - * "regexp": { - * "text": { - * "value": $pattern - * } - * } - * } - * } - * - * @param index - * @param pattern - * @return StatusOr - */ - virtual StatusOr regexp(const std::string& index, - const std::string& pattern, - int64_t size = -1, - int64_t timeout = -1); - - /** - * @brief - * - * Query: - * - * GET /_search - * { - * "query": { - * "wildcard": { - * "text": { - * "value": $pattern - * } - * } - * } - * } - * - * @param index - * @param pattern - * @return StatusOr - */ - virtual StatusOr wildcard(const std::string& index, - const std::string& pattern, - int64_t size = -1, - int64_t timeout = -1); - - // /** - // * @brief - // * - // * Query: - // * - // * GET /_search - // * { - // * "query": { - // * "bool": { - // * "must": [ - // * { - // * "term": { - // * "text": { - // * "value": $words[i] - // * } - // * } - // * }, - // * ... - // * ] - // * } - // * } - // * } - // * - // * @param index - // * @param words - // * @return StatusOr - // */ - // StatusOr term(const std::string& index, const std::vector& words); + virtual StatusOr queryString(const std::string& index, + const std::string& query, + int64_t from, + int64_t size); virtual StatusOr match_all(const std::string& index); StatusOr query(const std::string& index, diff --git a/src/common/plugin/fulltext/test/ElasticsearchTest.cpp b/src/common/plugin/fulltext/test/ElasticsearchTest.cpp index ce8ea5ad15b..4ec9dc6405b 100644 --- a/src/common/plugin/fulltext/test/ElasticsearchTest.cpp +++ b/src/common/plugin/fulltext/test/ElasticsearchTest.cpp @@ -154,16 +154,16 @@ TEST_F(ESTest, createIndex) { clients.push_back(client); plugin::ESAdapter adapter(std::move(clients)); { - auto result = adapter.createIndex("nebula_index_1"); + auto result = adapter.createIndex("nebula_index_1", {"a", "b", "c"}, ""); ASSERT_TRUE(result.ok()); } { - auto result = adapter.createIndex("nebula_index_1"); + auto result = adapter.createIndex("nebula_index_1", {"a", "b", "c"}, ""); ASSERT_FALSE(result.ok()); ASSERT_EQ(result.message(), R"({"reason":"mock error"})"); } { - auto result = adapter.createIndex("nebula_index_1"); + auto result = adapter.createIndex("nebula_index_1", {"a", "b", "c"}, ""); ASSERT_FALSE(result.ok()); ASSERT_EQ(result.message(), R"(curl error(7):mock error message)"); } @@ -301,138 +301,6 @@ content-length: 78 } } -TEST_F(ESTest, prefix) { - folly::dynamic prefixBody = folly::parseJson(R"( - { - "query":{ - "prefix":{ - "text":"abc" - } - } - } - )"); - MockHttpClient mockHttpClient; - - EXPECT_CALL(mockHttpClient, - post("http://127.0.0.1:9200/nebula_index_1/_search", - std::vector{"Content-Type: application/json"}, - folly::toJson(prefixBody), - std::string(""), - std::string(""))) - .Times(3) - .WillOnce(Return(queryResultResp_)) - .WillOnce(Return(esErrorResp_)) - .WillOnce(Return(curlErrorResp_)); - plugin::ESClient client(mockHttpClient, "http", "127.0.0.1:9200", "", ""); - plugin::ESAdapter adapter(std::vector({client})); - { - auto result = adapter.prefix("nebula_index_1", "abc", -1, -1); - ASSERT_TRUE(result.ok()); - auto esResult = std::move(result).value(); - ASSERT_EQ(esResult.items.size(), 2); - ASSERT_EQ(esResult.items[0], plugin::ESQueryResult::Item("1", "vertex text")); - ASSERT_EQ(esResult.items[1], plugin::ESQueryResult::Item("a", "b", 10, "edge text")); - } - { - auto result = adapter.prefix("nebula_index_1", "abc", -1, -1); - ASSERT_FALSE(result.ok()); - ASSERT_EQ(result.status().message(), R"({"reason":"mock error"})"); - } - { - auto result = adapter.prefix("nebula_index_1", "abc", -1, -1); - ASSERT_FALSE(result.ok()); - ASSERT_EQ(result.status().message(), R"(curl error(7):mock error message)"); - } -} - -TEST_F(ESTest, wildcard) { - folly::dynamic body = folly::parseJson(R"( - { - "query":{ - "wildcard":{ - "text":"abc" - } - } - } - )"); - MockHttpClient mockHttpClient; - - EXPECT_CALL(mockHttpClient, - post("http://127.0.0.1:9200/nebula_index_1/_search", - std::vector{"Content-Type: application/json"}, - folly::toJson(body), - "", - "")) - .Times(3) - .WillOnce(Return(queryResultResp_)) - .WillOnce(Return(esErrorResp_)) - .WillOnce(Return(curlErrorResp_)); - plugin::ESClient client(mockHttpClient, "http", "127.0.0.1:9200", "", ""); - plugin::ESAdapter adapter(std::vector({client})); - { - auto result = adapter.wildcard("nebula_index_1", "abc", -1, -1); - ASSERT_TRUE(result.ok()); - auto esResult = std::move(result).value(); - ASSERT_EQ(esResult.items.size(), 2); - ASSERT_EQ(esResult.items[0], plugin::ESQueryResult::Item("1", "vertex text")); - ASSERT_EQ(esResult.items[1], plugin::ESQueryResult::Item("a", "b", 10, "edge text")); - } - { - auto result = adapter.wildcard("nebula_index_1", "abc", -1, -1); - ASSERT_FALSE(result.ok()); - ASSERT_EQ(result.status().message(), R"({"reason":"mock error"})"); - } - { - auto result = adapter.wildcard("nebula_index_1", "abc", -1, -1); - ASSERT_FALSE(result.ok()); - ASSERT_EQ(result.status().message(), R"(curl error(7):mock error message)"); - } -} - -TEST_F(ESTest, regexp) { - folly::dynamic body = folly::parseJson(R"( - { - "query":{ - "regexp":{ - "text":"abc" - } - } - } - )"); - MockHttpClient mockHttpClient; - - EXPECT_CALL(mockHttpClient, - post("http://127.0.0.1:9200/nebula_index_1/_search", - std::vector{"Content-Type: application/json"}, - folly::toJson(body), - "", - "")) - .Times(3) - .WillOnce(Return(queryResultResp_)) - .WillOnce(Return(esErrorResp_)) - .WillOnce(Return(curlErrorResp_)); - plugin::ESClient client(mockHttpClient, "http", "127.0.0.1:9200", "", ""); - plugin::ESAdapter adapter(std::vector({client})); - { - auto result = adapter.regexp("nebula_index_1", "abc", -1, -1); - ASSERT_TRUE(result.ok()); - auto esResult = std::move(result).value(); - ASSERT_EQ(esResult.items.size(), 2); - ASSERT_EQ(esResult.items[0], plugin::ESQueryResult::Item("1", "vertex text")); - ASSERT_EQ(esResult.items[1], plugin::ESQueryResult::Item("a", "b", 10, "edge text")); - } - { - auto result = adapter.regexp("nebula_index_1", "abc", -1, -1); - ASSERT_FALSE(result.ok()); - ASSERT_EQ(result.status().message(), R"({"reason":"mock error"})"); - } - { - auto result = adapter.regexp("nebula_index_1", "abc", -1, -1); - ASSERT_FALSE(result.ok()); - ASSERT_EQ(result.status().message(), R"(curl error(7):mock error message)"); - } -} - TEST_F(ESTest, bulk) { MockHttpClient mockHttpClient; @@ -474,241 +342,4 @@ TEST_F(ESTest, bulk) { } } -TEST_F(ESTest, fuzzy) { - // folly::dynamic body = folly::parseJson(R"( - // { - // "query":{ - // "fuzzy":{ - // "text":{ - // "fuzziness": "2", - // "value": "abc" - // } - // } - // } - // } - // )"); - MockHttpClient mockHttpClient; - - EXPECT_CALL(mockHttpClient, - post("http://127.0.0.1:9200/nebula_index_1/_search", - std::vector{"Content-Type: application/json"}, - _, - "", - "")) - .Times(3) - .WillOnce(Return(queryResultResp_)) - .WillOnce(Return(esErrorResp_)) - .WillOnce(Return(curlErrorResp_)); - plugin::ESClient client(mockHttpClient, "http", "127.0.0.1:9200", "", ""); - plugin::ESAdapter adapter(std::vector({client})); - { - auto result = adapter.fuzzy("nebula_index_1", "abc", "2", -1, -1); - ASSERT_TRUE(result.ok()); - auto esResult = std::move(result).value(); - ASSERT_EQ(esResult.items.size(), 2); - ASSERT_EQ(esResult.items[0], plugin::ESQueryResult::Item("1", "vertex text")); - ASSERT_EQ(esResult.items[1], plugin::ESQueryResult::Item("a", "b", 10, "edge text")); - } - { - auto result = adapter.fuzzy("nebula_index_1", "abc", "2", -1, -1); - ASSERT_FALSE(result.ok()); - ASSERT_EQ(result.status().message(), R"({"reason":"mock error"})"); - } - { - auto result = adapter.fuzzy("nebula_index_1", "abc", "2", -1, -1); - ASSERT_FALSE(result.ok()); - ASSERT_EQ(result.status().message(), R"(curl error(7):mock error message)"); - } -} - -DEFINE_string(es_address, "192.168.8.211:9200", "address of elasticsearch"); -class RealESTest : public ::testing::Test {}; - -TEST_F(RealESTest, DISABLED_CREATE_DROP_INDEX) { - plugin::ESClient client(HttpClient::instance(), "http", FLAGS_es_address, "", ""); - plugin::ESAdapter adapter(std::vector({client})); - { - auto result = adapter.createIndex("nebula_index_1"); - ASSERT_TRUE(result.ok()) << result.message(); - } - { - auto result = adapter.createIndex("nebula_index_1"); - ASSERT_FALSE(result.ok()); - ASSERT_NE(result.message().find("resource_already_exists_exception"), std::string::npos); - } - { - auto result = adapter.dropIndex("nebula_index_1"); - ASSERT_TRUE(result.ok()); - } - { - auto result = adapter.dropIndex("nebula_index_1"); - ASSERT_FALSE(result.ok()); - ASSERT_NE(result.message().find("index_not_found_exception"), std::string::npos); - } -} - -TEST_F(RealESTest, DISABLED_QUERY) { - plugin::ESClient client(HttpClient::instance(), "http", FLAGS_es_address, "", ""); - std::string indexName = "nebula_index_2"; - plugin::ESAdapter adapter(std::vector({client})); - { - auto result = adapter.createIndex(indexName); - ASSERT_TRUE(result.ok()) << result.message(); - } - { - plugin::ESBulk bulk; - bulk.put(indexName, "1", "", "", 0, "abc"); - bulk.put(indexName, "2", "", "", 0, "abcd"); - auto result = adapter.bulk(bulk, true); - ASSERT_TRUE(result.ok()) << result.message(); - } - { - auto result = adapter.prefix(indexName, "a", -1, -1); - ASSERT_TRUE(result.ok()) << result.status().message(); - auto esResult = std::move(result).value(); - ASSERT_EQ(esResult.items.size(), 2); - std::sort(esResult.items.begin(), - esResult.items.end(), - [](const plugin::ESQueryResult::Item& a, const plugin::ESQueryResult::Item& b) { - return a.vid < b.vid; - }); - ASSERT_EQ(esResult.items[0], plugin::ESQueryResult::Item("1", "abc")); - ASSERT_EQ(esResult.items[1], plugin::ESQueryResult::Item("2", "abcd")); - } - { - auto result = adapter.regexp(indexName, "a.*", -1, -1); - ASSERT_TRUE(result.ok()) << result.status().message(); - auto esResult = std::move(result).value(); - ASSERT_EQ(esResult.items.size(), 2); - std::sort(esResult.items.begin(), - esResult.items.end(), - [](const plugin::ESQueryResult::Item& a, const plugin::ESQueryResult::Item& b) { - return a.vid < b.vid; - }); - ASSERT_EQ(esResult.items[0], plugin::ESQueryResult::Item("1", "abc")); - ASSERT_EQ(esResult.items[1], plugin::ESQueryResult::Item("2", "abcd")); - } - { - auto result = adapter.prefix(indexName, "abcd", -1, -1); - ASSERT_TRUE(result.ok()) << result.status().message(); - auto esResult = std::move(result).value(); - ASSERT_EQ(esResult.items.size(), 1); - ASSERT_EQ(esResult.items[0], plugin::ESQueryResult::Item("2", "abcd")); - } - { - plugin::ESBulk bulk; - bulk.put(indexName, "2", "", "", 0, "NebulaGraph is a graph database"); - bulk.put(indexName, "3", "", "", 0, "The best graph database is NebulaGraph"); - bulk.put(indexName, "4", "", "", 0, "Nebulagraph是最好的图数据库"); - bulk.put(indexName, "5", "", "", 0, "NebulaGraph是一个图数据库"); - auto result = adapter.bulk(bulk, true); - ASSERT_TRUE(result.ok()) << result.message(); - } - { - auto result = adapter.match_all(indexName); - ASSERT_TRUE(result.ok()) << result.status().message(); - auto esResult = std::move(result).value(); - ASSERT_EQ(esResult.items.size(), 5); - std::sort(esResult.items.begin(), - esResult.items.end(), - [](const plugin::ESQueryResult::Item& a, const plugin::ESQueryResult::Item& b) { - return a.vid < b.vid; - }); - ASSERT_EQ(esResult.items[0], plugin::ESQueryResult::Item("1", "abc")); - ASSERT_EQ(esResult.items[1], - plugin::ESQueryResult::Item("2", "NebulaGraph is a graph database")); - ASSERT_EQ(esResult.items[2], - plugin::ESQueryResult::Item("3", "The best graph database is NebulaGraph")); - ASSERT_EQ(esResult.items[3], plugin::ESQueryResult::Item("4", "Nebulagraph是最好的图数据库")); - ASSERT_EQ(esResult.items[4], plugin::ESQueryResult::Item("5", "NebulaGraph是一个图数据库")); - } - { - auto result = adapter.prefix(indexName, "NebulaGraph", -1, -1); - ASSERT_TRUE(result.ok()) << result.status().message(); - auto esResult = std::move(result).value(); - ASSERT_EQ(esResult.items.size(), 2); - std::sort(esResult.items.begin(), - esResult.items.end(), - [](const plugin::ESQueryResult::Item& a, const plugin::ESQueryResult::Item& b) { - return a.vid < b.vid; - }); - ASSERT_EQ(esResult.items[0], - plugin::ESQueryResult::Item("2", "NebulaGraph is a graph database")); - ASSERT_EQ(esResult.items[1], plugin::ESQueryResult::Item("5", "NebulaGraph是一个图数据库")); - } - { - auto result = adapter.regexp(indexName, "NebulaGraph.*(图数据库|database)", -1, -1); - ASSERT_TRUE(result.ok()) << result.status().message(); - auto esResult = std::move(result).value(); - ASSERT_EQ(esResult.items.size(), 2); - std::sort(esResult.items.begin(), - esResult.items.end(), - [](const plugin::ESQueryResult::Item& a, const plugin::ESQueryResult::Item& b) { - return a.vid < b.vid; - }); - ASSERT_EQ(esResult.items[0], - plugin::ESQueryResult::Item("2", "NebulaGraph is a graph database")); - ASSERT_EQ(esResult.items[1], plugin::ESQueryResult::Item("5", "NebulaGraph是一个图数据库")); - } - { - auto result = adapter.wildcard(indexName, "Nebula?raph是*", -1, -1); - ASSERT_TRUE(result.ok()) << result.status().message(); - auto esResult = std::move(result).value(); - ASSERT_EQ(esResult.items.size(), 2); - std::sort(esResult.items.begin(), - esResult.items.end(), - [](const plugin::ESQueryResult::Item& a, const plugin::ESQueryResult::Item& b) { - return a.vid < b.vid; - }); - ASSERT_EQ(esResult.items[0], plugin::ESQueryResult::Item("4", "Nebulagraph是最好的图数据库")); - ASSERT_EQ(esResult.items[1], plugin::ESQueryResult::Item("5", "NebulaGraph是一个图数据库")); - } - { - auto result = adapter.fuzzy(indexName, "Nebulagraph is a graph Database", "2", -1, -1); - ASSERT_TRUE(result.ok()) << result.status().message(); - auto esResult = std::move(result).value(); - ASSERT_EQ(esResult.items.size(), 1); - - ASSERT_EQ(esResult.items[0], - plugin::ESQueryResult::Item("2", "NebulaGraph is a graph database")); - } - { - plugin::ESBulk bulk; - bulk.delete_(indexName, "2", "", "", 0); - auto result = adapter.bulk(bulk, true); - ASSERT_TRUE(result.ok()) << result.message(); - } - { - auto result = adapter.prefix(indexName, "NebulaGraph", -1, -1); - ASSERT_TRUE(result.ok()) << result.status().message(); - auto esResult = std::move(result).value(); - ASSERT_EQ(esResult.items.size(), 1); - ASSERT_EQ(esResult.items[0], plugin::ESQueryResult::Item("5", "NebulaGraph是一个图数据库")); - } - { - auto result = adapter.clearIndex(indexName, true); - ASSERT_TRUE(result.ok()) << result.message(); - } - { - auto result = adapter.match_all(indexName); - ASSERT_TRUE(result.ok()) << result.status().message(); - auto esResult = std::move(result).value(); - ASSERT_EQ(esResult.items.size(), 0); - } - { - auto result = adapter.isIndexExist(indexName); - ASSERT_TRUE(result.ok()) << result.status().message(); - ASSERT_TRUE(result.value()); - } - { - auto result = adapter.dropIndex(indexName); - ASSERT_TRUE(result.ok()) << result.message(); - } - { - auto result = adapter.isIndexExist(indexName); - ASSERT_TRUE(result.ok()) << result.status().message(); - ASSERT_FALSE(result.value()); - } -} - } // namespace nebula diff --git a/src/graph/context/ast/QueryAstContext.h b/src/graph/context/ast/QueryAstContext.h index 92d07570bc3..5da3ad2ed8e 100644 --- a/src/graph/context/ast/QueryAstContext.h +++ b/src/graph/context/ast/QueryAstContext.h @@ -128,7 +128,7 @@ struct LookupContext final : public AstContext { // fulltext index bool isFulltextIndex{false}; - std::string fulltextIndex; + bool hasScore{false}; Expression* fulltextExpr{nullptr}; // order by diff --git a/src/graph/executor/maintain/FTIndexExecutor.h b/src/graph/executor/maintain/FTIndexExecutor.h index cbe65faddf3..fe5124f4faa 100644 --- a/src/graph/executor/maintain/FTIndexExecutor.h +++ b/src/graph/executor/maintain/FTIndexExecutor.h @@ -6,7 +6,7 @@ #define GRAPH_EXECUTOR_ADMIN_SHOW_FT_INDEXES_EXECUTOR_H_ #include "graph/executor/Executor.h" -// full-text indexes are used to do prefix, wildcard, regexp, and fuzzy search on a string property. +// full-text indexes are used to do "query_string" search on a string property. // you can use the WHERE clause to specify the search strings in LOOKUP statements. namespace nebula { namespace graph { diff --git a/src/graph/executor/query/FulltextIndexScanExecutor.cpp b/src/graph/executor/query/FulltextIndexScanExecutor.cpp index 42fd35309bc..8c2627c2e21 100644 --- a/src/graph/executor/query/FulltextIndexScanExecutor.cpp +++ b/src/graph/executor/query/FulltextIndexScanExecutor.cpp @@ -5,6 +5,7 @@ #include "graph/executor/query/FulltextIndexScanExecutor.h" #include "common/datatypes/DataSet.h" +#include "common/datatypes/Edge.h" #include "graph/planner/plan/Query.h" #include "graph/util/FTIndexUtils.h" @@ -21,7 +22,7 @@ folly::Future FulltextIndexScanExecutor::execute() { } esAdapter_ = std::move(esAdapterResult).value(); auto* ftIndexScan = asNode(node()); - auto esQueryResult = accessFulltextIndex(ftIndexScan->index(), ftIndexScan->searchExpression()); + auto esQueryResult = accessFulltextIndex(ftIndexScan->searchExpression()); if (!esQueryResult.ok()) { LOG(ERROR) << esQueryResult.status().message(); return esQueryResult.status(); @@ -30,15 +31,20 @@ folly::Future FulltextIndexScanExecutor::execute() { const auto& space = qctx()->rctx()->session()->space(); if (!isIntVidType(space)) { if (ftIndexScan->isEdge()) { - DataSet edges({kSrc, kRank, kDst}); + DataSet edges({"edge"}); for (auto& item : esResultValue.items) { - edges.emplace_back(Row({item.src, item.rank, item.dst})); + Edge edge; + edge.src = item.src; + edge.dst = item.dst; + edge.ranking = item.rank; + edge.type = ftIndexScan->schemaId(); + edges.emplace_back(Row({std::move(edge), item.score})); } finish(ResultBuilder().value(Value(std::move(edges))).iter(Iterator::Kind::kProp).build()); } else { DataSet vertices({kVid}); for (auto& item : esResultValue.items) { - vertices.emplace_back(Row({item.vid})); + vertices.emplace_back(Row({item.vid, item.score})); } finish(ResultBuilder().value(Value(std::move(vertices))).iter(Iterator::Kind::kProp).build()); } @@ -67,46 +73,18 @@ folly::Future FulltextIndexScanExecutor::execute() { } StatusOr FulltextIndexScanExecutor::accessFulltextIndex( - const std::string& index, TextSearchExpression* tsExpr) { + TextSearchExpression* tsExpr) { std::function()> execFunc; plugin::ESAdapter& esAdapter = esAdapter_; + auto* ftIndexScan = asNode(node()); switch (tsExpr->kind()) { - case Expression::Kind::kTSFuzzy: { - std::string pattern = tsExpr->arg()->val(); - int fuzziness = tsExpr->arg()->fuzziness(); - int64_t size = tsExpr->arg()->limit(); - int64_t timeout = tsExpr->arg()->timeout(); - execFunc = [&index, pattern, &esAdapter, fuzziness, size, timeout]() { - return esAdapter.fuzzy( - index, pattern, fuzziness < 0 ? "AUTO" : std::to_string(fuzziness), size, timeout); - }; - break; - } - case Expression::Kind::kTSPrefix: { - std::string pattern = tsExpr->arg()->val(); - int64_t size = tsExpr->arg()->limit(); - int64_t timeout = tsExpr->arg()->timeout(); - execFunc = [&index, pattern, &esAdapter, size, timeout]() { - return esAdapter.prefix(index, pattern, size, timeout); - }; - break; - } - case Expression::Kind::kTSRegexp: { - std::string pattern = tsExpr->arg()->val(); - int64_t size = tsExpr->arg()->limit(); - int64_t timeout = tsExpr->arg()->timeout(); - execFunc = [&index, pattern, &esAdapter, size, timeout]() { - return esAdapter.regexp(index, pattern, size, timeout); - }; - break; - } - case Expression::Kind::kTSWildcard: { - std::string pattern = tsExpr->arg()->val(); - int64_t size = tsExpr->arg()->limit(); - int64_t timeout = tsExpr->arg()->timeout(); - execFunc = [&index, pattern, &esAdapter, size, timeout]() { - return esAdapter.wildcard(index, pattern, size, timeout); - }; + case Expression::Kind::kESQUERY: { + auto arg = tsExpr->arg(); + auto index = arg->index(); + auto query = arg->query(); + int64_t offset = ftIndexScan->offset(); + int64_t count = ftIndexScan->limit(); + execFunc = [=, &esAdapter]() { return esAdapter.queryString(index, query, offset, count); }; break; } default: { diff --git a/src/graph/executor/query/FulltextIndexScanExecutor.h b/src/graph/executor/query/FulltextIndexScanExecutor.h index 7c05f443a3d..e199635f633 100644 --- a/src/graph/executor/query/FulltextIndexScanExecutor.h +++ b/src/graph/executor/query/FulltextIndexScanExecutor.h @@ -19,8 +19,7 @@ class FulltextIndexScanExecutor final : public Executor { folly::Future execute() override; private: - StatusOr accessFulltextIndex(const std::string& index, - TextSearchExpression* expr); + StatusOr accessFulltextIndex(TextSearchExpression* expr); bool isIntVidType(const SpaceInfo& space) const { return (*space.spaceDesc.vid_type_ref()).type == nebula::cpp2::PropertyType::INT64; diff --git a/src/graph/executor/query/InnerJoinExecutor.cpp b/src/graph/executor/query/InnerJoinExecutor.cpp index 37512d971f8..24e8cc13e9f 100644 --- a/src/graph/executor/query/InnerJoinExecutor.cpp +++ b/src/graph/executor/query/InnerJoinExecutor.cpp @@ -231,7 +231,7 @@ void InnerJoinExecutor::buildNewRow(const std::unordered_mapsecond.size() - 1); ++i) { + for (std::size_t i = 0, e = range->second.size() - 1; i < e; ++i) { if (exchange_) { ds.rows.emplace_back(newRow(rRow, *range->second[i])); } else { diff --git a/src/graph/optimizer/CMakeLists.txt b/src/graph/optimizer/CMakeLists.txt index 97f7b2c243b..d7a4f48f28f 100644 --- a/src/graph/optimizer/CMakeLists.txt +++ b/src/graph/optimizer/CMakeLists.txt @@ -18,8 +18,13 @@ nebula_add_library( rule/MergeGetVerticesAndProjectRule.cpp rule/MergeGetNbrsAndDedupRule.cpp rule/MergeGetNbrsAndProjectRule.cpp + rule/MergeLimitAndFulltextIndexScanRule.cpp rule/IndexScanRule.cpp rule/PushLimitDownGetNeighborsRule.cpp + rule/PushLimitDownGetVerticesRule.cpp + rule/PushLimitDownGetEdgesRule.cpp + rule/PushLimitDownFulltextIndexScanRule.cpp + rule/PushLimitDownFulltextIndexScanRule2.cpp rule/PushLimitDownExpandAllRule.cpp rule/PushStepSampleDownGetNeighborsRule.cpp rule/PushStepLimitDownGetNeighborsRule.cpp diff --git a/src/graph/optimizer/OptRule.cpp b/src/graph/optimizer/OptRule.cpp index 23c693c2fd0..d9e46e04e8f 100644 --- a/src/graph/optimizer/OptRule.cpp +++ b/src/graph/optimizer/OptRule.cpp @@ -55,8 +55,9 @@ Pattern Pattern::create(graph::PlanNode::Kind kind, std::initializer_list kinds, - std::initializer_list patterns) { +/*static*/ +Pattern Pattern::create(std::initializer_list kinds, + std::initializer_list patterns) { return Pattern(std::move(kinds), std::move(patterns)); } diff --git a/src/graph/optimizer/rule/MergeLimitAndFulltextIndexScanRule.cpp b/src/graph/optimizer/rule/MergeLimitAndFulltextIndexScanRule.cpp new file mode 100644 index 00000000000..7d8649b97fb --- /dev/null +++ b/src/graph/optimizer/rule/MergeLimitAndFulltextIndexScanRule.cpp @@ -0,0 +1,92 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "graph/optimizer/rule/MergeLimitAndFulltextIndexScanRule.h" + +#include "graph/optimizer/OptContext.h" +#include "graph/optimizer/OptGroup.h" +#include "graph/planner/plan/PlanNode.h" +#include "graph/planner/plan/Query.h" +#include "graph/util/ExpressionUtils.h" + +using nebula::graph::Explore; +using nebula::graph::FulltextIndexScan; +using nebula::graph::Limit; +using nebula::graph::PlanNode; +using nebula::graph::QueryContext; + +namespace nebula { +namespace opt { + +std::unique_ptr MergeLimitAndFulltextIndexScanRule::kInstance = + std::unique_ptr(new MergeLimitAndFulltextIndexScanRule()); + +MergeLimitAndFulltextIndexScanRule::MergeLimitAndFulltextIndexScanRule() { + RuleSet::QueryRules().addRule(this); +} + +const Pattern &MergeLimitAndFulltextIndexScanRule::pattern() const { + static Pattern pattern = Pattern::create( + PlanNode::Kind::kLimit, + {Pattern::create({graph::PlanNode::Kind::kGetVertices, graph::PlanNode::Kind::kGetEdges}, + {Pattern::create(graph::PlanNode::Kind::kFulltextIndexScan)})}); + return pattern; +} + +bool MergeLimitAndFulltextIndexScanRule::match(OptContext *, const MatchedResult &matched) const { + auto limit = static_cast(matched.planNode()); + if (limit->offset() <= 0) { + return false; + } + auto explore = static_cast(matched.planNode({0, 0})); + if (explore->limit() <= 0) { + return false; + } + auto ft = static_cast(matched.planNode({0, 0, 0})); + return ft->limit() >= 0 && ft->offset() < 0; +} + +StatusOr MergeLimitAndFulltextIndexScanRule::transform( + OptContext *octx, const MatchedResult &matched) const { + auto limitGroupNode = matched.result().node; + auto exploreGroupNode = matched.result({0, 0}).node; + auto ftGroupNode = matched.result({0, 0, 0}).node; + + const auto limit = static_cast(limitGroupNode->node()); + const auto explore = static_cast(exploreGroupNode->node()); + const auto ft = static_cast(ftGroupNode->node()); + + auto limitRows = limit->count() + limit->offset(); + if (limitRows != explore->limit() || limitRows != ft->limit()) { + return TransformResult::noTransform(); + } + + auto newExplore = static_cast(explore->clone()); + newExplore->setOutputVar(limit->outputVar()); + auto newExploreGroupNode = OptGroupNode::create(octx, newExplore, limitGroupNode->group()); + + auto newFt = static_cast(ft->clone()); + newFt->setOffset(limit->offset()); + auto newFtGroup = OptGroup::create(octx); + auto newFtGroupNode = newFtGroup->makeGroupNode(newFt); + + newExploreGroupNode->dependsOn(newFtGroup); + newExplore->setInputVar(newFt->outputVar()); + for (auto dep : ftGroupNode->dependencies()) { + newFtGroupNode->dependsOn(dep); + } + + TransformResult result; + result.eraseAll = true; + result.newGroupNodes.emplace_back(newExploreGroupNode); + return result; +} + +std::string MergeLimitAndFulltextIndexScanRule::toString() const { + return "MergeLimitAndFulltextIndexScanRule"; +} + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/MergeLimitAndFulltextIndexScanRule.h b/src/graph/optimizer/rule/MergeLimitAndFulltextIndexScanRule.h new file mode 100644 index 00000000000..77e6f8b95b8 --- /dev/null +++ b/src/graph/optimizer/rule/MergeLimitAndFulltextIndexScanRule.h @@ -0,0 +1,68 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef GRAPH_OPTIMIZER_RULE_MERGELIMITANDFULLTEXTINDEXSCAN_H_ +#define GRAPH_OPTIMIZER_RULE_MERGELIMITANDFULLTEXTINDEXSCAN_H_ + +#include "graph/optimizer/OptRule.h" + +namespace nebula { +namespace opt { + +// Embedding limit/offset to [[FulltextIndexScan]] +// Required conditions: +// 1. Match the pattern +// Benefits: +// 1. Limit data early to optimize performance +// +// Transformation: +// Before: +// +// +--------+------------+ +// | Limit | +// | (count=3, offset=1) | +// +--------+------------+ +// | +// +---------+------------+ +// | GetVertices/GetEdges | +// | (limit=4) | +// +---------+------------+ +// | +// +---------+---------+ +// | FulltextIndexScan | +// +---------+---------+ +// +// After: +// +// +---------+------------+ +// | GetVertices/GetEdges | +// | (limit=4) | +// +---------+------------+ +// | +// +---------+-----------+ +// | FulltextIndexScan | +// | (limit=4, offset=1) | +// +---------+-----------+ + +class MergeLimitAndFulltextIndexScanRule final : public OptRule { + public: + const Pattern &pattern() const override; + + bool match(OptContext *ctx, const MatchedResult &matched) const override; + StatusOr transform(OptContext *ctx, + const MatchedResult &matched) const override; + + std::string toString() const override; + + private: + MergeLimitAndFulltextIndexScanRule(); + + static std::unique_ptr kInstance; +}; + +} // namespace opt +} // namespace nebula + +#endif diff --git a/src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule.cpp b/src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule.cpp new file mode 100644 index 00000000000..3bbfdbe242f --- /dev/null +++ b/src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule.cpp @@ -0,0 +1,80 @@ +/* Copyright (c) 2023 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "graph/optimizer/rule/PushLimitDownFulltextIndexScanRule.h" + +#include "graph/optimizer/OptContext.h" +#include "graph/optimizer/OptGroup.h" +#include "graph/planner/plan/PlanNode.h" +#include "graph/planner/plan/Query.h" +#include "graph/util/ExpressionUtils.h" + +using nebula::graph::Explore; +using nebula::graph::FulltextIndexScan; +using nebula::graph::PlanNode; +using nebula::graph::QueryContext; + +namespace nebula { +namespace opt { + +std::unique_ptr PushLimitDownFulltextIndexScanRule::kInstance = + std::unique_ptr(new PushLimitDownFulltextIndexScanRule()); + +PushLimitDownFulltextIndexScanRule::PushLimitDownFulltextIndexScanRule() { + RuleSet::QueryRules().addRule(this); +} + +const Pattern &PushLimitDownFulltextIndexScanRule::pattern() const { + static Pattern pattern = + Pattern::create({graph::PlanNode::Kind::kGetVertices, graph::PlanNode::Kind::kGetEdges}, + {Pattern::create(graph::PlanNode::Kind::kFulltextIndexScan)}); + return pattern; +} + +StatusOr PushLimitDownFulltextIndexScanRule::transform( + OptContext *octx, const MatchedResult &matched) const { + auto *qctx = octx->qctx(); + auto exploreGroupNode = matched.node; + auto ftGroupNode = matched.dependencies.front().node; + + const auto explore = static_cast(exploreGroupNode->node()); + const auto ft = static_cast(ftGroupNode->node()); + + if (!graph::ExpressionUtils::isEvaluableExpr(explore->limitExpr())) { + return TransformResult::noTransform(); + } + int64_t limitRows = explore->limit(qctx); + auto ftLimit = ft->limit(qctx); + if (ftLimit >= 0 && limitRows >= ftLimit) { + return TransformResult::noTransform(); + } + + auto newExplore = static_cast(explore->clone()); + newExplore->setOutputVar(explore->outputVar()); + auto newExploreGroupNode = OptGroupNode::create(octx, newExplore, exploreGroupNode->group()); + + auto newFt = static_cast(ft->clone()); + newFt->setLimit(limitRows); + auto newFtGroup = OptGroup::create(octx); + auto newFtGroupNode = newFtGroup->makeGroupNode(newFt); + + newExploreGroupNode->dependsOn(newFtGroup); + newExplore->setInputVar(newFt->outputVar()); + for (auto dep : ftGroupNode->dependencies()) { + newFtGroupNode->dependsOn(dep); + } + + TransformResult result; + result.eraseAll = true; + result.newGroupNodes.emplace_back(newExploreGroupNode); + return result; +} + +std::string PushLimitDownFulltextIndexScanRule::toString() const { + return "PushLimitDownFulltextIndexScanRule"; +} + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule.h b/src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule.h new file mode 100644 index 00000000000..fa6861d86f9 --- /dev/null +++ b/src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule.h @@ -0,0 +1,62 @@ +/* Copyright (c) 2023 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef GRAPH_OPTIMIZER_RULE_PUSHLIMITDOWNGETEDGESRULE_H +#define GRAPH_OPTIMIZER_RULE_PUSHLIMITDOWNGETEDGESRULE_H + +#include "graph/optimizer/OptRule.h" + +namespace nebula { +namespace opt { + +// Embedding limit to [[FulltextIndexScan]] +// Required conditions: +// 1. Match the pattern +// Benefits: +// 1. Limit data early to optimize performance +// +// Transformation: +// Before: +// +// +----------------------+ +// | GetVertices/GetEdges | +// | (limit=3) | +// +---------+------------+ +// | +// +---------+---------+ +// | FulltextIndexScan | +// +---------+---------+ +// +// After: +// +// +----------------------+ +// | GetVertices/GetEdges | +// | (limit=3) | +// +---------+------------+ +// | +// +---------+---------+ +// | FulltextIndexScan | +// | (limit=3) | +// +---------+---------+ + +class PushLimitDownFulltextIndexScanRule final : public OptRule { + public: + const Pattern &pattern() const override; + + StatusOr transform(OptContext *ctx, + const MatchedResult &matched) const override; + + std::string toString() const override; + + private: + PushLimitDownFulltextIndexScanRule(); + + static std::unique_ptr kInstance; +}; + +} // namespace opt +} // namespace nebula + +#endif diff --git a/src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule2.cpp b/src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule2.cpp new file mode 100644 index 00000000000..72482519bbe --- /dev/null +++ b/src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule2.cpp @@ -0,0 +1,118 @@ +/* Copyright (c) 2023 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "graph/optimizer/rule/PushLimitDownFulltextIndexScanRule2.h" + +#include "graph/optimizer/OptContext.h" +#include "graph/optimizer/OptGroup.h" +#include "graph/planner/plan/Logic.h" +#include "graph/planner/plan/PlanNode.h" +#include "graph/planner/plan/Query.h" +#include "graph/util/ExpressionUtils.h" + +using nebula::graph::Argument; +using nebula::graph::Explore; +using nebula::graph::FulltextIndexScan; +using nebula::graph::HashInnerJoin; +using nebula::graph::Limit; +using nebula::graph::PlanNode; +using nebula::graph::QueryContext; + +namespace nebula { +namespace opt { + +std::unique_ptr PushLimitDownFulltextIndexScanRule2::kInstance = + std::unique_ptr(new PushLimitDownFulltextIndexScanRule2()); + +PushLimitDownFulltextIndexScanRule2::PushLimitDownFulltextIndexScanRule2() { + RuleSet::QueryRules().addRule(this); +} + +const Pattern &PushLimitDownFulltextIndexScanRule2::pattern() const { + static Pattern pattern = Pattern::create( + PlanNode::Kind::kLimit, + { + Pattern::create( + PlanNode::Kind::kHashInnerJoin, + { + Pattern::create(PlanNode::Kind::kFulltextIndexScan), + Pattern::create({PlanNode::Kind::kGetVertices, PlanNode::Kind::kGetEdges}, + { + Pattern::create(PlanNode::Kind::kArgument), + }), + }), + }); + return pattern; +} + +bool PushLimitDownFulltextIndexScanRule2::match(OptContext *, const MatchedResult &matched) const { + auto ft = static_cast(matched.planNode({0, 0, 0})); + return !ft->limitExpr() || ft->limit() < 0 || ft->offset() < 0; +} + +StatusOr PushLimitDownFulltextIndexScanRule2::transform( + OptContext *octx, const MatchedResult &matched) const { + auto limitGroupNode = matched.result().node; + const auto limit = static_cast(limitGroupNode->node()); + auto newLimit = static_cast(limit->clone()); + newLimit->setOutputVar(limit->outputVar()); + auto newLimitGroupNode = OptGroupNode::create(octx, newLimit, limitGroupNode->group()); + + auto joinGroupNode = matched.result({0, 0}).node; + auto join = static_cast(joinGroupNode->node()); + auto newJoin = static_cast(join->clone()); + auto newJoinGroup = OptGroup::create(octx); + auto newJoinGroupNode = newJoinGroup->makeGroupNode(newJoin); + + newLimitGroupNode->dependsOn(newJoinGroup); + newLimit->setInputVar(newJoin->outputVar()); + + auto ftGroupNode = matched.result({0, 0, 0}).node; + const auto ft = static_cast(ftGroupNode->node()); + auto newFt = static_cast(ft->clone()); + newFt->setLimit(limit->count() + limit->offset()); + newFt->setOffset(limit->offset()); + auto newFtGroup = OptGroup::create(octx); + auto newFtGroupNode = newFtGroup->makeGroupNode(newFt); + + newJoinGroupNode->dependsOn(newFtGroup); + newJoin->setLeftVar(newFt->outputVar()); + for (auto dep : ftGroupNode->dependencies()) { + newFtGroupNode->dependsOn(dep); + } + + auto exploreGroupNode = matched.result({0, 0, 1}).node; + auto explore = static_cast(exploreGroupNode->node()); + auto newExplore = static_cast(explore->clone()); + auto newExploreGroup = OptGroup::create(octx); + auto newExploreGroupNode = newExploreGroup->makeGroupNode(newExplore); + + newJoinGroupNode->dependsOn(newExploreGroup); + newJoin->setRightVar(newExplore->outputVar()); + + auto argGroupNode = matched.result({0, 0, 1, 0}).node; + auto arg = static_cast(argGroupNode->node()); + auto newArg = static_cast(arg->clone()); + auto newArgGroup = OptGroup::create(octx); + auto newArgGroupNode = newArgGroup->makeGroupNode(newArg); + + newExploreGroupNode->dependsOn(newArgGroup); + newExplore->setInputVar(newArg->outputVar()); + for (auto dep : argGroupNode->dependencies()) { + newArgGroupNode->dependsOn(dep); + } + + TransformResult result; + result.eraseAll = true; + result.newGroupNodes.emplace_back(newLimitGroupNode); + return result; +} + +std::string PushLimitDownFulltextIndexScanRule2::toString() const { + return "PushLimitDownFulltextIndexScanRule2"; +} + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule2.h b/src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule2.h new file mode 100644 index 00000000000..30bb300ad02 --- /dev/null +++ b/src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule2.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2023 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef GRAPH_OPTIMIZER_RULE_PUSHLIMITDOWNGETEDGESRULE2_H_ +#define GRAPH_OPTIMIZER_RULE_PUSHLIMITDOWNGETEDGESRULE2_H_ + +#include "graph/optimizer/OptRule.h" + +namespace nebula { +namespace opt { + +// Embedding limit to [[FulltextIndexScan]] +// Required conditions: +// 1. Match the pattern +// Benefits: +// 1. Limit data early to optimize performance +// +// Transformation: +// Before: +// +// Limit (count=3, offset=1) +// `- HashInnerJoin +// |- GetVertices/GetEdges +// | `- Argument +// `- FulltextIndexScan +// +// After: +// +// Limit (count=3, offset=1) +// `- HashInnerJoin +// |- GetVertices/GetEdges +// | `- Argument +// `- FulltextIndexScan (limit=4, offset=1) +// + +class PushLimitDownFulltextIndexScanRule2 final : public OptRule { + public: + const Pattern &pattern() const override; + + bool match(OptContext *ctx, const MatchedResult &matched) const override; + + StatusOr transform(OptContext *ctx, + const MatchedResult &matched) const override; + + std::string toString() const override; + + private: + PushLimitDownFulltextIndexScanRule2(); + + static std::unique_ptr kInstance; +}; + +} // namespace opt +} // namespace nebula + +#endif diff --git a/src/graph/optimizer/rule/PushLimitDownGetEdgesRule.cpp b/src/graph/optimizer/rule/PushLimitDownGetEdgesRule.cpp new file mode 100644 index 00000000000..0199de7a873 --- /dev/null +++ b/src/graph/optimizer/rule/PushLimitDownGetEdgesRule.cpp @@ -0,0 +1,79 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "graph/optimizer/rule/PushLimitDownGetEdgesRule.h" + +#include "graph/optimizer/OptContext.h" +#include "graph/optimizer/OptGroup.h" +#include "graph/planner/plan/PlanNode.h" +#include "graph/planner/plan/Query.h" +#include "graph/util/ExpressionUtils.h" + +using nebula::graph::GetEdges; +using nebula::graph::Limit; +using nebula::graph::PlanNode; +using nebula::graph::QueryContext; + +namespace nebula { +namespace opt { + +std::unique_ptr PushLimitDownGetEdgesRule::kInstance = + std::unique_ptr(new PushLimitDownGetEdgesRule()); + +PushLimitDownGetEdgesRule::PushLimitDownGetEdgesRule() { + RuleSet::QueryRules().addRule(this); +} + +const Pattern &PushLimitDownGetEdgesRule::pattern() const { + static Pattern pattern = Pattern::create(graph::PlanNode::Kind::kLimit, + {Pattern::create(graph::PlanNode::Kind::kGetEdges)}); + return pattern; +} + +StatusOr PushLimitDownGetEdgesRule::transform( + OptContext *octx, const MatchedResult &matched) const { + auto *qctx = octx->qctx(); + auto limitGroupNode = matched.node; + auto geGroupNode = matched.dependencies.front().node; + + const auto limit = static_cast(limitGroupNode->node()); + const auto ge = static_cast(geGroupNode->node()); + + if (!graph::ExpressionUtils::isEvaluableExpr(limit->countExpr())) { + return TransformResult::noTransform(); + } + int64_t limitRows = limit->offset() + limit->count(qctx); + auto geLimit = ge->limit(qctx); + if (geLimit >= 0 && limitRows >= geLimit) { + return TransformResult::noTransform(); + } + + auto newLimit = static_cast(limit->clone()); + newLimit->setOutputVar(limit->outputVar()); + auto newLimitGroupNode = OptGroupNode::create(octx, newLimit, limitGroupNode->group()); + + auto newGe = static_cast(ge->clone()); + newGe->setLimit(limitRows); + auto newGeGroup = OptGroup::create(octx); + auto newGeGroupNode = newGeGroup->makeGroupNode(newGe); + + newLimitGroupNode->dependsOn(newGeGroup); + newLimit->setInputVar(newGe->outputVar()); + for (auto dep : geGroupNode->dependencies()) { + newGeGroupNode->dependsOn(dep); + } + + TransformResult result; + result.eraseAll = true; + result.newGroupNodes.emplace_back(newLimitGroupNode); + return result; +} + +std::string PushLimitDownGetEdgesRule::toString() const { + return "PushLimitDownGetEdgesRule"; +} + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownGetEdgesRule.h b/src/graph/optimizer/rule/PushLimitDownGetEdgesRule.h new file mode 100644 index 00000000000..d196006b8d0 --- /dev/null +++ b/src/graph/optimizer/rule/PushLimitDownGetEdgesRule.h @@ -0,0 +1,62 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef GRAPH_OPTIMIZER_RULE_PUSHLIMITDOWNGETEDGESRULE_H +#define GRAPH_OPTIMIZER_RULE_PUSHLIMITDOWNGETEDGESRULE_H + +#include "graph/optimizer/OptRule.h" + +namespace nebula { +namespace opt { + +// Embedding limit to [[GetEdges]] +// Required conditions: +// 1. Match the pattern +// Benefits: +// 1. Limit data early to optimize performance +// +// Transformation: +// Before: +// +// +--------+--------+ +// | Limit | +// | (limit=3) | +// +--------+--------+ +// | +// +---------+---------+ +// | GetEdges | +// +---------+---------+ +// +// After: +// +// +--------+--------+ +// | Limit | +// | (limit=3) | +// +--------+--------+ +// | +// +---------+---------+ +// | GetEdges | +// | (limit=3) | +// +---------+---------+ + +class PushLimitDownGetEdgesRule final : public OptRule { + public: + const Pattern &pattern() const override; + + StatusOr transform(OptContext *ctx, + const MatchedResult &matched) const override; + + std::string toString() const override; + + private: + PushLimitDownGetEdgesRule(); + + static std::unique_ptr kInstance; +}; + +} // namespace opt +} // namespace nebula + +#endif diff --git a/src/graph/optimizer/rule/PushLimitDownGetVerticesRule.cpp b/src/graph/optimizer/rule/PushLimitDownGetVerticesRule.cpp new file mode 100644 index 00000000000..8c7a75dc241 --- /dev/null +++ b/src/graph/optimizer/rule/PushLimitDownGetVerticesRule.cpp @@ -0,0 +1,79 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "graph/optimizer/rule/PushLimitDownGetVerticesRule.h" + +#include "graph/optimizer/OptContext.h" +#include "graph/optimizer/OptGroup.h" +#include "graph/planner/plan/PlanNode.h" +#include "graph/planner/plan/Query.h" +#include "graph/util/ExpressionUtils.h" + +using nebula::graph::GetVertices; +using nebula::graph::Limit; +using nebula::graph::PlanNode; +using nebula::graph::QueryContext; + +namespace nebula { +namespace opt { + +std::unique_ptr PushLimitDownGetVerticesRule::kInstance = + std::unique_ptr(new PushLimitDownGetVerticesRule()); + +PushLimitDownGetVerticesRule::PushLimitDownGetVerticesRule() { + RuleSet::QueryRules().addRule(this); +} + +const Pattern &PushLimitDownGetVerticesRule::pattern() const { + static Pattern pattern = Pattern::create(graph::PlanNode::Kind::kLimit, + {Pattern::create(graph::PlanNode::Kind::kGetVertices)}); + return pattern; +} + +StatusOr PushLimitDownGetVerticesRule::transform( + OptContext *octx, const MatchedResult &matched) const { + auto *qctx = octx->qctx(); + auto limitGroupNode = matched.node; + auto gvGroupNode = matched.dependencies.front().node; + + const auto limit = static_cast(limitGroupNode->node()); + const auto gv = static_cast(gvGroupNode->node()); + + if (!graph::ExpressionUtils::isEvaluableExpr(limit->countExpr())) { + return TransformResult::noTransform(); + } + int64_t limitRows = limit->offset() + limit->count(qctx); + auto gvLimit = gv->limit(qctx); + if (gvLimit >= 0 && limitRows >= gvLimit) { + return TransformResult::noTransform(); + } + + auto newLimit = static_cast(limit->clone()); + newLimit->setOutputVar(limit->outputVar()); + auto newLimitGroupNode = OptGroupNode::create(octx, newLimit, limitGroupNode->group()); + + auto newGv = static_cast(gv->clone()); + newGv->setLimit(limitRows); + auto newGvGroup = OptGroup::create(octx); + auto newGvGroupNode = newGvGroup->makeGroupNode(newGv); + + newLimitGroupNode->dependsOn(newGvGroup); + newLimit->setInputVar(newGv->outputVar()); + for (auto dep : gvGroupNode->dependencies()) { + newGvGroupNode->dependsOn(dep); + } + + TransformResult result; + result.eraseAll = true; + result.newGroupNodes.emplace_back(newLimitGroupNode); + return result; +} + +std::string PushLimitDownGetVerticesRule::toString() const { + return "PushLimitDownGetVerticesRule"; +} + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownGetVerticesRule.h b/src/graph/optimizer/rule/PushLimitDownGetVerticesRule.h new file mode 100644 index 00000000000..9bde6fec6f8 --- /dev/null +++ b/src/graph/optimizer/rule/PushLimitDownGetVerticesRule.h @@ -0,0 +1,62 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef GRAPH_OPTIMIZER_RULE_PUSHLIMITDOWNGETVERTICESRULE_H +#define GRAPH_OPTIMIZER_RULE_PUSHLIMITDOWNGETVERTICESRULE_H + +#include "graph/optimizer/OptRule.h" + +namespace nebula { +namespace opt { + +// Embedding limit to [[GetVertices]] +// Required conditions: +// 1. Match the pattern +// Benefits: +// 1. Limit data early to optimize performance +// +// Transformation: +// Before: +// +// +--------+--------+ +// | Limit | +// | (limit=3) | +// +--------+--------+ +// | +// +---------+---------+ +// | GetVertcies | +// +---------+---------+ +// +// After: +// +// +--------+--------+ +// | Limit | +// | (limit=3) | +// +--------+--------+ +// | +// +---------+---------+ +// | GetVertices | +// | (limit=3) | +// +---------+---------+ + +class PushLimitDownGetVerticesRule final : public OptRule { + public: + const Pattern &pattern() const override; + + StatusOr transform(OptContext *ctx, + const MatchedResult &matched) const override; + + std::string toString() const override; + + private: + PushLimitDownGetVerticesRule(); + + static std::unique_ptr kInstance; +}; + +} // namespace opt +} // namespace nebula + +#endif diff --git a/src/graph/planner/Planner.cpp b/src/graph/planner/Planner.cpp index 47e5459b496..02dea081d29 100644 --- a/src/graph/planner/Planner.cpp +++ b/src/graph/planner/Planner.cpp @@ -10,13 +10,6 @@ namespace nebula { namespace graph { -const char* kSrcVID = "SrcVID"; -const char* kDstVID = "DstVID"; -const char* kRanking = "Ranking"; -const char* kVertexID = "VertexID"; -const char* kVertices = "_vertices"; -const char* kEdges = "_edges"; - std::ostream& operator<<(std::ostream& os, const SubPlan& subplan) { os << "root(" << subplan.root->toString() << "): " << subplan.root->outputVar() << ", tail(" << subplan.tail->toString() << "): " << subplan.tail->outputVar(); diff --git a/src/graph/planner/Planner.h b/src/graph/planner/Planner.h index 790b8dd7285..b5a2df54045 100644 --- a/src/graph/planner/Planner.h +++ b/src/graph/planner/Planner.h @@ -12,18 +12,12 @@ #include "graph/context/ast/AstContext.h" #include "graph/planner/plan/ExecutionPlan.h" #include "graph/planner/plan/PlanNode.h" +#include "graph/util/Constants.h" namespace nebula { namespace graph { class Planner; -extern const char* kSrcVID; -extern const char* kDstVID; -extern const char* kRanking; -extern const char* kVertexID; -extern const char* kVertices; -extern const char* kEdges; - std::ostream& operator<<(std::ostream& os, const SubPlan& subplan); using MatchFunc = std::function; diff --git a/src/graph/planner/ngql/LookupPlanner.cpp b/src/graph/planner/ngql/LookupPlanner.cpp index 41fbc9fa0dc..9d3637c505c 100644 --- a/src/graph/planner/ngql/LookupPlanner.cpp +++ b/src/graph/planner/ngql/LookupPlanner.cpp @@ -8,7 +8,9 @@ #include "common/base/Base.h" #include "graph/context/ast/QueryAstContext.h" #include "graph/planner/Planner.h" +#include "graph/planner/plan/Logic.h" #include "graph/planner/plan/Scan.h" +#include "graph/util/Constants.h" #include "parser/Clauses.h" #include "parser/TraverseSentences.h" @@ -27,45 +29,64 @@ StatusOr LookupPlanner::transform(AstContext* astCtx) { auto lookupCtx = static_cast(astCtx); auto qctx = lookupCtx->qctx; auto from = static_cast(lookupCtx->sentence)->from(); + const char* kIdColName = "id"; SubPlan plan; if (lookupCtx->isFulltextIndex) { auto expr = static_cast(lookupCtx->fulltextExpr); auto fulltextIndexScan = - FulltextIndexScan::make(qctx, lookupCtx->fulltextIndex, expr, lookupCtx->isEdge); + FulltextIndexScan::make(qctx, expr, lookupCtx->isEdge, lookupCtx->schemaId); plan.tail = fulltextIndexScan; plan.root = fulltextIndexScan; + if (lookupCtx->hasScore) { + fulltextIndexScan->setColNames({kIdColName, kScore}); + + auto argument = Argument::make(qctx, kIdColName); + argument->setInputVertexRequired(false); + plan.root = argument; + } else { + fulltextIndexScan->setColNames({kIdColName}); + } + + auto* pool = qctx->objPool(); + auto spaceId = lookupCtx->space.id; + + std::vector hashKeys, probeKeys; + if (lookupCtx->isEdge) { - auto* pool = qctx->objPool(); storage::cpp2::EdgeProp edgeProp; edgeProp.type_ref() = lookupCtx->schemaId; edgeProp.props_ref() = lookupCtx->idxReturnCols; auto edgeProps = std::make_unique>(); edgeProps->emplace_back(std::move(edgeProp)); - auto getEdges = GetEdges::make(qctx, - fulltextIndexScan, - lookupCtx->space.id, - ColumnExpression::make(pool, 0), - ConstantExpression::make(pool, Value(lookupCtx->schemaId)), - ColumnExpression::make(pool, 1), - ColumnExpression::make(pool, 2), - std::move(edgeProps)); - plan.root = getEdges; + + auto src = FunctionCallExpression::make(pool, "src", {ColumnExpression::make(pool, 0)}); + auto type = FunctionCallExpression::make(pool, "typeid", {ColumnExpression::make(pool, 0)}); + auto rank = FunctionCallExpression::make(pool, "rank", {ColumnExpression::make(pool, 0)}); + auto dst = FunctionCallExpression::make(pool, "dst", {ColumnExpression::make(pool, 0)}); + + plan.root = + GetEdges::make(qctx, plan.root, spaceId, src, type, rank, dst, std::move(edgeProps)); + + hashKeys = {ColumnExpression::make(pool, 0)}; + probeKeys = {ColumnExpression::make(pool, 0)}; } else { - auto* pool = qctx->objPool(); storage::cpp2::VertexProp vertexProp; vertexProp.tag_ref() = lookupCtx->schemaId; vertexProp.props_ref() = lookupCtx->idxReturnCols; auto vertexProps = std::make_unique>(); vertexProps->emplace_back(std::move(vertexProp)); - auto getVertices = GetVertices::make(qctx, - fulltextIndexScan, - lookupCtx->space.id, - ColumnExpression::make(pool, 0), - std::move(vertexProps)); - plan.root = getVertices; + plan.root = GetVertices::make( + qctx, plan.root, spaceId, ColumnExpression::make(pool, 0), std::move(vertexProps)); + + hashKeys = {ColumnExpression::make(pool, 0)}; + probeKeys = {FunctionCallExpression::make(pool, "id", {ColumnExpression::make(pool, 0)})}; } + if (lookupCtx->hasScore) { + plan.root = HashInnerJoin::make( + qctx, fulltextIndexScan, plan.root, std::move(hashKeys), std::move(probeKeys)); + } } else { if (lookupCtx->isEdge) { auto* edgeIndexFullScan = EdgeIndexFullScan::make(qctx, diff --git a/src/graph/planner/plan/Query.cpp b/src/graph/planner/plan/Query.cpp index fdc267ef181..e36d7cfedf0 100644 --- a/src/graph/planner/plan/Query.cpp +++ b/src/graph/planner/plan/Query.cpp @@ -21,19 +21,16 @@ namespace graph { int64_t Explore::limit(QueryContext* qctx) const { DCHECK(ExpressionUtils::isEvaluableExpr(limit_, qctx)); - return DCHECK_NOTNULL(limit_) - ->eval(QueryExpressionContext(qctx ? qctx->ectx() : nullptr)()) - .getInt(); + QueryExpressionContext ctx(qctx ? qctx->ectx() : nullptr); + return DCHECK_NOTNULL(limit_)->eval(ctx).getInt(); } std::unique_ptr Explore::explain() const { auto desc = SingleInputNode::explain(); addDescription("space", folly::to(space_), desc.get()); - addDescription("dedup", folly::toJson(util::toJson(dedup_)), desc.get()); - addDescription( - "limit", folly::to(limit_ == nullptr ? "" : limit_->toString()), desc.get()); - std::string filter = filter_ == nullptr ? "" : filter_->toString(); - addDescription("filter", filter, desc.get()); + addDescription("dedup", folly::to(dedup_), desc.get()); + addDescription("limit", limit_ ? limit_->toString() : "", desc.get()); + addDescription("filter", filter_ ? filter_->toString() : "", desc.get()); addDescription("orderBy", folly::toJson(util::toJson(orderBy_)), desc.get()); return desc; } @@ -1059,15 +1056,18 @@ PlanNode* PatternApply::clone() const { } PlanNode* FulltextIndexScan::clone() const { - auto ret = FulltextIndexScan::make(qctx_, index_, searchExpr_, isEdge_); + auto ret = FulltextIndexScan::make(qctx_, searchExpr_, isEdge_, schemaId_); ret->cloneMembers(*this); + ret->setOffset(offset_); return ret; } std::unique_ptr FulltextIndexScan::explain() const { auto desc = Explore::explain(); - addDescription("isEdge", folly::toJson(util::toJson(isEdge_)), desc.get()); - // TODO(hs.zhang): add all information + addDescription("isEdge", folly::to(isEdge_), desc.get()); + addDescription("offset", folly::to(offset_), desc.get()); + addDescription("schemaId", folly::to(schemaId_), desc.get()); + addDescription("searchExpr", searchExpr_->toString(), desc.get()); return desc; } diff --git a/src/graph/planner/plan/Query.h b/src/graph/planner/plan/Query.h index 784a22d18b7..17d8bcd6253 100644 --- a/src/graph/planner/plan/Query.h +++ b/src/graph/planner/plan/Query.h @@ -753,13 +753,10 @@ class IndexScan : public Explore { class FulltextIndexScan : public Explore { public: static FulltextIndexScan* make(QueryContext* qctx, - const std::string& index, TextSearchExpression* searchExpr, - bool isEdge) { - return qctx->objPool()->makeAndAdd(qctx, index, searchExpr, isEdge); - } - const std::string& index() const { - return index_; + bool isEdge, + int32_t schemaId) { + return qctx->objPool()->makeAndAdd(qctx, searchExpr, isEdge, schemaId); } TextSearchExpression* searchExpression() const { @@ -770,6 +767,18 @@ class FulltextIndexScan : public Explore { return isEdge_; } + int64_t offset() const { + return offset_; + } + + void setOffset(int64_t offset) { + offset_ = offset; + } + + int32_t schemaId() const { + return schemaId_; + } + PlanNode* clone() const override; std::unique_ptr explain() const override; @@ -777,16 +786,18 @@ class FulltextIndexScan : public Explore { protected: friend ObjectPool; FulltextIndexScan(QueryContext* qctx, - const std::string& index, TextSearchExpression* searchExpr, - bool isEdge) + bool isEdge, + int32_t schemaId) : Explore(qctx, Kind::kFulltextIndexScan, nullptr, 0, false, -1, nullptr, {}), - index_(index), searchExpr_(searchExpr), - isEdge_(isEdge) {} - std::string index_; + isEdge_(isEdge), + schemaId_(schemaId) {} + TextSearchExpression* searchExpr_{nullptr}; bool isEdge_{false}; + int64_t offset_{-1}; + int32_t schemaId_; }; // Scan vertices diff --git a/src/graph/util/CMakeLists.txt b/src/graph/util/CMakeLists.txt index ca60803907a..c3339df4bbf 100644 --- a/src/graph/util/CMakeLists.txt +++ b/src/graph/util/CMakeLists.txt @@ -5,6 +5,7 @@ nebula_add_library( util_obj OBJECT + Constants.cpp ExpressionUtils.cpp SchemaUtil.cpp IndexUtil.cpp diff --git a/src/graph/util/Constants.cpp b/src/graph/util/Constants.cpp new file mode 100644 index 00000000000..d0d662c5110 --- /dev/null +++ b/src/graph/util/Constants.cpp @@ -0,0 +1,20 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "graph/util/Constants.h" + +namespace nebula { +namespace graph { + +const char* kSrcVID = "SrcVID"; +const char* kDstVID = "DstVID"; +const char* kRanking = "Ranking"; +const char* kVertexID = "VertexID"; +const char* kVertices = "_vertices"; +const char* kEdges = "_edges"; +const char* kScore = "score"; + +} // namespace graph +} // namespace nebula diff --git a/src/graph/util/Constants.h b/src/graph/util/Constants.h new file mode 100644 index 00000000000..b37c7ae9311 --- /dev/null +++ b/src/graph/util/Constants.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef GRAPH_UTIL_CONSTANTS_H_ +#define GRAPH_UTIL_CONSTANTS_H_ + +namespace nebula { +namespace graph { + +extern const char* kSrcVID; +extern const char* kDstVID; +extern const char* kRanking; +extern const char* kVertexID; +extern const char* kVertices; +extern const char* kEdges; +extern const char* kScore; + +} // namespace graph +} // namespace nebula + +#endif // GRAPH_UTIL_CONSTANTS_H_ diff --git a/src/graph/util/ExpressionUtils.cpp b/src/graph/util/ExpressionUtils.cpp index 29d95db9755..9c2edbfa67f 100644 --- a/src/graph/util/ExpressionUtils.cpp +++ b/src/graph/util/ExpressionUtils.cpp @@ -1557,10 +1557,7 @@ bool ExpressionUtils::checkExprDepth(const Expression *expr) { case Expression::Kind::kUUID: case Expression::Kind::kPathBuild: case Expression::Kind::kColumn: - case Expression::Kind::kTSPrefix: - case Expression::Kind::kTSWildcard: - case Expression::Kind::kTSRegexp: - case Expression::Kind::kTSFuzzy: + case Expression::Kind::kESQUERY: case Expression::Kind::kAggregate: case Expression::Kind::kSubscriptRange: case Expression::Kind::kVersionedVar: diff --git a/src/graph/util/FTIndexUtils.cpp b/src/graph/util/FTIndexUtils.cpp index ab64e060d30..84e5a3d60b8 100644 --- a/src/graph/util/FTIndexUtils.cpp +++ b/src/graph/util/FTIndexUtils.cpp @@ -14,10 +14,7 @@ namespace graph { bool FTIndexUtils::needTextSearch(const Expression* expr) { switch (expr->kind()) { - case Expression::Kind::kTSFuzzy: - case Expression::Kind::kTSPrefix: - case Expression::Kind::kTSRegexp: - case Expression::Kind::kTSWildcard: { + case Expression::Kind::kESQUERY: { return true; } default: @@ -44,96 +41,6 @@ StatusOr<::nebula::plugin::ESAdapter> FTIndexUtils::getESAdapter(meta::MetaClien return ::nebula::plugin::ESAdapter(std::move(clients)); } -StatusOr FTIndexUtils::rewriteTSFilter(ObjectPool* pool, - bool isEdge, - Expression* expr, - const std::string& index, - ::nebula::plugin::ESAdapter& esAdapter) { - auto vRet = textSearch(expr, index, esAdapter); - if (!vRet.ok()) { - return vRet.status(); - } - auto result = std::move(vRet).value(); - if (result.items.empty()) { - return nullptr; - } - auto tsArg = static_cast(expr)->arg(); - Expression* propExpr; - if (isEdge) { - propExpr = EdgePropertyExpression::make(pool, tsArg->from(), tsArg->prop()); - } else { - propExpr = TagPropertyExpression::make(pool, tsArg->from(), tsArg->prop()); - } - std::vector rels; - for (auto& item : result.items) { - auto constExpr = ConstantExpression::make(pool, Value(item.text)); - rels.emplace_back(RelationalExpression::makeEQ(pool, propExpr, constExpr)); - } - if (rels.size() == 1) { - return rels.front(); - } - return ExpressionUtils::pushOrs(pool, rels); -} - -StatusOr FTIndexUtils::textSearch( - Expression* expr, const std::string& index, ::nebula::plugin::ESAdapter& esAdapter) { - auto tsExpr = static_cast(expr); - std::function()> execFunc; - switch (tsExpr->kind()) { - case Expression::Kind::kTSFuzzy: { - std::string pattern = tsExpr->arg()->val(); - int fuzziness = tsExpr->arg()->fuzziness(); - int64_t size = tsExpr->arg()->limit(); - int64_t timeout = tsExpr->arg()->timeout(); - execFunc = [&index, pattern, &esAdapter, fuzziness, size, timeout]() { - return esAdapter.fuzzy( - index, pattern, fuzziness < 0 ? "AUTO" : std::to_string(fuzziness), size, timeout); - }; - break; - } - case Expression::Kind::kTSPrefix: { - std::string pattern = tsExpr->arg()->val(); - int64_t size = tsExpr->arg()->limit(); - int64_t timeout = tsExpr->arg()->timeout(); - execFunc = [&index, pattern, &esAdapter, size, timeout]() { - return esAdapter.prefix(index, pattern, size, timeout); - }; - break; - } - case Expression::Kind::kTSRegexp: { - std::string pattern = tsExpr->arg()->val(); - int64_t size = tsExpr->arg()->limit(); - int64_t timeout = tsExpr->arg()->timeout(); - execFunc = [&index, pattern, &esAdapter, size, timeout]() { - return esAdapter.regexp(index, pattern, size, timeout); - }; - break; - } - case Expression::Kind::kTSWildcard: { - std::string pattern = tsExpr->arg()->val(); - int64_t size = tsExpr->arg()->limit(); - int64_t timeout = tsExpr->arg()->timeout(); - execFunc = [&index, pattern, &esAdapter, size, timeout]() { - return esAdapter.wildcard(index, pattern, size, timeout); - }; - break; - } - default: { - return Status::SemanticError("text search expression error"); - } - } - - auto retryCnt = FLAGS_ft_request_retry_times > 0 ? FLAGS_ft_request_retry_times : 1; - StatusOr result; - while (retryCnt-- > 0) { - result = execFunc(); - if (!result.ok()) { - continue; - } - break; - } - return result; -} } // namespace graph } // namespace nebula diff --git a/src/graph/util/ValidateUtil.cpp b/src/graph/util/ValidateUtil.cpp index 28af23d14a6..7cadb7b9dfb 100644 --- a/src/graph/util/ValidateUtil.cpp +++ b/src/graph/util/ValidateUtil.cpp @@ -12,6 +12,7 @@ #include "graph/planner/plan/Query.h" #include "graph/util/ExpressionUtils.h" #include "graph/util/SchemaUtil.h" +#include "graph/util/Utils.h" namespace nebula { namespace graph { @@ -96,14 +97,8 @@ Status ValidateUtil::validateOver(QueryContext* qctx, const OverClause* clause, Status ValidateUtil::invalidLabelIdentifiers(const Expression* expr) { auto labelExprs = ExpressionUtils::collectAll(expr, {Expression::Kind::kLabel}); if (!labelExprs.empty()) { - std::stringstream ss; - ss << "Invalid label identifiers: "; - for (auto* label : labelExprs) { - ss << label->toString() << ","; - } - auto errMsg = ss.str(); - errMsg.pop_back(); - return Status::SemanticError(std::move(errMsg)); + auto errMsg = util::join(labelExprs, [](auto* e) { return e->toString(); }); + return Status::SemanticError("Invalid label identifiers: %s", errMsg.c_str()); } return Status::OK(); } diff --git a/src/graph/util/test/CMakeLists.txt b/src/graph/util/test/CMakeLists.txt index 8ec346efe20..e8eadf9bca8 100644 --- a/src/graph/util/test/CMakeLists.txt +++ b/src/graph/util/test/CMakeLists.txt @@ -27,7 +27,6 @@ nebula_add_test( SOURCES ExpressionUtilsTest.cpp IdGeneratorTest.cpp - FTindexUtilsTest.cpp OBJECTS $ $ diff --git a/src/graph/util/test/FTindexUtilsTest.cpp b/src/graph/util/test/FTindexUtilsTest.cpp deleted file mode 100644 index 41dff6a0dd4..00000000000 --- a/src/graph/util/test/FTindexUtilsTest.cpp +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright (c) 2022 vesoft inc. All rights reserved. -// -// This source code is licensed under Apache 2.0 License. - -#include "common/plugin/fulltext/elasticsearch/ESAdapter.h" -#include "glog/logging.h" -#include "gmock/gmock.h" -#include "graph/util/ExpressionUtils.h" -#include "graph/util/FTIndexUtils.h" -#include "gtest/gtest.h" - -DECLARE_uint32(ft_request_retry_times); - -using ::testing::_; -using ::testing::Return; - -namespace nebula::graph { - -class MockESAdapter : public plugin::ESAdapter { - public: - MOCK_METHOD(StatusOr, - fuzzy, - (const std::string&, const std::string&, const std::string&, int64_t, int64_t), - (override)); - MOCK_METHOD(StatusOr, - prefix, - (const std::string&, const std::string&, int64_t, int64_t), - (override)); - MOCK_METHOD(StatusOr, - regexp, - (const std::string&, const std::string&, int64_t, int64_t), - (override)); - MOCK_METHOD(StatusOr, - wildcard, - (const std::string&, const std::string&, int64_t, int64_t), - (override)); -}; - -class FTIndexUtilsTest : public ::testing::Test { - public: - protected: - ObjectPool pool_; - Expression* eq(Expression* left, Expression* right) { - return RelationalExpression::makeEQ(&pool_, left, right); - } - Expression* or_(const std::vector& exprs) { - return ExpressionUtils::pushOrs(&pool_, exprs); - } - Expression* tagProp(const std::string& tag, const std::string& prop) { - return TagPropertyExpression::make(&pool_, tag, prop); - } - Expression* edgeProp(const std::string& edge, const std::string& prop) { - return EdgePropertyExpression::make(&pool_, edge, prop); - } - template - Expression* constant(const T& value) { - return ConstantExpression::make(&pool_, Value(value)); - } -}; - -TEST_F(FTIndexUtilsTest, rewriteTSFilter) { - ObjectPool pool; - std::string indexName = "test_index"; - std::string tagName = "tag1"; - std::string edgeName = "edge1"; - std::string propName = "prop1"; - using Item = plugin::ESQueryResult::Item; - std::vector items = {Item("1", "abc"), Item("2", "abc1"), Item("3", "abc")}; - plugin::ESQueryResult esResult; - esResult.items = items; - { - MockESAdapter mockESAdapter; - EXPECT_CALL(mockESAdapter, prefix(indexName, "prefix_pattern", 10000, -1)) - .WillOnce(Return(esResult)); - auto argument = TextSearchArgument::make(&pool, tagName, propName, "prefix_pattern"); - auto expr = TextSearchExpression::makePrefix(&pool, argument); - auto expect = or_({eq(tagProp(tagName, propName), constant("abc")), - eq(tagProp(tagName, propName), constant("abc1")), - eq(tagProp(tagName, propName), constant("abc"))}); - auto result = FTIndexUtils::rewriteTSFilter(&pool, false, expr, indexName, mockESAdapter); - ASSERT_TRUE(result.ok()) << result.status().message(); - ASSERT_EQ(*result.value(), *expect) << result.value()->toString() << "\t" << expect->toString(); - } - { - plugin::ESQueryResult emptyEsResult; - MockESAdapter mockESAdapter; - EXPECT_CALL(mockESAdapter, prefix(indexName, "prefix_pattern", 10000, -1)) - .WillOnce(Return(emptyEsResult)); - auto argument = TextSearchArgument::make(&pool, tagName, propName, "prefix_pattern"); - auto expr = TextSearchExpression::makePrefix(&pool, argument); - auto result = FTIndexUtils::rewriteTSFilter(&pool, false, expr, indexName, mockESAdapter); - ASSERT_TRUE(result.ok()) << result.status().message(); - ASSERT_EQ(result.value(), nullptr); - } - { - Status status = Status::Error("mock error"); - MockESAdapter mockESAdapter; - EXPECT_CALL(mockESAdapter, prefix(indexName, "prefix_pattern", 10000, -1)) - .Times(FLAGS_ft_request_retry_times) - .WillRepeatedly(Return(status)); - auto argument = TextSearchArgument::make(&pool, tagName, propName, "prefix_pattern"); - auto expr = TextSearchExpression::makePrefix(&pool, argument); - auto result = FTIndexUtils::rewriteTSFilter(&pool, false, expr, indexName, mockESAdapter); - ASSERT_FALSE(result.ok()); - ASSERT_EQ(result.status().message(), "mock error"); - } - { - MockESAdapter mockESAdapter; - EXPECT_CALL(mockESAdapter, wildcard(indexName, "wildcard_pattern", 10000, -1)) - .WillOnce(Return(esResult)); - auto argument = TextSearchArgument::make(&pool, edgeName, propName, "wildcard_pattern"); - auto expr = TextSearchExpression::makeWildcard(&pool, argument); - auto expect = or_({eq(edgeProp(edgeName, propName), constant("abc")), - eq(edgeProp(edgeName, propName), constant("abc1")), - eq(edgeProp(edgeName, propName), constant("abc"))}); - auto result = FTIndexUtils::rewriteTSFilter(&pool, true, expr, indexName, mockESAdapter); - ASSERT_TRUE(result.ok()) << result.status().message(); - ASSERT_EQ(*result.value(), *expect) << result.value()->toString() << "\t" << expect->toString(); - } - { - plugin::ESQueryResult singleEsResult; - singleEsResult.items = {Item("a", "b", 1, "edge text")}; - MockESAdapter mockESAdapter; - EXPECT_CALL(mockESAdapter, wildcard(indexName, "wildcard_pattern", 10000, -1)) - .WillOnce(Return(singleEsResult)); - auto argument = TextSearchArgument::make(&pool, edgeName, propName, "wildcard_pattern"); - auto expr = TextSearchExpression::makeWildcard(&pool, argument); - auto expect = eq(edgeProp(edgeName, propName), constant("edge text")); - auto result = FTIndexUtils::rewriteTSFilter(&pool, true, expr, indexName, mockESAdapter); - ASSERT_TRUE(result.ok()) << result.status().message(); - ASSERT_EQ(*result.value(), *expect) << result.value()->toString() << "\t" << expect->toString(); - } - { - MockESAdapter mockESAdapter; - EXPECT_CALL(mockESAdapter, regexp(indexName, "regexp_pattern", 10000, -1)) - .WillOnce(Return(esResult)); - auto argument = TextSearchArgument::make(&pool, edgeName, propName, "regexp_pattern"); - auto expr = TextSearchExpression::makeRegexp(&pool, argument); - auto expect = or_({eq(edgeProp(edgeName, propName), constant("abc")), - eq(edgeProp(edgeName, propName), constant("abc1")), - eq(edgeProp(edgeName, propName), constant("abc"))}); - auto result = FTIndexUtils::rewriteTSFilter(&pool, true, expr, indexName, mockESAdapter); - ASSERT_TRUE(result.ok()) << result.status().message(); - ASSERT_EQ(*result.value(), *expect) << result.value()->toString() << "\t" << expect->toString(); - } - { - MockESAdapter mockESAdapter; - EXPECT_CALL(mockESAdapter, fuzzy(indexName, "fuzzy_pattern", "1", 10000, -1)) - .WillOnce(Return(esResult)); - auto argument = TextSearchArgument::make(&pool, tagName, propName, "fuzzy_pattern"); - argument->setFuzziness(1); - auto expr = TextSearchExpression::makeFuzzy(&pool, argument); - auto expect = or_({eq(tagProp(tagName, propName), constant("abc")), - eq(tagProp(tagName, propName), constant("abc1")), - eq(tagProp(tagName, propName), constant("abc"))}); - auto result = FTIndexUtils::rewriteTSFilter(&pool, false, expr, indexName, mockESAdapter); - ASSERT_TRUE(result.ok()) << result.status().message(); - ASSERT_EQ(*result.value(), *expect) << result.value()->toString() << "\t" << expect->toString(); - } -} - -} // namespace nebula::graph diff --git a/src/graph/validator/FindPathValidator.cpp b/src/graph/validator/FindPathValidator.cpp index 1c53fbfbf07..8e7f679c01b 100644 --- a/src/graph/validator/FindPathValidator.cpp +++ b/src/graph/validator/FindPathValidator.cpp @@ -49,12 +49,8 @@ Status FindPathValidator::validateWhere(WhereClause* where) { auto undefinedParams = graph::ExpressionUtils::ExtractInnerVars(filterExpr, qctx_); if (!undefinedParams.empty()) { - return Status::SemanticError( - "Undefined parameters: " + - std::accumulate(++undefinedParams.begin(), - undefinedParams.end(), - *undefinedParams.begin(), - [](auto& lhs, auto& rhs) { return lhs + ", " + rhs; })); + auto msg = folly::join(", ", undefinedParams); + return Status::SemanticError("Undefined parameters: %s", msg.c_str()); } auto* newFilter = graph::ExpressionUtils::rewriteParameter(filterExpr, qctx_); diff --git a/src/graph/validator/GetSubgraphValidator.cpp b/src/graph/validator/GetSubgraphValidator.cpp index e3b0a0d5697..bf3a2779b42 100644 --- a/src/graph/validator/GetSubgraphValidator.cpp +++ b/src/graph/validator/GetSubgraphValidator.cpp @@ -154,12 +154,8 @@ Status GetSubgraphValidator::validateWhere(WhereClause* where) { auto undefinedParams = graph::ExpressionUtils::ExtractInnerVars(filterExpr, qctx_); if (!undefinedParams.empty()) { - return Status::SemanticError( - "Undefined parameters: " + - std::accumulate(++undefinedParams.begin(), - undefinedParams.end(), - *undefinedParams.begin(), - [](auto& lhs, auto& rhs) { return lhs + ", " + rhs; })); + auto msg = folly::join(", ", undefinedParams); + return Status::SemanticError("Undefined parameters: %s", msg.c_str()); } auto* newFilter = graph::ExpressionUtils::rewriteParameter(filterExpr, qctx_); diff --git a/src/graph/validator/LookupValidator.cpp b/src/graph/validator/LookupValidator.cpp index 4ec37299220..8bf7e3a7991 100644 --- a/src/graph/validator/LookupValidator.cpp +++ b/src/graph/validator/LookupValidator.cpp @@ -9,6 +9,7 @@ #include "common/meta/NebulaSchemaProvider.h" #include "graph/context/ast/QueryAstContext.h" #include "graph/planner/plan/Query.h" +#include "graph/util/Constants.h" #include "graph/util/ExpressionUtils.h" #include "graph/util/FTIndexUtils.h" #include "graph/util/SchemaUtil.h" @@ -230,35 +231,37 @@ Status LookupValidator::validateWhere() { } auto* filter = whereClause->filter(); - if (filter != nullptr) { - auto vars = graph::ExpressionUtils::ExtractInnerVars(filter, qctx_); - std::vector undefinedParams; - for (const auto& var : vars) { - if (!vctx_->existVar(var)) { - undefinedParams.emplace_back(var); - } - } - if (!undefinedParams.empty()) { - return Status::SemanticError( - "Undefined parameters: " + - std::accumulate(++undefinedParams.begin(), - undefinedParams.end(), - *undefinedParams.begin(), - [](auto& lhs, auto& rhs) { return lhs + ", " + rhs; })); - } - filter = graph::ExpressionUtils::rewriteParameter(filter, qctx_); - } if (FTIndexUtils::needTextSearch(filter)) { lookupCtx_->isFulltextIndex = true; lookupCtx_->fulltextExpr = filter; auto tsExpr = static_cast(filter); - auto prop = tsExpr->arg()->prop(); + auto arg = tsExpr->arg(); + std::string& index = arg->index(); auto metaClient = qctx_->getMetaClient(); - auto tsi = metaClient->getFTIndexFromCache(spaceId(), schemaId(), prop); - NG_RETURN_IF_ERROR(tsi); - auto tsName = tsi.value().first; - lookupCtx_->fulltextIndex = tsName; + meta::cpp2::FTIndex ftIndex; + auto result = metaClient->getFTIndexFromCache(spaceId(), schemaId()); + NG_RETURN_IF_ERROR(result); + auto indexes = std::move(result).value(); + auto iter = indexes.find(index); + if (iter == indexes.end()) { + return Status::Error("Index %s is not found", index.c_str()); + } + ftIndex = iter->second; } else { + if (filter != nullptr) { + auto vars = graph::ExpressionUtils::ExtractInnerVars(filter, qctx_); + std::vector undefinedParams; + for (const auto& var : vars) { + if (!vctx_->existVar(var)) { + undefinedParams.emplace_back(var); + } + } + if (!undefinedParams.empty()) { + auto msg = folly::join(", ", undefinedParams); + return Status::SemanticError("Undefined parameters: %s", msg.c_str()); + } + filter = graph::ExpressionUtils::rewriteParameter(filter, qctx_); + } auto ret = checkFilter(filter); NG_RETURN_IF_ERROR(ret); lookupCtx_->filter = std::move(ret).value(); @@ -438,8 +441,8 @@ StatusOr LookupValidator::rewriteRelExpr(RelationalExpression* expr } // Rewrite expression of geo search. -// Put geo expression to left, check validity of geo search, check schema validity, fold expression, -// rewrite attribute expression to fit semantic. +// Put geo expression to left, check validity of geo search, check schema validity, fold +// expression, rewrite attribute expression to fit semantic. StatusOr LookupValidator::rewriteGeoPredicate(Expression* expr) { // swap LHS and RHS of relExpr if LabelAttributeExpr in on the right, // so that LabelAttributeExpr is always on the left @@ -606,17 +609,6 @@ StatusOr LookupValidator::checkConstExpr(Expression* expr, return expr; } -// Check does test search contains properties search in test search expression -StatusOr LookupValidator::checkTSExpr(Expression* expr) { - auto tsExpr = static_cast(expr); - auto prop = tsExpr->arg()->prop(); - auto metaClient = qctx_->getMetaClient(); - auto tsi = metaClient->getFTIndexFromCache(spaceId(), schemaId(), prop); - NG_RETURN_IF_ERROR(tsi); - auto tsName = tsi.value().first; - return tsName; -} - // Reverse position of operands in relational expression and keep the origin semantic. // Transform (A > B) to (B < A) Expression* LookupValidator::reverseRelKind(RelationalExpression* expr) { @@ -700,15 +692,66 @@ Status LookupValidator::getSchemaProvider(shared_ptr return Status::OK(); } -// Generate text search filter, check validity and rewrite -StatusOr LookupValidator::genTsFilter(Expression* filter) { - auto esAdapterRet = FTIndexUtils::getESAdapter(qctx_->getMetaClient()); - NG_RETURN_IF_ERROR(esAdapterRet); - auto esAdapter = std::move(esAdapterRet).value(); - auto tsIndex = checkTSExpr(filter); - NG_RETURN_IF_ERROR(tsIndex); - return FTIndexUtils::rewriteTSFilter( - qctx_->objPool(), lookupCtx_->isEdge, filter, tsIndex.value(), esAdapter); +Status LookupValidator::validateYieldColumn(YieldColumn* col, bool isEdge) { + auto kind = isEdge ? Expression::Kind::kVertex : Expression::Kind::kEdge; + if (ExpressionUtils::hasAny(col->expr(), {kind})) { + return Status::SemanticError("illegal yield clauses `%s'", col->toString().c_str()); + } + + bool scoreCol = false; + switch (col->expr()->kind()) { + case Expression::Kind::kLabelAttribute: { + auto expr = static_cast(col->expr()); + const auto& schemaName = expr->left()->name(); + if (schemaName != sentence()->from()) { + return Status::SemanticError("Schema name error: %s", schemaName.c_str()); + } + break; + } + case Expression::Kind::kFunctionCall: { + auto funcExpr = static_cast(col->expr()); + if (funcExpr->name() == kScore) { + if (col->alias().empty()) { + return Status::SemanticError("Yield column should have an alias for score()"); + } + scoreCol = true; + } + break; + } + default: + break; + } + + Expression* colExpr = nullptr; + if (scoreCol) { + // Rewrite score() to $score + colExpr = VariablePropertyExpression::make(qctx_->objPool(), "", kScore); + col->setExpr(colExpr); + outputs_.emplace_back(col->name(), Value::Type::FLOAT); + lookupCtx_->yieldExpr->addColumn(col->clone().release()); + lookupCtx_->hasScore = true; + } else { + if (isEdge) { + colExpr = ExpressionUtils::rewriteLabelAttr2EdgeProp(col->expr()); + } else { + colExpr = ExpressionUtils::rewriteLabelAttr2TagProp(col->expr()); + } + + col->setExpr(colExpr); + NG_RETURN_IF_ERROR(ValidateUtil::invalidLabelIdentifiers(colExpr)); + + auto typeStatus = deduceExprType(colExpr); + NG_RETURN_IF_ERROR(typeStatus); + outputs_.emplace_back(col->name(), typeStatus.value()); + lookupCtx_->yieldExpr->addColumn(col->clone().release()); + if (isEdge) { + NG_RETURN_IF_ERROR(deduceProps(colExpr, exprProps_, nullptr, &schemaIds_)); + } else { + NG_RETURN_IF_ERROR(deduceProps(colExpr, exprProps_, &schemaIds_)); + } + } + + return Status::OK(); } } // namespace graph diff --git a/src/graph/validator/LookupValidator.h b/src/graph/validator/LookupValidator.h index 79ed4ce4e42..3b5a50df5e3 100644 --- a/src/graph/validator/LookupValidator.h +++ b/src/graph/validator/LookupValidator.h @@ -31,6 +31,7 @@ class LookupValidator final : public Validator { Status validateWhere(); Status validateYieldTag(); Status validateYieldEdge(); + Status validateYieldColumn(YieldColumn* col, bool isEdge); StatusOr checkFilter(Expression* expr); Status checkRelExpr(RelationalExpression* expr); diff --git a/src/graph/validator/MaintainValidator.cpp b/src/graph/validator/MaintainValidator.cpp index 95bde20293c..155cd7d836d 100644 --- a/src/graph/validator/MaintainValidator.cpp +++ b/src/graph/validator/MaintainValidator.cpp @@ -599,10 +599,7 @@ Status AddHostsIntoZoneValidator::toPlan() { Status CreateFTIndexValidator::validateImpl() { auto sentence = static_cast(sentence_); folly::StringPiece name = folly::StringPiece(*sentence->indexName()); - if (!name.startsWith(kFulltextIndexNamePrefix)) { - return Status::SyntaxError("Index name must begin with \"%s\"", - kFulltextIndexNamePrefix.c_str()); - } + if (name.size() > kFulltextIndexNameLength) { return Status::SyntaxError(fmt::format("Fulltext index name's length must less equal than {}", kFulltextIndexNameLength)); @@ -639,7 +636,10 @@ Status CreateFTIndexValidator::validateImpl() { } index_.space_id_ref() = space.id; index_.depend_schema_ref() = std::move(id); - index_.fields_ref()->push_back(sentence->field()); + for (auto &f : sentence->fields()) { + index_.fields_ref()->push_back(f); + } + index_.analyzer_ref() = sentence->analyzer(); return Status::OK(); } diff --git a/src/graph/validator/MaintainValidator.h b/src/graph/validator/MaintainValidator.h index c055b1795fa..de4be76c177 100644 --- a/src/graph/validator/MaintainValidator.h +++ b/src/graph/validator/MaintainValidator.h @@ -419,7 +419,6 @@ class CreateFTIndexValidator final : public Validator { private: meta::cpp2::FTIndex index_; - const std::string kFulltextIndexNamePrefix = "nebula_"; const size_t kFulltextIndexNameLength = 256; }; diff --git a/src/interface/meta.thrift b/src/interface/meta.thrift index 2916c8a0ba7..c973b134ba0 100644 --- a/src/interface/meta.thrift +++ b/src/interface/meta.thrift @@ -1187,6 +1187,7 @@ struct FTIndex { 1: common.GraphSpaceID space_id, 2: common.SchemaID depend_schema, 3: list fields, + 4: binary analyzer, } struct CreateFTIndexReq { diff --git a/src/meta/processors/index/FTIndexProcessor.cpp b/src/meta/processors/index/FTIndexProcessor.cpp index 278dee1e3ee..aadeaeecc11 100644 --- a/src/meta/processors/index/FTIndexProcessor.cpp +++ b/src/meta/processors/index/FTIndexProcessor.cpp @@ -125,7 +125,7 @@ void CreateFTIndexProcessor::process(const cpp2::CreateFTIndexReq& req) { HttpClient::instance(), protocol, client.get_host().toRawString(), user, password); } plugin::ESAdapter esAdapter(std::move(esClients)); - auto createIndexresult = esAdapter.createIndex(name); + auto createIndexresult = esAdapter.createIndex(name, index.get_fields(), index.get_analyzer()); if (!createIndexresult.ok()) { LOG(ERROR) << createIndexresult.message(); handleErrorCode(nebula::cpp2::ErrorCode::E_ACCESS_ES_FAILURE); diff --git a/src/parser/MaintainSentences.cpp b/src/parser/MaintainSentences.cpp index 15d99d535cf..b572c92d97b 100644 --- a/src/parser/MaintainSentences.cpp +++ b/src/parser/MaintainSentences.cpp @@ -497,8 +497,7 @@ std::string CreateFTIndexSentence::toString() const { buf += " ON "; buf += *schemaName_; buf += "("; - std::vector fieldDefs; - fieldDefs.emplace_back(field()); + std::vector fieldDefs = fields(); std::string fields; folly::join(", ", fieldDefs, fields); buf += fields; diff --git a/src/parser/MaintainSentences.h b/src/parser/MaintainSentences.h index a18f9cc2334..5c493fefac5 100644 --- a/src/parser/MaintainSentences.h +++ b/src/parser/MaintainSentences.h @@ -1125,11 +1125,15 @@ class CreateFTIndexSentence final : public Sentence { CreateFTIndexSentence(bool isEdge, std::string *indexName, std::string *schemaName, - std::string *field) { + NameLabelList *fields, + std::string *analyzer) { isEdge_ = isEdge; indexName_.reset(indexName); schemaName_.reset(schemaName); - field_.reset(field); + for (auto &f : fields->labels()) { + fields_.push_back(*f); + } + analyzer_.reset(analyzer); kind_ = Kind::kCreateFTIndex; } @@ -1146,15 +1150,24 @@ class CreateFTIndexSentence final : public Sentence { return schemaName_.get(); } - std::string field() const { - return *field_; + std::vector fields() const { + return fields_; + } + + std::string analyzer() const { + if (analyzer_ == nullptr) { + return ""; + } else { + return *analyzer_; + } } private: bool isEdge_; std::unique_ptr indexName_; std::unique_ptr schemaName_; - std::unique_ptr field_; + std::vector fields_; + std::unique_ptr analyzer_; }; class DropFTIndexSentence final : public Sentence { public: diff --git a/src/parser/parser.yy b/src/parser/parser.yy index 6e4740af8a2..529a2b1d72c 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -155,8 +155,6 @@ using namespace nebula; nebula::IndexFieldList *index_field_list; CaseList *case_list; nebula::TextSearchArgument *text_search_argument; - nebula::TextSearchArgument *base_text_search_argument; - nebula::TextSearchArgument *fuzzy_text_search_argument; nebula::meta::cpp2::ServiceClient *service_client_item; nebula::ServiceClientList *service_client_list; nebula::QueryUniqueIdentifier *query_unique_identifier; @@ -170,7 +168,7 @@ using namespace nebula; %destructor {} // Expression related memory will be managed by object pool %destructor {} -%destructor {} +%destructor {} %destructor {} %destructor {} %destructor { delete $$; } <*> @@ -211,7 +209,7 @@ using namespace nebula; %token KW_CASE KW_THEN KW_ELSE KW_END %token KW_GROUP KW_ZONE KW_GROUPS KW_ZONES KW_INTO KW_NEW %token KW_LISTENER KW_ELASTICSEARCH KW_FULLTEXT KW_HTTPS KW_HTTP KW_SYNC KW_DRAINER KW_DRAINERS -%token KW_AUTO KW_FUZZY KW_PREFIX KW_REGEXP KW_WILDCARD +%token KW_AUTO KW_ES_QUERY KW_ANALYZER %token KW_TEXT KW_SEARCH KW_CLIENTS KW_SIGN KW_SERVICE KW_TEXT_SEARCH %token KW_ANY KW_SINGLE KW_NONE %token KW_REDUCE @@ -237,7 +235,7 @@ using namespace nebula; %token DOUBLE %token STRING VARIABLE LABEL IPV4 -%type name_label unreserved_keyword predicate_name +%type name_label unreserved_keyword predicate_name opt_analyzer %type expression expression_internal %type property_expression %type vertex_prop_expression @@ -354,8 +352,6 @@ using namespace nebula; %type match_step_range %type match_order_by %type text_search_argument -%type base_text_search_argument -%type fuzzy_text_search_argument %type service_client_item %type service_client_list @@ -571,10 +567,7 @@ unreserved_keyword | KW_STATS { $$ = new std::string("stats"); } | KW_STATUS { $$ = new std::string("status"); } | KW_AUTO { $$ = new std::string("auto"); } - | KW_FUZZY { $$ = new std::string("fuzzy"); } - | KW_PREFIX { $$ = new std::string("prefix"); } - | KW_REGEXP { $$ = new std::string("regexp"); } - | KW_WILDCARD { $$ = new std::string("wildcard"); } + | KW_ES_QUERY { $$ = new std::string("es_query"); } | KW_TEXT { $$ = new std::string("text"); } | KW_SEARCH { $$ = new std::string("search"); } | KW_CLIENTS { $$ = new std::string("clients"); } @@ -2123,119 +2116,19 @@ get_variable_sentence } ; -base_text_search_argument - : name_label DOT name_label COMMA STRING { - auto arg = TextSearchArgument::make(qctx->objPool(), *$1, *$3, *$5); - $$ = arg; - delete $1; - delete $3; - delete $5; - } - ; - -fuzzy_text_search_argument - : base_text_search_argument COMMA KW_AUTO COMMA KW_AND { - $$ = $1; - $$->setFuzziness(-1); - $$->setOP("and"); - } - | base_text_search_argument COMMA KW_AUTO COMMA KW_OR { - $$ = $1; - $$->setFuzziness(-1); - $$->setOP("or"); - } - | base_text_search_argument COMMA legal_integer COMMA KW_AND { - if ($3 != 0 && $3 != 1 && $3 != 2) { - throw nebula::GraphParser::syntax_error(@3, "Out of range:"); - } - $$ = $1; - $$->setFuzziness($3); - $$->setOP("and"); - } - | base_text_search_argument COMMA legal_integer COMMA KW_OR { - if ($3 != 0 && $3 != 1 && $3 != 2) { - throw nebula::GraphParser::syntax_error(@3, "Out of range:"); - } - $$ = $1; - $$->setFuzziness($3); - $$->setOP("or"); - } text_search_argument - : base_text_search_argument { - $$ = $1; - } - | fuzzy_text_search_argument { - $$ = $1; - } - | base_text_search_argument COMMA legal_integer { - if ($3 < 1) { - throw nebula::GraphParser::syntax_error(@3, "Out of range:"); - } - $$ = $1; - $$->setLimit($3); - } - | base_text_search_argument COMMA legal_integer COMMA legal_integer { - if ($3 < 1) { - throw nebula::GraphParser::syntax_error(@3, "Out of range:"); - } - if ($5 < 1) { - throw nebula::GraphParser::syntax_error(@5, "Out of range:"); - } - $$ = $1; - $$->setLimit($3); - $$->setTimeout($5); - } - | fuzzy_text_search_argument COMMA legal_integer { - if ($3 < 1) { - throw nebula::GraphParser::syntax_error(@3, "Out of range:"); - } - $$ = $1; - $$->setLimit($3); - } - | fuzzy_text_search_argument COMMA legal_integer COMMA legal_integer { - if ($3 < 1) { - throw nebula::GraphParser::syntax_error(@3, "Out of range:"); - } - if ($5 < 1) { - throw nebula::GraphParser::syntax_error(@5, "Out of range:"); - } - $$ = $1; - $$->setLimit($3); - $$->setTimeout($5); + : name_label COMMA STRING { + auto args = TextSearchArgument::make(qctx->objPool(), *$1, *$3); + delete $1; + delete $3; + $$ = args; } ; text_search_expression - : KW_PREFIX L_PAREN text_search_argument R_PAREN { - if (!$3->op().empty()) { - throw nebula::GraphParser::syntax_error(@3, "argument error:"); - } - if ($3->fuzziness() != -2) { - throw nebula::GraphParser::syntax_error(@3, "argument error:"); - } - $$ = TextSearchExpression::makePrefix(qctx->objPool(), $3); - } - | KW_WILDCARD L_PAREN text_search_argument R_PAREN { - if (!$3->op().empty()) { - throw nebula::GraphParser::syntax_error(@3, "argument error:"); - } - if ($3->fuzziness() != -2) { - throw nebula::GraphParser::syntax_error(@3, "argument error:"); - } - $$ = TextSearchExpression::makeWildcard(qctx->objPool(), $3); - } - | KW_REGEXP L_PAREN text_search_argument R_PAREN { - if (!$3->op().empty()) { - throw nebula::GraphParser::syntax_error(@3, "argument error:"); - } - if ($3->fuzziness() != -2) { - throw nebula::GraphParser::syntax_error(@3, "argument error:"); - } - $$ = TextSearchExpression::makeRegexp(qctx->objPool(), $3); - } - | KW_FUZZY L_PAREN text_search_argument R_PAREN { - $$ = TextSearchExpression::makeFuzzy(qctx->objPool(), $3); + : KW_ES_QUERY L_PAREN text_search_argument R_PAREN { + $$ = TextSearchExpression::makeQuery(qctx->objPool(), $3); } ; @@ -2766,12 +2659,21 @@ create_edge_index_sentence } ; +opt_analyzer + : %empty { + $$ = nullptr; + } + | KW_ANALYZER ASSIGN STRING { + $$ = $3; + } + ; + create_fulltext_index_sentence - : KW_CREATE KW_FULLTEXT KW_TAG KW_INDEX name_label KW_ON name_label L_PAREN name_label R_PAREN { - $$ = new CreateFTIndexSentence(false, $5, $7, $9); + : KW_CREATE KW_FULLTEXT KW_TAG KW_INDEX name_label KW_ON name_label L_PAREN name_label_list R_PAREN opt_analyzer { + $$ = new CreateFTIndexSentence(false, $5, $7, $9, $11); } - | KW_CREATE KW_FULLTEXT KW_EDGE KW_INDEX name_label KW_ON name_label L_PAREN name_label R_PAREN { - $$ = new CreateFTIndexSentence(true, $5, $7, $9); + | KW_CREATE KW_FULLTEXT KW_EDGE KW_INDEX name_label KW_ON name_label L_PAREN name_label_list R_PAREN opt_analyzer { + $$ = new CreateFTIndexSentence(true, $5, $7, $9, $11); } ; diff --git a/src/parser/scanner.lex b/src/parser/scanner.lex index cedb49eb039..8bc12f56113 100644 --- a/src/parser/scanner.lex +++ b/src/parser/scanner.lex @@ -290,10 +290,7 @@ LABEL_FULL_WIDTH {CN_EN_FULL_WIDTH}{CN_EN_NUM_FULL_WIDTH}* "DRAINERS" { return TokenType::KW_DRAINERS; } "FULLTEXT" { return TokenType::KW_FULLTEXT; } "AUTO" { return TokenType::KW_AUTO; } -"FUZZY" { return TokenType::KW_FUZZY; } -"PREFIX" { return TokenType::KW_PREFIX; } -"REGEXP" { return TokenType::KW_REGEXP; } -"WILDCARD" { return TokenType::KW_WILDCARD; } +"ES_QUERY" { return TokenType::KW_ES_QUERY; } "TEXT" { return TokenType::KW_TEXT; } "SEARCH" { return TokenType::KW_SEARCH; } "CLIENTS" { return TokenType::KW_CLIENTS; } diff --git a/src/parser/test/ParserTest.cpp b/src/parser/test/ParserTest.cpp index 91432bab0b6..029c00661d7 100644 --- a/src/parser/test/ParserTest.cpp +++ b/src/parser/test/ParserTest.cpp @@ -2944,139 +2944,9 @@ TEST_F(ParserTest, Zone) { TEST_F(ParserTest, FullText) { { - std::string query = "LOOKUP ON t1 WHERE PREFIX(t1.c1, \"a\")"; + std::string query = "LOOKUP ON t1 WHERE ES_QUERY(abc, \"qwerty\")"; auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE WILDCARD(t1.c1, \"a\")"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE REGEXP(t1.c1, \"a\")"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE FUZZY(t1.c1, \"a\")"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE PREFIX(t1.c1, \"a\", 1)"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE WILDCARD(t1.c1, \"a\", 1)"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE REGEXP(t1.c1, \"a\", 1)"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE FUZZY(t1.c1, \"a\", 1)"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE PREFIX(t1.c1, \"a\", 1, 2)"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE WILDCARD(t1.c1, \"a\", 1, 2)"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE REGEXP(t1.c1, \"a\", 1, 2)"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE FUZZY(t1.c1, \"a\", 1, 2)"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE FUZZY(t1.c1, \"a\", AUTO, AND)"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE FUZZY(t1.c1, \"a\", AUTO, OR)"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE FUZZY(t1.c1, \"a\", 0, AND)"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE FUZZY(t1.c1, \"a\", 0, OR)"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE FUZZY(t1.c1, \"a\", 0, OR, 1)"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE FUZZY(t1.c1, \"a\", 0, OR, 1, 1)"; - auto result = parse(query); - ASSERT_TRUE(result.ok()) << result.status(); - } - { - std::string query = "LOOKUP ON t1 WHERE FUZZY(t1.c1, \"a\", 0, OR, -1, 1)"; - auto result = parse(query); - ASSERT_FALSE(result.ok()); - } - { - std::string query = "LOOKUP ON t1 WHERE FUZZY(t1.c1, \"a\", 0, OR, 1, -1)"; - auto result = parse(query); - ASSERT_FALSE(result.ok()); - } - { - std::string query = "LOOKUP ON t1 WHERE FUZZY(t1.c1, \"a\", -1, OR, 1, 1)"; - auto result = parse(query); - ASSERT_FALSE(result.ok()); - } - { - std::string query = "LOOKUP ON t1 WHERE FUZZY(t1.c1, \"a\", 4, OR, 1, 1)"; - auto result = parse(query); - ASSERT_FALSE(result.ok()); - } - { - std::string query = "LOOKUP ON t1 WHERE PREFIX(t1.c1, \"a\", -1, 2)"; - auto result = parse(query); - ASSERT_FALSE(result.ok()); - } - { - std::string query = "LOOKUP ON t1 WHERE PREFIX(t1.c1, \"a\", 1, -2)"; - auto result = parse(query); - ASSERT_FALSE(result.ok()); - } - { - std::string query = "LOOKUP ON t1 WHERE PREFIX(t1.c1, \"a\", AUTO, AND)"; - auto result = parse(query); - ASSERT_FALSE(result.ok()); - } - { - std::string query = "LOOKUP ON t1 WHERE WILDCARD(t1.c1, \"a\", AUTO, AND)"; - auto result = parse(query); - ASSERT_FALSE(result.ok()); - } - { - std::string query = "LOOKUP ON t1 WHERE REGEXP(t1.c1, \"a\", AUTO, AND)"; - auto result = parse(query); - ASSERT_FALSE(result.ok()); + EXPECT_TRUE(result.ok()) << result.status(); } } diff --git a/src/storage/query/QueryBaseProcessor-inl.h b/src/storage/query/QueryBaseProcessor-inl.h index a2ae8a34d33..11db18e9e96 100644 --- a/src/storage/query/QueryBaseProcessor-inl.h +++ b/src/storage/query/QueryBaseProcessor-inl.h @@ -628,10 +628,7 @@ nebula::cpp2::ErrorCode QueryBaseProcessor::checkExp( case Expression::Kind::kUUID: case Expression::Kind::kPathBuild: case Expression::Kind::kColumn: - case Expression::Kind::kTSPrefix: - case Expression::Kind::kTSWildcard: - case Expression::Kind::kTSRegexp: - case Expression::Kind::kTSFuzzy: + case Expression::Kind::kESQUERY: case Expression::Kind::kAggregate: case Expression::Kind::kSubscriptRange: case Expression::Kind::kVersionedVar: diff --git a/tests/tck/conftest.py b/tests/tck/conftest.py index b7efa4288a5..1854e7b5778 100644 --- a/tests/tck/conftest.py +++ b/tests/tck/conftest.py @@ -248,7 +248,7 @@ def add_listeners(request, exec_ctx): values = result.row_values(0) host = values[0] port = values[1] - add_listener = f"ADD LISTENER ELASTICSEARCH {host}:{port}" + add_listener = f"ADD LISTENER ELASTICSEARCH {host}:{port}" exec_ctx['result_set'] = [] exec_query(request, add_listener, exec_ctx) result = exec_ctx["result_set"][0] diff --git a/tests/tck/features/fulltext_index/FulltextIndexQuery1.feature b/tests/tck/features/fulltext_index/FulltextIndexQuery1.feature deleted file mode 100644 index f8ad3645bc4..00000000000 --- a/tests/tck/features/fulltext_index/FulltextIndexQuery1.feature +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (c) 2022 vesoft inc. All rights reserved. -# -# This source code is licensed under Apache 2.0 License. -Feature: FulltextIndexTest - - Background: - Given an empty graph - And create a space with following options: - | partition_num | 1 | - | replica_factor | 1 | - | vid_type | INT64 | - And add listeners to space - - @ft_index - Scenario: fulltext query1 - When executing query: - """ - CREATE TAG tag1(prop1 string,prop2 string); - CREATE EDGE edge1(prop1 string); - """ - Then the execution should be successful - And wait 3 seconds - When executing query: - """ - CREATE FULLTEXT TAG INDEX nebula_index_tag1_prop1 on tag1(prop1); - CREATE FULLTEXT TAG INDEX nebula_index_tag1_prop2 on tag1(prop2); - CREATE FULLTEXT EDGE INDEX nebula_index_edge1_prop1 on edge1(prop1); - """ - Then the execution should be successful - And wait 5 seconds - When executing query: - """ - INSERT VERTEX tag1(prop1,prop2) VALUES 1:("abc","nebula graph"); - INSERT VERTEX tag1(prop1,prop2) VALUES 2:("abcde","nebula-graph"); - INSERT VERTEX tag1(prop1,prop2) VALUES 3:("bcd","nebula database"); - INSERT VERTEX tag1(prop1,prop2) VALUES 4:("zyx","Nebula"); - INSERT VERTEX tag1(prop1,prop2) VALUES 5:("cba","neBula"); - INSERT VERTEX tag1(prop1,prop2) VALUES 6:("abcxyz","nebula graph"); - INSERT VERTEX tag1(prop1,prop2) VALUES 7:("xyz","nebula graph"); - INSERT VERTEX tag1(prop1,prop2) VALUES 8:("123456","nebula graph"); - """ - Then the execution should be successful - When executing query: - """ - INSERT EDGE edge1(prop1) VALUES 1->2@1:("一个可靠的分布式"); - INSERT EDGE edge1(prop1) VALUES 2->3@3:("性能高效的图数据库"); - INSERT EDGE edge1(prop1) VALUES 3->4@5:("高性能"); - INSERT EDGE edge1(prop1) VALUES 4->5@7:("高吞吐"); - INSERT EDGE edge1(prop1) VALUES 5->6@9:("低延时"); - INSERT EDGE edge1(prop1) VALUES 6->7@11:("易扩展"); - INSERT EDGE edge1(prop1) VALUES 7->8@13:("线性扩缩容"); - INSERT EDGE edge1(prop1) VALUES 8->1@15:("安全稳定"); - """ - Then the execution should be successful - And wait 10 seconds - When executing query: - """ - LOOKUP ON tag1 where prefix(tag1.prop1,"abc") YIELD id(vertex) as id, tag1.prop1 as prop1, tag1.prop2 as prop2 - """ - Then the result should be, in any order: - | id | prop1 | prop2 | - | 1 | "abc" | "nebula graph" | - | 2 | "abcde" | "nebula-graph" | - | 6 | "abcxyz" | "nebula graph" | - When executing query: - """ - LOOKUP ON tag1 where prefix(tag1.prop2,"nebula") YIELD id(vertex) as id, tag1.prop1 as prop1, tag1.prop2 as prop2 - """ - Then the result should be, in any order: - | id | prop1 | prop2 | - | 1 | "abc" | "nebula graph" | - | 2 | "abcde" | "nebula-graph" | - | 3 | "bcd" | "nebula database" | - | 6 | "abcxyz" | "nebula graph" | - | 7 | "xyz" | "nebula graph" | - | 8 | "123456" | "nebula graph" | - When executing query: - """ - LOOKUP ON edge1 where prefix(edge1.prop1,"高") YIELD src(edge) as src,dst(edge) as dst,rank(edge) as rank, edge1.prop1 as prop1 - """ - Then the result should be, in any order: - | src | dst | rank | prop1 | - | 3 | 4 | 5 | "高性能" | - | 4 | 5 | 7 | "高吞吐" | - When executing query: - """ - LOOKUP ON tag1 where regexp(tag1.prop2,"neBula.*") YIELD id(vertex) as id, tag1.prop1 as prop1 - """ - Then the result should be, in any order: - | id | prop1 | - | 5 | "cba" | - When executing query: - """ - LOOKUP ON edge1 where wildcard(edge1.prop1,"高??") YIELD src(edge) as src,dst(edge) as dst,rank(edge) as rank, edge1.prop1 as prop1 - """ - Then the result should be, in any order: - | src | dst | rank | prop1 | - | 3 | 4 | 5 | "高性能" | - | 4 | 5 | 7 | "高吞吐" | - When executing query: - """ - LOOKUP ON tag1 where fuzzy(edge1.prop2,"nebula") YIELD tag1.prop2 as prop2 - """ - Then the result should be, in any order: - | prop2 | - | "Nebula" | - | "neBula" | diff --git a/tests/tck/features/fulltext_index/FulltextIndexQuery2.feature b/tests/tck/features/fulltext_index/FulltextIndexQuery2.feature deleted file mode 100644 index 40a7bd90cec..00000000000 --- a/tests/tck/features/fulltext_index/FulltextIndexQuery2.feature +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) 2022 vesoft inc. All rights reserved. -# -# This source code is licensed under Apache 2.0 License. -Feature: FulltextIndexTest - - Background: - Given an empty graph - And create a space with following options: - | partition_num | 1 | - | replica_factor | 1 | - | vid_type | FIXED_STRING(30) | - And add listeners to space - - @ft_index - Scenario: fulltext query2 - When executing query: - """ - CREATE TAG tag2(prop1 string,prop2 string); - CREATE EDGE edge2(prop1 string); - """ - Then the execution should be successful - And wait 3 seconds - When executing query: - """ - CREATE FULLTEXT TAG INDEX nebula_index_tag2_prop1 on tag2(prop1); - CREATE FULLTEXT TAG INDEX nebula_index_tag2_prop2 on tag2(prop2); - CREATE FULLTEXT EDGE INDEX nebula_index_edge2_prop1 on edge2(prop1); - """ - Then the execution should be successful - And wait 5 seconds - When executing query: - """ - INSERT VERTEX tag2(prop1,prop2) VALUES "1":("abc","nebula graph"); - INSERT VERTEX tag2(prop1,prop2) VALUES "2":("abcde","nebula-graph"); - INSERT VERTEX tag2(prop1,prop2) VALUES "3":("bcd","nebula database"); - INSERT VERTEX tag2(prop1,prop2) VALUES "4":("zyx","Nebula"); - INSERT VERTEX tag2(prop1,prop2) VALUES "5":("cba","neBula"); - INSERT VERTEX tag2(prop1,prop2) VALUES "6":("abcxyz","nebula graph"); - INSERT VERTEX tag2(prop1,prop2) VALUES "7":("xyz","nebula graph"); - INSERT VERTEX tag2(prop1,prop2) VALUES "8":("123456","nebula graph"); - """ - Then the execution should be successful - When executing query: - """ - INSERT EDGE edge2(prop1) VALUES "1"->"2"@1:("一个可靠的分布式"); - INSERT EDGE edge2(prop1) VALUES "2"->"3"@3:("性能高效的图数据库"); - INSERT EDGE edge2(prop1) VALUES "3"->"4"@5:("高性能"); - INSERT EDGE edge2(prop1) VALUES "4"->"5"@7:("高吞吐"); - INSERT EDGE edge2(prop1) VALUES "5"->"6"@9:("低延时"); - INSERT EDGE edge2(prop1) VALUES "6"->"7"@11:("易扩展"); - INSERT EDGE edge2(prop1) VALUES "7"->"8"@13:("线性扩缩容"); - INSERT EDGE edge2(prop1) VALUES "8"->"1"@15:("安全稳定"); - """ - Then the execution should be successful - And wait 10 seconds - When executing query: - """ - LOOKUP ON tag2 where prefix(tag2.prop1,"abc") YIELD id(vertex) as id, tag2.prop1 as prop1, tag2.prop2 as prop2 - """ - Then the result should be, in any order: - | id | prop1 | prop2 | - | "1" | "abc" | "nebula graph" | - | "2" | "abcde" | "nebula-graph" | - | "6" | "abcxyz" | "nebula graph" | - When executing query: - """ - LOOKUP ON tag2 where prefix(tag2.prop2,"nebula") YIELD id(vertex) as id, tag2.prop1 as prop1, tag2.prop2 as prop2 - """ - Then the result should be, in any order: - | id | prop1 | prop2 | - | "1" | "abc" | "nebula graph" | - | "2" | "abcde" | "nebula-graph" | - | "3" | "bcd" | "nebula database" | - | "6" | "abcxyz" | "nebula graph" | - | "7" | "xyz" | "nebula graph" | - | "8" | "123456" | "nebula graph" | - When executing query: - """ - LOOKUP ON edge2 where prefix(edge2.prop1,"高") YIELD src(edge) as src,dst(edge) as dst,rank(edge) as rank, edge2.prop1 as prop1 - """ - Then the result should be, in any order: - | src | dst | rank | prop1 | - | "3" | "4" | 5 | "高性能" | - | "4" | "5" | 7 | "高吞吐" | diff --git a/tests/tck/features/yield/yield.IntVid.feature b/tests/tck/features/yield/yield.IntVid.feature index 7955f856c6b..d1b04750451 100644 --- a/tests/tck/features/yield/yield.IntVid.feature +++ b/tests/tck/features/yield/yield.IntVid.feature @@ -110,11 +110,6 @@ Feature: Yield Sentence Then the result should be, in any order: | [1,2,3][0..1] | | [1] | - When executing query: - """ - YIELD prefix(edge1.prop1,"高") - """ - Then a SyntaxError should be raised at runtime: syntax error near `(edge1.p' When executing query: """ YIELD (INT)"1" diff --git a/tests/tck/features/yield/yield.feature b/tests/tck/features/yield/yield.feature index d6f092cb00d..5d6b5bc05eb 100644 --- a/tests/tck/features/yield/yield.feature +++ b/tests/tck/features/yield/yield.feature @@ -110,11 +110,6 @@ Feature: Yield Sentence Then the result should be, in any order: | [1,2,3][0..1] | | [1] | - When executing query: - """ - YIELD prefix(edge1.prop1,"高") - """ - Then a SyntaxError should be raised at runtime: syntax error near `(edge1.p' When executing query: """ YIELD (int)"1"