diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index f891fb94c5b..851791b0f67 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -1714,6 +1714,7 @@ folly::Future> MetaClient::createTagIndex(GraphSpaceID spaceID std::string tagName, std::vector fields, bool ifNotExists, + const cpp2::IndexParams* indexParams, const std::string* comment) { cpp2::CreateTagIndexReq req; req.set_space_id(spaceID); @@ -1721,6 +1722,9 @@ folly::Future> MetaClient::createTagIndex(GraphSpaceID spaceID req.set_tag_name(std::move(tagName)); req.set_fields(std::move(fields)); req.set_if_not_exists(ifNotExists); + if (indexParams != nullptr) { + req.set_index_params(*indexParams); + } if (comment != nullptr) { req.set_comment(*comment); } @@ -1826,6 +1830,7 @@ folly::Future> MetaClient::createEdgeIndex( std::string edgeName, std::vector fields, bool ifNotExists, + const cpp2::IndexParams* indexParams, const std::string* comment) { cpp2::CreateEdgeIndexReq req; req.set_space_id(spaceID); @@ -1833,6 +1838,9 @@ folly::Future> MetaClient::createEdgeIndex( req.set_edge_name(std::move(edgeName)); req.set_fields(std::move(fields)); req.set_if_not_exists(ifNotExists); + if (indexParams != nullptr) { + req.set_index_params(*indexParams); + } if (comment != nullptr) { req.set_comment(*comment); } diff --git a/src/clients/meta/MetaClient.h b/src/clients/meta/MetaClient.h index 80f975e0181..5edefa2a000 100644 --- a/src/clients/meta/MetaClient.h +++ b/src/clients/meta/MetaClient.h @@ -311,12 +311,14 @@ class MetaClient { bool ifExists = false); // Operations for index - folly::Future> createTagIndex(GraphSpaceID spaceID, - std::string indexName, - std::string tagName, - std::vector fields, - bool ifNotExists = false, - const std::string* comment = nullptr); + folly::Future> createTagIndex( + GraphSpaceID spaceID, + std::string indexName, + std::string tagName, + std::vector fields, + bool ifNotExists = false, + const meta::cpp2::IndexParams* indexParams = nullptr, + const std::string* comment = nullptr); // Remove the define of tag index folly::Future> dropTagIndex(GraphSpaceID spaceId, @@ -336,6 +338,7 @@ class MetaClient { std::string edgeName, std::vector fields, bool ifNotExists = false, + const cpp2::IndexParams* indexParams = nullptr, const std::string* comment = nullptr); // Remove the definition of edge index diff --git a/src/common/datatypes/Geography.h b/src/common/datatypes/Geography.h index 45315df611b..cea5bd9c82e 100644 --- a/src/common/datatypes/Geography.h +++ b/src/common/datatypes/Geography.h @@ -141,7 +141,7 @@ struct Geography { bool needNormalize = false, bool verifyValidity = false); - Geography() {} + Geography() = default; Geography(const Point& v) : geo_(v) {} // NOLINT Geography(Point&& v) : geo_(std::move(v)) {} // NOLINT Geography(const LineString& v) : geo_(v) {} // NOLINT diff --git a/src/common/geo/GeoIndex.cpp b/src/common/geo/GeoIndex.cpp index 35df2e3fb09..ffd2a3753f6 100644 --- a/src/common/geo/GeoIndex.cpp +++ b/src/common/geo/GeoIndex.cpp @@ -30,7 +30,17 @@ namespace nebula { namespace geo { -nebula::storage::cpp2::IndexColumnHint ScanRange::toIndexColumnHint() { +bool ScanRange::operator==(const ScanRange& rhs) const { + if (isRangeScan != rhs.isRangeScan) { + return false; + } + if (isRangeScan) { + return rangeMin == rhs.rangeMin && rangeMax == rhs.rangeMax; + } + return rangeMin == rhs.rangeMin; +} + +nebula::storage::cpp2::IndexColumnHint ScanRange::toIndexColumnHint() const { nebula::storage::cpp2::IndexColumnHint hint; // column_name should be set by the caller if (isRangeScan) { diff --git a/src/common/geo/GeoIndex.h b/src/common/geo/GeoIndex.h index deaff94d028..d6dc47961c6 100644 --- a/src/common/geo/GeoIndex.h +++ b/src/common/geo/GeoIndex.h @@ -24,8 +24,8 @@ namespace geo { struct RegionCoverParams { // TODO(jie): Find the best default params - int minCellLevel_ = 4; - int maxCellLevel_ = 23; // About 1m + int minCellLevel_ = 0; + int maxCellLevel_ = 30; // About 1m int maxCellNum_ = 8; RegionCoverParams() = default; @@ -52,7 +52,9 @@ struct ScanRange { explicit ScanRange(uint64_t v) : rangeMin(v), isRangeScan(false) {} - nebula::storage::cpp2::IndexColumnHint toIndexColumnHint(); + bool operator==(const ScanRange& rhs) const; + + nebula::storage::cpp2::IndexColumnHint toIndexColumnHint() const; }; class GeoIndex { @@ -82,6 +84,7 @@ class GeoIndex { private: RegionCoverParams rcParams_; + // pointsOnly_ indicates whether the indexed column only has points. // For the column Geography(Point), we don't need to build ancestor cells bool pointsOnly_{false}; }; diff --git a/src/common/geo/test/GeoIndexTest.cpp b/src/common/geo/test/GeoIndexTest.cpp index fcd377b7770..2b6f6298e59 100644 --- a/src/common/geo/test/GeoIndexTest.cpp +++ b/src/common/geo/test/GeoIndexTest.cpp @@ -4,7 +4,9 @@ */ #include +#include +#include #include #include "common/base/Base.h" @@ -13,162 +15,294 @@ namespace nebula { namespace geo { -std::vector toUint64Vector(std::vector expect) { - auto reinterpretInt64AsUint64 = [](int64_t i) -> uint64_t { - const char* c = reinterpret_cast(&i); - uint64_t u = *reinterpret_cast(c); - return u; - }; - +std::vector asUint64Vec(std::vector expect) { std::vector transformedExpect; transformedExpect.reserve(expect.size()); for (int64_t i : expect) { - transformedExpect.push_back(reinterpretInt64AsUint64(i)); + transformedExpect.push_back(static_cast(i)); } return transformedExpect; } // The tested wkt data is generated by https://clydedacruz.github.io/openstreetmap-wkt-playground/ +// And the expected result comes from BigQuery TEST(indexCells, point) { - geo::RegionCoverParams rc; - geo::GeoIndex geoIndex(rc); - { - auto point = Geography::fromWKT("POINT(1.0 1.0)").value(); - auto cells = geoIndex.indexCells(point); - EXPECT_EQ(toUint64Vector({1153277837650709461}), cells); - } - { - auto point = Geography::fromWKT("POINT(179.9 -89.999)").value(); - auto cells = geoIndex.indexCells(point); - EXPECT_EQ(toUint64Vector({-5764607523209916331}), cells); - } + // s2_min_level=4, s2_max_level=23, s2_max_cells=8 { - auto point = Geography::fromWKT("POINT(0.0 0.0)").value(); - auto cells = geoIndex.indexCells(point); - EXPECT_EQ(toUint64Vector({1152921504606846977}), cells); + geo::RegionCoverParams rc(4, 23, 8); + geo::GeoIndex geoIndex(rc); + { + auto point = Geography::fromWKT("POINT(1.0 1.0)").value(); + auto cells = geoIndex.indexCells(point); + EXPECT_EQ(asUint64Vec({1153277837650709461}), cells); + } + { + auto point = Geography::fromWKT("POINT(179.9 -89.999)").value(); + auto cells = geoIndex.indexCells(point); + EXPECT_EQ(asUint64Vec({-5764607523209916331}), cells); + } + { + auto point = Geography::fromWKT("POINT(0.0 0.0)").value(); + auto cells = geoIndex.indexCells(point); + EXPECT_EQ(asUint64Vec({1152921504606846977}), cells); + } + { + auto point = Geography::fromWKT("POINT(-36.843143 79.9999999)").value(); + auto cells = geoIndex.indexCells(point); + EXPECT_EQ(asUint64Vec({5738492864430648919}), cells); + } } + // s2_min_level=0, s2_max_level=30, s2_max_cells=8 { - auto point = Geography::fromWKT("POINT(-36.843143 79.9999999)").value(); - auto cells = geoIndex.indexCells(point); - EXPECT_EQ(toUint64Vector({5738492864430648919}), cells); + geo::RegionCoverParams rc(0, 30, 8); + geo::GeoIndex geoIndex(rc); + { + auto point = Geography::fromWKT("POINT(1.0 1.0)").value(); + auto cells = geoIndex.indexCells(point); + EXPECT_EQ(asUint64Vec({1153277837650709461}), cells); + } + { + auto point = Geography::fromWKT("POINT(179.9 -89.999)").value(); + auto cells = geoIndex.indexCells(point); + EXPECT_EQ(asUint64Vec({-5764607523209916331}), cells); + } + { + auto point = Geography::fromWKT("POINT(0.0 0.0)").value(); + auto cells = geoIndex.indexCells(point); + EXPECT_EQ(asUint64Vec({1152921504606846977}), cells); + } + { + auto point = Geography::fromWKT("POINT(-36.843143 79.9999999)").value(); + auto cells = geoIndex.indexCells(point); + EXPECT_EQ(asUint64Vec({5738492864430648919}), cells); + } } } TEST(indexCells, lineString) { - geo::RegionCoverParams rc; - geo::GeoIndex geoIndex(rc); + // s2_min_level=4, s2_max_level=23, s2_max_cells=8 { - auto line = Geography::fromWKT("LINESTRING(1.0 1.0, 2.0 2.0)").value(); - auto cells = geoIndex.indexCells(line); - std::vector expect{ - 1153290940513779712, - 1154047404446580736, - 1154064996699734016, - 1154135365443911680, - 1154164685843464192, - 1154328879221964800, - 1154346471676444672, - }; - EXPECT_EQ(toUint64Vector(expect), cells); + geo::RegionCoverParams rc(4, 23, 8); + geo::GeoIndex geoIndex(rc); + { + auto line = Geography::fromWKT("LINESTRING(1.0 1.0, 2.0 2.0)").value(); + auto cells = geoIndex.indexCells(line); + std::vector expect{ + 1153290940513779712, + 1154047404446580736, + 1154064996699734016, + 1154135365443911680, + 1154164685843464192, + 1154328879221964800, + 1154346471676444672, + }; + EXPECT_EQ(asUint64Vec(expect), cells); + } + { + auto line = Geography::fromWKT( + "LINESTRING(-100.03601074218751 40.74400563300867,-96.96516036987305 " + "39.60634945766583,-91.84398651123048 39.706526341505366)") + .value(); + auto cells = geoIndex.indexCells(line); + std::vector expect{-8676255050873438208, + -8675903207152549888, + -8674214357292285952, + -8665770107990966272, + -8665207158037544960, + -8664644208084123648, + -8664081258130702336, + -8656692539992047616}; + EXPECT_EQ(asUint64Vec(expect), cells); + } + { + auto line = Geography::fromWKT( + "LINESTRING(-109.18024063110352 40.96952973563833,-102.11740493774414 " + "40.98832114106014,-102.00119018554688 37.07120386611709,-108.97098541259767 " + "37.00392356248513,-109.09063339233398 40.94178643285866)") + .value(); + auto cells = geoIndex.indexCells(line); + std::vector expect{-8715591178868752384, + -8714183803985199104, + -8712494954124935168, + -8702080379986640896, + -8699828580172955648, + -8693407432266743808, + -8688569581104529408, + -8686317781290844160}; + EXPECT_EQ(asUint64Vec(expect), cells); + } } + // s2_min_level=0, s2_max_level=30, s2_max_cells=8 { - auto line = Geography::fromWKT( - "LINESTRING(-100.03601074218751 40.74400563300867,-96.96516036987305 " - "39.60634945766583,-91.84398651123048 39.706526341505366)") - .value(); - auto cells = geoIndex.indexCells(line); - std::vector expect{-8676255050873438208, - -8675903207152549888, - -8674214357292285952, - -8665770107990966272, - -8665207158037544960, - -8664644208084123648, - -8664081258130702336, - -8656692539992047616}; - EXPECT_EQ(toUint64Vector(expect), cells); - } - { - auto line = Geography::fromWKT( - "LINESTRING(-109.18024063110352 40.96952973563833,-102.11740493774414 " - "40.98832114106014,-102.00119018554688 37.07120386611709,-108.97098541259767 " - "37.00392356248513,-109.09063339233398 40.94178643285866)") - .value(); - auto cells = geoIndex.indexCells(line); - std::vector expect{-8715591178868752384, - -8714183803985199104, - -8712494954124935168, - -8702080379986640896, - -8699828580172955648, - -8693407432266743808, - -8688569581104529408, - -8686317781290844160}; - EXPECT_EQ(toUint64Vector(expect), cells); + geo::RegionCoverParams rc(0, 30, 8); + geo::GeoIndex geoIndex(rc); + { + auto line = Geography::fromWKT("LINESTRING(1.0 1.0, 2.0 2.0)").value(); + auto cells = geoIndex.indexCells(line); + std::vector expect{ + 1153290940513779712, + 1154047404446580736, + 1154064996699734016, + 1154135365443911680, + 1154164685843464192, + 1154328879221964800, + 1154346471676444672, + }; + EXPECT_EQ(asUint64Vec(expect), cells); + } + { + auto line = Geography::fromWKT( + "LINESTRING(-100.03601074218751 40.74400563300867,-96.96516036987305 " + "39.60634945766583,-91.84398651123048 39.706526341505366)") + .value(); + auto cells = geoIndex.indexCells(line); + std::vector expect{-8676255050873438208, + -8675903207152549888, + -8674214357292285952, + -8665770107990966272, + -8665207158037544960, + -8664644208084123648, + -8664081258130702336, + -8656692539992047616}; + EXPECT_EQ(asUint64Vec(expect), cells); + } + { + auto line = Geography::fromWKT( + "LINESTRING(-109.18024063110352 40.96952973563833,-102.11740493774414 " + "40.98832114106014,-102.00119018554688 37.07120386611709,-108.97098541259767 " + "37.00392356248513,-109.09063339233398 40.94178643285866)") + .value(); + auto cells = geoIndex.indexCells(line); + std::vector expect{-8715591178868752384, + -8714183803985199104, + -8712494954124935168, + -8702080379986640896, + -8699828580172955648, + -8693407432266743808, + -8688569581104529408, + -8686317781290844160}; + EXPECT_EQ(asUint64Vec(expect), cells); + } } } + TEST(indexCells, polygon) { - geo::RegionCoverParams rc; - geo::GeoIndex geoIndex(rc); - { - auto polygon = Geography::fromWKT( - "POLYGON((-105.59286117553711 43.12955341892069,-98.76176834106447 " - "44.11877181138391,-93.97396087646486 " - "38.023348535033705,-105.59286117553711 43.12955341892069))") - .value(); - auto cells = geoIndex.indexCells(polygon); - std::vector expect{-8690821380918214656, - -8686317781290844160, - -8684065981477158912, - -8678436481942945792, - -8665770107990966272, - -8665207158037544960, - -8664644208084123648, - -8662955358223859712}; - EXPECT_EQ(toUint64Vector(expect), cells); - } + // s2_min_level=4, s2_max_level=23, s2_max_cells=8 { - auto polygon = - Geography::fromWKT( - "POLYGON((-107.24699020385744 45.21638951846552,-91.75283432006836 " - "46.158312926461235,-90.07295608520508 35.17914020576748,-109.77504730224612 " - "38.65334327823746,-107.24699020385744 45.21638951846552))") - .value(); - auto cells = geoIndex.indexCells(polygon); - std::vector expect{5958262307011166208, - 5967269506265907200, - 5994291104030130176, - 6002172403378028544, - -8714465278961909760, - -8702080379986640896, - -8696450880452427776, - -8687443681197686784, - -8678436481942945792, - -8669429282688204800, - -8660422083433463808, - -8651414884178722816}; - EXPECT_EQ(toUint64Vector(expect), cells); + geo::RegionCoverParams rc(4, 23, 8); + geo::GeoIndex geoIndex(rc); + { + auto polygon = Geography::fromWKT( + "POLYGON((-105.59286117553711 43.12955341892069,-98.76176834106447 " + "44.11877181138391,-93.97396087646486 " + "38.023348535033705,-105.59286117553711 43.12955341892069))") + .value(); + auto cells = geoIndex.indexCells(polygon); + std::vector expect{-8690821380918214656, + -8686317781290844160, + -8684065981477158912, + -8678436481942945792, + -8665770107990966272, + -8665207158037544960, + -8664644208084123648, + -8662955358223859712}; + EXPECT_EQ(asUint64Vec(expect), cells); + } + { + auto polygon = + Geography::fromWKT( + "POLYGON((-107.24699020385744 45.21638951846552,-91.75283432006836 " + "46.158312926461235,-90.07295608520508 35.17914020576748,-109.77504730224612 " + "38.65334327823746,-107.24699020385744 45.21638951846552))") + .value(); + auto cells = geoIndex.indexCells(polygon); + std::vector expect{5958262307011166208, + 5967269506265907200, + 5994291104030130176, + 6002172403378028544, + -8714465278961909760, + -8702080379986640896, + -8696450880452427776, + -8687443681197686784, + -8678436481942945792, + -8669429282688204800, + -8660422083433463808, + -8651414884178722816}; + EXPECT_EQ(asUint64Vec(expect), cells); + } + { + auto polygon = + Geography::fromWKT( + "POLYGON((-107.17094421386722 51.23698687887105,-100.24475097656253 " + "50.57407993312597,-101.63520812988283 47.57050358015326,-108.1597137451172 " + "47.614032638527846,-107.17094421386722 51.23698687887105),(-106.00682258605956 " + "50.35416859141216,-105.23014068603514 50.212503875989455,-105.55715560913085 " + "49.755319847594194,-106.36962890624999 49.95817799043337,-106.00682258605956 " + "50.35416859141216),(-103.90560150146483 49.21126151433475,-102.1109676361084 " + "49.32232483567492,-102.99759864807127 48.52160729809822,-103.90560150146483 " + "49.21126151433475))") + .value(); + auto cells = geoIndex.indexCells(polygon); + std::vector expect{5969732412312125440, + 5971192563753811968, + 5971491630916567040, + 5972899005800120320, + 5986409804682231808, + 5988661604495917056, + 5990913404309602304, + 5997668803750658048}; + EXPECT_EQ(asUint64Vec(expect), cells); + } } + // s2_min_level=0, s2_max_level=30, s2_max_cells=8 { - auto polygon = - Geography::fromWKT( - "POLYGON((-107.17094421386722 51.23698687887105,-100.24475097656253 " - "50.57407993312597,-101.63520812988283 47.57050358015326,-108.1597137451172 " - "47.614032638527846,-107.17094421386722 51.23698687887105),(-106.00682258605956 " - "50.35416859141216,-105.23014068603514 50.212503875989455,-105.55715560913085 " - "49.755319847594194,-106.36962890624999 49.95817799043337,-106.00682258605956 " - "50.35416859141216),(-103.90560150146483 49.21126151433475,-102.1109676361084 " - "49.32232483567492,-102.99759864807127 48.52160729809822,-103.90560150146483 " - "49.21126151433475))") - .value(); - auto cells = geoIndex.indexCells(polygon); - std::vector expect{5969732412312125440, - 5971192563753811968, - 5971491630916567040, - 5972899005800120320, - 5986409804682231808, - 5988661604495917056, - 5990913404309602304, - 5997668803750658048}; - EXPECT_EQ(toUint64Vector(expect), cells); + geo::RegionCoverParams rc(0, 30, 8); + geo::GeoIndex geoIndex(rc); + { + auto polygon = + Geography::fromWKT( + "POLYGON((-107.24699020385744 45.21638951846552,-91.75283432006836 " + "46.158312926461235,-90.07295608520508 35.17914020576748,-109.77504730224612 " + "38.65334327823746,-107.24699020385744 45.21638951846552))") + .value(); + auto cells = geoIndex.indexCells(polygon); + std::vector expect{ + 5957417882081034240, + 5959388206918008832, + 5966143606359064576, + 5968395406172749824, + 5994291104030130176, + 6002172403378028544, + -8700954480079798272, + -8664925683060834304, + }; + EXPECT_EQ(asUint64Vec(expect), cells); + } + { + auto polygon = + Geography::fromWKT( + "POLYGON((-107.17094421386722 51.23698687887105,-100.24475097656253 " + "50.57407993312597,-101.63520812988283 47.57050358015326,-108.1597137451172 " + "47.614032638527846,-107.17094421386722 51.23698687887105),(-106.00682258605956 " + "50.35416859141216,-105.23014068603514 50.212503875989455,-105.55715560913085 " + "49.755319847594194,-106.36962890624999 49.95817799043337,-106.00682258605956 " + "50.35416859141216),(-103.90560150146483 49.21126151433475,-102.1109676361084 " + "49.32232483567492,-102.99759864807127 48.52160729809822,-103.90560150146483 " + "49.21126151433475))") + .value(); + auto cells = geoIndex.indexCells(polygon); + std::vector expect{ + 5969732412312125440, + 5971192563753811968, + 5971491630916567040, + 5972899005800120320, + 5986409804682231808, + 5988661604495917056, + 5990913404309602304, + 5997668803750658048, + }; + EXPECT_EQ(asUint64Vec(expect), cells); + } } } diff --git a/src/common/utils/IndexKeyUtils.cpp b/src/common/utils/IndexKeyUtils.cpp index 7694db7f0c3..dd43757fb56 100644 --- a/src/common/utils/IndexKeyUtils.cpp +++ b/src/common/utils/IndexKeyUtils.cpp @@ -7,11 +7,14 @@ #include +#include "common/geo/GeoIndex.h" + namespace nebula { // static -std::vector IndexKeyUtils::encodeValues( - std::vector&& values, const std::vector& cols) { +std::vector IndexKeyUtils::encodeValues(std::vector&& values, + const meta::cpp2::IndexItem* indexItem) { + auto& cols = indexItem->get_fields(); bool hasNullCol = false; // An index has a maximum of 16 columns. 2 byte (16 bit) is enough. u_short nullableBitSet = 0; @@ -52,7 +55,17 @@ std::vector IndexKeyUtils::encodeValues( const auto& value = values.back(); if (!value.isNull()) { DCHECK(value.type() == Value::Type::GEOGRAPHY); - indexes = encodeGeography(value.getGeography()); + geo::RegionCoverParams rc; + const auto* indexParams = indexItem->get_index_params(); + if (indexParams) { + if (indexParams->s2_max_level_ref().has_value()) { + rc.maxCellLevel_ = indexParams->s2_max_level_ref().value(); + } + if (indexParams->s2_max_cells_ref().has_value()) { + rc.maxCellNum_ = indexParams->s2_max_cells_ref().value(); + } + } + indexes = encodeGeography(value.getGeography(), rc); } else { nullableBitSet |= 0x8000; auto type = IndexKeyUtils::toValueType(cols.back().type.get_type()); @@ -157,10 +170,11 @@ Value IndexKeyUtils::parseIndexTTL(const folly::StringPiece& raw) { // static StatusOr> IndexKeyUtils::collectIndexValues( - RowReader* reader, const std::vector& cols) { + RowReader* reader, const meta::cpp2::IndexItem* indexItem) { if (reader == nullptr) { return Status::Error("Invalid row reader"); } + auto& cols = indexItem->get_fields(); std::vector values; for (const auto& col : cols) { auto v = reader->getValueByName(col.get_name()); @@ -172,7 +186,7 @@ StatusOr> IndexKeyUtils::collectIndexValues( } values.emplace_back(std::move(v)); } - return encodeValues(std::move(values), cols); + return encodeValues(std::move(values), indexItem); } // static diff --git a/src/common/utils/IndexKeyUtils.h b/src/common/utils/IndexKeyUtils.h index c2d8e7d1276..75bb9c3f784 100644 --- a/src/common/utils/IndexKeyUtils.h +++ b/src/common/utils/IndexKeyUtils.h @@ -7,6 +7,7 @@ #define COMMON_UTILS_INDEXKEYUTILS_H_ #include +#include #include "codec/RowReader.h" #include "common/base/Base.h" @@ -185,6 +186,12 @@ class IndexKeyUtils final { return raw; } + static uint64_t decodeUint64(const folly::StringPiece& raw) { + auto val = *reinterpret_cast(raw.data()); + val = folly::Endian::big(val); + return val; + } + static std::string encodeRank(EdgeRanking rank) { return IndexKeyUtils::encodeInt64(rank); } static EdgeRanking decodeRank(const folly::StringPiece& raw) { @@ -302,19 +309,23 @@ class IndexKeyUtils final { return buf; } - static std::vector encodeGeography(const nebula::Geography& gg) { - // TODO(jie): Get index params from meta to construct RegionCoverParams - geo::RegionCoverParams rc; - // TODO(jie): Get schema meta to know if it's point only - geo::GeoIndex geoIndex(rc, false); + static std::vector encodeGeography(const nebula::Geography& gg, + const geo::RegionCoverParams& rc) { + geo::GeoIndex geoIndex(rc); auto cellIds = geoIndex.indexCells(gg); std::vector bufs; + bufs.reserve(cellIds.size()); for (auto cellId : cellIds) { bufs.emplace_back(encodeUint64(cellId)); } return bufs; } + // NOTE(jie): The decoded data is not the original Geography data, but the uint64 type S2CellID. + // decodeValue() should not call this function, it should turn for the data table instead. + // It's only used for tests. + static uint64_t decodeGeography(const folly::StringPiece& raw) { return decodeUint64(raw); } + static nebula::DateTime decodeDateTime(const folly::StringPiece& raw) { int16_t year = *reinterpret_cast(raw.data()); int8_t month = *reinterpret_cast(raw.data() + sizeof(int16_t)); @@ -496,8 +507,8 @@ class IndexKeyUtils final { /** * Generate vertex|edge index key for kv store **/ - static std::vector encodeValues( - std::vector&& values, const std::vector& cols); + static std::vector encodeValues(std::vector&& values, + const meta::cpp2::IndexItem* indexItem); /** * param valueTypes ： column type of each index column. If there are no @@ -530,7 +541,7 @@ class IndexKeyUtils final { static Value parseIndexTTL(const folly::StringPiece& raw); static StatusOr> collectIndexValues( - RowReader* reader, const std::vector& cols); + RowReader* reader, const meta::cpp2::IndexItem* indexItem); private: IndexKeyUtils() = delete; diff --git a/src/common/utils/test/IndexKeyUtilsTest.cpp b/src/common/utils/test/IndexKeyUtilsTest.cpp index 79b298bf2b3..6e8da0c39bc 100644 --- a/src/common/utils/test/IndexKeyUtilsTest.cpp +++ b/src/common/utils/test/IndexKeyUtilsTest.cpp @@ -5,7 +5,12 @@ #include +#include + #include "common/base/Base.h" +#include "common/datatypes/DataSet.h" +#include "common/datatypes/Geography.h" +#include "common/geo/GeoIndex.h" #include "common/utils/IndexKeyUtils.h" namespace nebula { @@ -96,6 +101,18 @@ bool evalDateTime(nebula::DateTime val) { return val == res.getDateTime(); } +std::vector evalGeography(nebula::Geography val) { + geo::RegionCoverParams rc(0, 30, 8); + auto vals = IndexKeyUtils::encodeGeography(val, rc); + std::vector got; + for (auto& str : vals) { + auto res = IndexKeyUtils::decodeGeography(str); + got.emplace_back(res); + } + // Return the actual cellIds + return got; +} + TEST(IndexKeyUtilsTest, encodeValue) { EXPECT_TRUE(evalInt64(1)); EXPECT_TRUE(evalInt64(0)); @@ -135,6 +152,49 @@ TEST(IndexKeyUtilsTest, encodeValue) { EXPECT_TRUE(evalDateTime(dt)); } +TEST(IndexKeyUtilsTest, encodeGeography) { + // The expected result comes from BigQuery + auto toUint64Vector = [](std::vector expect) { + auto asUint64 = [](int64_t i) -> uint64_t { + const char* c = reinterpret_cast(&i); + uint64_t u = *reinterpret_cast(c); + return u; + }; + + std::vector transformedExpect; + transformedExpect.reserve(expect.size()); + for (int64_t i : expect) { + transformedExpect.push_back(asUint64(i)); + } + return transformedExpect; + }; + + auto pt = Geography::fromWKT("POINT(108.1 32.5)").value(); + EXPECT_EQ(toUint64Vector({0x368981adc0392fe3}), evalGeography(pt)); + + auto ls = Geography::fromWKT("LINESTRING(68.9 48.9,76.1 35.5,125.7 28.2)").value(); + EXPECT_EQ(toUint64Vector({0x3440000000000000, + 0x34ff000000000000, + 0x3640000000000000, + 0x3684000000000000, + 0x37c0000000000000, + 0x3840000000000000, + 0x38c0000000000000, + 0x4240000000000000}), + evalGeography(ls)); + + auto pg = Geography::fromWKT( + "POLYGON((91.2 38.6,99.7 41.9,111.2 38.9,115.6 33.2,109.5 29.0,105.8 24.1,102.9 " + "30.5,93.0 28.1,95.4 32.8,86.1 33.6,85.3 38.8,91.2 38.6))") + .value(); + EXPECT_EQ(toUint64Vector({0x342b000000000000, + 0x35d0000000000000, + 0x3700000000000000, + 0x3830000000000000, + 0x39d0000000000000}), + evalGeography(pg)); +} + TEST(IndexKeyUtilsTest, encodeDouble) { EXPECT_TRUE(evalDouble(100.5)); EXPECT_TRUE(evalDouble(200.5)); @@ -225,7 +285,9 @@ TEST(IndexKeyUtilsTest, nullableValue) { cols.emplace_back(nullCol(folly::stringPrintf("col%ld", j), PropertyType::BOOL)); } // TODO(jie) Add index key tests for geography - auto raws = IndexKeyUtils::encodeValues(std::move(values), std::move(cols)); + auto indexItem = std::make_unique(); + indexItem->set_fields(cols); + auto raws = IndexKeyUtils::encodeValues(std::move(values), indexItem.get()); u_short s = 0xfc00; /* the binary is '11111100 00000000'*/ std::string expected; expected.append(reinterpret_cast(&s), sizeof(u_short)); @@ -240,7 +302,9 @@ TEST(IndexKeyUtilsTest, nullableValue) { for (int64_t j = 1; j <= 2; j++) { cols.emplace_back(nullCol(folly::stringPrintf("col%ld", j), PropertyType::BOOL)); } - auto raws = IndexKeyUtils::encodeValues(std::move(values), std::move(cols)); + auto indexItem = std::make_unique(); + indexItem->set_fields(cols); + auto raws = IndexKeyUtils::encodeValues(std::move(values), indexItem.get()); u_short s = 0x4000; /* the binary is '01000000 00000000'*/ std::string expected; expected.append(reinterpret_cast(&s), sizeof(u_short)); @@ -255,7 +319,9 @@ TEST(IndexKeyUtilsTest, nullableValue) { for (int64_t j = 1; j <= 2; j++) { cols.emplace_back(nullCol(folly::stringPrintf("col%ld", j), PropertyType::BOOL)); } - auto raws = IndexKeyUtils::encodeValues(std::move(values), std::move(cols)); + auto indexItem = std::make_unique(); + indexItem->set_fields(cols); + auto raws = IndexKeyUtils::encodeValues(std::move(values), indexItem.get()); u_short s = 0x0000; /* the binary is '01000000 00000000'*/ std::string expected; expected.append(reinterpret_cast(&s), sizeof(u_short)); @@ -269,8 +335,9 @@ TEST(IndexKeyUtilsTest, nullableValue) { values.emplace_back(Value(NullType::__NULL__)); cols.emplace_back(nullCol(folly::stringPrintf("col%ld", i), PropertyType::INT64)); } - - auto raws = IndexKeyUtils::encodeValues(std::move(values), std::move(cols)); + auto indexItem = std::make_unique(); + indexItem->set_fields(cols); + auto raws = IndexKeyUtils::encodeValues(std::move(values), indexItem.get()); u_short s = 0xfff0; /* the binary is '11111111 11110000'*/ std::string expected; expected.append(reinterpret_cast(&s), sizeof(u_short)); @@ -308,7 +375,9 @@ TEST(IndexKeyUtilsTest, nullableValue) { cols.emplace_back(nullCol(folly::stringPrintf("col_%ld_%ld", i, j), types[j])); } } - auto raws = IndexKeyUtils::encodeValues(std::move(values), cols); + auto indexItem = std::make_unique(); + indexItem->set_fields(cols); + auto raws = IndexKeyUtils::encodeValues(std::move(values), indexItem.get()); u_short s = 0xaaa0; /* the binary is '10101010 10100000'*/ std::string expected; expected.append(reinterpret_cast(&s), sizeof(u_short)); @@ -322,7 +391,9 @@ TEST(IndexKeyUtilsTest, nullableValue) { values.emplace_back(Value(NullType::__NULL__)); cols.emplace_back(nullCol(folly::stringPrintf("col%ld", i), PropertyType::BOOL)); } - auto raws = IndexKeyUtils::encodeValues(std::move(values), std::move(cols)); + auto indexItem = std::make_unique(); + indexItem->set_fields(cols); + auto raws = IndexKeyUtils::encodeValues(std::move(values), indexItem.get()); u_short s = 0xff80; /* the binary is '11111111 10000000'*/ std::string expected; expected.append(reinterpret_cast(&s), sizeof(u_short)); @@ -418,9 +489,11 @@ TEST(IndexKeyUtilsTest, getValueFromIndexKeyTest) { }; auto expected = vertices; + auto indexItem = std::make_unique(); + indexItem->set_fields(cols); std::vector indexKeys; for (auto& row : vertices) { - auto values = IndexKeyUtils::encodeValues(std::move(row.second), cols); + auto values = IndexKeyUtils::encodeValues(std::move(row.second), indexItem.get()); ASSERT_EQ(indexValueSize, values[0].size()); auto keys = IndexKeyUtils::vertexIndexKeys(vIdLen, partId, indexId, row.first, std::move(values)); @@ -440,9 +513,11 @@ TEST(IndexKeyUtilsTest, getValueFromIndexKeyTest) { }; auto expected = edges; + auto indexItem = std::make_unique(); + indexItem->set_fields(cols); std::vector indexKeys; for (auto& row : edges) { - auto values = IndexKeyUtils::encodeValues(std::move(row.second), cols); + auto values = IndexKeyUtils::encodeValues(std::move(row.second), indexItem.get()); ASSERT_EQ(indexValueSize, values[0].size()); auto keys = IndexKeyUtils::edgeIndexKeys( @@ -476,9 +551,11 @@ TEST(IndexKeyUtilsTest, getValueFromIndexKeyTest) { {"9", {null, null, null, null, null, null}}}; auto expected = vertices; + auto indexItem = std::make_unique(); + indexItem->set_fields(cols); std::vector indexKeys; for (auto& row : vertices) { - auto values = IndexKeyUtils::encodeValues(std::move(row.second), cols); + auto values = IndexKeyUtils::encodeValues(std::move(row.second), indexItem.get()); ASSERT_EQ(indexValueSize, values[0].size()); auto keys = IndexKeyUtils::vertexIndexKeys(vIdLen, partId, indexId, row.first, std::move(values)); @@ -503,9 +580,11 @@ TEST(IndexKeyUtilsTest, getValueFromIndexKeyTest) { {"9", {null, null, null, null, null, null}}}; auto expected = edges; + auto indexItem = std::make_unique(); + indexItem->set_fields(cols); std::vector indexKeys; for (auto& row : edges) { - auto values = IndexKeyUtils::encodeValues(std::move(row.second), cols); + auto values = IndexKeyUtils::encodeValues(std::move(row.second), indexItem.get()); ASSERT_EQ(indexValueSize, values[0].size()); auto keys = IndexKeyUtils::edgeIndexKeys( vIdLen, partId, indexId, row.first, 0, row.first, std::move(values)); diff --git a/src/graph/executor/maintain/EdgeIndexExecutor.cpp b/src/graph/executor/maintain/EdgeIndexExecutor.cpp index 798ff8640dc..cfe1b85ba7d 100644 --- a/src/graph/executor/maintain/EdgeIndexExecutor.cpp +++ b/src/graph/executor/maintain/EdgeIndexExecutor.cpp @@ -23,6 +23,7 @@ folly::Future CreateEdgeIndexExecutor::execute() { ceiNode->getSchemaName(), ceiNode->getFields(), ceiNode->getIfNotExists(), + ceiNode->getIndexParams(), ceiNode->getComment()) .via(runner()) .thenValue([ceiNode, spaceId](StatusOr resp) { diff --git a/src/graph/executor/maintain/TagIndexExecutor.cpp b/src/graph/executor/maintain/TagIndexExecutor.cpp index 3d8a8708e64..86c53301193 100644 --- a/src/graph/executor/maintain/TagIndexExecutor.cpp +++ b/src/graph/executor/maintain/TagIndexExecutor.cpp @@ -23,6 +23,7 @@ folly::Future CreateTagIndexExecutor::execute() { ctiNode->getSchemaName(), ctiNode->getFields(), ctiNode->getIfNotExists(), + ctiNode->getIndexParams(), ctiNode->getComment()) .via(runner()) .thenValue([ctiNode, spaceId](StatusOr resp) { diff --git a/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp b/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp index 3a0af33852e..3729afc85b2 100644 --- a/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp +++ b/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp @@ -80,10 +80,25 @@ StatusOr GeoPredicateIndexScanBaseRule::transform( DCHECK(secondVal.isGeography()); const auto& geog = secondVal.getGeography(); - // TODO(jie): Get index params from meta to construct RegionCoverParams + auto indexItem = indexItems.back(); + const auto& fields = indexItem->get_fields(); + DCHECK_EQ(fields.size(), 1); // geo field + auto& geoField = fields.back(); + auto& geoColumnTypeDef = geoField.get_type(); + bool isPointColumn = geoColumnTypeDef.geo_shape_ref().has_value() && + geoColumnTypeDef.geo_shape_ref().value() == meta::cpp2::GeoShape::POINT; + geo::RegionCoverParams rc; - // TODO(jie): Get schema meta to know if it's point only - geo::GeoIndex geoIndex(rc, false); + const auto* indexParams = indexItem->get_index_params(); + if (indexParams) { + if (indexParams->s2_max_level_ref().has_value()) { + rc.maxCellLevel_ = indexParams->s2_max_level_ref().value(); + } + if (indexParams->s2_max_cells_ref().has_value()) { + rc.maxCellNum_ = indexParams->s2_max_cells_ref().value(); + } + } + geo::GeoIndex geoIndex(rc, isPointColumn); std::vector scanRanges; if (geoPredicateName == "st_intersects") { scanRanges = geoIndex.intersects(geog); @@ -102,10 +117,7 @@ StatusOr GeoPredicateIndexScanBaseRule::transform( } std::vector idxCtxs; idxCtxs.reserve(scanRanges.size()); - auto indexItem = indexItems.back(); - const auto& fields = indexItem->get_fields(); - DCHECK_EQ(fields.size(), 1); // geo field - auto fieldName = fields.back().get_name(); + auto fieldName = geoField.get_name(); for (auto& scanRange : scanRanges) { IndexQueryContext ictx; auto indexColumnHint = scanRange.toIndexColumnHint(); diff --git a/src/graph/planner/plan/Maintain.cpp b/src/graph/planner/plan/Maintain.cpp index 900e433c0a5..4a108ce1cc0 100644 --- a/src/graph/planner/plan/Maintain.cpp +++ b/src/graph/planner/plan/Maintain.cpp @@ -52,6 +52,9 @@ std::unique_ptr CreateIndexNode::explain() const { } addDescription("fields", folly::toJson(util::toJson(fields)), desc.get()); addDescription("ifNotExists", folly::to(ifNotExists_), desc.get()); + if (indexParams_) { + addDescription("indexParams", folly::toJson(util::toJson(*indexParams_)), desc.get()); + } return desc; } diff --git a/src/graph/planner/plan/Maintain.h b/src/graph/planner/plan/Maintain.h index b93ee18ff2d..988be1e51bd 100644 --- a/src/graph/planner/plan/Maintain.h +++ b/src/graph/planner/plan/Maintain.h @@ -300,12 +300,14 @@ class CreateIndexNode : public SingleDependencyNode { std::string indexName, std::vector fields, bool ifNotExists, + std::unique_ptr indexParams, const std::string* comment) : SingleDependencyNode(qctx, kind, input), schemaName_(std::move(schemaName)), indexName_(std::move(indexName)), fields_(std::move(fields)), ifNotExists_(ifNotExists), + indexParams_(std::move(indexParams)), comment_(comment) {} public: @@ -317,6 +319,8 @@ class CreateIndexNode : public SingleDependencyNode { bool getIfNotExists() const { return ifNotExists_; } + const meta::cpp2::IndexParams* getIndexParams() const { return indexParams_.get(); } + const std::string* getComment() const { return comment_; } std::unique_ptr explain() const override; @@ -326,6 +330,7 @@ class CreateIndexNode : public SingleDependencyNode { std::string indexName_; std::vector fields_; bool ifNotExists_; + std::unique_ptr indexParams_; const std::string* comment_; }; @@ -337,6 +342,7 @@ class CreateTagIndex final : public CreateIndexNode { std::string indexName, std::vector fields, bool ifNotExists, + std::unique_ptr indexParams, const std::string* comment) { return qctx->objPool()->add(new CreateTagIndex(qctx, input, @@ -344,6 +350,7 @@ class CreateTagIndex final : public CreateIndexNode { std::move(indexName), std::move(fields), ifNotExists, + std::move(indexParams), comment)); } @@ -354,6 +361,7 @@ class CreateTagIndex final : public CreateIndexNode { std::string indexName, std::vector fields, bool ifNotExists, + std::unique_ptr indexParams, const std::string* comment) : CreateIndexNode(qctx, input, @@ -362,6 +370,7 @@ class CreateTagIndex final : public CreateIndexNode { std::move(indexName), std::move(fields), ifNotExists, + std::move(indexParams), comment) {} }; @@ -373,6 +382,7 @@ class CreateEdgeIndex final : public CreateIndexNode { std::string indexName, std::vector fields, bool ifNotExists, + std::unique_ptr indexParams, const std::string* comment) { return qctx->objPool()->add(new CreateEdgeIndex(qctx, input, @@ -380,6 +390,7 @@ class CreateEdgeIndex final : public CreateIndexNode { std::move(indexName), std::move(fields), ifNotExists, + std::move(indexParams), comment)); } @@ -390,6 +401,7 @@ class CreateEdgeIndex final : public CreateIndexNode { std::string indexName, std::vector fields, bool ifNotExists, + std::unique_ptr indexParams, const std::string* comment) : CreateIndexNode(qctx, input, @@ -398,6 +410,7 @@ class CreateEdgeIndex final : public CreateIndexNode { std::move(indexName), std::move(fields), ifNotExists, + std::move(indexParams), comment) {} }; diff --git a/src/graph/util/IndexUtil.cpp b/src/graph/util/IndexUtil.cpp index 57c98bf4bbc..67f2e7c0a50 100644 --- a/src/graph/util/IndexUtil.cpp +++ b/src/graph/util/IndexUtil.cpp @@ -5,6 +5,8 @@ #include "graph/util/IndexUtil.h" +#include + namespace nebula { namespace graph { @@ -21,6 +23,29 @@ Status IndexUtil::validateColumns(const std::vector &fields) { return Status::OK(); } +// static +Status IndexUtil::validateIndexParams(const std::vector ¶ms, + meta::cpp2::IndexParams &indexParams) { + for (auto *param : params) { + switch (param->getParamType()) { + case IndexParamItem::S2_MAX_LEVEL: { + auto ret = param->getS2MaxLevel(); + NG_RETURN_IF_ERROR(ret); + indexParams.set_s2_max_level(std::move(ret).value()); + break; + } + case IndexParamItem::S2_MAX_CELLS: { + auto ret2 = param->getS2MaxCells(); + NG_RETURN_IF_ERROR(ret2); + indexParams.set_s2_max_cells(std::move(ret2).value()); + break; + } + } + } + + return Status::OK(); +} + StatusOr IndexUtil::toDescIndex(const meta::cpp2::IndexItem &indexItem) { DataSet dataSet({"Field", "Type"}); for (auto &col : indexItem.get_fields()) { @@ -38,7 +63,7 @@ StatusOr IndexUtil::toShowCreateIndex(bool isTagIndex, DataSet dataSet; std::string createStr; createStr.reserve(1024); - std::string schemaName = indexItem.get_schema_name(); + const std::string &schemaName = indexItem.get_schema_name(); if (isTagIndex) { dataSet.colNames = {"Tag Index Name", "Create Tag Index"}; createStr = "CREATE TAG INDEX `" + indexName + "` ON `" + schemaName + "` (\n"; @@ -62,11 +87,31 @@ StatusOr IndexUtil::toShowCreateIndex(bool isTagIndex, createStr += "\n"; } createStr += ")"; + + const auto *indexParams = indexItem.get_index_params(); + std::vector params; + if (indexParams) { + if (indexParams->s2_max_level_ref().has_value()) { + params.emplace_back("s2_max_level = " + + std::to_string(indexParams->s2_max_level_ref().value())); + } + if (indexParams->s2_max_cells_ref().has_value()) { + params.emplace_back("s2_max_cells = " + + std::to_string(indexParams->s2_max_cells_ref().value())); + } + } + if (!params.empty()) { + createStr += " WITH ("; + createStr += folly::join(", ", params); + createStr += ")"; + } + if (indexItem.comment_ref().has_value()) { - createStr += " comment = \""; + createStr += " comment \""; createStr += *indexItem.get_comment(); createStr += "\""; } + row.emplace_back(std::move(createStr)); dataSet.rows.emplace_back(std::move(row)); return dataSet; diff --git a/src/graph/util/IndexUtil.h b/src/graph/util/IndexUtil.h index c1f08360a66..22100690363 100644 --- a/src/graph/util/IndexUtil.h +++ b/src/graph/util/IndexUtil.h @@ -19,6 +19,9 @@ class IndexUtil final { static Status validateColumns(const std::vector &fields); + static Status validateIndexParams(const std::vector ¶ms, + meta::cpp2::IndexParams &indexParams); + static StatusOr toDescIndex(const meta::cpp2::IndexItem &indexItem); static StatusOr toShowCreateIndex(bool isTagIndex, diff --git a/src/graph/util/SchemaUtil.cpp b/src/graph/util/SchemaUtil.cpp index 13e0ed7e896..f0ef3b93bae 100644 --- a/src/graph/util/SchemaUtil.cpp +++ b/src/graph/util/SchemaUtil.cpp @@ -8,6 +8,7 @@ #include #include "common/base/Base.h" +#include "common/base/Status.h" #include "graph/context/QueryContext.h" #include "graph/context/QueryExpressionContext.h" diff --git a/src/graph/util/ToJson.cpp b/src/graph/util/ToJson.cpp index e3ec2d5dcbe..6f7084662e6 100644 --- a/src/graph/util/ToJson.cpp +++ b/src/graph/util/ToJson.cpp @@ -96,6 +96,17 @@ folly::dynamic toJson(const meta::cpp2::SchemaProp &prop) { return object; } +folly::dynamic toJson(const meta::cpp2::IndexParams ¶ms) { + folly::dynamic object = folly::dynamic::object(); + if (params.s2_max_level_ref().has_value()) { + object.insert("s2_max_level", *params.s2_max_level_ref()); + } + if (params.s2_max_cells_ref().has_value()) { + object.insert("s2_max_cells", *params.s2_max_cells_ref()); + } + return object; +} + folly::dynamic toJson(const meta::cpp2::AlterSchemaItem &item) { folly::dynamic json = folly::dynamic::object(); if (item.schema_ref().is_set()) { diff --git a/src/graph/util/ToJson.h b/src/graph/util/ToJson.h index 01f39a54e10..eed062480a7 100644 --- a/src/graph/util/ToJson.h +++ b/src/graph/util/ToJson.h @@ -25,6 +25,7 @@ class AlterSchemaItem; class ColumnDef; class Schema; class SchemaProp; +class IndexParams; } // namespace cpp2 } // namespace meta @@ -68,6 +69,7 @@ folly::dynamic toJson(const meta::cpp2::SpaceDesc &desc); folly::dynamic toJson(const meta::cpp2::ColumnDef &column); folly::dynamic toJson(const meta::cpp2::Schema &schema); folly::dynamic toJson(const meta::cpp2::SchemaProp &prop); +folly::dynamic toJson(const meta::cpp2::IndexParams &indexParams); folly::dynamic toJson(const meta::cpp2::AlterSchemaItem &item); folly::dynamic toJson(const storage::cpp2::EdgeKey &edgeKey); folly::dynamic toJson(const storage::cpp2::NewTag &tag); diff --git a/src/graph/validator/MaintainValidator.cpp b/src/graph/validator/MaintainValidator.cpp index 61f30013106..56a906f15e3 100644 --- a/src/graph/validator/MaintainValidator.cpp +++ b/src/graph/validator/MaintainValidator.cpp @@ -5,6 +5,9 @@ #include "graph/validator/MaintainValidator.h" +#include + +#include "common/base/Status.h" #include "common/charset/Charset.h" #include "common/expression/ConstantExpression.h" #include "graph/planner/plan/Admin.h" @@ -15,6 +18,7 @@ #include "graph/util/FTIndexUtils.h" #include "graph/util/IndexUtil.h" #include "graph/util/SchemaUtil.h" +#include "interface/gen-cpp2/meta_types.h" #include "parser/MaintainSentences.h" namespace nebula { @@ -294,7 +298,12 @@ Status CreateTagIndexValidator::validateImpl() { index_ = *sentence->indexName(); fields_ = sentence->fields(); ifNotExist_ = sentence->isIfNotExist(); - // TODO(darion) Save the index + auto *indexParamList = sentence->getIndexParamList(); + if (indexParamList) { + meta::cpp2::IndexParams indexParams; + NG_RETURN_IF_ERROR(IndexUtil::validateIndexParams(indexParamList->getParams(), indexParams)); + indexParams_ = std::make_unique(std::move(indexParams)); + } return Status::OK(); } @@ -306,6 +315,7 @@ Status CreateTagIndexValidator::toPlan() { *sentence->indexName(), sentence->fields(), sentence->isIfNotExist(), + std::move(indexParams_), sentence->comment()); root_ = doNode; tail_ = root_; @@ -319,6 +329,12 @@ Status CreateEdgeIndexValidator::validateImpl() { fields_ = sentence->fields(); ifNotExist_ = sentence->isIfNotExist(); // TODO(darion) Save the index + auto *indexParamList = sentence->getIndexParamList(); + if (indexParamList) { + meta::cpp2::IndexParams indexParams; + NG_RETURN_IF_ERROR(IndexUtil::validateIndexParams(indexParamList->getParams(), indexParams)); + indexParams_ = std::make_unique(std::move(indexParams)); + } return Status::OK(); } @@ -330,6 +346,7 @@ Status CreateEdgeIndexValidator::toPlan() { *sentence->indexName(), sentence->fields(), sentence->isIfNotExist(), + std::move(indexParams_), sentence->comment()); root_ = doNode; tail_ = root_; diff --git a/src/graph/validator/MaintainValidator.h b/src/graph/validator/MaintainValidator.h index 4c577063536..508dd2c1598 100644 --- a/src/graph/validator/MaintainValidator.h +++ b/src/graph/validator/MaintainValidator.h @@ -9,6 +9,7 @@ #include "clients/meta/MetaClient.h" #include "graph/context/ast/QueryAstContext.h" #include "graph/validator/Validator.h" +#include "interface/gen-cpp2/meta_types.h" #include "parser/AdminSentences.h" namespace nebula { @@ -158,6 +159,7 @@ class CreateTagIndexValidator final : public Validator { std::string index_; std::vector fields_; bool ifNotExist_; + std::unique_ptr indexParams_; }; class CreateEdgeIndexValidator final : public Validator { @@ -174,6 +176,7 @@ class CreateEdgeIndexValidator final : public Validator { std::string index_; std::vector fields_; bool ifNotExist_; + std::unique_ptr indexParams_; }; class DropTagIndexValidator final : public Validator { diff --git a/src/interface/meta.thrift b/src/interface/meta.thrift index 353940088b8..931b014cf58 100644 --- a/src/interface/meta.thrift +++ b/src/interface/meta.thrift @@ -141,13 +141,19 @@ struct EdgeItem { 4: Schema schema, } +struct IndexParams { + 1: optional i32 s2_max_level, + 2: optional i32 s2_max_cells, +} + struct IndexItem { - 1: common.IndexID index_id, - 2: binary index_name, - 3: common.SchemaID schema_id - 4: binary schema_name, - 5: list fields, - 6: optional binary comment, + 1: common.IndexID index_id, + 2: binary index_name, + 3: common.SchemaID schema_id + 4: binary schema_name, + 5: list fields, + 6: optional binary comment, + 7: optional IndexParams index_params, } enum HostStatus { @@ -591,6 +597,7 @@ struct CreateTagIndexReq { 4: list fields, 5: bool if_not_exists, 6: optional binary comment, + 7: optional IndexParams index_params, } struct DropTagIndexReq { @@ -627,6 +634,7 @@ struct CreateEdgeIndexReq { 4: list fields, 5: bool if_not_exists, 6: optional binary comment, + 7: optional IndexParams index_params, } struct DropEdgeIndexReq { diff --git a/src/meta/processors/index/CreateEdgeIndexProcessor.cpp b/src/meta/processors/index/CreateEdgeIndexProcessor.cpp index e3de55f3445..b8a3451d57c 100644 --- a/src/meta/processors/index/CreateEdgeIndexProcessor.cpp +++ b/src/meta/processors/index/CreateEdgeIndexProcessor.cpp @@ -190,10 +190,12 @@ void CreateEdgeIndexProcessor::process(const cpp2::CreateEdgeIndexReq& req) { item.set_schema_id(schemaID); item.set_schema_name(edgeName); item.set_fields(std::move(columns)); + if (req.index_params_ref().has_value()) { + item.set_index_params(*req.index_params_ref()); + } if (req.comment_ref().has_value()) { item.set_comment(*req.comment_ref()); } - data.emplace_back(MetaKeyUtils::indexIndexKey(space, indexName), std::string(reinterpret_cast(&edgeIndex), sizeof(IndexID))); data.emplace_back(MetaKeyUtils::indexKey(space, edgeIndex), MetaKeyUtils::indexVal(item)); diff --git a/src/meta/processors/index/CreateTagIndexProcessor.cpp b/src/meta/processors/index/CreateTagIndexProcessor.cpp index 366a02243d6..6c094d0cfff 100644 --- a/src/meta/processors/index/CreateTagIndexProcessor.cpp +++ b/src/meta/processors/index/CreateTagIndexProcessor.cpp @@ -188,6 +188,9 @@ void CreateTagIndexProcessor::process(const cpp2::CreateTagIndexReq& req) { item.set_schema_id(schemaID); item.set_schema_name(tagName); item.set_fields(std::move(columns)); + if (req.index_params_ref().has_value()) { + item.set_index_params(*req.index_params_ref()); + } if (req.comment_ref().has_value()) { item.set_comment(*req.comment_ref()); } diff --git a/src/parser/MaintainSentences.cpp b/src/parser/MaintainSentences.cpp index b9ced49661c..ac70707fdf9 100644 --- a/src/parser/MaintainSentences.cpp +++ b/src/parser/MaintainSentences.cpp @@ -232,6 +232,31 @@ std::string DropEdgeSentence::toString() const { return folly::stringPrintf("DROP EDGE %s", name_.get()->c_str()); } +std::string IndexParamItem::toString() const { + switch (paramType_) { + case S2_MAX_LEVEL: + return folly::stringPrintf("s2_max_level = %ld", paramValue_.getInt()); + case S2_MAX_CELLS: + return folly::stringPrintf("s2_max_cells = \"%ld\"", paramValue_.getInt()); + } + DLOG(FATAL) << "Index param type illegal"; + return "Unknown"; +} + +std::string IndexParamList::toString() const { + std::string buf; + buf.reserve(256); + for (auto& item : items_) { + buf += " "; + buf += item->toString(); + buf += ","; + } + if (!buf.empty()) { + buf.resize(buf.size() - 1); + } + return buf; +} + std::string CreateTagIndexSentence::toString() const { std::string buf; buf.reserve(256); @@ -255,8 +280,17 @@ std::string CreateTagIndexSentence::toString() const { folly::join(", ", fieldDefs, fields); buf += fields; buf += ")"; + std::string params; + if (indexParams_ != nullptr) { + params = indexParams_->toString(); + } + if (!params.empty()) { + buf += "WITH ("; + buf += params; + buf += ")"; + } if (comment_ != nullptr) { - buf += "COMMENT = "; + buf += "COMMENT "; buf += *comment_; } return buf; @@ -285,8 +319,17 @@ std::string CreateEdgeIndexSentence::toString() const { folly::join(", ", fieldDefs, fields); buf += fields; buf += ")"; + std::string params; + if (indexParams_ != nullptr) { + params = indexParams_->toString(); + } + if (!params.empty()) { + buf += "WITH ("; + buf += params; + buf += ")"; + } if (comment_ != nullptr) { - buf += "COMMENT = "; + buf += "COMMENT "; buf += *comment_; } return buf; diff --git a/src/parser/MaintainSentences.h b/src/parser/MaintainSentences.h index ee7d6dde864..6b620d9a6ed 100644 --- a/src/parser/MaintainSentences.h +++ b/src/parser/MaintainSentences.h @@ -490,12 +490,65 @@ class IndexFieldList final { std::vector> fields_; }; +class IndexParamItem final { + public: + enum ParamType : uint8_t { S2_MAX_LEVEL, S2_MAX_CELLS }; + + IndexParamItem(ParamType op, Value val) { + paramType_ = op; + paramValue_ = val; + } + + ParamType getParamType() { return paramType_; } + + StatusOr getS2MaxLevel() { + if (paramType_ == S2_MAX_LEVEL) { + return paramValue_.getInt(); + } else { + return Status::Error("Not exists s2_max_level."); + } + } + + StatusOr getS2MaxCells() { + if (paramType_ == S2_MAX_CELLS) { + return paramValue_.getInt(); + } else { + return Status::Error("Not exists s2_max_cells."); + } + } + + std::string toString() const; + + private: + ParamType paramType_; + Value paramValue_; +}; + +class IndexParamList final { + public: + void add(IndexParamItem *item) { items_.emplace_back(item); } + + std::vector getParams() const { + std::vector result; + result.resize(items_.size()); + auto get = [](auto &ptr) { return ptr.get(); }; + std::transform(items_.begin(), items_.end(), result.begin(), get); + return result; + } + + std::string toString() const; + + private: + std::vector> items_; +}; + class CreateTagIndexSentence final : public CreateSentence { public: CreateTagIndexSentence(std::string *indexName, std::string *tagName, IndexFieldList *fields, bool ifNotExists, + IndexParamList *indexParams, std::string *comment) : CreateSentence(ifNotExists) { indexName_.reset(indexName); @@ -505,6 +558,7 @@ class CreateTagIndexSentence final : public CreateSentence { } else { fields_.reset(fields); } + indexParams_.reset(indexParams); comment_.reset(comment); kind_ = Kind::kCreateTagIndex; } @@ -524,12 +578,15 @@ class CreateTagIndexSentence final : public CreateSentence { return result; } + const IndexParamList *getIndexParamList() const { return indexParams_.get(); } + const std::string *comment() const { return comment_.get(); } private: std::unique_ptr indexName_; std::unique_ptr tagName_; std::unique_ptr fields_; + std::unique_ptr indexParams_; std::unique_ptr comment_; }; @@ -539,6 +596,7 @@ class CreateEdgeIndexSentence final : public CreateSentence { std::string *edgeName, IndexFieldList *fields, bool ifNotExists, + IndexParamList *indexParams, std::string *comment) : CreateSentence(ifNotExists) { indexName_.reset(indexName); @@ -548,6 +606,7 @@ class CreateEdgeIndexSentence final : public CreateSentence { } else { fields_.reset(fields); } + indexParams_.reset(indexParams); comment_.reset(comment); kind_ = Kind::kCreateEdgeIndex; } @@ -567,12 +626,15 @@ class CreateEdgeIndexSentence final : public CreateSentence { return result; } + const IndexParamList *getIndexParamList() const { return indexParams_.get(); } + const std::string *comment() const { return comment_.get(); } private: std::unique_ptr indexName_; std::unique_ptr edgeName_; std::unique_ptr fields_; + std::unique_ptr indexParams_; std::unique_ptr comment_; }; diff --git a/src/parser/parser.yy b/src/parser/parser.yy index cfe9c72b456..9026f103761 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -109,6 +109,8 @@ static constexpr size_t kCommentLengthLimit = 256; nebula::SchemaPropItem *create_schema_prop_item; nebula::SchemaPropList *alter_schema_prop_list; nebula::SchemaPropItem *alter_schema_prop_item; + nebula::IndexParamList *index_param_list; + nebula::IndexParamItem *index_param_item; nebula::OrderFactor *order_factor; nebula::OrderFactors *order_factors; nebula::meta::cpp2::ConfigModule config_module; @@ -169,7 +171,7 @@ static constexpr size_t kCommentLengthLimit = 256; %token KW_NO KW_OVERWRITE KW_IN KW_DESCRIBE KW_DESC KW_SHOW KW_HOST KW_HOSTS KW_PART KW_PARTS KW_ADD %token KW_PARTITION_NUM KW_REPLICA_FACTOR KW_CHARSET KW_COLLATE KW_COLLATION KW_VID_TYPE %token KW_ATOMIC_EDGE -%token KW_COMMENT +%token KW_COMMENT KW_S2_MAX_LEVEL KW_S2_MAX_CELLS %token KW_DROP KW_REMOVE KW_SPACES KW_INGEST KW_INDEX KW_INDEXES %token KW_IF KW_NOT KW_EXISTS KW_WITH %token KW_BY KW_DOWNLOAD KW_HDFS KW_UUID KW_CONFIGS KW_FORCE @@ -279,6 +281,8 @@ static constexpr size_t kCommentLengthLimit = 256; %type create_schema_prop_item %type alter_schema_prop_list %type alter_schema_prop_item +%type opt_with_index_param_list index_param_list +%type index_param_item %type order_factor %type order_factors %type config_module_enum @@ -329,7 +333,7 @@ static constexpr size_t kCommentLengthLimit = 256; %type legal_integer unary_integer rank port job_concurrency -%type opt_comment_prop_assignment comment_prop_assignment comment_prop +%type comment_prop_assignment comment_prop opt_comment_prop %type column_property %type column_properties %type column_spec @@ -525,6 +529,8 @@ unreserved_keyword | KW_RESET { $$ = new std::string("reset"); } | KW_PLAN { $$ = new std::string("plan"); } | KW_COMMENT { $$ = new std::string("comment"); } + | KW_S2_MAX_LEVEL { $$ = new std::string("s2_max_level"); } + | KW_S2_MAX_CELLS { $$ = new std::string("s2_max_cells"); } | KW_SESSION { $$ = new std::string("session"); } | KW_SESSIONS { $$ = new std::string("sessions"); } | KW_SAMPLE { $$ = new std::string("sample"); } @@ -2620,14 +2626,14 @@ opt_index_field_list ; create_tag_index_sentence - : KW_CREATE KW_TAG KW_INDEX opt_if_not_exists name_label KW_ON name_label L_PAREN opt_index_field_list R_PAREN opt_comment_prop_assignment { - $$ = new CreateTagIndexSentence($5, $7, $9, $4, $11); + : KW_CREATE KW_TAG KW_INDEX opt_if_not_exists name_label KW_ON name_label L_PAREN opt_index_field_list R_PAREN opt_with_index_param_list opt_comment_prop { + $$ = new CreateTagIndexSentence($5, $7, $9, $4, $11, $12); } ; create_edge_index_sentence - : KW_CREATE KW_EDGE KW_INDEX opt_if_not_exists name_label KW_ON name_label L_PAREN opt_index_field_list R_PAREN opt_comment_prop_assignment { - $$ = new CreateEdgeIndexSentence($5, $7, $9, $4, $11); + : KW_CREATE KW_EDGE KW_INDEX opt_if_not_exists name_label KW_ON name_label L_PAREN opt_index_field_list R_PAREN opt_with_index_param_list opt_comment_prop { + $$ = new CreateEdgeIndexSentence($5, $7, $9, $4, $11, $12); } ; @@ -2646,15 +2652,6 @@ drop_fulltext_index_sentence } ; -opt_comment_prop_assignment - : %empty { - $$ = nullptr; - } - | comment_prop_assignment { - $$ = $1; - } - ; - comment_prop_assignment : KW_COMMENT ASSIGN STRING { if ($3->size() > kCommentLengthLimit) { @@ -2683,6 +2680,51 @@ comment_prop } ; +opt_comment_prop + : %empty { + $$ = nullptr; + } + | comment_prop { + $$ = $1; + } + ; + +opt_with_index_param_list + : %empty { + $$ = nullptr; + } + | KW_WITH L_PAREN index_param_list R_PAREN { + $$ = $3; + } + ; + +index_param_list + : index_param_item { + $$ = new IndexParamList(); + $$->add($1); + } + | index_param_list COMMA index_param_item { + $$ = $1; + $$->add($3); + } + ; + +index_param_item + : KW_S2_MAX_LEVEL ASSIGN legal_integer { + if ($3 < 0 || $3 > 30) { + throw nebula::GraphParser::syntax_error(@3, "'s2_max_level' value must be between 0 and 30 inclusive"); + } + $$ = new IndexParamItem(IndexParamItem::S2_MAX_LEVEL, $3); + } + | KW_S2_MAX_CELLS ASSIGN legal_integer { + if ($3 < 1 || $3 > 32) { + throw nebula::GraphParser::syntax_error(@3, "'s2_max_cells' value must be between 1 and 32 inclusive"); + } + $$ = new IndexParamItem(IndexParamItem::S2_MAX_CELLS, $3); + } + ; + + drop_tag_index_sentence : KW_DROP KW_TAG KW_INDEX opt_if_exists name_label { $$ = new DropTagIndexSentence($5, $4); diff --git a/src/parser/scanner.lex b/src/parser/scanner.lex index fcd086be140..1a34292a08d 100644 --- a/src/parser/scanner.lex +++ b/src/parser/scanner.lex @@ -267,6 +267,8 @@ LABEL_FULL_WIDTH {CN_EN_FULL_WIDTH}{CN_EN_NUM_FULL_WIDTH}* "RESET" { return TokenType::KW_RESET; } "PLAN" { return TokenType::KW_PLAN; } "COMMENT" { return TokenType::KW_COMMENT; } +"S2_MAX_LEVEL" { return TokenType::KW_S2_MAX_LEVEL; } +"S2_MAX_CELLS" { return TokenType::KW_S2_MAX_CELLS; } "SESSIONS" { return TokenType::KW_SESSIONS; } "SESSION" { return TokenType::KW_SESSION; } "SAMPLE" { return TokenType::KW_SAMPLE; } diff --git a/src/storage/admin/RebuildEdgeIndexTask.cpp b/src/storage/admin/RebuildEdgeIndexTask.cpp index 0bb20fad91b..d778af46644 100644 --- a/src/storage/admin/RebuildEdgeIndexTask.cpp +++ b/src/storage/admin/RebuildEdgeIndexTask.cpp @@ -136,7 +136,7 @@ nebula::cpp2::ErrorCode RebuildEdgeIndexTask::buildIndexGlobal(GraphSpaceID spac for (const auto& item : items) { if (item->get_schema_id().get_edge_type() == edgeType) { - auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), item->get_fields()); + auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), item.get()); if (!valuesRet.ok()) { LOG(WARNING) << "Collect index value failed"; continue; diff --git a/src/storage/admin/RebuildTagIndexTask.cpp b/src/storage/admin/RebuildTagIndexTask.cpp index ff3336269a9..852592854d0 100644 --- a/src/storage/admin/RebuildTagIndexTask.cpp +++ b/src/storage/admin/RebuildTagIndexTask.cpp @@ -127,7 +127,7 @@ nebula::cpp2::ErrorCode RebuildTagIndexTask::buildIndexGlobal(GraphSpaceID space for (const auto& item : items) { if (item->get_schema_id().get_tag_id() == tagID) { - auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), item->get_fields()); + auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), item.get()); if (!valuesRet.ok()) { LOG(WARNING) << "Collect index value failed"; continue; diff --git a/src/storage/exec/UpdateNode.h b/src/storage/exec/UpdateNode.h index 261cce227a0..9527f0288cd 100644 --- a/src/storage/exec/UpdateNode.h +++ b/src/storage/exec/UpdateNode.h @@ -421,7 +421,7 @@ class UpdateTagNode : public UpdateNode { const VertexID& vId, RowReader* reader, std::shared_ptr index) { - auto values = IndexKeyUtils::collectIndexValues(reader, index->get_fields()); + auto values = IndexKeyUtils::collectIndexValues(reader, index.get()); if (!values.ok()) { return {}; } @@ -752,7 +752,7 @@ class UpdateEdgeNode : public UpdateNode { RowReader* reader, const cpp2::EdgeKey& edgeKey, std::shared_ptr index) { - auto values = IndexKeyUtils::collectIndexValues(reader, index->get_fields()); + auto values = IndexKeyUtils::collectIndexValues(reader, index.get()); if (!values.ok()) { return {}; } diff --git a/src/storage/mutate/AddEdgesProcessor.cpp b/src/storage/mutate/AddEdgesProcessor.cpp index 7743309b1a4..ad28cdc76bb 100644 --- a/src/storage/mutate/AddEdgesProcessor.cpp +++ b/src/storage/mutate/AddEdgesProcessor.cpp @@ -470,7 +470,7 @@ std::vector AddEdgesProcessor::indexKeys( RowReader* reader, const folly::StringPiece& rawKey, std::shared_ptr index) { - auto values = IndexKeyUtils::collectIndexValues(reader, index->get_fields()); + auto values = IndexKeyUtils::collectIndexValues(reader, index.get()); if (!values.ok()) { return {}; } diff --git a/src/storage/mutate/AddVerticesProcessor.cpp b/src/storage/mutate/AddVerticesProcessor.cpp index e6dcc9fa71a..a3a5fe810d5 100644 --- a/src/storage/mutate/AddVerticesProcessor.cpp +++ b/src/storage/mutate/AddVerticesProcessor.cpp @@ -331,7 +331,7 @@ std::vector AddVerticesProcessor::indexKeys( const VertexID& vId, RowReader* reader, std::shared_ptr index) { - auto values = IndexKeyUtils::collectIndexValues(reader, index->get_fields()); + auto values = IndexKeyUtils::collectIndexValues(reader, index.get()); if (!values.ok()) { return {}; } diff --git a/src/storage/mutate/DeleteEdgesProcessor.cpp b/src/storage/mutate/DeleteEdgesProcessor.cpp index 0c4964bf71b..47d63d24db3 100644 --- a/src/storage/mutate/DeleteEdgesProcessor.cpp +++ b/src/storage/mutate/DeleteEdgesProcessor.cpp @@ -161,7 +161,7 @@ ErrorOr DeleteEdgesProcessor::deleteEdges( return nebula::cpp2::ErrorCode::E_INVALID_DATA; } } - auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), index->get_fields()); + auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), index.get()); if (!valuesRet.ok()) { continue; } diff --git a/src/storage/mutate/DeleteTagsProcessor.cpp b/src/storage/mutate/DeleteTagsProcessor.cpp index af414c55ba4..3e48ae7e542 100644 --- a/src/storage/mutate/DeleteTagsProcessor.cpp +++ b/src/storage/mutate/DeleteTagsProcessor.cpp @@ -122,8 +122,7 @@ ErrorOr DeleteTagsProcessor::deleteTags( if (index->get_schema_id().get_tag_id() == tagId) { auto indexId = index->get_index_id(); - const auto& cols = index->get_fields(); - auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), cols); + auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), index.get()); if (!valuesRet.ok()) { return nebula::cpp2::ErrorCode::E_INVALID_DATA; } diff --git a/src/storage/mutate/DeleteVerticesProcessor.cpp b/src/storage/mutate/DeleteVerticesProcessor.cpp index 570cbb0672a..8e3ffc91a24 100644 --- a/src/storage/mutate/DeleteVerticesProcessor.cpp +++ b/src/storage/mutate/DeleteVerticesProcessor.cpp @@ -145,8 +145,7 @@ ErrorOr DeleteVerticesProcessor::deleteVer return nebula::cpp2::ErrorCode::E_INVALID_DATA; } } - const auto& cols = index->get_fields(); - auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), cols); + auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), index.get()); if (!valuesRet.ok()) { continue; } diff --git a/src/storage/test/IndexScanLimitTest.cpp b/src/storage/test/IndexScanLimitTest.cpp index 83e678c0d4c..f4988f494a7 100644 --- a/src/storage/test/IndexScanLimitTest.cpp +++ b/src/storage/test/IndexScanLimitTest.cpp @@ -122,26 +122,28 @@ class IndexScanLimitTest : public ::testing::Test { data.emplace_back(std::move(edgeKey), val); data.emplace_back(std::move(tagKey), std::move(val)); if (indexMan_ != nullptr) { + auto indexItem = std::make_unique(); + indexItem->set_fields(genCols()); if (indexMan_->getTagIndex(spaceId, tagIndex).ok()) { - auto vertexIndexKeys = - IndexKeyUtils::vertexIndexKeys(vertexLen, - pId, - tagIndex, - vertex, - IndexKeyUtils::encodeValues({col1Val}, genCols())); + auto vertexIndexKeys = IndexKeyUtils::vertexIndexKeys( + vertexLen, + pId, + tagIndex, + vertex, + IndexKeyUtils::encodeValues({col1Val}, indexItem.get())); for (auto& vertexIndexKey : vertexIndexKeys) { data.emplace_back(std::move(vertexIndexKey), ""); } } if (indexMan_->getEdgeIndex(spaceId, edgeIndex).ok()) { - auto edgeIndexKeys = - IndexKeyUtils::edgeIndexKeys(vertexLen, - pId, - edgeIndex, - vertex, - 0, - vertex, - IndexKeyUtils::encodeValues({col1Val}, genCols())); + auto edgeIndexKeys = IndexKeyUtils::edgeIndexKeys( + vertexLen, + pId, + edgeIndex, + vertex, + 0, + vertex, + IndexKeyUtils::encodeValues({col1Val}, indexItem.get())); for (auto& edgeIndexKey : edgeIndexKeys) { data.emplace_back(std::move(edgeIndexKey), ""); } diff --git a/src/storage/test/IndexTest.cpp b/src/storage/test/IndexTest.cpp index 2b269da938f..f0b7147e06a 100644 --- a/src/storage/test/IndexTest.cpp +++ b/src/storage/test/IndexTest.cpp @@ -3,7 +3,9 @@ * This source code is licensed under Apache 2.0 License. */ +#include #include +#include #include #include @@ -14,6 +16,8 @@ #include "common/expression/ConstantExpression.h" #include "common/expression/PropertyExpression.h" #include "common/expression/RelationalExpression.h" +#include "common/geo/GeoIndex.h" +#include "common/utils/IndexKeyUtils.h" #include "common/utils/NebulaKeyUtils.h" #include "kvstore/KVEngine.h" #include "kvstore/KVIterator.h" @@ -38,7 +42,7 @@ using std::string_literals::operator""s; * 1. Vertex/Edge * 2. back to table or not * 3. different value type - * a. int/float/bool/fix_string/time/date/datetime + * a. int/float/bool/fix_string/time/date/datetime/geography * b. compound index * 4. range/prefix * a. prefix(equal) @@ -163,10 +167,12 @@ class IndexScanTest : public ::testing::Test { RowReaderWrapper reader(schema.get(), folly::StringPiece(value), schemaVer); for (size_t j = 0; j < indices.size(); j++) { auto& index = indices[j]; - auto indexValue = IndexKeyUtils::collectIndexValues(&reader, index->get_fields()).value(); - auto indexKey = IndexKeyUtils::vertexIndexKeys( - 8, 0, index->get_index_id(), std::to_string(i), std::move(indexValue))[0]; - CHECK(ret[j + 1].insert({indexKey, ""}).second); + auto indexValue = IndexKeyUtils::collectIndexValues(&reader, index.get()).value(); + auto indexKeys = IndexKeyUtils::vertexIndexKeys( + 8, 0, index->get_index_id(), std::to_string(i), std::move(indexValue)); + for (auto& indexKey : indexKeys) { + CHECK(ret[j + 1].insert({indexKey, ""}).second); + } } } return ret; @@ -189,15 +195,17 @@ class IndexScanTest : public ::testing::Test { RowReaderWrapper reader(schema.get(), folly::StringPiece(value), schemaVer); for (size_t j = 0; j < indices.size(); j++) { auto& index = indices[j]; - auto indexValue = IndexKeyUtils::collectIndexValues(&reader, index->get_fields()).value(); - auto indexKey = IndexKeyUtils::edgeIndexKeys(8, - 0, - index->get_index_id(), - std::to_string(i), - i, - std::to_string(i), - std::move(indexValue))[0]; - CHECK(ret[j + 1].insert({indexKey, ""}).second); + auto indexValue = IndexKeyUtils::collectIndexValues(&reader, index.get()).value(); + auto indexKeys = IndexKeyUtils::edgeIndexKeys(8, + 0, + index->get_index_id(), + std::to_string(i), + i, + std::to_string(i), + std::move(indexValue)); + for (auto& indexKey : indexKeys) { + CHECK(ret[j + 1].insert({indexKey, ""}).second); + } } } return ret; @@ -1730,6 +1738,131 @@ TEST_F(IndexScanTest, Date) { TEST_F(IndexScanTest, DateTime) { // TODO(hs.zhang): add unittest } + +TEST_F(IndexScanTest, Geography) { + auto rows = R"( + geography + POINT(108.1 32.5) + POINT(103.8 32.3) + LINESTRING(68.9 48.9,76.1 35.5,125.7 28.2) + POLYGON((91.2 38.6,99.7 41.9,111.2 38.9,115.6 33.2,109.5 29.0,105.8 24.1,102.9 30.5,93.0 28.1,95.4 32.8,86.1 33.6,85.3 38.8,91.2 38.6)) + POLYGON((88.9 42.2,92.5 39.5,104.2 39.6,115.5 36.5,114.4 24.4,98.9 20.3,109.5 31.3,91.4 25.6,95.4 33.1,88.9 42.2)) + )"_row; + /* Format: WKT:[CellID...]. The expected result comes from BigQuery. + POINT(108.1 32.5): [0x368981adc0392fe3] + POINT(103.8 32.3): [0x36f0b347378c3683] + LINESTRING(68.9 48.9,76.1 35.5,125.7 28.2): + [0x3440000000000000,0x34ff000000000000,0x3640000000000000,0x3684000000000000, + 0x37c0000000000000,0x3840000000000000,0x38c0000000000000,0x4240000000000000] + POLYGON((91.2 38.6,99.7 41.9,111.2 38.9,115.6 33.2,109.5 29.0, + 105.8 24.1,102.9 30.5,93.0 28.1,95.4 32.8,86.1 33.6,85.3 38.8,91.2 38.6)): + [0x342b000000000000,0x35d0000000000000,0x3700000000000000,0x3830000000000000, + 0x39d0000000000000] + POLYGON((88.9 42.2,92.5 39.5,104.2 39.6,115.5 36.5,114.4 24.4,98.9 20.3, + 109.5 31.3,91.4 25.6,95.4 33.1,88.9 42.2)): + [0x30d4000000000000,0x3130000000000000,0x341c000000000000,0x3430000000000000, + 0x35d4000000000000,0x35dc000000000000,0x3700000000000000,0x381c000000000000] + */ + auto schema = R"( + geo | geography | | false + )"_schema; + auto indices = R"( + TAG(t,1) + (i1,2):geo + )"_index(schema); + auto kv = encodeTag(rows, 1, schema, indices); + auto kvstore = std::make_unique(); + for (auto& iter : kv) { + for (auto& item : iter) { + kvstore->put(item.first, item.second); + } + } + auto actual = [&](std::shared_ptr index, + const std::vector& columnHints) -> auto { + auto context = makeContext(1, 0); + auto scanNode = + std::make_unique(context.get(), 0, columnHints, kvstore.get()); + IndexScanTestHelper helper; + helper.setIndex(scanNode.get(), index); + helper.setTag(scanNode.get(), schema); + InitContext initCtx; + initCtx.requiredColumns = {kVid}; + scanNode->init(initCtx); + scanNode->execute(0); + + std::vector result; + while (true) { + auto res = scanNode->next(); + ASSERT(res.success()); + if (!res.hasData()) { + break; + } + result.emplace_back(std::move(res).row()); + } + return result; + }; + auto expect = [](auto... vidList) { + std::vector ret; + std::vector value; + (value.push_back(std::to_string(vidList)), ...); + for (auto& v : value) { + Row row; + row.emplace_back(v); + ret.emplace_back(std::move(row)); + } + return ret; + }; + auto encodeCellId = [](int64_t i) -> std::string { + // First, reinterpret the int64_t as uint64_t + uint64_t u = static_cast(i); + // Then, encode the uint64_t as string + return IndexKeyUtils::encodeUint64(u); + }; + // For the Geography type, there are only two cases: prefix and [x, y]. + /* Case 1: Prefix */ + // Explicitly specify the index column hints + { + std::vector columnHints; + columnHints = {makeColumnHint("geo", encodeCellId(0x368981adc0392fe3))}; + EXPECT_EQ(expect(0), actual(indices[0], columnHints)); + columnHints = {makeColumnHint("geo", encodeCellId(0x36f0b347378c3683))}; + EXPECT_EQ(expect(1), actual(indices[0], columnHints)); + columnHints = {makeColumnHint("geo", encodeCellId(0x381c000000000000))}; + EXPECT_EQ(expect(4), actual(indices[0], columnHints)); + columnHints = {makeColumnHint("geo", encodeCellId(0x3440000000000000))}; + EXPECT_EQ(expect(2), actual(indices[0], columnHints)); + columnHints = {makeColumnHint("geo", encodeCellId(0x4240000000000000))}; + EXPECT_EQ(expect(2), actual(indices[0], columnHints)); + columnHints = {makeColumnHint("geo", encodeCellId(0x342b000000000000))}; + EXPECT_EQ(expect(3), actual(indices[0], columnHints)); + columnHints = {makeColumnHint("geo", encodeCellId(0x3700000000000000))}; + EXPECT_EQ(expect(3, 4), actual(indices[0], columnHints)); + } + // Let GeoIndex generates the index column hints + { + // TODO(jie) + } /* Case 2: [x, y] */ + // Explicitly specify the index column hints + { + auto hint = [&encodeCellId](const char* name, int64_t begin, int64_t end) { + return std::vector{ + makeColumnHint(name, encodeCellId(begin), encodeCellId(end))}; + }; + auto columnHint = hint("geo", 0x36f0b347378c3683, 0x36f0b347378c3683); + EXPECT_EQ(expect(1), actual(indices[0], columnHint)); + columnHint = hint("geo", 0x35d0000000000000, 0x35dfffffffffffff); + EXPECT_EQ(expect(3, 4), actual(indices[0], columnHint)); + columnHint = hint("geo", 0x3600000000000000, 0x38ffffffffffffff); + EXPECT_EQ(expect(2, 0, 1, 3, 4), actual(indices[0], columnHint)); + columnHint = hint("geo", 0x36f0b00000000000, 0x3700000000000000); + EXPECT_EQ(expect(1, 3, 4), actual(indices[0], columnHint)); + } + // Let GeoIndex generates the index column hints + { + // TODO(jie) + } +} + TEST_F(IndexScanTest, Compound) { // TODO(hs.zhang): add unittest } diff --git a/src/storage/test/IndexTestUtil.h b/src/storage/test/IndexTestUtil.h index 73e457e6167..f5022ce3dd9 100644 --- a/src/storage/test/IndexTestUtil.h +++ b/src/storage/test/IndexTestUtil.h @@ -15,6 +15,7 @@ #include "common/meta/NebulaSchemaProvider.h" #include "folly/Conv.h" #include "folly/String.h" +#include "interface/gen-cpp2/common_types.h" #include "kvstore/KVIterator.h" #include "kvstore/KVStore.h" #include "storage/exec/IndexNode.h" @@ -436,7 +437,8 @@ class RowParser { {"int", [](const std::string& str) { return Value(std::stol(str)); }}, {"string", [](const std::string& str) { return Value(str); }}, {"float", [](const std::string& str) { return Value(folly::to(str)); }}, - {"bool", [](const std::string& str) { return Value(str == "true" ? true : false); }}}; + {"bool", [](const std::string& str) { return Value(str == "true" ? true : false); }}, + {"geography", [](const std::string& str) { return Value(Geography::fromWKT(str).value()); }}}; }; /** @@ -491,7 +493,8 @@ class SchemaParser { {"int", ::nebula::cpp2::PropertyType::INT64}, {"double", ::nebula::cpp2::PropertyType::DOUBLE}, {"string", ::nebula::cpp2::PropertyType::STRING}, - {"bool", ::nebula::cpp2::PropertyType::BOOL}}; + {"bool", ::nebula::cpp2::PropertyType::BOOL}, + {"geography", ::nebula::cpp2::PropertyType::GEOGRAPHY}}; }; /** diff --git a/src/storage/test/IndexWriteTest.cpp b/src/storage/test/IndexWriteTest.cpp index 91f30a0f53d..23a92c4e5c6 100644 --- a/src/storage/test/IndexWriteTest.cpp +++ b/src/storage/test/IndexWriteTest.cpp @@ -318,8 +318,9 @@ TEST(IndexTest, VerticesValueTest) { values.emplace_back(Value(date)); // col_date_null values.emplace_back(nullValue); - auto indexes = - IndexKeyUtils::encodeValues(std::move(values), mock::MockData::mockTypicaIndexColumns()); + auto indexItem = std::make_unique(); + indexItem->set_fields(mock::MockData::mockTypicaIndexColumns()); + auto indexes = IndexKeyUtils::encodeValues(std::move(values), indexItem.get()); for (auto partId = 1; partId <= 6; partId++) { auto prefix = IndexKeyUtils::indexPrefix(partId, indexId); diff --git a/src/tools/db-upgrade/DbUpgrader.cpp b/src/tools/db-upgrade/DbUpgrader.cpp index 40c638a6354..e5a6e3f6ec1 100644 --- a/src/tools/db-upgrade/DbUpgrader.cpp +++ b/src/tools/db-upgrade/DbUpgrader.cpp @@ -887,7 +887,7 @@ std::vector UpgraderSpace::indexVertexKeys( VertexID& vId, RowReader* reader, std::shared_ptr index) { - auto values = IndexKeyUtils::collectIndexValues(reader, index->get_fields()); + auto values = IndexKeyUtils::collectIndexValues(reader, index.get()); if (!values.ok()) { return {}; } @@ -943,7 +943,7 @@ std::vector UpgraderSpace::indexEdgeKeys( EdgeRanking rank, VertexID& dstId, std::shared_ptr index) { - auto values = IndexKeyUtils::collectIndexValues(reader, index->get_fields()); + auto values = IndexKeyUtils::collectIndexValues(reader, index.get()); if (!values.ok()) { return {}; } diff --git a/tests/tck/features/geo/GeoBase.feature b/tests/tck/features/geo/GeoBase.feature index d321432254e..a0a24e2203d 100644 --- a/tests/tck/features/geo/GeoBase.feature +++ b/tests/tck/features/geo/GeoBase.feature @@ -220,17 +220,17 @@ Feature: Geo base # Create index on geo column When executing query: """ - CREATE TAG INDEX any_shape_geo_index ON any_shape(geo); + CREATE TAG INDEX any_shape_geo_index ON any_shape(geo) with (s2_max_level=30, s2_max_cells=8) comment "test"; """ Then the execution should be successful When executing query: """ - CREATE TAG INDEX only_point_geo_index ON only_point(geo); + CREATE TAG INDEX only_point_geo_index ON only_point(geo) comment "test2"; """ Then the execution should be successful When executing query: """ - CREATE TAG INDEX only_linestring_geo_index ON only_linestring(geo); + CREATE TAG INDEX only_linestring_geo_index ON only_linestring(geo) with (s2_max_cells=12) comment "test3"; """ Then the execution should be successful When executing query: @@ -240,10 +240,18 @@ Feature: Geo base Then the execution should be successful When executing query: """ - CREATE EDGE INDEX any_shape_edge_geo_index ON any_shape_edge(geo); + CREATE EDGE INDEX any_shape_edge_geo_index ON any_shape_edge(geo) with (s2_max_level=23); """ Then the execution should be successful And wait 3 seconds + # Show create tag index + When executing query: + """ + SHOW CREATE TAG INDEX any_shape_geo_index; + """ + Then the result should be, in any order: + | Tag Index Name | Create Tag Index | + | "any_shape_geo_index" | "CREATE TAG INDEX `any_shape_geo_index` ON `any_shape` (\n `geo`\n) WITH (s2_max_level = 30, s2_max_cells = 8) comment \"test\"" | # Rebuild the geo index When submit a job: """ diff --git a/tests/tck/features/schema/Comment.feature b/tests/tck/features/schema/Comment.feature index 638478bed77..94c4ecc8911 100644 --- a/tests/tck/features/schema/Comment.feature +++ b/tests/tck/features/schema/Comment.feature @@ -148,7 +148,7 @@ Feature: Schema Comment When executing query: """ CREATE tag index test_comment_tag_index ON test_comment_tag(name(8)) - comment = 'The tag index of person name.'; + comment 'The tag index of person name.'; """ Then the execution should be successful When executing query: @@ -156,8 +156,8 @@ Feature: Schema Comment SHOW CREATE TAG INDEX test_comment_tag_index; """ Then the result should be, in any order: - | Tag Index Name | Create Tag Index | - | "test_comment_tag_index" | 'CREATE TAG INDEX `test_comment_tag_index` ON `test_comment_tag` (\n `name`(8)\n) comment = "The tag index of person name."' | + | Tag Index Name | Create Tag Index | + | "test_comment_tag_index" | 'CREATE TAG INDEX `test_comment_tag_index` ON `test_comment_tag` (\n `name`(8)\n) comment "The tag index of person name."' | # edge When executing query: """ @@ -212,7 +212,7 @@ Feature: Schema Comment When executing query: """ CREATE edge index test_comment_edge_index ON test_comment_edge(name(8)) - comment = 'The edge index of person name.'; + comment 'The edge index of person name.'; """ Then the execution should be successful When executing query: @@ -220,8 +220,8 @@ Feature: Schema Comment SHOW CREATE EDGE INDEX test_comment_edge_index; """ Then the result should be, in any order: - | Edge Index Name | Create Edge Index | - | "test_comment_edge_index" | 'CREATE EDGE INDEX `test_comment_edge_index` ON `test_comment_edge` (\n `name`(8)\n) comment = "The edge index of person name."' | + | Edge Index Name | Create Edge Index | + | "test_comment_edge_index" | 'CREATE EDGE INDEX `test_comment_edge_index` ON `test_comment_edge` (\n `name`(8)\n) comment "The edge index of person name."' | Examples: | tag_of_person_comment | tag_of_person_comment_modified | edge_of_person_comment | edge_of_person_comment_modified |