diff --git a/src/common/utils/IndexKeyUtils.cpp b/src/common/utils/IndexKeyUtils.cpp index dd43757fb56..29b34fb7913 100644 --- a/src/common/utils/IndexKeyUtils.cpp +++ b/src/common/utils/IndexKeyUtils.cpp @@ -8,6 +8,7 @@ #include #include "common/geo/GeoIndex.h" +#include "common/utils/DefaultValueContext.h" namespace nebula { @@ -170,18 +171,26 @@ Value IndexKeyUtils::parseIndexTTL(const folly::StringPiece& raw) { // static StatusOr> IndexKeyUtils::collectIndexValues( - RowReader* reader, const meta::cpp2::IndexItem* indexItem) { + RowReader* reader, + const meta::cpp2::IndexItem* indexItem, + const meta::SchemaProviderIf* latestSchema) { if (reader == nullptr) { return Status::Error("Invalid row reader"); } auto& cols = indexItem->get_fields(); std::vector values; for (const auto& col : cols) { - auto v = reader->getValueByName(col.get_name()); + auto propName = col.get_name(); + auto val = readValueWithLatestSche(reader, propName, latestSchema); + if (!val.ok()) { + LOG(ERROR) << "prop error by : " << propName << ". status : " << val.status(); + return val.status(); + } + auto v = val.value(); auto isNullable = col.nullable_ref().value_or(false); auto ret = checkValue(v, isNullable); if (!ret.ok()) { - LOG(ERROR) << "prop error by : " << col.get_name() << ". status : " << ret; + LOG(ERROR) << "prop error by : " << propName << ". status : " << ret; return ret; } values.emplace_back(std::move(v)); @@ -189,6 +198,28 @@ StatusOr> IndexKeyUtils::collectIndexValues( return encodeValues(std::move(values), indexItem); } +// static +StatusOr IndexKeyUtils::readValueWithLatestSche(RowReader* reader, + const std::string propName, + const meta::SchemaProviderIf* latestSchema) { + auto value = reader->getValueByName(propName); + if (latestSchema == nullptr || !value.isNull() || value.getNull() != NullType::UNKNOWN_PROP) { + return value; + } + auto field = latestSchema->field(propName); + if (field == nullptr) { + return Status::Error("Unknown prop"); + } + if (field->hasDefault()) { + DefaultValueContext expCtx; + auto expr = field->defaultValue()->clone(); + return Expression::eval(expr, expCtx); + } else if (field->nullable()) { + return NullType::__NULL__; + } + return Status::Error(folly::stringPrintf("Fail to read prop %s ", propName.c_str())); +} + // static Status IndexKeyUtils::checkValue(const Value& v, bool isNullable) { if (!v.isNull()) { diff --git a/src/common/utils/IndexKeyUtils.h b/src/common/utils/IndexKeyUtils.h index b3dac860ea5..981d4f70581 100644 --- a/src/common/utils/IndexKeyUtils.h +++ b/src/common/utils/IndexKeyUtils.h @@ -545,11 +545,17 @@ class IndexKeyUtils final { static Value parseIndexTTL(const folly::StringPiece& raw); static StatusOr> collectIndexValues( - RowReader* reader, const meta::cpp2::IndexItem* indexItem); + RowReader* reader, + const meta::cpp2::IndexItem* indexItem, + const meta::SchemaProviderIf* latestSchema = nullptr); private: IndexKeyUtils() = delete; + static StatusOr readValueWithLatestSche(RowReader* reader, + const std::string propName, + const meta::SchemaProviderIf* latestSchema); + static Status checkValue(const Value& v, bool isNullable); }; diff --git a/src/storage/admin/RebuildEdgeIndexTask.cpp b/src/storage/admin/RebuildEdgeIndexTask.cpp index d778af46644..687c8691671 100644 --- a/src/storage/admin/RebuildEdgeIndexTask.cpp +++ b/src/storage/admin/RebuildEdgeIndexTask.cpp @@ -136,7 +136,7 @@ nebula::cpp2::ErrorCode RebuildEdgeIndexTask::buildIndexGlobal(GraphSpaceID spac for (const auto& item : items) { if (item->get_schema_id().get_edge_type() == edgeType) { - auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), item.get()); + auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), item.get(), schema); if (!valuesRet.ok()) { LOG(WARNING) << "Collect index value failed"; continue; diff --git a/src/storage/admin/RebuildTagIndexTask.cpp b/src/storage/admin/RebuildTagIndexTask.cpp index 852592854d0..c5af2822adb 100644 --- a/src/storage/admin/RebuildTagIndexTask.cpp +++ b/src/storage/admin/RebuildTagIndexTask.cpp @@ -127,7 +127,7 @@ nebula::cpp2::ErrorCode RebuildTagIndexTask::buildIndexGlobal(GraphSpaceID space for (const auto& item : items) { if (item->get_schema_id().get_tag_id() == tagID) { - auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), item.get()); + auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), item.get(), schema); if (!valuesRet.ok()) { LOG(WARNING) << "Collect index value failed"; continue; diff --git a/src/storage/exec/UpdateNode.h b/src/storage/exec/UpdateNode.h index 9527f0288cd..f000c21e87d 100644 --- a/src/storage/exec/UpdateNode.h +++ b/src/storage/exec/UpdateNode.h @@ -421,7 +421,7 @@ class UpdateTagNode : public UpdateNode { const VertexID& vId, RowReader* reader, std::shared_ptr index) { - auto values = IndexKeyUtils::collectIndexValues(reader, index.get()); + auto values = IndexKeyUtils::collectIndexValues(reader, index.get(), schema_); if (!values.ok()) { return {}; } @@ -752,7 +752,7 @@ class UpdateEdgeNode : public UpdateNode { RowReader* reader, const cpp2::EdgeKey& edgeKey, std::shared_ptr index) { - auto values = IndexKeyUtils::collectIndexValues(reader, index.get()); + auto values = IndexKeyUtils::collectIndexValues(reader, index.get(), schema_); if (!values.ok()) { return {}; } diff --git a/src/storage/mutate/AddEdgesProcessor.cpp b/src/storage/mutate/AddEdgesProcessor.cpp index 0cdac63284c..0c060509fcb 100644 --- a/src/storage/mutate/AddEdgesProcessor.cpp +++ b/src/storage/mutate/AddEdgesProcessor.cpp @@ -252,7 +252,7 @@ void AddEdgesProcessor::doProcessWithIndex(const cpp2::AddEdgesRequest& req) { * step 1 , Delete old version index if exists. */ if (oReader != nullptr) { - auto ois = indexKeys(partId, oReader.get(), key, index); + auto ois = indexKeys(partId, oReader.get(), key, index, schema.get()); if (!ois.empty()) { // Check the index is building for the specified partition or not. auto indexState = env_->getIndexState(spaceId_, partId); @@ -276,7 +276,7 @@ void AddEdgesProcessor::doProcessWithIndex(const cpp2::AddEdgesRequest& req) { * step 2 , Insert new edge index */ if (nReader != nullptr) { - auto niks = indexKeys(partId, nReader.get(), key, index); + auto niks = indexKeys(partId, nReader.get(), key, index, schema.get()); if (!niks.empty()) { auto v = CommonUtils::ttlValue(schema.get(), nReader.get()); auto niv = v.ok() ? IndexKeyUtils::indexVal(std::move(v).value()) : ""; @@ -384,7 +384,7 @@ ErrorOr AddEdgesProcessor::addEdges( } if (!val.empty()) { - auto ois = indexKeys(partId, oReader.get(), e.first, index); + auto ois = indexKeys(partId, oReader.get(), e.first, index, schema.get()); if (!ois.empty()) { // Check the index is building for the specified partition or not. auto indexState = env_->getIndexState(spaceId_, partId); @@ -416,7 +416,7 @@ ErrorOr AddEdgesProcessor::addEdges( } } - auto niks = indexKeys(partId, nReader.get(), e.first, index); + auto niks = indexKeys(partId, nReader.get(), e.first, index, schema.get()); if (!niks.empty()) { auto v = CommonUtils::ttlValue(schema.get(), nReader.get()); auto niv = v.ok() ? IndexKeyUtils::indexVal(std::move(v).value()) : ""; @@ -473,8 +473,9 @@ std::vector AddEdgesProcessor::indexKeys( PartitionID partId, RowReader* reader, const folly::StringPiece& rawKey, - std::shared_ptr index) { - auto values = IndexKeyUtils::collectIndexValues(reader, index.get()); + std::shared_ptr index, + const meta::SchemaProviderIf* latestSchema) { + auto values = IndexKeyUtils::collectIndexValues(reader, index.get(), latestSchema); if (!values.ok()) { return {}; } diff --git a/src/storage/mutate/AddEdgesProcessor.h b/src/storage/mutate/AddEdgesProcessor.h index bacc212bd06..b8f75b6caec 100644 --- a/src/storage/mutate/AddEdgesProcessor.h +++ b/src/storage/mutate/AddEdgesProcessor.h @@ -46,7 +46,8 @@ class AddEdgesProcessor : public BaseProcessor { std::vector indexKeys(PartitionID partId, RowReader* reader, const folly::StringPiece& rawKey, - std::shared_ptr index); + std::shared_ptr index, + const meta::SchemaProviderIf* latestSchema); private: GraphSpaceID spaceId_; diff --git a/src/storage/mutate/AddVerticesProcessor.cpp b/src/storage/mutate/AddVerticesProcessor.cpp index 161dfb5e5e2..86a8ca9530e 100644 --- a/src/storage/mutate/AddVerticesProcessor.cpp +++ b/src/storage/mutate/AddVerticesProcessor.cpp @@ -230,7 +230,7 @@ void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& re * step 1 , Delete old version index if exists. */ if (oReader != nullptr) { - auto ois = indexKeys(partId, vid, oReader.get(), index); + auto ois = indexKeys(partId, vid, oReader.get(), index, schema.get()); if (!ois.empty()) { // Check the index is building for the specified partition or // not. @@ -256,7 +256,7 @@ void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& re * step 2 , Insert new vertex index */ if (nReader != nullptr) { - auto niks = indexKeys(partId, vid, nReader.get(), index); + auto niks = indexKeys(partId, vid, nReader.get(), index, schema.get()); if (!niks.empty()) { auto v = CommonUtils::ttlValue(schema.get(), nReader.get()); auto niv = v.ok() ? IndexKeyUtils::indexVal(std::move(v).value()) : ""; @@ -334,8 +334,9 @@ std::vector AddVerticesProcessor::indexKeys( PartitionID partId, const VertexID& vId, RowReader* reader, - std::shared_ptr index) { - auto values = IndexKeyUtils::collectIndexValues(reader, index.get()); + std::shared_ptr index, + const meta::SchemaProviderIf* latestSchema) { + auto values = IndexKeyUtils::collectIndexValues(reader, index.get(), latestSchema); if (!values.ok()) { return {}; } diff --git a/src/storage/mutate/AddVerticesProcessor.h b/src/storage/mutate/AddVerticesProcessor.h index ea8e34b56e4..fecca60516b 100644 --- a/src/storage/mutate/AddVerticesProcessor.h +++ b/src/storage/mutate/AddVerticesProcessor.h @@ -41,7 +41,8 @@ class AddVerticesProcessor : public BaseProcessor { std::vector indexKeys(PartitionID partId, const VertexID& vId, RowReader* reader, - std::shared_ptr index); + std::shared_ptr index, + const meta::SchemaProviderIf* latestSchema); private: GraphSpaceID spaceId_; diff --git a/src/storage/mutate/DeleteEdgesProcessor.cpp b/src/storage/mutate/DeleteEdgesProcessor.cpp index 6c68f5b9e0a..ac6071436cd 100644 --- a/src/storage/mutate/DeleteEdgesProcessor.cpp +++ b/src/storage/mutate/DeleteEdgesProcessor.cpp @@ -145,6 +145,7 @@ ErrorOr DeleteEdgesProcessor::deleteEdges( auto key = NebulaKeyUtils::edgeKey(spaceVidLen_, partId, srcId, type, rank, dstId); std::string val; auto ret = env_->kvstore_->get(spaceId_, partId, key, &val); + auto schema = env_->schemaMan_->getEdgeSchema(spaceId_, std::abs(type)); if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { /** @@ -162,7 +163,8 @@ ErrorOr DeleteEdgesProcessor::deleteEdges( return nebula::cpp2::ErrorCode::E_INVALID_DATA; } } - auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), index.get()); + auto valuesRet = + IndexKeyUtils::collectIndexValues(reader.get(), index.get(), schema.get()); if (!valuesRet.ok()) { continue; } diff --git a/src/storage/mutate/DeleteVerticesProcessor.cpp b/src/storage/mutate/DeleteVerticesProcessor.cpp index 873f6b8d299..6a1dccd2d3e 100644 --- a/src/storage/mutate/DeleteVerticesProcessor.cpp +++ b/src/storage/mutate/DeleteVerticesProcessor.cpp @@ -134,6 +134,7 @@ ErrorOr DeleteVerticesProcessor::deleteVer } target.emplace_back(std::move(l)); } + auto schema = env_->schemaMan_->getTagSchema(spaceId_, tagId); RowReaderWrapper reader; for (auto& index : indexes_) { if (index->get_schema_id().get_tag_id() == tagId) { @@ -147,7 +148,8 @@ ErrorOr DeleteVerticesProcessor::deleteVer return nebula::cpp2::ErrorCode::E_INVALID_DATA; } } - auto valuesRet = IndexKeyUtils::collectIndexValues(reader.get(), index.get()); + auto valuesRet = + IndexKeyUtils::collectIndexValues(reader.get(), index.get(), schema.get()); if (!valuesRet.ok()) { continue; } diff --git a/src/tools/db-upgrade/DbUpgrader.cpp b/src/tools/db-upgrade/DbUpgrader.cpp index a22d6d62cf6..daedb8451ba 100644 --- a/src/tools/db-upgrade/DbUpgrader.cpp +++ b/src/tools/db-upgrade/DbUpgrader.cpp @@ -660,7 +660,7 @@ void UpgraderSpace::encodeVertexValue(PartitionID partId, return; } for (auto& index : it->second) { - auto newIndexKeys = indexVertexKeys(partId, strVid, nReader.get(), index); + auto newIndexKeys = indexVertexKeys(partId, strVid, nReader.get(), index, schema); for (auto& newIndexKey : newIndexKeys) { data.emplace_back(std::move(newIndexKey), ""); } @@ -997,8 +997,9 @@ std::vector UpgraderSpace::indexVertexKeys( PartitionID partId, VertexID& vId, RowReader* reader, - std::shared_ptr index) { - auto values = IndexKeyUtils::collectIndexValues(reader, index.get()); + std::shared_ptr index, + const meta::SchemaProviderIf* latestSchema) { + auto values = IndexKeyUtils::collectIndexValues(reader, index.get(), latestSchema); if (!values.ok()) { return {}; } @@ -1039,7 +1040,7 @@ void UpgraderSpace::encodeEdgeValue(PartitionID partId, return; } for (auto& index : it->second) { - auto newIndexKeys = indexEdgeKeys(partId, nReader.get(), svId, rank, dstId, index); + auto newIndexKeys = indexEdgeKeys(partId, nReader.get(), svId, rank, dstId, index, schema); for (auto& newIndexKey : newIndexKeys) { data.emplace_back(std::move(newIndexKey), ""); } @@ -1053,8 +1054,9 @@ std::vector UpgraderSpace::indexEdgeKeys( VertexID& svId, EdgeRanking rank, VertexID& dstId, - std::shared_ptr index) { - auto values = IndexKeyUtils::collectIndexValues(reader, index.get()); + std::shared_ptr index, + const meta::SchemaProviderIf* latestSchema) { + auto values = IndexKeyUtils::collectIndexValues(reader, index.get(), latestSchema); if (!values.ok()) { return {}; } diff --git a/src/tools/db-upgrade/DbUpgrader.h b/src/tools/db-upgrade/DbUpgrader.h index 96c77dcacc0..b66543aa882 100644 --- a/src/tools/db-upgrade/DbUpgrader.h +++ b/src/tools/db-upgrade/DbUpgrader.h @@ -87,7 +87,8 @@ class UpgraderSpace { std::vector indexVertexKeys(PartitionID partId, VertexID& vId, RowReader* reader, - std::shared_ptr index); + std::shared_ptr index, + const meta::SchemaProviderIf* latestSchema); void encodeEdgeValue(PartitionID partId, RowReader* reader, @@ -104,7 +105,8 @@ class UpgraderSpace { VertexID& svId, EdgeRanking rank, VertexID& dstId, - std::shared_ptr index); + std::shared_ptr index, + const meta::SchemaProviderIf* latestSchema); WriteResult convertValue(const meta::NebulaSchemaProvider* newSchema, const meta::SchemaProviderIf* oldSchema, diff --git a/tests/tck/features/index/Index.feature b/tests/tck/features/index/Index.feature index bef4eae033e..a1e25e6c0a5 100644 --- a/tests/tck/features/index/Index.feature +++ b/tests/tck/features/index/Index.feature @@ -986,3 +986,154 @@ Feature: IndexTest_Vid_String """ Then the execution should be successful Then drop the used space + + Scenario: IndexTest rebuild tag index with old schema version value + Given an empty graph + And create a space with following options: + | partition_num | 9 | + | replica_factor | 1 | + | vid_type | FIXED_STRING(30) | + | charset | utf8 | + | collate | utf8_bin | + And having executed: + """ + CREATE TAG student(name string, age int); + """ + And wait 6 seconds + When executing query: + """ + INSERT VERTEX + student(name, age) + VALUES + "Alen" : ("Alen", 18), + "Bob" : ("Bob", 28), + "Candy" : ("Candy", 38) + """ + Then the execution should be successful + When executing query: + """ + ALTER TAG student ADD (teacher string) + """ + Then the execution should be successful + When executing query: + """ + ALTER TAG student ADD (alias string default "abc") + """ + Then the execution should be successful + And wait 6 seconds + When executing query: + """ + CREATE TAG INDEX student_name_teacher ON student(name(10), teacher(10)) + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX student_teacher ON student(teacher(10)) + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX student_alias ON student(alias(10)) + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX student_ta ON student(alias(10), teacher(10)) + """ + Then the execution should be successful + And wait 6 seconds + When submit a job: + """ + REBUILD TAG INDEX student_name_teacher + """ + Then wait the job to finish + When submit a job: + """ + REBUILD TAG INDEX student_teacher + """ + Then wait the job to finish + When submit a job: + """ + REBUILD TAG INDEX student_alias + """ + Then wait the job to finish + When submit a job: + """ + REBUILD TAG INDEX student_ta + """ + Then wait the job to finish + When executing query: + """ + LOOKUP ON student WHERE student.alias == "abc" YIELD id(vertex) as id + """ + Then the result should be, in any order: + | id | + | "Alen" | + | "Bob" | + | "Candy" | + When executing query: + """ + LOOKUP ON student WHERE student.name < "a" YIELD id(vertex) as id, student.name as name, student.age as age + """ + Then the result should be, in any order: + | id | name | age | + | "Alen" | "Alen" | 18 | + | "Bob" | "Bob" | 28 | + | "Candy" | "Candy" | 38 | + When executing query: + """ + LOOKUP ON student WHERE student.teacher < "a" YIELD id(vertex) as id + """ + Then the result should be, in any order: + | id | + When executing query: + """ + LOOKUP ON student WHERE student.teacher > "a" YIELD id(vertex) as id + """ + Then the result should be, in any order: + | id | + When executing query: + """ + LOOKUP ON student WHERE student.teacher == "a" YIELD id(vertex) as id + """ + Then the result should be, in any order: + | id | + When executing query: + """ + UPDATE VERTEX ON student "Alen" SET teacher = "Bob" + """ + Then the execution should be successful + When executing query: + """ + INSERT VERTEX + student(age, alias, name, teacher) + VALUES + "Bob" : (28, "abc", "Bob", "Candy") + """ + Then the execution should be successful + When executing query: + """ + LOOKUP ON student WHERE student.teacher < "a" YIELD id(vertex) as id + """ + Then the result should be, in any order: + | id | + | "Alen" | + | "Bob" | + When executing query: + """ + LOOKUP ON student WHERE student.alias == "abc" YIELD id(vertex) as id + """ + Then the result should be, in any order: + | id | + | "Alen" | + | "Bob" | + | "Candy" | + When executing query: + """ + LOOKUP ON student WHERE student.alias < "b" and student.teacher < "abc" YIELD DISTINCT id(vertex) as id, student.teacher, student.alias + """ + Then the result should be, in any order: + | id | student.teacher | student.alias | + | "Alen" | "Bob" | "abc" | + | "Bob" | "Candy" | "abc" | + Then drop the used space