Skip to content
This repository has been archived by the owner on Dec 1, 2022. It is now read-only.

Commit

Permalink
add test cases
Browse files Browse the repository at this point in the history
  • Loading branch information
@bright-starry-sky
bright-starry-sky committed Mar 24, 2021
1 parent e8e329c commit 30cb073
Show file tree
Hide file tree
Showing 6 changed files with 1,090 additions and 18 deletions.
36 changes: 28 additions & 8 deletions src/storage/admin/RebuildEdgeIndexTask.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,33 @@ kvstore::ResultCode RebuildEdgeIndexTask::buildIndexGlobal(GraphSpaceID space,
continue;
}

auto schema = env_->schemaMan_->getEdgeSchema(space, edgeType);
if (!schema) {
LOG(WARNING) << "Space " << space << ", edge " << edgeType << " invalid";
iter->next();
continue;
}

auto ttlProp = CommonUtils::ttlProps(schema.get());
if (ttlProp.first && CommonUtils::checkDataExpiredForTTL(schema.get(),
reader.get(),
ttlProp.second.second,
ttlProp.second.first)) {
VLOG(3) << "ttl expired : "
<< "Source " << source << " Destination " << destination
<< " Ranking " << ranking << " Edge Type " << edgeType;
iter->next();
continue;
}

std::string indexVal = "";
if (ttlProp.first) {
auto ttlValRet = CommonUtils::ttlValue(schema.get(), reader.get());
if (ttlValRet.ok()) {
indexVal = IndexKeyUtils::indexVal(std::move(ttlValRet).value());
}
}

for (const auto& item : items) {
if (item->get_schema_id().get_edge_type() == edgeType) {
auto valuesRet =
Expand All @@ -125,14 +152,7 @@ kvstore::ResultCode RebuildEdgeIndexTask::buildIndexGlobal(GraphSpaceID space,
ranking,
destination.toString(),
std::move(valuesRet).value());
auto schema = env_->schemaMan_->getEdgeSchema(space, edgeType);
if (!schema) {
LOG(WARNING) << "Space " << space << ", edge " << edgeType << " invalid";
continue;
}
auto ttlVal = CommonUtils::ttlValue(schema.get(), reader.get());
auto niv = ttlVal.ok() ? IndexKeyUtils::indexVal(std::move(ttlVal).value()) : "";
data.emplace_back(std::move(indexKey), std::move(niv));
data.emplace_back(std::move(indexKey), indexVal);
}
}
iter->next();
Expand Down
34 changes: 26 additions & 8 deletions src/storage/admin/RebuildTagIndexTask.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,31 @@ kvstore::ResultCode RebuildTagIndexTask::buildIndexGlobal(GraphSpaceID space,
continue;
}

auto schema = env_->schemaMan_->getTagSchema(space, tagID);
if (!schema) {
LOG(WARNING) << "Space " << space << ", tag " << tagID << " invalid";
iter->next();
continue;
}

auto ttlProp = CommonUtils::ttlProps(schema.get());
if (ttlProp.first && CommonUtils::checkDataExpiredForTTL(schema.get(),
reader.get(),
ttlProp.second.second,
ttlProp.second.first)) {
VLOG(3) << "ttl expired : " << "Tag ID " << tagID << " Vertex ID " << vertex;
iter->next();
continue;
}

std::string indexVal = "";
if (ttlProp.first) {
auto ttlValRet = CommonUtils::ttlValue(schema.get(), reader.get());
if (ttlValRet.ok()) {
indexVal = IndexKeyUtils::indexVal(std::move(ttlValRet).value());
}
}

for (const auto& item : items) {
if (item->get_schema_id().get_tag_id() == tagID) {
auto valuesRet =
Expand All @@ -110,14 +135,7 @@ kvstore::ResultCode RebuildTagIndexTask::buildIndexGlobal(GraphSpaceID space,
item->get_index_id(),
vertex.toString(),
std::move(valuesRet).value());
auto schema = env_->schemaMan_->getTagSchema(space, tagID);
if (!schema) {
LOG(WARNING) << "Space " << space << ", tag " << tagID << " invalid";
continue;
}
auto ttlVal = CommonUtils::ttlValue(schema.get(), reader.get());
auto niv = ttlVal.ok() ? IndexKeyUtils::indexVal(std::move(ttlVal).value()) : "";
data.emplace_back(std::move(indexKey), std::move(niv));
data.emplace_back(std::move(indexKey), indexVal);
}
}
iter->next();
Expand Down
15 changes: 15 additions & 0 deletions src/storage/test/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -628,6 +628,21 @@ nebula_add_test(
gtest
)

nebula_add_test(
NAME
index_ttl_test
SOURCES
IndexWithTTLTest.cpp
OBJECTS
${storage_test_deps}
LIBRARIES
${ROCKSDB_LIBRARIES}
${THRIFT_LIBRARIES}
${PROXYGEN_LIBRARIES}
wangle
gtest
)

nebula_add_executable(
NAME
es_bulk_example
Expand Down
141 changes: 141 additions & 0 deletions src/storage/test/CompactionTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,147 @@ TEST(CompactionFilterTest, DropIndexTest) {
checkIndexData(spaceId, 102, 6, env, 18);
}

TEST(CompactionFilterTest, TTLFilterDataIndexExpiredTest) {
FLAGS_mock_ttl_col = true;
FLAGS_mock_ttl_duration = 1;

fs::TempDir rootPath("/tmp/TTLFilterDataIndexExpiredTest.XXXXXX");
mock::MockCluster cluster;
cluster.initStorageKV(rootPath.path(), HostAddr("", 0),
1, true, false, {}, true);
auto* env = cluster.storageEnv_.get();
auto parts = cluster.getTotalParts();

GraphSpaceID spaceId = 1;
auto status = env->schemaMan_->getSpaceVidLen(spaceId);
ASSERT_TRUE(status.ok());
auto spaceVidLen = status.value();

// Add tag/edge data and index data
ASSERT_TRUE(QueryTestUtils::mockVertexData(env, parts, true));
ASSERT_TRUE(QueryTestUtils::mockEdgeData(env, parts, true));

LOG(INFO) << "Before compaction, check data...";
// check players data, data count is 51
checkTagVertexData(spaceVidLen, spaceId, 1, parts, env, 51);
// check teams data, data count is 30
checkTagVertexData(spaceVidLen, spaceId, 2, parts, env, 30);

// check serve positive data, data count is 167
checkEdgeData(spaceVidLen, spaceId, 101, parts, env, 167);
// check teammates positive data, data count is 18
checkEdgeData(spaceVidLen, spaceId, 102, parts, env, 18);

// check player indexId 1 data
checkIndexData(spaceId, 1, 6, env, 51);
// check teams indexId 2 data
checkIndexData(spaceId, 2, 6, env, 30);

// check serve indexId 101 data
checkIndexData(spaceId, 101, 6, env, 167);
// check teammates indexId 102 data
checkIndexData(spaceId, 102, 6, env, 18);

// wait ttl data Expire
sleep(FLAGS_mock_ttl_duration + 1);

LOG(INFO) << "Do compaction";
auto* ns = dynamic_cast<kvstore::NebulaStore*>(env->kvstore_);
ns->compact(spaceId);

LOG(INFO) << "Finish compaction, check index data...";
// check players data, data count is 0
checkTagVertexData(spaceVidLen, spaceId, 1, parts, env, 0);
// check teams data, data count is 30
checkTagVertexData(spaceVidLen, spaceId, 2, parts, env, 30);

// check serve positive data, data count is 0
checkEdgeData(spaceVidLen, spaceId, 101, parts, env, 0);
// check teammates positive data, data count is 18
checkEdgeData(spaceVidLen, spaceId, 102, parts, env, 18);

// check player indexId 1 data
checkIndexData(spaceId, 1, 6, env, 0);
// check teams indexId 2 data
checkIndexData(spaceId, 2, 6, env, 30);

// check serve indexId 101 data
checkIndexData(spaceId, 101, 6, env, 0);
// check teammates indexId 102 data
checkIndexData(spaceId, 102, 6, env, 18);

FLAGS_mock_ttl_col = false;
}

TEST(CompactionFilterTest, TTLFilterDataIndexNotExpiredTest) {
FLAGS_mock_ttl_col = true;
FLAGS_mock_ttl_duration = 1800;

fs::TempDir rootPath("/tmp/TTLFilterDataIndexNotExpiredTest.XXXXXX");
mock::MockCluster cluster;
cluster.initStorageKV(rootPath.path(), HostAddr("", 0),
1, true, false, {}, true);
auto* env = cluster.storageEnv_.get();
auto parts = cluster.getTotalParts();

GraphSpaceID spaceId = 1;
auto status = env->schemaMan_->getSpaceVidLen(spaceId);
ASSERT_TRUE(status.ok());
auto spaceVidLen = status.value();

// Add tag/edge data and index data
ASSERT_TRUE(QueryTestUtils::mockVertexData(env, parts, true));
ASSERT_TRUE(QueryTestUtils::mockEdgeData(env, parts, true));

LOG(INFO) << "Before compaction, check data...";
// check players data, data count is 51
checkTagVertexData(spaceVidLen, spaceId, 1, parts, env, 51);
// check teams data, data count is 30
checkTagVertexData(spaceVidLen, spaceId, 2, parts, env, 30);

// check serve positive data, data count is 167
checkEdgeData(spaceVidLen, spaceId, 101, parts, env, 167);
// check teammates positive data, data count is 18
checkEdgeData(spaceVidLen, spaceId, 102, parts, env, 18);

// check player indexId 1 data
checkIndexData(spaceId, 1, 6, env, 51);
// check teams indexId 2 data
checkIndexData(spaceId, 2, 6, env, 30);

// check serve indexId 101 data
checkIndexData(spaceId, 101, 6, env, 167);
// check teammates indexId 102 data
checkIndexData(spaceId, 102, 6, env, 18);

LOG(INFO) << "Do compaction";
auto* ns = dynamic_cast<kvstore::NebulaStore*>(env->kvstore_);
ns->compact(spaceId);

LOG(INFO) << "Finish compaction, check index data...";
// check players data, data count is 51
checkTagVertexData(spaceVidLen, spaceId, 1, parts, env, 51);
// check teams data, data count is 30
checkTagVertexData(spaceVidLen, spaceId, 2, parts, env, 30);

// check serve positive data, data count is 167
checkEdgeData(spaceVidLen, spaceId, 101, parts, env, 167);
// check teammates positive data, data count is 18
checkEdgeData(spaceVidLen, spaceId, 102, parts, env, 18);

// check player indexId 1 data
checkIndexData(spaceId, 1, 6, env, 51);
// check teams indexId 2 data
checkIndexData(spaceId, 2, 6, env, 30);

// check serve indexId 101 data
checkIndexData(spaceId, 101, 6, env, 167);
// check teammates indexId 102 data
checkIndexData(spaceId, 102, 6, env, 18);

FLAGS_mock_ttl_col = false;
}

} // namespace storage
} // namespace nebula

Expand Down
Loading

0 comments on commit 30cb073

Please sign in to comment.