Skip to content

Test fulltext index backup #28095

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kungasc merged 2 commits into from
Nov 1, 2025
Merged

Test fulltext index backup #28095

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0d1add1
Test fulltext index backup
kungasc Oct 31, 2025
4db5214
fix enable
kungasc Oct 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 93 additions & 52 deletions ydb/services/ydb/backup_ut/ydb_backup_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "ydb_common_ut.h"

#include <ydb/core/protos/flat_scheme_op.pb.h>
#include <ydb/core/util/aws.h>
#include <ydb/core/wrappers/ut_helpers/s3_mock.h>

Expand Down Expand Up @@ -287,26 +288,46 @@ auto CreateHasIndexChecker(const TString& indexName, EIndexType indexType, bool
if (indexDesc.GetIndexColumns().back() != "Value") {
continue;
}
if (indexType != NYdb::NTable::EIndexType::GlobalVectorKMeansTree) {
return true;
}
auto* settings = std::get_if<TKMeansTreeSettings>(&indexDesc.GetIndexSettings());
UNIT_ASSERT(settings);
if (settings->Settings.Metric != NYdb::NTable::TVectorIndexSettings::EMetric::InnerProduct) {
continue;
}
if (settings->Settings.VectorType != NYdb::NTable::TVectorIndexSettings::EVectorType::Float) {
continue;
}
if (settings->Settings.VectorDimension != 768) {
continue;
}
if (settings->Levels != 2) {
continue;
}
if (settings->Clusters != 80) {
continue;
}
switch (indexType) { // check settings
case EIndexType::GlobalSync:
case EIndexType::GlobalAsync:
case EIndexType::GlobalUnique:
UNIT_ASSERT(std::holds_alternative<std::monostate>(indexDesc.GetIndexSettings()));
break;
case EIndexType::GlobalVectorKMeansTree: {
Ydb::Table::KMeansTreeSettings settings;
std::get<TKMeansTreeSettings>(indexDesc.GetIndexSettings()).SerializeTo(settings);
Ydb::Table::KMeansTreeSettings expected;
expected.mutable_settings()->set_metric(Ydb::Table::VectorIndexSettings::SIMILARITY_INNER_PRODUCT);
expected.mutable_settings()->set_vector_type(Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT);
expected.mutable_settings()->set_vector_dimension(768);
expected.set_levels(2);
expected.set_clusters(80);
if (!google::protobuf::util::MessageDifferencer::Equals(settings, expected)) {
continue;
}
break;
}
case EIndexType::GlobalFulltext: {
Ydb::Table::FulltextIndexSettings settings;
std::get<TFulltextIndexSettings>(indexDesc.GetIndexSettings()).SerializeTo(settings);
Ydb::Table::FulltextIndexSettings expected;
expected.set_layout(Ydb::Table::FulltextIndexSettings::FLAT);
auto column = expected.add_columns();
column->set_column("Value");
column->mutable_analyzers()->set_tokenizer(Ydb::Table::FulltextIndexSettings::STANDARD);
column->mutable_analyzers()->set_use_filter_lowercase(true);
column->mutable_analyzers()->set_use_filter_length(true);
column->mutable_analyzers()->set_filter_length_max(42);
if (!google::protobuf::util::MessageDifferencer::Equals(settings, expected)) {
continue;
}
break;
}
case EIndexType::Unknown: {
UNIT_ASSERT(false);
}
}
return true;
}
return false;
Expand Down Expand Up @@ -681,6 +702,8 @@ NYdb::NTable::EIndexType ConvertIndexTypeToAPI(NKikimrSchemeOp::EIndexType index
return NYdb::NTable::EIndexType::GlobalUnique;
case NKikimrSchemeOp::EIndexTypeGlobalVectorKmeansTree:
return NYdb::NTable::EIndexType::GlobalVectorKMeansTree;
case NKikimrSchemeOp::EIndexTypeGlobalFulltext:
return NYdb::NTable::EIndexType::GlobalFulltext;
default:
UNIT_FAIL("No conversion to API for this index type");
return NYdb::NTable::EIndexType::Unknown;
Expand All @@ -692,39 +715,59 @@ void TestRestoreTableWithIndex(
TBackupFunction&& backup, TRestoreFunction&& restore
) {
TString query;
if (indexType == NKikimrSchemeOp::EIndexTypeGlobalVectorKmeansTree) {
if (prefix) {
query = Sprintf(R"(CREATE TABLE `%s` (
Key Uint32,
Group Uint32,
Value String,
PRIMARY KEY (Key),
INDEX %s GLOBAL USING vector_kmeans_tree
ON (Group, Value)
WITH (similarity=inner_product, vector_type=float, vector_dimension=768, levels=2, clusters=80)
);)", table, index);
} else {
switch (indexType) {
case NKikimrSchemeOp::EIndexTypeGlobal:
case NKikimrSchemeOp::EIndexTypeGlobalAsync:
case NKikimrSchemeOp::EIndexTypeGlobalUnique:
query = Sprintf(R"(
CREATE TABLE `%s` (
Key Uint32,
Group Uint32,
Value Uint32,
PRIMARY KEY (Key),
INDEX %s %s ON (Value)
);
)", table, index, ConvertIndexTypeToSQL(indexType));
break;
case NKikimrSchemeOp::EIndexTypeGlobalVectorKmeansTree:
if (prefix) {
query = Sprintf(R"(CREATE TABLE `%s` (
Key Uint32,
Group Uint32,
Value String,
PRIMARY KEY (Key),
INDEX %s GLOBAL USING vector_kmeans_tree
ON (Group, Value)
WITH (similarity=inner_product, vector_type=float, vector_dimension=768, levels=2, clusters=80)
);)", table, index);
} else {
query = Sprintf(R"(CREATE TABLE `%s` (
Key Uint32,
Group Uint32,
Value String,
PRIMARY KEY (Key),
INDEX %s GLOBAL USING vector_kmeans_tree
ON (Value)
WITH (similarity=inner_product, vector_type=float, vector_dimension=768, levels=2, clusters=80)
);)", table, index);
}
break;
case NKikimrSchemeOp::EIndexTypeGlobalFulltext:
query = Sprintf(R"(CREATE TABLE `%s` (
Key Uint32,
Group Uint32,
Value String,
PRIMARY KEY (Key),
INDEX %s GLOBAL USING vector_kmeans_tree
INDEX %s GLOBAL USING fulltext
ON (Value)
WITH (similarity=inner_product, vector_type=float, vector_dimension=768, levels=2, clusters=80)
WITH (layout=flat, tokenizer=standard, use_filter_lowercase=true, use_filter_length=true, filter_length_max=42)
);)", table, index);
}
} else {
query = Sprintf(R"(
CREATE TABLE `%s` (
Key Uint32,
Group Uint32,
Value Uint32,
PRIMARY KEY (Key),
INDEX %s %s ON (Value)
);
)", table, index, ConvertIndexTypeToSQL(indexType));
}
break;
default:
UNIT_FAIL("No creation this index type");
break;
};

ExecuteDataDefinitionQuery(session, query);

backup();
Expand Down Expand Up @@ -2329,6 +2372,7 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
NKikimrConfig::TAppConfig appConfig;
appConfig.MutableFeatureFlags()->SetEnableVectorIndex(true);
appConfig.MutableFeatureFlags()->SetEnableAddUniqueIndex(true);
appConfig.MutableFeatureFlags()->SetEnableFulltextIndex(true);
TKikimrWithGrpcAndRootSchema server{std::move(appConfig)};

auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort())).SetDatabase("/Root"));
Expand Down Expand Up @@ -2773,10 +2817,8 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
case EIndexTypeGlobalAsync:
case EIndexTypeGlobalUnique:
case EIndexTypeGlobalVectorKmeansTree:
return TestTableWithIndexBackupRestore(Value);
case EIndexTypeGlobalFulltext:
// TODO: will be added later
break;
return TestTableWithIndexBackupRestore(Value);
case EIndexTypeInvalid:
break; // not applicable
default:
Expand Down Expand Up @@ -3271,6 +3313,7 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) {
NKikimrConfig::TAppConfig appConfig;
appConfig.MutableFeatureFlags()->SetEnableVectorIndex(true);
appConfig.MutableFeatureFlags()->SetEnableAddUniqueIndex(true);
appConfig.MutableFeatureFlags()->SetEnableFulltextIndex(true);
return appConfig;
}())
, Driver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", Server.GetPort())).SetDatabase("/Root"))
Expand Down Expand Up @@ -3803,10 +3846,8 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) {
case EIndexTypeGlobalAsync:
case EIndexTypeGlobalUnique:
case EIndexTypeGlobalVectorKmeansTree:
TestTableWithIndexBackupRestore(Value);
break;
case EIndexTypeGlobalFulltext:
// TODO: will be added later
TestTableWithIndexBackupRestore(Value);
break;
case EIndexTypeInvalid:
break; // not applicable
Expand Down