diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 9ec80ddb2e3c..7213c4a41a2c 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -1837,6 +1837,7 @@ message TDescribeOptions { optional bool ReturnChannelsBinding = 8 [default = false]; optional bool ReturnRangeKey = 9 [default = true]; optional bool ReturnSetVal = 10 [default = false]; + optional bool ReturnIndexTableBoundaries = 11 [default = false]; } // Request to read scheme for a specific path diff --git a/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp index 4af5a86d53ea..d6fca52aecaf 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp @@ -76,6 +76,7 @@ static NKikimrSchemeOp::TPathDescription GetTableDescription(TSchemeShard* ss, c opts.SetReturnPartitioningInfo(false); opts.SetReturnPartitionConfig(true); opts.SetReturnBoundaries(true); + opts.SetReturnIndexTableBoundaries(true); auto desc = DescribePath(ss, TlsActivationContext->AsActorContext(), pathId, opts); auto record = desc->GetRecord(); diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp index 872956a98b6e..a71f5d5a442e 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp @@ -73,6 +73,79 @@ static void FillTableStats(NKikimrSchemeOp::TPathDescription& pathDescription, c FillTableMetrics(pathDescription.MutableTabletMetrics(), stats); } +static void FillColumns( + const TTableInfo& tableInfo, + google::protobuf::RepeatedPtrField& out +) { + bool familyNamesBuilt = false; + THashMap familyNames; + + out.Reserve(tableInfo.Columns.size()); + for (const auto& col : tableInfo.Columns) { + const auto& cinfo = col.second; + if (cinfo.IsDropped()) + continue; + + auto* colDescr = out.Add(); + colDescr->SetName(cinfo.Name); + colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod)); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod); + colDescr->SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *colDescr->MutableTypeInfo() = *columnType.TypeInfo; + } + colDescr->SetId(cinfo.Id); + colDescr->SetNotNull(cinfo.NotNull); + + if (cinfo.Family != 0) { + colDescr->SetFamily(cinfo.Family); + + if (!familyNamesBuilt) { + for (const auto& family : tableInfo.PartitionConfig().GetColumnFamilies()) { + if (family.HasName() && family.HasId()) { + familyNames[family.GetId()] = family.GetName(); + } + } + familyNamesBuilt = true; + } + + auto it = familyNames.find(cinfo.Family); + if (it != familyNames.end() && !it->second.empty()) { + colDescr->SetFamilyName(it->second); + } + } + + colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress); + + switch (cinfo.DefaultKind) { + case ETableColumnDefaultKind::None: + break; + case ETableColumnDefaultKind::FromSequence: + colDescr->SetDefaultFromSequence(cinfo.DefaultValue); + break; + case ETableColumnDefaultKind::FromLiteral: + Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString( + cinfo.DefaultValue)); + break; + } + } +} + +static void FillKeyColumns( + const TTableInfo& tableInfo, + google::protobuf::RepeatedPtrField& names, + google::protobuf::RepeatedField& ids +) { + Y_ABORT_UNLESS(!tableInfo.KeyColumnIds.empty()); + names.Reserve(tableInfo.KeyColumnIds.size()); + ids.Reserve(tableInfo.KeyColumnIds.size()); + for (ui32 keyColId : tableInfo.KeyColumnIds) { + *names.Add() = tableInfo.Columns.at(keyColId).Name; + *ids.Add() = keyColId; + } + +} + void TPathDescriber::FillPathDescr(NKikimrSchemeOp::TDirEntry* descr, TPathElement::TPtr pathEl, TPathElement::EPathSubType subType) { FillChildDescr(descr, pathEl); @@ -303,6 +376,7 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa bool returnBoundaries = false; bool returnRangeKey = true; bool returnSetVal = Params.GetOptions().GetReturnSetVal(); + bool returnIndexTableBoundaries = Params.GetOptions().GetReturnIndexTableBoundaries(); if (Params.HasOptions()) { returnConfig = Params.GetOptions().GetReturnPartitionConfig(); returnPartitioning = Params.GetOptions().GetReturnPartitioningInfo(); @@ -427,7 +501,9 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa switch (childPath->PathType) { case NKikimrSchemeOp::EPathTypeTableIndex: - Self->DescribeTableIndex(childPathId, childName, returnConfig, false, *entry->AddTableIndexes()); + Self->DescribeTableIndex( + childPathId, childName, returnConfig, returnIndexTableBoundaries, *entry->AddTableIndexes() + ); break; case NKikimrSchemeOp::EPathTypeCdcStream: Self->DescribeCdcStream(childPathId, childName, *entry->AddCdcStreams()); @@ -1189,67 +1265,10 @@ void TSchemeShard::DescribeTable( ) const { Y_UNUSED(typeRegistry); - THashMap familyNames; - bool familyNamesBuilt = false; entry->SetTableSchemaVersion(tableInfo.AlterVersion); - entry->MutableColumns()->Reserve(tableInfo.Columns.size()); - for (auto col : tableInfo.Columns) { - const auto& cinfo = col.second; - if (cinfo.IsDropped()) - continue; - - auto colDescr = entry->AddColumns(); - colDescr->SetName(cinfo.Name); - colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod)); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod); - colDescr->SetTypeId(columnType.TypeId); - if (columnType.TypeInfo) { - *colDescr->MutableTypeInfo() = *columnType.TypeInfo; - } - colDescr->SetId(cinfo.Id); - colDescr->SetNotNull(cinfo.NotNull); - - if (cinfo.Family != 0) { - colDescr->SetFamily(cinfo.Family); - - if (!familyNamesBuilt) { - for (const auto& family : tableInfo.PartitionConfig().GetColumnFamilies()) { - if (family.HasName() && family.HasId()) { - familyNames[family.GetId()] = family.GetName(); - } - } - familyNamesBuilt = true; - } - - auto it = familyNames.find(cinfo.Family); - if (it != familyNames.end() && !it->second.empty()) { - colDescr->SetFamilyName(it->second); - } - } - - colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress); - - switch (cinfo.DefaultKind) { - case ETableColumnDefaultKind::None: - break; - case ETableColumnDefaultKind::FromSequence: - colDescr->SetDefaultFromSequence(cinfo.DefaultValue); - break; - case ETableColumnDefaultKind::FromLiteral: - Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString( - cinfo.DefaultValue)); - break; - } - } - Y_ABORT_UNLESS(!tableInfo.KeyColumnIds.empty()); - - entry->MutableKeyColumnNames()->Reserve(tableInfo.KeyColumnIds.size()); - entry->MutableKeyColumnIds()->Reserve(tableInfo.KeyColumnIds.size()); - for (ui32 keyColId : tableInfo.KeyColumnIds) { - entry->AddKeyColumnNames(tableInfo.Columns.at(keyColId).Name); - entry->AddKeyColumnIds(keyColId); - } + FillColumns(tableInfo, *entry->MutableColumns()); + FillKeyColumns(tableInfo, *entry->MutableKeyColumnNames(), *entry->MutableKeyColumnIds()); if (fillConfig) { FillPartitionConfig(tableInfo.PartitionConfig(), *entry->MutablePartitionConfig()); @@ -1324,6 +1343,9 @@ void TSchemeShard::DescribeTableIndex(const TPathId& pathId, const TString& name FillPartitionConfig(tableInfo.PartitionConfig(), *tableDescription->MutablePartitionConfig()); } if (fillBoundaries) { + // column info is necessary for split boundary type conversion + FillColumns(tableInfo, *tableDescription->MutableColumns()); + FillKeyColumns(tableInfo, *tableDescription->MutableKeyColumnNames(), *tableDescription->MutableKeyColumnIds()); FillTableBoundaries(tableDescription->MutableSplitBoundary(), tableInfo); } } diff --git a/ydb/core/ydb_convert/table_description.cpp b/ydb/core/ydb_convert/table_description.cpp index 029c5dc1485f..10402d9329cc 100644 --- a/ydb/core/ydb_convert/table_description.cpp +++ b/ydb/core/ydb_convert/table_description.cpp @@ -852,8 +852,8 @@ void FillGlobalIndexSettings(Ydb::Table::GlobalIndexSettings& settings, NKikimrMiniKQL::TType splitKeyType; Ydb::Table::DescribeTableResult unused; FillColumnDescription(unused, splitKeyType, indexImplTableDescription); - FillTableBoundaryImpl( - *settings.mutable_partition_at_keys(), + FillTableBoundaryImpl( + settings, indexImplTableDescription, splitKeyType ); diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index a340d5dbbf84..ef0f593037e0 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -469,6 +469,10 @@ class TTableDescription::TImpl { Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, dataColumns)); } + void AddSecondaryIndex(const TIndexDescription& indexDescription) { + Indexes_.emplace_back(indexDescription); + } + void AddVectorIndex(const TString& indexName, EIndexType type, const TVector& indexColumns, const TVectorIndexSettings& vectorIndexSettings) { Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, {}, {}, vectorIndexSettings)); } @@ -749,6 +753,10 @@ void TTableDescription::AddSecondaryIndex(const TString& indexName, EIndexType t Impl_->AddSecondaryIndex(indexName, type, indexColumns, dataColumns); } +void TTableDescription::AddSecondaryIndex(const TIndexDescription& indexDescription) { + Impl_->AddSecondaryIndex(indexDescription); +} + void TTableDescription::AddSyncSecondaryIndex(const TString& indexName, const TVector& indexColumns) { AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns); } @@ -1173,6 +1181,11 @@ TTableBuilder& TTableBuilder::SetPrimaryKeyColumn(const TString& primaryKeyColum return *this; } +TTableBuilder& TTableBuilder::AddSecondaryIndex(const TIndexDescription& indexDescription) { + TableDescription_.AddSecondaryIndex(indexDescription); + return *this; +} + TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector& indexColumns, const TVector& dataColumns) { TableDescription_.AddSecondaryIndex(indexName, type, indexColumns, dataColumns); return *this; @@ -2364,7 +2377,7 @@ TVectorIndexSettings TVectorIndexSettings::FromProto(const TProto& proto) { default: return EVectorType::Unknown; } - }; + }; auto metricFromProto = [&](const auto& proto) -> TVectorIndexSettings::TMetric { @@ -2376,7 +2389,7 @@ TVectorIndexSettings TVectorIndexSettings::FromProto(const TProto& proto) { default: return {}; } - }; + }; return { .Metric = metricFromProto(proto), @@ -2424,8 +2437,8 @@ void TVectorIndexSettings::SerializeTo(Ydb::Table::VectorIndexSettings& settings return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UNSPECIFIED; } }; - - + + if (const auto* distance = std::get_if(&Metric)) { settings.set_distance(convertDistance(*distance)); } else if (const auto* similarity = std::get_if(&Metric)) { diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index 9284665cd76b..d87ccf9be983 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -174,10 +174,10 @@ struct TExplicitPartitions { using TSelf = TExplicitPartitions; FLUENT_SETTING_VECTOR(TValue, SplitPoints); - + template static TExplicitPartitions FromProto(const TProto& proto); - + void SerializeTo(Ydb::Table::ExplicitPartitions& proto) const; }; @@ -642,6 +642,7 @@ class TTableDescription { // common void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector& indexColumns); void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector& indexColumns, const TVector& dataColumns); + void AddSecondaryIndex(const TIndexDescription& indexDescription); // sync void AddSyncSecondaryIndex(const TString& indexName, const TVector& indexColumns); void AddSyncSecondaryIndex(const TString& indexName, const TVector& indexColumns, const TVector& dataColumns); @@ -855,6 +856,7 @@ class TTableBuilder { TTableBuilder& SetPrimaryKeyColumn(const TString& primaryKeyColumn); // common + TTableBuilder& AddSecondaryIndex(const TIndexDescription& indexDescription); TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector& indexColumns, const TVector& dataColumns); TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector& indexColumns); TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TString& indexColumn); diff --git a/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp b/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp index 90e0f55a8ebd..f7e9371cf9c3 100644 --- a/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp +++ b/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp @@ -3,32 +3,52 @@ #include #include +#include #include #include #include -#include #include #include #include #include +#include #include +#include using namespace NYdb; using namespace NYdb::NTable; +namespace NYdb::NTable { + +bool operator==(const TValue& lhs, const TValue& rhs) { + return google::protobuf::util::MessageDifferencer::Equals(lhs.GetProto(), rhs.GetProto()); +} + +bool operator==(const TKeyBound& lhs, const TKeyBound& rhs) { + return lhs.GetValue() == rhs.GetValue() && lhs.IsInclusive() == rhs.IsInclusive(); +} + +bool operator==(const TKeyRange& lhs, const TKeyRange& rhs) { + return lhs.From() == lhs.From() && lhs.To() == rhs.To(); +} + +} + namespace { +#define DEBUG_HINT (TStringBuilder() << "at line " << __LINE__) + void ExecuteDataDefinitionQuery(TSession& session, const TString& script) { const auto result = session.ExecuteSchemeQuery(script).ExtractValueSync(); UNIT_ASSERT_C(result.IsSuccess(), "script:\n" << script << "\nissues:\n" << result.GetIssues().ToString()); } TDataQueryResult ExecuteDataModificationQuery(TSession& session, - const TString& script, - const TExecDataQuerySettings& settings = {} + const TString& script, + const TExecDataQuerySettings& settings = {} ) { const auto result = session.ExecuteDataQuery( script, @@ -40,197 +60,363 @@ TDataQueryResult ExecuteDataModificationQuery(TSession& session, return result; } -TValue GetSingleResult(const TDataQueryResult& rawResults) { - auto resultSetParser = rawResults.GetResultSetParser(0); - UNIT_ASSERT(resultSetParser.TryNextRow()); - return resultSetParser.GetValue(0); +TDataQueryResult GetTableContent(TSession& session, const char* table) { + return ExecuteDataModificationQuery(session, Sprintf(R"( + SELECT * FROM `%s` ORDER BY Key; + )", table + )); +} + +void CompareResults(const TDataQueryResult& first, const TDataQueryResult& second) { + const auto& firstResults = first.GetResultSets(); + const auto& secondResults = second.GetResultSets(); + + UNIT_ASSERT_VALUES_EQUAL(firstResults.size(), secondResults.size()); + for (size_t i = 0; i < firstResults.size(); ++i) { + UNIT_ASSERT_STRINGS_EQUAL( + FormatResultSetYson(firstResults[i]), + FormatResultSetYson(secondResults[i]) + ); + } } -ui64 GetUint64(const TValue& value) { - return TValueParser(value).GetUint64(); +TTableDescription GetTableDescription(TSession& session, const TString& path, + const TDescribeTableSettings& settings = {} +) { + auto describeResult = session.DescribeTable(path, settings).ExtractValueSync(); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); + return describeResult.GetTableDescription(); } -auto CreateMinPartitionsChecker(ui64 expectedMinPartitions) { +auto CreateMinPartitionsChecker(ui32 expectedMinPartitions, const TString& debugHint = "") { return [=](const TTableDescription& tableDescription) { - return tableDescription.GetPartitioningSettings().GetMinPartitionsCount() == expectedMinPartitions; + UNIT_ASSERT_VALUES_EQUAL_C( + tableDescription.GetPartitioningSettings().GetMinPartitionsCount(), + expectedMinPartitions, + debugHint + ); }; } -void CheckTableDescription(TSession& session, const TString& path, auto&& checker) { - auto describeResult = session.DescribeTable(path).ExtractValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); - auto tableDescription = describeResult.GetTableDescription(); - Ydb::Table::CreateTableRequest descriptionProto; - // The purpose of translating to CreateTableRequest is solely to produce a clearer error message. - tableDescription.SerializeTo(descriptionProto); - UNIT_ASSERT_C( - checker(tableDescription), - descriptionProto.DebugString() +void CheckTableDescription(TSession& session, const TString& path, auto&& checker, + const TDescribeTableSettings& settings = {} +) { + checker(GetTableDescription(session, path, settings)); +} + +using TBackupFunction = std::function; +using TRestoreFunction = std::function; + +void TestTableContentIsPreserved( + const char* table, TSession& session, TBackupFunction&& backup, TRestoreFunction&& restore +) { + ExecuteDataDefinitionQuery(session, Sprintf(R"( + CREATE TABLE `%s` ( + Key Uint32, + Value Utf8, + PRIMARY KEY (Key) + ); + )", + table + )); + ExecuteDataModificationQuery(session, Sprintf(R"( + UPSERT INTO `%s` ( + Key, + Value + ) + VALUES + (1, "one"), + (2, "two"), + (3, "three"), + (4, "four"), + (5, "five"); + )", + table + )); + const auto originalContent = GetTableContent(session, table); + + backup(table); + + ExecuteDataDefinitionQuery(session, Sprintf(R"( + DROP TABLE `%s`; + )", table + )); + + restore(table); + CompareResults(GetTableContent(session, table), originalContent); +} + +void TestTablePartitioningSettingsArePreserved( + const char* table, ui32 minPartitions, TSession& session, TBackupFunction&& backup, TRestoreFunction&& restore +) { + ExecuteDataDefinitionQuery(session, Sprintf(R"( + CREATE TABLE `%s` ( + Key Uint32, + Value Utf8, + PRIMARY KEY (Key) + ) + WITH ( + AUTO_PARTITIONING_BY_LOAD = ENABLED, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %u + ); + )", + table, minPartitions + )); + CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions, DEBUG_HINT)); + + backup(table); + + ExecuteDataDefinitionQuery(session, Sprintf(R"( + DROP TABLE `%s`; + )", table + )); + + restore(table); + CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions, DEBUG_HINT)); +} + +void TestIndexTablePartitioningSettingsArePreserved( + const char* table, const char* index, ui32 minIndexPartitions, TSession& session, + TBackupFunction&& backup, TRestoreFunction&& restore +) { + const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable"); + + ExecuteDataDefinitionQuery(session, Sprintf(R"( + CREATE TABLE `%s` ( + Key Uint32, + Value Uint32, + PRIMARY KEY (Key), + INDEX %s GLOBAL ON (Value) + ); + )", + table, index + )); + ExecuteDataDefinitionQuery(session, Sprintf(R"( + ALTER TABLE `%s` ALTER INDEX %s SET ( + AUTO_PARTITIONING_BY_LOAD = ENABLED, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %u + ); + )", table, index, minIndexPartitions + )); + CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minIndexPartitions, DEBUG_HINT)); + + backup(table); + + ExecuteDataDefinitionQuery(session, Sprintf(R"( + DROP TABLE `%s`; + )", table + )); + + restore(table); + CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minIndexPartitions, DEBUG_HINT)); +} + +void TestTableSplitBoundariesArePreserved( + const char* table, ui64 partitions, TSession& session, TBackupFunction&& backup, TRestoreFunction&& restore +) { + ExecuteDataDefinitionQuery(session, Sprintf(R"( + CREATE TABLE `%s` ( + Key Uint32, + Value Utf8, + PRIMARY KEY (Key) + ) + WITH ( + PARTITION_AT_KEYS = (1, 2, 4, 8, 16, 32, 64, 128, 256) + ); + )", + table + )); + const auto describeSettings = TDescribeTableSettings() + .WithTableStatistics(true) + .WithKeyShardBoundary(true); + const auto originalTableDescription = GetTableDescription(session, table, describeSettings); + UNIT_ASSERT_VALUES_EQUAL(originalTableDescription.GetPartitionsCount(), partitions); + const auto& originalKeyRanges = originalTableDescription.GetKeyRanges(); + UNIT_ASSERT_VALUES_EQUAL(originalKeyRanges.size(), partitions); + + backup(table); + + ExecuteDataDefinitionQuery(session, Sprintf(R"( + DROP TABLE `%s`; + )", table + )); + + restore(table); + const auto restoredTableDescription = GetTableDescription(session, table, describeSettings); + UNIT_ASSERT_VALUES_EQUAL(restoredTableDescription.GetPartitionsCount(), partitions); + const auto& restoredKeyRanges = restoredTableDescription.GetKeyRanges(); + UNIT_ASSERT_VALUES_EQUAL(restoredKeyRanges.size(), partitions); + UNIT_ASSERT_EQUAL(restoredTableDescription.GetKeyRanges(), originalKeyRanges); +} + +void TestIndexTableSplitBoundariesArePreserved( + const char* table, const char* index, ui64 indexPartitions, TSession& session, + TBackupFunction&& backup, TRestoreFunction&& restore +) { + const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable"); + + { + TExplicitPartitions indexPartitionBoundaries; + for (ui32 boundary : {1, 2, 4, 8, 16, 32, 64, 128, 256}) { + indexPartitionBoundaries.AppendSplitPoints( + // split boundary is technically always a tuple + TValueBuilder().BeginTuple().AddElement().OptionalUint32(boundary).EndTuple().Build() + ); + } + // By default indexImplTable has auto partitioning by size enabled, + // so you must set min partition count for partitions to not merge immediately after indexImplTable is built. + TPartitioningSettingsBuilder partitioningSettingsBuilder; + partitioningSettingsBuilder + .SetMinPartitionsCount(indexPartitions) + .SetMaxPartitionsCount(indexPartitions); + + const auto indexSettings = TGlobalIndexSettings{ + .PartitioningSettings = partitioningSettingsBuilder.Build(), + .Partitions = std::move(indexPartitionBoundaries) + }; + + auto tableBuilder = TTableBuilder() + .AddNullableColumn("Key", EPrimitiveType::Uint32) + .AddNullableColumn("Value", EPrimitiveType::Uint32) + .SetPrimaryKeyColumn("Key") + .AddSecondaryIndex(TIndexDescription("byValue", EIndexType::GlobalSync, { "Value" }, {}, { indexSettings })); + + const auto result = session.CreateTable(table, tableBuilder.Build()).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + const auto describeSettings = TDescribeTableSettings() + .WithTableStatistics(true) + .WithKeyShardBoundary(true); + const auto originalIndexTableDescription = GetTableDescription( + session, indexTablePath, describeSettings ); + UNIT_ASSERT_VALUES_EQUAL(originalIndexTableDescription.GetPartitionsCount(), indexPartitions); + const auto& originalKeyRanges = originalIndexTableDescription.GetKeyRanges(); + UNIT_ASSERT_VALUES_EQUAL(originalKeyRanges.size(), indexPartitions); + + backup(table); + + ExecuteDataDefinitionQuery(session, Sprintf(R"( + DROP TABLE `%s`; + )", table + )); + + restore(table); + const auto restoredIndexTableDescription = GetTableDescription( + session, indexTablePath, describeSettings + ); + UNIT_ASSERT_VALUES_EQUAL(restoredIndexTableDescription.GetPartitionsCount(), indexPartitions); + const auto& restoredKeyRanges = restoredIndexTableDescription.GetKeyRanges(); + UNIT_ASSERT_VALUES_EQUAL(restoredKeyRanges.size(), indexPartitions); + UNIT_ASSERT_EQUAL(restoredKeyRanges, originalKeyRanges); } } Y_UNIT_TEST_SUITE(BackupRestore) { - + void Restore(NDump::TClient& client, const TFsPath& sourceFile, const TString& dbPath) { auto result = client.Restore(sourceFile, dbPath); UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); } - Y_UNIT_TEST(Basic) { + auto CreateBackupLambda(const TDriver& driver, const TFsPath& pathToBackup, bool schemaOnly = false) { + return [&driver, &pathToBackup, schemaOnly](const char* table) { + Y_UNUSED(table); + // TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder + NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, schemaOnly, false); + }; + } + + auto CreateRestoreLambda(const TDriver& driver, const TFsPath& pathToBackup) { + return [&driver, &pathToBackup](const char* table) { + Y_UNUSED(table); + NDump::TClient backupClient(driver); + Restore(backupClient, pathToBackup, "/Root"); + }; + } + + Y_UNIT_TEST(RestoreTableContent) { TKikimrWithGrpcAndRootSchema server; - auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%d", server.GetPort()))); + auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort()))); TTableClient tableClient(driver); auto session = tableClient.GetSession().ExtractValueSync().GetSession(); - - constexpr const char* table = "/Root/table"; - ExecuteDataDefinitionQuery(session, Sprintf(R"( - CREATE TABLE `%s` ( - Key Uint32, - Value Utf8, - PRIMARY KEY (Key) - ); - )", - table - )); - ExecuteDataModificationQuery(session, Sprintf(R"( - UPSERT INTO `%s` ( - Key, - Value - ) - VALUES - (1, "one"), - (2, "two"), - (3, "three"), - (4, "four"), - (5, "five"); - )", - table - )); - TTempDir tempDir; const auto& pathToBackup = tempDir.Path(); - // TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder - NYdb::NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, false, false); - - NDump::TClient backupClient(driver); - - // restore deleted rows in an existing table - ExecuteDataModificationQuery(session, Sprintf(R"( - DELETE FROM `%s` WHERE Key > 3; - )", table - )); - Restore(backupClient, pathToBackup, "/Root"); - { - auto result = ExecuteDataModificationQuery(session, Sprintf(R"( - SELECT COUNT(*) FROM `%s`; - )", table - )); - UNIT_ASSERT_VALUES_EQUAL(GetUint64(GetSingleResult(result)), 5ull); - } - // restore deleted table - ExecuteDataDefinitionQuery(session, Sprintf(R"( - DROP TABLE `%s`; - )", table - )); - Restore(backupClient, pathToBackup, "/Root"); - { - auto result = ExecuteDataModificationQuery(session, Sprintf(R"( - SELECT COUNT(*) FROM `%s`; - )", table - )); - UNIT_ASSERT_VALUES_EQUAL(GetUint64(GetSingleResult(result)), 5ull); - } + constexpr const char* table = "/Root/table"; + + TestTableContentIsPreserved( + table, + session, + CreateBackupLambda(driver, pathToBackup), + CreateRestoreLambda(driver, pathToBackup) + ); } - + Y_UNIT_TEST(RestoreTablePartitioningSettings) { TKikimrWithGrpcAndRootSchema server; - auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%d", server.GetPort()))); + auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort()))); TTableClient tableClient(driver); auto session = tableClient.GetSession().ExtractValueSync().GetSession(); - - constexpr const char* table = "/Root/table"; - constexpr int minPartitions = 10; - ExecuteDataDefinitionQuery(session, Sprintf(R"( - CREATE TABLE `%s` ( - Key Uint32, - Value Utf8, - PRIMARY KEY (Key) - ) - WITH ( - AUTO_PARTITIONING_BY_LOAD = ENABLED, - AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %d - ); - )", - table, minPartitions - )); - - CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions)); - TTempDir tempDir; const auto& pathToBackup = tempDir.Path(); - // TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder - NYdb::NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, false, false); - - NDump::TClient backupClient(driver); - - // restore deleted table - ExecuteDataDefinitionQuery(session, Sprintf(R"( - DROP TABLE `%s`; - )", table - )); - Restore(backupClient, pathToBackup, "/Root"); - CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions)); + + constexpr const char* table = "/Root/table"; + constexpr ui32 minPartitions = 10; + + TestTablePartitioningSettingsArePreserved( + table, + minPartitions, + session, + CreateBackupLambda(driver, pathToBackup, true), + CreateRestoreLambda(driver, pathToBackup) + ); } Y_UNIT_TEST(RestoreIndexTablePartitioningSettings) { TKikimrWithGrpcAndRootSchema server; - auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%d", server.GetPort()))); + auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort()))); TTableClient tableClient(driver); auto session = tableClient.GetSession().ExtractValueSync().GetSession(); + TTempDir tempDir; + const auto& pathToBackup = tempDir.Path(); constexpr const char* table = "/Root/table"; constexpr const char* index = "byValue"; - const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable"); - constexpr int minPartitions = 10; - ExecuteDataDefinitionQuery(session, Sprintf(R"( - CREATE TABLE `%s` ( - Key Uint32, - Value Uint32, - PRIMARY KEY (Key), - INDEX %s GLOBAL ON (Value) - ); - )", - table, index - )); - ExecuteDataDefinitionQuery(session, Sprintf(R"( - ALTER TABLE `%s` ALTER INDEX %s SET ( - AUTO_PARTITIONING_BY_LOAD = ENABLED, - AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %d - ); - )", table, index, minPartitions - )); + constexpr ui32 minIndexPartitions = 10; + + TestIndexTablePartitioningSettingsArePreserved( + table, + index, + minIndexPartitions, + session, + CreateBackupLambda(driver, pathToBackup, true), + CreateRestoreLambda(driver, pathToBackup) + ); + } - CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minPartitions)); - + Y_UNIT_TEST(RestoreTableSplitBoundaries) { + TKikimrWithGrpcAndRootSchema server; + auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort()))); + TTableClient tableClient(driver); + auto session = tableClient.GetSession().ExtractValueSync().GetSession(); TTempDir tempDir; const auto& pathToBackup = tempDir.Path(); - // TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder - NYdb::NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, false, false); - - NDump::TClient backupClient(driver); - - // restore deleted table - ExecuteDataDefinitionQuery(session, Sprintf(R"( - DROP TABLE `%s`; - )", table - )); - Restore(backupClient, pathToBackup, "/Root"); - CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minPartitions)); + + constexpr const char* table = "/Root/table"; + constexpr ui64 partitions = 10; + + TestTableSplitBoundariesArePreserved( + table, + partitions, + session, + CreateBackupLambda(driver, pathToBackup, true), + CreateRestoreLambda(driver, pathToBackup) + ); } + // TO DO: test index impl table split boundaries restoration from a backup } Y_UNIT_TEST_SUITE(BackupRestoreS3) { @@ -259,7 +445,7 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) { public: TS3TestEnv() - : driver(TDriverConfig().SetEndpoint(Sprintf("localhost:%d", server.GetPort()))) + : driver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort()))) , tableClient(driver) , session(tableClient.CreateSession().ExtractValueSync().GetSession()) , s3Port(server.GetPortManager().GetPort()) @@ -313,7 +499,7 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) { // The exact values for Bucket, AccessKey and SecretKey do not matter if the S3 backend is TS3Mock. // Any non-empty strings should do. const auto exportSettings = NExport::TExportToS3Settings() - .Endpoint(Sprintf("localhost:%d", s3Port)) + .Endpoint(Sprintf("localhost:%u", s3Port)) .Scheme(ES3Scheme::HTTP) .Bucket("test_bucket") .AccessKey("test_key") @@ -331,10 +517,10 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) { void ImportFromS3(NImport::TImportClient& importClient, ui16 s3Port, NOperation::TOperationClient& operationClient, const TString& source, const TString& destination ) { - // The exact values for Bucket, AccessKey and SecretKey do not matter if the S3 backend is TS3Mock. + // The exact values for Bucket, AccessKey and SecretKey do not matter if the S3 backend is TS3Mock. // Any non-empty strings should do. const auto importSettings = NImport::TImportFromS3Settings() - .Endpoint(Sprintf("localhost:%d", s3Port)) + .Endpoint(Sprintf("localhost:%u", s3Port)) .Scheme(ES3Scheme::HTTP) .Bucket("test_bucket") .AccessKey("test_key") @@ -349,134 +535,92 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) { ); } - Y_UNIT_TEST(Basic) { - TS3TestEnv testEnv; + auto CreateBackupLambda(const TDriver& driver, ui16 s3Port) { + return [&driver, s3Port](const char* table) { + NExport::TExportClient exportClient(driver); + NOperation::TOperationClient operationClient(driver); + ExportToS3(exportClient, s3Port, operationClient, table, "table"); + }; + } - constexpr const char* table = "/Root/table"; - ExecuteDataDefinitionQuery(testEnv.GetSession(), Sprintf(R"( - CREATE TABLE `%s` ( - Key Uint32, - Value Utf8, - PRIMARY KEY (Key) - ); - )", - table - )); - ExecuteDataModificationQuery(testEnv.GetSession(), Sprintf(R"( - UPSERT INTO `%s` ( - Key, - Value - ) - VALUES - (1, "one"), - (2, "two"), - (3, "three"), - (4, "four"), - (5, "five"); - )", - table - )); - - NExport::TExportClient exportClient(testEnv.GetDriver()); - NImport::TImportClient importClient(testEnv.GetDriver()); - NOperation::TOperationClient operationClient(testEnv.GetDriver()); - - ExportToS3(exportClient, testEnv.GetS3Port(), operationClient, table, "table"); - - // The table needs to be dropped before importing from S3 can proceed successfully. - ExecuteDataDefinitionQuery(testEnv.GetSession(), Sprintf(R"( - DROP TABLE `%s`; - )", table - )); - - ImportFromS3(importClient, testEnv.GetS3Port(), operationClient, "table", table); - { - auto result = ExecuteDataModificationQuery(testEnv.GetSession(), Sprintf(R"( - SELECT COUNT(*) FROM `%s`; - )", table - )); - UNIT_ASSERT_VALUES_EQUAL(GetUint64(GetSingleResult(result)), 5ull); - } + auto CreateRestoreLambda(const TDriver& driver, ui16 s3Port) { + return [&driver, s3Port](const char* table) { + NImport::TImportClient importClient(driver); + NOperation::TOperationClient operationClient(driver); + ImportFromS3(importClient, s3Port, operationClient, "table", table); + }; } - Y_UNIT_TEST(RestoreTablePartitioningSettings) { + Y_UNIT_TEST(RestoreTableContent) { TS3TestEnv testEnv; - constexpr const char* table = "/Root/table"; - constexpr int minPartitions = 10; - ExecuteDataDefinitionQuery(testEnv.GetSession(), Sprintf(R"( - CREATE TABLE `%s` ( - Key Uint32, - Value Utf8, - PRIMARY KEY (Key) - ) - WITH ( - AUTO_PARTITIONING_BY_LOAD = ENABLED, - AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %d - ); - )", - table, minPartitions - )); - CheckTableDescription(testEnv.GetSession(), table, CreateMinPartitionsChecker(minPartitions)); - - NExport::TExportClient exportClient(testEnv.GetDriver()); - NImport::TImportClient importClient(testEnv.GetDriver()); - NOperation::TOperationClient operationClient(testEnv.GetDriver()); - - ExportToS3(exportClient, testEnv.GetS3Port(), operationClient, table, "table"); - - // The table needs to be dropped before importing from S3 can proceed successfully. - ExecuteDataDefinitionQuery(testEnv.GetSession(), Sprintf(R"( - DROP TABLE `%s`; - )", table - )); + TestTableContentIsPreserved( + table, + testEnv.GetSession(), + CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()), + CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port()) + ); + } - ImportFromS3(importClient, testEnv.GetS3Port(), operationClient, "table", table); - CheckTableDescription(testEnv.GetSession(), table, CreateMinPartitionsChecker(minPartitions)); + Y_UNIT_TEST(RestoreTablePartitioningSettings) { + TS3TestEnv testEnv; + constexpr const char* table = "/Root/table"; + constexpr ui32 minPartitions = 10; + + TestTablePartitioningSettingsArePreserved( + table, + minPartitions, + testEnv.GetSession(), + CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()), + CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port()) + ); } Y_UNIT_TEST(RestoreIndexTablePartitioningSettings) { TS3TestEnv testEnv; - constexpr const char* table = "/Root/table"; constexpr const char* index = "byValue"; - const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable"); - constexpr int minPartitions = 10; - ExecuteDataDefinitionQuery(testEnv.GetSession(), Sprintf(R"( - CREATE TABLE `%s` ( - Key Uint32, - Value Uint32, - PRIMARY KEY (Key), - INDEX %s GLOBAL ON (Value) - ); - )", - table, index - )); - ExecuteDataDefinitionQuery(testEnv.GetSession(), Sprintf(R"( - ALTER TABLE `%s` ALTER INDEX %s SET ( - AUTO_PARTITIONING_BY_LOAD = ENABLED, - AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %d - ); - )", table, index, minPartitions - )); - - CheckTableDescription(testEnv.GetSession(), indexTablePath, CreateMinPartitionsChecker(minPartitions)); - - NExport::TExportClient exportClient(testEnv.GetDriver()); - NImport::TImportClient importClient(testEnv.GetDriver()); - NOperation::TOperationClient operationClient(testEnv.GetDriver()); - - ExportToS3(exportClient, testEnv.GetS3Port(), operationClient, table, "table"); + constexpr ui32 minIndexPartitions = 10; + + TestIndexTablePartitioningSettingsArePreserved( + table, + index, + minIndexPartitions, + testEnv.GetSession(), + CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()), + CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port()) + ); + } - // The table needs to be dropped before importing from S3 can proceed successfully. - ExecuteDataDefinitionQuery(testEnv.GetSession(), Sprintf(R"( - DROP TABLE `%s`; - )", table - )); + Y_UNIT_TEST(RestoreTableSplitBoundaries) { + TS3TestEnv testEnv; + constexpr const char* table = "/Root/table"; + constexpr ui64 partitions = 10; + + TestTableSplitBoundariesArePreserved( + table, + partitions, + testEnv.GetSession(), + CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()), + CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port()) + ); + } - ImportFromS3(importClient, testEnv.GetS3Port(), operationClient, "table", table); - CheckTableDescription(testEnv.GetSession(), indexTablePath, CreateMinPartitionsChecker(minPartitions)); + Y_UNIT_TEST(RestoreIndexTableSplitBoundaries) { + TS3TestEnv testEnv; + constexpr const char* table = "/Root/table"; + constexpr const char* index = "byValue"; + constexpr ui64 indexPartitions = 10; + + TestIndexTableSplitBoundariesArePreserved( + table, + index, + indexPartitions, + testEnv.GetSession(), + CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()), + CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port()) + ); } }