Skip to content

Restore indexes from backup with the original partitioning #7589

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jepett0 merged 8 commits into from
Aug 21, 2024
Merged

Restore indexes from backup with the original partitioning #7589

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f27a8e7
refactoring to reduce (future) code duplication
jepett0 Aug 16, 2024
c5142da
bugfix
jepett0 Aug 16, 2024
8ed6cc2
enable indexImplTable split boundaries export
jepett0 Aug 16, 2024
2440d3f
refactoring in tests
jepett0 Aug 16, 2024
c5b1522
supporting changes in SDK
jepett0 Aug 16, 2024
5798b1c
add new tests
jepett0 Aug 16, 2024
f0791ef
review fix #1
jepett0 Aug 20, 2024
8a1999f
review fix #2
jepett0 Aug 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ydb/core/protos/flat_scheme_op.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1837,6 +1837,7 @@ message TDescribeOptions {
optional bool ReturnChannelsBinding = 8 [default = false];
optional bool ReturnRangeKey = 9 [default = true];
optional bool ReturnSetVal = 10 [default = false];
optional bool ReturnIndexTableBoundaries = 11 [default = false];
}

// Request to read scheme for a specific path
Expand Down
1 change: 1 addition & 0 deletions ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ static NKikimrSchemeOp::TPathDescription GetTableDescription(TSchemeShard* ss, c
opts.SetReturnPartitioningInfo(false);
opts.SetReturnPartitionConfig(true);
opts.SetReturnBoundaries(true);
opts.SetReturnIndexTableBoundaries(true);

auto desc = DescribePath(ss, TlsActivationContext->AsActorContext(), pathId, opts);
auto record = desc->GetRecord();
Expand Down
142 changes: 82 additions & 60 deletions ydb/core/tx/schemeshard/schemeshard_path_describer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,79 @@ static void FillTableStats(NKikimrSchemeOp::TPathDescription& pathDescription, c
FillTableMetrics(pathDescription.MutableTabletMetrics(), stats);
}

static void FillColumns(
const TTableInfo& tableInfo,
google::protobuf::RepeatedPtrField<NKikimrSchemeOp::TColumnDescription>& out
) {
bool familyNamesBuilt = false;
THashMap<ui32, TString> familyNames;

out.Reserve(tableInfo.Columns.size());
for (const auto& col : tableInfo.Columns) {
const auto& cinfo = col.second;
if (cinfo.IsDropped())
continue;

auto* colDescr = out.Add();
colDescr->SetName(cinfo.Name);
colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod));
auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod);
colDescr->SetTypeId(columnType.TypeId);
if (columnType.TypeInfo) {
*colDescr->MutableTypeInfo() = *columnType.TypeInfo;
}
colDescr->SetId(cinfo.Id);
colDescr->SetNotNull(cinfo.NotNull);

if (cinfo.Family != 0) {
colDescr->SetFamily(cinfo.Family);

if (!familyNamesBuilt) {
for (const auto& family : tableInfo.PartitionConfig().GetColumnFamilies()) {
if (family.HasName() && family.HasId()) {
familyNames[family.GetId()] = family.GetName();
}
}
familyNamesBuilt = true;
}

auto it = familyNames.find(cinfo.Family);
if (it != familyNames.end() && !it->second.empty()) {
colDescr->SetFamilyName(it->second);
}
}

colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress);

switch (cinfo.DefaultKind) {
case ETableColumnDefaultKind::None:
break;
case ETableColumnDefaultKind::FromSequence:
colDescr->SetDefaultFromSequence(cinfo.DefaultValue);
break;
case ETableColumnDefaultKind::FromLiteral:
Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString(
cinfo.DefaultValue));
break;
}
}
}

static void FillKeyColumns(
const TTableInfo& tableInfo,
google::protobuf::RepeatedPtrField<TProtoStringType>& names,
google::protobuf::RepeatedField<ui32>& ids
) {
Y_ABORT_UNLESS(!tableInfo.KeyColumnIds.empty());
names.Reserve(tableInfo.KeyColumnIds.size());
ids.Reserve(tableInfo.KeyColumnIds.size());
for (ui32 keyColId : tableInfo.KeyColumnIds) {
*names.Add() = tableInfo.Columns.at(keyColId).Name;
*ids.Add() = keyColId;
}

}

void TPathDescriber::FillPathDescr(NKikimrSchemeOp::TDirEntry* descr, TPathElement::TPtr pathEl, TPathElement::EPathSubType subType) {
FillChildDescr(descr, pathEl);

Expand Down Expand Up @@ -303,6 +376,7 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa
bool returnBoundaries = false;
bool returnRangeKey = true;
bool returnSetVal = Params.GetOptions().GetReturnSetVal();
bool returnIndexTableBoundaries = Params.GetOptions().GetReturnIndexTableBoundaries();
if (Params.HasOptions()) {
returnConfig = Params.GetOptions().GetReturnPartitionConfig();
returnPartitioning = Params.GetOptions().GetReturnPartitioningInfo();
Expand Down Expand Up @@ -427,7 +501,9 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa

switch (childPath->PathType) {
case NKikimrSchemeOp::EPathTypeTableIndex:
Self->DescribeTableIndex(childPathId, childName, returnConfig, false, *entry->AddTableIndexes());
Self->DescribeTableIndex(
childPathId, childName, returnConfig, returnIndexTableBoundaries, *entry->AddTableIndexes()
);
break;
case NKikimrSchemeOp::EPathTypeCdcStream:
Self->DescribeCdcStream(childPathId, childName, *entry->AddCdcStreams());
Expand Down Expand Up @@ -1189,67 +1265,10 @@ void TSchemeShard::DescribeTable(
) const
{
Y_UNUSED(typeRegistry);
THashMap<ui32, TString> familyNames;
bool familyNamesBuilt = false;

entry->SetTableSchemaVersion(tableInfo.AlterVersion);
entry->MutableColumns()->Reserve(tableInfo.Columns.size());
for (auto col : tableInfo.Columns) {
const auto& cinfo = col.second;
if (cinfo.IsDropped())
continue;

auto colDescr = entry->AddColumns();
colDescr->SetName(cinfo.Name);
colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod));
auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod);
colDescr->SetTypeId(columnType.TypeId);
if (columnType.TypeInfo) {
*colDescr->MutableTypeInfo() = *columnType.TypeInfo;
}
colDescr->SetId(cinfo.Id);
colDescr->SetNotNull(cinfo.NotNull);

if (cinfo.Family != 0) {
colDescr->SetFamily(cinfo.Family);

if (!familyNamesBuilt) {
for (const auto& family : tableInfo.PartitionConfig().GetColumnFamilies()) {
if (family.HasName() && family.HasId()) {
familyNames[family.GetId()] = family.GetName();
}
}
familyNamesBuilt = true;
}

auto it = familyNames.find(cinfo.Family);
if (it != familyNames.end() && !it->second.empty()) {
colDescr->SetFamilyName(it->second);
}
}

colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress);

switch (cinfo.DefaultKind) {
case ETableColumnDefaultKind::None:
break;
case ETableColumnDefaultKind::FromSequence:
colDescr->SetDefaultFromSequence(cinfo.DefaultValue);
break;
case ETableColumnDefaultKind::FromLiteral:
Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString(
cinfo.DefaultValue));
break;
}
}
Y_ABORT_UNLESS(!tableInfo.KeyColumnIds.empty());

entry->MutableKeyColumnNames()->Reserve(tableInfo.KeyColumnIds.size());
entry->MutableKeyColumnIds()->Reserve(tableInfo.KeyColumnIds.size());
for (ui32 keyColId : tableInfo.KeyColumnIds) {
entry->AddKeyColumnNames(tableInfo.Columns.at(keyColId).Name);
entry->AddKeyColumnIds(keyColId);
}
FillColumns(tableInfo, *entry->MutableColumns());
FillKeyColumns(tableInfo, *entry->MutableKeyColumnNames(), *entry->MutableKeyColumnIds());

if (fillConfig) {
FillPartitionConfig(tableInfo.PartitionConfig(), *entry->MutablePartitionConfig());
Expand Down Expand Up @@ -1324,6 +1343,9 @@ void TSchemeShard::DescribeTableIndex(const TPathId& pathId, const TString& name
FillPartitionConfig(tableInfo.PartitionConfig(), *tableDescription->MutablePartitionConfig());
}
if (fillBoundaries) {
// column info is necessary for split boundary type conversion
FillColumns(tableInfo, *tableDescription->MutableColumns());
FillKeyColumns(tableInfo, *tableDescription->MutableKeyColumnNames(), *tableDescription->MutableKeyColumnIds());
FillTableBoundaries(tableDescription->MutableSplitBoundary(), tableInfo);
}
}
Expand Down
4 changes: 2 additions & 2 deletions ydb/core/ydb_convert/table_description.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -852,8 +852,8 @@ void FillGlobalIndexSettings(Ydb::Table::GlobalIndexSettings& settings,
NKikimrMiniKQL::TType splitKeyType;
Ydb::Table::DescribeTableResult unused;
FillColumnDescription(unused, splitKeyType, indexImplTableDescription);
FillTableBoundaryImpl(
*settings.mutable_partition_at_keys(),
FillTableBoundaryImpl<Ydb::Table::GlobalIndexSettings>(
settings,
indexImplTableDescription,
splitKeyType
);
Expand Down
21 changes: 17 additions & 4 deletions ydb/public/sdk/cpp/client/ydb_table/table.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,6 +469,10 @@ class TTableDescription::TImpl {
Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, dataColumns));
}

void AddSecondaryIndex(const TIndexDescription& indexDescription) {
Indexes_.emplace_back(indexDescription);
}

void AddVectorIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVectorIndexSettings& vectorIndexSettings) {
Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, {}, {}, vectorIndexSettings));
}
Expand Down Expand Up @@ -749,6 +753,10 @@ void TTableDescription::AddSecondaryIndex(const TString& indexName, EIndexType t
Impl_->AddSecondaryIndex(indexName, type, indexColumns, dataColumns);
}

void TTableDescription::AddSecondaryIndex(const TIndexDescription& indexDescription) {
Impl_->AddSecondaryIndex(indexDescription);
}

void TTableDescription::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) {
AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns);
}
Expand Down Expand Up @@ -1173,6 +1181,11 @@ TTableBuilder& TTableBuilder::SetPrimaryKeyColumn(const TString& primaryKeyColum
return *this;
}

TTableBuilder& TTableBuilder::AddSecondaryIndex(const TIndexDescription& indexDescription) {
TableDescription_.AddSecondaryIndex(indexDescription);
return *this;
}

TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) {
TableDescription_.AddSecondaryIndex(indexName, type, indexColumns, dataColumns);
return *this;
Expand Down Expand Up @@ -2364,7 +2377,7 @@ TVectorIndexSettings TVectorIndexSettings::FromProto(const TProto& proto) {
default:
return EVectorType::Unknown;
}
};
};


auto metricFromProto = [&](const auto& proto) -> TVectorIndexSettings::TMetric {
Expand All @@ -2376,7 +2389,7 @@ TVectorIndexSettings TVectorIndexSettings::FromProto(const TProto& proto) {
default:
return {};
}
};
};

return {
.Metric = metricFromProto(proto),
Expand Down Expand Up @@ -2424,8 +2437,8 @@ void TVectorIndexSettings::SerializeTo(Ydb::Table::VectorIndexSettings& settings
return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UNSPECIFIED;
}
};


if (const auto* distance = std::get_if<EDistance>(&Metric)) {
settings.set_distance(convertDistance(*distance));
} else if (const auto* similarity = std::get_if<ESimilarity>(&Metric)) {
Expand Down
6 changes: 4 additions & 2 deletions ydb/public/sdk/cpp/client/ydb_table/table.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,10 +174,10 @@ struct TExplicitPartitions {
using TSelf = TExplicitPartitions;

FLUENT_SETTING_VECTOR(TValue, SplitPoints);

template <typename TProto>
static TExplicitPartitions FromProto(const TProto& proto);

void SerializeTo(Ydb::Table::ExplicitPartitions& proto) const;
};

Expand Down Expand Up @@ -642,6 +642,7 @@ class TTableDescription {
// common
void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns);
void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
void AddSecondaryIndex(const TIndexDescription& indexDescription);
// sync
void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns);
void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
Expand Down Expand Up @@ -855,6 +856,7 @@ class TTableBuilder {
TTableBuilder& SetPrimaryKeyColumn(const TString& primaryKeyColumn);

// common
TTableBuilder& AddSecondaryIndex(const TIndexDescription& indexDescription);
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns);
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TString& indexColumn);
Expand Down
Loading