diff --git a/ydb/core/grpc_services/rpc_create_table.cpp b/ydb/core/grpc_services/rpc_create_table.cpp index 556004f466d0..88fed4de46d3 100644 --- a/ydb/core/grpc_services/rpc_create_table.cpp +++ b/ydb/core/grpc_services/rpc_create_table.cpp @@ -181,8 +181,28 @@ class TCreateTableRPC : public TRpcSchemeRequestActorcolumns()) { + switch (column.default_value_case()) { + case Ydb::Table::ColumnMeta::kFromSequence: { + auto* seqDesc = modifyScheme->MutableCreateIndexedTable()->MutableSequenceDescription()->Add(); + if (!FillSequenceDescription(*seqDesc, column.from_sequence(), code, error)) { + NYql::TIssues issues; + issues.AddIssue(NYql::TIssue(error)); + return Reply(code, issues, ctx); + } + hasSerial = true; + break; + } + default: break; + } + } + NKikimrSchemeOp::TTableDescription* tableDesc = nullptr; - if (req->indexesSize()) { + if (req->indexesSize() || hasSerial) { modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexedTable); tableDesc = modifyScheme->MutableCreateIndexedTable()->MutableTableDescription(); } else { @@ -192,9 +212,6 @@ class TCreateTableRPC : public TRpcSchemeRequestActorSetName(name); - StatusIds::StatusCode code = StatusIds::SUCCESS; - TString error; - if (!FillColumnDescription(*tableDesc, req->columns(), code, error)) { NYql::TIssues issues; issues.AddIssue(NYql::TIssue(error)); diff --git a/ydb/core/grpc_services/rpc_describe_table.cpp b/ydb/core/grpc_services/rpc_describe_table.cpp index 62e1dfd2d5c5..c4fa00af373b 100644 --- a/ydb/core/grpc_services/rpc_describe_table.cpp +++ b/ydb/core/grpc_services/rpc_describe_table.cpp @@ -142,6 +142,14 @@ class TDescribeTableRPC : public TRpcSchemeRequestActorRaiseIssue(NYql::TIssue(error)); + return Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx); + } + describeTableResult.mutable_primary_key()->CopyFrom(tableDescription.GetKeyColumnNames()); try { @@ -212,6 +220,10 @@ class TDescribeTableRPC : public TRpcSchemeRequestActorMutableOptions()->SetReturnPartitionStats(true); } + if (req->include_set_val()) { + record->MutableOptions()->SetReturnSetVal(true); + } + record->MutableOptions()->SetShowPrivateTable(ShowPrivatePath(path)); ctx.Send(MakeTxProxyID(), navigateRequest.release()); diff --git a/ydb/core/tx/datashard/export_common.cpp b/ydb/core/tx/datashard/export_common.cpp index c72bd371fd7a..cf999f1f76a7 100644 --- a/ydb/core/tx/datashard/export_common.cpp +++ b/ydb/core/tx/datashard/export_common.cpp @@ -66,7 +66,12 @@ TMaybe GenYdbScheme( FillPartitioningSettings(scheme, tableDesc); FillKeyBloomFilter(scheme, tableDesc); FillReadReplicasSettings(scheme, tableDesc); - FillSequenceDescription(scheme, tableDesc); + + TString error; + Ydb::StatusIds::StatusCode status; + if (!FillSequenceDescription(scheme, tableDesc, status, error)) { + return Nothing(); + } return scheme; } diff --git a/ydb/core/ydb_convert/table_description.cpp b/ydb/core/ydb_convert/table_description.cpp index 979b59f6fc60..2744041ea50d 100644 --- a/ydb/core/ydb_convert/table_description.cpp +++ b/ydb/core/ydb_convert/table_description.cpp @@ -1390,7 +1390,6 @@ bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out, const Ydb::Table::CreateTableRequest& in, const TTableProfiles& profiles, Ydb::StatusIds::StatusCode& status, TString& error, bool indexedTable) { - NKikimrSchemeOp::TTableDescription* tableDesc = nullptr; if (indexedTable) { tableDesc = out.MutableCreateIndexedTable()->MutableTableDescription(); @@ -1432,7 +1431,10 @@ bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out, return true; } -void FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in) { +template +bool FillSequenceDescriptionImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) { + Y_UNUSED(status); + Y_UNUSED(error); THashMap sequences; for (const auto& sequenceDescription : in.GetSequences()) { @@ -1478,6 +1480,45 @@ void FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrS default: break; } } + return true; +} + +bool FillSequenceDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) { + return FillSequenceDescriptionImpl(out, in, status, error); +} + +bool FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) { + return FillSequenceDescriptionImpl(out, in, status, error); +} + +bool FillSequenceDescription(NKikimrSchemeOp::TSequenceDescription& out, const Ydb::Table::SequenceDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) { + Y_UNUSED(status); + Y_UNUSED(error); + out.SetName(in.name()); + if (in.has_min_value()) { + out.SetMinValue(in.min_value()); + } + if (in.has_max_value()) { + out.SetMaxValue(in.max_value()); + } + if (in.has_start_value()) { + out.SetStartValue(in.start_value()); + } + if (in.has_cache()) { + out.SetCache(in.cache()); + } + if (in.has_increment()) { + out.SetIncrement(in.increment()); + } + if (in.has_cycle()) { + out.SetCycle(in.cycle()); + } + if (in.has_set_val()) { + auto* setVal = out.MutableSetVal(); + setVal->SetNextUsed(in.set_val().next_used()); + setVal->SetNextValue(in.set_val().next_value()); + } + return true; } } // namespace NKikimr diff --git a/ydb/core/ydb_convert/table_description.h b/ydb/core/ydb_convert/table_description.h index d76722e58446..aada5d1f767c 100644 --- a/ydb/core/ydb_convert/table_description.h +++ b/ydb/core/ydb_convert/table_description.h @@ -133,11 +133,13 @@ bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out, // out -void FillSequenceDescription(Ydb::Table::CreateTableRequest& out, - const NKikimrSchemeOp::TTableDescription& in); +bool FillSequenceDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error); -// out -void FillSequenceDescription(Ydb::Table::CreateTableRequest& out, - const NKikimrSchemeOp::TTableDescription& in); +bool FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error); + +// in +bool FillSequenceDescription( + NKikimrSchemeOp::TSequenceDescription& out, const Ydb::Table::SequenceDescription& in, + Ydb::StatusIds::StatusCode& status, TString& error); } // namespace NKikimr diff --git a/ydb/library/backup/backup.cpp b/ydb/library/backup/backup.cpp index 223837d36649..8f247809b16c 100644 --- a/ydb/library/backup/backup.cpp +++ b/ydb/library/backup/backup.cpp @@ -372,7 +372,7 @@ NTable::TTableDescription DescribeTable(TDriver driver, const TString& fullTable NTable::TTableClient client(driver); TStatus status = client.RetryOperationSync([fullTablePath, &desc](NTable::TSession session) { - auto settings = NTable::TDescribeTableSettings().WithKeyShardBoundary(true); + auto settings = NTable::TDescribeTableSettings().WithKeyShardBoundary(true).WithSetVal(true); auto result = session.DescribeTable(fullTablePath, settings).GetValueSync(); VerifyStatus(result); @@ -489,6 +489,7 @@ void BackupTable(TDriver driver, const TString& dbPrefix, const TString& backupP << " backupPrefix: " << backupPrefix << " path: " << path); auto desc = DescribeTable(driver, JoinDatabasePath(schemaOnly ? dbPrefix : backupPrefix, path)); + auto proto = ProtoFromTableDescription(desc, preservePoolKinds); TString schemaStr; @@ -718,19 +719,26 @@ void BackupFolder(TDriver driver, const TString& database, const TString& relDbP // Restore //////////////////////////////////////////////////////////////////////////////// -TString ProcessColumnType(const TString& name, TTypeParser parser, NTable::TTableBuilder *builder) { +TString ProcessColumnType(const TString& name, TTypeParser parser, NTable::TTableBuilder *builder, std::optional sequenceDescription) { TStringStream ss; ss << "name: " << name << "; "; if (parser.GetKind() == TTypeParser::ETypeKind::Optional) { ss << " optional; "; parser.OpenOptional(); } + if (sequenceDescription.has_value()) { + ss << "serial; "; + } ss << "kind: " << parser.GetKind() << "; "; switch (parser.GetKind()) { case TTypeParser::ETypeKind::Primitive: ss << " type_id: " << parser.GetPrimitive() << "; "; if (builder) { - builder->AddNullableColumn(name, parser.GetPrimitive()); + if (sequenceDescription.has_value()) { + builder->AddSerialColumn(name, parser.GetPrimitive(), std::move(*sequenceDescription)); + } else { + builder->AddNullableColumn(name, parser.GetPrimitive()); + } } break; case TTypeParser::ETypeKind::Decimal: @@ -751,8 +759,19 @@ TString ProcessColumnType(const TString& name, TTypeParser parser, NTable::TTabl NTable::TTableDescription TableDescriptionFromProto(const Ydb::Table::CreateTableRequest& proto) { NTable::TTableBuilder builder; + std::optional sequenceDescription; for (const auto &col : proto.Getcolumns()) { - LOG_DEBUG("AddNullableColumn: " << ProcessColumnType(col.Getname(), TType(col.Gettype()), &builder)); + if (col.from_sequence().name() == "_serial_column_" + col.name()) { + NTable::TSequenceDescription currentSequenceDescription; + if (col.from_sequence().has_set_val()) { + NTable::TSequenceDescription::TSetVal setVal; + setVal.NextUsed = col.from_sequence().set_val().next_used(); + setVal.NextValue = col.from_sequence().set_val().next_value(); + currentSequenceDescription.SetVal = std::move(setVal); + } + sequenceDescription = std::move(currentSequenceDescription); + } + LOG_DEBUG("AddColumn: " << ProcessColumnType(col.Getname(), TType(col.Gettype()), &builder, std::move(sequenceDescription))); } for (const auto &primary : proto.Getprimary_key()) { @@ -781,7 +800,7 @@ TString SerializeColumnsToString(const TVector& columns, TVector(); auto extractor = [promise, settings] diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index 7067585f5663..656f0b087e00 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -290,7 +290,24 @@ class TTableDescription::TImpl { if (col.has_not_null()) { not_null = col.not_null(); } - Columns_.emplace_back(col.name(), col.type(), col.family(), not_null); + std::optional sequenceDescription; + switch (col.default_value_case()) { + case Ydb::Table::ColumnMeta::kFromSequence: { + if (col.from_sequence().name() == "_serial_column_" + col.name()) { + TSequenceDescription currentSequenceDescription; + if (col.from_sequence().has_set_val()) { + TSequenceDescription::TSetVal setVal; + setVal.NextUsed = col.from_sequence().set_val().next_used(); + setVal.NextValue = col.from_sequence().set_val().next_value(); + currentSequenceDescription.SetVal = std::move(setVal); + } + sequenceDescription = std::move(currentSequenceDescription); + } + break; + } + default: break; + } + Columns_.emplace_back(col.name(), col.type(), col.family(), not_null, std::move(sequenceDescription)); } // indexes @@ -452,8 +469,8 @@ class TTableDescription::TImpl { return Proto_; } - void AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional notNull) { - Columns_.emplace_back(name, type, family, notNull); + void AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional notNull, std::optional sequenceDescription) { + Columns_.emplace_back(name, type, family, notNull, std::move(sequenceDescription)); } void SetPrimaryKeyColumns(const TVector& primaryKeyColumns) { @@ -724,8 +741,8 @@ const TVector& TTableDescription::GetKeyRanges() const { return Impl_->GetKeyRanges(); } -void TTableDescription::AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional notNull) { - Impl_->AddColumn(name, type, family, notNull); +void TTableDescription::AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional notNull, std::optional sequenceDescription) { + Impl_->AddColumn(name, type, family, notNull, std::move(sequenceDescription)); } void TTableDescription::SetPrimaryKeyColumns(const TVector& primaryKeyColumns) { @@ -889,6 +906,15 @@ void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) con if (column.NotNull.has_value()) { protoColumn.set_not_null(column.NotNull.value()); } + if (column.SequenceDescription.has_value()) { + auto* fromSequence = protoColumn.mutable_from_sequence(); + if (column.SequenceDescription->SetVal.has_value()) { + auto* setVal = fromSequence->mutable_set_val(); + setVal->set_next_value(column.SequenceDescription->SetVal->NextValue); + setVal->set_next_used(column.SequenceDescription->SetVal->NextUsed); + } + fromSequence->set_name("_serial_column_" + column.Name); + } } for (const auto& pk : Impl_->GetPrimaryKeyColumns()) { @@ -1096,7 +1122,7 @@ TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const EPrim .EndOptional() .Build(); - TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false); + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false, std::nullopt); return *this; } @@ -1106,7 +1132,7 @@ TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const TDeci .Decimal(type) .EndOptional() .Build(); - TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false); + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false, std::nullopt); return *this; } @@ -1115,7 +1141,7 @@ TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const TPgTy .Pg(type) .Build(); - TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false); + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false, std::nullopt); return *this; } @@ -1124,7 +1150,7 @@ TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const EP .Primitive(type) .Build(); - TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true); + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::nullopt); return *this; } @@ -1133,7 +1159,7 @@ TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const TD .Decimal(type) .Build(); - TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true); + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::nullopt); return *this; } @@ -1142,7 +1168,16 @@ TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const TP .Pg(type) .Build(); - TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true); + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::nullopt); + return *this; +} + +TTableBuilder& TTableBuilder::AddSerialColumn(const TString& name, const EPrimitiveType& type, TSequenceDescription sequenceDescription, const TString& family) { + auto columnType = TTypeBuilder() + .Primitive(type) + .Build(); + + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::move(sequenceDescription)); return *this; } diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index 4e20f5259f38..ba54abb380d8 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -99,19 +99,29 @@ class TKeyRange { TMaybe To_; }; +struct TSequenceDescription { + struct TSetVal { + i64 NextValue; + bool NextUsed; + }; + std::optional SetVal; +}; + struct TTableColumn { TString Name; TType Type; TString Family; std::optional NotNull; + std::optional SequenceDescription; TTableColumn() = default; - TTableColumn(TString name, TType type, TString family = TString(), std::optional notNull = std::nullopt) + TTableColumn(TString name, TType type, TString family = TString(), std::optional notNull = std::nullopt, std::optional sequenceDescription = std::nullopt) : Name(std::move(name)) , Type(std::move(type)) , Family(std::move(family)) , NotNull(std::move(notNull)) + , SequenceDescription(std::move(sequenceDescription)) { } // Conversion from TColumn for API compatibility @@ -593,7 +603,7 @@ class TTableDescription { TTableDescription(); explicit TTableDescription(const Ydb::Table::CreateTableRequest& request); - void AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional notNull); + void AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional notNull, std::optional sequenceDescription); void SetPrimaryKeyColumns(const TVector& primaryKeyColumns); // common @@ -807,6 +817,7 @@ class TTableBuilder { TTableBuilder& AddNonNullableColumn(const TString& name, const TPgType& type, const TString& family = TString()); TTableBuilder& SetPrimaryKeyColumns(const TVector& primaryKeyColumns); TTableBuilder& SetPrimaryKeyColumn(const TString& primaryKeyColumn); + TTableBuilder& AddSerialColumn(const TString& name, const EPrimitiveType& type, TSequenceDescription sequenceDescription, const TString& family = TString()); // common TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector& indexColumns, const TVector& dataColumns); @@ -1577,6 +1588,7 @@ struct TDescribeTableSettings : public TOperationRequestSettings { diff --git a/ydb/services/ydb/ydb_import_ut.cpp b/ydb/services/ydb/ydb_import_ut.cpp index a423246c30a5..3c93e2b57d23 100644 --- a/ydb/services/ydb/ydb_import_ut.cpp +++ b/ydb/services/ydb/ydb_import_ut.cpp @@ -176,7 +176,7 @@ Y_UNIT_TEST_SUITE(BackupRestore) { } void CheckTableDescription(TSession& session, const TString& path, auto&& checker) { - auto describeResult = session.DescribeTable(path).ExtractValueSync(); + auto describeResult = session.DescribeTable(path, TDescribeTableSettings().WithSetVal(true)).ExtractValueSync(); UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); auto tableDescription = describeResult.GetTableDescription(); Ydb::Table::CreateTableRequest descriptionProto; @@ -188,6 +188,21 @@ Y_UNIT_TEST_SUITE(BackupRestore) { ); } + auto CreateHasSerialChecker(i64 nextValue, bool nextUsed) { + return [=](const TTableDescription& tableDescription) { + for (const auto& column : tableDescription.GetTableColumns()) { + if (column.Name == "Key") { + UNIT_ASSERT(column.SequenceDescription.has_value()); + UNIT_ASSERT(column.SequenceDescription->SetVal.has_value()); + UNIT_ASSERT_VALUES_EQUAL(column.SequenceDescription->SetVal->NextValue, nextValue); + UNIT_ASSERT_VALUES_EQUAL(column.SequenceDescription->SetVal->NextUsed, nextUsed); + return true; + } + } + return false; + }; + } + Y_UNIT_TEST(Basic) { TKikimrWithGrpcAndRootSchema server; auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%d", server.GetPort()))); @@ -341,4 +356,46 @@ Y_UNIT_TEST_SUITE(BackupRestore) { CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minPartitions)); } + Y_UNIT_TEST(BasicRestoreTableWithSerial) { + TKikimrWithGrpcAndRootSchema server; + auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%d", server.GetPort()))); + TTableClient tableClient(driver); + auto session = tableClient.GetSession().ExtractValueSync().GetSession(); + + constexpr const char* table = "/Root/table"; + + ExecuteDataDefinitionQuery(session, Sprintf(R"( + CREATE TABLE `%s` ( + Key Serial, + Value Uint32, + PRIMARY KEY (Key) + ); + )", + table + )); + ExecuteDataModificationQuery(session, Sprintf(R"( + UPSERT INTO `%s` ( + Value + ) + VALUES (1), (2), (3), (4), (5), (6), (7); + )", + table + )); + + TTempDir tempDir; + const auto& pathToBackup = tempDir.Path(); + // TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder + NYdb::NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, false, false); + + NDump::TClient backupClient(driver); + + // restore deleted table + ExecuteDataDefinitionQuery(session, Sprintf(R"( + DROP TABLE `%s`; + )", table + )); + Restore(backupClient, pathToBackup, "/Root"); + CheckTableDescription(session, table, CreateHasSerialChecker(8, false)); + } + }