Support ydb dump for tables with serial types

ydb/core/grpc_services/rpc_create_table.cpp

@@ -181,8 +181,28 @@ class TCreateTableRPC : public TRpcSchemeRequestActor<TCreateTableRPC, TEvCreate
             return;
         }
 
+        StatusIds::StatusCode code = StatusIds::SUCCESS;
+        TString error;
+
+        bool hasSerial = false;
+        for (const auto& column : req->columns()) {
+            switch (column.default_value_case()) {
+                case Ydb::Table::ColumnMeta::kFromSequence: {
+                    auto* seqDesc = modifyScheme->MutableCreateIndexedTable()->MutableSequenceDescription()->Add();
+                    if (!FillSequenceDescription(*seqDesc, column.from_sequence(), code, error)) {
+                        NYql::TIssues issues;
+                        issues.AddIssue(NYql::TIssue(error));
+                        return Reply(code, issues, ctx);
+                    }
+                    hasSerial = true;
+                    break;
+                }
+                default: break;
+            }
+        }
+
         NKikimrSchemeOp::TTableDescription* tableDesc = nullptr;
-        if (req->indexesSize()) {
+        if (req->indexesSize() || hasSerial) {
             modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexedTable);
             tableDesc = modifyScheme->MutableCreateIndexedTable()->MutableTableDescription();
         } else {
@@ -192,9 +212,6 @@ class TCreateTableRPC : public TRpcSchemeRequestActor<TCreateTableRPC, TEvCreate
 
         tableDesc->SetName(name);
 
-        StatusIds::StatusCode code = StatusIds::SUCCESS;
-        TString error;
-
         if (!FillColumnDescription(*tableDesc, req->columns(), code, error)) {
             NYql::TIssues issues;
             issues.AddIssue(NYql::TIssue(error));

ydb/core/grpc_services/rpc_describe_table.cpp

@@ -142,6 +142,14 @@ class TDescribeTableRPC : public TRpcSchemeRequestActor<TDescribeTableRPC, TEvDe
                     return Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx);
                 }
 
+                StatusIds::StatusCode code = StatusIds::SUCCESS;
+                TString error;
+                if (!FillSequenceDescription(describeTableResult, tableDescription, code, error)) {
+                    LOG_ERROR(ctx, NKikimrServices::GRPC_SERVER, "Unable to fill sequence description: %s", error.c_str());
+                    Request_->RaiseIssue(NYql::TIssue(error));
+                    return Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx);
+                }
+
                 describeTableResult.mutable_primary_key()->CopyFrom(tableDescription.GetKeyColumnNames());
 
                 try {
@@ -212,6 +220,10 @@ class TDescribeTableRPC : public TRpcSchemeRequestActor<TDescribeTableRPC, TEvDe
             record->MutableOptions()->SetReturnPartitionStats(true);
         }
 
+        if (req->include_set_val()) {
+            record->MutableOptions()->SetReturnSetVal(true);
+        }
+
         record->MutableOptions()->SetShowPrivateTable(ShowPrivatePath(path));
 
         ctx.Send(MakeTxProxyID(), navigateRequest.release());

ydb/core/tx/datashard/export_common.cpp

@@ -66,7 +66,12 @@ TMaybe<Ydb::Table::CreateTableRequest> GenYdbScheme(
     FillPartitioningSettings(scheme, tableDesc);
     FillKeyBloomFilter(scheme, tableDesc);
     FillReadReplicasSettings(scheme, tableDesc);
-    FillSequenceDescription(scheme, tableDesc);
+
+    TString error;
+    Ydb::StatusIds::StatusCode status;
+    if (!FillSequenceDescription(scheme, tableDesc, status, error)) {
+        return Nothing();
+    }
 
     return scheme;
 }

ydb/core/ydb_convert/table_description.cpp

@@ -1390,7 +1390,6 @@ bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out,
         const Ydb::Table::CreateTableRequest& in, const TTableProfiles& profiles,
         Ydb::StatusIds::StatusCode& status, TString& error, bool indexedTable)
 {
-
     NKikimrSchemeOp::TTableDescription* tableDesc = nullptr;
     if (indexedTable) {
         tableDesc = out.MutableCreateIndexedTable()->MutableTableDescription();
@@ -1432,7 +1431,10 @@ bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out,
     return true;
 }
 
-void FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in) {
+template <typename TYdbProto>
+bool FillSequenceDescriptionImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) {
+    Y_UNUSED(status);
+    Y_UNUSED(error);
     THashMap<TString, NKikimrSchemeOp::TSequenceDescription> sequences;
 
     for (const auto& sequenceDescription : in.GetSequences()) {
@@ -1478,6 +1480,45 @@ void FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrS
             default: break;
         }
     }
+    return true;
+}
+
+bool FillSequenceDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) {
+    return FillSequenceDescriptionImpl(out, in, status, error);
+}
+
+bool FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) {
+    return FillSequenceDescriptionImpl(out, in, status, error);
+}
+
+bool FillSequenceDescription(NKikimrSchemeOp::TSequenceDescription& out, const Ydb::Table::SequenceDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) {
+    Y_UNUSED(status);
+    Y_UNUSED(error);
+    out.SetName(in.name());
+    if (in.has_min_value()) {
+        out.SetMinValue(in.min_value());
+    }
+    if (in.has_max_value()) {
+        out.SetMaxValue(in.max_value());
+    }
+    if (in.has_start_value()) {
+        out.SetStartValue(in.start_value());
+    }
+    if (in.has_cache()) {
+        out.SetCache(in.cache());
+    }
+    if (in.has_increment()) {
+        out.SetIncrement(in.increment());
+    }
+    if (in.has_cycle()) {
+        out.SetCycle(in.cycle());
+    }
+    if (in.has_set_val()) {
+        auto* setVal = out.MutableSetVal();
+        setVal->SetNextUsed(in.set_val().next_used());
+        setVal->SetNextValue(in.set_val().next_value());
+    }
+    return true;
 }
 
 } // namespace NKikimr

ydb/core/ydb_convert/table_description.h

@@ -133,11 +133,13 @@ bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out,
 
 
 // out
-void FillSequenceDescription(Ydb::Table::CreateTableRequest& out,
-    const NKikimrSchemeOp::TTableDescription& in);
+bool FillSequenceDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error);
 
-// out
-void FillSequenceDescription(Ydb::Table::CreateTableRequest& out,
-    const NKikimrSchemeOp::TTableDescription& in);
+bool FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error);
+
+// in
+bool FillSequenceDescription(
+    NKikimrSchemeOp::TSequenceDescription& out, const Ydb::Table::SequenceDescription& in, 
+    Ydb::StatusIds::StatusCode& status, TString& error);
 
 } // namespace NKikimr

ydb/library/backup/backup.cpp

@@ -372,7 +372,7 @@ NTable::TTableDescription DescribeTable(TDriver driver, const TString& fullTable
     NTable::TTableClient client(driver);
 
     TStatus status = client.RetryOperationSync([fullTablePath, &desc](NTable::TSession session) {
-        auto settings = NTable::TDescribeTableSettings().WithKeyShardBoundary(true);
+        auto settings = NTable::TDescribeTableSettings().WithKeyShardBoundary(true).WithSetVal(true);
         auto result = session.DescribeTable(fullTablePath, settings).GetValueSync();
 
         VerifyStatus(result);
@@ -489,6 +489,7 @@ void BackupTable(TDriver driver, const TString& dbPrefix, const TString& backupP
         << " backupPrefix: " << backupPrefix << " path: " << path);
 
     auto desc = DescribeTable(driver, JoinDatabasePath(schemaOnly ? dbPrefix : backupPrefix, path));
+
     auto proto = ProtoFromTableDescription(desc, preservePoolKinds);
 
     TString schemaStr;
@@ -718,19 +719,26 @@ void BackupFolder(TDriver driver, const TString& database, const TString& relDbP
 //                               Restore
 ////////////////////////////////////////////////////////////////////////////////
 
-TString ProcessColumnType(const TString& name, TTypeParser parser, NTable::TTableBuilder *builder) {
+TString ProcessColumnType(const TString& name, TTypeParser parser, NTable::TTableBuilder *builder, std::optional<NTable::TSequenceDescription> sequenceDescription) {
     TStringStream ss;
     ss << "name: " << name << "; ";
     if (parser.GetKind() == TTypeParser::ETypeKind::Optional) {
         ss << " optional; ";
         parser.OpenOptional();
     }
+    if (sequenceDescription.has_value()) {
+        ss << "serial; ";
+    }
     ss << "kind: " << parser.GetKind() << "; ";
     switch (parser.GetKind()) {
         case TTypeParser::ETypeKind::Primitive:
             ss << " type_id: " << parser.GetPrimitive() << "; ";
             if (builder) {
-                builder->AddNullableColumn(name, parser.GetPrimitive());
+                if (sequenceDescription.has_value()) {
+                    builder->AddSerialColumn(name, parser.GetPrimitive(), std::move(*sequenceDescription));
+                } else {
+                    builder->AddNullableColumn(name, parser.GetPrimitive());
+                }
             }
             break;
         case TTypeParser::ETypeKind::Decimal:
@@ -751,8 +759,19 @@ TString ProcessColumnType(const TString& name, TTypeParser parser, NTable::TTabl
 NTable::TTableDescription TableDescriptionFromProto(const Ydb::Table::CreateTableRequest& proto) {
     NTable::TTableBuilder builder;
 
+    std::optional<NTable::TSequenceDescription> sequenceDescription;
     for (const auto &col : proto.Getcolumns()) {
-        LOG_DEBUG("AddNullableColumn: " << ProcessColumnType(col.Getname(), TType(col.Gettype()), &builder));
+        if (col.from_sequence().name() == "_serial_column_" + col.name()) {
+            NTable::TSequenceDescription currentSequenceDescription;
+            if (col.from_sequence().has_set_val()) {
+                NTable::TSequenceDescription::TSetVal setVal;
+                setVal.NextUsed = col.from_sequence().set_val().next_used();
+                setVal.NextValue = col.from_sequence().set_val().next_value();
+                currentSequenceDescription.SetVal = std::move(setVal);
+            }
+            sequenceDescription = std::move(currentSequenceDescription);
+        }
+        LOG_DEBUG("AddColumn: " << ProcessColumnType(col.Getname(), TType(col.Gettype()), &builder, std::move(sequenceDescription)));
     }
 
     for (const auto &primary : proto.Getprimary_key()) {
@@ -781,7 +800,7 @@ TString SerializeColumnsToString(const TVector<TColumn>& columns, TVector<TStrin
         if (BinarySearch(primary.cbegin(), primary.cend(), col.Name)) {
             ss << "primary; ";
         }
-        ss << ProcessColumnType(col.Name, col.Type, nullptr) << Endl;
+        ss << ProcessColumnType(col.Name, col.Type, nullptr, std::nullopt) << Endl;
     }
     // Cerr << "Parse column to : " << ss.Str() << Endl;
     return ss.Str();

ydb/public/api/protos/ydb_table.proto

@@ -737,6 +737,8 @@ message DescribeTableRequest {
     bool include_table_stats = 6;
     // Includes partition statistics (required include_table_statistics)
     bool include_partition_stats = 7;
+    // Includes set_val settings for sequences
+    bool include_set_val = 8;
 }
 
 message DescribeTableResponse {

ydb/public/sdk/cpp/client/ydb_table/impl/table_client.cpp

@@ -525,6 +525,10 @@ TAsyncDescribeTableResult TTableClient::TImpl::DescribeTable(const TString& sess
         request.set_include_partition_stats(true);
     }
 
+    if (settings.WithSetVal_) {
+        request.set_include_set_val(true);
+    }
+
     auto promise = NewPromise<TDescribeTableResult>();
 
     auto extractor = [promise, settings]

ydb/public/sdk/cpp/client/ydb_table/table.cpp

@@ -290,7 +290,24 @@ class TTableDescription::TImpl {
             if (col.has_not_null()) {
                 not_null = col.not_null();
             }
-            Columns_.emplace_back(col.name(), col.type(), col.family(), not_null);
+            std::optional<TSequenceDescription> sequenceDescription;
+            switch (col.default_value_case()) {
+                case Ydb::Table::ColumnMeta::kFromSequence: {
+                    if (col.from_sequence().name() == "_serial_column_" + col.name()) {
+                        TSequenceDescription currentSequenceDescription;
+                        if (col.from_sequence().has_set_val()) {
+                            TSequenceDescription::TSetVal setVal;
+                            setVal.NextUsed = col.from_sequence().set_val().next_used();
+                            setVal.NextValue = col.from_sequence().set_val().next_value();
+                            currentSequenceDescription.SetVal = std::move(setVal);
+                        }
+                        sequenceDescription = std::move(currentSequenceDescription);
+                    }
+                    break;
+                }
+                default: break;
+            }
+            Columns_.emplace_back(col.name(), col.type(), col.family(), not_null, std::move(sequenceDescription));
         }
 
         // indexes
@@ -452,8 +469,8 @@ class TTableDescription::TImpl {
         return Proto_;
     }
 
-    void AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional<bool> notNull) {
-        Columns_.emplace_back(name, type, family, notNull);
+    void AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional<bool> notNull, std::optional<TSequenceDescription> sequenceDescription) {
+        Columns_.emplace_back(name, type, family, notNull, std::move(sequenceDescription));
     }
 
     void SetPrimaryKeyColumns(const TVector<TString>& primaryKeyColumns) {
@@ -724,8 +741,8 @@ const TVector<TKeyRange>& TTableDescription::GetKeyRanges() const {
     return Impl_->GetKeyRanges();
 }
 
-void TTableDescription::AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional<bool> notNull) {
-    Impl_->AddColumn(name, type, family, notNull);
+void TTableDescription::AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional<bool> notNull, std::optional<TSequenceDescription> sequenceDescription) {
+    Impl_->AddColumn(name, type, family, notNull, std::move(sequenceDescription));
 }
 
 void TTableDescription::SetPrimaryKeyColumns(const TVector<TString>& primaryKeyColumns) {
@@ -889,6 +906,15 @@ void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) con
         if (column.NotNull.has_value()) {
             protoColumn.set_not_null(column.NotNull.value());
         }
+        if (column.SequenceDescription.has_value()) {
+            auto* fromSequence = protoColumn.mutable_from_sequence();
+            if (column.SequenceDescription->SetVal.has_value()) {
+                auto* setVal = fromSequence->mutable_set_val();
+                setVal->set_next_value(column.SequenceDescription->SetVal->NextValue);
+                setVal->set_next_used(column.SequenceDescription->SetVal->NextUsed);
+            }
+            fromSequence->set_name("_serial_column_" + column.Name);
+        }
     }
 
     for (const auto& pk : Impl_->GetPrimaryKeyColumns()) {
@@ -1096,7 +1122,7 @@ TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const EPrim
         .EndOptional()
         .Build();
 
-    TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false);
+    TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false, std::nullopt);
     return *this;
 }
 
@@ -1106,7 +1132,7 @@ TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const TDeci
             .Decimal(type)
         .EndOptional()
         .Build();
-    TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false);
+    TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false, std::nullopt);
     return *this;
 }
 
@@ -1115,7 +1141,7 @@ TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const TPgTy
         .Pg(type)
         .Build();
 
-    TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false);
+    TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false, std::nullopt);
     return *this;
 }
 
@@ -1124,7 +1150,7 @@ TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const EP
         .Primitive(type)
         .Build();
 
-    TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true);
+    TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::nullopt);
     return *this;
 }
 
@@ -1133,7 +1159,7 @@ TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const TD
         .Decimal(type)
         .Build();
 
-    TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true);
+    TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::nullopt);
     return *this;
 }
 
@@ -1142,7 +1168,16 @@ TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const TP
         .Pg(type)
         .Build();
 
-    TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true);
+    TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::nullopt);
+    return *this;
+}
+
+TTableBuilder& TTableBuilder::AddSerialColumn(const TString& name, const EPrimitiveType& type, TSequenceDescription sequenceDescription, const TString& family) {
+    auto columnType = TTypeBuilder()
+        .Primitive(type)
+        .Build();
+
+    TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::move(sequenceDescription));
     return *this;
 }