diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index 2735e8a12ffa..bd58a9c7e75a 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -1086,6 +1086,9 @@ class TKiSinkCallableExecutionTransformer : public TAsyncCallbackTransformerCast(); SetColumnType(*add_column->mutable_type(), TString(dataType->GetName()), notNull); + ::NKikimrIndexBuilder::TColumnBuildSetting* columnBuild = nullptr; + bool hasDefaultValue = false; + bool hasNotNull = false; if (columnTuple.Size() > 2) { auto columnConstraints = columnTuple.Item(2).Cast(); for(const auto& constraint: columnConstraints.Value().Cast()) { @@ -1094,13 +1097,32 @@ class TKiSinkCallableExecutionTransformer : public TAsyncCallbackTransformeradd_column(); + if (columnBuild == nullptr) { + columnBuild = indexBuildSettings.mutable_column_build_operation()->add_column(); + } + columnBuild->SetColumnName(TString(columnName)); FillLiteralProto(constraint.Value().Cast(), *columnBuild->mutable_default_from_literal()); + hasDefaultValue = true; + } else if (constraint.Name().Value() == "not_null") { + if (columnBuild == nullptr) { + columnBuild = indexBuildSettings.mutable_column_build_operation()->add_column(); + } + + columnBuild->SetNotNull(true); + hasNotNull = true; } } } + if (hasNotNull && !hasDefaultValue) { + ctx.AddError( + YqlIssue(ctx.GetPosition(columnTuple.Pos()), + TIssuesIds::KIKIMR_BAD_REQUEST, + "Cannot add not null column without default value")); + return SyncError(); + } + if (columnTuple.Size() > 3) { auto families = columnTuple.Item(3).Cast(); if (families.Size() > 1) { @@ -1112,6 +1134,12 @@ class TKiSinkCallableExecutionTransformer : public TAsyncCallbackTransformerset_family(TString(family.Value())); } + + if (columnBuild) { + for (auto family : families) { + columnBuild->SetFamily(TString(family.Value())); + } + } } } } else if (name == "dropColumns") { diff --git a/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp index b954da884d1d..61f4e02f6889 100644 --- a/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp @@ -291,6 +291,7 @@ Y_UNIT_TEST_SUITE(KqpConstraints) { Y_UNIT_TEST(AlterTableAddColumnWithDefaultValue) { NKikimrConfig::TAppConfig appConfig; appConfig.MutableTableServiceConfig()->SetEnableSequences(false); + appConfig.MutableFeatureFlags()->SetEnableAddColumsWithDefaults(true); auto serverSettings = TKikimrSettings().SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); @@ -588,6 +589,8 @@ Y_UNIT_TEST_SUITE(KqpConstraints) { appConfig.MutableTableServiceConfig()->SetEnableSequences(false); appConfig.MutableTableServiceConfig()->SetEnableColumnsWithDefault(true); + appConfig.MutableFeatureFlags()->SetEnableAddColumsWithDefaults(true); + TKikimrRunner kikimr(TKikimrSettings().SetPQConfig(DefaultPQConfig()).SetAppConfig(appConfig)); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -661,7 +664,7 @@ Y_UNIT_TEST_SUITE(KqpConstraints) { fCompareTable(R"( [ - [[1u];["Old"];[1]];[[2u];["New"];[1]] + [[1u];["Old"];1];[[2u];["New"];1] ] )"); @@ -672,7 +675,7 @@ Y_UNIT_TEST_SUITE(KqpConstraints) { fCompareTable(R"( [ - [[1u];["Old"];[1]];[[2u];["New"];[2]] + [[1u];["Old"];1];[[2u];["New"];2] ] )"); @@ -686,7 +689,7 @@ Y_UNIT_TEST_SUITE(KqpConstraints) { fCompareTable(R"( [ - [[1u];["Old"];[1]];[[2u];["OldNew"];[2]];[[3u];["BrandNew"];[1]] + [[1u];["Old"];1];[[2u];["OldNew"];2];[[3u];["BrandNew"];1] ] )"); diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 0f30a84aca39..47fec6626cb7 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -5553,7 +5553,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { { auto alterQuery = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() << "`ADD COLUMN new_column Uint64 NOT NULL;"; auto alterResult = testHelper.GetSession().ExecuteSchemeQuery(alterQuery).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), EStatus::SCHEME_ERROR, alterResult.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), EStatus::BAD_REQUEST, alterResult.GetIssues().ToString()); } } diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index 7841c1403203..115bd8facc9b 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -127,4 +127,5 @@ message TFeatureFlags { optional bool EnableViews = 112 [default = false]; optional bool EnableServerlessExclusiveDynamicNodes = 113 [default = false]; optional bool EnableAccessServiceBulkAuthorization = 114 [default = false]; + optional bool EnableAddColumsWithDefaults = 115 [ default = false]; } diff --git a/ydb/core/protos/index_builder.proto b/ydb/core/protos/index_builder.proto index bf44465db04c..5370a7215336 100644 --- a/ydb/core/protos/index_builder.proto +++ b/ydb/core/protos/index_builder.proto @@ -12,6 +12,8 @@ option java_package = "ru.yandex.kikimr.proto"; message TColumnBuildSetting { optional string ColumnName = 1; optional Ydb.TypedValue default_from_literal = 2; + optional bool NotNull = 3; + optional string Family = 4; } message TColumnBuildSettings { diff --git a/ydb/core/testlib/basics/feature_flags.h b/ydb/core/testlib/basics/feature_flags.h index 3185f6ef877c..22086cb95595 100644 --- a/ydb/core/testlib/basics/feature_flags.h +++ b/ydb/core/testlib/basics/feature_flags.h @@ -56,6 +56,7 @@ class TTestFeatureFlagsHolder { FEATURE_FLAG_SETTER(EnableTablePgTypes) FEATURE_FLAG_SETTER(EnableServerlessExclusiveDynamicNodes) FEATURE_FLAG_SETTER(EnableAccessServiceBulkAuthorization) + FEATURE_FLAG_SETTER(EnableAddColumsWithDefaults) #undef FEATURE_FLAG_SETTER }; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp index 2aa57064a0e8..ce9a784c5424 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp @@ -87,11 +87,19 @@ TTableInfo::TAlterDataPtr ParseParams(const TPath& path, TTableInfo::TPtr table, // Ignore column ids if they were passed by user! for (auto& col : *copyAlter.MutableColumns()) { - if (col.GetNotNull()) { - errStr = Sprintf("Not null columns is not supported for alter command"); + bool hasDefault = col.HasDefaultFromLiteral(); + if (hasDefault && !context.SS->EnableAddColumsWithDefaults) { + errStr = Sprintf("Column addition with default value is not supported now."); status = NKikimrScheme::StatusInvalidParameter; return nullptr; } + + if (col.GetNotNull() && !hasDefault) { + errStr = Sprintf("Not null columns without defaults are not supported."); + status = NKikimrScheme::StatusInvalidParameter; + return nullptr; + } + col.ClearId(); } diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index.cpp index 6997656c3f73..08f72ca6c04f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index.cpp @@ -68,7 +68,9 @@ void TSchemeShard::PersistCreateBuildIndex(NIceDb::TNiceDb& db, const TIndexBuil db.Table().Key(info->Id, i).Update( NIceDb::TUpdate(info->BuildColumns[i].ColumnName), NIceDb::TUpdate( - TString(info->BuildColumns[i].DefaultFromLiteral.SerializeAsString())) + TString(info->BuildColumns[i].DefaultFromLiteral.SerializeAsString())), + NIceDb::TUpdate(info->BuildColumns[i].NotNull), + NIceDb::TUpdate(info->BuildColumns[i].FamilyName) ); } } diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp index ba1832c172a1..64b5f9c7a69d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp @@ -161,8 +161,11 @@ class TSchemeShard::TIndexBuilder::TTxCreate: public TSchemeShard::TIndexBuilder buildInfo->BuildColumns.reserve(settings.column_build_operation().column_size()); for(int i = 0; i < settings.column_build_operation().column_size(); i++) { const auto& colInfo = settings.column_build_operation().column(i); + bool notNull = colInfo.HasNotNull() && colInfo.GetNotNull(); + TString familyName = colInfo.HasFamily() ? colInfo.GetFamily() : ""; buildInfo->BuildColumns.push_back( - TIndexBuildInfo::TColumnBuildInfo(colInfo.GetColumnName(), colInfo.default_from_literal())); + TIndexBuildInfo::TColumnBuildInfo( + colInfo.GetColumnName(), colInfo.default_from_literal(), notNull, familyName)); } } diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index bc0596e6bfb1..37c0a91e8cfa 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -98,6 +98,15 @@ THolder AlterMainTablePropose( col->SetType(NScheme::TypeName(typeInfo, typeMod)); col->SetName(colInfo.ColumnName); col->MutableDefaultFromLiteral()->CopyFrom(colInfo.DefaultFromLiteral); + + if (!colInfo.FamilyName.empty()) { + col->SetFamilyName(colInfo.FamilyName); + } + + if (colInfo.NotNull) { + col->SetNotNull(colInfo.NotNull); + } + } } else { diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 55026ab3b282..fa2662f423bb 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -4271,6 +4271,7 @@ void TSchemeShard::OnActivateExecutor(const TActorContext &ctx) { EnableStatistics = appData->FeatureFlags.GetEnableStatistics(); EnableTablePgTypes = appData->FeatureFlags.GetEnableTablePgTypes(); EnableServerlessExclusiveDynamicNodes = appData->FeatureFlags.GetEnableServerlessExclusiveDynamicNodes(); + EnableAddColumsWithDefaults = appData->FeatureFlags.GetEnableAddColumsWithDefaults(); ConfigureCompactionQueues(appData->CompactionConfig, ctx); ConfigureStatsBatching(appData->SchemeShardConfig, ctx); @@ -6748,6 +6749,7 @@ void TSchemeShard::ApplyConsoleConfigs(const NKikimrConfig::TFeatureFlags& featu EnableStatistics = featureFlags.GetEnableStatistics(); EnableTablePgTypes = featureFlags.GetEnableTablePgTypes(); EnableServerlessExclusiveDynamicNodes = featureFlags.GetEnableServerlessExclusiveDynamicNodes(); + EnableAddColumsWithDefaults = featureFlags.GetEnableAddColumsWithDefaults(); } void TSchemeShard::ConfigureStatsBatching(const NKikimrConfig::TSchemeShardConfig& config, const TActorContext& ctx) { diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index e7f9c4e3150c..05c3f4cb7abd 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -271,6 +271,7 @@ class TSchemeShard bool EnableStatistics = false; bool EnableTablePgTypes = false; bool EnableServerlessExclusiveDynamicNodes = false; + bool EnableAddColumsWithDefaults = false; TShardDeleter ShardDeleter; diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 769a5dfa7961..8ae3ac154926 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -3007,22 +3007,30 @@ struct TIndexBuildInfo: public TSimpleRefCount { struct TColumnBuildInfo { TString ColumnName; Ydb::TypedValue DefaultFromLiteral; + bool NotNull = false; + TString FamilyName; - TColumnBuildInfo(const TString& name, const TString& serializedLiteral) + TColumnBuildInfo(const TString& name, const TString& serializedLiteral, bool notNull, const TString& familyName) : ColumnName(name) + , NotNull(notNull) + , FamilyName(familyName) { Y_ABORT_UNLESS(DefaultFromLiteral.ParseFromString(serializedLiteral)); } - TColumnBuildInfo(const TString& name, const Ydb::TypedValue& defaultFromLiteral) + TColumnBuildInfo(const TString& name, const Ydb::TypedValue& defaultFromLiteral, bool notNull, const TString& familyName) : ColumnName(name) , DefaultFromLiteral(defaultFromLiteral) + , NotNull(notNull) + , FamilyName(familyName) { } void SerializeToProto(NKikimrIndexBuilder::TColumnBuildSetting* setting) const { setting->SetColumnName(ColumnName); setting->mutable_default_from_literal()->CopyFrom(DefaultFromLiteral); + setting->SetNotNull(NotNull); + setting->SetFamily(FamilyName); } }; @@ -3140,7 +3148,9 @@ struct TIndexBuildInfo: public TSimpleRefCount { void AddBuildColumnInfo(const TRow& row){ TString columnName = row.template GetValue(); TString defaultFromLiteral = row.template GetValue(); - BuildColumns.push_back(TColumnBuildInfo(columnName, defaultFromLiteral)); + bool notNull = row.template GetValue(); + TString familyName = row.template GetValue(); + BuildColumns.push_back(TColumnBuildInfo(columnName, defaultFromLiteral, notNull, familyName)); } template diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index 2aade717ba0c..3651c4e37d7f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -1726,13 +1726,17 @@ struct Schema : NIceDb::Schema { struct ColumnNo : Column<2, NScheme::NTypeIds::Uint64> {}; struct ColumnName : Column<3, NScheme::NTypeIds::Utf8> {}; struct DefaultFromLiteral : Column<4, NScheme::NTypeIds::String> {}; + struct NotNull : Column<5, NScheme::NTypeIds::Bool> {}; + struct FamilyName : Column<6, NScheme::NTypeIds::String> {}; using TKey = TableKey; using TColumns = TableColumns< Id, ColumnNo, ColumnName, - DefaultFromLiteral + DefaultFromLiteral, + NotNull, + FamilyName >; }; diff --git a/ydb/core/tx/schemeshard/ut_column_build/ut_column_build.cpp b/ydb/core/tx/schemeshard/ut_column_build/ut_column_build.cpp index d5d498a0292e..451e05c5443f 100644 --- a/ydb/core/tx/schemeshard/ut_column_build/ut_column_build.cpp +++ b/ydb/core/tx/schemeshard/ut_column_build/ut_column_build.cpp @@ -12,7 +12,7 @@ using namespace NSchemeShardUT_Private; Y_UNIT_TEST_SUITE(ColumnBuildTest) { Y_UNIT_TEST(AlreadyExists) { TTestBasicRuntime runtime; - TTestEnv env(runtime); + TTestEnv env(runtime, TTestEnvOptions().EnableAddColumsWithDefaults(true)); ui64 txId = 100; TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", @@ -117,7 +117,7 @@ Y_UNIT_TEST_SUITE(ColumnBuildTest) { Y_UNIT_TEST(InvalidValue) { TTestBasicRuntime runtime; - TTestEnv env(runtime); + TTestEnv env(runtime, TTestEnvOptions().EnableAddColumsWithDefaults(true)); ui64 txId = 100; TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", @@ -230,7 +230,7 @@ Y_UNIT_TEST_SUITE(ColumnBuildTest) { Y_UNIT_TEST(BuildColumnDoesnotRestoreDeletedRows) { TTestBasicRuntime runtime(1, false); - TTestEnv env(runtime); + TTestEnv env(runtime, TTestEnvOptions().EnableAddColumsWithDefaults(true)); ui64 txId = 100; TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", @@ -400,7 +400,7 @@ Y_UNIT_TEST_SUITE(ColumnBuildTest) { Y_UNIT_TEST(BaseCase) { TTestBasicRuntime runtime; - TTestEnv env(runtime); + TTestEnv env(runtime, TTestEnvOptions().EnableAddColumsWithDefaults(true)); ui64 txId = 100; TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", @@ -545,7 +545,7 @@ Y_UNIT_TEST_SUITE(ColumnBuildTest) { Y_UNIT_TEST(CancelBuild) { TTestBasicRuntime runtime; - TTestEnv env(runtime); + TTestEnv env(runtime, TTestEnvOptions().EnableAddColumsWithDefaults(true)); ui64 txId = 100; // Just create main table diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp index 81054aa54004..f8eb23547139 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp @@ -536,6 +536,7 @@ NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime& runtime, const TTe app.SetEnableChangefeedDebeziumJsonFormat(opts.EnableChangefeedDebeziumJsonFormat_); app.SetEnableTablePgTypes(opts.EnableTablePgTypes_); app.SetEnableServerlessExclusiveDynamicNodes(opts.EnableServerlessExclusiveDynamicNodes_); + app.SetEnableAddColumsWithDefaults(opts.EnableAddColumsWithDefaults_); app.ColumnShardConfig.SetDisabledOnSchemeShard(false); diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.h b/ydb/core/tx/schemeshard/ut_helpers/test_env.h index c85081d07339..d0ff68ece860 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.h +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.h @@ -58,6 +58,7 @@ namespace NSchemeShardUT_Private { OPTION(std::optional, EnableChangefeedDebeziumJsonFormat, std::nullopt); OPTION(std::optional, EnableTablePgTypes, std::nullopt); OPTION(std::optional, EnableServerlessExclusiveDynamicNodes, std::nullopt); + OPTION(std::optional, EnableAddColumsWithDefaults, std::nullopt); #undef OPTION }; diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema index 05a8b981e54f..937270748b93 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema @@ -7145,6 +7145,16 @@ "ColumnId": 4, "ColumnName": "DefaultFromLiteral", "ColumnType": "String" + }, + { + "ColumnId": 5, + "ColumnName": "NotNull", + "ColumnType": "Bool" + }, + { + "ColumnId": 6, + "ColumnName": "FamilyName", + "ColumnType": "String" } ], "ColumnsDropped": [], @@ -7154,7 +7164,9 @@ 1, 2, 3, - 4 + 4, + 5, + 6 ], "RoomID": 0, "Codec": 0,