From 27ecbff8441515bb95ea60775326de3820d5f0a4 Mon Sep 17 00:00:00 2001 From: Vasily Gerasimov Date: Tue, 26 Dec 2023 15:57:28 +0000 Subject: [PATCH] Support DDL operations (including IF EXISTS/IF NOT EXISTS) for objects with type secret --- .../kqp/compile_service/kqp_compile_actor.cpp | 2 +- .../executer_actor/kqp_scheme_executer.cpp | 2 +- .../kqp/executer_actor/ut/kqp_executer_ut.cpp | 2 +- .../external_data_source/manager.cpp | 2 +- .../behaviour/external_data_source/manager.h | 2 +- .../gateway/behaviour/tablestore/manager.cpp | 2 +- .../gateway/behaviour/tablestore/manager.h | 2 +- .../kqp/gateway/behaviour/view/manager.cpp | 5 +- ydb/core/kqp/gateway/behaviour/view/manager.h | 1 + ydb/core/kqp/host/kqp_gateway_proxy.cpp | 3 + ydb/core/kqp/host/kqp_host.cpp | 8 +- ydb/core/kqp/host/kqp_host.h | 3 +- ydb/core/kqp/provider/yql_kikimr_provider.h | 10 +- .../kqp/session_actor/kqp_worker_actor.cpp | 2 +- .../generic/kqp_generic_provider_join_ut.cpp | 7 +- .../s3/kqp_federated_scheme_ut.cpp | 2 +- ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp | 2 +- ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp | 92 ++++++++++++++ ydb/core/protos/kqp_physical.proto | 15 +++ ydb/core/tx/tiering/rule/manager.cpp | 2 +- ydb/core/tx/tiering/rule/manager.h | 2 +- ydb/core/tx/tiering/tier/manager.cpp | 2 +- ydb/core/tx/tiering/tier/manager.h | 2 +- ydb/library/conclusion/result.h | 9 ++ ydb/library/conclusion/status.h | 19 ++- ydb/library/conclusion/ya.make | 1 + ydb/services/ext_index/metadata/manager.cpp | 2 +- ydb/services/ext_index/metadata/manager.h | 2 +- ydb/services/ext_index/service/activation.cpp | 10 +- ydb/services/ext_index/service/activation.h | 10 +- ydb/services/ext_index/service/deleting.cpp | 4 +- ydb/services/ext_index/service/deleting.h | 8 +- ydb/services/ext_index/service/executor.cpp | 4 +- .../metadata/initializer/accessor_init.cpp | 2 +- .../metadata/initializer/accessor_init.h | 2 +- ydb/services/metadata/initializer/common.h | 6 +- ydb/services/metadata/initializer/manager.cpp | 2 +- ydb/services/metadata/initializer/manager.h | 2 +- ydb/services/metadata/manager/abstract.h | 2 +- ydb/services/metadata/manager/alter.h | 33 ++++- .../metadata/manager/generic_manager.h | 116 ++++++++++++++++-- ydb/services/metadata/manager/modification.h | 22 +++- ydb/services/metadata/request/common.h | 12 +- ydb/services/metadata/request/request_actor.h | 10 +- .../metadata/request/request_actor_cb.h | 24 ++-- ydb/services/metadata/secret/manager.cpp | 14 +-- ydb/services/metadata/secret/manager.h | 10 +- 47 files changed, 375 insertions(+), 123 deletions(-) diff --git a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp index 5741fe4cfc8c..8f31da06827b 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp +++ b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp @@ -184,7 +184,7 @@ class TKqpCompileActor : public TActorBootstrapped { Config->FeatureFlags = AppData(ctx)->FeatureFlags; KqpHost = CreateKqpHost(Gateway, QueryId.Cluster, QueryId.Database, Config, ModuleResolverState->ModuleResolver, - FederatedQuerySetup, AppData(ctx)->FunctionRegistry, false, false, std::move(TempTablesState)); + FederatedQuerySetup, UserToken, AppData(ctx)->FunctionRegistry, false, false, std::move(TempTablesState)); IKqpHost::TPrepareSettings prepareSettings; prepareSettings.DocumentApiRestricted = QueryId.Settings.DocumentApiRestricted; diff --git a/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp b/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp index 97ea45997c56..450229c2f7b2 100644 --- a/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp @@ -259,7 +259,7 @@ class TKqpSchemeExecuter : public TActorBootstrapped { context.SetUserToken(*UserToken); } - auto resultFuture = cBehaviour->GetOperationsManager()->ExecutePrepared(schemeOp, cBehaviour, context); + auto resultFuture = cBehaviour->GetOperationsManager()->ExecutePrepared(schemeOp, SelfId().NodeId(), cBehaviour, context); using TResultFuture = NThreading::TFuture; resultFuture.Subscribe([actorSystem, selfId](const TResultFuture& f) { diff --git a/ydb/core/kqp/executer_actor/ut/kqp_executer_ut.cpp b/ydb/core/kqp/executer_actor/ut/kqp_executer_ut.cpp index 062add1ddb25..f395537d6945 100644 --- a/ydb/core/kqp/executer_actor/ut/kqp_executer_ut.cpp +++ b/ydb/core/kqp/executer_actor/ut/kqp_executer_ut.cpp @@ -28,7 +28,7 @@ NKqpProto::TKqpPhyTx BuildTxPlan(const TString& sql, TIntrusivePtr IModuleResolver::TPtr moduleResolver; UNIT_ASSERT(GetYqlDefaultModuleResolver(moduleCtx, moduleResolver)); - auto qp = CreateKqpHost(gateway, cluster, "/Root", config, moduleResolver, NYql::IHTTPGateway::Make(), nullptr, false, false, nullptr, actorSystem); + auto qp = CreateKqpHost(gateway, cluster, "/Root", config, moduleResolver, NYql::IHTTPGateway::Make(), nullptr, nullptr, false, false, nullptr, actorSystem); auto result = qp->SyncPrepareDataQuery(sql, IKqpHost::TPrepareSettings()); result.Issues().PrintTo(Cerr); UNIT_ASSERT(result.Success()); diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp index 10797a6b64d5..f13d1155d788 100644 --- a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp @@ -229,7 +229,7 @@ void TExternalDataSourceManager::PrepareDropExternalDataSource(NKqpProto::TKqpSc } NThreading::TFuture TExternalDataSourceManager::ExecutePrepared(const NKqpProto::TKqpSchemeOperation& schemeOperation, - const NMetadata::IClassBehaviour::TPtr& /*manager*/, const IOperationsManager::TExternalModificationContext& context) const { + const ui32 /*nodeId*/, const NMetadata::IClassBehaviour::TPtr& /*manager*/, const IOperationsManager::TExternalModificationContext& context) const { using TRequest = TEvTxUserProxy::TEvProposeTransaction; auto ev = MakeHolder(); diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.h b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.h index fcf7e7582862..83bc138b5382 100644 --- a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.h +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.h @@ -27,7 +27,7 @@ class TExternalDataSourceManager: public NMetadata::NModifications::IOperationsM const NMetadata::IClassBehaviour::TPtr& manager, IOperationsManager::TInternalModificationContext& context) const override; NThreading::TFuture ExecutePrepared(const NKqpProto::TKqpSchemeOperation& schemeOperation, - const NMetadata::IClassBehaviour::TPtr& manager, const IOperationsManager::TExternalModificationContext& context) const override; + const ui32 nodeId, const NMetadata::IClassBehaviour::TPtr& manager, const IOperationsManager::TExternalModificationContext& context) const override; public: using NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus; diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/manager.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/manager.cpp index 66d4fc26c974..844ef19344db 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/manager.cpp +++ b/ydb/core/kqp/gateway/behaviour/tablestore/manager.cpp @@ -77,7 +77,7 @@ NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus TTableStoreM } NThreading::TFuture TTableStoreManager::ExecutePrepared(const NKqpProto::TKqpSchemeOperation& /*schemeOperation*/, - const NMetadata::IClassBehaviour::TPtr& /*manager*/, const IOperationsManager::TExternalModificationContext& /*context*/) const { + const ui32 /*nodeId*/, const NMetadata::IClassBehaviour::TPtr& /*manager*/, const IOperationsManager::TExternalModificationContext& /*context*/) const { return NThreading::MakeFuture(NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus::Fail( "Execution of prepare operations for TABLE objects is not supported")); } diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/manager.h b/ydb/core/kqp/gateway/behaviour/tablestore/manager.h index 58fd0e57f885..06c555cfaeb4 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/manager.h +++ b/ydb/core/kqp/gateway/behaviour/tablestore/manager.h @@ -17,7 +17,7 @@ class TTableStoreManager: public NMetadata::NModifications::IOperationsManager { const NMetadata::IClassBehaviour::TPtr& manager, IOperationsManager::TInternalModificationContext& context) const override; NThreading::TFuture ExecutePrepared(const NKqpProto::TKqpSchemeOperation& schemeOperation, - const NMetadata::IClassBehaviour::TPtr& manager, const IOperationsManager::TExternalModificationContext& context) const override; + const ui32 nodeId, const NMetadata::IClassBehaviour::TPtr& manager, const IOperationsManager::TExternalModificationContext& context) const override; public: TTableStoreManager(bool isStandalone) : IsStandalone(isStandalone) diff --git a/ydb/core/kqp/gateway/behaviour/view/manager.cpp b/ydb/core/kqp/gateway/behaviour/view/manager.cpp index 0f2bdbd5d5e1..bd3cee0133d1 100644 --- a/ydb/core/kqp/gateway/behaviour/view/manager.cpp +++ b/ydb/core/kqp/gateway/behaviour/view/manager.cpp @@ -152,7 +152,7 @@ TViewManager::TYqlConclusionStatus TViewManager::DoPrepare(NKqpProto::TKqpScheme const NMetadata::IClassBehaviour::TPtr& manager, TInternalModificationContext& context) const { Y_UNUSED(manager); - + try { CheckFeatureFlag(context); switch (context.GetActivityType()) { @@ -176,9 +176,10 @@ TViewManager::TYqlConclusionStatus TViewManager::DoPrepare(NKqpProto::TKqpScheme } NThreading::TFuture TViewManager::ExecutePrepared(const NKqpProto::TKqpSchemeOperation& schemeOperation, + const ui32 nodeId, const NMetadata::IClassBehaviour::TPtr& manager, const TExternalModificationContext& context) const { - Y_UNUSED(manager); + Y_UNUSED(manager, nodeId); auto proposal = MakeHolder(); proposal->Record.SetDatabaseName(context.GetDatabase()); diff --git a/ydb/core/kqp/gateway/behaviour/view/manager.h b/ydb/core/kqp/gateway/behaviour/view/manager.h index 50c8bea954e8..a2329109f052 100644 --- a/ydb/core/kqp/gateway/behaviour/view/manager.h +++ b/ydb/core/kqp/gateway/behaviour/view/manager.h @@ -17,6 +17,7 @@ class TViewManager: public NMetadata::NModifications::IOperationsManager { TInternalModificationContext& context) const override; NThreading::TFuture ExecutePrepared(const NKqpProto::TKqpSchemeOperation& schemeOperation, + const ui32 nodeId, const NMetadata::IClassBehaviour::TPtr& manager, const TExternalModificationContext& context) const override; diff --git a/ydb/core/kqp/host/kqp_gateway_proxy.cpp b/ydb/core/kqp/host/kqp_gateway_proxy.cpp index 743c518c2c71..f3b2201ffac6 100644 --- a/ydb/core/kqp/host/kqp_gateway_proxy.cpp +++ b/ydb/core/kqp/host/kqp_gateway_proxy.cpp @@ -1022,6 +1022,9 @@ class TKqpGatewayProxy : public IKikimrGateway { NMetadata::NModifications::IOperationsManager::TExternalModificationContext context; context.SetDatabase(SessionCtx->GetDatabase()); context.SetActorSystem(ActorSystem); + if (SessionCtx->GetUserToken()) { + context.SetUserToken(*SessionCtx->GetUserToken()); + } auto& phyTx = phyTxRemover.Capture(SessionCtx->Query().PreparingQuery->MutablePhysicalQuery()); phyTx.SetType(NKqpProto::TKqpPhyTx::TYPE_SCHEME); diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp index 70bca19d12f3..750fe8120dbc 100644 --- a/ydb/core/kqp/host/kqp_host.cpp +++ b/ydb/core/kqp/host/kqp_host.cpp @@ -921,7 +921,7 @@ class TKqpHost : public IKqpHost { public: TKqpHost(TIntrusivePtr gateway, const TString& cluster, const TString& database, TKikimrConfiguration::TPtr config, IModuleResolver::TPtr moduleResolver, - std::optional federatedQuerySetup, + std::optional federatedQuerySetup, const TIntrusiveConstPtr& userToken, const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry, bool keepConfigChanges, bool isInternalCall, TKqpTempTablesState::TConstPtr tempTablesState = nullptr, NActors::TActorSystem* actorSystem = nullptr) @@ -932,7 +932,7 @@ class TKqpHost : public IKqpHost { , KeepConfigChanges(keepConfigChanges) , IsInternalCall(isInternalCall) , FederatedQuerySetup(federatedQuerySetup) - , SessionCtx(new TKikimrSessionContext(funcRegistry, config, TAppData::TimeProvider, TAppData::RandomProvider)) + , SessionCtx(new TKikimrSessionContext(funcRegistry, config, TAppData::TimeProvider, TAppData::RandomProvider, userToken)) , TypesCtx(MakeIntrusive()) , PlanBuilder(CreatePlanBuilder(*TypesCtx)) , FakeWorld(ExprCtx->NewWorld(TPosition())) @@ -1672,11 +1672,11 @@ Ydb::Table::QueryStatsCollection::Mode GetStatsMode(NYql::EKikimrStatsMode stats TIntrusivePtr CreateKqpHost(TIntrusivePtr gateway, const TString& cluster, const TString& database, TKikimrConfiguration::TPtr config, IModuleResolver::TPtr moduleResolver, - std::optional federatedQuerySetup, + std::optional federatedQuerySetup, const TIntrusiveConstPtr& userToken, const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry, bool keepConfigChanges, bool isInternalCall, TKqpTempTablesState::TConstPtr tempTablesState, NActors::TActorSystem* actorSystem) { - return MakeIntrusive(gateway, cluster, database, config, moduleResolver, federatedQuerySetup, funcRegistry, + return MakeIntrusive(gateway, cluster, database, config, moduleResolver, federatedQuerySetup, userToken, funcRegistry, keepConfigChanges, isInternalCall, std::move(tempTablesState), actorSystem); } diff --git a/ydb/core/kqp/host/kqp_host.h b/ydb/core/kqp/host/kqp_host.h index 445d77e8875e..3b8022221dff 100644 --- a/ydb/core/kqp/host/kqp_host.h +++ b/ydb/core/kqp/host/kqp_host.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -108,7 +109,7 @@ class IKqpHost : public TThrRefBase { TIntrusivePtr CreateKqpHost(TIntrusivePtr gateway, const TString& cluster, const TString& database, NYql::TKikimrConfiguration::TPtr config, NYql::IModuleResolver::TPtr moduleResolver, - std::optional federatedQuerySetup, const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry = nullptr, + std::optional federatedQuerySetup, const TIntrusiveConstPtr& userToken, const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry = nullptr, bool keepConfigChanges = false, bool isInternalCall = false, TKqpTempTablesState::TConstPtr tempTablesState = nullptr, NActors::TActorSystem* actorSystem = nullptr /*take from TLS by default*/); diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.h b/ydb/core/kqp/provider/yql_kikimr_provider.h index f662ce6becfe..2dd58916d108 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider.h @@ -445,11 +445,14 @@ class TKikimrSessionContext : public TThrRefBase { TKikimrConfiguration::TPtr config, TIntrusivePtr timeProvider, TIntrusivePtr randomProvider, + const TIntrusiveConstPtr& userToken, TIntrusivePtr txCtx = nullptr) : Configuration(config) , TablesData(MakeIntrusive()) , QueryCtx(MakeIntrusive(functionRegistry, timeProvider, randomProvider)) - , TxCtx(txCtx) {} + , TxCtx(txCtx) + , UserToken(userToken) + {} TKikimrSessionContext(const TKikimrSessionContext&) = delete; TKikimrSessionContext& operator=(const TKikimrSessionContext&) = delete; @@ -518,6 +521,10 @@ class TKikimrSessionContext : public TThrRefBase { TempTablesState = tempTablesState; } + const TIntrusiveConstPtr& GetUserToken() const { + return UserToken; + } + private: TString UserName; TString Cluster; @@ -527,6 +534,7 @@ class TKikimrSessionContext : public TThrRefBase { TIntrusivePtr QueryCtx; TIntrusivePtr TxCtx; NKikimr::NKqp::TKqpTempTablesState::TConstPtr TempTablesState; + TIntrusiveConstPtr UserToken; }; TIntrusivePtr CreateKikimrDataSource( diff --git a/ydb/core/kqp/session_actor/kqp_worker_actor.cpp b/ydb/core/kqp/session_actor/kqp_worker_actor.cpp index 0bffab8dba14..682ee146e8a3 100644 --- a/ydb/core/kqp/session_actor/kqp_worker_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_worker_actor.cpp @@ -187,7 +187,7 @@ class TKqpWorkerActor : public TActorBootstrapped { Config->FeatureFlags = AppData(ctx)->FeatureFlags; KqpHost = CreateKqpHost(Gateway, Settings.Cluster, Settings.Database, Config, ModuleResolverState->ModuleResolver, - FederatedQuerySetup, AppData(ctx)->FunctionRegistry, !Settings.LongSession, false); + FederatedQuerySetup, QueryState->RequestEv->GetUserToken(), AppData(ctx)->FunctionRegistry, !Settings.LongSession, false); auto& queryRequest = QueryState->RequestEv; QueryState->ProxyRequestId = proxyRequestId; diff --git a/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_join_ut.cpp b/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_join_ut.cpp index 63c62309b1c1..4c88715f2af6 100644 --- a/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_join_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_join_ut.cpp @@ -63,9 +63,7 @@ Y_UNIT_TEST_SUITE(FederatedQueryJoin) { } std::shared_ptr kikimr = MakeKikimrRunnerWithConnector(); - - auto tableCLient = kikimr->GetTableClient(); - auto session = tableCLient.CreateSession().GetValueSync().GetSession(); + auto queryClient = kikimr->GetQueryClient(); // external tables to pg/ch { @@ -104,7 +102,7 @@ Y_UNIT_TEST_SUITE(FederatedQueryJoin) { "ch_database"_a = GetChDatabase(), "ch_user"_a = GetChUser(), "ch_password"_a = GetChPassword()); - auto result = session.ExecuteSchemeQuery(sql).GetValueSync(); + auto result = queryClient.ExecuteQuery(sql, NYdb::NQuery::TTxControl::NoTx()).GetValueSync(); UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); } @@ -116,7 +114,6 @@ Y_UNIT_TEST_SUITE(FederatedQueryJoin) { WHERE ch.key > 998 )sql"; - auto queryClient = kikimr->GetQueryClient(); auto result = queryClient.ExecuteQuery(sql, NYdb::NQuery::TTxControl::BeginTx().CommitTx()).GetValueSync(); UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); diff --git a/ydb/core/kqp/ut/federated_query/s3/kqp_federated_scheme_ut.cpp b/ydb/core/kqp/ut/federated_query/s3/kqp_federated_scheme_ut.cpp index 6cc0f2a894ba..45506581cff1 100644 --- a/ydb/core/kqp/ut/federated_query/s3/kqp_federated_scheme_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/s3/kqp_federated_scheme_ut.cpp @@ -17,7 +17,7 @@ using namespace NTestUtils; using namespace fmt::literals; Y_UNIT_TEST_SUITE(KqpFederatedSchemeTest) { - Y_UNIT_TEST(CreateExternalTable) { + Y_UNIT_TEST(ExternalTableDdl) { enum EEx { Empty, IfExists, diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp index 6574e7a8a093..74214695a128 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp @@ -55,7 +55,7 @@ TIntrusivePtr CreateKikimrQueryProcessor(TIntrusivePtr ga auto federatedQuerySetup = std::make_optional({NYql::IHTTPGateway::Make(), nullptr, nullptr, nullptr, {}, {}}); return NKqp::CreateKqpHost(gateway, cluster, "/Root", kikimrConfig, moduleResolver, - federatedQuerySetup, funcRegistry, funcRegistry, keepConfigChanges, nullptr, actorSystem); + federatedQuerySetup, nullptr, funcRegistry, funcRegistry, keepConfigChanges, nullptr, actorSystem); } NYql::NNodes::TExprBase GetExpr(const TString& ast, NYql::TExprContext& ctx, NYql::IModuleResolver* moduleResolver) { diff --git a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp index 9caf25a81d4d..233e7932c542 100644 --- a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp @@ -1160,6 +1160,98 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { }); } + Y_UNIT_TEST(DdlSecret) { + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnablePreparedDdl(true); + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() + .SetAppConfig(appConfig) + .SetKqpSettings({setting}); + + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetQueryClient(); + + enum EEx { + Empty, + IfExists, + IfNotExists, + }; + + auto executeSql = [&](const TString& sql, bool expectSuccess) { + Cerr << "Execute SQL:\n" << sql << Endl; + + auto result = db.ExecuteQuery(sql, TTxControl::NoTx()).ExtractValueSync(); + if (expectSuccess) { + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } else { + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + UNIT_ASSERT(result.GetResultSets().empty()); + }; + + auto checkCreate = [&](bool expectSuccess, EEx exMode, int nameSuffix) { + UNIT_ASSERT_UNEQUAL(exMode, EEx::IfExists); + const TString ifNotExistsStatement = exMode == EEx::IfNotExists ? "IF NOT EXISTS" : ""; + const TString sql = fmt::format(R"sql( + CREATE OBJECT {if_not_exists} my_secret_{name_suffix} (TYPE SECRET) WITH (value="qwerty"); + )sql", + "if_not_exists"_a = ifNotExistsStatement, + "name_suffix"_a = nameSuffix + ); + + executeSql(sql, expectSuccess); + }; + + auto checkAlter = [&](bool expectSuccess, int nameSuffix) { + const TString sql = fmt::format(R"sql( + ALTER OBJECT my_secret_{name_suffix} (TYPE SECRET) SET value = "abcde"; + )sql", + "name_suffix"_a = nameSuffix + ); + + executeSql(sql, expectSuccess); + }; + + auto checkUpsert = [&](bool expectSuccess, int nameSuffix) { + const TString sql = fmt::format(R"sql( + UPSERT OBJECT my_secret_{name_suffix} (TYPE SECRET) WITH value = "edcba"; + )sql", + "name_suffix"_a = nameSuffix + ); + + executeSql(sql, expectSuccess); + }; + + auto checkDrop = [&](bool expectSuccess, EEx exMode, int nameSuffix) { + UNIT_ASSERT_UNEQUAL(exMode, EEx::IfNotExists); + const TString ifExistsStatement = exMode == EEx::IfExists ? "IF EXISTS" : ""; + const TString sql = fmt::format(R"sql( + DROP OBJECT {if_exists} my_secret_{name_suffix} (TYPE SECRET); + )sql", + "if_exists"_a = ifExistsStatement, + "name_suffix"_a = nameSuffix + ); + + executeSql(sql, expectSuccess); + }; + + checkCreate(true, EEx::Empty, 0); + checkCreate(false, EEx::Empty, 0); + checkAlter(true, 0); + checkAlter(false, 2); // not exists + checkDrop(true, EEx::Empty, 0); + checkDrop(true, EEx::Empty, 0); // we don't check object existence + + checkCreate(true, EEx::IfNotExists, 1); + checkCreate(true, EEx::IfNotExists, 1); + checkDrop(true, EEx::IfExists, 1); + checkDrop(true, EEx::IfExists, 1); + + checkUpsert(true, 2); + checkCreate(false, EEx::Empty, 2); // already exists + checkUpsert(true, 2); + } + Y_UNIT_TEST(DdlCache) { NKikimrConfig::TAppConfig appConfig; appConfig.MutableTableServiceConfig()->SetEnablePreparedDdl(true); diff --git a/ydb/core/protos/kqp_physical.proto b/ydb/core/protos/kqp_physical.proto index 74e28c92bcf2..be5d9d64842c 100644 --- a/ydb/core/protos/kqp_physical.proto +++ b/ydb/core/protos/kqp_physical.proto @@ -374,6 +374,17 @@ message TKqpPhyResult { optional uint32 QueryResultIndex = 5; } +message TKqpPhyMetadataOperation { + message TColumnValue { + string Column = 1; + Ydb.Value Value = 2; + } + repeated TColumnValue ColumnValues = 1; + + optional bool SuccessOnNotExist = 2; + optional bool SuccessOnAlreadyExists = 3; +} + message TKqpSchemeOperation { enum EFlags { FLAG_UNSPECIFIED = 0; @@ -409,6 +420,10 @@ message TKqpSchemeOperation { NKikimrSchemeOp.TModifyScheme CreateView = 20; NKikimrSchemeOp.TModifyScheme AlterView = 21; NKikimrSchemeOp.TModifyScheme DropView = 22; + TKqpPhyMetadataOperation CreateObject = 23; + TKqpPhyMetadataOperation UpsertObject = 24; + TKqpPhyMetadataOperation AlterObject = 25; + TKqpPhyMetadataOperation DropObject = 26; } } diff --git a/ydb/core/tx/tiering/rule/manager.cpp b/ydb/core/tx/tiering/rule/manager.cpp index 0b3c24d9e74c..0e543098a7ec 100644 --- a/ydb/core/tx/tiering/rule/manager.cpp +++ b/ydb/core/tx/tiering/rule/manager.cpp @@ -37,7 +37,7 @@ NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus TTieringRule } NThreading::TFuture TTieringRulesManager::ExecutePrepared(const NKqpProto::TKqpSchemeOperation& /*schemeOperation*/, - const NMetadata::IClassBehaviour::TPtr& /*manager*/, const IOperationsManager::TExternalModificationContext& /*context*/) const { + const ui32 /*nodeId*/, const NMetadata::IClassBehaviour::TPtr& /*manager*/, const IOperationsManager::TExternalModificationContext& /*context*/) const { return NThreading::MakeFuture(NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus::Fail( "Execution of prepare operations for TIERING_RULE objects is not supported")); } diff --git a/ydb/core/tx/tiering/rule/manager.h b/ydb/core/tx/tiering/rule/manager.h index 45e7f767ef37..45057cfa04ff 100644 --- a/ydb/core/tx/tiering/rule/manager.h +++ b/ydb/core/tx/tiering/rule/manager.h @@ -18,7 +18,7 @@ class TTieringRulesManager: public NMetadata::NModifications::TGenericOperations const NMetadata::IClassBehaviour::TPtr& manager, IOperationsManager::TInternalModificationContext& context) const override; virtual NThreading::TFuture ExecutePrepared(const NKqpProto::TKqpSchemeOperation& schemeOperation, - const NMetadata::IClassBehaviour::TPtr& manager, const IOperationsManager::TExternalModificationContext& context) const override; + const ui32 nodeId, const NMetadata::IClassBehaviour::TPtr& manager, const IOperationsManager::TExternalModificationContext& context) const override; }; } diff --git a/ydb/core/tx/tiering/tier/manager.cpp b/ydb/core/tx/tiering/tier/manager.cpp index b8ae6f8b9909..7c657b3df15e 100644 --- a/ydb/core/tx/tiering/tier/manager.cpp +++ b/ydb/core/tx/tiering/tier/manager.cpp @@ -74,7 +74,7 @@ NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus TTiersManage } NThreading::TFuture TTiersManager::ExecutePrepared(const NKqpProto::TKqpSchemeOperation& /*schemeOperation*/, - const NMetadata::IClassBehaviour::TPtr& /*manager*/, const IOperationsManager::TExternalModificationContext& /*context*/) const { + const ui32 /*nodeId*/, const NMetadata::IClassBehaviour::TPtr& /*manager*/, const IOperationsManager::TExternalModificationContext& /*context*/) const { return NThreading::MakeFuture(NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus::Fail( "Execution of prepare operations for TIER objects is not supported")); } diff --git a/ydb/core/tx/tiering/tier/manager.h b/ydb/core/tx/tiering/tier/manager.h index 3d279986e1f7..547e06ef0011 100644 --- a/ydb/core/tx/tiering/tier/manager.h +++ b/ydb/core/tx/tiering/tier/manager.h @@ -18,7 +18,7 @@ class TTiersManager: public NMetadata::NModifications::TGenericOperationsManager const NMetadata::IClassBehaviour::TPtr& manager, IOperationsManager::TInternalModificationContext& context) const override; virtual NThreading::TFuture ExecutePrepared(const NKqpProto::TKqpSchemeOperation& schemeOperation, - const NMetadata::IClassBehaviour::TPtr& manager, const IOperationsManager::TExternalModificationContext& context) const override; + const ui32 nodeId, const NMetadata::IClassBehaviour::TPtr& manager, const IOperationsManager::TExternalModificationContext& context) const override; public: }; diff --git a/ydb/library/conclusion/result.h b/ydb/library/conclusion/result.h index 36a19176f5a5..17d72180b3b5 100644 --- a/ydb/library/conclusion/result.h +++ b/ydb/library/conclusion/result.h @@ -76,6 +76,15 @@ class TConclusion { return status->GetErrorMessage(); } } + + Ydb::StatusIds::StatusCode GetStatus() const { + auto* status = std::get_if(&Result); + if (!status) { + return Ydb::StatusIds::SUCCESS; + } else { + return status->GetStatus(); + } + } }; } diff --git a/ydb/library/conclusion/status.h b/ydb/library/conclusion/status.h index 0dd3d66fb250..86e018cd3407 100644 --- a/ydb/library/conclusion/status.h +++ b/ydb/library/conclusion/status.h @@ -1,4 +1,6 @@ #pragma once +#include + #include #include @@ -7,14 +9,19 @@ namespace NKikimr { class TConclusionStatus { private: std::optional ErrorMessage; + Ydb::StatusIds::StatusCode Status = Ydb::StatusIds::SUCCESS; TConclusionStatus() = default; - TConclusionStatus(const TString& errorMessage) - : ErrorMessage(errorMessage) { + TConclusionStatus(const TString& errorMessage, Ydb::StatusIds::StatusCode status = Ydb::StatusIds::INTERNAL_ERROR) + : ErrorMessage(errorMessage) + , Status(status) + { Y_ABORT_UNLESS(!!ErrorMessage); } - TConclusionStatus(const char* errorMessage) - : ErrorMessage(errorMessage) { + TConclusionStatus(const char* errorMessage, Ydb::StatusIds::StatusCode status = Ydb::StatusIds::INTERNAL_ERROR) + : ErrorMessage(errorMessage) + , Status(status) + { Y_ABORT_UNLESS(!!ErrorMessage); } public: @@ -23,6 +30,10 @@ class TConclusionStatus { return ErrorMessage ? *ErrorMessage : Default(); } + Ydb::StatusIds::StatusCode GetStatus() const { + return Status; + } + static TConclusionStatus Fail(const char* errorMessage) { return TConclusionStatus(errorMessage); } diff --git a/ydb/library/conclusion/ya.make b/ydb/library/conclusion/ya.make index 3fcb4c0e9b2c..dd50013dc5a4 100644 --- a/ydb/library/conclusion/ya.make +++ b/ydb/library/conclusion/ya.make @@ -6,6 +6,7 @@ SRCS( ) PEERDIR( + ydb/public/api/protos ) END() diff --git a/ydb/services/ext_index/metadata/manager.cpp b/ydb/services/ext_index/metadata/manager.cpp index d1f1fc94743c..adebe4f71c14 100644 --- a/ydb/services/ext_index/metadata/manager.cpp +++ b/ydb/services/ext_index/metadata/manager.cpp @@ -97,7 +97,7 @@ NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus TManager::Do } NThreading::TFuture TManager::ExecutePrepared(const NKqpProto::TKqpSchemeOperation& /*schemeOperation*/, - const NMetadata::IClassBehaviour::TPtr& /*manager*/, const IOperationsManager::TExternalModificationContext& /*context*/) const { + const ui32 /*nodeId*/, const NMetadata::IClassBehaviour::TPtr& /*manager*/, const IOperationsManager::TExternalModificationContext& /*context*/) const { return NThreading::MakeFuture(NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus::Fail( "Execution of prepare operations for CS_EXT_INDEX objects is not supported")); } diff --git a/ydb/services/ext_index/metadata/manager.h b/ydb/services/ext_index/metadata/manager.h index 48c6070819b4..70bf63575554 100644 --- a/ydb/services/ext_index/metadata/manager.h +++ b/ydb/services/ext_index/metadata/manager.h @@ -19,7 +19,7 @@ class TManager: public NModifications::TGenericOperationsManager { const NMetadata::IClassBehaviour::TPtr& manager, IOperationsManager::TInternalModificationContext& context) const override; virtual NThreading::TFuture ExecutePrepared(const NKqpProto::TKqpSchemeOperation& schemeOperation, - const NMetadata::IClassBehaviour::TPtr& manager, const IOperationsManager::TExternalModificationContext& context) const override; + const ui32 nodeId, const NMetadata::IClassBehaviour::TPtr& manager, const IOperationsManager::TExternalModificationContext& context) const override; public: }; diff --git a/ydb/services/ext_index/service/activation.cpp b/ydb/services/ext_index/service/activation.cpp index 797cc0a7de77..e52bcb6bd947 100644 --- a/ydb/services/ext_index/service/activation.cpp +++ b/ydb/services/ext_index/service/activation.cpp @@ -33,13 +33,13 @@ NKikimr::NMetadata::NRequest::TDialogYQLRequest::TRequest TActivation::BuildUpda void TActivation::OnDescriptionSuccess(NMetadata::NProvider::TTableInfo&& result, const TString& /*requestId*/) { if (!result->ColumnTableInfo) { - ExternalController->OnActivationFailed("incorrect column table info", RequestId); + ExternalController->OnActivationFailed(Ydb::StatusIds::INTERNAL_ERROR, "incorrect column table info", RequestId); SelfContainer = nullptr; return; } Ydb::Table::CreateTableRequest request; if (!Object.TryProvideTtl(result->ColumnTableInfo->Description, &request)) { - ExternalController->OnActivationFailed("cannot convert ttl method from column tables", RequestId); + ExternalController->OnActivationFailed(Ydb::StatusIds::INTERNAL_ERROR, "cannot convert ttl method from column tables", RequestId); SelfContainer = nullptr; return; } @@ -61,7 +61,7 @@ void TActivation::OnDescriptionSuccess(NMetadata::NProvider::TTableInfo&& result column.set_name("pk_" + i.Name); auto primitiveType = NMetadata::NInternal::TYDBType::ConvertYQLToYDB(i.PType.GetTypeId()); if (!primitiveType) { - ExternalController->OnActivationFailed("cannot convert type yql -> ydb", RequestId); + ExternalController->OnActivationFailed(Ydb::StatusIds::INTERNAL_ERROR, "cannot convert type yql -> ydb", RequestId); SelfContainer = nullptr; return; } @@ -83,8 +83,8 @@ void TActivation::OnModificationFinished(const TString& modificationId) { } } -void TActivation::OnModificationFailed(const TString& errorMessage, const TString& /*modificationId*/) { - ExternalController->OnActivationFailed(errorMessage, RequestId); +void TActivation::OnModificationFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage, const TString& /*modificationId*/) { + ExternalController->OnActivationFailed(status, errorMessage, RequestId); SelfContainer = nullptr; } diff --git a/ydb/services/ext_index/service/activation.h b/ydb/services/ext_index/service/activation.h index a7dcaf74ff82..9df8b67c4ab1 100644 --- a/ydb/services/ext_index/service/activation.h +++ b/ydb/services/ext_index/service/activation.h @@ -12,7 +12,7 @@ class IActivationExternalController { public: using TPtr = std::shared_ptr; virtual ~IActivationExternalController() = default; - virtual void OnActivationFailed(const TString& errorMessage, const TString& requestId) = 0; + virtual void OnActivationFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage, const TString& requestId) = 0; virtual void OnActivationSuccess(const TString& requestId) = 0; }; @@ -31,22 +31,22 @@ class TActivation: public NMetadata::NProvider::ISchemeDescribeController, protected: virtual void OnDescriptionFailed(const TString& errorMessage, const TString& requestId) override { - ExternalController->OnActivationFailed(errorMessage, requestId); + ExternalController->OnActivationFailed(Ydb::StatusIds::INTERNAL_ERROR, errorMessage, requestId); SelfContainer = nullptr; } virtual void OnDescriptionSuccess(NMetadata::NProvider::TTableInfo&& result, const TString& requestId) override; virtual void OnModificationFinished(const TString& modificationId) override; - virtual void OnModificationFailed(const TString& errorMessage, const TString& modificationId) override; + virtual void OnModificationFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage, const TString& modificationId) override; virtual void OnRequestResult(NMetadata::NRequest::TDialogYQLRequest::TResponse&& /*result*/) override { ExternalController->OnActivationSuccess(RequestId); SelfContainer = nullptr; } - virtual void OnRequestFailed(const TString& errorMessage) override { - ExternalController->OnActivationFailed(errorMessage, RequestId); + virtual void OnRequestFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage) override { + ExternalController->OnActivationFailed(status, errorMessage, RequestId); SelfContainer = nullptr; } public: diff --git a/ydb/services/ext_index/service/deleting.cpp b/ydb/services/ext_index/service/deleting.cpp index ee744530e0f6..ebafd645f632 100644 --- a/ydb/services/ext_index/service/deleting.cpp +++ b/ydb/services/ext_index/service/deleting.cpp @@ -46,8 +46,8 @@ void TDeleting::OnModificationFinished(const TString& /*modificationId*/) { NMetadata::NRequest::TYQLRequestExecutor::Execute(BuildDeleteRequest(), NACLib::TSystemUsers::Metadata(), SelfContainer); } -void TDeleting::OnModificationFailed(const TString& errorMessage, const TString& /*modificationId*/) { - ExternalController->OnDeletingFailed(errorMessage, RequestId); +void TDeleting::OnModificationFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage, const TString& /*modificationId*/) { + ExternalController->OnDeletingFailed(status, errorMessage, RequestId); SelfContainer = nullptr; } diff --git a/ydb/services/ext_index/service/deleting.h b/ydb/services/ext_index/service/deleting.h index 61ac740c5aa9..5518c48a1ea7 100644 --- a/ydb/services/ext_index/service/deleting.h +++ b/ydb/services/ext_index/service/deleting.h @@ -13,7 +13,7 @@ class IDeletingExternalController { public: using TPtr = std::shared_ptr; virtual ~IDeletingExternalController() = default; - virtual void OnDeletingFailed(const TString& errorMessage, const TString& requestId) = 0; + virtual void OnDeletingFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage, const TString& requestId) = 0; virtual void OnDeletingSuccess(const TString& requestId) = 0; }; @@ -33,10 +33,10 @@ class TDeleting: protected: virtual void OnModificationFinished(const TString& modificationId) override; - virtual void OnModificationFailed(const TString& errorMessage, const TString& modificationId) override; + virtual void OnModificationFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage, const TString& modificationId) override; - virtual void OnRequestFailed(const TString& errorMessage) override { - ExternalController->OnDeletingFailed(errorMessage, RequestId); + virtual void OnRequestFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage) override { + ExternalController->OnDeletingFailed(status, errorMessage, RequestId); SelfContainer = nullptr; } diff --git a/ydb/services/ext_index/service/executor.cpp b/ydb/services/ext_index/service/executor.cpp index 47db37ba14ea..d5af1a98ac27 100644 --- a/ydb/services/ext_index/service/executor.cpp +++ b/ydb/services/ext_index/service/executor.cpp @@ -27,13 +27,13 @@ void TExecutor::Handle(TEvAddData::TPtr& ev) { class TIndexesController: public IActivationExternalController, public IDeletingExternalController { public: - virtual void OnActivationFailed(const TString& errorMessage, const TString& requestId) override { + virtual void OnActivationFailed(Ydb::StatusIds::StatusCode /*status*/, const TString& errorMessage, const TString& requestId) override { ALS_ERROR(NKikimrServices::EXT_INDEX) << "cannot activate index for " << requestId << ": " << errorMessage; } virtual void OnActivationSuccess(const TString& requestId) override { ALS_NOTICE(NKikimrServices::EXT_INDEX) << "index activated " << requestId; } - virtual void OnDeletingFailed(const TString& errorMessage, const TString& requestId) override { + virtual void OnDeletingFailed(Ydb::StatusIds::StatusCode /*status*/, const TString& errorMessage, const TString& requestId) override { ALS_ERROR(NKikimrServices::EXT_INDEX) << "cannot remove index for " << requestId << ": " << errorMessage; } virtual void OnDeletingSuccess(const TString& requestId) override { diff --git a/ydb/services/metadata/initializer/accessor_init.cpp b/ydb/services/metadata/initializer/accessor_init.cpp index 03a76588e9fc..01a09dd68f99 100644 --- a/ydb/services/metadata/initializer/accessor_init.cpp +++ b/ydb/services/metadata/initializer/accessor_init.cpp @@ -79,7 +79,7 @@ void TDSAccessorInitialized::OnAlteringProblem(const TString& errorMessage) { }, TDuration::Seconds(1)); } -void TDSAccessorInitialized::OnModificationFailed(const TString& errorMessage, const TString& modificationId) { +void TDSAccessorInitialized::OnModificationFailed(Ydb::StatusIds::StatusCode /*status*/, const TString& errorMessage, const TString& modificationId) { AFL_ERROR(NKikimrServices::METADATA_INITIALIZER)("event", "OnModificationFailed")("error", errorMessage)("modificationId", modificationId); NActors::ScheduleInvokeActivity([self = this->SelfPtr]() { Y_ABORT_UNLESS(self->Modifiers.size()); diff --git a/ydb/services/metadata/initializer/accessor_init.h b/ydb/services/metadata/initializer/accessor_init.h index a83f866c654f..7bff4adba61d 100644 --- a/ydb/services/metadata/initializer/accessor_init.h +++ b/ydb/services/metadata/initializer/accessor_init.h @@ -33,7 +33,7 @@ class TDSAccessorInitialized: public IInitializerInput, virtual void OnAlteringFinished() override; virtual void OnModificationFinished(const TString& modificationId) override; - virtual void OnModificationFailed(const TString& errorMessage, const TString& modificationId) override; + virtual void OnModificationFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage, const TString& modificationId) override; TDSAccessorInitialized(const NRequest::TConfig& config, const TString& componentId, diff --git a/ydb/services/metadata/initializer/common.h b/ydb/services/metadata/initializer/common.h index 6ddb0b1ac4ae..9a72628dcccd 100644 --- a/ydb/services/metadata/initializer/common.h +++ b/ydb/services/metadata/initializer/common.h @@ -11,7 +11,7 @@ class IModifierExternalController { using TPtr = std::shared_ptr; virtual ~IModifierExternalController() = default; virtual void OnModificationFinished(const TString& modificationId) = 0; - virtual void OnModificationFailed(const TString& errorMessage, const TString& modificationId) = 0; + virtual void OnModificationFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage, const TString& modificationId) = 0; }; class ITableModifier { @@ -55,8 +55,8 @@ class TGenericTableModifier: public ITableModifier { virtual void OnRequestResult(typename TDialogPolicy::TResponse&& /*result*/) override { ExternalController->OnModificationFinished(ModificationId); } - virtual void OnRequestFailed(const TString& errorMessage) override { - ExternalController->OnModificationFailed(errorMessage, ModificationId); + virtual void OnRequestFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage) override { + ExternalController->OnModificationFailed(status, errorMessage, ModificationId); } }; diff --git a/ydb/services/metadata/initializer/manager.cpp b/ydb/services/metadata/initializer/manager.cpp index 715580877fd0..3d6a1c54da0d 100644 --- a/ydb/services/metadata/initializer/manager.cpp +++ b/ydb/services/metadata/initializer/manager.cpp @@ -24,7 +24,7 @@ NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus TManager::Do } NThreading::TFuture TManager::ExecutePrepared(const NKqpProto::TKqpSchemeOperation& /*schemeOperation*/, - const NMetadata::IClassBehaviour::TPtr& /*manager*/, const IOperationsManager::TExternalModificationContext& /*context*/) const { + const ui32 /*nodeId*/, const NMetadata::IClassBehaviour::TPtr& /*manager*/, const IOperationsManager::TExternalModificationContext& /*context*/) const { return NThreading::MakeFuture(NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus::Fail( "Execution of prepare operations for INITIALIZATION objects is not supported")); } diff --git a/ydb/services/metadata/initializer/manager.h b/ydb/services/metadata/initializer/manager.h index 394e5490968b..3c4b04f783f2 100644 --- a/ydb/services/metadata/initializer/manager.h +++ b/ydb/services/metadata/initializer/manager.h @@ -24,7 +24,7 @@ class TManager: public NModifications::TGenericOperationsManager ExecutePrepared(const NKqpProto::TKqpSchemeOperation& schemeOperation, - const NMetadata::IClassBehaviour::TPtr& manager, const IOperationsManager::TExternalModificationContext& context) const override; + const ui32 nodeId, const NMetadata::IClassBehaviour::TPtr& manager, const IOperationsManager::TExternalModificationContext& context) const override; public: }; diff --git a/ydb/services/metadata/manager/abstract.h b/ydb/services/metadata/manager/abstract.h index 0db55d5b7b92..132afb8a8a12 100644 --- a/ydb/services/metadata/manager/abstract.h +++ b/ydb/services/metadata/manager/abstract.h @@ -117,7 +117,7 @@ class IOperationsManager { const TExternalModificationContext& context) const; virtual NThreading::TFuture ExecutePrepared(const NKqpProto::TKqpSchemeOperation& schemeOperation, - const IClassBehaviour::TPtr& manager, const TExternalModificationContext& context) const = 0; + const ui32 nodeId, const IClassBehaviour::TPtr& manager, const TExternalModificationContext& context) const = 0; const TTableSchema& GetSchema() const { Y_ABORT_UNLESS(!!ActualSchema); diff --git a/ydb/services/metadata/manager/alter.h b/ydb/services/metadata/manager/alter.h index 5283835959c1..481675db202d 100644 --- a/ydb/services/metadata/manager/alter.h +++ b/ydb/services/metadata/manager/alter.h @@ -50,11 +50,12 @@ template class TCreateObjectActor: public TModificationActor { private: using TBase = TModificationActor; + bool ExistingOk = false; protected: virtual bool ProcessPreparedObjects(NInternal::TTableRecords&& records) const override { TBase::Register(new TInsertObjectsActor(std::move(records), TBase::UserToken, TBase::InternalController, TBase::SessionId, TBase::TransactionId, - TBase::Context.GetExternalData().GetUserToken())); + TBase::Context.GetExternalData().GetUserToken(), ExistingOk)); return true; } @@ -63,6 +64,10 @@ class TCreateObjectActor: public TModificationActor { } public: using TBase::TBase; + + void SetExistingOk(bool existingOk) { + ExistingOk = existingOk; + } }; template @@ -126,13 +131,24 @@ template class TCreateObjectCommand: public IObjectModificationCommand { private: using TBase = IObjectModificationCommand; + bool ExistingOk = false; protected: virtual void DoExecute() const override { typename IObjectOperationsManager::TPtr manager = TBase::GetOperationsManagerFor(); - TActivationContext::AsActorContext().Register(new TCreateObjectActor(GetRecords(), GetController(), manager, GetContext())); + auto* actor = new TCreateObjectActor(GetRecords(), GetController(), manager, GetContext()); + actor->SetExistingOk(ExistingOk); + TActivationContext::AsActorContext().Register(actor); } public: - using TBase::TBase; + TCreateObjectCommand(const NInternal::TTableRecord& record, + IClassBehaviour::TPtr behaviour, + NModifications::IAlterController::TPtr controller, + const IOperationsManager::TInternalModificationContext& context, + bool existingOk) + : TBase(record, behaviour, controller, context) + , ExistingOk(existingOk) + { + } }; template @@ -152,13 +168,22 @@ template class TDeleteObjectCommand: public IObjectModificationCommand { private: using TBase = IObjectModificationCommand; + bool MissingOk = false; protected: virtual void DoExecute() const override { typename IObjectOperationsManager::TPtr manager = TBase::GetOperationsManagerFor(); TActivationContext::AsActorContext().Register(new TDeleteObjectActor(GetRecords(), GetController(), manager, GetContext())); } public: - using TBase::TBase; + TDeleteObjectCommand(const NInternal::TTableRecord& record, + IClassBehaviour::TPtr behaviour, + NModifications::IAlterController::TPtr controller, + const IOperationsManager::TInternalModificationContext& context, + bool missingOk) + : TBase(record, behaviour, controller, context) + , MissingOk(missingOk) + { + } }; } diff --git a/ydb/services/metadata/manager/generic_manager.h b/ydb/services/metadata/manager/generic_manager.h index 51062e89ea75..570317ca796a 100644 --- a/ydb/services/metadata/manager/generic_manager.h +++ b/ydb/services/metadata/manager/generic_manager.h @@ -8,7 +8,7 @@ class TOperationsController: public IAlterController { private: YDB_READONLY_DEF(NThreading::TPromise, Promise); public: - TOperationsController(NThreading::TPromise&& p) + TOperationsController(NThreading::TPromise p) : Promise(std::move(p)) { @@ -30,6 +30,8 @@ class TGenericOperationsManager: public IObjectOperationsManager { using IOperationsManager::TYqlConclusionStatus; public: using TInternalModificationContext = typename TBase::TInternalModificationContext; + using TExternalModificationContext = typename TBase::TExternalModificationContext; + using EActivityType = typename IOperationsManager::EActivityType; protected: virtual NThreading::TFuture DoModify( const NYql::TObjectSettingsImpl& settings, const ui32 nodeId, @@ -42,35 +44,123 @@ class TGenericOperationsManager: public IObjectOperationsManager { return NThreading::MakeFuture(TYqlConclusionStatus::Fail("modification is unavailable for " + manager->GetTypeId())); } auto promise = NThreading::NewPromise(); - auto result = promise.GetFuture(); { TOperationParsingResult patch(TBase::BuildPatchFromSettings(settings, context)); if (!patch.IsSuccess()) { return NThreading::MakeFuture(TYqlConclusionStatus::Fail(patch.GetErrorMessage())); } - auto controller = std::make_shared(std::move(promise)); + auto controller = std::make_shared(promise); IObjectModificationCommand::TPtr modifyObjectCommand; switch (context.GetActivityType()) { - case IOperationsManager::EActivityType::Upsert: - modifyObjectCommand = std::make_shared>(patch.GetResult(), manager, controller, context); + case EActivityType::Upsert: + modifyObjectCommand = std::make_shared>(patch.GetResult(), manager, std::move(controller), context); break; - case IOperationsManager::EActivityType::Create: - modifyObjectCommand = std::make_shared>(patch.GetResult(), manager, controller, context); + case EActivityType::Create: + modifyObjectCommand = std::make_shared>(patch.GetResult(), manager, std::move(controller), context, settings.GetExistingOk()); break; - case IOperationsManager::EActivityType::Alter: - modifyObjectCommand = std::make_shared>(patch.GetResult(), manager, controller, context); + case EActivityType::Alter: + modifyObjectCommand = std::make_shared>(patch.GetResult(), manager, std::move(controller), context); break; - case IOperationsManager::EActivityType::Drop: - modifyObjectCommand = std::make_shared>(patch.GetResult(), manager, controller, context); + case EActivityType::Drop: + modifyObjectCommand = std::make_shared>(patch.GetResult(), manager, std::move(controller), context, settings.GetMissingOk()); break; - case IOperationsManager::EActivityType::Undefined: + case EActivityType::Undefined: return NThreading::MakeFuture(TYqlConclusionStatus::Fail("undefined action type")); } TActivationContext::Send(new IEventHandle(NProvider::MakeServiceId(nodeId), {}, new NProvider::TEvObjectsOperation(modifyObjectCommand))); } + return promise; + } + + virtual TYqlConclusionStatus DoPrepare(NKqpProto::TKqpSchemeOperation& schemeOperation, const NYql::TObjectSettingsImpl& settings, + const IClassBehaviour::TPtr& manager, TInternalModificationContext& context) const override { + if (!manager) { + return TYqlConclusionStatus::Fail("modification object behaviour not initialized"); + } + if (!manager->GetOperationsManager()) { + return TYqlConclusionStatus::Fail("modification is unavailable for " + manager->GetTypeId()); + } + + TOperationParsingResult patch(TBase::BuildPatchFromSettings(settings, context)); + if (!patch.IsSuccess()) { + return TYqlConclusionStatus::Fail(patch.GetErrorMessage()); + } + + NKqpProto::TKqpPhyMetadataOperation* operationProto = nullptr; + switch (context.GetActivityType()) { + case EActivityType::Upsert: + operationProto = schemeOperation.MutableUpsertObject(); + break; + case EActivityType::Create: + operationProto = schemeOperation.MutableCreateObject(); + operationProto->SetSuccessOnAlreadyExists(settings.GetExistingOk()); + break; + case EActivityType::Alter: + operationProto = schemeOperation.MutableAlterObject(); + break; + case EActivityType::Drop: + operationProto = schemeOperation.MutableDropObject(); + operationProto->SetSuccessOnNotExist(settings.GetMissingOk()); + break; + case EActivityType::Undefined: + return TYqlConclusionStatus::Fail("undefined action type"); + } + Y_ENSURE(operationProto); + + for (const auto& [col, value] : patch.GetResult().GetValues()) { + auto* proto = operationProto->AddColumnValues(); + proto->SetColumn(col); + proto->MutableValue()->CopyFrom(value); + } + + return TYqlConclusionStatus::Success(); + } + + virtual NThreading::TFuture ExecutePrepared(const NKqpProto::TKqpSchemeOperation& schemeOperation, + const ui32 nodeId, const IClassBehaviour::TPtr& manager, const TExternalModificationContext& context) const override { + if (!manager) { + return NThreading::MakeFuture(TYqlConclusionStatus::Fail("modification object behaviour not initialized")); + } + if (!manager->GetOperationsManager()) { + return NThreading::MakeFuture(TYqlConclusionStatus::Fail("modification is unavailable for " + manager->GetTypeId())); + } + + auto promise = NThreading::NewPromise(); + auto controller = std::make_shared(promise); + IObjectModificationCommand::TPtr modifyObjectCommand; + TInternalModificationContext internalContext(context); + switch (schemeOperation.GetOperationCase()) { + case NKqpProto::TKqpSchemeOperation::kUpsertObject: + internalContext.SetActivityType(EActivityType::Upsert); + modifyObjectCommand = std::make_shared>(BuildPatchFromProto(schemeOperation.GetUpsertObject()), manager, std::move(controller), internalContext); + break; + case NKqpProto::TKqpSchemeOperation::kCreateObject: + internalContext.SetActivityType(EActivityType::Create); + modifyObjectCommand = std::make_shared>(BuildPatchFromProto(schemeOperation.GetCreateObject()), manager, std::move(controller), internalContext, schemeOperation.GetCreateObject().GetSuccessOnAlreadyExists()); + break; + case NKqpProto::TKqpSchemeOperation::kAlterObject: + internalContext.SetActivityType(EActivityType::Alter); + modifyObjectCommand = std::make_shared>(BuildPatchFromProto(schemeOperation.GetAlterObject()), manager, std::move(controller), internalContext); + break; + case NKqpProto::TKqpSchemeOperation::kDropObject: + internalContext.SetActivityType(EActivityType::Drop); + modifyObjectCommand = std::make_shared>(BuildPatchFromProto(schemeOperation.GetDropObject()), manager, std::move(controller), internalContext, schemeOperation.GetDropObject().GetSuccessOnNotExist()); + break; + default: + return NThreading::MakeFuture(TYqlConclusionStatus::Fail("undefined operation type")); + } + TActivationContext::Send(new IEventHandle(NProvider::MakeServiceId(nodeId), {}, new NProvider::TEvObjectsOperation(modifyObjectCommand))); + + return promise; + } +private: + static NInternal::TTableRecord BuildPatchFromProto(const NKqpProto::TKqpPhyMetadataOperation& op) { + NInternal::TTableRecord result; + for (const auto& val : op.GetColumnValues()) { + result.SetColumn(val.GetColumn(), val.GetValue()); + } return result; } -public: }; } diff --git a/ydb/services/metadata/manager/modification.h b/ydb/services/metadata/manager/modification.h index cf7a01618e3d..382fd23f6f27 100644 --- a/ydb/services/metadata/manager/modification.h +++ b/ydb/services/metadata/manager/modification.h @@ -19,8 +19,8 @@ class TModifyObjectsActor: public NActors::TActorBootstrapped UserToken; - std::deque Requests; protected: + std::deque Requests; NInternal::TTableRecords Objects; virtual Ydb::Table::ExecuteDataQueryRequest BuildModifyQuery() const = 0; virtual TString GetModifyType() const = 0; @@ -58,7 +58,7 @@ class TModifyObjectsActor: public NActors::TActorBootstrappedOnModificationProblem("cannot execute yql request for " + GetModifyType() + " objects: " + ev->Get()->GetErrorMessage()); @@ -152,6 +152,7 @@ template class TInsertObjectsActor: public TModifyObjectsActor { private: using TBase = TModifyObjectsActor; + bool ExistingOk = false; protected: virtual Ydb::Table::ExecuteDataQueryRequest BuildModifyQuery() const override { return TBase::Objects.BuildInsertQuery(TObject::GetBehaviour()->GetStorageTablePath()); @@ -159,8 +160,23 @@ class TInsertObjectsActor: public TModifyObjectsActor { virtual TString GetModifyType() const override { return "insert"; } + + void Handle(NRequest::TEvRequestFailed::TPtr& ev) override { + if (ev->Get()->GetStatus() == Ydb::StatusIds::PRECONDITION_FAILED && ExistingOk) { + NRequest::TDialogYQLRequest::TResponse resp; + this->Send(this->SelfId(), new NRequest::TEvRequestResult(std::move(resp))); + this->Requests.clear(); // Remove history request + return; + } + TBase::Handle(ev); + } public: - using TBase::TBase; + TInsertObjectsActor(NInternal::TTableRecords&& objects, const NACLib::TUserToken& systemUserToken, IModificationObjectsController::TPtr controller, const TString& sessionId, + const TString& transactionId, const std::optional& userToken, bool existingOk) + : TBase(std::move(objects), systemUserToken, std::move(controller), sessionId, transactionId, userToken) + , ExistingOk(existingOk) + { + } }; } diff --git a/ydb/services/metadata/request/common.h b/ydb/services/metadata/request/common.h index 05564a999309..cacfa187e413 100644 --- a/ydb/services/metadata/request/common.h +++ b/ydb/services/metadata/request/common.h @@ -36,7 +36,7 @@ enum EEvents { EvModifyPermissionsRequest, EvModifyPermissionsInternalResponse, EvModifyPermissionsResponse, - + EvRequestFinished, EvRequestFailed, EvRequestStart, @@ -72,7 +72,7 @@ class IExternalController { using TPtr = std::shared_ptr; virtual ~IExternalController() = default; virtual void OnRequestResult(typename TDialogPolicy::TResponse&& result) = 0; - virtual void OnRequestFailed(const TString& errorMessage) = 0; + virtual void OnRequestFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage) = 0; }; using TDialogCreatePath = TDialogPolicyImpl { private: + YDB_READONLY_DEF(Ydb::StatusIds::StatusCode, Status); YDB_READONLY_DEF(TString, ErrorMessage) public: - TEvRequestFailed(const TString& errorMessage) - : ErrorMessage(errorMessage) { - + TEvRequestFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage) + : Status(status) + , ErrorMessage(errorMessage) + { } }; diff --git a/ydb/services/metadata/request/request_actor.h b/ydb/services/metadata/request/request_actor.h index c502178ebbb0..80a1c2612066 100644 --- a/ydb/services/metadata/request/request_actor.h +++ b/ydb/services/metadata/request/request_actor.h @@ -36,7 +36,7 @@ class TYDBOneRequest: public NActors::TActorBootstrappedGet()->GetFuture().HasValue() || ev->Get()->GetFuture().HasException()) { ALS_ERROR(NKikimrServices::METADATA_PROVIDER) << "cannot receive result on initialization"; - OnInternalResultError("cannot receive result from future"); + OnInternalResultError(Ydb::StatusIds::INTERNAL_ERROR, "cannot receive result from future"); return; } auto f = ev->Get()->GetFuture(); @@ -64,7 +64,7 @@ class TYDBOneRequest: public NActors::TActorBootstrapped { const TConfig Config; ui32 Retry = 0; protected: - virtual void OnInternalResultError(const TString& errorMessage) override { - TBase::template Sender(errorMessage).SendTo(CallbackActorId); + virtual void OnInternalResultError(Ydb::StatusIds::StatusCode status, const TString& errorMessage) override { + TBase::template Sender(status, errorMessage).SendTo(CallbackActorId); TBase::PassAway(); } virtual void OnInternalResultSuccess(TResponse&& response) override { diff --git a/ydb/services/metadata/request/request_actor_cb.h b/ydb/services/metadata/request/request_actor_cb.h index 40b8d1ce75b2..07bea21c9992 100644 --- a/ydb/services/metadata/request/request_actor_cb.h +++ b/ydb/services/metadata/request/request_actor_cb.h @@ -26,15 +26,15 @@ class TYDBOneRequestSender { static void OnInternalResult(const NThreading::TFuture& f, typename IExternalController::TPtr externalController) { if (!f.HasValue() || f.HasException()) { ALS_ERROR(NKikimrServices::METADATA_PROVIDER) << "cannot receive result on initialization"; - externalController->OnRequestFailed("cannot receive result from future"); + externalController->OnRequestFailed(Ydb::StatusIds::INTERNAL_ERROR, "cannot receive result from future"); return; } TResponse response = f.GetValue(); if (!TOperatorChecker::IsSuccess(response)) { AFL_ERROR(NKikimrServices::METADATA_PROVIDER)("event", "unexpected reply")("response", response.DebugString()); - NYql::TIssues issue; - NYql::IssuesFromMessage(response.operation().issues(), issue); - externalController->OnRequestFailed(issue.ToString()); + NYql::TIssues issues; + NYql::IssuesFromMessage(response.operation().issues(), issues); + externalController->OnRequestFailed(response.operation().status(), issues.ToString()); return; } externalController->OnRequestResult(std::move(response)); @@ -74,14 +74,14 @@ class IChainController: public IExternalController { virtual void OnRequestResult(typename TCurrentDialogPolicy::TResponse&& result) override { TConclusion nextRequest = BuildNextRequest(std::move(result)); if (!nextRequest) { - OnRequestFailed(nextRequest.GetErrorMessage()); + OnRequestFailed(nextRequest.GetStatus(), nextRequest.GetErrorMessage()); } else { TYDBOneRequestSender req(*nextRequest, UserToken, NextController); req.Start(); } } - virtual void OnRequestFailed(const TString& errorMessage) override final { - NextController->OnRequestFailed(errorMessage); + virtual void OnRequestFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage) override final { + NextController->OnRequestFailed(status, errorMessage); } IChainController(const NACLib::TUserToken& userToken, std::shared_ptr nextController) : NextController(nextController) @@ -149,8 +149,8 @@ class TNaiveExternalController: public IExternalController { ActorId.Send(ActorId, new TEvRequestResult(std::move(result))); ActorId.Send(ActorId, new TEvRequestFinished); } - virtual void OnRequestFailed(const TString& errorMessage) override { - ActorId.Send(ActorId, new TEvRequestFailed(errorMessage)); + virtual void OnRequestFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage) override { + ActorId.Send(ActorId, new TEvRequestFailed(status, errorMessage)); } }; @@ -165,7 +165,7 @@ class TSessionDeleteResponseController: public IExternalControllerGetSessionId() << ", reason: " << errorMessage; } }; @@ -201,8 +201,8 @@ class TSessionDeleteController: public IExternalController { CloseSession(); } - virtual void OnRequestFailed(const TString& errorMessage) override { - ExternalController->OnRequestFailed(errorMessage); + virtual void OnRequestFailed(Ydb::StatusIds::StatusCode status, const TString& errorMessage) override { + ExternalController->OnRequestFailed(status, errorMessage); CloseSession(); } }; diff --git a/ydb/services/metadata/secret/manager.cpp b/ydb/services/metadata/secret/manager.cpp index 64e643b3dac9..01ea816b08e7 100644 --- a/ydb/services/metadata/secret/manager.cpp +++ b/ydb/services/metadata/secret/manager.cpp @@ -53,7 +53,7 @@ NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus TAccessManag } NThreading::TFuture TAccessManager::ExecutePrepared(const NKqpProto::TKqpSchemeOperation& /*schemeOperation*/, - const NMetadata::IClassBehaviour::TPtr& /*manager*/, const IOperationsManager::TExternalModificationContext& /*context*/) const { + const ui32 /*nodeId*/, const NMetadata::IClassBehaviour::TPtr& /*manager*/, const IOperationsManager::TExternalModificationContext& /*context*/) const { return NThreading::MakeFuture(NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus::Fail( "Execution of prepare operations for SECRET_ACCESS objects is not supported")); } @@ -112,16 +112,4 @@ void TSecretManager::DoPrepareObjectsBeforeModification(std::vector&& p TActivationContext::Register(new TSecretPreparationActor(std::move(patchedObjects), controller, context)); } -NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus TSecretManager::DoPrepare(NKqpProto::TKqpSchemeOperation& /*schemeOperation*/, const NYql::TObjectSettingsImpl& /*settings*/, - const NMetadata::IClassBehaviour::TPtr& /*manager*/, NMetadata::NModifications::IOperationsManager::TInternalModificationContext& /*context*/) const { - return NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus::Fail( - "Prepare operations for SECRET objects are not supported"); -} - -NThreading::TFuture TSecretManager::ExecutePrepared(const NKqpProto::TKqpSchemeOperation& /*schemeOperation*/, - const NMetadata::IClassBehaviour::TPtr& /*manager*/, const IOperationsManager::TExternalModificationContext& /*context*/) const { - return NThreading::MakeFuture(NMetadata::NModifications::IOperationsManager::TYqlConclusionStatus::Fail( - "Execution of prepare operations for SECRET objects is not supported")); -} - } diff --git a/ydb/services/metadata/secret/manager.h b/ydb/services/metadata/secret/manager.h index 5e0e4934b92e..d83911f67b19 100644 --- a/ydb/services/metadata/secret/manager.h +++ b/ydb/services/metadata/secret/manager.h @@ -14,14 +14,6 @@ class TSecretManager: public NModifications::TGenericOperationsManager virtual NModifications::TOperationParsingResult DoBuildPatchFromSettings( const NYql::TObjectSettingsImpl& settings, TInternalModificationContext& context) const override; - - virtual IOperationsManager::TYqlConclusionStatus DoPrepare(NKqpProto::TKqpSchemeOperation& schemeOperation, const NYql::TObjectSettingsImpl& settings, - const NMetadata::IClassBehaviour::TPtr& manager, IOperationsManager::TInternalModificationContext& context) const override; - - virtual NThreading::TFuture ExecutePrepared(const NKqpProto::TKqpSchemeOperation& schemeOperation, - const NMetadata::IClassBehaviour::TPtr& manager, const IOperationsManager::TExternalModificationContext& context) const override; - -public: }; class TAccessManager: public NModifications::TGenericOperationsManager { @@ -37,7 +29,7 @@ class TAccessManager: public NModifications::TGenericOperationsManager const NMetadata::IClassBehaviour::TPtr& manager, IOperationsManager::TInternalModificationContext& context) const override; virtual NThreading::TFuture ExecutePrepared(const NKqpProto::TKqpSchemeOperation& schemeOperation, - const NMetadata::IClassBehaviour::TPtr& manager, const IOperationsManager::TExternalModificationContext& context) const override; + const ui32 nodeId, const NMetadata::IClassBehaviour::TPtr& manager, const IOperationsManager::TExternalModificationContext& context) const override; }; }