diff --git a/ydb/core/external_sources/external_source_factory.cpp b/ydb/core/external_sources/external_source_factory.cpp index 98e4514f0242..4ef363f06cd9 100644 --- a/ydb/core/external_sources/external_source_factory.cpp +++ b/ydb/core/external_sources/external_source_factory.cpp @@ -32,12 +32,12 @@ struct TExternalSourceFactory : public IExternalSourceFactory { } -IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector& hostnamePatterns) { +IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector& hostnamePatterns, size_t pathsLimit) { std::vector hostnamePatternsRegEx(hostnamePatterns.begin(), hostnamePatterns.end()); return MakeIntrusive(TMap{ { ToString(NYql::EDatabaseType::ObjectStorage), - CreateObjectStorageExternalSource(hostnamePatternsRegEx) + CreateObjectStorageExternalSource(hostnamePatternsRegEx, pathsLimit) }, { ToString(NYql::EDatabaseType::ClickHouse), diff --git a/ydb/core/external_sources/external_source_factory.h b/ydb/core/external_sources/external_source_factory.h index f8444d8b1f33..9a7b3133d9e7 100644 --- a/ydb/core/external_sources/external_source_factory.h +++ b/ydb/core/external_sources/external_source_factory.h @@ -10,6 +10,6 @@ struct IExternalSourceFactory : public TThrRefBase { virtual IExternalSource::TPtr GetOrCreate(const TString& type) const = 0; }; -IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector& hostnamePatterns); +IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector& hostnamePatterns, size_t pathsLimit = 50000); } diff --git a/ydb/core/external_sources/object_storage.cpp b/ydb/core/external_sources/object_storage.cpp index 7986f69d9d12..40c2c324e8b5 100644 --- a/ydb/core/external_sources/object_storage.cpp +++ b/ydb/core/external_sources/object_storage.cpp @@ -20,8 +20,9 @@ namespace NKikimr::NExternalSource { namespace { struct TObjectStorageExternalSource : public IExternalSource { - explicit TObjectStorageExternalSource(const std::vector& hostnamePatterns) + explicit TObjectStorageExternalSource(const std::vector& hostnamePatterns, size_t pathsLimit) : HostnamePatterns(hostnamePatterns) + , PathsLimit(pathsLimit) {} virtual TString Pack(const NKikimrExternalSources::TSchema& schema, @@ -47,7 +48,7 @@ struct TObjectStorageExternalSource : public IExternalSource { } } - if (auto issues = Validate(schema, objectStorage)) { + if (auto issues = Validate(schema, objectStorage, PathsLimit)) { ythrow TExternalSourceException() << issues.ToString(); } @@ -116,7 +117,7 @@ struct TObjectStorageExternalSource : public IExternalSource { } template - static NYql::TIssues Validate(const TScheme& schema, const TObjectStorage& objectStorage, size_t pathsLimit = 50000) { + static NYql::TIssues Validate(const TScheme& schema, const TObjectStorage& objectStorage, size_t pathsLimit) { NYql::TIssues issues; issues.AddIssues(ValidateFormatSetting(objectStorage.format(), objectStorage.format_setting())); issues.AddIssues(ValidateRawFormat(objectStorage.format(), schema, objectStorage.partitioned_by())); @@ -472,12 +473,13 @@ struct TObjectStorageExternalSource : public IExternalSource { private: const std::vector HostnamePatterns; + const size_t PathsLimit; }; } -IExternalSource::TPtr CreateObjectStorageExternalSource(const std::vector& hostnamePatterns) { - return MakeIntrusive(hostnamePatterns); +IExternalSource::TPtr CreateObjectStorageExternalSource(const std::vector& hostnamePatterns, size_t pathsLimit) { + return MakeIntrusive(hostnamePatterns, pathsLimit); } NYql::TIssues Validate(const FederatedQuery::Schema& schema, const FederatedQuery::ObjectStorageBinding::Subset& objectStorage, size_t pathsLimit) { diff --git a/ydb/core/external_sources/object_storage.h b/ydb/core/external_sources/object_storage.h index dd02946a3ec5..e357be02d994 100644 --- a/ydb/core/external_sources/object_storage.h +++ b/ydb/core/external_sources/object_storage.h @@ -8,7 +8,7 @@ namespace NKikimr::NExternalSource { -IExternalSource::TPtr CreateObjectStorageExternalSource(const std::vector& hostnamePatterns); +IExternalSource::TPtr CreateObjectStorageExternalSource(const std::vector& hostnamePatterns, size_t pathsLimit); NYql::TIssues Validate(const FederatedQuery::Schema& schema, const FederatedQuery::ObjectStorageBinding::Subset& objectStorage, size_t pathsLimit); diff --git a/ydb/core/external_sources/object_storage_ut.cpp b/ydb/core/external_sources/object_storage_ut.cpp index 1e5f201792e2..b93907e6b7ae 100644 --- a/ydb/core/external_sources/object_storage_ut.cpp +++ b/ydb/core/external_sources/object_storage_ut.cpp @@ -8,14 +8,14 @@ namespace NKikimr { Y_UNIT_TEST_SUITE(ObjectStorageTest) { Y_UNIT_TEST(SuccessValidation) { - auto source = NExternalSource::CreateObjectStorageExternalSource({}); + auto source = NExternalSource::CreateObjectStorageExternalSource({}, 1000); NKikimrExternalSources::TSchema schema; NKikimrExternalSources::TGeneral general; UNIT_ASSERT_NO_EXCEPTION(source->Pack(schema, general)); } Y_UNIT_TEST(FailedCreate) { - auto source = NExternalSource::CreateObjectStorageExternalSource({}); + auto source = NExternalSource::CreateObjectStorageExternalSource({}, 1000); NKikimrExternalSources::TSchema schema; NKikimrExternalSources::TGeneral general; general.mutable_attributes()->insert({"a", "b"}); @@ -23,7 +23,7 @@ Y_UNIT_TEST_SUITE(ObjectStorageTest) { } Y_UNIT_TEST(FailedValidation) { - auto source = NExternalSource::CreateObjectStorageExternalSource({}); + auto source = NExternalSource::CreateObjectStorageExternalSource({}, 1000); NKikimrExternalSources::TSchema schema; NKikimrExternalSources::TGeneral general; general.mutable_attributes()->insert({"projection.h", "b"}); diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp index e019630a682b..49870a1554a8 100644 --- a/ydb/core/kqp/host/kqp_host.cpp +++ b/ydb/core/kqp/host/kqp_host.cpp @@ -1059,6 +1059,8 @@ class TKqpHost : public IKqpHost { SessionCtx->SetDatabase(database); SessionCtx->SetCluster(cluster); SessionCtx->SetTempTables(std::move(tempTablesState)); + + ExternalSourceFactory = NExternalSource::CreateExternalSourceFactory({}, FederatedQuerySetup->S3GatewayConfig.GetGeneratorPathsLimit()); } IAsyncQueryResultPtr ExecuteSchemeQuery(const TKqpQueryRef& query, bool isSql, const TExecSettings& settings) override { @@ -1907,7 +1909,7 @@ class TKqpHost : public IKqpHost { TIntrusivePtr ExecuteCtx; TIntrusivePtr TransformCtx; TIntrusivePtr KqpRunner; - NExternalSource::IExternalSourceFactory::TPtr ExternalSourceFactory{NExternalSource::CreateExternalSourceFactory({})}; + NExternalSource::IExternalSourceFactory::TPtr ExternalSourceFactory; TKqpTempTablesState::TConstPtr TempTablesState; NActors::TActorSystem* ActorSystem = nullptr; diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index b3a4f8181077..2c5c92617953 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -6875,7 +6876,10 @@ void TSchemeShard::ApplyConsoleConfigs(const NKikimrConfig::TAppConfig& appConfi if (appConfig.HasQueryServiceConfig()) { const auto& hostnamePatterns = appConfig.GetQueryServiceConfig().GetHostnamePatterns(); - ExternalSourceFactory = NExternalSource::CreateExternalSourceFactory(std::vector(hostnamePatterns.begin(), hostnamePatterns.end())); + ExternalSourceFactory = NExternalSource::CreateExternalSourceFactory( + std::vector(hostnamePatterns.begin(), hostnamePatterns.end()), + appConfig.GetQueryServiceConfig().GetS3().GetGeneratorPathsLimit() + ); } if (IsSchemeShardConfigured()) { diff --git a/ydb/core/tx/schemeshard/ya.make b/ydb/core/tx/schemeshard/ya.make index 6f47c84ed3da..ac8e6d436cb0 100644 --- a/ydb/core/tx/schemeshard/ya.make +++ b/ydb/core/tx/schemeshard/ya.make @@ -274,6 +274,7 @@ PEERDIR( ydb/library/login/protos ydb/library/protobuf_printer ydb/library/yql/minikql + ydb/library/yql/providers/common/proto ydb/services/bg_tasks )