Pass TS3GatewayConfig::GeneratorPathsLimit as pathsLimit (#4259)

ydb-platform · May 3, 2024 · 6636c4d · 6636c4d
1 parent 7031b1c
commit 6636c4d
Show file tree

Hide file tree

Showing 8 changed files with 23 additions and 14 deletions.
diff --git a/ydb/core/external_sources/external_source_factory.cpp b/ydb/core/external_sources/external_source_factory.cpp
@@ -32,12 +32,12 @@ struct TExternalSourceFactory : public IExternalSourceFactory {
 
 }
 
-IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TString>& hostnamePatterns) {
+IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TString>& hostnamePatterns, size_t pathsLimit) {
     std::vector<TRegExMatch> hostnamePatternsRegEx(hostnamePatterns.begin(), hostnamePatterns.end());
     return MakeIntrusive<TExternalSourceFactory>(TMap<TString, IExternalSource::TPtr>{
         {
             ToString(NYql::EDatabaseType::ObjectStorage),
-            CreateObjectStorageExternalSource(hostnamePatternsRegEx)
+            CreateObjectStorageExternalSource(hostnamePatternsRegEx, pathsLimit)
         },
         {
             ToString(NYql::EDatabaseType::ClickHouse),

diff --git a/ydb/core/external_sources/external_source_factory.h b/ydb/core/external_sources/external_source_factory.h
@@ -10,6 +10,6 @@ struct IExternalSourceFactory : public TThrRefBase {
     virtual IExternalSource::TPtr GetOrCreate(const TString& type) const = 0;
 };
 
-IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TString>& hostnamePatterns);
+IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TString>& hostnamePatterns, size_t pathsLimit = 50000);
 
 }
diff --git a/ydb/core/external_sources/object_storage.cpp b/ydb/core/external_sources/object_storage.cpp
@@ -20,8 +20,9 @@ namespace NKikimr::NExternalSource {
 namespace {
 
 struct TObjectStorageExternalSource : public IExternalSource {
-    explicit TObjectStorageExternalSource(const std::vector<TRegExMatch>& hostnamePatterns)
+    explicit TObjectStorageExternalSource(const std::vector<TRegExMatch>& hostnamePatterns, size_t pathsLimit)
         : HostnamePatterns(hostnamePatterns)
+        , PathsLimit(pathsLimit)
     {}
 
     virtual TString Pack(const NKikimrExternalSources::TSchema& schema,
@@ -47,7 +48,7 @@ struct TObjectStorageExternalSource : public IExternalSource {
             }
         }
 
-        if (auto issues = Validate(schema, objectStorage)) {
+        if (auto issues = Validate(schema, objectStorage, PathsLimit)) {
             ythrow TExternalSourceException() << issues.ToString();
         }
 
@@ -116,7 +117,7 @@ struct TObjectStorageExternalSource : public IExternalSource {
     }
 
     template<typename TScheme, typename TObjectStorage>
-    static NYql::TIssues Validate(const TScheme& schema, const TObjectStorage& objectStorage, size_t pathsLimit = 50000) {
+    static NYql::TIssues Validate(const TScheme& schema, const TObjectStorage& objectStorage, size_t pathsLimit) {
         NYql::TIssues issues;
         issues.AddIssues(ValidateFormatSetting(objectStorage.format(), objectStorage.format_setting()));
         issues.AddIssues(ValidateRawFormat(objectStorage.format(), schema, objectStorage.partitioned_by()));
@@ -472,12 +473,13 @@ struct TObjectStorageExternalSource : public IExternalSource {
 
 private:
     const std::vector<TRegExMatch> HostnamePatterns;
+    const size_t PathsLimit;
 };
 
 }
 
-IExternalSource::TPtr CreateObjectStorageExternalSource(const std::vector<TRegExMatch>& hostnamePatterns) {
-    return MakeIntrusive<TObjectStorageExternalSource>(hostnamePatterns);
+IExternalSource::TPtr CreateObjectStorageExternalSource(const std::vector<TRegExMatch>& hostnamePatterns, size_t pathsLimit) {
+    return MakeIntrusive<TObjectStorageExternalSource>(hostnamePatterns, pathsLimit);
 }
 
 NYql::TIssues Validate(const FederatedQuery::Schema& schema, const FederatedQuery::ObjectStorageBinding::Subset& objectStorage, size_t pathsLimit) {

diff --git a/ydb/core/external_sources/object_storage.h b/ydb/core/external_sources/object_storage.h
@@ -8,7 +8,7 @@
 
 namespace NKikimr::NExternalSource {
 
-IExternalSource::TPtr CreateObjectStorageExternalSource(const std::vector<TRegExMatch>& hostnamePatterns);
+IExternalSource::TPtr CreateObjectStorageExternalSource(const std::vector<TRegExMatch>& hostnamePatterns, size_t pathsLimit);
 
 NYql::TIssues Validate(const FederatedQuery::Schema& schema, const FederatedQuery::ObjectStorageBinding::Subset& objectStorage, size_t pathsLimit);
 

diff --git a/ydb/core/external_sources/object_storage_ut.cpp b/ydb/core/external_sources/object_storage_ut.cpp
@@ -8,22 +8,22 @@ namespace NKikimr {
 
 Y_UNIT_TEST_SUITE(ObjectStorageTest) {
     Y_UNIT_TEST(SuccessValidation) {
-        auto source = NExternalSource::CreateObjectStorageExternalSource({});
+        auto source = NExternalSource::CreateObjectStorageExternalSource({}, 1000);
         NKikimrExternalSources::TSchema schema;
         NKikimrExternalSources::TGeneral general;
         UNIT_ASSERT_NO_EXCEPTION(source->Pack(schema, general));
     }
 
     Y_UNIT_TEST(FailedCreate) {
-        auto source = NExternalSource::CreateObjectStorageExternalSource({});
+        auto source = NExternalSource::CreateObjectStorageExternalSource({}, 1000);
         NKikimrExternalSources::TSchema schema;
         NKikimrExternalSources::TGeneral general;
         general.mutable_attributes()->insert({"a", "b"});
         UNIT_ASSERT_EXCEPTION_CONTAINS(source->Pack(schema, general), NExternalSource::TExternalSourceException, "Unknown attribute a");
     }
 
     Y_UNIT_TEST(FailedValidation) {
-        auto source = NExternalSource::CreateObjectStorageExternalSource({});
+        auto source = NExternalSource::CreateObjectStorageExternalSource({}, 1000);
         NKikimrExternalSources::TSchema schema;
         NKikimrExternalSources::TGeneral general;
         general.mutable_attributes()->insert({"projection.h", "b"});

diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp
@@ -1059,6 +1059,8 @@ class TKqpHost : public IKqpHost {
         SessionCtx->SetDatabase(database);
         SessionCtx->SetCluster(cluster);
         SessionCtx->SetTempTables(std::move(tempTablesState));
+
+        ExternalSourceFactory = NExternalSource::CreateExternalSourceFactory({}, FederatedQuerySetup->S3GatewayConfig.GetGeneratorPathsLimit());
     }
 
     IAsyncQueryResultPtr ExecuteSchemeQuery(const TKqpQueryRef& query, bool isSql, const TExecSettings& settings) override {
@@ -1907,7 +1909,7 @@ class TKqpHost : public IKqpHost {
     TIntrusivePtr<TExecuteContext> ExecuteCtx;
     TIntrusivePtr<TKqlTransformContext> TransformCtx;
     TIntrusivePtr<IKqpRunner> KqpRunner;
-    NExternalSource::IExternalSourceFactory::TPtr ExternalSourceFactory{NExternalSource::CreateExternalSourceFactory({})};
+    NExternalSource::IExternalSourceFactory::TPtr ExternalSourceFactory;
 
     TKqpTempTablesState::TConstPtr TempTablesState;
     NActors::TActorSystem* ActorSystem = nullptr;

diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp
@@ -15,6 +15,7 @@
 #include <ydb/core/scheme/scheme_types_proto.h>
 #include <ydb/core/tx/scheme_board/events_schemeshard.h>
 #include <ydb/library/yql/minikql/mkql_type_ops.h>
+#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h>
 #include <util/random/random.h>
 #include <util/system/byteorder.h>
 #include <util/system/unaligned_mem.h>
@@ -6875,7 +6876,10 @@ void TSchemeShard::ApplyConsoleConfigs(const NKikimrConfig::TAppConfig& appConfi
 
     if (appConfig.HasQueryServiceConfig()) {
         const auto& hostnamePatterns = appConfig.GetQueryServiceConfig().GetHostnamePatterns();
-        ExternalSourceFactory = NExternalSource::CreateExternalSourceFactory(std::vector<TString>(hostnamePatterns.begin(), hostnamePatterns.end()));
+        ExternalSourceFactory = NExternalSource::CreateExternalSourceFactory(
+            std::vector<TString>(hostnamePatterns.begin(), hostnamePatterns.end()),
+            appConfig.GetQueryServiceConfig().GetS3().GetGeneratorPathsLimit()
+        );
     }
 
     if (IsSchemeShardConfigured()) {

diff --git a/ydb/core/tx/schemeshard/ya.make b/ydb/core/tx/schemeshard/ya.make
@@ -274,6 +274,7 @@ PEERDIR(
     ydb/library/login/protos
     ydb/library/protobuf_printer
     ydb/library/yql/minikql
+    ydb/library/yql/providers/common/proto
     ydb/services/bg_tasks
 )