Merge external_data_channels_count

ydb/core/kqp/host/kqp_gateway_proxy.cpp

@@ -292,6 +292,10 @@ bool ConvertCreateTableSettingsToProto(NYql::TKikimrTableMetadataPtr metadata, Y
         }
     }
 
+    if (const auto count = metadata->TableSettings.ExternalDataChannelsCount) {
+        proto.mutable_storage_settings()->set_external_data_channels_count(*count);
+    }
+
     proto.set_temporary(metadata->Temporary);
 
     return true;

ydb/core/kqp/provider/yql_kikimr_gateway.cpp

@@ -164,7 +164,7 @@ bool TTableSettings::IsSet() const {
     return CompactionPolicy || PartitionBy || AutoPartitioningBySize || UniformPartitions || PartitionAtKeys
         || PartitionSizeMb || AutoPartitioningByLoad || MinPartitions || MaxPartitions || KeyBloomFilter
         || ReadReplicasSettings || TtlSettings || DataSourcePath || Location || ExternalSourceParameters
-        || StoreExternalBlobs;
+        || StoreExternalBlobs || ExternalDataChannelsCount;
 }
 
 EYqlIssueCode YqlStatusFromYdbStatus(ui32 ydbStatus) {

ydb/core/kqp/provider/yql_kikimr_gateway.h

@@ -254,6 +254,7 @@ struct TTableSettings {
     TResetableSetting<TTtlSettings, void> TtlSettings;
     TMaybe<TString> PartitionByHashFunction;
     TMaybe<TString> StoreExternalBlobs;
+    TMaybe<ui64> ExternalDataChannelsCount;
 
     // These parameters are only used for external sources
     TMaybe<TString> DataSourcePath;

ydb/core/kqp/provider/yql_kikimr_type_ann.cpp

@@ -1277,6 +1277,10 @@ virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) over
                 );
             } else if (name == "storeExternalBlobs") {
                 meta->TableSettings.StoreExternalBlobs = TString(setting.Value().Cast<TCoAtom>().Value());
+            } else if (name == "externalDataChannelsCount") {
+                meta->TableSettings.ExternalDataChannelsCount = FromString<ui32>(
+                    setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value()
+                );
             } else {
                 ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()),
                     TStringBuilder() << "Unknown table profile setting: " << name));

ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp

@@ -2342,14 +2342,16 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
                 PRIMARY KEY (Key)
             )
             WITH (
-                STORE_EXTERNAL_BLOBS = ENABLED
+                STORE_EXTERNAL_BLOBS = ENABLED,
+                EXTERNAL_DATA_CHANNELS_COUNT = 7
             );)";
         auto result = session.ExecuteSchemeQuery(query).GetValueSync();
         UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
 
         auto describeResult = session.DescribeTable(tableName).GetValueSync();
         UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString());
         UNIT_ASSERT(describeResult.GetTableDescription().GetStorageSettings().GetStoreExternalBlobs().value_or(false));
+        UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetStorageSettings().GetExternalDataChannelsCount().value(), 7);
     }
 
     Y_UNIT_TEST(CreateAndAlterTableComplex) {

ydb/core/ydb_convert/column_families.h

@@ -133,6 +133,10 @@ namespace NKikimr {
                 }
             }
 
+            if (settings.has_external_data_channels_count()) {
+                MutableDefaultFamily()->MutableStorageConfig()->SetExternalChannelsCount(settings.external_data_channels_count());
+            }
+
             return true;
         }
 

ydb/core/ydb_convert/table_description.cpp

@@ -1378,6 +1378,10 @@ void FillStorageSettingsImpl(TYdbProto& out,
                 if (externalThreshold != 0 && externalThreshold != Max<ui32>()) {
                     settings->set_store_external_blobs(Ydb::FeatureFlag::ENABLED);
                 }
+
+                if (const auto externalChannelsCount = family.GetStorageConfig().GetExternalChannelsCount(); externalChannelsCount > 1U) {
+                    settings->set_external_data_channels_count(externalChannelsCount);
+                }
             }
 
             // Check legacy settings for enabled external blobs

ydb/core/ydb_convert/table_description_ut.cpp

@@ -124,6 +124,7 @@ PartitionConfig {
         AllowOtherKinds: false
       }
       ExternalThreshold: 1
+      ExternalChannelsCount: 7
     }
   }
 })", R"(
@@ -132,6 +133,7 @@ storage_settings {
     media: "hdd"
   }
   store_external_blobs: ENABLED
+  external_data_channels_count: 7
 }
 )");
     }

ydb/public/api/protos/ydb_table.proto

@@ -535,6 +535,10 @@ message StorageSettings {
     // * Table cannot be copied or backed up when this is enabled
     // * This feature cannot be disabled once enabled for a table
     Ydb.FeatureFlag.Status store_external_blobs = 5;
+
+    // This works when store_external_blobs is enabled
+    // and specifies the number of data channels to store "external blobs"
+    optional uint32 external_data_channels_count = 6;
 }
 
 message ColumnFamily {

ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp

@@ -733,7 +733,8 @@ namespace {
         const auto commitLog1 = settings.GetTabletCommitLog1();
         const auto external = settings.GetExternal();
         const auto storeExternalBlobs = settings.GetStoreExternalBlobs();
-        if (!commitLog0 && !commitLog1 && !external && !storeExternalBlobs.has_value()) {
+        const auto externalDataChannelsCount = settings.GetExternalDataChannelsCount();
+        if (!commitLog0 && !commitLog1 && !external && !storeExternalBlobs.has_value() && !externalDataChannelsCount.has_value()) {
             return;
         }
         Cout << Endl << "Storage settings: " << Endl;
@@ -750,6 +751,9 @@ namespace {
             Cout << "Store large values in \"external blobs\": "
                 << (storeExternalBlobs.value() ? "true" : "false") << Endl;
         }
+        if (externalDataChannelsCount) {
+            Cout << "External data channels: " << externalDataChannelsCount.value() << Endl;
+        }
     }
 
     void PrintColumnFamilies(const NTable::TTableDescription& tableDescription) {

ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/table.h

@@ -629,6 +629,7 @@ class TStorageSettings {
     std::optional<std::string> GetTabletCommitLog1() const;
     std::optional<std::string> GetExternal() const;
     std::optional<bool> GetStoreExternalBlobs() const;
+    std::optional<std::uint32_t> GetExternalDataChannelsCount() const;
 
 private:
     class TImpl;
@@ -784,6 +785,7 @@ class TStorageSettingsBuilder {
     TStorageSettingsBuilder& SetTabletCommitLog1(const std::string& media);
     TStorageSettingsBuilder& SetExternal(const std::string& media);
     TStorageSettingsBuilder& SetStoreExternalBlobs(bool enabled);
+    TStorageSettingsBuilder& SetExternalDataChannelsCount(uint32_t count);
 
     TStorageSettings Build() const;
 
@@ -858,6 +860,11 @@ class TTableStorageSettingsBuilder {
         return *this;
     }
 
+    TTableStorageSettingsBuilder& SetExternalDataChannelsCount(uint32_t count) {
+        Builder_.SetExternalDataChannelsCount(count);
+        return *this;
+    }
+
     TTableBuilder& EndStorageSettings();
 
 private:
@@ -1473,6 +1480,11 @@ class TAlterStorageSettingsBuilder {
         return *this;
     }
 
+    TAlterStorageSettingsBuilder& SetExternalDataChannelsCount(uint32_t count) {
+        Builder_.SetExternalDataChannelsCount(count);
+        return *this;
+    }
+
     TAlterTableSettings& EndAlterStorageSettings();
 
 private:

ydb/public/sdk/cpp/src/client/table/table.cpp

@@ -102,6 +102,14 @@ std::optional<bool> TStorageSettings::GetStoreExternalBlobs() const {
     }
 }
 
+std::optional<std::uint32_t> TStorageSettings::GetExternalDataChannelsCount() const {
+    if (GetProto().has_external_data_channels_count()) {
+        return GetProto().external_data_channels_count();
+    } else {
+        return { };
+    }
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 class TColumnFamilyDescription::TImpl {
@@ -995,6 +1003,11 @@ TStorageSettingsBuilder& TStorageSettingsBuilder::SetStoreExternalBlobs(bool ena
     return *this;
 }
 
+TStorageSettingsBuilder& TStorageSettingsBuilder::SetExternalDataChannelsCount(uint32_t count) {
+    Impl_->Proto.set_external_data_channels_count(count);
+    return *this;
+}
+
 TStorageSettings TStorageSettingsBuilder::Build() const {
     return TStorageSettings(Impl_->Proto);
 }

ydb/services/ydb/ydb_table_ut.cpp

@@ -3842,6 +3842,7 @@ R"___(<main>: Error: Transaction not found: , code: 2015
                     .SetTabletCommitLog1("ssd")
                     .SetExternal("hdd")
                     .SetStoreExternalBlobs(true)
+                    .SetExternalDataChannelsCount(7U)
                 .EndStorageSettings()
                 .BeginColumnFamily("default")
                     .SetData("ssd")
@@ -3875,6 +3876,7 @@ R"___(<main>: Error: Transaction not found: , code: 2015
             const auto& settings = res.GetTableDescription().GetStorageSettings();
             UNIT_ASSERT_VALUES_EQUAL(settings.GetExternal(), "hdd");
             UNIT_ASSERT_VALUES_EQUAL(settings.GetStoreExternalBlobs().value(), true);
+            UNIT_ASSERT_VALUES_EQUAL(settings.GetExternalDataChannelsCount().value(), 7U);
             const auto& families = res.GetTableDescription().GetColumnFamilies();
             UNIT_ASSERT_EQUAL(families.size(), 2);
             UNIT_ASSERT_VALUES_EQUAL(families[0].GetName(), "default");

yql/essentials/sql/v1/node.h

@@ -1159,6 +1159,7 @@ namespace NSQLTranslationV1 {
         TMaybe<TIdentifier> StoreType;
         TNodePtr PartitionByHashFunction;
         TMaybe<TIdentifier> StoreExternalBlobs;
+        TNodePtr ExternalDataChannelsCount;
 
         TNodePtr DataSourcePath;
         NYql::TResetableSetting<TNodePtr, void> Location;
@@ -1168,7 +1169,7 @@ namespace NSQLTranslationV1 {
             return CompactionPolicy || AutoPartitioningBySize || PartitionSizeMb || AutoPartitioningByLoad
                 || MinPartitions || MaxPartitions || UniformPartitions || PartitionAtKeys || KeyBloomFilter
                 || ReadReplicasSettings || TtlSettings || Tiering || StoreType || PartitionByHashFunction
-                || StoreExternalBlobs || DataSourcePath || Location || ExternalSourceParameters;
+                || StoreExternalBlobs || DataSourcePath || Location || ExternalSourceParameters || ExternalDataChannelsCount;
         }
     };
 

yql/essentials/sql/v1/query.cpp

@@ -281,6 +281,9 @@ static INode::TPtr CreateTableSettings(const TTableSettings& tableSettings, ETab
     if (tableSettings.PartitionByHashFunction && parsingMode == ETableSettingsParsingMode::Create) {
         settings = L(settings, Q(Y(Q("partitionByHashFunction"), tableSettings.PartitionByHashFunction)));
     }
+    if (tableSettings.ExternalDataChannelsCount) {
+        settings = L(settings, Q(Y(Q("externalDataChannelsCount"), tableSettings.ExternalDataChannelsCount)));
+    }
 
     return settings;
 }

yql/essentials/sql/v1/sql_translation.cpp

@@ -2308,6 +2308,15 @@ bool TSqlTranslation::StoreTableSettingsEntry(const TIdentifier& id, const TRule
             Ctx.Error() << to_upper(id.Name) << " value should be an identifier";
             return false;
         }
+    } else if (to_lower(id.Name) == "external_data_channels_count") {
+        if (reset) {
+            Ctx.Error() << to_upper(id.Name) << " reset is not supported";
+            return false;
+        }
+        if (!StoreInt(*value, settings.ExternalDataChannelsCount, Ctx)) {
+            Ctx.Error() << to_upper(id.Name) << " value should be an integer";
+            return false;
+        }
     } else {
         Ctx.Error() << "Unknown table setting: " << id.Name;
         return false;