Merge 2323646 into 308ab47

ydb-platform · Jun 25, 2024 · 21d1903 · 21d1903
2 parents 308ab47 + 2323646
commit 21d1903
Show file tree

Hide file tree

Showing 9 changed files with 77 additions and 3 deletions.
diff --git a/ydb/core/fq/libs/actors/clusters_from_connections.cpp b/ydb/core/fq/libs/actors/clusters_from_connections.cpp
@@ -271,6 +271,18 @@ void AddClustersFromConnections(
             clusters.emplace(connectionName, GenericProviderName);
             break;
         }
+        case FederatedQuery::ConnectionSetting::kGreenplumCluster: {
+            FillGenericClusterConfig(
+                common,
+                *gatewaysConfig.MutableGeneric()->AddClusterMapping(),
+                conn.content().setting().greenplum_cluster(),
+                connectionName,
+                NYql::NConnector::NApi::EDataSourceKind::GREENPLUM,
+                authToken,
+                accountIdSignatures);
+            clusters.emplace(connectionName, GenericProviderName);
+            break;
+        }
 
         // Do not replace with default. Adding a new connection should cause a compilation error
         case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET:

diff --git a/ydb/core/fq/libs/common/util.cpp b/ydb/core/fq/libs/common/util.cpp
@@ -126,6 +126,9 @@ TString ExtractServiceAccountId(const FederatedQuery::ConnectionSetting& setting
     case FederatedQuery::ConnectionSetting::kPostgresqlCluster: {
         return GetServiceAccountId(setting.postgresql_cluster().auth());
     }
+    case FederatedQuery::ConnectionSetting::kGreenplumCluster: {
+        return GetServiceAccountId(setting.greenplum_cluster().auth());
+    }
     // Do not replace with default. Adding a new connection should cause a compilation error
     case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET:
     break;
@@ -157,6 +160,8 @@ TMaybe<TString> GetLogin(const FederatedQuery::ConnectionSetting& setting) {
             return {};
         case FederatedQuery::ConnectionSetting::kPostgresqlCluster:
             return setting.postgresql_cluster().login();
+        case FederatedQuery::ConnectionSetting::kGreenplumCluster:
+            return setting.greenplum_cluster().login();
     }
 }
 
@@ -176,6 +181,8 @@ TMaybe<TString> GetPassword(const FederatedQuery::ConnectionSetting& setting) {
             return {};
         case FederatedQuery::ConnectionSetting::kPostgresqlCluster:
             return setting.postgresql_cluster().password();
+        case FederatedQuery::ConnectionSetting::kGreenplumCluster:
+            return setting.greenplum_cluster().password();
     }
 }
 
@@ -195,6 +202,8 @@ EYdbComputeAuth GetYdbComputeAuthMethod(const FederatedQuery::ConnectionSetting&
             return GetIamAuthMethod(setting.monitoring().auth());
         case FederatedQuery::ConnectionSetting::kPostgresqlCluster:
             return GetBasicAuthMethod(setting.postgresql_cluster().auth());
+        case FederatedQuery::ConnectionSetting::kGreenplumCluster:
+            return GetBasicAuthMethod(setting.greenplum_cluster().auth());
     }
 }
 
@@ -212,6 +221,8 @@ FederatedQuery::IamAuth GetAuth(const FederatedQuery::Connection& connection) {
         return connection.content().setting().monitoring().auth();
     case FederatedQuery::ConnectionSetting::kPostgresqlCluster:
         return connection.content().setting().postgresql_cluster().auth();
+    case FederatedQuery::ConnectionSetting::kGreenplumCluster:
+        return connection.content().setting().greenplum_cluster().auth();
     case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET:
         return FederatedQuery::IamAuth{};
     }

diff --git a/ydb/core/fq/libs/compute/common/config.h b/ydb/core/fq/libs/compute/common/config.h
@@ -164,6 +164,7 @@ class TComputeConfig {
             case FederatedQuery::ConnectionSetting::kObjectStorage:
             case FederatedQuery::ConnectionSetting::kClickhouseCluster:
             case FederatedQuery::ConnectionSetting::kPostgresqlCluster:
+            case FederatedQuery::ConnectionSetting::kGreenplumCluster:
             case FederatedQuery::ConnectionSetting::kYdbDatabase:
                 return true;
             case FederatedQuery::ConnectionSetting::kDataStreams:

diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp b/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp
@@ -215,8 +215,8 @@ TString MakeCreateExternalDataSourceQuery(
         }
         case FederatedQuery::ConnectionSetting::kMonitoring:
         break;
-        case FederatedQuery::ConnectionSetting::kPostgresqlCluster:
-            const auto schema = connectionContent.setting().postgresql_cluster().schema();
+        case FederatedQuery::ConnectionSetting::kPostgresqlCluster: {
+            const auto pgschema = connectionContent.setting().postgresql_cluster().schema();
             properties = fmt::format(
                 R"(
                     SOURCE_TYPE="PostgreSQL",
@@ -229,7 +229,25 @@ TString MakeCreateExternalDataSourceQuery(
                 "mdb_cluster_id"_a = EncloseAndEscapeString(connectionContent.setting().postgresql_cluster().database_id(), '"'),
                 "database_name"_a = EncloseAndEscapeString(connectionContent.setting().postgresql_cluster().database_name(), '"'),
                 "use_tls"_a = common.GetDisableSslForGenericDataSources() ? "false" : "true",
-                "schema"_a =  schema ? ", SCHEMA=" + EncloseAndEscapeString(schema, '"') : TString{});
+                "schema"_a =  pgschema ? ", SCHEMA=" + EncloseAndEscapeString(pgschema, '"') : TString{});
+        }
+        break;
+        case FederatedQuery::ConnectionSetting::kGreenplumCluster: {
+            const auto gpschema = connectionContent.setting().greenplum_cluster().schema();
+            properties = fmt::format(
+                R"(
+                    SOURCE_TYPE="Greenplum",
+                    MDB_CLUSTER_ID={mdb_cluster_id},
+                    DATABASE_NAME={database_name},
+                    USE_TLS="{use_tls}"
+                    {schema}
+                )",
+                "mdb_cluster_id"_a = EncloseAndEscapeString(connectionContent.setting().greenplum_cluster().database_id(), '"'),
+                "database_name"_a = EncloseAndEscapeString(connectionContent.setting().greenplum_cluster().database_name(), '"'),
+                "use_tls"_a = common.GetDisableSslForGenericDataSources() ? "false" : "true",
+                "schema"_a =  gpschema ? ", SCHEMA=" + EncloseAndEscapeString(gpschema, '"') : TString{});
+
+        }
         break;
     }
 

diff --git a/ydb/core/fq/libs/control_plane_proxy/utils/utils.h b/ydb/core/fq/libs/control_plane_proxy/utils/utils.h
@@ -31,6 +31,9 @@ TString ExtractServiceAccountIdWithConnection(const T& setting) {
     case FederatedQuery::ConnectionSetting::kPostgresqlCluster: {
         return GetServiceAccountId(setting.postgresql_cluster().auth());
     }
+    case FederatedQuery::ConnectionSetting::kGreenplumCluster: {
+        return GetServiceAccountId(setting.greenplum_cluster().auth());
+    }
     // Do not replace with default. Adding a new connection should cause a compilation error
     case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET:
     break;

diff --git a/ydb/core/fq/libs/control_plane_storage/request_validators.cpp b/ydb/core/fq/libs/control_plane_storage/request_validators.cpp
@@ -69,6 +69,21 @@ NYql::TIssues ValidateConnectionSetting(
         ValidateGenericConnectionSetting(setting.postgresql_cluster(), "postgresql", disableCurrentIam, passwordRequired, issues);
         break;
     }
+    case FederatedQuery::ConnectionSetting::kGreenplumCluster: {
+        const FederatedQuery::GreenplumCluster database = setting.greenplum_cluster(); 
+        if (!database.has_auth() || database.auth().identity_case() == FederatedQuery::IamAuth::IDENTITY_NOT_SET) {
+            issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "content.setting.greenplum_database.auth field is not specified"));
+        }
+
+        if (database.auth().identity_case() == FederatedQuery::IamAuth::kCurrentIam && disableCurrentIam) {
+            issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "current iam authorization is disabled"));
+        }
+
+        if (!database.database_id() && !database.database_name()) {
+            issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "content.setting.greenplum_database.{database_id or database_name} field is not specified"));
+        }
+        break;
+    }
     case FederatedQuery::ConnectionSetting::kObjectStorage: {
         const FederatedQuery::ObjectStorageConnection objectStorage = setting.object_storage();
         if (!objectStorage.has_auth() || objectStorage.auth().identity_case() == FederatedQuery::IamAuth::IDENTITY_NOT_SET) {

diff --git a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp
@@ -39,6 +39,8 @@ FederatedQuery::IamAuth::IdentityCase GetIamAuth(const FederatedQuery::Connectio
             return setting.monitoring().auth().identity_case();
         case FederatedQuery::ConnectionSetting::kPostgresqlCluster:
             return setting.postgresql_cluster().auth().identity_case();
+        case FederatedQuery::ConnectionSetting::kGreenplumCluster:
+            return setting.greenplum_cluster().auth().identity_case();
         case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET:
             return FederatedQuery::IamAuth::IDENTITY_NOT_SET;
     }

diff --git a/ydb/library/yql/providers/generic/connector/api/common/data_source.proto b/ydb/library/yql/providers/generic/connector/api/common/data_source.proto
@@ -34,6 +34,7 @@ enum EDataSourceKind {
     YDB = 4;
     MYSQL = 5;
     MS_SQL_SERVER = 6;
+    GREENPLUM = 7;
 }
 
 // EProtocol generalizes various kinds of network protocols supported by different databases.

diff --git a/ydb/public/api/protos/draft/fq.proto b/ydb/public/api/protos/draft/fq.proto
@@ -492,6 +492,15 @@ message PostgreSQLCluster {
     bool secure = 7;
 }
 
+message GreenplumCluster {
+    string database_id = 1 [(Ydb.length).le = 1024];
+    string database_name = 2 [(Ydb.length).le = 1024];
+    string login = 3 [(Ydb.length).le = 1024, (Ydb.sensitive) = true];
+    string password = 4 [(Ydb.length).le = 1024, (Ydb.sensitive) = true];
+    string schema = 5 [(Ydb.length).le = 1024];
+    IamAuth auth = 6;
+}
+
 message ConnectionSetting {
     enum ConnectionType {
         CONNECTION_TYPE_UNSPECIFIED = 0;
@@ -501,6 +510,7 @@ message ConnectionSetting {
         OBJECT_STORAGE = 4;
         MONITORING = 5;
         POSTGRESQL_CLUSTER = 6;
+        GREENPLUM_CLUSTER = 7;
     }
 
     oneof connection {
@@ -510,6 +520,7 @@ message ConnectionSetting {
         ObjectStorageConnection object_storage = 4;
         Monitoring monitoring = 5;
         PostgreSQLCluster postgresql_cluster = 6;
+        GreenplumCluster greenplum_cluster = 7;
     }
 }