From 06799773a46112d0db9c5c70a061746446aec0fe Mon Sep 17 00:00:00 2001 From: kang <35803862+ghkang98@users.noreply.github.com> Date: Fri, 9 May 2025 21:30:28 +0800 Subject: [PATCH 1/4] [fix](iceberg) fix iceberg hadoop type kerbros issue. (#50623) Support Pre-Execution Authentication for Hadoop Type Iceberg Catalog Operations Summary This PR fix the hadoop type iceberg catalog kerbros use the PreExecutionAuthenticator class, This is especially useful in environments where secure access is required, such as Kerberos-based Hadoop ecosystems. By integrating PreExecutionAuthenticator, each relevant operation will undergo an authentication step prior to execution, maintaining security compliance. - Behavior changed: No. - Does this need documentation? No. Co-authored-by: lik40 (cherry picked from commit 925699691021f0983ec82cd572300d089d55f32a) --- .../iceberg/IcebergHadoopExternalCatalog.java | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java index bf9e8c2b3f06fb..cd84093c74793a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java @@ -18,6 +18,8 @@ package org.apache.doris.datasource.iceberg; import org.apache.doris.catalog.HdfsResource; +import org.apache.doris.common.security.authentication.AuthenticationConfig; +import org.apache.doris.common.security.authentication.HadoopAuthenticator; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.property.PropertyConverter; @@ -52,15 +54,30 @@ public IcebergHadoopExternalCatalog(long catalogId, String name, String resource @Override protected void initCatalog() { icebergCatalogType = ICEBERG_HADOOP; - HadoopCatalog hadoopCatalog = new HadoopCatalog(); + Configuration conf = getConfiguration(); initS3Param(conf); + + //create the authenticator first + if (preExecutionAuthenticator.getHadoopAuthenticator() == null) { + AuthenticationConfig config = AuthenticationConfig.getKerberosConfig(conf); + HadoopAuthenticator authenticator = HadoopAuthenticator.getHadoopAuthenticator(config); + preExecutionAuthenticator.setHadoopAuthenticator(authenticator); + } + // initialize hadoop catalog - Map catalogProperties = catalogProperty.getProperties(); - String warehouse = catalogProperty.getHadoopProperties().get(CatalogProperties.WAREHOUSE_LOCATION); - hadoopCatalog.setConf(conf); - catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse); - hadoopCatalog.initialize(getName(), catalogProperties); - catalog = hadoopCatalog; + try { + this.catalog = preExecutionAuthenticator.execute(() -> { + Map catalogProperties = catalogProperty.getProperties(); + String warehouse = catalogProperty.getHadoopProperties().get(CatalogProperties.WAREHOUSE_LOCATION); + HadoopCatalog hadoopCatalog = new HadoopCatalog(); + hadoopCatalog.setConf(conf); + catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse); + hadoopCatalog.initialize(getName(), catalogProperties); + return hadoopCatalog; + }); + } catch (Exception e) { + throw new RuntimeException("Hadoop catalog init error!", e); + } } } From a7bd2910c1e51c2b60067433bc4ec2a038e0fa32 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 23 May 2025 11:39:46 +0800 Subject: [PATCH 2/4] branch-3.0: [Fix](Catalog)Ensure preExecutionAuthenticator is properly initialized (#50839) cherrypick #50839 --- .../PreExecutionAuthenticator.java | 15 ++++++++++++++ .../doris/datasource/ExternalCatalog.java | 19 ++++++++++++++++++ .../datasource/hive/HMSExternalCatalog.java | 17 +++++++--------- .../datasource/hive/HiveMetadataOps.java | 2 +- .../iceberg/IcebergExternalCatalog.java | 9 ++++++++- .../iceberg/IcebergHadoopExternalCatalog.java | 20 +++++-------------- 6 files changed, 55 insertions(+), 27 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/PreExecutionAuthenticator.java b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/PreExecutionAuthenticator.java index 6260833b7db558..0d7cf60c6f751d 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/PreExecutionAuthenticator.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/PreExecutionAuthenticator.java @@ -17,6 +17,8 @@ package org.apache.doris.common.security.authentication; +import org.apache.hadoop.conf.Configuration; + import java.security.PrivilegedExceptionAction; import java.util.concurrent.Callable; @@ -40,6 +42,19 @@ public class PreExecutionAuthenticator { public PreExecutionAuthenticator() { } + /** + * Constructor to initialize the PreExecutionAuthenticator object. + * This constructor is responsible for initializing the Hadoop authenticator required for Kerberos authentication + * based on the provided configuration information. + * + * @param configuration Configuration information used to obtain Kerberos authentication settings + */ + public PreExecutionAuthenticator(Configuration configuration) { + AuthenticationConfig config = AuthenticationConfig.getKerberosConfig(configuration); + this.hadoopAuthenticator = HadoopAuthenticator.getHadoopAuthenticator(config); + } + + /** * Executes the specified task with necessary authentication. *

If a HadoopAuthenticator is set, the task will be executed within a diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 00ed3284abbc71..a96221057ea01c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -176,6 +176,17 @@ public ExternalCatalog(long catalogId, String name, InitCatalogLog.Type logType, this.comment = Strings.nullToEmpty(comment); } + /** + * Initializes the PreExecutionAuthenticator instance. + * This method ensures that the authenticator is created only once in a thread-safe manner. + * If additional authentication logic is required, it should be extended and implemented in subclasses. + */ + protected synchronized void initPreExecutionAuthenticator() { + if (preExecutionAuthenticator == null) { + preExecutionAuthenticator = new PreExecutionAuthenticator(); + } + } + public Configuration getConfiguration() { // build configuration is costly, so we cache it. if (cachedConf != null) { @@ -213,6 +224,11 @@ protected List listDatabaseNames() { } } + public ExternalMetadataOps getMetadataOps() { + makeSureInitialized(); + return metadataOps; + } + // Will be called when creating catalog(so when as replaying) // to add some default properties if missing. public void setDefaultPropsIfMissing(boolean isReplay) { @@ -1137,6 +1153,9 @@ public void setAutoAnalyzePolicy(String dbName, String tableName, String policy) } public PreExecutionAuthenticator getPreExecutionAuthenticator() { + if (null == preExecutionAuthenticator) { + throw new RuntimeException("PreExecutionAuthenticator is null, please confirm it is initialized."); + } return preExecutionAuthenticator; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 61a7c030477ece..8179a7aab0f46e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -25,8 +25,6 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.Pair; import org.apache.doris.common.ThreadPoolManager; -import org.apache.doris.common.security.authentication.AuthenticationConfig; -import org.apache.doris.common.security.authentication.HadoopAuthenticator; import org.apache.doris.common.security.authentication.PreExecutionAuthenticator; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogProperty; @@ -91,8 +89,6 @@ public class HMSExternalCatalog extends ExternalCatalog { private static final int FILE_SYSTEM_EXECUTOR_THREAD_NUM = 16; private ThreadPoolExecutor fileSystemExecutor; - @Getter - private HadoopAuthenticator authenticator; private int hmsEventsBatchSizePerRpc = -1; private boolean enableHmsEventsIncrementalSync = false; @@ -170,14 +166,15 @@ public void checkProperties() throws DdlException { } @Override - protected void initLocalObjectsImpl() { - this.preExecutionAuthenticator = new PreExecutionAuthenticator(); - if (this.authenticator == null) { - AuthenticationConfig config = AuthenticationConfig.getKerberosConfig(getConfiguration()); - this.authenticator = HadoopAuthenticator.getHadoopAuthenticator(config); - this.preExecutionAuthenticator.setHadoopAuthenticator(authenticator); + protected synchronized void initPreExecutionAuthenticator() { + if (preExecutionAuthenticator == null) { + preExecutionAuthenticator = new PreExecutionAuthenticator(getConfiguration()); } + } + @Override + protected void initLocalObjectsImpl() { + initPreExecutionAuthenticator(); HiveConf hiveConf = null; JdbcClientConfig jdbcClientConfig = null; String hiveMetastoreType = catalogProperty.getOrDefault(HMSProperties.HIVE_METASTORE_TYPE, ""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java index c611462f10a90b..a4a6b1ebdabe75 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java @@ -72,7 +72,7 @@ public HiveMetadataOps(HiveConf hiveConf, JdbcClientConfig jdbcClientConfig, HMS this(catalog, createCachedClient(hiveConf, Math.max(MIN_CLIENT_POOL_SIZE, Config.max_external_cache_loader_thread_pool_size), jdbcClientConfig)); - hadoopAuthenticator = catalog.getAuthenticator(); + hadoopAuthenticator = catalog.getPreExecutionAuthenticator().getHadoopAuthenticator(); client.setHadoopAuthenticator(hadoopAuthenticator); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index f7966b9709ff13..7282de52aa0d22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -52,9 +52,16 @@ public IcebergExternalCatalog(long catalogId, String name, String comment) { // Create catalog based on catalog type protected abstract void initCatalog(); + @Override + protected synchronized void initPreExecutionAuthenticator() { + if (preExecutionAuthenticator == null) { + preExecutionAuthenticator = new PreExecutionAuthenticator(getConfiguration()); + } + } + @Override protected void initLocalObjectsImpl() { - preExecutionAuthenticator = new PreExecutionAuthenticator(); + initPreExecutionAuthenticator(); initCatalog(); IcebergMetadataOps ops = ExternalMetadataOperations.newIcebergMetadataOps(this, catalog); transactionManager = TransactionManagerFactory.createIcebergTransactionManager(ops); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java index cd84093c74793a..56470a4d071c2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java @@ -18,8 +18,6 @@ package org.apache.doris.datasource.iceberg; import org.apache.doris.catalog.HdfsResource; -import org.apache.doris.common.security.authentication.AuthenticationConfig; -import org.apache.doris.common.security.authentication.HadoopAuthenticator; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.property.PropertyConverter; @@ -57,22 +55,14 @@ protected void initCatalog() { Configuration conf = getConfiguration(); initS3Param(conf); - - //create the authenticator first - if (preExecutionAuthenticator.getHadoopAuthenticator() == null) { - AuthenticationConfig config = AuthenticationConfig.getKerberosConfig(conf); - HadoopAuthenticator authenticator = HadoopAuthenticator.getHadoopAuthenticator(config); - preExecutionAuthenticator.setHadoopAuthenticator(authenticator); - } - // initialize hadoop catalog + Map catalogProperties = catalogProperty.getProperties(); + String warehouse = catalogProperty.getHadoopProperties().get(CatalogProperties.WAREHOUSE_LOCATION); + HadoopCatalog hadoopCatalog = new HadoopCatalog(); + hadoopCatalog.setConf(conf); + catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse); try { this.catalog = preExecutionAuthenticator.execute(() -> { - Map catalogProperties = catalogProperty.getProperties(); - String warehouse = catalogProperty.getHadoopProperties().get(CatalogProperties.WAREHOUSE_LOCATION); - HadoopCatalog hadoopCatalog = new HadoopCatalog(); - hadoopCatalog.setConf(conf); - catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse); hadoopCatalog.initialize(getName(), catalogProperties); return hadoopCatalog; }); From 7a2d05335c8427ba1b69876f8536d8600cd35b58 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 27 May 2025 14:26:52 +0800 Subject: [PATCH 3/4] fix checkstyle --- .../org/apache/doris/datasource/hive/HMSExternalCatalog.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 8179a7aab0f46e..20d43263316cd1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -51,7 +51,6 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import lombok.Getter; import org.apache.commons.lang3.math.NumberUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.iceberg.hive.HiveCatalog; From e9237f2fb4f4ef28cecb031cd0a9011f9c63d215 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 27 May 2025 16:39:33 +0800 Subject: [PATCH 4/4] fix checkstyle --- .../security/authentication/HadoopAuthenticator.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopAuthenticator.java b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopAuthenticator.java index c3cab5f410be3a..88d32a593e14dc 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopAuthenticator.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopAuthenticator.java @@ -20,6 +20,7 @@ import org.apache.hadoop.security.UserGroupInformation; import java.io.IOException; +import java.lang.reflect.UndeclaredThrowableException; import java.security.PrivilegedExceptionAction; public interface HadoopAuthenticator { @@ -31,6 +32,12 @@ default T doAs(PrivilegedExceptionAction action) throws IOException { return getUGI().doAs(action); } catch (InterruptedException e) { throw new IOException(e); + } catch (UndeclaredThrowableException e) { + if (e.getCause() instanceof RuntimeException) { + throw (RuntimeException) e.getCause(); + } else { + throw new RuntimeException(e.getCause()); + } } }