properties) {
this.storageDesc = new StorageDesc(storageName, storageType, properties);
this.location = location;
- boolean convertedToS3 = BosProperties.tryConvertBosToS3(properties, storageType);
+ /*boolean convertedToS3 = BosProperties.tryConvertBosToS3(properties, storageType);
if (convertedToS3) {
this.storageDesc.setStorageType(StorageBackend.StorageType.S3);
this.location = BosProperties.convertPathToS3(location);
} else {
this.location = location;
- }
+ }*/
}
public void setStorageDesc(StorageDesc storageDesc) {
@@ -138,8 +101,8 @@ public String toSql() {
sb.append(" `").append(storageDesc.getName()).append("`");
}
sb.append(" ON LOCATION ").append(location).append(" PROPERTIES(")
- .append(new PrintableMap<>(storageDesc.getProperties(), " = ", true, false, true))
- .append(")");
+ .append(new PrintableMap<>(storageDesc.getProperties(), " = ", true, false, true))
+ .append(")");
return sb.toString();
}
@@ -185,6 +148,35 @@ public TStorageBackendType toThrift() {
return TStorageBackendType.BROKER;
}
}
+
+ /**
+ * A set of storage types that currently support parameter refactoring.
+ *
+ * Includes: S3 (referring to all systems compatible with the S3 protocol),
+ * HDFS, OFS, JFS, and AZURE. For S3, this is a generalized type that matches
+ * any system whose storage type name is returned as "s3" (or compatible)
+ * by {@link org.apache.doris.datasource.property.storage.StorageProperties#getStorageName()}.
+ *
+ * This set is a temporary solution. Once parameter refactoring is fully supported
+ * across all storage systems, this class can be removed.
+ */
+ public static final Set REFACTOR_STORAGE_TYPES =
+ ImmutableSet.of(StorageType.S3, StorageType.HDFS, StorageType.OFS, StorageType.JFS, StorageType.AZURE);
+
+ public static StorageType convertToStorageType(String storageName) {
+ switch (storageName.toLowerCase()) {
+ case "hdfs":
+ return StorageType.HDFS;
+ case "s3":
+ return StorageType.S3;
+ case "jfs":
+ return StorageType.JFS;
+ case "local":
+ return StorageType.LOCAL;
+ default:
+ throw new IllegalArgumentException("Invalid storage type: " + storageName);
+ }
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java
index 902f5d24b91820..bbfcd8e9d2c45b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java
@@ -17,7 +17,11 @@
package org.apache.doris.analysis;
+import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
import com.google.gson.annotations.SerializedName;
+import lombok.Getter;
import java.util.Map;
@@ -32,18 +36,33 @@
* The broker's StorageBackend.StorageType desc
*/
public class StorageDesc extends ResourceDesc {
+
+ @Deprecated
@SerializedName("st")
protected StorageBackend.StorageType storageType;
+ @Getter
+ protected StorageProperties storageProperties;
+
public StorageDesc() {
}
public StorageDesc(String name, StorageBackend.StorageType storageType, Map properties) {
this.name = name;
this.storageType = storageType;
+ if (!storageType.equals(StorageBackend.StorageType.BROKER)) {
+ this.storageProperties = StorageProperties.createPrimary(properties);
+ }
this.properties = properties;
}
+ public StorageDesc(String name, Map properties) throws UserException {
+ this.name = name;
+ this.properties = properties;
+ this.storageProperties = StorageProperties.createPrimary(properties);
+ this.storageType = StorageBackend.StorageType.convertToStorageType(storageProperties.getStorageName());
+ }
+
public void setName(String name) {
this.name = name;
}
@@ -67,4 +86,11 @@ public StorageBackend.StorageType getStorageType() {
public Map getProperties() {
return properties;
}
+
+ public Map getBackendConfigProperties() {
+ if (null == storageProperties) {
+ return properties;
+ }
+ return storageProperties.getBackendConfigProperties();
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
index 64ca3582db160e..399f996e314fd3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
@@ -19,7 +19,6 @@
import org.apache.doris.analysis.AbstractBackupStmt;
import org.apache.doris.analysis.AbstractBackupTableRefClause;
-import org.apache.doris.analysis.AlterRepositoryStmt;
import org.apache.doris.analysis.BackupStmt;
import org.apache.doris.analysis.BackupStmt.BackupType;
import org.apache.doris.analysis.CancelBackupStmt;
@@ -45,15 +44,17 @@
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.Pair;
+import org.apache.doris.common.UserException;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.DebugPointUtil;
import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.common.util.Util;
-import org.apache.doris.fs.FileSystemFactory;
-import org.apache.doris.fs.remote.AzureFileSystem;
-import org.apache.doris.fs.remote.RemoteFileSystem;
-import org.apache.doris.fs.remote.S3FileSystem;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+import org.apache.doris.fsv2.FileSystemFactory;
+import org.apache.doris.fsv2.remote.AzureFileSystem;
+import org.apache.doris.fsv2.remote.RemoteFileSystem;
+import org.apache.doris.fsv2.remote.S3FileSystem;
import org.apache.doris.nereids.trees.plans.commands.CancelBackupCommand;
import org.apache.doris.persist.BarrierLog;
import org.apache.doris.task.DirMoveTask;
@@ -215,10 +216,18 @@ public void createRepository(CreateRepositoryStmt stmt) throws DdlException {
"broker does not exist: " + stmt.getBrokerName());
}
- RemoteFileSystem fileSystem = FileSystemFactory.get(stmt.getBrokerName(), stmt.getStorageType(),
- stmt.getProperties());
+ RemoteFileSystem fileSystem;
+ try {
+ fileSystem = FileSystemFactory.get(stmt.getStorageType(), stmt.getProperties());
+ } catch (UserException e) {
+ throw new DdlException("Failed to initialize remote file system: " + e.getMessage());
+ }
+ org.apache.doris.fs.remote.RemoteFileSystem oldfs = org.apache.doris.fs.FileSystemFactory
+ .get(stmt.getBrokerName(), stmt.getStorageType(),
+ stmt.getProperties());
long repoId = env.getNextId();
- Repository repo = new Repository(repoId, stmt.getName(), stmt.isReadOnly(), stmt.getLocation(), fileSystem);
+ Repository repo = new Repository(repoId, stmt.getName(), stmt.isReadOnly(), stmt.getLocation(),
+ fileSystem, oldfs);
Status st = repoMgr.addAndInitRepoIfNotExist(repo, false);
if (!st.ok()) {
@@ -231,61 +240,104 @@ public void createRepository(CreateRepositoryStmt stmt) throws DdlException {
}
}
- public void alterRepository(AlterRepositoryStmt stmt) throws DdlException {
- alterRepositoryInternal(stmt.getName(), stmt.getProperties());
- }
-
- public void alterRepositoryInternal(String repoName, Map properties) throws DdlException {
+ /**
+ * Alters an existing repository by applying the given new properties.
+ *
+ * @param repoName The name of the repository to alter.
+ * @param newProps The new properties to apply to the repository.
+ * @param strictCheck If true, only allows altering S3 or Azure repositories and validates properties accordingly.
+ * TODO: Investigate why only S3 and Azure repositories are supported for alter operation
+ * @throws DdlException if the repository does not exist, fails to apply properties, or cannot connect
+ * to the updated repository.
+ */
+ public void alterRepository(String repoName, Map newProps, boolean strictCheck)
+ throws DdlException {
tryLock();
try {
- Repository repo = repoMgr.getRepo(repoName);
- if (repo == null) {
- ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Repository does not exist");
+ Repository oldRepo = repoMgr.getRepo(repoName);
+ if (oldRepo == null) {
+ throw new DdlException("Repository does not exist");
}
-
- if (repo.getRemoteFileSystem() instanceof S3FileSystem
- || repo.getRemoteFileSystem() instanceof AzureFileSystem) {
- Map oldProperties = new HashMap<>(properties);
- Status status = repo.alterRepositoryS3Properties(oldProperties);
- if (!status.ok()) {
- ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, status.getErrMsg());
- }
- RemoteFileSystem fileSystem = null;
- if (repo.getRemoteFileSystem() instanceof S3FileSystem) {
- fileSystem = FileSystemFactory.get(repo.getRemoteFileSystem().getName(),
- StorageBackend.StorageType.S3, oldProperties);
- } else if (repo.getRemoteFileSystem() instanceof AzureFileSystem) {
- fileSystem = FileSystemFactory.get(repo.getRemoteFileSystem().getName(),
- StorageBackend.StorageType.AZURE, oldProperties);
- }
-
- Repository newRepo = new Repository(repo.getId(), repo.getName(), repo.isReadOnly(),
- repo.getLocation(), fileSystem);
- if (!newRepo.ping()) {
- LOG.warn("Failed to connect repository {}. msg: {}", repo.getName(), repo.getErrorMsg());
- ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
- "Repo can not ping with new s3 properties");
- }
-
- Status st = repoMgr.alterRepo(newRepo, false /* not replay */);
- if (!st.ok()) {
- ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
- "Failed to alter repository: " + st.getErrMsg());
- }
- for (AbstractJob job : getAllCurrentJobs()) {
- if (!job.isDone() && job.getRepoId() == repo.getId()) {
- job.updateRepo(newRepo);
- }
- }
- } else {
- ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
- "Only support alter s3 or azure repository");
+ // Merge new properties with the existing repository's properties
+ Map mergedProps = mergeProperties(oldRepo, newProps, strictCheck);
+ // Create new remote file system with merged properties
+ RemoteFileSystem fileSystem = FileSystemFactory.get(StorageProperties.createPrimary(mergedProps));
+ org.apache.doris.fs.remote.RemoteFileSystem oldfs = null;
+ if (oldRepo.getRemoteFileSystem() instanceof S3FileSystem) {
+ oldfs = org.apache.doris.fs.FileSystemFactory.get(oldRepo.getRemoteFileSystem().getName(),
+ StorageBackend.StorageType.S3, mergedProps);
+ } else if (oldRepo.getRemoteFileSystem() instanceof AzureFileSystem) {
+ oldfs = org.apache.doris.fs.FileSystemFactory.get(oldRepo.getRemoteFileSystem().getName(),
+ StorageBackend.StorageType.AZURE, mergedProps);
+ }
+ // Create new Repository instance with updated file system
+ Repository newRepo = new Repository(
+ oldRepo.getId(), oldRepo.getName(), oldRepo.isReadOnly(),
+ oldRepo.getLocation(), fileSystem, oldfs
+ );
+ // Verify the repository can be connected with new settings
+ if (!newRepo.ping()) {
+ LOG.warn("Failed to connect repository {}. msg: {}", repoName, newRepo.getErrorMsg());
+ throw new DdlException("Repository ping failed with new properties");
+ }
+ // Apply the new repository metadata
+ Status st = repoMgr.alterRepo(newRepo, false /* not replay */);
+ if (!st.ok()) {
+ throw new DdlException("Failed to alter repository: " + st.getErrMsg());
}
+ // Update all running jobs that are using this repository
+ updateOngoingJobs(oldRepo.getId(), newRepo);
} finally {
seqlock.unlock();
}
}
+ /**
+ * Merges new user-provided properties into the existing repository's configuration.
+ * In strict mode, only supports S3 or Azure repositories and applies internal S3 merge logic.
+ *
+ * @param repo The existing repository.
+ * @param newProps New user-specified properties.
+ * @param strictCheck Whether to enforce S3/Azure-only and validate the new properties.
+ * @return A complete set of merged properties.
+ * @throws DdlException if the merge fails or the repository type is unsupported.
+ */
+ private Map mergeProperties(Repository repo, Map newProps, boolean strictCheck)
+ throws DdlException {
+ if (strictCheck) {
+ if (!(repo.getRemoteFileSystem() instanceof S3FileSystem
+ || repo.getRemoteFileSystem() instanceof AzureFileSystem)) {
+ throw new DdlException("Only support altering S3 or Azure repository");
+ }
+ // Let the repository validate and enrich the new S3/Azure properties
+ Map propsCopy = new HashMap<>(newProps);
+ Status status = repo.alterRepositoryS3Properties(propsCopy);
+ if (!status.ok()) {
+ throw new DdlException("Failed to merge S3 properties: " + status.getErrMsg());
+ }
+ return propsCopy;
+ } else {
+ // General case: just override old props with new ones
+ Map combined = new HashMap<>(repo.getRemoteFileSystem().getProperties());
+ combined.putAll(newProps);
+ return combined;
+ }
+ }
+
+ /**
+ * Updates all currently running jobs associated with the given repository ID.
+ * Used to ensure that all jobs operate on the new repository instance after alteration.
+ *
+ * @param repoId The ID of the altered repository.
+ * @param newRepo The new repository instance.
+ */
+ private void updateOngoingJobs(long repoId, Repository newRepo) {
+ for (AbstractJob job : getAllCurrentJobs()) {
+ if (!job.isDone() && job.getRepoId() == repoId) {
+ job.updateRepo(newRepo);
+ }
+ }
+ }
// handle drop repository stmt
public void dropRepository(DropRepositoryStmt stmt) throws DdlException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index b530d3b35e0a68..92e25ecc341503 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -39,7 +39,6 @@
import org.apache.doris.common.io.Text;
import org.apache.doris.common.util.DebugPointUtil;
import org.apache.doris.common.util.TimeUtils;
-import org.apache.doris.datasource.property.S3ClientBEProperties;
import org.apache.doris.persist.BarrierLog;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
@@ -388,7 +387,7 @@ public synchronized Status updateRepo(Repository repo) {
continue;
}
((UploadTask) task).updateBrokerProperties(
- S3ClientBEProperties.getBeFSProperties(repo.getRemoteFileSystem().getProperties()));
+ repo.getRemoteFileSystem().getStorageProperties().getBackendConfigProperties());
AgentTaskQueue.updateTask(beId, TTaskType.UPLOAD, signature, task);
}
LOG.info("finished to update upload job properties. {}", this);
@@ -783,7 +782,7 @@ private void uploadSnapshot() {
long signature = env.getNextId();
UploadTask task = new UploadTask(null, beId, signature, jobId, dbId, srcToDest,
brokers.get(0),
- S3ClientBEProperties.getBeFSProperties(repo.getRemoteFileSystem().getProperties()),
+ repo.getRemoteFileSystem().getStorageProperties().getBackendConfigProperties(),
repo.getRemoteFileSystem().getStorageType(), repo.getLocation());
batchTask.addTask(task);
unfinishedTaskIds.put(signature, beId);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
index 1450ef9a03459f..5154c74ab58294 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
@@ -25,19 +25,19 @@
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.Pair;
+import org.apache.doris.common.UserException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.property.constants.S3Properties;
-import org.apache.doris.fs.FileSystemFactory;
-import org.apache.doris.fs.PersistentFileSystem;
-import org.apache.doris.fs.remote.AzureFileSystem;
-import org.apache.doris.fs.remote.BrokerFileSystem;
-import org.apache.doris.fs.remote.RemoteFile;
-import org.apache.doris.fs.remote.RemoteFileSystem;
-import org.apache.doris.fs.remote.S3FileSystem;
-import org.apache.doris.fs.remote.dfs.DFSFileSystem;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+import org.apache.doris.fsv2.FileSystemFactory;
+import org.apache.doris.fsv2.PersistentFileSystem;
+import org.apache.doris.fsv2.remote.BrokerFileSystem;
+import org.apache.doris.fsv2.remote.RemoteFile;
+import org.apache.doris.fsv2.remote.RemoteFileSystem;
+import org.apache.doris.fsv2.remote.S3FileSystem;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.system.Backend;
@@ -132,18 +132,28 @@ public class Repository implements Writable, GsonPostProcessable {
private String location;
@SerializedName("fs")
+ private org.apache.doris.fs.PersistentFileSystem oldfs;
+
+ // Temporary field: currently still using the legacy fs config (oldfs).
+ // This field can be removed once the new fs configuration is fully enabled.
private PersistentFileSystem fileSystem;
+ public org.apache.doris.fs.PersistentFileSystem getOldfs() {
+ return oldfs;
+ }
+
private Repository() {
// for persist
}
- public Repository(long id, String name, boolean isReadOnly, String location, RemoteFileSystem fileSystem) {
+ public Repository(long id, String name, boolean isReadOnly, String location, RemoteFileSystem fileSystem,
+ org.apache.doris.fs.PersistentFileSystem oldFs) {
this.id = id;
this.name = name;
this.isReadOnly = isReadOnly;
this.location = location;
this.fileSystem = fileSystem;
+ this.oldfs = oldFs;
this.createTime = System.currentTimeMillis();
}
@@ -203,17 +213,53 @@ public static Repository read(DataInput in) throws IOException {
}
}
+ //todo why only support alter S3 properties
+ public Status alterRepositoryS3Properties(Map properties) {
+ if (this.fileSystem instanceof S3FileSystem) {
+ Map oldProperties = new HashMap<>(this.getRemoteFileSystem().getProperties());
+ oldProperties.remove(S3Properties.ACCESS_KEY);
+ oldProperties.remove(S3Properties.SECRET_KEY);
+ oldProperties.remove(S3Properties.SESSION_TOKEN);
+ oldProperties.remove(S3Properties.Env.ACCESS_KEY);
+ oldProperties.remove(S3Properties.Env.SECRET_KEY);
+ oldProperties.remove(S3Properties.Env.TOKEN);
+ for (Map.Entry entry : properties.entrySet()) {
+ if (Objects.equals(entry.getKey(), S3Properties.ACCESS_KEY)
+ || Objects.equals(entry.getKey(), S3Properties.Env.ACCESS_KEY)) {
+ oldProperties.putIfAbsent(S3Properties.ACCESS_KEY, entry.getValue());
+ }
+ if (Objects.equals(entry.getKey(), S3Properties.SECRET_KEY)
+ || Objects.equals(entry.getKey(), S3Properties.Env.SECRET_KEY)) {
+ oldProperties.putIfAbsent(S3Properties.SECRET_KEY, entry.getValue());
+ }
+ if (Objects.equals(entry.getKey(), S3Properties.SESSION_TOKEN)
+ || Objects.equals(entry.getKey(), S3Properties.Env.TOKEN)) {
+ oldProperties.putIfAbsent(S3Properties.SESSION_TOKEN, entry.getValue());
+ }
+ }
+ properties.clear();
+ properties.putAll(oldProperties);
+ return Status.OK;
+ } else {
+ return new Status(ErrCode.COMMON_ERROR, "Only support alter s3 repository");
+ }
+ }
+
@Override
public void gsonPostProcess() {
StorageBackend.StorageType type = StorageBackend.StorageType.BROKER;
- if (this.fileSystem.properties.containsKey(PersistentFileSystem.STORAGE_TYPE)) {
+ if (this.oldfs.properties.containsKey(org.apache.doris.fs.PersistentFileSystem.STORAGE_TYPE)) {
type = StorageBackend.StorageType.valueOf(
- this.fileSystem.properties.get(PersistentFileSystem.STORAGE_TYPE));
- this.fileSystem.properties.remove(PersistentFileSystem.STORAGE_TYPE);
+ this.oldfs.properties.get(org.apache.doris.fs.PersistentFileSystem.STORAGE_TYPE));
+ this.oldfs.properties.remove(org.apache.doris.fs.PersistentFileSystem.STORAGE_TYPE);
}
- this.fileSystem = FileSystemFactory.get(this.fileSystem.getName(),
+ this.oldfs = org.apache.doris.fs.FileSystemFactory.get(this.oldfs.getName(),
type,
- this.fileSystem.getProperties());
+ this.oldfs.getProperties());
+ if (!type.equals(StorageBackend.StorageType.BROKER)) {
+ StorageProperties storageProperties = StorageProperties.createPrimary(this.oldfs.properties);
+ this.fileSystem = FileSystemFactory.get(storageProperties);
+ }
}
public long getId() {
@@ -229,7 +275,18 @@ public boolean isReadOnly() {
}
public String getLocation() {
- return location;
+ if (null == fileSystem) {
+ return location;
+ }
+ try {
+ if (null == fileSystem.getStorageProperties()) {
+ return location;
+ } else {
+ return fileSystem.getStorageProperties().validateAndNormalizeUri(location);
+ }
+ } catch (UserException e) {
+ throw new RuntimeException(e);
+ }
}
public String getErrorMsg() {
@@ -277,7 +334,7 @@ public Status initRepository() {
if (name.compareTo((String) root.get("name")) != 0) {
return new Status(ErrCode.COMMON_ERROR,
"Invalid repository __repo_info, expected repo '" + name + "', but get name '"
- + (String) root.get("name") + "' from " + repoInfoFilePath);
+ + (String) root.get("name") + "' from " + repoInfoFilePath);
}
name = (String) root.get("name");
createTime = TimeUtils.timeStringToLong((String) root.get("create_time"));
@@ -307,54 +364,23 @@ public Status initRepository() {
}
}
- public Status alterRepositoryS3Properties(Map properties) {
- if (fileSystem instanceof S3FileSystem) {
- Map oldProperties = new HashMap<>(this.getRemoteFileSystem().getProperties());
- oldProperties.remove(S3Properties.ACCESS_KEY);
- oldProperties.remove(S3Properties.SECRET_KEY);
- oldProperties.remove(S3Properties.SESSION_TOKEN);
- oldProperties.remove(S3Properties.Env.ACCESS_KEY);
- oldProperties.remove(S3Properties.Env.SECRET_KEY);
- oldProperties.remove(S3Properties.Env.TOKEN);
- for (Map.Entry entry : properties.entrySet()) {
- if (Objects.equals(entry.getKey(), S3Properties.ACCESS_KEY)
- || Objects.equals(entry.getKey(), S3Properties.Env.ACCESS_KEY)) {
- oldProperties.putIfAbsent(S3Properties.ACCESS_KEY, entry.getValue());
- }
- if (Objects.equals(entry.getKey(), S3Properties.SECRET_KEY)
- || Objects.equals(entry.getKey(), S3Properties.Env.SECRET_KEY)) {
- oldProperties.putIfAbsent(S3Properties.SECRET_KEY, entry.getValue());
- }
- if (Objects.equals(entry.getKey(), S3Properties.SESSION_TOKEN)
- || Objects.equals(entry.getKey(), S3Properties.Env.TOKEN)) {
- oldProperties.putIfAbsent(S3Properties.SESSION_TOKEN, entry.getValue());
- }
- }
- properties.clear();
- properties.putAll(oldProperties);
- return Status.OK;
- } else {
- return new Status(ErrCode.COMMON_ERROR, "Only support alter s3 repository");
- }
- }
-
// eg: location/__palo_repository_repo_name/__repo_info
public String assembleRepoInfoFilePath() {
- return Joiner.on(PATH_DELIMITER).join(location,
+ return Joiner.on(PATH_DELIMITER).join(getLocation(),
joinPrefix(PREFIX_REPO, name),
FILE_REPO_INFO);
}
// eg: location/__palo_repository_repo_name/__my_sp1/__meta
public String assembleMetaInfoFilePath(String label) {
- return Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name),
+ return Joiner.on(PATH_DELIMITER).join(getLocation(), joinPrefix(PREFIX_REPO, name),
joinPrefix(PREFIX_SNAPSHOT_DIR, label),
FILE_META_INFO);
}
// eg: location/__palo_repository_repo_name/__my_sp1/__info_2018-01-01-08-00-00
public String assembleJobInfoFilePath(String label, long createTime) {
- return Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name),
+ return Joiner.on(PATH_DELIMITER).join(getLocation(), joinPrefix(PREFIX_REPO, name),
joinPrefix(PREFIX_SNAPSHOT_DIR, label),
jobInfoFileNameWithTimestamp(createTime));
}
@@ -362,7 +388,7 @@ public String assembleJobInfoFilePath(String label, long createTime) {
// eg:
// __palo_repository_repo_name/__ss_my_ss1/__ss_content/__db_10001/__tbl_10020/__part_10031/__idx_10020/__10022/
public String getRepoTabletPathBySnapshotInfo(String label, SnapshotInfo info) {
- String path = Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name),
+ String path = Joiner.on(PATH_DELIMITER).join(getLocation(), joinPrefix(PREFIX_REPO, name),
joinPrefix(PREFIX_SNAPSHOT_DIR, label),
DIR_SNAPSHOT_CONTENT,
joinPrefix(PREFIX_DB, info.getDbId()),
@@ -381,7 +407,7 @@ public String getRepoTabletPathBySnapshotInfo(String label, SnapshotInfo info) {
}
public String getRepoPath(String label, String childPath) {
- String path = Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name),
+ String path = Joiner.on(PATH_DELIMITER).join(getLocation(), joinPrefix(PREFIX_REPO, name),
joinPrefix(PREFIX_SNAPSHOT_DIR, label),
DIR_SNAPSHOT_CONTENT,
childPath);
@@ -568,7 +594,7 @@ public Status upload(String localFilePath, String remoteFilePath) {
if (!st.ok()) {
return st;
}
- } else if (fileSystem instanceof S3FileSystem || fileSystem instanceof AzureFileSystem) {
+ } else {
if (LOG.isDebugEnabled()) {
LOG.debug("get md5sum of file: {}. final remote path: {}", localFilePath, finalRemotePath);
}
@@ -577,20 +603,6 @@ public Status upload(String localFilePath, String remoteFilePath) {
return st;
}
- // upload final file
- st = fileSystem.upload(localFilePath, finalRemotePath);
- if (!st.ok()) {
- return st;
- }
- } else if (fileSystem instanceof DFSFileSystem) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("hdfs get md5sum of file: {}. final remote path: {}", localFilePath, finalRemotePath);
- }
- st = fileSystem.delete(finalRemotePath);
- if (!st.ok()) {
- return st;
- }
-
// upload final file
st = fileSystem.upload(localFilePath, finalRemotePath);
if (!st.ok()) {
@@ -637,7 +649,7 @@ public Status download(String remoteFilePath, String localFilePath) {
// 2. download
status = fileSystem.downloadWithFileSize(remoteFilePathWithChecksum, localFilePath,
- remoteFiles.get(0).getSize());
+ remoteFiles.get(0).getSize());
if (!status.ok()) {
return status;
}
@@ -855,7 +867,13 @@ public void readFields(DataInput in) throws IOException {
name = Text.readString(in);
isReadOnly = in.readBoolean();
location = Text.readString(in);
- fileSystem = PersistentFileSystem.read(in);
+ oldfs = org.apache.doris.fs.PersistentFileSystem.read(in);
+ try {
+ fileSystem = FileSystemFactory.get(oldfs.getStorageType(), oldfs.getProperties());
+ } catch (UserException e) {
+ // do we ignore this exception?
+ throw new IOException("Failed to create file system: " + e.getMessage());
+ }
createTime = in.readLong();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java
index 853c1841449046..d57593c5443098 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java
@@ -23,8 +23,8 @@
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.Daemon;
-import org.apache.doris.fs.remote.AzureFileSystem;
-import org.apache.doris.fs.remote.S3FileSystem;
+import org.apache.doris.fsv2.remote.AzureFileSystem;
+import org.apache.doris.fsv2.remote.S3FileSystem;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
@@ -104,6 +104,9 @@ public Repository getRepo(long repoId) {
return repoIdMap.get(repoId);
}
+ /**
+ * todo: why not support alter other file system like hdfs
+ */
public Status alterRepo(Repository newRepo, boolean isReplay) {
lock.lock();
try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index a3230de290e739..2643d0efacd5e4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -68,7 +68,6 @@
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.InternalCatalog;
-import org.apache.doris.datasource.property.S3ClientBEProperties;
import org.apache.doris.persist.ColocatePersistInfo;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
@@ -438,7 +437,7 @@ public synchronized Status updateRepo(Repository repo) {
continue;
}
((DownloadTask) task).updateBrokerProperties(
- S3ClientBEProperties.getBeFSProperties(repo.getRemoteFileSystem().getProperties()));
+ repo.getRemoteFileSystem().getStorageProperties().getBackendConfigProperties());
AgentTaskQueue.updateTask(beId, TTaskType.DOWNLOAD, signature, task);
}
LOG.info("finished to update download job properties. {}", this);
@@ -1864,7 +1863,7 @@ private void downloadRemoteSnapshots() {
long signature = env.getNextId();
DownloadTask task = new DownloadTask(null, beId, signature, jobId, dbId, srcToDest,
brokerAddrs.get(0),
- S3ClientBEProperties.getBeFSProperties(repo.getRemoteFileSystem().getProperties()),
+ repo.getRemoteFileSystem().getStorageProperties().getBackendConfigProperties(),
repo.getRemoteFileSystem().getStorageType(), repo.getLocation());
batchTask.addTask(task);
unfinishedSignatureToId.put(signature, beId);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerReader.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerReader.java
index 592d9f94d114b3..bb3162aa1e7f01 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerReader.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerReader.java
@@ -103,7 +103,7 @@ public TBrokerFD open(String path) throws IOException {
String clientId = NetUtils
.getHostPortInAccessibleFormat(FrontendOptions.getLocalHostAddress(), Config.rpc_port);
TBrokerOpenReaderRequest tOpenReaderRequest = new TBrokerOpenReaderRequest(
- TBrokerVersion.VERSION_ONE, path, 0, clientId, brokerDesc.getProperties());
+ TBrokerVersion.VERSION_ONE, path, 0, clientId, brokerDesc.getBackendConfigProperties());
TBrokerOpenReaderResponse tOpenReaderResponse = null;
try {
tOpenReaderResponse = client.openReader(tOpenReaderRequest);
@@ -137,7 +137,7 @@ public void close(TBrokerFD fd) {
public long getFileLength(String path) throws IOException {
TBrokerListPathRequest request = new TBrokerListPathRequest(
- TBrokerVersion.VERSION_ONE, path, false, brokerDesc.getProperties());
+ TBrokerVersion.VERSION_ONE, path, false, brokerDesc.getBackendConfigProperties());
TBrokerListResponse tBrokerListResponse = null;
try {
tBrokerListResponse = client.listPath(request);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java
index 617d346ba3daa3..e043b7a9fd6406 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java
@@ -28,9 +28,9 @@
import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.hive.HiveMetaStoreCache;
-import org.apache.doris.fs.FileSystemFactory;
-import org.apache.doris.fs.remote.RemoteFile;
-import org.apache.doris.fs.remote.RemoteFileSystem;
+import org.apache.doris.fsv2.FileSystemFactory;
+import org.apache.doris.fsv2.remote.RemoteFile;
+import org.apache.doris.fsv2.remote.RemoteFileSystem;
import org.apache.doris.service.FrontendOptions;
import org.apache.doris.thrift.TBrokerCheckPathExistRequest;
import org.apache.doris.thrift.TBrokerCheckPathExistResponse;
@@ -85,8 +85,7 @@ public class BrokerUtil {
public static void parseFile(String path, BrokerDesc brokerDesc, List fileStatuses)
throws UserException {
List rfiles = new ArrayList<>();
- try (RemoteFileSystem fileSystem = FileSystemFactory.get(
- brokerDesc.getName(), brokerDesc.getStorageType(), brokerDesc.getProperties())) {
+ try (RemoteFileSystem fileSystem = FileSystemFactory.get(brokerDesc)) {
Status st = fileSystem.globList(path, rfiles, false);
if (!st.ok()) {
throw new UserException(st.getErrMsg());
@@ -107,8 +106,7 @@ public static void parseFile(String path, BrokerDesc brokerDesc, List getLocationProperties() throws UserException;
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java
index 70d55a497e458a..2b028cf695ddb7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java
@@ -99,9 +99,10 @@ public FileLoadScanNode.ParamCreateContext createContext(Analyzer analyzer) thro
&& fileGroupInfo.getFileGroup().getFileFormat().equals("hive_text")) {
params.setTextSerdeType(TTextSerdeType.HIVE_TEXT_SERDE);
}
- params.setProperties(fileGroupInfo.getBrokerDesc().getProperties());
+ params.setProperties(fileGroupInfo.getBrokerDesc().getBackendConfigProperties());
if (fileGroupInfo.getBrokerDesc().getFileType() == TFileType.FILE_HDFS) {
- THdfsParams tHdfsParams = HdfsResource.generateHdfsParam(fileGroupInfo.getBrokerDesc().getProperties());
+ THdfsParams tHdfsParams = HdfsResource.generateHdfsParam(fileGroupInfo.getBrokerDesc()
+ .getBackendConfigProperties());
params.setHdfsParams(tHdfsParams);
}
TFileAttributes fileAttributes = new TFileAttributes();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectionProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectionProperties.java
index 7fe00cedec7984..2c86c67e4ce4a8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectionProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectionProperties.java
@@ -18,7 +18,7 @@
package org.apache.doris.datasource.property;
import org.apache.doris.common.CatalogConfigFileUtils;
-import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
@@ -55,7 +55,7 @@ protected ConnectionProperties(Map origProps) {
this.origProps = origProps;
}
- protected void initNormalizeAndCheckProps() throws UserException {
+ protected void initNormalizeAndCheckProps() {
List supportedProps = PropertyUtils.getConnectorProperties(this.getClass());
for (Field field : supportedProps) {
field.setAccessible(true);
@@ -67,7 +67,8 @@ protected void initNormalizeAndCheckProps() throws UserException {
field.set(this, origProps.get(name));
matchedProperties.put(name, origProps.get(name));
} catch (IllegalAccessException e) {
- throw new RuntimeException("Failed to set property " + name + ", " + e.getMessage(), e);
+ throw new StoragePropertiesException("Failed to set property " + name
+ + ", " + e.getMessage(), e);
}
break;
}
@@ -115,7 +116,8 @@ protected void checkRequiredProperties() {
throw new IllegalArgumentException("Property " + names[0] + " is required.");
}
} catch (IllegalAccessException e) {
- throw new RuntimeException(e);
+ throw new StoragePropertiesException("Failed to get property " + names[0]
+ + ", " + e.getMessage(), e);
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFProperties.java
index c0096baddc467b..37b79b7dcc7f22 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFProperties.java
@@ -17,7 +17,6 @@
package org.apache.doris.datasource.property.metastore;
-import org.apache.doris.common.UserException;
import org.apache.doris.datasource.property.ConnectorProperty;
import com.google.common.base.Strings;
@@ -67,7 +66,7 @@ public AliyunDLFProperties(Map origProps) {
}
@Override
- protected void initNormalizeAndCheckProps() throws UserException {
+ protected void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
// Other properties that start with "dlf." will be saved in otherDlfProps,
// and passed to the DLF client.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java
index b7c18a3e74a808..2702ae474918ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java
@@ -18,7 +18,6 @@
package org.apache.doris.datasource.property.metastore;
import org.apache.doris.common.CatalogConfigFileUtils;
-import org.apache.doris.common.UserException;
import org.apache.doris.datasource.property.ConnectorProperty;
import com.google.common.base.Strings;
@@ -95,7 +94,7 @@ protected void checkRequiredProperties() {
}
@Override
- protected void initNormalizeAndCheckProps() throws UserException {
+ protected void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
hiveConfParams = loadConfigFromFile(getResourceConfigPropName());
initHmsConnectionProperties();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
index 5685c278347867..3c0422954e54e7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
@@ -80,6 +80,12 @@ public abstract class AbstractS3CompatibleProperties extends StorageProperties i
@Getter
protected String forceParsingByStandardUrl = "false";
+ @Getter
+ @ConnectorProperty(names = {"s3.session_token", "session_token"},
+ required = false,
+ description = "The session token of S3.")
+ protected String sessionToken = "";
+
/**
* Constructor to initialize the object storage properties with the provided type and original properties map.
*
@@ -135,6 +141,9 @@ private Map doBuildS3Configuration(String maxConnections,
s3Props.put("AWS_REQUEST_TIMEOUT_MS", requestTimeoutMs);
s3Props.put("AWS_CONNECTION_TIMEOUT_MS", connectionTimeoutMs);
s3Props.put("use_path_style", usePathStyle);
+ if (StringUtils.isNotBlank(getSessionToken())) {
+ s3Props.put("AWS_TOKEN", getSessionToken());
+ }
return s3Props;
}
@@ -145,7 +154,7 @@ public Map getBackendConfigProperties() {
@Override
- protected void initNormalizeAndCheckProps() throws UserException {
+ protected void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
setEndpointIfNotSet();
if (!isValidEndpoint(getEndpoint())) {
@@ -184,7 +193,7 @@ private boolean isValidEndpoint(String endpoint) {
return endpointPattern().matcher(endpoint).matches();
}
- private void setEndpointIfNotSet() throws UserException {
+ private void setEndpointIfNotSet() {
if (StringUtils.isNotBlank(getEndpoint())) {
return;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AzureProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AzureProperties.java
index 512ed92cca9054..9d85c3657668c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AzureProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AzureProperties.java
@@ -100,7 +100,7 @@ public AzureProperties(Map origProps) {
private static final String AZURE_ENDPOINT_SUFFIX = ".blob.core.windows.net";
@Override
- protected void initNormalizeAndCheckProps() throws UserException {
+ protected void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
//check endpoint
if (!endpoint.endsWith(AZURE_ENDPOINT_SUFFIX)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/BrokerProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/BrokerProperties.java
new file mode 100644
index 00000000000000..b53a65ecdc7ac9
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/BrokerProperties.java
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.storage;
+
+import org.apache.doris.common.UserException;
+
+import java.util.Map;
+
+public class BrokerProperties extends StorageProperties {
+ public BrokerProperties(Map origProps) {
+ super(Type.BROKER, origProps);
+ }
+
+ @Override
+ public Map getBackendConfigProperties() {
+ return origProps;
+ }
+
+ @Override
+ public String validateAndNormalizeUri(String url) throws UserException {
+ return url;
+ }
+
+ @Override
+ public String validateAndGetUri(Map loadProps) throws UserException {
+ return loadProps.get("uri");
+ }
+
+ @Override
+ public String getStorageName() {
+ return "BROKER";
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
index 6dc72e1b27cc97..730a4bdb7b2fac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
@@ -25,6 +25,7 @@
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.regex.Pattern;
import java.util.stream.Stream;
@@ -45,7 +46,7 @@ public class COSProperties extends AbstractS3CompatibleProperties {
protected String region = "";
@Getter
- @ConnectorProperty(names = {"cos.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key"},
+ @ConnectorProperty(names = {"cos.access_key", "s3.access_key", "AWS_ACCESS_KEY", "access_key", "ACCESS_KEY"},
description = "The access key of COS.")
protected String accessKey = "";
@@ -75,12 +76,13 @@ protected static boolean guessIsMe(Map origProps) {
.findFirst()
.orElse(null);
if (!Strings.isNullOrEmpty(value)) {
- return ENDPOINT_PATTERN.matcher(value).matches();
+ return value.contains("myqcloud.com");
}
- if (!origProps.containsKey("uri")) {
- return false;
- }
- return origProps.get("uri").contains("myqcloud.com");
+ Optional uriValue = origProps.entrySet().stream()
+ .filter(e -> e.getKey().equalsIgnoreCase("uri"))
+ .map(Map.Entry::getValue)
+ .findFirst();
+ return uriValue.isPresent() && uriValue.get().contains("myqcloud.com");
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java
index d4722a3c1ad0f5..97c6c28db145a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java
@@ -58,6 +58,11 @@ public class HdfsProperties extends HdfsCompatibleProperties {
description = "Whether to enable the impersonation of HDFS.")
private boolean hdfsImpersonationEnabled = false;
+ @ConnectorProperty(names = {"ipc.client.fallback-to-simple-auth-allowed"},
+ required = false,
+ description = "Whether to allow fallback to simple authentication.")
+ private String allowFallbackToSimpleAuth = "";
+
private Map backendConfigProperties;
/**
@@ -69,6 +74,8 @@ public class HdfsProperties extends HdfsCompatibleProperties {
*/
private Map userOverriddenHdfsConfig;
+ public static final String HDFS_DEFAULT_FS_NAME = "fs.defaultFS";
+
private static final List HDFS_PROPERTIES_KEYS = Arrays.asList("hdfs.authentication.type",
"hadoop.security.authentication", "hadoop.username",
"hdfs.authentication.kerberos.principal", "hadoop.kerberos.principal", "dfs.nameservices");
@@ -81,6 +88,9 @@ public static boolean guessIsMe(Map props) {
if (MapUtils.isEmpty(props)) {
return false;
}
+ if (HdfsPropertiesUtils.validateUriIsHdfsUri(props)) {
+ return true;
+ }
if (HDFS_PROPERTIES_KEYS.stream().anyMatch(props::containsKey)) {
return true;
}
@@ -98,7 +108,7 @@ public static boolean guessIsMe(Map props) {
}
@Override
- protected void initNormalizeAndCheckProps() throws UserException {
+ protected void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
extractUserOverriddenHdfsConfig(origProps);
initHadoopConfiguration();
@@ -111,7 +121,7 @@ private void extractUserOverriddenHdfsConfig(Map origProps) {
}
userOverriddenHdfsConfig = new HashMap<>();
origProps.forEach((key, value) -> {
- if (key.startsWith("hadoop.") || key.startsWith("dfs.") || key.equals("fs.defaultFS")) {
+ if (key.startsWith("hadoop.") || key.startsWith("dfs.") || key.startsWith("fs.")) {
userOverriddenHdfsConfig.put(key, value);
}
});
@@ -132,11 +142,7 @@ protected void checkRequiredProperties() {
// fsDefaultFS is not strictly required here.
// This is a best-effort fallback to populate fsDefaultFS when possible.
if (StringUtils.isBlank(fsDefaultFS)) {
- try {
- this.fsDefaultFS = HdfsPropertiesUtils.validateAndGetUri(origProps);
- } catch (UserException e) {
- //ignore
- }
+ this.fsDefaultFS = HdfsPropertiesUtils.extractDefaultFsFromUri(origProps);
}
}
@@ -148,7 +154,12 @@ private void initHadoopConfiguration() {
userOverriddenHdfsConfig.forEach(conf::set);
}
if (StringUtils.isNotBlank(fsDefaultFS)) {
- conf.set("fs.defaultFS", fsDefaultFS);
+ conf.set(HDFS_DEFAULT_FS_NAME, fsDefaultFS);
+ }
+ if (StringUtils.isNotBlank(allowFallbackToSimpleAuth)) {
+ conf.set("ipc.client.fallback-to-simple-auth-allowed", allowFallbackToSimpleAuth);
+ } else {
+ conf.set("ipc.client.fallback-to-simple-auth-allowed", "true");
}
conf.set("hdfs.security.authentication", hdfsAuthenticationType);
if ("kerberos".equalsIgnoreCase(hdfsAuthenticationType)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java
index 5e97a2fd639fb4..77822f8426d093 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java
@@ -20,120 +20,101 @@
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.URI;
+import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
+import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang3.StringUtils;
-import java.util.HashSet;
import java.util.Map;
import java.util.Set;
public class HdfsPropertiesUtils {
private static final String URI_KEY = "uri";
+ private static final Set supportSchema = ImmutableSet.of("hdfs", "viewfs");
- private static Set supportSchema = new HashSet<>();
-
- static {
- supportSchema.add("hdfs");
- supportSchema.add("viewfs");
- }
-
- /**
- * Validates that the 'uri' property exists in the provided props map, and normalizes it.
- *
- * @param props the map of properties that must include a 'uri' entry
- * @return a normalized URI string like 'hdfs://host/path'
- * @throws UserException if the map is empty or does not contain the required 'uri' key
- *
- * Example:
- * Input: {"uri": "hdfs://namenode:9000/data/input"}
- * Output: "hdfs://namenode:9000/data/input"
- */
public static String validateAndGetUri(Map props) throws UserException {
if (props.isEmpty()) {
throw new UserException("props is empty");
}
- if (!props.containsKey(URI_KEY)) {
- throw new UserException("props must contain uri");
+ String uriStr = getUri(props);
+ if (StringUtils.isBlank(uriStr)) {
+ throw new StoragePropertiesException("props must contain uri");
}
- String uriStr = props.get(URI_KEY);
return validateAndNormalizeUri(uriStr);
}
- /**
- * Validates and normalizes a raw URI string.
- *
- * @param uriStr the URI string to validate
- * @return a normalized URI in the form of 'scheme://authority/path'
- * @throws UserException if the URI is invalid or unsupported
- *
- * Example:
- * Input: "viewfs://ns1/path/to/file"
- * Output: "viewfs://ns1/path/to/file"
- */
- public static String convertUrlToFilePath(String uriStr) throws UserException {
- return validateAndNormalizeUri(uriStr);
+ public static boolean validateUriIsHdfsUri(Map props) {
+ String uriStr = getUri(props);
+ if (StringUtils.isBlank(uriStr)) {
+ return false;
+ }
+ try {
+ URI uri = URI.create(uriStr);
+ String schema = uri.getScheme();
+ if (StringUtils.isBlank(schema)) {
+ throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null");
+ }
+ return isSupportedSchema(schema);
+ } catch (AnalysisException e) {
+ throw new IllegalArgumentException("Invalid uri: " + uriStr, e);
+ }
}
- /**
- * Constructs the default filesystem URI (scheme + authority) from a full URI string in the props map.
- *
- * @param props the map of properties, expected to contain a valid 'uri' entry
- * @return a URI prefix like 'hdfs://host:port', or null if the URI is missing or invalid
- *
- * Example:
- * Input: {"uri": "hdfs://namenode:8020/data"}
- * Output: "hdfs://namenode:8020"
- */
- public static String constructDefaultFsFromUri(Map props) {
- if (props.isEmpty()) {
+ public static String extractDefaultFsFromPath(String filePath) {
+ if (StringUtils.isBlank(filePath)) {
return null;
}
- if (!props.containsKey(URI_KEY)) {
- return null;
+ try {
+ URI uri = URI.create(filePath);
+ return uri.getScheme() + "://" + uri.getAuthority();
+ } catch (AnalysisException e) {
+ throw new IllegalArgumentException("Invalid file path: " + filePath, e);
}
- String uriStr = props.get(URI_KEY);
+ }
+
+ public static String extractDefaultFsFromUri(Map props) {
+ String uriStr = getUri(props);
if (StringUtils.isBlank(uriStr)) {
return null;
}
- URI uri = null;
try {
- uri = URI.create(uriStr);
+ URI uri = URI.create(uriStr);
+ if (!isSupportedSchema(uri.getScheme())) {
+ return null;
+ }
+ return uri.getScheme() + "://" + uri.getAuthority();
} catch (AnalysisException e) {
- return null;
- }
- String schema = uri.getScheme();
- if (StringUtils.isBlank(schema)) {
- throw new IllegalArgumentException("Invalid uri: " + uriStr + "extract schema is null");
- }
- if (!supportSchema.contains(schema.toLowerCase())) {
- throw new IllegalArgumentException("Invalid export path:"
- + schema + " , please use valid 'hdfs://' or 'viewfs://' path.");
+ throw new IllegalArgumentException("Invalid uri: " + uriStr, e);
}
- return uri.getScheme() + "://" + uri.getAuthority();
}
- /**
- * Internal method that validates and normalizes a URI string.
- * Ensures it has a valid scheme and is supported (e.g., hdfs, viewfs).
- *
- * @param uriStr the URI string to validate
- * @return the normalized URI string
- * @throws AnalysisException if the URI is blank or has an unsupported scheme
- *
- * Example:
- * Input: "hdfs://host:8020/user/data"
- * Output: "hdfs://host:8020/user/data"
- */
+ public static String convertUrlToFilePath(String uriStr) throws UserException {
+ return validateAndNormalizeUri(uriStr);
+ }
+
+ private static String getUri(Map props) {
+ return props.entrySet().stream()
+ .filter(e -> e.getKey().equalsIgnoreCase(URI_KEY))
+ .map(Map.Entry::getValue)
+ .filter(StringUtils::isNotBlank)
+ .findFirst()
+ .orElse(null);
+ }
+
+ private static boolean isSupportedSchema(String schema) {
+ return schema != null && supportSchema.contains(schema.toLowerCase());
+ }
+
private static String validateAndNormalizeUri(String uriStr) throws AnalysisException {
if (StringUtils.isBlank(uriStr)) {
- throw new IllegalArgumentException("uri is null, pls check your params");
+ throw new IllegalArgumentException("Properties 'uri' is required");
}
URI uri = URI.create(uriStr);
String schema = uri.getScheme();
if (StringUtils.isBlank(schema)) {
- throw new IllegalArgumentException("Invalid uri: " + uriStr + "extract schema is null");
+ throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null");
}
- if (!supportSchema.contains(schema.toLowerCase())) {
+ if (!isSupportedSchema(schema)) {
throw new IllegalArgumentException("Invalid export path:"
+ schema + " , please use valid 'hdfs://' or 'viewfs://' path.");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java
new file mode 100644
index 00000000000000..8f5c3f3dfa324a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.storage;
+
+import org.apache.doris.datasource.property.ConnectorProperty;
+
+import com.google.common.collect.ImmutableSet;
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+public class MinioProperties extends AbstractS3CompatibleProperties {
+ @Setter
+ @Getter
+ @ConnectorProperty(names = {"minio.endpoint", "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT"},
+ required = false, description = "The endpoint of Minio.")
+ protected String endpoint = "";
+
+ @Getter
+ @Setter
+ protected String region = "us-east-1";
+
+ @Getter
+ @ConnectorProperty(names = {"minio.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key", "s3.access_key"},
+ description = "The access key of Minio.")
+ protected String accessKey = "";
+
+ @Getter
+ @ConnectorProperty(names = {"minio.secret_key", "s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY"},
+ description = "The secret key of Minio.")
+ protected String secretKey = "";
+
+ private static final Set IDENTIFIERS = ImmutableSet.of("minio.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY",
+ "access_key", "s3.access_key", "minio.endpoint", "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT");
+
+ /**
+ * Constructor to initialize the object storage properties with the provided type and original properties map.
+ *
+ * @param origProps the original properties map.
+ */
+ protected MinioProperties(Map origProps) {
+ super(Type.MINIO, origProps);
+ }
+
+ public static boolean guessIsMe(Map origProps) {
+ //ugly, but we need to check if the user has set any of the identifiers
+ if (AzureProperties.guessIsMe(origProps) || COSProperties.guessIsMe(origProps)
+ || OSSProperties.guessIsMe(origProps) || S3Properties.guessIsMe(origProps)) {
+ return false;
+ }
+
+ return IDENTIFIERS.stream().map(origProps::get).anyMatch(value -> value != null && !value.isEmpty());
+ }
+
+
+ @Override
+ protected Pattern endpointPattern() {
+ return Pattern.compile("^(?:https?://)?[a-zA-Z0-9.-]+(?::\\d+)?$");
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
index 87ad9b5761c7ab..b99f36bcd30303 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
@@ -25,6 +25,7 @@
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.regex.Pattern;
import java.util.stream.Stream;
@@ -38,12 +39,12 @@ public class OBSProperties extends AbstractS3CompatibleProperties {
protected String endpoint = "";
@Getter
- @ConnectorProperty(names = {"obs.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key"},
+ @ConnectorProperty(names = {"obs.access_key", "s3.access_key", "AWS_ACCESS_KEY", "access_key", "ACCESS_KEY"},
description = "The access key of OBS.")
protected String accessKey = "";
@Getter
- @ConnectorProperty(names = {"obs.secret_key", "secret_key", "s3.secret_key"},
+ @ConnectorProperty(names = {"obs.secret_key", "s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY"},
description = "The secret key of OBS.")
protected String secretKey = "";
@@ -80,13 +81,13 @@ protected static boolean guessIsMe(Map origProps) {
.orElse(null);
if (!Strings.isNullOrEmpty(value)) {
- return ENDPOINT_PATTERN.matcher(value).matches();
+ return value.contains("myhuaweicloud.com");
}
- if (!origProps.containsKey("uri")) {
- return false;
- }
- // Check if the uri property contains "myhuaweicloud.com"
- return origProps.get("uri").contains("myhuaweicloud.com");
+ Optional uriValue = origProps.entrySet().stream()
+ .filter(e -> e.getKey().equalsIgnoreCase("uri"))
+ .map(Map.Entry::getValue)
+ .findFirst();
+ return uriValue.isPresent() && uriValue.get().contains("myhuaweicloud.com");
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
index e0e0def53f6d01..0a09a8602532dc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
@@ -82,7 +82,7 @@ protected void checkRequiredProperties() {
}
@Override
- protected void initNormalizeAndCheckProps() throws UserException {
+ protected void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
initConfigurationParams();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
index 34982f4b690039..26170dcef8bbad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
@@ -25,6 +25,7 @@
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.regex.Pattern;
import java.util.stream.Stream;
@@ -38,7 +39,7 @@ public class OSSProperties extends AbstractS3CompatibleProperties {
protected String endpoint = "";
@Getter
- @ConnectorProperty(names = {"oss.access_key", "s3.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key"},
+ @ConnectorProperty(names = {"oss.access_key", "s3.access_key", "AWS_ACCESS_KEY", "access_key", "ACCESS_KEY"},
description = "The access key of OSS.")
protected String accessKey = "";
@@ -78,12 +79,17 @@ protected static boolean guessIsMe(Map origProps) {
.findFirst()
.orElse(null);
if (!Strings.isNullOrEmpty(value)) {
- return ENDPOINT_PATTERN.matcher(value).matches();
+ return value.contains("aliyuncs.com");
}
- if (!origProps.containsKey("uri")) {
+ Optional uriValue = origProps.entrySet().stream()
+ .filter(e -> e.getKey().equalsIgnoreCase("uri"))
+ .map(Map.Entry::getValue)
+ .findFirst();
+ if (!uriValue.isPresent()) {
return false;
}
- return origProps.get("uri").contains("aliyuncs.com");
+ String uri = uriValue.get();
+ return uri.contains("aliyuncs.com") && (!uri.contains("oss-dls.aliyuncs.com"));
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
index 478f45ee3d06b5..343b4b29d39ef5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -28,6 +28,7 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.regex.Pattern;
import java.util.stream.Stream;
@@ -49,7 +50,7 @@ public class S3Properties extends AbstractS3CompatibleProperties {
protected String region = "";
@Getter
- @ConnectorProperty(names = {"s3.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key"},
+ @ConnectorProperty(names = {"s3.access_key", "AWS_ACCESS_KEY", "access_key", "ACCESS_KEY"},
description = "The access key of S3.")
protected String accessKey = "";
@@ -134,14 +135,20 @@ protected static boolean guessIsMe(Map origProps) {
.filter(Objects::nonNull)
.findFirst()
.orElse(null);
+ /**
+ * Check if the endpoint contains "amazonaws.com" to determine if it's an S3-compatible storage.
+ * Note: This check should not be overly strict, as a malformed or misconfigured endpoint may
+ * cause the type detection to fail, leading to missed recognition of valid S3 properties.
+ * A more robust approach would allow further validation downstream rather than failing early here.
+ */
if (!Strings.isNullOrEmpty(endpoint)) {
- return ENDPOINT_PATTERN.matcher(endpoint).matches();
+ return endpoint.contains("amazonaws.com");
}
- if (!origProps.containsKey("uri")) {
- return false;
- }
- String uri = origProps.get("uri");
- return uri.contains("amazonaws.com");
+ Optional uriValue = origProps.entrySet().stream()
+ .filter(e -> e.getKey().equalsIgnoreCase("uri"))
+ .map(Map.Entry::getValue)
+ .findFirst();
+ return uriValue.isPresent() && uriValue.get().contains("amazonaws.com");
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java
index 745838438dd012..0a4fda0bcdfe8f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java
@@ -19,10 +19,12 @@
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.S3URI;
+import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
import org.apache.commons.lang3.StringUtils;
import java.util.Map;
+import java.util.Optional;
public class S3PropertyUtils {
@@ -43,14 +45,28 @@ public class S3PropertyUtils {
*/
public static String constructEndpointFromUrl(Map props,
String stringUsePathStyle,
- String stringForceParsingByStandardUri) throws UserException {
- String uri = props.get(URI_KEY);
- if (uri == null || uri.isEmpty()) {
+ String stringForceParsingByStandardUri) {
+ Optional uriOptional = props.entrySet().stream()
+ .filter(e -> e.getKey().equalsIgnoreCase(URI_KEY))
+ .map(Map.Entry::getValue)
+ .findFirst();
+
+ if (!uriOptional.isPresent()) {
+ return null;
+ }
+ String uri = uriOptional.get();
+ if (StringUtils.isBlank(uri)) {
return null;
}
boolean usePathStyle = Boolean.parseBoolean(stringUsePathStyle);
boolean forceParsingByStandardUri = Boolean.parseBoolean(stringForceParsingByStandardUri);
- S3URI s3uri = S3URI.create(uri, usePathStyle, forceParsingByStandardUri);
+ S3URI s3uri;
+ try {
+ s3uri = S3URI.create(uri, usePathStyle, forceParsingByStandardUri);
+ } catch (UserException e) {
+ throw new IllegalArgumentException("Invalid S3 URI: " + uri + ",usePathStyle: " + usePathStyle
+ + " forceParsingByStandardUri: " + forceParsingByStandardUri, e);
+ }
return s3uri.getEndpoint().orElse(null);
}
@@ -68,14 +84,28 @@ public static String constructEndpointFromUrl(Map props,
*/
public static String constructRegionFromUrl(Map props,
String stringUsePathStyle,
- String stringForceParsingByStandardUri) throws UserException {
- String uri = props.get(URI_KEY);
- if (uri == null || uri.isEmpty()) {
+ String stringForceParsingByStandardUri) {
+ Optional uriOptional = props.entrySet().stream()
+ .filter(e -> e.getKey().equalsIgnoreCase(URI_KEY))
+ .map(Map.Entry::getValue)
+ .findFirst();
+
+ if (!uriOptional.isPresent()) {
+ return null;
+ }
+ String uri = uriOptional.get();
+ if (StringUtils.isBlank(uri)) {
return null;
}
boolean usePathStyle = Boolean.parseBoolean(stringUsePathStyle);
boolean forceParsingByStandardUri = Boolean.parseBoolean(stringForceParsingByStandardUri);
- S3URI s3uri = S3URI.create(uri, usePathStyle, forceParsingByStandardUri);
+ S3URI s3uri = null;
+ try {
+ s3uri = S3URI.create(uri, usePathStyle, forceParsingByStandardUri);
+ } catch (UserException e) {
+ throw new IllegalArgumentException("Invalid S3 URI: " + uri + ",usePathStyle: " + usePathStyle
+ + " forceParsingByStandardUri: " + forceParsingByStandardUri, e);
+ }
return s3uri.getRegion().orElse(null);
}
@@ -99,7 +129,7 @@ public static String validateAndNormalizeUri(String path,
String stringUsePathStyle,
String stringForceParsingByStandardUri) throws UserException {
if (StringUtils.isBlank(path)) {
- throw new UserException("path is null");
+ throw new StoragePropertiesException("path is null");
}
if (path.startsWith("s3://")) {
return path;
@@ -122,13 +152,18 @@ public static String validateAndNormalizeUri(String path,
* Input: {"uri": "s3://my-bucket/my-key"}
* Output: "s3://my-bucket/my-key"
*/
- public static String validateAndGetUri(Map props) throws UserException {
+ public static String validateAndGetUri(Map props) {
if (props.isEmpty()) {
- throw new UserException("props is empty");
+ throw new StoragePropertiesException("props is empty");
}
- if (!props.containsKey(URI_KEY)) {
- throw new UserException("props must contain uri");
+ Optional uriOptional = props.entrySet().stream()
+ .filter(e -> e.getKey().equalsIgnoreCase(URI_KEY))
+ .map(Map.Entry::getValue)
+ .findFirst();
+
+ if (!uriOptional.isPresent()) {
+ throw new StoragePropertiesException("props must contain uri");
}
- return props.get(URI_KEY);
+ return uriOptional.get();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
index 58cdedde869a5e..ae69deea400ba0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
@@ -20,6 +20,7 @@
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.property.ConnectionProperties;
import org.apache.doris.datasource.property.ConnectorProperty;
+import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
import lombok.Getter;
@@ -35,6 +36,7 @@ public abstract class StorageProperties extends ConnectionProperties {
public static final String FS_HDFS_SUPPORT = "fs.hdfs.support";
public static final String FS_S3_SUPPORT = "fs.s3.support";
public static final String FS_GCS_SUPPORT = "fs.gcs.support";
+ public static final String FS_MINIO_SUPPORT = "fs.minio.support";
public static final String FS_AZURE_SUPPORT = "fs.azure.support";
public static final String FS_OSS_SUPPORT = "fs.oss.support";
public static final String FS_OBS_SUPPORT = "fs.obs.support";
@@ -50,7 +52,9 @@ public enum Type {
OSS,
OBS,
COS,
+ MINIO,
AZURE,
+ BROKER,
UNKNOWN
}
@@ -98,7 +102,7 @@ public static List createAll(Map origProps) t
* @return a StorageProperties instance for the primary storage type
* @throws RuntimeException if no supported storage type is found
*/
- public static StorageProperties createPrimary(Map origProps) throws UserException {
+ public static StorageProperties createPrimary(Map origProps) {
for (Function, StorageProperties> func : PROVIDERS) {
StorageProperties p = func.apply(origProps);
if (p != null) {
@@ -106,7 +110,7 @@ public static StorageProperties createPrimary(Map origProps) thr
return p;
}
}
- throw new RuntimeException("No supported storage type found.");
+ throw new StoragePropertiesException("No supported storage type found. Please check your configuration.");
}
private static final List, StorageProperties>> PROVIDERS =
@@ -125,7 +129,9 @@ public static StorageProperties createPrimary(Map origProps) thr
props -> (isFsSupport(props, FS_COS_SUPPORT)
|| COSProperties.guessIsMe(props)) ? new COSProperties(props) : null,
props -> (isFsSupport(props, FS_AZURE_SUPPORT)
- || AzureProperties.guessIsMe(props)) ? new AzureProperties(props) : null
+ || AzureProperties.guessIsMe(props)) ? new AzureProperties(props) : null,
+ props -> (isFsSupport(props, FS_MINIO_SUPPORT)
+ || MinioProperties.guessIsMe(props)) ? new MinioProperties(props) : null
);
protected StorageProperties(Type type, Map origProps) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/exception/StoragePropertiesException.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/exception/StoragePropertiesException.java
new file mode 100644
index 00000000000000..00d8c46411f097
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/exception/StoragePropertiesException.java
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/**
+ * Custom exception class for handling storage property-related errors.
+ * This exception class extends RuntimeException and is used to handle errors related
+ * to storage properties at runtime. It provides two constructors: one that accepts only
+ * an error message, and another that also accepts a cause exception.
+ */
+
+package org.apache.doris.datasource.property.storage.exception;
+
+public class StoragePropertiesException extends RuntimeException {
+
+ /**
+ * Constructor that initializes the exception with an error message.
+ *
+ * @param message The error message describing the reason for the exception.
+ */
+ public StoragePropertiesException(String message) {
+ super(message);
+ }
+
+ /**
+ * Constructor that initializes the exception with a message and a cause.
+ *
+ * @param message The error message describing the reason for the exception.
+ * @param cause The underlying cause of the exception, typically another Throwable.
+ */
+ public StoragePropertiesException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
index 5e65093036517e..82f42ee110546b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
@@ -115,7 +115,7 @@ protected boolean isFileStreamType() {
@Override
public Map getLocationProperties() {
- return tableValuedFunction.getLocationProperties();
+ return tableValuedFunction.getBackendConnectProperties();
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/FileSystemFactory.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/FileSystemFactory.java
index 4e418ac217e057..319188c91091d5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/FileSystemFactory.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/FileSystemFactory.java
@@ -17,8 +17,11 @@
package org.apache.doris.fsv2;
+import org.apache.doris.analysis.BrokerDesc;
+import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.property.storage.StorageProperties;
+import org.apache.doris.fsv2.remote.BrokerFileSystem;
import org.apache.doris.fsv2.remote.RemoteFileSystem;
import java.util.Map;
@@ -30,7 +33,33 @@ public static RemoteFileSystem get(Map properties) throws UserEx
return get(storageProperties);
}
+ public static RemoteFileSystem get(StorageBackend.StorageType storageType, Map properties)
+ throws UserException {
+ if (storageType.equals(StorageBackend.StorageType.BROKER)) {
+ return new BrokerFileSystem("broker", properties);
+ }
+ StorageProperties storageProperties = StorageProperties.createPrimary(properties);
+ return get(storageProperties);
+ }
+
public static RemoteFileSystem get(StorageProperties storageProperties) {
return StorageTypeMapper.create(storageProperties);
}
+
+ // This method is a temporary workaround for handling properties.
+ // It will be removed when broker properties are officially supported.
+ public static RemoteFileSystem get(String name, Map properties) {
+ return new BrokerFileSystem(name, properties);
+ }
+
+ public static RemoteFileSystem get(BrokerDesc brokerDesc) {
+ if (null != brokerDesc.getStorageProperties()) {
+ return get(brokerDesc.getStorageProperties());
+ }
+ if (null != brokerDesc.getStorageType()
+ && brokerDesc.getStorageType().equals(StorageBackend.StorageType.BROKER)) {
+ return new BrokerFileSystem(brokerDesc.getName(), brokerDesc.getProperties());
+ }
+ throw new RuntimeException("Unexpected storage type: " + brokerDesc.getStorageType());
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/StorageTypeMapper.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/StorageTypeMapper.java
index 10c8daec002e78..5683934d9606d9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/StorageTypeMapper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/StorageTypeMapper.java
@@ -20,6 +20,7 @@
import org.apache.doris.datasource.property.storage.AzureProperties;
import org.apache.doris.datasource.property.storage.COSProperties;
import org.apache.doris.datasource.property.storage.HdfsProperties;
+import org.apache.doris.datasource.property.storage.MinioProperties;
import org.apache.doris.datasource.property.storage.OBSProperties;
import org.apache.doris.datasource.property.storage.OSSHdfsProperties;
import org.apache.doris.datasource.property.storage.OSSProperties;
@@ -37,6 +38,7 @@ public enum StorageTypeMapper {
OSS(OSSProperties.class, S3FileSystem::new),
OBS(OBSProperties.class, S3FileSystem::new),
COS(COSProperties.class, S3FileSystem::new),
+ MINIO(MinioProperties.class, S3FileSystem::new),
AZURE(AzureProperties.class, AzureFileSystem::new),
S3(S3Properties.class, S3FileSystem::new),
HDFS(HdfsProperties.class, DFSFileSystem::new),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java
index 9d04a077b622ba..76c12a85834df3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java
@@ -56,11 +56,14 @@
import java.io.File;
import java.io.InputStream;
import java.net.URI;
+import java.nio.file.FileSystems;
+import java.nio.file.PathMatcher;
+import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
-import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class S3ObjStorage implements ObjStorage {
@@ -101,57 +104,98 @@ public S3Client getClient() throws UserException {
CloudCredential credential = new CloudCredential();
credential.setAccessKey(s3Properties.getAccessKey());
credential.setSecretKey(s3Properties.getSecretKey());
-
- /* if (properties.containsKey(S3Properties.SESSION_TOKEN)) {
- credential.setSessionToken(properties.get(S3Properties.SESSION_TOKEN));
- }*/
+ if (StringUtils.isNotBlank(s3Properties.getSessionToken())) {
+ credential.setSessionToken(s3Properties.getSessionToken());
+ }
client = S3Util.buildS3Client(endpoint, s3Properties.getRegion(), credential, isUsePathStyle);
}
return client;
}
-
-
public Status globList(String remotePath, List result, boolean fileNameOnly) {
+ long roundCnt = 0;
+ long elementCnt = 0;
+ long matchCnt = 0;
+ long startTime = System.nanoTime();
try {
- remotePath = s3Properties.validateAndNormalizeUri(remotePath);
- URI uri = new URI(remotePath);
- String bucketName = uri.getHost();
- String prefix = uri.getPath().substring(1);
- int wildcardIndex = prefix.indexOf('*');
- String searchPrefix = wildcardIndex > 0 ? prefix.substring(0, wildcardIndex) : prefix;
- try (S3Client s3 = getClient()) {
- ListObjectsV2Request listRequest = ListObjectsV2Request.builder()
- .bucket(bucketName)
- .prefix(searchPrefix)
- .build();
+ S3URI uri = S3URI.create(remotePath, isUsePathStyle, forceParsingByStandardUri);
+ String bucket = uri.getBucket();
+ String globPath = uri.getKey(); // eg: path/to/*.csv
+ if (LOG.isDebugEnabled()) {
+ LOG.info("globList globPath:{}, remotePath:{}", globPath, remotePath);
+ }
- ListObjectsV2Response listResponse = s3.listObjectsV2(listRequest);
- String regex = prefix.replace(".", "\\.")
- .replace("*", ".*")
- .replace("?", ".");
- Pattern pattern = Pattern.compile(regex);
- List matchedFiles = listResponse.contents().stream()
- .filter(obj -> pattern.matcher(obj.key()).matches())
- .map(obj -> {
- String fullKey = obj.key();
- String fullPath = "s3://" + bucketName + "/" + fullKey;
- return new RemoteFile(
- fileNameOnly ? fullPath.substring(fullPath.lastIndexOf('/') + 1) : fullPath,
- true,
- obj.size(),
- -1,
- obj.lastModified().toEpochMilli()
- );
- })
- .collect(Collectors.toList());
+ java.nio.file.Path pathPattern = Paths.get(globPath);
+ PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:" + pathPattern);
+ HashSet directorySet = new HashSet<>();
+ String listPrefix = S3Util.getLongestPrefix(globPath); // similar to Azure
+ if (LOG.isDebugEnabled()) {
+ LOG.info("globList listPrefix: {}", listPrefix);
+ }
+ ListObjectsV2Request request = ListObjectsV2Request.builder()
+ .bucket(bucket)
+ .prefix(listPrefix)
+ .build();
+ boolean isTruncated;
+ do {
+ roundCnt++;
+ ListObjectsV2Response response = getClient().listObjectsV2(request);
+ for (S3Object obj : response.contents()) {
+ elementCnt++;
+ java.nio.file.Path objPath = Paths.get(obj.key());
- result.addAll(matchedFiles);
+ boolean isPrefix = false;
+ while (objPath != null && objPath.normalize().toString().startsWith(listPrefix)) {
+ if (!matcher.matches(objPath)) {
+ isPrefix = true;
+ objPath = objPath.getParent();
+ continue;
+ }
+ if (directorySet.contains(objPath.normalize().toString())) {
+ break;
+ }
+ if (isPrefix) {
+ directorySet.add(objPath.normalize().toString());
+ }
+
+ matchCnt++;
+ RemoteFile remoteFile = new RemoteFile(
+ fileNameOnly ? objPath.getFileName().toString() :
+ "s3://" + bucket + "/" + objPath,
+ !isPrefix,
+ isPrefix ? -1 : obj.size(),
+ isPrefix ? -1 : obj.size(),
+ isPrefix ? 0 : obj.lastModified().toEpochMilli()
+ );
+ result.add(remoteFile);
+ objPath = objPath.getParent();
+ isPrefix = true;
+ }
+ }
+
+ isTruncated = response.isTruncated();
+ if (isTruncated) {
+ request = request.toBuilder()
+ .continuationToken(response.nextContinuationToken())
+ .build();
+ }
+ } while (isTruncated);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("remotePath:{}, result:{}", remotePath, result);
}
return Status.OK;
} catch (Exception e) {
LOG.warn("Errors while getting file status", e);
return new Status(Status.ErrCode.COMMON_ERROR, "Errors while getting file status " + e.getMessage());
+ } finally {
+ long endTime = System.nanoTime();
+ long duration = endTime - startTime;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("process {} elements under prefix {} for {} round, match {} elements, take {} ms",
+ elementCnt, remotePath, roundCnt, matchCnt,
+ duration / 1000);
+ }
}
}
@@ -167,7 +211,7 @@ public Status headObject(String remotePath) {
S3URI uri = S3URI.create(remotePath, isUsePathStyle, forceParsingByStandardUri);
HeadObjectResponse response = getClient()
.headObject(HeadObjectRequest.builder().bucket(uri.getBucket()).key(uri.getKey()).build());
- LOG.info("head file " + remotePath + " success: " + response.toString());
+ LOG.info("head file {} success: {}", remotePath, response);
return Status.OK;
} catch (S3Exception e) {
if (e.statusCode() == HttpStatus.SC_NOT_FOUND) {
@@ -189,7 +233,7 @@ public Status getObject(String remoteFilePath, File localFile) {
S3URI uri = S3URI.create(remoteFilePath, isUsePathStyle, forceParsingByStandardUri);
GetObjectResponse response = getClient().getObject(
GetObjectRequest.builder().bucket(uri.getBucket()).key(uri.getKey()).build(), localFile.toPath());
- LOG.info("get file " + remoteFilePath + " success: " + response.toString());
+ LOG.info("get file {} success: {}", remoteFilePath, response);
return Status.OK;
} catch (S3Exception s3Exception) {
return new Status(
@@ -214,13 +258,13 @@ public Status putObject(String remotePath, @Nullable InputStream content, long c
.putObject(
PutObjectRequest.builder().bucket(uri.getBucket()).key(uri.getKey()).build(),
body);
- LOG.info("put object success: " + response.toString());
+ LOG.info("put object success: {}", response.toString());
return Status.OK;
} catch (S3Exception e) {
- LOG.error("put object failed:", e);
+ LOG.warn("put object failed:", e);
return new Status(Status.ErrCode.COMMON_ERROR, "put object failed: " + e.getMessage());
} catch (Exception ue) {
- LOG.error("connect to s3 failed: ", ue);
+ LOG.warn("connect to s3 failed: ", ue);
return new Status(Status.ErrCode.COMMON_ERROR, "connect to s3 failed: " + ue.getMessage());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/BrokerFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/BrokerFileSystem.java
index b5398f0102e6ea..88e0f90fc1911d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/BrokerFileSystem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/BrokerFileSystem.java
@@ -27,6 +27,7 @@
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.BrokerUtil;
import org.apache.doris.datasource.property.PropertyConverter;
+import org.apache.doris.datasource.property.storage.BrokerProperties;
import org.apache.doris.fs.operations.BrokerFileOperations;
import org.apache.doris.fs.operations.OpParams;
import org.apache.doris.service.FrontendOptions;
@@ -81,6 +82,8 @@ public BrokerFileSystem(String name, Map properties) {
properties.putAll(PropertyConverter.convertToHadoopFSProperties(properties));
this.properties = properties;
this.operations = new BrokerFileOperations(name, properties);
+ // support broker properties in future
+ this.storageProperties = new BrokerProperties(properties);
}
public Pair getBroker() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/RemoteFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/RemoteFileSystem.java
index d418fe90b19e1c..c7877d9ed8db69 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/RemoteFileSystem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/RemoteFileSystem.java
@@ -42,7 +42,7 @@ public abstract class RemoteFileSystem extends PersistentFileSystem implements C
// this field will be visited by multi-threads, better use volatile qualifier
protected volatile org.apache.hadoop.fs.FileSystem dfsFileSystem = null;
private final ReentrantLock fsLock = new ReentrantLock();
- protected static final AtomicBoolean closed = new AtomicBoolean(false);
+ protected AtomicBoolean closed = new AtomicBoolean(false);
public RemoteFileSystem(String name, StorageBackend.StorageType type) {
super(name, type);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/dfs/DFSFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/dfs/DFSFileSystem.java
index 53fcc9ba17c5e1..8c708399eadb64 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/dfs/DFSFileSystem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/dfs/DFSFileSystem.java
@@ -90,6 +90,9 @@ public FileSystem nativeFileSystem(String remotePath) throws UserException {
}
if (dfsFileSystem == null) {
Configuration conf = hdfsProperties.getHadoopConfiguration();
+ // TODO: Temporarily disable the HDFS file system cache to prevent instances from being closed by
+ // each other in V1. This line can be removed once V1 and V2 are unified.
+ conf.set("fs.hdfs.impl.disable.cache", "true");
authenticator = HadoopAuthenticator.getHadoopAuthenticator(conf);
try {
dfsFileSystem = authenticator.doAs(() -> {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
index b9e1ca2ead31ef..434deaf440be08 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
@@ -597,7 +597,7 @@ private Map convertOutfileProperties() {
outfileProperties.put(BROKER_PROPERTY_PREFIXES + "name", brokerDesc.getName());
brokerDesc.getProperties().forEach((k, v) -> outfileProperties.put(BROKER_PROPERTY_PREFIXES + k, v));
} else {
- for (Entry kv : brokerDesc.getProperties().entrySet()) {
+ for (Entry kv : brokerDesc.getBackendConfigProperties().entrySet()) {
outfileProperties.put(kv.getKey(), kv.getValue());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
index 6ce45167a0bc15..a89435d357c00b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
@@ -952,7 +952,7 @@ private void initTBrokerScanRange(DescriptorTable descTable, TupleDescriptor des
// scan range params
TBrokerScanRangeParams params = new TBrokerScanRangeParams();
params.setStrictMode(false);
- params.setProperties(brokerDesc.getProperties());
+ params.setProperties(brokerDesc.getBackendConfigProperties());
TupleDescriptor srcTupleDesc = descTable.createTupleDescriptor();
Map srcSlotDescByName = Maps.newHashMap();
for (Column column : columns) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java
index ac494398ac94f6..153c69bbd7ff46 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java
@@ -170,7 +170,7 @@ public TFileScanRangeParams toFileScanRangeParams(TUniqueId loadId, NereidsFileG
params.addToRequiredSlots(slotInfo);
}
- Map properties = fileGroupInfo.getBrokerDesc().getProperties();
+ Map properties = fileGroupInfo.getBrokerDesc().getBackendConfigProperties();
if (fileGroupInfo.getBrokerDesc().getFileType() == TFileType.FILE_HDFS) {
THdfsParams tHdfsParams = HdfsResource.generateHdfsParam(properties);
params.setHdfsParams(tHdfsParams);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java
index 3b810d72172057..78fe220fa23556 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java
@@ -30,7 +30,6 @@
import org.apache.doris.common.util.FileFormatConstants;
import org.apache.doris.common.util.FileFormatUtils;
import org.apache.doris.common.util.TimeUtils;
-import org.apache.doris.datasource.property.constants.S3Properties;
import org.apache.doris.job.base.JobExecuteType;
import org.apache.doris.job.base.JobExecutionConfiguration;
import org.apache.doris.job.extensions.insert.InsertJob;
@@ -469,10 +468,8 @@ private static Map getTvfProperties(BulkLoadDataDesc dataDesc, B
// TODO: support multi location by union
String listFilePath = filePaths.get(0);
if (bulkStorageDesc.getStorageType() == BulkStorageDesc.StorageType.S3) {
- S3Properties.convertToStdProperties(tvfProperties);
- tvfProperties.keySet().removeIf(S3Properties.Env.FS_KEYS::contains);
// TODO: check file path by s3 fs list status
- tvfProperties.put(S3TableValuedFunction.PROP_URI, listFilePath);
+ tvfProperties.put("uri", listFilePath);
}
final Map dataDescProps = dataDesc.getProperties();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/alter/AlterRepositoryCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/alter/AlterRepositoryCommand.java
index f97109076f8d77..d74c25b4708746 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/alter/AlterRepositoryCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/alter/AlterRepositoryCommand.java
@@ -73,8 +73,7 @@ public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
throw new UserException("alter repository only support ak/sk/token info of s3."
+ " unsupported properties: " + copyProperties.keySet());
}
-
- Env.getCurrentEnv().getBackupHandler().alterRepositoryInternal(name, properties);
+ Env.getCurrentEnv().getBackupHandler(). alterRepository(name, properties, true);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExportSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExportSink.java
index 0fd2535b62bff7..9f307a376b2f01 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExportSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExportSink.java
@@ -82,7 +82,7 @@ protected TDataSink toThrift() {
tExportSink.addToBrokerAddresses(new TNetworkAddress(broker.host, broker.port));
}
}
- tExportSink.setProperties(brokerDesc.getProperties());
+ tExportSink.setProperties(brokerDesc.getStorageProperties().getBackendConfigProperties());
tExportSink.setHeader(header);
result.setExportSink(tExportSink);
return result;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
index 6d3475dbebed03..d23e10739d241a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
@@ -435,7 +435,9 @@ public static void execute(Env env, DdlStmt ddlStmt) throws Exception {
DropAnalyzeJobStmt analyzeJobStmt = (DropAnalyzeJobStmt) ddlStmt;
Env.getCurrentEnv().getAnalysisManager().dropAnalyzeJob(analyzeJobStmt);
} else if (ddlStmt instanceof AlterRepositoryStmt) {
- env.getBackupHandler().alterRepository((AlterRepositoryStmt) ddlStmt);
+ AlterRepositoryStmt alterRepositoryStmt = (AlterRepositoryStmt) ddlStmt;
+ env.getBackupHandler().alterRepository(alterRepositoryStmt.getName(), alterRepositoryStmt.getProperties(),
+ false);
} else if (ddlStmt instanceof CreateStorageVaultStmt) {
env.getStorageVaultMgr().createStorageVaultResource((CreateStorageVaultStmt) ddlStmt);
} else if (ddlStmt instanceof CreateStageStmt) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
index d47b7f082569d2..c25263cee25d8f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
@@ -42,6 +42,7 @@
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties;
import org.apache.doris.datasource.property.fileformat.FileFormatProperties;
+import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.datasource.tvf.source.TVFScanNode;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.planner.PlanNodeId;
@@ -81,6 +82,7 @@
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -93,6 +95,7 @@
*/
public abstract class ExternalFileTableValuedFunction extends TableValuedFunctionIf {
public static final Logger LOG = LogManager.getLogger(ExternalFileTableValuedFunction.class);
+ protected static final String URI_KEY = "uri";
public static final String PROP_TABLE_ID = "table_id";
@@ -105,7 +108,10 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio
private List pathPartitionKeys;
protected List fileStatuses = Lists.newArrayList();
- protected Map locationProperties = Maps.newHashMap();
+ protected Map backendConnectProperties = Maps.newHashMap();
+ protected StorageProperties storageProperties;
+ // Processed parameters derived from user input; includes normalization and default value filling.
+ Map processedParams;
protected String filePath;
protected Optional resourceName = Optional.empty();
@@ -127,8 +133,8 @@ public TFileCompressType getTFileCompressType() {
return fileFormatProperties.getCompressionType();
}
- public Map getLocationProperties() {
- return locationProperties;
+ public Map getBackendConnectProperties() {
+ return backendConnectProperties;
}
public List getPathPartitionKeys() {
@@ -181,7 +187,7 @@ protected Map parseCommonProperties(Map properti
.map(String::trim)
.collect(Collectors.toList()))
.orElse(Lists.newArrayList());
-
+ this.processedParams = new HashMap<>(copiedProps);
return copiedProps;
}
@@ -370,7 +376,9 @@ private PFetchTableSchemaRequest getFetchTableStructureRequest() throws TExcepti
// set TFileScanRangeParams
TFileScanRangeParams fileScanRangeParams = new TFileScanRangeParams();
fileScanRangeParams.setFormatType(fileFormatProperties.getFileFormatType());
- fileScanRangeParams.setProperties(locationProperties);
+ Map beProperties = new HashMap<>();
+ beProperties.putAll(backendConnectProperties);
+ fileScanRangeParams.setProperties(beProperties);
if (fileFormatProperties instanceof CsvFileFormatProperties) {
fileScanRangeParams.setTextSerdeType(((CsvFileFormatProperties) fileFormatProperties).getTextSerdeType());
}
@@ -384,8 +392,8 @@ private PFetchTableSchemaRequest getFetchTableStructureRequest() throws TExcepti
}
if (getTFileType() == TFileType.FILE_HDFS) {
- THdfsParams tHdfsParams = HdfsResource.generateHdfsParam(locationProperties);
- String fsName = getLocationProperties().get(HdfsResource.HADOOP_FS_NAME);
+ THdfsParams tHdfsParams = HdfsResource.generateHdfsParam(storageProperties.getBackendConfigProperties());
+ String fsName = storageProperties.getBackendConfigProperties().get(HdfsResource.HADOOP_FS_NAME);
tHdfsParams.setFsName(fsName);
fileScanRangeParams.setHdfsParams(tHdfsParams);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/GroupCommitTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/GroupCommitTableValuedFunction.java
index a387902e94acac..bb2dd5f9e3dae9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/GroupCommitTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/GroupCommitTableValuedFunction.java
@@ -106,7 +106,7 @@ public String getFilePath() {
@Override
public BrokerDesc getBrokerDesc() {
- return new BrokerDesc("GroupCommitTvfBroker", StorageType.STREAM, locationProperties);
+ return new BrokerDesc("GroupCommitTvfBroker", StorageType.STREAM, processedParams);
}
// =========== implement abstract methods of TableValuedFunctionIf =================
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
index 80149e3d1380ae..e99340eb105bec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
@@ -18,15 +18,13 @@
package org.apache.doris.tablefunction;
import org.apache.doris.analysis.BrokerDesc;
-import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.analysis.StorageBackend.StorageType;
-import org.apache.doris.catalog.HdfsResource;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
-import org.apache.doris.common.util.URI;
+import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.thrift.TFileType;
-import com.google.common.base.Strings;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -39,7 +37,6 @@
public class HdfsTableValuedFunction extends ExternalFileTableValuedFunction {
public static final Logger LOG = LogManager.getLogger(HdfsTableValuedFunction.class);
public static final String NAME = "hdfs";
- private static final String PROP_URI = "uri";
public HdfsTableValuedFunction(Map properties) throws AnalysisException {
init(properties);
@@ -47,28 +44,15 @@ public HdfsTableValuedFunction(Map properties) throws AnalysisEx
private void init(Map properties) throws AnalysisException {
// 1. analyze common properties
- Map otherProps = super.parseCommonProperties(properties);
-
+ Map props = super.parseCommonProperties(properties);
// 2. analyze uri
- String uriStr = getOrDefaultAndRemove(otherProps, PROP_URI, null);
- if (Strings.isNullOrEmpty(uriStr)) {
- throw new AnalysisException(String.format("Properties '%s' is required.", PROP_URI));
- }
- URI uri = URI.create(uriStr);
- StorageBackend.checkUri(uri, StorageType.HDFS);
- filePath = uri.getScheme() + "://" + uri.getAuthority() + uri.getPath();
-
- // 3. analyze other properties
- for (String key : otherProps.keySet()) {
- if (HdfsResource.HADOOP_FS_NAME.equalsIgnoreCase(key)) {
- locationProperties.put(HdfsResource.HADOOP_FS_NAME, otherProps.get(key));
- } else {
- locationProperties.put(key, otherProps.get(key));
- }
- }
- // If the user does not specify the HADOOP_FS_NAME, we will use the uri's scheme and authority
- if (!locationProperties.containsKey(HdfsResource.HADOOP_FS_NAME)) {
- locationProperties.put(HdfsResource.HADOOP_FS_NAME, uri.getScheme() + "://" + uri.getAuthority());
+ try {
+ this.storageProperties = StorageProperties.createPrimary(props);
+ backendConnectProperties.putAll(storageProperties.getBackendConfigProperties());
+ String uri = storageProperties.validateAndGetUri(props);
+ filePath = storageProperties.validateAndNormalizeUri(uri);
+ } catch (UserException e) {
+ throw new AnalysisException("Failed check storage props, " + e.getMessage(), e);
}
if (!FeConstants.runningUnitTest) {
@@ -91,7 +75,7 @@ public String getFilePath() {
@Override
public BrokerDesc getBrokerDesc() {
- return new BrokerDesc("HdfsTvfBroker", StorageType.HDFS, locationProperties);
+ return new BrokerDesc("HdfsTvfBroker", StorageType.HDFS, processedParams);
}
// =========== implement abstract methods of TableValuedFunctionIf =================
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java
index 72573a2355f451..c473e174189975 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java
@@ -60,7 +60,7 @@ public String getFilePath() {
@Override
public BrokerDesc getBrokerDesc() {
- return new BrokerDesc("HttpStreamTvfBroker", StorageType.STREAM, locationProperties);
+ return new BrokerDesc("HttpStreamTvfBroker", StorageType.STREAM, processedParams);
}
// =========== implement abstract methods of TableValuedFunctionIf =================
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java
index 32f76b233fad68..c82552f1862a14 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java
@@ -141,7 +141,7 @@ public String getFilePath() {
@Override
public BrokerDesc getBrokerDesc() {
- return new BrokerDesc("LocalTvfBroker", StorageType.LOCAL, locationProperties);
+ return new BrokerDesc("LocalTvfBroker", StorageType.LOCAL, processedParams);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
index 93dc28d84a35b2..7d38c50f565ac2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
@@ -18,22 +18,12 @@
package org.apache.doris.tablefunction;
import org.apache.doris.analysis.BrokerDesc;
-import org.apache.doris.analysis.StorageBackend.StorageType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.UserException;
-import org.apache.doris.common.credentials.CloudCredentialWithEndpoint;
-import org.apache.doris.common.util.S3URI;
-import org.apache.doris.datasource.property.PropertyConverter;
-import org.apache.doris.datasource.property.S3ClientBEProperties;
-import org.apache.doris.datasource.property.constants.AzureProperties;
-import org.apache.doris.datasource.property.constants.S3Properties;
-import org.apache.doris.fs.FileSystemFactory;
+import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.thrift.TFileType;
-import com.google.common.base.Strings;
-import com.google.common.collect.ImmutableSet;
-
import java.util.Map;
/**
@@ -49,120 +39,28 @@
*/
public class S3TableValuedFunction extends ExternalFileTableValuedFunction {
public static final String NAME = "s3";
- public static final String PROP_URI = "uri";
-
- private static final ImmutableSet DEPRECATED_KEYS =
- ImmutableSet.of("access_key", "secret_key", "session_token", "region",
- "ACCESS_KEY", "SECRET_KEY", "SESSION_TOKEN", "REGION");
public S3TableValuedFunction(Map properties) throws AnalysisException {
// 1. analyze common properties
- Map otherProps = super.parseCommonProperties(properties);
-
- // 2. analyze uri and other properties
- String uriStr = getOrDefaultAndRemove(otherProps, PROP_URI, null);
- if (Strings.isNullOrEmpty(uriStr)) {
- throw new AnalysisException(String.format("Properties '%s' is required.", PROP_URI));
- }
- forwardCompatibleDeprecatedKeys(otherProps);
-
- String usePathStyle = getOrDefaultAndRemove(otherProps, PropertyConverter.USE_PATH_STYLE,
- PropertyConverter.USE_PATH_STYLE_DEFAULT_VALUE);
- String forceParsingByStandardUri = getOrDefaultAndRemove(otherProps,
- PropertyConverter.FORCE_PARSING_BY_STANDARD_URI,
- PropertyConverter.FORCE_PARSING_BY_STANDARD_URI_DEFAULT_VALUE);
-
- S3URI s3uri = getS3Uri(uriStr, Boolean.parseBoolean(usePathStyle.toLowerCase()),
- Boolean.parseBoolean(forceParsingByStandardUri.toLowerCase()));
-
- // get endpoint first from properties, if not present, get it from s3 uri.
- // If endpoint is missing, exception will be thrown.
- String endpoint = constructEndpoint(otherProps, s3uri);
- if (!otherProps.containsKey(S3Properties.REGION)) {
- String region;
- if (AzureProperties.checkAzureProviderPropertyExist(properties)) {
- // Azure could run without region
- region = s3uri.getRegion().orElse("DUMMY-REGION");
- } else {
- region = s3uri.getRegion().orElseThrow(() -> new AnalysisException(
- String.format("Properties '%s' is required.", S3Properties.REGION)));
- }
- otherProps.put(S3Properties.REGION, region);
- }
- checkNecessaryS3Properties(otherProps);
- CloudCredentialWithEndpoint credential = new CloudCredentialWithEndpoint(endpoint,
- getOrDefaultAndRemove(otherProps, S3Properties.REGION, ""),
- getOrDefaultAndRemove(otherProps, S3Properties.ACCESS_KEY, ""),
- getOrDefaultAndRemove(otherProps, S3Properties.SECRET_KEY, ""));
- if (otherProps.containsKey(S3Properties.SESSION_TOKEN)) {
- credential.setSessionToken(getOrDefaultAndRemove(otherProps, S3Properties.SESSION_TOKEN, ""));
- }
-
- locationProperties = S3Properties.credentialToMap(credential);
- locationProperties.put(PropertyConverter.USE_PATH_STYLE, usePathStyle);
- if (AzureProperties.checkAzureProviderPropertyExist(properties)) {
- // For Azure's compatibility, we need bucket to connect to the blob storage's container
- locationProperties.put(S3Properties.BUCKET, s3uri.getBucket());
- }
-
- if (properties.containsKey(S3Properties.ROLE_ARN)) {
- locationProperties.put(S3Properties.ROLE_ARN, properties.get(S3Properties.ROLE_ARN));
- if (properties.containsKey(S3Properties.EXTERNAL_ID)) {
- locationProperties.put(S3Properties.EXTERNAL_ID, properties.get(S3Properties.EXTERNAL_ID));
- }
+ Map props = super.parseCommonProperties(properties);
+ try {
+ this.storageProperties = StorageProperties.createPrimary(props);
+ this.backendConnectProperties.putAll(storageProperties.getBackendConfigProperties());
+ String uri = storageProperties.validateAndGetUri(props);
+ filePath = storageProperties.validateAndNormalizeUri(uri);
+ this.backendConnectProperties.put(URI_KEY, filePath);
+ } catch (UserException e) {
+ throw new RuntimeException(e);
}
-
- locationProperties.putAll(S3ClientBEProperties.getBeFSProperties(locationProperties));
- locationProperties.putAll(otherProps);
-
- filePath = NAME + S3URI.SCHEME_DELIM + s3uri.getBucket() + S3URI.PATH_DELIM + s3uri.getKey();
-
if (FeConstants.runningUnitTest) {
// Just check
- FileSystemFactory.getS3FileSystem(locationProperties);
+ // Fixme wait to be done #50320
+ // FileSystemFactory.get(storageProperties);
} else {
parseFile();
}
}
- private String constructEndpoint(Map properties, S3URI s3uri) throws AnalysisException {
- // get endpoint first from properties, if not present, get it from s3 uri.
- String endpoint = getOrDefaultAndRemove(properties, S3Properties.ENDPOINT, s3uri.getEndpoint().orElse(""));
- if (AzureProperties.checkAzureProviderPropertyExist(properties)) {
- String accountName = properties.getOrDefault(S3Properties.ACCESS_KEY, "");
- if (accountName.isEmpty()) {
- throw new AnalysisException(String.format("Properties '%s' is required.", S3Properties.ACCESS_KEY));
- }
- endpoint = AzureProperties.formatAzureEndpoint(endpoint, accountName);
- } else if (Strings.isNullOrEmpty(endpoint)) {
- throw new AnalysisException(String.format("Properties '%s' is required.", S3Properties.ENDPOINT));
- }
- return endpoint;
- }
-
- private void forwardCompatibleDeprecatedKeys(Map props) {
- for (String deprecatedKey : DEPRECATED_KEYS) {
- String value = props.remove(deprecatedKey);
- if (!Strings.isNullOrEmpty(value)) {
- props.put("s3." + deprecatedKey.toLowerCase(), value);
- }
- }
- }
-
- private void checkNecessaryS3Properties(Map props) throws AnalysisException {
- if (Strings.isNullOrEmpty(props.get(S3Properties.REGION))) {
- throw new AnalysisException(String.format("Properties '%s' is required.", S3Properties.REGION));
- }
- // do not check ak and sk, because we can read them from system environment.
- }
-
- private S3URI getS3Uri(String uri, boolean isPathStyle, boolean forceParsingStandardUri) throws AnalysisException {
- try {
- return S3URI.create(uri, isPathStyle, forceParsingStandardUri);
- } catch (UserException e) {
- throw new AnalysisException("parse s3 uri failed, uri = " + uri, e);
- }
- }
// =========== implement abstract methods of ExternalFileTableValuedFunction =================
@Override
@@ -178,7 +76,7 @@ public String getFilePath() {
@Override
public BrokerDesc getBrokerDesc() {
- return new BrokerDesc("S3TvfBroker", StorageType.S3, locationProperties);
+ return new BrokerDesc("S3TvfBroker", processedParams);
}
// =========== implement abstract methods of TableValuedFunctionIf =================
diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java
index cbee4a8bcecd5b..a8c2960868adde 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java
@@ -18,7 +18,6 @@
package org.apache.doris.backup;
import org.apache.doris.analysis.BackupStmt;
-import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.analysis.TableName;
import org.apache.doris.analysis.TableRef;
import org.apache.doris.backup.BackupJob.BackupJobState;
@@ -29,10 +28,11 @@
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.common.util.UnitTestUtil;
import org.apache.doris.datasource.InternalCatalog;
-import org.apache.doris.fs.FileSystemFactory;
+import org.apache.doris.fsv2.FileSystemFactory;
import org.apache.doris.persist.EditLog;
import org.apache.doris.task.AgentBatchTask;
import org.apache.doris.task.AgentTask;
@@ -96,6 +96,9 @@ public class BackupJobTest {
private MockRepositoryMgr repoMgr;
+ public BackupJobTest() throws UserException {
+ }
+
// Thread is not mockable in Jmockit, use subclass instead
private final class MockBackupHandler extends BackupHandler {
public MockBackupHandler(Env env) {
@@ -124,7 +127,7 @@ public Repository getRepo(long repoId) {
private EditLog editLog;
private Repository repo = new Repository(repoId, "repo", false, "my_repo",
- FileSystemFactory.get("broker", StorageBackend.StorageType.BROKER, Maps.newHashMap()));
+ FileSystemFactory.get("broker", Maps.newHashMap()), null);
@BeforeClass
public static void start() {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java
index a49d7e4328e94b..c8fec2b64033f6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java
@@ -18,13 +18,13 @@
package org.apache.doris.backup;
import org.apache.doris.analysis.ShowRepositoriesStmt;
-import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.catalog.BrokerMgr;
import org.apache.doris.catalog.FsBroker;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.fs.FileSystemFactory;
-import org.apache.doris.fs.remote.RemoteFile;
-import org.apache.doris.fs.remote.RemoteFileSystem;
+import org.apache.doris.common.UserException;
+import org.apache.doris.fsv2.FileSystemFactory;
+import org.apache.doris.fsv2.remote.RemoteFile;
+import org.apache.doris.fsv2.remote.RemoteFileSystem;
import org.apache.doris.service.FrontendOptions;
import com.google.common.collect.Lists;
@@ -36,6 +36,7 @@
import mockit.Mocked;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import java.io.DataInputStream;
@@ -80,6 +81,13 @@ String getLocalHostAddress() {
return "127.0.0.1";
}
};
+ new Expectations() {
+ {
+ fileSystem.getStorageProperties();
+ minTimes = 0;
+ result = null;
+ }
+ };
new MockUp() {
@Mock
@@ -92,7 +100,7 @@ public FsBroker getBroker(String name, String host) throws AnalysisException {
@Test
public void testGet() {
- repo = new Repository(10000, "repo", false, location, fileSystem);
+ repo = new Repository(10000, "repo", false, location, fileSystem, null);
Assert.assertEquals(repoId, repo.getId());
Assert.assertEquals(name, repo.getName());
@@ -103,7 +111,7 @@ public void testGet() {
}
@Test
- public void testInit() {
+ public void testInit() throws UserException {
new Expectations() {
{
fileSystem.globList(anyString, (List) any);
@@ -114,14 +122,13 @@ public Status list(String remotePath, List result) {
return Status.OK;
}
};
-
fileSystem.directUpload(anyString, anyString);
minTimes = 0;
result = Status.OK;
}
};
- repo = new Repository(10000, "repo", false, location, fileSystem);
+ repo = new Repository(10000, "repo", false, location, fileSystem, null);
Status st = repo.initRepository();
System.out.println(st);
@@ -130,7 +137,7 @@ public Status list(String remotePath, List result) {
@Test
public void testassemnblePath() throws MalformedURLException, URISyntaxException {
- repo = new Repository(10000, "repo", false, location, fileSystem);
+ repo = new Repository(10000, "repo", false, location, fileSystem, null);
// job info
String label = "label";
@@ -171,7 +178,7 @@ public void testPing() {
}
};
- repo = new Repository(10000, "repo", false, location, fileSystem);
+ repo = new Repository(10000, "repo", false, location, fileSystem, null);
Assert.assertTrue(repo.ping());
Assert.assertTrue(repo.getErrorMsg() == null);
}
@@ -192,7 +199,7 @@ public Status list(String remotePath, List result) {
}
};
- repo = new Repository(10000, "repo", false, location, fileSystem);
+ repo = new Repository(10000, "repo", false, location, fileSystem, null);
List snapshotNames = Lists.newArrayList();
Status st = repo.listSnapshots(snapshotNames);
Assert.assertTrue(st.ok());
@@ -218,7 +225,7 @@ public void testUpload() {
}
};
- repo = new Repository(10000, "repo", false, location, fileSystem);
+ repo = new Repository(10000, "repo", false, location, fileSystem, null);
String localFilePath = "./tmp_" + System.currentTimeMillis();
try (PrintWriter out = new PrintWriter(localFilePath)) {
out.print("a");
@@ -265,7 +272,7 @@ public Status list(String remotePath, List result) {
}
};
- repo = new Repository(10000, "repo", false, location, fileSystem);
+ repo = new Repository(10000, "repo", false, location, fileSystem, null);
String remoteFilePath = location + "/remote_file";
Status st = repo.download(remoteFilePath, localFilePath);
Assert.assertTrue(st.ok());
@@ -276,7 +283,7 @@ public Status list(String remotePath, List result) {
@Test
public void testGetInfo() {
- repo = new Repository(10000, "repo", false, location, fileSystem);
+ repo = new Repository(10000, "repo", false, location, fileSystem, null);
List infos = repo.getInfo();
Assert.assertTrue(infos.size() == ShowRepositoriesStmt.TITLE_NAMES.size());
}
@@ -304,7 +311,7 @@ public Status list(String remotePath, List result) {
}
};
- repo = new Repository(10000, "repo", false, location, fileSystem);
+ repo = new Repository(10000, "repo", false, location, fileSystem, null);
String snapshotName = "";
String timestamp = "";
try {
@@ -317,14 +324,15 @@ public Status list(String remotePath, List result) {
}
}
+ @Ignore("wait support")
@Test
- public void testPersist() {
+ public void testPersist() throws UserException {
Map properties = Maps.newHashMap();
properties.put("bos_endpoint", "http://gz.bcebos.com");
properties.put("bos_accesskey", "a");
properties.put("bos_secret_accesskey", "b");
- RemoteFileSystem fs = FileSystemFactory.get(brokerName, StorageBackend.StorageType.BROKER, properties);
- repo = new Repository(10000, "repo", false, location, fs);
+ RemoteFileSystem fs = FileSystemFactory.get(properties);
+ repo = new Repository(10000, "repo", false, location, fs, null);
File file = new File("./Repository");
try {
@@ -352,8 +360,9 @@ public void testPersist() {
@Test
public void testPathNormalize() {
+
String newLoc = "bos://cmy_bucket/bos_repo/";
- repo = new Repository(10000, "repo", false, newLoc, fileSystem);
+ repo = new Repository(10000, "repo", false, newLoc, fileSystem, null);
String path = repo.getRepoPath("label1", "/_ss_my_ss/_ss_content/__db_10000/");
Assert.assertEquals("bos://cmy_bucket/bos_repo/__palo_repository_repo/__ss_label1/__ss_content/_ss_my_ss/_ss_content/__db_10000/", path);
@@ -361,7 +370,7 @@ public void testPathNormalize() {
Assert.assertEquals("bos://cmy_bucket/bos_repo/__palo_repository_repo/__ss_label1/__ss_content/_ss_my_ss/_ss_content/__db_10000", path);
newLoc = "hdfs://path/to/repo";
- repo = new Repository(10000, "repo", false, newLoc, fileSystem);
+ repo = new Repository(10000, "repo", false, newLoc, fileSystem, null);
SnapshotInfo snapshotInfo = new SnapshotInfo(1, 2, 3, 4, 5, 6, 7, "/path", Lists.newArrayList());
path = repo.getRepoTabletPathBySnapshotInfo("label1", snapshotInfo);
Assert.assertEquals("hdfs://path/to/repo/__palo_repository_repo/__ss_label1/__ss_content/__db_1/__tbl_2/__part_3/__idx_4/__5", path);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
index d47be652861c44..db3c7944dd91d3 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
@@ -17,7 +17,6 @@
package org.apache.doris.backup;
-import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.backup.BackupJobInfo.BackupIndexInfo;
import org.apache.doris.backup.BackupJobInfo.BackupOlapTableInfo;
import org.apache.doris.backup.BackupJobInfo.BackupPartitionInfo;
@@ -38,9 +37,10 @@
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.MarkedCountDownLatch;
+import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.datasource.InternalCatalog;
-import org.apache.doris.fs.FileSystemFactory;
+import org.apache.doris.fsv2.FileSystemFactory;
import org.apache.doris.persist.EditLog;
import org.apache.doris.resource.Tag;
import org.apache.doris.system.SystemInfoService;
@@ -91,6 +91,9 @@ public class RestoreJobTest {
private MockRepositoryMgr repoMgr;
+ public RestoreJobTest() throws UserException {
+ }
+
// Thread is not mockable in Jmockit, use subclass instead
private final class MockBackupHandler extends BackupHandler {
public MockBackupHandler(Env env) {
@@ -122,7 +125,7 @@ public Repository getRepo(long repoId) {
@Injectable
private Repository repo = new Repository(repoId, "repo", false, "bos://my_repo",
- FileSystemFactory.get("broker", StorageBackend.StorageType.BROKER, Maps.newHashMap()));
+ FileSystemFactory.get("broker", Maps.newHashMap()), null);
private BackupMeta backupMeta;
@@ -245,7 +248,7 @@ boolean await(long timeout, TimeUnit unit) {
for (Tablet tablet : index.getTablets()) {
List files = Lists.newArrayList(tablet.getId() + ".dat",
- tablet.getId() + ".idx", tablet.getId() + ".hdr");
+ tablet.getId() + ".idx", tablet.getId() + ".hdr");
BackupTabletInfo tabletInfo = new BackupTabletInfo(tablet.getId(), files);
idxInfo.sortedTabletInfoList.add(tabletInfo);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
index ce8158d1bd6efd..b7912e449faa10 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
@@ -58,6 +58,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
@@ -93,10 +94,10 @@ public void testOutFileS3PropertiesConverter() throws Exception {
+ "into outfile 's3://bucket/mock_dir'\n"
+ "format as csv\n"
+ "properties(\n"
- + " 'AWS_ENDPOINT' = 'http://127.0.0.1:9000',\n"
+ + " 'AWS_ENDPOINT' = 's3.ap-northeast-1.amazonaws.com',\n"
+ " 'AWS_ACCESS_KEY' = 'akk',\n"
+ " 'AWS_SECRET_KEY'='akk',\n"
- + " 'AWS_REGION' = 'mock',\n"
+ + " 'AWS_REGION' = 'ap-northeast-1',\n"
+ " 'use_path_style' = 'true'\n"
+ ");";
QueryStmt analyzedOutStmt = createStmt(query);
@@ -112,7 +113,7 @@ public void testOutFileS3PropertiesConverter() throws Exception {
+ "into outfile 's3://bucket/mock_dir'\n"
+ "format as csv\n"
+ "properties(\n"
- + " 's3.endpoint' = 'http://127.0.0.1:9000',\n"
+ + " 's3.endpoint' = 'https://s3.ap-northeast-1.amazonaws.com',\n"
+ " 's3.access_key' = 'akk',\n"
+ " 's3.secret_key'='akk',\n"
+ " 'use_path_style' = 'true'\n"
@@ -181,7 +182,7 @@ public void testS3RepositoryPropertiesConverter() throws Exception {
CreateRepositoryStmt analyzedStmt = createStmt(s3Repo);
Assertions.assertEquals(analyzedStmt.getProperties().size(), 4);
Repository repository = getRepository(analyzedStmt, "s3_repo");
- Assertions.assertEquals(9, repository.getRemoteFileSystem().getProperties().size());
+ Assertions.assertEquals(4, repository.getRemoteFileSystem().getProperties().size());
String s3RepoNew = "CREATE REPOSITORY `s3_repo_new`\n"
+ "WITH S3\n"
@@ -195,7 +196,7 @@ public void testS3RepositoryPropertiesConverter() throws Exception {
CreateRepositoryStmt analyzedStmtNew = createStmt(s3RepoNew);
Assertions.assertEquals(analyzedStmtNew.getProperties().size(), 3);
Repository repositoryNew = getRepository(analyzedStmtNew, "s3_repo_new");
- Assertions.assertEquals(repositoryNew.getRemoteFileSystem().getProperties().size(), 5);
+ Assertions.assertEquals(repositoryNew.getRemoteFileSystem().getProperties().size(), 3);
}
private static Repository getRepository(CreateRepositoryStmt analyzedStmt, String name) throws DdlException {
@@ -203,6 +204,7 @@ private static Repository getRepository(CreateRepositoryStmt analyzedStmt, Strin
return Env.getCurrentEnv().getBackupHandler().getRepoMgr().getRepo(name);
}
+ @Disabled("not support")
@Test
public void testBosBrokerRepositoryPropertiesConverter() throws Exception {
FeConstants.runningUnitTest = true;
@@ -241,7 +243,7 @@ public void testS3TVFPropertiesConverter() throws Exception {
Assertions.assertEquals(analyzedStmt.getTableRefs().size(), 1);
TableValuedFunctionRef oldFuncTable = (TableValuedFunctionRef) analyzedStmt.getTableRefs().get(0);
S3TableValuedFunction s3Tvf = (S3TableValuedFunction) oldFuncTable.getTableFunction();
- Assertions.assertEquals(10, s3Tvf.getBrokerDesc().getProperties().size());
+ Assertions.assertEquals(5, s3Tvf.getBrokerDesc().getProperties().size());
String queryNew = "select * from s3(\n"
+ " 'uri' = 'http://s3.us-east-1.amazonaws.com/my-bucket/test.parquet',\n"
@@ -254,7 +256,7 @@ public void testS3TVFPropertiesConverter() throws Exception {
Assertions.assertEquals(analyzedStmtNew.getTableRefs().size(), 1);
TableValuedFunctionRef newFuncTable = (TableValuedFunctionRef) analyzedStmt.getTableRefs().get(0);
S3TableValuedFunction newS3Tvf = (S3TableValuedFunction) newFuncTable.getTableFunction();
- Assertions.assertEquals(10, newS3Tvf.getBrokerDesc().getProperties().size());
+ Assertions.assertEquals(5, newS3Tvf.getBrokerDesc().getProperties().size());
}
@Test
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java
index 32a212c5cf7bdc..1f953f6aefc604 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java
@@ -45,8 +45,8 @@ public void testS3TVFPropertiesPassThrough() throws Exception {
Assertions.assertEquals(analyzedStmt.getTableRefs().size(), 1);
TableValuedFunctionRef oldFuncTable = (TableValuedFunctionRef) analyzedStmt.getTableRefs().get(0);
S3TableValuedFunction s3Tvf = (S3TableValuedFunction) oldFuncTable.getTableFunction();
- Assertions.assertTrue(s3Tvf.getBrokerDesc().getProperties().containsKey("fs.s3a.list.version"));
- Assertions.assertTrue(s3Tvf.getBrokerDesc().getProperties().containsKey("test_property"));
+ Assertions.assertFalse(s3Tvf.getBrokerDesc().getBackendConfigProperties().containsKey("fs.s3a.list.version"));
+ Assertions.assertFalse(s3Tvf.getBrokerDesc().getBackendConfigProperties().containsKey("test_property"));
}
@Test
@@ -63,6 +63,6 @@ public void testHdfsTVFPropertiesPassThrough() throws Exception {
Assertions.assertEquals(analyzedStmt.getTableRefs().size(), 1);
TableValuedFunctionRef oldFuncTable = (TableValuedFunctionRef) analyzedStmt.getTableRefs().get(0);
HdfsTableValuedFunction hdfsTvf = (HdfsTableValuedFunction) oldFuncTable.getTableFunction();
- Assertions.assertTrue(hdfsTvf.getBrokerDesc().getProperties().containsKey("test_property"));
+ Assertions.assertFalse(hdfsTvf.getBrokerDesc().getBackendConfigProperties().containsKey("test_property"));
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/AzurePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/AzurePropertiesTest.java
index fadefd43f57c30..fd36521a82875d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/AzurePropertiesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/AzurePropertiesTest.java
@@ -18,6 +18,7 @@
package org.apache.doris.datasource.property.storage;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
@@ -81,8 +82,6 @@ public void testMissingProvider() throws UserException {
Assertions.assertEquals(HdfsProperties.class, storagePropertiesList.get(1).getClass());
Assertions.assertEquals(AzureProperties.class, storagePropertiesList.get(0).getClass());
origProps.put("s3.endpoint", "https://mystorageaccount.net");
- Assertions.assertThrows(RuntimeException.class, () ->
- StorageProperties.createPrimary(origProps), "No supported storage type found.");
// Expect an exception due to missing provider
origProps.put("provider", "azure");
Assertions.assertThrows(IllegalArgumentException.class, () ->
@@ -130,7 +129,7 @@ public void testParsingUri() throws Exception {
AzureProperties azureProperties = (AzureProperties) StorageProperties.createPrimary(origProps);
Assertions.assertEquals("s3://mycontainer/blob.txt",
azureProperties.validateAndNormalizeUri("https://mystorageaccount.blob.core.windows.net/mycontainer/blob.txt"));
- Assertions.assertThrowsExactly(UserException.class, () ->
+ Assertions.assertThrowsExactly(StoragePropertiesException.class, () ->
azureProperties.validateAndGetUri(origProps),
"props must contain uri");
origProps.put("uri", "https://mystorageaccount.blob.core.windows.net/mycontainer/blob.txt");
@@ -188,7 +187,7 @@ public void testEmptyPath() throws UserException {
AzureProperties azureProperties = (AzureProperties) StorageProperties.createPrimary(origProps);
// Expect an exception when the path is empty
- Assertions.assertThrows(UserException.class, () ->
+ Assertions.assertThrows(StoragePropertiesException.class, () ->
azureProperties.validateAndNormalizeUri(""), "Path cannot be empty.");
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
index d6f8dce27a5816..294f9c2bea97b1 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
@@ -18,6 +18,7 @@
package org.apache.doris.datasource.property.storage;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
@@ -140,6 +141,6 @@ public void testGetRegionWithDefault() throws UserException {
cosNoEndpointProps.put("cos.region", "ap-beijing");
origProps.put("uri", "s3://examplebucket-1250000000/test/file.txt");
//not support this case
- Assertions.assertThrowsExactly(RuntimeException.class, () -> StorageProperties.createPrimary(cosNoEndpointProps), "Property cos.endpoint is required.");
+ Assertions.assertThrowsExactly(StoragePropertiesException.class, () -> StorageProperties.createPrimary(cosNoEndpointProps), "Property cos.endpoint is required.");
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java
index a4dff23d2766d1..84f4c40cbdc53e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java
@@ -19,6 +19,7 @@
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
@@ -32,10 +33,12 @@
public class HdfsPropertiesTest {
-
@Test
public void testBasicHdfsCreate() throws UserException {
// Test 1: Check default authentication type (should be "simple")
+ Map simpleHdfsProperties = new HashMap<>();
+ simpleHdfsProperties.put("uri", "hdfs://test/1.orc");
+ Assertions.assertEquals(HdfsProperties.class, StorageProperties.createPrimary(simpleHdfsProperties).getClass());
Map origProps = createBaseHdfsProperties();
List storageProperties = StorageProperties.createAll(origProps);
HdfsProperties hdfsProperties = (HdfsProperties) storageProperties.get(0);
@@ -101,9 +104,9 @@ public void testBasicHdfsPropertiesCreateByConfigFile() throws UserException {
@Test
public void testNonParamsException() throws UserException {
Map origProps = new HashMap<>();
- Assertions.assertThrowsExactly(RuntimeException.class, () -> StorageProperties.createPrimary(origProps));
+ Assertions.assertThrowsExactly(StoragePropertiesException.class, () -> StorageProperties.createPrimary(origProps));
origProps.put("nonhdfs", "hdfs://localhost:9000");
- Assertions.assertThrowsExactly(RuntimeException.class, () -> {
+ Assertions.assertThrowsExactly(StoragePropertiesException.class, () -> {
StorageProperties.createPrimary(origProps);
});
origProps.put(StorageProperties.FS_HDFS_SUPPORT, "true");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java
index b01688cd264478..e150af31f3b62d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java
@@ -18,6 +18,7 @@
package org.apache.doris.datasource.property.storage;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -51,10 +52,10 @@ public void testCheckLoadPropsAndReturnUri_missingUriKey() {
Map props = new HashMap<>();
props.put("path", "xxx");
- Exception exception = Assertions.assertThrows(UserException.class, () -> {
+ Exception exception = Assertions.assertThrows(StoragePropertiesException.class, () -> {
HdfsPropertiesUtils.validateAndGetUri(props);
});
- Assertions.assertEquals("errCode = 2, detailMessage = props must contain uri", exception.getMessage());
+ Assertions.assertEquals("props must contain uri", exception.getMessage());
}
@Test
@@ -81,7 +82,7 @@ public void testConvertUrlToFilePath_blankUri() {
Exception exception = Assertions.assertThrows(IllegalArgumentException.class, () -> {
HdfsPropertiesUtils.convertUrlToFilePath(uri);
});
- Assertions.assertTrue(exception.getMessage().contains("uri is null"));
+ Assertions.assertTrue(exception.getMessage().contains("Properties 'uri' is required"));
}
@Test
@@ -89,7 +90,7 @@ public void testConstructDefaultFsFromUri_valid() {
Map props = new HashMap<>();
props.put("uri", "hdfs://localhost:8020/data");
- String result = HdfsPropertiesUtils.constructDefaultFsFromUri(props);
+ String result = HdfsPropertiesUtils.extractDefaultFsFromUri(props);
Assertions.assertEquals("hdfs://localhost:8020", result);
}
@@ -98,7 +99,7 @@ public void testConstructDefaultFsFromUri_viewfs() {
Map props = new HashMap<>();
props.put("uri", "viewfs://cluster/path");
- String result = HdfsPropertiesUtils.constructDefaultFsFromUri(props);
+ String result = HdfsPropertiesUtils.extractDefaultFsFromUri(props);
Assertions.assertEquals("viewfs://cluster", result);
}
@@ -106,17 +107,13 @@ public void testConstructDefaultFsFromUri_viewfs() {
public void testConstructDefaultFsFromUri_invalidSchema() {
Map props = new HashMap<>();
props.put("uri", "obs://bucket/test");
-
- Exception exception = Assertions.assertThrows(IllegalArgumentException.class, () -> {
- HdfsPropertiesUtils.constructDefaultFsFromUri(props);
- });
- Assertions.assertTrue(exception.getMessage().contains("Invalid export path"));
+ Assertions.assertNull(HdfsPropertiesUtils.extractDefaultFsFromUri(props));
}
@Test
public void testConstructDefaultFsFromUri_emptyProps() {
Map props = new HashMap<>();
- String result = HdfsPropertiesUtils.constructDefaultFsFromUri(props);
+ String result = HdfsPropertiesUtils.extractDefaultFsFromUri(props);
Assertions.assertNull(result);
}
@@ -125,7 +122,7 @@ public void testConstructDefaultFsFromUri_missingUri() {
Map props = new HashMap<>();
props.put("x", "y");
- String result = HdfsPropertiesUtils.constructDefaultFsFromUri(props);
+ String result = HdfsPropertiesUtils.extractDefaultFsFromUri(props);
Assertions.assertNull(result);
}
@@ -134,7 +131,7 @@ public void testConstructDefaultFsFromUri_blankUri() {
Map props = new HashMap<>();
props.put("uri", " ");
- String result = HdfsPropertiesUtils.constructDefaultFsFromUri(props);
+ String result = HdfsPropertiesUtils.extractDefaultFsFromUri(props);
Assertions.assertNull(result);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java
new file mode 100644
index 00000000000000..994a00ad1c8b11
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.storage;
+
+import org.apache.doris.common.UserException;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class MinioPropertiesTest {
+
+ private Map origProps;
+
+ @BeforeEach
+ public void setup() {
+ origProps = new HashMap<>();
+ }
+
+ @Test
+ public void testValidMinioConfiguration() throws UserException {
+ origProps.put("s3.endpoint", "http://localhost:9000");
+ origProps.put("s3.access_key", "minioAccessKey");
+ origProps.put("s3.secret_key", "minioSecretKey");
+
+ MinioProperties minioProperties = (MinioProperties) StorageProperties.createPrimary(origProps);
+
+ Assertions.assertEquals("http://localhost:9000", minioProperties.getEndpoint());
+ Assertions.assertEquals("minioAccessKey", minioProperties.getAccessKey());
+ Assertions.assertEquals("minioSecretKey", minioProperties.getSecretKey());
+ Assertions.assertEquals("us-east-1", minioProperties.getRegion());
+ origProps.remove("s3.endpoint");
+ origProps.put("uri", "http://localhost:9000/test/");
+ Assertions.assertThrows(IllegalArgumentException.class, () ->
+ StorageProperties.createPrimary(origProps), "Property endpoint is required.");
+ origProps.put("s3.endpoint", "http://localhost:9000");
+ Assertions.assertDoesNotThrow(() -> StorageProperties.createPrimary(origProps));
+
+
+ }
+
+ @Test
+ public void testGuessIsMeWithMinio() {
+ origProps.put("s3.access_key", "minioAccessKey");
+ Assertions.assertTrue(MinioProperties.guessIsMe(origProps));
+ }
+
+ @Test
+ public void testMissingAccessKey() {
+ origProps.put("s3.endpoint", "http://localhost:9000");
+ origProps.put("s3.secret_key", "minioSecretKey");
+ Assertions.assertThrows(IllegalArgumentException.class, () ->
+ StorageProperties.createPrimary(origProps), "Property s3.access_key is required.");
+ }
+
+ @Test
+ public void testEndpoint() {
+ origProps.put("s3.endpoint", "not-a-valid-url");
+ origProps.put("s3.access_key", "a");
+ origProps.put("s3.secret_key", "b");
+ Assertions.assertDoesNotThrow(() -> StorageProperties.createPrimary(origProps));
+ origProps.put("s3.endpoint", "http://localhost:9000");
+ Assertions.assertDoesNotThrow(() -> StorageProperties.createPrimary(origProps));
+ }
+
+ @Test
+ public void testBackendConfigProperties() {
+ origProps.put("s3.endpoint", "http://localhost:9000");
+ origProps.put("s3.access_key", "minioAccessKey");
+ origProps.put("s3.secret_key", "minioSecretKey");
+
+ MinioProperties minioProperties = (MinioProperties) StorageProperties.createPrimary(origProps);
+ Map backendProps = minioProperties.getBackendConfigProperties();
+
+ Assertions.assertEquals("http://localhost:9000", backendProps.get("AWS_ENDPOINT"));
+ Assertions.assertEquals("minioAccessKey", backendProps.get("AWS_ACCESS_KEY"));
+ Assertions.assertEquals("minioSecretKey", backendProps.get("AWS_SECRET_KEY"));
+ Assertions.assertEquals("us-east-1", backendProps.get("AWS_REGION"));
+ }
+}
+
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
index 03914f76693d3e..a5993742837c6d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
@@ -117,9 +117,9 @@ public void testGetRegionWithDefault() throws UserException {
cosNoEndpointProps.put("obs.access_key", "myCOSAccessKey");
cosNoEndpointProps.put("obs.secret_key", "myCOSSecretKey");
cosNoEndpointProps.put("obs.region", "ap-beijing");
- cosNoEndpointProps.put("uri", "s3://examplebucket-1250000000/test/file.txt");
+ cosNoEndpointProps.put("uri", "s3://examplebucket-1250000000/myhuaweicloud.com/test/file.txt");
//not support
- Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> StorageProperties.createAll(cosNoEndpointProps), "Property cos.endpoint is required.");
+ Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> StorageProperties.createPrimary(cosNoEndpointProps), "Property cos.endpoint is required.");
}
private static String obsAccessKey = "";
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java
index 6a076fdc664270..d499027904e5d2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java
@@ -116,7 +116,7 @@ public void testGetStorageName() throws UserException {
origProps.put("oss.secret_key", "testSecretKey");
origProps.put("oss.region", "cn-shanghai");
- OSSHdfsProperties props = (OSSHdfsProperties) StorageProperties.createAll(origProps).get(0);
+ OSSHdfsProperties props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
Assertions.assertEquals("HDFS", props.getStorageName());
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
index ad0d89f1db8acd..b52e888358b81f 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
@@ -18,6 +18,7 @@
package org.apache.doris.datasource.property.storage;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -46,8 +47,7 @@ public void testBasicCreateTest() {
origProps = new HashMap<>();
origProps.put("oss.endpoint", "https://oss.aliyuncs.com");
Map finalOrigProps2 = origProps;
- Assertions.assertThrowsExactly(RuntimeException.class, () -> StorageProperties.createPrimary(finalOrigProps2));
-
+ Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> StorageProperties.createPrimary(finalOrigProps2));
}
@@ -123,6 +123,6 @@ public void testGetRegionWithDefault() throws UserException {
cosNoEndpointProps.put("oss.region", "cn-hangzhou");
origProps.put("uri", "s3://examplebucket-1250000000/test/file.txt");
// not support
- Assertions.assertThrowsExactly(RuntimeException.class, () -> StorageProperties.createPrimary(cosNoEndpointProps), "Property cos.endpoint is required.");
+ Assertions.assertThrowsExactly(StoragePropertiesException.class, () -> StorageProperties.createPrimary(cosNoEndpointProps), "Property cos.endpoint is required.");
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
index 14a449ab91f4e9..9c397c358aac02 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
@@ -18,6 +18,7 @@
package org.apache.doris.datasource.property.storage;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
@@ -175,6 +176,6 @@ public void testGetRegionWithDefault() throws UserException {
s3EndpointProps.put("oss.region", "cn-hangzhou");
origProps.put("uri", "s3://examplebucket-1250000000/test/file.txt");
//not support
- Assertions.assertThrowsExactly(RuntimeException.class, () -> StorageProperties.createPrimary(s3EndpointProps), "Property cos.endpoint is required.");
+ Assertions.assertThrowsExactly(StoragePropertiesException.class, () -> StorageProperties.createPrimary(s3EndpointProps), "Property cos.endpoint is required.");
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertyUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertyUtilsTest.java
index 64795761383c3e..6427abd8fda5f9 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertyUtilsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertyUtilsTest.java
@@ -18,6 +18,7 @@
package org.apache.doris.datasource.property.storage;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -40,13 +41,13 @@ void testCheckLoadPropsAndReturnUri_success() throws UserException {
void testCheckLoadPropsAndReturnUri_missingKey() {
Map props = new HashMap<>();
Executable executable = () -> S3PropertyUtils.validateAndGetUri(props);
- UserException exception = Assertions.assertThrows(UserException.class, executable);
- Assertions.assertEquals("errCode = 2, detailMessage = props is empty", exception.getMessage());
+ StoragePropertiesException exception = Assertions.assertThrows(StoragePropertiesException.class, executable);
+ Assertions.assertEquals("props is empty", exception.getMessage());
props.put("someKey", "value");
executable = () -> S3PropertyUtils.validateAndGetUri(props);
- exception = Assertions.assertThrows(UserException.class, executable);
- Assertions.assertEquals("errCode = 2, detailMessage = props must contain uri", exception.getMessage());
+ exception = Assertions.assertThrows(StoragePropertiesException.class, executable);
+ Assertions.assertEquals("props must contain uri", exception.getMessage());
}
@Test
@@ -66,7 +67,7 @@ void testConstructEndpointFromUrl_nullOrBlank() throws UserException {
Assertions.assertNull(S3PropertyUtils.constructEndpointFromUrl(props, "false", "true"));
props.put("uri", "invalid uri without scheme");
- Assertions.assertThrowsExactly(UserException.class, () -> S3PropertyUtils.constructEndpointFromUrl(props, "true", "true"));
+ Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> S3PropertyUtils.constructEndpointFromUrl(props, "true", "true"));
}
@Test
@@ -86,7 +87,7 @@ void testConstructRegionFromUrl_nullOrInvalid() throws UserException {
Assertions.assertNull(S3PropertyUtils.constructRegionFromUrl(props, "false", "true"));
props.put("uri", "not a uri");
- Assertions.assertThrowsExactly(UserException.class, () -> S3PropertyUtils.constructRegionFromUrl(props, "false", "true"));
+ Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> S3PropertyUtils.constructRegionFromUrl(props, "false", "true"));
props.put("uri", "https://my-bucket.s3.us-west-1.amazonaws.com/test.txt");
Assertions.assertEquals("us-west-1", S3PropertyUtils.constructRegionFromUrl(props, "false", "true"));
}
@@ -103,8 +104,8 @@ void testConvertToS3Address_success() throws UserException {
@Test
void testConvertToS3Address_invalid() {
- Assertions.assertThrows(UserException.class, () -> S3PropertyUtils.validateAndNormalizeUri(null, "false", "true"));
- Assertions.assertThrows(UserException.class, () -> S3PropertyUtils.validateAndNormalizeUri("", "false", "false"));
+ Assertions.assertThrows(StoragePropertiesException.class, () -> S3PropertyUtils.validateAndNormalizeUri(null, "false", "true"));
+ Assertions.assertThrows(StoragePropertiesException.class, () -> S3PropertyUtils.validateAndNormalizeUri("", "false", "false"));
Assertions.assertThrows(UserException.class, () -> S3PropertyUtils.validateAndNormalizeUri("not a uri", "true", "true"));
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java b/fe/fe-core/src/test/java/org/apache/doris/fsv2/obj/S3FileSystemTest.java
similarity index 92%
rename from fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java
rename to fe/fe-core/src/test/java/org/apache/doris/fsv2/obj/S3FileSystemTest.java
index 442883573ce49a..b36d0b4aba46b9 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/fsv2/obj/S3FileSystemTest.java
@@ -15,16 +15,19 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.fs.obj;
+package org.apache.doris.fsv2.obj;
import org.apache.doris.backup.Repository;
import org.apache.doris.backup.Status;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.S3URI;
import org.apache.doris.datasource.property.PropertyConverter;
-import org.apache.doris.fs.FileSystemFactory;
-import org.apache.doris.fs.remote.RemoteFile;
-import org.apache.doris.fs.remote.S3FileSystem;
+import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+import org.apache.doris.fs.obj.MockedS3Client;
+import org.apache.doris.fsv2.FileSystemFactory;
+import org.apache.doris.fsv2.remote.RemoteFile;
+import org.apache.doris.fsv2.remote.S3FileSystem;
import mockit.Mock;
import mockit.MockUp;
@@ -70,7 +73,7 @@ public void setUp() throws Exception {
properties = new HashMap<>();
properties.put("AWS_ACCESS_KEY", System.getenv().getOrDefault("AWS_AK", ""));
properties.put("AWS_SECRET_KEY", System.getenv().getOrDefault("AWS_SK", ""));
- properties.put("AWS_ENDPOINT", "http://s3.bj.bcebos.com");
+ properties.put("AWS_ENDPOINT", "http://s3.ap-northeast-1.amazonaws.com");
properties.put(PropertyConverter.USE_PATH_STYLE, "false");
properties.put("AWS_REGION", "bj");
content =
@@ -102,10 +105,10 @@ public S3Client getClient() throws UserException {
return mockedClient;
}
};
- S3ObjStorage mockedStorage = new S3ObjStorage(properties);
+ S3ObjStorage mockedStorage = new S3ObjStorage((AbstractS3CompatibleProperties) StorageProperties.createPrimary(properties));
Assertions.assertTrue(mockedStorage.getClient() instanceof MockedS3Client);
// inject storage to file system.
- fileSystem = new S3FileSystem(mockedStorage);
+ fileSystem = (S3FileSystem) FileSystemFactory.get(properties);
new MockUp(S3FileSystem.class) {
@Mock
public Status globList(String remotePath, List result, boolean fileNameOnly) {
@@ -124,7 +127,7 @@ public Status globList(String remotePath, List result, boolean fileN
};
} else {
// can also real file system to test.
- fileSystem = (S3FileSystem) FileSystemFactory.getS3FileSystem(properties);
+ fileSystem = (S3FileSystem) FileSystemFactory.get(properties);
}
testFile = bucket + basePath + "/Ode_to_the_West_Wind";
Assertions.assertEquals(Status.OK, fileSystem.directUpload(content, testFile));
@@ -168,7 +171,8 @@ public void upload() throws IOException {
@Test
public void testRepositoryUpload() throws IOException {
- Repository repo = new Repository(10000, "repo", false, bucket + basePath, fileSystem);
+ Repository repo = new Repository(10000, "repo", false, bucket + basePath, fileSystem,
+ null);
File localFile = File.createTempFile("s3unittest", ".dat");
localFile.deleteOnExit();
String remote = bucket + basePath + "/" + localFile.getName();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java
index 59f86b38fbb3e3..90dfecf6ab3b65 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java
@@ -47,7 +47,7 @@ public void testEncryption() {
+ " \"max_file_size\" = \"2048MB\""
+ ")"
+ "WITH s3 ("
- + " \"s3.endpoint\" = \"abc\","
+ + " \"s3.endpoint\" = \"cos.ap-beijing.myqcloud.com\","
+ " \"s3.region\" = \"ap-beijing\","
+ " \"s3.secret_key\" = \"abc\","
+ " \"s3.access_key\" = \"abc\""
@@ -59,7 +59,7 @@ public void testEncryption() {
+ " \"max_file_size\" = \"2048MB\""
+ ")"
+ "WITH s3 ("
- + " \"s3.endpoint\" = \"abc\","
+ + " \"s3.endpoint\" = \"cos.ap-beijing.myqcloud.com\","
+ " \"s3.region\" = \"ap-beijing\","
+ " \"s3.secret_key\" = \"*XXX\","
+ " \"s3.access_key\" = \"abc\""
@@ -70,7 +70,7 @@ public void testEncryption() {
+ " INTO OUTFILE \"s3://abc/aaa\""
+ " FORMAT AS ORC"
+ " PROPERTIES ("
- + " \"s3.endpoint\" = \"abc\","
+ + " \"s3.endpoint\" = \"cos.ap-beijing.myqcloud.com\","
+ " \"s3.region\" = \"ap-beijing\","
+ " \"s3.secret_key\" = \"abc\","
+ " \"s3.access_key\" = \"abc\""
@@ -80,7 +80,7 @@ public void testEncryption() {
+ " INTO OUTFILE \"s3://abc/aaa\""
+ " FORMAT AS ORC"
+ " PROPERTIES ("
- + " \"s3.endpoint\" = \"abc\","
+ + " \"s3.endpoint\" = \"cos.ap-beijing.myqcloud.com\","
+ " \"s3.region\" = \"ap-beijing\","
+ " \"s3.secret_key\" = \"*XXX\","
+ " \"s3.access_key\" = \"abc\""
diff --git a/fe/pom.xml b/fe/pom.xml
index 14ec4bc07bde43..51222fb42ee5b8 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -340,6 +340,7 @@ under the License.
1.12.669
3.0.9
3.3.6
+ 1.8
1.2.0
1.1.1
2.4.9
@@ -628,6 +629,11 @@ under the License.