Skip to content

Commit

Permalink
add comments
Browse files Browse the repository at this point in the history
  • Loading branch information
kaijchen authored Aug 30, 2024
1 parent a1422e6 commit beca182
Show file tree
Hide file tree
Showing 7 changed files with 8 additions and 1 deletion.
1 change: 1 addition & 0 deletions be/src/util/s3_util.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,7 @@ Status S3ClientFactory::convert_properties_to_s3_conf(
}
}
if (auto it = properties.find(S3_PROVIDER); it != properties.end()) {
// S3 Provider properties should be case insensitive.
if (0 == strcasecmp(it->second.c_str(), AZURE_PROVIDER_STRING)) {
s3_conf->client_conf.provider = io::ObjStorageType::AZURE;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ private void analyzeStagePB(StagePB stagePB) throws AnalysisException {
}
brokerProperties.put(S3_BUCKET, objInfo.getBucket());
brokerProperties.put(S3_PREFIX, objInfo.getPrefix());
// S3 Provider properties should be case insensitive.
brokerProperties.put(S3Properties.PROVIDER, objInfo.getProvider().toString().toUpperCase());
StageProperties stageProperties = new StageProperties(stagePB.getPropertiesMap());
this.copyIntoProperties.mergeProperties(stageProperties);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -527,6 +527,7 @@ private String getProviderFromEndpoint() {
Map<String, String> properties = brokerDesc.getProperties();
for (Map.Entry<String, String> entry : properties.entrySet()) {
if (entry.getKey().equalsIgnoreCase(S3Properties.PROVIDER)) {
// S3 Provider properties should be case insensitive.
return entry.getValue().toUpperCase();
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@ private void analyzeStorageProperties() throws AnalysisException {
}
properties.put(PREFIX, prefix);
// analyze provider
// S3 Provider properties should be case insensitive.
String provider = properties.get(PROVIDER).toUpperCase();
if (!EnumUtils.isValidEnumIgnoreCase(ObjectStoreInfoPB.Provider.class, provider)) {
throw new AnalysisException("Property " + PROVIDER + " with invalid value " + provider);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@ public static Map<String, String> requiredS3TVFProperties(Map<String, String> pr
private static void checkProvider(Map<String, String> properties) throws DdlException {
if (properties.containsKey(PROVIDER)) {
properties.put(PROVIDER, properties.get(PROVIDER).toUpperCase());
// S3 Provider properties should be case insensitive.
if (!PROVIDERS.stream().anyMatch(s -> s.equals(properties.get(PROVIDER).toUpperCase()))) {
throw new DdlException("Provider must be one of OSS, OBS, AZURE, BOS, COS, S3, GCP");
}
Expand Down Expand Up @@ -330,6 +331,7 @@ public static Cloud.ObjectStoreInfoPB.Builder getObjStoreInfoPB(Map<String, Stri
builder.setExternalEndpoint(properties.get(S3Properties.EXTERNAL_ENDPOINT));
}
if (properties.containsKey(S3Properties.PROVIDER)) {
// S3 Provider properties should be case insensitive.
builder.setProvider(Provider.valueOf(properties.get(S3Properties.PROVIDER).toUpperCase()));
}
return builder;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ public void testAnalyzeStorageProperties() throws Exception {
+ "'prefix' = 'tmp_prefix', "
+ "'provider' = 'abc', "
+ "'ak'='tmp_ak', 'sk'='tmp_sk', 'access_type'='aksk');";
// S3 Provider will be converted to upper case.
parseAndAnalyzeWithException(sql, "Property provider with invalid value ABC");

// test getObjectInfoPB
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -630,7 +630,7 @@ class Config {
} else if (config.s3Source == "huawei") {
s3Provider = "OBS"
} else if (config.s3Source == "azure") {
s3Provider = "Azure"
s3Provider = "Azure" // case insensitive test
} else if (config.s3Source == "gcp") {
s3Provider = "GCP"
}
Expand Down

0 comments on commit beca182

Please sign in to comment.