Skip to content

Commit

Permalink
multiple prefixes
Browse files Browse the repository at this point in the history
  • Loading branch information
eric-maynard committed Oct 4, 2024
1 parent 6f66848 commit fac2d72
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -222,10 +222,12 @@ private static List<String> userSpecifiedWriteLocations(Map<String, String> prop

/** Validate if the provided allowed locations are valid for the storage type */
protected void validatePrefixForStorageType(String loc) {
if (!loc.toLowerCase(Locale.ROOT).startsWith(storageType.prefix)) {
if (storageType.prefixes.stream().noneMatch(p -> loc.toLowerCase(Locale.ROOT).startsWith(p))) {
throw new IllegalArgumentException(
String.format(
"Location prefix not allowed: '%s', expected prefix: '%s'", loc, storageType.prefix));
"Location prefix not allowed: '%s', expected prefixes: '%s'",
loc,
String.join(",", storageType.prefixes)));
}
}

Expand All @@ -240,19 +242,23 @@ public void validateMaxAllowedLocations(int maxAllowedLocations) {
/** Polaris' storage type, each has a fixed prefix for its location */
public enum StorageType {
S3("s3://"),
AZURE("abfs"), // abfs or abfss
AZURE(List.of("abfs", "wasb", "abfss", "wasbs")),
GCS("gs://"),
FILE("file://"),
;

final String prefix;
final List<String> prefixes;

StorageType(String prefix) {
this.prefix = prefix;
this.prefixes = List.of(prefix);
}

public String getPrefix() {
return prefix;
StorageType(List<String> prefixes) {
this.prefixes = prefixes;
}

public List<String> getPrefixes() {
return prefixes;
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@

/** This class represents all information for a azure location */
public class AzureLocation {
/** The pattern only allows abfs[s] now because the ResovlingFileIO only accept ADLSFileIO */
private static final Pattern URI_PATTERN = Pattern.compile("^abfss?://([^/?#]+)(.*)?$");
private static final Pattern URI_PATTERN = Pattern.compile("^(abfss?|wasbs?)://([^/?#]+)(.*)?$");

public static final String ADLS_ENDPOINT = "dfs.core.windows.net";

Expand All @@ -47,9 +46,10 @@ public class AzureLocation {
public AzureLocation(@NotNull String location) {
Matcher matcher = URI_PATTERN.matcher(location);
if (!matcher.matches()) {
throw new IllegalArgumentException("Invalid azure adls location uri " + location);
throw new IllegalArgumentException("Invalid azure location uri " + location);
}
String authority = matcher.group(1);
String scheme = matcher.group(1);
String authority = matcher.group(2);
// look for <container>@<storage account host>
String[] parts = authority.split("@", -1);

Expand All @@ -65,7 +65,7 @@ public AzureLocation(@NotNull String location) {
}
this.storageAccount = hostParts[0];
this.endpoint = hostParts[1];
String path = matcher.group(2);
String path = matcher.group(3);
filePath = path == null ? "" : path.startsWith("/") ? path.substring(1) : path;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public void testEmptyString() {
}

@ParameterizedTest
@ValueSource(strings = {"s3", "gcs", "abfs", "file"})
@ValueSource(strings = {"s3", "gcs", "abfs", "wasb", "file"})
public void testAbsolutePaths(String scheme) {
Assertions.assertThat(StorageUtil.getBucket(scheme + "://bucket/path/file.txt"))
.isEqualTo("bucket");
Expand Down

0 comments on commit fac2d72

Please sign in to comment.