From 18e7f8b4061b95c3d4130ce9ab60e19de555bd45 Mon Sep 17 00:00:00 2001 From: hdavidh Date: Wed, 29 Mar 2023 16:34:26 -0700 Subject: [PATCH 01/12] S3 URI Parser --- .../awssdk/services/s3/S3Utilities.java | 230 ++++++++++++++ .../awssdk/services/s3/parsing/S3Uri.java | 210 +++++++++++++ .../awssdk/services/s3/S3UtilitiesTest.java | 288 +++++++++++++++++- 3 files changed, 727 insertions(+), 1 deletion(-) create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java index 0a0bbc9cfc33..daea28489f87 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java @@ -16,14 +16,20 @@ package software.amazon.awssdk.services.s3; +import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; +import java.net.URLDecoder; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.function.Consumer; import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.annotations.SdkPublicApi; @@ -38,6 +44,7 @@ import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; import software.amazon.awssdk.core.client.config.SdkClientConfiguration; import software.amazon.awssdk.core.client.config.SdkClientOption; +import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; @@ -62,6 +69,7 @@ import software.amazon.awssdk.services.s3.internal.endpoints.UseGlobalEndpointResolver; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetUrlRequest; +import software.amazon.awssdk.services.s3.parsing.S3Uri; import software.amazon.awssdk.utils.AttributeMap; import software.amazon.awssdk.utils.Validate; @@ -494,4 +502,226 @@ public S3Utilities build() { return new S3Utilities(this); } } + + public S3Uri parseS3Uri(URI uri) { + return parseS3Uri(uri, true); + } + + public S3Uri parseS3Uri(URI uri, boolean urlEncode) { + if (uri == null) { + throw SdkClientException.create("URI must not be null"); + } + + Pattern accessPointPattern = Pattern.compile("^([a-zA-Z0-9\\-]+)\\.s3-accesspoint(-fips)?(\\.dualstack)?" + + "\\.([a-zA-Z0-9\\-]+)\\.amazonaws\\.com(.cn)?$"); + if (accessPointPattern.matcher(uri.toString()).find()) { + throw SdkClientException.create("AccessPoints URI parsing is not supported"); + } + + Pattern outpostPattern = Pattern.compile("^([a-zA-Z0-9\\-]+)\\.op\\-[0-9]+\\.s3-outposts\\.([a-zA-Z0-9\\-]+)" + + "\\.amazonaws\\.com(.cn)?$"); + if (outpostPattern.matcher(uri.toString()).find()) { + throw SdkClientException.create("Outposts URI parsing is not supported"); + } + + String bucket = null; + String key = null; + String region = null; + boolean isPathStyle = false; + Map queryParams = new HashMap<>(); + + if ("s3".equalsIgnoreCase(uri.getScheme())) { + if (uri.getAuthority() == null) { + throw SdkClientException.create("Invalid S3 URI: bucket not included"); + } + bucket = uri.getAuthority(); + + String path = uri.getPath(); + if (path.length() > 1) { + key = uri.getPath().substring(1); + } + + } else { + if (uri.getHost() == null) { + throw SdkClientException.create("Invalid S3 URI: hostname not included"); + } + + Pattern endpointPattern = Pattern.compile("^(.+\\.)?s3[.-]([a-z0-9-]+)\\."); + Matcher matcher = endpointPattern.matcher(uri.getHost()); + if (!matcher.find()) { + throw SdkClientException.create("Invalid S3 URI: hostname does not appear to be a valid S3 endpoint"); + } + + String prefix = matcher.group(1); + if (prefix == null || prefix.isEmpty()) { + isPathStyle = true; + String path = urlEncode ? uri.getPath() : uri.getRawPath(); + + if (!"".equals(path) && !"/".equals(path)) { + int index = path.indexOf('/', 1); + + if (index == -1) { + bucket = decode(path.substring(1)); + } else if (index == (path.length() - 1)) { + bucket = decode(path.substring(1, index)); + } else { + bucket = decode(path.substring(1, index)); + key = decode(path.substring(index + 1)); + } + } + } else { + bucket = prefix.substring(0, prefix.length() - 1); + String path = uri.getPath(); + if (path != null && !path.isEmpty() && !"/".equals(uri.getPath())) { + key = uri.getPath().substring(1); + } + } + + if (!"amazonaws".equals(matcher.group(2))) { + region = matcher.group(2); + } + } + + String queryPart = uri.getRawQuery(); + if (queryPart != null) { + parseQuery(queryParams, queryPart); + } + + return S3Uri.builder() + .uri(uri) + .bucket(bucket) + .key(key) + .region(region) + .isPathStyle(isPathStyle) + .queryParams(queryParams) + .build(); + } + + private void parseQuery(Map queryParams, String queryPart) { + String[] params = queryPart.split("&"); + for (String param: params) { + try { + String[] keyValuePair = param.split("=", 2); + String key = URLDecoder.decode(keyValuePair[0], "UTF-8"); + if (key.isEmpty()) { + continue; + } + String value = URLDecoder.decode(keyValuePair[1], "UTF-8"); + queryParams.put(key, value); + } catch (UnsupportedEncodingException e) { + // Param could not be decoded + } + + } + } + + /** + * Percent-decodes the given string, with a fast path for strings that + * are not percent-encoded. + * + * @param str the string to decode + * @return the decoded string + */ + private static String decode(final String str) { + if (str == null) { + return null; + } + + for (int i = 0; i < str.length(); ++i) { + if (str.charAt(i) == '%') { + return decode(str, i); + } + } + + return str; + } + + /** + * Percent-decodes the given string. + * + * @param str the string to decode + * @param firstPercent the index of the first '%' character in the string + * @return the decoded string + */ + private static String decode(final String str, final int firstPercent) { + StringBuilder builder = new StringBuilder(); + builder.append(str.substring(0, firstPercent)); + + appendDecoded(builder, str, firstPercent); + + for (int i = firstPercent + 3; i < str.length(); ++i) { + if (str.charAt(i) == '%') { + appendDecoded(builder, str, i); + i += 2; + } else { + builder.append(str.charAt(i)); + } + } + + return builder.toString(); + } + + /** + * Decodes the percent-encoded character at the given index in the string + * and appends the decoded value to the given {@code StringBuilder}. + * + * @param builder the string builder to append to + * @param str the string being decoded + * @param index the index of the '%' character in the string + */ + private static void appendDecoded(final StringBuilder builder, + final String str, + final int index) { + + if (index > str.length() - 3) { + throw new IllegalStateException("Invalid percent-encoded string:" + + "\"" + str + "\"."); + } + + char first = str.charAt(index + 1); + char second = str.charAt(index + 2); + + char decoded = (char) ((fromHex(first) << 4) | fromHex(second)); + builder.append(decoded); + } + + /** + * Converts a hex character (0-9A-Fa-f) into its corresponding quad value. + * + * @param c the hex character + * @return the quad value + */ + private static int fromHex(final char c) { + if (c < '0') { + throw new IllegalStateException( + "Invalid percent-encoded string: bad character '" + c + "' in " + + "escape sequence."); + } + if (c <= '9') { + return (c - '0'); + } + + if (c < 'A') { + throw new IllegalStateException( + "Invalid percent-encoded string: bad character '" + c + "' in " + + "escape sequence."); + } + if (c <= 'F') { + return (c - 'A') + 10; + } + + if (c < 'a') { + throw new IllegalStateException( + "Invalid percent-encoded string: bad character '" + c + "' in " + + "escape sequence."); + } + if (c <= 'f') { + return (c - 'a') + 10; + } + + throw new IllegalStateException( + "Invalid percent-encoded string: bad character '" + c + "' in " + + "escape sequence."); + } + } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java new file mode 100644 index 000000000000..fc577a51ff10 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java @@ -0,0 +1,210 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.parsing; + +import java.net.URI; +import java.util.Map; +import java.util.Objects; +import software.amazon.awssdk.annotations.Immutable; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Object that represents a parsed S3 URI. Can be used to easily retrieve the bucket, key, region, style, and query parameters + * of the URI. Only basic buket endpoints are supported, i.e., path-style and virtual-hosted-style URIs. + */ +@Immutable +@SdkPublicApi +public class S3Uri implements ToCopyableBuilder { + + private final URI uri; + private final String bucket; + private final String key; + private final String region; + private final boolean isPathStyle; + private final Map queryParams; + + private S3Uri(Builder builder) { + this.uri = builder.uri; + this.bucket = builder.bucket; + this.key = builder.key; + this.region = builder.region; + this.isPathStyle = builder.isPathStyle; + this.queryParams = builder.queryParams; + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public Builder toBuilder() { + return new Builder(this); + } + + /** + * Returns the original URI that was used to instantiate the S3Uri + */ + public URI uri() { + return uri; + } + + /** + * Returns the bucket specified in the URI. Returns null if no bucket is specified. + */ + public String bucket() { + return bucket; + } + + /** + * Returns the key specified in the URI. Returns null if no key is specified. + */ + public String key() { + return key; + } + + /** + * Returns the region specified in the URI. Returns null if no region is specified, i.e., is a global endpoint. + */ + public String region() { + return region; + } + + /** + * Returns true if the URI is path-style, false if the URI is virtual-hosted-style. + */ + public boolean isPathStyle() { + return isPathStyle; + } + + /** + * Returns a map of the query parameters specified in the URI. Returns an empty map if no queries are specified. + */ + public Map queryParams() { + return queryParams; + } + + @Override + public String toString() { + return uri.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + S3Uri s3Uri = (S3Uri) o; + return Objects.equals(uri, s3Uri.uri) + && Objects.equals(bucket, s3Uri.bucket) + && Objects.equals(key, s3Uri.key) + && Objects.equals(region, s3Uri.region) + && Objects.equals(queryParams, s3Uri.queryParams); + } + + @Override + public int hashCode() { + int result = uri != null ? uri.hashCode() : 0; + result = 31 * result + (bucket != null ? bucket.hashCode() : 0); + result = 31 * result + (key != null ? key.hashCode() : 0); + result = 31 * result + (region != null ? region.hashCode() : 0); + result = 31 * result + (queryParams != null ? queryParams.hashCode() : 0); + return result; + } + + /** + * A builder for creating a {@link S3Uri} + */ + public static final class Builder implements CopyableBuilder { + private URI uri; + private String bucket; + private String key; + private String region; + private boolean isPathStyle; + private Map queryParams; + + private Builder() { + } + + private Builder(S3Uri s3Uri) { + this.uri = s3Uri.uri; + this.bucket = s3Uri.bucket; + this.key = s3Uri.key; + this.region = s3Uri.region; + this.isPathStyle = s3Uri.isPathStyle; + this.queryParams = s3Uri.queryParams; + } + + /** + * Configure the URI + */ + public Builder uri(URI uri) { + this.uri = uri; + return this; + } + + /** + * Configure the bucket + */ + public Builder bucket(String bucket) { + this.bucket = bucket; + return this; + } + + /** + * Configure the key + */ + public Builder key(String key) { + this.key = key; + return this; + } + + /** + * Configure the region + */ + public Builder region(String region) { + this.region = region; + return this; + } + + /** + * Configure the path style flag + */ + public Builder isPathStyle(boolean isPathStyle) { + this.isPathStyle = isPathStyle; + return this; + } + + /** + * Configure the map of query parameters + */ + public Builder queryParams(Map queryParams) { + this.queryParams = queryParams; + return this; + } + + @Override + public S3Uri build() { + return new S3Uri(this); + } + } + +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java index 0dd1650c7bf2..b18ac19f50af 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java @@ -16,18 +16,20 @@ package software.amazon.awssdk.services.s3; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.net.MalformedURLException; import java.net.URI; -import java.net.URL; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.model.GetUrlRequest; +import software.amazon.awssdk.services.s3.parsing.S3Uri; public class S3UtilitiesTest { @@ -69,6 +71,290 @@ public static void cleanup() { asyncClient.close(); } + @Test + public void parseS3Uri_pathStyleWithRoot_shouldParseCorrectly() { + String uriString = "https://s3.amazonaws.com/"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isNull(); + assertThat(s3Uri.key()).isNull(); + assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.isPathStyle()).isTrue(); + assertThat(s3Uri.queryParams()).isEmpty(); + } + + @Test + public void parseS3Uri_pathStyleWithRootNoTrailingSlash_shouldParseCorrectly() { + String uriString = "https://s3.amazonaws.com"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isNull(); + assertThat(s3Uri.key()).isNull(); + assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.isPathStyle()).isTrue(); + assertThat(s3Uri.queryParams()).isEmpty(); + } + + @Test + public void parseS3Uri_pathStyleGlobalEndpoint_shouldParseCorrectly() { + String uriString = "https://s3.amazonaws.com/myBucket/resources/image1.png"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isEqualTo("resources/image1.png"); + assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.isPathStyle()).isTrue(); + assertThat(s3Uri.queryParams()).isEmpty(); + } + + @Test + public void parseS3Uri_virtualStyleGlobalEndpoint_shouldParseCorrectly() { + String uriString = "https://myBucket.s3.amazonaws.com/resources/image1.png"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isEqualTo("resources/image1.png"); + assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.isPathStyle()).isFalse(); + assertThat(s3Uri.queryParams()).isEmpty(); + } + + @Test + public void parseS3Uri_pathStyleWithDot_shouldParseCorrectly() { + String uriString = "https://s3.eu-west-2.amazonaws.com/myBucket/resources/image1.png"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isEqualTo("resources/image1.png"); + assertThat(s3Uri.region()).isEqualTo("eu-west-2"); + assertThat(s3Uri.isPathStyle()).isTrue(); + assertThat(s3Uri.queryParams()).isEmpty(); + } + + @Test + public void parseS3Uri_pathStyleWithDash_shouldParseCorrectly() { + String uriString = "https://s3-eu-west-2.amazonaws.com/myBucket/resources/image1.png"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isEqualTo("resources/image1.png"); + assertThat(s3Uri.region()).isEqualTo("eu-west-2"); + assertThat(s3Uri.isPathStyle()).isTrue(); + assertThat(s3Uri.queryParams()).isEmpty(); + } + + @Test + public void parseS3Uri_virtualHostedStyleWithDot_shouldParseCorrectly() { + String uriString = "https://my-bucket.s3.us-east-2.amazonaws.com/image.png"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("my-bucket"); + assertThat(s3Uri.key()).isEqualTo("image.png"); + assertThat(s3Uri.region()).isEqualTo("us-east-2"); + assertThat(s3Uri.isPathStyle()).isFalse(); + assertThat(s3Uri.queryParams()).isEmpty(); + } + + @Test + public void parseS3Uri_virtualHostedStyleWithDash_shouldParseCorrectly() { + String uriString = "https://my-bucket.s3-us-east-2.amazonaws.com/image.png"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("my-bucket"); + assertThat(s3Uri.key()).isEqualTo("image.png"); + assertThat(s3Uri.region()).isEqualTo("us-east-2"); + assertThat(s3Uri.isPathStyle()).isFalse(); + assertThat(s3Uri.queryParams()).isEmpty(); + } + + @Test + public void parses3Uri_pathStyleWithQuery_shouldParseCorrectly() { + String uriString = "https://s3.us-west-1.amazonaws.com/bucket/key?versionId=abc123"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("bucket"); + assertThat(s3Uri.key()).isEqualTo("key"); + assertThat(s3Uri.region()).isEqualTo("us-west-1"); + assertThat(s3Uri.isPathStyle()).isTrue(); + assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); + } + + @Test + public void parses3Uri_pathStyleWithEncodedQuery_shouldParseCorrectly() { + String uriString = "https://s3.us-west-1.amazonaws.com/bucket/key?versionId=%61%62%63%31%32%33"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("bucket"); + assertThat(s3Uri.key()).isEqualTo("key"); + assertThat(s3Uri.region()).isEqualTo("us-west-1"); + assertThat(s3Uri.isPathStyle()).isTrue(); + assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); + } + + @Test + public void parses3Uri_pathStyleWithMultipleQueries_shouldParseCorrectly() { + String uriString = "https://s3.us-west-1.amazonaws.com/bucket/key?versionId=abc123&partNumber=77"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("bucket"); + assertThat(s3Uri.key()).isEqualTo("key"); + assertThat(s3Uri.region()).isEqualTo("us-west-1"); + assertThat(s3Uri.isPathStyle()).isTrue(); + assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); + assertThat(s3Uri.queryParams()).containsEntry("partNumber", "77"); + } + + @Test + public void parses3Uri_virtualStyleWithQuery_shouldParseCorrectly() { + String uriString = "https://bucket.s3.us-west-1.amazonaws.com/key?versionId=abc123"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("bucket"); + assertThat(s3Uri.key()).isEqualTo("key"); + assertThat(s3Uri.region()).isEqualTo("us-west-1"); + assertThat(s3Uri.isPathStyle()).isFalse(); + assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); + } + + @Test + public void parses3Uri_virtualStyleWithEncodedQuery_shouldParseCorrectly() { + String uriString = "https://bucket.s3.us-west-1.amazonaws.com/key?versionId=%61%62%63%31%32%33"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("bucket"); + assertThat(s3Uri.key()).isEqualTo("key"); + assertThat(s3Uri.region()).isEqualTo("us-west-1"); + assertThat(s3Uri.isPathStyle()).isFalse(); + assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); + } + + @Test + public void parses3Uri_virtualStyleWithMultipleQueries_shouldParseCorrectly() { + String uriString = "https://bucket.s3.us-west-1.amazonaws.com/key?versionId=abc123&partNumber=77"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("bucket"); + assertThat(s3Uri.key()).isEqualTo("key"); + assertThat(s3Uri.region()).isEqualTo("us-west-1"); + assertThat(s3Uri.isPathStyle()).isFalse(); + assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); + assertThat(s3Uri.queryParams()).containsEntry("partNumber", "77"); + } + + @Test + public void parses3Uri_virtualStyleS3SchemeWithoutKey_shouldParseCorrectly() { + String uriString = "s3://myBucket"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isNull(); + assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.isPathStyle()).isFalse(); + } + + @Test + public void parses3Uri_virtualStyleS3SchemeWithKey_shouldParseCorrectly() { + String uriString = "s3://myBucket/resources/key"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isEqualTo("resources/key"); + assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.isPathStyle()).isFalse(); + } + + @Test + public void parseS3Uri_accessPointUri_shouldThrowProperErrorMessage() { + String accessPointUriString = "myendpoint-123456789012.s3-accesspoint.us-east-1.amazonaws.com"; + URI accessPointUri = URI.create(accessPointUriString); + + Exception exception = assertThrows(SdkClientException.class, () -> { + defaultUtilities.parseS3Uri(accessPointUri); + }); + + assertThat(exception.getMessage()).isEqualTo("AccessPoints URI parsing is not supported"); + } + + @Test + public void parseS3Uri_accessPointUriWithFipsDualstack_shouldThrowProperErrorMessage() { + String accessPointUriString = "myendpoint-123456789012.s3-accesspoint-fips.dualstack.us-gov-east-1.amazonaws.com"; + URI accessPointUri = URI.create(accessPointUriString); + + Exception exception = assertThrows(SdkClientException.class, () -> { + defaultUtilities.parseS3Uri(accessPointUri); + }); + + assertThat(exception.getMessage()).isEqualTo("AccessPoints URI parsing is not supported"); + } + + @Test + public void parseS3Uri_accessPointsUriWithChinaPartition_shouldThrowProperErrorMessage() { + String outpostsUriString = "myendpoint-123456789012.s3-accesspoint.cn-northwest-1.amazonaws.com.cn"; + URI outpostsUri = URI.create(outpostsUriString); + + Exception exception = assertThrows(SdkClientException.class, () -> { + defaultUtilities.parseS3Uri(outpostsUri); + }); + + assertThat(exception.getMessage()).isEqualTo("AccessPoints URI parsing is not supported"); + } + + @Test + public void parseS3Uri_outpostsUri_shouldThrowProperErrorMessage() { + String outpostsUriString = "myaccesspoint-123456789012.op-01234567890123456.s3-outposts.us-west-2.amazonaws.com"; + URI outpostsUri = URI.create(outpostsUriString); + + Exception exception = assertThrows(SdkClientException.class, () -> { + defaultUtilities.parseS3Uri(outpostsUri); + }); + + assertThat(exception.getMessage()).isEqualTo("Outposts URI parsing is not supported"); + } + + @Test + public void parseS3Uri_outpostsUriWithChinaPartition_shouldThrowProperErrorMessage() { + String outpostsUriString = "myaccesspoint-123456789012.op-01234567890123456.s3-outposts.cn-north-1.amazonaws.com.cn"; + URI outpostsUri = URI.create(outpostsUriString); + + Exception exception = assertThrows(SdkClientException.class, () -> { + defaultUtilities.parseS3Uri(outpostsUri); + }); + + assertThat(exception.getMessage()).isEqualTo("Outposts URI parsing is not supported"); + } + @Test public void test_utilities_createdThroughS3Client() throws MalformedURLException { assertThat(defaultUtilities.getUrl(requestWithoutSpaces()) From 20309babd474ede8999aaa387692a075f570cfd6 Mon Sep 17 00:00:00 2001 From: hdavidh Date: Thu, 30 Mar 2023 12:40:22 -0700 Subject: [PATCH 02/12] S3 URI Parser --- .../awssdk/services/s3/S3Utilities.java | 329 +++++--------- .../awssdk/services/s3/parsing/S3Uri.java | 12 +- .../awssdk/services/s3/S3UtilitiesTest.java | 427 ++++++++++-------- 3 files changed, 346 insertions(+), 422 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java index daea28489f87..fd3f2d20c964 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java @@ -259,6 +259,101 @@ public URL getUrl(GetUrlRequest getUrlRequest) { } } + /** + * Returns a parsed {@link S3Uri} with which a user can easily retrieve the the bucket, key, region, style, and query + * parameters of the URI. Only basic bucket endpoints are supported, i.e., path-style and virtual-hosted-style URLs. + * Encoded buckets, keys, and query parameters will be returned decoded. + * + * @param uri The URI to be parsed + * @return Parsed {@link S3Uri} + */ + public S3Uri parseS3Uri(URI uri) { + if (uri == null) { + throw SdkClientException.create("URI must not be null"); + } + + Pattern accessPointPattern = Pattern.compile("^([a-zA-Z0-9\\-]+)\\.s3-accesspoint(-fips)?(\\.dualstack)?" + + "\\.([a-zA-Z0-9\\-]+)\\.amazonaws\\.com(.cn)?$"); + if (accessPointPattern.matcher(uri.toString()).find()) { + throw SdkClientException.create("AccessPoints URI parsing is not supported"); + } + + Pattern outpostPattern = Pattern.compile("^([a-zA-Z0-9\\-]+)\\.op\\-[0-9]+\\.s3-outposts\\.([a-zA-Z0-9\\-]+)" + + "\\.amazonaws\\.com(.cn)?$"); + if (outpostPattern.matcher(uri.toString()).find()) { + throw SdkClientException.create("Outposts URI parsing is not supported"); + } + + String bucket = null; + String key = null; + String region = null; + boolean isPathStyle = false; + Map queryParams = new HashMap<>(); + String path = uri.getPath(); + + if ("s3".equalsIgnoreCase(uri.getScheme())) { + if (uri.getAuthority() == null) { + throw SdkClientException.create("Invalid S3 URI: bucket not included"); + } + bucket = uri.getAuthority(); + if (path.length() > 1) { + key = path.substring(1); + } + + } else { + if (uri.getHost() == null) { + throw SdkClientException.create("Invalid S3 URI: hostname not included"); + } + + Pattern endpointPattern = Pattern.compile("^(.+\\.)?s3[.-]([a-z0-9-]+)\\."); + Matcher matcher = endpointPattern.matcher(uri.getHost()); + if (!matcher.find()) { + throw SdkClientException.create("Invalid S3 URI: hostname does not appear to be a valid S3 endpoint"); + } + + String prefix = matcher.group(1); + if (prefix == null || prefix.isEmpty()) { + isPathStyle = true; + + if (!path.isEmpty() && !"/".equals(path)) { + int index = path.indexOf('/', 1); + + if (index == -1) { + bucket = path.substring(1); + } else if (index == (path.length() - 1)) { + bucket = path.substring(1, index); + } else { + bucket = path.substring(1, index); + key = path.substring(index + 1); + } + } + } else { + bucket = prefix.substring(0, prefix.length() - 1); + if (path != null && !path.isEmpty() && !"/".equals(path)) { + key = path.substring(1); + } + } + + if (!"amazonaws".equals(matcher.group(2))) { + region = matcher.group(2); + } + } + + String queryPart = uri.getQuery(); + if (queryPart != null) { + parseQuery(queryParams, queryPart); + } + + return S3Uri.builder() + .uri(uri) + .bucket(bucket) + .key(key) + .region(region) + .isPathStyle(isPathStyle) + .queryParams(queryParams) + .build(); + } + private Region resolveRegionForGetUrl(GetUrlRequest getUrlRequest) { if (getUrlRequest.region() == null && this.region == null) { throw new IllegalArgumentException("Region should be provided either in GetUrlRequest object or S3Utilities object"); @@ -376,6 +471,18 @@ private UseGlobalEndpointResolver createUseGlobalEndpointResolver() { return new UseGlobalEndpointResolver(config); } + private void parseQuery(Map queryParams, String queryPart) { + String[] params = queryPart.split("&"); + for (String param: params) { + String[] keyValuePair = param.split("=", 2); + String key = keyValuePair[0]; + if (key.isEmpty()) { + continue; + } + queryParams.put(key, keyValuePair[1]); + } + } + /** * Builder class to construct {@link S3Utilities} object */ @@ -502,226 +609,4 @@ public S3Utilities build() { return new S3Utilities(this); } } - - public S3Uri parseS3Uri(URI uri) { - return parseS3Uri(uri, true); - } - - public S3Uri parseS3Uri(URI uri, boolean urlEncode) { - if (uri == null) { - throw SdkClientException.create("URI must not be null"); - } - - Pattern accessPointPattern = Pattern.compile("^([a-zA-Z0-9\\-]+)\\.s3-accesspoint(-fips)?(\\.dualstack)?" - + "\\.([a-zA-Z0-9\\-]+)\\.amazonaws\\.com(.cn)?$"); - if (accessPointPattern.matcher(uri.toString()).find()) { - throw SdkClientException.create("AccessPoints URI parsing is not supported"); - } - - Pattern outpostPattern = Pattern.compile("^([a-zA-Z0-9\\-]+)\\.op\\-[0-9]+\\.s3-outposts\\.([a-zA-Z0-9\\-]+)" - + "\\.amazonaws\\.com(.cn)?$"); - if (outpostPattern.matcher(uri.toString()).find()) { - throw SdkClientException.create("Outposts URI parsing is not supported"); - } - - String bucket = null; - String key = null; - String region = null; - boolean isPathStyle = false; - Map queryParams = new HashMap<>(); - - if ("s3".equalsIgnoreCase(uri.getScheme())) { - if (uri.getAuthority() == null) { - throw SdkClientException.create("Invalid S3 URI: bucket not included"); - } - bucket = uri.getAuthority(); - - String path = uri.getPath(); - if (path.length() > 1) { - key = uri.getPath().substring(1); - } - - } else { - if (uri.getHost() == null) { - throw SdkClientException.create("Invalid S3 URI: hostname not included"); - } - - Pattern endpointPattern = Pattern.compile("^(.+\\.)?s3[.-]([a-z0-9-]+)\\."); - Matcher matcher = endpointPattern.matcher(uri.getHost()); - if (!matcher.find()) { - throw SdkClientException.create("Invalid S3 URI: hostname does not appear to be a valid S3 endpoint"); - } - - String prefix = matcher.group(1); - if (prefix == null || prefix.isEmpty()) { - isPathStyle = true; - String path = urlEncode ? uri.getPath() : uri.getRawPath(); - - if (!"".equals(path) && !"/".equals(path)) { - int index = path.indexOf('/', 1); - - if (index == -1) { - bucket = decode(path.substring(1)); - } else if (index == (path.length() - 1)) { - bucket = decode(path.substring(1, index)); - } else { - bucket = decode(path.substring(1, index)); - key = decode(path.substring(index + 1)); - } - } - } else { - bucket = prefix.substring(0, prefix.length() - 1); - String path = uri.getPath(); - if (path != null && !path.isEmpty() && !"/".equals(uri.getPath())) { - key = uri.getPath().substring(1); - } - } - - if (!"amazonaws".equals(matcher.group(2))) { - region = matcher.group(2); - } - } - - String queryPart = uri.getRawQuery(); - if (queryPart != null) { - parseQuery(queryParams, queryPart); - } - - return S3Uri.builder() - .uri(uri) - .bucket(bucket) - .key(key) - .region(region) - .isPathStyle(isPathStyle) - .queryParams(queryParams) - .build(); - } - - private void parseQuery(Map queryParams, String queryPart) { - String[] params = queryPart.split("&"); - for (String param: params) { - try { - String[] keyValuePair = param.split("=", 2); - String key = URLDecoder.decode(keyValuePair[0], "UTF-8"); - if (key.isEmpty()) { - continue; - } - String value = URLDecoder.decode(keyValuePair[1], "UTF-8"); - queryParams.put(key, value); - } catch (UnsupportedEncodingException e) { - // Param could not be decoded - } - - } - } - - /** - * Percent-decodes the given string, with a fast path for strings that - * are not percent-encoded. - * - * @param str the string to decode - * @return the decoded string - */ - private static String decode(final String str) { - if (str == null) { - return null; - } - - for (int i = 0; i < str.length(); ++i) { - if (str.charAt(i) == '%') { - return decode(str, i); - } - } - - return str; - } - - /** - * Percent-decodes the given string. - * - * @param str the string to decode - * @param firstPercent the index of the first '%' character in the string - * @return the decoded string - */ - private static String decode(final String str, final int firstPercent) { - StringBuilder builder = new StringBuilder(); - builder.append(str.substring(0, firstPercent)); - - appendDecoded(builder, str, firstPercent); - - for (int i = firstPercent + 3; i < str.length(); ++i) { - if (str.charAt(i) == '%') { - appendDecoded(builder, str, i); - i += 2; - } else { - builder.append(str.charAt(i)); - } - } - - return builder.toString(); - } - - /** - * Decodes the percent-encoded character at the given index in the string - * and appends the decoded value to the given {@code StringBuilder}. - * - * @param builder the string builder to append to - * @param str the string being decoded - * @param index the index of the '%' character in the string - */ - private static void appendDecoded(final StringBuilder builder, - final String str, - final int index) { - - if (index > str.length() - 3) { - throw new IllegalStateException("Invalid percent-encoded string:" - + "\"" + str + "\"."); - } - - char first = str.charAt(index + 1); - char second = str.charAt(index + 2); - - char decoded = (char) ((fromHex(first) << 4) | fromHex(second)); - builder.append(decoded); - } - - /** - * Converts a hex character (0-9A-Fa-f) into its corresponding quad value. - * - * @param c the hex character - * @return the quad value - */ - private static int fromHex(final char c) { - if (c < '0') { - throw new IllegalStateException( - "Invalid percent-encoded string: bad character '" + c + "' in " - + "escape sequence."); - } - if (c <= '9') { - return (c - '0'); - } - - if (c < 'A') { - throw new IllegalStateException( - "Invalid percent-encoded string: bad character '" + c + "' in " - + "escape sequence."); - } - if (c <= 'F') { - return (c - 'A') + 10; - } - - if (c < 'a') { - throw new IllegalStateException( - "Invalid percent-encoded string: bad character '" + c + "' in " - + "escape sequence."); - } - if (c <= 'f') { - return (c - 'a') + 10; - } - - throw new IllegalStateException( - "Invalid percent-encoded string: bad character '" + c + "' in " - + "escape sequence."); - } - } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java index fc577a51ff10..adf3d8d04040 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.services.s3.parsing; import java.net.URI; +import java.util.Collections; import java.util.Map; import java.util.Objects; import software.amazon.awssdk.annotations.Immutable; @@ -25,11 +26,12 @@ /** * Object that represents a parsed S3 URI. Can be used to easily retrieve the bucket, key, region, style, and query parameters - * of the URI. Only basic buket endpoints are supported, i.e., path-style and virtual-hosted-style URIs. + * of the URI. Only basic bucket endpoints are supported, i.e., path-style and virtual-hosted style URLs. Encoded buckets, keys, + * and query parameters will be returned decoded. */ @Immutable @SdkPublicApi -public class S3Uri implements ToCopyableBuilder { +public final class S3Uri implements ToCopyableBuilder { private final URI uri; private final String bucket; @@ -78,14 +80,14 @@ public String key() { } /** - * Returns the region specified in the URI. Returns null if no region is specified, i.e., is a global endpoint. + * Returns the region specified in the URI. Returns null if no region is specified, i.e., global endpoint. */ public String region() { return region; } /** - * Returns true if the URI is path-style, false if the URI is virtual-hosted-style. + * Returns true if the URI is path-style, false if the URI is virtual-hosted style. */ public boolean isPathStyle() { return isPathStyle; @@ -95,7 +97,7 @@ public boolean isPathStyle() { * Returns a map of the query parameters specified in the URI. Returns an empty map if no queries are specified. */ public Map queryParams() { - return queryParams; + return Collections.unmodifiableMap(queryParams); } @Override diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java index b18ac19f50af..485a3fa3932d 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java @@ -20,6 +20,8 @@ import java.net.MalformedURLException; import java.net.URI; +import java.net.URLDecoder; +import java.net.URLEncoder; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -72,8 +74,155 @@ public static void cleanup() { } @Test - public void parseS3Uri_pathStyleWithRoot_shouldParseCorrectly() { - String uriString = "https://s3.amazonaws.com/"; + public void test_utilities_createdThroughS3Client() throws MalformedURLException { + assertThat(defaultUtilities.getUrl(requestWithoutSpaces()) + .toExternalForm()) + .isEqualTo("https://foo-bucket.s3.us-west-2.amazonaws.com/key-without-spaces"); + + assertThat(defaultUtilities.getUrl(requestWithSpecialCharacters()) + .toExternalForm()) + .isEqualTo("https://foo-bucket.s3.us-west-2.amazonaws.com/key%20with%40spaces"); + } + + @Test + public void test_utilities_withPathStyleAccessEnabled() throws MalformedURLException { + S3Utilities pathStyleUtilities = S3Utilities.builder() + .region(Region.US_WEST_2) + .s3Configuration(PATH_STYLE_CONFIG) + .build(); + + assertThat(pathStyleUtilities.getUrl(requestWithoutSpaces()) + .toExternalForm()) + .isEqualTo("https://s3.us-west-2.amazonaws.com/foo-bucket/key-without-spaces"); + + assertThat(pathStyleUtilities.getUrl(requestWithSpecialCharacters()) + .toExternalForm()) + .isEqualTo("https://s3.us-west-2.amazonaws.com/foo-bucket/key%20with%40spaces"); + } + + @Test + public void test_withUsEast1Region() throws MalformedURLException { + S3Utilities usEastUtilities = S3Utilities.builder().region(Region.US_EAST_1).build(); + + assertThat(usEastUtilities.getUrl(requestWithoutSpaces()) + .toExternalForm()) + .isEqualTo("https://foo-bucket.s3.amazonaws.com/key-without-spaces"); + + assertThat(usEastUtilities.getUrl(requestWithSpecialCharacters()).toExternalForm()) + .isEqualTo("https://foo-bucket.s3.amazonaws.com/key%20with%40spaces"); + } + + @Test + public void test_RegionOnRequestTakesPrecendence() throws MalformedURLException { + S3Utilities utilities = S3Utilities.builder().region(Region.US_WEST_2).build(); + + assertThat(utilities.getUrl(b -> b.bucket("foo-bucket") + .key("key-without-spaces") + .region(Region.US_EAST_1)) + .toExternalForm()) + .isEqualTo("https://foo-bucket.s3.amazonaws.com/key-without-spaces"); + } + + @Test + public void test_EndpointOnRequestTakesPrecendence() throws MalformedURLException { + assertThat(defaultUtilities.getUrl(GetUrlRequest.builder() + .bucket("foo-bucket") + .key("key-without-spaces") + .endpoint(US_EAST_1_URI) + .build()) + .toExternalForm()) + .isEqualTo("https://foo-bucket.s3.amazonaws.com/key-without-spaces"); + } + + @Test + public void test_EndpointOverrideOnClientWorks() { + S3Utilities customizeUtilities = S3Client.builder() + .endpointOverride(URI.create("https://s3.custom.host")) + .build() + .utilities(); + assertThat(customizeUtilities.getUrl(GetUrlRequest.builder() + .bucket("foo-bucket") + .key("key-without-spaces") + .build()) + .toExternalForm()) + .isEqualTo("https://foo-bucket.s3.custom.host/key-without-spaces"); + } + + @Test + public void testWithAccelerateAndDualStackEnabled() throws MalformedURLException { + S3Utilities utilities = S3Client.builder() + .credentialsProvider(dummyCreds()) + .region(Region.US_WEST_2) + .serviceConfiguration(ACCELERATE_AND_DUALSTACK_ENABLED) + .build() + .utilities(); + + assertThat(utilities.getUrl(requestWithSpecialCharacters()) + .toExternalForm()) + .isEqualTo("https://foo-bucket.s3-accelerate.dualstack.amazonaws.com/key%20with%40spaces"); + } + + @Test + public void testWithAccelerateAndDualStackViaClientEnabled() throws MalformedURLException { + S3Utilities utilities = S3Client.builder() + .credentialsProvider(dummyCreds()) + .region(Region.US_WEST_2) + .serviceConfiguration(S3Configuration.builder() + .accelerateModeEnabled(true) + .build()) + .dualstackEnabled(true) + .build() + .utilities(); + + assertThat(utilities.getUrl(requestWithSpecialCharacters()) + .toExternalForm()) + .isEqualTo("https://foo-bucket.s3-accelerate.dualstack.amazonaws.com/key%20with%40spaces"); + } + + @Test + public void testWithDualStackViaUtilitiesBuilderEnabled() throws MalformedURLException { + S3Utilities utilities = S3Utilities.builder() + .region(Region.US_WEST_2) + .dualstackEnabled(true) + .build(); + + assertThat(utilities.getUrl(requestWithSpecialCharacters()) + .toExternalForm()) + .isEqualTo("https://foo-bucket.s3.dualstack.us-west-2.amazonaws.com/key%20with%40spaces"); + } + + @Test + public void testAsync() throws MalformedURLException { + assertThat(utilitiesFromAsyncClient.getUrl(requestWithoutSpaces()) + .toExternalForm()) + .isEqualTo("https://foo-bucket.s3.ap-northeast-2.amazonaws.com/key-without-spaces"); + + assertThat(utilitiesFromAsyncClient.getUrl(requestWithSpecialCharacters()) + .toExternalForm()) + .isEqualTo("https://foo-bucket.s3.ap-northeast-2.amazonaws.com/key%20with%40spaces"); + } + + @Test (expected = NullPointerException.class) + public void failIfRegionIsNotSetOnS3UtilitiesObject() throws MalformedURLException { + S3Utilities.builder().build(); + } + + @Test + public void getUrlWithVersionId() { + S3Utilities utilities = S3Utilities.builder().region(Region.US_WEST_2).build(); + + assertThat(utilities.getUrl(b -> b.bucket("foo").key("bar").versionId("1")) + .toExternalForm()) + .isEqualTo("https://foo.s3.us-west-2.amazonaws.com/bar?versionId=1"); + + assertThat(utilities.getUrl(b -> b.bucket("foo").key("bar").versionId("@1")) + .toExternalForm()) + .isEqualTo("https://foo.s3.us-west-2.amazonaws.com/bar?versionId=%401"); + } + + @Test + public void parseS3Uri_rootUri_shouldParseCorrectly() { + String uriString = "https://s3.amazonaws.com"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); @@ -86,8 +235,8 @@ public void parseS3Uri_pathStyleWithRoot_shouldParseCorrectly() { } @Test - public void parseS3Uri_pathStyleWithRootNoTrailingSlash_shouldParseCorrectly() { - String uriString = "https://s3.amazonaws.com"; + public void parseS3Uri_rootUriTrailingSlash_shouldParseCorrectly() { + String uriString = "https://s3.amazonaws.com/"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); @@ -99,6 +248,20 @@ public void parseS3Uri_pathStyleWithRootNoTrailingSlash_shouldParseCorrectly() { assertThat(s3Uri.queryParams()).isEmpty(); } + @Test + public void parseS3Uri_pathStyleTrailingSlash_shouldParseCorrectly() { + String uriString = "https://s3.us-east-1.amazonaws.com/myBucket/"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isNull(); + assertThat(s3Uri.region()).isEqualTo("us-east-1"); + assertThat(s3Uri.isPathStyle()).isTrue(); + assertThat(s3Uri.queryParams()).isEmpty(); + } + @Test public void parseS3Uri_pathStyleGlobalEndpoint_shouldParseCorrectly() { String uriString = "https://s3.amazonaws.com/myBucket/resources/image1.png"; @@ -157,12 +320,12 @@ public void parseS3Uri_pathStyleWithDash_shouldParseCorrectly() { @Test public void parseS3Uri_virtualHostedStyleWithDot_shouldParseCorrectly() { - String uriString = "https://my-bucket.s3.us-east-2.amazonaws.com/image.png"; + String uriString = "https://myBucket.s3.us-east-2.amazonaws.com/image.png"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("my-bucket"); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isEqualTo("image.png"); assertThat(s3Uri.region()).isEqualTo("us-east-2"); assertThat(s3Uri.isPathStyle()).isFalse(); @@ -171,12 +334,12 @@ public void parseS3Uri_virtualHostedStyleWithDot_shouldParseCorrectly() { @Test public void parseS3Uri_virtualHostedStyleWithDash_shouldParseCorrectly() { - String uriString = "https://my-bucket.s3-us-east-2.amazonaws.com/image.png"; + String uriString = "https://myBucket.s3-us-east-2.amazonaws.com/image.png"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("my-bucket"); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isEqualTo("image.png"); assertThat(s3Uri.region()).isEqualTo("us-east-2"); assertThat(s3Uri.isPathStyle()).isFalse(); @@ -184,93 +347,93 @@ public void parseS3Uri_virtualHostedStyleWithDash_shouldParseCorrectly() { } @Test - public void parses3Uri_pathStyleWithQuery_shouldParseCorrectly() { - String uriString = "https://s3.us-west-1.amazonaws.com/bucket/key?versionId=abc123"; + public void parseS3Uri_pathStyleWithQuery_shouldParseCorrectly() { + String uriString = "https://s3.us-west-1.amazonaws.com/myBucket/doc.txt?versionId=abc123"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("bucket"); - assertThat(s3Uri.key()).isEqualTo("key"); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isEqualTo("doc.txt"); assertThat(s3Uri.region()).isEqualTo("us-west-1"); assertThat(s3Uri.isPathStyle()).isTrue(); assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); } @Test - public void parses3Uri_pathStyleWithEncodedQuery_shouldParseCorrectly() { - String uriString = "https://s3.us-west-1.amazonaws.com/bucket/key?versionId=%61%62%63%31%32%33"; + public void parseS3Uri_pathStyleWithMultipleQueries_shouldParseCorrectly() { + String uriString = "https://s3.us-west-1.amazonaws.com/myBucket/doc.txt?versionId=abc123&partNumber=77"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("bucket"); - assertThat(s3Uri.key()).isEqualTo("key"); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isEqualTo("doc.txt"); assertThat(s3Uri.region()).isEqualTo("us-west-1"); assertThat(s3Uri.isPathStyle()).isTrue(); assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); + assertThat(s3Uri.queryParams()).containsEntry("partNumber", "77"); } @Test - public void parses3Uri_pathStyleWithMultipleQueries_shouldParseCorrectly() { - String uriString = "https://s3.us-west-1.amazonaws.com/bucket/key?versionId=abc123&partNumber=77"; + public void parseS3Uri_pathStyleWithEncoding_shouldParseCorrectly() { + String uriString = "https://s3.us-west-1.amazonaws.com/my%40Bucket/object%20key?versionId%3D%61%62%63%31%32%33"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("bucket"); - assertThat(s3Uri.key()).isEqualTo("key"); + assertThat(s3Uri.bucket()).isEqualTo("my@Bucket"); + assertThat(s3Uri.key()).isEqualTo("object key"); assertThat(s3Uri.region()).isEqualTo("us-west-1"); assertThat(s3Uri.isPathStyle()).isTrue(); assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); - assertThat(s3Uri.queryParams()).containsEntry("partNumber", "77"); } @Test - public void parses3Uri_virtualStyleWithQuery_shouldParseCorrectly() { - String uriString = "https://bucket.s3.us-west-1.amazonaws.com/key?versionId=abc123"; + public void parseS3Uri_virtualStyleWithQuery_shouldParseCorrectly() { + String uriString = "https://myBucket.s3.us-west-1.amazonaws.com/doc.txt?versionId=abc123"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("bucket"); - assertThat(s3Uri.key()).isEqualTo("key"); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isEqualTo("doc.txt"); assertThat(s3Uri.region()).isEqualTo("us-west-1"); assertThat(s3Uri.isPathStyle()).isFalse(); assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); } @Test - public void parses3Uri_virtualStyleWithEncodedQuery_shouldParseCorrectly() { - String uriString = "https://bucket.s3.us-west-1.amazonaws.com/key?versionId=%61%62%63%31%32%33"; + public void parseS3Uri_virtualStyleWithMultipleQueries_shouldParseCorrectly() { + String uriString = "https://myBucket.s3.us-west-1.amazonaws.com/doc.txt?versionId=abc123&partNumber=77"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("bucket"); - assertThat(s3Uri.key()).isEqualTo("key"); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isEqualTo("doc.txt"); assertThat(s3Uri.region()).isEqualTo("us-west-1"); assertThat(s3Uri.isPathStyle()).isFalse(); assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); + assertThat(s3Uri.queryParams()).containsEntry("partNumber", "77"); } @Test - public void parses3Uri_virtualStyleWithMultipleQueries_shouldParseCorrectly() { - String uriString = "https://bucket.s3.us-west-1.amazonaws.com/key?versionId=abc123&partNumber=77"; + public void parseS3Uri_virtualStyleWithEncoding_shouldParseCorrectly() { + String uriString = "https://myBucket.s3.us-west-1.amazonaws.com/object%20key?versionId%3D%61%62%63%31%32%33"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("bucket"); - assertThat(s3Uri.key()).isEqualTo("key"); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isEqualTo("object key"); assertThat(s3Uri.region()).isEqualTo("us-west-1"); assertThat(s3Uri.isPathStyle()).isFalse(); assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); - assertThat(s3Uri.queryParams()).containsEntry("partNumber", "77"); } @Test - public void parses3Uri_virtualStyleS3SchemeWithoutKey_shouldParseCorrectly() { + public void parseS3Uri_virtualStyleS3SchemeWithoutKey_shouldParseCorrectly() { String uriString = "s3://myBucket"; URI uri = URI.create(uriString); @@ -283,7 +446,20 @@ public void parses3Uri_virtualStyleS3SchemeWithoutKey_shouldParseCorrectly() { } @Test - public void parses3Uri_virtualStyleS3SchemeWithKey_shouldParseCorrectly() { + public void parseS3Uri_virtualStyleS3SchemeWithoutKeyWithTrailingSlash_shouldParseCorrectly() { + String uriString = "s3://myBucket/"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isNull(); + assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.isPathStyle()).isFalse(); + } + + @Test + public void parseS3Uri_virtualStyleS3SchemeWithKey_shouldParseCorrectly() { String uriString = "s3://myBucket/resources/key"; URI uri = URI.create(uriString); @@ -295,6 +471,19 @@ public void parses3Uri_virtualStyleS3SchemeWithKey_shouldParseCorrectly() { assertThat(s3Uri.isPathStyle()).isFalse(); } + @Test + public void parseS3Uri_virtualStyleS3SchemeWithEncoding_shouldParseCorrectly() { + String uriString = "s3://my%40Bucket/object%20key"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("my@Bucket"); + assertThat(s3Uri.key()).isEqualTo("object key"); + assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.isPathStyle()).isFalse(); + } + @Test public void parseS3Uri_accessPointUri_shouldThrowProperErrorMessage() { String accessPointUriString = "myendpoint-123456789012.s3-accesspoint.us-east-1.amazonaws.com"; @@ -303,7 +492,6 @@ public void parseS3Uri_accessPointUri_shouldThrowProperErrorMessage() { Exception exception = assertThrows(SdkClientException.class, () -> { defaultUtilities.parseS3Uri(accessPointUri); }); - assertThat(exception.getMessage()).isEqualTo("AccessPoints URI parsing is not supported"); } @@ -315,19 +503,6 @@ public void parseS3Uri_accessPointUriWithFipsDualstack_shouldThrowProperErrorMes Exception exception = assertThrows(SdkClientException.class, () -> { defaultUtilities.parseS3Uri(accessPointUri); }); - - assertThat(exception.getMessage()).isEqualTo("AccessPoints URI parsing is not supported"); - } - - @Test - public void parseS3Uri_accessPointsUriWithChinaPartition_shouldThrowProperErrorMessage() { - String outpostsUriString = "myendpoint-123456789012.s3-accesspoint.cn-northwest-1.amazonaws.com.cn"; - URI outpostsUri = URI.create(outpostsUriString); - - Exception exception = assertThrows(SdkClientException.class, () -> { - defaultUtilities.parseS3Uri(outpostsUri); - }); - assertThat(exception.getMessage()).isEqualTo("AccessPoints URI parsing is not supported"); } @@ -339,7 +514,6 @@ public void parseS3Uri_outpostsUri_shouldThrowProperErrorMessage() { Exception exception = assertThrows(SdkClientException.class, () -> { defaultUtilities.parseS3Uri(outpostsUri); }); - assertThat(exception.getMessage()).isEqualTo("Outposts URI parsing is not supported"); } @@ -351,155 +525,18 @@ public void parseS3Uri_outpostsUriWithChinaPartition_shouldThrowProperErrorMessa Exception exception = assertThrows(SdkClientException.class, () -> { defaultUtilities.parseS3Uri(outpostsUri); }); - assertThat(exception.getMessage()).isEqualTo("Outposts URI parsing is not supported"); } @Test - public void test_utilities_createdThroughS3Client() throws MalformedURLException { - assertThat(defaultUtilities.getUrl(requestWithoutSpaces()) - .toExternalForm()) - .isEqualTo("https://foo-bucket.s3.us-west-2.amazonaws.com/key-without-spaces"); - - assertThat(defaultUtilities.getUrl(requestWithSpecialCharacters()) - .toExternalForm()) - .isEqualTo("https://foo-bucket.s3.us-west-2.amazonaws.com/key%20with%40spaces"); - } - - @Test - public void test_utilities_withPathStyleAccessEnabled() throws MalformedURLException { - S3Utilities pathStyleUtilities = S3Utilities.builder() - .region(Region.US_WEST_2) - .s3Configuration(PATH_STYLE_CONFIG) - .build(); - - assertThat(pathStyleUtilities.getUrl(requestWithoutSpaces()) - .toExternalForm()) - .isEqualTo("https://s3.us-west-2.amazonaws.com/foo-bucket/key-without-spaces"); - - assertThat(pathStyleUtilities.getUrl(requestWithSpecialCharacters()) - .toExternalForm()) - .isEqualTo("https://s3.us-west-2.amazonaws.com/foo-bucket/key%20with%40spaces"); - } - - @Test - public void test_withUsEast1Region() throws MalformedURLException { - S3Utilities usEastUtilities = S3Utilities.builder().region(Region.US_EAST_1).build(); - - assertThat(usEastUtilities.getUrl(requestWithoutSpaces()) - .toExternalForm()) - .isEqualTo("https://foo-bucket.s3.amazonaws.com/key-without-spaces"); - - assertThat(usEastUtilities.getUrl(requestWithSpecialCharacters()).toExternalForm()) - .isEqualTo("https://foo-bucket.s3.amazonaws.com/key%20with%40spaces"); - } - - @Test - public void test_RegionOnRequestTakesPrecendence() throws MalformedURLException { - S3Utilities utilities = S3Utilities.builder().region(Region.US_WEST_2).build(); - - assertThat(utilities.getUrl(b -> b.bucket("foo-bucket") - .key("key-without-spaces") - .region(Region.US_EAST_1)) - .toExternalForm()) - .isEqualTo("https://foo-bucket.s3.amazonaws.com/key-without-spaces"); - } - - @Test - public void test_EndpointOnRequestTakesPrecendence() throws MalformedURLException { - assertThat(defaultUtilities.getUrl(GetUrlRequest.builder() - .bucket("foo-bucket") - .key("key-without-spaces") - .endpoint(US_EAST_1_URI) - .build()) - .toExternalForm()) - .isEqualTo("https://foo-bucket.s3.amazonaws.com/key-without-spaces"); - } - - @Test - public void test_EndpointOverrideOnClientWorks() { - S3Utilities customizeUtilities = S3Client.builder() - .endpointOverride(URI.create("https://s3.custom.host")) - .build() - .utilities(); - assertThat(customizeUtilities.getUrl(GetUrlRequest.builder() - .bucket("foo-bucket") - .key("key-without-spaces") - .build()) - .toExternalForm()) - .isEqualTo("https://foo-bucket.s3.custom.host/key-without-spaces"); - } - - @Test - public void testWithAccelerateAndDualStackEnabled() throws MalformedURLException { - S3Utilities utilities = S3Client.builder() - .credentialsProvider(dummyCreds()) - .region(Region.US_WEST_2) - .serviceConfiguration(ACCELERATE_AND_DUALSTACK_ENABLED) - .build() - .utilities(); - - assertThat(utilities.getUrl(requestWithSpecialCharacters()) - .toExternalForm()) - .isEqualTo("https://foo-bucket.s3-accelerate.dualstack.amazonaws.com/key%20with%40spaces"); - } - - @Test - public void testWithAccelerateAndDualStackViaClientEnabled() throws MalformedURLException { - S3Utilities utilities = S3Client.builder() - .credentialsProvider(dummyCreds()) - .region(Region.US_WEST_2) - .serviceConfiguration(S3Configuration.builder() - .accelerateModeEnabled(true) - .build()) - .dualstackEnabled(true) - .build() - .utilities(); - - assertThat(utilities.getUrl(requestWithSpecialCharacters()) - .toExternalForm()) - .isEqualTo("https://foo-bucket.s3-accelerate.dualstack.amazonaws.com/key%20with%40spaces"); - } - - @Test - public void testWithDualStackViaUtilitiesBuilderEnabled() throws MalformedURLException { - S3Utilities utilities = S3Utilities.builder() - .region(Region.US_WEST_2) - .dualstackEnabled(true) - .build(); - - assertThat(utilities.getUrl(requestWithSpecialCharacters()) - .toExternalForm()) - .isEqualTo("https://foo-bucket.s3.dualstack.us-west-2.amazonaws.com/key%20with%40spaces"); - } - - @Test - public void testAsync() throws MalformedURLException { - assertThat(utilitiesFromAsyncClient.getUrl(requestWithoutSpaces()) - .toExternalForm()) - .isEqualTo("https://foo-bucket.s3.ap-northeast-2.amazonaws.com/key-without-spaces"); - - assertThat(utilitiesFromAsyncClient.getUrl(requestWithSpecialCharacters()) - .toExternalForm()) - .isEqualTo("https://foo-bucket.s3.ap-northeast-2.amazonaws.com/key%20with%40spaces"); - } - - @Test (expected = NullPointerException.class) - public void failIfRegionIsNotSetOnS3UtilitiesObject() throws MalformedURLException { - S3Utilities.builder().build(); - } - - @Test - public void getUrlWithVersionId() { - S3Utilities utilities = S3Utilities.builder().region(Region.US_WEST_2).build(); - - assertThat(utilities.getUrl(b -> b.bucket("foo").key("bar").versionId("1")) - .toExternalForm()) - .isEqualTo("https://foo.s3.us-west-2.amazonaws.com/bar?versionId=1"); + public void parseS3Uri_withNonS3Uri_shouldThrowProperErrorMessage() { + String nonS3UriString = "https://www.amazon.com/"; + URI nonS3Uri = URI.create(nonS3UriString); - assertThat(utilities.getUrl(b -> b.bucket("foo").key("bar").versionId("@1")) - .toExternalForm()) - .isEqualTo("https://foo.s3.us-west-2.amazonaws.com/bar?versionId=%401"); + Exception exception = assertThrows(SdkClientException.class, () -> { + defaultUtilities.parseS3Uri(nonS3Uri); + }); + assertThat(exception.getMessage()).isEqualTo("Invalid S3 URI: hostname does not appear to be a valid S3 endpoint"); } private static GetUrlRequest requestWithoutSpaces() { From 996ac35fa2b76f4dce4d7b1ac15a15db8da28671 Mon Sep 17 00:00:00 2001 From: hdavidh Date: Thu, 30 Mar 2023 12:41:29 -0700 Subject: [PATCH 03/12] S3 URI Parser --- .../java/software/amazon/awssdk/services/s3/S3Utilities.java | 2 -- .../software/amazon/awssdk/services/s3/S3UtilitiesTest.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java index fd3f2d20c964..1858b694c5f4 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java @@ -16,11 +16,9 @@ package software.amazon.awssdk.services.s3; -import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; -import java.net.URLDecoder; import java.util.ArrayList; import java.util.HashMap; import java.util.List; diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java index 485a3fa3932d..58c7791329dd 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java @@ -20,8 +20,6 @@ import java.net.MalformedURLException; import java.net.URI; -import java.net.URLDecoder; -import java.net.URLEncoder; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; From 68a79dedf2c4100536c65d2287cf525ed46d0ae5 Mon Sep 17 00:00:00 2001 From: hdavidh Date: Thu, 30 Mar 2023 13:04:48 -0700 Subject: [PATCH 04/12] S3 URI Parser --- .changes/next-release/feature-AmazonS3-92ece24.json | 6 ++++++ .../software/amazon/awssdk/services/s3/S3Utilities.java | 6 +++--- .../software/amazon/awssdk/services/s3/parsing/S3Uri.java | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 .changes/next-release/feature-AmazonS3-92ece24.json diff --git a/.changes/next-release/feature-AmazonS3-92ece24.json b/.changes/next-release/feature-AmazonS3-92ece24.json new file mode 100644 index 000000000000..1373cde29d22 --- /dev/null +++ b/.changes/next-release/feature-AmazonS3-92ece24.json @@ -0,0 +1,6 @@ +{ + "category": "Amazon S3", + "contributor": "", + "type": "feature", + "description": "Adding feature for parsing S3 URIs" +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java index 1858b694c5f4..4b7795fc25e9 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java @@ -318,11 +318,11 @@ public S3Uri parseS3Uri(URI uri) { if (index == -1) { bucket = path.substring(1); - } else if (index == (path.length() - 1)) { - bucket = path.substring(1, index); } else { bucket = path.substring(1, index); - key = path.substring(index + 1); + if (index != path.length() - 1) { + key = path.substring(index + 1); + } } } } else { diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java index adf3d8d04040..023cdd7f58ca 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java @@ -59,7 +59,7 @@ public Builder toBuilder() { } /** - * Returns the original URI that was used to instantiate the S3Uri + * Returns the original URI that was used to instantiate the {@link S3Uri} */ public URI uri() { return uri; From 40fd6c63d7ab1bcc0e3acc7a38621eac3fc50917 Mon Sep 17 00:00:00 2001 From: hdavidh Date: Fri, 31 Mar 2023 13:39:29 -0700 Subject: [PATCH 05/12] Refactoring --- .../awssdk/services/s3/S3Utilities.java | 16 +- .../awssdk/services/s3/parsing/S3Uri.java | 45 +++++- .../awssdk/services/s3/S3UtilitiesTest.java | 147 +++++++++++++----- 3 files changed, 152 insertions(+), 56 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java index 4b7795fc25e9..e3c94cacde9c 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java @@ -20,6 +20,7 @@ import java.net.URI; import java.net.URL; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -265,7 +266,7 @@ public URL getUrl(GetUrlRequest getUrlRequest) { * @param uri The URI to be parsed * @return Parsed {@link S3Uri} */ - public S3Uri parseS3Uri(URI uri) { + public S3Uri parseUri(URI uri) { if (uri == null) { throw SdkClientException.create("URI must not be null"); } @@ -286,7 +287,7 @@ public S3Uri parseS3Uri(URI uri) { String key = null; String region = null; boolean isPathStyle = false; - Map queryParams = new HashMap<>(); + Map> queryParams = new HashMap<>(); String path = uri.getPath(); if ("s3".equalsIgnoreCase(uri.getScheme())) { @@ -342,11 +343,13 @@ public S3Uri parseS3Uri(URI uri) { parseQuery(queryParams, queryPart); } + Region uriRegion = region != null ? Region.of(region) : null; + return S3Uri.builder() .uri(uri) .bucket(bucket) .key(key) - .region(region) + .region(uriRegion) .isPathStyle(isPathStyle) .queryParams(queryParams) .build(); @@ -469,7 +472,7 @@ private UseGlobalEndpointResolver createUseGlobalEndpointResolver() { return new UseGlobalEndpointResolver(config); } - private void parseQuery(Map queryParams, String queryPart) { + private void parseQuery(Map> queryParams, String queryPart) { String[] params = queryPart.split("&"); for (String param: params) { String[] keyValuePair = param.split("=", 2); @@ -477,7 +480,10 @@ private void parseQuery(Map queryParams, String queryPart) { if (key.isEmpty()) { continue; } - queryParams.put(key, keyValuePair[1]); + List paramValues = queryParams.containsKey(key) ? queryParams.get(key) : new ArrayList<>(); + String[] valuesPart = keyValuePair[1].split(","); + Collections.addAll(paramValues, valuesPart); + queryParams.put(key, paramValues); } } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java index 023cdd7f58ca..58542478ccc1 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java @@ -16,11 +16,15 @@ package software.amazon.awssdk.services.s3.parsing; import java.net.URI; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Objects; import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.utils.builder.CopyableBuilder; import software.amazon.awssdk.utils.builder.ToCopyableBuilder; @@ -36,9 +40,9 @@ public final class S3Uri implements ToCopyableBuilder { private final URI uri; private final String bucket; private final String key; - private final String region; + private final Region region; private final boolean isPathStyle; - private final Map queryParams; + private final Map> queryParams; private S3Uri(Builder builder) { this.uri = builder.uri; @@ -82,7 +86,7 @@ public String key() { /** * Returns the region specified in the URI. Returns null if no region is specified, i.e., global endpoint. */ - public String region() { + public Region region() { return region; } @@ -96,10 +100,35 @@ public boolean isPathStyle() { /** * Returns a map of the query parameters specified in the URI. Returns an empty map if no queries are specified. */ - public Map queryParams() { + public Map> queryParams() { return Collections.unmodifiableMap(queryParams); } + /** + * Returns the list of values for a specified query parameter. A {@link SdkClientException} is thrown if the URI does not + * contain the specified query parameter. + */ + public List queryParamValues(String key) { + List queryValues = queryParams.get(key); + if (queryValues == null) { + throw SdkClientException.create("The URI does not contain the specified query parameter."); + } + List queryValuesCopy = Arrays.asList(new String[queryValues.size()]); + Collections.copy(queryValuesCopy, queryValues); + return queryValuesCopy; + } + + /** + * Returns the value for the specified query parameter. If there are multiple values for the query parameter, the first + * value is returned. A {@link SdkClientException} is thrown if the URI does not contain the specified query parameter. + */ + public String queryParamValue(String key) { + if (queryParams().get(key) == null) { + throw SdkClientException.create("The URI does not contain the specified query parameter."); + } + return queryParams.get(key).get(0); + } + @Override public String toString() { return uri.toString(); @@ -139,9 +168,9 @@ public static final class Builder implements CopyableBuilder { private URI uri; private String bucket; private String key; - private String region; + private Region region; private boolean isPathStyle; - private Map queryParams; + private Map> queryParams; private Builder() { } @@ -182,7 +211,7 @@ public Builder key(String key) { /** * Configure the region */ - public Builder region(String region) { + public Builder region(Region region) { this.region = region; return this; } @@ -198,7 +227,7 @@ public Builder isPathStyle(boolean isPathStyle) { /** * Configure the map of query parameters */ - public Builder queryParams(Map queryParams) { + public Builder queryParams(Map> queryParams) { this.queryParams = queryParams; return this; } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java index 58c7791329dd..2ba157f1d538 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java @@ -223,7 +223,7 @@ public void parseS3Uri_rootUri_shouldParseCorrectly() { String uriString = "https://s3.amazonaws.com"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isNull(); assertThat(s3Uri.key()).isNull(); @@ -237,7 +237,7 @@ public void parseS3Uri_rootUriTrailingSlash_shouldParseCorrectly() { String uriString = "https://s3.amazonaws.com/"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isNull(); assertThat(s3Uri.key()).isNull(); @@ -251,11 +251,11 @@ public void parseS3Uri_pathStyleTrailingSlash_shouldParseCorrectly() { String uriString = "https://s3.us-east-1.amazonaws.com/myBucket/"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isNull(); - assertThat(s3Uri.region()).isEqualTo("us-east-1"); + assertThat(s3Uri.region()).isEqualTo(Region.US_EAST_1); assertThat(s3Uri.isPathStyle()).isTrue(); assertThat(s3Uri.queryParams()).isEmpty(); } @@ -265,7 +265,7 @@ public void parseS3Uri_pathStyleGlobalEndpoint_shouldParseCorrectly() { String uriString = "https://s3.amazonaws.com/myBucket/resources/image1.png"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isEqualTo("resources/image1.png"); @@ -279,7 +279,7 @@ public void parseS3Uri_virtualStyleGlobalEndpoint_shouldParseCorrectly() { String uriString = "https://myBucket.s3.amazonaws.com/resources/image1.png"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isEqualTo("resources/image1.png"); @@ -293,11 +293,11 @@ public void parseS3Uri_pathStyleWithDot_shouldParseCorrectly() { String uriString = "https://s3.eu-west-2.amazonaws.com/myBucket/resources/image1.png"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isEqualTo("resources/image1.png"); - assertThat(s3Uri.region()).isEqualTo("eu-west-2"); + assertThat(s3Uri.region()).isEqualTo(Region.EU_WEST_2); assertThat(s3Uri.isPathStyle()).isTrue(); assertThat(s3Uri.queryParams()).isEmpty(); } @@ -307,11 +307,11 @@ public void parseS3Uri_pathStyleWithDash_shouldParseCorrectly() { String uriString = "https://s3-eu-west-2.amazonaws.com/myBucket/resources/image1.png"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isEqualTo("resources/image1.png"); - assertThat(s3Uri.region()).isEqualTo("eu-west-2"); + assertThat(s3Uri.region()).isEqualTo(Region.EU_WEST_2); assertThat(s3Uri.isPathStyle()).isTrue(); assertThat(s3Uri.queryParams()).isEmpty(); } @@ -321,11 +321,11 @@ public void parseS3Uri_virtualHostedStyleWithDot_shouldParseCorrectly() { String uriString = "https://myBucket.s3.us-east-2.amazonaws.com/image.png"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isEqualTo("image.png"); - assertThat(s3Uri.region()).isEqualTo("us-east-2"); + assertThat(s3Uri.region()).isEqualTo(Region.US_EAST_2); assertThat(s3Uri.isPathStyle()).isFalse(); assertThat(s3Uri.queryParams()).isEmpty(); } @@ -335,11 +335,11 @@ public void parseS3Uri_virtualHostedStyleWithDash_shouldParseCorrectly() { String uriString = "https://myBucket.s3-us-east-2.amazonaws.com/image.png"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isEqualTo("image.png"); - assertThat(s3Uri.region()).isEqualTo("us-east-2"); + assertThat(s3Uri.region()).isEqualTo(Region.US_EAST_2); assertThat(s3Uri.isPathStyle()).isFalse(); assertThat(s3Uri.queryParams()).isEmpty(); } @@ -349,13 +349,44 @@ public void parseS3Uri_pathStyleWithQuery_shouldParseCorrectly() { String uriString = "https://s3.us-west-1.amazonaws.com/myBucket/doc.txt?versionId=abc123"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isEqualTo("doc.txt"); - assertThat(s3Uri.region()).isEqualTo("us-west-1"); + assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); + assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + } + + @Test + public void parseS3Uri_pathStyleWithQueryMultipleValuesAmpersand_shouldParseCorrectly() { + String uriString = "https://s3.us-west-1.amazonaws.com/myBucket/doc.txt?versionId=abc123&versionId=def456"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseUri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isEqualTo("doc.txt"); + assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); + assertThat(s3Uri.isPathStyle()).isTrue(); + assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + assertThat(s3Uri.queryParamValues("versionId")).contains("def456"); + } + + @Test + public void parseS3Uri_pathStyleWithQueryMultipleValuesCommas_shouldParseCorrectly() { + String uriString = "https://s3.us-west-1.amazonaws.com/myBucket/doc.txt?versionId=abc123,def456,ghi789"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseUri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isEqualTo("doc.txt"); + assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); + assertThat(s3Uri.isPathStyle()).isTrue(); + assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + assertThat(s3Uri.queryParamValues("versionId")).contains("def456"); + assertThat(s3Uri.queryParamValues("versionId")).contains("ghi789"); } @Test @@ -363,14 +394,14 @@ public void parseS3Uri_pathStyleWithMultipleQueries_shouldParseCorrectly() { String uriString = "https://s3.us-west-1.amazonaws.com/myBucket/doc.txt?versionId=abc123&partNumber=77"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isEqualTo("doc.txt"); - assertThat(s3Uri.region()).isEqualTo("us-west-1"); + assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); - assertThat(s3Uri.queryParams()).containsEntry("partNumber", "77"); + assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + assertThat(s3Uri.queryParamValue("partNumber")).isEqualTo("77"); } @Test @@ -378,13 +409,13 @@ public void parseS3Uri_pathStyleWithEncoding_shouldParseCorrectly() { String uriString = "https://s3.us-west-1.amazonaws.com/my%40Bucket/object%20key?versionId%3D%61%62%63%31%32%33"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("my@Bucket"); assertThat(s3Uri.key()).isEqualTo("object key"); - assertThat(s3Uri.region()).isEqualTo("us-west-1"); + assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); + assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); } @Test @@ -392,13 +423,43 @@ public void parseS3Uri_virtualStyleWithQuery_shouldParseCorrectly() { String uriString = "https://myBucket.s3.us-west-1.amazonaws.com/doc.txt?versionId=abc123"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isEqualTo("doc.txt"); + assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); + assertThat(s3Uri.isPathStyle()).isFalse(); + assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + } + @Test + public void parseS3Uri_virtualStyleWithQueryMultipleValuesAmpersand_shouldParseCorrectly() { + String uriString = "https://myBucket.s3.us-west-1.amazonaws.com/doc.txt?versionId=abc123&versionId=def456"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseUri(uri); + assertThat(s3Uri.uri()).isEqualTo(uri); + assertThat(s3Uri.bucket()).isEqualTo("myBucket"); + assertThat(s3Uri.key()).isEqualTo("doc.txt"); + assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); + assertThat(s3Uri.isPathStyle()).isFalse(); + assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + assertThat(s3Uri.queryParamValues("versionId")).contains("def456"); + } + + @Test + public void parseS3Uri_virtualStyleWithQueryMultipleValuesCommas_shouldParseCorrectly() { + String uriString = "https://myBucket.s3.us-west-1.amazonaws.com/doc.txt?versionId=abc123,def456,ghi789"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isEqualTo("doc.txt"); - assertThat(s3Uri.region()).isEqualTo("us-west-1"); + assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); + assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + assertThat(s3Uri.queryParamValues("versionId")).contains("def456"); + assertThat(s3Uri.queryParamValues("versionId")).contains("ghi789"); } @Test @@ -406,14 +467,14 @@ public void parseS3Uri_virtualStyleWithMultipleQueries_shouldParseCorrectly() { String uriString = "https://myBucket.s3.us-west-1.amazonaws.com/doc.txt?versionId=abc123&partNumber=77"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isEqualTo("doc.txt"); - assertThat(s3Uri.region()).isEqualTo("us-west-1"); + assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); - assertThat(s3Uri.queryParams()).containsEntry("partNumber", "77"); + assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + assertThat(s3Uri.queryParamValue("partNumber")).isEqualTo("77"); } @Test @@ -421,13 +482,13 @@ public void parseS3Uri_virtualStyleWithEncoding_shouldParseCorrectly() { String uriString = "https://myBucket.s3.us-west-1.amazonaws.com/object%20key?versionId%3D%61%62%63%31%32%33"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isEqualTo("object key"); - assertThat(s3Uri.region()).isEqualTo("us-west-1"); + assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.queryParams()).containsEntry("versionId", "abc123"); + assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); } @Test @@ -435,7 +496,7 @@ public void parseS3Uri_virtualStyleS3SchemeWithoutKey_shouldParseCorrectly() { String uriString = "s3://myBucket"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isNull(); @@ -448,7 +509,7 @@ public void parseS3Uri_virtualStyleS3SchemeWithoutKeyWithTrailingSlash_shouldPar String uriString = "s3://myBucket/"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isNull(); @@ -461,7 +522,7 @@ public void parseS3Uri_virtualStyleS3SchemeWithKey_shouldParseCorrectly() { String uriString = "s3://myBucket/resources/key"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("myBucket"); assertThat(s3Uri.key()).isEqualTo("resources/key"); @@ -474,7 +535,7 @@ public void parseS3Uri_virtualStyleS3SchemeWithEncoding_shouldParseCorrectly() { String uriString = "s3://my%40Bucket/object%20key"; URI uri = URI.create(uriString); - S3Uri s3Uri = defaultUtilities.parseS3Uri(uri); + S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); assertThat(s3Uri.bucket()).isEqualTo("my@Bucket"); assertThat(s3Uri.key()).isEqualTo("object key"); @@ -488,7 +549,7 @@ public void parseS3Uri_accessPointUri_shouldThrowProperErrorMessage() { URI accessPointUri = URI.create(accessPointUriString); Exception exception = assertThrows(SdkClientException.class, () -> { - defaultUtilities.parseS3Uri(accessPointUri); + defaultUtilities.parseUri(accessPointUri); }); assertThat(exception.getMessage()).isEqualTo("AccessPoints URI parsing is not supported"); } @@ -499,7 +560,7 @@ public void parseS3Uri_accessPointUriWithFipsDualstack_shouldThrowProperErrorMes URI accessPointUri = URI.create(accessPointUriString); Exception exception = assertThrows(SdkClientException.class, () -> { - defaultUtilities.parseS3Uri(accessPointUri); + defaultUtilities.parseUri(accessPointUri); }); assertThat(exception.getMessage()).isEqualTo("AccessPoints URI parsing is not supported"); } @@ -510,7 +571,7 @@ public void parseS3Uri_outpostsUri_shouldThrowProperErrorMessage() { URI outpostsUri = URI.create(outpostsUriString); Exception exception = assertThrows(SdkClientException.class, () -> { - defaultUtilities.parseS3Uri(outpostsUri); + defaultUtilities.parseUri(outpostsUri); }); assertThat(exception.getMessage()).isEqualTo("Outposts URI parsing is not supported"); } @@ -521,7 +582,7 @@ public void parseS3Uri_outpostsUriWithChinaPartition_shouldThrowProperErrorMessa URI outpostsUri = URI.create(outpostsUriString); Exception exception = assertThrows(SdkClientException.class, () -> { - defaultUtilities.parseS3Uri(outpostsUri); + defaultUtilities.parseUri(outpostsUri); }); assertThat(exception.getMessage()).isEqualTo("Outposts URI parsing is not supported"); } @@ -532,7 +593,7 @@ public void parseS3Uri_withNonS3Uri_shouldThrowProperErrorMessage() { URI nonS3Uri = URI.create(nonS3UriString); Exception exception = assertThrows(SdkClientException.class, () -> { - defaultUtilities.parseS3Uri(nonS3Uri); + defaultUtilities.parseUri(nonS3Uri); }); assertThat(exception.getMessage()).isEqualTo("Invalid S3 URI: hostname does not appear to be a valid S3 endpoint"); } From 00afb77b608195efa291915d0db3bfe4844a305b Mon Sep 17 00:00:00 2001 From: hdavidh Date: Tue, 4 Apr 2023 10:47:58 -0700 Subject: [PATCH 06/12] Refactoring --- .../awssdk/services/s3/S3Utilities.java | 189 +++++++++------ .../awssdk/services/s3/parsing/S3Uri.java | 62 +++-- .../awssdk/services/s3/S3UtilitiesTest.java | 215 ++++++++++-------- 3 files changed, 274 insertions(+), 192 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java index e3c94cacde9c..2a21aaab160e 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java @@ -43,7 +43,6 @@ import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; import software.amazon.awssdk.core.client.config.SdkClientConfiguration; import software.amazon.awssdk.core.client.config.SdkClientOption; -import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; @@ -101,7 +100,7 @@ @SdkPublicApi public final class S3Utilities { private static final String SERVICE_NAME = "s3"; - + private static final Pattern ENDPOINT_PATTERN = Pattern.compile("^(.+\\.)?s3[.-]([a-z0-9-]+)\\."); private final Region region; private final URI endpoint; private final S3Configuration s3Configuration; @@ -259,30 +258,48 @@ public URL getUrl(GetUrlRequest getUrlRequest) { } /** - * Returns a parsed {@link S3Uri} with which a user can easily retrieve the the bucket, key, region, style, and query - * parameters of the URI. Only basic bucket endpoints are supported, i.e., path-style and virtual-hosted-style URLs. - * Encoded buckets, keys, and query parameters will be returned decoded. + * Returns a parsed {@link S3Uri} with which a user can easily retrieve the bucket, key, region, style, and query + * parameters of the URI. Only path-style and virtual-hosted-style URI parsing is supported, including CLI-style + * URIs, e.g., "s3://bucket/key". AccessPoints and Outposts URI parsing is not supported. If you work with object keys + * and/or query parameters with special characters, they must be URL-encoded, e.g., replace " " with "%20". If you work with + * bucket names that contain a dot, i.e., ".", the dot must not be URL-encoded. Encoded buckets, keys, and query parameters + * will be returned decoded. + * + *

+ * For more information on path-style and virtual-hosted-style URIs, see Methods for accessing a bucket. * * @param uri The URI to be parsed * @return Parsed {@link S3Uri} + * + *

Example Usage + *

+ * {@snippet : + * S3Client s3Client = S3Client.create(); + * S3Utilities s3Utilities = s3Client.utilities(); + * String uriString = "https://myBucket.s3.us-west-1.amazonaws.com/doc.txt?versionId=abc123"; + * URI uri = URI.create(uriString); + * S3Uri s3Uri = s3Utilities.parseUri(uri); + * + * String bucket = s3Uri.bucket().get(); // "myBucket" + * String key = s3Uri.key().get(); // "doc.txt" + * Region region = s3Uri.region().get(); // Region.US_WEST_1 + * boolean isPathStyle = s3Uri.isPathStyle(); // false + * String versionId = s3Uri.firstMatchingQueryParamValue("versionId").get(); // "abc123" + *} */ public S3Uri parseUri(URI uri) { - if (uri == null) { - throw SdkClientException.create("URI must not be null"); - } + validateUri(uri); - Pattern accessPointPattern = Pattern.compile("^([a-zA-Z0-9\\-]+)\\.s3-accesspoint(-fips)?(\\.dualstack)?" - + "\\.([a-zA-Z0-9\\-]+)\\.amazonaws\\.com(.cn)?$"); - if (accessPointPattern.matcher(uri.toString()).find()) { - throw SdkClientException.create("AccessPoints URI parsing is not supported"); + if ("s3".equalsIgnoreCase(uri.getScheme())) { + return parseAwsCliStyleUri(uri); } - Pattern outpostPattern = Pattern.compile("^([a-zA-Z0-9\\-]+)\\.op\\-[0-9]+\\.s3-outposts\\.([a-zA-Z0-9\\-]+)" - + "\\.amazonaws\\.com(.cn)?$"); - if (outpostPattern.matcher(uri.toString()).find()) { - throw SdkClientException.create("Outposts URI parsing is not supported"); - } + return parseStandardUri(uri); + } + private S3Uri parseStandardUri(URI uri) { String bucket = null; String key = null; String region = null; @@ -290,54 +307,43 @@ public S3Uri parseUri(URI uri) { Map> queryParams = new HashMap<>(); String path = uri.getPath(); - if ("s3".equalsIgnoreCase(uri.getScheme())) { - if (uri.getAuthority() == null) { - throw SdkClientException.create("Invalid S3 URI: bucket not included"); - } - bucket = uri.getAuthority(); - if (path.length() > 1) { - key = path.substring(1); - } - - } else { - if (uri.getHost() == null) { - throw SdkClientException.create("Invalid S3 URI: hostname not included"); - } + if (uri.getHost() == null) { + throw new IllegalArgumentException("Invalid S3 URI: no hostname: " + uri); + } - Pattern endpointPattern = Pattern.compile("^(.+\\.)?s3[.-]([a-z0-9-]+)\\."); - Matcher matcher = endpointPattern.matcher(uri.getHost()); - if (!matcher.find()) { - throw SdkClientException.create("Invalid S3 URI: hostname does not appear to be a valid S3 endpoint"); - } + Matcher matcher = ENDPOINT_PATTERN.matcher(uri.getHost()); + if (!matcher.find()) { + throw new IllegalArgumentException("Invalid S3 URI: hostname does not appear to be a valid S3 endpoint: " + uri); + } - String prefix = matcher.group(1); - if (prefix == null || prefix.isEmpty()) { - isPathStyle = true; + String prefix = matcher.group(1); + if (prefix == null || prefix.isEmpty()) { + isPathStyle = true; - if (!path.isEmpty() && !"/".equals(path)) { - int index = path.indexOf('/', 1); + if (!path.isEmpty() && !"/".equals(path)) { + int index = path.indexOf('/', 1); - if (index == -1) { - bucket = path.substring(1); - } else { - bucket = path.substring(1, index); - if (index != path.length() - 1) { - key = path.substring(index + 1); - } + if (index == -1) { + // No trailing slash, e.g., "https://s3.amazonaws.com/bucket" + bucket = path.substring(1); + } else { + bucket = path.substring(1, index); + if (index != path.length() - 1) { + key = path.substring(index + 1); } } - } else { - bucket = prefix.substring(0, prefix.length() - 1); - if (path != null && !path.isEmpty() && !"/".equals(path)) { - key = path.substring(1); - } } - - if (!"amazonaws".equals(matcher.group(2))) { - region = matcher.group(2); + } else { + bucket = prefix.substring(0, prefix.length() - 1); + if (path != null && !path.isEmpty() && !"/".equals(path)) { + key = path.substring(1); } } + if (!"amazonaws".equals(matcher.group(2))) { + region = matcher.group(2); + } + String queryPart = uri.getQuery(); if (queryPart != null) { parseQuery(queryParams, queryPart); @@ -355,6 +361,64 @@ public S3Uri parseUri(URI uri) { .build(); } + private S3Uri parseAwsCliStyleUri(URI uri) { + String key = null; + String bucket = uri.getAuthority(); + Region region = null; + boolean isPathStyle = false; + Map> queryParams = new HashMap<>(); + String path = uri.getPath(); + + if (bucket == null) { + throw new IllegalArgumentException("Invalid S3 URI: bucket not included: " + uri); + } + + if (path.length() > 1) { + key = path.substring(1); + } + + String queryPart = uri.getQuery(); + if (queryPart != null) { + parseQuery(queryParams, queryPart); + } + + return S3Uri.builder() + .uri(uri) + .bucket(bucket) + .key(key) + .region(region) + .isPathStyle(isPathStyle) + .queryParams(queryParams) + .build(); + } + + private void validateUri(URI uri) { + Validate.paramNotNull(uri, "uri"); + + if (uri.toString().contains(".s3-accesspoint")) { + throw new IllegalArgumentException("AccessPoints URI parsing is not supported: " + uri); + } + + if (uri.toString().contains(".s3-outposts")) { + throw new IllegalArgumentException("Outposts URI parsing is not supported: " + uri); + } + } + + private void parseQuery(Map> queryParams, String queryPart) { + String[] params = queryPart.split("&"); + for (String param: params) { + String[] keyValuePair = param.split("=", 2); + String key = keyValuePair[0]; + if (key.isEmpty()) { + continue; + } + List paramValues = queryParams.containsKey(key) ? queryParams.get(key) : new ArrayList<>(); + String[] valuesPart = keyValuePair[1].split(","); + Collections.addAll(paramValues, valuesPart); + queryParams.put(key, paramValues); + } + } + private Region resolveRegionForGetUrl(GetUrlRequest getUrlRequest) { if (getUrlRequest.region() == null && this.region == null) { throw new IllegalArgumentException("Region should be provided either in GetUrlRequest object or S3Utilities object"); @@ -472,21 +536,6 @@ private UseGlobalEndpointResolver createUseGlobalEndpointResolver() { return new UseGlobalEndpointResolver(config); } - private void parseQuery(Map> queryParams, String queryPart) { - String[] params = queryPart.split("&"); - for (String param: params) { - String[] keyValuePair = param.split("=", 2); - String key = keyValuePair[0]; - if (key.isEmpty()) { - continue; - } - List paramValues = queryParams.containsKey(key) ? queryParams.get(key) : new ArrayList<>(); - String[] valuesPart = keyValuePair[1].split(","); - Collections.addAll(paramValues, valuesPart); - queryParams.put(key, paramValues); - } - } - /** * Builder class to construct {@link S3Utilities} object */ diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java index 58542478ccc1..92d066c0d86e 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java @@ -16,22 +16,28 @@ package software.amazon.awssdk.services.s3.parsing; import java.net.URI; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkPublicApi; -import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.Validate; import software.amazon.awssdk.utils.builder.CopyableBuilder; import software.amazon.awssdk.utils.builder.ToCopyableBuilder; /** * Object that represents a parsed S3 URI. Can be used to easily retrieve the bucket, key, region, style, and query parameters - * of the URI. Only basic bucket endpoints are supported, i.e., path-style and virtual-hosted style URLs. Encoded buckets, keys, - * and query parameters will be returned decoded. + * of the URI. Only path-style and virtual-hosted-style URI parsing is supported, including CLI-style URIs, e.g., + * "s3://bucket/key". AccessPoints and Outposts URI parsing is not supported. If you work with object keys and/or query + * parameters with special characters, they must be URL-encoded, e.g., replace " " with "%20". If you work with bucket names + * that contain a dot, i.e., ".", the dot must not be URL-encoded. Encoded buckets, keys, and query parameters will be returned + * decoded. */ @Immutable @SdkPublicApi @@ -45,12 +51,12 @@ public final class S3Uri implements ToCopyableBuilder { private final Map> queryParams; private S3Uri(Builder builder) { - this.uri = builder.uri; + this.uri = Validate.notNull(builder.uri, "URI must not be null"); this.bucket = builder.bucket; this.key = builder.key; this.region = builder.region; - this.isPathStyle = builder.isPathStyle; - this.queryParams = builder.queryParams; + this.isPathStyle = Validate.notNull(builder.isPathStyle, "Path style flag must not be null"); + this.queryParams = Validate.notNull(builder.queryParams, "Query parameters map must not be null"); } public static Builder builder() { @@ -70,24 +76,24 @@ public URI uri() { } /** - * Returns the bucket specified in the URI. Returns null if no bucket is specified. + * Returns the bucket specified in the URI. Returns an empty optional if no bucket is specified. */ - public String bucket() { - return bucket; + public Optional bucket() { + return Optional.ofNullable(bucket); } /** - * Returns the key specified in the URI. Returns null if no key is specified. + * Returns the key specified in the URI. Returns an empty optional if no key is specified. */ - public String key() { - return key; + public Optional key() { + return Optional.ofNullable(key); } /** - * Returns the region specified in the URI. Returns null if no region is specified, i.e., global endpoint. + * Returns the region specified in the URI. Returns an empty optional if no region is specified, i.e., global endpoint. */ - public Region region() { - return region; + public Optional region() { + return Optional.ofNullable(region); } /** @@ -105,13 +111,13 @@ public Map> queryParams() { } /** - * Returns the list of values for a specified query parameter. A {@link SdkClientException} is thrown if the URI does not - * contain the specified query parameter. + * Returns the list of values for a specified query parameter. A empty list is returned if the URI does not contain the + * specified query parameter. */ public List queryParamValues(String key) { List queryValues = queryParams.get(key); if (queryValues == null) { - throw SdkClientException.create("The URI does not contain the specified query parameter."); + return new ArrayList<>(); } List queryValuesCopy = Arrays.asList(new String[queryValues.size()]); Collections.copy(queryValuesCopy, queryValues); @@ -120,18 +126,22 @@ public List queryParamValues(String key) { /** * Returns the value for the specified query parameter. If there are multiple values for the query parameter, the first - * value is returned. A {@link SdkClientException} is thrown if the URI does not contain the specified query parameter. + * value is returned. An empty optional is returned if the URI does not contain the specified query parameter. */ - public String queryParamValue(String key) { - if (queryParams().get(key) == null) { - throw SdkClientException.create("The URI does not contain the specified query parameter."); - } - return queryParams.get(key).get(0); + public Optional firstMatchingQueryParamValue(String key) { + return queryParams.get(key) == null ? Optional.empty() : Optional.of(queryParams.get(key).get(0)); } @Override public String toString() { - return uri.toString(); + return ToString.builder("S3Uri") + .add("uri", uri) + .add("bucket", bucket) + .add("key", key) + .add("region", region) + .add("isPathStyle", isPathStyle) + .add("queryParams", queryParams) + .build(); } @Override @@ -148,6 +158,7 @@ public boolean equals(Object o) { && Objects.equals(bucket, s3Uri.bucket) && Objects.equals(key, s3Uri.key) && Objects.equals(region, s3Uri.region) + && Objects.equals(isPathStyle, s3Uri.isPathStyle) && Objects.equals(queryParams, s3Uri.queryParams); } @@ -157,6 +168,7 @@ public int hashCode() { result = 31 * result + (bucket != null ? bucket.hashCode() : 0); result = 31 * result + (key != null ? key.hashCode() : 0); result = 31 * result + (region != null ? region.hashCode() : 0); + result = 31 * result + Boolean.hashCode(isPathStyle); result = 31 * result + (queryParams != null ? queryParams.hashCode() : 0); return result; } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java index 2ba157f1d538..43e81f9af916 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java @@ -20,13 +20,13 @@ import java.net.MalformedURLException; import java.net.URI; +import nl.jqno.equalsverifier.EqualsVerifier; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.model.GetUrlRequest; import software.amazon.awssdk.services.s3.parsing.S3Uri; @@ -225,9 +225,9 @@ public void parseS3Uri_rootUri_shouldParseCorrectly() { S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isNull(); - assertThat(s3Uri.key()).isNull(); - assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.bucket()).isEmpty(); + assertThat(s3Uri.key()).isEmpty(); + assertThat(s3Uri.region()).isEmpty(); assertThat(s3Uri.isPathStyle()).isTrue(); assertThat(s3Uri.queryParams()).isEmpty(); } @@ -239,9 +239,9 @@ public void parseS3Uri_rootUriTrailingSlash_shouldParseCorrectly() { S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isNull(); - assertThat(s3Uri.key()).isNull(); - assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.bucket()).isEmpty(); + assertThat(s3Uri.key()).isEmpty(); + assertThat(s3Uri.region()).isEmpty(); assertThat(s3Uri.isPathStyle()).isTrue(); assertThat(s3Uri.queryParams()).isEmpty(); } @@ -253,9 +253,9 @@ public void parseS3Uri_pathStyleTrailingSlash_shouldParseCorrectly() { S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isNull(); - assertThat(s3Uri.region()).isEqualTo(Region.US_EAST_1); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).isEmpty(); + assertThat(s3Uri.region()).contains(Region.US_EAST_1); assertThat(s3Uri.isPathStyle()).isTrue(); assertThat(s3Uri.queryParams()).isEmpty(); } @@ -267,9 +267,9 @@ public void parseS3Uri_pathStyleGlobalEndpoint_shouldParseCorrectly() { S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("resources/image1.png"); - assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("resources/image1.png"); + assertThat(s3Uri.region()).isEmpty(); assertThat(s3Uri.isPathStyle()).isTrue(); assertThat(s3Uri.queryParams()).isEmpty(); } @@ -281,9 +281,9 @@ public void parseS3Uri_virtualStyleGlobalEndpoint_shouldParseCorrectly() { S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("resources/image1.png"); - assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("resources/image1.png"); + assertThat(s3Uri.region()).isEmpty(); assertThat(s3Uri.isPathStyle()).isFalse(); assertThat(s3Uri.queryParams()).isEmpty(); } @@ -295,9 +295,9 @@ public void parseS3Uri_pathStyleWithDot_shouldParseCorrectly() { S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("resources/image1.png"); - assertThat(s3Uri.region()).isEqualTo(Region.EU_WEST_2); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("resources/image1.png"); + assertThat(s3Uri.region()).contains(Region.EU_WEST_2); assertThat(s3Uri.isPathStyle()).isTrue(); assertThat(s3Uri.queryParams()).isEmpty(); } @@ -309,9 +309,9 @@ public void parseS3Uri_pathStyleWithDash_shouldParseCorrectly() { S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("resources/image1.png"); - assertThat(s3Uri.region()).isEqualTo(Region.EU_WEST_2); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("resources/image1.png"); + assertThat(s3Uri.region()).contains(Region.EU_WEST_2); assertThat(s3Uri.isPathStyle()).isTrue(); assertThat(s3Uri.queryParams()).isEmpty(); } @@ -323,9 +323,9 @@ public void parseS3Uri_virtualHostedStyleWithDot_shouldParseCorrectly() { S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("image.png"); - assertThat(s3Uri.region()).isEqualTo(Region.US_EAST_2); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("image.png"); + assertThat(s3Uri.region()).contains(Region.US_EAST_2); assertThat(s3Uri.isPathStyle()).isFalse(); assertThat(s3Uri.queryParams()).isEmpty(); } @@ -337,9 +337,9 @@ public void parseS3Uri_virtualHostedStyleWithDash_shouldParseCorrectly() { S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("image.png"); - assertThat(s3Uri.region()).isEqualTo(Region.US_EAST_2); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("image.png"); + assertThat(s3Uri.region()).contains(Region.US_EAST_2); assertThat(s3Uri.isPathStyle()).isFalse(); assertThat(s3Uri.queryParams()).isEmpty(); } @@ -351,11 +351,11 @@ public void parseS3Uri_pathStyleWithQuery_shouldParseCorrectly() { S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("doc.txt"); - assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("doc.txt"); + assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); } @Test @@ -365,11 +365,11 @@ public void parseS3Uri_pathStyleWithQueryMultipleValuesAmpersand_shouldParseCorr S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("doc.txt"); - assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("doc.txt"); + assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); assertThat(s3Uri.queryParamValues("versionId")).contains("def456"); } @@ -380,11 +380,11 @@ public void parseS3Uri_pathStyleWithQueryMultipleValuesCommas_shouldParseCorrect S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("doc.txt"); - assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("doc.txt"); + assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); assertThat(s3Uri.queryParamValues("versionId")).contains("def456"); assertThat(s3Uri.queryParamValues("versionId")).contains("ghi789"); } @@ -396,12 +396,12 @@ public void parseS3Uri_pathStyleWithMultipleQueries_shouldParseCorrectly() { S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("doc.txt"); - assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("doc.txt"); + assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); - assertThat(s3Uri.queryParamValue("partNumber")).isEqualTo("77"); + assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); + assertThat(s3Uri.firstMatchingQueryParamValue("partNumber")).contains("77"); } @Test @@ -411,25 +411,25 @@ public void parseS3Uri_pathStyleWithEncoding_shouldParseCorrectly() { S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("my@Bucket"); - assertThat(s3Uri.key()).isEqualTo("object key"); - assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); + assertThat(s3Uri.bucket()).contains("my@Bucket"); + assertThat(s3Uri.key()).contains("object key"); + assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); } @Test public void parseS3Uri_virtualStyleWithQuery_shouldParseCorrectly() { - String uriString = "https://myBucket.s3.us-west-1.amazonaws.com/doc.txt?versionId=abc123"; + String uriString= "https://myBucket.s3.us-west-1.amazonaws.com/doc.txt?versionId=abc123"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("doc.txt"); - assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("doc.txt"); + assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); } @Test public void parseS3Uri_virtualStyleWithQueryMultipleValuesAmpersand_shouldParseCorrectly() { @@ -438,11 +438,11 @@ public void parseS3Uri_virtualStyleWithQueryMultipleValuesAmpersand_shouldParseC S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("doc.txt"); - assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("doc.txt"); + assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); assertThat(s3Uri.queryParamValues("versionId")).contains("def456"); } @@ -453,11 +453,11 @@ public void parseS3Uri_virtualStyleWithQueryMultipleValuesCommas_shouldParseCorr S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("doc.txt"); - assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("doc.txt"); + assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); assertThat(s3Uri.queryParamValues("versionId")).contains("def456"); assertThat(s3Uri.queryParamValues("versionId")).contains("ghi789"); } @@ -469,12 +469,12 @@ public void parseS3Uri_virtualStyleWithMultipleQueries_shouldParseCorrectly() { S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("doc.txt"); - assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("doc.txt"); + assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); - assertThat(s3Uri.queryParamValue("partNumber")).isEqualTo("77"); + assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); + assertThat(s3Uri.firstMatchingQueryParamValue("partNumber")).contains("77"); } @Test @@ -484,62 +484,62 @@ public void parseS3Uri_virtualStyleWithEncoding_shouldParseCorrectly() { S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("object key"); - assertThat(s3Uri.region()).isEqualTo(Region.US_WEST_1); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("object key"); + assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.queryParamValue("versionId")).isEqualTo("abc123"); + assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); } @Test - public void parseS3Uri_virtualStyleS3SchemeWithoutKey_shouldParseCorrectly() { + public void parseS3Uri_cliStyleWithoutKey_shouldParseCorrectly() { String uriString = "s3://myBucket"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isNull(); - assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).isEmpty(); + assertThat(s3Uri.region()).isEmpty(); assertThat(s3Uri.isPathStyle()).isFalse(); } @Test - public void parseS3Uri_virtualStyleS3SchemeWithoutKeyWithTrailingSlash_shouldParseCorrectly() { + public void parseS3Uri_cliStyleWithoutKeyWithTrailingSlash_shouldParseCorrectly() { String uriString = "s3://myBucket/"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isNull(); - assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).isEmpty(); + assertThat(s3Uri.region()).isEmpty(); assertThat(s3Uri.isPathStyle()).isFalse(); } @Test - public void parseS3Uri_virtualStyleS3SchemeWithKey_shouldParseCorrectly() { + public void parseS3Uri_cliStyleWithKey_shouldParseCorrectly() { String uriString = "s3://myBucket/resources/key"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("myBucket"); - assertThat(s3Uri.key()).isEqualTo("resources/key"); - assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.bucket()).contains("myBucket"); + assertThat(s3Uri.key()).contains("resources/key"); + assertThat(s3Uri.region()).isEmpty(); assertThat(s3Uri.isPathStyle()).isFalse(); } @Test - public void parseS3Uri_virtualStyleS3SchemeWithEncoding_shouldParseCorrectly() { + public void parseS3Uri_cliStyleWithEncoding_shouldParseCorrectly() { String uriString = "s3://my%40Bucket/object%20key"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseUri(uri); assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).isEqualTo("my@Bucket"); - assertThat(s3Uri.key()).isEqualTo("object key"); - assertThat(s3Uri.region()).isNull(); + assertThat(s3Uri.bucket()).contains("my@Bucket"); + assertThat(s3Uri.key()).contains("object key"); + assertThat(s3Uri.region()).isEmpty(); assertThat(s3Uri.isPathStyle()).isFalse(); } @@ -548,10 +548,11 @@ public void parseS3Uri_accessPointUri_shouldThrowProperErrorMessage() { String accessPointUriString = "myendpoint-123456789012.s3-accesspoint.us-east-1.amazonaws.com"; URI accessPointUri = URI.create(accessPointUriString); - Exception exception = assertThrows(SdkClientException.class, () -> { + Exception exception = assertThrows(IllegalArgumentException.class, () -> { defaultUtilities.parseUri(accessPointUri); }); - assertThat(exception.getMessage()).isEqualTo("AccessPoints URI parsing is not supported"); + assertThat(exception.getMessage()).isEqualTo("AccessPoints URI parsing is not supported: " + + "myendpoint-123456789012.s3-accesspoint.us-east-1.amazonaws.com"); } @Test @@ -559,10 +560,11 @@ public void parseS3Uri_accessPointUriWithFipsDualstack_shouldThrowProperErrorMes String accessPointUriString = "myendpoint-123456789012.s3-accesspoint-fips.dualstack.us-gov-east-1.amazonaws.com"; URI accessPointUri = URI.create(accessPointUriString); - Exception exception = assertThrows(SdkClientException.class, () -> { + Exception exception = assertThrows(IllegalArgumentException.class, () -> { defaultUtilities.parseUri(accessPointUri); }); - assertThat(exception.getMessage()).isEqualTo("AccessPoints URI parsing is not supported"); + assertThat(exception.getMessage()).isEqualTo("AccessPoints URI parsing is not supported: " + + "myendpoint-123456789012.s3-accesspoint-fips.dualstack.us-gov-east-1.amazonaws.com"); } @Test @@ -570,10 +572,11 @@ public void parseS3Uri_outpostsUri_shouldThrowProperErrorMessage() { String outpostsUriString = "myaccesspoint-123456789012.op-01234567890123456.s3-outposts.us-west-2.amazonaws.com"; URI outpostsUri = URI.create(outpostsUriString); - Exception exception = assertThrows(SdkClientException.class, () -> { + Exception exception = assertThrows(IllegalArgumentException.class, () -> { defaultUtilities.parseUri(outpostsUri); }); - assertThat(exception.getMessage()).isEqualTo("Outposts URI parsing is not supported"); + assertThat(exception.getMessage()).isEqualTo("Outposts URI parsing is not supported: " + + "myaccesspoint-123456789012.op-01234567890123456.s3-outposts.us-west-2.amazonaws.com"); } @Test @@ -581,10 +584,11 @@ public void parseS3Uri_outpostsUriWithChinaPartition_shouldThrowProperErrorMessa String outpostsUriString = "myaccesspoint-123456789012.op-01234567890123456.s3-outposts.cn-north-1.amazonaws.com.cn"; URI outpostsUri = URI.create(outpostsUriString); - Exception exception = assertThrows(SdkClientException.class, () -> { + Exception exception = assertThrows(IllegalArgumentException.class, () -> { defaultUtilities.parseUri(outpostsUri); }); - assertThat(exception.getMessage()).isEqualTo("Outposts URI parsing is not supported"); + assertThat(exception.getMessage()).isEqualTo("Outposts URI parsing is not supported: " + + "myaccesspoint-123456789012.op-01234567890123456.s3-outposts.cn-north-1.amazonaws.com.cn"); } @Test @@ -592,10 +596,27 @@ public void parseS3Uri_withNonS3Uri_shouldThrowProperErrorMessage() { String nonS3UriString = "https://www.amazon.com/"; URI nonS3Uri = URI.create(nonS3UriString); - Exception exception = assertThrows(SdkClientException.class, () -> { + Exception exception = assertThrows(IllegalArgumentException.class, () -> { defaultUtilities.parseUri(nonS3Uri); }); - assertThat(exception.getMessage()).isEqualTo("Invalid S3 URI: hostname does not appear to be a valid S3 endpoint"); + assertThat(exception.getMessage()).isEqualTo("Invalid S3 URI: hostname does not appear to be a valid S3 endpoint: " + + "https://www.amazon.com/"); + } + + @Test + public void S3Uri_toString_printsCorrectOutput() { + String uriString = "https://myBucket.s3.us-west-1.amazonaws.com/doc.txt?versionId=abc123&partNumber=77"; + URI uri = URI.create(uriString); + + S3Uri s3Uri = defaultUtilities.parseUri(uri); + assertThat(s3Uri.toString()).isEqualTo("S3Uri(uri=https://myBucket.s3.us-west-1.amazonaws.com/doc.txt?" + + "versionId=abc123&partNumber=77, bucket=myBucket, key=doc.txt, region=us-west-1," + + " isPathStyle=false, queryParams={versionId=[abc123], partNumber=[77]})"); + } + + @Test + public void S3Uri_testEqualsAndHashCodeContract() { + EqualsVerifier.forClass(S3Uri.class).verify(); } private static GetUrlRequest requestWithoutSpaces() { From 0c93692f7b015c40c9b4657b603594d60e345994 Mon Sep 17 00:00:00 2001 From: hdavidh Date: Tue, 4 Apr 2023 14:33:05 -0700 Subject: [PATCH 07/12] Refactoring --- .../awssdk/services/s3/S3Utilities.java | 97 ++++++++++--------- .../awssdk/services/s3/parsing/S3Uri.java | 12 +-- .../awssdk/services/s3/S3UtilitiesTest.java | 82 +++++----------- 3 files changed, 81 insertions(+), 110 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java index 2a21aaab160e..b49ca81f73ae 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java @@ -20,7 +20,6 @@ import java.net.URI; import java.net.URL; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -69,7 +68,9 @@ import software.amazon.awssdk.services.s3.model.GetUrlRequest; import software.amazon.awssdk.services.s3.parsing.S3Uri; import software.amazon.awssdk.utils.AttributeMap; +import software.amazon.awssdk.utils.StringUtils; import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.http.SdkHttpUtils; /** * Utilities for working with Amazon S3 objects. An instance of this class can be created by: @@ -262,8 +263,8 @@ public URL getUrl(GetUrlRequest getUrlRequest) { * parameters of the URI. Only path-style and virtual-hosted-style URI parsing is supported, including CLI-style * URIs, e.g., "s3://bucket/key". AccessPoints and Outposts URI parsing is not supported. If you work with object keys * and/or query parameters with special characters, they must be URL-encoded, e.g., replace " " with "%20". If you work with - * bucket names that contain a dot, i.e., ".", the dot must not be URL-encoded. Encoded buckets, keys, and query parameters - * will be returned decoded. + * virtual-hosted-style URIs with bucket names that contain a dot, i.e., ".", the dot must not be URL-encoded. Encoded + * buckets, keys, and query parameters will be returned decoded. * *

* For more information on path-style and virtual-hosted-style URIs, see > queryParams = new HashMap<>(); - String path = uri.getPath(); if (uri.getHost() == null) { throw new IllegalArgumentException("Invalid S3 URI: no hostname: " + uri); @@ -316,29 +351,16 @@ private S3Uri parseStandardUri(URI uri) { throw new IllegalArgumentException("Invalid S3 URI: hostname does not appear to be a valid S3 endpoint: " + uri); } + String[] parsed; String prefix = matcher.group(1); if (prefix == null || prefix.isEmpty()) { isPathStyle = true; - - if (!path.isEmpty() && !"/".equals(path)) { - int index = path.indexOf('/', 1); - - if (index == -1) { - // No trailing slash, e.g., "https://s3.amazonaws.com/bucket" - bucket = path.substring(1); - } else { - bucket = path.substring(1, index); - if (index != path.length() - 1) { - key = path.substring(index + 1); - } - } - } + parsed = parsePathStyleUri(uri); } else { - bucket = prefix.substring(0, prefix.length() - 1); - if (path != null && !path.isEmpty() && !"/".equals(path)) { - key = path.substring(1); - } + parsed = parseVirtualHostedStyleUri(uri, matcher); } + key = parsed[0]; + bucket = parsed[1]; if (!"amazonaws".equals(matcher.group(2))) { region = matcher.group(2); @@ -346,7 +368,7 @@ private S3Uri parseStandardUri(URI uri) { String queryPart = uri.getQuery(); if (queryPart != null) { - parseQuery(queryParams, queryPart); + queryParams = SdkHttpUtils.uriParams(uri); } Region uriRegion = region != null ? Region.of(region) : null; @@ -359,6 +381,7 @@ private S3Uri parseStandardUri(URI uri) { .isPathStyle(isPathStyle) .queryParams(queryParams) .build(); + } private S3Uri parseAwsCliStyleUri(URI uri) { @@ -377,11 +400,6 @@ private S3Uri parseAwsCliStyleUri(URI uri) { key = path.substring(1); } - String queryPart = uri.getQuery(); - if (queryPart != null) { - parseQuery(queryParams, queryPart); - } - return S3Uri.builder() .uri(uri) .bucket(bucket) @@ -404,21 +422,6 @@ private void validateUri(URI uri) { } } - private void parseQuery(Map> queryParams, String queryPart) { - String[] params = queryPart.split("&"); - for (String param: params) { - String[] keyValuePair = param.split("=", 2); - String key = keyValuePair[0]; - if (key.isEmpty()) { - continue; - } - List paramValues = queryParams.containsKey(key) ? queryParams.get(key) : new ArrayList<>(); - String[] valuesPart = keyValuePair[1].split(","); - Collections.addAll(paramValues, valuesPart); - queryParams.put(key, paramValues); - } - } - private Region resolveRegionForGetUrl(GetUrlRequest getUrlRequest) { if (getUrlRequest.region() == null && this.region == null) { throw new IllegalArgumentException("Region should be provided either in GetUrlRequest object or S3Utilities object"); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java index 92d066c0d86e..9896577d38fb 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java @@ -35,9 +35,9 @@ * Object that represents a parsed S3 URI. Can be used to easily retrieve the bucket, key, region, style, and query parameters * of the URI. Only path-style and virtual-hosted-style URI parsing is supported, including CLI-style URIs, e.g., * "s3://bucket/key". AccessPoints and Outposts URI parsing is not supported. If you work with object keys and/or query - * parameters with special characters, they must be URL-encoded, e.g., replace " " with "%20". If you work with bucket names - * that contain a dot, i.e., ".", the dot must not be URL-encoded. Encoded buckets, keys, and query parameters will be returned - * decoded. + * parameters with special characters, they must be URL-encoded, e.g., replace " " with "%20". If you work with + * virtual-hosted-style URIs with bucket names that contain a dot, i.e., ".", the dot must not be URL-encoded. Encoded buckets, + * keys, and query parameters will be returned decoded. */ @Immutable @SdkPublicApi @@ -106,7 +106,7 @@ public boolean isPathStyle() { /** * Returns a map of the query parameters specified in the URI. Returns an empty map if no queries are specified. */ - public Map> queryParams() { + public Map> rawQueryParameters() { return Collections.unmodifiableMap(queryParams); } @@ -114,7 +114,7 @@ public Map> queryParams() { * Returns the list of values for a specified query parameter. A empty list is returned if the URI does not contain the * specified query parameter. */ - public List queryParamValues(String key) { + public List firstMatchingRawQueryParameters(String key) { List queryValues = queryParams.get(key); if (queryValues == null) { return new ArrayList<>(); @@ -128,7 +128,7 @@ public List queryParamValues(String key) { * Returns the value for the specified query parameter. If there are multiple values for the query parameter, the first * value is returned. An empty optional is returned if the URI does not contain the specified query parameter. */ - public Optional firstMatchingQueryParamValue(String key) { + public Optional firstMatchingRawQueryParameter(String key) { return queryParams.get(key) == null ? Optional.empty() : Optional.of(queryParams.get(key).get(0)); } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java index 43e81f9af916..04e50c7f0411 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java @@ -229,7 +229,7 @@ public void parseS3Uri_rootUri_shouldParseCorrectly() { assertThat(s3Uri.key()).isEmpty(); assertThat(s3Uri.region()).isEmpty(); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.queryParams()).isEmpty(); + assertThat(s3Uri.rawQueryParameters()).isEmpty(); } @Test @@ -243,7 +243,7 @@ public void parseS3Uri_rootUriTrailingSlash_shouldParseCorrectly() { assertThat(s3Uri.key()).isEmpty(); assertThat(s3Uri.region()).isEmpty(); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.queryParams()).isEmpty(); + assertThat(s3Uri.rawQueryParameters()).isEmpty(); } @Test @@ -257,7 +257,7 @@ public void parseS3Uri_pathStyleTrailingSlash_shouldParseCorrectly() { assertThat(s3Uri.key()).isEmpty(); assertThat(s3Uri.region()).contains(Region.US_EAST_1); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.queryParams()).isEmpty(); + assertThat(s3Uri.rawQueryParameters()).isEmpty(); } @Test @@ -271,7 +271,7 @@ public void parseS3Uri_pathStyleGlobalEndpoint_shouldParseCorrectly() { assertThat(s3Uri.key()).contains("resources/image1.png"); assertThat(s3Uri.region()).isEmpty(); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.queryParams()).isEmpty(); + assertThat(s3Uri.rawQueryParameters()).isEmpty(); } @Test @@ -285,7 +285,7 @@ public void parseS3Uri_virtualStyleGlobalEndpoint_shouldParseCorrectly() { assertThat(s3Uri.key()).contains("resources/image1.png"); assertThat(s3Uri.region()).isEmpty(); assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.queryParams()).isEmpty(); + assertThat(s3Uri.rawQueryParameters()).isEmpty(); } @Test @@ -299,7 +299,7 @@ public void parseS3Uri_pathStyleWithDot_shouldParseCorrectly() { assertThat(s3Uri.key()).contains("resources/image1.png"); assertThat(s3Uri.region()).contains(Region.EU_WEST_2); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.queryParams()).isEmpty(); + assertThat(s3Uri.rawQueryParameters()).isEmpty(); } @Test @@ -313,7 +313,7 @@ public void parseS3Uri_pathStyleWithDash_shouldParseCorrectly() { assertThat(s3Uri.key()).contains("resources/image1.png"); assertThat(s3Uri.region()).contains(Region.EU_WEST_2); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.queryParams()).isEmpty(); + assertThat(s3Uri.rawQueryParameters()).isEmpty(); } @Test @@ -327,7 +327,7 @@ public void parseS3Uri_virtualHostedStyleWithDot_shouldParseCorrectly() { assertThat(s3Uri.key()).contains("image.png"); assertThat(s3Uri.region()).contains(Region.US_EAST_2); assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.queryParams()).isEmpty(); + assertThat(s3Uri.rawQueryParameters()).isEmpty(); } @Test @@ -341,7 +341,7 @@ public void parseS3Uri_virtualHostedStyleWithDash_shouldParseCorrectly() { assertThat(s3Uri.key()).contains("image.png"); assertThat(s3Uri.region()).contains(Region.US_EAST_2); assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.queryParams()).isEmpty(); + assertThat(s3Uri.rawQueryParameters()).isEmpty(); } @Test @@ -355,11 +355,11 @@ public void parseS3Uri_pathStyleWithQuery_shouldParseCorrectly() { assertThat(s3Uri.key()).contains("doc.txt"); assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); + assertThat(s3Uri.firstMatchingRawQueryParameter("versionId")).contains("abc123"); } @Test - public void parseS3Uri_pathStyleWithQueryMultipleValuesAmpersand_shouldParseCorrectly() { + public void parseS3Uri_pathStyleWithQueryMultipleValues_shouldParseCorrectly() { String uriString = "https://s3.us-west-1.amazonaws.com/myBucket/doc.txt?versionId=abc123&versionId=def456"; URI uri = URI.create(uriString); @@ -369,24 +369,8 @@ public void parseS3Uri_pathStyleWithQueryMultipleValuesAmpersand_shouldParseCorr assertThat(s3Uri.key()).contains("doc.txt"); assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); - assertThat(s3Uri.queryParamValues("versionId")).contains("def456"); - } - - @Test - public void parseS3Uri_pathStyleWithQueryMultipleValuesCommas_shouldParseCorrectly() { - String uriString = "https://s3.us-west-1.amazonaws.com/myBucket/doc.txt?versionId=abc123,def456,ghi789"; - URI uri = URI.create(uriString); - - S3Uri s3Uri = defaultUtilities.parseUri(uri); - assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).contains("myBucket"); - assertThat(s3Uri.key()).contains("doc.txt"); - assertThat(s3Uri.region()).contains(Region.US_WEST_1); - assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); - assertThat(s3Uri.queryParamValues("versionId")).contains("def456"); - assertThat(s3Uri.queryParamValues("versionId")).contains("ghi789"); + assertThat(s3Uri.firstMatchingRawQueryParameter("versionId")).contains("abc123"); + assertThat(s3Uri.firstMatchingRawQueryParameters("versionId")).contains("def456"); } @Test @@ -400,13 +384,13 @@ public void parseS3Uri_pathStyleWithMultipleQueries_shouldParseCorrectly() { assertThat(s3Uri.key()).contains("doc.txt"); assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); - assertThat(s3Uri.firstMatchingQueryParamValue("partNumber")).contains("77"); + assertThat(s3Uri.firstMatchingRawQueryParameter("versionId")).contains("abc123"); + assertThat(s3Uri.firstMatchingRawQueryParameter("partNumber")).contains("77"); } @Test public void parseS3Uri_pathStyleWithEncoding_shouldParseCorrectly() { - String uriString = "https://s3.us-west-1.amazonaws.com/my%40Bucket/object%20key?versionId%3D%61%62%63%31%32%33"; + String uriString = "https://s3.us-west-1.amazonaws.com/my%40Bucket/object%20key?versionId=%61%62%63%31%32%33"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseUri(uri); @@ -415,7 +399,7 @@ public void parseS3Uri_pathStyleWithEncoding_shouldParseCorrectly() { assertThat(s3Uri.key()).contains("object key"); assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isTrue(); - assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); + assertThat(s3Uri.firstMatchingRawQueryParameter("versionId")).contains("abc123"); } @Test @@ -429,10 +413,10 @@ public void parseS3Uri_virtualStyleWithQuery_shouldParseCorrectly() { assertThat(s3Uri.key()).contains("doc.txt"); assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); + assertThat(s3Uri.firstMatchingRawQueryParameter("versionId")).contains("abc123"); } @Test - public void parseS3Uri_virtualStyleWithQueryMultipleValuesAmpersand_shouldParseCorrectly() { + public void parseS3Uri_virtualStyleWithQueryMultipleValues_shouldParseCorrectly() { String uriString = "https://myBucket.s3.us-west-1.amazonaws.com/doc.txt?versionId=abc123&versionId=def456"; URI uri = URI.create(uriString); @@ -442,24 +426,8 @@ public void parseS3Uri_virtualStyleWithQueryMultipleValuesAmpersand_shouldParseC assertThat(s3Uri.key()).contains("doc.txt"); assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); - assertThat(s3Uri.queryParamValues("versionId")).contains("def456"); - } - - @Test - public void parseS3Uri_virtualStyleWithQueryMultipleValuesCommas_shouldParseCorrectly() { - String uriString = "https://myBucket.s3.us-west-1.amazonaws.com/doc.txt?versionId=abc123,def456,ghi789"; - URI uri = URI.create(uriString); - - S3Uri s3Uri = defaultUtilities.parseUri(uri); - assertThat(s3Uri.uri()).isEqualTo(uri); - assertThat(s3Uri.bucket()).contains("myBucket"); - assertThat(s3Uri.key()).contains("doc.txt"); - assertThat(s3Uri.region()).contains(Region.US_WEST_1); - assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); - assertThat(s3Uri.queryParamValues("versionId")).contains("def456"); - assertThat(s3Uri.queryParamValues("versionId")).contains("ghi789"); + assertThat(s3Uri.firstMatchingRawQueryParameter("versionId")).contains("abc123"); + assertThat(s3Uri.firstMatchingRawQueryParameters("versionId")).contains("def456"); } @Test @@ -473,13 +441,13 @@ public void parseS3Uri_virtualStyleWithMultipleQueries_shouldParseCorrectly() { assertThat(s3Uri.key()).contains("doc.txt"); assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); - assertThat(s3Uri.firstMatchingQueryParamValue("partNumber")).contains("77"); + assertThat(s3Uri.firstMatchingRawQueryParameter("versionId")).contains("abc123"); + assertThat(s3Uri.firstMatchingRawQueryParameter("partNumber")).contains("77"); } @Test public void parseS3Uri_virtualStyleWithEncoding_shouldParseCorrectly() { - String uriString = "https://myBucket.s3.us-west-1.amazonaws.com/object%20key?versionId%3D%61%62%63%31%32%33"; + String uriString = "https://myBucket.s3.us-west-1.amazonaws.com/object%20key?versionId=%61%62%63%31%32%33"; URI uri = URI.create(uriString); S3Uri s3Uri = defaultUtilities.parseUri(uri); @@ -488,7 +456,7 @@ public void parseS3Uri_virtualStyleWithEncoding_shouldParseCorrectly() { assertThat(s3Uri.key()).contains("object key"); assertThat(s3Uri.region()).contains(Region.US_WEST_1); assertThat(s3Uri.isPathStyle()).isFalse(); - assertThat(s3Uri.firstMatchingQueryParamValue("versionId")).contains("abc123"); + assertThat(s3Uri.firstMatchingRawQueryParameter("versionId")).contains("abc123"); } @Test From 1bc78943eef6707503bd1e6f69f1d6bed1e994b3 Mon Sep 17 00:00:00 2001 From: hdavidh Date: Tue, 4 Apr 2023 14:34:36 -0700 Subject: [PATCH 08/12] Refactoring --- .../awssdk/services/s3/S3Utilities.java | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java index b49ca81f73ae..e9ab9a18ca3e 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java @@ -300,44 +300,9 @@ public S3Uri parseUri(URI uri) { return parseStandardUri(uri); } - private String[] parsePathStyleUri(URI uri) { - String bucket = null; - String key = null; - String path = uri.getPath(); - - if (!StringUtils.isEmpty(path) && !"/".equals(path)) { - int index = path.indexOf('/', 1); - - if (index == -1) { - // No trailing slash, e.g., "https://s3.amazonaws.com/bucket" - bucket = path.substring(1); - } else { - bucket = path.substring(1, index); - if (index != path.length() - 1) { - key = path.substring(index + 1); - } - } - } - return new String[]{key, bucket}; - } - - private String[] parseVirtualHostedStyleUri(URI uri, Matcher matcher) { - String bucket = null; - String key = null; - String path = uri.getPath(); - String prefix = matcher.group(1); - - bucket = prefix.substring(0, prefix.length() - 1); - if (!StringUtils.isEmpty(path) && !"/".equals(path)) { - key = path.substring(1); - } - - return new String[]{key, bucket}; - } - private S3Uri parseStandardUri(URI uri) { - String bucket = null; - String key = null; + String bucket; + String key; String region = null; boolean isPathStyle = false; Map> queryParams = new HashMap<>(); @@ -384,6 +349,41 @@ private S3Uri parseStandardUri(URI uri) { } + private String[] parsePathStyleUri(URI uri) { + String bucket = null; + String key = null; + String path = uri.getPath(); + + if (!StringUtils.isEmpty(path) && !"/".equals(path)) { + int index = path.indexOf('/', 1); + + if (index == -1) { + // No trailing slash, e.g., "https://s3.amazonaws.com/bucket" + bucket = path.substring(1); + } else { + bucket = path.substring(1, index); + if (index != path.length() - 1) { + key = path.substring(index + 1); + } + } + } + return new String[]{key, bucket}; + } + + private String[] parseVirtualHostedStyleUri(URI uri, Matcher matcher) { + String bucket; + String key = null; + String path = uri.getPath(); + String prefix = matcher.group(1); + + bucket = prefix.substring(0, prefix.length() - 1); + if (!StringUtils.isEmpty(path) && !"/".equals(path)) { + key = path.substring(1); + } + + return new String[]{key, bucket}; + } + private S3Uri parseAwsCliStyleUri(URI uri) { String key = null; String bucket = uri.getAuthority(); From e929cb3b4274245bcef798da58467277416dae76 Mon Sep 17 00:00:00 2001 From: hdavidh Date: Tue, 4 Apr 2023 15:56:55 -0700 Subject: [PATCH 09/12] Refactoring --- .../software/amazon/awssdk/services/s3/parsing/S3Uri.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java index 9896577d38fb..cff0aafdd98d 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -26,6 +27,7 @@ import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.utils.CollectionUtils; import software.amazon.awssdk.utils.ToString; import software.amazon.awssdk.utils.Validate; import software.amazon.awssdk.utils.builder.CopyableBuilder; @@ -56,7 +58,7 @@ private S3Uri(Builder builder) { this.key = builder.key; this.region = builder.region; this.isPathStyle = Validate.notNull(builder.isPathStyle, "Path style flag must not be null"); - this.queryParams = Validate.notNull(builder.queryParams, "Query parameters map must not be null"); + this.queryParams = builder.queryParams == null ? new HashMap<>() : CollectionUtils.deepCopyMap(builder.queryParams); } public static Builder builder() { @@ -129,7 +131,7 @@ public List firstMatchingRawQueryParameters(String key) { * value is returned. An empty optional is returned if the URI does not contain the specified query parameter. */ public Optional firstMatchingRawQueryParameter(String key) { - return queryParams.get(key) == null ? Optional.empty() : Optional.of(queryParams.get(key).get(0)); + return Optional.ofNullable(queryParams.get(key)).map(q -> q.get(0)); } @Override From d1d323d95d160c561bb91661b996365979408df3 Mon Sep 17 00:00:00 2001 From: hdavidh Date: Tue, 4 Apr 2023 17:01:00 -0700 Subject: [PATCH 10/12] Refactoring --- .../awssdk/services/s3/S3Utilities.java | 64 ++++++++----------- .../awssdk/services/s3/parsing/S3Uri.java | 7 ++ 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java index e9ab9a18ca3e..4d7bcc4ebce9 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java @@ -301,11 +301,6 @@ public S3Uri parseUri(URI uri) { } private S3Uri parseStandardUri(URI uri) { - String bucket; - String key; - String region = null; - boolean isPathStyle = false; - Map> queryParams = new HashMap<>(); if (uri.getHost() == null) { throw new IllegalArgumentException("Invalid S3 URI: no hostname: " + uri); @@ -316,43 +311,35 @@ private S3Uri parseStandardUri(URI uri) { throw new IllegalArgumentException("Invalid S3 URI: hostname does not appear to be a valid S3 endpoint: " + uri); } - String[] parsed; + S3Uri.Builder builder = S3Uri.builder().uri(uri); + addRegionIfNeeded(builder, matcher.group(2)); + addQueryParamsIfNeeded(builder); + String prefix = matcher.group(1); - if (prefix == null || prefix.isEmpty()) { - isPathStyle = true; - parsed = parsePathStyleUri(uri); - } else { - parsed = parseVirtualHostedStyleUri(uri, matcher); + if (StringUtils.isEmpty(prefix)) { + return parsePathStyleUri(builder); } - key = parsed[0]; - bucket = parsed[1]; + return parseVirtualHostedStyleUri(builder, matcher); + } - if (!"amazonaws".equals(matcher.group(2))) { - region = matcher.group(2); + private S3Uri.Builder addRegionIfNeeded(S3Uri.Builder builder, String region) { + if (!"amazonaws".equals(region)) { + return builder.region(Region.of(region)); } + return builder; + } - String queryPart = uri.getQuery(); - if (queryPart != null) { - queryParams = SdkHttpUtils.uriParams(uri); + private S3Uri.Builder addQueryParamsIfNeeded(S3Uri.Builder builder) { + if (builder.uri().getQuery() != null) { + return builder.queryParams(SdkHttpUtils.uriParams(builder.uri())); } - - Region uriRegion = region != null ? Region.of(region) : null; - - return S3Uri.builder() - .uri(uri) - .bucket(bucket) - .key(key) - .region(uriRegion) - .isPathStyle(isPathStyle) - .queryParams(queryParams) - .build(); - + return builder; } - private String[] parsePathStyleUri(URI uri) { + private S3Uri parsePathStyleUri(S3Uri.Builder builder) { String bucket = null; String key = null; - String path = uri.getPath(); + String path = builder.uri().getPath(); if (!StringUtils.isEmpty(path) && !"/".equals(path)) { int index = path.indexOf('/', 1); @@ -367,13 +354,16 @@ private String[] parsePathStyleUri(URI uri) { } } } - return new String[]{key, bucket}; + return builder.key(key) + .bucket(bucket) + .isPathStyle(true) + .build(); } - private String[] parseVirtualHostedStyleUri(URI uri, Matcher matcher) { + private S3Uri parseVirtualHostedStyleUri(S3Uri.Builder builder, Matcher matcher) { String bucket; String key = null; - String path = uri.getPath(); + String path = builder.uri().getPath(); String prefix = matcher.group(1); bucket = prefix.substring(0, prefix.length() - 1); @@ -381,7 +371,9 @@ private String[] parseVirtualHostedStyleUri(URI uri, Matcher matcher) { key = path.substring(1); } - return new String[]{key, bucket}; + return builder.key(key) + .bucket(bucket) + .build(); } private S3Uri parseAwsCliStyleUri(URI uri) { diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java index cff0aafdd98d..9a0c0dcdcb14 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java @@ -246,6 +246,13 @@ public Builder queryParams(Map> queryParams) { return this; } + /** + * Return the configured URI + */ + public URI uri() { + return this.uri; + } + @Override public S3Uri build() { return new S3Uri(this); From 301be45caa43c5fc65d356ff4970bb1b0daf6c9b Mon Sep 17 00:00:00 2001 From: hdavidh Date: Wed, 5 Apr 2023 11:53:44 -0700 Subject: [PATCH 11/12] Refactoring --- .../awssdk/services/s3/S3Utilities.java | 20 +++++++++---------- .../awssdk/services/s3/parsing/S3Uri.java | 7 ------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java index 4d7bcc4ebce9..36a7b90211ec 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java @@ -313,13 +313,13 @@ private S3Uri parseStandardUri(URI uri) { S3Uri.Builder builder = S3Uri.builder().uri(uri); addRegionIfNeeded(builder, matcher.group(2)); - addQueryParamsIfNeeded(builder); + addQueryParamsIfNeeded(builder, uri); String prefix = matcher.group(1); if (StringUtils.isEmpty(prefix)) { - return parsePathStyleUri(builder); + return parsePathStyleUri(builder, uri); } - return parseVirtualHostedStyleUri(builder, matcher); + return parseVirtualHostedStyleUri(builder, uri, matcher); } private S3Uri.Builder addRegionIfNeeded(S3Uri.Builder builder, String region) { @@ -329,17 +329,17 @@ private S3Uri.Builder addRegionIfNeeded(S3Uri.Builder builder, String region) { return builder; } - private S3Uri.Builder addQueryParamsIfNeeded(S3Uri.Builder builder) { - if (builder.uri().getQuery() != null) { - return builder.queryParams(SdkHttpUtils.uriParams(builder.uri())); + private S3Uri.Builder addQueryParamsIfNeeded(S3Uri.Builder builder, URI uri) { + if (uri.getQuery() != null) { + return builder.queryParams(SdkHttpUtils.uriParams(uri)); } return builder; } - private S3Uri parsePathStyleUri(S3Uri.Builder builder) { + private S3Uri parsePathStyleUri(S3Uri.Builder builder, URI uri) { String bucket = null; String key = null; - String path = builder.uri().getPath(); + String path = uri.getPath(); if (!StringUtils.isEmpty(path) && !"/".equals(path)) { int index = path.indexOf('/', 1); @@ -360,10 +360,10 @@ private S3Uri parsePathStyleUri(S3Uri.Builder builder) { .build(); } - private S3Uri parseVirtualHostedStyleUri(S3Uri.Builder builder, Matcher matcher) { + private S3Uri parseVirtualHostedStyleUri(S3Uri.Builder builder, URI uri, Matcher matcher) { String bucket; String key = null; - String path = builder.uri().getPath(); + String path = uri.getPath(); String prefix = matcher.group(1); bucket = prefix.substring(0, prefix.length() - 1); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java index 9a0c0dcdcb14..cff0aafdd98d 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java @@ -246,13 +246,6 @@ public Builder queryParams(Map> queryParams) { return this; } - /** - * Return the configured URI - */ - public URI uri() { - return this.uri; - } - @Override public S3Uri build() { return new S3Uri(this); From 0c716b43c479270593de6f32bf66b0d5a9b332d9 Mon Sep 17 00:00:00 2001 From: hdavidh Date: Wed, 5 Apr 2023 15:32:10 -0700 Subject: [PATCH 12/12] Refactoring --- .../amazon/awssdk/services/s3/{parsing => }/S3Uri.java | 4 ++-- .../java/software/amazon/awssdk/services/s3/S3Utilities.java | 1 - .../software/amazon/awssdk/services/s3/S3UtilitiesTest.java | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) rename services/s3/src/main/java/software/amazon/awssdk/services/s3/{parsing => }/S3Uri.java (98%) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Uri.java similarity index 98% rename from services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java rename to services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Uri.java index cff0aafdd98d..b4ebc29fc939 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/parsing/S3Uri.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Uri.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.services.s3.parsing; +package software.amazon.awssdk.services.s3; import java.net.URI; import java.util.ArrayList; @@ -109,7 +109,7 @@ public boolean isPathStyle() { * Returns a map of the query parameters specified in the URI. Returns an empty map if no queries are specified. */ public Map> rawQueryParameters() { - return Collections.unmodifiableMap(queryParams); + return queryParams; } /** diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java index 36a7b90211ec..3e9c32b0ff4b 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java @@ -66,7 +66,6 @@ import software.amazon.awssdk.services.s3.internal.endpoints.UseGlobalEndpointResolver; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetUrlRequest; -import software.amazon.awssdk.services.s3.parsing.S3Uri; import software.amazon.awssdk.utils.AttributeMap; import software.amazon.awssdk.utils.StringUtils; import software.amazon.awssdk.utils.Validate; diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java index 04e50c7f0411..1829ab488665 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/S3UtilitiesTest.java @@ -29,7 +29,6 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.model.GetUrlRequest; -import software.amazon.awssdk.services.s3.parsing.S3Uri; public class S3UtilitiesTest {