S3: really call version 2 in list_objects_v2

Project.toml

@@ -16,6 +16,7 @@ Mocking = "78c3b35d-d492-501b-9361-3d52fe80e533"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Retry = "20febd7b-183b-5ae2-ac4a-720e7ce64774"
 Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
+URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 XMLDict = "228000da-037f-5747-90a9-8195ccbf91a5"
 
@@ -28,10 +29,12 @@ JSON = "0.18, 0.19, 0.20, 0.21"
 MbedTLS = "0.6, 0.7, 1"
 Mocking = "0.7"
 OrderedCollections = "1"
+URIs = "1"
 Retry = "0.3, 0.4"
 XMLDict = "0.3, 0.4"
 julia = "1"
 
+
 [extras]
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"

src/AWS.jl

@@ -11,6 +11,7 @@ using Retry
 using Sockets
 using UUIDs: UUIDs
 using XMLDict
+import URIs
 
 export @service
 export _merge, AbstractAWSConfig, AWSConfig, AWSExceptions, AWSServices, Request
@@ -501,6 +502,39 @@ function _flatten_query!(result::Vector{Pair{String, String}}, service::String,
     return result
 end
 
+"""
+    _clean_s3_uri(uri::AbstractString)
+
+Escape special AWS S3 characters in the path of the uri properly.
+
+AWS S3 allows for various special characters in file names, these characters are not being
+properly escaped before we make the requests.
+
+We cannot call `HTTP.escapeuri(request.uri)` because this will escape `/` characters which
+are used in the filepathing for sub-directories.
+
+# Arguments
+- `uri::AbstractString`: URI to to cleaned
+
+# Returns
+- `String`: URI with characters escaped
+"""
+function _clean_s3_uri(uri::AbstractString)
+    chars_to_clean = (
+        ' ' => "%20",
+        '!' => "%21",
+        ''' => "%27",
+        '(' => "%28",
+        ')' => "%29",
+        '*' => "%2A",
+        '+' => "%2B",
+        '=' => "%3D",
+    )
+    parsed_uri = URIs.URI(uri)
+    cleaned_path = reduce(replace, chars_to_clean, init=parsed_uri.path)
+    return string(URIs.URI(parsed_uri; path=cleaned_path))
+end
+
 """
     (service::RestXMLService)(
         request_method::String, request_uri::String, args::AbstractDict{String, <:Any}=Dict{String, String}();
@@ -524,36 +558,6 @@ function (service::RestXMLService)(
     request_method::String, request_uri::String, args::AbstractDict{String, <:Any}=Dict{String, Any}();
     aws_config::AbstractAWSConfig=global_aws_config(),
 )
-    """
-        _clean_s3_uri(uri::AbstractString)
-
-    Escape special AWS S3 characters properly.
-
-    AWS S3 allows for various special characters in file names, these characters are not being
-    properly escaped before we make the requests.
-
-    We cannot call `HTTP.escapeuri(request.uri)` because this will escape `/` characters which
-    are used in the filepathing for sub-directories.
-
-    # Arguments
-    - `uri::AbstractString`: URI to to cleaned
-
-    # Returns
-    - `String`: URI with characters escaped
-    """
-    function _clean_s3_uri(uri::AbstractString)
-        chars_to_clean = (
-            ' ' => "%20",
-            '!' => "%21",
-            ''' => "%27",
-            '(' => "%28",
-            ')' => "%29",
-            '*' => "%2A",
-            '+' => "%2B",
-            '=' => "%3D",
-        )
-        return reduce(replace, chars_to_clean, init=uri)
-    end
 
     request = Request(
         service=service.name,

test/AWS.jl

@@ -457,6 +457,24 @@ end
     end
 end
 
+@testset "_clean_s3_uri" begin
+    uri = "/test-bucket/*)=('! +.txt?list-objects=v2"
+    expected_uri = "/test-bucket/%2A%29%3D%28%27%21%20%2B.txt?list-objects=v2"
+    @test AWS._clean_s3_uri(uri) == expected_uri
+
+    # make sure that other parts of the uri aren't changed by `_clean_s3_uri`
+    for uri in (
+            "https://julialang.org",
+            "http://julialang.org",
+            "http://julialang.org:8080",
+            "/onlypath",
+            "/path?query=  +99",
+            "/anchor?query=yes#anchor1"
+    )
+        @test AWS._clean_s3_uri(uri) == uri
+    end
+end
+
 @testset "json" begin
     @testset "high-level secrets manager" begin
         @service Secrets_Manager

test/minio.jl

@@ -36,6 +36,12 @@ S3.put_object("anewbucket","foo/baz",Dict("body"=>"a secondnested object"))
 objs = S3.list_objects_v2("anewbucket")
 @test length(objs["Contents"]) == 4
 
+# Test api version 2 of list-objects
+objs_truncated = S3.list_objects_v2("anewbucket", Dict("max-keys"=>2))
+@test length(objs_truncated["Contents"]) == 2
+@test objs_truncated["IsTruncated"] == "true"
+@test haskey(objs_truncated,"NextContinuationToken")
+
 # Test listing with prefixes
 objs_prefix = S3.list_objects_v2("anewbucket", Dict("prefix"=>"", "delimiter"=>"/"))
 @test length(objs_prefix["Contents"]) == 2