added ListObjectsV2 call

src/AWSS3.jl

@@ -16,7 +16,7 @@ export S3Path, s3_arn, s3_put, s3_get, s3_get_file, s3_exists, s3_delete, s3_cop
        s3_create_bucket,
        s3_put_cors,
        s3_enable_versioning, s3_delete_bucket, s3_list_buckets,
-       s3_list_objects, s3_list_keys, s3_list_versions,
+       s3_list_objects, s3_list_objects_v2, s3_list_keys, s3_list_versions,
        s3_get_meta, s3_purge_versions,
        s3_sign_url, s3_begin_multipart_upload, s3_upload_part,
        s3_complete_multipart_upload, s3_multipart_upload,
@@ -538,8 +538,70 @@ function s3_list_objects(aws::AWSConfig, bucket, path_prefix=""; delimiter="/",
     end
 end
 
-s3_list_objects(a...) = s3_list_objects(default_aws_config(), a...)
+s3_list_objects(a...; kwargs...) = s3_list_objects(default_aws_config(), a...; kwargs...)
 
+"""
+    s3_list_objects_v2([::AWSConfig], bucket, [path_prefix]; delimiter="/", max_items=1000)
+
+[List Objects](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html)
+in `bucket` with optional `path_prefix`.
+
+Returns an iterator of `Dict`s with keys `Key`, `LastModified`, `ETag`, `Size`,
+`Owner`, `StorageClass`.
+
+This uses the `ListObjectV2` function call.
+"""
+function s3_list_objects_v2(aws::AWSS3.AWSConfig, bucket, path_prefix=""; delimiter="/", max_items=nothing)
+    return Channel() do chnl
+        more = true
+        num_objects = 0
+        contoken = ""
+
+        while more
+            q = Dict{String, String}("list-type"=>"2")
+            if path_prefix != ""
+                q["prefix"] = path_prefix
+            end
+            if delimiter != ""
+                q["delimiter"] = delimiter
+            end
+            if contoken ≠ ""
+                q["continuation-token"] = contoken
+            end
+            if max_items !== nothing
+                q["max-keys"] = string(max_items - num_objects)
+            end
+
+            @repeat 4 try
+                # Request objects
+                r = s3(aws, "GET", bucket; query = q)
+
+                # Add each object from the response and update our object count / marker
+                if haskey(r, "Contents")
+                    l = isa(r["Contents"], Vector) ? r["Contents"] : [r["Contents"]]
+                    for object in l
+                        put!(chnl, xml_dict(object))
+                        num_objects += 1
+                    end
+                # It's possible that the response doesn't have "Contents" and just has a prefix,
+                # in which case we should just save the next marker and iterate.
+                elseif haskey(r, "Prefix")
+                    put!(chnl, Dict("Key" => r["Prefix"]))
+                    num_objects += 1
+                end
+
+                contoken = get(r, "NextContinuationToken", "")
+
+                # Continue looping if the results were truncated and we haven't exceeded out max_items (if specified)
+                more = r["IsTruncated"] == "true" && (max_items === nothing || num_objects < max_items)
+            catch e
+                @delay_retry if ecode(e) in ["NoSuchBucket"] end
+            end
+        end
+    end
+end
+
+s3_list_objects_v2(a...; kwargs...) = s3_list_objects_v2(default_aws_config(), a...; kwargs...)
 
 """
     s3_list_keys([::AWSConfig], bucket, [path_prefix])