Merge #116

116: Return both body and header for all AWS requests r=mattBrzezinski a=mattBrzezinski

`AWSS3.jl` is having issues where the multi-part upload is not working properly, it attempts to return request headers back as both the `part number` and `etag` are required to successfully complete a multi-part upload.

`AWSCore.jl` currently does not return headers unless the operation is a `HEAD` type request. This merge request returns back the response `body` and `headers` for all requests, which accommodates for these requirements.

This change will allow use to begin resolving [AWSS3.jl #84](JuliaCloud/AWSS3.jl#84).

Co-authored-by: Matt Brzezinski <matt.brzezinski@invenia.ca>
Co-authored-by: Matt.jl <matt.brzezinski@invenia.ca>

Project.toml

@@ -1,6 +1,6 @@
 name = "AWSCore"
 uuid = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598"
-version = "0.6.9"
+version = "0.6.10"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

src/AWSCore.jl

@@ -225,7 +225,6 @@ end
 Process request for AWS "query" service protocol.
 """
 function service_query(aws::AWSConfig; args...)
-
  request = Dict{Symbol,Any}(args)
 
  request[:verb] = "POST"
@@ -257,7 +256,6 @@ end
 Process request for AWS "json" service protocol.
 """
 function service_json(aws::AWSConfig; args...)
-
  request = Dict{Symbol,Any}(args)
 
  request[:verb] = "POST"
@@ -301,7 +299,6 @@ end
 Process request for AWS "rest_json" service protocol.
 """
 function service_rest_json(aws::AWSConfig; args...)
-
  request = Dict{Symbol,Any}(args)
  args = Dict(request[:args])
 
@@ -323,7 +320,6 @@ end
 Process request for AWS "rest_xml" service protocol.
 """
 function service_rest_xml(aws::AWSConfig; args...)
-
  request = Dict{Symbol,Any}(args)
  args = stringdict(request[:args])
 
@@ -387,12 +383,11 @@ end
 
 
 """
- do_request(::AWSRequest)
+ do_request(::AWSRequest; return_headers=false)
 
-Submit an API request, return the result.
+Submit an API request, return the response body and headers if `return_headers=true`.
 """
-function do_request(r::AWSRequest)
-
+function do_request(r::AWSRequest; return_headers=false)
  response = nothing
 
  # Try request 3 times to deal with possible Redirect and ExiredToken...
@@ -507,7 +502,7 @@ function do_request(r::AWSRequest)
 
  # Return raw data if requested...
  if get(r, :return_raw, false)
- return response.body
+ return (return_headers ? (response.body, response.headers) : response.body)
  end
 
  # Parse response data according to mimetype...
@@ -521,17 +516,17 @@ function do_request(r::AWSRequest)
  body = String(copy(response.body))
 
  if occursin(r"/xml", mime)
- return parse_xml(body)
+ return (return_headers ? (parse_xml(body), Dict(response.headers)) : parse_xml(body))
  end
 
  if occursin(r"/x-amz-json-1.[01]$", mime)
  if isempty(response.body)
  return nothing
  end
  if get(r, :ordered_json_dict, true)
- return JSON.parse(body, dicttype=OrderedDict)
+ return (return_headers ? (JSON.parse(body, dicttype=OrderedDict), Dict(response.headers)) : JSON.parse(body, dicttype=OrderedDict))
  else
- return JSON.parse(body)
+ return (return_headers ? (JSON.parse(body), Dict(response.headers)) : JSON.parse(body))
  end
  end
 
@@ -551,15 +546,16 @@ function do_request(r::AWSRequest)
  catch e
  @ignore if typeof(e) == KeyError end
  end
- return info
+
+ return (return_headers ? (info, Dict(response.headers)) : info)
  end
 
  if occursin(r"^text/", mime)
- return body
+ return (return_headers ? (body, Dict(response.headers)) : body)
  end
 
  # Return raw data by default...
- return response.body
+ return (return_headers ? (response.body, nothing) : response.body)
 end
 
 

test/aws4.jl

@@ -221,17 +221,20 @@ end
  expected_data = "{\n \"zarr_format\": 2\n}"
 
  @test result["Contents"][1]["Key"] == expected_content_key
- @test String(AWSCore.Services.s3(
+
+ key_result = AWSCore.Services.s3(
  aws_google,
  "GET", "/{Bucket}/{Key+}",
  Bucket="cmip6",
- Key="$(data_prefix)/.zgroup")
- ) == expected_data
+ Key="$(data_prefix)/.zgroup"
+ )
+
+ @test String(key_result) == expected_data
  end
 
  @testset "Accessing OTC Object Store" begin
  aws_otc = aws_config(creds=nothing, region="eu-de", service_name="obs", service_host="otc.t-systems.com")
- result = AWSCore.Services.s3(aws_otc, "GET", "/{Bucket}?list-type=2", Dict(:Bucket=>"obs-esdc-v2.0.0",:prefix=>"",:delimiter=>"/"))
+ result = AWSCore.Services.s3(aws_otc, "GET", "/{Bucket}?list-type=2", Dict(:Bucket=>"obs-esdc-v2.0.0", :prefix=>"",:delimiter=>"/"))
 
  @test result["CommonPrefixes"][1]["Prefix"] == "esdc-8d-0.0083deg-184x60x60-2.0.0_colombia.zarr/"
  end