From 279bfc6eba58c8d484c807d7816ce682273fa788 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 28 Apr 2021 13:12:33 -0500 Subject: [PATCH 1/4] Improve EC2 instance metadata function --- Project.toml | 2 +- src/AWSCredentials.jl | 38 +++++++++++++++++++------------------- test/AWSCredentials.jl | 17 +++++++++++++++-- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/Project.toml b/Project.toml index abf15da261..bd165c2ca3 100644 --- a/Project.toml +++ b/Project.toml @@ -23,7 +23,7 @@ XMLDict = "228000da-037f-5747-90a9-8195ccbf91a5" [compat] Compat = "3" GitHub = "5" -HTTP = "0.8, 0.9" +HTTP = "0.9.6" IniFile = "0.5" JSON = "0.18, 0.19, 0.20, 0.21" MbedTLS = "0.6, 0.7, 1" diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 064f26a09c..25b9d3a0cf 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -291,29 +291,29 @@ end """ - _ec2_metadata(metadata_endpoint::String) -> Union{String, Nothing} + ec2_instance_metadata(path::AbstractString) -> Union{String, Nothing} -Retrieve the EC2 meta data from the local AWS endpoint. Return the EC2 metadata request -body, or `nothing` if not running on an EC2 instance. +Retrieve the AWS EC2 instance metadata as a string using the provided `path`. If no instance +metadata is available (typically due to not running within an EC2 instance) then `nothing` +will be returned. See the AWS documentation for details on what metadata is available: +https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html # Arguments -- `metadata_endpoint::String`: AWS internal meta data endpoint to hit - https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html#instance-metadata-ex-1 - -# Throws -- `StatusError`: If the response status is >= 300, except for 404 -- `ParsingError`: Invalid HTTP request target +- `path`: The URI path to used to specify that metadata to return """ -function _ec2_metadata(metadata_endpoint::String) - try - request = @mock HTTP.request("GET", "http://169.254.169.254/latest/meta-data/$metadata_endpoint") - - return request === nothing ? nothing : String(request.body) +function ec2_instance_metadata(path::AbstractString) + uri = HTTP.URI(scheme="http", host="169.254.169.254", path=path) + request = try + @mock HTTP.request("GET", uri; connect_timeout=1) catch e - e isa HTTP.IOError || e isa HTTP.StatusError && e.status == 404 || rethrow(e) + if e isa HTTP.ConnectionPool.ConnectTimeout + nothing + else + rethrow() + end end - return nothing + return request !== nothing ? String(request.body) : nothing end @@ -323,7 +323,7 @@ end Parse the EC2 metadata to retrieve AWSCredentials. """ function ec2_instance_credentials() - info = _ec2_metadata("iam/info") + info = ec2_instance_metadata("/latest/meta-data/iam/info") if info === nothing return nothing @@ -331,8 +331,8 @@ function ec2_instance_credentials() info = JSON.parse(info) - name = _ec2_metadata("iam/security-credentials/") - creds = _ec2_metadata("iam/security-credentials/$name") + name = ec2_instance_metadata("/latest/meta-data/iam/security-credentials/") + creds = ec2_instance_metadata("/latest/meta-data/iam/security-credentials/$name") new_creds = JSON.parse(creds) expiry = DateTime(rstrip(new_creds["Expiration"], 'Z')) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 7e478535aa..2f7c44fd78 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -44,6 +44,18 @@ end @test AWS._role_session_name("a" ^ 22, "b" ^ 22, "c" ^ 22) == "a" ^ 22 * "b" ^ 20 * "c" ^ 22 end +@testset "ec2_instance_metadata" begin + @testset "connect timeout" begin + patch = @patch function HTTP.request(method::String, url; kwargs...) + throw(HTTP.ConnectionPool.ConnectTimeout("169.254.169.254", "80")) + end + + apply(patch) do + @test AWS.ec2_instance_metadata("/latest/meta-data") === nothing + end + end +end + @testset "AWSCredentials" begin @testset "Defaults" begin creds = AWSCredentials("access_key_id" ,"secret_key") @@ -328,9 +340,10 @@ end "Security-Credentials" => "Test-Security-Credentials" ) - _http_request_patch = @patch function HTTP.request(method::String, url::String) + _http_request_patch = @patch function HTTP.request(method::String, url; kwargs...) security_credentials = test_values["Security-Credentials"] uri = test_values["URI"] + url = string(url) if url == "http://169.254.169.254/latest/meta-data/iam/info" instance_profile_arn = test_values["InstanceProfileArn"] @@ -533,7 +546,7 @@ end end @testset "Credentials Not Found" begin - _http_request_patch = @patch function HTTP.request(method::String, url::String) + _http_request_patch = @patch function HTTP.request(method::String, url; kwargs...) return nothing end From f18d78d93d34164cf9f0bea34be3e1a53c258eab Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 28 Apr 2021 13:22:57 -0500 Subject: [PATCH 2/4] Create ec2_instance_region function --- src/AWSCredentials.jl | 12 ++++++++++++ test/AWSCredentials.jl | 22 ++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 25b9d3a0cf..dab9881211 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -317,6 +317,18 @@ function ec2_instance_metadata(path::AbstractString) end +""" + ec2_instance_region() -> Union{String, Nothing} + +Determine the AWS region of the machine executing this code if running inside of an EC2 +instance, otherwise `nothing` is returned. +""" +function ec2_instance_region() + az = ec2_instance_metadata("/latest/meta-data/placement/availability-zone") + return az !== nothing ? chop(az) : nothing +end + + """ ec2_instance_credentials() -> AWSCredentials diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 2f7c44fd78..64c015e019 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -13,6 +13,10 @@ macro test_ecode(error_codes, expr) end end +metadata_timeout_patch = @patch function HTTP.request(method::String, url; kwargs...) + throw(HTTP.ConnectionPool.ConnectTimeout("169.254.169.254", "80")) +end + @testset "Load Credentials" begin user = aws_user_arn(aws) @test occursin(r"^arn:aws:(iam|sts)::[0-9]+:[^:]+$", user) @@ -46,12 +50,26 @@ end @testset "ec2_instance_metadata" begin @testset "connect timeout" begin + apply(metadata_timeout_patch) do + @test AWS.ec2_instance_metadata("/latest/meta-data") === nothing + end + end +end + +@testset "ec2_instance_region" begin + @testset "available" begin patch = @patch function HTTP.request(method::String, url; kwargs...) - throw(HTTP.ConnectionPool.ConnectTimeout("169.254.169.254", "80")) + return HTTP.Response("ap-atlantis-1a") # Fake availability zone end apply(patch) do - @test AWS.ec2_instance_metadata("/latest/meta-data") === nothing + @test AWS.ec2_instance_region() == "ap-atlantis-1" + end + end + + @testset "unavailable" begin + apply(metadata_timeout_patch) do + @test AWS.ec2_instance_region() === nothing end end end From c349663a1c26d48b93c966c4c343b74edb104b55 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 28 Apr 2021 14:00:39 -0500 Subject: [PATCH 3/4] Refactor "Credentials Not Found" testset --- test/AWSCredentials.jl | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 64c015e019..a58325c3ec 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -564,31 +564,19 @@ end end @testset "Credentials Not Found" begin - _http_request_patch = @patch function HTTP.request(method::String, url; kwargs...) - return nothing - end - - ACCESS_KEY = "AWS_ACCESS_KEY_ID" - CONTAINER_URI = "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" - - old_aws_access_key_id = get(ENV, ACCESS_KEY, "") - old_container_uri = get(ENV, CONTAINER_URI, "") - - try - delete!(ENV, "AWS_ACCESS_KEY_ID") - delete!(ENV, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") + patches = [ + @patch HTTP.request(method::String, url; kwargs...) = nothing + Patches._cred_file_patch + Patches._config_file_patch + ] - apply([_http_request_patch, Patches._cred_file_patch, Patches._config_file_patch]) do + withenv( + "AWS_ACCESS_KEY_ID" => nothing, + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => nothing, + ) do + apply(patches) do @test_throws NoCredentials AWSConfig() end - finally - if !isempty(old_aws_access_key_id) - ENV[ACCESS_KEY] = old_aws_access_key_id - end - - if !isempty(old_container_uri) - ENV[CONTAINER_URI] = old_container_uri - end end end From 3e99dcfd26d02712617f5c6b7a5d4b5f348383ec Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 29 Apr 2021 08:49:06 -0500 Subject: [PATCH 4/4] Code review changes --- src/AWS.jl | 1 + src/AWSCredentials.jl | 5 +---- test/AWSCredentials.jl | 10 +++------- test/patch.jl | 4 ++++ 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/AWS.jl b/src/AWS.jl index e8ccbe3699..377bcf462f 100644 --- a/src/AWS.jl +++ b/src/AWS.jl @@ -15,6 +15,7 @@ import URIs export @service export _merge, AbstractAWSConfig, AWSConfig, AWSExceptions, AWSServices, Request +export ec2_instance_metadata, ec2_instance_region export global_aws_config, generate_service_url, set_user_agent, sign!, sign_aws2!, sign_aws4! export JSONService, RestJSONService, RestXMLService, QueryService diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index dab9881211..eed746eaf1 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -323,10 +323,7 @@ end Determine the AWS region of the machine executing this code if running inside of an EC2 instance, otherwise `nothing` is returned. """ -function ec2_instance_region() - az = ec2_instance_metadata("/latest/meta-data/placement/availability-zone") - return az !== nothing ? chop(az) : nothing -end +ec2_instance_region() = ec2_instance_metadata("/latest/meta-data/placement/region") """ diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index a58325c3ec..e8199b1481 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -13,10 +13,6 @@ macro test_ecode(error_codes, expr) end end -metadata_timeout_patch = @patch function HTTP.request(method::String, url; kwargs...) - throw(HTTP.ConnectionPool.ConnectTimeout("169.254.169.254", "80")) -end - @testset "Load Credentials" begin user = aws_user_arn(aws) @test occursin(r"^arn:aws:(iam|sts)::[0-9]+:[^:]+$", user) @@ -50,7 +46,7 @@ end @testset "ec2_instance_metadata" begin @testset "connect timeout" begin - apply(metadata_timeout_patch) do + apply(Patches._instance_metadata_timeout_patch) do @test AWS.ec2_instance_metadata("/latest/meta-data") === nothing end end @@ -59,7 +55,7 @@ end @testset "ec2_instance_region" begin @testset "available" begin patch = @patch function HTTP.request(method::String, url; kwargs...) - return HTTP.Response("ap-atlantis-1a") # Fake availability zone + return HTTP.Response("ap-atlantis-1") # Made up region end apply(patch) do @@ -68,7 +64,7 @@ end end @testset "unavailable" begin - apply(metadata_timeout_patch) do + apply(Patches._instance_metadata_timeout_patch) do @test AWS.ec2_instance_region() === nothing end end diff --git a/test/patch.jl b/test/patch.jl index 6eb9f32f00..09b192500b 100644 --- a/test/patch.jl +++ b/test/patch.jl @@ -101,4 +101,8 @@ _github_tree_patch = @patch function tree(repo, tree_obj; kwargs...) end end +_instance_metadata_timeout_patch = @patch function HTTP.request(method::String, url; kwargs...) + throw(HTTP.ConnectionPool.ConnectTimeout("169.254.169.254", "80")) +end + end