Add support for EC2 IMDS endpoint from environment variable (#3504)

Adds support for specifying a custom EC2 IMDS endpoint from the
environment variable, `AWS_EC2_METADATA_SERVICE_ENDPOINT`.

The `aws/session#Options` struct also has a new field,
`EC2IMDSEndpoint`. This field can be used to configure the custom
endpoint of the EC2 IMDS client. The option only applies to EC2 IMDS
clients created after the Session with EC2IMDSEndpoint is specified.

```go
sess, err := session.NewSessionWithOptions(session.Options{
	EC2IMDSEndpoint: "http://[::1]",
})
```

CHANGELOG_PENDING.md

@@ -3,3 +3,6 @@
 ### SDK Enhancements
 
 ### SDK Bugs
+* `aws/ec2metadata`: Add support for EC2 IMDS endpoint from environment variable ([#3504](https://github.com/aws/aws-sdk-go/pull/3504))
+  * Adds support for specifying a custom EC2 IMDS endpoint from the environment variable, `AWS_EC2_METADATA_SERVICE_ENDPOINT`.
+  * The `aws/session#Options` struct also has a new field, `EC2IMDSEndpoint`. This field can be used to configure the custom endpoint of the EC2 IMDS client. The option only applies to EC2 IMDS clients created after the Session with `EC2IMDSEndpoint` is specified.

aws/defaults/defaults_test.go

@@ -120,7 +120,7 @@ func TestDefaultEC2RoleProvider(t *testing.T) {
 	if ec2Provider == nil {
 		t.Fatalf("expect provider not to be nil, but was")
 	}
-	if e, a := "http://169.254.169.254/latest", ec2Provider.Client.Endpoint; e != a {
+	if e, a := "http://169.254.169.254", ec2Provider.Client.Endpoint; e != a {
 		t.Errorf("expect %q endpoint, got %q", e, a)
 	}
 }

aws/ec2metadata/api.go

@@ -20,7 +20,7 @@ func (c *EC2Metadata) getToken(ctx aws.Context, duration time.Duration) (tokenOu
 	op := &request.Operation{
 		Name:       "GetToken",
 		HTTPMethod: "PUT",
-		HTTPPath:   "/api/token",
+		HTTPPath:   "/latest/api/token",
 	}
 
 	var output tokenOutput
@@ -62,7 +62,7 @@ func (c *EC2Metadata) GetMetadataWithContext(ctx aws.Context, p string) (string,
 	op := &request.Operation{
 		Name:       "GetMetadata",
 		HTTPMethod: "GET",
-		HTTPPath:   sdkuri.PathJoin("/meta-data", p),
+		HTTPPath:   sdkuri.PathJoin("/latest/meta-data", p),
 	}
 	output := &metadataOutput{}
 
@@ -88,7 +88,7 @@ func (c *EC2Metadata) GetUserDataWithContext(ctx aws.Context) (string, error) {
 	op := &request.Operation{
 		Name:       "GetUserData",
 		HTTPMethod: "GET",
-		HTTPPath:   "/user-data",
+		HTTPPath:   "/latest/user-data",
 	}
 
 	output := &metadataOutput{}
@@ -113,7 +113,7 @@ func (c *EC2Metadata) GetDynamicDataWithContext(ctx aws.Context, p string) (stri
 	op := &request.Operation{
 		Name:       "GetDynamicData",
 		HTTPMethod: "GET",
-		HTTPPath:   sdkuri.PathJoin("/dynamic", p),
+		HTTPPath:   sdkuri.PathJoin("/latest/dynamic", p),
 	}
 
 	output := &metadataOutput{}

aws/ec2metadata/api_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/awstesting/unit"
+	"github.com/aws/aws-sdk-go/internal/sdktesting"
 )
 
 const instanceIdentityDocument = `{
@@ -106,22 +107,22 @@ func newTestServer(t *testing.T, testType testType, testServer *testServer) *htt
 	switch testType {
 	case SecureTestType:
 		mux.HandleFunc("/latest/api/token", getTokenRequiredParams(t, testServer.secureGetTokenHandler))
-		mux.HandleFunc("/latest/", testServer.secureGetLatestHandler)
+		mux.HandleFunc("/", testServer.secureGetLatestHandler)
 	case InsecureTestType:
 		mux.HandleFunc("/latest/api/token", testServer.insecureGetTokenHandler)
-		mux.HandleFunc("/latest/", testServer.insecureGetLatestHandler)
+		mux.HandleFunc("/", testServer.insecureGetLatestHandler)
 	case BadRequestTestType:
 		mux.HandleFunc("/latest/api/token", getTokenRequiredParams(t, testServer.badRequestGetTokenHandler))
-		mux.HandleFunc("/latest/", testServer.badRequestGetLatestHandler)
+		mux.HandleFunc("/", testServer.badRequestGetLatestHandler)
 	case ServerErrorForTokenTestType:
 		mux.HandleFunc("/latest/api/token", getTokenRequiredParams(t, testServer.serverErrorGetTokenHandler))
-		mux.HandleFunc("/latest/", testServer.insecureGetLatestHandler)
+		mux.HandleFunc("/", testServer.insecureGetLatestHandler)
 	case pageNotFoundForTokenTestType:
 		mux.HandleFunc("/latest/api/token", getTokenRequiredParams(t, testServer.pageNotFoundGetTokenHandler))
-		mux.HandleFunc("/latest/", testServer.insecureGetLatestHandler)
+		mux.HandleFunc("/", testServer.insecureGetLatestHandler)
 	case pageNotFoundWith401TestType:
 		mux.HandleFunc("/latest/api/token", getTokenRequiredParams(t, testServer.pageNotFoundGetTokenHandler))
-		mux.HandleFunc("/latest/", testServer.unauthorizedGetLatestHandler)
+		mux.HandleFunc("/", testServer.unauthorizedGetLatestHandler)
 
 	}
 
@@ -204,17 +205,17 @@ func (opListProvider *operationListProvider) addToOperationPerformedList(r *requ
 }
 
 func TestEndpoint(t *testing.T) {
+	restoreEnvFn := sdktesting.StashEnv()
+	defer restoreEnvFn()
+
 	c := ec2metadata.New(unit.Session)
 	op := &request.Operation{
 		Name:       "GetMetadata",
 		HTTPMethod: "GET",
-		HTTPPath:   path.Join("/", "meta-data", "testpath"),
+		HTTPPath:   path.Join("/latest", "meta-data", "testpath"),
 	}
 
 	req := c.NewRequest(op, nil, nil)
-	if e, a := "http://169.254.169.254/latest", req.ClientInfo.Endpoint; e != a {
-		t.Errorf("expect %v, got %v", e, a)
-	}
 	if e, a := "http://169.254.169.254/latest/meta-data/testpath", req.HTTPRequest.URL.String(); e != a {
 		t.Errorf("expect %v, got %v", e, a)
 	}
@@ -289,7 +290,9 @@ func TestGetMetadata(t *testing.T) {
 
 			op := &operationListProvider{}
 
-			c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+			c := ec2metadata.New(unit.Session, &aws.Config{
+				Endpoint: aws.String(server.URL),
+			})
 			c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 
 			resp, err := c.GetMetadata("some/path")
@@ -340,7 +343,9 @@ func TestGetUserData_Error(t *testing.T) {
 	}))
 
 	defer server.Close()
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 
 	resp, err := c.GetUserData()
 	if err == nil {
@@ -425,7 +430,9 @@ func TestGetRegion(t *testing.T) {
 
 			op := &operationListProvider{}
 
-			c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+			c := ec2metadata.New(unit.Session, &aws.Config{
+				Endpoint: aws.String(server.URL),
+			})
 			c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 
 			resp, err := c.Region()
@@ -494,7 +501,9 @@ func TestMetadataIAMInfo_success(t *testing.T) {
 
 			op := &operationListProvider{}
 
-			c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+			c := ec2metadata.New(unit.Session, &aws.Config{
+				Endpoint: aws.String(server.URL),
+			})
 			c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 
 			iamInfo, err := c.IAMInfo()
@@ -570,7 +579,9 @@ func TestMetadataIAMInfo_failure(t *testing.T) {
 
 			op := &operationListProvider{}
 
-			c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+			c := ec2metadata.New(unit.Session, &aws.Config{
+				Endpoint: aws.String(server.URL),
+			})
 			c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 
 			iamInfo, err := c.IAMInfo()
@@ -675,7 +686,9 @@ func TestEC2RoleProviderInstanceIdentity(t *testing.T) {
 
 			op := &operationListProvider{}
 
-			c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+			c := ec2metadata.New(unit.Session, &aws.Config{
+				Endpoint: aws.String(server.URL),
+			})
 			c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 			doc, err := c.GetInstanceIdentityDocument()
 
@@ -719,7 +732,9 @@ func TestEC2MetadataRetryFailure(t *testing.T) {
 	server := httptest.NewServer(mux)
 	defer server.Close()
 
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 
 	c.Handlers.AfterRetry.PushBack(func(i *request.Request) {
 		t.Logf("%v received, retrying operation %v", i.HTTPResponse.StatusCode, i.Operation.Name)
@@ -774,7 +789,9 @@ func TestEC2MetadataRetryOnce(t *testing.T) {
 
 	server := httptest.NewServer(mux)
 	defer server.Close()
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 
 	// Handler on client that logs if retried
 	c.Handlers.AfterRetry.PushBack(func(i *request.Request) {
@@ -807,7 +824,9 @@ func TestEC2Metadata_Concurrency(t *testing.T) {
 	server := newTestServer(t, SecureTestType, ts)
 	defer server.Close()
 
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 
 	var wg sync.WaitGroup
 	wg.Add(10)
@@ -838,11 +857,13 @@ func TestRequestOnMetadata(t *testing.T) {
 	server := newTestServer(t, SecureTestType, ts)
 	defer server.Close()
 
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 	req := c.NewRequest(&request.Operation{
 		Name:            "Ec2Metadata request",
 		HTTPMethod:      "GET",
-		HTTPPath:        "/latest",
+		HTTPPath:        "/latest/foo",
 		Paginator:       nil,
 		BeforePresignFn: nil,
 	}, nil, nil)
@@ -878,7 +899,9 @@ func TestExhaustiveRetryToFetchToken(t *testing.T) {
 
 	op := &operationListProvider{}
 
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 	c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 
 	resp, err := c.GetMetadata("/some/path")
@@ -930,7 +953,9 @@ func TestExhaustiveRetryWith401(t *testing.T) {
 
 	op := &operationListProvider{}
 
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 	c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 
 	resp, err := c.GetMetadata("/some/path")
@@ -991,7 +1016,9 @@ func TestRequestTimeOut(t *testing.T) {
 
 	op := &operationListProvider{}
 
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 	// for test, change the timeout to 100 ms
 	c.Config.HTTPClient.Timeout = 100 * time.Millisecond
 
@@ -1068,7 +1095,9 @@ func TestTokenExpiredBehavior(t *testing.T) {
 
 	op := &operationListProvider{}
 
-	c := ec2metadata.New(unit.Session, &aws.Config{Endpoint: aws.String(server.URL + "/latest")})
+	c := ec2metadata.New(unit.Session, &aws.Config{
+		Endpoint: aws.String(server.URL),
+	})
 	c.Handlers.Complete.PushBack(op.addToOperationPerformedList)
 
 	resp, err := c.GetMetadata("/some/path")

aws/ec2metadata/service.go

@@ -5,13 +5,18 @@
 // variable "AWS_EC2_METADATA_DISABLED=true". This environment variable set to
 // true instructs the SDK to disable the EC2 Metadata client. The client cannot
 // be used while the environment variable is set to true, (case insensitive).
+//
+// The endpoint of the EC2 IMDS client can be configured via the environment
+// variable, AWS_EC2_METADATA_SERVICE_ENDPOINT when creating the client with a
+// Session. See aws/session#Options.EC2IMDSEndpoint for more details.
 package ec2metadata
 
 import (
 	"bytes"
 	"errors"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"strconv"
 	"strings"
@@ -69,6 +74,9 @@ func New(p client.ConfigProvider, cfgs ...*aws.Config) *EC2Metadata {
 // a client when not using a session. Generally using just New with a session
 // is preferred.
 //
+// Will remove the URL path from the endpoint provided to ensure the EC2 IMDS
+// client is able to communicate with the EC2 IMDS API.
+//
 // If an unmodified HTTP client is provided from the stdlib default, or no client
 // the EC2RoleProvider's EC2Metadata HTTP client's timeout will be shortened.
 // To disable this set Config.EC2MetadataDisableTimeoutOverride to false. Enabled by default.
@@ -86,6 +94,15 @@ func NewClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio
 		cfg.MaxRetries = aws.Int(2)
 	}
 
+	if u, err := url.Parse(endpoint); err == nil {
+		// Remove path from the endpoint since it will be added by requests.
+		// This is an artifact of the SDK adding `/latest` to the endpoint for
+		// EC2 IMDS, but this is now moved to the operation definition.
+		u.Path = ""
+		u.RawPath = ""
+		endpoint = u.String()
+	}
+
 	svc := &EC2Metadata{
 		Client: client.New(
 			cfg,

aws/ec2metadata/service_test.go

@@ -1,3 +1,5 @@
+// +build go1.7
+
 package ec2metadata_test
 
 import (
@@ -89,9 +91,7 @@ func TestClientDisableIMDS(t *testing.T) {
 
 	os.Setenv("AWS_EC2_METADATA_DISABLED", "true")
 
-	svc := ec2metadata.New(unit.Session, &aws.Config{
-		LogLevel: aws.LogLevel(aws.LogDebugWithHTTPBody),
-	})
+	svc := ec2metadata.New(unit.Session)
 	resp, err := svc.GetUserData()
 	if err == nil {
 		t.Fatalf("expect error, got none")
@@ -109,6 +109,37 @@ func TestClientDisableIMDS(t *testing.T) {
 	}
 }
 
+func TestClientStripPath(t *testing.T) {
+	cases := map[string]struct {
+		Endpoint string
+		Expect   string
+	}{
+		"no change": {
+			Endpoint: "http://example.aws",
+			Expect:   "http://example.aws",
+		},
+		"strip path": {
+			Endpoint: "http://example.aws/foo",
+			Expect:   "http://example.aws",
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			restoreEnvFn := sdktesting.StashEnv()
+			defer restoreEnvFn()
+
+			svc := ec2metadata.New(unit.Session, &aws.Config{
+				Endpoint: aws.String(c.Endpoint),
+			})
+
+			if e, a := c.Expect, svc.ClientInfo.Endpoint; e != a {
+				t.Errorf("expect %v endpoint, got %v", e, a)
+			}
+		})
+	}
+}
+
 func runEC2MetadataClients(t *testing.T, cfg *aws.Config, atOnce int) {
 	var wg sync.WaitGroup
 	wg.Add(atOnce)