adds support for s3 and s3control private link (#3770)

cr https://code.amazon.com/reviews/CR-42972833

CHANGELOG_PENDING.md

@@ -1,6 +1,7 @@
 ### SDK Features
 
 ### SDK Enhancements
+* `service/s3`: Amazon S3 now supports AWS PrivateLink, providing direct access to S3 via a private endpoint within your virtual private network.
 
 ### SDK Bugs
 * `aws/session`: Fixed a bug that prevented credentials from being sourced from the environment if the loaded shared config profile contained partial SSO configuration. ([#3769](https://github.com/aws/aws-sdk-go/pull/3769))

example/service/s3/usingPrivateLink/README.md

@@ -0,0 +1,33 @@
+# Example
+
+This example demonstrates how you can use the AWS SDK for Go's Amazon S3 client 
+to use AWS PrivateLink for Amazon S3.
+
+# Usage
+
+To access S3 bucket data using the s3 interface endpoints, prefix the vpc 
+endpoint with `bucket`. For eg, use endpoint url as `https://bucket.vpce-0xxxxxxx-xxx8xxg.s3.us-west-2.vpce.amazonaws.com` 
+to access S3 bucket data via the associated vpc endpoint. The SDK may mutate 
+this endpoint as per the input provided to work with ARNs. 
+
+To access S3 access point data using the s3 interface endpoints, prefix the vpc 
+endpoint with `accesspoint`. For eg, use endpoint url as `https://accesspoint.vpce-0xxxxxxx-xxxx8xxg.s3.us-west-2.vpce.amazonaws.com` 
+to access S3 access point data via the associated vpc endpoint. The SDK may 
+mutate this endpoint as per the input provided to work with ARNs.
+
+To work with S3 control using the s3 interface endpoints, prefix the vpc endpoint 
+with `control`. For eg, use endpoint url as `https://control.vpce-0xxxxxxx-xxx8xxg.s3.us-west-2.vpce.amazonaws.com` 
+to use S3 Control operations with the associated vpc endpoint. The SDK may mutate 
+this endpoint as per the input provided to work with ARNs.
+
+The example will create s3 client's that use appropriate vpc endpoint url. The example 
+will then create a bucket of the name provided in code. Replace the value of 
+the `accountID` const with the account ID for your AWS account. The 
+`vpcBucketEndpointUrl`, `vpcAccesspointEndpoint`, `vpcControlEndpoint`, `bucket`, 
+`keyName`, and `accessPoint` const variables need to be updated to match the name 
+of the appropriate vpc endpoint, Bucket, Object Key, and Access Point that will be 
+created by the example.
+
+```sh
+AWS_REGION=<region> go run -tags example usingPrivateLink.go
+```

example/service/s3/usingPrivateLink/usingPrivateLink.go

@@ -0,0 +1,105 @@
+// +build example
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/arn"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3control"
+)
+
+const (
+	bucketName  = "myBucketName"
+	keyName     = "myKeyName"
+	accountID   = "123456789012"
+	accessPoint = "accesspointname"
+
+	// vpcBucketEndpoint will be used by the SDK to resolve an endpoint, when making a call to
+	// access `bucket` data using s3 interface endpoint. This endpoint may be mutated by the SDK,
+	// as per the input provided to work with ARNs.
+	vpcBucketEndpoint = "https://bucket.vpce-0xxxxxxx-xxx8xxg.s3.us-west-2.vpce.amazonaws.com"
+
+	// vpcAccesspointEndpoint will be used by the SDK to resolve an endpoint, when making a call to
+	// access `access-point` data using s3 interface endpoint. This endpoint may be mutated by the SDK,
+	// as per the input provided to work with ARNs.
+	vpcAccesspointEndpoint = "https://accesspoint.vpce-0xxxxxxx-xxx8xxg.s3.us-west-2.vpce.amazonaws.com"
+
+	// vpcControlEndpoint will be used by the SDK to resolve an endpoint, when making a call to
+	// access `control` data using s3 interface endpoint. This endpoint may be mutated by the SDK,
+	// as per the input provided to work with ARNs.
+	vpcControlEndpoint = "https://control.vpce-0xxxxxxx-xxx8xxg.s3.us-west-2.vpce.amazonaws.com"
+)
+
+func main() {
+	sess := session.Must(session.NewSession())
+
+	s3BucketSvc := s3.New(sess, &aws.Config{
+		Endpoint: aws.String(vpcBucketEndpoint),
+	})
+
+	s3AccesspointSvc := s3.New(sess, &aws.Config{
+		Endpoint: aws.String(vpcAccesspointEndpoint),
+	})
+
+	s3ControlSvc := s3control.New(sess, &aws.Config{
+		Endpoint: aws.String(vpcControlEndpoint),
+	})
+
+	// Create an S3 Bucket
+	fmt.Println("create s3 bucket")
+	_, err := s3BucketSvc.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err != nil {
+		panic(fmt.Errorf("failed to create bucket: %v", err))
+	}
+
+	// Wait for S3 Bucket to Exist
+	fmt.Println("wait for s3 bucket to exist")
+	err = s3BucketSvc.WaitUntilBucketExists(&s3.HeadBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err != nil {
+		panic(fmt.Sprintf("bucket failed to materialize: %v", err))
+	}
+
+	// Create an Access Point referring to the bucket
+	fmt.Println("create an access point")
+	_, err = s3ControlSvc.CreateAccessPoint(&s3control.CreateAccessPointInput{
+		AccountId: aws.String(accountID),
+		Bucket:    aws.String(bucketName),
+		Name:      aws.String(accessPoint),
+	})
+	if err != nil {
+		panic(fmt.Sprintf("failed to create access point: %v", err))
+	}
+
+	// Use the SDK's ARN builder to create an ARN for the Access Point.
+	apARN := arn.ARN{
+		Partition: "aws",
+		Service:   "s3",
+		Region:    aws.StringValue(sess.Config.Region),
+		AccountID: accountID,
+		Resource:  "accesspoint/" + accessPoint,
+	}
+
+	// And Use Access Point ARN where bucket parameters are accepted
+	fmt.Println("get object using access point")
+	getObjectOutput, err := s3AccesspointSvc.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(apARN.String()),
+		Key:    aws.String("somekey"),
+	})
+	if err != nil {
+		panic(fmt.Sprintf("failed get object request: %v", err))
+	}
+
+	_, err = ioutil.ReadAll(getObjectOutput.Body)
+	if err != nil {
+		panic(fmt.Sprintf("failed to read object body: %v", err))
+	}
+}

private/model/api/customization_passes.go

@@ -94,6 +94,10 @@ func supressSmokeTest(a *API) error {
 
 // Customizes the API generation to replace values specific to S3.
 func s3Customizations(a *API) error {
+
+	// back-fill signing name as 's3'
+	a.Metadata.SigningName = "s3"
+
 	var strExpires *Shape
 
 	var keepContentMD5Ref = map[string]struct{}{

service/s3/endpoint.go

@@ -98,7 +98,7 @@ func endpointHandler(req *request.Request) {
 		Request:  req,
 	}
 
-	if resReq.IsCrossPartition() {
+	if len(resReq.Request.ClientInfo.PartitionID) != 0 && resReq.IsCrossPartition() {
 		req.Error = s3shared.NewClientPartitionMismatchError(resource,
 			req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
 		return
@@ -110,11 +110,6 @@ func endpointHandler(req *request.Request) {
 		return
 	}
 
-	if resReq.HasCustomEndpoint() {
-		req.Error = s3shared.NewInvalidARNWithCustomEndpointError(resource, nil)
-		return
-	}
-
 	switch tv := resource.(type) {
 	case arn.AccessPointARN:
 		err = updateRequestAccessPointEndpoint(req, tv)
@@ -155,8 +150,7 @@ func updateRequestAccessPointEndpoint(req *request.Request, accessPoint arn.Acce
 			req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
 	}
 
-	// Ignore the disable host prefix for access points since custom endpoints
-	// are not supported.
+	// Ignore the disable host prefix for access points
 	req.Config.DisableEndpointHostPrefix = aws.Bool(false)
 
 	if err := accessPointEndpointBuilder(accessPoint).build(req); err != nil {
@@ -181,8 +175,7 @@ func updateRequestOutpostAccessPointEndpoint(req *request.Request, accessPoint a
 			req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
 	}
 
-	// Ignore the disable host prefix for access points since custom endpoints
-	// are not supported.
+	// Ignore the disable host prefix for access points
 	req.Config.DisableEndpointHostPrefix = aws.Bool(false)
 
 	if err := outpostAccessPointEndpointBuilder(accessPoint).build(req); err != nil {

service/s3/endpoint_builder.go

@@ -22,6 +22,11 @@ const (
 	outpostAccessPointPrefixTemplate = accessPointPrefixTemplate + "{" + outpostPrefixLabel + "}."
 )
 
+// hasCustomEndpoint returns true if endpoint is a custom endpoint
+func hasCustomEndpoint(r *request.Request) bool {
+	return len(aws.StringValue(r.Config.Endpoint)) > 0
+}
+
 // accessPointEndpointBuilder represents the endpoint builder for access point arn
 type accessPointEndpointBuilder arn.AccessPointARN
 
@@ -55,16 +60,19 @@ func (a accessPointEndpointBuilder) build(req *request.Request) error {
 			req.ClientInfo.PartitionID, cfgRegion, err)
 	}
 
-	if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
-		return err
-	}
+	endpoint.URL = endpoints.AddScheme(endpoint.URL, aws.BoolValue(req.Config.DisableSSL))
 
-	const serviceEndpointLabel = "s3-accesspoint"
+	if !hasCustomEndpoint(req) {
+		if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
+			return err
+		}
+		const serviceEndpointLabel = "s3-accesspoint"
 
-	// dual stack provided by endpoint resolver
-	cfgHost := req.HTTPRequest.URL.Host
-	if strings.HasPrefix(cfgHost, "s3") {
-		req.HTTPRequest.URL.Host = serviceEndpointLabel + cfgHost[2:]
+		// dual stack provided by endpoint resolver
+		cfgHost := req.HTTPRequest.URL.Host
+		if strings.HasPrefix(cfgHost, "s3") {
+			req.HTTPRequest.URL.Host = serviceEndpointLabel + cfgHost[2:]
+		}
 	}
 
 	protocol.HostPrefixBuilder{
@@ -116,14 +124,17 @@ func (o outpostAccessPointEndpointBuilder) build(req *request.Request) error {
 			req.ClientInfo.PartitionID, resolveRegion, err)
 	}
 
-	if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
-		return err
-	}
+	endpoint.URL = endpoints.AddScheme(endpoint.URL, aws.BoolValue(req.Config.DisableSSL))
 
-	// add url host as s3-outposts
-	cfgHost := req.HTTPRequest.URL.Host
-	if strings.HasPrefix(cfgHost, endpointsID) {
-		req.HTTPRequest.URL.Host = resolveService + cfgHost[len(endpointsID):]
+	if !hasCustomEndpoint(req) {
+		if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
+			return err
+		}
+		// add url host as s3-outposts
+		cfgHost := req.HTTPRequest.URL.Host
+		if strings.HasPrefix(cfgHost, endpointsID) {
+			req.HTTPRequest.URL.Host = resolveService + cfgHost[len(endpointsID):]
+		}
 	}
 
 	protocol.HostPrefixBuilder{
@@ -159,7 +170,6 @@ func resolveRegionalEndpoint(r *request.Request, region string, endpointsID stri
 }
 
 func updateRequestEndpoint(r *request.Request, endpoint string) (err error) {
-	endpoint = endpoints.AddScheme(endpoint, aws.BoolValue(r.Config.DisableSSL))
 
 	r.HTTPRequest.URL, err = url.Parse(endpoint + r.Operation.HTTPPath)
 	if err != nil {