plugins/rest: SigV4 Signing for any AWS service

This adds a new `service` option to the `s3_signing` config, allowing for other AWS services (such
as API Gateway endpoints) to be used for bundles, decision logs etc.

For example:

```
services:
  decision-log-service:
    url: https://myrestapi.execute-api.ap-southeast-2.amazonaws.com/prod/
    credentials:
      s3_signing:
        service: execute-api
        environment_credentials: {}

decision_logs:
  service: decision-log-service
  reporting:
    min_delay_seconds: 300
    max_delay_seconds: 600
```

If no service is specified, we default to `s3` to maintain backwards compatibility.

This updates the sigv4 signer to include the specified service in the signature, and to sign all
request headers for better compatibility with other AWS services, except an explicit ignore list,
as per https://github.com/aws/aws-sdk-go/blob/master/aws/signer/v4/v4.go#L92

Additionally, this fixes a bug in the signer where the body ReadCloser was consumed and not reset,
meaning requests that were signed were always sent with an empty body!

Fixes #3193

Signed-off-by: Jack Stevenson <jacsteve@amazon.com>

docs/content/configuration.md

@@ -384,10 +384,16 @@ keys:
 
 #### AWS Signature
 
-OPA will authenticate with an [AWS4 HMAC](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html) signature. Two methods of obtaining the
+OPA will authenticate with an [AWS4 HMAC](https://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html) signature. Several methods of obtaining the
 necessary credentials are available; exactly one must be specified to use the AWS signature
 authentication method.
 
+The AWS service for which to sign the request can be specified in the `service` field. If omitted, the default is `s3`.
+
+| Field | Type | Required | Description |
+| --- | --- | --- | --- |
+| `services[_].credentials.s3_signing.service` | `string` | No | The AWS service to sign requests with, eg `execute-api` or `s3`. Default: `s3` |
+
 ##### Using Static Environment Credentials
 If specifying `environment_credentials`, OPA will expect to find environment variables
 for `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_REGION`, in accordance with the

plugins/rest/aws.go

@@ -5,6 +5,7 @@
 package rest
 
 import (
+	"bytes"
 	"crypto/hmac"
 	"crypto/sha256"
 	"encoding/json"
@@ -47,6 +48,15 @@ const (
 	awsWebIdentityTokenFileEnvVar = "AWS_WEB_IDENTITY_TOKEN_FILE"
 )
 
+// Headers that may be mutated before reaching an aws service (eg by a proxy) should be added here to omit them from
+// the sigv4 canonical request
+// ref. https://github.com/aws/aws-sdk-go/blob/master/aws/signer/v4/v4.go#L92
+var awsSigv4IgnoredHeaders = map[string]bool{
+	"authorization":   true,
+	"user-agent":      true,
+	"x-amzn-trace-id": true,
+}
+
 // awsCredentials represents the credentials obtained from an AWS credential provider
 type awsCredentials struct {
 	AccessKey    string
@@ -384,7 +394,7 @@ func sha256MAC(message []byte, key []byte) []byte {
 	return mac.Sum(nil)
 }
 
-func sortKeys(strMap map[string]string) []string {
+func sortKeys(strMap map[string][]string) []string {
 	keys := make([]string, len(strMap))
 
 	i := 0
@@ -397,7 +407,11 @@ func sortKeys(strMap map[string]string) []string {
 }
 
 // signV4 modifies an http.Request to include an AWS V4 signature based on a credential provider
-func signV4(req *http.Request, credService awsCredentialService, theTime time.Time) error {
+func signV4(req *http.Request, service string, credService awsCredentialService, theTime time.Time) error {
+	// General ref. https://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
+	// S3 ref. https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
+	// APIGateway ref. https://docs.aws.amazon.com/apigateway/api-reference/signing-requests/
+
 	var body []byte
 	if req.Body == nil {
 		body = []byte("")
@@ -407,6 +421,9 @@ func signV4(req *http.Request, credService awsCredentialService, theTime time.Ti
 		if err != nil {
 			return errors.New("error getting request body: " + err.Error())
 		}
+		// Since ReadAll consumed the body ReadCloser, we must create a new ReadCloser for the request so that the
+		// subsequent read starts from the beginning
+		req.Body = ioutil.NopCloser(bytes.NewReader(body))
 	}
 	creds, err := credService.credentials()
 	if err != nil {
@@ -422,9 +439,13 @@ func signV4(req *http.Request, credService awsCredentialService, theTime time.Ti
 	iso8601Now := now.Format("20060102T150405Z")
 
 	awsHeaders := map[string]string{
-		"host":                 req.URL.Host,
-		"x-amz-content-sha256": bodyHexHash,
-		"x-amz-date":           iso8601Now,
+		"host":       req.URL.Host,
+		"x-amz-date": iso8601Now,
+	}
+
+	// s3 and glacier require the extra x-amz-content-sha256 header. other services do not.
+	if service == "s3" || service == "glacier" {
+		awsHeaders["x-amz-content-sha256"] = bodyHexHash
 	}
 
 	// the security token header is necessary for ephemeral credentials, e.g. from
@@ -433,7 +454,20 @@ func signV4(req *http.Request, credService awsCredentialService, theTime time.Ti
 		awsHeaders["x-amz-security-token"] = creds.SessionToken
 	}
 
-	// ref. https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
+	headersToSign := map[string][]string{}
+
+	// sign all of the aws headers
+	for k, v := range awsHeaders {
+		headersToSign[k] = []string{v}
+	}
+
+	// sign all of the request's headers, except for those in the ignore list
+	for k, v := range req.Header {
+		lowerCaseHeader := strings.ToLower(k)
+		if !awsSigv4IgnoredHeaders[lowerCaseHeader] {
+			headersToSign[lowerCaseHeader] = v
+		}
+	}
 
 	// the "canonical request" is the normalized version of the AWS service access
 	// that we're attempting to perform; in this case, a GET from an S3 bucket
@@ -442,9 +476,9 @@ func signV4(req *http.Request, credService awsCredentialService, theTime time.Ti
 	canonicalReq += "\n"                         // query string; not implemented
 
 	// include the values for the signed headers
-	orderedKeys := sortKeys(awsHeaders)
+	orderedKeys := sortKeys(headersToSign)
 	for _, k := range orderedKeys {
-		canonicalReq += k + ":" + awsHeaders[k] + "\n"
+		canonicalReq += k + ":" + strings.Join(headersToSign[k], ",") + "\n"
 	}
 	canonicalReq += "\n" // linefeed to terminate headers
 
@@ -455,17 +489,17 @@ func signV4(req *http.Request, credService awsCredentialService, theTime time.Ti
 
 	// the "string to sign" is a time-bounded, scoped request token which
 	// is linked to the "canonical request" by inclusion of its SHA-256 hash
-	strToSign := "AWS4-HMAC-SHA256\n"                                    // V4 signing with SHA-256 HMAC
-	strToSign += iso8601Now + "\n"                                       // ISO 8601 time
-	strToSign += dateNow + "/" + creds.RegionName + "/s3/aws4_request\n" // scoping for signature
-	strToSign += fmt.Sprintf("%x", sha256.Sum256([]byte(canonicalReq)))  // SHA-256 of canonical request
+	strToSign := "AWS4-HMAC-SHA256\n"                                                 // V4 signing with SHA-256 HMAC
+	strToSign += iso8601Now + "\n"                                                    // ISO 8601 time
+	strToSign += dateNow + "/" + creds.RegionName + "/" + service + "/aws4_request\n" // scoping for signature
+	strToSign += fmt.Sprintf("%x", sha256.Sum256([]byte(canonicalReq)))               // SHA-256 of canonical request
 
 	// the "signing key" is generated by repeated HMAC-SHA256 based on the same
 	// scoping that's included in the "string to sign"; but including the secret key
 	// to allow AWS to validate it
 	signingKey := sha256MAC([]byte(dateNow), []byte("AWS4"+creds.SecretKey))
 	signingKey = sha256MAC([]byte(creds.RegionName), signingKey)
-	signingKey = sha256MAC([]byte("s3"), signingKey)
+	signingKey = sha256MAC([]byte(service), signingKey)
 	signingKey = sha256MAC([]byte("aws4_request"), signingKey)
 
 	// the "signature" is finally the "string to sign" signed by the "signing key"
@@ -474,15 +508,15 @@ func signV4(req *http.Request, credService awsCredentialService, theTime time.Ti
 	// required format of Authorization header; n.b. the access key corresponding to
 	// the secret key is included here
 	authHdr := "AWS4-HMAC-SHA256 Credential=" + creds.AccessKey + "/" + dateNow
-	authHdr += "/" + creds.RegionName + "/s3/aws4_request,"
+	authHdr += "/" + creds.RegionName + "/" + service + "/aws4_request,"
 	authHdr += "SignedHeaders=" + headerList + ","
 	authHdr += "Signature=" + fmt.Sprintf("%x", signature)
 
 	// add the computed Authorization
-	req.Header.Add("Authorization", authHdr)
+	req.Header.Set("Authorization", authHdr)
 
 	// populate the other signed headers into the request
-	for _, k := range orderedKeys {
+	for k := range awsHeaders {
 		req.Header.Add(k, awsHeaders[k])
 	}
 

plugins/rest/aws_test.go

@@ -271,7 +271,7 @@ func TestV4Signing(t *testing.T) {
 		logger:          sdk.NewStandardLogger(),
 	}
 	req, _ := http.NewRequest("GET", "https://mybucket.s3.amazonaws.com/bundle.tar.gz", strings.NewReader(""))
-	err := signV4(req, cs, time.Unix(1556129697, 0))
+	err := signV4(req, "s3", cs, time.Unix(1556129697, 0))
 
 	assertErr("error getting AWS credentials: metadata HTTP request returned unexpected status: 404 Not Found", err, t)
 
@@ -290,10 +290,10 @@ func TestV4Signing(t *testing.T) {
 		Token:           "MYAWSSECURITYTOKENGOESHERE",
 		Expiration:      time.Now().UTC().Add(time.Minute * 2)}
 	req, _ = http.NewRequest("GET", "https://mybucket.s3.amazonaws.com/bundle.tar.gz", strings.NewReader(""))
-	err = signV4(req, cs, time.Unix(1556129697, 0))
+	err = signV4(req, "s3", cs, time.Unix(1556129697, 0))
 
 	if err != nil {
-		t.Error("unexpected error during signing")
+		t.Fatal("unexpected error during signing")
 	}
 
 	// expect mandatory headers
@@ -308,6 +308,89 @@ func TestV4Signing(t *testing.T) {
 	assertEq(req.Header.Get("X-Amz-Security-Token"), "MYAWSSECURITYTOKENGOESHERE", t)
 }
 
+func TestV4SigningForApiGateway(t *testing.T) {
+	ts := ec2CredTestServer{}
+	ts.start()
+	defer ts.stop()
+
+	cs := &awsMetadataCredentialService{
+		RoleName:        "my_iam_role", // not present
+		RegionName:      "us-east-1",
+		credServicePath: ts.server.URL + "/latest/meta-data/iam/security-credentials/",
+		tokenPath:       ts.server.URL + "/latest/api/token",
+		logger:          sdk.NewStandardLogger(),
+	}
+	ts.payload = metadataPayload{
+		AccessKeyID:     "MYAWSACCESSKEYGOESHERE",
+		SecretAccessKey: "MYAWSSECRETACCESSKEYGOESHERE",
+		Code:            "Success",
+		Token:           "MYAWSSECURITYTOKENGOESHERE",
+		Expiration:      time.Now().UTC().Add(time.Minute * 2)}
+	req, _ := http.NewRequest("POST", "https://myrestapi.execute-api.us-east-1.amazonaws.com/prod/logs",
+		strings.NewReader("{ \"payload\": 42 }"))
+	req.Header.Set("Content-Type", "application/json")
+
+	err := signV4(req, "execute-api", cs, time.Unix(1556129697, 0))
+
+	if err != nil {
+		t.Fatal("unexpected error during signing")
+	}
+
+	// expect mandatory headers
+	assertEq(req.Header.Get("Host"), "myrestapi.execute-api.us-east-1.amazonaws.com", t)
+	assertEq(req.Header.Get("Authorization"),
+		"AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20190424/us-east-1/execute-api/aws4_request,"+
+			"SignedHeaders=content-type;host;x-amz-date;x-amz-security-token,"+
+			"Signature=c8ee72cc45050b255bcbf19defc693f7cd788959b5380fa0985de6e865635339", t)
+	// no content sha should be set, since this is specific to s3 and glacier
+	assertEq(req.Header.Get("X-Amz-Content-Sha256"), "", t)
+	assertEq(req.Header.Get("X-Amz-Date"), "20190424T181457Z", t)
+	assertEq(req.Header.Get("X-Amz-Security-Token"), "MYAWSSECURITYTOKENGOESHERE", t)
+}
+
+func TestV4SigningOmitsIgnoredHeaders(t *testing.T) {
+	ts := ec2CredTestServer{}
+	ts.start()
+	defer ts.stop()
+
+	cs := &awsMetadataCredentialService{
+		RoleName:        "my_iam_role", // not present
+		RegionName:      "us-east-1",
+		credServicePath: ts.server.URL + "/latest/meta-data/iam/security-credentials/",
+		tokenPath:       ts.server.URL + "/latest/api/token",
+		logger:          sdk.NewStandardLogger(),
+	}
+	ts.payload = metadataPayload{
+		AccessKeyID:     "MYAWSACCESSKEYGOESHERE",
+		SecretAccessKey: "MYAWSSECRETACCESSKEYGOESHERE",
+		Code:            "Success",
+		Token:           "MYAWSSECURITYTOKENGOESHERE",
+		Expiration:      time.Now().UTC().Add(time.Minute * 2)}
+	req, _ := http.NewRequest("POST", "https://myrestapi.execute-api.us-east-1.amazonaws.com/prod/logs",
+		strings.NewReader("{ \"payload\": 42 }"))
+	req.Header.Set("Content-Type", "application/json")
+
+	// These are headers that should never be included in the signed headers
+	req.Header.Set("User-Agent", "Unit Tests!")
+	req.Header.Set("Authorization", "Auth header will be overwritten, and shouldn't be signed")
+	req.Header.Set("X-Amzn-Trace-Id", "Some trace id")
+
+	err := signV4(req, "execute-api", cs, time.Unix(1556129697, 0))
+
+	if err != nil {
+		t.Fatal("unexpected error during signing")
+	}
+
+	// Check the signed headers doesn't include user-agent, authorization or x-amz-trace-id
+	assertEq(req.Header.Get("Authorization"),
+		"AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20190424/us-east-1/execute-api/aws4_request,"+
+			"SignedHeaders=content-type;host;x-amz-date;x-amz-security-token,"+
+			"Signature=c8ee72cc45050b255bcbf19defc693f7cd788959b5380fa0985de6e865635339", t)
+	// The headers omitted from signing should still be present in the request
+	assertEq(req.Header.Get("User-Agent"), "Unit Tests!", t)
+	assertEq(req.Header.Get("X-Amzn-Trace-Id"), "Some trace id", t)
+}
+
 func TestV4SigningCustomPort(t *testing.T) {
 	ts := ec2CredTestServer{}
 	ts.start()
@@ -327,10 +410,10 @@ func TestV4SigningCustomPort(t *testing.T) {
 		Token:           "MYAWSSECURITYTOKENGOESHERE",
 		Expiration:      time.Now().UTC().Add(time.Minute * 2)}
 	req, _ := http.NewRequest("GET", "https://custom.s3.server:9000/bundle.tar.gz", strings.NewReader(""))
-	err := signV4(req, cs, time.Unix(1556129697, 0))
+	err := signV4(req, "s3", cs, time.Unix(1556129697, 0))
 
 	if err != nil {
-		t.Error("unexpected error during signing")
+		t.Fatal("unexpected error during signing")
 	}
 
 	// expect mandatory headers
@@ -345,6 +428,77 @@ func TestV4SigningCustomPort(t *testing.T) {
 	assertEq(req.Header.Get("X-Amz-Security-Token"), "MYAWSSECURITYTOKENGOESHERE", t)
 }
 
+func TestV4SigningDoesNotMutateBody(t *testing.T) {
+	ts := ec2CredTestServer{}
+	ts.start()
+	defer ts.stop()
+
+	cs := &awsMetadataCredentialService{
+		RoleName:        "my_iam_role", // not present
+		RegionName:      "us-east-1",
+		credServicePath: ts.server.URL + "/latest/meta-data/iam/security-credentials/",
+		tokenPath:       ts.server.URL + "/latest/api/token",
+		logger:          sdk.NewStandardLogger(),
+	}
+	ts.payload = metadataPayload{
+		AccessKeyID:     "MYAWSACCESSKEYGOESHERE",
+		SecretAccessKey: "MYAWSSECRETACCESSKEYGOESHERE",
+		Code:            "Success",
+		Token:           "MYAWSSECURITYTOKENGOESHERE",
+		Expiration:      time.Now().UTC().Add(time.Minute * 2)}
+	req, _ := http.NewRequest("POST", "https://myrestapi.execute-api.us-east-1.amazonaws.com/prod/logs",
+		strings.NewReader("{ \"payload\": 42 }"))
+
+	err := signV4(req, "execute-api", cs, time.Unix(1556129697, 0))
+
+	if err != nil {
+		t.Fatal("unexpected error during signing")
+	}
+
+	// Read the body and check that it was not mutated
+	body, _ := ioutil.ReadAll(req.Body)
+	assertEq(string(body), "{ \"payload\": 42 }", t)
+}
+
+func TestV4SigningWithMultiValueHeaders(t *testing.T) {
+	ts := ec2CredTestServer{}
+	ts.start()
+	defer ts.stop()
+
+	cs := &awsMetadataCredentialService{
+		RoleName:        "my_iam_role", // not present
+		RegionName:      "us-east-1",
+		credServicePath: ts.server.URL + "/latest/meta-data/iam/security-credentials/",
+		tokenPath:       ts.server.URL + "/latest/api/token",
+		logger:          sdk.NewStandardLogger(),
+	}
+	ts.payload = metadataPayload{
+		AccessKeyID:     "MYAWSACCESSKEYGOESHERE",
+		SecretAccessKey: "MYAWSSECRETACCESSKEYGOESHERE",
+		Code:            "Success",
+		Token:           "MYAWSSECURITYTOKENGOESHERE",
+		Expiration:      time.Now().UTC().Add(time.Minute * 2)}
+	req, _ := http.NewRequest("POST", "https://myrestapi.execute-api.us-east-1.amazonaws.com/prod/logs",
+		strings.NewReader("{ \"payload\": 42 }"))
+	req.Header.Add("Accept", "text/plain")
+	req.Header.Add("Accept", "text/html")
+
+	err := signV4(req, "execute-api", cs, time.Unix(1556129697, 0))
+
+	if err != nil {
+		t.Fatal("unexpected error during signing")
+	}
+
+	// Check the signed headers includes our multi-value 'accept' header
+	assertEq(req.Header.Get("Authorization"),
+		"AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20190424/us-east-1/execute-api/aws4_request,"+
+			"SignedHeaders=accept;host;x-amz-date;x-amz-security-token,"+
+			"Signature=0237b0c789cad36212f0efba70c02549e1f659ab9caaca16423930cc7236c046", t)
+	// The multi-value headers are preserved
+	assertEq(req.Header.Values("Accept")[0], "text/plain", t)
+	assertEq(req.Header.Values("Accept")[1], "text/html", t)
+}
+
 // simulate EC2 metadata service
 type ec2CredTestServer struct {
 	t         *testing.T