From 0e9c270a3632fb21d5d69f267a480df2aa01819a Mon Sep 17 00:00:00 2001 From: Jack Stevenson Date: Wed, 10 Mar 2021 10:05:55 +1100 Subject: [PATCH] plugins/rest: SigV4 Signing for any AWS service This adds a new `service` option to the `s3_signing` config, allowing for other AWS services (such as API Gateway endpoints) to be used for bundles, decision logs etc. For example: ``` services: decision-log-service: url: https://myrestapi.execute-api.ap-southeast-2.amazonaws.com/prod/ credentials: s3_signing: service: execute-api environment_credentials: {} decision_logs: service: decision-log-service reporting: min_delay_seconds: 300 max_delay_seconds: 600 ``` If no service is specified, we default to `s3` to maintain backwards compatibility. This updates the sigv4 signer to include the specified service in the signature, and to sign all request headers for better compatibility with other AWS services, except an explicit ignore list, as per https://github.com/aws/aws-sdk-go/blob/master/aws/signer/v4/v4.go#L92 Additionally, this fixes a bug in the signer where the body ReadCloser was consumed and not reset, meaning requests that were signed were always sent with an empty body! Fixes #3193 Signed-off-by: Jack Stevenson --- docs/content/configuration.md | 8 +- plugins/rest/aws.go | 66 ++++++++++---- plugins/rest/aws_test.go | 164 ++++++++++++++++++++++++++++++++-- plugins/rest/rest_auth.go | 13 ++- plugins/rest/rest_test.go | 13 +++ 5 files changed, 241 insertions(+), 23 deletions(-) diff --git a/docs/content/configuration.md b/docs/content/configuration.md index ef23d0d2fb..29efb8dff2 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -384,10 +384,16 @@ keys: #### AWS Signature -OPA will authenticate with an [AWS4 HMAC](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html) signature. Two methods of obtaining the +OPA will authenticate with an [AWS4 HMAC](https://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html) signature. Several methods of obtaining the necessary credentials are available; exactly one must be specified to use the AWS signature authentication method. +The AWS service for which to sign the request can be specified in the `service` field. If omitted, the default is `s3`. + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `services[_].credentials.s3_signing.service` | `string` | No | The AWS service to sign requests with, eg `execute-api` or `s3`. Default: `s3` | + ##### Using Static Environment Credentials If specifying `environment_credentials`, OPA will expect to find environment variables for `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_REGION`, in accordance with the diff --git a/plugins/rest/aws.go b/plugins/rest/aws.go index 52a73345cf..8af5f32c71 100644 --- a/plugins/rest/aws.go +++ b/plugins/rest/aws.go @@ -5,6 +5,7 @@ package rest import ( + "bytes" "crypto/hmac" "crypto/sha256" "encoding/json" @@ -47,6 +48,15 @@ const ( awsWebIdentityTokenFileEnvVar = "AWS_WEB_IDENTITY_TOKEN_FILE" ) +// Headers that may be mutated before reaching an aws service (eg by a proxy) should be added here to omit them from +// the sigv4 canonical request +// ref. https://github.com/aws/aws-sdk-go/blob/master/aws/signer/v4/v4.go#L92 +var awsSigv4IgnoredHeaders = map[string]bool{ + "authorization": true, + "user-agent": true, + "x-amzn-trace-id": true, +} + // awsCredentials represents the credentials obtained from an AWS credential provider type awsCredentials struct { AccessKey string @@ -384,7 +394,7 @@ func sha256MAC(message []byte, key []byte) []byte { return mac.Sum(nil) } -func sortKeys(strMap map[string]string) []string { +func sortKeys(strMap map[string][]string) []string { keys := make([]string, len(strMap)) i := 0 @@ -397,7 +407,11 @@ func sortKeys(strMap map[string]string) []string { } // signV4 modifies an http.Request to include an AWS V4 signature based on a credential provider -func signV4(req *http.Request, credService awsCredentialService, theTime time.Time) error { +func signV4(req *http.Request, service string, credService awsCredentialService, theTime time.Time) error { + // General ref. https://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html + // S3 ref. https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html + // APIGateway ref. https://docs.aws.amazon.com/apigateway/api-reference/signing-requests/ + var body []byte if req.Body == nil { body = []byte("") @@ -407,6 +421,9 @@ func signV4(req *http.Request, credService awsCredentialService, theTime time.Ti if err != nil { return errors.New("error getting request body: " + err.Error()) } + // Since ReadAll consumed the body ReadCloser, we must create a new ReadCloser for the request so that the + // subsequent read starts from the beginning + req.Body = ioutil.NopCloser(bytes.NewReader(body)) } creds, err := credService.credentials() if err != nil { @@ -422,9 +439,13 @@ func signV4(req *http.Request, credService awsCredentialService, theTime time.Ti iso8601Now := now.Format("20060102T150405Z") awsHeaders := map[string]string{ - "host": req.URL.Host, - "x-amz-content-sha256": bodyHexHash, - "x-amz-date": iso8601Now, + "host": req.URL.Host, + "x-amz-date": iso8601Now, + } + + // s3 and glacier require the extra x-amz-content-sha256 header. other services do not. + if service == "s3" || service == "glacier" { + awsHeaders["x-amz-content-sha256"] = bodyHexHash } // the security token header is necessary for ephemeral credentials, e.g. from @@ -433,7 +454,20 @@ func signV4(req *http.Request, credService awsCredentialService, theTime time.Ti awsHeaders["x-amz-security-token"] = creds.SessionToken } - // ref. https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html + headersToSign := map[string][]string{} + + // sign all of the aws headers + for k, v := range awsHeaders { + headersToSign[k] = []string{v} + } + + // sign all of the request's headers, except for those in the ignore list + for k, v := range req.Header { + lowerCaseHeader := strings.ToLower(k) + if !awsSigv4IgnoredHeaders[lowerCaseHeader] { + headersToSign[lowerCaseHeader] = v + } + } // the "canonical request" is the normalized version of the AWS service access // that we're attempting to perform; in this case, a GET from an S3 bucket @@ -442,9 +476,9 @@ func signV4(req *http.Request, credService awsCredentialService, theTime time.Ti canonicalReq += "\n" // query string; not implemented // include the values for the signed headers - orderedKeys := sortKeys(awsHeaders) + orderedKeys := sortKeys(headersToSign) for _, k := range orderedKeys { - canonicalReq += k + ":" + awsHeaders[k] + "\n" + canonicalReq += k + ":" + strings.Join(headersToSign[k], ",") + "\n" } canonicalReq += "\n" // linefeed to terminate headers @@ -455,17 +489,17 @@ func signV4(req *http.Request, credService awsCredentialService, theTime time.Ti // the "string to sign" is a time-bounded, scoped request token which // is linked to the "canonical request" by inclusion of its SHA-256 hash - strToSign := "AWS4-HMAC-SHA256\n" // V4 signing with SHA-256 HMAC - strToSign += iso8601Now + "\n" // ISO 8601 time - strToSign += dateNow + "/" + creds.RegionName + "/s3/aws4_request\n" // scoping for signature - strToSign += fmt.Sprintf("%x", sha256.Sum256([]byte(canonicalReq))) // SHA-256 of canonical request + strToSign := "AWS4-HMAC-SHA256\n" // V4 signing with SHA-256 HMAC + strToSign += iso8601Now + "\n" // ISO 8601 time + strToSign += dateNow + "/" + creds.RegionName + "/" + service + "/aws4_request\n" // scoping for signature + strToSign += fmt.Sprintf("%x", sha256.Sum256([]byte(canonicalReq))) // SHA-256 of canonical request // the "signing key" is generated by repeated HMAC-SHA256 based on the same // scoping that's included in the "string to sign"; but including the secret key // to allow AWS to validate it signingKey := sha256MAC([]byte(dateNow), []byte("AWS4"+creds.SecretKey)) signingKey = sha256MAC([]byte(creds.RegionName), signingKey) - signingKey = sha256MAC([]byte("s3"), signingKey) + signingKey = sha256MAC([]byte(service), signingKey) signingKey = sha256MAC([]byte("aws4_request"), signingKey) // the "signature" is finally the "string to sign" signed by the "signing key" @@ -474,15 +508,15 @@ func signV4(req *http.Request, credService awsCredentialService, theTime time.Ti // required format of Authorization header; n.b. the access key corresponding to // the secret key is included here authHdr := "AWS4-HMAC-SHA256 Credential=" + creds.AccessKey + "/" + dateNow - authHdr += "/" + creds.RegionName + "/s3/aws4_request," + authHdr += "/" + creds.RegionName + "/" + service + "/aws4_request," authHdr += "SignedHeaders=" + headerList + "," authHdr += "Signature=" + fmt.Sprintf("%x", signature) // add the computed Authorization - req.Header.Add("Authorization", authHdr) + req.Header.Set("Authorization", authHdr) // populate the other signed headers into the request - for _, k := range orderedKeys { + for k := range awsHeaders { req.Header.Add(k, awsHeaders[k]) } diff --git a/plugins/rest/aws_test.go b/plugins/rest/aws_test.go index c9c6a6c728..a169ee412a 100644 --- a/plugins/rest/aws_test.go +++ b/plugins/rest/aws_test.go @@ -271,7 +271,7 @@ func TestV4Signing(t *testing.T) { logger: sdk.NewStandardLogger(), } req, _ := http.NewRequest("GET", "https://mybucket.s3.amazonaws.com/bundle.tar.gz", strings.NewReader("")) - err := signV4(req, cs, time.Unix(1556129697, 0)) + err := signV4(req, "s3", cs, time.Unix(1556129697, 0)) assertErr("error getting AWS credentials: metadata HTTP request returned unexpected status: 404 Not Found", err, t) @@ -290,10 +290,10 @@ func TestV4Signing(t *testing.T) { Token: "MYAWSSECURITYTOKENGOESHERE", Expiration: time.Now().UTC().Add(time.Minute * 2)} req, _ = http.NewRequest("GET", "https://mybucket.s3.amazonaws.com/bundle.tar.gz", strings.NewReader("")) - err = signV4(req, cs, time.Unix(1556129697, 0)) + err = signV4(req, "s3", cs, time.Unix(1556129697, 0)) if err != nil { - t.Error("unexpected error during signing") + t.Fatal("unexpected error during signing") } // expect mandatory headers @@ -308,6 +308,89 @@ func TestV4Signing(t *testing.T) { assertEq(req.Header.Get("X-Amz-Security-Token"), "MYAWSSECURITYTOKENGOESHERE", t) } +func TestV4SigningForApiGateway(t *testing.T) { + ts := ec2CredTestServer{} + ts.start() + defer ts.stop() + + cs := &awsMetadataCredentialService{ + RoleName: "my_iam_role", // not present + RegionName: "us-east-1", + credServicePath: ts.server.URL + "/latest/meta-data/iam/security-credentials/", + tokenPath: ts.server.URL + "/latest/api/token", + logger: sdk.NewStandardLogger(), + } + ts.payload = metadataPayload{ + AccessKeyID: "MYAWSACCESSKEYGOESHERE", + SecretAccessKey: "MYAWSSECRETACCESSKEYGOESHERE", + Code: "Success", + Token: "MYAWSSECURITYTOKENGOESHERE", + Expiration: time.Now().UTC().Add(time.Minute * 2)} + req, _ := http.NewRequest("POST", "https://myrestapi.execute-api.us-east-1.amazonaws.com/prod/logs", + strings.NewReader("{ \"payload\": 42 }")) + req.Header.Set("Content-Type", "application/json") + + err := signV4(req, "execute-api", cs, time.Unix(1556129697, 0)) + + if err != nil { + t.Fatal("unexpected error during signing") + } + + // expect mandatory headers + assertEq(req.Header.Get("Host"), "myrestapi.execute-api.us-east-1.amazonaws.com", t) + assertEq(req.Header.Get("Authorization"), + "AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20190424/us-east-1/execute-api/aws4_request,"+ + "SignedHeaders=content-type;host;x-amz-date;x-amz-security-token,"+ + "Signature=c8ee72cc45050b255bcbf19defc693f7cd788959b5380fa0985de6e865635339", t) + // no content sha should be set, since this is specific to s3 and glacier + assertEq(req.Header.Get("X-Amz-Content-Sha256"), "", t) + assertEq(req.Header.Get("X-Amz-Date"), "20190424T181457Z", t) + assertEq(req.Header.Get("X-Amz-Security-Token"), "MYAWSSECURITYTOKENGOESHERE", t) +} + +func TestV4SigningOmitsIgnoredHeaders(t *testing.T) { + ts := ec2CredTestServer{} + ts.start() + defer ts.stop() + + cs := &awsMetadataCredentialService{ + RoleName: "my_iam_role", // not present + RegionName: "us-east-1", + credServicePath: ts.server.URL + "/latest/meta-data/iam/security-credentials/", + tokenPath: ts.server.URL + "/latest/api/token", + logger: sdk.NewStandardLogger(), + } + ts.payload = metadataPayload{ + AccessKeyID: "MYAWSACCESSKEYGOESHERE", + SecretAccessKey: "MYAWSSECRETACCESSKEYGOESHERE", + Code: "Success", + Token: "MYAWSSECURITYTOKENGOESHERE", + Expiration: time.Now().UTC().Add(time.Minute * 2)} + req, _ := http.NewRequest("POST", "https://myrestapi.execute-api.us-east-1.amazonaws.com/prod/logs", + strings.NewReader("{ \"payload\": 42 }")) + req.Header.Set("Content-Type", "application/json") + + // These are headers that should never be included in the signed headers + req.Header.Set("User-Agent", "Unit Tests!") + req.Header.Set("Authorization", "Auth header will be overwritten, and shouldn't be signed") + req.Header.Set("X-Amzn-Trace-Id", "Some trace id") + + err := signV4(req, "execute-api", cs, time.Unix(1556129697, 0)) + + if err != nil { + t.Fatal("unexpected error during signing") + } + + // Check the signed headers doesn't include user-agent, authorization or x-amz-trace-id + assertEq(req.Header.Get("Authorization"), + "AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20190424/us-east-1/execute-api/aws4_request,"+ + "SignedHeaders=content-type;host;x-amz-date;x-amz-security-token,"+ + "Signature=c8ee72cc45050b255bcbf19defc693f7cd788959b5380fa0985de6e865635339", t) + // The headers omitted from signing should still be present in the request + assertEq(req.Header.Get("User-Agent"), "Unit Tests!", t) + assertEq(req.Header.Get("X-Amzn-Trace-Id"), "Some trace id", t) +} + func TestV4SigningCustomPort(t *testing.T) { ts := ec2CredTestServer{} ts.start() @@ -327,10 +410,10 @@ func TestV4SigningCustomPort(t *testing.T) { Token: "MYAWSSECURITYTOKENGOESHERE", Expiration: time.Now().UTC().Add(time.Minute * 2)} req, _ := http.NewRequest("GET", "https://custom.s3.server:9000/bundle.tar.gz", strings.NewReader("")) - err := signV4(req, cs, time.Unix(1556129697, 0)) + err := signV4(req, "s3", cs, time.Unix(1556129697, 0)) if err != nil { - t.Error("unexpected error during signing") + t.Fatal("unexpected error during signing") } // expect mandatory headers @@ -345,6 +428,77 @@ func TestV4SigningCustomPort(t *testing.T) { assertEq(req.Header.Get("X-Amz-Security-Token"), "MYAWSSECURITYTOKENGOESHERE", t) } +func TestV4SigningDoesNotMutateBody(t *testing.T) { + ts := ec2CredTestServer{} + ts.start() + defer ts.stop() + + cs := &awsMetadataCredentialService{ + RoleName: "my_iam_role", // not present + RegionName: "us-east-1", + credServicePath: ts.server.URL + "/latest/meta-data/iam/security-credentials/", + tokenPath: ts.server.URL + "/latest/api/token", + logger: sdk.NewStandardLogger(), + } + ts.payload = metadataPayload{ + AccessKeyID: "MYAWSACCESSKEYGOESHERE", + SecretAccessKey: "MYAWSSECRETACCESSKEYGOESHERE", + Code: "Success", + Token: "MYAWSSECURITYTOKENGOESHERE", + Expiration: time.Now().UTC().Add(time.Minute * 2)} + req, _ := http.NewRequest("POST", "https://myrestapi.execute-api.us-east-1.amazonaws.com/prod/logs", + strings.NewReader("{ \"payload\": 42 }")) + + err := signV4(req, "execute-api", cs, time.Unix(1556129697, 0)) + + if err != nil { + t.Fatal("unexpected error during signing") + } + + // Read the body and check that it was not mutated + body, _ := ioutil.ReadAll(req.Body) + assertEq(string(body), "{ \"payload\": 42 }", t) +} + +func TestV4SigningWithMultiValueHeaders(t *testing.T) { + ts := ec2CredTestServer{} + ts.start() + defer ts.stop() + + cs := &awsMetadataCredentialService{ + RoleName: "my_iam_role", // not present + RegionName: "us-east-1", + credServicePath: ts.server.URL + "/latest/meta-data/iam/security-credentials/", + tokenPath: ts.server.URL + "/latest/api/token", + logger: sdk.NewStandardLogger(), + } + ts.payload = metadataPayload{ + AccessKeyID: "MYAWSACCESSKEYGOESHERE", + SecretAccessKey: "MYAWSSECRETACCESSKEYGOESHERE", + Code: "Success", + Token: "MYAWSSECURITYTOKENGOESHERE", + Expiration: time.Now().UTC().Add(time.Minute * 2)} + req, _ := http.NewRequest("POST", "https://myrestapi.execute-api.us-east-1.amazonaws.com/prod/logs", + strings.NewReader("{ \"payload\": 42 }")) + req.Header.Add("Accept", "text/plain") + req.Header.Add("Accept", "text/html") + + err := signV4(req, "execute-api", cs, time.Unix(1556129697, 0)) + + if err != nil { + t.Fatal("unexpected error during signing") + } + + // Check the signed headers includes our multi-value 'accept' header + assertEq(req.Header.Get("Authorization"), + "AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20190424/us-east-1/execute-api/aws4_request,"+ + "SignedHeaders=accept;host;x-amz-date;x-amz-security-token,"+ + "Signature=0237b0c789cad36212f0efba70c02549e1f659ab9caaca16423930cc7236c046", t) + // The multi-value headers are preserved + assertEq(req.Header.Values("Accept")[0], "text/plain", t) + assertEq(req.Header.Values("Accept")[1], "text/html", t) +} + // simulate EC2 metadata service type ec2CredTestServer struct { t *testing.T diff --git a/plugins/rest/rest_auth.go b/plugins/rest/rest_auth.go index 24c2fdecdd..f8c9f20770 100644 --- a/plugins/rest/rest_auth.go +++ b/plugins/rest/rest_auth.go @@ -28,6 +28,11 @@ import ( "github.com/open-policy-agent/opa/sdk" ) +const ( + // Default to s3 when the service for sigv4 signing is not specified for backwards compatibility + awsSigv4SigningDefaultService = "s3" +) + // DefaultTLSConfig defines standard TLS configurations based on the Config func DefaultTLSConfig(c Config) (*tls.Config, error) { t := &tls.Config{} @@ -417,6 +422,7 @@ type awsSigningAuthPlugin struct { AWSEnvironmentCredentials *awsEnvironmentCredentialService `json:"environment_credentials,omitempty"` AWSMetadataCredentials *awsMetadataCredentialService `json:"metadata_credentials,omitempty"` AWSWebIdentityCredentials *awsWebIdentityCredentialService `json:"web_identity_credentials,omitempty"` + AWSService string `json:"service,omitempty"` logger sdk.Logger } @@ -459,11 +465,16 @@ func (ap *awsSigningAuthPlugin) NewClient(c Config) (*http.Client, error) { return nil, err } } + + if ap.AWSService == "" { + ap.AWSService = awsSigv4SigningDefaultService + } + return DefaultRoundTripperClient(t, *c.ResponseHeaderTimeoutSeconds), nil } func (ap *awsSigningAuthPlugin) Prepare(req *http.Request) error { ap.logger.Debug("Signing request with AWS credentials.") - err := signV4(req, ap.awsCredentialService(), time.Now()) + err := signV4(req, ap.AWSService, ap.awsCredentialService(), time.Now()) return err } diff --git a/plugins/rest/rest_test.go b/plugins/rest/rest_test.go index cc4ae289c6..5669452a42 100644 --- a/plugins/rest/rest_test.go +++ b/plugins/rest/rest_test.go @@ -135,6 +135,19 @@ func TestNew(t *testing.T) { } }`, }, + { + name: "ValidApiGatewayEnvCreds", + input: `{ + "name": "foo", + "url": "http://localhost", + "credentials": { + "s3_signing": { + "service": "execute-api", + "environment_credentials": {} + } + } + }`, + }, { name: "ValidS3MetadataCredsWithRole", input: `{