diff --git a/CHANGELOG.md b/CHANGELOG.md index b2130fb419..6f2e522876 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ * Initialize empty schedule metrics on server init (#1054, @cbeneke) * Update CHANGELOGs (#1063, @wwitzel3) * Remove default token from all service accounts (#1048, @ncdc) - + * Allow to use AWS Signature v1 for creating signed AWS urls (#811, @bashofmann) + ## Current release: * [CHANGELOG-0.10.md][8] diff --git a/docs/api-types/backupstoragelocation.md b/docs/api-types/backupstoragelocation.md index 5729c9611e..fcb17a84d7 100644 --- a/docs/api-types/backupstoragelocation.md +++ b/docs/api-types/backupstoragelocation.md @@ -51,6 +51,7 @@ The configurable parameters are as follows: | `s3Url` | string | Required field for non-AWS-hosted storage| *Example*: http://minio:9000

You can specify the AWS S3 URL here for explicitness, but Ark can already generate it from `region`, and `bucket`. This field is primarily for local storage services like Minio.| | `publicUrl` | string | Empty | *Example*: https://minio.mycluster.com

If specified, use this instead of `s3Url` when generating download URLs (e.g., for logs). This field is primarily for local storage services like Minio.| | `kmsKeyId` | string | Empty | *Example*: "502b409c-4da1-419f-a16e-eif453b3i49f" or "alias/``"

Specify an [AWS KMS key][10] id or alias to enable encryption of the backups stored in S3. Only works with AWS S3 and may require explicitly granting key usage rights.| +| `signatureVersion` | string | `"4"` | Version of the signature algorithm used to create signed URLs that are used by ark cli to download backups or fetch logs. Possible versions are "1" and "4". Usually the default version 4 is correct, but some S3-compatible providers like Quobyte only support version 1.| #### Azure diff --git a/docs/support-matrix.md b/docs/support-matrix.md index 6a39a2759b..05e8e1cc25 100644 --- a/docs/support-matrix.md +++ b/docs/support-matrix.md @@ -20,6 +20,9 @@ _Note that these providers are not regularly tested by the Ark team._ * [Minio][9] * Ceph RADOS v12.2.7 * [DigitalOcean][7] + * Quobyte + +_Some storage providers, like Quobyte, may need a different [signature algorithm version][15]._ ## Volume Snapshot Providers @@ -52,3 +55,4 @@ After you publish your plugin, open a PR that adds your plugin to the appropriat [12]: https://github.com/aws/aws-sdk-go/aws [13]: https://portworx.slack.com/messages/px-k8s [14]: https://github.com/portworx/ark-plugin/issues +[15]: api-types/backupstoragelocation.md#aws diff --git a/pkg/cloudprovider/aws/object_store.go b/pkg/cloudprovider/aws/object_store.go index b0b934286c..a657e80f4f 100644 --- a/pkg/cloudprovider/aws/object_store.go +++ b/pkg/cloudprovider/aws/object_store.go @@ -24,6 +24,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/endpoints" + "github.com/aws/aws-sdk-go/aws/signer/v4" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/pkg/errors" @@ -38,20 +39,30 @@ const ( kmsKeyIDKey = "kmsKeyId" s3ForcePathStyleKey = "s3ForcePathStyle" bucketKey = "bucket" + signatureVersionKey = "signatureVersion" ) type objectStore struct { - log logrus.FieldLogger - s3 *s3.S3 - preSignS3 *s3.S3 - s3Uploader *s3manager.Uploader - kmsKeyID string + log logrus.FieldLogger + s3 *s3.S3 + preSignS3 *s3.S3 + s3Uploader *s3manager.Uploader + kmsKeyID string + signatureVersion string } func NewObjectStore(logger logrus.FieldLogger) cloudprovider.ObjectStore { return &objectStore{log: logger} } +func isValidSignatureVersion(signatureVersion string) bool { + switch signatureVersion { + case "1", "4": + return true + } + return false +} + func (o *objectStore) Init(config map[string]string) error { var ( region = config[regionKey] @@ -59,6 +70,7 @@ func (o *objectStore) Init(config map[string]string) error { publicURL = config[publicURLKey] kmsKeyID = config[kmsKeyIDKey] s3ForcePathStyleVal = config[s3ForcePathStyleKey] + signatureVersion = config[signatureVersionKey] // note that bucket is automatically added to the config map // by the server from the ObjectStorageProviderConfig so @@ -100,6 +112,13 @@ func (o *objectStore) Init(config map[string]string) error { o.s3Uploader = s3manager.NewUploader(serverSession) o.kmsKeyID = kmsKeyID + if signatureVersion != "" { + if !isValidSignatureVersion(signatureVersion) { + return errors.Errorf("invalid signature version: %s", signatureVersion) + } + o.signatureVersion = signatureVersion + } + if publicURL != "" { publicConfig, err := newAWSConfig(publicURL, region, s3ForcePathStyle) if err != nil { @@ -239,5 +258,10 @@ func (o *objectStore) CreateSignedURL(bucket, key string, ttl time.Duration) (st Key: aws.String(key), }) + if o.signatureVersion == "1" { + req.Handlers.Sign.Remove(v4.SignRequestHandler) + req.Handlers.Sign.PushBackNamed(v1SignRequestHandler) + } + return req.Presign(ttl) } diff --git a/pkg/cloudprovider/aws/object_store_test.go b/pkg/cloudprovider/aws/object_store_test.go new file mode 100644 index 0000000000..2bb70938d7 --- /dev/null +++ b/pkg/cloudprovider/aws/object_store_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsValidSignatureVersion(t *testing.T) { + assert.True(t, isValidSignatureVersion("1")) + assert.True(t, isValidSignatureVersion("4")) + assert.False(t, isValidSignatureVersion("3")) +} diff --git a/pkg/cloudprovider/aws/v1_sign_request_handler.go b/pkg/cloudprovider/aws/v1_sign_request_handler.go new file mode 100644 index 0000000000..7442914f86 --- /dev/null +++ b/pkg/cloudprovider/aws/v1_sign_request_handler.go @@ -0,0 +1,147 @@ +/* +Copyright 2018 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "fmt" + "net/url" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/pkg/errors" +) + +var ( + errInvalidMethod = errors.New("v1 signer only handles HTTP GET") +) + +type signer struct { + // Values that must be populated from the request + request *request.Request + time time.Time + credentials *credentials.Credentials + debug aws.LogLevelType + logger aws.Logger + + query url.Values + stringToSign string + signature string +} + +// SignRequestHandler is a named request handler the SDK will use to sign +// service client request with using the V4 signature. +var v1SignRequestHandler = request.NamedHandler{ + Name: "v1.SignRequestHandler", Fn: signSDKRequest, +} + +func signSDKRequest(req *request.Request) { + // If the request does not need to be signed ignore the signing of the + // request if the AnonymousCredentials object is used. + if req.Config.Credentials == credentials.AnonymousCredentials { + return + } + + if req.HTTPRequest.Method != "GET" { + // The V1 signer only supports GET + req.Error = errInvalidMethod + return + } + + v1 := signer{ + request: req, + time: req.Time, + credentials: req.Config.Credentials, + debug: req.Config.LogLevel.Value(), + logger: req.Config.Logger, + } + + req.Error = v1.sign() + + if req.Error != nil { + return + } + + req.HTTPRequest.URL.RawQuery = v1.query.Encode() +} + +func (v1 *signer) sign() error { + credentialsValue, err := v1.credentials.Get() + if err != nil { + return errors.Wrap(err, "error getting credentials") + } + + httpRequest := v1.request.HTTPRequest + + v1.query = httpRequest.URL.Query() + + // Set new query parameters + v1.query.Set("AWSAccessKeyId", credentialsValue.AccessKeyID) + if credentialsValue.SessionToken != "" { + v1.query.Set("SecurityToken", credentialsValue.SessionToken) + } + + // in case this is a retry, ensure no signature present + v1.query.Del("Signature") + + method := httpRequest.Method + path := httpRequest.URL.Path + if path == "" { + path = "/" + } + + duration := int64(v1.request.ExpireTime / time.Second) + expires := strconv.FormatInt(duration, 10) + // build the canonical string for the v1 signature + v1.stringToSign = strings.Join([]string{ + method, + "", + "", + expires, + path, + }, "\n") + + hash := hmac.New(sha1.New, []byte(credentialsValue.SecretAccessKey)) + hash.Write([]byte(v1.stringToSign)) + v1.signature = base64.StdEncoding.EncodeToString(hash.Sum(nil)) + v1.query.Set("Signature", v1.signature) + v1.query.Set("Expires", expires) + + if v1.debug.Matches(aws.LogDebugWithSigning) { + v1.logSigningInfo() + } + + return nil +} + +const logSignInfoMsg = `DEBUG: Request Signature: +---[ STRING TO SIGN ]-------------------------------- +%s +---[ SIGNATURE ]------------------------------------- +%s +-----------------------------------------------------` + +func (v1 *signer) logSigningInfo() { + msg := fmt.Sprintf(logSignInfoMsg, v1.stringToSign, v1.query.Get("Signature")) + v1.logger.Log(msg) +}