From 2d15c8d2e1a2db197fe5ecd97b97a8acdb31bbce Mon Sep 17 00:00:00 2001 From: Bradley Walters Date: Thu, 7 Jan 2021 16:06:33 -0700 Subject: [PATCH] fix(s3-deployment): Fix duplicate x-amz-meta- prefix for user metadata Without this fix, keys are prefixed with x-amzn-meta- by the TypeScript code, then the Python code runs and prefixes metadata keys again with x-amz-meta-. Then, the Python shells out to aws-cli which makes a request to the S3 service. There, the keys are prefixed *yet again* with x-amz-meta- _unconditionally_. Thus no matter what keys the user specified, the keys in S3 would be: x-amz-meta-x-amz-meta-x-amzn-meta-. This issue was originally reported as #8459. After this change, neither the TypeScript code nor the Python code will attempt to prepend to the metadata key. Instead we rely on the S3 service to do it. --- .../ec2/integ.environment-file.expected.json | 18 +++++++------- packages/@aws-cdk/aws-s3-deployment/README.md | 2 +- .../lib/bucket-deployment.ts | 7 ++---- .../aws-s3-deployment/lib/lambda/index.py | 2 +- .../test/bucket-deployment.test.ts | 2 +- ...bucket-deployment-cloudfront.expected.json | 18 +++++++------- .../integ.bucket-deployment.expected.json | 24 +++++++++---------- .../aws-s3-deployment/test/lambda/test.py | 2 +- 8 files changed, 36 insertions(+), 39 deletions(-) diff --git a/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json b/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json index 34017cbb94b10..9376c507c20cf 100644 --- a/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json +++ b/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json @@ -1219,7 +1219,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3BucketFD1BBE00" + "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C" }, "S3Key": { "Fn::Join": [ @@ -1232,7 +1232,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76" + "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" } ] } @@ -1245,7 +1245,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76" + "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" } ] } @@ -1348,17 +1348,17 @@ "Type": "String", "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3BucketFD1BBE00": { + "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": { "Type": "String", - "Description": "S3 bucket for asset \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\"" + "Description": "S3 bucket for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" }, - "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76": { + "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": { "Type": "String", - "Description": "S3 key for asset version \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\"" + "Description": "S3 key for asset version \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" }, - "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aArtifactHash595EC1E7": { + "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": { "Type": "String", - "Description": "Artifact hash for asset \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\"" + "Description": "Artifact hash for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" }, "AssetParameters972240f9dd6e036a93d5f081af9a24315b2053828ac049b3b19b2fa12d7ae64aS3Bucket1F1A8472": { "Type": "String", diff --git a/packages/@aws-cdk/aws-s3-deployment/README.md b/packages/@aws-cdk/aws-s3-deployment/README.md index 2509372e55c11..c3ff0ae61e724 100644 --- a/packages/@aws-cdk/aws-s3-deployment/README.md +++ b/packages/@aws-cdk/aws-s3-deployment/README.md @@ -109,7 +109,7 @@ new BucketDeployment(this, 'HTMLBucketDeployment', { You can specify metadata to be set on all the objects in your deployment. There are 2 types of metadata in S3: system-defined metadata and user-defined metadata. System-defined metadata have a special purpose, for example cache-control defines how long to keep an object cached. -User-defined metadata are not used by S3 and keys always begin with `x-amzn-meta-` (if this is not provided, it is added automatically). +User-defined metadata are not used by S3 and keys always begin with `x-amz-meta-` (this prefix is added automatically). System defined metadata keys include the following: diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts index 9e1c20ae02ef7..bbf526b79df86 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts +++ b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts @@ -257,10 +257,7 @@ export class BucketDeployment extends CoreConstruct { */ function mapUserMetadata(metadata: UserDefinedObjectMetadata) { - const mapKey = (key: string) => - key.toLowerCase().startsWith('x-amzn-meta-') - ? key.toLowerCase() - : `x-amzn-meta-${key.toLowerCase()}`; + const mapKey = (key: string) => key.toLowerCase(); return Object.keys(metadata).reduce((o, key) => ({ ...o, [mapKey(key)]: metadata[key] }), {}); } @@ -358,7 +355,7 @@ export class Expires { export interface UserDefinedObjectMetadata { /** * Arbitrary metadata key-values - * Keys must begin with `x-amzn-meta-` (will be added automatically if not provided) + * The `x-amz-meta-` prefix will automatically be added to keys. * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#UserMetadata */ readonly [key: string]: string; diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py b/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py index bf16d84608517..3935e3122529d 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py +++ b/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py @@ -169,7 +169,7 @@ def create_metadata_args(raw_user_metadata, raw_system_metadata): return [] format_system_metadata_key = lambda k: k.lower() - format_user_metadata_key = lambda k: k.lower() if k.lower().startswith("x-amz-meta-") else f"x-amz-meta-{k.lower()}" + format_user_metadata_key = lambda k: k.lower() system_metadata = { format_system_metadata_key(k): v for k, v in raw_system_metadata.items() } user_metadata = { format_user_metadata_key(k): v for k, v in raw_user_metadata.items() } diff --git a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts index a1e43dfd10eb3..5cb7cdc1e7cdf 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts +++ b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts @@ -297,7 +297,7 @@ test('user metadata is correctly transformed', () => { // THEN expect(stack).toHaveResource('Custom::CDKBucketDeployment', { - UserMetadata: { 'x-amzn-meta-a': '1', 'x-amzn-meta-b': '2' }, + UserMetadata: { a: '1', b: '2' }, }); }); diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json index 3e138f405e0d6..fa5e26b9d57e9 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json @@ -295,7 +295,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3BucketFD1BBE00" + "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C" }, "S3Key": { "Fn::Join": [ @@ -308,7 +308,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76" + "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" } ] } @@ -321,7 +321,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76" + "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" } ] } @@ -365,17 +365,17 @@ "Type": "String", "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3BucketFD1BBE00": { + "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": { "Type": "String", - "Description": "S3 bucket for asset \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\"" + "Description": "S3 bucket for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" }, - "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76": { + "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": { "Type": "String", - "Description": "S3 key for asset version \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\"" + "Description": "S3 key for asset version \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" }, - "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aArtifactHash595EC1E7": { + "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": { "Type": "String", - "Description": "Artifact hash for asset \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\"" + "Description": "Artifact hash for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" }, "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3Bucket9CD8B20A": { "Type": "String", diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json index 9d52b89269f5a..92d6c5bb8514b 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json @@ -304,7 +304,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3BucketFD1BBE00" + "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C" }, "S3Key": { "Fn::Join": [ @@ -317,7 +317,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76" + "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" } ] } @@ -330,7 +330,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76" + "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" } ] } @@ -572,9 +572,9 @@ "RetainOnDelete": false, "Prune": true, "UserMetadata": { - "x-amzn-meta-a": "aaa", - "x-amzn-meta-b": "bbb", - "x-amzn-meta-c": "ccc" + "a": "aaa", + "b": "bbb", + "c": "ccc" }, "SystemMetadata": { "cache-control": "public, max-age=60", @@ -700,17 +700,17 @@ "Type": "String", "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3BucketFD1BBE00": { + "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": { "Type": "String", - "Description": "S3 bucket for asset \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\"" + "Description": "S3 bucket for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" }, - "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76": { + "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": { "Type": "String", - "Description": "S3 key for asset version \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\"" + "Description": "S3 key for asset version \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" }, - "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aArtifactHash595EC1E7": { + "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": { "Type": "String", - "Description": "Artifact hash for asset \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\"" + "Description": "Artifact hash for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" }, "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3Bucket9CD8B20A": { "Type": "String", diff --git a/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py b/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py index fcd79f18af4d5..7813a13db3859 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py +++ b/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py @@ -122,7 +122,7 @@ def test_create_update_with_metadata(self): self.assertAwsCommands( ["s3", "cp", "s3:///", "archive.zip"], - ["s3", "sync", "--delete", "contents.zip", "s3:///", "--content-type", "text/html", "--content-language", "en", "--metadata", "{\"x-amz-meta-best\":\"game\"}", "--metadata-directive", "REPLACE"] + ["s3", "sync", "--delete", "contents.zip", "s3:///", "--content-type", "text/html", "--content-language", "en", "--metadata", "{\"best\":\"game\"}", "--metadata-directive", "REPLACE"] ) def test_delete_no_retain(self):