Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(s3-deployment): User metadata keys have redundant triple x-amz prefix #12414

Merged
merged 4 commits into from
Jan 21, 2021
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1219,7 +1219,7 @@
"Properties": {
"Code": {
"S3Bucket": {
"Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3BucketFD1BBE00"
"Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C"
},
"S3Key": {
"Fn::Join": [
Expand All @@ -1232,7 +1232,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76"
"Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70"
}
]
}
Expand All @@ -1245,7 +1245,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76"
"Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70"
}
]
}
Expand Down Expand Up @@ -1348,17 +1348,17 @@
"Type": "String",
"Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\""
},
"AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3BucketFD1BBE00": {
"AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": {
"Type": "String",
"Description": "S3 bucket for asset \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\""
"Description": "S3 bucket for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\""
},
"AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76": {
"AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": {
"Type": "String",
"Description": "S3 key for asset version \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\""
"Description": "S3 key for asset version \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\""
},
"AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aArtifactHash595EC1E7": {
"AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": {
"Type": "String",
"Description": "Artifact hash for asset \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\""
"Description": "Artifact hash for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\""
},
"AssetParameters972240f9dd6e036a93d5f081af9a24315b2053828ac049b3b19b2fa12d7ae64aS3Bucket1F1A8472": {
"Type": "String",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-s3-deployment/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ new BucketDeployment(this, 'HTMLBucketDeployment', {
You can specify metadata to be set on all the objects in your deployment.
There are 2 types of metadata in S3: system-defined metadata and user-defined metadata.
System-defined metadata have a special purpose, for example cache-control defines how long to keep an object cached.
User-defined metadata are not used by S3 and keys always begin with `x-amzn-meta-` (if this is not provided, it is added automatically).
User-defined metadata are not used by S3 and keys always begin with `x-amz-meta-` (this prefix is added automatically).

System defined metadata keys include the following:

Expand Down
7 changes: 2 additions & 5 deletions packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -257,10 +257,7 @@ export class BucketDeployment extends CoreConstruct {
*/

function mapUserMetadata(metadata: UserDefinedObjectMetadata) {
const mapKey = (key: string) =>
key.toLowerCase().startsWith('x-amzn-meta-')
? key.toLowerCase()
: `x-amzn-meta-${key.toLowerCase()}`;
const mapKey = (key: string) => key.toLowerCase();

return Object.keys(metadata).reduce((o, key) => ({ ...o, [mapKey(key)]: metadata[key] }), {});
}
Expand Down Expand Up @@ -358,7 +355,7 @@ export class Expires {
export interface UserDefinedObjectMetadata {
/**
* Arbitrary metadata key-values
* Keys must begin with `x-amzn-meta-` (will be added automatically if not provided)
* The `x-amz-meta-` prefix will automatically be added to keys.
* @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#UserMetadata
*/
readonly [key: string]: string;
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ def create_metadata_args(raw_user_metadata, raw_system_metadata):
return []

format_system_metadata_key = lambda k: k.lower()
format_user_metadata_key = lambda k: k.lower() if k.lower().startswith("x-amz-meta-") else f"x-amz-meta-{k.lower()}"
format_user_metadata_key = lambda k: k.lower()

system_metadata = { format_system_metadata_key(k): v for k, v in raw_system_metadata.items() }
user_metadata = { format_user_metadata_key(k): v for k, v in raw_user_metadata.items() }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ test('user metadata is correctly transformed', () => {

// THEN
expect(stack).toHaveResource('Custom::CDKBucketDeployment', {
UserMetadata: { 'x-amzn-meta-a': '1', 'x-amzn-meta-b': '2' },
UserMetadata: { a: '1', b: '2' },
});
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@
"Properties": {
"Code": {
"S3Bucket": {
"Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3BucketFD1BBE00"
"Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C"
},
"S3Key": {
"Fn::Join": [
Expand All @@ -308,7 +308,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76"
"Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70"
}
]
}
Expand All @@ -321,7 +321,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76"
"Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70"
}
]
}
Expand Down Expand Up @@ -365,17 +365,17 @@
"Type": "String",
"Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\""
},
"AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3BucketFD1BBE00": {
"AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": {
"Type": "String",
"Description": "S3 bucket for asset \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\""
"Description": "S3 bucket for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\""
},
"AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76": {
"AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": {
"Type": "String",
"Description": "S3 key for asset version \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\""
"Description": "S3 key for asset version \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\""
},
"AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aArtifactHash595EC1E7": {
"AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": {
"Type": "String",
"Description": "Artifact hash for asset \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\""
"Description": "Artifact hash for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\""
},
"AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3Bucket9CD8B20A": {
"Type": "String",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,7 @@
"Properties": {
"Code": {
"S3Bucket": {
"Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3BucketFD1BBE00"
"Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C"
},
"S3Key": {
"Fn::Join": [
Expand All @@ -317,7 +317,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76"
"Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70"
}
]
}
Expand All @@ -330,7 +330,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76"
"Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70"
}
]
}
Expand Down Expand Up @@ -572,9 +572,9 @@
"RetainOnDelete": false,
"Prune": true,
"UserMetadata": {
"x-amzn-meta-a": "aaa",
"x-amzn-meta-b": "bbb",
"x-amzn-meta-c": "ccc"
"a": "aaa",
"b": "bbb",
"c": "ccc"
},
"SystemMetadata": {
"cache-control": "public, max-age=60",
Expand Down Expand Up @@ -700,17 +700,17 @@
"Type": "String",
"Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\""
},
"AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3BucketFD1BBE00": {
"AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": {
"Type": "String",
"Description": "S3 bucket for asset \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\""
"Description": "S3 bucket for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\""
},
"AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aS3VersionKey6E54DC76": {
"AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": {
"Type": "String",
"Description": "S3 key for asset version \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\""
"Description": "S3 key for asset version \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\""
},
"AssetParameters8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59aArtifactHash595EC1E7": {
"AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": {
"Type": "String",
"Description": "Artifact hash for asset \"8bda025b845a88fbeb54ef75e52048aa9f3378463116cb413f12f6014673a59a\""
"Description": "Artifact hash for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\""
},
"AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3Bucket9CD8B20A": {
"Type": "String",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ def test_create_update_with_metadata(self):

self.assertAwsCommands(
["s3", "cp", "s3://<source-bucket>/<source-object-key>", "archive.zip"],
["s3", "sync", "--delete", "contents.zip", "s3://<dest-bucket-name>/<dest-key-prefix>", "--content-type", "text/html", "--content-language", "en", "--metadata", "{\"x-amz-meta-best\":\"game\"}", "--metadata-directive", "REPLACE"]
["s3", "sync", "--delete", "contents.zip", "s3://<dest-bucket-name>/<dest-key-prefix>", "--content-type", "text/html", "--content-language", "en", "--metadata", "{\"best\":\"game\"}", "--metadata-directive", "REPLACE"]
)

def test_delete_no_retain(self):
Expand Down