From d42e89e01034dcba08c8f8ac0390a743143c4531 Mon Sep 17 00:00:00 2001 From: David Tucker Date: Thu, 19 Aug 2021 11:06:41 -0400 Subject: [PATCH] feat(s3-deployment): exclude and include filters (#16054) This construct only enables integration with the two existing CLI options (`--exclude` and `--include`) that are supported for the `s3 sync` command. There are a few situations where this can prove valuable: 1. Situations where you want to deploy a subset of files from an archive - This can be handled by leveraging the bundling option for a source, although in some situations the `exclude` filter would be significantly easier. 2. Situations where you want to leverage `prune` but have specific files excluded - This is the situation that cannot be solved with current tools. The most common scenario (and one I detailed in #14362 ) is where you manage a web app config file with a custom CloudFormation resource (to pass in API endpoint, user pool, etc...) and then manage a web application using this construct. Closes #14362 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../ec2/integ.environment-file.expected.json | 32 ++--- packages/@aws-cdk/aws-s3-deployment/README.md | 23 ++++ .../lib/bucket-deployment.ts | 25 ++++ .../aws-s3-deployment/lib/lambda/index.py | 14 +- .../test/bucket-deployment.test.ts | 76 +++++++++++ ...bucket-deployment-cloudfront.expected.json | 20 +-- .../integ.bucket-deployment.expected.json | 125 ++++++++++++++++-- .../test/integ.bucket-deployment.ts | 7 + .../aws-s3-deployment/test/lambda/aws | 9 +- .../aws-s3-deployment/test/lambda/test.py | 116 ++++++++++++++++ 10 files changed, 408 insertions(+), 39 deletions(-) diff --git a/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json b/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json index c309c8a70f249..dc3aad690d805 100644 --- a/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json +++ b/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json @@ -108,15 +108,15 @@ "VpcPublicSubnet1NATGateway4D7517AA": { "Type": "AWS::EC2::NatGateway", "Properties": { + "SubnetId": { + "Ref": "VpcPublicSubnet1Subnet5C2D37C4" + }, "AllocationId": { "Fn::GetAtt": [ "VpcPublicSubnet1EIPD7E02669", "AllocationId" ] }, - "SubnetId": { - "Ref": "VpcPublicSubnet1Subnet5C2D37C4" - }, "Tags": [ { "Key": "Name", @@ -205,15 +205,15 @@ "VpcPublicSubnet2NATGateway9182C01D": { "Type": "AWS::EC2::NatGateway", "Properties": { + "SubnetId": { + "Ref": "VpcPublicSubnet2Subnet691E08A3" + }, "AllocationId": { "Fn::GetAtt": [ "VpcPublicSubnet2EIP3C605A87", "AllocationId" ] }, - "SubnetId": { - "Ref": "VpcPublicSubnet2Subnet691E08A3" - }, "Tags": [ { "Key": "Name", @@ -1219,7 +1219,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3BucketD8D20B9A" }, "S3Key": { "Fn::Join": [ @@ -1232,7 +1232,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3" } ] } @@ -1245,7 +1245,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3" } ] } @@ -1348,17 +1348,17 @@ "Type": "String", "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3BucketD8D20B9A": { "Type": "String", - "Description": "S3 bucket for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "S3 bucket for asset \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3": { "Type": "String", - "Description": "S3 key for asset version \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "S3 key for asset version \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaArtifactHash3943F7F3": { "Type": "String", - "Description": "Artifact hash for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "Artifact hash for asset \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, "AssetParameters972240f9dd6e036a93d5f081af9a24315b2053828ac049b3b19b2fa12d7ae64aS3Bucket1F1A8472": { "Type": "String", @@ -1385,4 +1385,4 @@ "Description": "Artifact hash for asset \"872561bf078edd1685d50c9ff821cdd60d2b2ddfb0013c4087e79bf2bb50724d\"" } } -} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/README.md b/packages/@aws-cdk/aws-s3-deployment/README.md index a896d55cfc575..74b31d98c41fd 100644 --- a/packages/@aws-cdk/aws-s3-deployment/README.md +++ b/packages/@aws-cdk/aws-s3-deployment/README.md @@ -138,6 +138,29 @@ new BucketDeployment(this, 'HTMLBucketDeployment', { }); ``` +## Exclude and Include Filters + +There are two points at which filters are evaluated in a deployment: asset bundling and the actual deployment. If you simply want to exclude files in the asset bundling process, you should leverage the `exclude` property of `AssetOptions` when defining your source: + +```ts +new BucketDeployment(this, 'HTMLBucketDeployment', { + sources: [Source.asset('./website', { exclude: ['*', '!index.html'] })], + destinationBucket: bucket, +}); +``` + +If you want to specify filters to be used in the deployment process, you can use the `exclude` and `include` filters on `BucketDeployment`. If excluded, these files will not be deployed to the destination bucket. In addition, if the file already exists in the destination bucket, it will not be deleted if you are using the `prune` option: + +```ts +new s3deploy.BucketDeployment(this, 'DeployButExcludeSpecificFiles', { + sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], + destinationBucket, + exclude: ['*.txt'] +}); +``` + +These filters follow the same format that is used for the AWS CLI. See the CLI documentation for information on [Using Include and Exclude Filters](https://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters). + ## Objects metadata You can specify metadata to be set on all the objects in your deployment. diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts index 990a1ec359d0b..4281a82b74140 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts +++ b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts @@ -35,6 +35,29 @@ export interface BucketDeploymentProps { */ readonly destinationKeyPrefix?: string; + /** + * If this is set, matching files or objects will be excluded from the deployment's sync + * command. This can be used to exclude a file from being pruned in the destination bucket. + * + * If you want to just exclude files from the deployment package (which excludes these files + * evaluated when invalidating the asset), you should leverage the `exclude` property of + * `AssetOptions` when defining your source. + * + * @default - No exclude filters are used + * @see https://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters + */ + readonly exclude?: string[] + + /** + * If this is set, matching files or objects will be included with the deployment's sync + * command. Since all files from the deployment package are included by default, this property + * is usually leveraged alongside an `exclude` filter. + * + * @default - No include filters are used and all files are included with the sync command + * @see https://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters + */ + readonly include?: string[] + /** * If this is set to false, files in the destination bucket that * do not exist in the asset, will NOT be deleted during deployment (create/update). @@ -245,6 +268,8 @@ export class BucketDeployment extends CoreConstruct { DestinationBucketKeyPrefix: props.destinationKeyPrefix, RetainOnDelete: props.retainOnDelete, Prune: props.prune ?? true, + Exclude: props.exclude, + Include: props.include, UserMetadata: props.metadata ? mapUserMetadata(props.metadata) : undefined, SystemMetadata: mapSystemMetadata(props), DistributionId: props.distribution?.distributionId, diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py b/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py index 3935e3122529d..ed7e1d9c5d7e9 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py +++ b/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py @@ -49,6 +49,8 @@ def cfn_error(message=None): user_metadata = props.get('UserMetadata', {}) system_metadata = props.get('SystemMetadata', {}) prune = props.get('Prune', 'true').lower() == 'true' + exclude = props.get('Exclude', []) + include = props.get('Include', []) default_distribution_path = dest_bucket_prefix if not default_distribution_path.endswith("/"): @@ -100,7 +102,7 @@ def cfn_error(message=None): aws_command("s3", "rm", old_s3_dest, "--recursive") if request_type == "Update" or request_type == "Create": - s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune) + s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include) if distribution_id: cloudfront_invalidate(distribution_id, distribution_paths) @@ -114,7 +116,7 @@ def cfn_error(message=None): #--------------------------------------------------------------------------------------------------- # populate all files from s3_source_zips to a destination bucket -def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune): +def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include): # create a temporary working directory workdir=tempfile.mkdtemp() logger.info("| workdir: %s" % workdir) @@ -139,6 +141,14 @@ def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune): if prune: s3_command.append("--delete") + if exclude: + for filter in exclude: + s3_command.extend(["--exclude", filter]) + + if include: + for filter in include: + s3_command.extend(["--include", filter]) + s3_command.extend([contents_dir, s3_dest]) s3_command.extend(create_metadata_args(user_metadata, system_metadata)) aws_command(*s3_command) diff --git a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts index 19af404bbfa7b..cd40925261668 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts +++ b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts @@ -689,6 +689,82 @@ test('deploy without deleting missing files from destination', () => { }); }); +test('deploy with excluded files from destination', () => { + + // GIVEN + const stack = new cdk.Stack(); + const bucket = new s3.Bucket(stack, 'Dest'); + + // WHEN + new s3deploy.BucketDeployment(stack, 'Deploy', { + sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], + destinationBucket: bucket, + exclude: ['sample.js'], + }); + + expect(stack).toHaveResourceLike('Custom::CDKBucketDeployment', { + Exclude: ['sample.js'], + }); +}); + +test('deploy with included files from destination', () => { + + // GIVEN + const stack = new cdk.Stack(); + const bucket = new s3.Bucket(stack, 'Dest'); + + // WHEN + new s3deploy.BucketDeployment(stack, 'Deploy', { + sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], + destinationBucket: bucket, + include: ['sample.js'], + }); + + expect(stack).toHaveResourceLike('Custom::CDKBucketDeployment', { + Include: ['sample.js'], + }); +}); + +test('deploy with excluded and included files from destination', () => { + + // GIVEN + const stack = new cdk.Stack(); + const bucket = new s3.Bucket(stack, 'Dest'); + + // WHEN + new s3deploy.BucketDeployment(stack, 'Deploy', { + sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], + destinationBucket: bucket, + exclude: ['sample/*'], + include: ['sample/include.json'], + }); + + expect(stack).toHaveResourceLike('Custom::CDKBucketDeployment', { + Exclude: ['sample/*'], + Include: ['sample/include.json'], + }); +}); + +test('deploy with multiple exclude and include filters', () => { + + // GIVEN + const stack = new cdk.Stack(); + const bucket = new s3.Bucket(stack, 'Dest'); + + // WHEN + new s3deploy.BucketDeployment(stack, 'Deploy', { + sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], + destinationBucket: bucket, + exclude: ['sample/*', 'another/*'], + include: ['sample/include.json', 'another/include.json'], + }); + + expect(stack).toHaveResourceLike('Custom::CDKBucketDeployment', { + Exclude: ['sample/*', 'another/*'], + Include: ['sample/include.json', 'another/include.json'], + }); +}); + test('deployment allows vpc to be implicitly supplied to lambda', () => { // GIVEN diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json index 76f906941f2bd..a0d375ee6e3a1 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json @@ -295,7 +295,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3BucketD8D20B9A" }, "S3Key": { "Fn::Join": [ @@ -308,7 +308,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3" } ] } @@ -321,7 +321,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3" } ] } @@ -365,17 +365,17 @@ "Type": "String", "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3BucketD8D20B9A": { "Type": "String", - "Description": "S3 bucket for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "S3 bucket for asset \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3": { "Type": "String", - "Description": "S3 key for asset version \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "S3 key for asset version \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaArtifactHash3943F7F3": { "Type": "String", - "Description": "Artifact hash for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "Artifact hash for asset \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3Bucket9CD8B20A": { "Type": "String", @@ -390,4 +390,4 @@ "Description": "Artifact hash for asset \"fc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222e\"" } } -} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json index 877298773816b..320bf196acb7b 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json @@ -304,7 +304,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3BucketD8D20B9A" }, "S3Key": { "Fn::Join": [ @@ -317,7 +317,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3" } ] } @@ -330,7 +330,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3" } ] } @@ -685,6 +685,111 @@ }, "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" + }, + "DeployMeWithExcludedFilesOnDestinationAwsCliLayer68F5E11D": { + "Type": "AWS::Lambda::LayerVersion", + "Properties": { + "Content": { + "S3Bucket": { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + } + ] + } + ] + } + ] + ] + } + }, + "Description": "/opt/awscli/aws" + } + }, + "DeployMeWithExcludedFilesOnDestinationCustomResource48D69581": { + "Type": "Custom::CDKBucketDeployment", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756C81C01536", + "Arn" + ] + }, + "SourceBucketNames": [ + { + "Ref": "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3Bucket9CD8B20A" + } + ], + "SourceObjectKeys": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3VersionKeyA58D380C" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3VersionKeyA58D380C" + } + ] + } + ] + } + ] + ] + } + ], + "DestinationBucketName": { + "Ref": "Destination920A3C57" + }, + "RetainOnDelete": false, + "Prune": true, + "Exclude": [ + "*.gif" + ] + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" } }, "Parameters": { @@ -700,17 +805,17 @@ "Type": "String", "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3BucketD8D20B9A": { "Type": "String", - "Description": "S3 bucket for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "S3 bucket for asset \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3": { "Type": "String", - "Description": "S3 key for asset version \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "S3 key for asset version \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaArtifactHash3943F7F3": { "Type": "String", - "Description": "Artifact hash for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "Artifact hash for asset \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3Bucket9CD8B20A": { "Type": "String", @@ -725,4 +830,4 @@ "Description": "Artifact hash for asset \"fc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222e\"" } } -} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.ts b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.ts index b2b1754083bd7..b6f20f62e6024 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.ts +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.ts @@ -46,6 +46,13 @@ class TestBucketDeployment extends cdk.Stack { retainOnDelete: false, }); + new s3deploy.BucketDeployment(this, 'DeployMeWithExcludedFilesOnDestination', { + sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], + destinationBucket, + exclude: ['*.gif'], + retainOnDelete: false, + }); + } } diff --git a/packages/@aws-cdk/aws-s3-deployment/test/lambda/aws b/packages/@aws-cdk/aws-s3-deployment/test/lambda/aws index 969bb982cd08c..5cfd9275eeb71 100755 --- a/packages/@aws-cdk/aws-s3-deployment/test/lambda/aws +++ b/packages/@aws-cdk/aws-s3-deployment/test/lambda/aws @@ -21,7 +21,14 @@ if sys.argv[2] == "cp" and not sys.argv[4].startswith("s3://"): sys.argv[4] = "archive.zip" if sys.argv[2] == "sync": - sys.argv[4 if '--delete' in sys.argv else 3] = "contents.zip" + contentsIdx = 3 + if '--include' in sys.argv: + contentsIdx += sys.argv.count("--include") * 2 + if '--exclude' in sys.argv: + contentsIdx += sys.argv.count("--exclude") * 2 + if '--delete' in sys.argv: + contentsIdx += 1 + sys.argv[contentsIdx] = "contents.zip" with open("./aws.out", "a") as myfile: myfile.write(json.dumps(sys.argv[1:]) + "\n") diff --git a/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py b/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py index 7813a13db3859..c7a371fafa01e 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py +++ b/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py @@ -68,6 +68,122 @@ def test_update_no_delete(self): ["s3", "sync", "contents.zip", "s3:///"] ) + def test_create_exclude(self): + invoke_handler("Create", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Exclude": ["sample.json"] + }) + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--exclude", "sample.json", "contents.zip", "s3:///"] + ) + + def test_update_exclude(self): + invoke_handler("Update", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Exclude": ["sample.json"] + }, old_resource_props={ + "DestinationBucketName": "", + }, physical_id="") + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--exclude", "sample.json", "contents.zip", "s3:///"] + ) + + def test_create_include(self): + invoke_handler("Create", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Include": ["/sample/*.json"] + }) + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--include", "/sample/*.json", "contents.zip", "s3:///"] + ) + + def test_update_include(self): + invoke_handler("Update", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Include": ["/sample/*.json"] + }, old_resource_props={ + "DestinationBucketName": "", + }, physical_id="") + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--include", "/sample/*.json", "contents.zip", "s3:///"] + ) + + def test_create_include_exclude(self): + invoke_handler("Create", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Exclude": ["/sample/*"], + "Include": ["/sample/*.json"] + }) + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--exclude", "/sample/*", "--include", "/sample/*.json", "contents.zip", "s3:///"] + ) + + def test_update_include_exclude(self): + invoke_handler("Update", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Exclude": ["/sample/*"], + "Include": ["/sample/*.json"] + }, old_resource_props={ + "DestinationBucketName": "", + }, physical_id="") + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--exclude", "/sample/*", "--include", "/sample/*.json", "contents.zip", "s3:///"] + ) + + def test_create_multiple_include_exclude(self): + invoke_handler("Create", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Exclude": ["/sample/*", "/another/*"], + "Include": ["/sample/*.json", "/another/*.json"] + }) + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--exclude", "/sample/*", "--exclude", "/another/*", "--include", "/sample/*.json", "--include", "/another/*.json", "contents.zip", "s3:///"] + ) + + def test_update_multiple_include_exclude(self): + invoke_handler("Update", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Exclude": ["/sample/*", "/another/*"], + "Include": ["/sample/*.json", "/another/*.json"] + }, old_resource_props={ + "DestinationBucketName": "", + }, physical_id="") + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--exclude", "/sample/*", "--exclude", "/another/*", "--include", "/sample/*.json", "--include", "/another/*.json", "contents.zip", "s3:///"] + ) + def test_create_update_multiple_sources(self): invoke_handler("Create", { "SourceBucketNames": ["", ""],