feat(s3-deployment): deploy data with deploy-time values

packages/@aws-cdk/aws-s3-deployment/README.md

@@ -49,6 +49,10 @@ The following source types are supported for bucket deployments:
 - Local .zip file: `s3deploy.Source.asset('/path/to/local/file.zip')`
 - Local directory: `s3deploy.Source.asset('/path/to/local/directory')`
 - Another bucket: `s3deploy.Source.bucket(bucket, zipObjectKey)`
+- String data: `s3deploy.Source.data('object-key.txt', 'hello, world!')`
+ (supports [deploy-time values](#data-with-deploy-time-values))
+- JSON data: `s3deploy.Source.jsonData('object-key.json', { json: 'object' })`
+ (supports [deploy-time values](#data-with-deploy-time-values))
 
 To create a source from a single file, you can pass `AssetOptions` to exclude
 all but a single file:
@@ -268,6 +272,32 @@ new s3deploy.BucketDeployment(this, 'DeployMeWithEfsStorage', {
 });
 ```
 
+## Data with deploy-time values
+
+The content passed to `Source.data()` or `Source.jsonData()` can include
+references that will get resolved only during deployment.
+
+For example:
+
+```ts
+declare const destinationBucket: s3.Bucket;
+declare const topic: sns.Topic;
+
+const appConfig = {
+ topic_arn: topic.topicArn,
+ base_url: 'https://my-endpoint',
+};
+
+new s3deploy.BucketDeployment(this, 'BucketDeployment', {
+ sources: [s3deploy.Source.jsonData('config.json', appConfig)],
+ destinationBucket,
+});
+```
+
+The value in `topic.topicArn` is a deploy-time value. It only gets resolved
+during deployment by placing a marker in the generated source file and
+substituting it when its deployed to the destination with the actual value.
+
 ## Notes
 
 - This library uses an AWS CloudFormation custom resource which about 10MiB in
@@ -282,7 +312,7 @@ new s3deploy.BucketDeployment(this, 'DeployMeWithEfsStorage', {
  be good enough: the custom resource will simply not run if the properties don't
  change.
  - If you use assets (`s3deploy.Source.asset()`) you don't need to worry
- about this: the asset system will make sure that if the files have changed, 
+ about this: the asset system will make sure that if the files have changed,
  the file name is unique and the deployment will run.
 
 ## Development

packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts

@@ -323,13 +323,18 @@ export class BucketDeployment extends CoreConstruct {
  }));
  }
 
+ // to avoid redundant stack updates, only include "SourceMarkers" if one of
+ // the sources actually has markers.
+ const hasMarkers = sources.some(source => source.markers);
+
  const crUniqueId = `CustomResource${this.renderUniqueId(props.memoryLimit, props.vpc)}`;
  const cr = new cdk.CustomResource(this, crUniqueId, {
  serviceToken: handler.functionArn,
  resourceType: 'Custom::CDKBucketDeployment',
  properties: {
  SourceBucketNames: sources.map(source => source.bucket.bucketName),
  SourceObjectKeys: sources.map(source => source.zipObjectKey),
+ SourceMarkers: hasMarkers ? sources.map(source => source.markers ?? {}) : undefined,
  DestinationBucketName: props.destinationBucket.bucketName,
  DestinationBucketKeyPrefix: props.destinationKeyPrefix,
  RetainOnDelete: props.retainOnDelete,

packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py

@@ -20,6 +20,7 @@
 CFN_SUCCESS = "SUCCESS"
 CFN_FAILED = "FAILED"
 ENV_KEY_MOUNT_PATH = "MOUNT_PATH"
+ENV_KEY_SKIP_CLEANUP = "SKIP_CLEANUP"
 
 CUSTOM_RESOURCE_OWNER_TAG = "aws-cdk:cr-owned"
 
@@ -45,6 +46,7 @@ def cfn_error(message=None):
  try:
  source_bucket_names = props['SourceBucketNames']
  source_object_keys = props['SourceObjectKeys']
+ source_markers = props.get('SourceMarkers', None)
  dest_bucket_name = props['DestinationBucketName']
  dest_bucket_prefix = props.get('DestinationBucketKeyPrefix', '')
  retain_on_delete = props.get('RetainOnDelete', "true") == "true"
@@ -55,6 +57,11 @@ def cfn_error(message=None):
  exclude = props.get('Exclude', [])
  include = props.get('Include', [])
 
+ # backwards compatibility - if "SourceMarkers" is not specified,
+ # assume all sources have an empty market map
+ if source_markers is None:
+ source_markers = [{} for i in range(len(source_bucket_names))]
+
  default_distribution_path = dest_bucket_prefix
  if not default_distribution_path.endswith("/"):
  default_distribution_path += "/"
@@ -71,7 +78,7 @@ def cfn_error(message=None):
  if dest_bucket_prefix == "/":
  dest_bucket_prefix = ""
 
- s3_source_zips = map(lambda name, key: "s3://%s/%s" % (name, key), source_bucket_names, source_object_keys)
+ s3_source_zips = list(map(lambda name, key: "s3://%s/%s" % (name, key), source_bucket_names, source_object_keys))
  s3_dest = "s3://%s/%s" % (dest_bucket_name, dest_bucket_prefix)
  old_s3_dest = "s3://%s/%s" % (old_props.get("DestinationBucketName", ""), old_props.get("DestinationBucketKeyPrefix", ""))
 
@@ -106,7 +113,7 @@ def cfn_error(message=None):
  aws_command("s3", "rm", old_s3_dest, "--recursive")
 
  if request_type == "Update" or request_type == "Create":
- s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include)
+ s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers)
 
  if distribution_id:
  cloudfront_invalidate(distribution_id, distribution_paths)
@@ -120,7 +127,11 @@ def cfn_error(message=None):
 
 #---------------------------------------------------------------------------------------------------
 # populate all files from s3_source_zips to a destination bucket
-def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include):
+def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers):
+ # list lengths are equal
+ if len(s3_source_zips) != len(source_markers):
+ raise Exception("'source_markers' and 's3_source_zips' must be the same length")
+
  # create a temporary working directory in /tmp or if enabled an attached efs volume
  if ENV_KEY_MOUNT_PATH in os.environ:
  workdir = os.getenv(ENV_KEY_MOUNT_PATH) + "/" + str(uuid4())
@@ -136,13 +147,16 @@ def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, ex
 
  try:
  # download the archive from the source and extract to "contents"
- for s3_source_zip in s3_source_zips:
+ for i in range(len(s3_source_zips)):
+ s3_source_zip = s3_source_zips[i]
+ markers = source_markers[i]
+
  archive=os.path.join(workdir, str(uuid4()))
  logger.info("archive: %s" % archive)
  aws_command("s3", "cp", s3_source_zip, archive)
  logger.info("| extracting archive to: %s\n" % contents_dir)
- with ZipFile(archive, "r") as zip:
-  zip.extractall(contents_dir)
+ logger.info("| markers: %s" % markers)
+ extract_and_replace_markers(archive, contents_dir, markers)
 
  # sync from "contents" to destination
 
@@ -163,7 +177,8 @@ def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, ex
  s3_command.extend(create_metadata_args(user_metadata, system_metadata))
  aws_command(*s3_command)
  finally:
- shutil.rmtree(workdir)
+ if not os.getenv(ENV_KEY_SKIP_CLEANUP):
+ shutil.rmtree(workdir)
 
 #---------------------------------------------------------------------------------------------------
 # invalidate files in the CloudFront distribution edge caches
@@ -257,3 +272,29 @@ def bucket_owned(bucketName, keyPrefix):
  logger.info("| error getting tags from bucket")
  logger.exception(e)
  return False
+
+# extract archive and replace markers in output files
+def extract_and_replace_markers(archive, contents_dir, markers):
+ with ZipFile(archive, "r") as zip:
+ zip.extractall(contents_dir)
+
+ # replace markers for this source
+ for file in zip.namelist():
+ file_path = os.path.join(contents_dir, file)
+ if os.path.isdir(file_path): continue
+ replace_markers(file_path, markers) 
+
+def replace_markers(filename, markers):
+ # convert the dict of string markers to binary markers
+ replace_tokens = dict([(k.encode('utf-8'), v.encode('utf-8')) for k, v in markers.items()])
+
+ outfile = filename + '.new'
+ with open(filename, 'rb') as fi, open(outfile, 'wb') as fo:
+ for line in fi:
+ for token in replace_tokens:
+ line = line.replace(token, replace_tokens[token])
+ fo.write(line)
+
+ # # delete the original file and rename the new one to the original
+ os.remove(filename)
+ os.rename(outfile, filename)

packages/@aws-cdk/aws-s3-deployment/lib/render-data.ts

@@ -0,0 +1,81 @@
+import { Stack } from '@aws-cdk/core';
+
+// keep this import separate from other imports to reduce chance for merge conflicts with v2-main
+// eslint-disable-next-line no-duplicate-imports, import/order
+import { Construct } from '@aws-cdk/core';
+
+export interface Content {
+ readonly text: string;
+ readonly markers: Record<string, any>;
+}
+
+/**
+ * Renders the given string data as deployable content with markers substituted
+ * for all "Ref" and "Fn::GetAtt" objects.
+ *
+ * @param scope Construct scope
+ * @param data The input data
+ * @returns The markered text (`text`) and a map that maps marker names to their
+ * values (`markers`).
+ */
+export function renderData(scope: Construct, data: string): Content {
+ const obj = Stack.of(scope).resolve(data);
+ if (typeof(obj) === 'string') {
+ return { text: obj, markers: {} };
+ }
+
+ if (typeof(obj) !== 'object') {
+ throw new Error(`Unexpected: after resolve() data must either be a string or a CloudFormation intrinsic. Got: ${JSON.stringify(obj)}`);
+ }
+
+ let markerIndex = 0;
+ const markers: Record<string, FnJoinPart> = {};
+ const result = new Array<string>();
+ const fnJoin: FnJoin | undefined = obj['Fn::Join'];
+
+ if (fnJoin) {
+ const sep = fnJoin[0];
+ const parts = fnJoin[1];
+
+ if (sep !== '') {
+ throw new Error(`Unexpected "Fn::Join", expecting separator to be an empty string but got "${sep}"`);
+ }
+
+ for (const part of parts) {
+ if (typeof (part) === 'string') {
+ result.push(part);
+ continue;
+ }
+
+ if (typeof (part) === 'object') {
+ addMarker(part);
+ continue;
+ }
+
+ throw new Error(`Unexpected "Fn::Join" part, expecting string or object but got ${typeof (part)}`);
+ }
+
+ } else if (obj.Ref || obj['Fn::GetAtt']) {
+ addMarker(obj);
+ } else {
+ throw new Error('Unexpected: Expecting `resolve()` to return "Fn::Join", "Ref" or "Fn::GetAtt"');
+ }
+
+ function addMarker(part: Ref | GetAtt) {
+ const keys = Object.keys(part);
+ if (keys.length !== 1 || (keys[0] != 'Ref' && keys[0] != 'Fn::GetAtt')) {
+ throw new Error(`Invalid CloudFormation reference. "Ref" or "Fn::GetAtt". Got ${JSON.stringify(part)}`);
+ }
+
+ const marker = `<<marker:0xbaba:${markerIndex++}>>`;
+ result.push(marker);
+ markers[marker] = part;
+ }
+
+ return { text: result.join(''), markers };
+}
+
+type FnJoin = [string, FnJoinPart[]];
+type FnJoinPart = string | Ref | GetAtt;
+type Ref = { Ref: string };
+type GetAtt = { 'Fn::GetAtt': [string, string] };

packages/@aws-cdk/aws-s3-deployment/lib/source.ts

@@ -1,6 +1,10 @@
+import * as fs from 'fs';
+import { join, dirname } from 'path';
 import * as iam from '@aws-cdk/aws-iam';
 import * as s3 from '@aws-cdk/aws-s3';
 import * as s3_assets from '@aws-cdk/aws-s3-assets';
+import { FileSystem, Stack } from '@aws-cdk/core';
+import { renderData } from './render-data';
 
 // keep this import separate from other imports to reduce chance for merge conflicts with v2-main
 // eslint-disable-next-line no-duplicate-imports, import/order
@@ -19,6 +23,12 @@ export interface SourceConfig {
  * An S3 object key in the source bucket that points to a zip file.
  */
  readonly zipObjectKey: string;
+
+ /**
+ * A set of markers to substitute in the source content.
+ * @default - no markers
+ */
+ readonly markers?: Record<string, any>;
 }
 
 /**
@@ -50,6 +60,8 @@ export interface ISource {
  * Source.bucket(bucket, key)
  * Source.asset('/local/path/to/directory')
  * Source.asset('/local/path/to/a/file.zip')
+ * Source.data('hello/world/file.txt', 'Hello, world!')
+ * Source.data('config.json', { baz: topic.topicArn })
  *
  */
 export class Source {
@@ -110,5 +122,51 @@ export class Source {
  };
  }
 
+ /**
+ * Deploys an object with the specified string contents into the bucket. The
+ * content can include deploy-time values (such as `snsTopic.topicArn`) that
+ * will get resolved only during deployment.
+ *
+ * To store a JSON object use `Source.jsonData()`.
+ *
+ * @param objectKey The destination S3 object key (relative to the root of the
+ * S3 deployment).
+ * @param data The data to be stored in the object.
+ */
+ public static data(objectKey: string, data: string): ISource {
+ return {
+ bind: (scope: Construct, context?: DeploymentSourceContext) => {
+ const workdir = FileSystem.mkdtemp('s3-deployment');
+ const outputPath = join(workdir, objectKey);
+ const rendered = renderData(scope, data);
+ fs.mkdirSync(dirname(outputPath), { recursive: true });
+ fs.writeFileSync(outputPath, rendered.text);
+ const asset = this.asset(workdir).bind(scope, context);
+ return {
+ bucket: asset.bucket,
+ zipObjectKey: asset.zipObjectKey,
+ markers: rendered.markers,
+ };
+ },
+ };
+ }
+
+ /**
+ * Deploys an object with the specified JSON object into the bucket. The
+ * object can include deploy-time values (such as `snsTopic.topicArn`) that
+ * will get resolved only during deployment.
+ *
+ * @param objectKey The destination S3 object key (relative to the root of the
+ * S3 deployment).
+ * @param obj A JSON object.
+ */
+ public static jsonData(objectKey: string, obj: any): ISource {
+ return {
+ bind: (scope: Construct, context?: DeploymentSourceContext) => {
+ return Source.data(objectKey, Stack.of(scope).toJsonString(obj)).bind(scope, context);
+ },
+ };
+ }
+
  private constructor() { }
 }