feat(aws-s3-deployment): support specifying objects metadata (#4288)

* feat(aws-s3-deployment): support specifying objects metadata objects metadata can now be given as part of a deployment which can be useful for example in setting content-type to text/html for static web files * docs(aws-s3-deployment): section in README for objects metadata * chore(aws-s3-deployment): typo in docs * chore(aws-s3-deployment): full stop in docs * feat(aws-s3-deployment): simplify objects metadata api metadata is now defined per deployment, system-defined metadata helpers * chore(aws-s3-deployment): use date for expires system metadata value * chore(aws-s3-deployment): update metadata docs * chore(aws-s3-deployment): use interface instead of types * chore(aws-s3-deployment): add missing dependency fs * chore(aws-s3-deployment): make objects metadata properties readonly * feat(aws-s3-deployment): flatten metadata types, provide access to all available system metadata * chore(aws-s3-deployment): fix builds errors * chore(aws-s3-deployment): public and private are reserved words so use setPublic and setPrivate instead of CacheControl * chore(aws-s3-deployment): add s-max-age to cache-control * chore(aws-s3-deployment): fix style issues * chore(aws-s3-deployment): fix style issues * chore(aws-s3-deployment): fix style issues * chore(aws-s3-deployment): fix whitespace * chore(aws-s3-deployment): better docs for optional metadata properties * chore(aws-s3-deployment): fix whitespace issue * chore(aws-s3-deployment): stricter test for metadata * chore(aws-s3-deployment): import order in test * chore(aws-s3-deployment): import order in test * chore(aws-s3-deployment): update docs, increase test coverage * chore(aws-s3-deployment): add all system-defined metadata keys to README * fix(aws-s3-deployment): handle expires system metadata key * chore(aws-s3-deployment): style fixes * chore(aws-s3-deployment): add keys to metadata test * chore(aws-s3-deployment): docs for metadata classes/enums, change userMetadata to metadata for consistency with aws cli * chore(aws-s3-deployment): re-run integration tests * chore(aws-s3-deployment): fix name of metadata key in test * chore(aws-s3-deployment): shorten comment * chore(aws-s3-deployment): update output from integration tests * chore(aws-s3-deployment): remove trailing whitespace
aws · Oct 11, 2019 · 63cb2da · 63cb2da
1 parent be0d2c6
commit 63cb2da
Show file tree

Hide file tree

Showing 8 changed files with 352 additions and 37 deletions.
diff --git a/packages/@aws-cdk/aws-s3-deployment/README.md b/packages/@aws-cdk/aws-s3-deployment/README.md
@@ -65,6 +65,48 @@ By default, the contents of the destination bucket will be deleted when the
 changed. You can use the option `retainOnDelete: true` to disable this behavior,
 in which case the contents will be retained.
 
+## Objects metadata
+
+You can specify metadata to be set on all the objects in your deployment.
+There are 2 types of metadata in S3: system-defined metadata and user-defined metadata.
+System-defined metadata have a special purpose, for example cache-control defines how long to keep an object cached.
+User-defined metadata are not used by S3 and keys always begin with `x-amzn-meta-` (if this is not provided, it is added automatically).
+
+System defined metadata keys include the following:
+
+- cache-control
+- content-disposition
+- content-encoding
+- content-language
+- content-type
+- expires
+- server-side-encryption
+- storage-class
+- website-redirect-location
+- ssekms-key-id
+- sse-customer-algorithm
+
+```ts
+const websiteBucket = new s3.Bucket(this, 'WebsiteBucket', {
+  websiteIndexDocument: 'index.html',
+  publicReadAccess: true
+});
+
+new s3deploy.BucketDeployment(this, 'DeployWebsite', {
+  sources: [s3deploy.Source.asset('./website-dist')],
+  destinationBucket: websiteBucket,
+  destinationKeyPrefix: 'web/static', // optional prefix in destination bucket
+  userMetadata: { "A": "1", "b": "2" }, // user-defined metadata
+
+  // system-defined metadata
+  contentType: "text/html",
+  contentLanguage: "en",
+  storageClass: StorageClass.INTELLIGENT_TIERING,
+  serverSideEncryption: ServerSideEncryption.AES_256,
+  cacheControl: [CacheControl.setPublic(), CacheControl.maxAge(cdk.Duration.hours(1))],
+});
+```
+
 ## CloudFront Invalidation
 
 You can provide a CloudFront distribution and optional paths to invalidate after the bucket deployment finishes.

diff --git a/packages/@aws-cdk/aws-s3-deployment/lambda/src/index.py b/packages/@aws-cdk/aws-s3-deployment/lambda/src/index.py
@@ -41,10 +41,12 @@ def cfn_error(message=None):
         try:
             source_bucket_names = props['SourceBucketNames']
             source_object_keys  = props['SourceObjectKeys']
-            dest_bucket_name   = props['DestinationBucketName']
-            dest_bucket_prefix = props.get('DestinationBucketKeyPrefix', '')
-            retain_on_delete   = props.get('RetainOnDelete', "true") == "true"
-            distribution_id    = props.get('DistributionId', '')
+            dest_bucket_name    = props['DestinationBucketName']
+            dest_bucket_prefix  = props.get('DestinationBucketKeyPrefix', '')
+            retain_on_delete    = props.get('RetainOnDelete', "true") == "true"
+            distribution_id     = props.get('DistributionId', '')
+            user_metadata       = props.get('UserMetadata', {})
+            system_metadata     = props.get('SystemMetadata', {})
 
             default_distribution_path = dest_bucket_prefix
             if not default_distribution_path.endswith("/"):
@@ -96,7 +98,7 @@ def cfn_error(message=None):
             aws_command("s3", "rm", old_s3_dest, "--recursive")
 
         if request_type == "Update" or request_type == "Create":
-            s3_deploy(s3_source_zips, s3_dest)
+            s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata)
 
         if distribution_id:
             cloudfront_invalidate(distribution_id, distribution_paths)
@@ -110,7 +112,7 @@ def cfn_error(message=None):
 
 #---------------------------------------------------------------------------------------------------
 # populate all files from s3_source_zips to a destination bucket
-def s3_deploy(s3_source_zips, s3_dest):
+def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata):
     # create a temporary working directory
     workdir=tempfile.mkdtemp()
     logger.info("| workdir: %s" % workdir)
@@ -129,7 +131,7 @@ def s3_deploy(s3_source_zips, s3_dest):
           zip.extractall(contents_dir)
 
     # sync from "contents" to destination
-    aws_command("s3", "sync", "--delete", contents_dir, s3_dest)
+    aws_command("s3", "sync", "--delete", contents_dir, s3_dest, *create_metadata_args(user_metadata, system_metadata))
     shutil.rmtree(workdir)
 
 #---------------------------------------------------------------------------------------------------
@@ -149,6 +151,23 @@ def cloudfront_invalidate(distribution_id, distribution_paths):
         DistributionId=distribution_id,
         Id=invalidation_resp['Invalidation']['Id'])
 
+#---------------------------------------------------------------------------------------------------
+# set metadata
+def create_metadata_args(raw_user_metadata, raw_system_metadata):
+    if len(raw_user_metadata) == 0 and len(raw_system_metadata) == 0:
+        return []
+
+    format_system_metadata_key = lambda k: k.lower()
+    format_user_metadata_key = lambda k: k.lower() if k.lower().startswith("x-amzn-meta-") else f"x-amzn-meta-{k.lower()}" 
+
+    system_metadata = { format_system_metadata_key(k): v for k, v in raw_system_metadata.items() }
+    user_metadata = { format_user_metadata_key(k): v for k, v in raw_user_metadata.items() }
+
+    system_args = [f"--{k} '{v}'" for k, v in system_metadata.items()]
+    user_args = ["--metadata", f"'{json.dumps(user_metadata)}'"] if len(user_metadata) > 0 else []
+
+    return system_args + user_args + ["--metadata-directive", "REPLACE"]
+
 #---------------------------------------------------------------------------------------------------
 # executes an "aws" cli command
 def aws_command(*args):

diff --git a/packages/@aws-cdk/aws-s3-deployment/lambda/test/aws b/packages/@aws-cdk/aws-s3-deployment/lambda/test/aws
@@ -15,8 +15,8 @@ import shutil
 
 scriptdir=os.path.dirname(os.path.realpath(__file__))
 
-# if "cp" is called, copy a test zip file to the destination
-if sys.argv[2] == "cp":
+# if "cp" is called with a local destination, copy a test zip file to the destination or 
+if sys.argv[2] == "cp" and not sys.argv[4].startswith("s3://"):
     shutil.copyfile(os.path.join(scriptdir, 'test.zip'), sys.argv[4])
     sys.argv[4] = "archive.zip"
 

diff --git a/packages/@aws-cdk/aws-s3-deployment/lambda/test/test.py b/packages/@aws-cdk/aws-s3-deployment/lambda/test/test.py
@@ -79,6 +79,21 @@ def test_create_update_with_dest_key(self):
             "s3 sync --delete contents.zip s3://<dest-bucket-name>/<dest-key-prefix>"
         )
 
+    def test_create_update_with_metadata(self):
+        invoke_handler("Create", {
+            "SourceBucketNames": ["<source-bucket>"],
+            "SourceObjectKeys": ["<source-object-key>"],
+            "DestinationBucketName": "<dest-bucket-name>",
+            "DestinationBucketKeyPrefix": "<dest-key-prefix>",
+            "UserMetadata": { "best": "game" },
+            "SystemMetadata": { "content-type": "text/html", "content-language": "en" }
+        })
+
+        self.assertAwsCommands(
+            "s3 cp s3://<source-bucket>/<source-object-key> archive.zip",
+            "s3 sync --delete contents.zip s3://<dest-bucket-name>/<dest-key-prefix> --content-type 'text/html' --content-language 'en' --metadata '{\"x-amzn-meta-best\": \"game\"}' --metadata-directive REPLACE"
+        )
+
     def test_delete_no_retain(self):
         invoke_handler("Delete", {
             "SourceBucketNames": ["<source-bucket>"],

diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts
@@ -1,16 +1,16 @@
-import cloudformation = require('@aws-cdk/aws-cloudformation');
-import cloudfront = require('@aws-cdk/aws-cloudfront');
-import iam = require('@aws-cdk/aws-iam');
-import lambda = require('@aws-cdk/aws-lambda');
-import s3 = require('@aws-cdk/aws-s3');
-import cdk = require('@aws-cdk/core');
-import { Token } from '@aws-cdk/core';
+import cloudformation = require("@aws-cdk/aws-cloudformation");
+import cloudfront = require("@aws-cdk/aws-cloudfront");
+import iam = require("@aws-cdk/aws-iam");
+import lambda = require("@aws-cdk/aws-lambda");
+import s3 = require("@aws-cdk/aws-s3");
+import cdk = require("@aws-cdk/core");
+import { Token } from "@aws-cdk/core";
 import crypto = require('crypto');
 import fs = require('fs');
-import path = require('path');
-import { ISource, SourceConfig } from './source';
+import path = require("path");
+import { ISource, SourceConfig } from "./source";
 
-const handlerCodeBundle = path.join(__dirname, '..', 'lambda', 'bundle.zip');
+const handlerCodeBundle = path.join(__dirname, "..", "lambda", "bundle.zip");
 const handlerSourceDirectory = path.join(__dirname, '..', 'lambda', 'src');
 
 export interface BucketDeploymentProps {
@@ -77,6 +77,80 @@ export interface BucketDeploymentProps {
    * @default - A role is automatically created
    */
   readonly role?: iam.IRole;
+
+  /**
+   * User-defined object metadata to be set on all objects in the deployment
+   * @default - No user metadata is set
+   * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#UserMetadata
+   */
+  readonly metadata?: UserDefinedObjectMetadata;
+
+  /**
+   * System-defined cache-control metadata to be set on all objects in the deployment.
+   * @default - Not set.
+   * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
+   */
+  readonly cacheControl?: CacheControl[];
+  /**
+   * System-defined cache-disposition metadata to be set on all objects in the deployment.
+   * @default - Not set.
+   * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
+   */
+  readonly contentDisposition?: string;
+  /**
+   * System-defined content-encoding metadata to be set on all objects in the deployment.
+   * @default - Not set.
+   * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
+   */
+  readonly contentEncoding?: string;
+  /**
+   * System-defined content-language metadata to be set on all objects in the deployment.
+   * @default - Not set.
+   * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
+   */
+  readonly contentLanguage?: string;
+  /**
+   * System-defined content-type metadata to be set on all objects in the deployment.
+   * @default - Not set.
+   * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
+   */
+  readonly contentType?: string;
+  /**
+   * System-defined expires metadata to be set on all objects in the deployment.
+   * @default - The objects in the distribution will not expire.
+   * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
+   */
+  readonly expires?: Expires;
+  /**
+   * System-defined x-amz-server-side-encryption metadata to be set on all objects in the deployment.
+   * @default - Server side encryption is not used.
+   * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
+   */
+  readonly serverSideEncryption?: ServerSideEncryption;
+  /**
+   * System-defined x-amz-storage-class metadata to be set on all objects in the deployment.
+   * @default - Default storage-class for the bucket is used.
+   * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
+   */
+  readonly storageClass?: StorageClass;
+  /**
+   * System-defined x-amz-website-redirect-location metadata to be set on all objects in the deployment.
+   * @default - No website redirection.
+   * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
+   */
+  readonly websiteRedirectLocation?: string;
+  /**
+   * System-defined x-amz-server-side-encryption-aws-kms-key-id metadata to be set on all objects in the deployment.
+   * @default - Not set.
+   * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
+   */
+  readonly serverSideEncryptionAwsKmsKeyId?: string;
+  /**
+   * System-defined x-amz-server-side-encryption-customer-algorithm metadata to be set on all objects in the deployment.
+   * @default - Not set.
+   * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
+   */
+  readonly serverSideEncryptionCustomerAlgorithm?: string;
 }
 
 export class BucketDeployment extends cdk.Construct {
@@ -123,6 +197,8 @@ export class BucketDeployment extends cdk.Construct {
         DestinationBucketName: props.destinationBucket.bucketName,
         DestinationBucketKeyPrefix: props.destinationKeyPrefix,
         RetainOnDelete: props.retainOnDelete,
+        UserMetadata: props.metadata ? mapUserMetadata(props.metadata) : undefined,
+        SystemMetadata: mapSystemMetadata(props),
         DistributionId: props.distribution ? props.distribution.distributionId : undefined,
         DistributionPaths: props.distributionPaths
       }
@@ -165,3 +241,129 @@ function calcSourceHash(srcDir: string): string {
 
   return sha.digest('hex');
 }
+
+/**
+ * Metadata
+ */
+
+function mapUserMetadata(metadata: UserDefinedObjectMetadata) {
+  const mapKey = (key: string) =>
+    key.toLowerCase().startsWith("x-amzn-meta-")
+      ? key.toLowerCase()
+      : `x-amzn-meta-${key.toLowerCase()}`;
+
+  return Object.keys(metadata).reduce((o, key) => ({ ...o, [mapKey(key)]: metadata[key] }), {});
+}
+
+function mapSystemMetadata(metadata: BucketDeploymentProps) {
+  function mapCacheControlDirective(cacheControl: CacheControl) {
+    const { value } = cacheControl;
+
+    if (typeof value === "string") { return value; }
+    if ("max-age" in value) { return `max-age=${value["max-age"].toSeconds()}`; }
+    if ("s-max-age" in value) { return `s-max-age=${value["s-max-age"].toSeconds()}`; }
+
+    throw new Error(`Unsupported cache-control directive ${value}`);
+  }
+  function mapExpires(expires: Expires) {
+    const { value } = expires;
+
+    if (typeof value === "string") { return value; }
+    if (value instanceof Date) { return value.toUTCString(); }
+    if (value instanceof cdk.Duration) { return new Date(Date.now() + value.toMilliseconds()).toUTCString(); }
+
+    throw new Error(`Unsupported expires ${expires}`);
+  }
+
+  const res: { [key: string]: string } = {};
+
+  if (metadata.cacheControl) { res["cache-control"] = metadata.cacheControl.map(mapCacheControlDirective).join(", "); }
+  if (metadata.expires) { res.expires = mapExpires(metadata.expires); }
+  if (metadata.contentDisposition) { res["content-disposition"] = metadata.contentDisposition; }
+  if (metadata.contentEncoding) { res["content-encoding"] = metadata.contentEncoding; }
+  if (metadata.contentLanguage) { res["content-language"] = metadata.contentLanguage; }
+  if (metadata.contentType) { res["content-type"] = metadata.contentType; }
+  if (metadata.serverSideEncryption) { res["server-side-encryption"] = metadata.serverSideEncryption; }
+  if (metadata.storageClass) { res["storage-class"] = metadata.storageClass; }
+  if (metadata.websiteRedirectLocation) { res["website-redirect-location"] = metadata.websiteRedirectLocation; }
+  if (metadata.serverSideEncryptionAwsKmsKeyId) { res["ssekms-key-id"] = metadata.serverSideEncryptionAwsKmsKeyId; }
+  if (metadata.serverSideEncryptionCustomerAlgorithm) { res["sse-customer-algorithm"] = metadata.serverSideEncryptionCustomerAlgorithm; }
+
+  return Object.keys(res).length === 0 ? undefined : res;
+}
+
+/**
+ * Used for HTTP cache-control header, which influences downstream caches.
+ * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
+ */
+export class CacheControl {
+  public static mustRevalidate() { return new CacheControl("must-revalidate"); }
+  public static noCache() { return new CacheControl("no-cache"); }
+  public static noTransform() { return new CacheControl("no-transform"); }
+  public static setPublic() { return new CacheControl("public"); }
+  public static setPrivate() { return new CacheControl("private"); }
+  public static proxyRevalidate() { return new CacheControl("proxy-revalidate"); }
+  public static maxAge(t: cdk.Duration) { return new CacheControl({ "max-age": t }); }
+  public static sMaxAge(t: cdk.Duration) { return new CacheControl({ "s-max-age": t }); }
+  public static fromString(s: string) {  return new CacheControl(s); }
+
+  private constructor(public value: any) {}
+}
+
+/**
+ * Indicates whether server-side encryption is enabled for the object, and whether that encryption is
+ * from the AWS Key Management Service (AWS KMS) or from Amazon S3 managed encryption (SSE-S3).
+ * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
+ */
+export enum ServerSideEncryption {
+  AES_256 = 'AES256',
+  AWS_KMS = 'aws:kms'
+}
+
+/**
+ * Storage class used for storing the object.
+ * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
+ */
+export enum StorageClass {
+  STANDARD = 'STANDARD',
+  REDUCED_REDUNDANCY = 'REDUCED_REDUNDANCY',
+  STANDARD_IA = 'STANDARD_IA',
+  ONEZONE_IA = 'ONEZONE_IA',
+  INTELLIGENT_TIERING = 'INTELLIGENT_TIERING',
+  GLACIER = 'GLACIER',
+  DEEP_ARCHIVE = 'DEEP_ARCHIVE'
+}
+
+/**
+ * Used for HTTP expires header, which influences downstream caches. Does NOT influence deletion of the object.
+ * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#SysMetadata
+ */
+export class Expires {
+  /**
+   * Expire at the specified date
+   * @param d date to expire at
+   */
+  public static atDate(d: Date) { return new Expires(d); }
+  /**
+   * Expire at the specified timestamp
+   * @param t timestamp in unix milliseconds
+   */
+  public static atTimestamp(t: number) { return new Expires(t); }
+  /**
+   * Expire once the specified duration has passed since deployment time
+   * @param t the duration to wait before expiring
+   */
+  public static after(t: cdk.Duration) { return new Expires(t); }
+  public static fromString(s: string) { return new Expires(s); }
+
+  private constructor(public value: any) {}
+}
+
+export interface UserDefinedObjectMetadata {
+  /**
+   * Arbitrary metadata key-values
+   * Keys must begin with `x-amzn-meta-` (will be added automatically if not provided)
+   * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#UserMetadata
+   */
+  readonly [key: string]: string;
+}