feat(cli): Add an option to import existing resources (currently S3 b…

…uckets only)

This is an initial proposal to support existing resources import into
CDK stacks. As a PoC, this PR shows a working solution for S3 buckets.
This is achieved by introducing `-i` / `--import-resources` CLI option
to the `cdk deploy` command. If specified, the newly added resources
will not be created, but attempted to be imported (adopted) instead. If
the resource definition contains the full resource identifier, this
happens automatically. For resources that can't be identified (e.g. an
S3 bucket without an explicit `bucketName`), user will be prompted for
the necessary information.

packages/aws-cdk/bin/cdk.ts

@@ -124,7 +124,8 @@ async function parseCommandLineArguments() {
         desc: 'Continuously observe the project files, ' +
           'and deploy the given stack(s) automatically when changes are detected. ' +
           'Implies --hotswap by default',
-      }),
+      })
+      .option('import-resources', { type: 'boolean', alias: 'i', desc: 'Import existing resources in the stack' }),
     )
     .command('watch [STACKS..]', "Shortcut for 'deploy --watch'", yargs => yargs
       // I'm fairly certain none of these options, present for 'deploy', make sense for 'watch':
@@ -375,6 +376,7 @@ async function initCommandLine() {
           rollback: configuration.settings.get(['rollback']),
           hotswap: args.hotswap,
           watch: args.watch,
+          importResources: args['import-resources'],
         });
 
       case 'watch':

packages/aws-cdk/lib/api/cloudformation-deployments.ts

@@ -6,7 +6,7 @@ import { publishAssets } from '../util/asset-publishing';
 import { Mode, SdkProvider } from './aws-auth';
 import { deployStack, DeployStackResult, destroyStack } from './deploy-stack';
 import { ToolkitInfo } from './toolkit-info';
-import { CloudFormationStack, Template } from './util/cloudformation';
+import { CloudFormationStack, Template, ResourcesToImport } from './util/cloudformation';
 import { StackActivityProgress } from './util/cloudformation/stack-activity-monitor';
 
 /**
@@ -152,6 +152,11 @@ export interface DeployStackOptions {
    * @default - nothing extra is appended to the User-Agent header
    */
   readonly extraUserAgent?: string;
+
+  /**
+   * List of existing resources to be IMPORTED into the stack, instead of being CREATED
+   */
+  readonly resourcesToImport?: ResourcesToImport;
 }
 
 export interface DestroyStackOptions {
@@ -230,6 +235,7 @@ export class CloudFormationDeployments {
       rollback: options.rollback,
       hotswap: options.hotswap,
       extraUserAgent: options.extraUserAgent,
+      resourcesToImport: options.resourcesToImport,
     });
   }
 

packages/aws-cdk/lib/api/deploy-stack.ts

@@ -14,7 +14,7 @@ import { CfnEvaluationException } from './hotswap/evaluate-cloudformation-templa
 import { ToolkitInfo } from './toolkit-info';
 import {
   changeSetHasNoChanges, CloudFormationStack, TemplateParameters, waitForChangeSet,
-  waitForStackDeploy, waitForStackDelete, ParameterValues, ParameterChanges,
+  waitForStackDeploy, waitForStackDelete, ParameterValues, ParameterChanges, ResourcesToImport,
 } from './util/cloudformation';
 import { StackActivityMonitor, StackActivityProgress } from './util/cloudformation/stack-activity-monitor';
 
@@ -189,6 +189,12 @@ export interface DeployStackOptions {
    * @default - nothing extra is appended to the User-Agent header
    */
   readonly extraUserAgent?: string;
+
+  /**
+   * If set, change set of type IMPORT will be created, and resourcesToImport
+   * passed to it.
+   */
+  readonly resourcesToImport?: ResourcesToImport;
 }
 
 const LARGE_TEMPLATE_SIZE_KB = 50;
@@ -294,7 +300,8 @@ async function prepareAndExecuteChangeSet(
   const changeSet = await cfn.createChangeSet({
     StackName: deployName,
     ChangeSetName: changeSetName,
-    ChangeSetType: update ? 'UPDATE' : 'CREATE',
+    ChangeSetType: options.resourcesToImport ? 'IMPORT' : update ? 'UPDATE' : 'CREATE',
+    ResourcesToImport: options.resourcesToImport,
     Description: `CDK Changeset for execution ${executionId}`,
     TemplateBody: bodyParameter.TemplateBody,
     TemplateURL: bodyParameter.TemplateURL,

packages/aws-cdk/lib/api/util/cloudformation.ts

@@ -16,6 +16,8 @@ interface TemplateParameter {
   [key: string]: any;
 }
 
+export type ResourcesToImport = CloudFormation.ResourcesToImport;
+
 /**
  * Represents an (existing) Stack in CloudFormation
  *

packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts

@@ -34,7 +34,7 @@ export class StackStatus {
   }
 
   get isDeploySuccess(): boolean {
-    return !this.isNotFound && (this.name === 'CREATE_COMPLETE' || this.name === 'UPDATE_COMPLETE');
+    return !this.isNotFound && (this.name === 'CREATE_COMPLETE' || this.name === 'UPDATE_COMPLETE' || this.name === 'IMPORT_COMPLETE');
   }
 
   public toString(): string {

packages/aws-cdk/lib/cdk-toolkit.ts

@@ -11,8 +11,10 @@ import { Bootstrapper, BootstrapEnvironmentOptions } from './api/bootstrap';
 import { CloudFormationDeployments } from './api/cloudformation-deployments';
 import { CloudAssembly, DefaultSelection, ExtendedStackSelection, StackCollection, StackSelector } from './api/cxapp/cloud-assembly';
 import { CloudExecutable } from './api/cxapp/cloud-executable';
+import { ResourcesToImport } from './api/util/cloudformation';
 import { StackActivityProgress } from './api/util/cloudformation/stack-activity-monitor';
 import { printSecurityDiff, printStackDiff, RequireApproval } from './diff';
+import { prepareResourcesToImport } from './import';
 import { data, debug, error, highlight, print, success, warning } from './logging';
 import { deserializeStructure } from './serialize';
 import { Configuration, PROJECT_CONFIG } from './settings';
@@ -117,6 +119,11 @@ export class CdkToolkit {
       return this.watch(options);
     }
 
+    // TODO - print more intelligent message
+    if (options.importResources) {
+      warning('Import resources flag was set');
+    }
+
     const stacks = await this.selectStacksForDeploy(options.selector, options.exclusively, options.cacheCloudAssembly);
 
     const requireApproval = options.requireApproval ?? RequireApproval.Broadening;
@@ -167,6 +174,17 @@ export class CdkToolkit {
         continue;
       }
 
+      let resourcesToImport: ResourcesToImport | undefined = undefined;
+      if (options.importResources) {
+        const currentTemplate = await this.props.cloudFormation.readCurrentTemplate(stack);
+        resourcesToImport = await prepareResourcesToImport(currentTemplate, stack);
+
+        // There's a CloudFormation limitation that on import operation, no other changes are allowed:
+        // As CDK always changes the CDKMetadata resource with a new value, as a workaround, we override
+        // the template's metadata with currently deployed version
+        stack.template.Resources.CDKMetadata = currentTemplate.Resources.CDKMetadata;
+      }
+
       if (requireApproval !== RequireApproval.Never) {
         const currentTemplate = await this.props.cloudFormation.readCurrentTemplate(stack);
         if (printSecurityDiff(currentTemplate, stack, requireApproval)) {
@@ -209,6 +227,7 @@ export class CdkToolkit {
           rollback: options.rollback,
           hotswap: options.hotswap,
           extraUserAgent: options.extraUserAgent,
+          resourcesToImport,
         });
 
         const message = result.noOp
@@ -799,6 +818,14 @@ export interface DeployOptions extends WatchOptions {
    * @default true
    */
   readonly cacheCloudAssembly?: boolean;
+
+  /**
+   * Whether to import matching existing resources for newly defined constructs in the stack,
+   * rather than creating new ones
+   *
+   * @default false
+   */
+  readonly importResources?: boolean;
 }
 
 export interface DestroyOptions {

packages/aws-cdk/lib/import.ts

@@ -0,0 +1,53 @@
+import * as cfnDiff from '@aws-cdk/cloudformation-diff';
+import * as cxapi from '@aws-cdk/cx-api';
+import * as promptly from 'promptly';
+import { ResourcesToImport } from './api/util/cloudformation';
+
+// Basic idea: we want to have a structure (ideally auto-generated from CFN definitions) that lists all resource types
+// that support importing and for each type, the identification information
+//
+// For each resource that is to be added in the new template:
+// - look up the identification information for the resource type [if not found, fail "type not supported"]
+// - look up the physical resource (perhaps using cloud control API?) [if not found, fail "resource to be imported does not exist"]
+// - assembe and return "resources to import" object to be passed on to changeset creation
+//
+// TEST: can we have a CFN changeset that both creates resources and import other resources?
+const RESOURCE_IDENTIFIERS: { [key: string]: string[] } = {
+  'AWS::S3::Bucket': ['BucketName'],
+};
+
+export async function prepareResourcesToImport(oldTemplate: any, newTemplate: cxapi.CloudFormationStackArtifact): Promise<ResourcesToImport> {
+  const diff = cfnDiff.diffTemplate(oldTemplate, newTemplate.template);
+
+  const additions: { [key: string]: cfnDiff.ResourceDifference } = {};
+  diff.resources.forEachDifference((id, chg) => {
+    if (chg.isAddition) {
+      additions[id] = chg;
+    }
+  });
+
+  const resourcesToImport: ResourcesToImport = [];
+  for (let [id, chg] of Object.entries(additions)) {
+    if (chg.newResourceType === undefined || !(chg.newResourceType in RESOURCE_IDENTIFIERS)) {
+      throw new Error(`Resource ${id} is of type ${chg.newResourceType} that is not supported for import`);
+    }
+
+    let identifier: { [key: string]: string } = {};
+    for (let idpart of RESOURCE_IDENTIFIERS[chg.newResourceType]) {
+      if (chg.newProperties && (idpart in chg.newProperties)) {
+        identifier[idpart] = chg.newProperties[idpart];
+      } else {
+        const displayName : string = newTemplate.template?.Resources?.[id]?.Metadata?.['aws:cdk:path'] ?? id;
+        identifier[idpart] = await promptly.prompt(`Please enter ${idpart} of ${chg.newResourceType} to import as ${displayName.replace(/\/Resource$/, '')}: `);
+      }
+    }
+
+    resourcesToImport.push({
+      LogicalResourceId: id,
+      ResourceType: chg.newResourceType,
+      ResourceIdentifier: identifier,
+    });
+  }
+
+  return resourcesToImport;
+}