feat(deadline): change SEP construct to use launch templates instead …

…of launch specifications

examples/deadline/All-In-AWS-Infrastructure-SEP/python/package/app.py

@@ -45,6 +45,7 @@ def main():
     sep_props = sep_stack.SEPStackProps(
         docker_recipes_stage_path=os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, 'stage'),
         worker_machine_image=MachineImage.generic_linux(config.deadline_client_linux_ami_map),
+        deadline_resource_tracker_exists=config.deadline_resource_tracker_exists,
     )
     service = sep_stack.SEPStack(app, 'SEPStack', props=sep_props, env=env)
 

examples/deadline/All-In-AWS-Infrastructure-SEP/python/package/config.py

@@ -19,5 +19,11 @@ def __init__(self):
         #  should match the one used for staging the render queue and usage based licensing recipes.
         self.deadline_client_linux_ami_map: Mapping[str, str] = {'us-west-2': 'ami-0c8431fc72742c110'}
 
+        # Whether the DeadlineResourceTracker stack and supporting resources already exist in the account/region you are deploying to.
+        #
+        # If this is false, resources required by the Deadline Resource Tracker will be deployed into your account.
+        # If this is true, these resources will be skipped.
+        self.deadline_resource_tracker_exists: bool = False
+
 
 config: AppConfig = AppConfig()

examples/deadline/All-In-AWS-Infrastructure-SEP/python/package/lib/sep_stack.py

@@ -60,6 +60,8 @@ class SEPStackProps(StackProps):
     docker_recipes_stage_path: str
     # The IMachineImage to use for Workers (needs Deadline Client installed).
     worker_machine_image: IMachineImage
+    # Whether the DeadlineResourceTracker stack and supporting resources already exist or not.
+    deadline_resource_tracker_exists: bool
 
 
 class SEPStack(Stack):
@@ -159,15 +161,15 @@ def __init__(self, scope: Construct, stack_id: str, *, props: SEPStackProps, **k
             ),
         )
 
-        # Creates the Resource Tracker Access role. This role is required to exist in your account so the resource tracker will work properly
-        # Note: If you already have a Resource Tracker IAM role in your account you can remove this code.
-        Role(
-            self,
-            'ResourceTrackerRole',
-            assumed_by=ServicePrincipal('lambda.amazonaws.com'),
-            managed_policies= [ManagedPolicy.from_aws_managed_policy_name('AWSThinkboxDeadlineResourceTrackerAccessPolicy')],
-            role_name= 'DeadlineResourceTrackerAccessRole',
-        )
+        if not props.deadline_resource_tracker_exists:
+            # Creates the Resource Tracker Access role. This role is required to exist in your account so the resource tracker will work properly
+            Role(
+                self,
+                'ResourceTrackerRole',
+                assumed_by=ServicePrincipal('lambda.amazonaws.com'),
+                managed_policies= [ManagedPolicy.from_aws_managed_policy_name('AWSThinkboxDeadlineResourceTrackerAccessPolicy')],
+                role_name= 'DeadlineResourceTrackerAccessRole',
+            )
 
         fleet = SpotEventPluginFleet(
             self,

examples/deadline/All-In-AWS-Infrastructure-SEP/ts/bin/config.ts

@@ -16,6 +16,16 @@ class AppConfig {
    * is filled in. It can be used as-is, added to, or replaced.
    */
   public readonly deadlineClientLinuxAmiMap: Record<string, string> = {['us-west-2']: 'ami-0c8431fc72742c110'};
+
+  /**
+   * Whether the DeadlineResourceTracker stack and supporting resources already exist in the account/region you are deploying to.
+   *
+   * If this is false, resources required by the Deadline Resource Tracker will be deployed into your account.
+   * If this is true, these resources will be skipped.
+   *
+   * @default false
+   */
+  public readonly deadlineResourceTrackerExists: boolean = false;
 }
 
 export const config = new AppConfig();

examples/deadline/All-In-AWS-Infrastructure-SEP/ts/lib/sep-stack.ts

@@ -34,6 +34,7 @@ import {
   ThinkboxDockerRecipes,
 } from 'aws-rfdk/deadline';
 import { X509CertificatePem } from 'aws-rfdk';
+import { config } from '../bin/config';
 
 /**
  * Properties for {@link SEPStack}.
@@ -126,15 +127,16 @@ export class SEPStack extends Stack {
       trafficEncryption,
     });
 
-    // Creates the Resource Tracker Access role. This role is required to exist in your account so the resource tracker will work properly
-    // Note: If you already have a Resource Tracker IAM role in your account you can remove this code.
-    new Role(this, 'ResourceTrackerRole', {
-      assumedBy: new ServicePrincipal('lambda.amazonaws.com'),
-      managedPolicies: [
-        ManagedPolicy.fromAwsManagedPolicyName('AWSThinkboxDeadlineResourceTrackerAccessPolicy'),
-      ],
-      roleName: 'DeadlineResourceTrackerAccessRole',
-    });
+    if (!config.deadlineResourceTrackerExists) {
+      // Creates the Resource Tracker Access role. This role is required to exist in your account so the resource tracker will work properly
+      new Role(this, 'ResourceTrackerRole', {
+        assumedBy: new ServicePrincipal('lambda.amazonaws.com'),
+        managedPolicies: [
+          ManagedPolicy.fromAwsManagedPolicyName('AWSThinkboxDeadlineResourceTrackerAccessPolicy'),
+        ],
+        roleName: 'DeadlineResourceTrackerAccessRole',
+      });
+    }
 
     const fleet = new SpotEventPluginFleet(this, 'SpotEventPluginFleet', {
       vpc,

packages/aws-rfdk/lib/deadline/lib/configure-spot-event-plugin.ts

@@ -5,12 +5,9 @@
 
 import * as path from 'path';
 
-import {
-  BlockDevice,
-  BlockDeviceVolume,
-} from '@aws-cdk/aws-autoscaling';
 import {
   IVpc,
+  LaunchTemplate,
   SubnetSelection,
   SubnetType,
 } from '@aws-cdk/aws-ec2';
@@ -26,24 +23,22 @@ import {
 } from '@aws-cdk/aws-lambda';
 import { RetentionDays } from '@aws-cdk/aws-logs';
 import {
+  Aspects,
   Construct,
   CustomResource,
   Duration,
-  Fn,
   IResolvable,
   Lazy,
   Stack,
+  Tag,
+  Tags,
 } from '@aws-cdk/core';
 import {
   PluginSettings,
   SEPConfiguratorResourceProps,
-  LaunchSpecification,
   SpotFleetRequestConfiguration,
   SpotFleetRequestProps,
-  SpotFleetSecurityGroupId,
   SpotFleetTagSpecification,
-  BlockDeviceMappingProperty,
-  BlockDeviceProperty,
 } from '../../lambdas/nodejs/configure-spot-event-plugin';
 import { IRenderQueue, RenderQueue } from './render-queue';
 import { SpotEventPluginFleet } from './spot-event-plugin-fleet';
@@ -335,6 +330,9 @@ export interface ConfigureSpotEventPluginProps {
  *  - A policy to pass a fleet and instance role
  *  - A policy to create tags for spot fleet requests
  *
+ * The Spot Fleet Requests that this construct configures Deadline to create will always use the latest version of the
+ * corresponding EC2 Launch Template that was created for them.
+ *
  * ![architecture diagram](/diagrams/deadline/ConfigureSpotEventPlugin.svg)
  *
  * Resources Deployed
@@ -343,6 +341,7 @@ export interface ConfigureSpotEventPluginProps {
  * - A CloudFormation Custom Resource that triggers execution of the Lambda on stack deployment, update, and deletion.
  * - An Amazon CloudWatch log group that records history of the AWS Lambda's execution.
  * - An IAM Policy attached to Render Queue's Role.
+ * - EC2 Launch Templates for each Spot Event Plugin fleet.
  *
  * Security Considerations
  * ------------------------
@@ -522,51 +521,52 @@ export class ConfigureSpotEventPlugin extends Construct {
    * Each congiguration is a mapping between one Deadline Group and one Spot Fleet Request Configuration.
    */
   private generateSpotFleetRequestConfig(fleet: SpotEventPluginFleet): SpotFleetRequestConfiguration[] {
-    const securityGroupsToken = Lazy.any({ produce: () => {
-      return fleet.securityGroups.map(sg => {
-        const securityGroupId: SpotFleetSecurityGroupId = {
-          GroupId: sg.securityGroupId,
-        };
-        return securityGroupId;
-      });
-    }});
-
-    const userDataToken = Lazy.string({ produce: () => Fn.base64(fleet.userData.render()) });
-
-    const blockDeviceMappings = (fleet.blockDevices !== undefined ?
-      this.synthesizeBlockDeviceMappings(fleet.blockDevices) : undefined);
-
     const { subnetIds } = fleet.subnets;
     const subnetId = subnetIds.join(',');
 
-    const instanceTagsToken = this.tagSpecifications(fleet, SpotFleetResourceType.INSTANCE);
     const spotFleetRequestTagsToken = this.tagSpecifications(fleet, SpotFleetResourceType.SPOT_FLEET_REQUEST);
 
-    const launchSpecifications: LaunchSpecification[] = [];
-
-    fleet.instanceTypes.map(instanceType => {
-      const launchSpecification: LaunchSpecification = {
-        BlockDeviceMappings: blockDeviceMappings,
-        IamInstanceProfile: {
-          Arn: fleet.instanceProfile.attrArn,
-        },
-        ImageId: fleet.imageId,
-        KeyName: fleet.keyName,
-        // Need to convert from IResolvable to bypass TypeScript
-        SecurityGroups: (securityGroupsToken  as unknown) as SpotFleetSecurityGroupId[],
-        SubnetId: subnetId,
-        // Need to convert from IResolvable to bypass TypeScript
-        TagSpecifications: (instanceTagsToken as unknown) as SpotFleetTagSpecification[],
-        UserData: userDataToken,
-        InstanceType: instanceType.toString(),
-      };
-      launchSpecifications.push(launchSpecification);
+    const launchTemplates: LaunchTemplate[] = fleet.instanceTypes.map((instanceType, idx) => {
+      const launchTemplate = new LaunchTemplate(this, `${fleet.node.id}${instanceType}LaunchTemplate${idx}`, {
+        blockDevices: fleet.blockDevices,
+        role: fleet.fleetInstanceRole,
+        machineImage: fleet.machineImage,
+        keyName: fleet.keyName,
+        securityGroup: fleet.securityGroups[0],
+        userData: fleet.userData,
+        instanceType,
+      });
+      if (fleet.securityGroups.length > 1) {
+        launchTemplate.connections.addSecurityGroup(...fleet.securityGroups.slice(1));
+      }
+
+      // Add tags that are added via the TagManager API
+      /* istanbul ignore if */
+      if (fleet.tags.hasTags()) {
+        const tags = fleet.tags.renderTags() as {[key: string]: string}[];
+        tags.forEach(tag => Tags.of(launchTemplate).add(tag.Key, tag.Value));
+      }
+
+      // Add tags that are added via the Tags API which uses Aspects and is a layer above the TagManager API
+      const tagAspects = Aspects.of(fleet).aspects.filter(aspect => aspect instanceof Tag) as Tag[];
+      tagAspects.forEach(tagAspect => Tags.of(launchTemplate).add(tagAspect.key, tagAspect.value));
+
+      return launchTemplate;
     });
 
     const spotFleetRequestProps: SpotFleetRequestProps = {
       AllocationStrategy: fleet.allocationStrategy,
       IamFleetRole: fleet.fleetRole.roleArn,
-      LaunchSpecifications: launchSpecifications,
+      LaunchTemplateConfigs: launchTemplates.map(launchTemplate => {
+        return {
+          LaunchTemplateSpecification: {
+            Version: '$Latest',
+            LaunchTemplateId: launchTemplate.launchTemplateId,
+            LaunchTemplateName: launchTemplate.launchTemplateName,
+          },
+          Overrides: [{ SubnetId: subnetId }],
+        };
+      }),
       ReplaceUnhealthyInstances: true,
       // In order to work with Deadline, the 'Target Capacity' of the Spot fleet Request is
       // the maximum number of Workers that Deadline will start.
@@ -589,49 +589,6 @@ export class ConfigureSpotEventPlugin extends Construct {
     return spotFleetRequestConfigurations;
   }
 
-  /**
-   * Synthesize an array of block device mappings from a list of block devices
-   *
-   * @param blockDevices list of block devices
-   */
-  private synthesizeBlockDeviceMappings(blockDevices: BlockDevice[]): BlockDeviceMappingProperty[] {
-    return blockDevices.map(({ deviceName, volume, mappingEnabled }) => {
-      const { virtualName, ebsDevice: ebs } = volume;
-
-      if (volume === BlockDeviceVolume._NO_DEVICE || mappingEnabled === false) {
-        return {
-          DeviceName: deviceName,
-          // To omit the device from the block device mapping, specify an empty string.
-          // See https://docs.aws.amazon.com/cli/latest/reference/ec2/request-spot-fleet.html
-          NoDevice: '',
-        };
-      }
-
-      let Ebs: BlockDeviceProperty | undefined;
-
-      if (ebs) {
-        const { iops, volumeType, volumeSize, snapshotId, deleteOnTermination } = ebs;
-
-        Ebs = {
-          DeleteOnTermination: deleteOnTermination,
-          Iops: iops,
-          SnapshotId: snapshotId,
-          VolumeSize: volumeSize,
-          VolumeType: volumeType,
-          // encrypted is not exposed as part of ebsDeviceProps so we need to access it via [].
-          // eslint-disable-next-line dot-notation
-          Encrypted: 'encrypted' in ebs ? ebs['encrypted'] : undefined,
-        };
-      }
-
-      return {
-        DeviceName: deviceName,
-        Ebs,
-        VirtualName: virtualName,
-      };
-    });
-  }
-
   private mergeSpotFleetRequestConfigs(spotFleets?: SpotEventPluginFleet[]): SpotFleetRequestConfiguration | undefined {
     if (!spotFleets || spotFleets.length === 0) {
       return undefined;

packages/aws-rfdk/lib/deadline/lib/spot-event-plugin-fleet.ts

@@ -349,6 +349,11 @@ export class SpotEventPluginFleet extends Construct implements ISpotEventPluginF
    */
   public readonly imageId: string;
 
+  /**
+   * The Worker AMI.
+   */
+  public readonly machineImage: IMachineImage;
+
   /**
    * The tags to apply during creation of instances and of the Spot Fleet Request.
    */
@@ -457,6 +462,7 @@ export class SpotEventPluginFleet extends Construct implements ISpotEventPluginF
     this.osType = imageConfig.osType;
     this.userData = props.userData ?? imageConfig.userData;
     this.imageId = imageConfig.imageId;
+    this.machineImage = props.workerMachineImage;
 
     const workerConfig = new WorkerInstanceConfiguration(this, id, {
       worker: this,