feat(stepfunctions-tasks): support allocation strategies in EMR Creat…

…eCluster (aws#16296) fixes aws#16252 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
TikiTDO · Sep 6, 2021 · 8e7b717 · 8e7b717
1 parent 7a0ac89
commit 8e7b717
Show file tree

Hide file tree

Showing 3 changed files with 141 additions and 2 deletions.
diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emr/emr-create-cluster.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emr/emr-create-cluster.ts
@@ -544,7 +544,7 @@ export namespace EmrCreateCluster {
    *
    */
   export enum SpotTimeoutAction {
-    /**\
+    /**
      * SWITCH_TO_ON_DEMAND
      */
     SWITCH_TO_ON_DEMAND = 'SWITCH_TO_ON_DEMAND',
@@ -554,17 +554,38 @@ export namespace EmrCreateCluster {
     TERMINATE_CLUSTER = 'TERMINATE_CLUSTER',
   }
 
+  /**
+   * Spot Allocation Strategies
+   *
+   * Specifies the strategy to use in launching Spot Instance fleets. For example, "capacity-optimized" launches instances from Spot Instance pools with optimal capacity for the number of instances that are launching.
+   *
+   * @see https://docs.aws.amazon.com/emr/latest/APIReference/API_SpotProvisioningSpecification.html
+   *
+   */
+  export enum SpotAllocationStrategy {
+    /**
+     * Capacity-optimized, which launches instances from Spot Instance pools with optimal capacity for the number of instances that are launching.
+     */
+    CAPACITY_OPTIMIZED = 'capacity-optimized',
+  }
+
   /**
    * The launch specification for Spot instances in the instance fleet, which determines the defined duration and provisioning timeout behavior.
    *
    * @see https://docs.aws.amazon.com/emr/latest/APIReference/API_SpotProvisioningSpecification.html
    *
    */
   export interface SpotProvisioningSpecificationProperty {
+    /**
+     * Specifies the strategy to use in launching Spot Instance fleets.
+     *
+     * @default - No allocation strategy, i.e. spot instance type will be chosen based on current price only
+     */
+    readonly allocationStrategy?: SpotAllocationStrategy;
     /**
      * The defined duration for Spot instances (also known as Spot blocks) in minutes.
      *
-     * @default No blockDurationMinutes
+     * @default - No blockDurationMinutes
      */
     readonly blockDurationMinutes?: number;
 

diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emr/private/cluster-utils.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emr/private/cluster-utils.ts
@@ -123,6 +123,7 @@ export function InstanceTypeConfigPropertyToJson(property: EmrCreateCluster.Inst
 export function InstanceFleetProvisioningSpecificationsPropertyToJson(property: EmrCreateCluster.InstanceFleetProvisioningSpecificationsProperty) {
   return {
     SpotSpecification: {
+      AllocationStrategy: cdk.stringToCloudFormation(property.spotSpecification.allocationStrategy),
       BlockDurationMinutes: cdk.numberToCloudFormation(property.spotSpecification.blockDurationMinutes),
       TimeoutAction: cdk.stringToCloudFormation(property.spotSpecification.timeoutAction?.valueOf()),
       TimeoutDurationMinutes: cdk.numberToCloudFormation(property.spotSpecification.timeoutDurationMinutes),

diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/test/emr/emr-create-cluster.test.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/test/emr/emr-create-cluster.test.ts
@@ -641,6 +641,123 @@ test('Create Cluster with Instances configuration', () => {
   });
 });
 
+test('Create Cluster with InstanceFleet with allocation strategy=capacity-optimized', () => {
+  // WHEN
+  const task = new EmrCreateCluster(stack, 'Task', {
+    instances: {
+      instanceFleets: [{
+        instanceFleetType: EmrCreateCluster.InstanceRoleType.MASTER,
+        instanceTypeConfigs: [{
+          bidPrice: '1',
+          bidPriceAsPercentageOfOnDemandPrice: 1,
+          configurations: [{
+            classification: 'Classification',
+            properties: {
+              Key: 'Value',
+            },
+          }],
+          ebsConfiguration: {
+            ebsBlockDeviceConfigs: [{
+              volumeSpecification: {
+                iops: 1,
+                volumeSize: cdk.Size.gibibytes(1),
+                volumeType: EmrCreateCluster.EbsBlockDeviceVolumeType.STANDARD,
+              },
+              volumesPerInstance: 1,
+            }],
+            ebsOptimized: true,
+          },
+          instanceType: 'm5.xlarge',
+          weightedCapacity: 1,
+        }],
+        launchSpecifications: {
+          spotSpecification: {
+            allocationStrategy: EmrCreateCluster.SpotAllocationStrategy.CAPACITY_OPTIMIZED,
+            blockDurationMinutes: 1,
+            timeoutAction: EmrCreateCluster.SpotTimeoutAction.TERMINATE_CLUSTER,
+            timeoutDurationMinutes: 1,
+          },
+        },
+        name: 'Master',
+        targetOnDemandCapacity: 1,
+        targetSpotCapacity: 1,
+      }],
+    },
+    clusterRole,
+    name: 'Cluster',
+    serviceRole,
+    integrationPattern: sfn.IntegrationPattern.REQUEST_RESPONSE,
+  });
+
+  // THEN
+  expect(stack.resolve(task.toStateJson())).toEqual({
+    Type: 'Task',
+    Resource: {
+      'Fn::Join': [
+        '',
+        [
+          'arn:',
+          {
+            Ref: 'AWS::Partition',
+          },
+          ':states:::elasticmapreduce:createCluster',
+        ],
+      ],
+    },
+    End: true,
+    Parameters: {
+      Name: 'Cluster',
+      Instances: {
+        KeepJobFlowAliveWhenNoSteps: true,
+        InstanceFleets: [{
+          InstanceFleetType: 'MASTER',
+          InstanceTypeConfigs: [{
+            BidPrice: '1',
+            BidPriceAsPercentageOfOnDemandPrice: 1,
+            Configurations: [{
+              Classification: 'Classification',
+              Properties: {
+                Key: 'Value',
+              },
+            }],
+            EbsConfiguration: {
+              EbsBlockDeviceConfigs: [{
+                VolumeSpecification: {
+                  Iops: 1,
+                  SizeInGB: 1,
+                  VolumeType: 'standard',
+                },
+                VolumesPerInstance: 1,
+              }],
+              EbsOptimized: true,
+            },
+            InstanceType: 'm5.xlarge',
+            WeightedCapacity: 1,
+          }],
+          LaunchSpecifications: {
+            SpotSpecification: {
+              AllocationStrategy: 'capacity-optimized',
+              BlockDurationMinutes: 1,
+              TimeoutAction: 'TERMINATE_CLUSTER',
+              TimeoutDurationMinutes: 1,
+            },
+          },
+          Name: 'Master',
+          TargetOnDemandCapacity: 1,
+          TargetSpotCapacity: 1,
+        }],
+      },
+      VisibleToAllUsers: true,
+      JobFlowRole: {
+        Ref: 'ClusterRoleD9CA7471',
+      },
+      ServiceRole: {
+        Ref: 'ServiceRole4288B192',
+      },
+    },
+  });
+});
+
 test('Create Cluster with InstanceFleet', () => {
   // WHEN
   const task = new EmrCreateCluster(stack, 'Task', {