Skip to content

Commit

Permalink
feat(stepfunctions-tasks): add AutoTerminationPolicy to `EmrCreateC…
Browse files Browse the repository at this point in the history 
…luster` (#16976)

Auto-termination is supported with Amazon EMR versions 5.30.0 and 6.1.0 and later.
This change enables CDK users to specify an auto-termination policy on EMR clusters created through the aws-stepfunctions task EmrCreateCluster.

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
  • Loading branch information
@jacobfi
jacobfi authored Nov 9, 2021
1 parent 2627939 commit 27ad7d8
Show file tree
Hide file tree
Showing 4 changed files with 100 additions and 38 deletions.
14 changes: 14 additions & 0 deletions packages/@aws-cdk/aws-stepfunctions-tasks/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -700,6 +700,20 @@ new tasks.EmrCreateCluster(this, 'Create Cluster', {
});
```

If you want to use an [auto-termination policy](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-auto-termination-policy.html),
you can specify the `autoTerminationPolicy` property. Set the `idleTimeout` as a `Duration` between 60 seconds and 7 days.
`autoTerminationPolicy` requires the EMR release label to be 5.30.0 or above.

```ts
new tasks.EmrCreateCluster(this, 'Create Cluster', {
instances: {},
name: 'ClusterName',
autoTerminationPolicy: {
idleTimeout: Duration.seconds(120),
},
});
```

### Termination Protection

Locks a cluster (job flow) so the EC2 instances in the cluster cannot be
Expand Down
25 changes: 25 additions & 0 deletions packages/@aws-cdk/aws-stepfunctions-tasks/lib/emr/emr-create-cluster.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { Construct } from 'constructs';
import { integrationResourceArn, validatePatternSupported } from '../private/task-utils';
import {
ApplicationConfigPropertyToJson,
AutoTerminationPolicyPropertyToJson,
BootstrapActionConfigToJson,
ConfigurationPropertyToJson,
InstancesConfigPropertyToJson,
Expand Down Expand Up @@ -67,6 +68,15 @@ export interface EmrCreateClusterProps extends sfn.TaskStateBaseProps {
*/
readonly autoScalingRole?: iam.IRole;

/**
* An auto-termination policy for an Amazon EMR cluster. An auto-termination policy defines the amount of
* idle time in seconds after which a cluster automatically terminates. The value must be between
* 60 seconds and 7 days.
*
* @default - None
*/
readonly autoTerminationPolicy?: EmrCreateCluster.AutoTerminationPolicyProperty;

/**
* A list of bootstrap actions to run before Hadoop starts on the cluster nodes.
*
Expand Down Expand Up @@ -269,6 +279,7 @@ export class EmrCreateCluster extends sfn.TaskStateBase {
AdditionalInfo: cdk.stringToCloudFormation(this.props.additionalInfo),
Applications: cdk.listMapper(ApplicationConfigPropertyToJson)(this.props.applications),
AutoScalingRole: cdk.stringToCloudFormation(this._autoScalingRole?.roleName),
AutoTerminationPolicy: this.props.autoTerminationPolicy ? AutoTerminationPolicyPropertyToJson(this.props.autoTerminationPolicy) : undefined,
BootstrapActions: cdk.listMapper(BootstrapActionConfigToJson)(this.props.bootstrapActions),
Configurations: cdk.listMapper(ConfigurationPropertyToJson)(this.props.configurations),
CustomAmiId: cdk.stringToCloudFormation(this.props.customAmiId),
Expand Down Expand Up @@ -1389,6 +1400,20 @@ export namespace EmrCreateCluster {
readonly version?: string;
}

/**
* Auto-termination policy for the EMR cluster.
*
* @see https://docs.aws.amazon.com/emr/latest/APIReference/API_AutoTerminationPolicy.html
*
*/
export interface AutoTerminationPolicyProperty {

/**
* Specifies the amount of idle time after which the cluster automatically terminates.
*/
readonly idleTimeout: cdk.Duration;
}

/**
* Configuration of the script to run during a bootstrap action.
*
Expand Down
47 changes: 9 additions & 38 deletions packages/@aws-cdk/aws-stepfunctions-tasks/lib/emr/private/cluster-utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ import { EmrModifyInstanceGroupByName } from '../emr-modify-instance-group-by-na

/**
* Render the KerberosAttributesProperty as JSON
*
* @param property
*/
export function KerberosAttributesPropertyToJson(property: EmrCreateCluster.KerberosAttributesProperty) {
return {
Expand All @@ -19,8 +17,6 @@ export function KerberosAttributesPropertyToJson(property: EmrCreateCluster.Kerb

/**
* Render the InstancesConfigProperty to JSON
*
* @param property
*/
export function InstancesConfigPropertyToJson(property: EmrCreateCluster.InstancesConfigProperty) {
return {
Expand All @@ -46,8 +42,6 @@ export function InstancesConfigPropertyToJson(property: EmrCreateCluster.Instanc

/**
* Render the ApplicationConfigProperty as JSON
*
* @param property
*/
export function ApplicationConfigPropertyToJson(property: EmrCreateCluster.ApplicationConfigProperty) {
return {
Expand All @@ -58,10 +52,17 @@ export function ApplicationConfigPropertyToJson(property: EmrCreateCluster.Appli
};
}

/**
* Render the AutoTerminationPolicyProperty as JSON
*/
export function AutoTerminationPolicyPropertyToJson(property: EmrCreateCluster.AutoTerminationPolicyProperty) {
return {
IdleTimeout: cdk.numberToCloudFormation(property.idleTimeout.toSeconds()),
};
}

/**
* Render the ConfigurationProperty as JSON
*
* @param property
*/
export function ConfigurationPropertyToJson(property: EmrCreateCluster.ConfigurationProperty) {
return {
Expand All @@ -73,8 +74,6 @@ export function ConfigurationPropertyToJson(property: EmrCreateCluster.Configura

/**
* Render the EbsBlockDeviceConfigProperty as JSON
*
* @param property
*/
export function EbsBlockDeviceConfigPropertyToJson(property: EmrCreateCluster.EbsBlockDeviceConfigProperty) {
return {
Expand All @@ -89,8 +88,6 @@ export function EbsBlockDeviceConfigPropertyToJson(property: EmrCreateCluster.Eb

/**
* Render the EbsConfigurationProperty to JSON
*
* @param property
*/
export function EbsConfigurationPropertyToJson(property: EmrCreateCluster.EbsConfigurationProperty) {
return {
Expand All @@ -117,8 +114,6 @@ export function InstanceTypeConfigPropertyToJson(property: EmrCreateCluster.Inst

/**
* Render the InstanceFleetProvisioningSpecificationsProperty to JSON
*
* @param property
*/
export function InstanceFleetProvisioningSpecificationsPropertyToJson(property: EmrCreateCluster.InstanceFleetProvisioningSpecificationsProperty) {
return {
Expand All @@ -133,8 +128,6 @@ export function InstanceFleetProvisioningSpecificationsPropertyToJson(property:

/**
* Render the InstanceFleetConfigProperty as JSON
*
* @param property
*/
export function InstanceFleetConfigPropertyToJson(property: EmrCreateCluster.InstanceFleetConfigProperty) {
return {
Expand All @@ -152,8 +145,6 @@ export function InstanceFleetConfigPropertyToJson(property: EmrCreateCluster.Ins

/**
* Render the MetricDimensionProperty as JSON
*
* @param property
*/
export function MetricDimensionPropertyToJson(property: EmrCreateCluster.MetricDimensionProperty) {
return {
Expand All @@ -164,8 +155,6 @@ export function MetricDimensionPropertyToJson(property: EmrCreateCluster.MetricD

/**
* Render the ScalingTriggerProperty to JSON
*
* @param property
*/
export function ScalingTriggerPropertyToJson(property: EmrCreateCluster.ScalingTriggerProperty) {
return {
Expand All @@ -185,8 +174,6 @@ export function ScalingTriggerPropertyToJson(property: EmrCreateCluster.ScalingT

/**
* Render the ScalingActionProperty to JSON
*
* @param property
*/
export function ScalingActionPropertyToJson(property: EmrCreateCluster.ScalingActionProperty) {
return {
Expand All @@ -201,8 +188,6 @@ export function ScalingActionPropertyToJson(property: EmrCreateCluster.ScalingAc

/**
* Render the ScalingRuleProperty to JSON
*
* @param property
*/
export function ScalingRulePropertyToJson(property: EmrCreateCluster.ScalingRuleProperty) {
return {
Expand All @@ -215,8 +200,6 @@ export function ScalingRulePropertyToJson(property: EmrCreateCluster.ScalingRule

/**
* Render the AutoScalingPolicyProperty to JSON
*
* @param property
*/
export function AutoScalingPolicyPropertyToJson(property: EmrCreateCluster.AutoScalingPolicyProperty) {
return {
Expand All @@ -230,8 +213,6 @@ export function AutoScalingPolicyPropertyToJson(property: EmrCreateCluster.AutoS

/**
* Render the InstanceGroupConfigProperty to JSON
*
* @param property
*/
export function InstanceGroupConfigPropertyToJson(property: EmrCreateCluster.InstanceGroupConfigProperty) {
return {
Expand All @@ -250,8 +231,6 @@ export function InstanceGroupConfigPropertyToJson(property: EmrCreateCluster.Ins

/**
* Render the PlacementTypeProperty to JSON
*
* @param property
*/
export function PlacementTypePropertyToJson(property: EmrCreateCluster.PlacementTypeProperty) {
return {
Expand All @@ -262,8 +241,6 @@ export function PlacementTypePropertyToJson(property: EmrCreateCluster.Placement

/**
* Render the BootstrapActionProperty as JSON
*
* @param property
*/
export function BootstrapActionConfigToJson(property: EmrCreateCluster.BootstrapActionConfigProperty) {
return {
Expand All @@ -277,8 +254,6 @@ export function BootstrapActionConfigToJson(property: EmrCreateCluster.Bootstrap

/**
* Render the InstanceGroupModifyConfigProperty to JSON
*
* @param property
*/
export function InstanceGroupModifyConfigPropertyToJson(property: EmrModifyInstanceGroupByName.InstanceGroupModifyConfigProperty) {
return {
Expand All @@ -291,8 +266,6 @@ export function InstanceGroupModifyConfigPropertyToJson(property: EmrModifyInsta

/**
* Render the ShrinkPolicyProperty to JSON
*
* @param property
*/
function ShrinkPolicyPropertyToJson(property: EmrModifyInstanceGroupByName.ShrinkPolicyProperty) {
return {
Expand All @@ -303,8 +276,6 @@ function ShrinkPolicyPropertyToJson(property: EmrModifyInstanceGroupByName.Shrin

/**
* Render the InstanceResizePolicyProperty to JSON
*
* @param property
*/
function InstanceResizePolicyPropertyToJson(property: EmrModifyInstanceGroupByName.InstanceResizePolicyProperty) {
return {
Expand Down
52 changes: 52 additions & 0 deletions packages/@aws-cdk/aws-stepfunctions-tasks/test/emr/emr-create-cluster.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -864,6 +864,58 @@ test('Create Cluster with InstanceFleet with allocation strategy=capacity-optimi
});
});

test('Create Cluster with AutoTerminationPolicy', () => {
// WHEN
const task = new EmrCreateCluster(stack, 'Task', {
instances: {},
clusterRole,
name: 'Cluster',
serviceRole,
autoScalingRole,
autoTerminationPolicy: {
idleTimeout: cdk.Duration.seconds(120),
},
integrationPattern: sfn.IntegrationPattern.REQUEST_RESPONSE,
});

// THEN
expect(stack.resolve(task.toStateJson())).toEqual({
Type: 'Task',
Resource: {
'Fn::Join': [
'',
[
'arn:',
{
Ref: 'AWS::Partition',
},
':states:::elasticmapreduce:createCluster',
],
],
},
End: true,
Parameters: {
Name: 'Cluster',
Instances: {
KeepJobFlowAliveWhenNoSteps: true,
},
VisibleToAllUsers: true,
JobFlowRole: {
Ref: 'ClusterRoleD9CA7471',
},
ServiceRole: {
Ref: 'ServiceRole4288B192',
},
AutoScalingRole: {
Ref: 'AutoScalingRole015ADA0A',
},
AutoTerminationPolicy: {
IdleTimeout: 120,
},
},
});
});

test('Create Cluster with InstanceFleet', () => {
// WHEN
const task = new EmrCreateCluster(stack, 'Task', {
Expand Down

0 comments on commit 27ad7d8

Please sign in to comment.