Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(aws-ec2): AutoScalingGroup rolling updates #595

Merged
merged 5 commits into from
Aug 22, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 39 additions & 12 deletions packages/@aws-cdk/assert/lib/assertions/have-resource.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,24 @@ import { StackInspector } from "../inspector";
* - An object, in which case its properties will be compared to those of the actual resource found
* - A callable, in which case it will be treated as a predicate that is applied to the Properties of the found resources.
*/
export function haveResource(resourceType: string, properties?: any): Assertion<StackInspector> {
return new HaveResourceAssertion(resourceType, properties);
export function haveResource(resourceType: string, properties?: any, comparison?: ResourcePart): Assertion<StackInspector> {
return new HaveResourceAssertion(resourceType, properties, comparison);
}

type PropertyPredicate = (props: any) => boolean;

class HaveResourceAssertion extends Assertion<StackInspector> {
private inspected: any[] = [];
private readonly part: ResourcePart;
private readonly predicate: PropertyPredicate;

constructor(private readonly resourceType: string,
private readonly properties?: any) {
private readonly properties?: any,
part?: ResourcePart) {
super();

this.predicate = typeof properties === 'function' ? properties : makeSuperObjectPredicate(properties);
this.part = part !== undefined ? part : ResourcePart.Properties;
}

public assertUsing(inspector: StackInspector): boolean {
Expand All @@ -27,16 +35,9 @@ class HaveResourceAssertion extends Assertion<StackInspector> {
if (resource.Type === this.resourceType) {
this.inspected.push(resource);

let matches: boolean;
if (typeof this.properties === 'function') {
// If 'properties' is a callable, invoke it
matches = this.properties(resource.Properties);
} else {
// Otherwise treat as property bag that we check superset of
matches = isSuperObject(resource.Properties, this.properties);
}
const propsToCheck = this.part === ResourcePart.Properties ? resource.Properties : resource;

if (matches) {
if (this.predicate(propsToCheck)) {
return true;
}
}
Expand All @@ -57,6 +58,15 @@ class HaveResourceAssertion extends Assertion<StackInspector> {
}
}

/**
* Make a predicate that checks property superset
*/
function makeSuperObjectPredicate(obj: any) {
return (resourceProps: any) => {
return isSuperObject(resourceProps, obj);
};
}

/**
* Return whether `superObj` is a super-object of `obj`.
*
Expand Down Expand Up @@ -89,3 +99,20 @@ export function isSuperObject(superObj: any, obj: any): boolean {
}
return superObj === obj;
}

/**
* What part of the resource to compare
*/
export enum ResourcePart {
/**
* Only compare the resource's properties
*/
Properties,

/**
* Check the entire CloudFormation config
*
* (including UpdateConfig, DependsOn, etc.)
*/
CompleteDefinition
}
272 changes: 272 additions & 0 deletions packages/@aws-cdk/aws-autoscaling/lib/auto-scaling-group.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,63 @@ export interface AutoScalingGroupProps {
* @default true
*/
allowAllOutbound?: boolean;

/**
* What to do when an AutoScalingGroup's instance configuration is changed
*
* This is applied when any of the settings on the ASG are changed that
* affect how the instances should be created (VPC, instance type, startup
* scripts, etc.). It indicates how the existing instances should be
* replaced with new instances matching the new config. By default, nothing
* is done and only new instances are launched with the new config.
*
* @default UpdateType.None
*/
updateType?: UpdateType;

/**
* Configuration for rolling updates
*
* Only used if updateType == UpdateType.RollingUpdate.
*/
rollingUpdateConfiguration?: RollingUpdateConfiguration;

/**
* Configuration for replacing updates.
*
* Only used if updateType == UpdateType.ReplacingUpdate. Specifies how
* many instances must signal success for the update to succeed.
*/
replacingUpdateMinSuccessfulInstancesPercent?: number;

/**
* If the ASG has scheduled actions, don't reset unchanged group sizes
*
* Only used if the ASG has scheduled actions (which may scale your ASG up
* or down regardless of cdk deployments). If true, the size of the group
* will only be reset if it has been changed in the CDK app. If false, the
* sizes will always be changed back to what they were in the CDK app
* on deployment.
*
* @default true
*/
ignoreUnmodifiedSizeProperties?: boolean;

/**
* How many ResourceSignal calls CloudFormation expects before the resource is considered created
*
* @default 1
*/
resourceSignalCount?: number;

/**
* The length of time to wait for the resourceSignalCount
*
* The maximum value is 43200 (12 hours).
*
* @default 300 (5 minutes)
*/
resourceSignalTimeoutSec?: number;
}

/**
Expand Down Expand Up @@ -136,6 +193,10 @@ export class AutoScalingGroup extends cdk.Construct implements ec2.IClassicLoadB
const maxSize = props.maxSize || 1;
const desiredCapacity = props.desiredCapacity || 1;

if (desiredCapacity < minSize || desiredCapacity > maxSize) {
throw new Error(`Should have minSize (${minSize}) <= desiredCapacity (${desiredCapacity}) <= maxSize (${maxSize})`);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Like!

}

const asgProps: cloudformation.AutoScalingGroupResourceProps = {
minSize: minSize.toString(),
maxSize: maxSize.toString(),
Expand All @@ -162,6 +223,8 @@ export class AutoScalingGroup extends cdk.Construct implements ec2.IClassicLoadB

this.autoScalingGroup = new cloudformation.AutoScalingGroupResource(this, 'ASG', asgProps);
this.osType = machineImage.os.type;

this.applyUpdatePolicies(props);
}

public attachToClassicLB(loadBalancer: ec2.ClassicLoadBalancer): void {
Expand All @@ -186,4 +249,213 @@ export class AutoScalingGroup extends cdk.Construct implements ec2.IClassicLoadB
public addToRolePolicy(statement: cdk.PolicyStatement) {
this.role.addToPolicy(statement);
}

/**
* Apply CloudFormation update policies for the AutoScalingGroup
*/
private applyUpdatePolicies(props: AutoScalingGroupProps) {
if (props.updateType === UpdateType.ReplacingUpdate) {
this.asgUpdatePolicy.autoScalingReplacingUpdate = { willReplace: true };

if (props.replacingUpdateMinSuccessfulInstancesPercent !== undefined) {
// Yes, this goes on CreationPolicy, not as a process parameter to ReplacingUpdate.
// It's a little confusing, but the docs seem to explicitly state it will only be used
// during the update?
//
// https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-attribute-creationpolicy.html
this.asgCreationPolicy.autoScalingCreationPolicy = {
Copy link
Contributor

@moofish32 moofish32 Aug 21, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there is another creation policy that we use to prevent false positives when userdata must complete successfully:

  "ResourceSignal" : {    
    "Count" : Integer,
    "Timeout" : String
  }

Should we support this variation in this PR, or another PR? I am actually running into this issue trying to port our bastion pattern that requires some userdata to associate an EIP for consistent A records.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minSuccessfulInstancesPercent: validatePercentage(props.replacingUpdateMinSuccessfulInstancesPercent)
};
}
} else if (props.updateType === UpdateType.RollingUpdate) {
this.asgUpdatePolicy.autoScalingRollingUpdate = renderRollingUpdateConfig(props.rollingUpdateConfiguration);
}

// undefined is treated as 'true'
if (props.ignoreUnmodifiedSizeProperties !== false) {
this.asgUpdatePolicy.autoScalingScheduledAction = { ignoreUnmodifiedGroupSizeProperties: true };
}

if (props.resourceSignalCount !== undefined || props.resourceSignalTimeoutSec !== undefined) {
this.asgCreationPolicy.resourceSignal = {
count: props.resourceSignalCount,
timeout: props.resourceSignalTimeoutSec !== undefined ? renderIsoDuration(props.resourceSignalTimeoutSec) : undefined,
};
}
}

/**
* Create and return the ASG update policy
*/
private get asgUpdatePolicy() {
if (this.autoScalingGroup.options.updatePolicy === undefined) {
this.autoScalingGroup.options.updatePolicy = {};
}
return this.autoScalingGroup.options.updatePolicy;
}

/**
* Create and return the ASG creation policy
*/
private get asgCreationPolicy() {
if (this.autoScalingGroup.options.creationPolicy === undefined) {
this.autoScalingGroup.options.creationPolicy = {};
}
return this.autoScalingGroup.options.creationPolicy;
}
}

/**
* The type of update to perform on instances in this AutoScalingGroup
*/
export enum UpdateType {
/**
* Don't do anything
*/
None = 'None',

/**
* Replace the entire AutoScalingGroup
*
* Builds a new AutoScalingGroup first, then delete the old one.
*/
ReplacingUpdate = 'Replace',

/**
* Replace the instances in the AutoScalingGroup.
*/
RollingUpdate = 'RollingUpdate',
}

/**
* Additional settings when a rolling update is selected
*/
export interface RollingUpdateConfiguration {
/**
* The maximum number of instances that AWS CloudFormation updates at once.
*
* @default 1
*/
maxBatchSize?: number;

/**
* The minimum number of instances that must be in service before more instances are replaced.
*
* This number affects the speed of the replacement.
*
* @default 0
*/
minInstancesInService?: number;

/**
* The percentage of instances that must signal success for an update to succeed.
*
* If an instance doesn't send a signal within the time specified in the
* pauseTime property, AWS CloudFormation assumes that the instance wasn't
* updated.
*
* This number affects the success of the replacement.
*
* If you specify this property, you must also enable the
* waitOnResourceSignals and pauseTime properties.
*
* @default 100
*/
minSuccessfulInstancesPercent?: number;

/**
* The pause time after making a change to a batch of instances.
*
* This is intended to give those instances time to start software applications.
*
* Specify PauseTime in the ISO8601 duration format (in the format
* PT#H#M#S, where each # is the number of hours, minutes, and seconds,
* respectively). The maximum PauseTime is one hour (PT1H).
*
* @default 300 if the waitOnResourceSignals property is true, otherwise 0
*/
pauseTimeSec?: number;

/**
* Specifies whether the Auto Scaling group waits on signals from new instances during an update.
*
* AWS CloudFormation must receive a signal from each new instance within
* the specified PauseTime before continuing the update.
*
* To have instances wait for an Elastic Load Balancing health check before
* they signal success, add a health-check verification by using the
* cfn-init helper script. For an example, see the verify_instance_health
* command in the Auto Scaling rolling updates sample template.
*
* @default true if you specified the minSuccessfulInstancesPercent property, false otherwise
*/
waitOnResourceSignals?: boolean;

/**
* Specifies the Auto Scaling processes to suspend during a stack update.
*
* Suspending processes prevents Auto Scaling from interfering with a stack
* update.
*
* @default HealthCheck, ReplaceUnhealthy, AZRebalance, AlarmNotification, ScheduledActions.
*/
suspendProcesses?: ScalingProcess[];
}

export enum ScalingProcess {
Launch = 'Launch',
Terminate = 'Terminate',
HealthCheck = 'HealthCheck',
ReplaceUnhealthy = 'ReplaceUnhealthy',
AZRebalance = 'AZRebalance',
AlarmNotification = 'AlarmNotification',
ScheduledActions = 'ScheduledActions',
AddToLoadBalancer = 'AddToLoadBalancer'
}

/**
* Render the rolling update configuration into the appropriate object
*/
function renderRollingUpdateConfig(config: RollingUpdateConfiguration = {}): cdk.AutoScalingRollingUpdate {
const waitOnResourceSignals = config.minSuccessfulInstancesPercent !== undefined ? true : false;
const pauseTimeSec = config.pauseTimeSec !== undefined ? config.pauseTimeSec : (waitOnResourceSignals ? 300 : 0);

return {
maxBatchSize: config.maxBatchSize,
minInstancesInService: config.minInstancesInService,
minSuccessfulInstancesPercent: validatePercentage(config.minSuccessfulInstancesPercent),
waitOnResourceSignals,
pauseTime: renderIsoDuration(pauseTimeSec),
suspendProcesses: config.suspendProcesses !== undefined ? config.suspendProcesses :
// Recommended list of processes to suspend from here:
// https://aws.amazon.com/premiumsupport/knowledge-center/auto-scaling-group-rolling-updates/
[ScalingProcess.HealthCheck, ScalingProcess.ReplaceUnhealthy, ScalingProcess.AZRebalance,
ScalingProcess.AlarmNotification, ScalingProcess.ScheduledActions],
};
}

/**
* Render a number of seconds to a PTnX string.
*/
function renderIsoDuration(seconds: number): string {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is PT<seconds>S not okay if the amount of seconds is large? I'm (very) mildly surprised you have to go normalize like that...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did not test, but I fully expect the validator on the other end to be strict about it.

const ret: string[] = [];

if (seconds >= 3600) {
ret.push(`${Math.floor(seconds / 3600)}H`);
seconds %= 3600;
}
if (seconds >= 60) {
ret.push(`${Math.floor(seconds / 60)}M`);
seconds %= 60;
}
if (seconds > 0) {
ret.push(`${seconds}S`);
}

return 'PT' + ret.join('');
}

function validatePercentage(x?: number): number | undefined {
if (x === undefined || (0 <= x && x <= 100)) { return x; }
throw new Error(`Expected: a percentage 0..100, got: ${x}`);
}
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,11 @@
"Ref": "VPCPrivateSubnet3Subnet3EDCD457"
}
]
},
"UpdatePolicy": {
"AutoScalingScheduledAction": {
"IgnoreUnmodifiedGroupSizeProperties": true
}
}
},
"LBSecurityGroup8A41EA2B": {
Expand Down
Loading