From e8c773311fff833d7cc923b070205f99aa18faf6 Mon Sep 17 00:00:00 2001 From: Radek Smogura Date: Wed, 2 Dec 2020 21:28:52 +0100 Subject: [PATCH 1/6] feat(ec2): introduce multipart user data Add support for multiparat (MIME) user data for Linux environments. This type is more versatile type of user data, and some AWS service (i.e. AWS Batch) requires it in order to customize the launch behaviour. Change was tested in integ environment to check if all user data parts has been executed correctly and with proper charset encoding. fixes #8315 --- packages/@aws-cdk/aws-ec2/lib/user-data.ts | 183 ++++- ....instance-multipart-userdata.expected.json | 694 ++++++++++++++++++ .../test/integ.instance-multipart-userdata.ts | 65 ++ .../@aws-cdk/aws-ec2/test/userdata.test.ts | 23 + 4 files changed, 963 insertions(+), 2 deletions(-) create mode 100644 packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.expected.json create mode 100644 packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.ts diff --git a/packages/@aws-cdk/aws-ec2/lib/user-data.ts b/packages/@aws-cdk/aws-ec2/lib/user-data.ts index 20061bd609636..416722b326989 100644 --- a/packages/@aws-cdk/aws-ec2/lib/user-data.ts +++ b/packages/@aws-cdk/aws-ec2/lib/user-data.ts @@ -1,5 +1,7 @@ +import * as crypto from 'crypto'; + import { IBucket } from '@aws-cdk/aws-s3'; -import { CfnElement, Resource, Stack } from '@aws-cdk/core'; +import { CfnElement, Fn, Resource, Stack } from '@aws-cdk/core'; import { OperatingSystemType } from './machine-image'; /** @@ -61,7 +63,7 @@ export interface ExecuteFileOptions { /** * Instance User Data */ -export abstract class UserData { +export abstract class UserData implements IMultipartUserDataPartProducer { /** * Create a userdata object for Linux hosts */ @@ -108,6 +110,13 @@ export abstract class UserData { */ public abstract render(): string; + /** + * Render the user data as a part for `MultipartUserData`. Not all subclasses supports this. + */ + public renderAsMimePart(_renderOpts?: MultipartRenderOptions): MutlipartUserDataPart { + throw new Error('This class does not support rendering as MIME part'); + } + /** * Adds commands to download a file from S3 * @@ -151,6 +160,15 @@ class LinuxUserData extends UserData { return [shebang, ...(this.renderOnExitLines()), ...this.lines].join('\n'); } + public renderAsMimePart(renderOpts?: MultipartRenderOptions): MutlipartUserDataPart { + const contentType = renderOpts?.contentType || 'text/x-shellscript'; + return new MutlipartUserDataPart({ + body: Fn.base64(this.render()), // Wrap into base64, to support UTF-8 encoding (actually decoding) + contentType: `${contentType}; charset="utf-8"`, + transferEncoding: 'base64', + }); + } + public addS3DownloadCommand(params: S3DownloadOptions): string { const s3Path = `s3://${params.bucket.bucketName}/${params.bucketKey}`; const localPath = ( params.localFile && params.localFile.length !== 0 ) ? params.localFile : `/tmp/${ params.bucketKey }`; @@ -276,3 +294,164 @@ class CustomUserData extends UserData { throw new Error('CustomUserData does not support addSignalOnExitCommand, use UserData.forLinux() or UserData.forWindows() instead.'); } } + +/** + * Options when creating `MutlipartUserDataPart`. + */ +export interface MutlipartUserDataPartOptions { + /** + * The body of message. + * + * @default undefined - body will not be added to part + */ + readonly body?: string, + + /** + * `Content-Type` header of this part. + * + * For Linux shell scripts use `text/x-shellscript` + */ + readonly contentType: string; + + /** + * `Content-Transfer-Encoding` header specifing part encoding. + * + * @default undefined - don't add this header + */ + readonly transferEncoding?: string; +} + +/** + * The raw part of multip-part user data, which can be added to {@link MultipartUserData}. + */ +export class MutlipartUserDataPart implements IMultipartUserDataPartProducer { + /** The body of this MIME part. */ + public readonly body?: string; + + /** `Content-Type` header of this part */ + public readonly contentType: string; + + /** + * `Content-Transfer-Encoding` header specifing part encoding. + * + * @default undefined - don't add this header + */ + readonly transferEncoding?: string; + + public constructor(props: MutlipartUserDataPartOptions) { + this.body = props.body; + this.contentType = props.contentType; + this.transferEncoding = props.transferEncoding; + } + + renderAsMimePart(_renderOpts?: MultipartRenderOptions): MutlipartUserDataPart { + return this; + } +} + +/** + * Render options for parts of multipart user data. + */ +export interface MultipartRenderOptions { + /** + * Can be used to override default content type (without charset part) used when producing + * part by `IMultipartUserDataPartProducer`. + * + * @default undefined - leave content type unchanged + */ + readonly contentType?: string; +} + +/** + * Class implementing this interface can produce `MutlipartUserDataPart` and can be added + * to `MultipartUserData`. + */ +export interface IMultipartUserDataPartProducer { + /** + * Creats the `MutlipartUserDataPart. + */ + renderAsMimePart(renderOpts?: MultipartRenderOptions): MutlipartUserDataPart; +} + +/** + * Mime multipart user data. + * + * This class represents MIME multipart user data, as described in. + * [Specifying Multiple User Data Blocks Using a MIME Multi Part Archive](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/bootstrap_container_instance.html#multi-part_user_data) + * + */ +export class MultipartUserData extends UserData { + private static readonly USE_PART_ERROR = 'MultipartUserData does not support this operation. Please add part using addPart.'; + + private parts: IMultipartUserDataPartProducer[] = []; + + /** + * Adds a class which can producer `MutlipartUserDataPart`. I. e. `UserData.forLinux()`. + */ + public addPart(producer: IMultipartUserDataPartProducer): this { + this.parts.push(producer); + + return this; + } + + public render(): string { + const renderedParts: MutlipartUserDataPart[] = this.parts.map(producer => producer.renderAsMimePart()); + + // Hash the message content, it will be used as boundry. The boundry should be + // so much unique not to be in message text, and stable so the text of archive will + // not be changed only due to change of boundry (may cause redeploys of resources) + const hash = crypto.createHash('sha256'); + renderedParts.forEach(part => { + hash + .update(part.contentType) + .update(part.body || 'empty-body') + .update(part.transferEncoding || ''); + }); + hash.update('salt-boundary-rado'); + + const boundary = '-' + hash.digest('base64') + '-'; + + // Now build final MIME archive - there are few changes from MIME message which are accepted by cloud-init: + // - MIME RFC uses CRLF to separarte lines - cloud-init is fine with LF \n only + var resultArchive = `Content-Type: multipart/mixed; boundary="${boundary}"\n`; + resultArchive = resultArchive + 'MIME-Version: 1.0\n'; + + // Add parts - each part starts with boundry + renderedParts.forEach(part => { + resultArchive = resultArchive + '\n--' + boundary + '\n' + 'Content-Type: ' + part.contentType + '\n'; + + if (part.transferEncoding != null) { + resultArchive = resultArchive + `Content-Transfer-Encoding: ${part.transferEncoding}\n`; + } + + if (part.body != null) { + resultArchive = resultArchive + '\n' + part.body; + } + }); + + // Add closing boundry + resultArchive = resultArchive + `\n--${boundary}--\n`; + + return resultArchive; + } + + public addS3DownloadCommand(_params: S3DownloadOptions): string { + throw new Error(MultipartUserData.USE_PART_ERROR); + } + + public addExecuteFileCommand(_params: ExecuteFileOptions): void { + throw new Error(MultipartUserData.USE_PART_ERROR); + } + + public addSignalOnExitCommand(_resource: Resource): void { + throw new Error(MultipartUserData.USE_PART_ERROR); + } + + public addCommands(..._commands: string[]): void { + throw new Error(MultipartUserData.USE_PART_ERROR); + } + + public addOnExitCommands(..._commands: string[]): void { + throw new Error(MultipartUserData.USE_PART_ERROR); + } +} diff --git a/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.expected.json b/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.expected.json new file mode 100644 index 0000000000000..f78b3762b75f4 --- /dev/null +++ b/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.expected.json @@ -0,0 +1,694 @@ +{ + "Resources": { + "VPCB9E5F0B4": { + "Type": "AWS::EC2::VPC", + "Properties": { + "CidrBlock": "10.0.0.0/16", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, + "InstanceTenancy": "default", + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC" + } + ] + } + }, + "VPCPublicSubnet1SubnetB4246D30": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.0.0/19", + "VpcId": { + "Ref": "VPCB9E5F0B4" + }, + "AvailabilityZone": "test-region-1a", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PublicSubnet1" + } + ] + } + }, + "VPCPublicSubnet1RouteTableFEE4B781": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "VPCB9E5F0B4" + }, + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PublicSubnet1" + } + ] + } + }, + "VPCPublicSubnet1RouteTableAssociation0B0896DC": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VPCPublicSubnet1RouteTableFEE4B781" + }, + "SubnetId": { + "Ref": "VPCPublicSubnet1SubnetB4246D30" + } + } + }, + "VPCPublicSubnet1DefaultRoute91CEF279": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VPCPublicSubnet1RouteTableFEE4B781" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "VPCIGWB7E252D3" + } + }, + "DependsOn": [ + "VPCVPCGW99B986DC" + ] + }, + "VPCPublicSubnet1EIP6AD938E8": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PublicSubnet1" + } + ] + } + }, + "VPCPublicSubnet1NATGatewayE0556630": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "AllocationId": { + "Fn::GetAtt": [ + "VPCPublicSubnet1EIP6AD938E8", + "AllocationId" + ] + }, + "SubnetId": { + "Ref": "VPCPublicSubnet1SubnetB4246D30" + }, + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PublicSubnet1" + } + ] + } + }, + "VPCPublicSubnet2Subnet74179F39": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.32.0/19", + "VpcId": { + "Ref": "VPCB9E5F0B4" + }, + "AvailabilityZone": "test-region-1b", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PublicSubnet2" + } + ] + } + }, + "VPCPublicSubnet2RouteTable6F1A15F1": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "VPCB9E5F0B4" + }, + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PublicSubnet2" + } + ] + } + }, + "VPCPublicSubnet2RouteTableAssociation5A808732": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VPCPublicSubnet2RouteTable6F1A15F1" + }, + "SubnetId": { + "Ref": "VPCPublicSubnet2Subnet74179F39" + } + } + }, + "VPCPublicSubnet2DefaultRouteB7481BBA": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VPCPublicSubnet2RouteTable6F1A15F1" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "VPCIGWB7E252D3" + } + }, + "DependsOn": [ + "VPCVPCGW99B986DC" + ] + }, + "VPCPublicSubnet2EIP4947BC00": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PublicSubnet2" + } + ] + } + }, + "VPCPublicSubnet2NATGateway3C070193": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "AllocationId": { + "Fn::GetAtt": [ + "VPCPublicSubnet2EIP4947BC00", + "AllocationId" + ] + }, + "SubnetId": { + "Ref": "VPCPublicSubnet2Subnet74179F39" + }, + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PublicSubnet2" + } + ] + } + }, + "VPCPublicSubnet3Subnet631C5E25": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.64.0/19", + "VpcId": { + "Ref": "VPCB9E5F0B4" + }, + "AvailabilityZone": "test-region-1c", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PublicSubnet3" + } + ] + } + }, + "VPCPublicSubnet3RouteTable98AE0E14": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "VPCB9E5F0B4" + }, + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PublicSubnet3" + } + ] + } + }, + "VPCPublicSubnet3RouteTableAssociation427FE0C6": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VPCPublicSubnet3RouteTable98AE0E14" + }, + "SubnetId": { + "Ref": "VPCPublicSubnet3Subnet631C5E25" + } + } + }, + "VPCPublicSubnet3DefaultRouteA0D29D46": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VPCPublicSubnet3RouteTable98AE0E14" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "VPCIGWB7E252D3" + } + }, + "DependsOn": [ + "VPCVPCGW99B986DC" + ] + }, + "VPCPublicSubnet3EIPAD4BC883": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PublicSubnet3" + } + ] + } + }, + "VPCPublicSubnet3NATGatewayD3048F5C": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "AllocationId": { + "Fn::GetAtt": [ + "VPCPublicSubnet3EIPAD4BC883", + "AllocationId" + ] + }, + "SubnetId": { + "Ref": "VPCPublicSubnet3Subnet631C5E25" + }, + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PublicSubnet3" + } + ] + } + }, + "VPCPrivateSubnet1Subnet8BCA10E0": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.96.0/19", + "VpcId": { + "Ref": "VPCB9E5F0B4" + }, + "AvailabilityZone": "test-region-1a", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PrivateSubnet1" + } + ] + } + }, + "VPCPrivateSubnet1RouteTableBE8A6027": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "VPCB9E5F0B4" + }, + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PrivateSubnet1" + } + ] + } + }, + "VPCPrivateSubnet1RouteTableAssociation347902D1": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VPCPrivateSubnet1RouteTableBE8A6027" + }, + "SubnetId": { + "Ref": "VPCPrivateSubnet1Subnet8BCA10E0" + } + } + }, + "VPCPrivateSubnet1DefaultRouteAE1D6490": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VPCPrivateSubnet1RouteTableBE8A6027" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "VPCPublicSubnet1NATGatewayE0556630" + } + } + }, + "VPCPrivateSubnet2SubnetCFCDAA7A": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.128.0/19", + "VpcId": { + "Ref": "VPCB9E5F0B4" + }, + "AvailabilityZone": "test-region-1b", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PrivateSubnet2" + } + ] + } + }, + "VPCPrivateSubnet2RouteTable0A19E10E": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "VPCB9E5F0B4" + }, + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PrivateSubnet2" + } + ] + } + }, + "VPCPrivateSubnet2RouteTableAssociation0C73D413": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VPCPrivateSubnet2RouteTable0A19E10E" + }, + "SubnetId": { + "Ref": "VPCPrivateSubnet2SubnetCFCDAA7A" + } + } + }, + "VPCPrivateSubnet2DefaultRouteF4F5CFD2": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VPCPrivateSubnet2RouteTable0A19E10E" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "VPCPublicSubnet2NATGateway3C070193" + } + } + }, + "VPCPrivateSubnet3Subnet3EDCD457": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.160.0/19", + "VpcId": { + "Ref": "VPCB9E5F0B4" + }, + "AvailabilityZone": "test-region-1c", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PrivateSubnet3" + } + ] + } + }, + "VPCPrivateSubnet3RouteTable192186F8": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "VPCB9E5F0B4" + }, + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC/PrivateSubnet3" + } + ] + } + }, + "VPCPrivateSubnet3RouteTableAssociationC28D144E": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VPCPrivateSubnet3RouteTable192186F8" + }, + "SubnetId": { + "Ref": "VPCPrivateSubnet3Subnet3EDCD457" + } + } + }, + "VPCPrivateSubnet3DefaultRoute27F311AE": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VPCPrivateSubnet3RouteTable192186F8" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "VPCPublicSubnet3NATGatewayD3048F5C" + } + } + }, + "VPCIGWB7E252D3": { + "Type": "AWS::EC2::InternetGateway", + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/VPC" + } + ] + } + }, + "VPCVPCGW99B986DC": { + "Type": "AWS::EC2::VPCGatewayAttachment", + "Properties": { + "VpcId": { + "Ref": "VPCB9E5F0B4" + }, + "InternetGatewayId": { + "Ref": "VPCIGWB7E252D3" + } + } + }, + "InstanceInstanceSecurityGroupF0E2D5BE": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "TestStackMultipartUserData/Instance/InstanceSecurityGroup", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "SecurityGroupIngress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "from 0.0.0.0/0:ICMP Type 8", + "FromPort": 8, + "IpProtocol": "icmp", + "ToPort": -1 + } + ], + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/Instance" + } + ], + "VpcId": { + "Ref": "VPCB9E5F0B4" + } + } + }, + "InstanceInstanceRoleE9785DE5": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": { + "Fn::Join": [ + "", + [ + "ec2.", + { + "Ref": "AWS::URLSuffix" + } + ] + ] + } + } + } + ], + "Version": "2012-10-17" + }, + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/Instance" + } + ] + } + }, + "InstanceInstanceRoleDefaultPolicy4ACE9290": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ssm:*", + "ssmmessages:*", + "ec2messages:GetMessages" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "InstanceInstanceRoleDefaultPolicy4ACE9290", + "Roles": [ + { + "Ref": "InstanceInstanceRoleE9785DE5" + } + ] + } + }, + "InstanceInstanceProfileAB5AEF02": { + "Type": "AWS::IAM::InstanceProfile", + "Properties": { + "Roles": [ + { + "Ref": "InstanceInstanceRoleE9785DE5" + } + ] + } + }, + "InstanceC1063A87": { + "Type": "AWS::EC2::Instance", + "Properties": { + "AvailabilityZone": "test-region-1a", + "IamInstanceProfile": { + "Ref": "InstanceInstanceProfileAB5AEF02" + }, + "ImageId": { + "Ref": "SsmParameterValueawsserviceamiamazonlinuxlatestamzn2amihvmx8664gp2C96584B6F00A464EAD1953AFF4B05118Parameter" + }, + "InstanceType": "t3.nano", + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "InstanceInstanceSecurityGroupF0E2D5BE", + "GroupId" + ] + } + ], + "SubnetId": { + "Ref": "VPCPrivateSubnet1Subnet8BCA10E0" + }, + "Tags": [ + { + "Key": "Name", + "Value": "TestStackMultipartUserData/Instance" + } + ], + "UserData": { + "Fn::Base64": { + "Fn::Join": [ + "", + [ + "Content-Type: multipart/mixed; boundary=\"-uU56KH4CuLBnf9GVhBq+SFx/hFRXS5S9umauYjYeBxQ=-\"\nMIME-Version: 1.0\n\n---uU56KH4CuLBnf9GVhBq+SFx/hFRXS5S9umauYjYeBxQ=-\nContent-Type: text/x-shellscript; charset=\"utf-8\"\nContent-Transfer-Encoding: base64\n\n", + { + "Fn::Base64": "#!/bin/bash\necho 大らと > /var/tmp/echo1\ncp /var/tmp/echo1 /var/tmp/echo1-copy" + }, + "\n---uU56KH4CuLBnf9GVhBq+SFx/hFRXS5S9umauYjYeBxQ=-\nContent-Type: text/x-shellscript; charset=\"utf-8\"\nContent-Transfer-Encoding: base64\n\n", + { + "Fn::Base64": { + "Fn::Join": [ + "", + [ + "#!/bin/bash\necho 大らと ", + { + "Ref": "VPCB9E5F0B4" + }, + " > /var/tmp/echo2" + ] + ] + } + }, + "\n---uU56KH4CuLBnf9GVhBq+SFx/hFRXS5S9umauYjYeBxQ=-\nContent-Type: text/cloud-boothook; charset=\"utf-8\"\nContent-Transfer-Encoding: base64\n\n", + { + "Fn::Base64": "#!/bin/bash\necho \"Boothook2\" > /var/tmp/boothook\ncloud-init-per once docker_options echo 'OPTIONS=\"${OPTIONS} --storage-opt dm.basesize=20G\"' >> /etc/sysconfig/docker" + }, + "\n---uU56KH4CuLBnf9GVhBq+SFx/hFRXS5S9umauYjYeBxQ=-\nContent-Type: text/x-shellscript\n\necho \"RawPart\" > /var/tmp/rawPart1\n---uU56KH4CuLBnf9GVhBq+SFx/hFRXS5S9umauYjYeBxQ=-\nContent-Type: text/x-shellscript\n\necho \"RawPart ", + { + "Ref": "VPCB9E5F0B4" + }, + "\" > /var/tmp/rawPart2\n---uU56KH4CuLBnf9GVhBq+SFx/hFRXS5S9umauYjYeBxQ=---\n" + ] + ] + } + } + }, + "DependsOn": [ + "InstanceInstanceRoleDefaultPolicy4ACE9290", + "InstanceInstanceRoleE9785DE5" + ] + } + }, + "Parameters": { + "SsmParameterValueawsserviceamiamazonlinuxlatestamzn2amihvmx8664gp2C96584B6F00A464EAD1953AFF4B05118Parameter": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.ts b/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.ts new file mode 100644 index 0000000000000..ed859cccc9d96 --- /dev/null +++ b/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.ts @@ -0,0 +1,65 @@ +/// !cdk-integ * +import { PolicyStatement } from '@aws-cdk/aws-iam'; +import * as cdk from '@aws-cdk/core'; +import * as ec2 from '../lib'; + +const app = new cdk.App(); + +class TestStack extends cdk.Stack { + constructor(scope: cdk.App, id: string, props?: cdk.StackProps) { + super(scope, id, props); + + const vpc = new ec2.Vpc(this, 'VPC'); + + const multipartUserData = new ec2.MultipartUserData(); + + const userData1 = ec2.UserData.forLinux(); + userData1.addCommands('echo 大らと > /var/tmp/echo1'); + userData1.addCommands('cp /var/tmp/echo1 /var/tmp/echo1-copy'); + + const userData2 = ec2.UserData.forLinux(); + userData2.addCommands(`echo 大らと ${vpc.vpcId} > /var/tmp/echo2`); + + const rawPart1 = new ec2.MutlipartUserDataPart({ + contentType: 'text/x-shellscript', + body: 'echo "RawPart" > /var/tmp/rawPart1', + }); + + const rawPart2 = new ec2.MutlipartUserDataPart({ + contentType: 'text/x-shellscript', + body: `echo "RawPart ${vpc.vpcId}" > /var/tmp/rawPart2`, + }); + + const bootHook = ec2.UserData.forLinux(); + bootHook.addCommands( + 'echo "Boothook2" > /var/tmp/boothook', + 'cloud-init-per once docker_options echo \'OPTIONS="${OPTIONS} --storage-opt dm.basesize=20G"\' >> /etc/sysconfig/docker', + ); + + multipartUserData.addPart(userData1); + multipartUserData.addPart(userData2); + multipartUserData.addPart(bootHook.renderAsMimePart({ + contentType: 'text/cloud-boothook', + })); + multipartUserData.addPart(rawPart1); + multipartUserData.addPart(rawPart2); + + const instance = new ec2.Instance(this, 'Instance', { + vpc, + instanceType: ec2.InstanceType.of(ec2.InstanceClass.T3, ec2.InstanceSize.NANO), + machineImage: new ec2.AmazonLinuxImage({ generation: ec2.AmazonLinuxGeneration.AMAZON_LINUX_2 }), + userData: multipartUserData, + }); + + instance.addToRolePolicy(new PolicyStatement({ + actions: ['ssm:*', 'ssmmessages:*', 'ec2messages:GetMessages'], + resources: ['*'], + })); + + instance.connections.allowFromAnyIpv4(ec2.Port.icmpPing()); + } +} + +new TestStack(app, 'TestStackMultipartUserData'); + +app.synth(); diff --git a/packages/@aws-cdk/aws-ec2/test/userdata.test.ts b/packages/@aws-cdk/aws-ec2/test/userdata.test.ts index 883794bd5c585..4469e303a9105 100644 --- a/packages/@aws-cdk/aws-ec2/test/userdata.test.ts +++ b/packages/@aws-cdk/aws-ec2/test/userdata.test.ts @@ -273,4 +273,27 @@ nodeunitShim({ test.done(); }, + 'Linux user rendering multipart headers'(test: Test) { + // GIVEN + const linuxUserData = ec2.UserData.forLinux(); + linuxUserData.addCommands('echo "Hello world"'); + + // WHEN + const defaultRender1 = linuxUserData.renderAsMimePart(); + const defaultRender2 = linuxUserData.renderAsMimePart({}); + const overriden = linuxUserData.renderAsMimePart({ + contentType: 'text/cloud-boothook', + }); + + // THEN + test.equals(defaultRender1.contentType, 'text/x-shellscript; charset=\"utf-8\"'); + test.equals(defaultRender2.contentType, 'text/x-shellscript; charset=\"utf-8\"'); + test.equals(overriden.contentType, 'text/cloud-boothook; charset=\"utf-8\"'); + + test.equals(defaultRender1.transferEncoding, 'base64'); + test.equals(defaultRender2.transferEncoding, 'base64'); + test.equals(overriden.transferEncoding, 'base64'); + + test.done(); + }, }); From b84fa3a589df9151db4e37a856102a653e3acfbc Mon Sep 17 00:00:00 2001 From: Radek Smogura Date: Thu, 21 Jan 2021 00:41:52 +0100 Subject: [PATCH 2/6] Replace interface approacg with adaptor * Remove `IMultipartUserDataPartProducer` * Add `MultipartUserDataPart` & `IMultipart` * Concrete types to represent raw part and UserData wrapper can be created with `MultipartUserDataPart.fromUserData` & `MultipartUserDataPart.fromRawBody` * Removed auto-generation of separator (as with tokens hash codes can differ when tokens are not resolved) --- packages/@aws-cdk/aws-ec2/lib/user-data.ts | 214 ++++++++++++------ ....instance-multipart-userdata.expected.json | 10 +- .../test/integ.instance-multipart-userdata.ts | 16 +- .../@aws-cdk/aws-ec2/test/userdata.test.ts | 13 +- 4 files changed, 165 insertions(+), 88 deletions(-) diff --git a/packages/@aws-cdk/aws-ec2/lib/user-data.ts b/packages/@aws-cdk/aws-ec2/lib/user-data.ts index 416722b326989..ef2bbe0986666 100644 --- a/packages/@aws-cdk/aws-ec2/lib/user-data.ts +++ b/packages/@aws-cdk/aws-ec2/lib/user-data.ts @@ -1,5 +1,3 @@ -import * as crypto from 'crypto'; - import { IBucket } from '@aws-cdk/aws-s3'; import { CfnElement, Fn, Resource, Stack } from '@aws-cdk/core'; import { OperatingSystemType } from './machine-image'; @@ -63,7 +61,7 @@ export interface ExecuteFileOptions { /** * Instance User Data */ -export abstract class UserData implements IMultipartUserDataPartProducer { +export abstract class UserData { /** * Create a userdata object for Linux hosts */ @@ -110,13 +108,6 @@ export abstract class UserData implements IMultipartUserDataPartProducer { */ public abstract render(): string; - /** - * Render the user data as a part for `MultipartUserData`. Not all subclasses supports this. - */ - public renderAsMimePart(_renderOpts?: MultipartRenderOptions): MutlipartUserDataPart { - throw new Error('This class does not support rendering as MIME part'); - } - /** * Adds commands to download a file from S3 * @@ -160,15 +151,6 @@ class LinuxUserData extends UserData { return [shebang, ...(this.renderOnExitLines()), ...this.lines].join('\n'); } - public renderAsMimePart(renderOpts?: MultipartRenderOptions): MutlipartUserDataPart { - const contentType = renderOpts?.contentType || 'text/x-shellscript'; - return new MutlipartUserDataPart({ - body: Fn.base64(this.render()), // Wrap into base64, to support UTF-8 encoding (actually decoding) - contentType: `${contentType}; charset="utf-8"`, - transferEncoding: 'base64', - }); - } - public addS3DownloadCommand(params: S3DownloadOptions): string { const s3Path = `s3://${params.bucket.bucketName}/${params.bucketKey}`; const localPath = ( params.localFile && params.localFile.length !== 0 ) ? params.localFile : `/tmp/${ params.bucketKey }`; @@ -296,25 +278,72 @@ class CustomUserData extends UserData { } /** - * Options when creating `MutlipartUserDataPart`. + * Suggested content types, however any value is allowed. + */ +export type MultipartContentType = 'text/x-shellscript; charset="utf-8"' | 'text/cloud-boothook; charset="utf-8"' | string; + +/** + * Options when creating `MultipartUserDataPart`. */ -export interface MutlipartUserDataPartOptions { +export interface MultipartUserDataPartOptions { + + /** + * `Content-Type` header of this part. + * + * For Linux shell scripts use `text/x-shellscript` + */ + readonly contentType: MultipartContentType; + + /** + * `Content-Transfer-Encoding` header specifying part encoding. + * + * @default undefined - don't add this header + */ + readonly transferEncoding?: string; +} + +/** + * Options when creating `MultipartUserDataPart`. + */ +export interface MultipartUserDataPartOptionsWithBody extends MultipartUserDataPartOptions { /** * The body of message. * * @default undefined - body will not be added to part */ readonly body?: string, +} +/** + * Options when creating `MultipartUserDataPartWrapper`. + */ +export interface MultipartUserDataPartWrapperOptions { /** * `Content-Type` header of this part. * - * For Linux shell scripts use `text/x-shellscript` + * For Linux shell scripts typically it's `text/x-shellscript`. + * + * @default 'text/x-shellscript; charset="utf-8"' + */ + readonly contentType?: MultipartContentType; +} + +/** + * Interface representing part of `MultipartUserData` user data. + */ +export interface IMultipart { + /** + * The body of this MIME part. + */ + readonly body: string | undefined; + + /** + * `Content-Type` header of this part. */ readonly contentType: string; /** - * `Content-Transfer-Encoding` header specifing part encoding. + * `Content-Transfer-Encoding` header specifying part encoding. * * @default undefined - don't add this header */ @@ -322,55 +351,96 @@ export interface MutlipartUserDataPartOptions { } /** - * The raw part of multip-part user data, which can be added to {@link MultipartUserData}. + * The base class for all classes which can be used as {@link MultipartUserData}. */ -export class MutlipartUserDataPart implements IMultipartUserDataPartProducer { +export abstract class MultipartUserDataPart implements IMultipart { + + /** + * Constructs the new `MultipartUserDataPart` wrapping existing `UserData`. Modification to `UserData` are reflected + * in subsequent renders of the part. + * + * For more information about content types see `MultipartUserDataPartOptionsWithBody` + */ + public static fromUserData(userData: UserData, opts?: MultipartUserDataPartWrapperOptions): MultipartUserDataPart { + opts = opts || {}; + return new MultipartUserDataPartWrapper(userData, opts); + } + + /** + * Constructs the raw `MultipartUserDataPart` using specified body, content type and transfer encoding. + * + * When transfer encoding is specified (typically as Base64), it's caller responsibility to convert body to + * Base64 either by wrapping with `Fn.base64` or by converting it by other converters. + */ + public static fromRawBody(opts: MultipartUserDataPartOptionsWithBody): MultipartUserDataPart { + return new MultipartUserDataPartRaw(opts); + } + + protected static readonly DEFAULT_CONTENT_TYPE = 'text/x-shellscript; charset="utf-8"'; + /** The body of this MIME part. */ - public readonly body?: string; + public abstract get body(): string | undefined; /** `Content-Type` header of this part */ public readonly contentType: string; /** - * `Content-Transfer-Encoding` header specifing part encoding. + * `Content-Transfer-Encoding` header specifying part encoding. * * @default undefined - don't add this header */ - readonly transferEncoding?: string; + public readonly transferEncoding?: string; - public constructor(props: MutlipartUserDataPartOptions) { - this.body = props.body; + public constructor(props: MultipartUserDataPartOptions) { this.contentType = props.contentType; this.transferEncoding = props.transferEncoding; } +} - renderAsMimePart(_renderOpts?: MultipartRenderOptions): MutlipartUserDataPart { - return this; +/** + * The raw part of multi-part user data, which can be added to {@link MultipartUserData}. + */ +class MultipartUserDataPartRaw extends MultipartUserDataPart { + private _body : string | undefined; + + public constructor(props: MultipartUserDataPartOptionsWithBody) { + super(props); + this._body = props.body; + } + + public get body(): string | undefined { + return this._body; } } /** - * Render options for parts of multipart user data. + * Wrapper for `UserData`. */ -export interface MultipartRenderOptions { - /** - * Can be used to override default content type (without charset part) used when producing - * part by `IMultipartUserDataPartProducer`. - * - * @default undefined - leave content type unchanged - */ - readonly contentType?: string; +class MultipartUserDataPartWrapper extends MultipartUserDataPart { + public constructor(public readonly userData: UserData, opts: MultipartUserDataPartWrapperOptions) { + super({ + contentType: opts.contentType || MultipartUserDataPart.DEFAULT_CONTENT_TYPE, + // Force Base64 in case userData will contain UTF-8 characters + transferEncoding: 'base64', + }); + } + + public get body(): string { + // Wrap rendered user data with Base64 function, in case data contains tokens + return Fn.base64(this.userData.render()); + } } /** - * Class implementing this interface can produce `MutlipartUserDataPart` and can be added - * to `MultipartUserData`. + * Options for creating {@link MultipartUserData} */ -export interface IMultipartUserDataPartProducer { +export interface MultipartUserDataOptions { /** - * Creats the `MutlipartUserDataPart. + * The string used to separate parts in multipart user data archive (it's like MIME boundary). + * + * This string should contain [a-zA-Z0-9] characters only, and should not be present in any part, or in text content of archive. */ - renderAsMimePart(renderOpts?: MultipartRenderOptions): MutlipartUserDataPart; + readonly partsSeparator: string; } /** @@ -383,41 +453,47 @@ export interface IMultipartUserDataPartProducer { export class MultipartUserData extends UserData { private static readonly USE_PART_ERROR = 'MultipartUserData does not support this operation. Please add part using addPart.'; - private parts: IMultipartUserDataPartProducer[] = []; + private parts: IMultipart[] = []; + + private opts: MultipartUserDataOptions; + + constructor(opts: MultipartUserDataOptions) { + super(); + this.opts = { + ...opts, + }; + } /** - * Adds a class which can producer `MutlipartUserDataPart`. I. e. `UserData.forLinux()`. + * Adds existing `UserData`. Modification to `UserData` are reflected in subsequent renders of the part. + * + * For more information about content types see `MultipartUserDataPartOptionsWithBody` */ - public addPart(producer: IMultipartUserDataPartProducer): this { - this.parts.push(producer); + public addUserDataPart(userData: UserData, opts?: MultipartUserDataPartWrapperOptions): this { + this.parts.push(MultipartUserDataPart.fromUserData(userData, opts)); return this; } - public render(): string { - const renderedParts: MutlipartUserDataPart[] = this.parts.map(producer => producer.renderAsMimePart()); - - // Hash the message content, it will be used as boundry. The boundry should be - // so much unique not to be in message text, and stable so the text of archive will - // not be changed only due to change of boundry (may cause redeploys of resources) - const hash = crypto.createHash('sha256'); - renderedParts.forEach(part => { - hash - .update(part.contentType) - .update(part.body || 'empty-body') - .update(part.transferEncoding || ''); - }); - hash.update('salt-boundary-rado'); + /** + * Adds the 'raw' part using provided options. + */ + public addPart(opts: MultipartUserDataPartOptionsWithBody): this { + this.parts.push(MultipartUserDataPart.fromRawBody(opts)); - const boundary = '-' + hash.digest('base64') + '-'; + return this; + } + + public render(): string { + const boundary = this.opts.partsSeparator; // Now build final MIME archive - there are few changes from MIME message which are accepted by cloud-init: - // - MIME RFC uses CRLF to separarte lines - cloud-init is fine with LF \n only + // - MIME RFC uses CRLF to separate lines - cloud-init is fine with LF \n only var resultArchive = `Content-Type: multipart/mixed; boundary="${boundary}"\n`; resultArchive = resultArchive + 'MIME-Version: 1.0\n'; - // Add parts - each part starts with boundry - renderedParts.forEach(part => { + // Add parts - each part starts with boundary + this.parts.forEach(part => { resultArchive = resultArchive + '\n--' + boundary + '\n' + 'Content-Type: ' + part.contentType + '\n'; if (part.transferEncoding != null) { @@ -429,7 +505,7 @@ export class MultipartUserData extends UserData { } }); - // Add closing boundry + // Add closing boundary resultArchive = resultArchive + `\n--${boundary}--\n`; return resultArchive; diff --git a/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.expected.json b/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.expected.json index f78b3762b75f4..f09a2b3ca0eb9 100644 --- a/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.expected.json +++ b/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.expected.json @@ -646,11 +646,11 @@ "Fn::Join": [ "", [ - "Content-Type: multipart/mixed; boundary=\"-uU56KH4CuLBnf9GVhBq+SFx/hFRXS5S9umauYjYeBxQ=-\"\nMIME-Version: 1.0\n\n---uU56KH4CuLBnf9GVhBq+SFx/hFRXS5S9umauYjYeBxQ=-\nContent-Type: text/x-shellscript; charset=\"utf-8\"\nContent-Transfer-Encoding: base64\n\n", + "Content-Type: multipart/mixed; boundary=\"---separator---\"\nMIME-Version: 1.0\n\n-----separator---\nContent-Type: text/x-shellscript; charset=\"utf-8\"\nContent-Transfer-Encoding: base64\n\n", { "Fn::Base64": "#!/bin/bash\necho 大らと > /var/tmp/echo1\ncp /var/tmp/echo1 /var/tmp/echo1-copy" }, - "\n---uU56KH4CuLBnf9GVhBq+SFx/hFRXS5S9umauYjYeBxQ=-\nContent-Type: text/x-shellscript; charset=\"utf-8\"\nContent-Transfer-Encoding: base64\n\n", + "\n-----separator---\nContent-Type: text/x-shellscript; charset=\"utf-8\"\nContent-Transfer-Encoding: base64\n\n", { "Fn::Base64": { "Fn::Join": [ @@ -665,15 +665,15 @@ ] } }, - "\n---uU56KH4CuLBnf9GVhBq+SFx/hFRXS5S9umauYjYeBxQ=-\nContent-Type: text/cloud-boothook; charset=\"utf-8\"\nContent-Transfer-Encoding: base64\n\n", + "\n-----separator---\nContent-Type: text/cloud-boothook\nContent-Transfer-Encoding: base64\n\n", { "Fn::Base64": "#!/bin/bash\necho \"Boothook2\" > /var/tmp/boothook\ncloud-init-per once docker_options echo 'OPTIONS=\"${OPTIONS} --storage-opt dm.basesize=20G\"' >> /etc/sysconfig/docker" }, - "\n---uU56KH4CuLBnf9GVhBq+SFx/hFRXS5S9umauYjYeBxQ=-\nContent-Type: text/x-shellscript\n\necho \"RawPart\" > /var/tmp/rawPart1\n---uU56KH4CuLBnf9GVhBq+SFx/hFRXS5S9umauYjYeBxQ=-\nContent-Type: text/x-shellscript\n\necho \"RawPart ", + "\n-----separator---\nContent-Type: text/x-shellscript\n\necho \"RawPart\" > /var/tmp/rawPart1\n-----separator---\nContent-Type: text/x-shellscript\n\necho \"RawPart ", { "Ref": "VPCB9E5F0B4" }, - "\" > /var/tmp/rawPart2\n---uU56KH4CuLBnf9GVhBq+SFx/hFRXS5S9umauYjYeBxQ=---\n" + "\" > /var/tmp/rawPart2\n-----separator-----\n" ] ] } diff --git a/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.ts b/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.ts index ed859cccc9d96..1c5cd48a97683 100644 --- a/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.ts +++ b/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.ts @@ -11,7 +11,9 @@ class TestStack extends cdk.Stack { const vpc = new ec2.Vpc(this, 'VPC'); - const multipartUserData = new ec2.MultipartUserData(); + const multipartUserData = new ec2.MultipartUserData({ + partsSeparator: '---separator---', + }); const userData1 = ec2.UserData.forLinux(); userData1.addCommands('echo 大らと > /var/tmp/echo1'); @@ -20,12 +22,12 @@ class TestStack extends cdk.Stack { const userData2 = ec2.UserData.forLinux(); userData2.addCommands(`echo 大らと ${vpc.vpcId} > /var/tmp/echo2`); - const rawPart1 = new ec2.MutlipartUserDataPart({ + const rawPart1 = ec2.MultipartUserDataPart.fromRawBody({ contentType: 'text/x-shellscript', body: 'echo "RawPart" > /var/tmp/rawPart1', }); - const rawPart2 = new ec2.MutlipartUserDataPart({ + const rawPart2 = ec2.MultipartUserDataPart.fromRawBody({ contentType: 'text/x-shellscript', body: `echo "RawPart ${vpc.vpcId}" > /var/tmp/rawPart2`, }); @@ -36,11 +38,11 @@ class TestStack extends cdk.Stack { 'cloud-init-per once docker_options echo \'OPTIONS="${OPTIONS} --storage-opt dm.basesize=20G"\' >> /etc/sysconfig/docker', ); - multipartUserData.addPart(userData1); - multipartUserData.addPart(userData2); - multipartUserData.addPart(bootHook.renderAsMimePart({ + multipartUserData.addUserDataPart(userData1); + multipartUserData.addUserDataPart(userData2); + multipartUserData.addUserDataPart(bootHook, { contentType: 'text/cloud-boothook', - })); + }); multipartUserData.addPart(rawPart1); multipartUserData.addPart(rawPart2); diff --git a/packages/@aws-cdk/aws-ec2/test/userdata.test.ts b/packages/@aws-cdk/aws-ec2/test/userdata.test.ts index 4469e303a9105..d7e7730930867 100644 --- a/packages/@aws-cdk/aws-ec2/test/userdata.test.ts +++ b/packages/@aws-cdk/aws-ec2/test/userdata.test.ts @@ -279,20 +279,19 @@ nodeunitShim({ linuxUserData.addCommands('echo "Hello world"'); // WHEN - const defaultRender1 = linuxUserData.renderAsMimePart(); - const defaultRender2 = linuxUserData.renderAsMimePart({}); - const overriden = linuxUserData.renderAsMimePart({ - contentType: 'text/cloud-boothook', + const defaultRender1 = ec2.MultipartUserDataPart.fromUserData(linuxUserData); + const defaultRender2 = ec2.MultipartUserDataPart.fromUserData(linuxUserData, { + contentType: 'text/cloud-boothook; charset=\"utf-8\"', }); + const defaultRender3 = ec2.MultipartUserDataPart.fromUserData(linuxUserData, {}); // THEN test.equals(defaultRender1.contentType, 'text/x-shellscript; charset=\"utf-8\"'); - test.equals(defaultRender2.contentType, 'text/x-shellscript; charset=\"utf-8\"'); - test.equals(overriden.contentType, 'text/cloud-boothook; charset=\"utf-8\"'); + test.equals(defaultRender2.contentType, 'text/cloud-boothook; charset=\"utf-8\"'); + test.equals(defaultRender3.contentType, 'text/x-shellscript; charset=\"utf-8\"'); test.equals(defaultRender1.transferEncoding, 'base64'); test.equals(defaultRender2.transferEncoding, 'base64'); - test.equals(overriden.transferEncoding, 'base64'); test.done(); }, From a0131384b3e3681d22c86e0d3a653b22e6cf04b4 Mon Sep 17 00:00:00 2001 From: Radek Smogura Date: Thu, 21 Jan 2021 01:03:26 +0100 Subject: [PATCH 3/6] Add readme --- packages/@aws-cdk/aws-ec2/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/@aws-cdk/aws-ec2/README.md b/packages/@aws-cdk/aws-ec2/README.md index 49fff4b5c4f63..2737a3d7f7081 100644 --- a/packages/@aws-cdk/aws-ec2/README.md +++ b/packages/@aws-cdk/aws-ec2/README.md @@ -981,6 +981,16 @@ instance.userData.addExecuteFileCommand({ asset.grantRead( instance.role ); ``` +In addition to above the `MultipartUserData` can be used to change instance startup behavior. Multipart user data are composed +from separate parts forming archive. The moment, and behavior of each part can be controlled with `Content-Type`, and it's wider +than executing shell scripts. + +Some services (like AWS Batch) allows only `MultipartUserData`. + +For more information see +[Specifying Multiple User Data Blocks Using a MIME Multi Part Archive](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/bootstrap_container_instance.html#multi-part_user_data) + + ## Importing existing subnet To import an existing Subnet, call `Subnet.fromSubnetAttributes()` or From f50d10bfd07c2372cc1a11fd4cedffef74516e71 Mon Sep 17 00:00:00 2001 From: Rado Smogura Date: Thu, 4 Mar 2021 19:45:28 +0100 Subject: [PATCH 4/6] Refactor code: - remove `MultipartContentType` - remove `MultipartUserDataPartWrapperOptions` - remove `IMultipart` - rename `MultipartUserDataPart` -> `MultipartBody` - other removals - restructure other classes - moved part rendering to part class - set default separator to hard codeded string - added validation of boundry --- packages/@aws-cdk/aws-ec2/README.md | 42 +++- packages/@aws-cdk/aws-ec2/lib/user-data.ts | 214 +++++++++--------- ....instance-multipart-userdata.expected.json | 10 +- .../test/integ.instance-multipart-userdata.ts | 21 +- .../@aws-cdk/aws-ec2/test/userdata.test.ts | 80 ++++++- 5 files changed, 233 insertions(+), 134 deletions(-) diff --git a/packages/@aws-cdk/aws-ec2/README.md b/packages/@aws-cdk/aws-ec2/README.md index 2737a3d7f7081..d6b206be697cc 100644 --- a/packages/@aws-cdk/aws-ec2/README.md +++ b/packages/@aws-cdk/aws-ec2/README.md @@ -981,11 +981,47 @@ instance.userData.addExecuteFileCommand({ asset.grantRead( instance.role ); ``` +### Multipart user data + In addition to above the `MultipartUserData` can be used to change instance startup behavior. Multipart user data are composed -from separate parts forming archive. The moment, and behavior of each part can be controlled with `Content-Type`, and it's wider -than executing shell scripts. +from separate parts forming archive. The most common parts are scripts executed during instance set-up. However, there are other +kinds, too. + +The advantage of multipart archive is in flexibility when it's needed to add additional parts or to use specialized parts to +fine tune instance startup. Some services (like AWS Batch) supports only `MultipartUserData`. + +The parts can be executed at different moment of instance start-up and can server different purposes. This is controlled by `contentType` property. + +However, most common parts are script parts which can be created by `MultipartUserData.fromUserData`, and which have `contentType` `text/x-shellscript; charset="utf-8"`. + + +In order to create archive the `MultipartUserData` has to be instantiated. Than user can add parts to multipart archive using `addPart`. The `MultipartBody` contains methods supporting creation of body parts. + +If the custom parts is required, it can be created using `MultipartUserData.fromRawBody`, in this case full control over content type, +transfer encoding, and body properties is given to the user. + +Below is an example for creating multipart user data with single body part responsible for installing `awscli` -Some services (like AWS Batch) allows only `MultipartUserData`. +```ts +const bootHookConf = ec2.UserData.forLinux(); +bootHookConf.addCommands('cloud-init-per once docker_options echo \'OPTIONS="${OPTIONS} --storage-opt dm.basesize=40G"\' >> /etc/sysconfig/docker'); + +const setupCommands = ec2.UserData.forLinux(); +setupCommands.addCommands('sudo yum install awscli && echo Packages installed らと > /var/tmp/setup'); + +const multipartUserData = new ec2.MultipartUserData(); +// The docker has to be configured at early stage, so content type is overridden to boothook +multipartUserData.addPart(ec2.MultipartBody.fromUserData(bootHookConf, 'text/cloud-boothook; charset="us-ascii"')); +// Execute the rest of setup +multipartUserData.addPart(ec2.MultipartBody.fromUserData(setupCommands)); + +new ec2.LaunchTemplate(stack, '', { + userData: multipartUserData, + blockDevices: [ + // Block device configuration rest + ] +}); +``` For more information see [Specifying Multiple User Data Blocks Using a MIME Multi Part Archive](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/bootstrap_container_instance.html#multi-part_user_data) diff --git a/packages/@aws-cdk/aws-ec2/lib/user-data.ts b/packages/@aws-cdk/aws-ec2/lib/user-data.ts index ef2bbe0986666..8a7604b5af9fe 100644 --- a/packages/@aws-cdk/aws-ec2/lib/user-data.ts +++ b/packages/@aws-cdk/aws-ec2/lib/user-data.ts @@ -278,21 +278,20 @@ class CustomUserData extends UserData { } /** - * Suggested content types, however any value is allowed. + * Options when creating `MultipartBody`. */ -export type MultipartContentType = 'text/x-shellscript; charset="utf-8"' | 'text/cloud-boothook; charset="utf-8"' | string; - -/** - * Options when creating `MultipartUserDataPart`. - */ -export interface MultipartUserDataPartOptions { +export interface MultipartBodyOptions { /** * `Content-Type` header of this part. * - * For Linux shell scripts use `text/x-shellscript` + * Some examples of content types: + * * `text/x-shellscript; charset="utf-8"` (shell script) + * * `text/cloud-boothook; charset="utf-8"` (shell script executed during boot phase) + * + * For Linux shell scripts use `text/x-shellscript`. */ - readonly contentType: MultipartContentType; + readonly contentType: string; /** * `Content-Transfer-Encoding` header specifying part encoding. @@ -300,12 +299,7 @@ export interface MultipartUserDataPartOptions { * @default undefined - don't add this header */ readonly transferEncoding?: string; -} -/** - * Options when creating `MultipartUserDataPart`. - */ -export interface MultipartUserDataPartOptionsWithBody extends MultipartUserDataPartOptions { /** * The body of message. * @@ -314,66 +308,32 @@ export interface MultipartUserDataPartOptionsWithBody extends MultipartUserDataP readonly body?: string, } -/** - * Options when creating `MultipartUserDataPartWrapper`. - */ -export interface MultipartUserDataPartWrapperOptions { - /** - * `Content-Type` header of this part. - * - * For Linux shell scripts typically it's `text/x-shellscript`. - * - * @default 'text/x-shellscript; charset="utf-8"' - */ - readonly contentType?: MultipartContentType; -} - -/** - * Interface representing part of `MultipartUserData` user data. - */ -export interface IMultipart { - /** - * The body of this MIME part. - */ - readonly body: string | undefined; - - /** - * `Content-Type` header of this part. - */ - readonly contentType: string; - - /** - * `Content-Transfer-Encoding` header specifying part encoding. - * - * @default undefined - don't add this header - */ - readonly transferEncoding?: string; -} - /** * The base class for all classes which can be used as {@link MultipartUserData}. */ -export abstract class MultipartUserDataPart implements IMultipart { +export abstract class MultipartBody { /** - * Constructs the new `MultipartUserDataPart` wrapping existing `UserData`. Modification to `UserData` are reflected + * Constructs the new `MultipartBody` wrapping existing `UserData`. Modification to `UserData` are reflected * in subsequent renders of the part. * - * For more information about content types see `MultipartUserDataPartOptionsWithBody` + * For more information about content types see {@link MultipartBodyOptions.contentType}. + * + * @param userData user data to wrap into body part + * @param contentType optional content type, if default one should not be used */ - public static fromUserData(userData: UserData, opts?: MultipartUserDataPartWrapperOptions): MultipartUserDataPart { - opts = opts || {}; - return new MultipartUserDataPartWrapper(userData, opts); + public static fromUserData(userData: UserData, contentType?: string): MultipartBody { + return new MultipartBodyUserDataWrapper(userData, contentType); } /** - * Constructs the raw `MultipartUserDataPart` using specified body, content type and transfer encoding. + * Constructs the raw `MultipartBody` using specified body, content type and transfer encoding. * * When transfer encoding is specified (typically as Base64), it's caller responsibility to convert body to * Base64 either by wrapping with `Fn.base64` or by converting it by other converters. */ - public static fromRawBody(opts: MultipartUserDataPartOptionsWithBody): MultipartUserDataPart { - return new MultipartUserDataPartRaw(opts); + public static fromRawBody(opts: MultipartBodyOptions): MultipartBody { + return new MultipartBodyRaw(opts); } protected static readonly DEFAULT_CONTENT_TYPE = 'text/x-shellscript; charset="utf-8"'; @@ -382,51 +342,76 @@ export abstract class MultipartUserDataPart implements IMultipart { public abstract get body(): string | undefined; /** `Content-Type` header of this part */ - public readonly contentType: string; + public abstract get contentType(): string; /** * `Content-Transfer-Encoding` header specifying part encoding. * * @default undefined - don't add this header */ - public readonly transferEncoding?: string; + public abstract get transferEncoding(): string | undefined; - public constructor(props: MultipartUserDataPartOptions) { - this.contentType = props.contentType; - this.transferEncoding = props.transferEncoding; + public constructor() { + } + + /** + * Render body part as the string. + * + * Subclasses should not add leading nor trailing new line characters (\r \n) + */ + public renderBodyPart(): string { + const result: string[] = []; + + result.push(`Content-Type: ${this.contentType}`); + + if (this.transferEncoding != null) { + result.push(`Content-Transfer-Encoding: ${this.transferEncoding}`); + } + // One line free after separator + result.push(''); + + if (this.body != null) { + result.push(this.body); + // The new line added after join will be consumed by encapsulating or closing boundary + } + + return result.join('\n'); } } /** * The raw part of multi-part user data, which can be added to {@link MultipartUserData}. */ -class MultipartUserDataPartRaw extends MultipartUserDataPart { - private _body : string | undefined; +class MultipartBodyRaw extends MultipartBody { + public readonly body: string | undefined; + public readonly contentType: string; + public readonly transferEncoding: string | undefined; - public constructor(props: MultipartUserDataPartOptionsWithBody) { - super(props); - this._body = props.body; - } + public constructor(props: MultipartBodyOptions) { + super(); - public get body(): string | undefined { - return this._body; + this.body = props.body; + this.contentType = props.contentType; } } /** * Wrapper for `UserData`. */ -class MultipartUserDataPartWrapper extends MultipartUserDataPart { - public constructor(public readonly userData: UserData, opts: MultipartUserDataPartWrapperOptions) { - super({ - contentType: opts.contentType || MultipartUserDataPart.DEFAULT_CONTENT_TYPE, - // Force Base64 in case userData will contain UTF-8 characters - transferEncoding: 'base64', - }); +class MultipartBodyUserDataWrapper extends MultipartBody { + + public readonly contentType: string; + public readonly transferEncoding: string | undefined; + + public constructor(public readonly userData: UserData, contentType?: string) { + super(); + + this.contentType = contentType || MultipartBody.DEFAULT_CONTENT_TYPE; + this.transferEncoding = 'base64'; } public get body(): string { - // Wrap rendered user data with Base64 function, in case data contains tokens + // Wrap rendered user data with Base64 function, in case data contains non ASCII characters return Fn.base64(this.userData.render()); } } @@ -438,9 +423,11 @@ export interface MultipartUserDataOptions { /** * The string used to separate parts in multipart user data archive (it's like MIME boundary). * - * This string should contain [a-zA-Z0-9] characters only, and should not be present in any part, or in text content of archive. + * This string should contain [a-zA-Z0-9()+,-./:=?] characters only, and should not be present in any part, or in text content of archive. + * + * @default `+AWS+CDK+User+Data+Separator==` */ - readonly partsSeparator: string; + readonly partsSeparator?: string; } /** @@ -452,63 +439,66 @@ export interface MultipartUserDataOptions { */ export class MultipartUserData extends UserData { private static readonly USE_PART_ERROR = 'MultipartUserData does not support this operation. Please add part using addPart.'; + private static readonly BOUNDRY_PATTERN = '[^a-zA-Z0-9()+,-./:=?]'; - private parts: IMultipart[] = []; + private parts: MultipartBody[] = []; private opts: MultipartUserDataOptions; - constructor(opts: MultipartUserDataOptions) { + constructor(opts?: MultipartUserDataOptions) { super(); + let partsSeparator: string; + + // Validate separator + if (opts?.partsSeparator != null) { + if (new RegExp(MultipartUserData.BOUNDRY_PATTERN).test(opts!.partsSeparator)) { + throw new Error(`Invalid characters in separator. Separator has to match pattern ${MultipartUserData.BOUNDRY_PATTERN}`); + } else { + partsSeparator = opts!.partsSeparator; + } + } else { + partsSeparator = '+AWS+CDK+User+Data+Separator=='; + } + this.opts = { - ...opts, + partsSeparator: partsSeparator, }; } - /** - * Adds existing `UserData`. Modification to `UserData` are reflected in subsequent renders of the part. - * - * For more information about content types see `MultipartUserDataPartOptionsWithBody` - */ - public addUserDataPart(userData: UserData, opts?: MultipartUserDataPartWrapperOptions): this { - this.parts.push(MultipartUserDataPart.fromUserData(userData, opts)); - - return this; - } /** - * Adds the 'raw' part using provided options. + * Adds a part to the list of parts. */ - public addPart(opts: MultipartUserDataPartOptionsWithBody): this { - this.parts.push(MultipartUserDataPart.fromRawBody(opts)); + public addPart(part: MultipartBody): this { + this.parts.push(part); return this; } public render(): string { const boundary = this.opts.partsSeparator; - // Now build final MIME archive - there are few changes from MIME message which are accepted by cloud-init: // - MIME RFC uses CRLF to separate lines - cloud-init is fine with LF \n only - var resultArchive = `Content-Type: multipart/mixed; boundary="${boundary}"\n`; - resultArchive = resultArchive + 'MIME-Version: 1.0\n'; + // Note: new lines matters, matters a lot. + var resultArchive = new Array(); + resultArchive.push(`Content-Type: multipart/mixed; boundary="${boundary}"`); + resultArchive.push('MIME-Version: 1.0'); + + // Add new line, the next one will be boundary (encapsulating or closing) + // so this line will count into it. + resultArchive.push(''); // Add parts - each part starts with boundary this.parts.forEach(part => { - resultArchive = resultArchive + '\n--' + boundary + '\n' + 'Content-Type: ' + part.contentType + '\n'; - - if (part.transferEncoding != null) { - resultArchive = resultArchive + `Content-Transfer-Encoding: ${part.transferEncoding}\n`; - } - - if (part.body != null) { - resultArchive = resultArchive + '\n' + part.body; - } + resultArchive.push(`--${boundary}`); + resultArchive.push(part.renderBodyPart()); }); // Add closing boundary - resultArchive = resultArchive + `\n--${boundary}--\n`; + resultArchive.push(`--${boundary}--`); + resultArchive.push(''); // Force new line at the end - return resultArchive; + return resultArchive.join('\n'); } public addS3DownloadCommand(_params: S3DownloadOptions): string { diff --git a/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.expected.json b/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.expected.json index f09a2b3ca0eb9..371a30e7456f6 100644 --- a/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.expected.json +++ b/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.expected.json @@ -646,11 +646,11 @@ "Fn::Join": [ "", [ - "Content-Type: multipart/mixed; boundary=\"---separator---\"\nMIME-Version: 1.0\n\n-----separator---\nContent-Type: text/x-shellscript; charset=\"utf-8\"\nContent-Transfer-Encoding: base64\n\n", + "Content-Type: multipart/mixed; boundary=\"+AWS+CDK+User+Data+Separator==\"\nMIME-Version: 1.0\n\n--+AWS+CDK+User+Data+Separator==\nContent-Type: text/x-shellscript; charset=\"utf-8\"\nContent-Transfer-Encoding: base64\n\n", { "Fn::Base64": "#!/bin/bash\necho 大らと > /var/tmp/echo1\ncp /var/tmp/echo1 /var/tmp/echo1-copy" }, - "\n-----separator---\nContent-Type: text/x-shellscript; charset=\"utf-8\"\nContent-Transfer-Encoding: base64\n\n", + "\n--+AWS+CDK+User+Data+Separator==\nContent-Type: text/x-shellscript; charset=\"utf-8\"\nContent-Transfer-Encoding: base64\n\n", { "Fn::Base64": { "Fn::Join": [ @@ -665,15 +665,15 @@ ] } }, - "\n-----separator---\nContent-Type: text/cloud-boothook\nContent-Transfer-Encoding: base64\n\n", + "\n--+AWS+CDK+User+Data+Separator==\nContent-Type: text/cloud-boothook\nContent-Transfer-Encoding: base64\n\n", { "Fn::Base64": "#!/bin/bash\necho \"Boothook2\" > /var/tmp/boothook\ncloud-init-per once docker_options echo 'OPTIONS=\"${OPTIONS} --storage-opt dm.basesize=20G\"' >> /etc/sysconfig/docker" }, - "\n-----separator---\nContent-Type: text/x-shellscript\n\necho \"RawPart\" > /var/tmp/rawPart1\n-----separator---\nContent-Type: text/x-shellscript\n\necho \"RawPart ", + "\n--+AWS+CDK+User+Data+Separator==\nContent-Type: text/x-shellscript\n\necho \"RawPart\" > /var/tmp/rawPart1\n--+AWS+CDK+User+Data+Separator==\nContent-Type: text/x-shellscript\n\necho \"RawPart ", { "Ref": "VPCB9E5F0B4" }, - "\" > /var/tmp/rawPart2\n-----separator-----\n" + "\" > /var/tmp/rawPart2\n--+AWS+CDK+User+Data+Separator==\nContent-Type: text/x-shellscript\n\ncp $0 /var/tmp/upstart # Should be one line file no new line at the end and beginning\n--+AWS+CDK+User+Data+Separator==--\n" ] ] } diff --git a/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.ts b/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.ts index 1c5cd48a97683..9038166b93e39 100644 --- a/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.ts +++ b/packages/@aws-cdk/aws-ec2/test/integ.instance-multipart-userdata.ts @@ -11,9 +11,8 @@ class TestStack extends cdk.Stack { const vpc = new ec2.Vpc(this, 'VPC'); - const multipartUserData = new ec2.MultipartUserData({ - partsSeparator: '---separator---', - }); + // Here we test default separator as probably most useful + const multipartUserData = new ec2.MultipartUserData(); const userData1 = ec2.UserData.forLinux(); userData1.addCommands('echo 大らと > /var/tmp/echo1'); @@ -22,12 +21,12 @@ class TestStack extends cdk.Stack { const userData2 = ec2.UserData.forLinux(); userData2.addCommands(`echo 大らと ${vpc.vpcId} > /var/tmp/echo2`); - const rawPart1 = ec2.MultipartUserDataPart.fromRawBody({ + const rawPart1 = ec2.MultipartBody.fromRawBody({ contentType: 'text/x-shellscript', body: 'echo "RawPart" > /var/tmp/rawPart1', }); - const rawPart2 = ec2.MultipartUserDataPart.fromRawBody({ + const rawPart2 = ec2.MultipartBody.fromRawBody({ contentType: 'text/x-shellscript', body: `echo "RawPart ${vpc.vpcId}" > /var/tmp/rawPart2`, }); @@ -38,13 +37,17 @@ class TestStack extends cdk.Stack { 'cloud-init-per once docker_options echo \'OPTIONS="${OPTIONS} --storage-opt dm.basesize=20G"\' >> /etc/sysconfig/docker', ); - multipartUserData.addUserDataPart(userData1); - multipartUserData.addUserDataPart(userData2); - multipartUserData.addUserDataPart(bootHook, { - contentType: 'text/cloud-boothook', + multipartUserData.addPart(ec2.MultipartBody.fromUserData(userData1)); + multipartUserData.addPart(ec2.MultipartBody.fromUserData(userData2)); + multipartUserData.addPart(ec2.MultipartBody.fromUserData(bootHook, 'text/cloud-boothook')); + + const rawPart3 = ec2.MultipartBody.fromRawBody({ + contentType: 'text/x-shellscript', + body: 'cp $0 /var/tmp/upstart # Should be one line file no new line at the end and beginning', }); multipartUserData.addPart(rawPart1); multipartUserData.addPart(rawPart2); + multipartUserData.addPart(rawPart3); const instance = new ec2.Instance(this, 'Instance', { vpc, diff --git a/packages/@aws-cdk/aws-ec2/test/userdata.test.ts b/packages/@aws-cdk/aws-ec2/test/userdata.test.ts index d7e7730930867..2c5a79ab26c05 100644 --- a/packages/@aws-cdk/aws-ec2/test/userdata.test.ts +++ b/packages/@aws-cdk/aws-ec2/test/userdata.test.ts @@ -279,11 +279,10 @@ nodeunitShim({ linuxUserData.addCommands('echo "Hello world"'); // WHEN - const defaultRender1 = ec2.MultipartUserDataPart.fromUserData(linuxUserData); - const defaultRender2 = ec2.MultipartUserDataPart.fromUserData(linuxUserData, { - contentType: 'text/cloud-boothook; charset=\"utf-8\"', - }); - const defaultRender3 = ec2.MultipartUserDataPart.fromUserData(linuxUserData, {}); + const defaultRender1 = ec2.MultipartBody.fromUserData(linuxUserData); + const defaultRender2 = ec2.MultipartBody.fromUserData(linuxUserData, 'text/cloud-boothook; charset=\"utf-8\"'); + + const defaultRender3 = ec2.MultipartBody.fromUserData(linuxUserData); // THEN test.equals(defaultRender1.contentType, 'text/x-shellscript; charset=\"utf-8\"'); @@ -295,4 +294,75 @@ nodeunitShim({ test.done(); }, + + 'Default parts separator used, if not specified'(test: Test) { + // GIVEN + const multipart = new ec2.MultipartUserData(); + + multipart.addPart(ec2.MultipartBody.fromRawBody({ + contentType: 'CT', + })); + + // WHEN + const out = multipart.render(); + + // WHEN + test.equals(out, [ + 'Content-Type: multipart/mixed; boundary=\"+AWS+CDK+User+Data+Separator==\"', + 'MIME-Version: 1.0', + '', + '--+AWS+CDK+User+Data+Separator==', + 'Content-Type: CT', + '', + '--+AWS+CDK+User+Data+Separator==--', + '', + ].join('\n')); + + test.done(); + }, + + 'Non-default parts separator used, if not specified'(test: Test) { + // GIVEN + const multipart = new ec2.MultipartUserData({ + partsSeparator: '//', + }); + + multipart.addPart(ec2.MultipartBody.fromRawBody({ + contentType: 'CT', + })); + + // WHEN + const out = multipart.render(); + + // WHEN + test.equals(out, [ + 'Content-Type: multipart/mixed; boundary=\"//\"', + 'MIME-Version: 1.0', + '', + '--//', + 'Content-Type: CT', + '', + '--//--', + '', + ].join('\n')); + + test.done(); + }, + + 'Multipart separator validation'(test: Test) { + // Happy path + new ec2.MultipartUserData(); + new ec2.MultipartUserData({ + partsSeparator: 'a-zA-Z0-9()+,-./:=?', + }); + + [' ', '\n', '\r', '[', ']', '<', '>', '違う'].forEach(s => test.throws(() => { + new ec2.MultipartUserData({ + partsSeparator: s, + }); + }, /Invalid characters in separator/)); + + test.done(); + }, + }); From 5338f1d7d4fa9345b635ddf0610ea4eabf787334 Mon Sep 17 00:00:00 2001 From: Rado Smogura Date: Thu, 4 Mar 2021 20:46:40 +0100 Subject: [PATCH 5/6] Fix wording and spelling in Readme --- packages/@aws-cdk/aws-ec2/README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/@aws-cdk/aws-ec2/README.md b/packages/@aws-cdk/aws-ec2/README.md index d6b206be697cc..90b5b4be9cfb8 100644 --- a/packages/@aws-cdk/aws-ec2/README.md +++ b/packages/@aws-cdk/aws-ec2/README.md @@ -983,24 +983,23 @@ asset.grantRead( instance.role ); ### Multipart user data -In addition to above the `MultipartUserData` can be used to change instance startup behavior. Multipart user data are composed +In addition, to above the `MultipartUserData` can be used to change instance startup behavior. Multipart user data are composed from separate parts forming archive. The most common parts are scripts executed during instance set-up. However, there are other kinds, too. The advantage of multipart archive is in flexibility when it's needed to add additional parts or to use specialized parts to fine tune instance startup. Some services (like AWS Batch) supports only `MultipartUserData`. -The parts can be executed at different moment of instance start-up and can server different purposes. This is controlled by `contentType` property. +The parts can be executed at different moment of instance start-up and can serve a different purposes. This is controlled by `contentType` property. +For common scripts, `text/x-shellscript; charset="utf-8"` can be used as content type. -However, most common parts are script parts which can be created by `MultipartUserData.fromUserData`, and which have `contentType` `text/x-shellscript; charset="utf-8"`. +In order to create archive the `MultipartUserData` has to be instantiated. Than, user can add parts to multipart archive using `addPart`. The `MultipartBody` contains methods supporting creation of body parts. - -In order to create archive the `MultipartUserData` has to be instantiated. Than user can add parts to multipart archive using `addPart`. The `MultipartBody` contains methods supporting creation of body parts. - -If the custom parts is required, it can be created using `MultipartUserData.fromRawBody`, in this case full control over content type, +If the very custom part is required, it can be created using `MultipartUserData.fromRawBody`, in this case full control over content type, transfer encoding, and body properties is given to the user. -Below is an example for creating multipart user data with single body part responsible for installing `awscli` +Below is an example for creating multipart user data with single body part responsible for installing `awscli` and configuring maximum size +of storage used by Docker containers: ```ts const bootHookConf = ec2.UserData.forLinux(); From 256364888699b6f37d637d676739efcd700a941b Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Mon, 8 Mar 2021 13:20:38 +0000 Subject: [PATCH 6/6] Small simplifications --- packages/@aws-cdk/aws-ec2/lib/user-data.ts | 107 ++++++++++-------- .../@aws-cdk/aws-ec2/test/userdata.test.ts | 21 ++-- 2 files changed, 71 insertions(+), 57 deletions(-) diff --git a/packages/@aws-cdk/aws-ec2/lib/user-data.ts b/packages/@aws-cdk/aws-ec2/lib/user-data.ts index 8a7604b5af9fe..418b6d671846d 100644 --- a/packages/@aws-cdk/aws-ec2/lib/user-data.ts +++ b/packages/@aws-cdk/aws-ec2/lib/user-data.ts @@ -296,7 +296,7 @@ export interface MultipartBodyOptions { /** * `Content-Transfer-Encoding` header specifying part encoding. * - * @default undefined - don't add this header + * @default undefined - body is not encoded */ readonly transferEncoding?: string; @@ -312,6 +312,15 @@ export interface MultipartBodyOptions { * The base class for all classes which can be used as {@link MultipartUserData}. */ export abstract class MultipartBody { + /** + * Content type for shell scripts + */ + public static readonly SHELL_SCRIPT = 'text/x-shellscript; charset="utf-8"'; + + /** + * Content type for boot hooks + */ + public static readonly CLOUD_BOOTHOOK = 'text/cloud-boothook; charset="utf-8"'; /** * Constructs the new `MultipartBody` wrapping existing `UserData`. Modification to `UserData` are reflected @@ -336,62 +345,45 @@ export abstract class MultipartBody { return new MultipartBodyRaw(opts); } - protected static readonly DEFAULT_CONTENT_TYPE = 'text/x-shellscript; charset="utf-8"'; - - /** The body of this MIME part. */ - public abstract get body(): string | undefined; - - /** `Content-Type` header of this part */ - public abstract get contentType(): string; + public constructor() { + } /** - * `Content-Transfer-Encoding` header specifying part encoding. + * Render body part as the string. * - * @default undefined - don't add this header + * Subclasses should not add leading nor trailing new line characters (\r \n) */ - public abstract get transferEncoding(): string | undefined; + public abstract renderBodyPart(): string[]; +} - public constructor() { +/** + * The raw part of multi-part user data, which can be added to {@link MultipartUserData}. + */ +class MultipartBodyRaw extends MultipartBody { + public constructor(private readonly props: MultipartBodyOptions) { + super(); } /** * Render body part as the string. - * - * Subclasses should not add leading nor trailing new line characters (\r \n) */ - public renderBodyPart(): string { + public renderBodyPart(): string[] { const result: string[] = []; - result.push(`Content-Type: ${this.contentType}`); + result.push(`Content-Type: ${this.props.contentType}`); - if (this.transferEncoding != null) { - result.push(`Content-Transfer-Encoding: ${this.transferEncoding}`); + if (this.props.transferEncoding != null) { + result.push(`Content-Transfer-Encoding: ${this.props.transferEncoding}`); } // One line free after separator result.push(''); - if (this.body != null) { - result.push(this.body); + if (this.props.body != null) { + result.push(this.props.body); // The new line added after join will be consumed by encapsulating or closing boundary } - return result.join('\n'); - } -} - -/** - * The raw part of multi-part user data, which can be added to {@link MultipartUserData}. - */ -class MultipartBodyRaw extends MultipartBody { - public readonly body: string | undefined; - public readonly contentType: string; - public readonly transferEncoding: string | undefined; - - public constructor(props: MultipartBodyOptions) { - super(); - - this.body = props.body; - this.contentType = props.contentType; + return result; } } @@ -399,20 +391,26 @@ class MultipartBodyRaw extends MultipartBody { * Wrapper for `UserData`. */ class MultipartBodyUserDataWrapper extends MultipartBody { + private readonly contentType: string; - public readonly contentType: string; - public readonly transferEncoding: string | undefined; - - public constructor(public readonly userData: UserData, contentType?: string) { + public constructor(private readonly userData: UserData, contentType?: string) { super(); - this.contentType = contentType || MultipartBody.DEFAULT_CONTENT_TYPE; - this.transferEncoding = 'base64'; + this.contentType = contentType || MultipartBody.SHELL_SCRIPT; } - public get body(): string { - // Wrap rendered user data with Base64 function, in case data contains non ASCII characters - return Fn.base64(this.userData.render()); + /** + * Render body part as the string. + */ + public renderBodyPart(): string[] { + const result: string[] = []; + + result.push(`Content-Type: ${this.contentType}`); + result.push('Content-Transfer-Encoding: base64'); + result.push(''); + result.push(Fn.base64(this.userData.render())); + + return result; } } @@ -469,10 +467,21 @@ export class MultipartUserData extends UserData { /** * Adds a part to the list of parts. */ - public addPart(part: MultipartBody): this { + public addPart(part: MultipartBody) { this.parts.push(part); + } - return this; + /** + * Adds a multipart part based on a UserData object + * + * This is the same as calling: + * + * ```ts + * multiPart.addPart(MultipartBody.fromUserData(userData, contentType)); + * ``` + */ + public addUserDataPart(userData: UserData, contentType?: string) { + this.addPart(MultipartBody.fromUserData(userData, contentType)); } public render(): string { @@ -491,7 +500,7 @@ export class MultipartUserData extends UserData { // Add parts - each part starts with boundary this.parts.forEach(part => { resultArchive.push(`--${boundary}`); - resultArchive.push(part.renderBodyPart()); + resultArchive.push(...part.renderBodyPart()); }); // Add closing boundary diff --git a/packages/@aws-cdk/aws-ec2/test/userdata.test.ts b/packages/@aws-cdk/aws-ec2/test/userdata.test.ts index 2c5a79ab26c05..26493962cbbb8 100644 --- a/packages/@aws-cdk/aws-ec2/test/userdata.test.ts +++ b/packages/@aws-cdk/aws-ec2/test/userdata.test.ts @@ -275,6 +275,7 @@ nodeunitShim({ 'Linux user rendering multipart headers'(test: Test) { // GIVEN + const stack = new Stack(); const linuxUserData = ec2.UserData.forLinux(); linuxUserData.addCommands('echo "Hello world"'); @@ -282,15 +283,19 @@ nodeunitShim({ const defaultRender1 = ec2.MultipartBody.fromUserData(linuxUserData); const defaultRender2 = ec2.MultipartBody.fromUserData(linuxUserData, 'text/cloud-boothook; charset=\"utf-8\"'); - const defaultRender3 = ec2.MultipartBody.fromUserData(linuxUserData); - // THEN - test.equals(defaultRender1.contentType, 'text/x-shellscript; charset=\"utf-8\"'); - test.equals(defaultRender2.contentType, 'text/cloud-boothook; charset=\"utf-8\"'); - test.equals(defaultRender3.contentType, 'text/x-shellscript; charset=\"utf-8\"'); - - test.equals(defaultRender1.transferEncoding, 'base64'); - test.equals(defaultRender2.transferEncoding, 'base64'); + expect(stack.resolve(defaultRender1.renderBodyPart())).toEqual([ + 'Content-Type: text/x-shellscript; charset=\"utf-8\"', + 'Content-Transfer-Encoding: base64', + '', + { 'Fn::Base64': '#!/bin/bash\necho \"Hello world\"' }, + ]); + expect(stack.resolve(defaultRender2.renderBodyPart())).toEqual([ + 'Content-Type: text/cloud-boothook; charset=\"utf-8\"', + 'Content-Transfer-Encoding: base64', + '', + { 'Fn::Base64': '#!/bin/bash\necho \"Hello world\"' }, + ]); test.done(); },