diff --git a/stacker_blueprints/efs.py b/stacker_blueprints/efs.py index 159b66bf..0f668db8 100644 --- a/stacker_blueprints/efs.py +++ b/stacker_blueprints/efs.py @@ -1,14 +1,17 @@ from troposphere import ec2, efs -from troposphere import Join, Output, Ref +from troposphere import Join, Output, Ref, Tags from stacker.blueprints.base import Blueprint -from stacker.blueprints.variables.types import EC2VPCId, TroposphereType +from stacker.blueprints.variables.types import TroposphereType +from stacker.exceptions import ValidatorError + +from stacker_blueprints.util import merge_tags class ElasticFileSystem(Blueprint): VARIABLES = { 'VpcId': { - 'type': EC2VPCId, + 'type': str, 'description': 'VPC ID to create resources' }, 'PerformanceMode': { @@ -16,85 +19,126 @@ class ElasticFileSystem(Blueprint): 'description': 'The performance mode of the file system', 'default': 'generalPurpose' }, - 'FileSystemTags': { + 'Tags': { 'type': dict, - 'description': 'Tags to associate with the file system.', + 'description': 'Tags to associate with the created resources', 'default': {} }, 'Subnets': { - 'type': str, - 'description': 'Comma-delimited list of subnets to deploy private ' - 'mount targets in.' + 'type': list, + 'description': 'List of subnets to deploy private mount targets in' }, - 'IPAddresses': { - 'type': str, + 'IpAddresses': { + 'type': list, 'description': 'List of IP addresses to assign to mount targets. ' - 'Omit to assign automatically. ' + 'Omit or make empty to assign automatically. ' 'Corresponds to Subnets listed in the same order.', - 'default': '' + 'default': [] }, 'SecurityGroups': { 'type': TroposphereType(ec2.SecurityGroup, many=True, - validate=False), - 'description': 'Definition of SecurityGroups to be created and ' - 'assigned to the Mount Targets. The VpcId property ' - 'will be filled from the similarly named variable ' - 'of this blueprint, so it can be ommited. ' - 'Omit this parameter entirely, or make it an empty ' - 'list to avoid creating any groups (and use the ' - 'ExtraSecurityGroups variable instead)', - 'default': [] + optional=True, validate=False), + 'description': "Dictionary of titles to SecurityGroups " + "definitions to be created and assigned to this " + "filesystem's MountTargets. " + "The VpcId property will be filled automatically, " + "so it should not be included. \n" + "The IDs of the created groups will be exported as " + "a comma-separated list in the " + "EfsNewSecurityGroupIds output.\n" + "Omit this parameter or set it to an empty " + "dictionary to not create any groups. In that " + "case the ExistingSecurityGroups variable must not " + "be empty", + 'default': {} }, 'ExtraSecurityGroups': { - 'type': str, - 'description': 'Comma-separated list of existing SecurityGroups ' - 'to be assigned to the EFS.', - 'default': '' + 'type': list, + 'description': "List of existing SecurityGroup IDs to be asigned " + "to this filesystem's MountTargets", + 'default': [] } } - def create_efs_filesystem(self): - t = self.template + def validate_efs_security_groups(self): + validator = '{}.{}'.format(type(self).__name__, + 'validate_efs_security_groups') + v = self.get_variables() + count = len(v['SecurityGroups'] or []) + len(v['ExtraSecurityGroups']) + + if count == 0: + raise ValidatorError( + 'SecurityGroups,ExtraSecurityGroups', validator, count, + 'At least one SecurityGroup must be provided') + elif count > 5: + raise ValidatorError( + 'SecurityGroups,ExtraSecurityGroups', validator, count, + 'At most five total SecurityGroups must be provided') + + def validate_efs_subnets(self): + validator = '{}.{}'.format(type(self).__name__, 'validate_efs_subnets') v = self.get_variables() - fs = t.add_resource(efs.FileSystem( - 'EfsFileSystem', - FileSystemTags=efs.Tags(v['FileSystemTags']), - PerformanceMode=v['PerformanceMode'])) + subnet_count = len(v['Subnets']) + if not subnet_count: + raise ValidatorError( + 'Subnets', validator, v['Subnets'], + 'At least one Subnet must be provided') - t.add_output(Output( - 'EfsFileSystemId', - Value=Ref(fs))) + ip_count = len(v['IpAddresses']) + if ip_count and ip_count != subnet_count: + raise ValidatorError( + 'IpAddresses', validator, v['IpAddresses'], + 'The number of IpAddresses must match the number of Subnets') - return fs + def resolve_variables(self, provided_variables): + super(ElasticFileSystem, self).resolve_variables(provided_variables) + + self.validate_efs_security_groups() + self.validate_efs_subnets() - def create_efs_security_groups(self): + def prepare_efs_security_groups(self): t = self.template v = self.get_variables() - new_sgs = [] + created_groups = [] for sg in v['SecurityGroups']: - sg.VpcId = Ref('VpcId') - sg.validate() + sg.VpcId = v['VpcId'] + sg.Tags = merge_tags(v['Tags'], getattr(sg, 'Tags', {})) - t.add_resource(sg) - new_sgs.append(Ref(sg)) + sg = t.add_resource(sg) + created_groups.append(sg) + created_group_ids = list(map(Ref, created_groups)) t.add_output(Output( - 'EfsSecurityGroupIds', - Value=Join(',', new_sgs))) + 'EfsNewSecurityGroupIds', + Value=Join(',', created_group_ids))) - existing_sgs = v['ExtraSecurityGroups'].split(',') - return new_sgs + existing_sgs + groups_ids = created_group_ids + v['ExtraSecurityGroups'] + return groups_ids + + def create_efs_filesystem(self): + t = self.template + v = self.get_variables() + + fs = t.add_resource(efs.FileSystem( + 'EfsFileSystem', + FileSystemTags=Tags(v['Tags']), + PerformanceMode=v['PerformanceMode'])) + + t.add_output(Output( + 'EfsFileSystemId', + Value=Ref(fs))) + + return fs - def create_efs_mount_targets(self, fs, sgs): + def create_efs_mount_targets(self, fs): t = self.template v = self.get_variables() - subnets = v['Subnets'].split(',') - ips = v['IPAddresses'] and v['IPAddresses'].split(',') - if ips and len(ips) != len(subnets): - raise ValueError('Subnets and IPAddresses must have same count') + groups = self.prepare_efs_security_groups() + subnets = v['Subnets'] + ips = v['IpAddresses'] mount_targets = [] for i, subnet in enumerate(subnets): @@ -102,12 +146,12 @@ def create_efs_mount_targets(self, fs, sgs): 'EfsMountTarget{}'.format(i + 1), FileSystemId=Ref(fs), SubnetId=subnet, - SecurityGroups=sgs) + SecurityGroups=groups) if ips: mount_target.IpAddress = ips[i] - t.add_resource(mount_target) + mount_target = t.add_resource(mount_target) mount_targets.append(mount_target) t.add_output(Output( @@ -116,5 +160,4 @@ def create_efs_mount_targets(self, fs, sgs): def create_template(self): fs = self.create_efs_filesystem() - sgs = self.create_efs_security_groups() - self.create_efs_mount_targets(fs, sgs) + self.create_efs_mount_targets(fs) diff --git a/stacker_blueprints/util.py b/stacker_blueprints/util.py index b13ddc00..bb2079e2 100644 --- a/stacker_blueprints/util.py +++ b/stacker_blueprints/util.py @@ -1,3 +1,8 @@ +from collections import Mapping + +from troposphere import Tags + + def check_properties(properties, allowed_properties, resource): """Checks the list of properties in the properties variable against the property list provided by the allowed_properties variable. If any property @@ -15,3 +20,37 @@ def check_properties(properties, allowed_properties, resource): raise ValueError( "%s is not a valid property of %s" % (key, resource) ) + + +def _tags_to_dict(tag_list): + return dict((tag['Key'], tag['Value']) for tag in tag_list) + + +def merge_tags(left, right, factory=Tags): + """ + Merge two sets of tags into a new troposphere object + + Args: + left (Union[dict, troposphere.Tags]): dictionary or Tags object to be + merged with lower priority + right (Union[dict, troposphere.Tags]): dictionary or Tags object to be + merged with higher priority + factory (type): Type of object to create. Defaults to the troposphere + Tags class. + """ + + if isinstance(left, Mapping): + tags = dict(left) + elif hasattr(left, 'tags'): + tags = _tags_to_dict(left.tags) + else: + tags = _tags_to_dict(left) + + if isinstance(right, Mapping): + tags.update(right) + elif hasattr(left, 'tags'): + tags.update(_tags_to_dict(right.tags)) + else: + tags.update(_tags_to_dict(right)) + + return factory(**tags) diff --git a/tests/fixtures/blueprints/test_efs_ElasticFileSystem.json b/tests/fixtures/blueprints/test_efs_ElasticFileSystem.json index 4f8fb321..146bd2a1 100644 --- a/tests/fixtures/blueprints/test_efs_ElasticFileSystem.json +++ b/tests/fixtures/blueprints/test_efs_ElasticFileSystem.json @@ -20,7 +20,7 @@ ] } }, - "EfsSecurityGroupIds": { + "EfsNewSecurityGroupIds": { "Value": { "Fn::Join": [ ",", @@ -100,9 +100,17 @@ "ToPort": 2049 } ], - "VpcId": { - "Ref": "VpcId" - } + "Tags": [ + { + "Key": "Foo", + "Value": "Bar" + }, + { + "Key": "Hello", + "Value": "World" + } + ], + "VpcId": "vpc-11111111" }, "Type": "AWS::EC2::SecurityGroup" }, @@ -111,17 +119,21 @@ "GroupDescription": "EFS SG 2", "SecurityGroupIngress": [ { - "FromPort": 2048, + "FromPort": 2049, "IpProtocol": "tcp", "SourceSecurityGroupId": "sg-11111111", "ToPort": 2049 } ], - "VpcId": { - "Ref": "VpcId" - } + "Tags": [ + { + "Key": "Hello", + "Value": "World" + } + ], + "VpcId": "vpc-11111111" }, "Type": "AWS::EC2::SecurityGroup" } } -} \ No newline at end of file +} diff --git a/tests/test_efs.py b/tests/test_efs.py index 27726972..a3f10dfc 100644 --- a/tests/test_efs.py +++ b/tests/test_efs.py @@ -2,48 +2,90 @@ from stacker.blueprints.testutil import BlueprintTestCase from stacker.context import Context +from stacker.exceptions import ValidatorError from stacker.variables import Variable from stacker_blueprints.efs import ElasticFileSystem +EFS_VARIABLES = { + 'VpcId': 'vpc-11111111', + 'PerformanceMode': 'generalPurpose', + 'Tags': { + 'Hello': 'World' + }, + 'Subnets': ['subnet-11111111', 'subnet-22222222'], + 'IpAddresses': ['172.16.1.10', '172.16.2.10'], + 'SecurityGroups': { + 'EfsSg1': { + 'GroupDescription': 'EFS SG 1', + 'SecurityGroupIngress': [ + {'IpProtocol': 'tcp', 'FromPort': 2049, 'ToPort': 2049, + 'CidrIp': '172.16.0.0/12'} + ], + 'Tags': [{'Key': 'Foo', 'Value': 'Bar'}] + }, + 'EfsSg2': { + 'GroupDescription': 'EFS SG 2', + 'SecurityGroupIngress': [ + {'IpProtocol': 'tcp', 'FromPort': 2049, 'ToPort': 2049, + 'SourceSecurityGroupId': 'sg-11111111'} + ] + } + }, + 'ExtraSecurityGroups': ['sg-22222222', 'sg-33333333'] +} + + class TestElasticFileSystem(BlueprintTestCase): def setUp(self): self.ctx = Context({'namespace': 'test'}) def test_create_template(self): blueprint = ElasticFileSystem('test_efs_ElasticFileSystem', self.ctx) - variables = { - 'VpcId': 'vpc-11111111', - 'PerformanceMode': 'generalPurpose', - 'FileSystemTags': { - 'Hello': 'World' - }, - 'Subnets': 'subnet-11111111,subnet-22222222', - 'IPAddresses': '172.16.1.10,172.16.2.10', - 'SecurityGroups': { - 'EfsSg1': { - 'GroupDescription': 'EFS SG 1', - 'SecurityGroupIngress': [ - {'IpProtocol': 'tcp', 'FromPort': 2049, 'ToPort': 2049, - 'CidrIp': '172.16.0.0/12'} - ] - }, - 'EfsSg2': { - 'GroupDescription': 'EFS SG 2', - 'SecurityGroupIngress': [ - {'IpProtocol': 'tcp', 'FromPort': 2049, 'ToPort': 2049, - 'SourceSecurityGroupId': 'sg-11111111'} - ] - } - }, - 'ExtraSecurityGroups': 'sg-22222222,sg-33333333' - } + variables = EFS_VARIABLES blueprint.resolve_variables( [Variable(k, v) for k, v in variables.items()]) blueprint.create_template() self.assertRenderedBlueprint(blueprint) + def test_validate_security_group_count_empty(self): + blueprint = ElasticFileSystem('test_efs_ElasticFileSystem', self.ctx) + variables = EFS_VARIABLES.copy() + variables['SecurityGroups'] = {} + variables['ExtraSecurityGroups'] = [] + + with self.assertRaises(ValidatorError): + blueprint.resolve_variables( + [Variable(k, v) for k, v in variables.items()]) + + def test_validate_security_group_count_exceeded(self): + blueprint = ElasticFileSystem('test_efs_ElasticFileSystem', self.ctx) + variables = EFS_VARIABLES.copy() + variables['ExtraSecurityGroups'] = ['sg-22222222'] * 4 + + with self.assertRaises(ValidatorError): + blueprint.resolve_variables( + [Variable(k, v) for k, v in variables.items()]) + + def test_validate_subnets_empty(self): + blueprint = ElasticFileSystem('test_efs_ElasticFileSystem', self.ctx) + variables = EFS_VARIABLES.copy() + variables['Subnets'] = [] + + with self.assertRaises(ValidatorError): + blueprint.resolve_variables( + [Variable(k, v) for k, v in variables.items()]) + + def test_validate_subnets_ip_addresses_unmatching(self): + blueprint = ElasticFileSystem('test_efs_ElasticFileSystem', self.ctx) + variables = EFS_VARIABLES.copy() + variables['IpAddresses'] = ['172.16.1.10'] + + with self.assertRaises(ValidatorError): + blueprint.resolve_variables( + [Variable(k, v) for k, v in variables.items()]) + if __name__ == '__main__': unittest.main()