Skip to content
This repository has been archived by the owner on Jul 7, 2021. It is now read-only.

Commit

Permalink
efs: improve blueprint variables and validations
Browse files Browse the repository at this point in the history 
- Apply Tags to all supported resources, not just the filesystem
- Validate the number of SecurityGroups, Subnets and IpAddresses
  • Loading branch information
@danielkza
danielkza committed Apr 22, 2018
1 parent 0962d76 commit 573cc18
Show file tree
Hide file tree
Showing 4 changed files with 225 additions and 89 deletions.
151 changes: 97 additions & 54 deletions stacker_blueprints/efs.py
View file
Original file line number Diff line number Diff line change
@@ -1,113 +1,157 @@
from troposphere import ec2, efs
from troposphere import Join, Output, Ref
from troposphere import Join, Output, Ref, Tags

from stacker.blueprints.base import Blueprint
from stacker.blueprints.variables.types import EC2VPCId, TroposphereType
from stacker.blueprints.variables.types import TroposphereType
from stacker.exceptions import ValidatorError

from stacker_blueprints.util import merge_tags


class ElasticFileSystem(Blueprint):
VARIABLES = {
'VpcId': {
'type': EC2VPCId,
'type': str,
'description': 'VPC ID to create resources'
},
'PerformanceMode': {
'type': str,
'description': 'The performance mode of the file system',
'default': 'generalPurpose'
},
'FileSystemTags': {
'Tags': {
'type': dict,
'description': 'Tags to associate with the file system.',
'description': 'Tags to associate with the created resources',
'default': {}
},
'Subnets': {
'type': str,
'description': 'Comma-delimited list of subnets to deploy private '
'mount targets in.'
'type': list,
'description': 'List of subnets to deploy private mount targets in'
},
'IPAddresses': {
'type': str,
'IpAddresses': {
'type': list,
'description': 'List of IP addresses to assign to mount targets. '
'Omit to assign automatically. '
'Omit or make empty to assign automatically. '
'Corresponds to Subnets listed in the same order.',
'default': ''
'default': []
},
'SecurityGroups': {
'type': TroposphereType(ec2.SecurityGroup, many=True,
validate=False),
'description': 'Definition of SecurityGroups to be created and '
'assigned to the Mount Targets. The VpcId property '
'will be filled from the similarly named variable '
'of this blueprint, so it can be ommited. '
'Omit this parameter entirely, or make it an empty '
'list to avoid creating any groups (and use the '
'ExtraSecurityGroups variable instead)',
'default': []
optional=True, validate=False),
'description': "Dictionary of titles to SecurityGroups "
"definitions to be created and assigned to this "
"filesystem's MountTargets. "
"The VpcId property will be filled automatically, "
"so it should not be included. \n"
"The IDs of the created groups will be exported as "
"a comma-separated list in the "
"EfsNewSecurityGroupIds output.\n"
"Omit this parameter or set it to an empty "
"dictionary to not create any groups. In that "
"case the ExistingSecurityGroups variable must not "
"be empty",
'default': {}
},
'ExtraSecurityGroups': {
'type': str,
'description': 'Comma-separated list of existing SecurityGroups '
'to be assigned to the EFS.',
'default': ''
'type': list,
'description': "List of existing SecurityGroup IDs to be asigned "
"to this filesystem's MountTargets",
'default': []
}
}

def create_efs_filesystem(self):
t = self.template
def validate_efs_security_groups(self):
validator = '{}.{}'.format(type(self).__name__,
'validate_efs_security_groups')
v = self.get_variables()
count = len(v['SecurityGroups'] or []) + len(v['ExtraSecurityGroups'])

if count == 0:
raise ValidatorError(
'SecurityGroups,ExtraSecurityGroups', validator, count,
'At least one SecurityGroup must be provided')
elif count > 5:
raise ValidatorError(
'SecurityGroups,ExtraSecurityGroups', validator, count,
'At most five total SecurityGroups must be provided')

def validate_efs_subnets(self):
validator = '{}.{}'.format(type(self).__name__, 'validate_efs_subnets')
v = self.get_variables()

fs = t.add_resource(efs.FileSystem(
'EfsFileSystem',
FileSystemTags=efs.Tags(v['FileSystemTags']),
PerformanceMode=v['PerformanceMode']))
subnet_count = len(v['Subnets'])
if not subnet_count:
raise ValidatorError(
'Subnets', validator, v['Subnets'],
'At least one Subnet must be provided')

t.add_output(Output(
'EfsFileSystemId',
Value=Ref(fs)))
ip_count = len(v['IpAddresses'])
if ip_count and ip_count != subnet_count:
raise ValidatorError(
'IpAddresses', validator, v['IpAddresses'],
'The number of IpAddresses must match the number of Subnets')

return fs
def resolve_variables(self, provided_variables):
super(ElasticFileSystem, self).resolve_variables(provided_variables)

self.validate_efs_security_groups()
self.validate_efs_subnets()

def create_efs_security_groups(self):
def prepare_efs_security_groups(self):
t = self.template
v = self.get_variables()

new_sgs = []
created_groups = []
for sg in v['SecurityGroups']:
sg.VpcId = Ref('VpcId')
sg.validate()
sg.VpcId = v['VpcId']
sg.Tags = merge_tags(v['Tags'], getattr(sg, 'Tags', {}))

t.add_resource(sg)
new_sgs.append(Ref(sg))
sg = t.add_resource(sg)
created_groups.append(sg)

created_group_ids = list(map(Ref, created_groups))
t.add_output(Output(
'EfsSecurityGroupIds',
Value=Join(',', new_sgs)))
'EfsNewSecurityGroupIds',
Value=Join(',', created_group_ids)))

existing_sgs = v['ExtraSecurityGroups'].split(',')
return new_sgs + existing_sgs
groups_ids = created_group_ids + v['ExtraSecurityGroups']
return groups_ids

def create_efs_filesystem(self):
t = self.template
v = self.get_variables()

fs = t.add_resource(efs.FileSystem(
'EfsFileSystem',
FileSystemTags=Tags(v['Tags']),
PerformanceMode=v['PerformanceMode']))

t.add_output(Output(
'EfsFileSystemId',
Value=Ref(fs)))

return fs

def create_efs_mount_targets(self, fs, sgs):
def create_efs_mount_targets(self, fs):
t = self.template
v = self.get_variables()

subnets = v['Subnets'].split(',')
ips = v['IPAddresses'] and v['IPAddresses'].split(',')
if ips and len(ips) != len(subnets):
raise ValueError('Subnets and IPAddresses must have same count')
groups = self.prepare_efs_security_groups()
subnets = v['Subnets']
ips = v['IpAddresses']

mount_targets = []
for i, subnet in enumerate(subnets):
mount_target = efs.MountTarget(
'EfsMountTarget{}'.format(i + 1),
FileSystemId=Ref(fs),
SubnetId=subnet,
SecurityGroups=sgs)
SecurityGroups=groups)

if ips:
mount_target.IpAddress = ips[i]

t.add_resource(mount_target)
mount_target = t.add_resource(mount_target)
mount_targets.append(mount_target)

t.add_output(Output(
Expand All @@ -116,5 +160,4 @@ def create_efs_mount_targets(self, fs, sgs):

def create_template(self):
fs = self.create_efs_filesystem()
sgs = self.create_efs_security_groups()
self.create_efs_mount_targets(fs, sgs)
self.create_efs_mount_targets(fs)
39 changes: 39 additions & 0 deletions stacker_blueprints/util.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
from collections import Mapping

from troposphere import Tags


def check_properties(properties, allowed_properties, resource):
"""Checks the list of properties in the properties variable against the
property list provided by the allowed_properties variable. If any property
Expand All @@ -15,3 +20,37 @@ def check_properties(properties, allowed_properties, resource):
raise ValueError(
"%s is not a valid property of %s" % (key, resource)
)


def _tags_to_dict(tag_list):
return dict((tag['Key'], tag['Value']) for tag in tag_list)


def merge_tags(left, right, factory=Tags):
"""
Merge two sets of tags into a new troposphere object
Args:
left (Union[dict, troposphere.Tags]): dictionary or Tags object to be
merged with lower priority
right (Union[dict, troposphere.Tags]): dictionary or Tags object to be
merged with higher priority
factory (type): Type of object to create. Defaults to the troposphere
Tags class.
"""

if isinstance(left, Mapping):
tags = dict(left)
elif hasattr(left, 'tags'):
tags = _tags_to_dict(left.tags)
else:
tags = _tags_to_dict(left)

if isinstance(right, Mapping):
tags.update(right)
elif hasattr(left, 'tags'):
tags.update(_tags_to_dict(right.tags))
else:
tags.update(_tags_to_dict(right))

return factory(**tags)
30 changes: 21 additions & 9 deletions tests/fixtures/blueprints/test_efs_ElasticFileSystem.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
]
}
},
"EfsSecurityGroupIds": {
"EfsNewSecurityGroupIds": {
"Value": {
"Fn::Join": [
",",
Expand Down Expand Up @@ -100,9 +100,17 @@
"ToPort": 2049
}
],
"VpcId": {
"Ref": "VpcId"
}
"Tags": [
{
"Key": "Foo",
"Value": "Bar"
},
{
"Key": "Hello",
"Value": "World"
}
],
"VpcId": "vpc-11111111"
},
"Type": "AWS::EC2::SecurityGroup"
},
Expand All @@ -111,17 +119,21 @@
"GroupDescription": "EFS SG 2",
"SecurityGroupIngress": [
{
"FromPort": 2048,
"FromPort": 2049,
"IpProtocol": "tcp",
"SourceSecurityGroupId": "sg-11111111",
"ToPort": 2049
}
],
"VpcId": {
"Ref": "VpcId"
}
"Tags": [
{
"Key": "Hello",
"Value": "World"
}
],
"VpcId": "vpc-11111111"
},
"Type": "AWS::EC2::SecurityGroup"
}
}
}
}
Loading

0 comments on commit 573cc18

Please sign in to comment.