From 6dd8ea8ec9db01589130b65e96414bfc751e38c2 Mon Sep 17 00:00:00 2001 From: Jessie Nadler Date: Thu, 12 Jan 2017 15:11:09 -0500 Subject: [PATCH 1/4] Start _RedshiftService implementation --- awslimitchecker/services/__init__.py | 1 + awslimitchecker/services/redshift.py | 115 +++++++++++++++ .../tests/services/result_fixtures.py | 66 +++++++++ .../tests/services/test_redshift.py | 132 ++++++++++++++++++ .../awslimitchecker.services.redshift.rst | 7 + docs/source/awslimitchecker.services.rst | 1 + 6 files changed, 322 insertions(+) create mode 100644 awslimitchecker/services/redshift.py create mode 100644 awslimitchecker/tests/services/test_redshift.py create mode 100644 docs/source/awslimitchecker.services.redshift.rst diff --git a/awslimitchecker/services/__init__.py b/awslimitchecker/services/__init__.py index ba28eec5..5c0a4997 100644 --- a/awslimitchecker/services/__init__.py +++ b/awslimitchecker/services/__init__.py @@ -51,6 +51,7 @@ from awslimitchecker.services.ses import _SesService from awslimitchecker.services.cloudformation import _CloudformationService from awslimitchecker.services.firehose import _FirehoseService +from awslimitchecker.services.redshift import _RedshiftService # dynamically generate the service name to class dict _services = {} diff --git a/awslimitchecker/services/redshift.py b/awslimitchecker/services/redshift.py new file mode 100644 index 00000000..af1216fb --- /dev/null +++ b/awslimitchecker/services/redshift.py @@ -0,0 +1,115 @@ +""" +awslimitchecker/services/redshift.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2016 Jessie Nadler + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jessie Nadler +################################################################################ +""" + +import abc # noqa +import logging + +from .base import _AwsService +from ..limit import AwsLimit + +logger = logging.getLogger(__name__) + + +class _RedshiftService(_AwsService): + + service_name = 'Redshift' + api_name = 'redshift' + + def find_usage(self): + """ + Determine the current usage for each limit of this service, + and update corresponding Limit via + :py:meth:`~.AwsLimit._add_current_usage`. + """ + logger.debug("Checking usage for service %s", self.service_name) + self.connect() + for lim in self.limits.values(): + lim._reset_usage() + self._find_cluster_manual_snapshots() + self._have_usage = True + logger.debug("Done checking usage.") + + def _find_cluster_manual_snapshots(self): + snapshots = self.conn.describe_cluster_snapshots(SnapshotType='manual') + usage = len(snapshots['Snapshots']) + while snapshots.get('Marker'): + marker = snapshots['Marker'] + snapshots = self.conn.describe_cluster_snapshots( + Marker=marker, SnapshotType='manual') + usage += len(snapshots['Snapshots']) + self.limits['Redshift manual snapshots']._add_current_usage( + usage, + resource_id=self.region, + aws_type='AWS::Redshift::Snapshot', + ) + + def get_limits(self): + """ + Return all known limits for this service, as a dict of their names + to :py:class:`~.AwsLimit` objects. + + :returns: dict of limit names to :py:class:`~.AwsLimit` objects + :rtype: dict + """ + if self.limits != {}: + return self.limits + limits = {} + limits['Redshift manual snapshots'] = AwsLimit( + 'Redshift manual snapshots', + self, + 20, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::Redshift::Snapshot', + ) + self.limits = limits + return limits + + def required_iam_permissions(self): + """ + Return a list of IAM Actions required for this Service to function + properly. All Actions will be shown with an Effect of "Allow" + and a Resource of "*". + + :returns: list of IAM Action strings + :rtype: list + """ + return [ + "redshift:DescribeClusterSnapshots", + ] diff --git a/awslimitchecker/tests/services/result_fixtures.py b/awslimitchecker/tests/services/result_fixtures.py index 0adb30ba..c8dc17a2 100644 --- a/awslimitchecker/tests/services/result_fixtures.py +++ b/awslimitchecker/tests/services/result_fixtures.py @@ -2041,3 +2041,69 @@ class Firehose(object): } } ] + + +class Redshift(object): + + test_describe_cluster_snapshots = { + 'Snapshots': [ + { + "EstimatedSecondsToCompletion": 0, + "OwnerAccount": "123456789", + "CurrentBackupRateInMegaBytesPerSecond": 1.0, + "ActualIncrementalBackupSizeInMegaBytes": 1.0, + "NumberOfNodes": 1, + "Status": "available", + "VpcId": "vpc-123456", + "ClusterVersion": "1.0", + "Tags": [], + "MasterUsername": "username", + "TotalBackupSizeInMegaBytes": 10.0, + "DBName": "test", + "BackupProgressInMegaBytes": 4.0, + "ClusterCreateTime": "2017-01-01T00:00:00.000Z", + "RestorableNodeTypes": [ + "dc1.large" + ], + "EncryptedWithHSM": False, + "ClusterIdentifier": "test12346", + "SnapshotCreateTime": "2017-01-04T00:00:00.000Z", + "AvailabilityZone": "us-east-1e", + "NodeType": "dc1.large", + "Encrypted": False, + "ElapsedTimeInSeconds": 0, + "SnapshotType": "manual", + "Port": 1234, + "SnapshotIdentifier": "snapshot1" + }, + { + "EstimatedSecondsToCompletion": 0, + "OwnerAccount": "123456789", + "CurrentBackupRateInMegaBytesPerSecond": 1.0, + "ActualIncrementalBackupSizeInMegaBytes": 1.0, + "NumberOfNodes": 1, + "Status": "available", + "VpcId": "vpc-123456", + "ClusterVersion": "1.0", + "Tags": [], + "MasterUsername": "username", + "TotalBackupSizeInMegaBytes": 10.0, + "DBName": "test", + "BackupProgressInMegaBytes": 4.0, + "ClusterCreateTime": "2017-01-01T00:00:00.000Z", + "RestorableNodeTypes": [ + "dc1.large" + ], + "EncryptedWithHSM": False, + "ClusterIdentifier": "test12346", + "SnapshotCreateTime": "2017-01-04T00:00:00.000Z", + "AvailabilityZone": "us-east-1e", + "NodeType": "dc1.large", + "Encrypted": False, + "ElapsedTimeInSeconds": 0, + "SnapshotType": "manual", + "Port": 1234, + "SnapshotIdentifier": "snapshot2" + } + ] + } diff --git a/awslimitchecker/tests/services/test_redshift.py b/awslimitchecker/tests/services/test_redshift.py new file mode 100644 index 00000000..44b24c9e --- /dev/null +++ b/awslimitchecker/tests/services/test_redshift.py @@ -0,0 +1,132 @@ +""" +awslimitchecker/tests/services/test_redshift.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015 Jessie Nadler + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jessie Nadler +################################################################################ +""" + +import sys +from awslimitchecker.tests.services import result_fixtures +from awslimitchecker.services.redshift import _RedshiftService + +# https://code.google.com/p/mock/issues/detail?id=249 +# py>=3.4 should use unittest.mock not the mock package on pypi +if ( + sys.version_info[0] < 3 or + sys.version_info[0] == 3 and sys.version_info[1] < 4 +): + from mock import patch, call, Mock, DEFAULT +else: + from unittest.mock import patch, call, Mock, DEFAULT + + +pbm = 'awslimitchecker.services.redshift' # module patch base +pb = '%s._RedshiftService' % pbm # class patch pase + + +class Test_RedshiftService(object): + + def test_init(self): + """test __init__()""" + cls = _RedshiftService(21, 43) + assert cls.service_name == 'Redshift' + assert cls.api_name == 'redshift' + assert cls.conn is None + assert cls.warning_threshold == 21 + assert cls.critical_threshold == 43 + + def test_get_limits(self): + cls = _RedshiftService(21, 43) + cls.limits = {} + res = cls.get_limits() + assert sorted(res.keys()) == sorted([ + 'Redshift manual snapshots', + ]) + for name, limit in res.items(): + assert limit.service == cls + assert limit.def_warning_threshold == 21 + assert limit.def_critical_threshold == 43 + + def test_get_limits_again(self): + """test that existing limits dict is returned on subsequent calls""" + mock_limits = Mock() + cls = _RedshiftService(21, 43) + cls.limits = mock_limits + res = cls.get_limits() + assert res == mock_limits + + def test_find_usage(self): + """test find usage method calls other methods""" + mock_conn = Mock() + with patch('%s.connect' % pb) as mock_connect: + with patch.multiple( + pb, + _find_cluster_manual_snapshots=DEFAULT, + ) as mocks: + cls = _RedshiftService(21, 43) + cls.conn = mock_conn + assert cls._have_usage is False + cls.find_usage() + assert mock_connect.mock_calls == [call()] + assert cls._have_usage is True + assert mock_conn.mock_calls == [] + for x in [ + '_find_cluster_manual_snapshots', + ]: + assert mocks[x].mock_calls == [call()] + + def test_find_usage_manual_snapshots(self): + response = result_fixtures.Redshift.test_describe_cluster_snapshots + limit_key = 'Redshift manual snapshots' + + mock_conn = Mock() + mock_conn.describe_cluster_snapshots.return_value = response + + cls = _RedshiftService(21, 43) + cls.conn = mock_conn + cls._find_cluster_manual_snapshots() + + assert mock_conn.mock_calls == [ + call.describe_cluster_snapshots(SnapshotType='manual') + ] + assert len(cls.limits[limit_key].get_current_usage()) == 1 + assert cls.limits[limit_key].get_current_usage()[ + 0].get_value() == 2 + + def test_required_iam_permissions(self): + cls = _RedshiftService(21, 43) + assert cls.required_iam_permissions() == [ + "redshift:DescribeClusterSnapshots", + ] diff --git a/docs/source/awslimitchecker.services.redshift.rst b/docs/source/awslimitchecker.services.redshift.rst new file mode 100644 index 00000000..854134d6 --- /dev/null +++ b/docs/source/awslimitchecker.services.redshift.rst @@ -0,0 +1,7 @@ +awslimitchecker.services.redshift module +======================================== + +.. automodule:: awslimitchecker.services.redshift + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.rst b/docs/source/awslimitchecker.services.rst index 799eb451..b90c1940 100644 --- a/docs/source/awslimitchecker.services.rst +++ b/docs/source/awslimitchecker.services.rst @@ -22,6 +22,7 @@ Submodules awslimitchecker.services.firehose awslimitchecker.services.iam awslimitchecker.services.rds + awslimitchecker.services.redshift awslimitchecker.services.s3 awslimitchecker.services.ses awslimitchecker.services.vpc From fcd0d173b1c1b9d47e52d810ac0e939567f52516 Mon Sep 17 00:00:00 2001 From: Jessie Nadler Date: Thu, 12 Jan 2017 15:15:36 -0500 Subject: [PATCH 2/4] Add limit for redshift cluster subnet groups --- awslimitchecker/services/redshift.py | 24 ++++++ .../tests/services/result_fixtures.py | 74 +++++++++++++++++++ .../tests/services/test_redshift.py | 20 +++++ 3 files changed, 118 insertions(+) diff --git a/awslimitchecker/services/redshift.py b/awslimitchecker/services/redshift.py index af1216fb..c253c88e 100644 --- a/awslimitchecker/services/redshift.py +++ b/awslimitchecker/services/redshift.py @@ -62,6 +62,7 @@ def find_usage(self): for lim in self.limits.values(): lim._reset_usage() self._find_cluster_manual_snapshots() + self._find_cluster_subnet_groups() self._have_usage = True logger.debug("Done checking usage.") @@ -79,6 +80,20 @@ def _find_cluster_manual_snapshots(self): aws_type='AWS::Redshift::Snapshot', ) + def _find_cluster_subnet_groups(self): + subnet_groups = self.conn.describe_cluster_subnet_groups() + usage = len(subnet_groups['ClusterSubnetGroups']) + while subnet_groups.get('Marker'): + marker = subnet_groups['Marker'] + subnet_groups = self.conn.describe_cluster_subnet_groups( + Marker=marker) + usage += len(subnet_groups['ClusterSubnetGroups']) + self.limits['Redshift subnet groups']._add_current_usage( + usage, + resource_id=self.region, + aws_type='AWS::Redshift::SubnetGroup', + ) + def get_limits(self): """ Return all known limits for this service, as a dict of their names @@ -98,6 +113,14 @@ def get_limits(self): self.critical_threshold, limit_type='AWS::Redshift::Snapshot', ) + limits['Redshift subnet groups'] = AwsLimit( + 'Redshift subnet groups', + self, + 20, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::Redshift::SubnetGroup', + ) self.limits = limits return limits @@ -112,4 +135,5 @@ def required_iam_permissions(self): """ return [ "redshift:DescribeClusterSnapshots", + "redshift:DescribeClusterSubnetGroups", ] diff --git a/awslimitchecker/tests/services/result_fixtures.py b/awslimitchecker/tests/services/result_fixtures.py index c8dc17a2..29b9c4e3 100644 --- a/awslimitchecker/tests/services/result_fixtures.py +++ b/awslimitchecker/tests/services/result_fixtures.py @@ -2107,3 +2107,77 @@ class Redshift(object): } ] } + + test_describe_cluster_subnet_groups = { + "ClusterSubnetGroups": [ + { + "Subnets": [ + { + "SubnetStatus": "Active", + "SubnetIdentifier": "subnet-1", + "SubnetAvailabilityZone": { + "Name": "region-name" + } + }, + { + "SubnetStatus": "Active", + "SubnetIdentifier": "subnet-2", + "SubnetAvailabilityZone": { + "Name": "alt-region-name" + } + } + ], + "VpcId": "vpc-1", + "Description": "Redshift Subnet Group for Test1", + "Tags": [], + "SubnetGroupStatus": "Complete", + "ClusterSubnetGroupName": "groupname1" + }, + { + "Subnets": [ + { + "SubnetStatus": "Active", + "SubnetIdentifier": "subnet-3", + "SubnetAvailabilityZone": { + "Name": "alt-region-name" + } + }, + { + "SubnetStatus": "Active", + "SubnetIdentifier": "subnet-4", + "SubnetAvailabilityZone": { + "Name": "region-name" + } + } + ], + "VpcId": "vpc-2", + "Description": "Redshift Subnet Group for Test2", + "Tags": [], + "SubnetGroupStatus": "Complete", + "ClusterSubnetGroupName": "groupname2" + }, + { + "Subnets": [ + { + "SubnetStatus": "Active", + "SubnetIdentifier": "subnet-5", + "SubnetAvailabilityZone": { + "Name": "alt-region-name" + } + }, + { + "SubnetStatus": "Active", + "SubnetIdentifier": "subnet-6", + "SubnetAvailabilityZone": { + "Name": "region-name" + } + } + ], + "VpcId": "vpc-3", + "Description": "Redshift Subnet Group for Test3", + "Tags": [], + "SubnetGroupStatus": "Complete", + "ClusterSubnetGroupName": "groupname3" + } + ] + } diff --git a/awslimitchecker/tests/services/test_redshift.py b/awslimitchecker/tests/services/test_redshift.py index 44b24c9e..53fdce2a 100644 --- a/awslimitchecker/tests/services/test_redshift.py +++ b/awslimitchecker/tests/services/test_redshift.py @@ -73,6 +73,7 @@ def test_get_limits(self): res = cls.get_limits() assert sorted(res.keys()) == sorted([ 'Redshift manual snapshots', + 'Redshift subnet groups', ]) for name, limit in res.items(): assert limit.service == cls @@ -94,6 +95,7 @@ def test_find_usage(self): with patch.multiple( pb, _find_cluster_manual_snapshots=DEFAULT, + _find_cluster_subnet_groups=DEFAULT, ) as mocks: cls = _RedshiftService(21, 43) cls.conn = mock_conn @@ -104,6 +106,7 @@ def test_find_usage(self): assert mock_conn.mock_calls == [] for x in [ '_find_cluster_manual_snapshots', + '_find_cluster_subnet_groups', ]: assert mocks[x].mock_calls == [call()] @@ -125,8 +128,25 @@ def test_find_usage_manual_snapshots(self): assert cls.limits[limit_key].get_current_usage()[ 0].get_value() == 2 + def test_find_usage_subnet_groups(self): + response = result_fixtures.Redshift.test_describe_cluster_subnet_groups + limit_key = 'Redshift subnet groups' + + mock_conn = Mock() + mock_conn.describe_cluster_subnet_groups.return_value = response + + cls = _RedshiftService(21, 43) + cls.conn = mock_conn + cls._find_cluster_subnet_groups() + + assert mock_conn.mock_calls == [call.describe_cluster_subnet_groups()] + assert len(cls.limits[limit_key].get_current_usage()) == 1 + assert cls.limits[limit_key].get_current_usage()[ + 0].get_value() == 3 + def test_required_iam_permissions(self): cls = _RedshiftService(21, 43) assert cls.required_iam_permissions() == [ "redshift:DescribeClusterSnapshots", + "redshift:DescribeClusterSubnetGroups", ] From 172312d7de43b264eb704c2347f592962f14bccd Mon Sep 17 00:00:00 2001 From: Jessie Nadler Date: Thu, 12 Jan 2017 15:18:47 -0500 Subject: [PATCH 3/4] Refactor _RedshiftService functions to use paginate_dict --- awslimitchecker/services/redshift.py | 32 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/awslimitchecker/services/redshift.py b/awslimitchecker/services/redshift.py index c253c88e..f1b940a1 100644 --- a/awslimitchecker/services/redshift.py +++ b/awslimitchecker/services/redshift.py @@ -42,6 +42,7 @@ from .base import _AwsService from ..limit import AwsLimit +from ..utils import paginate_dict logger = logging.getLogger(__name__) @@ -67,29 +68,28 @@ def find_usage(self): logger.debug("Done checking usage.") def _find_cluster_manual_snapshots(self): - snapshots = self.conn.describe_cluster_snapshots(SnapshotType='manual') - usage = len(snapshots['Snapshots']) - while snapshots.get('Marker'): - marker = snapshots['Marker'] - snapshots = self.conn.describe_cluster_snapshots( - Marker=marker, SnapshotType='manual') - usage += len(snapshots['Snapshots']) + results = paginate_dict( + self.conn.describe_cluster_snapshots, + alc_marker_path=['Marker'], + alc_data_path=['Snapshots'], + alc_marker_param='Marker', + SnapshotType='manual' + ) self.limits['Redshift manual snapshots']._add_current_usage( - usage, + len(results['Snapshots']), resource_id=self.region, aws_type='AWS::Redshift::Snapshot', ) def _find_cluster_subnet_groups(self): - subnet_groups = self.conn.describe_cluster_subnet_groups() - usage = len(subnet_groups['ClusterSubnetGroups']) - while subnet_groups.get('Marker'): - marker = subnet_groups['Marker'] - subnet_groups = self.conn.describe_cluster_subnet_groups( - Marker=marker) - usage += len(subnet_groups['ClusterSubnetGroups']) + results = paginate_dict( + self.conn.describe_cluster_subnet_groups, + alc_marker_path=['Marker'], + alc_data_path=['ClusterSubnetGroups'], + alc_marker_param='Marker' + ) self.limits['Redshift subnet groups']._add_current_usage( - usage, + len(results['ClusterSubnetGroups']), resource_id=self.region, aws_type='AWS::Redshift::SubnetGroup', ) From 193bec13fb6ed592c1044816ce5a3fe46cc9b53c Mon Sep 17 00:00:00 2001 From: Jessie Nadler Date: Wed, 18 Jan 2017 15:10:48 -0500 Subject: [PATCH 4/4] Assign resource_id from boto3 connection kwargs --- awslimitchecker/services/redshift.py | 4 ++-- awslimitchecker/tests/services/test_redshift.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/awslimitchecker/services/redshift.py b/awslimitchecker/services/redshift.py index f1b940a1..0451edf2 100644 --- a/awslimitchecker/services/redshift.py +++ b/awslimitchecker/services/redshift.py @@ -77,7 +77,7 @@ def _find_cluster_manual_snapshots(self): ) self.limits['Redshift manual snapshots']._add_current_usage( len(results['Snapshots']), - resource_id=self.region, + resource_id=self._boto3_connection_kwargs['region_name'], aws_type='AWS::Redshift::Snapshot', ) @@ -90,7 +90,7 @@ def _find_cluster_subnet_groups(self): ) self.limits['Redshift subnet groups']._add_current_usage( len(results['ClusterSubnetGroups']), - resource_id=self.region, + resource_id=self._boto3_connection_kwargs['region_name'], aws_type='AWS::Redshift::SubnetGroup', ) diff --git a/awslimitchecker/tests/services/test_redshift.py b/awslimitchecker/tests/services/test_redshift.py index 53fdce2a..58be1a17 100644 --- a/awslimitchecker/tests/services/test_redshift.py +++ b/awslimitchecker/tests/services/test_redshift.py @@ -117,7 +117,7 @@ def test_find_usage_manual_snapshots(self): mock_conn = Mock() mock_conn.describe_cluster_snapshots.return_value = response - cls = _RedshiftService(21, 43) + cls = _RedshiftService(21, 43, {'region_name': 'us-west-2'}) cls.conn = mock_conn cls._find_cluster_manual_snapshots() @@ -135,7 +135,7 @@ def test_find_usage_subnet_groups(self): mock_conn = Mock() mock_conn.describe_cluster_subnet_groups.return_value = response - cls = _RedshiftService(21, 43) + cls = _RedshiftService(21, 43, {'region_name': 'us-west-2'}) cls.conn = mock_conn cls._find_cluster_subnet_groups()