Merge pull request #119 from /issues/95

Issues/95

.travis.yml

@@ -20,12 +20,15 @@ env:
 - TOXENV=pypy3-versioncheck PIP_DOWNLOAD_CACHE=$HOME/.pip-cache
 - TOXENV=docs PIP_DOWNLOAD_CACHE=$HOME/.pip-cache
 - TOXENV=integration
+- TOXENV=integration3
 install:
+- virtualenv --version; test $TOXENV = "py32-unit" -o $TOXENV = "py32-versioncheck" -o $TOXENV = "pypy3-unit" -o $TOXENV = "pypy3-versioncheck" && pip install --upgrade virtualenv==13.1.2 || /bin/true
 - git config --global user.email "travisci@jasonantman.com"
 - git config --global user.name "travisci"
 - pip install tox
 - pip install codecov
 - pip freeze
+- virtualenv --version
 script:
 - tox -r
 after_success:

README.rst

@@ -57,7 +57,7 @@ Develop:
    :target: https://readthedocs.org/projects/awslimitchecker/?badge=develop
    :alt: sphinx documentation for develop branch
 
-A script and python module to check your AWS service limits and usage using `boto <http://docs.pythonboto.org/en/latest/>`_.
+A script and python module to check your AWS service limits and usage using `boto3 <http://boto3.readthedocs.org/>`_.
 
 Users building out scalable services in Amazon AWS often run into AWS' `service limits <http://docs.aws.amazon.com/general/latest/gr/aws_service_limits.html>`_ -
 often at the least convenient time (i.e. mid-deploy or when autoscaling fails). Amazon's `Trusted Advisor <https://aws.amazon.com/premiumsupport/trustedadvisor/>`_
@@ -71,8 +71,11 @@ Full project documentation is available at `http://awslimitchecker.readthedocs.o
 Status
 ------
 
-This project is currently in very early development. At this time please consider it beta code and not fully tested in all situations;
-furthermore its API may be changing rapidly. I hope to have this stabilized soon.
+This project has just undergone a relatively major refactor to migrate from
+[boto](http://boto.readthedocs.org) to [boto3](http://boto3.readthedocs.org/),
+along with a refactor of much of the connection and usage gathering code. Until
+it's been running in production for a while, please consider this to be "beta"
+and make every effort to manually confirm the results for your environment.
 
 What It Does
 ------------
@@ -91,10 +94,9 @@ What It Does
 Requirements
 ------------
 
-* Python 2.6 through 3.4. Python 2.x is recommended, as `boto <http://docs.pythonboto.org/en/latest/>`_ (the AWS client library) currently has
-  incomplete Python3 support. See the `boto documentation <http://boto.readthedocs.org/en/latest/>`_ for a list of AWS services that are Python3-compatible.
+* Python 2.6 through 3.5.
 * Python `VirtualEnv <http://www.virtualenv.org/>`_ and ``pip`` (recommended installation method; your OS/distribution should have packages for these)
-* `boto <http://docs.pythonboto.org/en/latest/>`_ >= 2.32.0
+* `boto3 <http://boto3.readthedocs.org/>`_ >= 1.2.3
 
 Installation
 ------------
@@ -114,7 +116,7 @@ Credentials
 Aside from STS, awslimitchecker does nothing with AWS credentials, it leaves that to boto itself.
 You must either have your credentials configured in one of boto's supported config
 files, or set as environment variables. See
-`boto config <http://docs.pythonboto.org/en/latest/boto_config_tut.html>`_
+`boto3 config <http://boto3.readthedocs.org/en/latest/guide/configuration.html#guide-configuration>`_
 and
 `this project's documentation <http://awslimitchecker.readthedocs.org/en/latest/getting_started.html#credentials>`_
 for further information.

awslimitchecker/checker.py

@@ -194,7 +194,7 @@ def get_service_names(self):
     def find_usage(self, service=None, use_ta=True):
         """
         For each limit in the specified service (or all services if
-        ``service`` is ``None``), query the AWS API via :py:mod:`boto`
+        ``service`` is ``None``), query the AWS API via ``boto3``
         and find the current usage amounts for that limit.
 
         This method updates the ``current_usage`` attribute of the

awslimitchecker/connectable.py

@@ -38,11 +38,29 @@
 """
 
 import logging
-import boto.sts
+import boto3
 
 logger = logging.getLogger(__name__)
 
 
+class ConnectableCredentials(object):
+    """
+    boto's (2.x) :py:meth:`boto.sts.STSConnection.assume_role` returns a
+    :py:class:`boto.sts.credentials.Credentials` object, but boto3's
+    `boto3.sts.STSConnection.assume_role <https://boto3.readthedocs.org/en/
+    latest/reference/services/sts.html#STS.Client.assume_role>`_ just returns
+    a dict. This class provides a compatible interface for boto3.
+    """
+
+    def __init__(self, creds_dict):
+        self.access_key = creds_dict['Credentials']['AccessKeyId']
+        self.secret_key = creds_dict['Credentials']['SecretAccessKey']
+        self.session_token = creds_dict['Credentials']['SessionToken']
+        self.expiration = creds_dict['Credentials']['Expiration']
+        self.assumed_role_id = creds_dict['AssumedRoleUser']['AssumedRoleId']
+        self.assumed_role_arn = creds_dict['AssumedRoleUser']['Arn']
+
+
 class Connectable(object):
 
     """
@@ -53,62 +71,97 @@ class Connectable(object):
     # Class attribute to reuse credentials between calls
     credentials = None
 
-    def connect_via(self, driver):
+    @property
+    def _boto3_connection_kwargs(self):
         """
-        Connect to an AWS API and return the connection object. If
-        ``self.account_id`` is None, call ``driver(self.region)``. Otherwise,
-        call :py:meth:`~._get_sts_token` to get STS token credentials using
-        :py:meth:`boto.sts.STSConnection.assume_role` and call ``driver()`` with
-        those credentials to use an assumed role.
-
-        :param driver: the connect_to_region() function of the boto
-          submodule to use to create this connection
-        :type driver: :py:obj:`function`
-        :returns: connected boto service class instance
+        Generate keyword arguments for boto3 connection functions.
+        If ``self.account_id`` is None, this will just include
+        ``region_name=self.region``. Otherwise, call
+        :py:meth:`~._get_sts_token_boto3` to get STS token credentials using
+        `boto3.STS.Client.assume_role <https://boto3.readthedocs.org/en/
+        latest/reference/services/sts.html#STS.Client.assume_role>`_ and include
+        those credentials in the return value.
+
+        :return: keyword arguments for boto3 connection functions
+        :rtype: dict
         """
+        kwargs = {'region_name': self.region}
         if self.account_id is not None:
             if Connectable.credentials is None:
-                logger.debug("Connecting to %s for account %s (STS; %s)",
-                             self.service_name, self.account_id, self.region)
-                Connectable.credentials = self._get_sts_token()
+                logger.debug("Connecting for account %s role '%s' with STS "
+                             "(region: %s)", self.account_id, self.account_role,
+                             self.region)
+                Connectable.credentials = self._get_sts_token_boto3()
             else:
                 logger.debug("Reusing previous STS credentials for account %s",
                              self.account_id)
-
-            conn = driver(
-                self.region,
-                aws_access_key_id=Connectable.credentials.access_key,
-                aws_secret_access_key=Connectable.credentials.secret_key,
-                security_token=Connectable.credentials.session_token)
+            kwargs['aws_access_key_id'] = Connectable.credentials.access_key
+            kwargs['aws_secret_access_key'] = Connectable.credentials.secret_key
+            kwargs['aws_session_token'] = Connectable.credentials.session_token
         else:
-            logger.debug("Connecting to %s (%s)",
-                         self.service_name, self.region)
-            conn = driver(self.region)
-        logger.info("Connected to %s", self.service_name)
-        return conn
+            logger.debug("Connecting to region %s", self.region)
+        return kwargs
 
-    def _get_sts_token(self):
+    def connect(self):
+        """
+        Connect to an AWS API via boto3 low-level client and set ``self.conn``
+        to the `boto3.client <https://boto3.readthed
+        ocs.org/en/latest/reference/core/boto3.html#boto3.client>`_ object
+        (a ``botocore.client.*`` instance). If ``self.conn`` is not None,
+        do nothing. This connects to the API name given by ``self.api_name``.
+
+        :returns: None
+        """
+        if self.conn is not None:
+            return
+        kwargs = self._boto3_connection_kwargs
+        self.conn = boto3.client(self.api_name, **kwargs)
+        logger.info("Connected to %s in region %s", self.api_name,
+                    self.conn._client_config.region_name)
+
+    def connect_resource(self):
+        """
+        Connect to an AWS API via boto3 high-level resource connection and set
+        ``self.resource_conn`` to the `boto3.resource <https://boto3.readthed
+        ocs.org/en/latest/reference/core/boto3.html#boto3.resource>`_ object
+        (a ``boto3.resources.factory.*.ServiceResource`` instance).
+        If ``self.resource_conn`` is not None,
+        do nothing. This connects to the API name given by ``self.api_name``.
+
+        :returns: None
+        """
+        if self.resource_conn is not None:
+            return
+        kwargs = self._boto3_connection_kwargs
+        self.resource_conn = boto3.resource(self.api_name, **kwargs)
+        logger.info("Connected to %s (resource) in region %s", self.api_name,
+                    self.resource_conn.meta.client._client_config.region_name)
+
+    def _get_sts_token_boto3(self):
         """
         Assume a role via STS and return the credentials.
 
-        First connect to STS via :py:func:`boto.sts.connect_to_region`, then
-        assume a role using :py:meth:`boto.sts.STSConnection.assume_role`
+        First connect to STS via :py:func:`boto3.client`, then
+        assume a role using `boto3.STS.Client.assume_role <https://boto3.readthe
+        docs.org/en/latest/reference/services/sts.html#STS.Client.assume_role>`_
         using ``self.account_id`` and ``self.account_role`` (and optionally
         ``self.external_id``, ``self.mfa_serial_number``, ``self.mfa_token``).
-        Return the resulting :py:class:`boto.sts.credentials.Credentials`
+        Return the resulting :py:class:`~.ConnectableCredentials`
         object.
 
         :returns: STS assumed role credentials
-        :rtype: :py:class:`boto.sts.credentials.Credentials`
+        :rtype: :py:class:`~.ConnectableCredentials`
         """
         logger.debug("Connecting to STS in region %s", self.region)
-        sts = boto.sts.connect_to_region(self.region)
+        sts = boto3.client('sts', region_name=self.region)
         arn = "arn:aws:iam::%s:role/%s" % (self.account_id, self.account_role)
         logger.debug("STS assume role for %s", arn)
-        role = sts.assume_role(arn, "awslimitchecker",
-                               external_id=self.external_id,
-                               mfa_serial_number=self.mfa_serial_number,
-                               mfa_token=self.mfa_token)
+        role = sts.assume_role(RoleArn=arn,
+                               RoleSessionName="awslimitchecker",
+                               ExternalId=self.external_id,
+                               SerialNumber=self.mfa_serial_number,
+                               TokenCode=self.mfa_token)
+        creds = ConnectableCredentials(role)
         logger.debug("Got STS credentials for role; access_key_id=%s",
-                     role.credentials.access_key)
-        return role.credentials
+                     creds.access_key)
+        return creds

awslimitchecker/runner.py

@@ -50,10 +50,15 @@
 logging.basicConfig(level=logging.WARNING)
 logger = logging.getLogger()
 
-# suppress boto internal logging below WARNING level
-boto_log = logging.getLogger("boto")
-boto_log.setLevel(logging.WARNING)
-boto_log.propagate = True
+# suppress boto3 internal logging below WARNING level
+boto3_log = logging.getLogger("boto3")
+boto3_log.setLevel(logging.WARNING)
+boto3_log.propagate = True
+
+# suppress botocore internal logging below WARNING level
+botocore_log = logging.getLogger("botocore")
+botocore_log.setLevel(logging.WARNING)
+botocore_log.propagate = True
 
 
 class Runner(object):
@@ -73,7 +78,7 @@ def parse_args(self, argv):
         :returns: parsed arguments
         :rtype: :py:class:`argparse.Namespace`
         """
-        desc = 'Report on AWS service limits and usage via boto, optionally ' \
+        desc = 'Report on AWS service limits and usage via boto3, optionally ' \
                'warn about any services with usage nearing or exceeding their' \
                ' limits. For further help, see ' \
                '<http://awslimitchecker.readthedocs.org/>'

awslimitchecker/services/autoscaling.py

@@ -38,8 +38,6 @@
 """
 
 import abc  # noqa
-import boto
-import boto.ec2.autoscale
 import logging
 
 from .base import _AwsService
@@ -52,15 +50,7 @@
 class _AutoscalingService(_AwsService):
 
     service_name = 'AutoScaling'
-
-    def connect(self):
-        """Connect to API if not already connected; set self.conn."""
-        if self.conn is not None:
-            return
-        elif self.region:
-            self.conn = self.connect_via(boto.ec2.autoscale.connect_to_region)
-        else:
-            self.conn = boto.connect_autoscale()
+    api_name = 'autoscaling'
 
     def find_usage(self):
         """
@@ -75,16 +65,24 @@ def find_usage(self):
 
         self.limits['Auto Scaling groups']._add_current_usage(
             len(
-                boto_query_wrapper(self.conn.get_all_groups)
+                boto_query_wrapper(
+                    self.conn.describe_auto_scaling_groups,
+                    alc_marker_path=['NextToken'],
+                    alc_data_path=['AutoScalingGroups'],
+                    alc_marker_param='NextToken'
+                )['AutoScalingGroups']
             ),
             aws_type='AWS::AutoScaling::AutoScalingGroup',
         )
 
         self.limits['Launch configurations']._add_current_usage(
             len(
                 boto_query_wrapper(
-                    self.conn.get_all_launch_configurations
-                )
+                    self.conn.describe_launch_configurations,
+                    alc_marker_path=['NextToken'],
+                    alc_data_path=['LaunchConfigurations'],
+                    alc_marker_param='NextToken'
+                )['LaunchConfigurations']
             ),
             aws_type='AWS::AutoScaling::LaunchConfiguration',
         )
@@ -145,9 +143,8 @@ def _update_limits_from_api(self):
         """
         self.connect()
         logger.info("Querying EC2 DescribeAccountAttributes for limits")
-        lims = boto_query_wrapper(self.conn.get_account_limits)
+        lims = boto_query_wrapper(self.conn.describe_account_limits)
         self.limits['Auto Scaling groups']._set_api_limit(
-            lims.max_autoscaling_groups)
+            lims['MaxNumberOfAutoScalingGroups'])
         self.limits['Launch configurations']._set_api_limit(
-            lims.max_launch_configurations
-        )
+            lims['MaxNumberOfLaunchConfigurations'])

awslimitchecker/services/base.py

@@ -48,6 +48,7 @@ class _AwsService(Connectable):
     __metaclass__ = abc.ABCMeta
 
     service_name = 'baseclass'
+    api_name = 'baseclass'
 
     def __init__(self, warning_threshold, critical_threshold, account_id=None,
                  account_role=None, region=None, external_id=None,
@@ -104,27 +105,9 @@ def __init__(self, warning_threshold, critical_threshold, account_id=None,
         self.limits = {}
         self.limits = self.get_limits()
         self.conn = None
+        self.resource_conn = None
         self._have_usage = False
 
-    @abc.abstractmethod
-    def connect(self):
-        """
-        If not already done, establish a connection to the relevant AWS service
-        and save as ``self.conn``. If ``self.region`` is defined, call
-        ``self.connect_via()`` (:py:meth:`~.Connectable.connect_via`)
-        passing the appripriate boto ``connect_to_region()`` function as the
-        argument, else call the boto.connect_SERVICE_NAME() method directly.
-        """
-        """
-        if self.conn is not None:
-            return
-        elif self.region:
-            self.conn = self.connect_via(boto.ec2.connect_to_region)
-        else:
-            self.conn = boto.connect_ec2()
-        """
-        raise NotImplementedError('abstract base class')
-
     @abc.abstractmethod
     def find_usage(self):
         """