Merge pull request #92 from secynic/dev

0.11.2

.travis.yml

@@ -1,19 +1,15 @@
-# Build file
 language: python
 python:
-  - 3.5
-  - 3.4 
-  - 3.3 
-  - 2.7
   - 2.6
-# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
+  - 2.7
+  - 3.3
+  - 3.4
+  - 3.5
 install:
   - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements/python2.txt --use-mirrors; fi
   - if [[ $TRAVIS_PYTHON_VERSION == 3.3* ]]; then pip install -r requirements/python3.txt --use-mirrors; fi
   - if [[ $TRAVIS_PYTHON_VERSION == 3.4* ]]; then pip install -r requirements/python3.txt --use-mirrors; fi
   - if [[ $TRAVIS_PYTHON_VERSION == 3.5* ]]; then pip install -r requirements/python3.txt; fi
   - python setup.py -q install
-# command to run tests, e.g. python setup.py test
 script:
-  - nosetests -v  -w ipwhois
-
+  - nosetests -v -w ipwhois

CHANGES.rst

@@ -1,6 +1,17 @@
 Changelog
 =========
 
+0.11.2 (2016-02-25)
+-------------------
+
+- Added allow_permutations parameter (bool) to net.Net() and ipwhois.IPWhois()
+  to allow alternate ASN lookups if DNS lookups fail. (FirefighterBlu3)
+- Fixed ASN DNS resolver timeout/retry_count support. Retry count is used as a
+  multiplier of timeout, to determine a limetime interval. (FirefighterBlu3)
+- Fixed bug where remarks would return None if missing a title.
+- Added CONTRIBUTING.rst
+- Added tests
+
 0.11.1 (2015-12-17)
 -------------------
 

CONTRIBUTING.rst

@@ -0,0 +1,80 @@
+============
+Contributing
+============
+
+****************
+Issue submission
+****************
+
+| Issues are tracked on GitHub:
+| https://github.com/secynic/ipwhois/issues
+
+
+Follow the guidelines detailed in the appropriate section below. As a general
+rule of thumb, provide as much information as possible when submitting issues.
+
+Bug reports
+===========
+
+- Title should be a short, descriptive summary of the bug
+- Include the Python and ipwhois versions affected
+- Provide a context (with code example) in the description of your issue. What
+  are you attempting to do?
+- Include the full obfuscated output. Make sure to set DEBUG logging:
+  ::
+
+    import logging
+    LOG_FORMAT = ('[%(asctime)s] [%(levelname)s] [%(filename)s:%(lineno)s] '
+       '[%(funcName)s()] %(message)s')
+    logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
+- Include sources of information with links or screenshots
+- Do you have a suggestion on how to fix the bug?
+
+Feature Requests
+================
+
+- Title should be a short, descriptive summary of the feature requested
+- Provide use case examples
+- Include sources of information with links or screenshots
+- Do you have a suggestion on how to implement the feature?
+
+Questions
+=========
+
+I am happy to answer any questions and provide assistance where possible.
+Please be clear and concise. Provide examples when possible. Check the
+ipwhois `documentation <https://secynic.github.io/ipwhois>`_ and the
+`issue tracker <https://github.com/secynic/ipwhois/issues>`_ before asking a
+question.
+
+*************
+Pull Requests
+*************
+
+What to include
+===============
+
+Aside from the core code changes, it is helpful to provide the following
+(where applicable):
+
+- Unit tests
+- Examples
+- Sphinx configuration changes in /data
+- Requirements (python2.txt, python3.txt)
+
+Guidelines
+==========
+
+- Title should be a short, descriptive summary of the changes
+- Follow `PEP 8 <https://www.python.org/dev/peps/pep-0008/>`_ where possible.
+- Follow the `Google docstring style guide
+  <https://google.github.io/styleguide/pyguide.html#Comments>`_ for
+  comments
+- Must be compatible with Python 2.6, 2.7, and 3.3+
+- Break out reusable code to functions
+- Make your code easy to read and comment where necessary
+- Reference the GitHub issue number in the description (e.g., Issue #01)
+- When running nosetests, make sure to add the following arguments:
+  ::
+
+    --verbosity=3 --nologcapture --include=online --cover-erase

LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013, 2014, 2015 Philip Hane
+Copyright (c) 2013, 2014, 2015, 2016 Philip Hane
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

ipwhois/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2013, 2014, 2015 Philip Hane
+# Copyright (c) 2013, 2014, 2015, 2016 Philip Hane
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -22,7 +22,7 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-__version__ = '0.11.1'
+__version__ = '0.11.2'
 
 from .exceptions import *
 from .net import Net

ipwhois/docs/source/CONTRIBUTING.rst

@@ -0,0 +1 @@
+.. include:: ../../../CONTRIBUTING.rst

ipwhois/docs/source/conf.py

@@ -54,16 +54,16 @@
 
 # General information about the project.
 project = 'ipwhois'
-copyright = '2015, Philip Hane'
+copyright = '2016, Philip Hane'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '0.11.1'
+version = '0.11.2'
 # The full version, including alpha/beta/rc tags.
-release = '0.11.1'
+release = '0.11.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

ipwhois/docs/source/index.rst

@@ -13,6 +13,11 @@ Project Info
 
    Readme <README>
 
+.. toctree::
+   :maxdepth: 2
+
+   Contributing <CONTRIBUTING>
+
 .. toctree::
    :maxdepth: 2
 

ipwhois/exceptions.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2013, 2014, 2015 Philip Hane
+# Copyright (c) 2013, 2014, 2015, 2016 Philip Hane
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

ipwhois/ipwhois.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2013, 2014, 2015 Philip Hane
+# Copyright (c) 2013, 2014, 2015, 2016 Philip Hane
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -39,11 +39,14 @@ class IPWhois:
         timeout: The default timeout for socket connections in seconds.
         proxy_opener: The urllib.request.OpenerDirector request for proxy
             support or None.
+        allow_permutations: allow net.Net() to use additional methods if DNS
+            lookups to Cymru fail.
     """
 
-    def __init__(self, address, timeout=5, proxy_opener=None):
+    def __init__(self, address, timeout=5, proxy_opener=None,
+                 allow_permutations=True):
 
-        self.net = Net(address, timeout, proxy_opener)
+        self.net = Net(address, timeout, proxy_opener, allow_permutations)
 
         self.address = self.net.address
         self.timeout = self.net.timeout

ipwhois/net.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2013, 2014, 2015 Philip Hane
+# Copyright (c) 2013, 2014, 2015, 2016 Philip Hane
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -89,13 +89,16 @@ class Net:
         timeout: The default timeout for socket connections in seconds.
         proxy_opener: The urllib.request.OpenerDirector request for proxy
             support or None.
+        allow_permutations: Use additional methods if DNS lookups to Cymru
+            fail.
 
     Raises:
         IPDefinedError: The address provided is defined (does not need to be
             resolved).
     """
 
-    def __init__(self, address, timeout=5, proxy_opener=None):
+    def __init__(self, address, timeout=5, proxy_opener=None,
+                 allow_permutations=True):
 
         # IPv4Address or IPv6Address
         if isinstance(address, IPv4Address) or isinstance(
@@ -111,8 +114,11 @@ def __init__(self, address, timeout=5, proxy_opener=None):
         # Default timeout for socket connections.
         self.timeout = timeout
 
+        # Allow other than DNS lookups for ASNs.
+        self.allow_permutations = allow_permutations
+
         self.dns_resolver = dns.resolver.Resolver()
-        self.dns_resolver.timeout  = timeout
+        self.dns_resolver.timeout = timeout
         self.dns_resolver.lifetime = timeout
 
         # Proxy opener.
@@ -244,6 +250,14 @@ def get_asn_dns(self, result=None):
 
             raise
 
+        except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers,
+                dns.resolver.NoAnswer, dns.exception.Timeout) as e:
+
+            raise ASNLookupError(
+                'ASN lookup failed (DNS %s) for %r.' % (
+                    e.__class__.__name__, self.address_str)
+            )
+
         except:
 
             raise ASNLookupError(
@@ -587,11 +601,17 @@ def lookup_asn(self, retry_count=3):
                 retry_count and retry_count or 1)
             asn_data = self.get_asn_dns()
 
-        except (ASNLookupError, ASNRegistryError):
+        except (ASNLookupError, ASNRegistryError) as e:
+
+            if not self.allow_permutations:
+
+                raise ASNRegistryError('ASN registry lookup failed. '
+                                       'Permutations not allowed.')
 
             try:
 
-                log.debug('ASN DNS lookup failed, trying ASN WHOIS')
+                log.debug('ASN DNS lookup failed, trying ASN WHOIS: '
+                          '{0}'.format(e))
                 asn_data = self.get_asn_whois(retry_count)
 
             except (ASNLookupError, ASNRegistryError):  # pragma: no cover

ipwhois/rdap.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2013, 2014, 2015 Philip Hane
+# Copyright (c) 2013, 2014, 2015, 2016 Philip Hane
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -324,11 +324,27 @@ def summarize_notices(self, notices_json):
         for notices_dict in notices_json:
 
             tmp = {
-                'title': notices_dict['title'],
-                'description': '\n'.join(notices_dict['description']),
+                'title': None,
+                'description': None,
                 'links': None
             }
 
+            try:
+
+                tmp['title'] = notices_dict['title']
+
+            except (KeyError, ValueError, TypeError):
+
+                pass
+
+            try:
+
+                tmp['description'] = '\n'.join(notices_dict['description'])
+
+            except (KeyError, ValueError, TypeError):
+
+                pass
+
             try:
 
                 tmp['links'] = self.summarize_links(notices_dict['links'])
@@ -337,7 +353,9 @@ def summarize_notices(self, notices_json):
 
                 pass
 
-            ret.append(tmp)
+            if len(tmp.values()) > 1:
+
+                ret.append(tmp)
 
         return ret
 
@@ -720,7 +738,8 @@ def lookup(self, inc_raw=False, retry_count=3, asn_data=None, depth=0,
 
             for ent in response['entities']:
 
-                if ent['handle'] not in [results['entities'], excluded_entities]:
+                if ent['handle'] not in [results['entities'],
+                                         excluded_entities]:
 
                     result_ent = _RDAPEntity(ent)
                     result_ent.parse()
@@ -755,14 +774,12 @@ def lookup(self, inc_raw=False, retry_count=3, asn_data=None, depth=0,
                                        excluded_entities):
 
                             if bootstrap:
-
                                 entity_url = '{0}/entity/{1}'.format(
                                     BOOTSTRAP_URL, ent)
-
                             else:
-
-                                entity_url = str(RIR_RDAP[asn_data[
-                                    'asn_registry']]['entity_url']).format(ent)
+                                tmp_reg = asn_data['asn_registry']
+                                entity_url = RIR_RDAP[tmp_reg]['entity_url']
+                                entity_url = str(entity_url).format(ent)
 
                             try:
 

ipwhois/tests/online/test_ipwhois.py

@@ -143,3 +143,7 @@ def test_lookup_rdap(self):
         opener = build_opener(handler)
         result = IPWhois('74.125.225.229', 0, opener)
         self.assertRaises(HTTPLookupError, result.lookup_rdap)
+
+        log.debug('Testing allow_permutations')
+        result = IPWhois('74.125.225.229', timeout=0, allow_permutations=False)
+        self.assertRaises(ASNRegistryError, result.lookup_rdap)

ipwhois/tests/online/test_net.py

@@ -118,3 +118,6 @@ def test_lookup_asn(self):
             raise e
         except Exception as e:
             self.fail('Unexpected exception raised: %r' % e)
+
+        result = Net('74.125.225.229', timeout=0, allow_permutations=False)
+        self.assertRaises(ASNRegistryError, result.lookup_asn)

ipwhois/tests/test_net.py

@@ -81,6 +81,10 @@ def test_get_asn_dns(self):
         result = Net('74.125.225.229')
         self.assertRaises(ASNRegistryError, result.get_asn_dns, data)
 
+        data = []
+        result = Net('74.125.225.229')
+        self.assertRaises(ASNLookupError, result.get_asn_dns, data)
+
     def test_get_asn_whois(self):
         data = ('15169   | 74.125.225.229   | 74.125.225.0/24     | US | arin'
                 '     | 2007-03-13')