diff --git a/.coveragerc b/.coveragerc index 5f6fa559..3aab48d0 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,5 @@ [report] +show_missing = True omit = */python?.?/* */site-packages/nose/* diff --git a/.github/workflows/python-publish-test.yml b/.github/workflows/python-publish-test.yml new file mode 100644 index 00000000..fd6650eb --- /dev/null +++ b/.github/workflows/python-publish-test.yml @@ -0,0 +1,26 @@ +name: Upload Python Package to Test PyPi + +on: workflow_dispatch + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: secynic + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload --repository testpypi dist/* diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 00000000..6de16a55 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,28 @@ +name: Upload Python Package to PyPi + +on: + release: + types: [created] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: secynic + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.gitignore b/.gitignore index 690d6bea..8b4d2934 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,7 @@ nosetests.xml .pydevproject MANIFEST -.idea \ No newline at end of file +.idea +.history +.vscode +.venv \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index a363b5ec..845f994b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -sudo: required +os: linux dist: xenial python: - 2.7 @@ -7,6 +7,7 @@ python: - 3.5 - 3.6 - 3.7 + - 3.8 install: - pip install --upgrade setuptools - pip install --upgrade pip diff --git a/ASN.rst b/ASN.rst index f2ef8f16..23691ede 100644 --- a/ASN.rst +++ b/ASN.rst @@ -5,6 +5,25 @@ IP ASN Lookups This is new functionality as of v0.15.0. This functionality was migrated from net.Net and is still used by IPWhois.lookup*(). +.. note:: + + Cymru ASN data should not be considered a primary source for data points + like country code. + + Message from the Cymru site:: + + The country code, registry, and allocation date are all based on data + obtained directly from the regional registries including: ARIN, RIPE, + AFRINIC, APNIC, LACNIC. The information returned relating to these + categories will only be as accurate as the data present in the RIR + databases. + + IMPORTANT NOTE: Country codes are likely to vary significantly from + actual IP locations, and we must strongly advise that the IP to ASN + mapping tool not be used as an IP geolocation (GeoIP) service. + + https://team-cymru.com/community-services/ip-asn-mapping/ + .. _ip-asn-input: IP ASN Input @@ -24,12 +43,6 @@ Arguments supported by IPASN.lookup(). | | | resets, etc. are encountered. | | | | Defaults to 3. | +------------------------+--------+-------------------------------------------+ -| asn_alts | list | Additional lookup types to attempt if the | -| | | ASN dns lookup fails. Allow permutations | -| | | must be enabled. If None, defaults to all | -| | | ['whois', 'http']. *WARNING* deprecated | -| | | in favor of new argument asn_methods. | -+------------------------+--------+-------------------------------------------+ | extra_org_map | dict | Dictionary mapping org handles to RIRs. | | | | This is for limited cases where ARIN | | | | REST (ASN fallback HTTP lookup) does not | @@ -157,12 +170,6 @@ Arguments supported by ASNOrigin.lookup(). | | | ['description', 'maintainer', 'updated', | | | | 'source']. If None, defaults to all. | +------------------------+--------+-------------------------------------------+ -| asn_alts | list | Additional lookup types to attempt if the | -| | | ASN dns lookup fails. Allow permutations | -| | | must be enabled. If None, defaults to all | -| | | ['http']. *WARNING* deprecated | -| | | in favor of new argument asn_methods. | -+------------------------+--------+-------------------------------------------+ | asn_methods | list | ASN lookup types to attempt, in order. If | | | | None, defaults to all ['whois', 'http']. | +------------------------+--------+-------------------------------------------+ diff --git a/CHANGES.rst b/CHANGES.rst index 0a2ec4f0..52519b34 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,38 @@ Changelog ========= +1.2.0 (2020-09-17) +------------------ + +- Removed deprecated functions: asn.IPASN._parse_fields_http, + asn.IPASN._parse_fields_dns, asn.IPASN._parse_fields_whois, + asn.ASNOrigin._parse_fields, asn.ASNOrigin._get_nets_radb, + net.Net.lookup_asn, whois.Whois._parse_fields, whois.Whois._get_nets_arin + whois.Whois._get_nets_lacnic, whois.Whois._get_nets_other, + nir.NIRWhois._parse_fields, nir.NIRWhois._get_nets_jpnic + nir.NIRWhois._get_nets_krnic, nir.NIRWhois._get_contact (#230) +- Removed deprecated asn_alts parameter (#230) +- Removed deprecated allow_permutations parameter (#230) +- Fixed ASNOrigin lookups (#216) +- Fixed bug in ASNOrigin lookups when multiple asn_methods provided (#216) +- Fixed bug in KRNIC queries due to a change in their service (#243) +- Fixed bug in experimental.bulk_lookup_rdap where only the last + result was returned (#262 - ameidatou) +- Fixed deprecation warnings due to invalid escape sequences + (#272 - tirkarthi) +- Fixed bug in root and sub-entities not getting queried/data (#247) +- Fixed NIR datetime parsing issue if only date is returned (#284) +- Added new argument root_ent_check to IPWhois.lookup_rdap and + RDAP.lookup. Set this to False to revert to old functionality - missing data, + but less queries (#247) +- Added support for Python 3.8 (#267) +- Fixed travis build warnings (#268) +- Pinned requirements (#274) +- Added ip_failed_total key to stats dictionary in + experimental.bulk_lookup_rdap (#235) +- Added ipv4_generate_random and ipv6_generate_random to utils CLI (#236) +- Added documentation note for ASN data (#278) + 1.1.0 (2019-02-01) ------------------ @@ -217,4 +249,4 @@ Changelog - Added support for IPv4Address or IPv6Address as the address arg in IPWhois. - Fixed file open encoding bug. Moved from open to io.open. - Fixed parameter in IPWhois ip defined checks. -- Fixed TestIPWhois.test_ip_invalid() assertions. \ No newline at end of file +- Fixed TestIPWhois.test_ip_invalid() assertions. diff --git a/CLI.rst b/CLI.rst index cf4a986e..9c6934fd 100644 --- a/CLI.rst +++ b/CLI.rst @@ -22,7 +22,7 @@ ipwhois_cli.py [-h] [--whois] [--exclude_nir] [--json] [--hr] [--proxy_http "PROXY_HTTP"] [--proxy_https "PROXY_HTTPS"] [--inc_raw] [--retry_count RETRY_COUNT] - [--asn_alts "ASN_ALTS"] [--asn_methods "ASN_METHODS"] + [--asn_methods "ASN_METHODS"] [--extra_org_map "EXTRA_ORG_MAP"] [--skip_asn_description] [--depth COLOR_DEPTH] [--excluded_entities "EXCLUDED_ENTITIES"] [--bootstrap] @@ -66,12 +66,6 @@ Common settings (RDAP & Legacy Whois): --retry_count RETRY_COUNT The number of times to retry in case socket errors, timeouts, connection resets, etc. are encountered. - --asn_alts ASN_ALTS - A comma delimited list of additional lookup types to - attempt if the ASN dns lookup fails. Allow - permutations must be enabled. Defaults to all: - "whois,http". *WARNING* deprecated in favor of new - argument asn_methods. --asn_methods ASN_METHODS List of ASN lookup types to attempt, in order. Defaults to all ['dns', 'whois', 'http']. @@ -174,6 +168,12 @@ optional arguments: --ipv6_is_defined IPADDRESS Check if an IPv6 address is defined (in a reserved address range). + --ipv4_generate_random TOTAL + Generate random, unique IPv4 addresses that are not + defined (can be looked up using ipwhois). + --ipv6_generate_random TOTAL + Generate random, unique IPv6 addresses that are not + defined (can be looked up using ipwhois). --unique_everseen ITERABLE List unique elements from input iterable, preserving the order. @@ -267,6 +267,32 @@ ipv6_is_defined Name: Unique Local Unicast RFC: RFC 4193 +ipv4_generate_random +^^^^^^^^^^^^^^^^^^^^ + +:: + + >>>> ipwhois_utils_cli.py --ipv4_generate_random 5 + + 119.224.47.74 + 128.106.183.195 + 54.97.0.158 + 52.206.105.37 + 126.180.201.81 + +ipv6_generate_random +^^^^^^^^^^^^^^^^^^^^ + +:: + + >>>> ipwhois_utils_cli.py --ipv6_generate_random 5 + + 3e8c:dc93:49c8:57fd:31dd:2963:6332:426e + 2e3d:fd84:b57b:9282:91e6:5d4d:18d5:34f1 + 21d4:9d25:7dd6:e28b:77d7:7ce9:f85f:b34f + 3659:2b9:12ed:1eac:fd40:5756:3753:6d2d + 2e05:6d47:83fd:5de8:c6cb:85cb:912:fdb1 + unique_everseen ^^^^^^^^^^^^^^^ diff --git a/EXPERIMENTAL.rst b/EXPERIMENTAL.rst index d95f6350..0c6179fb 100644 --- a/EXPERIMENTAL.rst +++ b/EXPERIMENTAL.rst @@ -68,21 +68,21 @@ Basic usage >>>> pprint(results.split('\n')) [ - "Bulk mode; whois.cymru.com [2017-07-30 23:02:21 +0000]", - "15169 | 74.125.225.229 | 74.125.225.0/24 | US | arin | 2007-03-13 | GOOGLE - Google Inc., US", - "15169 | 2001:4860:4860::8888 | 2001:4860::/32 | US | arin | 2005-03-14 | GOOGLE - Google Inc., US", + "Bulk mode; whois.cymru.com [2020-09-15 16:42:29 +0000]", + "15169 | 74.125.225.229 | 74.125.225.0/24 | US | arin | 2007-03-13 | GOOGLE, US", + "15169 | 2001:4860:4860::8888 | 2001:4860::/32 | US | arin | 2005-03-14 | GOOGLE, US", "2856 | 62.239.237.1 | 62.239.0.0/16 | GB | ripencc | 2001-01-02 | BT-UK-AS BTnet UK Regional network, GB", "2856 | 2a00:2381:ffff::1 | 2a00:2380::/25 | GB | ripencc | 2007-08-29 | BT-UK-AS BTnet UK Regional network, GB", - "3786 | 210.107.73.73 | 210.107.0.0/17 | KR | apnic | | LGDACOM LG DACOM Corporation, KR", + "3786 | 210.107.73.73 | 210.107.0.0/17 | KR | apnic | 1997-08-29 | LGDACOM LG DACOM Corporation, KR", "2497 | 2001:240:10c:1::ca20:9d1d | 2001:240::/32 | JP | apnic | 2000-03-08 | IIJ Internet Initiative Japan Inc., JP", "19373 | 200.57.141.161 | 200.57.128.0/20 | MX | lacnic | 2000-12-04 | Triara.com, S.A. de C.V., MX", "NA | 2801:10:c000:: | NA | CO | lacnic | 2013-10-29 | NA", - "12091 | 196.11.240.215 | 196.11.240.0/24 | ZA | afrinic | | MTNNS-1, ZA", + "12091 | 196.11.240.215 | 196.11.240.0/24 | ZA | afrinic | 1994-07-21 | MTNNS-1, ZA", "37578 | 2001:43f8:7b0:: | 2001:43f8:7b0::/48 | KE | afrinic | 2013-03-22 | Tespok, KE", - "4730 | 133.1.2.5 | 133.1.0.0/16 | JP | apnic | | ODINS Osaka University, JP", - "4134 | 115.1.2.3 | 115.0.0.0/14 | KR | apnic | 2008-07-01 | CHINANET-BACKBONE No.31,Jin-rong Street, CN", + "4730 | 133.1.2.5 | 133.1.0.0/16 | JP | apnic | 1997-03-01 | ODINS Osaka University, JP", + "4766 | 115.1.2.3 | 115.0.0.0/12 | KR | apnic | 2008-07-01 | KIXS-AS-KR Korea Telecom, KR", "" - } + ] .. GET_BULK_ASN_WHOIS_OUTPUT_BASIC END @@ -175,11 +175,14 @@ The stats dictionary returned by ipwhois.experimental.bulk_lookup_rdap() 'ip_lookup_total' (int) - The total number of addresses that lookups were attempted for, excluding any that failed ASN registry checks. + 'ip_failed_total' (int) - The total number of addresses that + lookups failed for. Excludes any that failed initially, but + succeeded after further retries. 'lacnic' (dict) - { 'failed' (list) - The addresses that failed to lookup. Excludes any that failed initially, but succeeded after - futher retries. + further retries. 'rate_limited' (list) - The addresses that encountered rate-limiting. Unless an address is also in 'failed', it eventually succeeded. @@ -222,9 +225,7 @@ Basic usage "total": 2 }, "apnic": { - "failed": [ - "115.1.2.3" - ], + "failed": [], "rate_limited": [], "total": 4 }, @@ -233,6 +234,7 @@ Basic usage "rate_limited": [], "total": 2 }, + "ip_failed_total": 0, "ip_input_total": 12, "ip_lookup_total": 12, "ip_unique_total": 12, diff --git a/LICENSE.txt b/LICENSE.txt index 0da70ecd..be67aadd 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2013-2019 Philip Hane +Copyright (c) 2013-2020 Philip Hane All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/RDAP.rst b/RDAP.rst index f216b3a9..0f8d0fad 100644 --- a/RDAP.rst +++ b/RDAP.rst @@ -42,12 +42,6 @@ Arguments supported by IPWhois.lookup_rdap(). | | | when a rate limit notice is returned via | | | | rdap+json. Defaults to 120. | +--------------------+--------+-----------------------------------------------+ -| asn_alts | list | Additional lookup types to attempt if the ASN | -| | | dns lookup fails. Allow permutations must be | -| | | enabled. If None, defaults to all | -| | | ['whois', 'http']. *WARNING* deprecated in | -| | | favor of new argument asn_methods. | -+--------------------+--------+-----------------------------------------------+ | extra_org_map | dict | Dictionary mapping org handles to RIRs. | | | | This is for limited cases where ARIN REST | | | | (ASN fallback HTTP lookup) does not show an | @@ -79,6 +73,10 @@ Arguments supported by IPWhois.lookup_rdap(). | | | pulling ASN information via dns, in order to | | | | get the ASN description. Defaults to True. | +--------------------+--------+-----------------------------------------------+ +| root_ent_check | bool | If True, will perform additional RDAP HTTP | +| | | queries for missing entity data at the root | +| | | level. Defaults to True. | ++--------------------+--------+-----------------------------------------------+ .. _rdap-output: @@ -599,3 +597,12 @@ this very low for bulk queries, or disable completely by setting retry_count=0. Note that setting this result too low may cause a larger number of IP lookups to fail. + +root_ent_check +^^^^^^^^^^^^^^ + +When root level entities (depth=0) are missing vcard data, additional +entity specific HTTP lookups are performed. In the past, you would expect +depth=0 to mean a single lookup per IP. This was a bug and has been fixed as of +v1.2.0. Set this to False to revert back to the old method, although you will be +missing entity specific data. diff --git a/README.rst b/README.rst index eca1a080..524bcd78 100644 --- a/README.rst +++ b/README.rst @@ -7,8 +7,10 @@ ipwhois .. image:: https://coveralls.io/repos/github/secynic/ipwhois/badge.svg?branch= master :target: https://coveralls.io/github/secynic/ipwhois?branch=master +.. image:: https://img.shields.io/github/issues-raw/secynic/ipwhois + :target: https://github.com/secynic/ipwhois/issues .. image:: https://codeclimate.com/github/secynic/ipwhois/badges/issue_count.svg - :target: https://codeclimate.com/github/secynic/ipwhois + :target: https://codeclimate.com/github/secynic/ipwhois .. image:: https://img.shields.io/badge/license-BSD%202--Clause-blue.svg :target: https://github.com/secynic/ipwhois/tree/master/LICENSE.txt .. image:: https://img.shields.io/badge/python-2.7%2C%203.4+-blue.svg @@ -170,11 +172,6 @@ Input | proxy_opener | object | The urllib.request.OpenerDirector request for | | | | proxy support or None. | +--------------------+--------+-----------------------------------------------+ -| allow_permutations | bool | Allow net.Net() to use additional methods if | -| | | DNS lookups to Cymru fail. *WARNING* | -| | | deprecated in favor of new argument | -| | | asn_methods. Defaults to False. | -+--------------------+--------+-----------------------------------------------+ RDAP (HTTP) ----------- diff --git a/UPGRADING.rst b/UPGRADING.rst index 679db1a4..300da87e 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -9,6 +9,26 @@ any changes that may affect user experience when upgrading to a new release. This page is new as of version 1.0.0. Any information on older versions is likely missing or incomplete. +****** +v1.2.0 +****** + +- Removed deprecated functions: asn.IPASN._parse_fields_http, + asn.IPASN._parse_fields_dns, asn.IPASN._parse_fields_whois, + asn.ASNOrigin._parse_fields, asn.ASNOrigin._get_nets_radb, + net.Net.lookup_asn, whois.Whois._parse_fields, whois.Whois._get_nets_arin + whois.Whois._get_nets_lacnic, whois.Whois._get_nets_other, + nir.NIRWhois._parse_fields, nir.NIRWhois._get_nets_jpnic + nir.NIRWhois._get_nets_krnic, nir.NIRWhois._get_contact +- Removed deprecated asn_alts parameter +- Removed deprecated allow_permutations parameter +- Added new argument root_ent_check to IPWhois.lookup_rdap and + RDAP.lookup. Set this to False to revert to old functionality - missing data, + but less queries. If you leave this set to default of True, you will notice + more queries and potentially more rate-limiting. +- Added support for Python 3.8 +- Pinned requirements + ****** v1.1.0 ****** diff --git a/WHOIS.rst b/WHOIS.rst index bb7a9b0f..0d7b4963 100644 --- a/WHOIS.rst +++ b/WHOIS.rst @@ -53,12 +53,6 @@ Arguments supported by IPWhois.lookup_whois(). | | | 'postal_code', 'emails', 'created', | | | | 'updated']. If None, defaults to all. | +------------------------+--------+-------------------------------------------+ -| asn_alts | list | Additional lookup types to attempt if the | -| | | ASN dns lookup fails. Allow permutations | -| | | must be enabled. If None, defaults to all | -| | | ['whois', 'http']. *WARNING* deprecated | -| | | in favor of new argument asn_methods. | -+------------------------+--------+-------------------------------------------+ | extra_org_map | dict | Dictionary mapping org handles to RIRs. | | | | This is for limited cases where ARIN | | | | REST (ASN fallback HTTP lookup) does not | diff --git a/ipwhois/__init__.py b/ipwhois/__init__.py index 440666f9..2a1402bd 100644 --- a/ipwhois/__init__.py +++ b/ipwhois/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -26,4 +26,4 @@ from .net import Net from .ipwhois import IPWhois -__version__ = '1.1.0' +__version__ = '1.2.0' diff --git a/ipwhois/asn.py b/ipwhois/asn.py index 03946fe3..71b56758 100644 --- a/ipwhois/asn.py +++ b/ipwhois/asn.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -61,21 +61,21 @@ ASN_ORIGIN_HTTP = { 'radb': { - 'url': 'http://www.radb.net/query/', + 'url': 'http://www.radb.net/query', 'form_data_asn_field': 'keywords', 'form_data': { 'advanced_query': '1', 'query': 'Query', - '-T option': 'inet-rtr', + # '-T option': 'inet-rtr', 'ip_option': '', '-i': '1', '-i option': 'origin' }, 'fields': { - 'description': r'(descr):[^\S\n]+(?P.+?)\', - 'maintainer': r'(mnt-by):[^\S\n]+(?P.+?)\', - 'updated': r'(changed):[^\S\n]+(?P.+?)\', - 'source': r'(source):[^\S\n]+(?P.+?)\', + 'description': r'(descr):[^\S\n]+(?P.+?)\n', + 'maintainer': r'(mnt-by):[^\S\n]+(?P.+?)\n', + 'updated': r'(changed):[^\S\n]+(?P.+?)\n', + 'source': r'(source):[^\S\n]+(?P.+?)\<', } }, } @@ -169,16 +169,6 @@ def parse_fields_dns(self, response): return ret - def _parse_fields_dns(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('IPASN._parse_fields_dns() has been deprecated and will be ' - 'removed. You should now use IPASN.parse_fields_dns().') - return self.parse_fields_dns(*args, **kwargs) - def parse_fields_verbose_dns(self, response): """ The function for parsing ASN fields from a verbose dns response. @@ -293,16 +283,6 @@ def parse_fields_whois(self, response): return ret - def _parse_fields_whois(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('IPASN._parse_fields_whois() has been deprecated and will be ' - 'removed. You should now use IPASN.parse_fields_whois().') - return self.parse_fields_whois(*args, **kwargs) - def parse_fields_http(self, response, extra_org_map=None): """ The function for parsing ASN fields from a http response. @@ -403,19 +383,8 @@ def parse_fields_http(self, response, extra_org_map=None): return asn_data - def _parse_fields_http(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('IPASN._parse_fields_http() has been deprecated and will be ' - 'removed. You should now use IPASN.parse_fields_http().') - return self.parse_fields_http(*args, **kwargs) - - def lookup(self, inc_raw=False, retry_count=3, asn_alts=None, - extra_org_map=None, asn_methods=None, - get_asn_description=True): + def lookup(self, inc_raw=False, retry_count=3, extra_org_map=None, + asn_methods=None, get_asn_description=True): """ The wrapper function for retrieving and parsing ASN information for an IP address. @@ -426,10 +395,6 @@ def lookup(self, inc_raw=False, retry_count=3, asn_alts=None, retry_count (:obj:`int`): The number of times to retry in case socket errors, timeouts, connection resets, etc. are encountered. Defaults to 3. - asn_alts (:obj:`list`): Additional lookup types to attempt if the - ASN dns lookup fails. Allow permutations must be enabled. - Defaults to all ['whois', 'http']. *WARNING* deprecated in - favor of new argument asn_methods. Defaults to None. extra_org_map (:obj:`dict`): Mapping org handles to RIRs. This is for limited cases where ARIN REST (ASN fallback HTTP lookup) does not show an RIR as the org handle e.g., DNIC (which is @@ -466,17 +431,7 @@ def lookup(self, inc_raw=False, retry_count=3, asn_alts=None, if asn_methods is None: - if asn_alts is None: - - lookups = ['dns', 'whois', 'http'] - - else: - - from warnings import warn - warn('IPASN.lookup() asn_alts argument has been deprecated ' - 'and will be removed. You should now use the asn_methods ' - 'argument.') - lookups = ['dns'] + asn_alts + lookups = ['dns', 'whois', 'http'] else: @@ -492,12 +447,6 @@ def lookup(self, inc_raw=False, retry_count=3, asn_alts=None, dns_success = False for index, lookup_method in enumerate(lookups): - if index > 0 and not asn_methods and not ( - self._net.allow_permutations): - - raise ASNRegistryError('ASN registry lookup failed. ' - 'Permutations not allowed.') - if lookup_method == 'dns': try: @@ -706,16 +655,6 @@ def parse_fields(self, response, fields_dict, net_start=None, return ret - def _parse_fields(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('ASNOrigin._parse_fields() has been deprecated and will be ' - 'removed. You should now use ASNOrigin.parse_fields().') - return self.parse_fields(*args, **kwargs) - def get_nets_radb(self, response, is_http=False): """ The function for parsing network blocks from ASN origin data. @@ -743,7 +682,7 @@ def get_nets_radb(self, response, is_http=False): nets = [] if is_http: - regex = r'route(?:6)?:[^\S\n]+(?P.+?)
' + regex = r'route(?:6)?:[^\S\n]+(?P.+?)\n' else: regex = r'^route(?:6)?:[^\S\n]+(?P.+|.+)$' @@ -769,18 +708,8 @@ def get_nets_radb(self, response, is_http=False): return nets - def _get_nets_radb(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('ASNOrigin._get_nets_radb() has been deprecated and will be ' - 'removed. You should now use ASNOrigin.get_nets_radb().') - return self.get_nets_radb(*args, **kwargs) - def lookup(self, asn=None, inc_raw=False, retry_count=3, response=None, - field_list=None, asn_alts=None, asn_methods=None): + field_list=None, asn_methods=None): """ The function for retrieving and parsing ASN origin whois information via port 43/tcp (WHOIS). @@ -797,9 +726,6 @@ def lookup(self, asn=None, inc_raw=False, retry_count=3, response=None, field_list (:obj:`list`): If provided, fields to parse: ['description', 'maintainer', 'updated', 'source'] If None, defaults to all. - asn_alts (:obj:`list`): Additional lookup types to attempt if the - ASN whois lookup fails. If None, defaults to all ['http']. - *WARNING* deprecated in favor of new argument asn_methods. asn_methods (:obj:`list`): ASN lookup types to attempt, in order. If None, defaults to all ['whois', 'http']. @@ -828,17 +754,7 @@ def lookup(self, asn=None, inc_raw=False, retry_count=3, response=None, if asn_methods is None: - if asn_alts is None: - - lookups = ['whois', 'http'] - - else: - - from warnings import warn - warn('ASNOrigin.lookup() asn_alts argument has been deprecated' - ' and will be removed. You should now use the asn_methods' - ' argument.') - lookups = ['whois'] + asn_alts + lookups = ['whois', 'http'] else: @@ -875,6 +791,8 @@ def lookup(self, asn=None, inc_raw=False, retry_count=3, response=None, asn=asn, retry_count=retry_count ) + break + except (WhoisLookupError, WhoisRateLimitError) as e: log.debug('ASN origin WHOIS lookup failed: {0}' @@ -888,17 +806,22 @@ def lookup(self, asn=None, inc_raw=False, retry_count=3, response=None, log.debug('Response not given, perform ASN origin ' 'HTTP lookup for: {0}'.format(asn)) - tmp = ASN_ORIGIN_HTTP['radb']['form_data'] - tmp[str(ASN_ORIGIN_HTTP['radb']['form_data_asn_field'] - )] = asn + # tmp = ASN_ORIGIN_HTTP['radb']['form_data'] + # tmp[str( + # ASN_ORIGIN_HTTP['radb']['form_data_asn_field'] + # )] = asn response = self._net.get_http_raw( - url=ASN_ORIGIN_HTTP['radb']['url'], + url=('{0}?advanced_query=1&keywords={1}&-T+option' + '=&ip_option=&-i=1&-i+option=origin' + ).format(ASN_ORIGIN_HTTP['radb']['url'], asn), retry_count=retry_count, - request_type='POST', - form_data=tmp + request_type='GET', + # form_data=tmp ) is_http = True # pragma: no cover + break + except HTTPLookupError as e: log.debug('ASN origin HTTP lookup failed: {0}' diff --git a/ipwhois/docs/requirements.txt b/ipwhois/docs/requirements.txt index 8aba4d07..9a90c110 100644 --- a/ipwhois/docs/requirements.txt +++ b/ipwhois/docs/requirements.txt @@ -1,4 +1,3 @@ sphinx -sphinxcontrib-napoleon sphinx_rtd_theme -dnspython +dnspython<=2.0.0 diff --git a/ipwhois/docs/source/conf.py b/ipwhois/docs/source/conf.py index 164be02c..1039de6d 100644 --- a/ipwhois/docs/source/conf.py +++ b/ipwhois/docs/source/conf.py @@ -40,7 +40,7 @@ 'sphinx.ext.doctest', 'sphinx.ext.coverage', 'sphinx.ext.viewcode', - 'sphinxcontrib.napoleon' + 'sphinx.ext.napoleon' ] napoleon_google_docstring = True @@ -60,16 +60,16 @@ # General information about the project. project = 'ipwhois' -copyright = '2013-2019, Philip Hane' +copyright = '2013-2020, Philip Hane' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '1.1.0' +version = '1.2.0' # The full version, including alpha/beta/rc tags. -release = '1.1.0' +release = '1.2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/ipwhois/exceptions.py b/ipwhois/exceptions.py index 066814a6..88251876 100644 --- a/ipwhois/exceptions.py +++ b/ipwhois/exceptions.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/ipwhois/experimental.py b/ipwhois/experimental.py index a048bdf9..8f6ce230 100644 --- a/ipwhois/experimental.py +++ b/ipwhois/experimental.py @@ -158,11 +158,14 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, 'ip_lookup_total' (int) - The total number of addresses that lookups were attempted for, excluding any that failed ASN registry checks. + 'ip_failed_total' (int) - The total number of addresses that + lookups failed for. Excludes any that failed initially, but + succeeded after further retries. 'lacnic' (dict) - { 'failed' (list) - The addresses that failed to lookup. Excludes any that failed initially, but succeeded after - futher retries. + further retries. 'rate_limited' (list) - The addresses that encountered rate-limiting. Unless an address is also in 'failed', it eventually succeeded. @@ -196,6 +199,7 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, 'ip_input_total': len(addresses), 'ip_unique_total': 0, 'ip_lookup_total': 0, + 'ip_failed_total': 0, 'lacnic': {'failed': [], 'rate_limited': [], 'total': 0}, 'ripencc': {'failed': [], 'rate_limited': [], 'total': 0}, 'apnic': {'failed': [], 'rate_limited': [], 'total': 0}, @@ -253,15 +257,15 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, try: - results = ipasn.parse_fields_whois(asn_result) + asn_parsed = ipasn.parse_fields_whois(asn_result) except ASNRegistryError: # pragma: no cover continue # Add valid IP ASN result to asn_parsed_results for RDAP lookup - asn_parsed_results[ip] = results - stats[results['asn_registry']]['total'] += 1 + asn_parsed_results[ip] = asn_parsed + stats[asn_parsed['asn_registry']]['total'] += 1 # Set the list of IPs that are not allocated/failed ASN lookup stats['unallocated_addresses'] = list(k for k in addresses if k not in @@ -362,7 +366,7 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, # Perform the RDAP lookup. retry_count is set to 0 # here since we handle that in this function - results = rdap.lookup( + rdap_result = rdap.lookup( inc_raw=inc_raw, retry_count=0, asn_data=asn_data, depth=depth, excluded_entities=excluded_entities ) @@ -373,7 +377,9 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, # Lookup was successful, add to result. Set the nir # key to None as this is not supported # (yet - requires more queries) - results[ip] = results + results[ip] = asn_data + results[ip].update(rdap_result) + results[ip]['nir'] = None # Remove the IP from the lookup queue @@ -423,6 +429,7 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, del asn_parsed_results[ip] stats[rir]['failed'].append(ip) + stats['ip_failed_total'] += 1 if rir == 'lacnic': diff --git a/ipwhois/hr.py b/ipwhois/hr.py index b51196e4..31383b7b 100644 --- a/ipwhois/hr.py +++ b/ipwhois/hr.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/ipwhois/ipwhois.py b/ipwhois/ipwhois.py index 4a4d56bd..946c64b5 100644 --- a/ipwhois/ipwhois.py +++ b/ipwhois/ipwhois.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -42,17 +42,12 @@ class IPWhois: seconds. Defaults to 5. proxy_opener (:obj:`urllib.request.OpenerDirector`): The request for proxy support. Defaults to None. - allow_permutations (:obj:`bool`): Allow net.Net() to use additional - methods if DNS lookups to Cymru fail. *WARNING* deprecated in - favor of new argument asn_methods. Defaults to False. """ - def __init__(self, address, timeout=5, proxy_opener=None, - allow_permutations=False): + def __init__(self, address, timeout=5, proxy_opener=None): self.net = Net( - address=address, timeout=timeout, proxy_opener=proxy_opener, - allow_permutations=allow_permutations + address=address, timeout=timeout, proxy_opener=proxy_opener ) self.ipasn = IPASN(self.net) @@ -71,7 +66,7 @@ def __repr__(self): def lookup_whois(self, inc_raw=False, retry_count=3, get_referral=False, extra_blacklist=None, ignore_referral_errors=False, - field_list=None, asn_alts=None, extra_org_map=None, + field_list=None, extra_org_map=None, inc_nir=True, nir_field_list=None, asn_methods=None, get_asn_description=True): """ @@ -95,10 +90,6 @@ def lookup_whois(self, inc_raw=False, retry_count=3, get_referral=False, ['name', 'handle', 'description', 'country', 'state', 'city', 'address', 'postal_code', 'emails', 'created', 'updated'] If None, defaults to all. - asn_alts (:obj:`list`): Additional lookup types to attempt if the - ASN dns lookup fails. Allow permutations must be enabled. - If None, defaults to all ['whois', 'http']. *WARNING* - deprecated in favor of new argument asn_methods. extra_org_map (:obj:`dict`): Dictionary mapping org handles to RIRs. This is for limited cases where ARIN REST (ASN fallback HTTP lookup) does not show an RIR as the org handle e.g., DNIC @@ -161,7 +152,7 @@ def lookup_whois(self, inc_raw=False, retry_count=3, get_referral=False, log.debug('ASN lookup for {0}'.format(self.address_str)) asn_data = self.ipasn.lookup( - inc_raw=inc_raw, retry_count=retry_count, asn_alts=asn_alts, + inc_raw=inc_raw, retry_count=retry_count, extra_org_map=extra_org_map, asn_methods=asn_methods, get_asn_description=get_asn_description ) @@ -206,9 +197,9 @@ def lookup_whois(self, inc_raw=False, retry_count=3, get_referral=False, def lookup_rdap(self, inc_raw=False, retry_count=3, depth=0, excluded_entities=None, bootstrap=False, - rate_limit_timeout=120, asn_alts=None, extra_org_map=None, + rate_limit_timeout=120, extra_org_map=None, inc_nir=True, nir_field_list=None, asn_methods=None, - get_asn_description=True): + get_asn_description=True, root_ent_check=True): """ The function for retrieving and parsing whois information for an IP address via HTTP (RDAP). @@ -233,10 +224,6 @@ def lookup_rdap(self, inc_raw=False, retry_count=3, depth=0, rate_limit_timeout (:obj:`int`): The number of seconds to wait before retrying when a rate limit notice is returned via rdap+json. Defaults to 120. - asn_alts (:obj:`list`): Additional lookup types to attempt if the - ASN dns lookup fails. Allow permutations must be enabled. - If None, defaults to all ['whois', 'http']. *WARNING* - deprecated in favor of new argument asn_methods. extra_org_map (:obj:`dict`): Dictionary mapping org handles to RIRs. This is for limited cases where ARIN REST (ASN fallback HTTP lookup) does not show an RIR as the org handle e.g., DNIC @@ -260,6 +247,9 @@ def lookup_rdap(self, inc_raw=False, retry_count=3, depth=0, get_asn_description (:obj:`bool`): Whether to run an additional query when pulling ASN information via dns, in order to get the ASN description. Defaults to True. + root_ent_check (:obj:`bool`): If True, will perform + additional RDAP HTTP queries for missing entity data at the + root level. Defaults to True. Returns: dict: The IP RDAP lookup results @@ -303,7 +293,7 @@ def lookup_rdap(self, inc_raw=False, retry_count=3, depth=0, # Retrieve the ASN information. log.debug('ASN lookup for {0}'.format(self.address_str)) asn_data = self.ipasn.lookup( - inc_raw=inc_raw, retry_count=retry_count, asn_alts=asn_alts, + inc_raw=inc_raw, retry_count=retry_count, extra_org_map=extra_org_map, asn_methods=asn_methods, get_asn_description=get_asn_description ) @@ -318,7 +308,8 @@ def lookup_rdap(self, inc_raw=False, retry_count=3, depth=0, inc_raw=inc_raw, retry_count=retry_count, asn_data=asn_data, depth=depth, excluded_entities=excluded_entities, response=response, bootstrap=bootstrap, - rate_limit_timeout=rate_limit_timeout + rate_limit_timeout=rate_limit_timeout, + root_ent_check=root_ent_check ) # Add the RDAP information to the return dictionary. diff --git a/ipwhois/net.py b/ipwhois/net.py index 87939213..c7ce39cd 100644 --- a/ipwhois/net.py +++ b/ipwhois/net.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -103,17 +103,13 @@ class Net: seconds. Defaults to 5. proxy_opener (:obj:`urllib.request.OpenerDirector`): The request for proxy support. Defaults to None. - allow_permutations (:obj:`bool`): Allow net.Net() to use additional - methods if DNS lookups to Cymru fail. *WARNING* deprecated in - favor of new argument asn_methods. Defaults to False. Raises: IPDefinedError: The address provided is defined (does not need to be resolved). """ - def __init__(self, address, timeout=5, proxy_opener=None, - allow_permutations=False): + def __init__(self, address, timeout=5, proxy_opener=None): # IPv4Address or IPv6Address if isinstance(address, IPv4Address) or isinstance( @@ -129,16 +125,6 @@ def __init__(self, address, timeout=5, proxy_opener=None, # Default timeout for socket connections. self.timeout = timeout - # Allow other than DNS lookups for ASNs. - self.allow_permutations = allow_permutations - - if self.allow_permutations: - - from warnings import warn - warn('allow_permutations has been deprecated and will be removed. ' - 'It is no longer needed, due to the deprecation of asn_alts, ' - 'and the addition of the asn_methods argument.') - self.dns_resolver = dns.resolver.Resolver() self.dns_resolver.timeout = timeout self.dns_resolver.lifetime = timeout @@ -219,21 +205,6 @@ def __init__(self, address, timeout=5, proxy_opener=None, self.dns_zone = IPV6_DNS_ZONE.format(self.reversed) - def lookup_asn(self, *args, **kwargs): - """ - Temporary wrapper for IP ASN lookups (moved to - asn.IPASN.lookup()). This will be removed in a future - release. - """ - - from warnings import warn - warn('Net.lookup_asn() has been deprecated and will be removed. ' - 'You should now use asn.IPASN.lookup() for IP ASN lookups.') - from .asn import IPASN - response = None - ipasn = IPASN(self) - return ipasn.lookup(*args, **kwargs), response - def get_asn_dns(self): """ The function for retrieving ASN information for an IP address from diff --git a/ipwhois/nir.py b/ipwhois/nir.py index 910b4f8c..b3c79efb 100644 --- a/ipwhois/nir.py +++ b/ipwhois/nir.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -87,9 +87,9 @@ 'updated': r'(\[Last Update\])[^\S\n]+(?P.*?)\n', 'nameservers': r'(\[Nameserver\])[^\S\n]+(?P.*?)\n', 'contact_admin': r'(\[Administrative Contact\])[^\S\n]+.+?\>' - '(?P.+?)\<\/A\>\n', + '(?P.+?)\\<\\/A\\>\n', 'contact_tech': r'(\[Technical Contact\])[^\S\n]+.+?\>' - '(?P.+?)\<\/A\>\n' + '(?P.+?)\\<\\/A\\>\n' }, 'contact_fields': { 'name': r'(\[Last, First\])[^\S\n]+(?P.*?)\n', @@ -108,9 +108,14 @@ }, 'krnic': { 'country_code': 'KR', - 'url': 'https://whois.kisa.or.kr/eng/whois.jsc', + 'url': 'https://xn--c79as89aj0e29b77z.xn--3e0b707e/eng/whois.jsc', 'request_type': 'POST', - 'request_headers': {'Accept': 'text/html'}, + 'request_headers': { + 'Accept': 'text/html', + 'Referer': ( + 'https://xn--c79as89aj0e29b77z.xn--3e0b707e/eng/whois.jsp' + ), + }, 'form_data_ip_field': 'query', 'fields': { 'name': r'(Organization Name)[\s]+\:[^\S\n]+(?P.+?)\n', @@ -120,9 +125,9 @@ 'postal_code': r'(Zip Code)[\s]+\:[^\S\n]+(?P.+?)\n', 'created': r'(Registration Date)[\s]+\:[^\S\n]+(?P.+?)\n', 'contact_admin': r'(id="eng_isp_contact").+?\>(?P.*?)\<' - '\/div\>\n', + '\\/div\\>\n', 'contact_tech': r'(id="eng_user_contact").+?\>(?P.*?)\<' - '\/div\>\n' + '\\/div\\>\n' }, 'contact_fields': { 'name': r'(Name)[^\S\n]+?:[^\S\n]+?(?P.*?)\n', @@ -260,12 +265,20 @@ def parse_fields(self, response, fields_dict, net_start=None, if field in ['created', 'updated'] and dt_format: - value = ( - datetime.strptime( - values[0], - str(dt_format) - ) - timedelta(hours=hourdelta) - ).isoformat('T') + try: + value = ( + datetime.strptime( + values[0], + str(dt_format) + ) - timedelta(hours=hourdelta) + ).isoformat('T') + except ValueError: + value = ( + datetime.strptime( + values[0], + '%Y/%m/%d' + ) + ).isoformat('T') elif field in ['nameservers']: @@ -286,16 +299,6 @@ def parse_fields(self, response, fields_dict, net_start=None, return ret - def _parse_fields(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('NIRWhois._parse_fields() has been deprecated and will be ' - 'removed. You should now use NIRWhois.parse_fields().') - return self.parse_fields(*args, **kwargs) - def get_nets_jpnic(self, response): """ The function for parsing network blocks from jpnic whois data. @@ -359,16 +362,6 @@ def get_nets_jpnic(self, response): return nets - def _get_nets_jpnic(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('NIRWhois._get_nets_jpnic() has been deprecated and will be ' - 'removed. You should now use NIRWhois.get_nets_jpnic().') - return self.get_nets_jpnic(*args, **kwargs) - def get_nets_krnic(self, response): """ The function for parsing network blocks from krnic whois data. @@ -394,7 +387,7 @@ def get_nets_krnic(self, response): # and the start and end positions. for match in re.finditer( r'^(IPv4 Address)[\s]+:[^\S\n]+((.+?)[^\S\n]-[^\S\n](.+?)' - '[^\S\n]\((.+?)\)|.+)$', + '[^\\S\n]\\((.+?)\\)|.+)$', response, re.MULTILINE ): @@ -434,16 +427,6 @@ def get_nets_krnic(self, response): return nets - def _get_nets_krnic(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('NIRWhois._get_nets_krnic() has been deprecated and will be ' - 'removed. You should now use NIRWhois.get_nets_krnic().') - return self.get_nets_krnic(*args, **kwargs) - def get_contact(self, response=None, nir=None, handle=None, retry_count=3, dt_format=None): """ @@ -491,16 +474,6 @@ def get_contact(self, response=None, nir=None, handle=None, is_contact=True ) - def _get_contact(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('NIRWhois._get_contact() has been deprecated and will be ' - 'removed. You should now use NIRWhois.get_contact().') - return self.get_contact(*args, **kwargs) - def lookup(self, nir=None, inc_raw=False, retry_count=3, response=None, field_list=None, is_offline=False): """ diff --git a/ipwhois/rdap.py b/ipwhois/rdap.py index 9bccbe1f..3a65aef2 100644 --- a/ipwhois/rdap.py +++ b/ipwhois/rdap.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,6 +28,7 @@ from .net import ip_address import logging import json +from collections import namedtuple log = logging.getLogger(__name__) @@ -553,7 +554,7 @@ def parse(self): self.vars[v] = self.json[v].strip() - except (KeyError, ValueError): + except (KeyError, ValueError, AttributeError): pass @@ -688,9 +689,95 @@ def __init__(self, net): raise NetError('The provided net parameter is not an instance of ' 'ipwhois.net.Net') + def _get_entity(self, entity=None, roles=None, inc_raw=False, retry_count=3, + asn_data=None, bootstrap=False, rate_limit_timeout=120): + """ + The function for retrieving and parsing information for an entity via + RDAP (HTTP). + + Args: + entity (:obj:`str`): The entity name to lookup. + roles (:obj:`dict`): The mapping of entity handles to roles. + inc_raw (:obj:`bool`, optional): Whether to include the raw + results in the returned dictionary. Defaults to False. + retry_count (:obj:`int`): The number of times to retry in case + socket errors, timeouts, connection resets, etc. are + encountered. Defaults to 3. + asn_data (:obj:`dict`): Result from + :obj:`ipwhois.asn.IPASN.lookup`. Optional if the bootstrap + parameter is True. + bootstrap (:obj:`bool`): If True, performs lookups via ARIN + bootstrap rather than lookups based on ASN data. Defaults to + False. + rate_limit_timeout (:obj:`int`): The number of seconds to wait + before retrying when a rate limit notice is returned via + rdap+json. Defaults to 120. + + Returns: + namedtuple: + + :result (dict): Consists of the fields listed in the + ipwhois.rdap._RDAPEntity dict. The raw result is included for + each object if the inc_raw parameter is True. + :roles (dict): The mapping of entity handles to roles. + """ + + result = {} + + if bootstrap: + entity_url = '{0}/entity/{1}'.format( + BOOTSTRAP_URL, entity) + else: + tmp_reg = asn_data['asn_registry'] + entity_url = RIR_RDAP[tmp_reg]['entity_url'] + entity_url = str(entity_url).format(entity) + + try: + + # RDAP entity query + response = self._net.get_http_json( + url=entity_url, retry_count=retry_count, + rate_limit_timeout=rate_limit_timeout + ) + + # Parse the entity + result_ent = _RDAPEntity(response) + result_ent.parse() + result = result_ent.vars + + result['roles'] = None + try: + + result['roles'] = roles[entity] + + except KeyError: # pragma: no cover + + pass + + try: + + for tmp in response['entities']: + + if tmp['handle'] not in roles: + roles[tmp['handle']] = tmp['roles'] + + except (IndexError, KeyError): + + pass + + if inc_raw: + result['raw'] = response + + except (HTTPLookupError, InvalidEntityObject): + + pass + + return_tuple = namedtuple('return_tuple', ['result', 'roles']) + return return_tuple(result, roles) + def lookup(self, inc_raw=False, retry_count=3, asn_data=None, depth=0, excluded_entities=None, response=None, bootstrap=False, - rate_limit_timeout=120): + rate_limit_timeout=120, root_ent_check=True): """ The function for retrieving and parsing information for an IP address via RDAP (HTTP). @@ -716,6 +803,9 @@ def lookup(self, inc_raw=False, retry_count=3, asn_data=None, depth=0, rate_limit_timeout (:obj:`int`): The number of seconds to wait before retrying when a rate limit notice is returned via rdap+json. Defaults to 120. + root_ent_check (:obj:`bool`): If True, will perform + additional RDAP HTTP queries for missing entity data at the + root level. Defaults to True. Returns: dict: The IP RDAP lookup results @@ -792,10 +882,23 @@ def lookup(self, inc_raw=False, retry_count=3, asn_data=None, depth=0, if ent['handle'] not in [results['entities'], excluded_entities]: - result_ent = _RDAPEntity(ent) - result_ent.parse() + if 'vcardArray' not in ent and root_ent_check: + entity_object, roles = self._get_entity( + entity=ent['handle'], + roles=roles, + inc_raw=inc_raw, + retry_count=retry_count, + asn_data=asn_data, + bootstrap=bootstrap, + rate_limit_timeout=rate_limit_timeout + ) + results['objects'][ent['handle']] = entity_object - results['objects'][ent['handle']] = result_ent.vars + else: + result_ent = _RDAPEntity(ent) + result_ent.parse() + + results['objects'][ent['handle']] = result_ent.vars results['entities'].append(ent['handle']) @@ -835,57 +938,18 @@ def lookup(self, inc_raw=False, retry_count=3, asn_data=None, depth=0, list(new_objects.keys()) + excluded_entities): - if bootstrap: - entity_url = '{0}/entity/{1}'.format( - BOOTSTRAP_URL, ent) - else: - tmp_reg = asn_data['asn_registry'] - entity_url = RIR_RDAP[tmp_reg]['entity_url'] - entity_url = str(entity_url).format(ent) - - try: - - # RDAP entity query - response = self._net.get_http_json( - url=entity_url, retry_count=retry_count, - rate_limit_timeout=rate_limit_timeout - ) - - # Parse the entity - result_ent = _RDAPEntity(response) - result_ent.parse() - new_objects[ent] = result_ent.vars - - new_objects[ent]['roles'] = None - try: - - new_objects[ent]['roles'] = roles[ent] - - except KeyError: # pragma: no cover - - pass - - try: - - for tmp in response['entities']: - - if tmp['handle'] not in roles: - - roles[tmp['handle']] = tmp['roles'] - - except (IndexError, KeyError): - - pass - - if inc_raw: - - new_objects[ent]['raw'] = response - - except (HTTPLookupError, InvalidEntityObject): - - pass - - except TypeError: + entity_object, roles = self._get_entity( + entity=ent, + roles=roles, + inc_raw=inc_raw, + retry_count=retry_count, + asn_data=asn_data, + bootstrap=bootstrap, + rate_limit_timeout=rate_limit_timeout + ) + new_objects[ent] = entity_object + + except (KeyError, TypeError): pass diff --git a/ipwhois/scripts/docs/generate_examples.py b/ipwhois/scripts/docs/generate_examples.py index 68e21373..c5a6ea2b 100644 --- a/ipwhois/scripts/docs/generate_examples.py +++ b/ipwhois/scripts/docs/generate_examples.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/ipwhois/scripts/ipwhois_cli.py b/ipwhois/scripts/ipwhois_cli.py index 5a06e9ab..c36b7919 100644 --- a/ipwhois/scripts/ipwhois_cli.py +++ b/ipwhois/scripts/ipwhois_cli.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -166,17 +166,6 @@ help='The number of times to retry in case socket errors, timeouts, ' 'connection resets, etc. are encountered.' ) -group.add_argument( - '--asn_alts', - type=str, - nargs=1, - default='whois,http', - metavar='"ASN_ALTS"', - help='A comma delimited list of additional lookup types to attempt if the ' - 'ASN dns lookup fails. Allow permutations must be enabled. ' - 'Defaults to all: "whois,http" *WARNING* deprecated in ' - 'favor of new argument asn_methods.' -) group.add_argument( '--asn_methods', type=str, @@ -1456,9 +1445,6 @@ def lookup_whois(self, hr=True, show_name=False, colorize=True, **kwargs): field_list=script_args.field_list.split(',') if ( script_args.field_list and len(script_args.field_list) > 0) else None, - asn_alts=script_args.asn_alts.split(',') if ( - script_args.asn_alts and not script_args.asn_methods and - len(script_args.asn_alts) > 0) else None, extra_org_map=script_args.extra_org_map, inc_nir=(not script_args.exclude_nir), nir_field_list=script_args.nir_field_list.split(',') if ( @@ -1484,9 +1470,6 @@ def lookup_whois(self, hr=True, show_name=False, colorize=True, **kwargs): len(script_args.excluded_entities) > 0) else None, bootstrap=script_args.bootstrap, rate_limit_timeout=script_args.rate_limit_timeout, - asn_alts=script_args.asn_alts.split(',') if ( - script_args.asn_alts and not script_args.asn_methods and - len(script_args.asn_alts) > 0) else None, extra_org_map=script_args.extra_org_map, inc_nir=(not script_args.exclude_nir), nir_field_list=script_args.nir_field_list.split(',') if ( diff --git a/ipwhois/scripts/ipwhois_utils_cli.py b/ipwhois/scripts/ipwhois_utils_cli.py index 7f86b95a..d2895d2b 100644 --- a/ipwhois/scripts/ipwhois_utils_cli.py +++ b/ipwhois/scripts/ipwhois_utils_cli.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,8 +28,9 @@ from collections import OrderedDict import json from ipwhois.utils import (ipv4_lstrip_zeros, calculate_cidr, get_countries, - ipv4_is_defined, ipv6_is_defined, unique_everseen, - unique_addresses) + ipv4_is_defined, ipv6_is_defined, + ipv4_generate_random, ipv6_generate_random, + unique_everseen, unique_addresses) # CLI ANSI rendering ANSI = { @@ -86,6 +87,22 @@ metavar='"IP ADDRESS"', help='Check if an IPv6 address is defined (in a reserved address range).' ) +parser.add_argument( + '--ipv4_generate_random', + type=int, + nargs=1, + metavar='TOTAL', + help='Generate random, unique IPv4 addresses that are not defined (can be ' + 'looked up using ipwhois).' +) +parser.add_argument( + '--ipv6_generate_random', + type=int, + nargs=1, + metavar='TOTAL', + help='Generate random, unique IPv6 addresses that are not defined (can be ' + 'looked up using ipwhois).' +) parser.add_argument( '--unique_everseen', type=json.loads, @@ -224,6 +241,34 @@ print('{0}Error{1}: {2}'.format(ANSI['red'], ANSI['end'], str(e))) +elif script_args.ipv4_generate_random: + + try: + + result = ipv4_generate_random(total=script_args.ipv4_generate_random[0]) + + for random_ip in result: + + print(random_ip) + + except Exception as e: + + print('{0}Error{1}: {2}'.format(ANSI['red'], ANSI['end'], str(e))) + +elif script_args.ipv6_generate_random: + + try: + + result = ipv6_generate_random(total=script_args.ipv6_generate_random[0]) + + for random_ip in result: + + print(random_ip) + + except Exception as e: + + print('{0}Error{1}: {2}'.format(ANSI['red'], ANSI['end'], str(e))) + elif script_args.unique_everseen: try: diff --git a/ipwhois/tests/online/test_asn.py b/ipwhois/tests/online/test_asn.py index 62cd8049..76bc8ce0 100644 --- a/ipwhois/tests/online/test_asn.py +++ b/ipwhois/tests/online/test_asn.py @@ -96,4 +96,5 @@ def test_lookup(self): net = Net(address='74.125.225.229') asnorigin = ASNOrigin(net) - asnorigin.lookup(asn='15169', asn_methods=['whois', 'http']) + asnorigin.lookup(asn='15169', asn_methods=['whois']) + asnorigin.lookup(asn='15169', asn_methods=['http']) diff --git a/ipwhois/tests/online/test_experimental.py b/ipwhois/tests/online/test_experimental.py index 31ad57ec..6ea9b473 100644 --- a/ipwhois/tests/online/test_experimental.py +++ b/ipwhois/tests/online/test_experimental.py @@ -67,8 +67,23 @@ def test_bulk_lookup_rdap(self): '115.1.2.3' # KRNIC ] + expected_stats = {'ip_input_total': 12, 'ip_unique_total': 12, + 'ip_lookup_total': 12, 'ip_failed_total': 0, + 'lacnic': {'failed': [], 'rate_limited': [], 'total': 2}, + 'ripencc': {'failed': [], 'rate_limited': [], 'total': 2}, + 'apnic': {'failed': [], 'rate_limited': [], 'total': 4}, + 'afrinic': {'failed': [], 'rate_limited': [], 'total': 2}, + 'arin': {'failed': [], 'rate_limited': [], 'total': 2}, + 'unallocated_addresses': []} + try: - self.assertIsInstance(bulk_lookup_rdap(addresses=ips), tuple) + result = bulk_lookup_rdap(addresses=ips) + self.assertIsInstance(result, tuple) + + results, stats = result + self.assertEqual(stats, expected_stats) + self.assertEqual(len(results), 12) + except ASNLookupError: pass except AssertionError as e: diff --git a/ipwhois/tests/online/test_ipwhois.py b/ipwhois/tests/online/test_ipwhois.py index d9f24a79..3627e9d7 100644 --- a/ipwhois/tests/online/test_ipwhois.py +++ b/ipwhois/tests/online/test_ipwhois.py @@ -174,8 +174,3 @@ def test_lookup_rdap(self): result = IPWhois(address='74.125.225.229', timeout=0, proxy_opener=opener) self.assertRaises(ASNRegistryError, result.lookup_rdap) - - log.debug('Testing allow_permutations') - result = IPWhois(address='74.125.225.229', timeout=0, - allow_permutations=False) - self.assertRaises(ASNRegistryError, result.lookup_rdap) diff --git a/ipwhois/tests/test_asn.py b/ipwhois/tests/test_asn.py index e27a1b18..bb730512 100644 --- a/ipwhois/tests/test_asn.py +++ b/ipwhois/tests/test_asn.py @@ -136,8 +136,29 @@ def test_parse_fields_http(self): self.fail('Unexpected exception raised: {0}'.format(e)) def test_lookup(self): - # TODO: need to modify asn.json for this. - return NotImplemented + data_dir = path.dirname(__file__) + + with io.open(str(data_dir) + '/asn.json', 'r') as \ + data_file: + data = json.load(data_file) + + for key, val in data.items(): + + log.debug('Testing: {0}'.format(key)) + net = Net(key) + obj = IPASN(net) + + try: + + self.assertIsInstance(obj.lookup(), dict) + + except AssertionError as e: + + raise e + + except Exception as e: + + self.fail('Unexpected exception raised: {0}'.format(e)) class TestASNOrigin(TestCommon): @@ -243,7 +264,10 @@ def test__get_nets_radb(self): obj.get_nets_radb(multi_net_response) self.assertEqual(obj.get_nets_radb(multi_net_response, is_http=True), - []) + [{'cidr': '66.249.64.0/20', 'description': None, 'maintainer': None, 'updated': None, + 'source': None, 'start': 2, 'end': 29}, + {'cidr': '66.249.80.0/20', 'description': None, 'maintainer': None, 'updated': None, + 'source': None, 'start': 175, 'end': 202}]) net = Net('2001:43f8:7b0::') obj = ASNOrigin(net) diff --git a/ipwhois/tests/test_nir.py b/ipwhois/tests/test_nir.py index 6cecce27..85d2199b 100644 --- a/ipwhois/tests/test_nir.py +++ b/ipwhois/tests/test_nir.py @@ -45,6 +45,11 @@ def test_lookup(self): inc_raw=True), dict) + self.assertIsInstance(obj.lookup( + nir=val['nir'], + response=val['response']), + dict) + except AssertionError as e: raise e diff --git a/ipwhois/tests/test_rdap.py b/ipwhois/tests/test_rdap.py index 2ccdb451..6d4277db 100644 --- a/ipwhois/tests/test_rdap.py +++ b/ipwhois/tests/test_rdap.py @@ -82,7 +82,8 @@ def test_lookup(self): 'endAddress': '74.125.225.229' }, asn_data=val['asn_data'], - depth=0), dict) + depth=0, + root_ent_check=False), dict) log.debug('Testing rdap.lookup entitiy checks') net = Net('74.125.225.229') @@ -99,7 +100,8 @@ def test_lookup(self): 'entities': entity }, asn_data=val['asn_data'], - depth=1), dict) + depth=0, + root_ent_check=False), dict) self.assertIsInstance(obj.lookup(response={ 'handle': 'test', @@ -109,9 +111,10 @@ def test_lookup(self): 'entities': entity }, asn_data=val['asn_data'], - depth=1, + depth=0, bootstrap=True, - inc_raw=True), dict) + inc_raw=True, + root_ent_check=False), dict) # No sub entities. This is for coverage, but won't error out. entity = [{'handle': 'test', 'roles': [ @@ -125,7 +128,8 @@ def test_lookup(self): 'entities': entity }, asn_data=val['asn_data'], - depth=1), dict) + depth=0, + root_ent_check=False), dict) class TestRDAPContact(TestCommon): diff --git a/ipwhois/tests/test_utils.py b/ipwhois/tests/test_utils.py index 7a26950c..9dc041f4 100644 --- a/ipwhois/tests/test_utils.py +++ b/ipwhois/tests/test_utils.py @@ -64,35 +64,35 @@ def test_ipv4_is_defined(self): self.assertRaises(ValueError, ipv4_is_defined, '192.168.0.256') self.assertRaises(AddressValueError, ipv4_is_defined, 1234) - self.assertEquals(ipv4_is_defined('74.125.225.229'), (False, '', '')) + self.assertEqual(ipv4_is_defined('74.125.225.229'), (False, '', '')) - self.assertEquals(ipv4_is_defined('0.0.0.0'), + self.assertEqual(ipv4_is_defined('0.0.0.0'), (True, 'This Network', 'RFC 1122, Section 3.2.1.3')) - self.assertEquals(ipv4_is_defined('127.0.0.0'), + self.assertEqual(ipv4_is_defined('127.0.0.0'), (True, 'Loopback', 'RFC 1122, Section 3.2.1.3')) - self.assertEquals(ipv4_is_defined('169.254.0.0'), + self.assertEqual(ipv4_is_defined('169.254.0.0'), (True, 'Link Local', 'RFC 3927')) - self.assertEquals(ipv4_is_defined('192.0.0.0'), + self.assertEqual(ipv4_is_defined('192.0.0.0'), (True, 'IETF Protocol Assignments', 'RFC 5736')) - self.assertEquals(ipv4_is_defined('192.0.2.0'), + self.assertEqual(ipv4_is_defined('192.0.2.0'), (True, 'TEST-NET-1', 'RFC 5737')) - self.assertEquals(ipv4_is_defined('192.88.99.0'), + self.assertEqual(ipv4_is_defined('192.88.99.0'), (True, '6to4 Relay Anycast', 'RFC 3068')) - self.assertEquals(ipv4_is_defined('198.18.0.0'), + self.assertEqual(ipv4_is_defined('198.18.0.0'), (True, 'Network Interconnect Device Benchmark Testing', 'RFC 2544')) - self.assertEquals(ipv4_is_defined('198.51.100.0'), + self.assertEqual(ipv4_is_defined('198.51.100.0'), (True, 'TEST-NET-2', 'RFC 5737')) - self.assertEquals(ipv4_is_defined('203.0.113.0'), + self.assertEqual(ipv4_is_defined('203.0.113.0'), (True, 'TEST-NET-3', 'RFC 5737')) - self.assertEquals(ipv4_is_defined('224.0.0.0'), + self.assertEqual(ipv4_is_defined('224.0.0.0'), (True, 'Multicast', 'RFC 3171')) - self.assertEquals(ipv4_is_defined('255.255.255.255'), + self.assertEqual(ipv4_is_defined('255.255.255.255'), (True, 'Limited Broadcast', 'RFC 919, Section 7')) - self.assertEquals(ipv4_is_defined('192.168.0.1'), + self.assertEqual(ipv4_is_defined('192.168.0.1'), (True, 'Private-Use Networks', 'RFC 1918')) - self.assertEquals(ipv4_is_defined('198.97.38.0'), + self.assertEqual(ipv4_is_defined('198.97.38.0'), (True, 'IANA Reserved', '')) def test_ipv6_is_defined(self): @@ -105,31 +105,31 @@ def test_ipv6_is_defined(self): '2001:4860:4860::8888::1234') self.assertRaises(AddressValueError, ipv6_is_defined, 1234) - self.assertEquals(ipv6_is_defined('2001:4860:4860::8888'), + self.assertEqual(ipv6_is_defined('2001:4860:4860::8888'), (False, '', '')) - self.assertEquals(ipv6_is_defined('ff00::'), + self.assertEqual(ipv6_is_defined('ff00::'), (True, 'Multicast', 'RFC 4291, Section 2.7')) - self.assertEquals(ipv6_is_defined('0:0:0:0:0:0:0:0'), + self.assertEqual(ipv6_is_defined('0:0:0:0:0:0:0:0'), (True, 'Unspecified', 'RFC 4291, Section 2.5.2')) - self.assertEquals(ipv6_is_defined('0:0:0:0:0:0:0:1'), + self.assertEqual(ipv6_is_defined('0:0:0:0:0:0:0:1'), (True, 'Loopback', 'RFC 4291, Section 2.5.3')) - self.assertEquals(ipv6_is_defined('100::'), + self.assertEqual(ipv6_is_defined('100::'), (True, 'Reserved', 'RFC 4291')) - self.assertEquals(ipv6_is_defined('fe80::'), + self.assertEqual(ipv6_is_defined('fe80::'), (True, 'Link-Local', 'RFC 4291, Section 2.5.6')) - self.assertEquals(ipv6_is_defined('fec0::'), + self.assertEqual(ipv6_is_defined('fec0::'), (True, 'Site-Local', 'RFC 4291, Section 2.5.7')) - self.assertEquals(ipv6_is_defined('fc00::'), + self.assertEqual(ipv6_is_defined('fc00::'), (True, 'Unique Local Unicast', 'RFC 4193')) def test_unique_everseen(self): input_list = ['b', 'a', 'c', 'a', 'b', 'x', 'a'] - self.assertEquals(list(unique_everseen(input_list)), + self.assertEqual(list(unique_everseen(input_list)), ['b', 'a', 'c', 'x']) - self.assertEquals(list(unique_everseen(input_list, str.lower)), + self.assertEqual(list(unique_everseen(input_list, str.lower)), ['b', 'a', 'c', 'x']) def test_unique_addresses(self): @@ -150,14 +150,14 @@ def test_unique_addresses(self): '2001:4860:4860::8888': {'count': 2, 'ports': {'443': 1}} } - self.assertEquals(unique_addresses(input_data), expected_result) + self.assertEqual(unique_addresses(input_data), expected_result) data_dir = path.dirname(__file__) fp = str(data_dir) + '/rdap.json' # Expected result is different on 2.x vs 3.x, possible issues with # ipaddr vs ipaddress output. Investigation pending... - if sys.version_info >= (3, 3): + if (3, 3) <= sys.version_info < (3, 8): fp_expected_result = { '74.125.225.0/24': {'count': 1, 'ports': {}}, @@ -203,8 +203,60 @@ def test_unique_addresses(self): '210.0.0.0/8': {'count': 1, 'ports': {}} } - self.assertEquals(unique_addresses(file_path=fp), - fp_expected_result) + self.assertEqual(unique_addresses(file_path=fp), + fp_expected_result) + + elif sys.version_info >= (3, 8): + + fp_expected_result = { + '196.0.0.0': {'count': 1, 'ports': {}}, + '196.11.239.0': {'count': 2, 'ports': {}}, + '196.11.240.0/23': {'count': 1, 'ports': {}}, + '196.11.240.215': {'count': 2, 'ports': {}}, + '196.11.246.255': {'count': 2, 'ports': {}}, + '196.255.255.255': {'count': 1, 'ports': {}}, + '200.57.128.0/20': {'count': 1, 'ports': {}}, + '200.57.141.161': {'count': 7, 'ports': {}}, + '2001:200::/23': {'count': 2, 'ports': {}}, + '2001:240:10c:1::ca20:9d1d': + {'count': 2, 'ports': {}}, + '2001:240::': {'count': 1, 'ports': {}}, + '2001:240::/32': {'count': 6, 'ports': {}}, + '2001:240:ffff:ffff:ffff:ffff:ffff:ffff': + {'count': 1, 'ports': {}}, + '2001:4200::/23': {'count': 1, 'ports': {}}, + '2001:43f8:7b0::': {'count': 3, 'ports': {}}, + '2001:43f8:7b0:ffff:ffff:ffff:ffff:ffff': + {'count': 1, 'ports': {}}, + '2001:4860:4860::8888': {'count': 10, 'ports': {}}, + '2001:4860::': {'count': 2, 'ports': {}}, + '2001:4860::/32': {'count': 1, 'ports': {}}, + '2001:4860:ffff:ffff:ffff:ffff:ffff:ffff': + {'count': 1, 'ports': {}}, + '210.0.0.0': {'count': 1, 'ports': {}}, + '210.0.0.0/8': {'count': 1, 'ports': {}}, + '210.107.0.0': {'count': 2, 'ports': {}}, + '210.107.0.0/17': {'count': 6, 'ports': {}}, + '210.107.127.255': {'count': 2, 'ports': {}}, + '210.107.73.73': {'count': 2, 'ports': {}}, + '210.255.255.255': {'count': 1, 'ports': {}}, + '2801:10:c000::': {'count': 7, 'ports': {}}, + '2a00:2380::/25': {'count': 1, 'ports': {}}, + '2a00:2381:ffff::1': {'count': 4, 'ports': {}}, + '62.239.0.0/16': {'count': 1, 'ports': {}}, + '62.239.237.0': {'count': 1, 'ports': {}}, + '62.239.237.0/32': {'count': 1, 'ports': {}}, + '62.239.237.1': {'count': 4, 'ports': {}}, + '62.239.237.255': {'count': 1, 'ports': {}}, + '62.239.237.255/32': {'count': 1, 'ports': {}}, + '74.125.0.0': {'count': 2, 'ports': {}}, + '74.125.225.0/24': {'count': 1, 'ports': {}}, + '74.125.225.229': {'count': 8, 'ports': {}}, + '74.125.255.255': {'count': 1, 'ports': {}} + } + + self.assertEqual(unique_addresses(file_path=fp), + fp_expected_result) else: @@ -261,8 +313,8 @@ def test_unique_addresses(self): def test_ipv4_generate_random(self): - self.assertEquals(len(list(ipv4_generate_random(1000))), 1000) + self.assertEqual(len(list(ipv4_generate_random(1000))), 1000) def test_ipv6_generate_random(self): - self.assertEquals(len(list(ipv6_generate_random(1000))), 1000) + self.assertEqual(len(list(ipv6_generate_random(1000))), 1000) diff --git a/ipwhois/utils.py b/ipwhois/utils.py index 1df4a917..f3c0a76e 100644 --- a/ipwhois/utils.py +++ b/ipwhois/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -87,30 +87,30 @@ IP_REGEX = ( r'(?P' # IPv4 - '(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.)){3}' - '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' + r'(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.)){3}' + r'(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' # IPv6 - '|\[?(((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:)' - '{6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|' - '2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]' - '{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d' - '\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|' - '((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|' - '2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]' - '{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)' - '(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}((' - '(:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1' - '\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|((' - '[0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4})' - '{0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]' - '?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((' - '25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})' - ')|:)))(%.+)?))\]?' + r'|\[?(((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:)' + r'{6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|' + r'2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]' + r'{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d' + r'\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|' + r'((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|' + r'2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]' + r'{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)' + r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}((' + r'(:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1' + r'\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|((' + r'[0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4})' + r'{0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]' + r'?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((' + r'25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})' + r')|:)))(%.+)?))\]?' # Optional IPv4 Port - '((:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3}' + r'((:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3}' # Optional CIDR block - '))|(\/(?:[012]\d?|3[012]?|[4-9])))?' - ')' + r'))|(\/(?:[012]\d?|3[012]?|[4-9])))?' + r')' ) @@ -212,6 +212,7 @@ def get_countries(is_legacy_xml=False): # Read the file. data = f.read() + f.close() # Check if there is data. if not data: # pragma: no cover @@ -258,6 +259,8 @@ def get_countries(is_legacy_xml=False): # Add to the countries dictionary. countries[code] = name + f.close() + return countries @@ -506,6 +509,7 @@ def unique_addresses(data=None, file_path=None): # Read the file. file_data = f.read() + f.close() pattern = re.compile( str(IP_REGEX), diff --git a/ipwhois/whois.py b/ipwhois/whois.py index 882dd908..40c131d3 100644 --- a/ipwhois/whois.py +++ b/ipwhois/whois.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -67,7 +67,7 @@ 'name': r'(NetName):[^\S\n]+(?P.+?)\n', 'handle': r'(NetHandle):[^\S\n]+(?P.+?)\n', 'description': r'(OrgName|CustName):[^\S\n]+(?P.+?)' - '(?=(\n\S):?)', + '(?=(\n\\S):?)', 'country': r'(Country):[^\S\n]+(?P.+?)\n', 'state': r'(StateProv):[^\S\n]+(?P.+?)\n', 'city': r'(City):[^\S\n]+(?P.+?)\n', @@ -75,7 +75,7 @@ 'postal_code': r'(PostalCode):[^\S\n]+(?P.+?)\n', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), 'created': r'(RegDate):[^\S\n]+(?P.+?)\n', 'updated': r'(Updated):[^\S\n]+(?P.+?)\n', @@ -92,7 +92,7 @@ 'address': r'(address):[^\S\n]+(?P.+?)(?=(\n\S):?)', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), 'created': ( r'(created):[^\S\n]+(?P[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]' @@ -115,7 +115,7 @@ 'address': r'(address):[^\S\n]+(?P.+?)(?=(\n\S):?)', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), 'updated': r'(changed):[^\S\n]+.*(?P[0-9]{8}).*?\n' }, @@ -129,7 +129,7 @@ 'country': r'(country):[^\S\n]+(?P.+?)\n', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), 'created': r'(created):[^\S\n]+(?P[0-9]{8}).*?\n', 'updated': r'(changed):[^\S\n]+(?P[0-9]{8}).*?\n' @@ -146,7 +146,7 @@ 'address': r'(address):[^\S\n]+(?P.+?)(?=(\n\S):?)', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), } } @@ -166,7 +166,7 @@ 'postal_code': r'(network:Postal-Code):(?P.+?)\n', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), 'created': r'(network:Created):(?P.+?)\n', 'updated': r'(network:Updated):(?P.+?)\n' @@ -324,16 +324,6 @@ def parse_fields(self, response, fields_dict, net_start=None, return ret - def _parse_fields(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('Whois._parse_fields() has been deprecated and will be ' - 'removed. You should now use Whois.parse_fields().') - return self.parse_fields(*args, **kwargs) - def get_nets_arin(self, response): """ The function for parsing network blocks from ARIN whois data. @@ -415,16 +405,6 @@ def get_nets_arin(self, response): return nets - def _get_nets_arin(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('Whois._get_nets_arin() has been deprecated and will be ' - 'removed. You should now use Whois.get_nets_arin().') - return self.get_nets_arin(*args, **kwargs) - def get_nets_lacnic(self, response): """ The function for parsing network blocks from LACNIC whois data. @@ -474,7 +454,7 @@ def get_nets_lacnic(self, response): for addr in net_range.split(', '): count = addr.count('.') - if count is not 0 and count < 4: + if count != 0 and count < 4: addr_split = addr.strip().split('/') for i in range(count + 1, 4): @@ -495,16 +475,6 @@ def get_nets_lacnic(self, response): return nets - def _get_nets_lacnic(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('Whois._get_nets_lacnic() has been deprecated and will be ' - 'removed. You should now use Whois.get_nets_lacnic().') - return self.get_nets_lacnic(*args, **kwargs) - def get_nets_other(self, response): """ The function for parsing network blocks from generic whois data. @@ -577,16 +547,6 @@ def get_nets_other(self, response): return nets - def _get_nets_other(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('Whois._get_nets_other() has been deprecated and will be ' - 'removed. You should now use Whois.get_nets_other().') - return self.get_nets_other(*args, **kwargs) - def lookup(self, inc_raw=False, retry_count=3, response=None, get_referral=False, extra_blacklist=None, ignore_referral_errors=False, asn_data=None, @@ -667,7 +627,7 @@ def lookup(self, inc_raw=False, retry_count=3, response=None, # Only fetch the response if we haven't already. if response is None or (not is_offline and - asn_data['asn_registry'] is not 'arin'): + asn_data['asn_registry'] != 'arin'): log.debug('Response not given, perform WHOIS lookup for {0}' .format(self._net.address_str)) diff --git a/requirements/python2.txt b/requirements/python2.txt index b7f70e79..6f6db6bc 100644 --- a/requirements/python2.txt +++ b/requirements/python2.txt @@ -1,2 +1,2 @@ -dnspython -ipaddr +dnspython<=2.0.0 +ipaddr==2.2.0 diff --git a/requirements/python3.txt b/requirements/python3.txt index 2f735967..337b63d0 100644 --- a/requirements/python3.txt +++ b/requirements/python3.txt @@ -1 +1 @@ -dnspython +dnspython<=2.0.0 diff --git a/setup.py b/setup.py index 9103dec1..ef22a097 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import io NAME = 'ipwhois' -VERSION = '1.1.0' +VERSION = '1.2.0' AUTHOR = 'Philip Hane' AUTHOR_EMAIL = 'secynic@gmail.com' DESCRIPTION = 'Retrieve and parse whois data for IPv4 and IPv6 addresses.' @@ -58,6 +58,7 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Topic :: Internet', 'Topic :: Software Development', ] @@ -66,7 +67,7 @@ PACKAGE_DATA = {'ipwhois': ['data/*.xml', 'data/*.csv']} -INSTALL_REQUIRES = ['dnspython', 'ipaddr;python_version<"3.3"'] +INSTALL_REQUIRES = ['dnspython<=2.0.0', 'ipaddr==2.2.0;python_version<"3.3"'] setup( name=NAME,