diff --git a/.coveragerc b/.coveragerc index a0b7c21..5f6fa55 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,6 +2,9 @@ omit = */python?.?/* */site-packages/nose/* - ipwhois/hr.py - ipwhois/tests/* + ipwhois/data/* + ipwhois/docs/* ipwhois/examples/* + ipwhois/scripts/* + ipwhois/tests/* + ipwhois/hr.py diff --git a/CHANGES.rst b/CHANGES.rst index 8693ec2..00c0fc5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,8 +1,8 @@ Changelog ========= -0.14.0 (TBD) ------------- +0.14.0 (2016-08-29) +------------------- - Changed legacy whois emails output type to list (#133) - Fixed retry count non-decrementing infinite loop in diff --git a/NIR.rst b/NIR.rst index 828f58c..bb40863 100644 --- a/NIR.rst +++ b/NIR.rst @@ -6,6 +6,8 @@ IPWhois.nir provides functionality for national registries which restrict information on regional registries. Currently, JPNIC (Japan) and KRNIC (South Korea) are supported. +.. _nir-input-ipwhois-wrapper: + Input (IPWhois Wrapper) ======================= @@ -21,6 +23,8 @@ Legacy Whois documentation: https://ipwhois.readthedocs.io/en/latest/WHOIS.html#input +.. _nir-input-direct: + Input (Direct) ============== @@ -51,12 +55,16 @@ arguments for that function call: | | | Primarily used for testing. | +-------------+--------+------------------------------------------------------+ +.. _nir-output: + Output ====== If calling via an IPWhois wrapper, the NIR results are added to the RDAP/WHOIS result dictionary under the key 'nir'. +.. _nir-results-dictionary: + Results Dictionary ------------------ @@ -69,16 +77,18 @@ IPWhois.lookup_whois() results. | query | String | The IP address input | +------------------+--------+-------------------------------------------------+ | nets | List | List of network dictionaries. | -| | | See `Network Dictionary <#network-dictionary>`_.| +| | | See :ref:`nir-network-dictionary`. | +------------------+--------+-------------------------------------------------+ | raw | String | Raw NIR whois results if inc_raw is True. | +------------------+--------+-------------------------------------------------+ +.. _nir-network-dictionary: + Network Dictionary ^^^^^^^^^^^^^^^^^^ The dictionary mapped to the nets key in the -`Results Dictionary <#results-dictionary>`_. +:ref:`nir-results-dictionary`. +-------------+--------+------------------------------------------------------+ | **Key** |**Type**| **Description** | @@ -107,15 +117,17 @@ The dictionary mapped to the nets key in the +-------------+--------+------------------------------------------------------+ | contacts | Dict | Dictionary with keys: admin, tech. Values map to | | | | contact dictionaries if found. See | -| | | `Contact Dictionary <#contact-dictionary>`_. | +| | | :ref:`nir-contact-dictionary`. | +-------------+--------+------------------------------------------------------+ +.. _nir-contact-dictionary: + Contact Dictionary ^^^^^^^^^^^^^^^^^^ The contact information dictionary registered to a NIR network object. This is 'contacts' -> 'admin'/'tech' key in -`Network Dictionary <#network-dictionary>`_. +:ref:`nir-network-dictionary`. +--------------+--------+-----------------------------------------------------+ | **Key** |**Type**| **Description** | @@ -139,6 +151,8 @@ The contact information dictionary registered to a NIR network object. This is | title | String | The contact's position or job title. | +--------------+--------+-----------------------------------------------------+ +.. _nir-usage-examples: + Usage Examples ============== diff --git a/RDAP.rst b/RDAP.rst index fd6e8ee..7116a05 100644 --- a/RDAP.rst +++ b/RDAP.rst @@ -8,6 +8,8 @@ implementation). RDAP queries allow for parsing of contact information and details for users, organizations, and groups. RDAP also provides more detailed network information. +.. _rdap-input: + Input ===== @@ -54,9 +56,13 @@ Arguments supported by IPWhois.lookup_rdap(). | | | 'ARIN', 'RIPE', 'apnic', 'lacnic', 'afrinic' | +--------------------+--------+-----------------------------------------------+ +.. _rdap-output: + Output ====== +.. _rdap-results-dictionary: + Results Dictionary ------------------ @@ -81,7 +87,7 @@ and dictionaries, detailed below this section. +------------------+--------+-------------------------------------------------+ | network | Dict | The assigned network for an IP address. May be | | | | a parent or child network. See | -| | | `Network Dictionary <#network-dictionary>`_. | +| | | :ref:`rdap-network-dictionary`. | +------------------+--------+-------------------------------------------------+ | entities | List | List of object names referenced by an RIR | | | | network. Map these to the objects dict keys. | @@ -90,17 +96,19 @@ and dictionaries, detailed below this section. | | | network or by other entities (depending on | | | | depth parameter). Keys are the object names | | | | with values as | -| | | `Objects Dictionary <#objects-dictionary>`_. | +| | | :ref:`rdap-objects-dictionary`. | +------------------+--------+-------------------------------------------------+ | raw | Dict | The raw results dictionary (JSON) if | | | | inc_raw is True. | +------------------+--------+-------------------------------------------------+ +.. _rdap-network-dictionary: + Network Dictionary ^^^^^^^^^^^^^^^^^^ The dictionary mapped to the network key in the objects list within -`Results Dictionary <#results-dictionary>`_. +:ref:`rdap-results-dictionary`. +---------------+--------+----------------------------------------------------+ | **Key** |**Type**| **Description** | @@ -113,7 +121,7 @@ The dictionary mapped to the network key in the objects list within | end_address | String | The last IP address in a network block. | +---------------+--------+----------------------------------------------------+ | events | List | List of event dictionaries. See | -| | | `Events Dictionary <#events-dictionary>`_. | +| | | :ref:`rdap-events-dictionary`. | +---------------+--------+----------------------------------------------------+ | handle | String | Unique identifier for a registered object. | +---------------+--------+----------------------------------------------------+ @@ -125,13 +133,13 @@ The dictionary mapped to the network key in the objects list within | | | registration for an IP address. | +---------------+--------+----------------------------------------------------+ | notices | List | List of notice dictionaries. See | -| | | `Notices Dictionary <#notices-dictionary>`_. | +| | | :ref:`rdap-notices-dictionary`. | +---------------+--------+----------------------------------------------------+ | parent_handle | String | Unique identifier for the parent network of a | | | | registered network. | +---------------+--------+----------------------------------------------------+ | remarks | List | List of remark (notice) dictionaries. See | -| | | `Notices Dictionary <#notices-dictionary>`_. | +| | | :ref:`rdap-notices-dictionary`. | +---------------+--------+----------------------------------------------------+ | start_address | String | The first IP address in a network block. | +---------------+--------+----------------------------------------------------+ @@ -140,48 +148,52 @@ The dictionary mapped to the network key in the objects list within | type | String | The RIR classification of a registered network. | +---------------+--------+----------------------------------------------------+ +.. _rdap-objects-dictionary: + Objects Dictionary ^^^^^^^^^^^^^^^^^^ The dictionary mapped to the object (entity) key in the objects list within -`Results Dictionary <#results-dictionary>`_. +:ref:`rdap-results-dictionary`. +--------------+--------+-----------------------------------------------------+ | **Key** |**Type**| **Description** | +--------------+--------+-----------------------------------------------------+ | contact | Dict | Contact information registered with an RIR object. | | | | See | -| | | `Contact Dictionary <#objects-contact-dictionary>`_.| +| | | :ref:`rdap-objects-contact-dictionary`. | +--------------+--------+-----------------------------------------------------+ | entities | List | List of object names referenced by an RIR object. | | | | Map these to other objects dictionary keys. | +--------------+--------+-----------------------------------------------------+ | events | List | List of event dictionaries. See | -| | | `Events Dictionary <#events-dictionary>`_. | +| | | :ref:`rdap-events-dictionary`. | +--------------+--------+-----------------------------------------------------+ | events_actor | List | List of event (no actor) dictionaries. See | -| | | `Events Dictionary <#events-dictionary>`_. | +| | | :ref:`rdap-events-dictionary`. | +--------------+--------+-----------------------------------------------------+ | handle | String | Unique identifier for a registered object. | +--------------+--------+-----------------------------------------------------+ | links | List | List of HTTP/HTTPS links provided for an RIR object.| +--------------+--------+-----------------------------------------------------+ | notices | List | List of notice dictionaries. See | -| | | `Notices Dictionary <#notices-dictionary>`_. | +| | | :ref:`rdap-notices-dictionary`. | +--------------+--------+-----------------------------------------------------+ | remarks | List | List of remark (notice) dictionaries. See | -| | | `Notices Dictionary <#notices-dictionary>`_. | +| | | :ref:`rdap-notices-dictionary`. | +--------------+--------+-----------------------------------------------------+ | roles | List | List of roles assigned to a registered object. | +--------------+--------+-----------------------------------------------------+ | status | List | List indicating the state of a registered object. | +--------------+--------+-----------------------------------------------------+ +.. _rdap-objects-contact-dictionary: + Objects Contact Dictionary ^^^^^^^^^^^^^^^^^^^^^^^^^^ The contact information dictionary registered to an RIR object. This is the -contact key contained in `Objects Dictionary <#objects-dictionary>`_. +contact key contained in :ref:`rdap-objects-dictionary`. +---------+--------+----------------------------------------------------------+ | **Key** |**Type**| **Description** | @@ -204,11 +216,13 @@ contact key contained in `Objects Dictionary <#objects-dictionary>`_. | title | String | The contact's position or job title. | +---------+--------+----------------------------------------------------------+ +.. _rdap-events-dictionary: + Events Dictionary ^^^^^^^^^^^^^^^^^ -Common to lists in `Network <#network-dictionary>`_ and -`Objects <#objects-dictionary>`_. +Common to lists in :ref:`rdap-network-dictionary` and +:ref:`rdap-objects-dictionary`. Contained in events and events_actor (no actor). +-----------+--------+-------------------------------------------------+ @@ -221,11 +235,13 @@ Contained in events and events_actor (no actor). | actor | String | The identifier for an event initiator (if any). | +-----------+--------+-------------------------------------------------+ +.. _rdap-notices-dictionary: + Notices Dictionary ^^^^^^^^^^^^^^^^^^ -Common to lists in `Network <#network-dictionary>`_ and -`Objects <#objects-dictionary>`_. Contained in notices and remarks. +Common to lists in :ref:`rdap-network-dictionary` and +:ref:`rdap-objects-dictionary`. Contained in notices and remarks. +-------------+--------+-------------------------------------------------+ | **Key** |**Type**| **Description** | @@ -237,6 +253,8 @@ Common to lists in `Network <#network-dictionary>`_ and | links | List | List of HTTP/HTTPS links provided for a notice. | +-------------+--------+-------------------------------------------------+ +.. _rdap-upgrading-from-0-10-to-0-11: + Upgrading from 0.10 to 0.11 =========================== @@ -249,6 +267,8 @@ RDAP return data is different in nearly every way from the legacy whois data. For information on raw RDAP responses, please see the RFC: https://tools.ietf.org/html/rfc7483 +.. _rdap-usage-examples: + Usage Examples ============== @@ -459,16 +479,16 @@ Use a proxy :: - >>>> from urllib import request - >>>> from ipwhois import IPWhois - >>>> handler = request.ProxyHandler({'http': 'http://192.168.0.1:80/'}) - >>>> opener = request.build_opener(handler) - >>>> obj = IPWhois('74.125.225.229', proxy_opener = opener) + >>>> from urllib import request + >>>> from ipwhois import IPWhois + >>>> handler = request.ProxyHandler({'http': 'http://192.168.0.1:80/'}) + >>>> opener = request.build_opener(handler) + >>>> obj = IPWhois('74.125.225.229', proxy_opener = opener) Optimizing queries for your network ----------------------------------- -Multiple factors will slow your queries down. Several `Input <#input>`_ +Multiple factors will slow your queries down. Several :ref:`rdap-input` arguments assist in optimizing query performance: bootstrap diff --git a/README.rst b/README.rst index d0f015d..34a3bb8 100644 --- a/README.rst +++ b/README.rst @@ -20,24 +20,34 @@ ipwhois ipwhois is a Python package focused on retrieving and parsing whois data for IPv4 and IPv6 addresses. -.. attention:: - - RDAP (IPWhois.lookup_rdap()) is the recommended query method as of v0.11.0. - If you are upgrading from earlier than 0.11.0, please see the - `upgrade info `_. - .. attention:: NIR (National Internet Registry) lookups are now enabled by default. This is currently only performed for JPNIC and KRNIC addresses. To disable, set inc_nir=False in your IPWhois.lookup_*() query. -.. warning:: +.. attention:: The 'nets' -> 'emails' key in IPWhois.lookup_whois() has been changed from a '\\n' separated string to a list. + +.. important:: + + RDAP (IPWhois.lookup_rdap()) is the recommended query method as of v0.11.0. + If you are upgrading from earlier than 0.11.0, please see the + `upgrade info `_. + +.. note:: + + If you are experiencing latency issues, it is likely related to rate + limiting. Profiling the tests, I see most time spent attributed to network + latency. Rate limiting is based on your source IP, which may be a problem + with multiple users behind the same proxy. Additionally, LACNIC implements + aggressive rate limiting. Bulk query optimization is on the roadmap + (https://github.com/secynic/ipwhois/issues/134) + Features ======== diff --git a/WHOIS.rst b/WHOIS.rst index 5f5a1a0..d4acb82 100644 --- a/WHOIS.rst +++ b/WHOIS.rst @@ -6,7 +6,7 @@ IPWhois.lookup() is deprecated as of v0.12.0 and will be removed. Legacy whois lookups were moved to IPWhois.lookup_whois(). Parsing is currently limited to the keys in the output -`below <#results-dictionary>`_. +:ref:`whois-results-dictionary`. This is assuming that those fields are present (for both whois and rwhois). Some IPs have parent networks listed. The parser attempts to recognize this, @@ -17,6 +17,8 @@ Sometimes, you will see whois information with multiple consecutive same name fields, e.g., Description: some text\\nDescription: more text. The parser will recognize this and the returned result will have the values separated by '\\n'. +.. _whois-input: + Input ===== @@ -65,9 +67,13 @@ Arguments supported by IPWhois.lookup_whois(). | | | 'apnic', 'lacnic', 'afrinic' | +------------------------+--------+-------------------------------------------+ +.. _whois-output: + Output ====== +.. _whois-results-dictionary: + Results Dictionary ------------------ @@ -90,23 +96,25 @@ The output dictionary from IPWhois.lookup_whois(). | asn_registry | String | ASN assigned regional internet registry. | +------------------+--------+-------------------------------------------------+ | nets | List | List of network dictionaries. | -| | | See `Network Dictionary <#network-dictionary>`_.| +| | | See :ref:`whois-network-dictionary`. | +------------------+--------+-------------------------------------------------+ | raw | String | Raw whois results if inc_raw is True. | +------------------+--------+-------------------------------------------------+ | referral | Dict | Referral whois information if get_referral | | | | is True and the server isn't blacklisted. See | -| | | `Referral Dictionary <#referral-dictionary>`_. | +| | | :ref:`whois-referral-dictionary`. | +------------------+--------+-------------------------------------------------+ | raw_referral | String | Raw referral whois results if the inc_raw | | | | parameter is True. | +------------------+--------+-------------------------------------------------+ +.. _whois-network-dictionary: + Network Dictionary ^^^^^^^^^^^^^^^^^^ The dictionary mapped to the nets key in the -`Results Dictionary <#results-dictionary>`_. +:ref:`whois-results-dictionary`. +-------------+--------+------------------------------------------------------+ | **Key** |**Type**| **Description** | @@ -140,11 +148,13 @@ The dictionary mapped to the nets key in the | updated | String | Network registration updated date in ISO 8601 format.| +-------------+--------+------------------------------------------------------+ +.. _whois-referral-dictionary: + Referral Dictionary ^^^^^^^^^^^^^^^^^^^ The dictionary mapped to the referral key in the -`Results Dictionary <#results-dictionary>`_. +:ref:`whois-results-dictionary`. +-------------+--------+------------------------------------------------------+ | **Key** |**Type**| **Description** | @@ -175,6 +185,8 @@ The dictionary mapped to the referral key in the | updated | String | Network registration updated date in ISO 8601 format.| +-------------+--------+------------------------------------------------------+ +.. _whois-usage-examples: + Usage Examples ============== @@ -183,20 +195,20 @@ Basic usage :: - >>>> from ipwhois import IPWhois - >>>> from pprint import pprint + >>>> from ipwhois import IPWhois + >>>> from pprint import pprint - >>>> obj = IPWhois('74.125.225.229') - >>>> results = obj.lookup_whois() - >>>> pprint(results) + >>>> obj = IPWhois('74.125.225.229') + >>>> results = obj.lookup_whois() + >>>> pprint(results) - { - 'asn': '15169', - 'asn_cidr': '74.125.225.0/24', - 'asn_country_code': 'US', - 'asn_date': '2007-03-13', - 'asn_registry': 'arin', - 'nets': [{'address': '1600 Amphitheatre Parkway', + { + 'asn': '15169', + 'asn_cidr': '74.125.225.0/24', + 'asn_country_code': 'US', + 'asn_date': '2007-03-13', + 'asn_registry': 'arin', + 'nets': [{'address': '1600 Amphitheatre Parkway', 'cidr': '74.125.0.0/16', 'city': 'Mountain View', 'country': 'US', @@ -212,11 +224,11 @@ Basic usage 'range': '74.125.0.0 - 74.125.255.255', 'state': 'CA', 'updated': '2012-02-24'}], - 'query': '74.125.225.229', - 'raw': None, - 'raw_referral': None, - 'referral': None - } + 'query': '74.125.225.229', + 'raw': None, + 'raw_referral': None, + 'referral': None + } Multiple networks listed and referral whois ------------------------------------------- diff --git a/ipwhois/examples/elastic_search/README.rst b/ipwhois/examples/elastic_search/README.rst index 2309011..9ea3883 100644 --- a/ipwhois/examples/elastic_search/README.rst +++ b/ipwhois/examples/elastic_search/README.rst @@ -17,6 +17,11 @@ https://dev.maxmind.com/geoip/geoip2/geolite2/ Dependencies ============ +Tested using:: + + ElasticSearch 2.3.5 + Kibana 4.5.4 + Python 2.6 (requirements26.txt - geopy is not supported):: ipwhois @@ -66,14 +71,14 @@ Usage Examples ============== Create the ipwhois ElasticSearch index ---------------------------------------- +-------------------------------------- :: elastic_search.py --create Delete the ipwhois ElasticSearch index ---------------------------------------- +-------------------------------------- :: @@ -122,7 +127,7 @@ Import Kibana config (dashboard, search, visualization, ipwhois) from json file elastic_search.py --kimport "/tmp/ipwhois-kibana.json" Create ipwhois index on custom ElasticSearch host and port ------------------------------------------------------------ +---------------------------------------------------------- :: diff --git a/ipwhois/examples/redis_cache/README.rst b/ipwhois/examples/redis_cache/README.rst index 1c32f70..1b1457e 100644 --- a/ipwhois/examples/redis_cache/README.rst +++ b/ipwhois/examples/redis_cache/README.rst @@ -11,6 +11,10 @@ class embedded in this file (IPWhoisRedisCache). Dependencies ============ +Tested using:: + + Redis 3.2.1 + Python 2.6 (requirements26.txt):: ipwhois diff --git a/ipwhois/nir.py b/ipwhois/nir.py index 579819a..e03697a 100644 --- a/ipwhois/nir.py +++ b/ipwhois/nir.py @@ -295,7 +295,7 @@ def _get_nets_jpnic(self, response): # Iterate through all of the networks found, storing the CIDR value # and the start and end positions. for match in re.finditer( - r'^.+?(\[Network Number\])[^\S\n]+.+?>(?P.+?)$', + r'^.*?(\[Network Number\])[^\S\n]+.+?>(?P.+?)$', response, re.MULTILINE ): diff --git a/setup.py b/setup.py index 4fa35fa..1c3ee32 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,7 @@ "Python", "WHOIS", "RWhois", + "Referral Whois", "ASN", "IP Address", "IP", @@ -26,8 +27,12 @@ "Lacnic", "Afrinic", "NIC", + "National Information Center", "RDAP", - "RIR" + "RIR", + "Regional Internet Registry" + "NIR", + "National Intgernet Registry" ] README = io.open(file='README.rst', mode='r', encoding='utf-8').read()