Merge pull request #38 from openaddresses/proxyRequest

Proxied Requests

esridump/cli.py

@@ -32,6 +32,8 @@ def _parse_args(args):
  parser.add_argument("outfile",
  type=argparse.FileType('w'),
  help="Output file name (use - for stdout)")
+ parser.add_argument("--proxy",
+ help="Proxy string to send requests through ie: https://example.com/proxy.ashx?<SERVER>")
  parser.add_argument("--jsonlines",
  action='store_true',
  default=False,
@@ -87,6 +89,7 @@ def main():
  extra_headers=headers,
  fields=requested_fields,
  request_geometry=args.request_geometry,
+ proxy=args.proxy,
  parent_logger=logger)
 
  if args.jsonlines:

esridump/dumper.py

@@ -2,6 +2,7 @@
 import requests
 import simplejson as json
 import socket
+import urllib
 
 from esridump import esri2geojson
 from esridump.errors import EsriDownloadError
@@ -10,14 +11,15 @@ class EsriDumper(object):
  def __init__(self, url, parent_logger=None,
  extra_query_args=None, extra_headers=None,
  timeout=None, fields=None, request_geometry=True,
- outSR=None):
+ outSR=None, proxy=None):
  self._layer_url = url
  self._query_params = extra_query_args or {}
  self._headers = extra_headers or {}
  self._http_timeout = timeout or 30
  self._fields = fields or None
  self._outSR = outSR or '4326'
  self._request_geometry = request_geometry
+ self._proxy = proxy or None
 
  if parent_logger:
  self._logger = parent_logger.getChild('esridump')
@@ -27,6 +29,14 @@ def __init__(self, url, parent_logger=None,
  def _request(self, method, url, **kwargs):
  try:
  self._logger.debug("Requesting %s with args %s", url, kwargs.get('params') or kwargs.get('data'))
+
+ if self._proxy:
+ url = self._proxy + url
+
+ if kwargs.get('params'):
+ url += '?' + urllib.urlencode(kwargs.get('params'))
+ del kwargs['params']
+
  return requests.request(method, url, timeout=self._http_timeout, **kwargs)
  except requests.exceptions.SSLError:
  self._logger.warning("Retrying %s without SSL verification", url)

requirements.txt

@@ -3,7 +3,7 @@ funcsigs==1.0.2
 mock==2.0.0
 nose==1.3.7
 pbr==1.10.0
-requests==2.11.0
+requests==2.13.0
 responses==0.5.1
 simplejson==3.8.2
 six==1.10.0

tests/cli_tests.py

@@ -63,6 +63,7 @@ def setUp(self):
  self.parse_return.request_geometry = True
  self.parse_return.headers = []
  self.parse_return.params = []
+ self.parse_return.proxy = None
  self.mock_parseargs.return_value = self.parse_return
 
  self.fake_url = 'http://example.com'