Add extra_site_dirs config option to solve #566

Makefile

@@ -34,7 +34,7 @@ clean:
 
 test: init
 	@echo $(TAG)Running tests on the current Python interpreter with coverage $(END)
-	py.test --cov ./httpie --cov ./tests --doctest-modules --verbose ./httpie ./tests
+	py.test --cov ./httpie --cov ./tests --doctest-modules --verbose --ignore=./tests/test_plugins ./httpie ./tests
 	@echo
 
 

README.rst

@@ -774,7 +774,8 @@ Here's a few picks:
 * `httpie-oauth <https://github.com/httpie/httpie-oauth>`_: OAuth
 * `requests-hawk <https://github.com/mozilla-services/requests-hawk>`_: Hawk
 
-
+If HTTPie is not finding your installed plugins then you may need to set the `extra_site_dirs`_
+config option.
 
 
 HTTP redirects
@@ -1504,6 +1505,30 @@ Or you could change the implicit request content type from JSON to form by
 adding ``--form`` to the list.
 
 
+``extra_site_dirs``
+~~~~~~~~~~~~~~~~~~~
+
+An ``Array`` (by default empty) of directories that will be appended to Pythons
+``sys.path``. This is helpful when you want to use a plugin from outside the
+default ``sys.path``.
+
+Perhaps the most common use will be on macOS when HTTPie is installed via
+Homebrew and plugins are installed via ``pip``, you'll need to set
+``extra_site_dirs`` to something like the example below so that HTTPie can find
+your plugins:
+
+.. code-block:: json
+
+    "extra_site_dirs": [
+        "/usr/local/lib/python3.7/site-packages",
+        "~/Library/Python/3.7/lib/python/site-packages"
+    ]
+
+If you are having trouble with HTTPie not finding your plugins then run
+``http --debug`` to see the list of dirs being searched, the list of loaded
+plugins and what dir each comes from.
+
+
 ``__meta__``
 ~~~~~~~~~~~~
 

httpie/config.py

@@ -84,7 +84,8 @@ class Config(BaseConfigDict):
     about = 'HTTPie configuration file'
 
     DEFAULTS = {
-        'default_options': []
+        'default_options': [],
+        'extra_site_dirs': [],
     }
 
     def __init__(self, directory=DEFAULT_CONFIG_DIR):

httpie/core.py

@@ -12,8 +12,11 @@
 """
 import sys
 import errno
+import os
 import platform
+import site
 
+from pkg_resources import working_set
 import requests
 from requests import __version__ as requests_version
 from pygments import __version__ as pygments_version
@@ -56,6 +59,10 @@ def print_debug_info(env):
     ])
     env.stderr.write('\n\n')
     env.stderr.write(repr(env))
+    env.stderr.write('\n\n')
+    env.stderr.write('Looking for plugins in these directories:\n')
+    for p in sys.path:
+        env.stderr.write('  %s\n' % p)
     env.stderr.write('\n')
 
 
@@ -180,7 +187,26 @@ def main(args=sys.argv[1:], env=Environment(), custom_log_error=None):
 
     """
     args = decode_args(args, env.stdin_encoding)
-    plugin_manager.load_installed_plugins()
+
+    sys_path_length = len(sys.path)
+
+    for sitedir in env.config.extra_site_dirs:
+        parts = sitedir.split(os.sep)
+        if parts[0].startswith('~'):
+            parts[0] = os.path.expanduser(parts[0])
+        env.stderr.write('addsitedir ' + os.sep.join(parts) + '\n')
+        site.addsitedir(os.sep.join(parts))
+
+    for new_path in sys.path[sys_path_length:]:
+        env.stderr.write('add_entry ' + new_path + '\n')
+        working_set.add_entry(new_path)
+
+    include_debug_info = '--debug' in args
+
+    if include_debug_info:
+        print_debug_info(env)
+
+    plugin_manager.load_installed_plugins(env.stderr if include_debug_info else None)
 
     def log_error(msg, *args, **kwargs):
         msg = msg % args
@@ -196,13 +222,10 @@ def log_error(msg, *args, **kwargs):
     if custom_log_error:
         log_error = custom_log_error
 
-    include_debug_info = '--debug' in args
     include_traceback = include_debug_info or '--traceback' in args
 
-    if include_debug_info:
-        print_debug_info(env)
-        if args == ['--debug']:
-            return ExitStatus.OK
+    if args == ['--debug']:
+        return ExitStatus.OK
 
     exit_status = ExitStatus.OK
 

httpie/plugins/manager.py

@@ -27,12 +27,19 @@ def register(self, *plugins):
     def unregister(self, plugin):
         self._plugins.remove(plugin)
 
-    def load_installed_plugins(self):
+    def load_installed_plugins(self, debug_stream):
+        loaded_any_plugins = False
         for entry_point_name in ENTRY_POINT_NAMES:
             for entry_point in iter_entry_points(entry_point_name):
+                if debug_stream:
+                    debug_stream.write("Loading plugin '{}' from '{}' ...\n".format(
+                        entry_point.dist, entry_point.dist.location))
+                loaded_any_plugins = True
                 plugin = entry_point.load()
                 plugin.package_name = entry_point.dist.key
                 self.register(entry_point.load())
+        if debug_stream and loaded_any_plugins:
+            debug_stream.write('\n')
 
     # Auth
     def get_auth_plugins(self):

tests/test_config.py

@@ -1,6 +1,13 @@
+from os import path
+import sys
+
 from httpie import __version__
-from utils import MockEnvironment, http
+from utils import MockEnvironment, http, HTTP_OK
 from httpie.context import Environment
+import pkg_resources
+import pytest
+
+TEST_PLUGINS_PATH = path.join(path.abspath(path.dirname(__file__)), 'test_plugins')
 
 
 def test_default_options(httpbin):
@@ -38,3 +45,44 @@ def test_migrate_implicit_content_type():
 def test_current_version():
     version = Environment().config['__meta__']['httpie']
     assert version == __version__
+
+
+@pytest.fixture()
+def sandbox():
+    # this is based on setuptools.sandbox.setup_context
+    # but it seems the kind of thing likely to break!
+    saved_pkg_resources = pkg_resources.__getstate__()
+    saved_modules = sys.modules.copy()
+    saved_path = sys.path[:]
+    try:
+        yield
+    finally:
+        sys.path[:] = saved_path
+        sys.modules.update(saved_modules)
+        for mod_name in list(sys.modules.keys()):
+            if mod_name not in saved_modules and not mod_name.startswith('encodings.'):
+                del sys.modules[mod_name]
+        pkg_resources.__setstate__(saved_pkg_resources)
+
+
+def test_extra_site_dirs_plugin_loading(sandbox):
+    assert TEST_PLUGINS_PATH not in sys.path
+    env = MockEnvironment()
+    env.config['extra_site_dirs'] = [TEST_PLUGINS_PATH]
+    env.config.save()
+    r = http('--debug', '--help', env=env)
+    assert "Loading plugin 'foo 0.0.1' from " in r.stderr
+    assert '--auth-type {basic,digest,foo}' in r
+    assert TEST_PLUGINS_PATH in sys.path
+
+
+def test_extra_site_dirs_plugin_works(httpbin, sandbox):
+    assert TEST_PLUGINS_PATH not in sys.path
+    env = MockEnvironment()
+    env.config['extra_site_dirs'] = [TEST_PLUGINS_PATH]
+    env.config.save()
+    r = http('--auth-type=foo', '--auth=user:password',
+             'GET', httpbin + '/basic-auth/user/password-foo', env=env)
+    assert HTTP_OK in r
+    assert r.json == {'authenticated': True, 'user': 'user'}
+    assert TEST_PLUGINS_PATH in sys.path

tests/test_plugins/foo-0.0.1.dist-info/METADATA

@@ -0,0 +1,3 @@
+Metadata-Version: 2.1
+Name: foo
+Version: 0.0.1

tests/test_plugins/foo-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[httpie.plugins.auth.v1]
+foo = foo:FooPlugin

tests/test_plugins/foo-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+foo

tests/test_plugins/foo.py

@@ -0,0 +1,10 @@
+from httpie.plugins import AuthPlugin
+from httpie.plugins.builtin import HTTPBasicAuth
+
+
+class FooPlugin(AuthPlugin):
+    auth_type = 'foo'
+    name = 'foo HTTP auth'
+
+    def get_auth(self, username=None, password=None):
+        return HTTPBasicAuth(username, password + '-foo')

tox.ini

@@ -20,4 +20,5 @@ commands =
     py.test \
         --verbose \
         --doctest-modules \
+        --ignore=./tests/test_plugins \
         {posargs:./httpie ./tests}