From 35554d3be4872ae39b199906a9e09682b77ea818 Mon Sep 17 00:00:00 2001 From: matthew16550 Date: Thu, 6 Sep 2018 13:32:59 +1200 Subject: [PATCH 1/5] Add extra_site_dirs config option to solve Homebrew installed HTTPie not finding pip installed plugins (#566). --- Makefile | 2 +- README.rst | 27 +++++++++- httpie/config.py | 3 +- httpie/core.py | 35 ++++++++++--- httpie/plugins/manager.py | 9 +++- tests/test_config.py | 50 ++++++++++++++++++- .../test_plugins/foo-0.0.1.dist-info/METADATA | 3 ++ .../foo-0.0.1.dist-info/entry_points.txt | 2 + .../foo-0.0.1.dist-info/top_level.txt | 1 + tests/test_plugins/foo.py | 10 ++++ tox.ini | 1 + 11 files changed, 132 insertions(+), 11 deletions(-) create mode 100644 tests/test_plugins/foo-0.0.1.dist-info/METADATA create mode 100644 tests/test_plugins/foo-0.0.1.dist-info/entry_points.txt create mode 100644 tests/test_plugins/foo-0.0.1.dist-info/top_level.txt create mode 100644 tests/test_plugins/foo.py diff --git a/Makefile b/Makefile index 402017a252..dd451cc502 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ clean: test: init @echo $(TAG)Running tests on the current Python interpreter with coverage $(END) - py.test --cov ./httpie --cov ./tests --doctest-modules --verbose ./httpie ./tests + py.test --cov ./httpie --cov ./tests --doctest-modules --verbose --ignore=./tests/test_plugins ./httpie ./tests @echo diff --git a/README.rst b/README.rst index d6850af183..20ba6ae8ae 100644 --- a/README.rst +++ b/README.rst @@ -774,7 +774,8 @@ Here's a few picks: * `httpie-oauth `_: OAuth * `requests-hawk `_: Hawk - +If HTTPie is not finding your installed plugins then you may need to set the `extra_site_dirs`_ +config option. HTTP redirects @@ -1504,6 +1505,30 @@ Or you could change the implicit request content type from JSON to form by adding ``--form`` to the list. +``extra_site_dirs`` +~~~~~~~~~~~~~~~~~~~ + +An ``Array`` (by default empty) of directories that will be appended to Pythons +``sys.path``. This is helpful when you want to use a plugin from outside the +default ``sys.path``. + +Perhaps the most common use will be on macOS when HTTPie is installed via +Homebrew and plugins are installed via ``pip``, you'll need to set +``extra_site_dirs`` to something like the example below so that HTTPie can find +your plugins: + +.. code-block:: json + + "extra_site_dirs": [ + "/usr/local/lib/python3.7/site-packages", + "~/Library/Python/3.7/lib/python/site-packages" + ] + +If you are having trouble with HTTPie not finding your plugins then run +``http --debug`` to see the list of dirs being searched, the list of loaded +plugins and what dir each comes from. + + ``__meta__`` ~~~~~~~~~~~~ diff --git a/httpie/config.py b/httpie/config.py index 7621878fba..314b2ff23c 100644 --- a/httpie/config.py +++ b/httpie/config.py @@ -84,7 +84,8 @@ class Config(BaseConfigDict): about = 'HTTPie configuration file' DEFAULTS = { - 'default_options': [] + 'default_options': [], + 'extra_site_dirs': [], } def __init__(self, directory=DEFAULT_CONFIG_DIR): diff --git a/httpie/core.py b/httpie/core.py index 8442d614bd..2a3b32e4e3 100644 --- a/httpie/core.py +++ b/httpie/core.py @@ -12,8 +12,11 @@ """ import sys import errno +import os import platform +import site +from pkg_resources import working_set import requests from requests import __version__ as requests_version from pygments import __version__ as pygments_version @@ -56,6 +59,10 @@ def print_debug_info(env): ]) env.stderr.write('\n\n') env.stderr.write(repr(env)) + env.stderr.write('\n\n') + env.stderr.write('Looking for plugins in these directories:\n') + for p in sys.path: + env.stderr.write(' %s\n' % p) env.stderr.write('\n') @@ -180,7 +187,26 @@ def main(args=sys.argv[1:], env=Environment(), custom_log_error=None): """ args = decode_args(args, env.stdin_encoding) - plugin_manager.load_installed_plugins() + + sys_path_length = len(sys.path) + + for sitedir in env.config.extra_site_dirs: + parts = sitedir.split(os.sep) + if parts[0].startswith('~'): + parts[0] = os.path.expanduser(parts[0]) + env.stderr.write('addsitedir ' + os.sep.join(parts) + '\n') + site.addsitedir(os.sep.join(parts)) + + for new_path in sys.path[sys_path_length:]: + env.stderr.write('add_entry ' + new_path + '\n') + working_set.add_entry(new_path) + + include_debug_info = '--debug' in args + + if include_debug_info: + print_debug_info(env) + + plugin_manager.load_installed_plugins(env.stderr if include_debug_info else None) def log_error(msg, *args, **kwargs): msg = msg % args @@ -196,13 +222,10 @@ def log_error(msg, *args, **kwargs): if custom_log_error: log_error = custom_log_error - include_debug_info = '--debug' in args include_traceback = include_debug_info or '--traceback' in args - if include_debug_info: - print_debug_info(env) - if args == ['--debug']: - return ExitStatus.OK + if args == ['--debug']: + return ExitStatus.OK exit_status = ExitStatus.OK diff --git a/httpie/plugins/manager.py b/httpie/plugins/manager.py index c6390f2606..9e84188a90 100644 --- a/httpie/plugins/manager.py +++ b/httpie/plugins/manager.py @@ -27,12 +27,19 @@ def register(self, *plugins): def unregister(self, plugin): self._plugins.remove(plugin) - def load_installed_plugins(self): + def load_installed_plugins(self, debug_stream): + loaded_any_plugins = False for entry_point_name in ENTRY_POINT_NAMES: for entry_point in iter_entry_points(entry_point_name): + if debug_stream: + debug_stream.write("Loading plugin '{}' from '{}' ...\n".format( + entry_point.dist, entry_point.dist.location)) + loaded_any_plugins = True plugin = entry_point.load() plugin.package_name = entry_point.dist.key self.register(entry_point.load()) + if debug_stream and loaded_any_plugins: + debug_stream.write('\n') # Auth def get_auth_plugins(self): diff --git a/tests/test_config.py b/tests/test_config.py index d31ae1f38f..169a8da060 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,6 +1,13 @@ +from os import path +import sys + from httpie import __version__ -from utils import MockEnvironment, http +from utils import MockEnvironment, http, HTTP_OK from httpie.context import Environment +import pkg_resources +import pytest + +TEST_PLUGINS_PATH = path.join(path.abspath(path.dirname(__file__)), 'test_plugins') def test_default_options(httpbin): @@ -38,3 +45,44 @@ def test_migrate_implicit_content_type(): def test_current_version(): version = Environment().config['__meta__']['httpie'] assert version == __version__ + + +@pytest.fixture() +def sandbox(): + # this is based on setuptools.sandbox.setup_context + # but it seems the kind of thing likely to break! + saved_pkg_resources = pkg_resources.__getstate__() + saved_modules = sys.modules.copy() + saved_path = sys.path[:] + try: + yield + finally: + sys.path[:] = saved_path + sys.modules.update(saved_modules) + for mod_name in list(sys.modules.keys()): + if mod_name not in saved_modules and not mod_name.startswith('encodings.'): + del sys.modules[mod_name] + pkg_resources.__setstate__(saved_pkg_resources) + + +def test_extra_site_dirs_plugin_loading(sandbox): + assert TEST_PLUGINS_PATH not in sys.path + env = MockEnvironment() + env.config['extra_site_dirs'] = [TEST_PLUGINS_PATH] + env.config.save() + r = http('--debug', '--help', env=env) + assert "Loading plugin 'foo 0.0.1' from " in r.stderr + assert '--auth-type {basic,digest,foo}' in r + assert TEST_PLUGINS_PATH in sys.path + + +def test_extra_site_dirs_plugin_works(httpbin, sandbox): + assert TEST_PLUGINS_PATH not in sys.path + env = MockEnvironment() + env.config['extra_site_dirs'] = [TEST_PLUGINS_PATH] + env.config.save() + r = http('--auth-type=foo', '--auth=user:password', + 'GET', httpbin + '/basic-auth/user/password-foo', env=env) + assert HTTP_OK in r + assert r.json == {'authenticated': True, 'user': 'user'} + assert TEST_PLUGINS_PATH in sys.path diff --git a/tests/test_plugins/foo-0.0.1.dist-info/METADATA b/tests/test_plugins/foo-0.0.1.dist-info/METADATA new file mode 100644 index 0000000000..4e3c7c8f18 --- /dev/null +++ b/tests/test_plugins/foo-0.0.1.dist-info/METADATA @@ -0,0 +1,3 @@ +Metadata-Version: 2.1 +Name: foo +Version: 0.0.1 diff --git a/tests/test_plugins/foo-0.0.1.dist-info/entry_points.txt b/tests/test_plugins/foo-0.0.1.dist-info/entry_points.txt new file mode 100644 index 0000000000..76e51bff6c --- /dev/null +++ b/tests/test_plugins/foo-0.0.1.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[httpie.plugins.auth.v1] +foo = foo:FooPlugin diff --git a/tests/test_plugins/foo-0.0.1.dist-info/top_level.txt b/tests/test_plugins/foo-0.0.1.dist-info/top_level.txt new file mode 100644 index 0000000000..257cc5642c --- /dev/null +++ b/tests/test_plugins/foo-0.0.1.dist-info/top_level.txt @@ -0,0 +1 @@ +foo diff --git a/tests/test_plugins/foo.py b/tests/test_plugins/foo.py new file mode 100644 index 0000000000..44a2d039d4 --- /dev/null +++ b/tests/test_plugins/foo.py @@ -0,0 +1,10 @@ +from httpie.plugins import AuthPlugin +from httpie.plugins.builtin import HTTPBasicAuth + + +class FooPlugin(AuthPlugin): + auth_type = 'foo' + name = 'foo HTTP auth' + + def get_auth(self, username=None, password=None): + return HTTPBasicAuth(username, password + '-foo') diff --git a/tox.ini b/tox.ini index 8c888ed40c..ff46d2031e 100644 --- a/tox.ini +++ b/tox.ini @@ -20,4 +20,5 @@ commands = py.test \ --verbose \ --doctest-modules \ + --ignore=./tests/test_plugins \ {posargs:./httpie ./tests} From e6081340d365a88f502d314c3d614f715492b783 Mon Sep 17 00:00:00 2001 From: matthew16550 Date: Fri, 7 Sep 2018 12:38:51 +1200 Subject: [PATCH 2/5] An attempt to fix Travis build failure where python3.7 not found on osx --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4ad14764aa..9d21767f38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -61,7 +61,11 @@ install: if [[ $TRAVIS_OS_NAME == 'osx' ]]; then if [[ -n "$BREW_PYTHON_PACKAGE" ]]; then brew update - brew install "$BREW_PYTHON_PACKAGE" + if ! brew list --versions "$BREW_PYTHON_PACKAGE" >/dev/null; then + brew install "$BREW_PYTHON_PACKAGE" + elif ! brew outdated "$BREW_PYTHON_PACKAGE"; then + brew upgrade "$BREW_PYTHON_PACKAGE" + fi fi sudo pip2 install tox fi From fa7258bf7cfcee661ff981ee0d5fde39adcebe3a Mon Sep 17 00:00:00 2001 From: matthew16550 Date: Fri, 7 Sep 2018 15:49:26 +1200 Subject: [PATCH 3/5] Undo previous commit --- .travis.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9d21767f38..4ad14764aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -61,11 +61,7 @@ install: if [[ $TRAVIS_OS_NAME == 'osx' ]]; then if [[ -n "$BREW_PYTHON_PACKAGE" ]]; then brew update - if ! brew list --versions "$BREW_PYTHON_PACKAGE" >/dev/null; then - brew install "$BREW_PYTHON_PACKAGE" - elif ! brew outdated "$BREW_PYTHON_PACKAGE"; then - brew upgrade "$BREW_PYTHON_PACKAGE" - fi + brew install "$BREW_PYTHON_PACKAGE" fi sudo pip2 install tox fi From 3ca78085df6d37ad2ac61df547ab8e98cb230925 Mon Sep 17 00:00:00 2001 From: matthew16550 Date: Wed, 12 Sep 2018 19:24:13 +1200 Subject: [PATCH 4/5] Remove temporary debugging code --- httpie/core.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/httpie/core.py b/httpie/core.py index 2a3b32e4e3..83e19827f2 100644 --- a/httpie/core.py +++ b/httpie/core.py @@ -194,11 +194,9 @@ def main(args=sys.argv[1:], env=Environment(), custom_log_error=None): parts = sitedir.split(os.sep) if parts[0].startswith('~'): parts[0] = os.path.expanduser(parts[0]) - env.stderr.write('addsitedir ' + os.sep.join(parts) + '\n') site.addsitedir(os.sep.join(parts)) for new_path in sys.path[sys_path_length:]: - env.stderr.write('add_entry ' + new_path + '\n') working_set.add_entry(new_path) include_debug_info = '--debug' in args From 0ea66f88d531c61cf5b3c299955d6689ba5b1e83 Mon Sep 17 00:00:00 2001 From: matthew16550 Date: Fri, 1 Nov 2019 23:48:51 +1100 Subject: [PATCH 5/5] Finish merging upstream --- httpie/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/httpie/config.py b/httpie/config.py index 4ac181f2f6..4a063d2eb8 100644 --- a/httpie/config.py +++ b/httpie/config.py @@ -98,3 +98,7 @@ def _get_path(self) -> Path: @property def default_options(self) -> list: return self['default_options'] + + @property + def extra_site_dirs(self) -> list: + return self['extra_site_dirs']