From a64a55f65b9547d2d0bf3b3571e10278b13eb974 Mon Sep 17 00:00:00 2001 From: immerrr Date: Sun, 5 Apr 2015 00:19:11 +0200 Subject: [PATCH] Improve virtualenv support & egg-link resolution - add virtual_envs= kwarg to Script & Evaluator constructors - store sys_path for each evaluator instance - query an actual python interpreter for sys.path contents - use old path extension variant for fallback - use addsitedir to load .pth extension files in old variant - look for egg-link files in all directories in path --- jedi/api/__init__.py | 9 +++- jedi/evaluate/__init__.py | 11 +++- jedi/evaluate/compiled/__init__.py | 12 ++--- jedi/evaluate/imports.py | 9 ++-- jedi/evaluate/sys_path.py | 87 +++++++++++++++++++++++------- 5 files changed, 95 insertions(+), 33 deletions(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 4f10b0f2e..0dca64151 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -75,7 +75,8 @@ class Script(object): :type encoding: str """ def __init__(self, source=None, line=None, column=None, path=None, - encoding='utf-8', source_path=None, source_encoding=None): + encoding='utf-8', source_path=None, source_encoding=None, + virtual_envs=None): if source_path is not None: warnings.warn("Use path instead of source_path.", DeprecationWarning) path = source_path @@ -108,7 +109,11 @@ def __init__(self, source=None, line=None, column=None, path=None, self._user_context = UserContext(self.source, self._pos) self._parser = UserContextParser(self._grammar, self.source, path, self._pos, self._user_context) - self._evaluator = Evaluator(self._grammar) + if virtual_envs is None: + venv = os.getenv('VIRTUAL_ENV') + if venv: + virtual_envs = [venv] + self._evaluator = Evaluator(self._grammar, virtual_envs=virtual_envs) debug.speed('init') @property diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 697082705..1186aa985 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -61,6 +61,7 @@ """ import copy +import sys from itertools import chain from jedi.parser import tree as pr @@ -76,10 +77,11 @@ from jedi.evaluate import precedence from jedi.evaluate import param from jedi.evaluate import helpers +from jedi.evaluate.sys_path import get_venv_path class Evaluator(object): - def __init__(self, grammar): + def __init__(self, grammar, virtual_envs=None): self.grammar = grammar self.memoize_cache = {} # for memoize decorators self.import_cache = {} # like `sys.modules`. @@ -87,6 +89,13 @@ def __init__(self, grammar): self.recursion_detector = recursion.RecursionDetector() self.execution_recursion_detector = recursion.ExecutionRecursionDetector() self.analysis = [] + if virtual_envs: + self.sys_path = [] + for venv in virtual_envs: + self.sys_path = [p for p in get_venv_path(venv) + if p != "" and p not in self.sys_path] + else: + self.sys_path = [p for p in sys.path if p != ""] def find_types(self, scope, name_str, position=None, search_global=False, is_goto=False): diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index bf62c6439..9c05e2ea0 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -10,7 +10,6 @@ from jedi._compatibility import builtins as _builtins, unicode from jedi import debug from jedi.cache import underscore_memoization, memoize_method -from jedi.evaluate.sys_path import get_sys_path from jedi.parser.tree import Param, Base, Operator, zero_position_modifier from jedi.evaluate.helpers import FakeName from . import fake @@ -304,15 +303,12 @@ def parent(self, value): pass # Just ignore this, FakeName tries to overwrite the parent attribute. -def dotted_from_fs_path(fs_path, sys_path=None): +def dotted_from_fs_path(fs_path, sys_path): """ Changes `/usr/lib/python3.4/email/utils.py` to `email.utils`. I.e. compares the path with sys.path and then returns the dotted_path. If the path is not in the sys.path, just returns None. """ - if sys_path is None: - sys_path = get_sys_path() - # prefer # - UNIX # /path/to/pythonX.Y/lib-dynload @@ -332,13 +328,13 @@ def dotted_from_fs_path(fs_path, sys_path=None): return _path_re.sub('', fs_path[len(path):].lstrip(os.path.sep)).replace(os.path.sep, '.') -def load_module(path, name): +def load_module(path, name, evaluator): + sys_path = evaluator.sys_path if path is not None: - dotted_path = dotted_from_fs_path(path) + dotted_path = dotted_from_fs_path(path, sys_path=sys_path) else: dotted_path = name - sys_path = get_sys_path() if dotted_path is None: p, _, dotted_path = path.partition(os.path.sep) sys_path.insert(0, p) diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 78acc98c2..5c973cd92 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -22,7 +22,7 @@ from jedi import cache from jedi.parser import fast from jedi.parser import tree as pr -from jedi.evaluate.sys_path import get_sys_path, sys_path_with_modifications +from jedi.evaluate.sys_path import sys_path_with_modifications from jedi.evaluate import helpers from jedi import settings from jedi.common import source_to_unicode @@ -277,7 +277,7 @@ def _real_follow_file_system(self): sys_path_mod.append(temp_path) old_path, temp_path = temp_path, os.path.dirname(temp_path) else: - sys_path_mod = list(get_sys_path()) + sys_path_mod = list(self._evaluator.sys_path) from jedi.evaluate.representation import ModuleWrapper module, rest = self._follow_sys_path(sys_path_mod) @@ -488,12 +488,15 @@ def load(source): with open(path, 'rb') as f: source = f.read() else: - return compiled.load_module(path, name) + return compiled.load_module(path, name, evaluator) p = path or name p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p) cache.save_parser(path, name, p) return p.module + if sys_path is None: + sys_path = evaluator.sys_path + cached = cache.load_parser(path, name) return load(source) if cached is None else cached.module diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index 095278278..4af2ffb96 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -1,6 +1,9 @@ import glob import os import sys +from subprocess import check_output +from ast import literal_eval +from site import addsitedir from jedi._compatibility import exec_function, unicode from jedi.parser import tree as pr @@ -10,24 +13,66 @@ from jedi import common -def get_sys_path(): - def check_virtual_env(sys_path): - """ Add virtualenv's site-packages to the `sys.path`.""" - venv = os.getenv('VIRTUAL_ENV') - if not venv: - return - venv = os.path.abspath(venv) - p = _get_venv_sitepackages(venv) - if p not in sys_path: - sys_path.insert(0, p) - - # Add all egg-links from the virtualenv. +def get_venv_path(venv): + """Get sys.path for specified virtual environment.""" + try: + sys_path = _get_venv_path_online(venv) + except Exception as e: + debug.warning("Error when getting venv path: %s" % e) + sys_path = _get_venv_path_offline(venv) + with common.ignored(ValueError): + sys_path.remove('') + return _get_sys_path_with_egglinks(sys_path) + + +def _get_sys_path_with_egglinks(sys_path): + """Find all paths including those referenced by egg-links. + + Egg-link-referenced directories are inserted into path immediately after + the directory on which their links were found. Such directories are not + taken into consideration by normal import mechanism, but they are traversed + when doing pkg_resources.require. + """ + result = [] + for p in sys_path: + result.append(p) for egg_link in glob.glob(os.path.join(p, '*.egg-link')): with open(egg_link) as fd: - sys_path.insert(0, fd.readline().rstrip()) + for line in fd: + line = line.strip() + if line: + result.append(os.path.join(p, line)) + # pkg_resources package only interprets the first + # non-empty line in egg-link files. + break + return result + + +def _get_venv_path_offline(venv): + """Get sys.path for venv without starting up the interpreter.""" + venv = os.path.abspath(venv) + sitedir = _get_venv_sitepackages(venv) + sys.path, old_sys_path = [], sys.path + try: + addsitedir(sitedir) + return sys.path + finally: + sys.path = old_sys_path + + +def _get_venv_path_online(venv): + """Get sys.path for venv by running its python interpreter.""" + venv = os.path.abspath(os.path.expanduser(venv)) + for python_binary in ('python', 'python3', 'python.exe', + 'python3.exe'): + python_path = os.path.join(venv, 'bin', python_binary) + if os.path.isfile(python_path): + break + else: + raise RuntimeError("Cannot find python executable in venv: %s" % venv) + command = [python_path, '-c', 'import sys; print(sys.path)'] + return literal_eval(check_output(command)) - check_virtual_env(sys.path) - return [p for p in sys.path if p != ""] def _get_venv_sitepackages(venv): if os.name == 'nt': @@ -106,8 +151,7 @@ def _paths_from_list_modifications(module_path, trailer1, trailer2): name = trailer1.children[1].value if name not in ['insert', 'append']: - return [] - + retimport arg = trailer2.children[1] if name == 'insert' and len(arg.children) in (3, 4): # Possible trailing comma. arg = arg.children[2] @@ -115,6 +159,9 @@ def _paths_from_list_modifications(module_path, trailer1, trailer2): def _check_module(evaluator, module): + """ + Detect sys.path modifications within module. + """ def get_sys_path_powers(names): for name in names: power = name.parent.parent @@ -126,10 +173,12 @@ def get_sys_path_powers(names): if isinstance(n, pr.Name) and n.value == 'path': yield name, power - sys_path = list(get_sys_path()) # copy + sys_path = list(evaluator.sys_path) # copy try: possible_names = module.used_names['path'] except KeyError: + # module.used_names is MergedNamesDict whose getitem never throws + # keyerror, this is superfluous. pass else: for name, power in get_sys_path_powers(possible_names): @@ -146,7 +195,7 @@ def sys_path_with_modifications(evaluator, module): if module.path is None: # Support for modules without a path is bad, therefore return the # normal path. - return list(get_sys_path()) + return list(evaluator.sys_path) curdir = os.path.abspath(os.curdir) with common.ignored(OSError):