Improve virtualenv support & egg-link resolution

- add virtual_envs= kwarg to Script & Evaluator constructors

- store sys_path for each evaluator instance

- query an actual python interpreter for sys.path contents

- use old path extension variant for fallback

- use addsitedir to load .pth extension files in old variant

- look for egg-link files in all directories in path

jedi/api/__init__.py

@@ -75,7 +75,8 @@ class Script(object):
     :type encoding: str
     """
     def __init__(self, source=None, line=None, column=None, path=None,
-                 encoding='utf-8', source_path=None, source_encoding=None):
+                 encoding='utf-8', source_path=None, source_encoding=None,
+                 virtual_envs=None):
         if source_path is not None:
             warnings.warn("Use path instead of source_path.", DeprecationWarning)
             path = source_path
@@ -108,7 +109,11 @@ def __init__(self, source=None, line=None, column=None, path=None,
         self._user_context = UserContext(self.source, self._pos)
         self._parser = UserContextParser(self._grammar, self.source, path,
                                          self._pos, self._user_context)
-        self._evaluator = Evaluator(self._grammar)
+        if virtual_envs is None:
+            venv = os.getenv('VIRTUAL_ENV')
+            if venv:
+                virtual_envs = [venv]
+        self._evaluator = Evaluator(self._grammar, virtual_envs=virtual_envs)
         debug.speed('init')
 
     @property

jedi/evaluate/__init__.py

@@ -61,6 +61,7 @@
 """
 
 import copy
+import sys
 from itertools import chain
 
 from jedi.parser import tree as pr
@@ -76,17 +77,25 @@
 from jedi.evaluate import precedence
 from jedi.evaluate import param
 from jedi.evaluate import helpers
+from jedi.evaluate.sys_path import get_venv_path
 
 
 class Evaluator(object):
-    def __init__(self, grammar):
+    def __init__(self, grammar, virtual_envs=None):
         self.grammar = grammar
         self.memoize_cache = {}  # for memoize decorators
         self.import_cache = {}  # like `sys.modules`.
         self.compiled_cache = {}  # see `compiled.create()`
         self.recursion_detector = recursion.RecursionDetector()
         self.execution_recursion_detector = recursion.ExecutionRecursionDetector()
         self.analysis = []
+        if virtual_envs:
+            self.sys_path = []
+            for venv in virtual_envs:
+                self.sys_path = [p for p in get_venv_path(venv)
+                                 if p != "" and p not in self.sys_path]
+        else:
+            self.sys_path = [p for p in sys.path if p != ""]
 
     def find_types(self, scope, name_str, position=None, search_global=False,
                    is_goto=False):

jedi/evaluate/compiled/__init__.py

@@ -10,7 +10,6 @@
 from jedi._compatibility import builtins as _builtins, unicode
 from jedi import debug
 from jedi.cache import underscore_memoization, memoize_method
-from jedi.evaluate.sys_path import get_sys_path
 from jedi.parser.tree import Param, Base, Operator, zero_position_modifier
 from jedi.evaluate.helpers import FakeName
 from . import fake
@@ -304,15 +303,12 @@ def parent(self, value):
         pass  # Just ignore this, FakeName tries to overwrite the parent attribute.
 
 
-def dotted_from_fs_path(fs_path, sys_path=None):
+def dotted_from_fs_path(fs_path, sys_path):
     """
     Changes `/usr/lib/python3.4/email/utils.py` to `email.utils`.  I.e.
     compares the path with sys.path and then returns the dotted_path. If the
     path is not in the sys.path, just returns None.
     """
-    if sys_path is None:
-        sys_path = get_sys_path()
-
     # prefer
     #   - UNIX
     #     /path/to/pythonX.Y/lib-dynload
@@ -332,13 +328,13 @@ def dotted_from_fs_path(fs_path, sys_path=None):
     return _path_re.sub('', fs_path[len(path):].lstrip(os.path.sep)).replace(os.path.sep, '.')
 
 
-def load_module(path, name):
+def load_module(path, name, evaluator):
+    sys_path = evaluator.sys_path
     if path is not None:
-        dotted_path = dotted_from_fs_path(path)
+        dotted_path = dotted_from_fs_path(path, sys_path=sys_path)
     else:
         dotted_path = name
 
-    sys_path = get_sys_path()
     if dotted_path is None:
         p, _, dotted_path = path.partition(os.path.sep)
         sys_path.insert(0, p)

jedi/evaluate/imports.py

@@ -22,7 +22,7 @@
 from jedi import cache
 from jedi.parser import fast
 from jedi.parser import tree as pr
-from jedi.evaluate.sys_path import get_sys_path, sys_path_with_modifications
+from jedi.evaluate.sys_path import sys_path_with_modifications
 from jedi.evaluate import helpers
 from jedi import settings
 from jedi.common import source_to_unicode
@@ -277,7 +277,7 @@ def _real_follow_file_system(self):
                 sys_path_mod.append(temp_path)
                 old_path, temp_path = temp_path, os.path.dirname(temp_path)
         else:
-            sys_path_mod = list(get_sys_path())
+            sys_path_mod = list(self._evaluator.sys_path)
 
         from jedi.evaluate.representation import ModuleWrapper
         module, rest = self._follow_sys_path(sys_path_mod)
@@ -488,12 +488,15 @@ def load(source):
                 with open(path, 'rb') as f:
                     source = f.read()
         else:
-            return compiled.load_module(path, name)
+            return compiled.load_module(path, name, evaluator)
         p = path or name
         p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p)
         cache.save_parser(path, name, p)
         return p.module
 
+    if sys_path is None:
+        sys_path = evaluator.sys_path
+
     cached = cache.load_parser(path, name)
     return load(source) if cached is None else cached.module
 

jedi/evaluate/sys_path.py

@@ -1,6 +1,9 @@
 import glob
 import os
 import sys
+from subprocess import check_output
+from ast import literal_eval
+from site import addsitedir
 
 from jedi._compatibility import exec_function, unicode
 from jedi.parser import tree as pr
@@ -10,24 +13,66 @@
 from jedi import common
 
 
-def get_sys_path():
-    def check_virtual_env(sys_path):
-        """ Add virtualenv's site-packages to the `sys.path`."""
-        venv = os.getenv('VIRTUAL_ENV')
-        if not venv:
-            return
-        venv = os.path.abspath(venv)
-        p = _get_venv_sitepackages(venv)
-        if p not in sys_path:
-            sys_path.insert(0, p)
-
-        # Add all egg-links from the virtualenv.
+def get_venv_path(venv):
+    """Get sys.path for specified virtual environment."""
+    try:
+        sys_path = _get_venv_path_online(venv)
+    except Exception as e:
+        debug.warning("Error when getting venv path: %s" % e)
+        sys_path = _get_venv_path_offline(venv)
+    with common.ignored(ValueError):
+        sys_path.remove('')
+    return _get_sys_path_with_egglinks(sys_path)
+
+
+def _get_sys_path_with_egglinks(sys_path):
+    """Find all paths including those referenced by egg-links.
+
+    Egg-link-referenced directories are inserted into path immediately after
+    the directory on which their links were found.  Such directories are not
+    taken into consideration by normal import mechanism, but they are traversed
+    when doing pkg_resources.require.
+    """
+    result = []
+    for p in sys_path:
+        result.append(p)
         for egg_link in glob.glob(os.path.join(p, '*.egg-link')):
             with open(egg_link) as fd:
-                sys_path.insert(0, fd.readline().rstrip())
+                for line in fd:
+                    line = line.strip()
+                    if line:
+                        result.append(os.path.join(p, line))
+                        # pkg_resources package only interprets the first
+                        # non-empty line in egg-link files.
+                        break
+    return result
+
+
+def _get_venv_path_offline(venv):
+    """Get sys.path for venv without starting up the interpreter."""
+    venv = os.path.abspath(venv)
+    sitedir = _get_venv_sitepackages(venv)
+    sys.path, old_sys_path = [], sys.path
+    try:
+        addsitedir(sitedir)
+        return sys.path
+    finally:
+        sys.path = old_sys_path
+
+
+def _get_venv_path_online(venv):
+    """Get sys.path for venv by running its python interpreter."""
+    venv = os.path.abspath(os.path.expanduser(venv))
+    for python_binary in ('python', 'python3', 'python.exe',
+                          'python3.exe'):
+        python_path = os.path.join(venv, 'bin', python_binary)
+        if os.path.isfile(python_path):
+            break
+    else:
+        raise RuntimeError("Cannot find python executable in venv: %s" % venv)
+    command = [python_path, '-c', 'import sys; print(sys.path)']
+    return literal_eval(check_output(command))
 
-    check_virtual_env(sys.path)
-    return [p for p in sys.path if p != ""]
 
 def _get_venv_sitepackages(venv):
     if os.name == 'nt':
@@ -106,15 +151,17 @@ def _paths_from_list_modifications(module_path, trailer1, trailer2):
 
     name = trailer1.children[1].value
     if name not in ['insert', 'append']:
-        return []
-
+        retimport 
     arg = trailer2.children[1]
     if name == 'insert' and len(arg.children) in (3, 4):  # Possible trailing comma.
         arg = arg.children[2]
     return _execute_code(module_path, arg.get_code())
 
 
 def _check_module(evaluator, module):
+    """
+    Detect sys.path modifications within module.
+    """
     def get_sys_path_powers(names):
         for name in names:
             power = name.parent.parent
@@ -126,10 +173,12 @@ def get_sys_path_powers(names):
                     if isinstance(n, pr.Name) and n.value == 'path':
                         yield name, power
 
-    sys_path = list(get_sys_path())  # copy
+    sys_path = list(evaluator.sys_path)  # copy
     try:
         possible_names = module.used_names['path']
     except KeyError:
+        # module.used_names is MergedNamesDict whose getitem never throws
+        # keyerror, this is superfluous.
         pass
     else:
         for name, power in get_sys_path_powers(possible_names):
@@ -146,7 +195,7 @@ def sys_path_with_modifications(evaluator, module):
     if module.path is None:
         # Support for modules without a path is bad, therefore return the
         # normal path.
-        return list(get_sys_path())
+        return list(evaluator.sys_path)
 
     curdir = os.path.abspath(os.curdir)
     with common.ignored(OSError):