Merge from 5.x: PR #16111

Fixes #15648
Fixes #14871

external-deps/python-lsp-server/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/python-lsp/python-lsp-server.git
 	branch = develop
-	commit = 32bbc067f4a0f97ddabe69bc80059fa7481d622e
-	parent = bb30c7cb3dd40c963a50d3b6730b0f6479e56187
+	commit = 03ab6efdec1d53b9d2359caa376ef12269b81cb5
+	parent = 1a0510697d444347aa5244bf9bb68531a796470f
 	method = merge
 	cmdver = 0.4.3

external-deps/python-lsp-server/CONFIGURATION.md

@@ -32,6 +32,7 @@ This server can be configured using `workspace/didChangeConfiguration` method. E
 | `pylsp.plugins.jedi_signature_help.enabled` | `boolean` | Enable or disable the plugin. | `true` |
 | `pylsp.plugins.jedi_symbols.enabled` | `boolean` | Enable or disable the plugin. | `true` |
 | `pylsp.plugins.jedi_symbols.all_scopes` | `boolean` | If True lists the names of all scopes instead of only the module namespace. | `true` |
+| `pylsp.plugins.jedi_symbols.include_import_symbols` | `boolean` | If True includes symbols imported from other libraries. | `true` |
 | `pylsp.plugins.mccabe.enabled` | `boolean` | Enable or disable the plugin. | `true` |
 | `pylsp.plugins.mccabe.threshold` | `number`  | The minimum threshold that triggers warnings about cyclomatic complexity. | `15` |
 | `pylsp.plugins.preload.enabled` | `boolean` | Enable or disable the plugin. | `true` |

external-deps/python-lsp-server/pylsp/config/schema.json

@@ -157,6 +157,11 @@
       "default": true,
       "description": "If True lists the names of all scopes instead of only the module namespace."
     },
+    "pylsp.plugins.jedi_symbols.include_import_symbols": {
+      "type": "boolean",
+      "default": true,
+      "description": "If True includes symbols imported from other libraries."
+    },
     "pylsp.plugins.mccabe.enabled": {
       "type": "boolean",
       "default": true,

external-deps/python-lsp-server/pylsp/plugins/symbols.py

@@ -19,28 +19,46 @@ def pylsp_document_symbols(config, document):
     symbols_settings = config.plugin_settings('jedi_symbols')
     all_scopes = symbols_settings.get('all_scopes', True)
     add_import_symbols = symbols_settings.get('include_import_symbols', True)
-
-    use_document_path = False
-    document_dir = os.path.normpath(os.path.dirname(document.path))
-    if not os.path.isfile(os.path.join(document_dir, '__init__.py')):
-        use_document_path = True
-
-    definitions = document.jedi_names(use_document_path, all_scopes=all_scopes)
-    module_name = document.dot_path
+    definitions = document.jedi_names(all_scopes=all_scopes)
     symbols = []
     exclude = set({})
     redefinitions = {}
     while definitions != []:
         d = definitions.pop(0)
+
+        # Skip symbols imported from other modules.
         if not add_import_symbols:
+            # Skip if there's an import in the code the symbol is defined.
+            code = d.get_line_code()
+            if ' import ' in code or 'import ' in code:
+                continue
+
+            # Skip comparing module names.
             sym_full_name = d.full_name
+            module_name = document.dot_path
             if sym_full_name is not None:
-                if (not sym_full_name.startswith(module_name) and
-                        not sym_full_name.startswith('__main__')):
-                    continue
+                # module_name returns where the symbol is imported, whereas
+                # full_name says where it really comes from. So if the parent
+                # modules in full_name are not in module_name, it means the
+                # symbol was not defined there.
+                # Note: The last element of sym_full_name is the symbol itself,
+                # so we don't need to use it below.
+                imported_symbol = True
+                for mod in sym_full_name.split('.')[:-1]:
+                    if mod in module_name:
+                        imported_symbol = False
+
+                # When there's no __init__.py next to a file or in one of its
+                # parents, the check above fails. However, Jedi has a nice way
+                # to tell if the symbol was declared in the same file: if
+                # full_name starts by __main__.
+                if imported_symbol:
+                    if not sym_full_name.startswith('__main__'):
+                        continue
+
         try:
             docismodule = os.path.samefile(document.path, d.module_path)
-        except TypeError:
+        except (TypeError, FileNotFoundError):
             # Python 2 on Windows has no .samefile, but then these are
             # strings for sure
             docismodule = document.path == d.module_path

external-deps/python-lsp-server/pylsp/python_lsp.py

@@ -383,7 +383,8 @@ def m_text_document__signature_help(self, textDocument=None, position=None, **_k
         return self.signature_help(textDocument['uri'], position)
 
     def m_workspace__did_change_configuration(self, settings=None):
-        self.config.update((settings or {}).get('pylsp', {}))
+        if self.config is not None:
+            self.config.update((settings or {}).get('pylsp', {}))
         for workspace in self.workspaces.values():
             workspace.update_config(settings)
             for doc_uri in workspace.documents:

external-deps/python-lsp-server/pylsp/workspace.py

@@ -239,8 +239,8 @@ def word_at_position(self, position):
         return m_start[0] + m_end[-1]
 
     @lock
-    def jedi_names(self, use_document_path, all_scopes=False, definitions=True, references=False):
-        script = self.jedi_script(use_document_path=use_document_path)
+    def jedi_names(self, all_scopes=False, definitions=True, references=False):
+        script = self.jedi_script()
         return script.get_names(all_scopes=all_scopes, definitions=definitions,
                                 references=references)
 

external-deps/python-lsp-server/test/test_language_server.py

@@ -7,6 +7,7 @@
 import sys
 from threading import Thread
 
+from flaky import flaky
 from pylsp_jsonrpc.exceptions import JsonRpcMethodNotFound
 import pytest
 
@@ -75,6 +76,7 @@ def client_exited_server():
     assert client_server_pair.process.is_alive() is False
 
 
+@flaky(max_runs=10, min_passes=1)
 @pytest.mark.skipif(sys.platform == 'darwin', reason='Too flaky on Mac')
 def test_initialize(client_server):  # pylint: disable=redefined-outer-name
     response = client_server._endpoint.request('initialize', {

spyder/app/tests/test_mainwindow.py

@@ -247,25 +247,52 @@ def main_window(request, tmpdir):
 
     if preload_project:
         # Create project
-        project_path = str(tmpdir.mkdir('test_project'))
-        project = EmptyProject(project_path)
+        project = tmpdir.mkdir('test_project')
+        project_subdir = project.mkdir('subdir')
+
+        # Create directories out of the project
+        out_of_project_1 = tmpdir.mkdir('out_of_project_1')
+        out_of_project_2 = tmpdir.mkdir('out_of_project_2')
+        out_of_project_1_subdir = out_of_project_1.mkdir('subdir')
+        out_of_project_2_subdir = out_of_project_2.mkdir('subdir')
+
+        project_path = str(project)
+        spy_project = EmptyProject(project_path)
         CONF.set('project_explorer', 'current_project_path', project_path)
 
-        # Add some files to project
-        filenames = [
-            osp.join(project_path, f) for f in
-            ['file1.py', 'file2.py', 'file3.txt']
-        ]
+        # Add some files to project. This is necessary to test that we get
+        # symgbols for all these files.
+        abs_filenames = []
+        filenames_to_create = {
+            project: ['file1.py', 'file2.py', 'file3.txt', '__init__.py'],
+            project_subdir: ['a.py', '__init__.py'],
+            out_of_project_1: ['b.py'],
+            out_of_project_2: ['c.py', '__init__.py'],
+            out_of_project_1_subdir: ['d.py', '__init__.py'],
+            out_of_project_2_subdir: ['e.py']
+        }
 
-        for filename in filenames:
-            with open(filename, 'w') as f:
+        for path in filenames_to_create.keys():
+            filenames = filenames_to_create[path]
+            for filename in filenames:
+                f = path.join(filename)
+                abs_filenames.append(str(f))
                 if osp.splitext(filename)[1] == '.py':
-                    f.write("def f(x):\n"
-                            "    return x\n")
+                    code = dedent(
+                        """
+                        from math import cos
+                        from numpy import (
+                           linspace)
+
+                        def f(x):
+                            return x
+                        """
+                    )
+                    f.write(code)
                 else:
                     f.write("Hello world!")
 
-        project.set_recent_files(filenames)
+        spy_project.set_recent_files(abs_filenames)
     else:
         CONF.set('project_explorer', 'current_project_path', None)
 
@@ -3816,7 +3843,8 @@ def test_tour_message(main_window, qtbot):
 @flaky(max_runs=3)
 @pytest.mark.use_introspection
 @pytest.mark.preload_project
-@pytest.mark.skipif(os.name == 'nt', reason="Fails on Windows")
+@pytest.mark.skipif(not sys.platform.startswith('linux'),
+                    reason="Only works on Linux")
 def test_update_outline(main_window, qtbot, tmpdir):
     """
     Test that files in the Outline pane are updated at startup and
@@ -3834,12 +3862,12 @@ def test_update_outline(main_window, qtbot, tmpdir):
     ]
 
     # Wait a bit for trees to be filled
-    qtbot.wait(5000)
+    qtbot.wait(20000)
 
     # Assert all Python editors are filled
     assert all(
         [
-            len(treewidget.editor_tree_cache[editor.get_id()]) > 0
+            len(treewidget.editor_tree_cache[editor.get_id()]) == 1
             for editor in editors_py
         ]
     )

spyder/plugins/completion/providers/languageserver/client.py

@@ -15,6 +15,7 @@
 import logging
 import os
 import os.path as osp
+import pathlib
 import signal
 import sys
 import time
@@ -26,6 +27,7 @@
 import psutil
 
 # Local imports
+from spyder.api.config.mixins import SpyderConfigurationAccessor
 from spyder.config.base import (DEV, get_conf_path, get_debug_level,
                                 running_under_pytest)
 from spyder.plugins.completion.api import (
@@ -38,14 +40,8 @@
     MessageKind)
 from spyder.plugins.completion.providers.languageserver.providers import (
     LSPMethodProviderMixIn)
-from spyder.py3compat import PY2
 from spyder.utils.misc import getcwd_or_home, select_port
 
-# Conditional imports
-if PY2:
-    import pathlib2 as pathlib
-else:
-    import pathlib
 
 # Main constants
 LOCATION = osp.realpath(osp.join(os.getcwd(),
@@ -63,7 +59,7 @@
 
 
 @class_register
-class LSPClient(QObject, LSPMethodProviderMixIn):
+class LSPClient(QObject, LSPMethodProviderMixIn, SpyderConfigurationAccessor):
     """Language Server Protocol v3.0 client implementation."""
     #: Signal to inform the editor plugin that the client has
     #  started properly and it's ready to be used.
@@ -162,6 +158,26 @@ def _get_log_filename(self, kind):
 
         return location
 
+    def _clean_sys_path(self):
+        """
+        Remove from sys.path entries that come from our config system.
+
+        They will be passed to the server in the extra_paths option
+        and are not needed for the transport layer.
+        """
+        spyder_pythonpath = self.get_conf(
+            'spyder_pythonpath',
+            section='main',
+            default=[]
+        )
+
+        sys_path = sys.path[:]
+        for path in spyder_pythonpath:
+            if path in sys_path:
+                sys_path.remove(path)
+
+        return sys_path
+
     @property
     def server_log_file(self):
         """
@@ -265,7 +281,8 @@ def start_server(self):
         if DEV or running_under_pytest():
             running_in_ci = bool(os.environ.get('CI'))
             if os.name != 'nt' or os.name == 'nt' and not running_in_ci:
-                env.insert('PYTHONPATH', os.pathsep.join(sys.path)[:])
+                sys_path = self._clean_sys_path()
+                env.insert('PYTHONPATH', os.pathsep.join(sys_path)[:])
 
         # Adjustments for the Python language server.
         if self.language == 'python':
@@ -326,10 +343,11 @@ def start_transport(self):
         # Modifying PYTHONPATH to run transport in development mode or
         # tests
         if DEV or running_under_pytest():
+            sys_path = self._clean_sys_path()
             if running_under_pytest():
-                env.insert('PYTHONPATH', os.pathsep.join(sys.path)[:])
+                env.insert('PYTHONPATH', os.pathsep.join(sys_path)[:])
             else:
-                env.insert('PYTHONPATH', os.pathsep.join(sys.path)[1:])
+                env.insert('PYTHONPATH', os.pathsep.join(sys_path)[1:])
             self.transport.setProcessEnvironment(env)
 
         # Set up transport
@@ -433,6 +451,10 @@ def send(self, method, params, kind):
         if self.is_down():
             return
 
+        # Don't send requests to the server before it's been initialized.
+        if not self.initialized and method != 'initialize':
+            return
+
         if ClientConstants.CANCEL in params:
             return
         _id = self.request_seq

spyder/plugins/outlineexplorer/plugin.py

@@ -79,8 +79,3 @@ def stop_symbol_services(self, language):
         """Disable LSP symbols functionality."""
         explorer = self.get_widget()
         explorer.stop_symbol_services(language)
-
-    def update_all_editors(self):
-        """Update all editors with an associated LSP server."""
-        explorer = self.get_widget()
-        explorer.update_all_editors()

spyder/plugins/outlineexplorer/widgets.py

@@ -375,13 +375,13 @@ def toggle_show_all_files(self, state):
     def toggle_show_comments(self, state):
         self.show_comments = state
         self.sig_update_configuration.emit()
-        self.update_all_editors(reset_info=True)
+        self.update_editors(language='python')
 
     @on_conf_change(option='group_cells')
     def toggle_group_cells(self, state):
         self.group_cells = state
         self.sig_update_configuration.emit()
-        self.update_all_editors(reset_info=True)
+        self.update_editors(language='python')
 
     @on_conf_change(option='display_variables')
     def toggle_variables(self, state):
@@ -489,7 +489,7 @@ def set_current_editor(self, editor, update):
 
         # Update tree with currently stored info or require symbols if
         # necessary.
-        if (editor.get_language() in self._languages and
+        if (editor.get_language().lower() in self._languages and
                 len(self.editor_tree_cache[editor_id]) == 0):
             if editor.info is not None:
                 self.update_editor(editor.info)
@@ -552,12 +552,6 @@ def update_editors(self, language):
                 self.editors_to_update[language].remove(editor)
             self.update_timers[language].start()
 
-    def update_all_editors(self, reset_info=False):
-        """Update all editors with LSP support."""
-        for language in self._languages:
-            self.set_editors_to_update(language, reset_info=reset_info)
-            self.update_timers[language].start()
-
     @Slot(list)
     def update_editor(self, items, editor=None):
         """
@@ -567,14 +561,13 @@ def update_editor(self, items, editor=None):
         if items is None:
             return
 
-        plugin_base = self.parent().parent()
         if editor is None:
             editor = self.current_editor
         editor_id = editor.get_id()
         language = editor.get_language()
         update = self.update_tree(items, editor_id, language)
 
-        if getattr(plugin_base, "_isvisible", True) and update:
+        if update:
             self.save_expanded_state()
             self.restore_expanded_state()
             self.do_follow_cursor()
@@ -1027,7 +1020,3 @@ def start_symbol_services(self, language):
     def stop_symbol_services(self, language):
         """Disable LSP symbols functionality."""
         self.treewidget.stop_symbol_services(language)
-
-    def update_all_editors(self):
-        """Update all editors with an associated LSP server."""
-        self.treewidget.update_all_editors()