Hide pydevd threads from threading module. Fixes #202

src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_saved_modules.py

@@ -123,3 +123,6 @@ def check(self, module, expected_attributes):
     with VerifyShadowedImport('http.server') as verify_shadowed:
         import http.server as BaseHTTPServer;    verify_shadowed.check(BaseHTTPServer, ['BaseHTTPRequestHandler'])
 
+# If set, this is a version of the threading.enumerate that doesn't have the patching to remove the pydevd threads.
+# Note: as it can't be set during execution, don't import the name (import the module and access it through its name).
+pydevd_saved_threading_enumerate = None

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_code_to_source.py

@@ -7,7 +7,7 @@
 
 import dis
 
-from _pydevd_bundle.pydevd_collect_bytecode_info import _iter_instructions
+from _pydevd_bundle.pydevd_collect_bytecode_info import iter_instructions
 from _pydevd_bundle.pydevd_constants import dict_iter_items, IS_PY2
 from _pydev_bundle import pydev_log
 import sys
@@ -536,7 +536,7 @@ def __init__(self, co, memo=None):
             memo = {}
         self.memo = memo
         self.co = co
-        self.instructions = list(_iter_instructions(co))
+        self.instructions = list(iter_instructions(co))
         self.stack = _Stack()
         self.writer = _Writer()
 

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_collect_bytecode_info.py

@@ -141,7 +141,7 @@ def _iter_as_bytecode_as_instructions_py2(co):
                 yield _Instruction(curr_op_name, op, _get_line(op_offset_to_line, initial_bytecode_offset, 0), oparg, is_jump_target, initial_bytecode_offset, str(oparg))
 
 
-def _iter_instructions(co):
+def iter_instructions(co):
     if sys.version_info[0] < 3:
         iter_in = _iter_as_bytecode_as_instructions_py2(co)
     else:
@@ -168,7 +168,7 @@ def collect_return_info(co, use_func_first_line=False):
 
     lst = []
     op_offset_to_line = dict(dis.findlinestarts(co))
-    for instruction in _iter_instructions(co):
+    for instruction in iter_instructions(co):
         curr_op_name = instruction.opname
         if curr_op_name == 'RETURN_VALUE':
             lst.append(ReturnInfo(_get_line(op_offset_to_line, instruction.offset, firstlineno, search=True)))
@@ -192,7 +192,7 @@ def collect_try_except_info(co, use_func_first_line=False):
 
         op_offset_to_line = dict(dis.findlinestarts(co))
 
-        for instruction in _iter_instructions(co):
+        for instruction in iter_instructions(co):
             curr_op_name = instruction.opname
 
             if curr_op_name in ('SETUP_EXCEPT', 'SETUP_FINALLY'):
@@ -351,7 +351,7 @@ def collect_try_except_info(co, use_func_first_line=False):
 
         offset_to_instruction_idx = {}
 
-        instructions = list(_iter_instructions(co))
+        instructions = list(iter_instructions(co))
 
         for i, instruction in enumerate(instructions):
             offset_to_instruction_idx[instruction.offset] = i
@@ -471,7 +471,7 @@ def collect_try_except_info(co, use_func_first_line=False):
 
         offset_to_instruction_idx = {}
 
-        instructions = list(_iter_instructions(co))
+        instructions = list(iter_instructions(co))
 
         for i, instruction in enumerate(instructions):
             offset_to_instruction_idx[instruction.offset] = i
@@ -656,7 +656,7 @@ def __init__(self, co, firstlineno, level=0):
         self.co = co
         self.firstlineno = firstlineno
         self.level = level
-        self.instructions = list(_iter_instructions(co))
+        self.instructions = list(iter_instructions(co))
         op_offset_to_line = self.op_offset_to_line = dict(dis.findlinestarts(co))
 
         # Update offsets so that all offsets have the line index (and update it based on

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py

@@ -317,6 +317,9 @@ def as_float_in_env(env_key, default):
 # on how the thread interruption works (there are some caveats related to it).
 PYDEVD_INTERRUPT_THREAD_TIMEOUT = as_float_in_env('PYDEVD_INTERRUPT_THREAD_TIMEOUT', -1)
 
+# If PYDEVD_APPLY_PATCHING_TO_HIDE_PYDEVD_THREADS is set to False, the patching to hide pydevd threads won't be applied.
+PYDEVD_APPLY_PATCHING_TO_HIDE_PYDEVD_THREADS = os.getenv('PYDEVD_APPLY_PATCHING_TO_HIDE_PYDEVD_THREADS', 'true').lower() in ENV_TRUE_LOWER_VALUES
+
 EXCEPTION_TYPE_UNHANDLED = 'UNHANDLED'
 EXCEPTION_TYPE_USER_UNHANDLED = 'USER_UNHANDLED'
 EXCEPTION_TYPE_HANDLED = 'HANDLED'

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py

@@ -1,11 +1,14 @@
 from _pydev_imps._pydev_saved_modules import threading
+from _pydev_imps import _pydev_saved_modules
 from _pydevd_bundle.pydevd_utils import notify_about_gevent_if_needed
 import weakref
-from _pydevd_bundle.pydevd_constants import IS_JYTHON
+from _pydevd_bundle.pydevd_constants import IS_JYTHON, IS_IRONPYTHON, \
+    PYDEVD_APPLY_PATCHING_TO_HIDE_PYDEVD_THREADS
 from _pydev_bundle.pydev_log import exception as pydev_log_exception
 import sys
 from _pydev_bundle import pydev_log
 import pydevd_tracing
+from _pydevd_bundle.pydevd_collect_bytecode_info import iter_instructions
 
 if IS_JYTHON:
     import org.python.core as JyCore  # @UnresolvedImport
@@ -67,7 +70,111 @@ def _stop_trace(self):
             pydevd_tracing.SetTrace(None)  # no debugging on this thread
 
 
+def _collect_load_names(func):
+    found_load_names = set()
+    for instruction in iter_instructions(func.__code__):
+        if instruction.opname in ('LOAD_GLOBAL', 'LOAD_ATTR', 'LOAD_METHOD'):
+            found_load_names.add(instruction.argrepr)
+    return found_load_names
+
+
+def _patch_threading_to_hide_pydevd_threads():
+    '''
+    Patches the needed functions on the `threading` module so that the pydevd threads are hidden.
+
+    Note that we patch the functions __code__ to avoid issues if some code had already imported those
+    variables prior to the patching.
+    '''
+    found_load_names = _collect_load_names(threading.enumerate)
+    # i.e.: we'll only apply the patching if the function seems to be what we expect.
+
+    new_threading_enumerate = None
+
+    if found_load_names == set(('_active_limbo_lock', '_limbo', '_active', 'values', 'list')):
+        pydev_log.debug('Applying patching to hide pydevd threads (Py3 version).')
+
+        def new_threading_enumerate():
+            with _active_limbo_lock:
+                ret = list(_active.values()) + list(_limbo.values())
+
+            return [t for t in ret if not getattr(t, 'is_pydev_daemon_thread', False)]
+
+    elif found_load_names == set(('_active_limbo_lock', '_limbo', '_active', 'values')):
+        pydev_log.debug('Applying patching to hide pydevd threads (Py2 version).')
+
+        def new_threading_enumerate():
+            with _active_limbo_lock:
+                ret = _active.values() + _limbo.values()
+
+            return [t for t in ret if not getattr(t, 'is_pydev_daemon_thread', False)]
+
+    else:
+        pydev_log.info('Unable to hide pydevd threads. Found names in threading.enumerate: %s', found_load_names)
+
+    if new_threading_enumerate is not None:
+
+        def pydevd_saved_threading_enumerate():
+            with threading._active_limbo_lock:
+                return list(threading._active.values()) + list(threading._limbo.values())
+
+        _pydev_saved_modules.pydevd_saved_threading_enumerate = pydevd_saved_threading_enumerate
+
+        threading.enumerate.__code__ = new_threading_enumerate.__code__
+
+        # We also need to patch the active count (to match what we have in the enumerate).
+        def new_active_count():
+            # Note: as this will be executed in the `threading` module, `enumerate` will
+            # actually be threading.enumerate.
+            return len(enumerate())
+
+        threading.active_count.__code__ = new_active_count.__code__
+
+        # When shutting down, Python (on some versions) may do something as:
+        #
+        # def _pickSomeNonDaemonThread():
+        #     for t in enumerate():
+        #         if not t.daemon and t.is_alive():
+        #             return t
+        #     return None
+        #
+        # But in this particular case, we do want threads with `is_pydev_daemon_thread` to appear
+        # explicitly due to the pydevd `CheckAliveThread` (because we want the shutdown to wait on it).
+        # So, it can't rely on the `enumerate` for that anymore as it's patched to not return pydevd threads.
+        if hasattr(threading, '_pickSomeNonDaemonThread'):
+
+            def new_pick_some_non_daemon_thread():
+                with _active_limbo_lock:
+                    # Ok for py2 and py3.
+                    threads = list(_active.values()) + list(_limbo.values())
+
+                for t in threads:
+                    if not t.daemon and t.is_alive():
+                        return t
+                return None
+
+            threading._pickSomeNonDaemonThread.__code__ = new_pick_some_non_daemon_thread.__code__
+
+
+_patched_threading_to_hide_pydevd_threads = False
+
+
 def mark_as_pydevd_daemon_thread(thread):
+    if not IS_JYTHON and not IS_IRONPYTHON and PYDEVD_APPLY_PATCHING_TO_HIDE_PYDEVD_THREADS:
+        global _patched_threading_to_hide_pydevd_threads
+        if not _patched_threading_to_hide_pydevd_threads:
+            # When we mark the first thread as a pydevd daemon thread, we also change the threading
+            # functions to hide pydevd threads.
+            # Note: we don't just "hide" the pydevd threads from the threading module by not using it
+            # (i.e.: just using the `thread.start_new_thread` instead of `threading.Thread`)
+            # because there's 1 thread (the `CheckAliveThread`) which is a pydevd thread but
+            # isn't really a daemon thread (so, we need CPython to wait on it for shutdown,
+            # in which case it needs to be in `threading` and the patching would be needed anyways).
+            _patched_threading_to_hide_pydevd_threads = True
+            try:
+                _patch_threading_to_hide_pydevd_threads()
+            except:
+                pydev_log.exception('Error applying patching to hide pydevd threads.')
+
     thread.pydev_do_not_trace = True
     thread.is_pydev_daemon_thread = True
     thread.daemon = True

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py

@@ -3,6 +3,7 @@
 import warnings
 from _pydev_bundle import pydev_log
 from _pydev_imps._pydev_saved_modules import thread
+from _pydev_imps import _pydev_saved_modules
 import signal
 import os
 import ctypes
@@ -177,7 +178,11 @@ def dump_threads(stream=None, show_pydevd_threads=True):
         stream = sys.stderr
     thread_id_to_name_and_is_pydevd_thread = {}
     try:
-        for t in threading.enumerate():
+        threading_enumerate = _pydev_saved_modules.pydevd_saved_threading_enumerate
+        if threading_enumerate is None:
+            threading_enumerate = threading.enumerate
+
+        for t in threading_enumerate():
             is_pydevd_thread = getattr(t, 'is_pydev_daemon_thread', False)
             thread_id_to_name_and_is_pydevd_thread[t.ident] = (
                 '%s  (daemon: %s, pydevd thread: %s)' % (t.name, t.daemon, is_pydevd_thread),
@@ -297,6 +302,7 @@ def hasattr_checked(obj, name):
     else:
         return True
 
+
 def getattr_checked(obj, name):
     try:
         return getattr(obj, name)

src/debugpy/_vendored/pydevd/tests_python/test_debugger.py

@@ -4372,6 +4372,33 @@ def get_environ(writer):
 
     assert ('the module "%s" could not be imported because it is shadowed by:' % (module_name.split('.')[0])) in writer.get_stderr()
 
+
+def test_debugger_hide_pydevd_threads(case_setup, pyfile):
+
+    @pyfile
+    def target_file():
+        import threading
+        from _pydevd_bundle import pydevd_constants
+        found_pydevd_thread = False
+        for t in threading.enumerate():
+            if getattr(t, 'is_pydev_daemon_thread', False):
+                found_pydevd_thread = True
+
+        if pydevd_constants.IS_CPYTHON:
+            assert not found_pydevd_thread
+        else:
+            assert found_pydevd_thread
+        print('TEST SUCEEDED')
+
+    with case_setup.test_file(target_file) as writer:
+        line = writer.get_line_index_with_content('TEST SUCEEDED')
+        writer.write_add_breakpoint(line)
+        writer.write_make_initial_run()
+
+        hit = writer.wait_for_breakpoint_hit(line=line)
+        writer.write_run_thread(hit.thread_id)
+        writer.finished_ok = True
+
 # Jython needs some vars to be set locally.
 # set JAVA_HOME=c:\bin\jdk1.8.0_172
 # set PATH=%PATH%;C:\bin\jython2.7.0\bin

src/debugpy/_vendored/pydevd/tests_python/test_utilities.py

@@ -3,7 +3,7 @@
 from _pydevd_bundle.pydevd_utils import convert_dap_log_message_to_expression
 from tests_python.debug_constants import IS_PY26, IS_PY3K, TEST_GEVENT, IS_CPYTHON
 import sys
-from _pydevd_bundle.pydevd_constants import IS_WINDOWS, IS_PY2, IS_PYPY
+from _pydevd_bundle.pydevd_constants import IS_WINDOWS, IS_PY2, IS_PYPY, IS_JYTHON
 import pytest
 import os
 import codecs
@@ -424,7 +424,7 @@ def test_find_main_thread_id():
     )
 
 
-@pytest.mark.skipif(not IS_WINDOWS, reason='Windows-only test.')
+@pytest.mark.skipif(not IS_WINDOWS or IS_JYTHON, reason='Windows-only test.')
 def test_get_ppid():
     from _pydevd_bundle.pydevd_api import PyDevdAPI
     api = PyDevdAPI()
@@ -522,3 +522,38 @@ def __init__(self, offset):
     assert get_smart_step_into_variant_from_frame_offset(2, variants) is variants[0]
     assert get_smart_step_into_variant_from_frame_offset(3, variants) is variants[1]
     assert get_smart_step_into_variant_from_frame_offset(4, variants) is variants[1]
+
+
+def test_threading_hide_pydevd():
+
+    class T(threading.Thread):
+
+        def __init__(self, is_pydev_daemon_thread):
+            from _pydevd_bundle.pydevd_daemon_thread import mark_as_pydevd_daemon_thread
+            threading.Thread.__init__(self)
+            if is_pydev_daemon_thread:
+                mark_as_pydevd_daemon_thread(self)
+            else:
+                self.daemon = True
+            self.event = threading.Event()
+
+        def run(self):
+            self.event.wait(10)
+
+    current_count = threading.active_count()
+    t0 = T(True)
+    t1 = T(False)
+    t0.start()
+    t1.start()
+
+    # i.e.: the patching doesn't work for other implementations.
+    if IS_CPYTHON:
+        assert threading.active_count() == current_count + 1
+        assert t0 not in threading.enumerate()
+    else:
+        assert threading.active_count() == current_count + 2
+        assert t0 in threading.enumerate()
+
+    assert t1 in threading.enumerate()
+    t0.event.set()
+    t1.event.set()