From 655e35642c357b18344d589c1ae0751f90860060 Mon Sep 17 00:00:00 2001 From: Fabio Zadrozny Date: Fri, 29 Oct 2021 10:20:29 -0300 Subject: [PATCH] Hide pydevd threads from threading module. Fixes #202 --- .../_pydev_imps/_pydev_saved_modules.py | 3 + .../_pydevd_bundle/pydevd_code_to_source.py | 4 +- .../pydevd_collect_bytecode_info.py | 12 +- .../pydevd/_pydevd_bundle/pydevd_constants.py | 3 + .../_pydevd_bundle/pydevd_daemon_thread.py | 109 +++++++++++++++++- .../pydevd/_pydevd_bundle/pydevd_utils.py | 8 +- .../pydevd/tests_python/test_debugger.py | 27 +++++ .../pydevd/tests_python/test_utilities.py | 39 ++++++- 8 files changed, 193 insertions(+), 12 deletions(-) diff --git a/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_saved_modules.py b/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_saved_modules.py index 9b01a6513..3866e752e 100644 --- a/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_saved_modules.py +++ b/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_saved_modules.py @@ -123,3 +123,6 @@ def check(self, module, expected_attributes): with VerifyShadowedImport('http.server') as verify_shadowed: import http.server as BaseHTTPServer; verify_shadowed.check(BaseHTTPServer, ['BaseHTTPRequestHandler']) +# If set, this is a version of the threading.enumerate that doesn't have the patching to remove the pydevd threads. +# Note: as it can't be set during execution, don't import the name (import the module and access it through its name). +pydevd_saved_threading_enumerate = None diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_code_to_source.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_code_to_source.py index da1b53864..848af1885 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_code_to_source.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_code_to_source.py @@ -7,7 +7,7 @@ import dis -from _pydevd_bundle.pydevd_collect_bytecode_info import _iter_instructions +from _pydevd_bundle.pydevd_collect_bytecode_info import iter_instructions from _pydevd_bundle.pydevd_constants import dict_iter_items, IS_PY2 from _pydev_bundle import pydev_log import sys @@ -536,7 +536,7 @@ def __init__(self, co, memo=None): memo = {} self.memo = memo self.co = co - self.instructions = list(_iter_instructions(co)) + self.instructions = list(iter_instructions(co)) self.stack = _Stack() self.writer = _Writer() diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_collect_bytecode_info.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_collect_bytecode_info.py index aa577a63a..39da7150c 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_collect_bytecode_info.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_collect_bytecode_info.py @@ -141,7 +141,7 @@ def _iter_as_bytecode_as_instructions_py2(co): yield _Instruction(curr_op_name, op, _get_line(op_offset_to_line, initial_bytecode_offset, 0), oparg, is_jump_target, initial_bytecode_offset, str(oparg)) -def _iter_instructions(co): +def iter_instructions(co): if sys.version_info[0] < 3: iter_in = _iter_as_bytecode_as_instructions_py2(co) else: @@ -168,7 +168,7 @@ def collect_return_info(co, use_func_first_line=False): lst = [] op_offset_to_line = dict(dis.findlinestarts(co)) - for instruction in _iter_instructions(co): + for instruction in iter_instructions(co): curr_op_name = instruction.opname if curr_op_name == 'RETURN_VALUE': lst.append(ReturnInfo(_get_line(op_offset_to_line, instruction.offset, firstlineno, search=True))) @@ -192,7 +192,7 @@ def collect_try_except_info(co, use_func_first_line=False): op_offset_to_line = dict(dis.findlinestarts(co)) - for instruction in _iter_instructions(co): + for instruction in iter_instructions(co): curr_op_name = instruction.opname if curr_op_name in ('SETUP_EXCEPT', 'SETUP_FINALLY'): @@ -351,7 +351,7 @@ def collect_try_except_info(co, use_func_first_line=False): offset_to_instruction_idx = {} - instructions = list(_iter_instructions(co)) + instructions = list(iter_instructions(co)) for i, instruction in enumerate(instructions): offset_to_instruction_idx[instruction.offset] = i @@ -471,7 +471,7 @@ def collect_try_except_info(co, use_func_first_line=False): offset_to_instruction_idx = {} - instructions = list(_iter_instructions(co)) + instructions = list(iter_instructions(co)) for i, instruction in enumerate(instructions): offset_to_instruction_idx[instruction.offset] = i @@ -656,7 +656,7 @@ def __init__(self, co, firstlineno, level=0): self.co = co self.firstlineno = firstlineno self.level = level - self.instructions = list(_iter_instructions(co)) + self.instructions = list(iter_instructions(co)) op_offset_to_line = self.op_offset_to_line = dict(dis.findlinestarts(co)) # Update offsets so that all offsets have the line index (and update it based on diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py index c9a32c5b1..977f729b9 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py @@ -317,6 +317,9 @@ def as_float_in_env(env_key, default): # on how the thread interruption works (there are some caveats related to it). PYDEVD_INTERRUPT_THREAD_TIMEOUT = as_float_in_env('PYDEVD_INTERRUPT_THREAD_TIMEOUT', -1) +# If PYDEVD_APPLY_PATCHING_TO_HIDE_PYDEVD_THREADS is set to False, the patching to hide pydevd threads won't be applied. +PYDEVD_APPLY_PATCHING_TO_HIDE_PYDEVD_THREADS = os.getenv('PYDEVD_APPLY_PATCHING_TO_HIDE_PYDEVD_THREADS', 'true').lower() in ENV_TRUE_LOWER_VALUES + EXCEPTION_TYPE_UNHANDLED = 'UNHANDLED' EXCEPTION_TYPE_USER_UNHANDLED = 'USER_UNHANDLED' EXCEPTION_TYPE_HANDLED = 'HANDLED' diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py index 1eb40311d..6599836aa 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py @@ -1,11 +1,14 @@ from _pydev_imps._pydev_saved_modules import threading +from _pydev_imps import _pydev_saved_modules from _pydevd_bundle.pydevd_utils import notify_about_gevent_if_needed import weakref -from _pydevd_bundle.pydevd_constants import IS_JYTHON +from _pydevd_bundle.pydevd_constants import IS_JYTHON, IS_IRONPYTHON, \ + PYDEVD_APPLY_PATCHING_TO_HIDE_PYDEVD_THREADS from _pydev_bundle.pydev_log import exception as pydev_log_exception import sys from _pydev_bundle import pydev_log import pydevd_tracing +from _pydevd_bundle.pydevd_collect_bytecode_info import iter_instructions if IS_JYTHON: import org.python.core as JyCore # @UnresolvedImport @@ -67,7 +70,111 @@ def _stop_trace(self): pydevd_tracing.SetTrace(None) # no debugging on this thread +def _collect_load_names(func): + found_load_names = set() + for instruction in iter_instructions(func.__code__): + if instruction.opname in ('LOAD_GLOBAL', 'LOAD_ATTR', 'LOAD_METHOD'): + found_load_names.add(instruction.argrepr) + return found_load_names + + +def _patch_threading_to_hide_pydevd_threads(): + ''' + Patches the needed functions on the `threading` module so that the pydevd threads are hidden. + + Note that we patch the functions __code__ to avoid issues if some code had already imported those + variables prior to the patching. + ''' + found_load_names = _collect_load_names(threading.enumerate) + # i.e.: we'll only apply the patching if the function seems to be what we expect. + + new_threading_enumerate = None + + if found_load_names == set(('_active_limbo_lock', '_limbo', '_active', 'values', 'list')): + pydev_log.debug('Applying patching to hide pydevd threads (Py3 version).') + + def new_threading_enumerate(): + with _active_limbo_lock: + ret = list(_active.values()) + list(_limbo.values()) + + return [t for t in ret if not getattr(t, 'is_pydev_daemon_thread', False)] + + elif found_load_names == set(('_active_limbo_lock', '_limbo', '_active', 'values')): + pydev_log.debug('Applying patching to hide pydevd threads (Py2 version).') + + def new_threading_enumerate(): + with _active_limbo_lock: + ret = _active.values() + _limbo.values() + + return [t for t in ret if not getattr(t, 'is_pydev_daemon_thread', False)] + + else: + pydev_log.info('Unable to hide pydevd threads. Found names in threading.enumerate: %s', found_load_names) + + if new_threading_enumerate is not None: + + def pydevd_saved_threading_enumerate(): + with threading._active_limbo_lock: + return list(threading._active.values()) + list(threading._limbo.values()) + + _pydev_saved_modules.pydevd_saved_threading_enumerate = pydevd_saved_threading_enumerate + + threading.enumerate.__code__ = new_threading_enumerate.__code__ + + # We also need to patch the active count (to match what we have in the enumerate). + def new_active_count(): + # Note: as this will be executed in the `threading` module, `enumerate` will + # actually be threading.enumerate. + return len(enumerate()) + + threading.active_count.__code__ = new_active_count.__code__ + + # When shutting down, Python (on some versions) may do something as: + # + # def _pickSomeNonDaemonThread(): + # for t in enumerate(): + # if not t.daemon and t.is_alive(): + # return t + # return None + # + # But in this particular case, we do want threads with `is_pydev_daemon_thread` to appear + # explicitly due to the pydevd `CheckAliveThread` (because we want the shutdown to wait on it). + # So, it can't rely on the `enumerate` for that anymore as it's patched to not return pydevd threads. + if hasattr(threading, '_pickSomeNonDaemonThread'): + + def new_pick_some_non_daemon_thread(): + with _active_limbo_lock: + # Ok for py2 and py3. + threads = list(_active.values()) + list(_limbo.values()) + + for t in threads: + if not t.daemon and t.is_alive(): + return t + return None + + threading._pickSomeNonDaemonThread.__code__ = new_pick_some_non_daemon_thread.__code__ + + +_patched_threading_to_hide_pydevd_threads = False + + def mark_as_pydevd_daemon_thread(thread): + if not IS_JYTHON and not IS_IRONPYTHON and PYDEVD_APPLY_PATCHING_TO_HIDE_PYDEVD_THREADS: + global _patched_threading_to_hide_pydevd_threads + if not _patched_threading_to_hide_pydevd_threads: + # When we mark the first thread as a pydevd daemon thread, we also change the threading + # functions to hide pydevd threads. + # Note: we don't just "hide" the pydevd threads from the threading module by not using it + # (i.e.: just using the `thread.start_new_thread` instead of `threading.Thread`) + # because there's 1 thread (the `CheckAliveThread`) which is a pydevd thread but + # isn't really a daemon thread (so, we need CPython to wait on it for shutdown, + # in which case it needs to be in `threading` and the patching would be needed anyways). + _patched_threading_to_hide_pydevd_threads = True + try: + _patch_threading_to_hide_pydevd_threads() + except: + pydev_log.exception('Error applying patching to hide pydevd threads.') + thread.pydev_do_not_trace = True thread.is_pydev_daemon_thread = True thread.daemon = True diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py index 5b3d539db..6e4af794f 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py @@ -3,6 +3,7 @@ import warnings from _pydev_bundle import pydev_log from _pydev_imps._pydev_saved_modules import thread +from _pydev_imps import _pydev_saved_modules import signal import os import ctypes @@ -177,7 +178,11 @@ def dump_threads(stream=None, show_pydevd_threads=True): stream = sys.stderr thread_id_to_name_and_is_pydevd_thread = {} try: - for t in threading.enumerate(): + threading_enumerate = _pydev_saved_modules.pydevd_saved_threading_enumerate + if threading_enumerate is None: + threading_enumerate = threading.enumerate + + for t in threading_enumerate(): is_pydevd_thread = getattr(t, 'is_pydev_daemon_thread', False) thread_id_to_name_and_is_pydevd_thread[t.ident] = ( '%s (daemon: %s, pydevd thread: %s)' % (t.name, t.daemon, is_pydevd_thread), @@ -297,6 +302,7 @@ def hasattr_checked(obj, name): else: return True + def getattr_checked(obj, name): try: return getattr(obj, name) diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py b/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py index cba32237b..8190d3560 100644 --- a/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py +++ b/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py @@ -4372,6 +4372,33 @@ def get_environ(writer): assert ('the module "%s" could not be imported because it is shadowed by:' % (module_name.split('.')[0])) in writer.get_stderr() + +def test_debugger_hide_pydevd_threads(case_setup, pyfile): + + @pyfile + def target_file(): + import threading + from _pydevd_bundle import pydevd_constants + found_pydevd_thread = False + for t in threading.enumerate(): + if getattr(t, 'is_pydev_daemon_thread', False): + found_pydevd_thread = True + + if pydevd_constants.IS_CPYTHON: + assert not found_pydevd_thread + else: + assert found_pydevd_thread + print('TEST SUCEEDED') + + with case_setup.test_file(target_file) as writer: + line = writer.get_line_index_with_content('TEST SUCEEDED') + writer.write_add_breakpoint(line) + writer.write_make_initial_run() + + hit = writer.wait_for_breakpoint_hit(line=line) + writer.write_run_thread(hit.thread_id) + writer.finished_ok = True + # Jython needs some vars to be set locally. # set JAVA_HOME=c:\bin\jdk1.8.0_172 # set PATH=%PATH%;C:\bin\jython2.7.0\bin diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_utilities.py b/src/debugpy/_vendored/pydevd/tests_python/test_utilities.py index ba21361d2..4f3e65f99 100644 --- a/src/debugpy/_vendored/pydevd/tests_python/test_utilities.py +++ b/src/debugpy/_vendored/pydevd/tests_python/test_utilities.py @@ -3,7 +3,7 @@ from _pydevd_bundle.pydevd_utils import convert_dap_log_message_to_expression from tests_python.debug_constants import IS_PY26, IS_PY3K, TEST_GEVENT, IS_CPYTHON import sys -from _pydevd_bundle.pydevd_constants import IS_WINDOWS, IS_PY2, IS_PYPY +from _pydevd_bundle.pydevd_constants import IS_WINDOWS, IS_PY2, IS_PYPY, IS_JYTHON import pytest import os import codecs @@ -424,7 +424,7 @@ def test_find_main_thread_id(): ) -@pytest.mark.skipif(not IS_WINDOWS, reason='Windows-only test.') +@pytest.mark.skipif(not IS_WINDOWS or IS_JYTHON, reason='Windows-only test.') def test_get_ppid(): from _pydevd_bundle.pydevd_api import PyDevdAPI api = PyDevdAPI() @@ -522,3 +522,38 @@ def __init__(self, offset): assert get_smart_step_into_variant_from_frame_offset(2, variants) is variants[0] assert get_smart_step_into_variant_from_frame_offset(3, variants) is variants[1] assert get_smart_step_into_variant_from_frame_offset(4, variants) is variants[1] + + +def test_threading_hide_pydevd(): + + class T(threading.Thread): + + def __init__(self, is_pydev_daemon_thread): + from _pydevd_bundle.pydevd_daemon_thread import mark_as_pydevd_daemon_thread + threading.Thread.__init__(self) + if is_pydev_daemon_thread: + mark_as_pydevd_daemon_thread(self) + else: + self.daemon = True + self.event = threading.Event() + + def run(self): + self.event.wait(10) + + current_count = threading.active_count() + t0 = T(True) + t1 = T(False) + t0.start() + t1.start() + + # i.e.: the patching doesn't work for other implementations. + if IS_CPYTHON: + assert threading.active_count() == current_count + 1 + assert t0 not in threading.enumerate() + else: + assert threading.active_count() == current_count + 2 + assert t0 in threading.enumerate() + + assert t1 in threading.enumerate() + t0.event.set() + t1.event.set()