Support newer versions of gevent / show paused greenlets (based on GE…

…VENT_SHOW_PAUSED_GREENLETS env var). Fixes microsoft#515

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py

@@ -264,6 +264,11 @@ def as_int_in_env(env_key, default):
 # If true in env, use gevent mode.
 SUPPORT_GEVENT = is_true_in_env('GEVENT_SUPPORT')
 
+# Opt-in support to show gevent paused greenlets. False by default because if too many greenlets are
+# paused the UI can slow-down (i.e.: if 1000 greenlets are paused, each one would be shown separate
+# as a different thread, but if the UI isn't optimized for that the experience is lacking...).
+GEVENT_SHOW_PAUSED_GREENLETS = is_true_in_env('GEVENT_SHOW_PAUSED_GREENLETS')
+
 GEVENT_SUPPORT_NOT_SET_MSG = os.getenv(
  'GEVENT_SUPPORT_NOT_SET_MSG',
  'It seems that the gevent monkey-patching is being used.\n'

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_dont_trace_files.py

@@ -113,6 +113,7 @@
  'pydevd_frame_eval_main.py': PYDEV_FILE,
  'pydevd_frame_tracing.py': PYDEV_FILE,
  'pydevd_frame_utils.py': PYDEV_FILE,
+ 'pydevd_gevent_integration.py': PYDEV_FILE,
  'pydevd_helpers.py': PYDEV_FILE,
  'pydevd_import_class.py': PYDEV_FILE,
  'pydevd_io.py': PYDEV_FILE,

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_gevent_integration.py

@@ -0,0 +1,93 @@
+import pydevd_tracing
+import greenlet
+import gevent
+from _pydev_imps._pydev_saved_modules import threading
+from _pydevd_bundle.pydevd_custom_frames import add_custom_frame, update_custom_frame, remove_custom_frame
+from _pydevd_bundle.pydevd_constants import GEVENT_SHOW_PAUSED_GREENLETS, get_global_debugger, \
+ thread_get_ident
+from _pydev_bundle import pydev_log
+from pydevd_file_utils import basename
+
+_saved_greenlets_to_custom_frame_thread_id = {}
+
+if GEVENT_SHOW_PAUSED_GREENLETS:
+
+ def _get_paused_name(py_db, g):
+ frame = g.gr_frame
+ use_frame = frame
+
+ # i.e.: Show in the description of the greenlet the last user-code found.
+ while use_frame is not None:
+ if py_db.apply_files_filter(use_frame, use_frame.f_code.co_filename, True):
+ frame = use_frame
+ use_frame = use_frame.f_back
+ else:
+ break
+
+ if use_frame is None:
+ use_frame = frame
+
+ return '%s: %s - %s' % (type(g).__name__, use_frame.f_code.co_name, basename(use_frame.f_code.co_filename))
+
+ def greenlet_events(event, args):
+ if event in ('switch', 'throw'):
+ py_db = get_global_debugger()
+ origin, target = args
+
+ if not origin.dead and origin.gr_frame is not None:
+ frame_custom_thread_id = _saved_greenlets_to_custom_frame_thread_id.get(origin)
+ if frame_custom_thread_id is None:
+ _saved_greenlets_to_custom_frame_thread_id[origin] = add_custom_frame(
+ origin.gr_frame, _get_paused_name(py_db, origin), thread_get_ident())
+ else:
+ update_custom_frame(
+ frame_custom_thread_id, origin.gr_frame, _get_paused_name(py_db, origin), thread_get_ident())
+ else:
+ frame_custom_thread_id = _saved_greenlets_to_custom_frame_thread_id.pop(origin, None)
+ if frame_custom_thread_id is not None:
+ remove_custom_frame(frame_custom_thread_id)
+
+ # This one will be resumed, so, remove custom frame from it.
+ frame_custom_thread_id = _saved_greenlets_to_custom_frame_thread_id.pop(target, None)
+ if frame_custom_thread_id is not None:
+ remove_custom_frame(frame_custom_thread_id)
+
+ # The tracing needs to be reapplied for each greenlet as gevent
+ # clears the tracing set through sys.settrace for each greenlet.
+ pydevd_tracing.reapply_settrace()
+
+else:
+
+ # i.e.: no logic related to showing paused greenlets is needed.
+ def greenlet_events(event, args):
+ pydevd_tracing.reapply_settrace()
+
+
+def enable_gevent_integration():
+ # References:
+ # https://greenlet.readthedocs.io/en/latest/api.html#greenlet.settrace
+ # https://greenlet.readthedocs.io/en/latest/tracing.html
+
+ # Note: gevent.version_info is WRONG (gevent.__version__ must be used).
+ try:
+ if tuple(int(x) for x in gevent.__version__.split('.')[:2]) <= (20, 0):
+ if not GEVENT_SHOW_PAUSED_GREENLETS:
+ return
+
+ if not hasattr(greenlet, 'settrace'):
+ # In older versions it was optional.
+ # We still try to use if available though (because without it
+ pydev_log.debug('greenlet.settrace not available. GEVENT_SHOW_PAUSED_GREENLETS will have no effect.')
+ return
+ try:
+ greenlet.settrace(greenlet_events)
+ except:
+ pydev_log.exception('Error with greenlet.settrace.')
+ except:
+ pydev_log.exception('Error setting up gevent %s.', gevent.__version__)
+
+
+def log_gevent_debug_info():
+ pydev_log.debug('Greenlet version: %s', greenlet.__version__)
+ pydev_log.debug('Gevent version: %s', gevent.__version__)
+ pydev_log.debug('Gevent install location: %s', gevent.__file__)

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_net_command_factory_json.py

@@ -124,8 +124,18 @@ def make_thread_created_message(self, thread):
 
  return NetCommand(CMD_THREAD_CREATE, 0, msg, is_json=True)
 
+ @overrides(NetCommandFactory.make_custom_frame_created_message)
+ def make_custom_frame_created_message(self, frame_id, frame_description):
+ self._additional_thread_id_to_thread_name[frame_id] = frame_description
+ msg = pydevd_schema.ThreadEvent(
+ pydevd_schema.ThreadEventBody('started', frame_id),
+ )
+
+ return NetCommand(CMD_THREAD_CREATE, 0, msg, is_json=True)
+
  @overrides(NetCommandFactory.make_thread_killed_message)
  def make_thread_killed_message(self, tid):
+ self._additional_thread_id_to_thread_name.pop(tid, None)
  msg = pydevd_schema.ThreadEvent(
  pydevd_schema.ThreadEventBody('exited', tid),
  )
@@ -145,6 +155,10 @@ def make_list_threads_message(self, py_db, seq):
  thread_schema = pydevd_schema.Thread(id=thread_id, name=thread.name)
  threads.append(thread_schema.to_dict())
 
+ for thread_id, thread_name in list(self._additional_thread_id_to_thread_name.items()):
+ thread_schema = pydevd_schema.Thread(id=thread_id, name=thread_name)
+ threads.append(thread_schema.to_dict())
+
  body = pydevd_schema.ThreadsResponseBody(threads)
  response = pydevd_schema.ThreadsResponse(
  request_seq=seq, success=True, command='threads', body=body)

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_net_command_factory_xml.py

@@ -45,11 +45,14 @@ def unquote(s):
 #=======================================================================================================================
 class NetCommandFactory(object):
 
+ def __init__(self):
+ self._additional_thread_id_to_thread_name = {}
+
  def _thread_to_xml(self, thread):
  """ thread information as XML """
  name = pydevd_xml.make_valid_xml_value(thread.name)
- cmdText = '<thread name="%s" id="%s" />' % (quote(name), get_thread_id(thread))
- return cmdText
+ cmd_text = '<thread name="%s" id="%s" />' % (quote(name), get_thread_id(thread))
+ return cmd_text
 
  def make_error_message(self, seq, text):
  cmd = NetCommand(CMD_ERROR, seq, text)
@@ -75,6 +78,7 @@ def make_show_cython_warning_message(self):
  return self.make_error_message(0, get_exception_traceback_str())
 
  def make_custom_frame_created_message(self, frame_id, frame_description):
+ self._additional_thread_id_to_thread_name[frame_id] = frame_description
  frame_description = pydevd_xml.make_valid_xml_value(frame_description)
  return NetCommand(CMD_THREAD_CREATE, 0, '<xml><thread name="%s" id="%s"/></xml>' % (frame_description, frame_id))
 
@@ -87,6 +91,11 @@ def make_list_threads_message(self, py_db, seq):
  for thread in threads:
  if is_thread_alive(thread):
  append(self._thread_to_xml(thread))
+
+ for thread_id, thread_name in list(self._additional_thread_id_to_thread_name.items()):
+ name = pydevd_xml.make_valid_xml_value(thread_name)
+ append('<thread name="%s" id="%s" />' % (quote(name), thread_id))
+
  append("</xml>")
  return NetCommand(CMD_RETURN, seq, ''.join(cmd_text))
  except:
@@ -153,6 +162,7 @@ def make_version_message(self, seq):
  return self.make_error_message(seq, get_exception_traceback_str())
 
  def make_thread_killed_message(self, tid):
+ self._additional_thread_id_to_thread_name.pop(tid, None)
  try:
  return NetCommand(CMD_THREAD_KILL, 0, str(tid))
  except:

src/debugpy/_vendored/pydevd/pydevd.py

@@ -55,7 +55,7 @@
  dict_keys, dict_iter_items, DebugInfoHolder, PYTHON_SUSPEND, STATE_SUSPEND, STATE_RUN, get_frame,
  clear_cached_thread_id, INTERACTIVE_MODE_AVAILABLE, SHOW_DEBUG_INFO_ENV, IS_PY34_OR_GREATER, IS_PY2, NULL,
  NO_FTRACE, IS_IRONPYTHON, JSON_PROTOCOL, IS_CPYTHON, HTTP_JSON_PROTOCOL, USE_CUSTOM_SYS_CURRENT_FRAMES_MAP, call_only_once,
- ForkSafeLock, IGNORE_BASENAMES_STARTING_WITH, EXCEPTION_TYPE_UNHANDLED)
+ ForkSafeLock, IGNORE_BASENAMES_STARTING_WITH, EXCEPTION_TYPE_UNHANDLED, SUPPORT_GEVENT)
 from _pydevd_bundle.pydevd_defaults import PydevdCustomization # Note: import alias used on pydev_monkey.
 from _pydevd_bundle.pydevd_custom_frames import CustomFramesContainer, custom_frames_container_init
 from _pydevd_bundle.pydevd_dont_trace_files import DONT_TRACE, PYDEV_FILE, LIB_FILE, DONT_TRACE_DIRS
@@ -95,6 +95,18 @@
 from _pydevd_bundle.pydevd_timeout import TimeoutTracker
 from _pydevd_bundle.pydevd_thread_lifecycle import suspend_all_threads, mark_thread_suspended
 
+pydevd_gevent_integration = None
+
+if SUPPORT_GEVENT:
+ try:
+ from _pydevd_bundle import pydevd_gevent_integration
+ except:
+ pydev_log.exception(
+ 'pydevd: GEVENT_SUPPORT is set but gevent is not available in the environment.\n'
+ 'Please unset GEVENT_SUPPORT from the environment variables or install gevent.')
+ else:
+ pydevd_gevent_integration.log_gevent_debug_info()
+
 if USE_CUSTOM_SYS_CURRENT_FRAMES_MAP:
  from _pydevd_bundle.pydevd_constants import constructed_tid_to_last_frame
 
@@ -169,6 +181,7 @@ def pydevd_breakpointhook(*args, **kwargs):
 _CACHE_FILE_TYPE = {}
 
 pydev_log.debug('Using GEVENT_SUPPORT: %s', pydevd_constants.SUPPORT_GEVENT)
+pydev_log.debug('Using GEVENT_SHOW_PAUSED_GREENLETS: %s', pydevd_constants.GEVENT_SHOW_PAUSED_GREENLETS)
 pydev_log.debug('pydevd __file__: %s', os.path.abspath(__file__))
 
 
@@ -1046,6 +1059,9 @@ def enable_tracing(self, thread_trace_func=None, apply_to_all_threads=False):
  this function is called on a multi-threaded program (either programmatically or attach
  to pid).
  '''
+ if pydevd_gevent_integration is not None:
+ pydevd_gevent_integration.enable_gevent_integration()
+
  if self.frame_eval_func is not None:
  self.frame_eval_func()
  pydevd_tracing.SetTrace(self.dummy_trace_dispatch)

src/debugpy/_vendored/pydevd/pydevd_tracing.py

@@ -78,7 +78,12 @@ def _internal_set_trace(tracing_func):
  TracingFunctionHolder._original_tracing(tracing_func)
 
 
+_last_tracing_func_thread_local = threading.local()
+
+
 def SetTrace(tracing_func):
+ _last_tracing_func_thread_local.tracing_func = tracing_func
+
  if tracing_func is not None:
  if set_trace_to_threads(tracing_func, thread_idents=[thread.get_ident()], create_dummy_thread=False) == 0:
  # If we can use our own tracer instead of the one from sys.settrace, do it (the reason
@@ -94,6 +99,15 @@ def SetTrace(tracing_func):
  set_trace(tracing_func)
 
 
+def reapply_settrace():
+ try:
+ tracing_func = _last_tracing_func_thread_local.tracing_func
+ except AttributeError:
+ return
+ else:
+ SetTrace(tracing_func)
+
+
 def replace_sys_set_trace_func():
  if TracingFunctionHolder._original_tracing is None:
  TracingFunctionHolder._original_tracing = sys.settrace

src/debugpy/_vendored/pydevd/tests_python/resources/_debugger_case_gevent_simple.py

@@ -0,0 +1,21 @@
+import gevent
+
+
+def foo():
+ print('Running in foo')
+ gevent.sleep(0)
+ print('Explicit context switch to foo again')
+
+
+def bar():
+ print('Explicit context to bar')
+ gevent.sleep(0) # break here
+ print('Implicit context switch back to bar')
+
+
+if __name__ == '__main__':
+ gevent.joinall([
+ gevent.spawn(foo),
+ gevent.spawn(bar),
+ ])
+ print('TEST SUCEEDED')

src/debugpy/_vendored/pydevd/tests_python/test_debugger.py

@@ -3208,6 +3208,36 @@ def get_environ(writer):
  writer.finished_ok = True
 
 
+@pytest.mark.skipif(not TEST_GEVENT, reason='Gevent not installed.')
+@pytest.mark.parametrize('show', [True, False])
+def test_gevent_show_paused_greenlets(case_setup, show):
+
+ def get_environ(writer):
+ env = os.environ.copy()
+ env['GEVENT_SUPPORT'] = 'True'
+ if show:
+ env['GEVENT_SHOW_PAUSED_GREENLETS'] = 'True'
+ else:
+ env['GEVENT_SHOW_PAUSED_GREENLETS'] = 'False'
+ return env
+
+ with case_setup.test_file('_debugger_case_gevent_simple.py', get_environ=get_environ) as writer:
+ writer.write_add_breakpoint(writer.get_line_index_with_content('break here'))
+ writer.write_make_initial_run()
+ hit = writer.wait_for_breakpoint_hit(name='bar')
+ writer.write_run_thread(hit.thread_id)
+
+ seq = writer.write_list_threads()
+ msg = writer.wait_for_list_threads(seq)
+
+ if show:
+ assert len(msg) > 1
+ else:
+ assert len(msg) == 1
+
+ writer.finished_ok = True
+
+
 @pytest.mark.skipif(not TEST_GEVENT, reason='Gevent not installed.')
 def test_gevent_remote(case_setup_remote):