Support for Python 3.11: cython. WIP #939

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_collect_bytecode_info.py

@@ -669,7 +669,7 @@ def _lookahead(self):
  )
  return RESTART_FROM_LOOKAHEAD
 
- if next_instruction.opname == 'CALL_FUNCTION':
+ if next_instruction.opname in ('CALL_FUNCTION', 'PRECALL'):
  if len(found) == next_instruction.argval + 1:
  force_restart = False
  delta = 0
@@ -680,7 +680,10 @@ def _lookahead(self):
  else:
  return None # This is odd
 
- del self.instructions[delta:delta + next_instruction.argval + 2] # +2 = NAME / CALL_FUNCTION
+ del_upto = delta + next_instruction.argval + 2 # +2 = NAME / CALL_FUNCTION
+ if next_instruction.opname == 'PRECALL':
+ del_upto += 1 # Also remove the CALL right after the PRECALL.
+ del self.instructions[delta:del_upto]
 
  found = iter(found[delta:])
  call_func = next(found)
@@ -779,8 +782,12 @@ def _create_msg_part(self, instruction, tok=None, line=None):
  dec = self._decorate_jump_target
  if line is None or line in (self.BIG_LINE_INT, self.SMALL_LINE_INT):
  line = self.op_offset_to_line[instruction.offset]
+
+ argrepr = instruction.argrepr
+ if isinstance(argrepr, str) and argrepr.startswith('NULL + '):
+ argrepr = argrepr[7:]
  return _MsgPart(
- line, tok if tok is not None else dec(instruction, instruction.argrepr))
+ line, tok if tok is not None else dec(instruction, argrepr))
 
  def _next_instruction_to_str(self, line_to_contents):
  # indent = ''
@@ -797,6 +804,9 @@ def _next_instruction_to_str(self, line_to_contents):
 
  instruction = self.instructions.pop(0)
 
+ if instruction.opname in 'RESUME':
+ return None
+
  if instruction.opname in ('LOAD_GLOBAL', 'LOAD_FAST', 'LOAD_CONST', 'LOAD_NAME'):
  next_instruction = self.instructions[0]
  if next_instruction.opname in ('STORE_FAST', 'STORE_NAME'):
@@ -855,6 +865,8 @@ def build_line_to_contents(self):
  s = self._next_instruction_to_str(line_to_contents)
  if s is RESTART_FROM_LOOKAHEAD:
  continue
+ if s is None:
+ continue
 
  _MsgPart.add_to_line_to_contents(s, line_to_contents)
  m = self.max_line(s)

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_cython.pyx

@@ -704,7 +704,11 @@ cdef class PyDBFrame:
  # if DEBUG: print('frame trace_dispatch %s %s %s %s %s %s, stop: %s' % (frame.f_lineno, frame.f_code.co_name, frame.f_code.co_filename, event, constant_to_str(info.pydev_step_cmd), arg, info.pydev_step_stop))
  try:
  info.is_tracing += 1
- line = frame.f_lineno
+
+ # TODO: This shouldn't be needed. The fact that frame.f_lineno
+ # is None seems like a bug in Python 3.11.
+ # Reported in: https://github.com/python/cpython/issues/94485
+ line = frame.f_lineno or 0 # Workaround or case where frame.f_lineno is None
  line_cache_key = (frame_cache_key, line)
 
  if main_debugger.pydb_disposed:

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_frame.py

@@ -571,7 +571,11 @@ def trace_dispatch(self, frame, event, arg):
  # if DEBUG: print('frame trace_dispatch %s %s %s %s %s %s, stop: %s' % (frame.f_lineno, frame.f_code.co_name, frame.f_code.co_filename, event, constant_to_str(info.pydev_step_cmd), arg, info.pydev_step_stop))
  try:
  info.is_tracing += 1
- line = frame.f_lineno
+
+ # TODO: This shouldn't be needed. The fact that frame.f_lineno
+ # is None seems like a bug in Python 3.11.
+ # Reported in: https://github.com/python/cpython/issues/94485
+ line = frame.f_lineno or 0 # Workaround or case where frame.f_lineno is None
  line_cache_key = (frame_cache_key, line)
 
  if main_debugger.pydb_disposed:

src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_evaluator.pxd

@@ -52,6 +52,12 @@ cdef extern from "code.h":
  ctypedef void freefunc(void *)
  int _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra)
  int _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
+
+# TODO: Things are in a different place for Python 3.11.
+# cdef extern from "cpython/code.h":
+# ctypedef void freefunc(void *)
+# int _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra)
+# int _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
 
 cdef extern from "Python.h":
  void Py_INCREF(object o)

src/debugpy/_vendored/pydevd/setup_pydevd_cython.py

@@ -16,6 +16,7 @@
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
 IS_PY36_OR_GREATER = sys.version_info > (3, 6)
+TODO_PY311 = sys.version_info > (3, 11)
 
 
 def process_args():
@@ -234,7 +235,7 @@ def build_extension(dir_name, extension_name, target_pydevd_name, force_cython,
  target_pydevd_name = extension_name
 build_extension("_pydevd_bundle", extension_name, target_pydevd_name, force_cython, extension_folder, True)
 
-if IS_PY36_OR_GREATER:
+if IS_PY36_OR_GREATER and not TODO_PY311:
  extension_name = "pydevd_frame_evaluator"
  if target_frame_eval is None:
  target_frame_eval = extension_name

src/debugpy/_vendored/pydevd/tests_python/debug_constants.py

@@ -6,8 +6,11 @@
 PYDEVD_TEST_VM = os.getenv('PYDEVD_TEST_VM', None)
 
 IS_PY36_OR_GREATER = sys.version_info[0:2] >= (3, 6)
+IS_PY311_OR_GREATER = sys.version_info[0:2] >= (3, 11)
 IS_CPYTHON = platform.python_implementation() == 'CPython'
 
+TODO_PY311 = IS_PY311_OR_GREATER # Code which needs to be fixed in 3.11 should use this constant.
+
 IS_PY36 = False
 if sys.version_info[0] == 3 and sys.version_info[1] == 6:
  IS_PY36 = True

src/debugpy/_vendored/pydevd/tests_python/test_bytecode_manipulation.py

@@ -5,9 +5,13 @@
 
 import pytest
 
-from tests_python.debug_constants import IS_PY36_OR_GREATER, IS_CPYTHON, TEST_CYTHON
+from tests_python.debug_constants import IS_PY36_OR_GREATER, IS_CPYTHON, TEST_CYTHON, TODO_PY311
 
-pytestmark = pytest.mark.skipif(not IS_PY36_OR_GREATER or not IS_CPYTHON or not TEST_CYTHON, reason='Requires CPython >= 3.6')
+pytestmark = pytest.mark.skipif(
+ not IS_PY36_OR_GREATER or
+ TODO_PY311 or
+ not IS_CPYTHON or
+ not TEST_CYTHON, reason='Requires CPython >= 3.6 < 3.11')
 
 
 class _Tracer(object):

src/debugpy/_vendored/pydevd/tests_python/test_collect_bytecode_info.py

@@ -8,6 +8,7 @@
  collect_return_info, code_to_bytecode_representation
 from tests_python.debugger_unittest import IS_CPYTHON, IS_PYPY
 from _pydevd_bundle.pydevd_constants import IS_PY38_OR_GREATER, IS_JYTHON
+from tests_python.debug_constants import IS_PY311_OR_GREATER
 
 
 def _method_call_with_error():
@@ -192,7 +193,7 @@ def test_collect_try_except_info(data_regression, pyfile):
  info = collect_try_except_info(method.__code__, use_func_first_line=True)
  method_to_info[key] = sorted(str(x) for x in info)
 
- if sys.version_info[:2] != (3, 10):
+ if sys.version_info[:2] not in ((3, 10), (3, 11)):
  data_regression.check(method_to_info)
 
  data_regression.check(method_to_info_from_source)
@@ -385,6 +386,8 @@ def try_except_with():
 
 
 def test_collect_try_except_info_async_for():
+ if IS_PY311_OR_GREATER:
+ pytest.skip('On Python 3.11 we just support collecting info from the AST.')
 
  # Not valid on Python 2.
  code_str = '''
@@ -482,7 +485,8 @@ def method4():
  3,
  call('tnh %s' % 1))
 
- assert code_to_bytecode_representation(method4.__code__, use_func_first_line=True).count('\n') == 4
+ contents = code_to_bytecode_representation(method4.__code__, use_func_first_line=True)
+ assert contents.count('\n') == 4, 'Found:%s' % (contents,)
 
 
 @pytest.mark.skipif(IS_JYTHON, reason='Jython does not have bytecode support.')
@@ -537,6 +541,7 @@ def method4():
  new_repr = code_to_bytecode_representation(method4.__code__, use_func_first_line=True)
  assert 'call()' in new_repr
  assert 'call(1, 2, 3, a, b, \'x\')' in new_repr
+ assert 'NULL' not in new_repr
 
 
 @pytest.mark.skipif(IS_JYTHON, reason='Jython does not have bytecode support.')

src/debugpy/_vendored/pydevd/tests_python/test_debugger.py

@@ -1689,6 +1689,8 @@ def test_case_get_next_statement_targets(case_setup):
  # it's also not that bad (that line has no code in the source and
  # executing it will just set the tracing for the method).
  targets.discard(20)
+ # On Python 3.11 there's now a line 1 (which should be harmless).
+ targets.discard(1)
  expected = set((2, 3, 5, 8, 9, 10, 12, 13, 14, 15, 17, 18, 19, 21))
  assert targets == expected, 'Expected targets to be %s, was: %s' % (expected, targets)
 
@@ -3457,7 +3459,7 @@ def test_frame_eval_limitations(case_setup, filename, break_at_lines):
  hit = writer.wait_for_breakpoint_hit()
  thread_id = hit.thread_id
 
- if IS_PY36_OR_GREATER and TEST_CYTHON:
+ if (IS_PY36_OR_GREATER and TEST_CYTHON) and not TODO_PY311:
  assert hit.suspend_type == break_mode
  else:
  # Before 3.6 frame eval is not available.
@@ -3496,6 +3498,7 @@ def test_step_return_my_code(case_setup):
  writer.finished_ok = True
 
 
+@pytest.mark.skipif(TODO_PY311, reason='Needs bytecode support in Python 3.11')
 def test_smart_step_into_case1(case_setup):
  with case_setup.test_file('_debugger_case_smart_step_into.py') as writer:
  line = writer.get_line_index_with_content('break here')
@@ -3518,6 +3521,7 @@ def test_smart_step_into_case1(case_setup):
  writer.finished_ok = True
 
 
+@pytest.mark.skipif(TODO_PY311, reason='Needs bytecode support in Python 3.11')
 def test_smart_step_into_case2(case_setup):
  with case_setup.test_file('_debugger_case_smart_step_into2.py') as writer:
  line = writer.get_line_index_with_content('break here')
@@ -3546,6 +3550,7 @@ def test_smart_step_into_case2(case_setup):
  writer.finished_ok = True
 
 
+@pytest.mark.skipif(TODO_PY311, reason='Needs bytecode support in Python 3.11')
 def test_smart_step_into_case3(case_setup):
  with case_setup.test_file('_debugger_case_smart_step_into3.py') as writer:
  line = writer.get_line_index_with_content('break here')

src/debugpy/_vendored/pydevd/tests_python/test_debugger_json.py

@@ -22,7 +22,7 @@
  IS_PYPY, GENERATED_LEN_ATTR_NAME, IS_WINDOWS, IS_LINUX, IS_MAC, IS_PY38_OR_GREATER)
 from tests_python import debugger_unittest
 from tests_python.debug_constants import TEST_CHERRYPY, TEST_DJANGO, TEST_FLASK, \
- IS_CPYTHON, TEST_GEVENT, TEST_CYTHON
+ IS_CPYTHON, TEST_GEVENT, TEST_CYTHON, TODO_PY311
 from tests_python.debugger_unittest import (IS_JYTHON, IS_APPVEYOR, overrides,
  get_free_port, wait_for_condition)
 from _pydevd_bundle.pydevd_utils import DAPGrouper
@@ -4977,7 +4977,7 @@ def test_pydevd_systeminfo(case_setup):
  if use_cython is not None:
  using_cython = use_cython == 'YES'
  assert body['pydevd']['usingCython'] == using_cython
- assert body['pydevd']['usingFrameEval'] == (using_cython and IS_PY36_OR_GREATER)
+ assert body['pydevd']['usingFrameEval'] == (using_cython and IS_PY36_OR_GREATER and not TODO_PY311)
 
  json_facade.write_continue()
 
@@ -5709,7 +5709,13 @@ def get_environ(self):
  writer.finished_ok = True
 
 
-@pytest.mark.skipif(not IS_WINDOWS or not IS_PY36_OR_GREATER or not IS_CPYTHON or not TEST_CYTHON, reason='Windows only test and only Python 3.6 onwards.')
+@pytest.mark.skipif(
+ not IS_WINDOWS or
+ not IS_PY36_OR_GREATER or
+ not IS_CPYTHON or
+ not TEST_CYTHON or
+ TODO_PY311, # Requires frame-eval mode (still not available for Python 3.11).
+ reason='Windows only test and only Python 3.6 onwards.')
 def test_native_threads(case_setup, pyfile):
 
  @pyfile
@@ -5795,6 +5801,7 @@ def do_something():
  writer.finished_ok = True
 
 
+@pytest.mark.skipif(TODO_PY311, reason='Needs bytecode support in Python 3.11')
 def test_step_into_target_basic(case_setup):
  with case_setup.test_file('_debugger_case_smart_step_into.py') as writer:
  json_facade = JsonFacade(writer)
@@ -5819,6 +5826,7 @@ def test_step_into_target_basic(case_setup):
  writer.finished_ok = True
 
 
+@pytest.mark.skipif(TODO_PY311, reason='Needs bytecode support in Python 3.11')
 def test_step_into_target_multiple(case_setup):
  with case_setup.test_file('_debugger_case_smart_step_into2.py') as writer:
  json_facade = JsonFacade(writer)
@@ -5843,6 +5851,7 @@ def test_step_into_target_multiple(case_setup):
  writer.finished_ok = True
 
 
+@pytest.mark.skipif(TODO_PY311, reason='Needs bytecode support in Python 3.11')
 def test_step_into_target_genexpr(case_setup):
  with case_setup.test_file('_debugger_case_smart_step_into3.py') as writer:
  json_facade = JsonFacade(writer)

src/debugpy/_vendored/pydevd/tests_python/test_frame_eval_and_tracing.py

@@ -5,13 +5,13 @@
 
 from contextlib import contextmanager
 from tests_python.debugger_unittest import IS_PY36_OR_GREATER, IS_CPYTHON
-from tests_python.debug_constants import TEST_CYTHON
+from tests_python.debug_constants import TEST_CYTHON, TODO_PY311
 
 pytest_plugins = [
  str('tests_python.debugger_fixtures'),
 ]
 
-pytestmark = pytest.mark.skipif(not IS_PY36_OR_GREATER or not IS_CPYTHON or not TEST_CYTHON, reason='Requires CPython >= 3.6')
+pytestmark = pytest.mark.skipif(not IS_PY36_OR_GREATER or not IS_CPYTHON or not TEST_CYTHON or TODO_PY311, reason='Requires CPython >= 3.6')
 
 
 @pytest.fixture

src/debugpy/_vendored/pydevd/tests_python/test_frame_evaluator.py

@@ -2,9 +2,9 @@
 import threading
 import pytest
 from tests_python.debugger_unittest import IS_PY36_OR_GREATER, IS_CPYTHON
-from tests_python.debug_constants import TEST_CYTHON
+from tests_python.debug_constants import TEST_CYTHON, TODO_PY311
 
-pytestmark = pytest.mark.skipif(not IS_PY36_OR_GREATER or not IS_CPYTHON or not TEST_CYTHON, reason='Requires CPython >= 3.6')
+pytestmark = pytest.mark.skipif(not IS_PY36_OR_GREATER or not IS_CPYTHON or not TEST_CYTHON or TODO_PY311, reason='Requires CPython >= 3.6')
 
 
 def get_foo_frame():

src/debugpy/_vendored/pydevd/tests_python/test_smart_step_into_bytecode.py

@@ -1,11 +1,14 @@
 import sys
+from tests_python.debug_constants import TODO_PY311
 try:
  from _pydevd_bundle import pydevd_bytecode_utils
 except ImportError:
  pass
 import pytest
 
-pytestmark = pytest.mark.skipif(sys.version_info[0] < 3, reason='Only available for Python 3.')
+pytestmark = pytest.mark.skipif(
+ sys.version_info[0] < 3 or
+ TODO_PY311, reason='Only available for Python 3. / Requires bytecode support in Python 3.11')
 
 
 @pytest.fixture(autouse=True, scope='function')