Support step into target. Fixes microsoft#288

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_additional_thread_info_regular.py

@@ -28,7 +28,6 @@ class PyDBAdditionalThreadInfo(object):
         'pydev_original_step_cmd',
         'pydev_step_cmd',
         'pydev_notify_kill',
-        'pydev_smart_step_stop',
         'pydev_django_resolve_frame',
         'pydev_call_from_jinja2',
         'pydev_call_inside_jinja2',
@@ -44,6 +43,14 @@ class PyDBAdditionalThreadInfo(object):
         'top_level_thread_tracer_unhandled',
         'thread_tracer',
         'step_in_initial_location',
+
+        # Used for CMD_SMART_STEP_INTO (to know which smart step into variant to use)
+        'pydev_smart_parent_offset',
+
+        # Used for CMD_SMART_STEP_INTO (list[_pydevd_bundle.pydevd_bytecode_utils.Variant])
+        # Filled when the cmd_get_smart_step_into_variants is requested (so, this is a copy
+        # of the last request for a given thread and pydev_smart_parent_offset relies on it).
+        'pydev_smart_step_into_variants',
     ]
     # ENDIF
 
@@ -61,7 +68,6 @@ def __init__(self):
         self.pydev_step_cmd = -1  # Something as CMD_STEP_INTO, CMD_STEP_OVER, etc.
 
         self.pydev_notify_kill = False
-        self.pydev_smart_step_stop = None
         self.pydev_django_resolve_frame = False
         self.pydev_call_from_jinja2 = None
         self.pydev_call_inside_jinja2 = None
@@ -77,6 +83,8 @@ def __init__(self):
         self.top_level_thread_tracer_unhandled = None
         self.thread_tracer = None
         self.step_in_initial_location = None
+        self.pydev_smart_parent_offset = -1
+        self.pydev_smart_step_into_variants = ()
 
     def get_topmost_frame(self, thread):
         '''

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_api.py

@@ -11,9 +11,9 @@
     internal_get_description, internal_get_frame, internal_evaluate_expression, InternalConsoleExec,
     internal_get_variable_json, internal_change_variable, internal_change_variable_json,
     internal_evaluate_expression_json, internal_set_expression_json, internal_get_exception_details_json,
-    internal_step_in_thread)
+    internal_step_in_thread, internal_smart_step_into)
 from _pydevd_bundle.pydevd_comm_constants import (CMD_THREAD_SUSPEND, file_system_encoding,
-    CMD_STEP_INTO_MY_CODE, CMD_STOP_ON_START)
+    CMD_STEP_INTO_MY_CODE, CMD_STOP_ON_START, CMD_SMART_STEP_INTO)
 from _pydevd_bundle.pydevd_constants import (get_current_thread_id, set_protocol, get_protocol,
     HTTP_JSON_PROTOCOL, JSON_PROTOCOL, IS_PY3K, DebugInfoHolder, dict_keys, dict_items, IS_WINDOWS)
 from _pydevd_bundle.pydevd_net_command_factory_json import NetCommandFactoryJson
@@ -228,8 +228,28 @@ def request_step(self, py_db, thread_id, step_cmd_id):
         elif thread_id.startswith('__frame__:'):
             sys.stderr.write("Can't make tasklet step command: %s\n" % (thread_id,))
 
+    def request_smart_step_into(self, py_db, seq, thread_id, offset):
+        t = pydevd_find_thread_by_id(thread_id)
+        if t:
+            py_db.post_method_as_internal_command(
+                thread_id, internal_smart_step_into, thread_id, offset, set_additional_thread_info=set_additional_thread_info)
+        elif thread_id.startswith('__frame__:'):
+            sys.stderr.write("Can't set next statement in tasklet: %s\n" % (thread_id,))
+
+    def request_smart_step_into_by_func_name(self, py_db, seq, thread_id, line, func_name):
+        # Same thing as set next, just with a different cmd id.
+        self.request_set_next(py_db, seq, thread_id, CMD_SMART_STEP_INTO, None, line, func_name)
+
     def request_set_next(self, py_db, seq, thread_id, set_next_cmd_id, original_filename, line, func_name):
         '''
+        set_next_cmd_id may actually be one of:
+
+        CMD_RUN_TO_LINE
+        CMD_SET_NEXT_STATEMENT
+
+        CMD_SMART_STEP_INTO -- note: request_smart_step_into is preferred if it's possible
+                               to work with bytecode offset.
+
         :param Optional[str] original_filename:
             If available, the filename may be source translated, otherwise no translation will take
             place (the set next just needs the line afterwards as it executes locally, but for

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_bytecode_utils.py

@@ -0,0 +1,297 @@
+"""Bytecode analysing utils. Originally added for using in smart step into."""
+import dis
+import inspect
+from collections import namedtuple
+
+from _pydevd_bundle.pydevd_constants import IS_PY3K, KeyifyList
+from bisect import bisect
+
+_LOAD_OPNAMES = {
+    'LOAD_BUILD_CLASS',
+    'LOAD_CONST',
+    'LOAD_NAME',
+    'LOAD_ATTR',
+    'LOAD_GLOBAL',
+    'LOAD_FAST',
+    'LOAD_CLOSURE',
+    'LOAD_DEREF',
+}
+
+_CALL_OPNAMES = {
+    'CALL_FUNCTION',
+    'CALL_FUNCTION_KW',
+}
+
+if IS_PY3K:
+    for opname in ('LOAD_CLASSDEREF', 'LOAD_METHOD'):
+        _LOAD_OPNAMES.add(opname)
+    for opname in ('CALL_FUNCTION_EX', 'CALL_METHOD'):
+        _CALL_OPNAMES.add(opname)
+else:
+    _LOAD_OPNAMES.add('LOAD_LOCALS')
+    for opname in ('CALL_FUNCTION_VAR', 'CALL_FUNCTION_VAR_KW'):
+        _CALL_OPNAMES.add(opname)
+
+_BINARY_OPS = set([opname for opname in dis.opname if opname.startswith('BINARY_')])
+
+_BINARY_OP_MAP = {
+    'BINARY_POWER': '__pow__',
+    'BINARY_MULTIPLY': '__mul__',
+    'BINARY_MATRIX_MULTIPLY': '__matmul__',
+    'BINARY_FLOOR_DIVIDE': '__floordiv__',
+    'BINARY_TRUE_DIVIDE': '__div__',
+    'BINARY_MODULO': '__mod__',
+    'BINARY_ADD': '__add__',
+    'BINARY_SUBTRACT': '__sub__',
+    'BINARY_LSHIFT': '__lshift__',
+    'BINARY_RSHIFT': '__rshift__',
+    'BINARY_AND': '__and__',
+    'BINARY_OR': '__or__',
+    'BINARY_XOR': '__xor__',
+    'BINARY_SUBSCR': '__getitem__',
+}
+
+if not IS_PY3K:
+    _BINARY_OP_MAP['BINARY_DIVIDE'] = '__div__'
+
+_UNARY_OPS = set([opname for opname in dis.opname if opname.startswith('UNARY_') and opname != 'UNARY_NOT'])
+
+_UNARY_OP_MAP = {
+    'UNARY_POSITIVE': '__pos__',
+    'UNARY_NEGATIVE': '__neg__',
+    'UNARY_INVERT': '__invert__',
+}
+
+_MAKE_OPS = set([opname for opname in dis.opname if opname.startswith('MAKE_')])
+
+_COMP_OP_MAP = {
+    '<': '__lt__',
+    '<=': '__le__',
+    '==': '__eq__',
+    '!=': '__ne__',
+    '>': '__gt__',
+    '>=': '__ge__',
+    'in': '__contains__',
+    'not in': '__contains__',
+}
+
+
+def _is_load_opname(opname):
+    return opname in _LOAD_OPNAMES
+
+
+def _is_call_opname(opname):
+    return opname in _CALL_OPNAMES
+
+
+def _is_binary_opname(opname):
+    return opname in _BINARY_OPS
+
+
+def _is_unary_opname(opname):
+    return opname in _UNARY_OPS
+
+
+def _is_make_opname(opname):
+    return opname in _MAKE_OPS
+
+
+# Similar to :py:class:`dis._Instruction` but without fields we don't use. Also :py:class:`dis._Instruction`
+# is not available in Python 2.
+Instruction = namedtuple("Instruction", ["opname", "opcode", "arg", "argval", "lineno", "offset"])
+
+if IS_PY3K:
+    long = int
+
+try:
+    _unpack_opargs = dis._unpack_opargs
+except AttributeError:
+
+    def _unpack_opargs(code):
+        n = len(code)
+        i = 0
+        extended_arg = 0
+        while i < n:
+            c = code[i]
+            op = ord(c)
+            offset = i
+            arg = None
+            i += 1
+            if op >= dis.HAVE_ARGUMENT:
+                arg = ord(code[i]) + ord(code[i + 1]) * 256 + extended_arg
+                extended_arg = 0
+                i += 2
+                if op == dis.EXTENDED_ARG:
+                    extended_arg = arg * long(65536)
+            yield (offset, op, arg)
+
+
+def _code_to_name(inst):
+    """If thw instruction's ``argval`` is :py:class:`types.CodeType`, replace it with the name and return the updated instruction.
+    :type inst: :py:class:`Instruction`
+    :rtype: :py:class:`Instruction`
+    """
+    if inspect.iscode(inst.argval):
+        return inst._replace(argval=inst.argval.co_name)
+    return inst
+
+
+def _get_smart_step_into_candidates(code):
+    """Iterate through the bytecode and return a list of instructions which can be smart step into candidates.
+    :param code: A code object where we searching for calls.
+    :type code: :py:class:`types.CodeType`
+    :return: list of :py:class:`~Instruction` that represents the objects that were called
+      by one of the Python call instructions.
+    :raise: :py:class:`RuntimeError` if failed to parse the bytecode or if dis cannot be used.
+    """
+    try:
+        linestarts = dict(dis.findlinestarts(code))
+    except Exception:
+        raise RuntimeError("Unable to get smart step into candidates because dis.findlinestarts is not available.")
+
+    varnames = code.co_varnames
+    names = code.co_names
+    constants = code.co_consts
+    freevars = code.co_freevars
+    lineno = None
+    stk = []  # only the instructions related to calls are pushed in the stack
+    result = []
+
+    for offset, op, arg in _unpack_opargs(code.co_code):
+        try:
+            if linestarts is not None:
+                lineno = linestarts.get(offset, None) or lineno
+            opname = dis.opname[op]
+            argval = None
+            if arg is None:
+                if _is_binary_opname(opname):
+                    stk.pop()
+                    result.append(Instruction(opname, op, arg, _BINARY_OP_MAP[opname], lineno, offset))
+                elif _is_unary_opname(opname):
+                    result.append(Instruction(opname, op, arg, _UNARY_OP_MAP[opname], lineno, offset))
+            if opname == 'COMPARE_OP':
+                stk.pop()
+                cmp_op = dis.cmp_op[arg]
+                if cmp_op not in ('exception match', 'BAD'):
+                    result.append(Instruction(opname, op, arg, _COMP_OP_MAP.get(cmp_op, cmp_op), lineno, offset))
+            if _is_load_opname(opname):
+                if opname == 'LOAD_CONST':
+                    argval = constants[arg]
+                elif opname == 'LOAD_NAME' or opname == 'LOAD_GLOBAL':
+                    argval = names[arg]
+                elif opname == 'LOAD_ATTR':
+                    stk.pop()
+                    argval = names[arg]
+                elif opname == 'LOAD_FAST':
+                    argval = varnames[arg]
+                elif IS_PY3K and opname == 'LOAD_METHOD':
+                    stk.pop()
+                    argval = names[arg]
+                elif opname == 'LOAD_DEREF':
+                    argval = freevars[arg]
+                stk.append(Instruction(opname, op, arg, argval, lineno, offset))
+            elif _is_make_opname(opname):
+                tos = stk.pop()  # qualified name of the function or function code in Python 2
+                argc = 0
+                if IS_PY3K:
+                    stk.pop()  # function code
+                    for flag in (0x01, 0x02, 0x04, 0x08):
+                        if arg & flag:
+                            argc += 1  # each flag means one extra element to pop
+                else:
+                    argc = arg
+                    tos = _code_to_name(tos)
+                while argc > 0:
+                    stk.pop()
+                    argc -= 1
+                stk.append(tos)
+            elif _is_call_opname(opname):
+                argc = arg  # the number of the function or method arguments
+                if opname == 'CALL_FUNCTION_KW' or not IS_PY3K and opname == 'CALL_FUNCTION_VAR':
+                    stk.pop()  # pop the mapping or iterable with arguments or parameters
+                elif not IS_PY3K and opname == 'CALL_FUNCTION_VAR_KW':
+                    stk.pop()  # pop the mapping with arguments
+                    stk.pop()  # pop the iterable with parameters
+                elif not IS_PY3K and opname == 'CALL_FUNCTION':
+                    argc = arg & 0xff  # positional args
+                    argc += ((arg >> 8) * 2)  # keyword args
+                elif opname == 'CALL_FUNCTION_EX':
+                    has_keyword_args = arg & 0x01
+                    if has_keyword_args:
+                        stk.pop()
+                    stk.pop()  # positional args
+                    argc = 0
+                while argc > 0:
+                    stk.pop()  # popping args from the stack
+                    argc -= 1
+                tos = _code_to_name(stk[-1])
+                if tos.opname == 'LOAD_BUILD_CLASS':
+                    # an internal `CALL_FUNCTION` for building a class
+                    continue
+                result.append(tos._replace(offset=offset))  # the actual offset is not when a function was loaded but when it was called
+        except:
+            err_msg = "Bytecode parsing error at: offset(%d), opname(%s), arg(%d)" % (offset, dis.opname[op], arg)
+            raise RuntimeError(err_msg)
+    return result
+
+
+# Note that the offset is unique within the frame (so, we can use it as the target id).
+# Also, as the offset is the instruction offset within the frame, it's possible to
+# to inspect the parent frame for frame.f_lasti to know where we actually are (as the
+# caller name may not always match the new frame name).
+Variant = namedtuple('Variant', ['name', 'is_visited', 'line', 'offset', 'call_order'])
+
+
+def calculate_smart_step_into_variants(frame, start_line, end_line, base=0):
+    """
+    Calculate smart step into variants for the given line range.
+    :param frame:
+    :type frame: :py:class:`types.FrameType`
+    :param start_line:
+    :param end_line:
+    :return: A list of call names from the first to the last.
+    :note: it's guaranteed that the offsets appear in order.
+    :raise: :py:class:`RuntimeError` if failed to parse the bytecode or if dis cannot be used.
+    """
+    variants = []
+    is_context_reached = False
+    code = frame.f_code
+    lasti = frame.f_lasti
+
+    call_order_cache = {}
+
+    for inst in _get_smart_step_into_candidates(code):
+        if inst.lineno and inst.lineno > end_line:
+            break
+        if not is_context_reached and inst.lineno is not None and inst.lineno >= start_line:
+            is_context_reached = True
+        if not is_context_reached:
+            continue
+
+        call_order = call_order_cache.get(inst.argval, 0) + 1
+        call_order_cache[inst.argval] = call_order
+        variants.append(
+            Variant(
+                inst.argval, inst.offset <= lasti, inst.lineno - base, inst.offset, call_order))
+    return variants
+
+
+def get_smart_step_into_variant_from_frame_offset(frame_f_lasti, variants):
+    """
+    Given the frame.f_lasti, return the related `Variant`.
+
+    :note: if the offset is found before any variant available or no variants are
+           available, None is returned.
+
+    :rtype: Variant|NoneType
+    """
+    if not variants:
+        return None
+
+    i = bisect(KeyifyList(variants, lambda entry:entry.offset), frame_f_lasti)
+
+    if i == 0:
+        return None
+
+    else:
+        return variants[i - 1]