Properly validate/change django/jinja2 template breakpoints lines. Fixes

 microsoft#213

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_api.py

@@ -370,22 +370,31 @@ def __init__(self, filename):
     ADD_BREAKPOINT_FILE_NOT_FOUND = 1
     ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS = 2
 
+    # This means that the breakpoint couldn't be fully validated (more runtime
+    # information may be needed).
+    ADD_BREAKPOINT_LAZY_VALIDATION = 3
+    ADD_BREAKPOINT_INVALID_LINE = 4
+
     class _AddBreakpointResult(object):
 
         # :see: ADD_BREAKPOINT_NO_ERROR = 0
         # :see: ADD_BREAKPOINT_FILE_NOT_FOUND = 1
         # :see: ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS = 2
+        # :see: ADD_BREAKPOINT_LAZY_VALIDATION = 3
+        # :see: ADD_BREAKPOINT_INVALID_LINE = 4
 
-        __slots__ = ['error_code', 'translated_filename', 'translated_line']
+        __slots__ = ['error_code', 'breakpoint_id', 'translated_filename', 'translated_line', 'original_line']
 
-        def __init__(self, translated_filename, translated_line):
+        def __init__(self, breakpoint_id, translated_filename, translated_line, original_line):
             self.error_code = PyDevdAPI.ADD_BREAKPOINT_NO_ERROR
+            self.breakpoint_id = breakpoint_id
             self.translated_filename = translated_filename
             self.translated_line = translated_line
+            self.original_line = original_line
 
     def add_breakpoint(
             self, py_db, original_filename, breakpoint_type, breakpoint_id, line, condition, func_name,
-            expression, suspend_policy, hit_condition, is_logpoint, adjust_line=False):
+            expression, suspend_policy, hit_condition, is_logpoint, adjust_line=False, on_changed_breakpoint_state=None):
         '''
         :param str original_filename:
             Note: must be sent as it was received in the protocol. It may be translated in this
@@ -423,11 +432,27 @@ def add_breakpoint(
             If True and an expression is passed, pydevd will create an io message command with the
             result of the evaluation.
 
+        :param bool adjust_line:
+            If True, the breakpoint line should be adjusted if the current line doesn't really
+            match an executable line (if possible).
+
+        :param callable on_changed_breakpoint_state:
+            This is called when something changed internally on the breakpoint after it was initially
+            added (for instance, template file_to_line_to_breakpoints could be signaled as invalid initially and later
+            when the related template is loaded, if the line is valid it could be marked as valid).
+
+            The signature for the callback should be:
+                on_changed_breakpoint_state(breakpoint_id: int, add_breakpoint_result: _AddBreakpointResult)
+
+                Note that the add_breakpoint_result should not be modified by the callback (the
+                implementation may internally reuse the same instance multiple times).
+
         :return _AddBreakpointResult:
         '''
         assert original_filename.__class__ == str, 'Expected str, found: %s' % (original_filename.__class__,)  # i.e.: bytes on py2 and str on py3
 
         pydev_log.debug('Request for breakpoint in: %s line: %s', original_filename, line)
+        original_line = line
         # Parameters to reapply breakpoint.
         api_add_breakpoint_params = (original_filename, breakpoint_type, breakpoint_id, line, condition, func_name,
             expression, suspend_policy, hit_condition, is_logpoint)
@@ -449,7 +474,7 @@ def add_breakpoint(
             # (we want the outside world to see the line in the original file and not in the ipython
             # cell, otherwise the editor wouldn't be correct as the returned line is the line to
             # which the breakpoint will be moved in the editor).
-            result = self._AddBreakpointResult(original_filename, line)
+            result = self._AddBreakpointResult(breakpoint_id, original_filename, line, original_line)
 
             # If a multi-mapping was applied, consider it the canonical / source mapped version (translated to ipython cell).
             translated_absolute_filename = source_mapped_filename
@@ -461,7 +486,7 @@ def add_breakpoint(
             canonical_normalized_filename = pydevd_file_utils.canonical_normalized_path(translated_filename)
 
             if adjust_line and not translated_absolute_filename.startswith('<'):
-                # Validate breakpoints and adjust their positions.
+                # Validate file_to_line_to_breakpoints and adjust their positions.
                 try:
                     lines = sorted(_get_code_lines(translated_absolute_filename))
                 except Exception:
@@ -473,7 +498,7 @@ def add_breakpoint(
                         if idx > 0:
                             line = lines[idx - 1]
 
-            result = self._AddBreakpointResult(original_filename, line)
+            result = self._AddBreakpointResult(breakpoint_id, original_filename, line, original_line)
 
         py_db.api_received_breakpoints[(original_filename, breakpoint_id)] = (canonical_normalized_filename, api_add_breakpoint_params)
 
@@ -501,8 +526,10 @@ def add_breakpoint(
                 result.error_code = self.ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS
 
         if breakpoint_type == 'python-line':
-            added_breakpoint = LineBreakpoint(line, condition, func_name, expression, suspend_policy, hit_condition=hit_condition, is_logpoint=is_logpoint)
-            breakpoints = py_db.breakpoints
+            added_breakpoint = LineBreakpoint(
+                breakpoint_id, line, condition, func_name, expression, suspend_policy, hit_condition=hit_condition, is_logpoint=is_logpoint)
+
+            file_to_line_to_breakpoints = py_db.breakpoints
             file_to_id_to_breakpoint = py_db.file_to_id_to_line_breakpoint
             supported_type = True
 
@@ -511,11 +538,13 @@ def add_breakpoint(
             plugin = py_db.get_plugin_lazy_init()
             if plugin is not None:
                 add_plugin_breakpoint_result = plugin.add_breakpoint(
-                    'add_line_breakpoint', py_db, breakpoint_type, canonical_normalized_filename, line, condition, expression, func_name, hit_condition=hit_condition, is_logpoint=is_logpoint)
+                    'add_line_breakpoint', py_db, breakpoint_type, canonical_normalized_filename,
+                    breakpoint_id, line, condition, expression, func_name, hit_condition=hit_condition, is_logpoint=is_logpoint,
+                    add_breakpoint_result=result, on_changed_breakpoint_state=on_changed_breakpoint_state)
 
             if add_plugin_breakpoint_result is not None:
                 supported_type = True
-                added_breakpoint, breakpoints = add_plugin_breakpoint_result
+                added_breakpoint, file_to_line_to_breakpoints = add_plugin_breakpoint_result
                 file_to_id_to_breakpoint = py_db.file_to_id_to_plugin_breakpoint
             else:
                 supported_type = False
@@ -532,9 +561,10 @@ def add_breakpoint(
             id_to_pybreakpoint = file_to_id_to_breakpoint[canonical_normalized_filename] = {}
 
         id_to_pybreakpoint[breakpoint_id] = added_breakpoint
-        py_db.consolidate_breakpoints(canonical_normalized_filename, id_to_pybreakpoint, breakpoints)
+        py_db.consolidate_breakpoints(canonical_normalized_filename, id_to_pybreakpoint, file_to_line_to_breakpoints)
         if py_db.plugin is not None:
             py_db.has_plugin_line_breaks = py_db.plugin.has_line_breaks()
+            py_db.plugin.after_breakpoints_consolidated(py_db, canonical_normalized_filename, id_to_pybreakpoint, file_to_line_to_breakpoints)
 
         py_db.on_breakpoints_changed()
         return result
@@ -626,14 +656,14 @@ def remove_breakpoint(self, py_db, received_filename, breakpoint_type, breakpoin
         canonical_normalized_filename = pydevd_file_utils.canonical_normalized_path(received_filename)
 
         if breakpoint_type == 'python-line':
-            breakpoints = py_db.breakpoints
+            file_to_line_to_breakpoints = py_db.breakpoints
             file_to_id_to_breakpoint = py_db.file_to_id_to_line_breakpoint
 
         elif py_db.plugin is not None:
             result = py_db.plugin.get_breakpoints(py_db, breakpoint_type)
             if result is not None:
                 file_to_id_to_breakpoint = py_db.file_to_id_to_plugin_breakpoint
-                breakpoints = result
+                file_to_line_to_breakpoints = result
 
         if file_to_id_to_breakpoint is None:
             pydev_log.critical('Error removing breakpoint. Cannot handle breakpoint of type %s', breakpoint_type)
@@ -647,9 +677,10 @@ def remove_breakpoint(self, py_db, received_filename, breakpoint_type, breakpoin
                         canonical_normalized_filename, existing.line, existing.func_name.encode('utf-8'), breakpoint_id))
 
                 del id_to_pybreakpoint[breakpoint_id]
-                py_db.consolidate_breakpoints(canonical_normalized_filename, id_to_pybreakpoint, breakpoints)
+                py_db.consolidate_breakpoints(canonical_normalized_filename, id_to_pybreakpoint, file_to_line_to_breakpoints)
                 if py_db.plugin is not None:
                     py_db.has_plugin_line_breaks = py_db.plugin.has_line_breaks()
+                    py_db.plugin.after_breakpoints_consolidated(py_db, canonical_normalized_filename, id_to_pybreakpoint, file_to_line_to_breakpoints)
 
             except KeyError:
                 pydev_log.info("Error removing breakpoint: Breakpoint id not found: %s id: %s. Available ids: %s\n",

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_breakpoints.py

@@ -48,7 +48,8 @@ def handle_hit_condition(self, frame):
 
 class LineBreakpoint(object):
 
-    def __init__(self, line, condition, func_name, expression, suspend_policy="NONE", hit_condition=None, is_logpoint=False):
+    def __init__(self, breakpoint_id, line, condition, func_name, expression, suspend_policy="NONE", hit_condition=None, is_logpoint=False):
+        self.breakpoint_id = breakpoint_id
         self.line = line
         self.condition = condition
         self.func_name = func_name

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_dont_trace_files.py

@@ -117,6 +117,7 @@
     'pydevd_import_class.py': PYDEV_FILE,
     'pydevd_io.py': PYDEV_FILE,
     'pydevd_json_debug_options.py': PYDEV_FILE,
+    'pydevd_line_validation.py': PYDEV_FILE,
     'pydevd_modify_bytecode.py': PYDEV_FILE,
     'pydevd_net_command.py': PYDEV_FILE,
     'pydevd_net_command_factory_json.py': PYDEV_FILE,

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_frame_utils.py

@@ -29,6 +29,7 @@ def __init__(self, name, filename):
         self.co_name = name
         self.co_filename = filename
         self.co_firstlineno = 1
+        self.co_flags = 0
 
 
 def add_exception_to_frame(frame, exception_info):

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command.py

@@ -303,21 +303,42 @@ def cmd_set_break(self, py_db, cmd_id, seq, text):
         if hit_condition is not None and (len(hit_condition) <= 0 or hit_condition == u"None"):
             hit_condition = None
 
-        result = self.api.add_breakpoint(
-            py_db, self.api.filename_to_str(filename), btype, breakpoint_id, line, condition, func_name, expression, suspend_policy, hit_condition, is_logpoint)
-        error_code = result.error_code
+        def on_changed_breakpoint_state(breakpoint_id, add_breakpoint_result):
+            error_code = add_breakpoint_result.error_code
+
+            translated_line = add_breakpoint_result.translated_line
+            translated_filename = add_breakpoint_result.translated_filename
+            msg = ''
+            if error_code:
+
+                if error_code == self.api.ADD_BREAKPOINT_FILE_NOT_FOUND:
+                    msg = 'pydev debugger: Trying to add breakpoint to file that does not exist: %s (will have no effect).\n' % (translated_filename,)
+
+                elif error_code == self.api.ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS:
+                    msg = 'pydev debugger: Trying to add breakpoint to file that is excluded by filters: %s (will have no effect).\n' % (translated_filename,)
+
+                elif error_code == self.api.ADD_BREAKPOINT_LAZY_VALIDATION:
+                    msg = ''  # Ignore this here (if/when loaded, it'll call on_changed_breakpoint_state again accordingly).
 
-        if error_code:
-            translated_filename = result.translated_filename
-            if error_code == self.api.ADD_BREAKPOINT_FILE_NOT_FOUND:
-                pydev_log.critical('pydev debugger: warning: Trying to add breakpoint to file that does not exist: %s (will have no effect).' % (translated_filename,))
+                elif error_code == self.api.ADD_BREAKPOINT_INVALID_LINE:
+                    msg = 'pydev debugger: Trying to add breakpoint to line (%s) that is not valid in: %s.\n' % (translated_line, translated_filename,)
 
-            elif error_code == self.api.ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS:
-                pydev_log.critical('pydev debugger: warning: Trying to add breakpoint to file that is excluded by filters: %s (will have no effect).' % (translated_filename,))
+                else:
+                    # Shouldn't get here.
+                    msg = 'pydev debugger: Breakpoint not validated (reason unknown -- please report as error): %s (%s).\n' % (translated_filename, translated_line)
 
             else:
-                # Shouldn't get here.
-                pydev_log.critical('pydev debugger: warning: Breakpoint not validated (reason unknown -- please report as error): %s.' % (translated_filename,))
+                if add_breakpoint_result.original_line != translated_line:
+                    msg = 'pydev debugger (info): Breakpoint in line: %s moved to line: %s (in %s).\n' % (add_breakpoint_result.original_line, translated_line, translated_filename)
+
+            if msg:
+                py_db.writer.add_command(py_db.cmd_factory.make_warning_message(msg))
+
+        result = self.api.add_breakpoint(
+            py_db, self.api.filename_to_str(filename), btype, breakpoint_id, line, condition, func_name,
+            expression, suspend_policy, hit_condition, is_logpoint, on_changed_breakpoint_state=on_changed_breakpoint_state)
+
+        on_changed_breakpoint_state(breakpoint_id, result)
 
     def cmd_remove_break(self, py_db, cmd_id, seq, text):
         # command to remove some breakpoint

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command_json.py

@@ -16,7 +16,8 @@
     SetVariableResponseBody, SourceBreakpoint, SourceResponseBody,
     VariablesResponseBody, SetBreakpointsResponseBody, Response,
     Capabilities, PydevdAuthorizeRequest, Request, StepInTargetsResponse, StepInTarget,
-    StepInTargetsResponseBody, SetFunctionBreakpointsResponseBody)
+    StepInTargetsResponseBody, SetFunctionBreakpointsResponseBody, BreakpointEvent,
+    BreakpointEventBody)
 from _pydevd_bundle.pydevd_api import PyDevdAPI
 from _pydevd_bundle.pydevd_breakpoints import get_exception_class, FunctionBreakpoint
 from _pydevd_bundle.pydevd_comm_constants import (
@@ -762,7 +763,7 @@ def on_setbreakpoints_request(self, py_db, request):
             source_breakpoint = SourceBreakpoint(**source_breakpoint)
             line = source_breakpoint.line
             condition = source_breakpoint.condition
-            breakpoint_id = line
+            breakpoint_id = self._next_breakpoint_id()
 
             hit_condition = self._get_hit_condition_expression(source_breakpoint.hitCondition)
             log_message = source_breakpoint.logMessage
@@ -773,36 +774,56 @@ def on_setbreakpoints_request(self, py_db, request):
                 is_logpoint = True
                 expression = convert_dap_log_message_to_expression(log_message)
 
+            on_changed_breakpoint_state = partial(self._on_changed_breakpoint_state, py_db, arguments.source)
             result = self.api.add_breakpoint(
-                py_db, filename, btype, breakpoint_id, line, condition, func_name, expression, suspend_policy, hit_condition, is_logpoint, adjust_line=True)
-            error_code = result.error_code
+                py_db, filename, btype, breakpoint_id, line, condition, func_name, expression,
+                suspend_policy, hit_condition, is_logpoint, adjust_line=True, on_changed_breakpoint_state=on_changed_breakpoint_state)
 
-            if error_code:
-                if error_code == self.api.ADD_BREAKPOINT_FILE_NOT_FOUND:
-                    error_msg = 'Breakpoint in file that does not exist.'
+            bp = self._create_breakpoint_from_add_breakpoint_result(py_db, arguments.source, breakpoint_id, result)
+            breakpoints_set.append(bp)
 
-                elif error_code == self.api.ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS:
-                    error_msg = 'Breakpoint in file excluded by filters.'
-                    if py_db.get_use_libraries_filter():
-                        error_msg += ('\nNote: may be excluded because of "justMyCode" option (default == true).'
-                                      'Try setting \"justMyCode\": false in the debug configuration (e.g., launch.json).\n')
+        body = {'breakpoints': breakpoints_set}
+        set_breakpoints_response = pydevd_base_schema.build_response(request, kwargs={'body': body})
+        return NetCommand(CMD_RETURN, 0, set_breakpoints_response, is_json=True)
 
-                else:
-                    # Shouldn't get here.
-                    error_msg = 'Breakpoint not validated (reason unknown -- please report as bug).'
+    def _on_changed_breakpoint_state(self, py_db, source, breakpoint_id, result):
+        bp = self._create_breakpoint_from_add_breakpoint_result(py_db, source, breakpoint_id, result)
+        body = BreakpointEventBody(
+            reason='changed',
+            breakpoint=bp,
+        )
+        event = BreakpointEvent(body)
+        event_id = 0  # Actually ignored in this case
+        py_db.writer.add_command(NetCommand(event_id, 0, event, is_json=True))
+
+    def _create_breakpoint_from_add_breakpoint_result(self, py_db, source, breakpoint_id, result):
+        error_code = result.error_code
+
+        if error_code:
+            if error_code == self.api.ADD_BREAKPOINT_FILE_NOT_FOUND:
+                error_msg = 'Breakpoint in file that does not exist.'
+
+            elif error_code == self.api.ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS:
+                error_msg = 'Breakpoint in file excluded by filters.'
+                if py_db.get_use_libraries_filter():
+                    error_msg += ('\nNote: may be excluded because of "justMyCode" option (default == true).'
+                                  'Try setting \"justMyCode\": false in the debug configuration (e.g., launch.json).\n')
+
+            elif error_code == self.api.ADD_BREAKPOINT_LAZY_VALIDATION:
+                error_msg = 'Waiting for code to be loaded to verify breakpoint.'
+
+            elif error_code == self.api.ADD_BREAKPOINT_INVALID_LINE:
+                error_msg = 'Breakpoint added to invalid line.'
 
-                breakpoints_set.append(pydevd_schema.Breakpoint(
-                    verified=False, line=result.translated_line, message=error_msg, source=arguments.source).to_dict())
             else:
-                # Note that the id is made up (the id for pydevd is unique only within a file, so, the
-                # line is used for it).
-                # Also, the id is currently not used afterwards, so, we don't even keep a mapping.
-                breakpoints_set.append(pydevd_schema.Breakpoint(
-                    verified=True, id=self._next_breakpoint_id(), line=result.translated_line, source=arguments.source).to_dict())
+                # Shouldn't get here.
+                error_msg = 'Breakpoint not validated (reason unknown -- please report as bug).'
 
-        body = {'breakpoints': breakpoints_set}
-        set_breakpoints_response = pydevd_base_schema.build_response(request, kwargs={'body': body})
-        return NetCommand(CMD_RETURN, 0, set_breakpoints_response, is_json=True)
+            return pydevd_schema.Breakpoint(
+                verified=False, id=breakpoint_id, line=result.translated_line, message=error_msg, source=source).to_dict()
+        else:
+            return pydevd_schema.Breakpoint(
+                verified=True, id=breakpoint_id, line=result.translated_line, source=source).to_dict()
 
     def on_setexceptionbreakpoints_request(self, py_db, request):
         '''