Properly validate/change django/jinja2 template breakpoints lines. Fixes

 #213

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_api.py

@@ -370,22 +370,31 @@ def __init__(self, filename):
  ADD_BREAKPOINT_FILE_NOT_FOUND = 1
  ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS = 2
 
+ # This means that the breakpoint couldn't be fully validated (more runtime
+ # information may be needed).
+ ADD_BREAKPOINT_LAZY_VALIDATION = 3
+ ADD_BREAKPOINT_INVALID_LINE = 4
+
  class _AddBreakpointResult(object):
 
  # :see: ADD_BREAKPOINT_NO_ERROR = 0
  # :see: ADD_BREAKPOINT_FILE_NOT_FOUND = 1
  # :see: ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS = 2
+ # :see: ADD_BREAKPOINT_LAZY_VALIDATION = 3
+ # :see: ADD_BREAKPOINT_INVALID_LINE = 4
 
- __slots__ = ['error_code', 'translated_filename', 'translated_line']
+ __slots__ = ['error_code', 'breakpoint_id', 'translated_filename', 'translated_line', 'original_line']
 
- def __init__(self, translated_filename, translated_line):
+ def __init__(self, breakpoint_id, translated_filename, translated_line, original_line):
  self.error_code = PyDevdAPI.ADD_BREAKPOINT_NO_ERROR
+ self.breakpoint_id = breakpoint_id
  self.translated_filename = translated_filename
  self.translated_line = translated_line
+ self.original_line = original_line
 
  def add_breakpoint(
  self, py_db, original_filename, breakpoint_type, breakpoint_id, line, condition, func_name,
- expression, suspend_policy, hit_condition, is_logpoint, adjust_line=False):
+ expression, suspend_policy, hit_condition, is_logpoint, adjust_line=False, on_changed_breakpoint_state=None):
  '''
  :param str original_filename:
  Note: must be sent as it was received in the protocol. It may be translated in this
@@ -423,11 +432,27 @@ def add_breakpoint(
  If True and an expression is passed, pydevd will create an io message command with the
  result of the evaluation.
 
+ :param bool adjust_line:
+ If True, the breakpoint line should be adjusted if the current line doesn't really
+ match an executable line (if possible).
+
+ :param callable on_changed_breakpoint_state:
+ This is called when something changed internally on the breakpoint after it was initially
+ added (for instance, template file_to_line_to_breakpoints could be signaled as invalid initially and later
+ when the related template is loaded, if the line is valid it could be marked as valid).
+
+ The signature for the callback should be:
+ on_changed_breakpoint_state(breakpoint_id: int, add_breakpoint_result: _AddBreakpointResult)
+
+ Note that the add_breakpoint_result should not be modified by the callback (the
+ implementation may internally reuse the same instance multiple times).
+
  :return _AddBreakpointResult:
  '''
  assert original_filename.__class__ == str, 'Expected str, found: %s' % (original_filename.__class__,) # i.e.: bytes on py2 and str on py3
 
  pydev_log.debug('Request for breakpoint in: %s line: %s', original_filename, line)
+ original_line = line
  # Parameters to reapply breakpoint.
  api_add_breakpoint_params = (original_filename, breakpoint_type, breakpoint_id, line, condition, func_name,
  expression, suspend_policy, hit_condition, is_logpoint)
@@ -449,7 +474,7 @@ def add_breakpoint(
  # (we want the outside world to see the line in the original file and not in the ipython
  # cell, otherwise the editor wouldn't be correct as the returned line is the line to
  # which the breakpoint will be moved in the editor).
- result = self._AddBreakpointResult(original_filename, line)
+ result = self._AddBreakpointResult(breakpoint_id, original_filename, line, original_line)
 
  # If a multi-mapping was applied, consider it the canonical / source mapped version (translated to ipython cell).
  translated_absolute_filename = source_mapped_filename
@@ -461,7 +486,7 @@ def add_breakpoint(
  canonical_normalized_filename = pydevd_file_utils.canonical_normalized_path(translated_filename)
 
  if adjust_line and not translated_absolute_filename.startswith('<'):
- # Validate breakpoints and adjust their positions.
+ # Validate file_to_line_to_breakpoints and adjust their positions.
  try:
  lines = sorted(_get_code_lines(translated_absolute_filename))
  except Exception:
@@ -473,7 +498,7 @@ def add_breakpoint(
  if idx > 0:
  line = lines[idx - 1]
 
- result = self._AddBreakpointResult(original_filename, line)
+ result = self._AddBreakpointResult(breakpoint_id, original_filename, line, original_line)
 
  py_db.api_received_breakpoints[(original_filename, breakpoint_id)] = (canonical_normalized_filename, api_add_breakpoint_params)
 
@@ -501,8 +526,10 @@ def add_breakpoint(
  result.error_code = self.ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS
 
  if breakpoint_type == 'python-line':
- added_breakpoint = LineBreakpoint(line, condition, func_name, expression, suspend_policy, hit_condition=hit_condition, is_logpoint=is_logpoint)
- breakpoints = py_db.breakpoints
+ added_breakpoint = LineBreakpoint(
+ breakpoint_id, line, condition, func_name, expression, suspend_policy, hit_condition=hit_condition, is_logpoint=is_logpoint)
+
+ file_to_line_to_breakpoints = py_db.breakpoints
  file_to_id_to_breakpoint = py_db.file_to_id_to_line_breakpoint
  supported_type = True
 
@@ -511,11 +538,13 @@ def add_breakpoint(
  plugin = py_db.get_plugin_lazy_init()
  if plugin is not None:
  add_plugin_breakpoint_result = plugin.add_breakpoint(
- 'add_line_breakpoint', py_db, breakpoint_type, canonical_normalized_filename, line, condition, expression, func_name, hit_condition=hit_condition, is_logpoint=is_logpoint)
+ 'add_line_breakpoint', py_db, breakpoint_type, canonical_normalized_filename,
+ breakpoint_id, line, condition, expression, func_name, hit_condition=hit_condition, is_logpoint=is_logpoint,
+ add_breakpoint_result=result, on_changed_breakpoint_state=on_changed_breakpoint_state)
 
  if add_plugin_breakpoint_result is not None:
  supported_type = True
- added_breakpoint, breakpoints = add_plugin_breakpoint_result
+ added_breakpoint, file_to_line_to_breakpoints = add_plugin_breakpoint_result
  file_to_id_to_breakpoint = py_db.file_to_id_to_plugin_breakpoint
  else:
  supported_type = False
@@ -532,9 +561,10 @@ def add_breakpoint(
  id_to_pybreakpoint = file_to_id_to_breakpoint[canonical_normalized_filename] = {}
 
  id_to_pybreakpoint[breakpoint_id] = added_breakpoint
- py_db.consolidate_breakpoints(canonical_normalized_filename, id_to_pybreakpoint, breakpoints)
+ py_db.consolidate_breakpoints(canonical_normalized_filename, id_to_pybreakpoint, file_to_line_to_breakpoints)
  if py_db.plugin is not None:
  py_db.has_plugin_line_breaks = py_db.plugin.has_line_breaks()
+ py_db.plugin.after_breakpoints_consolidated(py_db, canonical_normalized_filename, id_to_pybreakpoint, file_to_line_to_breakpoints)
 
  py_db.on_breakpoints_changed()
  return result
@@ -626,14 +656,14 @@ def remove_breakpoint(self, py_db, received_filename, breakpoint_type, breakpoin
  canonical_normalized_filename = pydevd_file_utils.canonical_normalized_path(received_filename)
 
  if breakpoint_type == 'python-line':
- breakpoints = py_db.breakpoints
+ file_to_line_to_breakpoints = py_db.breakpoints
  file_to_id_to_breakpoint = py_db.file_to_id_to_line_breakpoint
 
  elif py_db.plugin is not None:
  result = py_db.plugin.get_breakpoints(py_db, breakpoint_type)
  if result is not None:
  file_to_id_to_breakpoint = py_db.file_to_id_to_plugin_breakpoint
- breakpoints = result
+ file_to_line_to_breakpoints = result
 
  if file_to_id_to_breakpoint is None:
  pydev_log.critical('Error removing breakpoint. Cannot handle breakpoint of type %s', breakpoint_type)
@@ -647,9 +677,10 @@ def remove_breakpoint(self, py_db, received_filename, breakpoint_type, breakpoin
  canonical_normalized_filename, existing.line, existing.func_name.encode('utf-8'), breakpoint_id))
 
  del id_to_pybreakpoint[breakpoint_id]
- py_db.consolidate_breakpoints(canonical_normalized_filename, id_to_pybreakpoint, breakpoints)
+ py_db.consolidate_breakpoints(canonical_normalized_filename, id_to_pybreakpoint, file_to_line_to_breakpoints)
  if py_db.plugin is not None:
  py_db.has_plugin_line_breaks = py_db.plugin.has_line_breaks()
+ py_db.plugin.after_breakpoints_consolidated(py_db, canonical_normalized_filename, id_to_pybreakpoint, file_to_line_to_breakpoints)
 
  except KeyError:
  pydev_log.info("Error removing breakpoint: Breakpoint id not found: %s id: %s. Available ids: %s\n",

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_breakpoints.py

@@ -48,7 +48,8 @@ def handle_hit_condition(self, frame):
 
 class LineBreakpoint(object):
 
- def __init__(self, line, condition, func_name, expression, suspend_policy="NONE", hit_condition=None, is_logpoint=False):
+ def __init__(self, breakpoint_id, line, condition, func_name, expression, suspend_policy="NONE", hit_condition=None, is_logpoint=False):
+ self.breakpoint_id = breakpoint_id
  self.line = line
  self.condition = condition
  self.func_name = func_name

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_dont_trace_files.py

@@ -117,6 +117,7 @@
  'pydevd_import_class.py': PYDEV_FILE,
  'pydevd_io.py': PYDEV_FILE,
  'pydevd_json_debug_options.py': PYDEV_FILE,
+ 'pydevd_line_validation.py': PYDEV_FILE,
  'pydevd_modify_bytecode.py': PYDEV_FILE,
  'pydevd_net_command.py': PYDEV_FILE,
  'pydevd_net_command_factory_json.py': PYDEV_FILE,

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_frame_utils.py

@@ -29,6 +29,7 @@ def __init__(self, name, filename):
  self.co_name = name
  self.co_filename = filename
  self.co_firstlineno = 1
+ self.co_flags = 0
 
 
 def add_exception_to_frame(frame, exception_info):

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command.py

@@ -303,21 +303,42 @@ def cmd_set_break(self, py_db, cmd_id, seq, text):
  if hit_condition is not None and (len(hit_condition) <= 0 or hit_condition == u"None"):
  hit_condition = None
 
- result = self.api.add_breakpoint(
- py_db, self.api.filename_to_str(filename), btype, breakpoint_id, line, condition, func_name, expression, suspend_policy, hit_condition, is_logpoint)
- error_code = result.error_code
+ def on_changed_breakpoint_state(breakpoint_id, add_breakpoint_result):
+ error_code = add_breakpoint_result.error_code
+
+ translated_line = add_breakpoint_result.translated_line
+ translated_filename = add_breakpoint_result.translated_filename
+ msg = ''
+ if error_code:
+
+ if error_code == self.api.ADD_BREAKPOINT_FILE_NOT_FOUND:
+ msg = 'pydev debugger: Trying to add breakpoint to file that does not exist: %s (will have no effect).\n' % (translated_filename,)
+
+ elif error_code == self.api.ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS:
+ msg = 'pydev debugger: Trying to add breakpoint to file that is excluded by filters: %s (will have no effect).\n' % (translated_filename,)
+
+ elif error_code == self.api.ADD_BREAKPOINT_LAZY_VALIDATION:
+ msg = '' # Ignore this here (if/when loaded, it'll call on_changed_breakpoint_state again accordingly).
 
- if error_code:
- translated_filename = result.translated_filename
- if error_code == self.api.ADD_BREAKPOINT_FILE_NOT_FOUND:
- pydev_log.critical('pydev debugger: warning: Trying to add breakpoint to file that does not exist: %s (will have no effect).' % (translated_filename,))
+ elif error_code == self.api.ADD_BREAKPOINT_INVALID_LINE:
+ msg = 'pydev debugger: Trying to add breakpoint to line (%s) that is not valid in: %s.\n' % (translated_line, translated_filename,)
 
- elif error_code == self.api.ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS:
- pydev_log.critical('pydev debugger: warning: Trying to add breakpoint to file that is excluded by filters: %s (will have no effect).' % (translated_filename,))
+ else:
+ # Shouldn't get here.
+ msg = 'pydev debugger: Breakpoint not validated (reason unknown -- please report as error): %s (%s).\n' % (translated_filename, translated_line)
 
  else:
- # Shouldn't get here.
- pydev_log.critical('pydev debugger: warning: Breakpoint not validated (reason unknown -- please report as error): %s.' % (translated_filename,))
+ if add_breakpoint_result.original_line != translated_line:
+ msg = 'pydev debugger (info): Breakpoint in line: %s moved to line: %s (in %s).\n' % (add_breakpoint_result.original_line, translated_line, translated_filename)
+
+ if msg:
+ py_db.writer.add_command(py_db.cmd_factory.make_warning_message(msg))
+
+ result = self.api.add_breakpoint(
+ py_db, self.api.filename_to_str(filename), btype, breakpoint_id, line, condition, func_name,
+ expression, suspend_policy, hit_condition, is_logpoint, on_changed_breakpoint_state=on_changed_breakpoint_state)
+
+ on_changed_breakpoint_state(breakpoint_id, result)
 
  def cmd_remove_break(self, py_db, cmd_id, seq, text):
  # command to remove some breakpoint

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command_json.py

@@ -16,7 +16,8 @@
  SetVariableResponseBody, SourceBreakpoint, SourceResponseBody,
  VariablesResponseBody, SetBreakpointsResponseBody, Response,
  Capabilities, PydevdAuthorizeRequest, Request, StepInTargetsResponse, StepInTarget,
- StepInTargetsResponseBody, SetFunctionBreakpointsResponseBody)
+ StepInTargetsResponseBody, SetFunctionBreakpointsResponseBody, BreakpointEvent,
+ BreakpointEventBody)
 from _pydevd_bundle.pydevd_api import PyDevdAPI
 from _pydevd_bundle.pydevd_breakpoints import get_exception_class, FunctionBreakpoint
 from _pydevd_bundle.pydevd_comm_constants import (
@@ -762,7 +763,7 @@ def on_setbreakpoints_request(self, py_db, request):
  source_breakpoint = SourceBreakpoint(**source_breakpoint)
  line = source_breakpoint.line
  condition = source_breakpoint.condition
- breakpoint_id = line
+ breakpoint_id = self._next_breakpoint_id()
 
  hit_condition = self._get_hit_condition_expression(source_breakpoint.hitCondition)
  log_message = source_breakpoint.logMessage
@@ -773,36 +774,56 @@ def on_setbreakpoints_request(self, py_db, request):
  is_logpoint = True
  expression = convert_dap_log_message_to_expression(log_message)
 
+ on_changed_breakpoint_state = partial(self._on_changed_breakpoint_state, py_db, arguments.source)
  result = self.api.add_breakpoint(
- py_db, filename, btype, breakpoint_id, line, condition, func_name, expression, suspend_policy, hit_condition, is_logpoint, adjust_line=True)
- error_code = result.error_code
+ py_db, filename, btype, breakpoint_id, line, condition, func_name, expression,
+  suspend_policy, hit_condition, is_logpoint, adjust_line=True, on_changed_breakpoint_state=on_changed_breakpoint_state)
 
- if error_code:
- if error_code == self.api.ADD_BREAKPOINT_FILE_NOT_FOUND:
- error_msg = 'Breakpoint in file that does not exist.'
+ bp = self._create_breakpoint_from_add_breakpoint_result(py_db, arguments.source, breakpoint_id, result)
+ breakpoints_set.append(bp)
 
- elif error_code == self.api.ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS:
- error_msg = 'Breakpoint in file excluded by filters.'
- if py_db.get_use_libraries_filter():
- error_msg += ('\nNote: may be excluded because of "justMyCode" option (default == true).'
- 'Try setting \"justMyCode\": false in the debug configuration (e.g., launch.json).\n')
+ body = {'breakpoints': breakpoints_set}
+ set_breakpoints_response = pydevd_base_schema.build_response(request, kwargs={'body': body})
+ return NetCommand(CMD_RETURN, 0, set_breakpoints_response, is_json=True)
 
- else:
- # Shouldn't get here.
- error_msg = 'Breakpoint not validated (reason unknown -- please report as bug).'
+ def _on_changed_breakpoint_state(self, py_db, source, breakpoint_id, result):
+ bp = self._create_breakpoint_from_add_breakpoint_result(py_db, source, breakpoint_id, result)
+ body = BreakpointEventBody(
+ reason='changed',
+ breakpoint=bp,
+ )
+ event = BreakpointEvent(body)
+ event_id = 0 # Actually ignored in this case
+ py_db.writer.add_command(NetCommand(event_id, 0, event, is_json=True))
+
+ def _create_breakpoint_from_add_breakpoint_result(self, py_db, source, breakpoint_id, result):
+ error_code = result.error_code
+
+ if error_code:
+ if error_code == self.api.ADD_BREAKPOINT_FILE_NOT_FOUND:
+ error_msg = 'Breakpoint in file that does not exist.'
+
+ elif error_code == self.api.ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS:
+ error_msg = 'Breakpoint in file excluded by filters.'
+ if py_db.get_use_libraries_filter():
+ error_msg += ('\nNote: may be excluded because of "justMyCode" option (default == true).'
+ 'Try setting \"justMyCode\": false in the debug configuration (e.g., launch.json).\n')
+
+ elif error_code == self.api.ADD_BREAKPOINT_LAZY_VALIDATION:
+ error_msg = 'Waiting for code to be loaded to verify breakpoint.'
+
+ elif error_code == self.api.ADD_BREAKPOINT_INVALID_LINE:
+ error_msg = 'Breakpoint added to invalid line.'
 
- breakpoints_set.append(pydevd_schema.Breakpoint(
- verified=False, line=result.translated_line, message=error_msg, source=arguments.source).to_dict())
  else:
- # Note that the id is made up (the id for pydevd is unique only within a file, so, the
- # line is used for it).
- # Also, the id is currently not used afterwards, so, we don't even keep a mapping.
- breakpoints_set.append(pydevd_schema.Breakpoint(
- verified=True, id=self._next_breakpoint_id(), line=result.translated_line, source=arguments.source).to_dict())
+ # Shouldn't get here.
+ error_msg = 'Breakpoint not validated (reason unknown -- please report as bug).'
 
- body = {'breakpoints': breakpoints_set}
- set_breakpoints_response = pydevd_base_schema.build_response(request, kwargs={'body': body})
- return NetCommand(CMD_RETURN, 0, set_breakpoints_response, is_json=True)
+ return pydevd_schema.Breakpoint(
+ verified=False, id=breakpoint_id, line=result.translated_line, message=error_msg, source=source).to_dict()
+ else:
+ return pydevd_schema.Breakpoint(
+ verified=True, id=breakpoint_id, line=result.translated_line, source=source).to_dict()
 
  def on_setexceptionbreakpoints_request(self, py_db, request):
  '''