sourceReference should be 0 for jupyter cells. Fixes microsoft#113

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py

@@ -1223,7 +1223,7 @@ def build_exception_info_response(dbg, thread_id, request_seq, set_additional_th
                 exc_type = frames_list.exc_type
                 exc_desc = frames_list.exc_desc
                 trace_obj = frames_list.trace_obj
-                for frame_id, frame, method_name, original_filename, filename_in_utf8, lineno in iter_visible_frames_info(
+                for frame_id, frame, method_name, original_filename, filename_in_utf8, lineno, _applied_mapping in iter_visible_frames_info(
                         dbg, frames_list):
 
                     line_text = linecache.getline(original_filename, lineno)

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_net_command_factory_json.py

@@ -14,7 +14,7 @@
     CMD_STEP_RETURN, CMD_STEP_CAUGHT_EXCEPTION, CMD_ADD_EXCEPTION_BREAK, CMD_SET_BREAK, \
     CMD_SET_NEXT_STATEMENT, CMD_THREAD_SUSPEND_SINGLE_NOTIFICATION, \
     CMD_THREAD_RESUME_SINGLE_NOTIFICATION, CMD_THREAD_KILL, CMD_STOP_ON_START, CMD_INPUT_REQUESTED, \
-    CMD_EXIT, CMD_STEP_INTO_COROUTINE, constant_to_str, CMD_STEP_RETURN_MY_CODE
+    CMD_EXIT, CMD_STEP_INTO_COROUTINE, CMD_STEP_RETURN_MY_CODE
 from _pydevd_bundle.pydevd_constants import get_thread_id, dict_values, ForkSafeLock
 from _pydevd_bundle.pydevd_net_command import NetCommand, NULL_NET_COMMAND
 from _pydevd_bundle.pydevd_net_command_factory_xml import NetCommandFactory
@@ -23,7 +23,6 @@
 from _pydevd_bundle.pydevd_comm import build_exception_info_response, pydevd_find_thread_by_id
 from _pydevd_bundle.pydevd_additional_thread_info import set_additional_thread_info
 from _pydevd_bundle import pydevd_frame_utils
-from _pydev_bundle import pydev_log
 import linecache
 
 
@@ -210,7 +209,7 @@ def make_get_thread_stack_message(self, py_db, seq, thread_id, topmost_frame, fm
                 else:
                     frames_list = pydevd_frame_utils.create_frames_list_from_frame(topmost_frame)
 
-            for frame_id, frame, method_name, original_filename, filename_in_utf8, lineno in self._iter_visible_frames_info(
+            for frame_id, frame, method_name, original_filename, filename_in_utf8, lineno, applied_mapping in self._iter_visible_frames_info(
                     py_db, frames_list
                 ):
 
@@ -232,17 +231,17 @@ def make_get_thread_stack_message(self, py_db, seq, thread_id, topmost_frame, fm
                 formatted_name = self._format_frame_name(fmt, method_name, module_name, lineno, filename_in_utf8)
                 source_reference = pydevd_file_utils.get_client_filename_source_reference(filename_in_utf8)
 
-                if not source_reference:
-                    # Check if someone added a source reference to the linecache (Python attrs does this).
-                    if linecache.getline(original_filename, 0):
-                        source_reference = pydevd_file_utils.create_source_reference_for_linecache(
-                            original_filename, filename_in_utf8)
-
-                    if not source_reference:
-                        if getattr(frame.f_code, 'co_lnotab', None):
-                            if not os.path.exists(original_filename):
-                                # Create a source-reference to be used where we provide the source by decompiling the code.
-                                source_reference = pydevd_file_utils.create_source_reference_for_frame_id(frame_id)
+                if not source_reference and not applied_mapping and not os.path.exists(original_filename):
+                    if getattr(frame.f_code, 'co_lnotab', None):
+                        # Create a source-reference to be used where we provide the source by decompiling the code.
+                        # Note: When the time comes to retrieve the source reference in this case, we'll
+                        # check the linecache first (see: get_decompiled_source_from_frame_id).
+                        source_reference = pydevd_file_utils.create_source_reference_for_frame_id(frame_id)
+                    else:
+                        # Check if someone added a source reference to the linecache (Python attrs does this).
+                        if linecache.getline(original_filename, 1):
+                            source_reference = pydevd_file_utils.create_source_reference_for_linecache(
+                                original_filename)
 
                 frames.append(pydevd_schema.StackFrame(
                     frame_id, formatted_name, lineno, column=1, source={

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_net_command_factory_xml.py

@@ -172,9 +172,10 @@ def _iter_visible_frames_info(self, py_db, frames_list):
             lineno = frames_list.frame_id_to_lineno.get(frame_id, frame.f_lineno)
 
             filename_in_utf8, lineno, changed = py_db.source_mapping.map_to_client(abs_path_real_path_and_base[0], lineno)
-            filename_in_utf8 = pydevd_file_utils.norm_file_to_client(filename_in_utf8)
+            new_filename_in_utf8, applied_mapping = pydevd_file_utils.norm_file_to_client(filename_in_utf8)
+            applied_mapping = applied_mapping or changed
 
-            yield frame_id, frame, method_name, abs_path_real_path_and_base[0], filename_in_utf8, lineno
+            yield frame_id, frame, method_name, abs_path_real_path_and_base[0], new_filename_in_utf8, lineno, applied_mapping
 
     def make_thread_stack_str(self, py_db, frames_list):
         assert frames_list.__class__ == FramesList
@@ -183,7 +184,7 @@ def make_thread_stack_str(self, py_db, frames_list):
         append = cmd_text_list.append
 
         try:
-            for frame_id, frame, method_name, _original_filename, filename_in_utf8, lineno in self._iter_visible_frames_info(
+            for frame_id, frame, method_name, _original_filename, filename_in_utf8, lineno, _applied_mapping in self._iter_visible_frames_info(
                     py_db, frames_list
                 ):
 

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_source_mapping.py

@@ -75,15 +75,17 @@ def set_source_mapping(self, source_filename, mapping):
                 return 'Cannot apply mapping from %s to %s (it conflicts with mapping: %s to %s)' % (
                     source_filename, map_entry.runtime_source, existing_source_filename, map_entry.runtime_source)
 
-        current_mapping = self._mappings_to_server.get(source_filename, [])
-        for map_entry in current_mapping:
-            del self._mappings_to_client[map_entry.runtime_source]
-
-        self._mappings_to_server[source_filename] = sorted(mapping, key=lambda entry:entry.line)
+        try:
+            current_mapping = self._mappings_to_server.get(source_filename, [])
+            for map_entry in current_mapping:
+                del self._mappings_to_client[map_entry.runtime_source]
 
-        for map_entry in mapping:
-            self._mappings_to_client[map_entry.runtime_source] = source_filename
+            self._mappings_to_server[source_filename] = sorted(mapping, key=lambda entry:entry.line)
 
+            for map_entry in mapping:
+                self._mappings_to_client[map_entry.runtime_source] = source_filename
+        finally:
+            self._cache.clear()
         return ''
 
     def map_to_client(self, filename, lineno):
@@ -96,9 +98,11 @@ def map_to_client(self, filename, lineno):
                 for map_entry in mapping:
                     if map_entry.runtime_source == filename:
                         if map_entry.contains_runtime_line(lineno):
-                            return source_filename, map_entry.line + (lineno - map_entry.runtime_line), True
+                            self._cache[key] = (source_filename, map_entry.line + (lineno - map_entry.runtime_line), True)
+                            return self._cache[key]
 
-            return filename, lineno, False
+            self._cache[key] = (filename, lineno, False)
+            return self._cache[key]
 
     def map_to_server(self, filename, lineno):
         # Note: the filename must be already normalized to the server at this point.

src/debugpy/_vendored/pydevd/pydevd_concurrency_analyser/pydevd_concurrency_logger.py

@@ -52,10 +52,10 @@ def get_text_list_for_frame(frame):
 
             filename = pydevd_file_utils.get_abs_path_real_path_and_base_from_frame(curFrame)[1]
 
-            myFile = pydevd_file_utils.norm_file_to_client(filename)
+            my_file, _applied_mapping = pydevd_file_utils.norm_file_to_client(filename)
 
-            # print "file is ", myFile
-            # myFile = inspect.getsourcefile(curFrame) or inspect.getfile(frame)
+            # print "file is ", my_file
+            # my_file = inspect.getsourcefile(curFrame) or inspect.getfile(frame)
 
             myLine = str(curFrame.f_lineno)
             # print "line is ", myLine
@@ -65,7 +65,7 @@ def get_text_list_for_frame(frame):
 
             variables = ''
             cmdTextList.append('<frame id="%s" name="%s" ' % (myId , pydevd_xml.make_valid_xml_value(myName)))
-            cmdTextList.append('file="%s" line="%s">' % (quote(myFile, '/>_= \t'), myLine))
+            cmdTextList.append('file="%s" line="%s">' % (quote(my_file, '/>_= \t'), myLine))
             cmdTextList.append(variables)
             cmdTextList.append("</frame>")
             curFrame = curFrame.f_back

src/debugpy/_vendored/pydevd/pydevd_file_utils.py

@@ -529,7 +529,7 @@ def _original_file_to_client(filename, cache={}):
         return cache[filename]
     except KeyError:
         translated = _path_to_expected_str(get_path_with_real_case(_AbsFile(filename)))
-        cache[filename] = translated
+        cache[filename] = (translated, False)
     return cache[filename]
 
 
@@ -726,7 +726,7 @@ def _norm_file_to_client(filename, cache=norm_filename_to_client_container):
 
             # The resulting path is not in the python process, so, we cannot do a _NormFile here,
             # only at the beginning of this method.
-            cache[filename] = translated
+            cache[filename] = (translated, path_mapping_applied)
 
             if translated not in _client_filename_in_utf8_to_source_reference:
                 if path_mapping_applied:
@@ -736,7 +736,7 @@ def _norm_file_to_client(filename, cache=norm_filename_to_client_container):
                 _client_filename_in_utf8_to_source_reference[translated] = source_reference
                 _source_reference_to_server_filename[source_reference] = filename
 
-            return translated
+            return (translated, path_mapping_applied)
 
     norm_file_to_server = _norm_file_to_server
     norm_file_to_client = _norm_file_to_client

src/debugpy/_vendored/pydevd/tests/test_pydevconsole.py

@@ -98,10 +98,10 @@ def test_console_requests(self):
                 if c[0] == 'RuntimeError':
                     self.fail('Did not expect to find RuntimeError there')
 
-            self.assertTrue(('__doc__', None, '', '3') not in interpreter.getCompletions('foo.CO', 'foo.'))
+            assert ('__doc__', None, '', '3') not in interpreter.getCompletions('foo.CO', 'foo.')
 
             comps = interpreter.getCompletions('va', 'va')
-            self.assertTrue(('val', '', '', '3') in comps or ('val', '', '', '4') in comps)
+            assert ('val', '', '', '3') in comps or ('val', '', '', '4') in comps
 
             interpreter.add_exec(CodeFragment('s = "mystring"'))
 

src/debugpy/_vendored/pydevd/tests_python/debugger_unittest.py

@@ -749,11 +749,13 @@ def get_pydevconsole_file(self):
         dirname = os.path.dirname(dirname)
         return os.path.abspath(os.path.join(dirname, 'pydevconsole.py'))
 
-    def get_line_index_with_content(self, line_content):
+    def get_line_index_with_content(self, line_content, filename=None):
         '''
         :return the line index which has the given content (1-based).
         '''
-        with open(self.TEST_FILE, 'r') as stream:
+        if filename is None:
+            filename = self.TEST_FILE
+        with open(filename, 'r') as stream:
             for i_line, line in enumerate(stream):
                 if line_content in line:
                     return i_line + 1

src/debugpy/_vendored/pydevd/tests_python/resources/_debugger_case_linecache_existing_file.py

@@ -0,0 +1,12 @@
+import linecache
+import sys
+import os
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+import _debugger_case_stepping
+
+linecache.updatecache(_debugger_case_stepping.__file__)
+assert linecache.getline(_debugger_case_stepping.__file__, 1)
+_debugger_case_stepping.Call()
+
+print('TEST SUCEEDED')

src/debugpy/_vendored/pydevd/tests_python/resources/_debugger_case_source_mapping_and_reference.py

@@ -0,0 +1,57 @@
+def full_function():
+    # Note that this function is not called, it's there just to make the mapping explicit.
+    a = 1  # map to cell1, line 2
+    b = 2  # map to cell1, line 3
+
+    c = 3  # map to cell2, line 2
+    d = 4  # map to cell2, line 3
+
+
+def create_code():
+    cell1_code = compile(''' # line 1
+a = 1  # line 2
+b = 2  # line 3
+''', '<cell1>', 'exec')
+
+    cell2_code = compile('''# line 1
+c = 3  # line 2
+d = 4  # line 3
+''', '<cell2>', 'exec')
+
+    # Set up the source in linecache. Python doesn't have a public API for
+    # this, so we have to hack around it, similar to what IPython does.
+    import linecache
+    import time
+    code = ''' # line 1
+a = 1  # line 2
+b = 2  # line 3
+'''
+    linecache.cache['<cell1>'] = (
+        len(code),
+        time.time(),
+        [line + '\n' for line in code.splitlines()],
+        '<cell1>',
+    )
+
+    code = '''# line 1
+c = 3  # line 2
+d = 4  # line 3
+'''
+    linecache.cache['<cell2>'] = (
+        len(code),
+        time.time(),
+        [line + '\n' for line in code.splitlines()],
+        '<cell2>',
+    )
+
+    return {'cell1': cell1_code, 'cell2': cell2_code}
+
+
+if __name__ == '__main__':
+    code = create_code()
+    exec(code['cell1'])
+    exec(code['cell1'])
+
+    exec(code['cell2'])
+    exec(code['cell2'])
+    print('TEST SUCEEDED')