Relax Pygments parsing on lexing failures

CHANGES

@@ -66,6 +66,7 @@ Bugs fixed
 * #11546: Translated nodes identical to their original text are now marked
   with the ``translated=True`` attribute.
 * #10049: html: Change "Permalink" to "Link" for title text in link anchors.
+* #4225: Relax Pygments parsing on lexing failures.
 
 Testing
 -------

sphinx/highlighting.py

@@ -164,17 +164,23 @@ def highlight_block(self, source: str, lang: str, opts: dict | None = None,
         formatter = self.get_formatter(**kwargs)
         try:
             hlsource = highlight(source, lexer, formatter)
-        except ErrorToken:
+        except ErrorToken as err:
             # this is most probably not the selected language,
-            # so let it pass unhighlighted
+            # so let it pass un highlighted
             if lang == 'default':
-                pass  # automatic highlighting failed.
+                lang = 'none'  # automatic highlighting failed.
             else:
-                logger.warning(__('Could not lex literal_block %r as "%s". '
-                                  'Highlighting skipped.'), source, lang,
-                               type='misc', subtype='highlighting_failure',
-                               location=location)
-            lexer = self.get_lexer(source, 'none', opts, force, location)
+                logger.warning(
+                    __('Lexing literal_block %r as "%s" resulted in an error at token: %r. '
+                       'Retrying in relaxed mode.'),
+                    source, lang, str(err),
+                    type='misc', subtype='highlighting_failure',
+                    location=location)
+                if force:
+                    lang = 'none'
+                else:
+                    force = True
+            lexer = self.get_lexer(source, lang, opts, force, location)
             hlsource = highlight(source, lexer, formatter)
 
         if self.dest == 'html':

tests/test_highlighting.py

@@ -97,8 +97,8 @@ def test_default_highlight(logger):
 
     # python: raises error if highlighting failed
     ret = bridge.highlight_block('reST ``like`` text', 'python')
-    logger.warning.assert_called_with('Could not lex literal_block %r as "%s". '
-                                      'Highlighting skipped.',
-                                      'reST ``like`` text', 'python',
+    logger.warning.assert_called_with('Lexing literal_block %r as "%s" resulted in an error at token: %r. '
+                                      'Retrying in relaxed mode.',
+                                      'reST ``like`` text', 'python', '`',
                                       type='misc', subtype='highlighting_failure',
                                       location=None)