Auto merge of #33199 - mitaa:tokenize-responsibly, r=nrc

Make some fatal lexer errors recoverable

I've kept the changes to a minimum since I'm not really sure if this approach is a acceptable.

fixes #12834

cc @nrc

Author: bors

Diff for: src/librustdoc/html/highlight.rs

@@ -29,25 +29,27 @@ pub fn render_with_highlighting(src: &str, class: Option<&str>, id: Option<&str>
 
     let mut out = Vec::new();
     write_header(class, id, &mut out).unwrap();
-    write_source(&sess,
-                 lexer::StringReader::new(&sess.span_diagnostic, fm),
-                 &mut out).unwrap();
+    if let Err(_) = write_source(&sess,
+                                 lexer::StringReader::new(&sess.span_diagnostic, fm),
+                                 &mut out) {
+        return format!("<pre>{}</pre>", src)
+    }
     write_footer(&mut out).unwrap();
     String::from_utf8_lossy(&out[..]).into_owned()
 }
 
 /// Highlights `src`, returning the HTML output. Returns only the inner html to
 /// be inserted into an element. C.f., `render_with_highlighting` which includes
 /// an enclosing `<pre>` block.
-pub fn render_inner_with_highlighting(src: &str) -> String {
+pub fn render_inner_with_highlighting(src: &str) -> io::Result<String> {
     let sess = parse::ParseSess::new();
     let fm = sess.codemap().new_filemap("<stdin>".to_string(), src.to_string());
 
     let mut out = Vec::new();
     write_source(&sess,
                  lexer::StringReader::new(&sess.span_diagnostic, fm),
-                 &mut out).unwrap();
-    String::from_utf8_lossy(&out[..]).into_owned()
+                 &mut out)?;
+    Ok(String::from_utf8_lossy(&out[..]).into_owned())
 }
 
 /// Exhausts the `lexer` writing the output into `out`.
@@ -65,7 +67,17 @@ fn write_source(sess: &parse::ParseSess,
     let mut is_macro = false;
     let mut is_macro_nonterminal = false;
     loop {
-        let next = lexer.next_token();
+        let next = match lexer.try_next_token() {
+            Ok(tok) => tok,
+            Err(_) => {
+                lexer.emit_fatal_errors();
+                lexer.span_diagnostic.struct_warn("Backing out of syntax highlighting")
+                                     .note("You probably did not intend to render this \
+                                            as a rust code-block")
+                                     .emit();
+                return Err(io::Error::new(io::ErrorKind::Other, ""))
+            },
+        };
 
         let snip = |sp| sess.codemap().span_to_snippet(sp).unwrap();
 

Diff for: src/libsyntax/errors/mod.rs

@@ -177,6 +177,7 @@ impl error::Error for ExplicitBug {
 
 /// Used for emitting structured error messages and other diagnostic information.
 #[must_use]
+#[derive(Clone)]
 pub struct DiagnosticBuilder<'a> {
     emitter: &'a RefCell<Box<Emitter>>,
     level: Level,
@@ -187,6 +188,7 @@ pub struct DiagnosticBuilder<'a> {
 }
 
 /// For example a note attached to an error.
+#[derive(Clone)]
 struct SubDiagnostic {
     level: Level,
     message: String,

Diff for: src/libsyntax/ext/tt/transcribe.rs

@@ -12,7 +12,7 @@ use self::LockstepIterSize::*;
 use ast;
 use ast::{TokenTree, Ident, Name};
 use codemap::{Span, DUMMY_SP};
-use errors::Handler;
+use errors::{Handler, DiagnosticBuilder};
 use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
 use parse::token::{DocComment, MatchNt, SubstNt};
 use parse::token::{Token, NtIdent, SpecialMacroVar};
@@ -50,6 +50,7 @@ pub struct TtReader<'a> {
     pub cur_span: Span,
     /// Transform doc comments. Only useful in macro invocations
     pub desugar_doc_comments: bool,
+    pub fatal_errs: Vec<DiagnosticBuilder<'a>>,
 }
 
 /// This can do Macro-By-Example transcription. On the other hand, if
@@ -99,6 +100,7 @@ pub fn new_tt_reader_with_doc_flag(sp_diag: &Handler,
         /* dummy values, never read: */
         cur_tok: token::Eof,
         cur_span: DUMMY_SP,
+        fatal_errs: Vec::new(),
     };
     tt_next_token(&mut r); /* get cur_tok and cur_span set up */
     r