Skip to content

Commit

Permalink
adapt rustdoc to infailable lexer
Browse files Browse the repository at this point in the history
  • Loading branch information
@matklad
matklad committed Aug 5, 2019
1 parent 58ac81a commit b3e8c8b
Show file tree
Hide file tree
Showing 3 changed files with 177 additions and 50 deletions.
27 changes: 14 additions & 13 deletions src/librustdoc/html/highlight.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ pub fn render_with_highlighting(

let mut highlighted_source = vec![];
if classifier.write_source(&mut highlighted_source).is_err() {
Err(classifier.lexer.buffer_fatal_errors())
Err(())
} else {
Ok(String::from_utf8_lossy(&highlighted_source).into_owned())
}
Expand All @@ -59,14 +59,9 @@ pub fn render_with_highlighting(
}
write_footer(&mut out).unwrap();
}
Err(errors) => {
// If errors are encountered while trying to highlight, cancel the errors and just emit
// the unhighlighted source. The errors will have already been reported in the
// `check-code-block-syntax` pass.
for mut error in errors {
error.cancel();
}

Err(()) => {
// If errors are encountered while trying to highlight, just emit
// the unhighlighted source.
write!(out, "<pre><code>{}</code></pre>", src).unwrap();
}
}
Expand Down Expand Up @@ -192,14 +187,20 @@ impl<'a> Classifier<'a> {
if let Some(token) = self.peek_token.take() {
return Ok(token);
}
self.lexer.try_next_token().map_err(|()| HighlightError::LexError)
let token = self.lexer.next_token();
if let token::Unknown(..) = &token.kind {
return Err(HighlightError::LexError);
}
Ok(token)
}

fn peek(&mut self) -> Result<&Token, HighlightError> {
if self.peek_token.is_none() {
self.peek_token = Some(
self.lexer.try_next_token().map_err(|()| HighlightError::LexError)?
);
let token = self.lexer.next_token();
if let token::Unknown(..) = &token.kind {
return Err(HighlightError::LexError);
}
self.peek_token = Some(token);
}
Ok(self.peek_token.as_ref().unwrap())
}
Expand Down
32 changes: 9 additions & 23 deletions src/librustdoc/passes/check_code_block_syntax.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,24 +32,20 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
dox[code_block.code].to_owned(),
);

let errors = {
let has_errors = {
let mut has_errors = false;
let mut lexer = Lexer::new(&sess, source_file, None);
while let Ok(token::Token { kind, .. }) = lexer.try_next_token() {
if kind == token::Eof {
break;
loop {
match lexer.next_token().kind {
token::Eof => break,
token::Unknown(..) => has_errors = true,
_ => (),
}
}

let errors = lexer.buffer_fatal_errors();

if !errors.is_empty() {
Err(errors)
} else {
Ok(())
}
has_errors
};

if let Err(errors) = errors {
if has_errors {
let mut diag = if let Some(sp) =
super::source_span_for_markdown_range(self.cx, &dox, &code_block.range, &item.attrs)
{
Expand All @@ -58,11 +54,6 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
.sess()
.struct_span_warn(sp, "could not parse code block as Rust code");

for mut err in errors {
diag.note(&format!("error from rustc: {}", err.message()));
err.cancel();
}

if code_block.syntax.is_none() && code_block.is_fenced {
let sp = sp.from_inner(InnerSpan::new(0, 3));
diag.span_suggestion(
Expand All @@ -82,11 +73,6 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
"doc comment contains an invalid Rust code block",
);

for mut err in errors {
// Don't bother reporting the error, because we can't show where it happened.
err.cancel();
}

if code_block.syntax.is_none() && code_block.is_fenced {
diag.help("mark blocks that do not contain Rust code as text: ```text");
}
Expand Down
168 changes: 154 additions & 14 deletions src/test/rustdoc-ui/invalid-syntax.stderr
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,21 @@
error: unknown start of token: \
--> <doctest>:1:1
|
1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/
| ^

error: unknown start of token: \
--> <doctest>:1:43
|
1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/
| ^

error: unknown start of token: \
--> <doctest>:1:60
|
1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/
| ^

warning: could not parse code block as Rust code
--> $DIR/invalid-syntax.rs:3:5
|
Expand All @@ -6,13 +24,31 @@ LL | /// ```
LL | | /// \__________pkt->size___________/ \_result->size_/ \__pkt->size__/
LL | | /// ```
| |_______^
|
= note: error from rustc: unknown start of token: \
help: mark blocks that do not contain Rust code as text
|
LL | /// ```text
| ^^^^^^^

error: unknown start of token: `
--> <doctest>:3:30
|
3 | | ^^^^^^ did you mean `baz::foobar`?
| ^
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
|
3 | | ^^^^^^ did you mean 'baz::foobar`?
| ^

error: unknown start of token: `
--> <doctest>:3:42
|
3 | | ^^^^^^ did you mean `baz::foobar`?
| ^
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
|
3 | | ^^^^^^ did you mean `baz::foobar'?
| ^

warning: could not parse code block as Rust code
--> $DIR/invalid-syntax.rs:8:5
|
Expand All @@ -23,13 +59,17 @@ LL | | /// LL | use foobar::Baz;
LL | | /// | ^^^^^^ did you mean `baz::foobar`?
LL | | /// ```
| |_______^
|
= note: error from rustc: unknown start of token: `
help: mark blocks that do not contain Rust code as text
|
LL | /// ```text
| ^^^^^^^

error: unknown start of token: \
--> <doctest>:1:1
|
1 | \_
| ^

warning: could not parse code block as Rust code
--> $DIR/invalid-syntax.rs:19:5
|
Expand All @@ -38,13 +78,17 @@ LL | /// ```
LL | | /// \_
LL | | /// ```
| |_______^
|
= note: error from rustc: unknown start of token: \
help: mark blocks that do not contain Rust code as text
|
LL | /// ```text
| ^^^^^^^

error: unknown start of token: \
--> <doctest>:1:1
|
1 | \_
| ^

warning: could not parse code block as Rust code
--> $DIR/invalid-syntax.rs:32:5
|
Expand All @@ -53,8 +97,12 @@ LL | /// ```rust
LL | | /// \_
LL | | /// ```
| |_______^
|
= note: error from rustc: unknown start of token: \

error: unknown start of token: \
--> <doctest>:2:5
|
2 | \_
| ^

warning: could not parse code block as Rust code
--> $DIR/invalid-syntax.rs:41:9
Expand All @@ -63,16 +111,48 @@ LL | /// code with bad syntax
| _________^
LL | | /// \_
| |__________^
|
= note: error from rustc: unknown start of token: \

error: unknown start of token: `
--> <doctest>:1:1
|
1 | ```
| ^
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
|
1 | '``
| ^

error: unknown start of token: `
--> <doctest>:1:2
|
1 | ```
| ^
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
|
1 | `'`
| ^

error: unknown start of token: `
--> <doctest>:1:3
|
1 | ```
| ^
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
|
1 | ``'
| ^

warning: could not parse code block as Rust code
--> $DIR/invalid-syntax.rs:55:9
|
LL | /// ```
| ^^^
|
= note: error from rustc: unknown start of token: `

error: unknown start of token: \
--> <doctest>:1:1
|
1 | \_
| ^

warning: could not parse code block as Rust code
--> $DIR/invalid-syntax.rs:58:5
Expand All @@ -82,8 +162,12 @@ LL | /// ```edition2018
LL | | /// \_
LL | | /// ```
| |_______^
|
= note: error from rustc: unknown start of token: \

error: unknown start of token: \
--> <doctest>:1:1
|
1 | \_
| ^

warning: doc comment contains an invalid Rust code block
--> $DIR/invalid-syntax.rs:63:1
Expand All @@ -95,3 +179,59 @@ LL | | #[doc = "```"]
|
= help: mark blocks that do not contain Rust code as text: ```text

error: unknown start of token: \
--> <rustdoc-highlighting>:1:1
|
1 | \_
| ^

error: unknown start of token: \
--> <rustdoc-highlighting>:1:1
|
1 | \_
| ^

error: unknown start of token: `
--> <rustdoc-highlighting>:1:1
|
1 | ```
| ^
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
|
1 | '``
| ^

error: unknown start of token: \
--> <rustdoc-highlighting>:2:1
|
2 | \_
| ^

error: unknown start of token: \
--> <rustdoc-highlighting>:1:1
|
1 | \_
| ^

error: unknown start of token: \
--> <rustdoc-highlighting>:1:1
|
1 | \_
| ^

error: unknown start of token: `
--> <rustdoc-highlighting>:3:30
|
3 | | ^^^^^^ did you mean `baz::foobar`?
| ^
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
|
3 | | ^^^^^^ did you mean 'baz::foobar`?
| ^

error: unknown start of token: \
--> <rustdoc-highlighting>:1:1
|
1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/
| ^

0 comments on commit b3e8c8b

Please sign in to comment.