Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merge idents when generating source content #83992

Merged
merged 2 commits into from
Apr 8, 2021
Merged

Merge idents when generating source content #83992

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2ee97bd
Merge idents if they are part of a path
GuillaumeGomez Apr 8, 2021
e2708b4
Add test for idents merge
GuillaumeGomez Apr 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 107 additions & 18 deletions src/librustdoc/html/highlight.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,16 @@ impl Iterator for TokenIter<'a> {
}
}

fn get_real_ident_class(text: &str, edition: Edition) -> Class {
match text {
"ref" | "mut" => Class::RefKeyWord,
"self" | "Self" => Class::Self_,
"false" | "true" => Class::Bool,
_ if Symbol::intern(text).is_reserved(|| edition) => Class::KeyWord,
_ => Class::Ident,
}
}

/// Processes program tokens, classifying strings of text by highlighting
/// category (`Class`).
struct Classifier<'a> {
Expand All @@ -144,6 +154,8 @@ struct Classifier<'a> {
in_macro: bool,
in_macro_nonterminal: bool,
edition: Edition,
byte_pos: u32,
src: &'a str,
}

impl<'a> Classifier<'a> {
Expand All @@ -155,6 +167,68 @@ impl<'a> Classifier<'a> {
in_macro: false,
in_macro_nonterminal: false,
edition,
byte_pos: 0,
src,
}
}

/// Concatenate colons and idents as one when possible.
fn get_full_ident_path(&mut self) -> Vec<(TokenKind, usize, usize)> {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about ArrayVec? librustdoc already depends on it, and these Vecs have exactly zero, one, or two items in them.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good idea!

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually just gave it a try, no difference and it's actually not great in case we have less than 2 elements. Keeping it as is for the moment. If you want to give it another try later on, don't hesitate!

let start = self.byte_pos as usize;
let mut pos = start;
let mut has_ident = false;
let edition = self.edition;

loop {
let mut nb = 0;
while let Some((TokenKind::Colon, _)) = self.tokens.peek() {
self.tokens.next();
nb += 1;
}
// Ident path can start with "::" but if we already have content in the ident path,
// the "::" is mandatory.
if has_ident && nb == 0 {
return vec![(TokenKind::Ident, start, pos)];
} else if nb != 0 && nb != 2 {
if has_ident {
return vec![(TokenKind::Ident, start, pos), (TokenKind::Colon, pos, pos + nb)];
} else {
return vec![(TokenKind::Colon, pos, pos + nb)];
}
}

if let Some((Class::Ident, text)) = self.tokens.peek().map(|(token, text)| {
if *token == TokenKind::Ident {
let class = get_real_ident_class(text, edition);
(class, text)
} else {
// Doesn't matter which Class we put in here...
(Class::Comment, text)
}
}) {
// We only "add" the colon if there is an ident behind.
pos += text.len() + nb;
has_ident = true;
self.tokens.next();
} else if nb > 0 && has_ident {
return vec![(TokenKind::Ident, start, pos), (TokenKind::Colon, pos, pos + nb)];
} else if nb > 0 {
return vec![(TokenKind::Colon, pos, pos + nb)];
} else if has_ident {
return vec![(TokenKind::Ident, start, pos)];
} else {
return Vec::new();
}
}
}

/// Wraps the tokens iteration to ensure that the byte_pos is always correct.
fn next(&mut self) -> Option<(TokenKind, &'a str)> {
if let Some((kind, text)) = self.tokens.next() {
self.byte_pos += text.len() as u32;
Some((kind, text))
} else {
None
}
}

Expand All @@ -165,8 +239,25 @@ impl<'a> Classifier<'a> {
/// token is used.
fn highlight(mut self, sink: &mut dyn FnMut(Highlight<'a>)) {
with_default_session_globals(|| {
while let Some((token, text)) = self.tokens.next() {
self.advance(token, text, sink);
loop {
if self
.tokens
.peek()
.map(|t| matches!(t.0, TokenKind::Colon | TokenKind::Ident))
.unwrap_or(false)
{
let tokens = self.get_full_ident_path();
for (token, start, end) in tokens {
let text = &self.src[start..end];
self.advance(token, text, sink);
self.byte_pos += text.len() as u32;
}
}
if let Some((token, text)) = self.next() {
self.advance(token, text, sink);
} else {
break;
}
}
})
}
Expand Down Expand Up @@ -203,12 +294,12 @@ impl<'a> Classifier<'a> {
},
TokenKind::And => match lookahead {
Some(TokenKind::And) => {
let _and = self.tokens.next();
self.next();
sink(Highlight::Token { text: "&&", class: Some(Class::Op) });
return;
}
Some(TokenKind::Eq) => {
let _eq = self.tokens.next();
self.next();
sink(Highlight::Token { text: "&=", class: Some(Class::Op) });
return;
}
Expand Down Expand Up @@ -260,7 +351,7 @@ impl<'a> Classifier<'a> {
match lookahead {
// Case 1: #![inner_attribute]
Some(TokenKind::Bang) => {
let _not = self.tokens.next().unwrap();
self.next();
if let Some(TokenKind::OpenBracket) = self.peek() {
self.in_attribute = true;
sink(Highlight::EnterSpan { class: Class::Attribute });
Expand Down Expand Up @@ -304,19 +395,17 @@ impl<'a> Classifier<'a> {
sink(Highlight::Token { text, class: None });
return;
}
TokenKind::Ident => match text {
"ref" | "mut" => Class::RefKeyWord,
"self" | "Self" => Class::Self_,
"false" | "true" => Class::Bool,
"Option" | "Result" => Class::PreludeTy,
"Some" | "None" | "Ok" | "Err" => Class::PreludeVal,
// Keywords are also included in the identifier set.
_ if Symbol::intern(text).is_reserved(|| self.edition) => Class::KeyWord,
_ if self.in_macro_nonterminal => {
self.in_macro_nonterminal = false;
Class::MacroNonTerminal
}
_ => Class::Ident,
TokenKind::Ident => match get_real_ident_class(text, self.edition) {
Class::Ident => match text {
"Option" | "Result" => Class::PreludeTy,
"Some" | "None" | "Ok" | "Err" => Class::PreludeVal,
_ if self.in_macro_nonterminal => {
self.in_macro_nonterminal = false;
Class::MacroNonTerminal
}
_ => Class::Ident,
},
c => c,
},
TokenKind::RawIdent => Class::Ident,
TokenKind::Lifetime { .. } => Class::Lifetime,
Expand Down
10 changes: 10 additions & 0 deletions src/librustdoc/html/highlight/fixtures/sample.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
</style>
<pre><code><span class="attribute">#![<span class="ident">crate_type</span> <span class="op">=</span> <span class="string">&quot;lib&quot;</span>]</span>

<span class="kw">use</span> <span class="ident">std::path</span>::{<span class="ident">Path</span>, <span class="ident">PathBuf</span>};

<span class="attribute">#[<span class="ident">cfg</span>(<span class="ident">target_os</span> <span class="op">=</span> <span class="string">&quot;linux&quot;</span>)]</span>
<span class="kw">fn</span> <span class="ident">main</span>() {
<span class="kw">let</span> <span class="ident">foo</span> <span class="op">=</span> <span class="bool-val">true</span> <span class="op">&amp;&amp;</span> <span class="bool-val">false</span> <span class="op">|</span><span class="op">|</span> <span class="bool-val">true</span>;
Expand All @@ -19,6 +21,14 @@
<span class="kw">let</span> <span class="kw">_</span> <span class="op">=</span> <span class="kw-2">*</span><span class="ident">foo</span>;
<span class="macro">mac!</span>(<span class="ident">foo</span>, <span class="kw-2">&amp;</span><span class="kw-2">mut</span> <span class="ident">bar</span>);
<span class="macro">assert!</span>(<span class="self">self</span>.<span class="ident">length</span> <span class="op">&lt;</span> <span class="ident">N</span> <span class="op">&amp;&amp;</span> <span class="ident">index</span> <span class="op">&lt;</span><span class="op">=</span> <span class="self">self</span>.<span class="ident">length</span>);
<span class="ident">::std::env::var</span>(<span class="string">&quot;gateau&quot;</span>).<span class="ident">is_ok</span>();
<span class="attribute">#[<span class="ident">rustfmt::skip</span>]</span>
<span class="kw">let</span> <span class="ident">s</span>:<span class="ident">std</span><span class="ident">::path::PathBuf</span> <span class="op">=</span> <span class="ident">std::path::PathBuf::new</span>();
<span class="kw">let</span> <span class="kw-2">mut</span> <span class="ident">s</span> <span class="op">=</span> <span class="ident">String::new</span>();

<span class="kw">match</span> <span class="kw-2">&amp;</span><span class="ident">s</span> {
<span class="kw-2">ref</span> <span class="kw-2">mut</span> <span class="ident">x</span> <span class="op">=</span><span class="op">&gt;</span> {}
}
}

<span class="macro">macro_rules!</span> <span class="ident">bar</span> {
Expand Down
10 changes: 10 additions & 0 deletions src/librustdoc/html/highlight/fixtures/sample.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#![crate_type = "lib"]

use std::path::{Path, PathBuf};

#[cfg(target_os = "linux")]
fn main() {
let foo = true && false || true;
Expand All @@ -9,6 +11,14 @@ fn main() {
let _ = *foo;
mac!(foo, &mut bar);
assert!(self.length < N && index <= self.length);
::std::env::var("gateau").is_ok();
#[rustfmt::skip]
let s:std::path::PathBuf = std::path::PathBuf::new();
let mut s = String::new();

match &s {
ref mut x => {}
}
}

macro_rules! bar {
Expand Down