Skip to content

Better tokentree pretty printer #39079

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

Better tokentree pretty printer #39079

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9d6b7c7
Better tokentree pretty printer
bjorn3 Jan 17, 2017
74c639b
Even better tokentree pretty printer
bjorn3 Feb 5, 2017
0380fc3
Prettier
bjorn3 Feb 6, 2017
c010b1e
Space after comma
bjorn3 Feb 10, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 147 additions & 44 deletions src/libsyntax/print/pprust.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1450,57 +1450,126 @@ impl<'a> State<'a> {
}
}

/// This doesn't deserve to be called "pretty" printing, but it should be
/// meaning-preserving. A quick hack that might help would be to look at the
/// spans embedded in the TTs to decide where to put spaces and newlines.
/// But it'd be better to parse these according to the grammar of the
/// appropriate macro, transcribe back into the grammar we just parsed from,
/// and then pretty-print the resulting AST nodes (so, e.g., we print
/// expression arguments as expressions). It can be done! I think.
/// Forwards to print_tts
pub fn print_tt(&mut self, tt: &tokenstream::TokenTree) -> io::Result<()> {
match *tt {
TokenTree::Token(_, ref tk) => {
word(&mut self.s, &token_to_string(tk))?;
match *tk {
parse::token::DocComment(..) => {
hardbreak(&mut self.s)
}
_ => Ok(())
}
self.print_tts(&[tt.clone()])
}

/// This uses heuristics to be meaning-preserving while making
/// it look nicer than just printing the seperate tokens
pub fn print_tts(&mut self, tts: &[tokenstream::TokenTree]) -> io::Result<()> {
let mut tts_iter = tts.into_iter().peekable();
while let Some(tt) = tts_iter.next() {
fn is_dot(token: &Token) -> bool {
*token == Token::Dot || *token == Token::DotDot || *token == Token::DotDotDot
}
TokenTree::Delimited(_, ref delimed) => {
word(&mut self.s, &token_to_string(&delimed.open_token()))?;
space(&mut self.s)?;
self.print_tts(&delimed.tts)?;
space(&mut self.s)?;
word(&mut self.s, &token_to_string(&delimed.close_token()))
},
TokenTree::Sequence(_, ref seq) => {
word(&mut self.s, "$(")?;
for tt_elt in &seq.tts {
self.print_tt(tt_elt)?;

match *tt {
TokenTree::Token(_, ref token) => {
if *token == Token::Semi {
word(&mut self.s, ";")?;
hardbreak(&mut self.s)?;
} else {
word(&mut self.s, &token_to_string(token))?;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sad. I guess it's pre-existing, though.

match (token, tts_iter.peek()) {
(_, None) => {} // {abc}
// ^^^
(&parse::token::DocComment(..), _) => hardbreak(&mut self.s)?, // ///abc
// ^^^---
(&Token::Comma, _) => {} // abc(a, b);
// ^-
(_, Some(&&TokenTree::Token(_, Token::Comma))) => { // abc(a, b);
space(&mut self.s, 1)? // ^ -
},
(_, Some(&&TokenTree::Token(_, Token::Semi))) => {} // let a = 0;
// ^-
(ref a, Some(&&TokenTree::Token(_, ref b))) if is_dot(a) &&
!is_dot(b) => {} // ..a
// ^^-
(ref a, Some(&&TokenTree::Token(_, ref b))) if is_dot(a) &&
is_dot(b) => { // ... ..
self.nbsp()? // ^^^ --
}
(&Token::Ident(_), Some(&&TokenTree::Token(_, Token::Not))) => {
// abc!
// ^^^-
}
(&Token::Literal(_, _), Some(&&TokenTree::Token(_, ref a)))
// self.0 .0
// ^ -
if is_dot(a) => {
self.nbsp()?
}
(_, Some(&&TokenTree::Delimited(_, ref delimed)))
if delimed.delim ==
parse::token::DelimToken::Paren => {} // abc()
// ^^^--
(_, Some(&&TokenTree::Token(_, Token::Dot))) => {} // a.
// ^-
_ => self.nbsp()?,
}
}
}
word(&mut self.s, ")")?;
if let Some(ref tk) = seq.separator {
word(&mut self.s, &token_to_string(tk))?;
TokenTree::Delimited(_, ref delimed) => {
if delimed.delim == parse::token::DelimToken::Brace {
if delimed.tts.is_empty() { // {}
// ++
word(&mut self.s, "{}")?;
zerobreak(&mut self.s)?;
} else {
word(&mut self.s, "{")?;
hardbreak(&mut self.s)?;
self.cbox(4)?;
space(&mut self.s)?;

self.print_tts(&delimed.tts)?;

self.end()?;
hardbreak(&mut self.s)?;
word(&mut self.s, "}")?;

match tts_iter.peek(){
None => {},
Some(&&TokenTree::Token(_, Token::Semi)) => { // {abc};
// ^^^^^-
},
_ => {
space(&mut self.s)?;
hardbreak(&mut self.s)?;
}
}
}
} else {
self.ibox(0)?;

word(&mut self.s, &token_to_string(&delimed.open_token()))?;
self.print_tts(&delimed.tts)?;
word(&mut self.s, &token_to_string(&delimed.close_token()))?;
if let Some(&&TokenTree::Token(_, Token::Semi)) = tts_iter.peek() {
} else {
space(&mut self.s)?;
}

self.end()?;
}
}
match seq.op {
tokenstream::KleeneOp::ZeroOrMore => word(&mut self.s, "*"),
tokenstream::KleeneOp::OneOrMore => word(&mut self.s, "+"),
TokenTree::Sequence(_, ref seq) => {
word(&mut self.s, "$(")?;
space(&mut self.s)?;
self.print_tts(&seq.tts)?;
space(&mut self.s)?;
word(&mut self.s, ")")?;
if let Some(ref tk) = seq.separator {
word(&mut self.s, &token_to_string(tk))?;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there no way to print a single token? I guess the same is a problem above.

}
match seq.op {
tokenstream::KleeneOp::ZeroOrMore => self.word_nbsp("*")?,
tokenstream::KleeneOp::OneOrMore => self.word_nbsp("+")?,
}
}
}
}
}

pub fn print_tts(&mut self, tts: &[tokenstream::TokenTree]) -> io::Result<()> {
self.ibox(0)?;
for (i, tt) in tts.iter().enumerate() {
if i != 0 {
space(&mut self.s)?;
}
self.print_tt(tt)?;
}
self.end()
Ok(())
}

pub fn print_variant(&mut self, v: &ast::Variant) -> io::Result<()> {
Expand Down Expand Up @@ -3116,4 +3185,38 @@ mod tests {
let varstr = variant_to_string(&var);
assert_eq!(varstr, "principal_skinner");
}

#[test]
fn test_pretty_print_tokentrees() {
use parse::parse_tts_from_source_str as parse_tts;
let original = r#"fn main() {

let program = "+ + * - /";
let mut accumulator = 0;
. .. ...;
a.b;
.. .a;
a. ..;
for token in program.chars() {

match token {

'+' => accumulator += 1
, '-' => accumulator -= 1
, '*' => accumulator *= 2
, '/' => accumulator /= 2
, _ => {}

}
}

println!("The program \"{}\" calculates the value {}", program
, accumulator);

}"#;
let sess = ParseSess::new();
let tts = parse_tts("<test>".to_string(), original.to_string(), &sess).unwrap();
let pretty = tts_to_string(&*tts);
assert_eq!(original, &*pretty);
}
}
3 changes: 3 additions & 0 deletions src/test/run-fail/while-panic.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// ignore-pretty
// pretty printer wants to add a newline after line 21 which tidy dislikes

#![allow(while_true)]

// error-pattern:giraffe
Expand Down
2 changes: 1 addition & 1 deletion src/test/run-make/trace-macros-flag/hello.trace
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
println! { "Hello, World!" }
print! { concat ! ( "Hello, World!" , "\n" ) }
print! { concat!("Hello, World!" , "\n") }
4 changes: 2 additions & 2 deletions src/test/run-pass-fulldeps/proc-macro/auxiliary/attr-args.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ use proc_macro::TokenStream;
pub fn attr_with_args(args: TokenStream, input: TokenStream) -> TokenStream {
let args = args.to_string();

assert_eq!(args, r#"( text = "Hello, world!" )"#);
assert_eq!(args, r#"(text = "Hello, world!")"#);

let input = input.to_string();

assert_eq!(input, "fn foo ( ) { }");
assert_eq!(input, "fn foo() {}");

r#"
fn foo() -> &'static str { "Hello, world!" }
Expand Down
4 changes: 2 additions & 2 deletions src/test/run-pass/syntax-extension-source-utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ pub fn main() {
assert_eq!(column!(), 4);
assert_eq!(indirect_line!(), 26);
assert!((file!().ends_with("syntax-extension-source-utils.rs")));
assert_eq!(stringify!((2*3) + 5).to_string(), "( 2 * 3 ) + 5".to_string());
assert_eq!(stringify!((2*3) + 5).to_string(), "(2 * 3) + 5".to_string());
assert!(include!("syntax-extension-source-utils-files/includeme.\
fragment").to_string()
== "victory robot 6".to_string());
Expand All @@ -40,5 +40,5 @@ pub fn main() {
// The Windows tests are wrapped in an extra module for some reason
assert!((m1::m2::where_am_i().ends_with("m1::m2")));

assert_eq!((43, "( 2 * 3 ) + 5"), (line!(), stringify!((2*3) + 5)));
assert_eq!((43, "(2 * 3) + 5"), (line!(), stringify!((2*3) + 5)));
}