From 78eb516dda4ed811bfb010f7a69e8eac0fe6ce80 Mon Sep 17 00:00:00 2001 From: Sergio Benitez Date: Fri, 16 Nov 2018 23:37:23 -0800 Subject: [PATCH] Ignore non-semantic tokens for 'probably_eq' streams. --- src/libsyntax/parse/token.rs | 5 +- src/libsyntax/tokenstream.rs | 30 +++++++++-- .../proc-macro/auxiliary/span-preservation.rs | 12 +++++ .../proc-macro/span-preservation.rs | 51 +++++++++++++++++++ .../proc-macro/span-preservation.stderr | 49 ++++++++++++++++++ 5 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 src/test/ui-fulldeps/proc-macro/auxiliary/span-preservation.rs create mode 100644 src/test/ui-fulldeps/proc-macro/span-preservation.rs create mode 100644 src/test/ui-fulldeps/proc-macro/span-preservation.stderr diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 1c6fc1ac1853f..46fc6454d1c3b 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -570,8 +570,9 @@ impl Token { // // Instead the "probably equal" check here is "does each token // recursively have the same discriminant?" We basically don't look at - // the token values here and assume that such fine grained modifications - // of token streams doesn't happen. + // the token values here and assume that such fine grained token stream + // modifications, including adding/removing typically non-semantic + // tokens such as extra braces and commas, don't happen. if let Some(tokens) = tokens { if tokens.probably_equal_for_proc_macro(&tokens_for_real) { return tokens diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 29bd63d28c5ec..a7f8c692675f4 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -26,7 +26,7 @@ use syntax_pos::{BytePos, Mark, Span, DUMMY_SP}; use ext::base; use ext::tt::{macro_parser, quoted}; use parse::Directory; -use parse::token::{self, Token}; +use parse::token::{self, DelimToken, Token}; use print::pprust; use serialize::{Decoder, Decodable, Encoder, Encodable}; use util::RcVec; @@ -38,7 +38,7 @@ use std::{fmt, iter, mem}; #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] pub struct Delimited { /// The type of delimiter - pub delim: token::DelimToken, + pub delim: DelimToken, /// The delimited sequence of token trees pub tts: ThinTokenStream, } @@ -368,8 +368,30 @@ impl TokenStream { // This is otherwise the same as `eq_unspanned`, only recursing with a // different method. pub fn probably_equal_for_proc_macro(&self, other: &TokenStream) -> bool { - let mut t1 = self.trees(); - let mut t2 = other.trees(); + // When checking for `probably_eq`, we ignore certain tokens that aren't + // preserved in the AST. Because they are not preserved, the pretty + // printer arbitrarily adds or removes them when printing as token + // streams, making a comparison between a token stream generated from an + // AST and a token stream which was parsed into an AST more reliable. + fn semantic_tree(tree: &TokenTree) -> bool { + match tree { + // The pretty printer tends to add trailing commas to + // everything, and in particular, after struct fields. + | TokenTree::Token(_, Token::Comma) + // The pretty printer emits `NoDelim` as whitespace. + | TokenTree::Token(_, Token::OpenDelim(DelimToken::NoDelim)) + | TokenTree::Token(_, Token::CloseDelim(DelimToken::NoDelim)) + // The pretty printer collapses many semicolons into one. + | TokenTree::Token(_, Token::Semi) + // The pretty printer collapses whitespace arbitrarily and can + // introduce whitespace from `NoDelim`. + | TokenTree::Token(_, Token::Whitespace) => false, + _ => true + } + } + + let mut t1 = self.trees().filter(semantic_tree); + let mut t2 = other.trees().filter(semantic_tree); for (t1, t2) in t1.by_ref().zip(t2.by_ref()) { if !t1.probably_equal_for_proc_macro(&t2) { return false; diff --git a/src/test/ui-fulldeps/proc-macro/auxiliary/span-preservation.rs b/src/test/ui-fulldeps/proc-macro/auxiliary/span-preservation.rs new file mode 100644 index 0000000000000..65ed9cf737254 --- /dev/null +++ b/src/test/ui-fulldeps/proc-macro/auxiliary/span-preservation.rs @@ -0,0 +1,12 @@ +// no-prefer-dynamic + +#![crate_type = "proc-macro"] + +extern crate proc_macro; + +use proc_macro::TokenStream; + +#[proc_macro_attribute] +pub fn foo(_: TokenStream, input: TokenStream) -> TokenStream { + input.into_iter().collect() +} diff --git a/src/test/ui-fulldeps/proc-macro/span-preservation.rs b/src/test/ui-fulldeps/proc-macro/span-preservation.rs new file mode 100644 index 0000000000000..adcb42ab2d1cc --- /dev/null +++ b/src/test/ui-fulldeps/proc-macro/span-preservation.rs @@ -0,0 +1,51 @@ +// aux-build:span-preservation.rs + +// For each of these, we should get the appropriate type mismatch error message, +// and the function should be echoed. + +extern crate span_preservation as foo; + +use foo::foo; + +#[foo] +fn a() { + let x: usize = "hello";;;;; +} + +#[foo] +fn b(x: Option) -> usize { + match x { + Some(x) => { return x }, + None => 10 + } +} + +#[foo] +fn c() { + struct Foo { + a: usize + } + + struct Bar { + a: usize, + b: usize + } + + let x = Foo { a: 10isize }; + let y = Foo { a: 10, b: 10isize }; +} + +// FIXME: This doesn't work at the moment. See the one below. The pretty-printer +// injects a "C" between `extern` and `fn` which causes a "probably_eq" +// `TokenStream` mismatch. The lack of `"C"` should be preserved in the AST. +#[foo] +extern fn bar() { + 0 +} + +#[foo] +extern "C" fn baz() { + 0 +} + +fn main() {} diff --git a/src/test/ui-fulldeps/proc-macro/span-preservation.stderr b/src/test/ui-fulldeps/proc-macro/span-preservation.stderr new file mode 100644 index 0000000000000..f33245aec3720 --- /dev/null +++ b/src/test/ui-fulldeps/proc-macro/span-preservation.stderr @@ -0,0 +1,49 @@ +error[E0308]: mismatched types + | + = note: expected type `()` + found type `{integer}` + +error[E0308]: mismatched types + --> $DIR/span-preservation.rs:12:20 + | +LL | let x: usize = "hello";;;;; + | ^^^^^^^ expected usize, found reference + | + = note: expected type `usize` + found type `&'static str` + +error[E0308]: mismatched types + --> $DIR/span-preservation.rs:18:29 + | +LL | Some(x) => { return x }, + | ^ expected usize, found isize + +error[E0308]: mismatched types + --> $DIR/span-preservation.rs:34:22 + | +LL | let x = Foo { a: 10isize }; + | ^^^^^^^ expected usize, found isize + +error[E0560]: struct `c::Foo` has no field named `b` + --> $DIR/span-preservation.rs:35:26 + | +LL | let y = Foo { a: 10, b: 10isize }; + | ^ `c::Foo` does not have this field + | + = note: available fields are: `a` + +error[E0308]: mismatched types + --> $DIR/span-preservation.rs:48:5 + | +LL | extern "C" fn baz() { + | - possibly return type missing here? +LL | 0 + | ^ expected (), found integral variable + | + = note: expected type `()` + found type `{integer}` + +error: aborting due to 6 previous errors + +Some errors occurred: E0308, E0560. +For more information about an error, try `rustc --explain E0308`.