Inconsistent tokenization of procedural macros with macro_rules

[alexcrichton]

Given a procedural macro like so:

// foo.rs
#![crate_type = "proc-macro"]
#![feature(proc_macro)]

extern crate proc_macro;

use proc_macro::*;

#[proc_macro_attribute]
pub fn attr(_a: TokenStream, b: TokenStream) -> TokenStream {
    println!("============== invocation =========================");
    println!("**** ToString *****\n{}", b);
    // println!("**** Debug *****\n{:?}", b);
    let b = recollect(b);
    println!("**** recollect ToString *****\n{}", b);
    // println!("**** recollect Debug *****\n{:?}", b);
    return b
}

fn recollect(a: TokenStream) -> TokenStream {
    a.into_iter()
        .map(|tt| {
            match tt {
                TokenTree::Group(tt) => {
                    let mut g = Group::new(tt.delimiter(), recollect(tt.stream()));
                    g.set_span(tt.span());
                    TokenTree::from(g)
                }
                other => other,
            }
        })
        .collect()
}

and an invocation:

// bar.rs
#![crate_type = "rlib"]
#![feature(proc_macro)]

extern crate foo;

use foo::attr;

#[attr]
pub fn bar() {}

macro_rules! a {
    ($i:item) => ($i)
}

a! {
    #[attr]
    pub fn baz() {}
}

you get the following when compiling:

$ rustc +nightly foo.rs
$ rustc +nightly bar.rs -L .
============== invocation =========================
**** ToString *****
pub fn bar() { }
**** recollect ToString *****
pub fn bar (  ) {  }
============== invocation =========================
**** ToString *****
pub fn baz() { }
**** recollect ToString *****
  # [ attr ] pub fn baz (  ) {  }  
thread 'main' panicked at 'index out of bounds: the len is 0 but the index is 1', /checkout/src/libcore/slice/mod.rs:871:14

In addition to the panic (oh dear!) we can see here that #[attr] is showing up in the tokens by accident. The attribute should have been removed during the tokenization when passing to the procedural macro!

I believe this is another instance of #43081

[alexcrichton]

cc @dtolnay