Iterating over parsed tokens causes char boundary index panic

[frewsxcv]

extern crate proc_macro2;

fn main() {
    let s = std::str::from_utf8(b"b\'\xc2\x86  \x00\x00\x00^\"").unwrap();
    if let Ok(token_stream) = s.parse::<proc_macro2::TokenStream>() {
        for _ in token_stream { }
    }
}

> cargo run
    Finished dev [unoptimized + debuginfo] target(s) in 0.0 secs
     Running `target/debug/proc-macro2-foo`
thread 'main' panicked at 'byte index 1 is not a char boundary; it is inside '\u{86}' (bytes 0..2) of `�  ^"`', src/libcore/str/mod.rs:2234:4
stack backtrace:
   0: std::sys::unix::backtrace::tracing::imp::unwind_backtrace
   1: std::sys_common::backtrace::print
   2: std::panicking::default_hook::{{closure}}
   3: std::panicking::default_hook
   4: std::panicking::rust_panic_with_hook
   5: std::panicking::begin_panic
   6: std::panicking::begin_panic_fmt
   7: rust_begin_unwind
   8: core::panicking::panic_fmt
   9: core::str::slice_error_fail
  10: core::str::traits::<impl core::slice::SliceIndex<str> for core::ops::range::RangeFrom<usize>>::index::{{closure}}
             at /Users/travis/build/rust-lang/rust/src/libcore/str/mod.rs:1987
  11: <core::option::Option<T>>::unwrap_or_else
             at /Users/travis/build/rust-lang/rust/src/libcore/option.rs:376
  12: core::str::traits::<impl core::slice::SliceIndex<str> for core::ops::range::RangeFrom<usize>>::index
             at /Users/travis/build/rust-lang/rust/src/libcore/str/mod.rs:1987
  13: core::str::traits::<impl core::ops::index::Index<core::ops::range::RangeFrom<usize>> for str>::index
             at /Users/travis/build/rust-lang/rust/src/libcore/str/mod.rs:1734
  14: proc_macro2::strnom::Cursor::advance
             at /Users/corey/.cargo/registry/src/github.com-1ecc6299db9ec823/proc-macro2-0.2.1/src/strnom.rs:20
  15: proc_macro2::imp::cooked_byte
             at /Users/corey/.cargo/registry/src/github.com-1ecc6299db9ec823/proc-macro2-0.2.1/src/stable.rs:907
  16: proc_macro2::imp::byte
             at /Users/corey/.cargo/registry/src/github.com-1ecc6299db9ec823/proc-macro2-0.2.1/src/strnom.rs:160
  17: proc_macro2::imp::literal_nocapture
             at /Users/corey/.cargo/registry/src/github.com-1ecc6299db9ec823/proc-macro2-0.2.1/src/strnom.rs:160
  18: proc_macro2::imp::literal
             at /Users/corey/.cargo/registry/src/github.com-1ecc6299db9ec823/proc-macro2-0.2.1/src/stable.rs:697
  19: proc_macro2::imp::token_kind
             at /Users/corey/.cargo/registry/src/github.com-1ecc6299db9ec823/proc-macro2-0.2.1/src/strnom.rs:160
  20: proc_macro2::imp::token_tree
             at /Users/corey/.cargo/registry/src/github.com-1ecc6299db9ec823/proc-macro2-0.2.1/src/stable.rs:596
  21: proc_macro2::imp::token_stream
             at /Users/corey/.cargo/registry/src/github.com-1ecc6299db9ec823/proc-macro2-0.2.1/src/strnom.rs:160
  22: proc_macro2::imp::get_cursor
             at /Users/corey/.cargo/registry/src/github.com-1ecc6299db9ec823/proc-macro2-0.2.1/src/stable.rs:66
  23: <str as core::str::StrExt>::parse
             at /Users/travis/build/rust-lang/rust/src/libcore/str/mod.rs:2511
  24: alloc::str::<impl str>::parse
             at /Users/travis/build/rust-lang/rust/src/liballoc/str.rs:1793
  25: alloc::slice::<impl [T]>::as_mut_ptr
             at /Users/corey/.cargo/registry/src/github.com-1ecc6299db9ec823/proc-macro2-0.2.1/src/lib.rs:59
  26: <str as core::str::StrExt>::parse
             at /Users/travis/build/rust-lang/rust/src/libcore/str/mod.rs:2511
  27: alloc::str::<impl str>::parse
             at /Users/travis/build/rust-lang/rust/src/liballoc/str.rs:1793
  28: proc_macro2_foo::main
             at src/main.rs:5
  29: __rust_maybe_catch_panic
  30: std::rt::lang_start
  31: proc_macro2_foo::main

found via afl.rs

[alexcrichton]

Thanks for the report!