Float parsing regression (repeated panic, basically a crash)

[Veetaha]

rust-analyzer version: 5d5bbec 2022-05-09 stable
rustc version: 1.60.0 (7737e0b5c 2022-04-04)

When vscode automatically upgraded me to 2022-05-09 stable I started seeing RA crashing constantly with this in the logs:

Verbose backtrace

thread '<unnamed>' panicked at 'Punct {
    char: 'i',
    spacing: Joint,
    id: TokenId(
        15,
    ),
} is not a valid punct', crates/mbe/src/to_parser_input.rs:61:48
stack backtrace:
   0:     0x559bd6f76df4 - std::backtrace_rs::backtrace::libunwind::trace::hee598835bc88d35b
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
   1:     0x559bd6f76df4 - std::backtrace_rs::backtrace::trace_unsynchronized::h9cdc730ba5cf5d72
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0x559bd6f76df4 - std::sys_common::backtrace::_print_fmt::h75aeaf7ed30e43fa
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/std/src/sys_common/backtrace.rs:66:5
   3:     0x559bd6f76df4 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h606862f787600875
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/std/src/sys_common/backtrace.rs:45:22
   4:     0x559bd605e60c - core::fmt::write::he803f0f418caf762
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/core/src/fmt/mod.rs:1190:17
   5:     0x559bd6f7052a - std::io::Write::write_fmt::h70bc45872f37e7bb
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/std/src/io/mod.rs:1657:15
   6:     0x559bd6f79ce6 - std::sys_common::backtrace::_print::h64d038cf8ac3e13e
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/std/src/sys_common/backtrace.rs:48:5
   7:     0x559bd6f79ce6 - std::sys_common::backtrace::print::h359300b4a7fccf65
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/std/src/sys_common/backtrace.rs:35:9
   8:     0x559bd6f79ce6 - std::panicking::default_hook::{{closure}}::hf51be35e2f510149
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/std/src/panicking.rs:295:22
   9:     0x559bd6f79a02 - std::panicking::default_hook::h03ca0f22e1d2d25e
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/std/src/panicking.rs:314:9
  10:     0x559bd6f8dd14 - std::thread::local::LocalKey<T>::with::hefa9eb3b771d736c
  11:     0x559bd6f7a4f8 - std::panicking::rust_panic_with_hook::h3b7380e99b825b63
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/std/src/panicking.rs:702:17
  12:     0x559bd6f7a1a4 - std::panicking::begin_panic_handler::{{closure}}::h8e849d0710154ce0
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/std/src/panicking.rs:588:13
  13:     0x559bd6f77334 - std::sys_common::backtrace::__rust_end_short_backtrace::hedcdaddbd4c46cc5
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/std/src/sys_common/backtrace.rs:138:18
  14:     0x559bd6f79f0d - rust_begin_unwind
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/std/src/panicking.rs:584:5
  15:     0x559bd5f5d933 - core::panicking::panic_fmt::he1bbc7336d49a357
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/core/src/panicking.rs:143:14
  16:     0x559bd6a4d79c - mbe::to_parser_input::to_parser_input::h5c24f2cca3f4d8c9
  17:     0x559bd6a3fc5c - mbe::syntax_bridge::token_tree_to_syntax_node::h2612d7cc7f520ef7
  18:     0x559bd634c860 - <hir_expand::db::ParseMacroExpansionQuery as salsa::plumbing::QueryFunction>::execute::hed714650d6356cb9
  19:     0x559bd638f829 - salsa::runtime::Runtime::execute_query_implementation::h3abcf0097d0eda06
  20:     0x559bd6320d81 - salsa::derived::slot::Slot<Q,MP>::read_upgrade::h21fee402daa458fd
  21:     0x559bd631f5ef - salsa::derived::slot::Slot<Q,MP>::read::hd183788e83c4f994
  22:     0x559bd6355416 - <salsa::derived::DerivedStorage<Q,MP> as salsa::plumbing::QueryStorageOps<Q>>::try_fetch::h9589847ffd149f08
  23:     0x559bd639126b - salsa::QueryTable<Q>::get::hcbc2cfbb7192712e
  24:     0x559bd69bccd8 - <DB as hir_expand::db::AstDatabase>::parse_macro_expansion::h8e7766d91b50d253
  25:     0x559bd634b239 - hir_expand::db::parse_or_expand::h8d7a025a8b0d48a5
  26:     0x559bd61a5989 - hir_def::item_tree::ItemTree::file_item_tree_query::he342a62917b6bfc2
  27:     0x559bd6276262 - salsa::runtime::Runtime::execute_query_implementation::hba5d10389616a88c
  28:     0x559bd616a255 - salsa::derived::slot::Slot<Q,MP>::read_upgrade::ha50f34a787dac487
  29:     0x559bd613b8d7 - salsa::derived::slot::Slot<Q,MP>::read::h227ea14d6234eccf
  30:     0x559bd62a8f4f - <salsa::derived::DerivedStorage<Q,MP> as salsa::plumbing::QueryStorageOps<Q>>::try_fetch::h30c2b72781f47bd1
  31:     0x559bd61d3fd8 - salsa::QueryTable<Q>::get::hd5569f743f415687
  32:     0x559bd69be803 - <DB as hir_def::db::DefDatabase>::file_item_tree::h9c84ded800c3bca8
  33:     0x559bd622c665 - hir_def::nameres::collector::DefCollector::collect_macro_expansion::h85fd4aa08adfc59c
  34:     0x559bd6225e01 - hir_def::nameres::collector::collect_defs::h43644ac7e93119fe
  35:     0x559bd61ada1f - hir_def::nameres::DefMap::crate_def_map_query::h1c61683a910ed5a1
  36:     0x559bd62765af - salsa::runtime::Runtime::execute_query_implementation::hbdab97d6a9cd1afb
  37:     0x559bd614c045 - salsa::derived::slot::Slot<Q,MP>::read_upgrade::h1bff562a530fb15b
  38:     0x559bd6146ae7 - salsa::derived::slot::Slot<Q,MP>::read::hc066ca83861ef907
  39:     0x559bd62a85d7 - <salsa::derived::DerivedStorage<Q,MP> as salsa::plumbing::QueryStorageOps<Q>>::try_fetch::h279fd47361670b48
  40:     0x559bd61d10a4 - salsa::QueryTable<Q>::get::h186bf05db14d439f
  41:     0x559bd69be858 - <DB as hir_def::db::DefDatabase>::crate_def_map_query::h21431f41e6777788
  42:     0x559bd62cf72e - hir_def::db::crate_def_map_wait::h65ae064872bb2b18
  43:     0x559bd611bcc0 - hir::semantics::source_to_def::SourceToDefCtx::file_to_def::h390ab70e584985c0
  44:     0x559bd69b6016 - hir::semantics::Semantics<DB>::to_module_def::hc079a38e5c6ce36f
  45:     0x559bd699f95f - ide_diagnostics::diagnostics::hc20b93caa6faaf52
  46:     0x559bd6623403 - std::panicking::try::hdb88ab80e595a633
  47:     0x559bd66f846e - ide::Analysis::diagnostics::hcacb9785369bdf56
  48:     0x559bd6e2578d - rust_analyzer::handlers::publish_diagnostics::hc3c3d56fb724fbaa
  49:     0x559bd6cb8e85 - core::ops::function::impls::<impl core::ops::function::FnMut<A> for &mut F>::call_mut::h22196717306b7087
  50:     0x559bd6e41199 - alloc::vec::source_iter_marker::<impl alloc::vec::spec_from_iter::SpecFromIter<T,I> for alloc::vec::Vec<T>>::from_iter::h634503b448fa3326
  51:     0x559bd6d055c9 - <F as threadpool::FnBox>::call_box::hb17e0fd6a5cdc02a
  52:     0x559bd6fdb819 - std::sys_common::backtrace::__rust_begin_short_backtrace::hc0c373d19bb64616
  53:     0x559bd6fdee48 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h901a3a04eb5bd37d
  54:     0x559bd6f80025 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hf70ac038171e3e1a
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/alloc/src/boxed.rs:1853:9
  55:     0x559bd6f80025 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::he6690128792365ad
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/alloc/src/boxed.rs:1853:9
  56:     0x559bd6f80025 - std::sys::unix::thread::Thread::new::thread_start::ha07928d93d5a5ec9
                               at /rustc/7737e0b5c4103216d6fd8cf941b7ab9bdbaace7c/library/std/src/sys/unix/thread.rs:108:17
  57:     0x7f3a63523609 - start_thread
  58:     0x7f3a632fb293 - clone
  59:                0x0 - <unknown>

Debugging the cause

I tried to debug the cause by manually modifying RA's code and compiling it.

I changed this line

rust-analyzer/crates/mbe/src/to_parser_input.rs

Lines 59 to 66 in 4d94cf3

	tt::Leaf::Punct(punct) => {
	let kind = SyntaxKind::from_char(punct.char)
	.unwrap_or_else(|| panic!("{:#?} is not a valid punct", punct));
	res.push(kind);
	if punct.spacing == tt::Spacing::Joint {
	res.was_joint();
	}
	}

this way:

- panic!("{:#?} is not a valid punct", punct)
+ panic!("{:#?} is not a valid punct\n{buffer:#?}", punct)

It allowed me to identify where in the whole world of a project I get the crash. It definitely helped, and I was able to minimize it to the following reproduction steps.

Reproduction

It requires two files. The first file with the following code included.rs:

#[async_trait]
impl Foo for WrapBj {
    async fn foo(&self) {
        // Notice how there is a gap between `self.0.` and `id()`
        // removing this gap prevents the crash
        self.0. id().await;
    }
}

and the second one lib.rs

use async_trait::async_trait;

include!("included.rs");

You also obviously need to add async-trait = "0.1" to the dependencies.

I think the regression was introduced in #12149. Previous versions of RA (i.e. 2022-05-02 and older) should work fine.

For that reason, @jonas-schievink please take a look at this issue. I think you can tell where the root problem is.

Workaround

As a temporary workaround I added parentheses around self.0 in generated code that we include!() in our project

[jonas-schievink]

Hmm, I only get "expected identifier or integer" with the code you posted, which does seem like a bug, but it didn't crash. When I remove the space after .0. it does expand correctly, as you noted.

[Veetaha]

Here is what I see in vscode.

Maybe my settings.json could influence this somehow?
Could it be an ignore for async_trait macro that causes this?

{
    "rust-analyzer.inlayHints.chainingHints": false,
    "rust-analyzer.inlayHints.closureReturnTypeHints": true,
    "rust-analyzer.inlayHints.maxLength": 10,
    "rust-analyzer.inlayHints.parameterHints": false,
    "rust-analyzer.inlayHints.reborrowHints": true,
    "rust-analyzer.inlayHints.typeHints": false,

    "rust-analyzer.assist.importGranularity": "module",
    "rust-analyzer.assist.importGroup": true,
    "rust-analyzer.cargo.allFeatures": true,
    "rust-analyzer.cargo.runBuildScripts": true,
    "rust-analyzer.completion.addCallParenthesis": false,
    "rust-analyzer.completion.postfix.enable": false,
    "rust-analyzer.diagnostics.enable": true,
    "rust-analyzer.experimental.procAttrMacros": true,
    "rust-analyzer.highlightRelated.breakPoints": true,
    "rust-analyzer.highlightRelated.exitPoints": true,
    "rust-analyzer.highlightRelated.references": true,
    "rust-analyzer.highlightRelated.yieldPoints": true,
    "rust-analyzer.lruCapacity": 200000,
    "rust-analyzer.procMacro.enable": true,
    "rust-analyzer.trace.extension": true,
    "rust-analyzer.trace.server": "off",
    "rust-analyzer.server.extraEnv": {
        "CARGO_TARGET_DIR": "/home/veetaha/junk/ra-target-dir-cache"
    },
    "rust-analyzer.procMacro.ignored": {
        "async-recursion": ["async_recursion"],
        "async-trait": ["async_trait"],
        "tokio-macros": ["test"],
        "tracing-attributes": ["instrument"],
        "trix-test-macros": ["test"],
        // This is not exactly right, because this macro changes the signature
        // of the function from async to non-async, but it is mostly used in
        // `Drop::drop()` methods
        "tokiox-macros": ["block_in_place"],
        "serial_test": ["serial"],
        "cucumber_rust_codegen": ["given", "when", "then"],
    },
}

I've also pushed the exact code used to a reproduction repo.

cc @jonas-schievink

[jonas-schievink]

Could it be an ignore for async_trait macro that causes this?

Yup, with that I can reproduce, thanks!