Auto merge of #108260 - lnicola:rust-analyzer-2023-02-20, r=lnicola

⬆️ rust-analyzer

r? `@ghost`

src/tools/rust-analyzer/Cargo.lock

@@ -711,6 +711,7 @@ dependencies = [
  "limit",
  "memchr",
  "once_cell",
+ "oorandom",
  "parser",
  "profile",
  "rayon",
@@ -932,9 +933,9 @@ dependencies = [
 
 [[package]]
 name = "lsp-types"
-version = "0.93.2"
+version = "0.94.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9be6e9c7e2d18f651974370d7aff703f9513e0df6e464fd795660edc77e6ca51"
+checksum = "0b63735a13a1f9cd4f4835223d828ed9c2e35c8c5e61837774399f558b6a1237"
 dependencies = [
  "bitflags",
  "serde",
@@ -1173,6 +1174,7 @@ dependencies = [
  "limit",
  "rustc-ap-rustc_lexer",
  "sourcegen",
+ "stdx",
 ]
 
 [[package]]

src/tools/rust-analyzer/Cargo.toml

@@ -74,3 +74,5 @@ toolchain = { path = "./crates/toolchain", version = "0.0.0" }
 tt = { path = "./crates/tt", version = "0.0.0" }
 vfs-notify = { path = "./crates/vfs-notify", version = "0.0.0" }
 vfs = { path = "./crates/vfs", version = "0.0.0" }
+# non-local crates
+smallvec = { version = "1.10.0", features = ["const_new", "union", "const_generics"] }

src/tools/rust-analyzer/crates/hir-def/Cargo.toml

@@ -27,7 +27,7 @@ itertools = "0.10.5"
 la-arena = { version = "0.3.0", path = "../../lib/la-arena" }
 once_cell = "1.17.0"
 rustc-hash = "1.1.0"
-smallvec = "1.10.0"
+smallvec.workspace = true
 tracing = "0.1.35"
 
 rustc_abi = { version = "0.0.20221221", package = "hkalbasi-rustc-ap-rustc_abi", default-features = false }

src/tools/rust-analyzer/crates/hir-def/src/adt.rs

@@ -2,9 +2,10 @@
 
 use std::sync::Arc;
 
-use crate::tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree};
 use base_db::CrateId;
+use cfg::CfgOptions;
 use either::Either;
+
 use hir_expand::{
     name::{AsName, Name},
     HirFileId, InFile,
@@ -24,12 +25,12 @@ use crate::{
     src::HasChildSource,
     src::HasSource,
     trace::Trace,
+    tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree},
     type_ref::TypeRef,
     visibility::RawVisibility,
     EnumId, LocalEnumVariantId, LocalFieldId, LocalModuleId, Lookup, ModuleId, StructId, UnionId,
     VariantId,
 };
-use cfg::CfgOptions;
 
 /// Note that we use `StructData` for unions as well!
 #[derive(Debug, Clone, PartialEq, Eq)]

src/tools/rust-analyzer/crates/hir-def/src/body.rs

@@ -19,7 +19,7 @@ use la_arena::{Arena, ArenaMap};
 use limit::Limit;
 use profile::Count;
 use rustc_hash::FxHashMap;
-use syntax::{ast, AstPtr, SyntaxNodePtr};
+use syntax::{ast, AstPtr, SyntaxNode, SyntaxNodePtr};
 
 use crate::{
     attr::Attrs,
@@ -51,7 +51,8 @@ pub struct Expander {
     def_map: Arc<DefMap>,
     current_file_id: HirFileId,
     module: LocalModuleId,
-    recursion_limit: usize,
+    /// `recursion_depth == usize::MAX` indicates that the recursion limit has been reached.
+    recursion_depth: usize,
 }
 
 impl CfgExpander {
@@ -84,7 +85,7 @@ impl Expander {
             def_map,
             current_file_id,
             module: module.local_id,
-            recursion_limit: 0,
+            recursion_depth: 0,
         }
     }
 
@@ -93,47 +94,52 @@ impl Expander {
         db: &dyn DefDatabase,
         macro_call: ast::MacroCall,
     ) -> Result<ExpandResult<Option<(Mark, T)>>, UnresolvedMacro> {
-        if self.recursion_limit(db).check(self.recursion_limit + 1).is_err() {
-            cov_mark::hit!(your_stack_belongs_to_me);
-            return Ok(ExpandResult::only_err(ExpandError::Other(
-                "reached recursion limit during macro expansion".into(),
-            )));
+        let mut unresolved_macro_err = None;
+
+        let result = self.within_limit(db, |this| {
+            let macro_call = InFile::new(this.current_file_id, &macro_call);
+
+            let resolver =
+                |path| this.resolve_path_as_macro(db, &path).map(|it| macro_id_to_def_id(db, it));
+
+            let mut err = None;
+            let call_id = match macro_call.as_call_id_with_errors(
+                db,
+                this.def_map.krate(),
+                resolver,
+                &mut |e| {
+                    err.get_or_insert(e);
+                },
+            ) {
+                Ok(call_id) => call_id,
+                Err(resolve_err) => {
+                    unresolved_macro_err = Some(resolve_err);
+                    return ExpandResult { value: None, err: None };
+                }
+            };
+            ExpandResult { value: call_id.ok(), err }
+        });
+
+        if let Some(err) = unresolved_macro_err {
+            Err(err)
+        } else {
+            Ok(result)
         }
-
-        let macro_call = InFile::new(self.current_file_id, &macro_call);
-
-        let resolver =
-            |path| self.resolve_path_as_macro(db, &path).map(|it| macro_id_to_def_id(db, it));
-
-        let mut err = None;
-        let call_id =
-            macro_call.as_call_id_with_errors(db, self.def_map.krate(), resolver, &mut |e| {
-                err.get_or_insert(e);
-            })?;
-        let call_id = match call_id {
-            Ok(it) => it,
-            Err(_) => {
-                return Ok(ExpandResult { value: None, err });
-            }
-        };
-
-        Ok(self.enter_expand_inner(db, call_id, err))
     }
 
     pub fn enter_expand_id<T: ast::AstNode>(
         &mut self,
         db: &dyn DefDatabase,
         call_id: MacroCallId,
     ) -> ExpandResult<Option<(Mark, T)>> {
-        self.enter_expand_inner(db, call_id, None)
+        self.within_limit(db, |_this| ExpandResult::ok(Some(call_id)))
     }
 
-    fn enter_expand_inner<T: ast::AstNode>(
-        &mut self,
+    fn enter_expand_inner(
         db: &dyn DefDatabase,
         call_id: MacroCallId,
         mut err: Option<ExpandError>,
-    ) -> ExpandResult<Option<(Mark, T)>> {
+    ) -> ExpandResult<Option<(HirFileId, SyntaxNode)>> {
         if err.is_none() {
             err = db.macro_expand_error(call_id);
         }
@@ -154,29 +160,21 @@ impl Expander {
             }
         };
 
-        let node = match T::cast(raw_node) {
-            Some(it) => it,
-            None => {
-                // This can happen without being an error, so only forward previous errors.
-                return ExpandResult { value: None, err };
-            }
-        };
-
-        tracing::debug!("macro expansion {:#?}", node.syntax());
-
-        self.recursion_limit += 1;
-        let mark =
-            Mark { file_id: self.current_file_id, bomb: DropBomb::new("expansion mark dropped") };
-        self.cfg_expander.hygiene = Hygiene::new(db.upcast(), file_id);
-        self.current_file_id = file_id;
-
-        ExpandResult { value: Some((mark, node)), err }
+        ExpandResult { value: Some((file_id, raw_node)), err }
     }
 
     pub fn exit(&mut self, db: &dyn DefDatabase, mut mark: Mark) {
         self.cfg_expander.hygiene = Hygiene::new(db.upcast(), mark.file_id);
         self.current_file_id = mark.file_id;
-        self.recursion_limit -= 1;
+        if self.recursion_depth == usize::MAX {
+            // Recursion limit has been reached somewhere in the macro expansion tree. Reset the
+            // depth only when we get out of the tree.
+            if !self.current_file_id.is_macro() {
+                self.recursion_depth = 0;
+            }
+        } else {
+            self.recursion_depth -= 1;
+        }
         mark.bomb.defuse();
     }
 
@@ -215,6 +213,50 @@ impl Expander {
         #[cfg(test)]
         return Limit::new(std::cmp::min(32, limit));
     }
+
+    fn within_limit<F, T: ast::AstNode>(
+        &mut self,
+        db: &dyn DefDatabase,
+        op: F,
+    ) -> ExpandResult<Option<(Mark, T)>>
+    where
+        F: FnOnce(&mut Self) -> ExpandResult<Option<MacroCallId>>,
+    {
+        if self.recursion_depth == usize::MAX {
+            // Recursion limit has been reached somewhere in the macro expansion tree. We should
+            // stop expanding other macro calls in this tree, or else this may result in
+            // exponential number of macro expansions, leading to a hang.
+            //
+            // The overflow error should have been reported when it occurred (see the next branch),
+            // so don't return overflow error here to avoid diagnostics duplication.
+            cov_mark::hit!(overflow_but_not_me);
+            return ExpandResult::only_err(ExpandError::RecursionOverflowPosioned);
+        } else if self.recursion_limit(db).check(self.recursion_depth + 1).is_err() {
+            self.recursion_depth = usize::MAX;
+            cov_mark::hit!(your_stack_belongs_to_me);
+            return ExpandResult::only_err(ExpandError::Other(
+                "reached recursion limit during macro expansion".into(),
+            ));
+        }
+
+        let ExpandResult { value, err } = op(self);
+        let Some(call_id) = value else {
+            return ExpandResult { value: None, err };
+        };
+
+        Self::enter_expand_inner(db, call_id, err).map(|value| {
+            value.and_then(|(new_file_id, node)| {
+                let node = T::cast(node)?;
+
+                self.recursion_depth += 1;
+                self.cfg_expander.hygiene = Hygiene::new(db.upcast(), new_file_id);
+                let old_file_id = std::mem::replace(&mut self.current_file_id, new_file_id);
+                let mark =
+                    Mark { file_id: old_file_id, bomb: DropBomb::new("expansion mark dropped") };
+                Some((mark, node))
+            })
+        })
+    }
 }
 
 #[derive(Debug)]

src/tools/rust-analyzer/crates/hir-def/src/body/lower.rs

@@ -624,6 +624,10 @@ impl ExprCollector<'_> {
                         krate: *krate,
                     });
                 }
+                Some(ExpandError::RecursionOverflowPosioned) => {
+                    // Recursion limit has been reached in the macro expansion tree, but not in
+                    // this very macro call. Don't add diagnostics to avoid duplication.
+                }
                 Some(err) => {
                     self.source_map.diagnostics.push(BodyDiagnostic::MacroError {
                         node: InFile::new(outer_file, syntax_ptr),
@@ -636,6 +640,8 @@ impl ExprCollector<'_> {
 
         match res.value {
             Some((mark, expansion)) => {
+                // Keep collecting even with expansion errors so we can provide completions and
+                // other services in incomplete macro expressions.
                 self.source_map.expansions.insert(macro_call_ptr, self.expander.current_file_id);
                 let prev_ast_id_map = mem::replace(
                     &mut self.ast_id_map,

src/tools/rust-analyzer/crates/hir-def/src/body/tests.rs

@@ -61,6 +61,19 @@ fn main() { n_nuple!(1,2,3); }
     );
 }
 
+#[test]
+fn your_stack_belongs_to_me2() {
+    cov_mark::check!(overflow_but_not_me);
+    lower(
+        r#"
+macro_rules! foo {
+    () => {{ foo!(); foo!(); }}
+}
+fn main() { foo!(); }
+"#,
+    );
+}
+
 #[test]
 fn recursion_limit() {
     cov_mark::check!(your_stack_belongs_to_me);

src/tools/rust-analyzer/crates/hir-def/src/item_tree/lower.rs

@@ -659,15 +659,16 @@ fn desugar_future_path(orig: TypeRef) -> Path {
     let path = path![core::future::Future];
     let mut generic_args: Vec<_> =
         std::iter::repeat(None).take(path.segments().len() - 1).collect();
-    let mut last = GenericArgs::empty();
     let binding = AssociatedTypeBinding {
         name: name![Output],
         args: None,
         type_ref: Some(orig),
-        bounds: Vec::new(),
+        bounds: Box::default(),
     };
-    last.bindings.push(binding);
-    generic_args.push(Some(Interned::new(last)));
+    generic_args.push(Some(Interned::new(GenericArgs {
+        bindings: Box::new([binding]),
+        ..GenericArgs::empty()
+    })));
 
     Path::from_known_path(path, generic_args)
 }

src/tools/rust-analyzer/crates/hir-def/src/macro_expansion_tests/mbe.rs

@@ -1476,7 +1476,7 @@ macro_rules! m {
 /* parse error: expected identifier */
 /* parse error: expected SEMICOLON */
 /* parse error: expected SEMICOLON */
-/* parse error: expected expression */
+/* parse error: expected expression, item or let statement */
 fn f() {
     K::(C("0"));
 }

src/tools/rust-analyzer/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs

@@ -830,8 +830,7 @@ macro_rules! rgb_color {
 /* parse error: expected COMMA */
 /* parse error: expected R_ANGLE */
 /* parse error: expected SEMICOLON */
-/* parse error: expected SEMICOLON */
-/* parse error: expected expression */
+/* parse error: expected expression, item or let statement */
 pub fn new() {
     let _ = 0as u32<<(8+8);
 }
@@ -848,21 +847,21 @@ pub fn new() {
 //     BLOCK_EXPR@10..31
 //       STMT_LIST@10..31
 //         L_CURLY@10..11 "{"
-//         LET_STMT@11..27
+//         LET_STMT@11..28
 //           LET_KW@11..14 "let"
 //           WILDCARD_PAT@14..15
 //             UNDERSCORE@14..15 "_"
 //           EQ@15..16 "="
-//           CAST_EXPR@16..27
+//           CAST_EXPR@16..28
 //             LITERAL@16..17
 //               INT_NUMBER@16..17 "0"
 //             AS_KW@17..19 "as"
-//             PATH_TYPE@19..27
-//               PATH@19..27
-//                 PATH_SEGMENT@19..27
+//             PATH_TYPE@19..28
+//               PATH@19..28
+//                 PATH_SEGMENT@19..28
 //                   NAME_REF@19..22
 //                     IDENT@19..22 "u32"
-//                   GENERIC_ARG_LIST@22..27
+//                   GENERIC_ARG_LIST@22..28
 //                     L_ANGLE@22..23 "<"
 //                     TYPE_ARG@23..27
 //                       DYN_TRAIT_TYPE@23..27
@@ -877,9 +876,9 @@ pub fn new() {
 //                                     ERROR@25..26
 //                                       INT_NUMBER@25..26 "8"
 //                           PLUS@26..27 "+"
-//         EXPR_STMT@27..28
-//           LITERAL@27..28
-//             INT_NUMBER@27..28 "8"
+//                     CONST_ARG@27..28
+//                       LITERAL@27..28
+//                         INT_NUMBER@27..28 "8"
 //         ERROR@28..29
 //           R_PAREN@28..29 ")"
 //         SEMICOLON@29..30 ";"