Auto merge of #14781 - lowr:patch/macro-subns-and-prelude, r=Veykril

Introduce macro sub-namespaces and `macro_use` prelude

This PR implements two mechanisms needed for correct macro name resolution: macro sub-namespace and `macro_use` prelude.

- [macro sub-namespaces][subns-ref]

  Macros have two sub-namespaces: one for function-like macro and the other for those in attributes (including custom derive macros). When we're resolving a macro name for function-like macro, we should ignore non-function-like macros, and vice versa.

  This helps resolve single-segment macro names because we can (and should, as rustc does) fallback to names in preludes when the name in the current module scope is in different sub-namespace.

- [`macro_use` prelude][prelude-ref]

  `#[macro_use]`'d extern crate declarations (including the standard library) bring their macros into scope, but they should not be prioritized over local macros (those defined in place and those explicitly imported).

  We have been bringing them into legacy (textual) macro scope, which has the highest precedence in name resolution. This PR introduces the `macro_use` prelude in crate-level `DefMap`s, whose precedence is lower than local macros but higher than the standard library prelude.

The first 3 commits are drive-by fixes/refactors.

Fixes #8828 (prelude)
Fixes #12505 (prelude)
Fixes #12734 (prelude)
Fixes #13683 (prelude)
Fixes #13821 (prelude)
Fixes #13974 (prelude)
Fixes #14254 (namespace)

[subns-ref]: https://doc.rust-lang.org/reference/names/namespaces.html#sub-namespaces
[prelude-ref]: https://doc.rust-lang.org/reference/names/preludes.html#macro_use-prelude

Author: bors

crates/hir-def/src/body/lower.rs

@@ -37,7 +37,7 @@ use crate::{
     item_scope::BuiltinShadowMode,
     lang_item::LangItem,
     lower::LowerCtx,
-    nameres::DefMap,
+    nameres::{DefMap, MacroSubNs},
     path::{GenericArgs, Path},
     type_ref::{Mutability, Rawness, TypeRef},
     AdtId, BlockId, BlockLoc, ModuleDefId, UnresolvedMacro,
@@ -800,7 +800,13 @@ impl ExprCollector<'_> {
         let module = self.expander.module.local_id;
         let res = self.expander.enter_expand(self.db, mcall, |path| {
             self.def_map
-                .resolve_path(self.db, module, &path, crate::item_scope::BuiltinShadowMode::Other)
+                .resolve_path(
+                    self.db,
+                    module,
+                    &path,
+                    crate::item_scope::BuiltinShadowMode::Other,
+                    Some(MacroSubNs::Bang),
+                )
                 .0
                 .take_macros()
         });
@@ -1056,6 +1062,7 @@ impl ExprCollector<'_> {
                         self.expander.module.local_id,
                         &name.clone().into(),
                         BuiltinShadowMode::Other,
+                        None,
                     );
                     match resolved.take_values() {
                         Some(ModuleDefId::ConstId(_)) => (None, Pat::Path(name.into())),

crates/hir-def/src/data.rs

@@ -22,7 +22,7 @@ use crate::{
         attr_resolution::ResolvedAttr,
         diagnostics::DefDiagnostic,
         proc_macro::{parse_macro_name_and_helper_attrs, ProcMacroKind},
-        DefMap,
+        DefMap, MacroSubNs,
     },
     type_ref::{TraitRef, TypeBound, TypeRef},
     visibility::RawVisibility,
@@ -673,6 +673,7 @@ impl<'a> AssocItemCollector<'a> {
                             module,
                             &path,
                             crate::item_scope::BuiltinShadowMode::Other,
+                            Some(MacroSubNs::Bang),
                         )
                         .0
                         .take_macros()

crates/hir-def/src/find_path.rs

@@ -543,6 +543,7 @@ mod tests {
                 module.local_id,
                 &mod_path,
                 crate::item_scope::BuiltinShadowMode::Module,
+                None,
             )
             .0
             .take_types()

crates/hir-def/src/generics.rs

@@ -22,7 +22,7 @@ use crate::{
     dyn_map::{keys, DynMap},
     expander::Expander,
     lower::LowerCtx,
-    nameres::DefMap,
+    nameres::{DefMap, MacroSubNs},
     src::{HasChildSource, HasSource},
     type_ref::{LifetimeRef, TypeBound, TypeRef},
     AdtId, ConstParamId, GenericDefId, HasModule, LifetimeParamId, LocalLifetimeParamId,
@@ -361,6 +361,7 @@ impl GenericParams {
                             module,
                             &path,
                             crate::item_scope::BuiltinShadowMode::Other,
+                            Some(MacroSubNs::Bang),
                         )
                         .0
                         .take_macros()

crates/hir-def/src/macro_expansion_tests/builtin_fn_macro.rs

@@ -13,12 +13,12 @@ macro_rules! column {() => {}}
 
 fn main() { column!(); }
 "#,
-        expect![[r##"
+        expect![[r#"
 #[rustc_builtin_macro]
 macro_rules! column {() => {}}
 
-fn main() { 0; }
-"##]],
+fn main() { 0 as u32; }
+"#]],
     );
 }
 
@@ -31,12 +31,12 @@ macro_rules! line {() => {}}
 
 fn main() { line!() }
 "#,
-        expect![[r##"
+        expect![[r#"
 #[rustc_builtin_macro]
 macro_rules! line {() => {}}
 
-fn main() { 0 }
-"##]],
+fn main() { 0 as u32 }
+"#]],
     );
 }
 

crates/hir-def/src/macro_expansion_tests/mbe.rs

@@ -922,7 +922,7 @@ macro_rules! m {
 
 fn bar() -> &'a Baz<u8> {}
 
-fn bar() -> extern "Rust"fn() -> Ret {}
+fn bar() -> extern "Rust" fn() -> Ret {}
 "#]],
     );
 }
@@ -1333,7 +1333,7 @@ macro_rules! matches {
 }
 fn main() {
     match 0 {
-        0|1if true =>true , _=>false
+        0|1 if true =>true , _=>false
     };
 }
  "#]],

crates/hir-def/src/macro_expansion_tests/mbe/matching.rs

@@ -73,7 +73,7 @@ fn main() {
 macro_rules! asi { ($($stmt:stmt)*) => ($($stmt)*); }
 
 fn main() {
-    let a = 2let b = 5drop(b-a)println!("{}", a+b)
+    let a = 2 let b = 5 drop(b-a)println!("{}", a+b)
 }
 "#]],
     )

crates/hir-def/src/macro_expansion_tests/mbe/regression.rs

@@ -297,55 +297,55 @@ macro_rules! impl_fn_for_zst  {
 
 #[derive(Clone)] struct CharEscapeDebugContinue;
 impl Fn<(char, )> for CharEscapeDebugContinue {
-    #[inline] extern "rust-call"fn call(&self , (c, ): (char, )) -> char::EscapeDebug { {
+    #[inline] extern "rust-call" fn call(&self , (c, ): (char, )) -> char::EscapeDebug { {
             c.escape_debug_ext(false )
         }
     }
 }
 impl FnMut<(char, )> for CharEscapeDebugContinue {
-    #[inline] extern "rust-call"fn call_mut(&mut self , (c, ): (char, )) -> char::EscapeDebug {
+    #[inline] extern "rust-call" fn call_mut(&mut self , (c, ): (char, )) -> char::EscapeDebug {
         Fn::call(&*self , (c, ))
     }
 }
 impl FnOnce<(char, )> for CharEscapeDebugContinue {
     type Output = char::EscapeDebug;
-    #[inline] extern "rust-call"fn call_once(self , (c, ): (char, )) -> char::EscapeDebug {
+    #[inline] extern "rust-call" fn call_once(self , (c, ): (char, )) -> char::EscapeDebug {
         Fn::call(&self , (c, ))
     }
 }
 #[derive(Clone)] struct CharEscapeUnicode;
 impl Fn<(char, )> for CharEscapeUnicode {
-    #[inline] extern "rust-call"fn call(&self , (c, ): (char, )) -> char::EscapeUnicode { {
+    #[inline] extern "rust-call" fn call(&self , (c, ): (char, )) -> char::EscapeUnicode { {
             c.escape_unicode()
         }
     }
 }
 impl FnMut<(char, )> for CharEscapeUnicode {
-    #[inline] extern "rust-call"fn call_mut(&mut self , (c, ): (char, )) -> char::EscapeUnicode {
+    #[inline] extern "rust-call" fn call_mut(&mut self , (c, ): (char, )) -> char::EscapeUnicode {
         Fn::call(&*self , (c, ))
     }
 }
 impl FnOnce<(char, )> for CharEscapeUnicode {
     type Output = char::EscapeUnicode;
-    #[inline] extern "rust-call"fn call_once(self , (c, ): (char, )) -> char::EscapeUnicode {
+    #[inline] extern "rust-call" fn call_once(self , (c, ): (char, )) -> char::EscapeUnicode {
         Fn::call(&self , (c, ))
     }
 }
 #[derive(Clone)] struct CharEscapeDefault;
 impl Fn<(char, )> for CharEscapeDefault {
-    #[inline] extern "rust-call"fn call(&self , (c, ): (char, )) -> char::EscapeDefault { {
+    #[inline] extern "rust-call" fn call(&self , (c, ): (char, )) -> char::EscapeDefault { {
             c.escape_default()
         }
     }
 }
 impl FnMut<(char, )> for CharEscapeDefault {
-    #[inline] extern "rust-call"fn call_mut(&mut self , (c, ): (char, )) -> char::EscapeDefault {
+    #[inline] extern "rust-call" fn call_mut(&mut self , (c, ): (char, )) -> char::EscapeDefault {
         Fn::call(&*self , (c, ))
     }
 }
 impl FnOnce<(char, )> for CharEscapeDefault {
     type Output = char::EscapeDefault;
-    #[inline] extern "rust-call"fn call_once(self , (c, ): (char, )) -> char::EscapeDefault {
+    #[inline] extern "rust-call" fn call_once(self , (c, ): (char, )) -> char::EscapeDefault {
         Fn::call(&self , (c, ))
     }
 }
@@ -833,7 +833,7 @@ macro_rules! rgb_color {
 /* parse error: expected SEMICOLON */
 /* parse error: expected expression, item or let statement */
 pub fn new() {
-    let _ = 0as u32<<(8+8);
+    let _ = 0 as u32<<(8+8);
 }
 // MACRO_ITEMS@0..31
 //   FN@0..31

crates/hir-def/src/macro_expansion_tests/mod.rs

@@ -33,8 +33,13 @@ use syntax::{
 use tt::token_id::{Subtree, TokenId};
 
 use crate::{
-    db::DefDatabase, macro_id_to_def_id, nameres::ModuleSource, resolver::HasResolver,
-    src::HasSource, test_db::TestDB, AdtId, AsMacroCall, Lookup, ModuleDefId,
+    db::DefDatabase,
+    macro_id_to_def_id,
+    nameres::{MacroSubNs, ModuleSource},
+    resolver::HasResolver,
+    src::HasSource,
+    test_db::TestDB,
+    AdtId, AsMacroCall, Lookup, ModuleDefId,
 };
 
 #[track_caller]
@@ -127,7 +132,9 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream
         let macro_call = InFile::new(source.file_id, &macro_call);
         let res = macro_call
             .as_call_id_with_errors(&db, krate, |path| {
-                resolver.resolve_path_as_macro(&db, &path).map(|it| macro_id_to_def_id(&db, it))
+                resolver
+                    .resolve_path_as_macro(&db, &path, Some(MacroSubNs::Bang))
+                    .map(|it| macro_id_to_def_id(&db, it))
             })
             .unwrap();
         let macro_call_id = res.value.unwrap();
@@ -280,6 +287,7 @@ fn pretty_print_macro_expansion(expn: SyntaxNode, map: Option<&TokenMap>) -> Str
         let curr_kind = token.kind();
         let space = match (prev_kind, curr_kind) {
             _ if prev_kind.is_trivia() || curr_kind.is_trivia() => "",
+            _ if prev_kind.is_literal() && !curr_kind.is_punct() => " ",
             (T!['{'], T!['}']) => "",
             (T![=], _) | (_, T![=]) => " ",
             (_, T!['{']) => " ",

crates/hir-def/src/nameres.rs

@@ -59,7 +59,7 @@ mod tests;
 
 use std::{cmp::Ord, ops::Deref};
 
-use base_db::{CrateId, Edition, FileId};
+use base_db::{CrateId, Edition, FileId, ProcMacroKind};
 use hir_expand::{name::Name, InFile, MacroCallId, MacroDefId};
 use itertools::Itertools;
 use la_arena::Arena;
@@ -77,7 +77,8 @@ use crate::{
     path::ModPath,
     per_ns::PerNs,
     visibility::Visibility,
-    AstId, BlockId, BlockLoc, FunctionId, LocalModuleId, MacroId, ModuleId, ProcMacroId,
+    AstId, BlockId, BlockLoc, FunctionId, LocalModuleId, Lookup, MacroExpander, MacroId, ModuleId,
+    ProcMacroId,
 };
 
 /// Contains the results of (early) name resolution.
@@ -105,6 +106,9 @@ pub struct DefMap {
     prelude: Option<ModuleId>,
     /// The extern prelude is only populated for non-block DefMaps
     extern_prelude: FxHashMap<Name, ModuleId>,
+    /// `macro_use` prelude that contains macros from `#[macro_use]`'d external crates. Note that
+    /// this contains all kinds of macro, not just `macro_rules!` macro.
+    macro_use_prelude: FxHashMap<Name, MacroId>,
 
     /// Side table for resolving derive helpers.
     exported_derives: FxHashMap<MacroDefId, Box<[Name]>>,
@@ -277,6 +281,7 @@ impl DefMap {
             edition,
             recursion_limit: None,
             extern_prelude: FxHashMap::default(),
+            macro_use_prelude: FxHashMap::default(),
             exported_derives: FxHashMap::default(),
             fn_proc_macro_mapping: FxHashMap::default(),
             proc_macro_loading_error: None,
@@ -376,9 +381,16 @@ impl DefMap {
         original_module: LocalModuleId,
         path: &ModPath,
         shadow: BuiltinShadowMode,
+        expected_macro_subns: Option<MacroSubNs>,
     ) -> (PerNs, Option<usize>) {
-        let res =
-            self.resolve_path_fp_with_macro(db, ResolveMode::Other, original_module, path, shadow);
+        let res = self.resolve_path_fp_with_macro(
+            db,
+            ResolveMode::Other,
+            original_module,
+            path,
+            shadow,
+            expected_macro_subns,
+        );
         (res.resolved_def, res.segment_index)
     }
 
@@ -395,6 +407,7 @@ impl DefMap {
             original_module,
             path,
             shadow,
+            None, // Currently this function isn't used for macro resolution.
         );
         (res.resolved_def, res.segment_index)
     }
@@ -489,6 +502,7 @@ impl DefMap {
             _c: _,
             exported_derives,
             extern_prelude,
+            macro_use_prelude,
             diagnostics,
             modules,
             registered_attrs,
@@ -507,6 +521,7 @@ impl DefMap {
         } = self;
 
         extern_prelude.shrink_to_fit();
+        macro_use_prelude.shrink_to_fit();
         exported_derives.shrink_to_fit();
         diagnostics.shrink_to_fit();
         modules.shrink_to_fit();
@@ -562,3 +577,48 @@ pub enum ModuleSource {
     Module(ast::Module),
     BlockExpr(ast::BlockExpr),
 }
+
+/// See `sub_namespace_match()`.
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub enum MacroSubNs {
+    /// Function-like macros, suffixed with `!`.
+    Bang,
+    /// Macros inside attributes, i.e. attribute macros and derive macros.
+    Attr,
+}
+
+impl MacroSubNs {
+    fn from_id(db: &dyn DefDatabase, macro_id: MacroId) -> Self {
+        let expander = match macro_id {
+            MacroId::Macro2Id(it) => it.lookup(db).expander,
+            MacroId::MacroRulesId(it) => it.lookup(db).expander,
+            MacroId::ProcMacroId(it) => {
+                return match it.lookup(db).kind {
+                    ProcMacroKind::CustomDerive | ProcMacroKind::Attr => Self::Attr,
+                    ProcMacroKind::FuncLike => Self::Bang,
+                };
+            }
+        };
+
+        // Eager macros aren't *guaranteed* to be bang macros, but they *are* all bang macros currently.
+        match expander {
+            MacroExpander::Declarative
+            | MacroExpander::BuiltIn(_)
+            | MacroExpander::BuiltInEager(_) => Self::Bang,
+            MacroExpander::BuiltInAttr(_) | MacroExpander::BuiltInDerive(_) => Self::Attr,
+        }
+    }
+}
+
+/// Quoted from [rustc]:
+/// Macro namespace is separated into two sub-namespaces, one for bang macros and
+/// one for attribute-like macros (attributes, derives).
+/// We ignore resolutions from one sub-namespace when searching names in scope for another.
+///
+/// [rustc]: https://github.com/rust-lang/rust/blob/1.69.0/compiler/rustc_resolve/src/macros.rs#L75
+fn sub_namespace_match(candidate: Option<MacroSubNs>, expected: Option<MacroSubNs>) -> bool {
+    match (candidate, expected) {
+        (Some(candidate), Some(expected)) => candidate == expected,
+        _ => true,
+    }
+}

crates/hir-def/src/nameres/attr_resolution.rs

@@ -14,7 +14,7 @@ use crate::{
     AstIdWithPath, LocalModuleId, UnresolvedMacro,
 };
 
-use super::DefMap;
+use super::{DefMap, MacroSubNs};
 
 pub enum ResolvedAttr {
     /// Attribute resolved to an attribute macro.
@@ -43,9 +43,12 @@ impl DefMap {
             original_module,
             &ast_id.path,
             BuiltinShadowMode::Module,
+            Some(MacroSubNs::Attr),
         );
         let def = match resolved_res.resolved_def.take_macros() {
             Some(def) => {
+                // `MacroSubNs` is just a hint, so the path may still resolve to a custom derive
+                // macro, or even function-like macro when the path is qualified.
                 if def.is_attribute(db) {
                     def
                 } else {