Merge #6897

6897: Basic support for macros 2.0 r=jonas-schievink a=jonas-schievink

This adds support for (built-in-only) macros 2.0, and removes some hacks used for builtin derives, which are declared via macros 2.0 in libcore.

First steps for #2248.

Blocked on rust-analyzer/ungrammar#16.

Co-authored-by: Jonas Schievink <jonasschievink@gmail.com>
Co-authored-by: Jonas Schievink <jonas.schievink@ferrous-systems.com>

crates/hir/src/code_model.rs

@@ -970,7 +970,7 @@ impl MacroDef {
     /// defines this macro. The reasons for this is that macros are expanded
     /// early, in `hir_expand`, where modules simply do not exist yet.
     pub fn module(self, db: &dyn HirDatabase) -> Option<Module> {
-        let krate = self.id.krate?;
+        let krate = self.id.krate;
         let module_id = db.crate_def_map(krate).root;
         Some(Module::new(Crate { id: krate }, module_id))
     }

crates/hir/src/has_source.rs

@@ -110,8 +110,8 @@ impl HasSource for TypeAlias {
     }
 }
 impl HasSource for MacroDef {
-    type Ast = ast::MacroRules;
-    fn source(self, db: &dyn HirDatabase) -> InFile<ast::MacroRules> {
+    type Ast = ast::Macro;
+    fn source(self, db: &dyn HirDatabase) -> InFile<ast::Macro> {
         InFile {
             file_id: self.id.ast_id.expect("MacroDef without ast_id").file_id,
             value: self.id.ast_id.expect("MacroDef without ast_id").to_node(db.upcast()),

crates/hir/src/semantics/source_to_def.rs

@@ -157,8 +157,8 @@ impl SourceToDefCtx<'_, '_> {
         let file_id = src.file_id.original_file(self.db.upcast());
         let krate = self.file_to_def(file_id)?.krate;
         let file_ast_id = self.db.ast_id_map(src.file_id).ast_id(&src.value);
-        let ast_id = Some(AstId::new(src.file_id, file_ast_id));
-        Some(MacroDefId { krate: Some(krate), ast_id, kind, local_inner: false })
+        let ast_id = Some(AstId::new(src.file_id, file_ast_id.upcast()));
+        Some(MacroDefId { krate, ast_id, kind, local_inner: false })
     }
 
     pub(super) fn find_container(&mut self, src: InFile<&SyntaxNode>) -> Option<ChildContainer> {

crates/hir_def/src/body/lower.rs

@@ -772,7 +772,10 @@ impl ExprCollector<'_> {
                     | ast::Item::Module(_)
                     | ast::Item::MacroCall(_) => return None,
                     ast::Item::MacroRules(def) => {
-                        return Some(Either::Right(def));
+                        return Some(Either::Right(ast::Macro::from(def)));
+                    }
+                    ast::Item::MacroDef(def) => {
+                        return Some(Either::Right(ast::Macro::from(def)));
                     }
                 };
 
@@ -800,7 +803,7 @@ impl ExprCollector<'_> {
                 }
                 Either::Right(e) => {
                     let mac = MacroDefId {
-                        krate: Some(self.expander.module.krate),
+                        krate: self.expander.module.krate,
                         ast_id: Some(self.expander.ast_id(&e)),
                         kind: MacroDefKind::Declarative,
                         local_inner: false,

crates/hir_def/src/item_scope.rs

@@ -363,7 +363,7 @@ impl ItemInNs {
                 ModuleDefId::TypeAliasId(id) => id.lookup(db).module(db).krate,
                 ModuleDefId::BuiltinType(_) => return None,
             },
-            ItemInNs::Macros(id) => return id.krate,
+            ItemInNs::Macros(id) => return Some(id.krate),
         })
     }
 }

crates/hir_def/src/item_tree.rs

@@ -143,6 +143,7 @@ impl ItemTree {
                 mods,
                 macro_calls,
                 macro_rules,
+                macro_defs,
                 exprs,
                 vis,
                 generics,
@@ -164,6 +165,7 @@ impl ItemTree {
             mods.shrink_to_fit();
             macro_calls.shrink_to_fit();
             macro_rules.shrink_to_fit();
+            macro_defs.shrink_to_fit();
             exprs.shrink_to_fit();
 
             vis.arena.shrink_to_fit();
@@ -283,6 +285,7 @@ struct ItemTreeData {
     mods: Arena<Mod>,
     macro_calls: Arena<MacroCall>,
     macro_rules: Arena<MacroRules>,
+    macro_defs: Arena<MacroDef>,
     exprs: Arena<Expr>,
 
     vis: ItemVisibilities,
@@ -431,6 +434,7 @@ mod_items! {
     Mod in mods -> ast::Module,
     MacroCall in macro_calls -> ast::MacroCall,
     MacroRules in macro_rules -> ast::MacroRules,
+    MacroDef in macro_defs -> ast::MacroDef,
 }
 
 macro_rules! impl_index {
@@ -640,7 +644,7 @@ pub struct MacroCall {
 
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub struct MacroRules {
-    /// For `macro_rules!` declarations, this is the name of the declared macro.
+    /// The name of the declared macro.
     pub name: Name,
     /// Has `#[macro_export]`.
     pub is_export: bool,
@@ -651,6 +655,16 @@ pub struct MacroRules {
     pub ast_id: FileAstId<ast::MacroRules>,
 }
 
+/// "Macros 2.0" macro definition.
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct MacroDef {
+    pub name: Name,
+    pub visibility: RawVisibilityId,
+    /// Has `#[rustc_builtin_macro]`.
+    pub is_builtin: bool,
+    pub ast_id: FileAstId<ast::MacroDef>,
+}
+
 // NB: There's no `FileAstId` for `Expr`. The only case where this would be useful is for array
 // lengths, but we don't do much with them yet.
 #[derive(Debug, Clone, Eq, PartialEq)]
@@ -680,7 +694,8 @@ impl ModItem {
             | ModItem::Trait(_)
             | ModItem::Impl(_)
             | ModItem::Mod(_)
-            | ModItem::MacroRules(_) => None,
+            | ModItem::MacroRules(_)
+            | ModItem::MacroDef(_) => None,
             ModItem::MacroCall(call) => Some(AssocItem::MacroCall(*call)),
             ModItem::Const(konst) => Some(AssocItem::Const(*konst)),
             ModItem::TypeAlias(alias) => Some(AssocItem::TypeAlias(*alias)),
@@ -708,6 +723,7 @@ impl ModItem {
             ModItem::Mod(it) => tree[it.index].ast_id().upcast(),
             ModItem::MacroCall(it) => tree[it.index].ast_id().upcast(),
             ModItem::MacroRules(it) => tree[it.index].ast_id().upcast(),
+            ModItem::MacroDef(it) => tree[it.index].ast_id().upcast(),
         }
     }
 }

crates/hir_def/src/item_tree/lower.rs

@@ -101,7 +101,8 @@ impl Ctx {
             | ast::Item::ExternCrate(_)
             | ast::Item::Use(_)
             | ast::Item::MacroCall(_)
-            | ast::Item::MacroRules(_) => {}
+            | ast::Item::MacroRules(_)
+            | ast::Item::MacroDef(_) => {}
         };
 
         let attrs = Attrs::new(item, &self.hygiene);
@@ -122,6 +123,7 @@ impl Ctx {
             ast::Item::ExternCrate(ast) => self.lower_extern_crate(ast).map(Into::into),
             ast::Item::MacroCall(ast) => self.lower_macro_call(ast).map(Into::into),
             ast::Item::MacroRules(ast) => self.lower_macro_rules(ast).map(Into::into),
+            ast::Item::MacroDef(ast) => self.lower_macro_def(ast).map(Into::into),
             ast::Item::ExternBlock(ast) => {
                 Some(ModItems(self.lower_extern_block(ast).into_iter().collect::<SmallVec<_>>()))
             }
@@ -561,6 +563,18 @@ impl Ctx {
         Some(id(self.data().macro_rules.alloc(res)))
     }
 
+    fn lower_macro_def(&mut self, m: &ast::MacroDef) -> Option<FileItemTreeId<MacroDef>> {
+        let name = m.name().map(|it| it.as_name())?;
+        let attrs = Attrs::new(m, &self.hygiene);
+
+        let ast_id = self.source_ast_id_map.ast_id(m);
+        let visibility = self.lower_visibility(m);
+
+        let is_builtin = attrs.by_key("rustc_builtin_macro").exists();
+        let res = MacroDef { name, is_builtin, ast_id, visibility };
+        Some(id(self.data().macro_defs.alloc(res)))
+    }
+
     fn lower_extern_block(&mut self, block: &ast::ExternBlock) -> Vec<ModItem> {
         block.extern_item_list().map_or(Vec::new(), |list| {
             list.extern_items()

crates/hir_def/src/nameres/collector.rs

@@ -309,13 +309,13 @@ impl DefCollector<'_> {
         let macro_def = match self.proc_macros.iter().find(|(n, _)| n == name) {
             Some((_, expander)) => MacroDefId {
                 ast_id: None,
-                krate: Some(self.def_map.krate),
+                krate: self.def_map.krate,
                 kind: MacroDefKind::ProcMacro(*expander),
                 local_inner: false,
             },
             None => MacroDefId {
                 ast_id: None,
-                krate: Some(self.def_map.krate),
+                krate: self.def_map.krate,
                 kind: MacroDefKind::ProcMacro(ProcMacroExpander::dummy(self.def_map.krate)),
                 local_inner: false,
             },
@@ -784,14 +784,6 @@ impl DefCollector<'_> {
         directive: &DeriveDirective,
         path: &ModPath,
     ) -> Option<MacroDefId> {
-        if let Some(name) = path.as_ident() {
-            // FIXME this should actually be handled with the normal name
-            // resolution; the std lib defines built-in stubs for the derives,
-            // but these are new-style `macro`s, which we don't support yet
-            if let Some(def_id) = find_builtin_derive(name) {
-                return Some(def_id);
-            }
-        }
         let resolved_res = self.def_map.resolve_path_fp_with_macro(
             self.db,
             ResolveMode::Other,
@@ -976,6 +968,35 @@ impl ModCollector<'_, '_> {
                 }
                 ModItem::MacroCall(mac) => self.collect_macro_call(&self.item_tree[mac]),
                 ModItem::MacroRules(mac) => self.collect_macro_rules(&self.item_tree[mac]),
+                ModItem::MacroDef(id) => {
+                    let mac = &self.item_tree[id];
+                    let ast_id = InFile::new(self.file_id, mac.ast_id.upcast());
+
+                    // "Macro 2.0" is not currently supported by rust-analyzer, but libcore uses it
+                    // to define builtin macros, so we support at least that part.
+                    if mac.is_builtin {
+                        let krate = self.def_collector.def_map.krate;
+                        let macro_id = find_builtin_macro(&mac.name, krate, ast_id)
+                            .or_else(|| find_builtin_derive(&mac.name, krate, ast_id));
+                        if let Some(macro_id) = macro_id {
+                            let vis = self
+                                .def_collector
+                                .def_map
+                                .resolve_visibility(
+                                    self.def_collector.db,
+                                    self.module_id,
+                                    &self.item_tree[mac.visibility],
+                                )
+                                .unwrap_or(Visibility::Public);
+                            self.def_collector.update(
+                                self.module_id,
+                                &[(Some(mac.name.clone()), PerNs::macros(macro_id, vis))],
+                                vis,
+                                ImportType::Named,
+                            );
+                        }
+                    }
+                }
                 ModItem::Impl(imp) => {
                     let module = ModuleId {
                         krate: self.def_collector.def_map.krate,
@@ -1280,7 +1301,7 @@ impl ModCollector<'_, '_> {
     }
 
     fn collect_macro_rules(&mut self, mac: &MacroRules) {
-        let ast_id = InFile::new(self.file_id, mac.ast_id);
+        let ast_id = InFile::new(self.file_id, mac.ast_id.upcast());
 
         // Case 1: builtin macros
         if mac.is_builtin {
@@ -1299,7 +1320,7 @@ impl ModCollector<'_, '_> {
         // Case 2: normal `macro_rules!` macro
         let macro_id = MacroDefId {
             ast_id: Some(ast_id),
-            krate: Some(self.def_collector.def_map.krate),
+            krate: self.def_collector.def_map.krate,
             kind: MacroDefKind::Declarative,
             local_inner: mac.is_local_inner,
         };

crates/hir_def/src/nameres/tests/macros.rs

@@ -633,14 +633,43 @@ pub struct bar;
 fn expand_derive() {
     let map = compute_crate_def_map(
         "
-        //- /main.rs
+        //- /main.rs crate:main deps:core
+        use core::*;
+
         #[derive(Copy, Clone)]
         struct Foo;
+
+        //- /core.rs crate:core
+        #[rustc_builtin_macro]
+        pub macro Copy {}
+
+        #[rustc_builtin_macro]
+        pub macro Clone {}
         ",
     );
     assert_eq!(map.modules[map.root].scope.impls().len(), 2);
 }
 
+#[test]
+fn resolve_builtin_derive() {
+    check(
+        r#"
+//- /main.rs crate:main deps:core
+use core::*;
+
+//- /core.rs crate:core
+#[rustc_builtin_macro]
+pub macro Clone {}
+
+pub trait Clone {}
+"#,
+        expect![[r#"
+            crate
+            Clone: t m
+        "#]],
+    );
+}
+
 #[test]
 fn macro_expansion_overflow() {
     mark::check!(macro_expansion_overflow);

crates/hir_expand/src/builtin_derive.rs

@@ -8,7 +8,7 @@ use syntax::{
     match_ast,
 };
 
-use crate::{db::AstDatabase, name, quote, LazyMacroId, MacroDefId, MacroDefKind};
+use crate::{db::AstDatabase, name, quote, AstId, CrateId, LazyMacroId, MacroDefId, MacroDefKind};
 
 macro_rules! register_builtin {
     ( $($trait:ident => $expand:ident),* ) => {
@@ -29,16 +29,15 @@ macro_rules! register_builtin {
                 };
                 expander(db, id, tt)
             }
-        }
-
-        pub fn find_builtin_derive(ident: &name::Name) -> Option<MacroDefId> {
-            let kind = match ident {
-                $( id if id == &name::name![$trait] => BuiltinDeriveExpander::$trait, )*
-                 _ => return None,
-            };
 
-            Some(MacroDefId { krate: None, ast_id: None, kind: MacroDefKind::BuiltInDerive(kind), local_inner: false })
+            fn find_by_name(name: &name::Name) -> Option<Self> {
+                match name {
+                    $( id if id == &name::name![$trait] => Some(BuiltinDeriveExpander::$trait), )*
+                     _ => None,
+                }
+            }
         }
+
     };
 }
 
@@ -54,6 +53,20 @@ register_builtin! {
     PartialEq => partial_eq_expand
 }
 
+pub fn find_builtin_derive(
+    ident: &name::Name,
+    krate: CrateId,
+    ast_id: AstId<ast::Macro>,
+) -> Option<MacroDefId> {
+    let expander = BuiltinDeriveExpander::find_by_name(ident)?;
+    Some(MacroDefId {
+        krate,
+        ast_id: Some(ast_id),
+        kind: MacroDefKind::BuiltInDerive(expander),
+        local_inner: false,
+    })
+}
+
 struct BasicAdtInfo {
     name: tt::Ident,
     type_params: usize,
@@ -261,7 +274,7 @@ mod tests {
     use super::*;
 
     fn expand_builtin_derive(s: &str, name: Name) -> String {
-        let def = find_builtin_derive(&name).unwrap();
+        let expander = BuiltinDeriveExpander::find_by_name(&name).unwrap();
         let fixture = format!(
             r#"//- /main.rs crate:main deps:core
 <|>
@@ -283,7 +296,12 @@ mod tests {
         let attr_id = AstId::new(file_id.into(), ast_id_map.ast_id(&items[0]));
 
         let loc = MacroCallLoc {
-            def,
+            def: MacroDefId {
+                krate: CrateId(0),
+                ast_id: None,
+                kind: MacroDefKind::BuiltInDerive(expander),
+                local_inner: false,
+            },
             krate: CrateId(0),
             kind: MacroCallKind::Attr(attr_id, name.to_string()),
         };