Auto merge of #133540 - ehuss:compiletest-proc-macro, r=jieyouxu

Compiletest: add proc-macro header

This adds a `proc-macro` header to simplify using proc-macros, and to reduce boilerplate. This header works similar to the `aux-build` header where you pass a path for a proc-macro to be built.

This allows the `force-host`, `no-prefer-dynamic` headers, and `crate_type` attribute to be removed. Additionally it uses `--extern` like `aux_crate` (allows implicit `extern crate` in 2018) and `--extern proc_macro` (to place in the prelude in 2018).

~~This also includes a secondary change which defaults the edition of proc-macros to 2024. This further reduces boilerplate (removing `extern crate proc_macro;`), and allows using modern Rust syntax. I was a little on the fence including this. I personally prefer it, but I can imagine it might be confusing to others.~~ EDIT: Removed

Some tests were changed so that when there is a chain of dependencies A→B→C, that the `@ proc-macro` is placed in `B` instead of `A` so that the `--extern` flag works correctly (previously it depended on `-L` to find `C`). I think this is better to make the dependencies more explicit. None of these tests looked like the were actually testing this behavior.

There is one test that had an unexplained output change: `tests/ui/macros/same-sequence-span.rs`. I do not know why it changed, but it didn't look like it was particularly important. Perhaps there was a normalization issue?

This is currently not compatible with the rustdoc `build-aux-docs` header. It can probably be fixed, I'm just not feeling motivated to do that right now.

### Implementation steps

- [x] Document this new behavior in rustc-dev-guide once we figure out the specifics. rust-lang/rustc-dev-guide#2149

src/tools/compiletest/src/directive-list.rs

@@ -215,6 +215,7 @@ const KNOWN_DIRECTIVE_NAMES: &[&str] = &[
     "pp-exact",
     "pretty-compare-only",
     "pretty-mode",
+    "proc-macro",
     "reference",
     "regex-error-pattern",
     "remap-src-base",

src/tools/compiletest/src/header.rs

@@ -221,6 +221,7 @@ mod directives {
     pub const AUX_BIN: &'static str = "aux-bin";
     pub const AUX_BUILD: &'static str = "aux-build";
     pub const AUX_CRATE: &'static str = "aux-crate";
+    pub const PROC_MACRO: &'static str = "proc-macro";
     pub const AUX_CODEGEN_BACKEND: &'static str = "aux-codegen-backend";
     pub const EXEC_ENV: &'static str = "exec-env";
     pub const RUSTC_ENV: &'static str = "rustc-env";

src/tools/compiletest/src/header/auxiliary.rs

@@ -4,7 +4,7 @@
 use std::iter;
 
 use crate::common::Config;
-use crate::header::directives::{AUX_BIN, AUX_BUILD, AUX_CODEGEN_BACKEND, AUX_CRATE};
+use crate::header::directives::{AUX_BIN, AUX_BUILD, AUX_CODEGEN_BACKEND, AUX_CRATE, PROC_MACRO};
 
 /// Properties parsed from `aux-*` test directives.
 #[derive(Clone, Debug, Default)]
@@ -17,6 +17,8 @@ pub(crate) struct AuxProps {
     /// Similar to `builds`, but a list of NAME=somelib.rs of dependencies
     /// to build and pass with the `--extern` flag.
     pub(crate) crates: Vec<(String, String)>,
+    /// Same as `builds`, but for proc-macros.
+    pub(crate) proc_macros: Vec<String>,
     /// Similar to `builds`, but also uses the resulting dylib as a
     /// `-Zcodegen-backend` when compiling the test file.
     pub(crate) codegen_backend: Option<String>,
@@ -26,26 +28,29 @@ impl AuxProps {
     /// Yields all of the paths (relative to `./auxiliary/`) that have been
     /// specified in `aux-*` directives for this test.
     pub(crate) fn all_aux_path_strings(&self) -> impl Iterator<Item = &str> {
-        let Self { builds, bins, crates, codegen_backend } = self;
+        let Self { builds, bins, crates, proc_macros, codegen_backend } = self;
 
         iter::empty()
             .chain(builds.iter().map(String::as_str))
             .chain(bins.iter().map(String::as_str))
             .chain(crates.iter().map(|(_, path)| path.as_str()))
+            .chain(proc_macros.iter().map(String::as_str))
             .chain(codegen_backend.iter().map(String::as_str))
     }
 }
 
 /// If the given test directive line contains an `aux-*` directive, parse it
 /// and update [`AuxProps`] accordingly.
 pub(super) fn parse_and_update_aux(config: &Config, ln: &str, aux: &mut AuxProps) {
-    if !ln.starts_with("aux-") {
+    if !(ln.starts_with("aux-") || ln.starts_with("proc-macro")) {
         return;
     }
 
     config.push_name_value_directive(ln, AUX_BUILD, &mut aux.builds, |r| r.trim().to_string());
     config.push_name_value_directive(ln, AUX_BIN, &mut aux.bins, |r| r.trim().to_string());
     config.push_name_value_directive(ln, AUX_CRATE, &mut aux.crates, parse_aux_crate);
+    config
+        .push_name_value_directive(ln, PROC_MACRO, &mut aux.proc_macros, |r| r.trim().to_string());
     if let Some(r) = config.parse_name_value_directive(ln, AUX_CODEGEN_BACKEND) {
         aux.codegen_backend = Some(r.trim().to_owned());
     }

src/tools/compiletest/src/runtest.rs

@@ -102,7 +102,7 @@ fn get_lib_name(name: &str, aux_type: AuxType) -> Option<String> {
         // In this case, the only path we can pass
         // with '--extern-meta' is the '.rlib' file
         AuxType::Lib => Some(format!("lib{name}.rlib")),
-        AuxType::Dylib => Some(dylib_name(name)),
+        AuxType::Dylib | AuxType::ProcMacro => Some(dylib_name(name)),
     }
 }
 
@@ -1097,7 +1097,9 @@ impl<'test> TestCx<'test> {
     }
 
     fn has_aux_dir(&self) -> bool {
-        !self.props.aux.builds.is_empty() || !self.props.aux.crates.is_empty()
+        !self.props.aux.builds.is_empty()
+            || !self.props.aux.crates.is_empty()
+            || !self.props.aux.proc_macros.is_empty()
     }
 
     fn aux_output_dir(&self) -> PathBuf {
@@ -1118,31 +1120,48 @@ impl<'test> TestCx<'test> {
 
     fn build_all_auxiliary(&self, of: &TestPaths, aux_dir: &Path, rustc: &mut Command) {
         for rel_ab in &self.props.aux.builds {
-            self.build_auxiliary(of, rel_ab, &aux_dir, false /* is_bin */);
+            self.build_auxiliary(of, rel_ab, &aux_dir, None);
         }
 
         for rel_ab in &self.props.aux.bins {
-            self.build_auxiliary(of, rel_ab, &aux_dir, true /* is_bin */);
+            self.build_auxiliary(of, rel_ab, &aux_dir, Some(AuxType::Bin));
         }
 
+        let path_to_crate_name = |path: &str| -> String {
+            path.rsplit_once('/')
+                .map_or(path, |(_, tail)| tail)
+                .trim_end_matches(".rs")
+                .replace('-', "_")
+        };
+
+        let add_extern =
+            |rustc: &mut Command, aux_name: &str, aux_path: &str, aux_type: AuxType| {
+                let lib_name = get_lib_name(&path_to_crate_name(aux_path), aux_type);
+                if let Some(lib_name) = lib_name {
+                    rustc.arg("--extern").arg(format!(
+                        "{}={}/{}",
+                        aux_name,
+                        aux_dir.display(),
+                        lib_name
+                    ));
+                }
+            };
+
         for (aux_name, aux_path) in &self.props.aux.crates {
-            let aux_type = self.build_auxiliary(of, &aux_path, &aux_dir, false /* is_bin */);
-            let lib_name =
-                get_lib_name(&aux_path.trim_end_matches(".rs").replace('-', "_"), aux_type);
-            if let Some(lib_name) = lib_name {
-                rustc.arg("--extern").arg(format!(
-                    "{}={}/{}",
-                    aux_name,
-                    aux_dir.display(),
-                    lib_name
-                ));
-            }
+            let aux_type = self.build_auxiliary(of, &aux_path, &aux_dir, None);
+            add_extern(rustc, aux_name, aux_path, aux_type);
+        }
+
+        for proc_macro in &self.props.aux.proc_macros {
+            self.build_auxiliary(of, proc_macro, &aux_dir, Some(AuxType::ProcMacro));
+            let crate_name = path_to_crate_name(proc_macro);
+            add_extern(rustc, &crate_name, proc_macro, AuxType::ProcMacro);
         }
 
         // Build any `//@ aux-codegen-backend`, and pass the resulting library
         // to `-Zcodegen-backend` when compiling the test file.
         if let Some(aux_file) = &self.props.aux.codegen_backend {
-            let aux_type = self.build_auxiliary(of, aux_file, aux_dir, false);
+            let aux_type = self.build_auxiliary(of, aux_file, aux_dir, None);
             if let Some(lib_name) = get_lib_name(aux_file.trim_end_matches(".rs"), aux_type) {
                 let lib_path = aux_dir.join(&lib_name);
                 rustc.arg(format!("-Zcodegen-backend={}", lib_path.display()));
@@ -1209,17 +1228,23 @@ impl<'test> TestCx<'test> {
     }
 
     /// Builds an aux dependency.
+    ///
+    /// If `aux_type` is `None`, then this will determine the aux-type automatically.
     fn build_auxiliary(
         &self,
         of: &TestPaths,
         source_path: &str,
         aux_dir: &Path,
-        is_bin: bool,
+        aux_type: Option<AuxType>,
     ) -> AuxType {
         let aux_testpaths = self.compute_aux_test_paths(of, source_path);
-        let aux_props = self.props.from_aux_file(&aux_testpaths.file, self.revision, self.config);
+        let mut aux_props =
+            self.props.from_aux_file(&aux_testpaths.file, self.revision, self.config);
+        if aux_type == Some(AuxType::ProcMacro) {
+            aux_props.force_host = true;
+        }
         let mut aux_dir = aux_dir.to_path_buf();
-        if is_bin {
+        if aux_type == Some(AuxType::Bin) {
             // On unix, the binary of `auxiliary/foo.rs` will be named
             // `auxiliary/foo` which clashes with the _dir_ `auxiliary/foo`, so
             // put bins in a `bin` subfolder.
@@ -1250,8 +1275,12 @@ impl<'test> TestCx<'test> {
             aux_rustc.env_remove(key);
         }
 
-        let (aux_type, crate_type) = if is_bin {
+        let (aux_type, crate_type) = if aux_type == Some(AuxType::Bin) {
             (AuxType::Bin, Some("bin"))
+        } else if aux_type == Some(AuxType::ProcMacro) {
+            (AuxType::ProcMacro, Some("proc-macro"))
+        } else if aux_type.is_some() {
+            panic!("aux_type {aux_type:?} not expected");
         } else if aux_props.no_prefer_dynamic {
             (AuxType::Dylib, None)
         } else if self.config.target.contains("emscripten")
@@ -1287,6 +1316,11 @@ impl<'test> TestCx<'test> {
             aux_rustc.args(&["--crate-type", crate_type]);
         }
 
+        if aux_type == AuxType::ProcMacro {
+            // For convenience, but this only works on 2018.
+            aux_rustc.args(&["--extern", "proc_macro"]);
+        }
+
         aux_rustc.arg("-L").arg(&aux_dir);
 
         let auxres = aux_cx.compose_and_run(
@@ -2768,8 +2802,10 @@ enum LinkToAux {
     No,
 }
 
+#[derive(Debug, PartialEq)]
 enum AuxType {
     Bin,
     Lib,
     Dylib,
+    ProcMacro,
 }

tests/codegen/debuginfo-proc-macro/auxiliary/macro_def.rs

@@ -1,7 +1,3 @@
-//@ force-host
-//@ no-prefer-dynamic
-#![crate_type = "proc-macro"]
-
 extern crate proc_macro;
 use proc_macro::*;
 

tests/codegen/debuginfo-proc-macro/mir_inlined_twice_var_locs.rs

@@ -3,7 +3,7 @@
 // MSVC is different because of the individual allocas.
 //@ ignore-msvc
 
-//@ aux-build:macro_def.rs
+//@ proc-macro: macro_def.rs
 
 // Find the variable.
 // CHECK-DAG: ![[#var_dbg:]] = !DILocalVariable(name: "n",{{( arg: 1,)?}} scope: ![[#var_scope:]]

tests/incremental/auxiliary/incremental_proc_macro_aux.rs

@@ -1,8 +1,3 @@
-//@ force-host
-//@ no-prefer-dynamic
-
-#![crate_type = "proc-macro"]
-
 extern crate proc_macro;
 
 use proc_macro::TokenStream;

tests/incremental/auxiliary/issue-49482-macro-def.rs

@@ -1,7 +1,3 @@
-//@ force-host
-//@ no-prefer-dynamic
-
-#![crate_type="proc-macro"]
 #![allow(non_snake_case)]
 
 extern crate proc_macro;

tests/incremental/auxiliary/issue-49482-reexport.rs

@@ -1,3 +1,4 @@
+//@ proc-macro: issue-49482-macro-def.rs
 #[macro_use]
 extern crate issue_49482_macro_def;
 

tests/incremental/auxiliary/issue-54059.rs

@@ -1,9 +1,5 @@
-//@ force-host
-//@ no-prefer-dynamic
-
 // check that having extern "C" functions in a proc macro doesn't crash.
 
-#![crate_type="proc-macro"]
 #![allow(non_snake_case)]
 
 extern crate proc_macro;

tests/incremental/incremental_proc_macro.rs

@@ -1,4 +1,4 @@
-//@ aux-build:incremental_proc_macro_aux.rs
+//@ proc-macro: incremental_proc_macro_aux.rs
 //@ revisions: cfail1 cfail2
 //@ build-pass (FIXME(62277): could be check-pass?)
 

tests/incremental/issue-110457-same-span-closures/auxiliary/egui_inspect_derive.rs

@@ -1,7 +1,3 @@
-//@ force-host
-//@ no-prefer-dynamic
-#![crate_type = "proc-macro"]
-
 extern crate proc_macro;
 
 use proc_macro::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};

tests/incremental/issue-110457-same-span-closures/main.rs

@@ -1,4 +1,4 @@
-//@ aux-build: egui_inspect_derive.rs
+//@ proc-macro: egui_inspect_derive.rs
 //@ revisions: cpass1 cpass2
 
 extern crate egui_inspect_derive;

tests/incremental/issue-49482.rs

@@ -1,4 +1,3 @@
-//@ aux-build:issue-49482-macro-def.rs
 //@ aux-build:issue-49482-reexport.rs
 //@ revisions: rpass1
 

tests/incremental/issue-54059.rs

@@ -1,4 +1,4 @@
-//@ aux-build:issue-54059.rs
+//@ proc-macro: issue-54059.rs
 //@ ignore-windows - dealing with weird symbols issues on dylibs isn't worth it
 //@ revisions: rpass1
 

tests/incremental/issue-85197-invalid-span/auxiliary/invalid-span-helper-lib.rs

@@ -1,3 +1,4 @@
+//@ proc-macro: respan.rs
 //@ revisions: rpass1 rpass2
 
 extern crate respan;

tests/incremental/issue-85197-invalid-span/auxiliary/respan.rs

@@ -1,8 +1,3 @@
-//@ force-host
-//@ no-prefer-dynamic
-
-#![crate_type = "proc-macro"]
-
 extern crate proc_macro;
 use proc_macro::TokenStream;
 

tests/incremental/issue-85197-invalid-span/invalid_span_main.rs

@@ -1,5 +1,4 @@
 //@ revisions: rpass1 rpass2
-//@ aux-build:respan.rs
 //@ aux-build:invalid-span-helper-lib.rs
 
 // This issue has several different parts. The high level idea is:

tests/pretty/attr-derive.rs

@@ -1,4 +1,4 @@
-//@ aux-build:derive-foo.rs
+//@ proc-macro: derive-foo.rs
 //@ pp-exact
 // Testing that both the inner item and next outer item are
 // preserved, and that the first outer item parsed in main is not

tests/pretty/auxiliary/derive-foo.rs

@@ -1,8 +1,3 @@
-//@ force-host
-//@ no-prefer-dynamic
-
-#![crate_type = "proc-macro"]
-
 extern crate proc_macro;
 
 use proc_macro::TokenStream;

tests/ui/annotate-snippet/auxiliary/multispan.rs

@@ -1,7 +1,3 @@
-//@ force-host
-//@ no-prefer-dynamic
-
-#![crate_type = "proc-macro"]
 #![feature(proc_macro_diagnostic, proc_macro_span, proc_macro_def_site)]
 
 extern crate proc_macro;

tests/ui/annotate-snippet/multispan.rs

@@ -1,4 +1,4 @@
-//@ aux-build:multispan.rs
+//@ proc-macro: multispan.rs
 //@ error-pattern:hello to you, too!
 //@ compile-flags: --error-format human-annotate-rs -Z unstable-options
 

tests/ui/async-await/issues/auxiliary/issue-60674.rs

@@ -1,7 +1,3 @@
-//@ force-host
-//@ no-prefer-dynamic
-#![crate_type = "proc-macro"]
-
 extern crate proc_macro;
 use proc_macro::TokenStream;
 

tests/ui/async-await/issues/issue-60674.rs

@@ -1,4 +1,4 @@
-//@ aux-build:issue-60674.rs
+//@ proc-macro: issue-60674.rs
 //@ build-pass (FIXME(62277): could be check-pass?)
 //@ edition:2018
 

tests/ui/attributes/auxiliary/key-value-expansion.rs

@@ -1,8 +1,3 @@
-//@ force-host
-//@ no-prefer-dynamic
-
-#![crate_type = "proc-macro"]
-
 extern crate proc_macro;
 use proc_macro::*;
 

tests/ui/attributes/key-value-expansion.rs

@@ -1,7 +1,7 @@
 // Regression tests for issue #55414, expansion happens in the value of a key-value attribute,
 // and the expanded expression is more complex than simply a macro call.
 
-//@ aux-build:key-value-expansion.rs
+//@ proc-macro: key-value-expansion.rs
 
 #![feature(rustc_attrs)]
 

tests/ui/attributes/main-removed-2/auxiliary/tokyo.rs

@@ -1,8 +1,3 @@
-//@ force-host
-//@ no-prefer-dynamic
-
-#![crate_type = "proc-macro"]
-
 extern crate proc_macro;
 use proc_macro::TokenStream;
 

tests/ui/attributes/main-removed-2/main.rs

@@ -1,5 +1,5 @@
 //@ run-pass
-//@ aux-build:tokyo.rs
+//@ proc-macro: tokyo.rs
 //@ compile-flags:--extern tokyo
 //@ edition:2021
 

tests/ui/autodiff/auxiliary/my_macro.rs

@@ -1,7 +1,3 @@
-//@ force-host
-//@ no-prefer-dynamic
-#![crate_type = "proc-macro"]
-
 extern crate proc_macro;
 use proc_macro::TokenStream;