Auto merge of #16751 - Veykril:codegen, r=Veykril

internal: Move diagnostics docs generation and lint definition generation into xtask/codegen

crates/ide-assists/Cargo.toml

@@ -33,8 +33,5 @@ expect-test = "1.4.0"
 test-utils.workspace = true
 test-fixture.workspace = true
 
-[features]
-in-rust-tree = []
-
 [lints]
 workspace = true

crates/ide-db/Cargo.toml

@@ -44,13 +44,10 @@ line-index.workspace = true
 
 [dev-dependencies]
 expect-test = "1.4.0"
-oorandom = "11.1.3"
-xshell.workspace = true
 
 # local deps
 test-utils.workspace = true
 test-fixture.workspace = true
-sourcegen.workspace = true
 
 [lints]
 workspace = true

crates/ide-db/src/lib.rs

@@ -412,9 +412,3 @@ impl SnippetCap {
         }
     }
 }
-
-#[cfg(test)]
-mod tests {
-    mod line_index;
-    mod sourcegen_lints;
-}

crates/ide-diagnostics/Cargo.toml

@@ -33,10 +33,6 @@ expect-test = "1.4.0"
 # local deps
 test-utils.workspace = true
 test-fixture.workspace = true
-sourcegen.workspace = true
-
-[features]
-in-rust-tree = []
 
 [lints]
 workspace = true

crates/ide-diagnostics/src/tests.rs

@@ -1,6 +1,4 @@
 #![allow(clippy::print_stderr)]
-#[cfg(not(feature = "in-rust-tree"))]
-mod sourcegen;
 
 use ide_db::{
     assists::AssistResolveStrategy, base_db::SourceDatabaseExt, LineIndexDatabase, RootDatabase,

crates/ide/Cargo.toml

@@ -51,8 +51,5 @@ expect-test = "1.4.0"
 test-utils.workspace = true
 test-fixture.workspace = true
 
-[features]
-in-rust-tree = ["ide-assists/in-rust-tree", "ide-diagnostics/in-rust-tree"]
-
 [lints]
 workspace = true

crates/ide/src/hover/tests.rs

@@ -5434,13 +5434,62 @@ fn hover_feature() {
 
             The tracking issue for this feature is: None.
 
-            Intrinsics are never intended to be stable directly, but intrinsics are often
+            Intrinsics are rarely intended to be stable directly, but are usually
             exported in some sort of stable manner. Prefer using the stable interfaces to
             the intrinsic directly when you can.
 
             ------------------------
 
 
+            ## Intrinsics with fallback logic
+
+            Many intrinsics can be written in pure rust, albeit inefficiently or without supporting
+            some features that only exist on some backends. Backends can simply not implement those
+            intrinsics without causing any code miscompilations or failures to compile.
+            All intrinsic fallback bodies are automatically made cross-crate inlineable (like `#[inline]`)
+            by the codegen backend, but not the MIR inliner.
+
+            ```rust
+            #![feature(rustc_attrs, effects)]
+            #![allow(internal_features)]
+
+            #[rustc_intrinsic]
+            const unsafe fn const_deallocate(_ptr: *mut u8, _size: usize, _align: usize) {}
+            ```
+
+            Since these are just regular functions, it is perfectly ok to create the intrinsic twice:
+
+            ```rust
+            #![feature(rustc_attrs, effects)]
+            #![allow(internal_features)]
+
+            #[rustc_intrinsic]
+            const unsafe fn const_deallocate(_ptr: *mut u8, _size: usize, _align: usize) {}
+
+            mod foo {
+                #[rustc_intrinsic]
+                const unsafe fn const_deallocate(_ptr: *mut u8, _size: usize, _align: usize) {
+                    panic!("noisy const dealloc")
+                }
+            }
+
+            ```
+
+            The behaviour on backends that override the intrinsic is exactly the same. On other
+            backends, the intrinsic behaviour depends on which implementation is called, just like
+            with any regular function.
+
+            ## Intrinsics lowered to MIR instructions
+
+            Various intrinsics have native MIR operations that they correspond to. Instead of requiring
+            backends to implement both the intrinsic and the MIR operation, the `lower_intrinsics` pass
+            will convert the calls to the MIR operation. Backends do not need to know about these intrinsics
+            at all.
+
+            ## Intrinsics without fallback logic
+
+            These must be implemented by all backends.
+
             These are imported as if they were FFI functions, with the special
             `rust-intrinsic` ABI. For example, if one was in a freestanding
             context, but wished to be able to `transmute` between types, and
@@ -5459,7 +5508,8 @@ fn hover_feature() {
             }
             ```
 
-            As with any other FFI functions, these are always `unsafe` to call.
+            As with any other FFI functions, these are by default always `unsafe` to call.
+            You can add `#[rustc_safe_intrinsic]` to the intrinsic to make it safe to call.
 
         "#]],
     )

crates/rust-analyzer/Cargo.toml

@@ -85,7 +85,6 @@ force-always-assert = ["always-assert/force"]
 sysroot-abi = []
 in-rust-tree = [
     "sysroot-abi",
-    "ide/in-rust-tree",
     "syntax/in-rust-tree",
     "parser/in-rust-tree",
     "hir/in-rust-tree",

lib/line-index/Cargo.toml

@@ -10,5 +10,8 @@ edition = "2021"
 text-size = "1.1.1"
 nohash-hasher = "0.2.0"
 
+[dev-dependencies]
+oorandom = "11.1.3"
+
 [lints]
-workspace = true
+workspace = true

lib/line-index/src/tests.rs

@@ -142,3 +142,56 @@ fn test_to_wide() {
     let wide_line_col = line_index.to_wide(WideEncoding::Utf16, line_col.unwrap());
     assert_eq!(wide_line_col, Some(WideLineCol { line: 5, col: 4 }));
 }
+
+#[test]
+fn test_every_chars() {
+    let text: String = {
+        let mut chars: Vec<char> = ((0 as char)..char::MAX).collect(); // Neat!
+        chars.extend("\n".repeat(chars.len() / 16).chars());
+        let seed = std::hash::Hasher::finish(&std::hash::BuildHasher::build_hasher(
+            #[allow(clippy::disallowed_types)]
+            &std::collections::hash_map::RandomState::new(),
+        ));
+        let mut rng = oorandom::Rand32::new(seed);
+        let mut rand_index = |i| rng.rand_range(0..i as u32) as usize;
+        let mut remaining = chars.len() - 1;
+        while remaining > 0 {
+            let index = rand_index(remaining);
+            chars.swap(remaining, index);
+            remaining -= 1;
+        }
+        chars.into_iter().collect()
+    };
+    assert!(text.contains('💩')); // Sanity check.
+
+    let line_index = LineIndex::new(&text);
+
+    let mut lin_col = LineCol { line: 0, col: 0 };
+    let mut col_utf16 = 0;
+    let mut col_utf32 = 0;
+    for (offset, c) in text.char_indices() {
+        let got_offset = line_index.offset(lin_col).unwrap();
+        assert_eq!(usize::from(got_offset), offset);
+
+        let got_lin_col = line_index.line_col(got_offset);
+        assert_eq!(got_lin_col, lin_col);
+
+        for (enc, col) in [(WideEncoding::Utf16, col_utf16), (WideEncoding::Utf32, col_utf32)] {
+            let wide_lin_col = line_index.to_wide(enc, lin_col).unwrap();
+            let got_lin_col = line_index.to_utf8(enc, wide_lin_col).unwrap();
+            assert_eq!(got_lin_col, lin_col);
+            assert_eq!(wide_lin_col.col, col)
+        }
+
+        if c == '\n' {
+            lin_col.line += 1;
+            lin_col.col = 0;
+            col_utf16 = 0;
+            col_utf32 = 0;
+        } else {
+            lin_col.col += c.len_utf8() as u32;
+            col_utf16 += c.len_utf16() as u32;
+            col_utf32 += 1;
+        }
+    }
+}

xtask/src/codegen.rs

@@ -8,14 +8,21 @@ use xshell::{cmd, Shell};
 use crate::{flags, project_root};
 
 pub(crate) mod assists_doc_tests;
+pub(crate) mod diagnostics_docs;
+mod lints;
 
 impl flags::Codegen {
     pub(crate) fn run(self, _sh: &Shell) -> anyhow::Result<()> {
         match self.codegen_type.unwrap_or_default() {
             flags::CodegenType::All => {
+                diagnostics_docs::generate(self.check);
                 assists_doc_tests::generate(self.check);
+                // lints::generate(self.check) Updating clones the rust repo, so don't run it unless
+                // explicitly asked for
             }
             flags::CodegenType::AssistsDocTests => assists_doc_tests::generate(self.check),
+            flags::CodegenType::DiagnosticsDocs => diagnostics_docs::generate(self.check),
+            flags::CodegenType::LintDefinitions => lints::generate(self.check),
         }
         Ok(())
     }

crates/ide-diagnostics/src/tests/sourcegen.rs → xtask/src/codegen/diagnostics_docs.rs

@@ -2,22 +2,26 @@
 
 use std::{fmt, fs, io, path::PathBuf};
 
-use sourcegen::project_root;
+use crate::{
+    codegen::{add_preamble, list_rust_files, CommentBlock, Location},
+    project_root,
+};
 
-#[test]
-fn sourcegen_diagnostic_docs() {
+pub(crate) fn generate(check: bool) {
     let diagnostics = Diagnostic::collect().unwrap();
-    let contents =
-        diagnostics.into_iter().map(|it| it.to_string()).collect::<Vec<_>>().join("\n\n");
-    let contents = sourcegen::add_preamble("sourcegen_diagnostic_docs", contents);
-    let dst = project_root().join("docs/user/generated_diagnostic.adoc");
-    fs::write(dst, contents).unwrap();
+    if !check {
+        let contents =
+            diagnostics.into_iter().map(|it| it.to_string()).collect::<Vec<_>>().join("\n\n");
+        let contents = add_preamble("sourcegen_diagnostic_docs", contents);
+        let dst = project_root().join("docs/user/generated_diagnostic.adoc");
+        fs::write(dst, contents).unwrap();
+    }
 }
 
 #[derive(Debug)]
 struct Diagnostic {
     id: String,
-    location: sourcegen::Location,
+    location: Location,
     doc: String,
 }
 
@@ -26,23 +30,23 @@ impl Diagnostic {
         let handlers_dir = project_root().join("crates/ide-diagnostics/src/handlers");
 
         let mut res = Vec::new();
-        for path in sourcegen::list_rust_files(&handlers_dir) {
+        for path in list_rust_files(&handlers_dir) {
             collect_file(&mut res, path)?;
         }
         res.sort_by(|lhs, rhs| lhs.id.cmp(&rhs.id));
         return Ok(res);
 
         fn collect_file(acc: &mut Vec<Diagnostic>, path: PathBuf) -> io::Result<()> {
             let text = fs::read_to_string(&path)?;
-            let comment_blocks = sourcegen::CommentBlock::extract("Diagnostic", &text);
+            let comment_blocks = CommentBlock::extract("Diagnostic", &text);
 
             for block in comment_blocks {
                 let id = block.id;
                 if let Err(msg) = is_valid_diagnostic_name(&id) {
                     panic!("invalid diagnostic name: {id:?}:\n  {msg}")
                 }
                 let doc = block.contents.join("\n");
-                let location = sourcegen::Location { file: path.clone(), line: block.line };
+                let location = Location { file: path.clone(), line: block.line };
                 acc.push(Diagnostic { id, location, doc })
             }