Merge #7657

7657: utf8 r=matklad a=matklad

- Prepare for utf-8 offsets
- reduce code duplication in tests
- Make utf8 default, implement utf16 in terms of it
- Make it easy to add additional context for offset conversion
- Implement utf8 offsets

closes #7453

Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>

crates/ide/src/lib.rs

@@ -95,7 +95,7 @@ pub use ide_db::{
     },
     call_info::CallInfo,
     label::Label,
-    line_index::{LineCol, LineIndex},
+    line_index::{LineCol, LineColUtf16, LineIndex},
     search::{ReferenceAccess, SearchScope},
     source_change::{FileSystemEdit, SourceChange},
     symbol_index::Query,

crates/ide_db/src/line_index.rs

@@ -15,11 +15,19 @@ pub struct LineIndex {
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
-pub struct LineCol {
+pub struct LineColUtf16 {
     /// Zero-based
     pub line: u32,
     /// Zero-based
-    pub col_utf16: u32,
+    pub col: u32,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct LineCol {
+    /// Zero-based
+    pub line: u32,
+    /// Zero-based utf8 offset
+    pub col: u32,
 }
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
@@ -92,14 +100,21 @@ impl LineIndex {
         let line = partition_point(&self.newlines, |&it| it <= offset) - 1;
         let line_start_offset = self.newlines[line];
         let col = offset - line_start_offset;
-
-        LineCol { line: line as u32, col_utf16: self.utf8_to_utf16_col(line as u32, col) as u32 }
+        LineCol { line: line as u32, col: col.into() }
     }
 
     pub fn offset(&self, line_col: LineCol) -> TextSize {
-        //FIXME: return Result
-        let col = self.utf16_to_utf8_col(line_col.line, line_col.col_utf16);
-        self.newlines[line_col.line as usize] + col
+        self.newlines[line_col.line as usize] + TextSize::from(line_col.col)
+    }
+
+    pub fn to_utf16(&self, line_col: LineCol) -> LineColUtf16 {
+        let col = self.utf8_to_utf16_col(line_col.line, line_col.col.into());
+        LineColUtf16 { line: line_col.line, col: col as u32 }
+    }
+
+    pub fn to_utf8(&self, line_col: LineColUtf16) -> LineCol {
+        let col = self.utf16_to_utf8_col(line_col.line, line_col.col);
+        LineCol { line: line_col.line, col: col.into() }
     }
 
     pub fn lines(&self, range: TextRange) -> impl Iterator<Item = TextRange> + '_ {

crates/ide_db/src/line_index/tests.rs

@@ -3,24 +3,29 @@ use super::*;
 #[test]
 fn test_line_index() {
     let text = "hello\nworld";
+    let table = [
+        (00, 0, 0),
+        (01, 0, 1),
+        (05, 0, 5),
+        (06, 1, 0),
+        (07, 1, 1),
+        (08, 1, 2),
+        (10, 1, 4),
+        (11, 1, 5),
+        (12, 1, 6),
+    ];
+
     let index = LineIndex::new(text);
-    assert_eq!(index.line_col(0.into()), LineCol { line: 0, col_utf16: 0 });
-    assert_eq!(index.line_col(1.into()), LineCol { line: 0, col_utf16: 1 });
-    assert_eq!(index.line_col(5.into()), LineCol { line: 0, col_utf16: 5 });
-    assert_eq!(index.line_col(6.into()), LineCol { line: 1, col_utf16: 0 });
-    assert_eq!(index.line_col(7.into()), LineCol { line: 1, col_utf16: 1 });
-    assert_eq!(index.line_col(8.into()), LineCol { line: 1, col_utf16: 2 });
-    assert_eq!(index.line_col(10.into()), LineCol { line: 1, col_utf16: 4 });
-    assert_eq!(index.line_col(11.into()), LineCol { line: 1, col_utf16: 5 });
-    assert_eq!(index.line_col(12.into()), LineCol { line: 1, col_utf16: 6 });
+    for &(offset, line, col) in &table {
+        assert_eq!(index.line_col(offset.into()), LineCol { line, col });
+    }
 
     let text = "\nhello\nworld";
+    let table = [(0, 0, 0), (1, 1, 0), (2, 1, 1), (6, 1, 5), (7, 2, 0)];
     let index = LineIndex::new(text);
-    assert_eq!(index.line_col(0.into()), LineCol { line: 0, col_utf16: 0 });
-    assert_eq!(index.line_col(1.into()), LineCol { line: 1, col_utf16: 0 });
-    assert_eq!(index.line_col(2.into()), LineCol { line: 1, col_utf16: 1 });
-    assert_eq!(index.line_col(6.into()), LineCol { line: 1, col_utf16: 5 });
-    assert_eq!(index.line_col(7.into()), LineCol { line: 2, col_utf16: 0 });
+    for &(offset, line, col) in &table {
+        assert_eq!(index.line_col(offset.into()), LineCol { line, col });
+    }
 }
 
 #[test]

crates/rust-analyzer/src/bin/main.rs

@@ -8,7 +8,7 @@ use std::{convert::TryFrom, env, fs, path::PathBuf, process};
 
 use lsp_server::Connection;
 use project_model::ProjectManifest;
-use rust_analyzer::{cli, config::Config, from_json, Result};
+use rust_analyzer::{cli, config::Config, from_json, lsp_ext::supports_utf8, Result};
 use vfs::AbsPathBuf;
 
 #[cfg(all(feature = "mimalloc"))]
@@ -127,7 +127,11 @@ fn run_server() -> Result<()> {
             name: String::from("rust-analyzer"),
             version: Some(String::from(env!("REV"))),
         }),
-        offset_encoding: None,
+        offset_encoding: if supports_utf8(&initialize_params.capabilities) {
+            Some("utf-8".to_string())
+        } else {
+            None
+        },
     };
 
     let initialize_result = serde_json::to_value(initialize_result).unwrap();

crates/rust-analyzer/src/caps.rs

@@ -37,6 +37,8 @@ pub fn server_capabilities(client_caps: &ClientCapabilities) -> ServerCapabiliti
             all_commit_characters: None,
             completion_item: None,
             work_done_progress_options: WorkDoneProgressOptions { work_done_progress: None },
+            all_commit_characters: None,
+            completion_item: None,
         }),
         signature_help_provider: Some(SignatureHelpOptions {
             trigger_characters: Some(vec!["(".to_string(), ",".to_string()]),

crates/rust-analyzer/src/cli/analysis_bench.rs

@@ -97,7 +97,7 @@ impl BenchCmd {
                 let offset = host
                     .analysis()
                     .file_line_index(file_id)?
-                    .offset(LineCol { line: pos.line - 1, col_utf16: pos.column });
+                    .offset(LineCol { line: pos.line - 1, col: pos.column });
                 let file_position = FilePosition { file_id, offset };
 
                 if is_completion {

crates/rust-analyzer/src/cli/analysis_stats.rs

@@ -218,9 +218,9 @@ impl AnalysisStatsCmd {
                         bar.println(format!(
                             "{}:{}-{}:{}: {}",
                             start.line + 1,
-                            start.col_utf16,
+                            start.col,
                             end.line + 1,
-                            end.col_utf16,
+                            end.col,
                             ty.display(db)
                         ));
                     } else {
@@ -250,9 +250,9 @@ impl AnalysisStatsCmd {
                                 "{} {}:{}-{}:{}: Expected {}, got {}",
                                 path,
                                 start.line + 1,
-                                start.col_utf16,
+                                start.col,
                                 end.line + 1,
-                                end.col_utf16,
+                                end.col,
                                 mismatch.expected.display(db),
                                 mismatch.actual.display(db)
                             ));

crates/rust-analyzer/src/config.rs

@@ -23,7 +23,10 @@ use rustc_hash::FxHashSet;
 use serde::{de::DeserializeOwned, Deserialize};
 use vfs::AbsPathBuf;
 
-use crate::{caps::completion_item_edit_resolve, diagnostics::DiagnosticsMapConfig};
+use crate::{
+    caps::completion_item_edit_resolve, diagnostics::DiagnosticsMapConfig,
+    line_index::OffsetEncoding, lsp_ext::supports_utf8,
+};
 
 config_data! {
     struct ConfigData {
@@ -415,6 +418,13 @@ impl Config {
             false
         )
     }
+    pub fn offset_encoding(&self) -> OffsetEncoding {
+        if supports_utf8(&self.caps) {
+            OffsetEncoding::Utf8
+        } else {
+            OffsetEncoding::Utf16
+        }
+    }
 
     fn experimental(&self, index: &'static str) -> bool {
         try_or!(self.caps.experimental.as_ref()?.get(index)?.as_bool()?, false)

crates/rust-analyzer/src/from_proto.rs

@@ -1,12 +1,17 @@
 //! Conversion lsp_types types to rust-analyzer specific ones.
 use std::convert::TryFrom;
 
-use ide::{Annotation, AnnotationKind, AssistKind, LineCol, LineIndex};
+use ide::{Annotation, AnnotationKind, AssistKind, LineCol, LineColUtf16};
 use ide_db::base_db::{FileId, FilePosition, FileRange};
 use syntax::{TextRange, TextSize};
 use vfs::AbsPathBuf;
 
-use crate::{from_json, global_state::GlobalStateSnapshot, lsp_ext, Result};
+use crate::{
+    from_json,
+    global_state::GlobalStateSnapshot,
+    line_index::{LineIndex, OffsetEncoding},
+    lsp_ext, Result,
+};
 
 pub(crate) fn abs_path(url: &lsp_types::Url) -> Result<AbsPathBuf> {
     let path = url.to_file_path().map_err(|()| "url is not a file")?;
@@ -18,8 +23,17 @@ pub(crate) fn vfs_path(url: &lsp_types::Url) -> Result<vfs::VfsPath> {
 }
 
 pub(crate) fn offset(line_index: &LineIndex, position: lsp_types::Position) -> TextSize {
-    let line_col = LineCol { line: position.line as u32, col_utf16: position.character as u32 };
-    line_index.offset(line_col)
+    let line_col = match line_index.encoding {
+        OffsetEncoding::Utf8 => {
+            LineCol { line: position.line as u32, col: position.character as u32 }
+        }
+        OffsetEncoding::Utf16 => {
+            let line_col =
+                LineColUtf16 { line: position.line as u32, col: position.character as u32 };
+            line_index.index.to_utf8(line_col)
+        }
+    };
+    line_index.index.offset(line_col)
 }
 
 pub(crate) fn text_range(line_index: &LineIndex, range: lsp_types::Range) -> TextRange {
@@ -37,8 +51,8 @@ pub(crate) fn file_position(
     tdpp: lsp_types::TextDocumentPositionParams,
 ) -> Result<FilePosition> {
     let file_id = file_id(world, &tdpp.text_document.uri)?;
-    let line_index = world.analysis.file_line_index(file_id)?;
-    let offset = offset(&*line_index, tdpp.position);
+    let line_index = world.file_line_index(file_id)?;
+    let offset = offset(&line_index, tdpp.position);
     Ok(FilePosition { file_id, offset })
 }
 
@@ -48,7 +62,7 @@ pub(crate) fn file_range(
     range: lsp_types::Range,
 ) -> Result<FileRange> {
     let file_id = file_id(world, &text_document_identifier.uri)?;
-    let line_index = world.analysis.file_line_index(file_id)?;
+    let line_index = world.file_line_index(file_id)?;
     let range = text_range(&line_index, range);
     Ok(FileRange { file_id, range })
 }
@@ -78,7 +92,7 @@ pub(crate) fn annotation(
         lsp_ext::CodeLensResolveData::Impls(params) => {
             let file_id =
                 world.url_to_file_id(&params.text_document_position_params.text_document.uri)?;
-            let line_index = world.analysis.file_line_index(file_id)?;
+            let line_index = world.file_line_index(file_id)?;
 
             Ok(Annotation {
                 range: text_range(&line_index, code_lens.range),
@@ -90,7 +104,7 @@ pub(crate) fn annotation(
         }
         lsp_ext::CodeLensResolveData::References(params) => {
             let file_id = world.url_to_file_id(&params.text_document.uri)?;
-            let line_index = world.analysis.file_line_index(file_id)?;
+            let line_index = world.file_line_index(file_id)?;
 
             Ok(Annotation {
                 range: text_range(&line_index, code_lens.range),

crates/rust-analyzer/src/global_state.rs

@@ -7,7 +7,7 @@ use std::{sync::Arc, time::Instant};
 
 use crossbeam_channel::{unbounded, Receiver, Sender};
 use flycheck::FlycheckHandle;
-use ide::{Analysis, AnalysisHost, Change, FileId};
+use ide::{Analysis, AnalysisHost, Cancelable, Change, FileId};
 use ide_db::base_db::{CrateId, VfsPath};
 use lsp_types::{SemanticTokens, Url};
 use parking_lot::{Mutex, RwLock};
@@ -22,7 +22,7 @@ use crate::{
     diagnostics::{CheckFixes, DiagnosticCollection},
     document::DocumentData,
     from_proto,
-    line_endings::LineEndings,
+    line_index::{LineEndings, LineIndex},
     main_loop::Task,
     op_queue::OpQueue,
     reload::SourceRootConfig,
@@ -271,8 +271,11 @@ impl GlobalStateSnapshot {
         file_id_to_url(&self.vfs.read().0, id)
     }
 
-    pub(crate) fn file_line_endings(&self, id: FileId) -> LineEndings {
-        self.vfs.read().1[&id]
+    pub(crate) fn file_line_index(&self, file_id: FileId) -> Cancelable<LineIndex> {
+        let endings = self.vfs.read().1[&file_id];
+        let index = self.analysis.file_line_index(file_id)?;
+        let res = LineIndex { index, endings, encoding: self.config.offset_encoding() };
+        Ok(res)
     }
 
     pub(crate) fn url_file_version(&self, url: &Url) -> Option<i32> {