fix: the distance for multivector query is not correct (#3522)

the dist should be `dist = sum(1 - sim)` for multivector query, but we
set it `dist = 1 - sum(sim)`.

The order of results is still correct but let's make it consistent

Signed-off-by: BubbleCal <bubble-cal@outlook.com>

Author: BubbleCal

python/python/tests/test_vector_index.py

@@ -558,6 +558,17 @@ def test_multivec_ann(indexed_multivec_dataset: lance.LanceDataset):
     assert results["vector"].type == pa.list_(pa.list_(pa.float32(), 128))
     assert len(results["vector"][0]) == 5
 
+    query = [query, query]
+    doubled_results = indexed_multivec_dataset.to_table(
+        nearest={"column": "vector", "q": query, "k": 100}
+    )
+    assert len(results) == len(doubled_results)
+    for i in range(len(results)):
+        assert (
+            results["_distance"][i].as_py() * 2
+            == doubled_results["_distance"][i].as_py()
+        )
+
     # query with a vector that dim not match
     query = np.random.rand(256)
     with pytest.raises(ValueError, match="does not match index column size"):

rust/lance/src/io/exec/knn.rs

@@ -813,6 +813,7 @@ impl ExecutionPlan for MultivectorScoringExec {
 
         let k = self.query.k;
         let refactor = self.query.refine_factor.unwrap_or(1) as usize;
+        let num_queries = self.inputs.len() as f32;
         let stream = stream::once(async move {
             // at most, we will have k * refine_factor results for each query
             let mut results = HashMap::with_capacity(k * refactor);
@@ -850,7 +851,7 @@ impl ExecutionPlan for MultivectorScoringExec {
             let dists = sims
                 .into_iter()
                 // it's similarity, so we need to convert it back to distance
-                .map(|sim| 1.0 - sim)
+                .map(|sim| num_queries - sim)
                 .collect::<Vec<_>>();
             let row_ids = UInt64Array::from(row_ids);
             let dists = Float32Array::from(dists);