fix: flat FTS panic with prefilter (#3470)

Author: BubbleCal

python/python/tests/test_scalar_index.py

@@ -303,6 +303,15 @@ def test_indexed_filter_with_fts_index(tmp_path):
     ds.create_scalar_index("text", "INVERTED")
     ds.create_scalar_index("sentiment", "BITMAP")
 
+    # append more data to test flat FTS
+    data = pa.table(
+        {
+            "text": ["flat", "search"],
+            "sentiment": ["positive", "positive"],
+        }
+    )
+    ds = lance.write_dataset(data, tmp_path, mode="append")
+
     results = ds.to_table(
         full_text_query="puppy",
         filter="sentiment='positive'",

rust/lance-index/src/scalar/inverted/index.rs

@@ -1063,8 +1063,8 @@ pub fn flat_bm25_search(
 
     let score_col = Arc::new(Float32Array::from(scores)) as ArrayRef;
     let batch = batch
-        .drop_column(doc_col)?
-        .try_with_column(SCORE_FIELD.clone(), score_col)?;
+        .try_with_column(SCORE_FIELD.clone(), score_col)?
+        .project_by_schema(&FTS_SCHEMA)?; // the scan node would probably scan some extra columns for prefilter, drop them here
     Ok(batch)
 }