feat!: create index in v3 version by default (#3477)

BREAKING CHANGE: changed vector index default from
`IndexFileVersion::Legacy` to `IndexFileVersion::V3`. To keep the older
behavior, you can change `VectorIndexParams::version` in Rust, or
_____________ in Python.

---------

Signed-off-by: BubbleCal <bubble-cal@outlook.com>

Author: BubbleCal

Cargo.lock

@@ -21,7 +21,7 @@ exclude = ["python"]
 resolver = "2"
 
 [workspace.package]
-version = "0.24.2"
+version = "0.25.0"
 edition = "2021"
 authors = ["Lance Devs <dev@lancedb.com>"]
 license = "Apache-2.0"
@@ -44,21 +44,21 @@ categories = [
 rust-version = "1.81.0"
 
 [workspace.dependencies]
-lance = { version = "=0.24.2", path = "./rust/lance" }
-lance-arrow = { version = "=0.24.2", path = "./rust/lance-arrow" }
-lance-core = { version = "=0.24.2", path = "./rust/lance-core" }
-lance-datafusion = { version = "=0.24.2", path = "./rust/lance-datafusion" }
-lance-datagen = { version = "=0.24.2", path = "./rust/lance-datagen" }
-lance-encoding = { version = "=0.24.2", path = "./rust/lance-encoding" }
-lance-encoding-datafusion = { version = "=0.24.2", path = "./rust/lance-encoding-datafusion" }
-lance-file = { version = "=0.24.2", path = "./rust/lance-file" }
-lance-index = { version = "=0.24.2", path = "./rust/lance-index" }
-lance-io = { version = "=0.24.2", path = "./rust/lance-io" }
-lance-jni = { version = "=0.24.2", path = "./java/core/lance-jni" }
-lance-linalg = { version = "=0.24.2", path = "./rust/lance-linalg" }
-lance-table = { version = "=0.24.2", path = "./rust/lance-table" }
-lance-test-macros = { version = "=0.24.2", path = "./rust/lance-test-macros" }
-lance-testing = { version = "=0.24.2", path = "./rust/lance-testing" }
+lance = { version = "=0.25.0", path = "./rust/lance" }
+lance-arrow = { version = "=0.25.0", path = "./rust/lance-arrow" }
+lance-core = { version = "=0.25.0", path = "./rust/lance-core" }
+lance-datafusion = { version = "=0.25.0", path = "./rust/lance-datafusion" }
+lance-datagen = { version = "=0.25.0", path = "./rust/lance-datagen" }
+lance-encoding = { version = "=0.25.0", path = "./rust/lance-encoding" }
+lance-encoding-datafusion = { version = "=0.25.0", path = "./rust/lance-encoding-datafusion" }
+lance-file = { version = "=0.25.0", path = "./rust/lance-file" }
+lance-index = { version = "=0.25.0", path = "./rust/lance-index" }
+lance-io = { version = "=0.25.0", path = "./rust/lance-io" }
+lance-jni = { version = "=0.25.0", path = "./java/core/lance-jni" }
+lance-linalg = { version = "=0.25.0", path = "./rust/lance-linalg" }
+lance-table = { version = "=0.25.0", path = "./rust/lance-table" }
+lance-test-macros = { version = "=0.25.0", path = "./rust/lance-test-macros" }
+lance-testing = { version = "=0.25.0", path = "./rust/lance-testing" }
 approx = "0.5.1"
 # Note that this one does not include pyarrow
 arrow = { version = "54.1", optional = false, features = ["prettyprint"] }
@@ -117,7 +117,7 @@ datafusion-physical-expr = { version = "45.0" }
 deepsize = "0.2.0"
 dirs = "5.0.0"
 either = "1.0"
-fsst = { version = "=0.24.2", path = "./rust/lance-encoding/src/compression_algo/fsst" }
+fsst = { version = "=0.25.0", path = "./rust/lance-encoding/src/compression_algo/fsst" }
 futures = "0.3"
 http = "1.1.0"
 hyperloglogplus = { version = "0.4.1", features = ["const-loop"] }

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pylance"
-version = "0.24.2"
+version = "0.25.0"
 edition = "2021"
 authors = ["Lance Devs <dev@lancedb.com>"]
 rust-version = "1.65"

python/Cargo.lock

@@ -589,6 +589,7 @@ def test_pre_populated_ivf_centroids(dataset, tmp_path: Path):
     dataset_with_index = dataset.create_index(
         ["vector"],
         index_type="IVF_PQ",
+        metric="cosine",
         ivf_centroids=centroids,
         num_partitions=5,
         num_sub_vectors=8,
@@ -613,14 +614,15 @@ def test_pre_populated_ivf_centroids(dataset, tmp_path: Path):
         "index_type": "IVF_PQ",
         "uuid": index_uuid,
         "uri": expected_filepath,
-        "metric_type": "l2",
+        "metric_type": "cosine",
         "num_partitions": 5,
         "sub_index": {
             "dimension": 128,
             "index_type": "PQ",
             "metric_type": "l2",
             "nbits": 8,
             "num_sub_vectors": 8,
+            "transposed": True,
         },
     }
 

python/Cargo.toml

@@ -56,6 +56,8 @@ lazy_static! {
             Field::new(DIST_COL, arrow_schema::DataType::Float32, false),
             ROW_ID_FIELD.clone(),
         ]));
+    pub static ref PART_ID_FIELD: arrow_schema::Field =
+        arrow_schema::Field::new(PART_ID_COLUMN, arrow_schema::DataType::UInt32, true);
 }
 
 /// Query parameters for the vector indices