Skip to content

Commit 2b784b3

Browse files
wjones127wjones127
authored
fix!: delta index fragment bitmaps contained previous index coverage (lancedb#3377)
BREAKING CHANGE: delta index fragment bitmaps will now only contain the fragment ids covered by the delta, not the full index. To get the full bitmap, make sure to union with all index segments with the same name. Old datasets will still show previous fragment ids, until a write is done, which forces a migration. **If corrupted fragment ids are present in a dataset, then the `dataset.index_statistics` will return an error. Before using `dataset.index_statistics()`, call `dataset.validate()` to check the integrity and use `dataset.delete("false")` to force a migration.** Fixes lancedb#3374
1 parent 4cb37c1 commit 2b784b3

File tree

19 files changed

+306
-96
lines changed

19 files changed

+306
-96
lines changed

‎Cargo.lock

+16-16
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

+17-17
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ exclude = ["python"]
2121
resolver = "2"
2222

2323
[workspace.package]
24-
version = "0.22.1"
24+
version = "0.23.0"
2525
edition = "2021"
2626
authors = ["Lance Devs <dev@lancedb.com>"]
2727
license = "Apache-2.0"
@@ -44,21 +44,21 @@ categories = [
4444
rust-version = "1.80.1"
4545

4646
[workspace.dependencies]
47-
lance = { version = "=0.22.1", path = "./rust/lance" }
48-
lance-arrow = { version = "=0.22.1", path = "./rust/lance-arrow" }
49-
lance-core = { version = "=0.22.1", path = "./rust/lance-core" }
50-
lance-datafusion = { version = "=0.22.1", path = "./rust/lance-datafusion" }
51-
lance-datagen = { version = "=0.22.1", path = "./rust/lance-datagen" }
52-
lance-encoding = { version = "=0.22.1", path = "./rust/lance-encoding" }
53-
lance-encoding-datafusion = { version = "=0.22.1", path = "./rust/lance-encoding-datafusion" }
54-
lance-file = { version = "=0.22.1", path = "./rust/lance-file" }
55-
lance-index = { version = "=0.22.1", path = "./rust/lance-index" }
56-
lance-io = { version = "=0.22.1", path = "./rust/lance-io" }
57-
lance-jni = { version = "=0.22.1", path = "./java/core/lance-jni" }
58-
lance-linalg = { version = "=0.22.1", path = "./rust/lance-linalg" }
59-
lance-table = { version = "=0.22.1", path = "./rust/lance-table" }
60-
lance-test-macros = { version = "=0.22.1", path = "./rust/lance-test-macros" }
61-
lance-testing = { version = "=0.22.1", path = "./rust/lance-testing" }
47+
lance = { version = "=0.23.0", path = "./rust/lance" }
48+
lance-arrow = { version = "=0.23.0", path = "./rust/lance-arrow" }
49+
lance-core = { version = "=0.23.0", path = "./rust/lance-core" }
50+
lance-datafusion = { version = "=0.23.0", path = "./rust/lance-datafusion" }
51+
lance-datagen = { version = "=0.23.0", path = "./rust/lance-datagen" }
52+
lance-encoding = { version = "=0.23.0", path = "./rust/lance-encoding" }
53+
lance-encoding-datafusion = { version = "=0.23.0", path = "./rust/lance-encoding-datafusion" }
54+
lance-file = { version = "=0.23.0", path = "./rust/lance-file" }
55+
lance-index = { version = "=0.23.0", path = "./rust/lance-index" }
56+
lance-io = { version = "=0.23.0", path = "./rust/lance-io" }
57+
lance-jni = { version = "=0.23.0", path = "./java/core/lance-jni" }
58+
lance-linalg = { version = "=0.23.0", path = "./rust/lance-linalg" }
59+
lance-table = { version = "=0.23.0", path = "./rust/lance-table" }
60+
lance-test-macros = { version = "=0.23.0", path = "./rust/lance-test-macros" }
61+
lance-testing = { version = "=0.23.0", path = "./rust/lance-testing" }
6262
approx = "0.5.1"
6363
# Note that this one does not include pyarrow
6464
arrow = { version = "53.2", optional = false, features = ["prettyprint"] }
@@ -114,7 +114,7 @@ datafusion-physical-expr = { version = "44.0" }
114114
deepsize = "0.2.0"
115115
dirs = "5.0.0"
116116
either = "1.0"
117-
fsst = { version = "=0.22.1", path = "./rust/lance-encoding/src/compression_algo/fsst" }
117+
fsst = { version = "=0.23.0", path = "./rust/lance-encoding/src/compression_algo/fsst" }
118118
futures = "0.3"
119119
http = "1.1.0"
120120
hyperloglogplus = { version = "0.4.1", features = ["const-loop"] }

‎java/core/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
<parent>
99
<groupId>com.lancedb</groupId>
1010
<artifactId>lance-parent</artifactId>
11-
<version>0.21.2</version>
11+
<version>0.23.0</version>
1212
<relativePath>../pom.xml</relativePath>
1313
</parent>
1414

‎java/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
<groupId>com.lancedb</groupId>
88
<artifactId>lance-parent</artifactId>
9-
<version>0.21.2</version>
9+
<version>0.23.0</version>
1010
<packaging>pom</packaging>
1111

1212
<name>Lance Parent</name>

‎java/spark/pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
<parent>
99
<groupId>com.lancedb</groupId>
1010
<artifactId>lance-parent</artifactId>
11-
<version>0.21.2</version>
11+
<version>0.23.0</version>
1212
<relativePath>../pom.xml</relativePath>
1313
</parent>
1414

@@ -112,7 +112,7 @@
112112
<dependency>
113113
<groupId>com.lancedb</groupId>
114114
<artifactId>lance-core</artifactId>
115-
<version>0.21.2</version>
115+
<version>0.23.0</version>
116116
</dependency>
117117
<dependency>
118118
<groupId>org.apache.spark</groupId>

‎python/Cargo.lock

+13-13
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎python/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "pylance"
3-
version = "0.22.1"
3+
version = "0.23.0"
44
edition = "2021"
55
authors = ["Lance Devs <dev@lancedb.com>"]
66
rust-version = "1.65"

0 commit comments

Comments
 (0)