refactor: remove unused dependencies (#1853)

Most of these dependencies are used in some other crate and so the
overall build time of the workspace didn't get much faster. This PR is
mostly just good hygiene. Building `lance-core` by itself though is now
200% faster.

Author: westonpace

rust/Cargo.toml

@@ -94,7 +94,7 @@ mock_instant = { version = "0.3.1", features = ["sync"] }
 moka = "0.11"
 num-traits = "0.2"
 num_cpus = "1.0"
-object_store = { version = "0.9.0", features = ["aws", "gcp", "azure"] }
+object_store = { version = "0.9.0" }
 parquet = "49.0"
 pin-project = "1.0"
 pprof = { version = "0.13", features = ["flamegraph", "criterion"] }

rust/lance-core/Cargo.toml

@@ -12,52 +12,35 @@ categories.workspace = true
 rust-version.workspace = true
 
 [dependencies]
-arrow-arith.workspace = true
 arrow-array.workspace = true
 arrow-buffer.workspace = true
-arrow-cast.workspace = true
-arrow-data.workspace = true
-arrow-ipc.workspace = true
 arrow-schema.workspace = true
-arrow-select.workspace = true
-async-recursion.workspace = true
 async-trait.workspace = true
 lance-arrow.workspace = true
 byteorder.workspace = true
 bytes.workspace = true
 chrono.workspace = true
-datafusion-common.workspace = true
-datafusion-sql.workspace = true
+datafusion-common = { workspace = true, optional = true }
+datafusion-sql = { workspace = true, optional = true }
 futures.workspace = true
-http.workspace = true
 lazy_static.workspace = true
-log.workspace = true
 mock_instant.workspace = true
 moka.workspace = true
-num_cpus.workspace = true
-num-traits.workspace = true
-object_store.workspace = true
+object_store = { workspace = true }
 pin-project.workspace = true
-prost-types.workspace = true
 prost.workspace = true
 rand.workspace = true
 roaring.workspace = true
 serde_json.workspace = true
-serde.workspace = true
-shellexpand.workspace = true
 snafu.workspace = true
 tokio.workspace = true
 tracing.workspace = true
 url.workspace = true
-uuid.workspace = true
 
 [dev-dependencies]
-arrow = "49.0"
-rand.workspace = true
 tempfile.workspace = true
 lance-testing.workspace = true
-parquet.workspace = true
 proptest.workspace = true
 
-[build-dependencies]
-prost-build.workspace = true
+[features]
+datafusion = ["datafusion-common", "datafusion-sql"]

rust/lance-core/src/error.rs

@@ -235,6 +235,7 @@ impl From<Error> for ArrowError {
     }
 }
 
+#[cfg(feature = "datafusion")]
 impl From<datafusion_sql::sqlparser::parser::ParserError> for Error {
     #[track_caller]
     fn from(e: datafusion_sql::sqlparser::parser::ParserError) -> Self {
@@ -245,6 +246,7 @@ impl From<datafusion_sql::sqlparser::parser::ParserError> for Error {
     }
 }
 
+#[cfg(feature = "datafusion")]
 impl From<datafusion_sql::sqlparser::tokenizer::TokenizerError> for Error {
     #[track_caller]
     fn from(e: datafusion_sql::sqlparser::tokenizer::TokenizerError) -> Self {
@@ -255,13 +257,15 @@ impl From<datafusion_sql::sqlparser::tokenizer::TokenizerError> for Error {
     }
 }
 
+#[cfg(feature = "datafusion")]
 impl From<Error> for datafusion_common::DataFusionError {
     #[track_caller]
     fn from(e: Error) -> Self {
         Self::Execution(e.to_string())
     }
 }
 
+#[cfg(feature = "datafusion")]
 impl From<datafusion_common::DataFusionError> for Error {
     #[track_caller]
     fn from(e: datafusion_common::DataFusionError) -> Self {

rust/lance-datafusion/Cargo.toml

@@ -17,9 +17,8 @@ arrow-ord.workspace = true
 async-trait.workspace = true
 datafusion.workspace = true
 datafusion-common.workspace = true
-datafusion-expr.workspace = true
 datafusion-physical-expr.workspace = true
 futures.workspace = true
 lance-arrow.workspace = true
-lance-core.workspace = true
+lance-core = { workspace = true, features = ["datafusion"] }
 tokio.workspace = true

rust/lance-file/Cargo.toml

@@ -18,14 +18,10 @@ lance-io.workspace = true
 arrow-arith.workspace = true
 arrow-array.workspace = true
 arrow-buffer.workspace = true
-arrow-cast.workspace = true
-arrow-data.workspace = true
 arrow-schema.workspace = true
 arrow-select.workspace = true
 async-recursion.workspace = true
 async-trait.workspace = true
-byteorder.workspace = true
-bytes.workspace = true
 datafusion-common.workspace = true
 futures.workspace = true
 num_cpus.workspace = true

rust/lance-index/Cargo.toml

@@ -16,7 +16,6 @@ arrow.workspace = true
 arrow-array.workspace = true
 arrow-ord.workspace = true
 arrow-schema.workspace = true
-arrow-arith.workspace = true
 arrow-select.workspace = true
 async-recursion.workspace = true
 async-trait.workspace = true
@@ -37,7 +36,6 @@ log.workspace = true
 num_cpus.workspace = true
 num-traits.workspace = true
 object_store.workspace = true
-pin-project.workspace = true
 prost.workspace = true
 rand.workspace = true
 roaring.workspace = true

rust/lance-io/Cargo.toml

@@ -31,7 +31,7 @@ chrono.workspace = true
 futures.workspace = true
 lazy_static.workspace = true
 num_cpus.workspace = true
-object_store.workspace = true
+object_store = { workspace = true, features = ["aws", "gcp", "azure"] }
 pin-project.workspace = true
 prost.workspace = true
 shellexpand.workspace = true

rust/lance-linalg/Cargo.toml

@@ -13,7 +13,6 @@ categories = { workspace = true }
 arrow-array = { workspace = true }
 arrow-ord = { workspace = true }
 arrow-schema = { workspace = true }
-arrow-select = { workspace = true }
 futures = { workspace = true }
 half = { workspace = true }
 lance-arrow = { workspace = true }

rust/lance/Cargo.toml

@@ -28,9 +28,6 @@ lance-table = { workspace = true }
 arrow-arith = { workspace = true }
 arrow-array = { workspace = true }
 arrow-buffer = { workspace = true }
-arrow-cast = { workspace = true }
-arrow-data = { workspace = true }
-arrow-ipc = { workspace = true }
 arrow-ord = { workspace = true }
 arrow-row = { workspace = true }
 arrow-schema = { workspace = true }
@@ -46,20 +43,16 @@ dashmap = "5"
 # matches arrow-rs use
 half.workspace = true
 itertools.workspace = true
-http.workspace = true
-object_store.workspace = true
-aws-config.workspace = true
+object_store = { workspace = true, features = ["aws", "gcp", "azure"] }
 aws-credential-types.workspace = true
 pin-project.workspace = true
 prost.workspace = true
-prost-types.workspace = true
 roaring.workspace = true
 tokio.workspace = true
 url.workspace = true
 rand.workspace = true
 futures.workspace = true
 uuid.workspace = true
-shellexpand.workspace = true
 arrow.workspace = true
 num_cpus.workspace = true
 # TODO: use datafusion sub-modules to reduce build size?
@@ -68,7 +61,6 @@ datafusion-physical-expr.workspace = true
 lapack = { version = "0.19.0", optional = true }
 cblas = { version = "0.4.0", optional = true }
 lru_time_cache = "0.11"
-num-traits.workspace = true
 ordered-float = "3.6.0"
 snafu = { workspace = true }
 log = { workspace = true }
@@ -80,7 +72,6 @@ aws-sdk-dynamodb = { workspace = true, optional = true }
 tempfile.workspace = true
 tracing.workspace = true
 lazy_static = { workspace = true }
-base64 = "0.21.4"
 async_cell = "0.2.2"
 
 [target.'cfg(target_os = "macos")'.dependencies]