Skip to content

Commit

Permalink
refactor: port DaftContext to rust side (#3767)
Browse files Browse the repository at this point in the history
ports the `DaftContext` to the rust side so we can more easily reuse it
for connect & catalog work.


notes for reviewers: 
- after removing `daft-local-execution` from daft-connect, it made
`cargo check` fail, so some of the changes to unrelated `Cargo.toml`'s
was to properly feature flag them. We missed a couple that were causing
things to break.
- @rchowell I requested review from you specifically to take a look at
[context.rs](https://github.com/Eventual-Inc/Daft/pull/3767/files#diff-5b536482a8303505c0c91a561a632218591bf8f02fff21f9f8a3535a3f0ff8a5).
This contains the "context"/"session", and the global state that was
previously only available in python. I added the `catalog` to this state
as well.
  • Loading branch information
universalmind303 authored Feb 6, 2025
1 parent 203f031 commit d40aef8
Show file tree
Hide file tree
Showing 25 changed files with 950 additions and 477 deletions.
20 changes: 16 additions & 4 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 8 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ daft-catalog = {path = "src/daft-catalog", default-features = false}
daft-catalog-python-catalog = {path = "src/daft-catalog/python-catalog", optional = true}
daft-compression = {path = "src/daft-compression", default-features = false}
daft-connect = {path = "src/daft-connect", optional = true}
daft-context = {path = "src/daft-context", default-features = false}
daft-core = {path = "src/daft-core", default-features = false}
daft-csv = {path = "src/daft-csv", default-features = false}
daft-dsl = {path = "src/daft-dsl", default-features = false}
Expand All @@ -31,7 +32,7 @@ daft-micropartition = {path = "src/daft-micropartition", default-features = fals
daft-minhash = {path = "src/daft-minhash", default-features = false}
daft-parquet = {path = "src/daft-parquet", default-features = false}
daft-physical-plan = {path = "src/daft-physical-plan", default-features = false}
daft-ray-execution = {path = "src/daft-ray-execution", default-features = false}
daft-py-runners = {path = "src/daft-py-runners", default-features = false}
daft-scan = {path = "src/daft-scan", default-features = false}
daft-scheduler = {path = "src/daft-scheduler", default-features = false}
daft-sql = {path = "src/daft-sql", default-features = false}
Expand Down Expand Up @@ -77,8 +78,10 @@ python = [
"daft-stats/python",
"daft-table/python",
"daft-writers/python",
"daft-context/python",
"dep:daft-catalog-python-catalog",
"dep:daft-connect",
"daft-connect/python",
"dep:pyo3",
"dep:pyo3-log"
]
Expand Down Expand Up @@ -173,7 +176,8 @@ members = [
# "src/spark-connect-script",
"src/generated/spark-connect",
"src/common/partitioning",
"src/daft-ray-execution"
"src/daft-py-runners",
"src/daft-context"
]

[workspace.dependencies]
Expand All @@ -195,13 +199,14 @@ common-display = {path = "src/common/display", default-features = false}
common-error = {path = "src/common/error", default-features = false}
common-file-formats = {path = "src/common/file-formats"}
common-runtime = {path = "src/common/runtime", default-features = false}
daft-context = {path = "src/daft-context"}
daft-core = {path = "src/daft-core"}
daft-dsl = {path = "src/daft-dsl"}
daft-hash = {path = "src/daft-hash"}
daft-local-execution = {path = "src/daft-local-execution"}
daft-logical-plan = {path = "src/daft-logical-plan"}
daft-micropartition = {path = "src/daft-micropartition"}
daft-ray-execution = {path = "src/daft-ray-execution"}
daft-py-runners = {path = "src/daft-py-runners"}
daft-scan = {path = "src/daft-scan"}
daft-schema = {path = "src/daft-schema"}
daft-sql = {path = "src/daft-sql"}
Expand Down
Loading

0 comments on commit d40aef8

Please sign in to comment.