fix: update file path logic

theseus-rs · Dec 16, 2024 · 5b135b3 · 5b135b3
1 parent 9c18a62
commit 5b135b3
Show file tree

Hide file tree

Showing 14 changed files with 137 additions and 91 deletions.
diff --git a/README.md b/README.md
@@ -69,26 +69,26 @@ rsql --url "<url>" -- "<query>"
 
 | Driver             | URL                                                                                                                       |
 |--------------------|---------------------------------------------------------------------------------------------------------------------------|
-| arrow (polars)     | `arrow://?file=<file>`                                                                                                    |
-| avro (polars)      | `avro://?file=<file>`                                                                                                     |
+| arrow (polars)     | `arrow://<file>`                                                                                                          |
+| avro (polars)      | `avro://<file>`                                                                                                           |
 | cockroachdb (sqlx) | `cockroachdb://<user[:password>]@<host>[:<port>]/<database>`                                                              |
-| csv (polars)       | `csv://?file=<file>[&has_header=<true/false>][&quote=<char>][&skip_rows=<n>]`                                             |
-| delimited (polars) | `delimited://?file=<file>[&separator=<char>][&has_header=<true/false>][&quote=<char>][&skip_rows=<n>]`                    |
+| csv (polars)       | `csv://<file>[?has_header=<true/false>][&quote=<char>][&skip_rows=<n>]`                                                   |
+| delimited (polars) | `delimited://<file>[?separator=<char>][&has_header=<true/false>][&quote=<char>][&skip_rows=<n>]`                          |
 | duckdb             | `duckdb://?<memory=true>[&file=<database_file>]`                                                                          |
-| json (polars)      | `json://?file=<file>`                                                                                                     |
-| jsonl (polars)     | `jsonl://?file=<file>`                                                                                                    |
+| json (polars)      | `json://<file>`                                                                                                           |
+| jsonl (polars)     | `jsonl://<file>`                                                                                                          |
 | libsql¹            | `libsql://<host>?[<memory=true>][&file=<database_file>][&auth_token=<token>]`                                             |
 | mariadb (sqlx)     | `mariadb://<user>[:<password>]@<host>[:<port>]/<database>`                                                                |
 | mysql (sqlx)       | `mysql://<user>[:<password>]@<host>[:<port>]/<database>`                                                                  |
-| parquet (polars)   | `parquet://?file=<file>`                                                                                                  |
+| parquet (polars)   | `parquet://<file>`                                                                                                        |
 | postgres           | `postgres://<user>[:<password>]@<host>[:<port>]/<database>?<embedded=true>`                                               |
 | postgresql (sqlx)  | `postgresql://<user>[:<password>]@<host>[:<port>]/<database>?<embedded=true>`                                             |
 | redshift (sqlx)    | `redshift://<user[:password>]@<host>[:<port>]/<database>`                                                                 |
 | rusqlite           | `rusqlite://?<memory=true>[&file=<database_file>]`                                                                        |
 | snowflake          | `snowflake://<user>[:<token>]@<account>.snowflakecomputing.com/[?private_key_file=pkey_file&public_key_file=pubkey_file]` |
 | sqlite (sqlx)      | `sqlite://?<memory=true>[&file=<database_file>]`                                                                          |
 | sqlserver          | `sqlserver://<user>[:<password>]@<host>[:<port>]/<database>`                                                              |
-| tsv (polars)       | `tsv://?file=<file>[&has_header=<true/false>][&quote=<char>][&skip_rows=<n>]`                                             |
+| tsv (polars)       | `tsv://<file>[?has_header=<true/false>][&quote=<char>][&skip_rows=<n>]`                                                   |
 
 ¹ `libsql` needs to be enabled with the `libsql` feature flag; it is disabled by default as it conflicts
 with `rusqlite`.

diff --git a/rsql_cli/docs/src/chapter2/drivers/index.md b/rsql_cli/docs/src/chapter2/drivers/index.md
@@ -12,26 +12,26 @@ The drivers command displays the available database drivers.
 
 | Driver        | Description                                                                                            | URL                                                                                                                       |
 |---------------|--------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------|
-| `arrow`       | Arrow IPC file driver provided by [Polars](https://github.com/pola-rs/polars)                          | `arrow://?file=<file>`                                                                                                    |
-| `avro`        | Avro file driver provided by [Polars](https://github.com/pola-rs/polars)                               | `avro://?file=<file>`                                                                                                     |
-| `cockroachdb` | CockroachDB driver provided by [SQLx](https://github.com/launchbadge/sqlx)                             | `redshift://<user>[:<password>]@<host>[:<port>]/<database>`                                                               |
-| `csv`         | Comma Separated Value (CSV) file driver provided by [Polars](https://github.com/pola-rs/polars)        | `csv://?file=<file>[&has_header=<true/false>][&quote=<char>][&skip_rows=<n>]`                                             |
-| `delimited`   | Delimited file driver provided by [Polars](https://github.com/pola-rs/polars)                          | `delimited://?file=<file>[&separator=<char>][&has_header=<true/false>][&quote=<char>][&skip_rows=<n>]`                    |
+| `arrow`       | Arrow IPC file driver provided by [Polars](https://github.com/pola-rs/polars)                          | `arrow://<file>`                                                                                                          |
+| `avro`        | Avro file driver provided by [Polars](https://github.com/pola-rs/polars)                               | `avro://<file>`                                                                                                           |
+| `cockroachdb` | CockroachDB driver provided by [SQLx](https://github.com/launchbadge/sqlx)                             | `cockroachdb://<user>[:<password>]@<host>[:<port>]/<database>`                                                            |
+| `csv`         | Comma Separated Value (CSV) file driver provided by [Polars](https://github.com/pola-rs/polars)        | `csv://<file>[?has_header=<true/false>][&quote=<char>][&skip_rows=<n>]`                                                   |
+| `delimited`   | Delimited file driver provided by [Polars](https://github.com/pola-rs/polars)                          | `delimited://<file>[?separator=<char>][&has_header=<true/false>][&quote=<char>][&skip_rows=<n>]`                          |
 | `duckdb`      | DuckDB provided by [DuckDB](https://duckdb.org/)                                                       | `duckdb://?<memory=true>[&file=<database_file>]`                                                                          |
-| `json`        | JSON file driver provided by [Polars](https://github.com/pola-rs/polars)                               | `json://?file=<file>`                                                                                                     |
-| `jsonl`       | JSONL file driver provided by [Polars](https://github.com/pola-rs/polars)                              | `jsonl://?file=<file>`                                                                                                    |
+| `json`        | JSON file driver provided by [Polars](https://github.com/pola-rs/polars)                               | `json://<file>`                                                                                                           |
+| `jsonl`       | JSONL file driver provided by [Polars](https://github.com/pola-rs/polars)                              | `jsonl://<file>`                                                                                                          |
 | `libsql`      | LibSQL provided by [Turso](https://github.com/tursodatabase/libsql)                                    | `libsql://<host>?[<memory=true>][&file=<database_file>][&auth_token=<token>]`                                             |
 | `mariadb`     | MariaDB provided by [SQLx](https://github.com/launchbadge/sqlx)                                        | `mariadb://<user>[:<password>]@<host>[:<port>]/<database>`                                                                |
 | `mysql`       | MySQL provided by [SQLx](https://github.com/launchbadge/sqlx)                                          | `mysql://<user>[:<password>]@<host>[:<port>]/<database>`                                                                  |
-| `parquet`     | Parquet file driver provided by [Polars](https://github.com/pola-rs/polars)                            | `parquet://?file=<file>`                                                                                                  |
+| `parquet`     | Parquet file driver provided by [Polars](https://github.com/pola-rs/polars)                            | `parquet://<file>`                                                                                                        |
 | `postgres`    | PostgreSQL driver provided by [rust-postgres](https://github.com/sfackler/rust-postgres)               | `postgres://<user>[:<password>]@<host>[:<port>]/<database>?<embedded=true>`                                               |
 | `postgresql`  | PostgreSQL driver provided by [SQLx](https://github.com/launchbadge/sqlx)                              | `postgresql://<user>[:<password>]@<host>[:<port>]/<database>?<embedded=true>`                                             |
 | `redshift`    | Redshift driver provided by [SQLx](https://github.com/launchbadge/sqlx)                                | `redshift://<user>[:<password>]@<host>[:<port>]/<database>`                                                               |
 | `rusqlite`    | SQLite provided by [Rusqlite](https://github.com/rusqlite/rusqlite?tab=readme-ov-file#rusqlite)        | `rusqlite://?<memory=true>[&file=<database_file>]`                                                                        |
 | `snowflake`   | Snowflake provided by [Snowflake SQL API](https://docs.snowflake.com/en/developer-guide/sql-api/index) | `snowflake://<user>[:<token>]@<account>.snowflakecomputing.com/[?private_key_file=pkey_file&public_key_file=pubkey_file]` |
 | `sqlite`      | SQLite provided by [SQLx](https://github.com/launchbadge/sqlx)                                         | `sqlite://?<memory=true>[&file=<database_file>]`                                                                          |
 | `sqlserver`   | SQL Server provided by [Tiberius](https://github.com/prisma/tiberius)                                  | `sqlserver://<user>[:<password>]@<host>[:<port>]/<database>`                                                              |
-| `tsv`         | Tab Separated Value (TSV) file driver provided by [Polars](https://github.com/pola-rs/polars)          | `tsv://?file=<file>[&has_header=<true/false>][&quote=<char>][&skip_rows=<n>]`                                             |
+| `tsv`         | Tab Separated Value (TSV) file driver provided by [Polars](https://github.com/pola-rs/polars)          | `tsv://<file>[?has_header=<true/false>][&quote=<char>][&skip_rows=<n>]`                                                   |
 
 ### Examples
 

diff --git a/rsql_drivers/src/arrow/driver.rs b/rsql_drivers/src/arrow/driver.rs
@@ -1,11 +1,10 @@
 use crate::error::Result;
 use crate::polars::Connection;
-use crate::Error::InvalidUrl;
+use crate::url::UrlExtension;
 use async_trait::async_trait;
 use polars::io::SerReader;
 use polars::prelude::{IntoLazy, IpcReader};
 use polars_sql::SQLContext;
-use std::collections::HashMap;
 use std::fs::File;
 use url::Url;
 
@@ -24,14 +23,8 @@ impl crate::Driver for Driver {
         _password: Option<String>,
     ) -> Result<Box<dyn crate::Connection>> {
         let parsed_url = Url::parse(url.as_str())?;
-        let query_parameters: HashMap<String, String> =
-            parsed_url.query_pairs().into_owned().collect();
-
-        // Read Options
-        let file_name = query_parameters
-            .get("file")
-            .ok_or(InvalidUrl("Missing file parameter".to_string()))?;
-        let file = File::open(file_name)?;
+        let file_name = parsed_url.to_file()?.to_string_lossy().to_string();
+        let file = File::open(&file_name)?;
 
         let data_frame = IpcReader::new(file).set_rechunk(true).finish()?;
 
@@ -46,12 +39,11 @@ impl crate::Driver for Driver {
 
 #[cfg(test)]
 mod test {
+    use crate::test::dataset_url;
     use crate::{DriverManager, Value};
 
-    const CRATE_DIRECTORY: &str = env!("CARGO_MANIFEST_DIR");
-
     fn database_url() -> String {
-        format!("arrow://?file={CRATE_DIRECTORY}/../datasets/users.arrow")
+        dataset_url("arrow", "users.arrow")
     }
 
     #[tokio::test]

diff --git a/rsql_drivers/src/avro/driver.rs b/rsql_drivers/src/avro/driver.rs
@@ -1,12 +1,11 @@
 use crate::error::Result;
 use crate::polars::Connection;
-use crate::Error::InvalidUrl;
+use crate::url::UrlExtension;
 use async_trait::async_trait;
 use polars::io::avro::AvroReader;
 use polars::io::SerReader;
 use polars::prelude::IntoLazy;
 use polars_sql::SQLContext;
-use std::collections::HashMap;
 use std::fs::File;
 use url::Url;
 
@@ -25,14 +24,8 @@ impl crate::Driver for Driver {
         _password: Option<String>,
     ) -> Result<Box<dyn crate::Connection>> {
         let parsed_url = Url::parse(url.as_str())?;
-        let query_parameters: HashMap<String, String> =
-            parsed_url.query_pairs().into_owned().collect();
-
-        // Read Options
-        let file_name = query_parameters
-            .get("file")
-            .ok_or(InvalidUrl("Missing file parameter".to_string()))?;
-        let file = File::open(file_name)?;
+        let file_name = parsed_url.to_file()?.to_string_lossy().to_string();
+        let file = File::open(&file_name)?;
 
         let data_frame = AvroReader::new(file).set_rechunk(true).finish()?;
 
@@ -47,12 +40,11 @@ impl crate::Driver for Driver {
 
 #[cfg(test)]
 mod test {
+    use crate::test::dataset_url;
     use crate::{DriverManager, Value};
 
-    const CRATE_DIRECTORY: &str = env!("CARGO_MANIFEST_DIR");
-
     fn database_url() -> String {
-        format!("avro://?file={CRATE_DIRECTORY}/../datasets/users.avro")
+        dataset_url("avro", "users.avro")
     }
 
     #[tokio::test]

diff --git a/rsql_drivers/src/csv/driver.rs b/rsql_drivers/src/csv/driver.rs
@@ -16,27 +16,26 @@ impl crate::Driver for Driver {
         url: String,
         password: Option<String>,
     ) -> Result<Box<dyn crate::Connection>> {
-        let url = format!("{url}&separator=,");
+        let url = format!("{url}?separator=,");
         DelimitedDriver.connect(url, password).await
     }
 }
 
 #[cfg(test)]
 mod test {
+    use crate::test::dataset_url;
     use crate::{DriverManager, Value};
 
-    const CRATE_DIRECTORY: &str = env!("CARGO_MANIFEST_DIR");
-
     fn database_url() -> String {
-        format!("csv://?file={CRATE_DIRECTORY}/../datasets/users.csv")
+        dataset_url("csv", "users.csv")
     }
 
     #[tokio::test]
     async fn test_driver_connect() -> anyhow::Result<()> {
         let database_url = database_url();
         let driver_manager = DriverManager::default();
         let mut connection = driver_manager.connect(&database_url).await?;
-        let expected_url = format!("{database_url}&separator=,");
+        let expected_url = format!("{database_url}?separator=,");
         assert_eq!(&expected_url, connection.url());
         connection.close().await?;
         Ok(())

diff --git a/rsql_drivers/src/delimited/driver.rs b/rsql_drivers/src/delimited/driver.rs
@@ -1,6 +1,7 @@
 use crate::error::Result;
 use crate::polars::Connection;
-use crate::Error::{ConversionError, InvalidUrl};
+use crate::url::UrlExtension;
+use crate::Error::ConversionError;
 use async_trait::async_trait;
 use polars::io::SerReader;
 use polars::prelude::{CsvParseOptions, CsvReadOptions, IntoLazy};
@@ -29,10 +30,8 @@ impl crate::Driver for Driver {
             parsed_url.query_pairs().into_owned().collect();
 
         // Read Options
-        let file_name = query_parameters
-            .get("file")
-            .ok_or(InvalidUrl("Missing file parameter".to_string()))?;
-        let file = File::open(file_name)?;
+        let file_name = parsed_url.to_file()?.to_string_lossy().to_string();
+        let file = File::open(&file_name)?;
         let has_header = query_parameters
             .get("has_header")
             .map_or(true, |v| v == "true");
@@ -118,12 +117,12 @@ fn string_to_ascii_char(value: &String) -> Result<u8> {
 
 #[cfg(test)]
 mod test {
+    use crate::test::dataset_url;
     use crate::{DriverManager, Value};
 
-    const CRATE_DIRECTORY: &str = env!("CARGO_MANIFEST_DIR");
-
     fn database_url() -> String {
-        format!("delimited://?file={CRATE_DIRECTORY}/../datasets/users.pipe&separator=|")
+        let path = dataset_url("delimited", "users.pipe");
+        format!("{path}?separator=|")
     }
 
     #[tokio::test]

diff --git a/rsql_drivers/src/json/driver.rs b/rsql_drivers/src/json/driver.rs
@@ -1,6 +1,7 @@
 use crate::error::Result;
 use crate::polars::Connection;
-use crate::Error::{ConversionError, InvalidUrl};
+use crate::url::UrlExtension;
+use crate::Error::ConversionError;
 use async_trait::async_trait;
 use polars::io::SerReader;
 use polars::prelude::{IntoLazy, JsonReader};
@@ -29,10 +30,8 @@ impl crate::Driver for Driver {
             parsed_url.query_pairs().into_owned().collect();
 
         // Read Options
-        let file_name = query_parameters
-            .get("file")
-            .ok_or(InvalidUrl("Missing file parameter".to_string()))?;
-        let file = File::open(file_name)?;
+        let file_name = parsed_url.to_file()?.to_string_lossy().to_string();
+        let file = File::open(&file_name)?;
         let ignore_errors = query_parameters
             .get("ignore_errors")
             .map_or(false, |v| v == "true");
@@ -67,12 +66,11 @@ impl crate::Driver for Driver {
 
 #[cfg(test)]
 mod test {
+    use crate::test::dataset_url;
     use crate::{DriverManager, Value};
 
-    const CRATE_DIRECTORY: &str = env!("CARGO_MANIFEST_DIR");
-
     fn database_url() -> String {
-        format!("json://?file={CRATE_DIRECTORY}/../datasets/users.json")
+        dataset_url("json", "users.json")
     }
 
     #[tokio::test]

diff --git a/rsql_drivers/src/jsonl/driver.rs b/rsql_drivers/src/jsonl/driver.rs
@@ -1,6 +1,7 @@
 use crate::error::Result;
 use crate::polars::Connection;
-use crate::Error::{ConversionError, InvalidUrl};
+use crate::url::UrlExtension;
+use crate::Error::ConversionError;
 use async_trait::async_trait;
 use polars::io::SerReader;
 use polars::prelude::{IntoLazy, JsonLineReader};
@@ -29,10 +30,8 @@ impl crate::Driver for Driver {
             parsed_url.query_pairs().into_owned().collect();
 
         // Read Options
-        let file_name = query_parameters
-            .get("file")
-            .ok_or(InvalidUrl("Missing file parameter".to_string()))?;
-        let file = File::open(file_name)?;
+        let file_name = parsed_url.to_file()?.to_string_lossy().to_string();
+        let file = File::open(&file_name)?;
         let ignore_errors = query_parameters
             .get("ignore_errors")
             .map_or(false, |v| v == "true");
@@ -67,12 +66,11 @@ impl crate::Driver for Driver {
 
 #[cfg(test)]
 mod test {
+    use crate::test::dataset_url;
     use crate::{DriverManager, Value};
 
-    const CRATE_DIRECTORY: &str = env!("CARGO_MANIFEST_DIR");
-
     fn database_url() -> String {
-        format!("jsonl://?file={CRATE_DIRECTORY}/../datasets/users.jsonl")
+        dataset_url("jsonl", "users.jsonl")
     }
 
     #[tokio::test]