diff --git a/rust/tests/data/http_requests/_delta_log/00000000000000000000.json b/rust/tests/data/http_requests/_delta_log/00000000000000000000.json new file mode 100644 index 0000000000..08fa6636a9 --- /dev/null +++ b/rust/tests/data/http_requests/_delta_log/00000000000000000000.json @@ -0,0 +1,3 @@ +{"commitInfo":{"delta-rs":"0.9.0","timestamp":1681604880998}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":1}} +{"metaData":{"id":"1c863a88-7231-4ffc-a24b-3bbfad87d80a","name":"http_requests","description":"","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"date\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ClientIP\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ClientRequestHost\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ClientRequestMethod\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ClientRequestURI\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"EdgeEndTimestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"EdgeResponseBytes\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"EdgeResponseStatus\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"EdgeStartTimestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["date"],"createdTime":1681604880998,"configuration":{}}} \ No newline at end of file diff --git a/rust/tests/data/http_requests/_delta_log/00000000000000000001.json b/rust/tests/data/http_requests/_delta_log/00000000000000000001.json new file mode 100644 index 0000000000..86934cfc80 --- /dev/null +++ b/rust/tests/data/http_requests/_delta_log/00000000000000000001.json @@ -0,0 +1,3 @@ +{"add":{"path":"date=2023-04-14/part-00000-731ab1b3-85a8-4bc3-92e5-96347fe3fd84-c000.snappy.parquet","size":5976,"partitionValues":{"date":"2023-04-14"},"modificationTime":1681604881837,"dataChange":true,"stats":"{\"numRecords\":1437,\"minValues\":{\"EdgeEndTimestamp\":\"2023-04-14T00:00:00.000Z\",\"ClientRequestHost\":\"example.com\",\"EdgeResponseStatus\":200,\"ClientRequestMethod\":\"GET\",\"EdgeStartTimestamp\":\"2023-04-14T00:00:00.000Z\",\"ClientIP\":\"127.0.0.1\",\"ClientRequestURI\":\"/\",\"EdgeResponseBytes\":303},\"maxValues\":{\"EdgeResponseBytes\":307,\"ClientRequestURI\":\"/\",\"EdgeResponseStatus\":200,\"EdgeStartTimestamp\":\"2023-04-14T00:00:45.000Z\",\"ClientIP\":\"127.0.0.1\",\"ClientRequestHost\":\"example.com\",\"ClientRequestMethod\":\"GET\",\"EdgeEndTimestamp\":\"2023-04-14T00:00:45.000Z\"},\"nullCount\":{\"ClientRequestHost\":0,\"ClientRequestURI\":0,\"ClientRequestMethod\":0,\"EdgeResponseStatus\":0,\"ClientIP\":0,\"EdgeEndTimestamp\":0,\"EdgeResponseBytes\":0,\"EdgeStartTimestamp\":0}}","tags":null}} +{"add":{"path":"date=2023-04-13/part-00000-e853fe2e-6f42-450c-8af1-4145b73a96c7-c000.snappy.parquet","size":3780,"partitionValues":{"date":"2023-04-13"},"modificationTime":1681604881849,"dataChange":true,"stats":"{\"numRecords\":144,\"minValues\":{\"EdgeStartTimestamp\":\"2023-04-13T23:58:58.000Z\",\"ClientRequestMethod\":\"GET\",\"ClientIP\":\"127.0.0.1\",\"EdgeEndTimestamp\":\"2023-04-13T23:58:58.000Z\",\"ClientRequestURI\":\"/\",\"ClientRequestHost\":\"example.com\",\"EdgeResponseBytes\":303,\"EdgeResponseStatus\":200},\"maxValues\":{\"ClientRequestHost\":\"example.com\",\"ClientRequestURI\":\"/\",\"ClientIP\":\"127.0.0.1\",\"EdgeResponseStatus\":200,\"ClientRequestMethod\":\"GET\",\"EdgeStartTimestamp\":\"2023-04-13T23:59:59.000Z\",\"EdgeEndTimestamp\":\"2023-04-14T00:00:00.000Z\",\"EdgeResponseBytes\":307},\"nullCount\":{\"ClientRequestMethod\":0,\"ClientIP\":0,\"EdgeResponseStatus\":0,\"ClientRequestURI\":0,\"EdgeEndTimestamp\":0,\"EdgeStartTimestamp\":0,\"EdgeResponseBytes\":0,\"ClientRequestHost\":0}}","tags":null}} +{"commitInfo":{"timestamp":1681604881849,"clientVersion":"delta-rs.0.9.0"}} \ No newline at end of file diff --git a/rust/tests/data/http_requests/date=2023-04-13/part-00000-e853fe2e-6f42-450c-8af1-4145b73a96c7-c000.snappy.parquet b/rust/tests/data/http_requests/date=2023-04-13/part-00000-e853fe2e-6f42-450c-8af1-4145b73a96c7-c000.snappy.parquet new file mode 100644 index 0000000000..ec0d1e367f Binary files /dev/null and b/rust/tests/data/http_requests/date=2023-04-13/part-00000-e853fe2e-6f42-450c-8af1-4145b73a96c7-c000.snappy.parquet differ diff --git a/rust/tests/data/http_requests/date=2023-04-14/part-00000-731ab1b3-85a8-4bc3-92e5-96347fe3fd84-c000.snappy.parquet b/rust/tests/data/http_requests/date=2023-04-14/part-00000-731ab1b3-85a8-4bc3-92e5-96347fe3fd84-c000.snappy.parquet new file mode 100644 index 0000000000..60f28fc97f Binary files /dev/null and b/rust/tests/data/http_requests/date=2023-04-14/part-00000-731ab1b3-85a8-4bc3-92e5-96347fe3fd84-c000.snappy.parquet differ diff --git a/rust/tests/datafusion_test.rs b/rust/tests/datafusion_test.rs index 769cf962e7..bc54a73f61 100644 --- a/rust/tests/datafusion_test.rs +++ b/rust/tests/datafusion_test.rs @@ -763,3 +763,65 @@ async fn test_datafusion_scan_timestamps() -> Result<()> { Ok(()) } + +#[tokio::test] +async fn test_issue_1292_datafusion_sql_projection() -> Result<()> { + let ctx = SessionContext::new(); + let table = deltalake::open_table("./tests/data/http_requests") + .await + .unwrap(); + ctx.register_table("http_requests", Arc::new(table))?; + + let batches = ctx + .sql("SELECT \"ClientRequestURI\" FROM http_requests LIMIT 5") + .await? + .collect() + .await?; + + let expected = vec![ + "+------------------+", + "| ClientRequestURI |", + "+------------------+", + "| / |", + "| / |", + "| / |", + "| / |", + "| / |", + "+------------------+", + ]; + + assert_batches_sorted_eq!(&expected, &batches); + + Ok(()) +} + +#[tokio::test] +async fn test_issue_1291_datafusion_sql_partitioned_data() -> Result<()> { + let ctx = SessionContext::new(); + let table = deltalake::open_table("./tests/data/http_requests") + .await + .unwrap(); + ctx.register_table("http_requests", Arc::new(table))?; + + let batches = ctx + .sql("SELECT * FROM http_requests WHERE date >= '2023-04-13' LIMIT 5") + .await? + .collect() + .await?; + + let expected = vec![ + "+------------------+", + "| ClientRequestURI |", + "+------------------+", + "| / |", + "| / |", + "| / |", + "| / |", + "| / |", + "+------------------+", + ]; + + assert_batches_sorted_eq!(&expected, &batches); + + Ok(()) +}