From dad66aab63871c7b65c8266e763ffd8fc18dbcc1 Mon Sep 17 00:00:00 2001 From: rui-mo Date: Tue, 24 Dec 2024 11:00:36 +0800 Subject: [PATCH] Pass timezone --- .../datasources/velox/VeloxParquetWriterInjects.scala | 5 +++++ cpp/velox/operators/writer/VeloxParquetDataSource.cc | 1 + ep/build-velox/src/get_velox.sh | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala index a75fec02853a..a66b271cb11b 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala @@ -39,6 +39,11 @@ class VeloxParquetWriterInjects extends VeloxFormatWriterInjects { GlutenConfig.PARQUET_BLOCK_ROWS, GlutenConfig.get.columnarParquetWriteBlockRows.toString) sparkOptions.put(GlutenConfig.PARQUET_BLOCK_ROWS, blockRows) + sparkOptions.put( + SQLConf.SESSION_LOCAL_TIMEZONE.key, + options.getOrElse( + SQLConf.SESSION_LOCAL_TIMEZONE.key, + SQLConf.SESSION_LOCAL_TIMEZONE.defaultValueString)) options .get(GlutenConfig.PARQUET_GZIP_WINDOW_SIZE) .foreach(sparkOptions.put(GlutenConfig.PARQUET_GZIP_WINDOW_SIZE, _)) diff --git a/cpp/velox/operators/writer/VeloxParquetDataSource.cc b/cpp/velox/operators/writer/VeloxParquetDataSource.cc index af8e233ad082..15082b8b8281 100644 --- a/cpp/velox/operators/writer/VeloxParquetDataSource.cc +++ b/cpp/velox/operators/writer/VeloxParquetDataSource.cc @@ -97,6 +97,7 @@ void VeloxParquetDataSource::init(const std::unordered_map( maxRowGroupRows_, maxRowGroupBytes_, [&]() { return false; }); }; + writeOption.parquetWriteTimestampTimeZone = getConfigValue(sparkConfs, kSessionTimezone, std::nullopt); auto schema = gluten::fromArrowSchema(schema_); parquetWriter_ = std::make_unique(std::move(sink_), writeOption, pool_, asRowType(schema)); diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 7c51af3fd294..5b094136d58e 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2025_02_11 +VELOX_BRANCH=2025_02_11_fix VELOX_HOME="" OS=`uname -s`