From 418ae63a68461143afd34214ad541056546cfa2d Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Mon, 16 Sep 2024 19:54:27 -0700 Subject: [PATCH 1/2] Add support for TIMESTAMP_NTZ in FlintDataType Signed-off-by: Chase Engelbrecht --- .../org/apache/spark/sql/flint/datatype/FlintDataType.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flint-spark-integration/src/main/scala/org/apache/spark/sql/flint/datatype/FlintDataType.scala b/flint-spark-integration/src/main/scala/org/apache/spark/sql/flint/datatype/FlintDataType.scala index 0db0f6da6..a4b23bd46 100644 --- a/flint-spark-integration/src/main/scala/org/apache/spark/sql/flint/datatype/FlintDataType.scala +++ b/flint-spark-integration/src/main/scala/org/apache/spark/sql/flint/datatype/FlintDataType.scala @@ -144,7 +144,7 @@ object FlintDataType { case FloatType => JObject("type" -> JString("float")) // Date - case TimestampType => + case TimestampType | _: TimestampNTZType => JObject( "type" -> JString("date"), "format" -> JString("strict_date_optional_time_nanos")); From 783e01a98600c9ee12e65b6ed945df4c2afb1124 Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Mon, 16 Sep 2024 19:59:22 -0700 Subject: [PATCH 2/2] Add unit test Signed-off-by: Chase Engelbrecht --- .../sql/flint/datatype/FlintDataTypeSuite.scala | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/flint-spark-integration/src/test/scala/org/apache/spark/sql/flint/datatype/FlintDataTypeSuite.scala b/flint-spark-integration/src/test/scala/org/apache/spark/sql/flint/datatype/FlintDataTypeSuite.scala index 9cf9c553f..94f4839d6 100644 --- a/flint-spark-integration/src/test/scala/org/apache/spark/sql/flint/datatype/FlintDataTypeSuite.scala +++ b/flint-spark-integration/src/test/scala/org/apache/spark/sql/flint/datatype/FlintDataTypeSuite.scala @@ -197,6 +197,21 @@ class FlintDataTypeSuite extends FlintSuite with Matchers { FlintDataType.serialize(sparkStructType) shouldBe compactJson(flintDataType) } + test("flint timestamp_ntz serialize") { + val flintDataType = """{ + | "properties": { + | "timestampField": { + | "type": "date", + | "format": "strict_date_optional_time_nanos" + | } + | } + |}""".stripMargin + val sparkStructType = StructType( + StructField("timestampField", TimestampNTZType, true) :: + Nil) + FlintDataType.serialize(sparkStructType) shouldBe compactJson(flintDataType) + } + test("spark array type map to should map to array element type in OpenSearch") { val flintDataType = """{ | "properties": {