From a9b0d751fd7262d39f935161fa07e5577fc33cbf Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sat, 18 Jan 2025 12:04:20 -0500 Subject: [PATCH 1/4] Set nanoseconds in DateTime64 max to zeros and not nines --- .../clickhouse/sink/connector/metadata/DataTypeRange.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/metadata/DataTypeRange.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/metadata/DataTypeRange.java index dae61dbae..f56877fe9 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/metadata/DataTypeRange.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/metadata/DataTypeRange.java @@ -22,7 +22,7 @@ public class DataTypeRange public static final Instant CLICKHOUSE_MIN_SUPPORTED_DATETIME64 = from(ofEpochMilli (DATETIME64_MIN * 1000).atZone(ZoneId.of("UTC"))).plusNanos(DATETIME64_MIN * 1000 % 1_000); public static final Instant CLICKHOUSE_MAX_SUPPORTED_DATETIME64 = from(ofEpochMilli - (DATETIME64_MAX * 1000).atZone(ZoneId.of("UTC")).withHour(23).withMinute(59).withSecond(59).withNano(999999999)); + (DATETIME64_MAX * 1000).atZone(ZoneId.of("UTC")).withHour(23).withMinute(59).withSecond(59).withNano(000000)); // DateTime and DateTime32 From d619bc9c72678f8a960768081a68f4d9aeab99ec Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sun, 19 Jan 2025 17:23:22 -0500 Subject: [PATCH 2/4] Fixed unit tests. --- .../clickhouse/sink/connector/metadata/DataTypeRange.java | 2 +- .../sink/connector/converters/DebeziumConverterTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/metadata/DataTypeRange.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/metadata/DataTypeRange.java index f56877fe9..c20b1909d 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/metadata/DataTypeRange.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/metadata/DataTypeRange.java @@ -45,6 +45,6 @@ public class DataTypeRange public static final String DATETIME_6_MAX = "2299-12-31 23:59:59.999999"; - public static final String DATETIME64_6_MAX = "2299-12-31 23:59:59.99999999"; + public static final String DATETIME64_6_MAX = "2299-12-31 23:59:59.00000000"; } diff --git a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverterTest.java b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverterTest.java index 336ece636..41a596c74 100644 --- a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverterTest.java +++ b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverterTest.java @@ -136,11 +136,11 @@ public void testMicroTimestampConverterMax() { // DateTime64 and America/Chicago timezone. String formattedTimestampChicagoTZ = DebeziumConverter.MicroTimestampConverter.convert(timestampEpoch, ZoneId.of("America/Chicago"), ClickHouseDataType.DateTime64); - Assert.assertTrue(formattedTimestampChicagoTZ.equalsIgnoreCase("2299-12-31 17:59:59.99999999")); + Assert.assertTrue(formattedTimestampChicagoTZ.equalsIgnoreCase("2299-12-31 17:59:59.00000000")); // DateTime64 and America/Los Angeles timezone. String formattedTimestampLATZ = DebeziumConverter.MicroTimestampConverter.convert(timestampEpoch, ZoneId.of("America/Los_Angeles"), ClickHouseDataType.DateTime64); - Assert.assertTrue(formattedTimestampLATZ.equalsIgnoreCase("2299-12-31 15:59:59.99999999")); + Assert.assertTrue(formattedTimestampLATZ.equalsIgnoreCase("2299-12-31 15:59:59.00000000")); // DateTime32 and UTC timezone String formattedTimestampDate32 = DebeziumConverter.MicroTimestampConverter.convert(timestampEpoch, ZoneId.of("UTC"), ClickHouseDataType.DateTime); From fc77d07c9a4876b157629daa56c22160fb322134 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 28 Jan 2025 14:41:41 +0400 Subject: [PATCH 3/4] try to fix datetime tests after precision fix --- .../tests/integration/tests/datatypes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sink-connector-lightweight/tests/integration/tests/datatypes.py b/sink-connector-lightweight/tests/integration/tests/datatypes.py index b59e72e74..267b105b1 100644 --- a/sink-connector-lightweight/tests/integration/tests/datatypes.py +++ b/sink-connector-lightweight/tests/integration/tests/datatypes.py @@ -94,18 +94,18 @@ def check_datetime_column(self, precision, data): if precision != "0": assert ( clickhouse_values.output.strip().replace('"', "") - == f"2299-12-31 23:59:59.{'9'*int(precision)}" + == f"2299-12-31 23:59:59.{'0'*int(precision)}" ), error() else: assert ( clickhouse_values.output.strip().replace('"', "") == "2299-12-31 23:59:59" ), error() - elif data[:19] == "2299-12-31 23:59:59.9": + elif data[:19] == "2299-12-31 23:59:59.0": if precision != "0": assert ( clickhouse_values.output.strip().replace('"', "") - == f"2299-12-31 23:59:59.{'9'*int(precision)}" + == f"2299-12-31 23:59:59.{'0'*int(precision)}" ), error() else: assert ( From 5c8ee11ad48058a1c0a6f5f1835027c2474bf8d3 Mon Sep 17 00:00:00 2001 From: Selfeer Date: Tue, 28 Jan 2025 18:34:43 +0400 Subject: [PATCH 4/4] fix datetime tests --- .../tests/integration/tests/datatypes.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/sink-connector-lightweight/tests/integration/tests/datatypes.py b/sink-connector-lightweight/tests/integration/tests/datatypes.py index 267b105b1..16e6936cb 100644 --- a/sink-connector-lightweight/tests/integration/tests/datatypes.py +++ b/sink-connector-lightweight/tests/integration/tests/datatypes.py @@ -20,13 +20,23 @@ def adjust_precision(datetime_str, precision): if len(parts) == 1: return f"{main_part}.{'0' * int(precision)}" - microseconds_part = parts[1] + microseconds_part = parts[1] #9999 adjusted_microseconds = microseconds_part[: int(precision)].ljust( int(precision), "0" ) return f"{main_part}.{adjusted_microseconds}" +def replace_nines_with_zeroes(datetime_str): + """Replace nines in the precision part of a datetime string with zeroes.""" + parts = datetime_str.split(".") + if len(parts) == 1: + return datetime_str + + main_part = parts[0] + precision_part = parts[1].replace("9", "0") + + return f"{main_part}.{precision_part}" @TestStep(Given) def create_table_with_datetime_column(self, table_name, data, precision): @@ -52,6 +62,9 @@ def check_datetime_column(self, precision, data): table_name = "table_" + getuid() clickhouse_node = self.context.clickhouse_node + if data == "2299-12-31 23:59:59.999999": + data = replace_nines_with_zeroes(data) + data = adjust_precision(datetime_str=data, precision=precision) with Given(