Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set nanoseconds in DateTime64 max to zeros and not nines #972

Open
wants to merge 4 commits into
base: 2.5.1
Choose a base branch
from
Open

Set nanoseconds in DateTime64 max to zeros and not nines #972

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a9b0d75
Set nanoseconds in DateTime64 max to zeros and not nines
subkanthi Jan 18, 2025
d619bc9
Fixed unit tests.
subkanthi Jan 19, 2025
fc77d07
try to fix datetime tests after precision fix
Jan 28, 2025
5c8ee11
fix datetime tests
Jan 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions sink-connector-lightweight/tests/integration/tests/datatypes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,23 @@ def adjust_precision(datetime_str, precision):
if len(parts) == 1:
return f"{main_part}.{'0' * int(precision)}"

microseconds_part = parts[1]
microseconds_part = parts[1] #9999
adjusted_microseconds = microseconds_part[: int(precision)].ljust(
int(precision), "0"
)

return f"{main_part}.{adjusted_microseconds}"

def replace_nines_with_zeroes(datetime_str):
"""Replace nines in the precision part of a datetime string with zeroes."""
parts = datetime_str.split(".")
if len(parts) == 1:
return datetime_str

main_part = parts[0]
precision_part = parts[1].replace("9", "0")

return f"{main_part}.{precision_part}"

@TestStep(Given)
def create_table_with_datetime_column(self, table_name, data, precision):
Expand All @@ -52,6 +62,9 @@ def check_datetime_column(self, precision, data):
table_name = "table_" + getuid()
clickhouse_node = self.context.clickhouse_node

if data == "2299-12-31 23:59:59.999999":
data = replace_nines_with_zeroes(data)

data = adjust_precision(datetime_str=data, precision=precision)

with Given(
Expand Down Expand Up @@ -94,18 +107,18 @@ def check_datetime_column(self, precision, data):
if precision != "0":
assert (
clickhouse_values.output.strip().replace('"', "")
== f"2299-12-31 23:59:59.{'9'*int(precision)}"
== f"2299-12-31 23:59:59.{'0'*int(precision)}"
), error()
else:
assert (
clickhouse_values.output.strip().replace('"', "")
== "2299-12-31 23:59:59"
), error()
elif data[:19] == "2299-12-31 23:59:59.9":
elif data[:19] == "2299-12-31 23:59:59.0":
if precision != "0":
assert (
clickhouse_values.output.strip().replace('"', "")
== f"2299-12-31 23:59:59.{'9'*int(precision)}"
== f"2299-12-31 23:59:59.{'0'*int(precision)}"
), error()
else:
assert (
Expand Down
4 changes: 2 additions & 2 deletions ...onnector/src/main/java/com/altinity/clickhouse/sink/connector/metadata/DataTypeRange.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public class DataTypeRange
public static final Instant CLICKHOUSE_MIN_SUPPORTED_DATETIME64 = from(ofEpochMilli
(DATETIME64_MIN * 1000).atZone(ZoneId.of("UTC"))).plusNanos(DATETIME64_MIN * 1000 % 1_000);
public static final Instant CLICKHOUSE_MAX_SUPPORTED_DATETIME64 = from(ofEpochMilli
(DATETIME64_MAX * 1000).atZone(ZoneId.of("UTC")).withHour(23).withMinute(59).withSecond(59).withNano(999999999));
(DATETIME64_MAX * 1000).atZone(ZoneId.of("UTC")).withHour(23).withMinute(59).withSecond(59).withNano(000000));


// DateTime and DateTime32
Expand All @@ -45,6 +45,6 @@ public class DataTypeRange

public static final String DATETIME_6_MAX = "2299-12-31 23:59:59.999999";

public static final String DATETIME64_6_MAX = "2299-12-31 23:59:59.99999999";
public static final String DATETIME64_6_MAX = "2299-12-31 23:59:59.00000000";

}
4 changes: 2 additions & 2 deletions ...rc/test/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,11 +136,11 @@ public void testMicroTimestampConverterMax() {

// DateTime64 and America/Chicago timezone.
String formattedTimestampChicagoTZ = DebeziumConverter.MicroTimestampConverter.convert(timestampEpoch, ZoneId.of("America/Chicago"), ClickHouseDataType.DateTime64);
Assert.assertTrue(formattedTimestampChicagoTZ.equalsIgnoreCase("2299-12-31 17:59:59.99999999"));
Assert.assertTrue(formattedTimestampChicagoTZ.equalsIgnoreCase("2299-12-31 17:59:59.00000000"));

// DateTime64 and America/Los Angeles timezone.
String formattedTimestampLATZ = DebeziumConverter.MicroTimestampConverter.convert(timestampEpoch, ZoneId.of("America/Los_Angeles"), ClickHouseDataType.DateTime64);
Assert.assertTrue(formattedTimestampLATZ.equalsIgnoreCase("2299-12-31 15:59:59.99999999"));
Assert.assertTrue(formattedTimestampLATZ.equalsIgnoreCase("2299-12-31 15:59:59.00000000"));

// DateTime32 and UTC timezone
String formattedTimestampDate32 = DebeziumConverter.MicroTimestampConverter.convert(timestampEpoch, ZoneId.of("UTC"), ClickHouseDataType.DateTime);
Expand Down
Loading