Skip to content

Commit

Permalink
use precise timer when getting timestamps, fixes test for py 3.13
Browse files Browse the repository at this point in the history
  • Loading branch information
rudolfix committed Feb 2, 2025
1 parent f8d8c36 commit e8077bc
Show file tree
Hide file tree
Showing 3 changed files with 10 additions and 6 deletions.
2 changes: 1 addition & 1 deletion dlt/common/storages/load_package.py
Original file line number Diff line number Diff line change
Expand Up @@ -739,7 +739,7 @@ def is_package_partially_loaded(package_info: LoadPackageInfo) -> bool:

@staticmethod
def _job_elapsed_time_seconds(file_path: str, now_ts: float = None) -> float:
return (now_ts or pendulum.now().timestamp()) - os.path.getmtime(file_path)
return (now_ts or precise_time()) - os.path.getmtime(file_path)

@staticmethod
def filter_jobs_for_table(
Expand Down
5 changes: 3 additions & 2 deletions dlt/destinations/impl/dummy/dummy.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
WithStagingDataset,
LoadJob,
)
from dlt.common.time import precise_time
from dlt.destinations.sql_jobs import SqlMergeFollowupJob

from dlt.destinations.exceptions import (
Expand All @@ -47,7 +48,7 @@ class LoadDummyBaseJob(RunnableLoadJob):
def __init__(self, file_name: str, config: DummyClientConfiguration) -> None:
super().__init__(file_name)
self.config = copy(config)
self.start_time: float = pendulum.now().timestamp()
self.start_time = precise_time()

if self.config.fail_terminally_in_init:
raise DestinationTerminalException(self._exception)
Expand All @@ -63,7 +64,7 @@ def run(self) -> None:
raise Exception("Dummy job status raised exception")

# timeout condition (terminal)
n = pendulum.now().timestamp()
n = precise_time()
if n - self.start_time > self.config.timeout:
# this will make the the job go to a failed state
raise DestinationTerminalException("failed due to timeout")
Expand Down
9 changes: 6 additions & 3 deletions tests/common/storages/test_load_package.py
Original file line number Diff line number Diff line change
Expand Up @@ -225,17 +225,20 @@ def test_job_elapsed_time_seconds(load_storage: LoadStorage) -> None:
load_storage.normalized_packages.get_job_file_path(load_id, "started_jobs", fn)
)
elapsed = PackageStorage._job_elapsed_time_seconds(fp)
sleep(0.3)
sleep(0.5)
# do not touch file
elapsed_2 = PackageStorage._job_elapsed_time_seconds(fp)
assert elapsed_2 - elapsed >= 0.3
# Python 3.13 + Windows often shows elapsed values less than 0.5
# it may be related to https://github.com/python/cpython/issues/65501
# most probably
assert elapsed_2 - elapsed >= 0.47
# rename the file
fp = load_storage.normalized_packages.retry_job(load_id, fn)
# retry_job increases retry number in file name so the line below does not work
# fp = storage.storage._make_path(storage._get_job_file_path(load_id, "new_jobs", fn))
elapsed_2 = PackageStorage._job_elapsed_time_seconds(fp)
# it should keep its mod original date after rename
assert elapsed_2 - elapsed >= 0.3
assert elapsed_2 - elapsed >= 0.47


def test_retry_job(load_storage: LoadStorage) -> None:
Expand Down

0 comments on commit e8077bc

Please sign in to comment.