From 3eba11918cdeacd5de82422397eb9e6d73daf4ed Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 21 Nov 2023 09:50:19 -0600 Subject: [PATCH] fix: ensure query job retry has longer deadline than API request deadline In cases where we can't disambiguate API failure from job failure, this ensures we can still retry the job at least once. --- google/cloud/bigquery/retry.py | 9 +++++++-- tests/unit/test_retry.py | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/google/cloud/bigquery/retry.py b/google/cloud/bigquery/retry.py index d0830ed13..b01c0662c 100644 --- a/google/cloud/bigquery/retry.py +++ b/google/cloud/bigquery/retry.py @@ -34,7 +34,12 @@ auth_exceptions.TransportError, ) -_DEFAULT_JOB_DEADLINE = 60.0 * 10.0 # seconds +_DEFAULT_RETRY_DEADLINE = 10.0 * 60.0 # 10 minutes + +# Allow for a few retries after the API request times out. This relevant for +# rateLimitExceeded errors, which can be raised either by the Google load +# balancer or the BigQuery job server. +_DEFAULT_JOB_DEADLINE = 3.0 * _DEFAULT_RETRY_DEADLINE def _should_retry(exc): @@ -51,7 +56,7 @@ def _should_retry(exc): return reason in _RETRYABLE_REASONS -DEFAULT_RETRY = retry.Retry(predicate=_should_retry, deadline=600.0) +DEFAULT_RETRY = retry.Retry(predicate=_should_retry, deadline=_DEFAULT_RETRY_DEADLINE) """The default retry object. Any method with a ``retry`` parameter will be retried automatically, diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index 60d04de89..1109b7ff2 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -125,6 +125,7 @@ def test_DEFAULT_JOB_RETRY_predicate(): def test_DEFAULT_JOB_RETRY_deadline(): - from google.cloud.bigquery.retry import DEFAULT_JOB_RETRY + from google.cloud.bigquery.retry import DEFAULT_JOB_RETRY, DEFAULT_RETRY - assert DEFAULT_JOB_RETRY._deadline == 600 + # Make sure we can retry the job at least once. + assert DEFAULT_JOB_RETRY._deadline > DEFAULT_RETRY._deadline