8000 feat(api_core): allow setting Retry deadline as strict · googleapis/google-cloud-python@8cb31da · GitHub
[go: up one dir, main page]
Skip to content

Commit 8cb31da

Browse files
plamutplamut
committed
feat(api_core): allow setting Retry deadline as strict
If a deadline is set as strict, Retry will shorten the last sleep period to end at the given deadline, and not possibly stretch beyond it.
1 parent 5894632 commit 8cb31da

File tree

2 files changed

+166
-23
lines changed

2 files changed

+166
-23
lines changed

‎api_core/google/api_core/retry.py

Lines changed: 40 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,9 @@ def exponential_sleep_generator(initial, maximum, multiplier=_DEFAULT_DELAY_MULT
141141
delay = delay * multiplier
142142

143143

144-
def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
144+< 8000 /span>
def retry_target(
145+
target, predicate, sleep_generator, deadline, on_error=None, strict_deadline=False
146+
):
145147
"""Call a function and retry if it fails.
146148
147149
This is the lowest-level retry helper. Generally, you'll use the
@@ -156,9 +158,12 @@ def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
156158
sleep_generator (Iterable[float]): An infinite iterator that determines
157159
how long to sleep between retries.
158160
deadline (float): How long to keep retrying the target.
159-
on_error (Callable): A function to call while processing a retryable
160-
exception. Any error raised by this function will *not* be
161-
caught.
161+
on_error (Callable[Exception]): A function to call while processing a
162+
retryable exception. Any error raised by this function will *not*
163+
be caught.
164+
strict_deadline (bool): If :data:`True`, the last retry will run at
165+
``deadline``, shortening the last sleep interval as necessary.
166+
Defaults to :data:`False`.
162167
163168
Returns:
164169
Any: the return value of the target function.
@@ -191,16 +196,21 @@ def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
191196
on_error(exc)
192197

193198
now = datetime_helpers.utcnow()
194-
if deadline_datetime is not None and deadline_datetime < now:
195-
six.raise_from(
196-
exceptions.RetryError(
197-
"Deadline of {:.1f}s exceeded while calling {}".format(
198-
deadline, target
199+
200+
if deadline_datetime is not None:
201+
if deadline_datetime <= now:
202+
six.raise_from(
203+
exceptions.RetryError(
204+
"Deadline of {:.1f}s exceeded while calling {}".format(
205+
deadline, target
206+
),
207+
last_exc,
199208
),
200209
last_exc,
201-
),
202-
last_exc,
203-
)
210+
)
211+
elif strict_deadline:
212+
time_to_deadline = (deadline_datetime - now).total_seconds()
213+
sleep = min(time_to_deadline, sleep)
204214

205215
_LOGGER.debug(
206216
"Retrying due to {}, sleeping {:.1f}s ...".format(last_exc, sleep)
@@ -228,6 +238,9 @@ class Retry(object):
228238
maximum (float): The maximum amout of time to delay in seconds.
229239
multiplier (float): The multiplier applied to the delay.
230240
deadline (float): How long to keep retrying in seconds.
241+
strict_deadline (bool): If :data:`True`, the last retry will run at
242+
``deadline``, shortening the last sleep interval as necessary.
243+
Defaults to :data:`False`.
231244
"""
232245

233246
def __init__(
@@ -238,21 +251,23 @@ def __init__(
238251
multiplier=_DEFAULT_DELAY_MULTIPLIER,
239252
deadline=_DEFAULT_DEADLINE,
240253
on_error=None,
254+
strict_deadline=False,
241255
):
242256
self._predicate = predicate
243257
self._initial = initial
244258
self._multiplier = multiplier
245259
self._maximum = maximum
246260
self._deadline = deadline
247261
self._on_error = on_error
262+
self._strict_deadline = strict_deadline
248263

249264
def __call__(self, func, on_error=None):
250265
"""Wrap a callable with retry behavior.
251266
252267
Args:
253268
func (Callable): The callable to add retry behavior to.
254-
on_error (Callable): A function to call while processing a
255-
retryable exception. Any error raised by this function will
269+
on_error (Callable[Exception]): A function to call while processing
270+
a retryable exception. Any error raised by this function will
256271
*not* be caught.
257272
258273
Returns:
@@ -275,15 +290,19 @@ def retry_wrapped_func(*args, **kwargs):
275290
sleep_generator,
276291
self._deadline,
277292
on_error=on_error,
293+
strict_deadline=self._strict_deadline
278294
)
279295

280296
return retry_wrapped_func
281297

282-
def with_deadline(self, deadline):
298+
def with_deadline(self, deadline, strict_deadline=False):
283299
"""Return a copy of this retry with the given deadline.
284300
285301
Args:
286302
deadline (float): How long to keep retrying.
303+
strict_deadline (bool): If :data:`True`, the last retry will run at
304+
``deadline``, shortening the last sleep interval as necessary.
305+
Defaults to :data:`False`.
287306
288307
Returns:
289308
Retry: A new retry instance with the given deadline.
@@ -295,6 +314,7 @@ def with_deadline(self, deadline):
295314
multiplier=self._multiplier,
296315
deadline=deadline,
297316
on_error=self._on_error,
317+
strict_deadline=strict_deadline,
298318
)
299319

300320
def with_predicate(self, predicate):
@@ -314,6 +334,7 @@ def with_predicate(self, predicate):
314334
multiplier=self._multiplier,
315335
deadline=self._deadline,
316336
on_error=self._on_error,
337+
strict_deadline=self._strict_deadline,
317338
)
318339

319340
def with_delay(self, initial=None, maximum=None, multiplier=None):
@@ -335,17 +356,20 @@ def with_delay(self, initial=None, maximum=None, multiplier=None):
335356
multiplier=multiplier if maximum is not None else self._multiplier,
336357
deadline=self._deadline,
337358
on_error=self._on_error,
359+
strict_deadline=self._strict_deadline,
338360
)
339361

340362
def __str__(self):
341363
return (
342364
"<Retry predicate={}, initial={:.1f}, maximum={:.1f}, "
343-
"multiplier={:.1f}, deadline={:.1f}, on_error={}>".format(
365+
"multiplier={:.1f}, deadline={:.1f}, on_error={}, "
366+
"strict_deadline={}>".format(
344367
self._predicate,
345368
self._initial,
346369
self._maximum,
347370
self._multiplier,
348371
self._deadline,
349372
self._on_error,
373+
self._strict_deadline,
350374
)
351375
)

‎api_core/tests/unit/test_retry.py

Lines changed: 126 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ def test_constructor_defaults(self):
162162
assert retry_._multiplier == 2
163163
assert retry_._deadline == 120
164164
assert retry_._on_error is None
165+
assert retry_._strict_deadline is False
165166

166167
def test_constructor_options(self):
167168
_some_function = mock.Mock()
@@ -173,49 +174,118 @@ def test_constructor_options(self):
173174
multiplier=3,
174175
deadline=4,
175176
on_error=_some_function,
177+
strict_deadline=True,
176178
)
177179
assert retry_._predicate == mock.sentinel.predicate
178180
assert retry_._initial == 1
179181
assert retry_._maximum == 2
180182
assert retry_._multiplier == 3
181183
assert retry_._deadline == 4
182184
assert retry_._on_error is _some_function
185+
assert retry_._strict_deadline is True
183186

184187
def test_with_deadline(self):
185-
retry_ = retry.Retry()
186-
new_retry = retry_.with_deadline(42)
188+
retry_ = retry.Retry(
189+
predicate=mock.sentinel.predicate,
190+
initial=1,
191+
maximum=2,
192+
multiplier=3,
193+
deadline=4,
194+
on_error=mock.sentinel.on_error,
195+
strict_deadline=True,
196+
)
197+
new_retry = retry_.with_deadline(42, strict_deadline=True)
187198
assert retry_ is not new_retry
188199
assert new_retry._deadline == 42
200+
assert new_retry._strict_deadline is True
201+
202+
# the rest of the attributes should remain the same
203+
assert new_retry._predicate is retry_._predicate
204+
assert new_retry._initial == retry_._initial
205+
assert new_retry._maximum == retry_._maximum
206+
assert new_retry._multiplier == retry_._multiplier
207+
assert new_retry._on_error is retry_._on_error
189208

190209
def test_with_predicate(self):
191-
retry_ = retry.Retry()
210+
retry_ = retry.Retry(
211+
predicate=mock.sentinel.predicate,
212+
initial=1,
213+
maximum=2,
214+
multiplier=3,
215+
deadline=4,
216+
on_error=mock.sentinel.on_error,
217+
strict_deadline=True,
218+
)
192219
new_retry = retry_.with_predicate(mock.sentinel.predicate)
193220
assert retry_ is not new_retry
194221
assert new_retry._predicate == mock.sentinel.predicate
195222

223+
# the rest of the attributes should remain the same
224+
assert new_retry._deadline == retry_._deadline
225+
assert new_retry._strict_deadline == retry_._strict_deadline
226+
assert new_retry._initial == retry_._initial
227+
assert new_retry._maximum == retry_._maximum
228+
assert new_retry._multiplier == retry_._multiplier
229+
assert new_retry._on_error is retry_._on_error
230+
196231
def test_with_delay_noop(self):
197-
retry_ = retry.Retry()
232+
retry_ = retry.Retry(
233+
predicate=mock.sentinel.predicate,
234+
initial=1,
235+
maximum=2,
236+
multiplier=3,
237+
deadline=4,
238+
on_error=mock.sentinel.on_error,
239+
strict_deadline=True,
240+
)
198241
new_retry = retry_.with_delay()
199242
assert retry_ is not new_retry
200243
assert new_retry._initial == retry_._initial
201244
assert new_retry._maximum == retry_._maximum
202245
assert new_retry._multiplier == retry_._multiplier
203246

204247
def test_with_delay(self):
205-
retry_ = retry.Retry()
248+
retry_ = retry.Retry(
249+
predicate=mock.sentinel.predicate,
250+
initial=1,
251+
maximum=2,
252+
multiplier=3,
253+
deadline=4,
254+
on_error=mock.sentinel.on_error,
255+
strict_deadline=True,
256+
)
206257
new_retry = retry_.with_delay(initial=1, maximum=2, multiplier=3)
207258
assert retry_ is not new_retry
208259
assert new_retry._initial == 1
209260
assert new_retry._maximum == 2
210261
assert new_retry._multiplier == 3
211262

263+
# the rest of the attributes should remain the same
264+
assert new_retry._deadline == retry_._deadline
265+
assert new_retry._strict_deadline == retry_._strict_deadline
266+
assert new_retry._predicate is retry_._predicate
267+
assert new_retry._on_error is retry_._on_error
268+
212269
def test___str__(self):
213-
retry_ = retry.Retry()
270+
def if_exception_type(exc):
271+
return bool(exc) # pragma: NO COVER
272+
273+
# Explicitly set all attributes as changed Retry defaults should not
274+
# cause this test to start failing.
275+
retry_ = retry.Retry(
276+
predicate=if_exception_type,
277+
initial=1.0,
278+
maximum=60.0,
279+
multiplier=2.0,
280+
deadline=120.0,
281+
on_error=None,
282+
strict_deadline=False,
283+
)
214284
assert re.match(
215285
(
216286
r"<Retry predicate=<function.*?if_exception_type.*?>, "
217287
r"initial=1.0, maximum=60.0, multiplier=2.0, deadline=120.0, "
218-
r"on_error=None>"
288+
r"on_error=None, strict_deadline=False>"
219289
),
220290
str(retry_),
221291
)
@@ -259,6 +329,55 @@ def test___call___and_execute_retry(self, sleep, uniform):
259329
sleep.assert_called_once_with(retry_._initial)
260330
assert on_error.call_count == 1
261331

332+
# Make uniform return half of its maximum, which is the calculated sleep time.
333+
@mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n / 2.0)
334+
@mock.patch("time.sleep", autospec=True)
335+
def test___call___and_execute_retry_strict_deadline(self, sleep, uniform):
336+
337+
on_error = mock.Mock(spec=["__call__"], side_effect=[None] * 10)
338+
retry_ = retry.Retry(
339+
predicate=retry.if_exception_type(ValueError),
340+
initial=1.0,
341+
maximum=1024.0,
342+
multiplier=2.0,
343+
deadline=9.9,
344+
strict_deadline=True,
345+
)
346+
347+
utcnow = datetime.datetime.utcnow()
348+
utcnow_patcher = mock.patch(
349+
"google.api_core.datetime_helpers.utcnow", return_value=utcnow
350+
)
351+
352+
target = mock.Mock(spec=["__call__"], side_effect=[ValueError()] * 10)
353+
# __name__ is needed by functools.partial.
354+
target.__name__ = "target"
355+
356+
decorated = retry_(target, on_error=on_error)
357+
target.assert_not_called()
358+
359+
with utcnow_patcher as patched_utcnow:
360+
# Make sure that calls to fake time.sleep() also advance the mocked
361+
# time clock.
362+
def increase_time(sleep_delay):
363+
patched_utcnow.return_value += datetime.timedelta(seconds=sleep_delay)
364+
sleep.side_effect = increase_time
365+
366+
with pytest.raises(exceptions.RetryError):
367+
decorated("meep")
368+
369+
assert target.call_count == 5
370+
target.assert_has_calls([mock.call("meep")] * 5)
371+
assert on_error.call_count == 5
372+
373+
# check the delays
374+
assert sleep.call_count == 4 # once between each successive target calls
375+
last_wait = sleep.call_args.args[0]
376+
total_wait = sum(call_args.args[0] for call_args in sleep.call_args_list)
377+
378+
assert last_wait == 2.9 # and not 8.0, because the last delay was shortened
379+
assert total_wait == 9.9 # the same as the (strict) deadline
380+
262381
@mock.patch("time.sleep", autospec=True)
263382
def test___init___without_retry_executed(self, sleep):
264383
_some_function = mock.Mock()

0 commit comments

Comments
 (0)
0