8000 Inject extra logs for backoffs · apify/apify-client-python@5855814 · GitHub
[go: up one dir, main page]
Skip to content

Commit 5855814

Browse files
PijukatelPijukatel
committed
Inject extra logs for backoffs
1 parent a03af42 commit 5855814

File tree

2 files changed

+18
-10
lines changed

2 files changed

+18
-10
lines changed

‎src/apify_client/_http_client.py

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -190,17 +190,17 @@ def _make_request(stop_retrying: Callable, attempt: int) -> httpx.Response:
190190
self.stats.add_rate_limit_error(attempt)
191191

192192
except Exception as e:
193-
logger.debug('Request threw exception', exc_info=e)
193+
logger.warning('Request threw exception', exc_info=e)
194194
if not is_retryable_error(e):
195-
logger.debug('Exception is not retryable', exc_info=e)
195+
logger.warning('Exception is not retryable', exc_info=e)
196196
stop_retrying()
197197
raise
198198

199199
# We want to retry only requests which are server errors (status >= 500) and could resolve on their own,
200200
# and also retry rate limited requests that throw 429 Too Many Requests errors
201-
logger.debug('Request unsuccessful', extra={'status_code': response.status_code})
201+
logger.warning('Request unsuccessful', extra={'status_code': response.status_code})
202202
if response.status_code < 500 and response.status_code != HTTPStatus.TOO_MANY_REQUESTS: # noqa: PLR2004
203-
logger.debug('Status code is not retryable', extra={'status_code': response.status_code})
203+
logger.warning('Status code is not retryable', extra={'status_code': response.status_code})
204204
stop_retrying()
205205
raise ApifyApiError(response, attempt)
206206

@@ -269,17 +269,17 @@ async def _make_request(stop_retrying: Callable, attempt: int) -> httpx.Response
269269
self.stats.add_rate_limit_error(attempt)
270270

271271
except Exception as e:
272-
logger.debug('Request threw exception', exc_info=e)
272+
logger.warning('Request threw exception', exc_info=e)
273273
if not is_retryable_error(e):
274-
logger.debug('Exception is not retryable', exc_info=e)
274+
logger.warning('Exception is not retryable', exc_info=e)
275275
stop_retrying()
276276
raise
277277

278278
# We want to retry only requests which are server errors (status >= 500) and could resolve on their own,
279279
# and also retry rate limited requests that throw 429 Too Many Requests errors
280-
logger.debug('Request unsuccessful', extra={'status_code': response.status_code})
280+
logger.warning('Request unsuccessful', extra={'status_code': response.status_code})
281281
if response.status_code < 500 and response.status_code != HTTPStatus.TOO_MANY_REQUESTS: # noqa: PLR2004
282-
logger.debug('Status code is not retryable', extra={'status_code': response.status_code})
282+
logger.warning('Status code is not retryable', extra={'status_code': response.status_code})
283283
stop_retrying()
284284
raise ApifyApiError(response, attempt)
285285

‎src/apify_client/_utils.py

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,15 @@
33
import asyncio
44
import base64
55
import json
6+
import logging
67
import random
78
import time
89
from http import HTTPStatus
910
from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
1011

1112
from apify_shared.utils import is_file_or_bytes, maybe_extract_enum_member_value
12-
13+
from apify_client._logging import log_context, logger_name
14+
logger = logging.getLogger(logger_name)
1315
if TYPE_CHECKING:
1416
from collections.abc import Awaitable
1517

@@ -73,6 +75,7 @@ def stop_retrying() -> None:
7375
backoff_exp_factor = backoff_factor ** (attempt - 1)
7476

7577
sleep_time_secs = random_sleep_factor * backoff_base_secs * backoff_exp_factor
78+
logger.warning(f'Backoff sleep {sleep_time_secs}. for attempt {attempt}')
7679
time.sleep(sleep_time_secs)
7780

7881
return func(stop_retrying, max_retries + 1)
@@ -96,7 +99,11 @@ def stop_retrying() -> None:
9699

97100
for attempt in range(1, max_retries + 1):
98101
try:
99-
return await async_func(stop_retrying, attempt)
102+
response = await async_func(stop_retrying, attempt)
103+
if attempt > 1 :
104+
logger.warning(response)
105+
logger.warning(response.text)
106+
return response
100107
except Exception:
101108
if not swallow:
102109
raise
@@ -106,6 +113,7 @@ def stop_retrying() -> None:
106113
backoff_exp_factor = backoff_factor ** (attempt - 1)
107114

108115
sleep_time_secs = random_sleep_factor * backoff_base_secs * backoff_exp_factor
116+
logger.warning(f'Backoff sleep {sleep_time_secs}. for attempt {attempt}. Max retries: {max_retries}')
109117
await asyncio.sleep(sleep_time_secs)
110118

111119
return await async_func(stop_retrying, max_retries + 1)

0 commit comments

Comments
 (0)
0