feat: support timeouts in the `send_request` context helper (#2174) · apify/crawlee-python@c2013a0 · GitHub
Skip to content

Commit c2013a0

Browse files
authored
feat: support timeouts in the send_request context helper (#2174)
Closes #2138 ## Summary `SendRequestFunction` (used via `context.send_request()` in request handlers) only accepted `url`, `method`, `payload` and `headers`, so a handler could not bound how long an extra HTTP call may take. All four HTTP client implementations (`HttpxHttpClient`, `CurlImpersonateHttpClient`, `ImpitHttpClient`, `PlaywrightHttpClient`) already support a per-request `timeout: timedelta | None` — this PR just exposes it through the public contract. ## Changes - `src/crawlee/_types.py`: add `timeout: timedelta | None = None` to the `SendRequestFunction` protocol + docstring. - `src/crawlee/crawlers/_basic/_basic_crawler.py`: thread `timeout` through the `_prepare_send_request_function` closure into `HttpClient.send_request`. - `src/crawlee/crawlers/_playwright/_playwright_http_client.py`: **fix** — `PlaywrightHttpClient` passed `timeout.total_seconds()` to Playwright's `APIRequestContext.fetch`, which expects **milliseconds**. Without this fix, any timeout passed via `context.send_request()` on a Playwright crawler would be ~1000x too short (e.g. `timedelta(seconds=12)` → 12ms). - `tests/unit/crawlers/_basic/test_basic_crawler.py`: regression test — `context.send_request('/slow?delay=2', timeout=timedelta(milliseconds=100))` raises `asyncio.TimeoutError`; plus a unit test asserting `timeout` is forwarded to `HttpClient.send_request`. - `tests/unit/crawlers/_playwright/test_playwright_http_client.py`: regression test asserting the seconds→milliseconds conversion. ## Validation - `pytest tests/unit/crawlers/_basic/test_basic_crawler.py` → 112 passed, 1 skipped - New tests (forwarding, real timeout, Playwright ms conversion) all pass - `ruff check` + `ruff format --check` clean on all touched files Note: no new dependency; `timeout` uses the existing `timedelta` convention across the Python HTTP clients (the JS implementation names it `timeoutMillis`, but the Python clients are timedelta-based).
1 parent 3332897 commit c2013a0

5 files changed

Lines changed: 87 additions & 2 deletions

File tree

‎src/crawlee/_types.py‎

Lines changed: 3 additions & 0 deletions

‎src/crawlee/crawlers/_basic/_basic_crawler.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1321,6 +1321,7 @@ async def send_request(
13211321
method: HttpMethod = 'GET',
13221322
payload: HttpPayload | None = None,
13231323
headers: HttpHeaders | dict[str, str] | None = None,
1324+
timeout: timedelta | None = None,
13241325
) -> HttpResponse:
13251326
return await self._http_client.send_request(
13261327
url=url,
@@ -1329,6 +1330,7 @@ async def send_request(
13291330
headers=headers,
13301331
session=session,
13311332
proxy_info=proxy_info,
1333+
timeout=timeout,
13321334
)
13331335

13341336
return send_request

‎src/crawlee/crawlers/_playwright/_playwright_http_client.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ async def send_request(
9999
method=method.lower(),
100100
headers=dict(headers) if headers else None,
101101
data=payload,
102-
timeout=timeout.total_seconds() if timeout is not None else None,
102+
timeout=timeout.total_seconds() * 1000 if timeout is not None else None,
103103
)
104104

105105
return await PlaywrightHttpResponse.from_playwright_response(response, protocol='')

‎tests/unit/crawlers/_basic/test_basic_crawler.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
from crawlee.crawlers import BasicCrawler
2929
from crawlee.errors import RequestCollisionError, SessionError, UserDefinedErrorHandlerError
3030
from crawlee.events import Event, EventCrawlerStatusData, LocalEventManager
31+
from crawlee.http_clients import HttpClient
3132
from crawlee.request_loaders import RequestList, RequestManagerTandem, ThrottlingRequestManager
3233
from crawlee.sessions import Session, SessionPool
3334
from crawlee.statistics import FinalStatistics, StatisticsState
@@ -523,6 +524,58 @@ async def handler(context: BasicCrawlingContext) -> None:
523524
assert content_type == 'application/json'
524525

525526

527+
async def test_send_request_forwards_timeout_to_http_client() -> None:
528+
http_client = AsyncMock(spec=HttpClient)
529+
response = Mock()
530+
http_client.send_request.return_value = response
531+
crawler = BasicCrawler(http_client=http_client, use_session_pool=False, max_requests_per_crawl=1)
532+
timeout = timedelta(seconds=12)
533+
result = None
534+
535+
@crawler.router.default_handler
536+
async def handler(context: BasicCrawlingContext) -> None:
537+
nonlocal result
538+
result = await context.send_request(
539+
'https://example.com',
540+
method='POST',
541+
payload=b'Hello, world!',
542+
headers={'Content-Type': 'text/plain'},
543+
timeout=timeout,
544+
)
545+
546+
await crawler.run(['https://a.placeholder.com'])
547+
548+
assert result is response
549+
http_client.send_request.assert_awaited_once_with(
550+
url='https://example.com',
551+
method='POST',
552+
payload=b'Hello, world!',
553+
headers={'Content-Type': 'text/plain'},
554+
session=None,
555+
proxy_info=None,
556+
timeout=timeout,
557+
)
558+
559+
560+
async def test_send_request_respects_timeout(server_url: URL) -> None:
561+
request_timed_out = asyncio.Event()
562+
563+
crawler = BasicCrawler(max_request_retries=3)
564+
565+
@crawler.router.default_handler
566+
async def handler(context: BasicCrawlingContext) -> None:
567+
with pytest.raises(asyncio.TimeoutError):
568+
await context.send_request(
569+
str(server_url / 'slow') + '?delay=2',
570+
timeout=timedelta(milliseconds=100),
571+
)
572+
request_timed_out.set()
573+
574+
await crawler.run(['https://a.placeholder.com'])
575+
576+
assert request_timed_out.is_set()
577+
578+
526579
@dataclass
527580
class AddRequestsTestInput:
528581
start_url: str

‎tests/unit/crawlers/_playwright/test_playwright_http_client.py‎

Lines changed: 28 additions & 1 deletion

0 commit comments

Comments
 (0)