fix: add explicit err msgs for missing pckg extras during import (#165) · apify/crawlee-python@200ebfa · GitHub
Skip to content

Commit 200ebfa

Browse files
vdusekvdusek
authored
fix: add explicit err msgs for missing pckg extras during import (#165)
## Description - Add explicit error messages for missing package extras during import. - It applies to BrowserPool, BeautifulsoupCrawler, and PlaywrightCrawler. - Also until now the Playwright was mandatory for even `BasicCrawler`, fixing it. ## Related issues - Closes #155 ## Testing ### PlaywrightCrawler ``` ImportError Traceback (most recent call last) Cell In[3], line 1 ----> 1 from crawlee.playwright_crawler import PlaywrightCrawler File ~/Projects/crawlee-py/src/crawlee/playwright_crawler/__init__.py:5 3 from .types import PlaywrightCrawlingContext 4 except ImportError as exc: ----> 5 raise ImportError( 6 'To use this module, you need to install the "playwright" extra. Run "pip install crawlee[playwright]".', 7 ) from exc ImportError: To use this module, you need to install the "playwright" extra. Run "pip install crawlee[playwright]". ``` ### BeautifulsoupCrawler ``` ImportError Traceback (most recent call last) Cell In[1], line 1 ----> 1 from crawlee.beautifulsoup_crawler import BeautifulSoupCrawler File ~/Projects/crawlee-py/src/crawlee/beautifulsoup_crawler/__init__.py:5 3 from .types import BeautifulSoupCrawlingContext 4 except ImportError as exc: ----> 5 raise ImportError( 6 'To use this module, you need to install the "beautifulsoup" extra. Run "pip install crawlee[beautifulsoup]".', 7 ) from exc ImportError: To use this module, you need to install the "beautifulsoup" extra. Run "pip install crawlee[beautifulsoup]". ``` ### BrowserPool ``` ImportError Traceback (most recent call last) Cell In[2], line 1 ----> 1 from crawlee.browsers import BrowserPool File ~/Projects/crawlee-py/src/crawlee/browsers/__init__.py:5 3 from .playwright_browser_plugin import PlaywrightBrowserPlugin 4 except ImportError as exc: ----> 5 raise ImportError( 6 'To use this module, you need to install the "playwright" extra. Run "pip install crawlee[playwright]".', 7 ) from exc ImportError: To use this module, you need to install the "playwright" extra. Run "pip install crawlee[playwright]". ``` ## Checklist - [x] Changes are described in the `CHANGELOG.md` - [x] CI passed
1 parent 382b6f4 commit 200ebfa

9 files changed

Lines changed: 83 additions & 55 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 3 additions & 1 deletion

‎pyproject.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,3 +208,6 @@ reportUnusedVariable = false
208208
reportCallInDefaultInitializer = false
209209
reportImplicitStringConcatenation = false
210210
reportAny = false
211+
212+
[tool.ipdb]
213+
context = 7

‎src/crawlee/basic_crawler/basic_crawler.py‎

Lines changed: 9 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from datetime import timedelta
88
from functools import partial
99
from logging import getLogger
10-
from typing import TYPE_CHECKING, Any, Callable, Generic, Union, cast
10+
from typing import TYPE_CHECKING, Any, AsyncContextManager, Callable, Generic, Union, cast
1111

1212
import httpx
1313
from tldextract import TLDExtract
@@ -18,9 +18,7 @@
1818
from crawlee.autoscaling import AutoscaledPool, ConcurrencySettings
1919
from crawlee.autoscaling.snapshotter import Snapshotter
2020
from crawlee.autoscaling.system_status import SystemStatus
21-
from crawlee.basic_crawler.context_pipeline import (
22-
ContextPipeline,
23-
)
21+
from crawlee.basic_crawler.context_pipeline import ContextPipeline
2422
from crawlee.basic_crawler.errors import (
2523
ContextPipelineInitializationError,
2624
ContextPipelineInterruptedError,
@@ -29,12 +27,7 @@
2927
UserDefinedErrorHandlerError,
3028
)
3129
from crawlee.basic_crawler.router import Router
32-
from crawlee.basic_crawler.types import (
33-
BasicCrawlingContext,
34-
RequestHandlerRunResult,
35-
SendRequestFunction,
36-
)
37-
from crawlee.browsers import BrowserPool
30+
from crawlee.basic_crawler.types import BasicCrawlingContext, RequestHandlerRunResult, SendRequestFunction
3831
from crawlee.configuration import Configuration
3932
from crawlee.enqueue_strategy import EnqueueStrategy
4033
from crawlee.events.local_event_manager import LocalEventManager
@@ -76,9 +69,8 @@ class BasicCrawlerOptions(TypedDict, Generic[TCrawlingContext]):
7669
retry_on_blocked: NotRequired[bool]
7770
proxy_configuration: NotRequired[ProxyConfiguration]
7871
statistics: NotRequired[Statistics[StatisticsState]]
79-
browser_pool: NotRequired[BrowserPool]
80-
use_browser_pool: NotRequired[bool]
8172
_context_pipeline: NotRequired[ContextPipeline[TCrawlingContext]]
73+
_additional_context_managers: NotRequired[Sequence[AsyncContextManager]]
8274

8375

8476
class BasicCrawler(Generic[TCrawlingContext]):
@@ -108,9 +100,8 @@ def __init__(
108100
retry_on_blocked: bool = True,
109101
proxy_configuration: ProxyConfiguration | None = None,
110102
statistics: Statistics | None = None,
111-
browser_pool: BrowserPool | None = None,
112-
use_browser_pool: bool = False,
113103
_context_pipeline: ContextPipeline[TCrawlingContext] | None = None,
104+
_additional_context_managers: Sequence[AsyncContextManager] | None = None,
114105
) -> None:
115106
"""Initialize the BasicCrawler.
116107
@@ -131,9 +122,9 @@ def __init__(
131122
proxy_configuration: A HTTP proxy configuration to be used for making requests
132123
statistics: A preconfigured `Statistics` instance if you wish to use non-default configuration
133124
browser_pool: A preconfigured `BrowserPool` instance for browser crawling.
134-
use_browser_pool: Enables using the browser pool for crawling.
135125
_context_pipeline: Allows extending the request lifecycle and modifying the crawling context.
136126
This parameter is meant to be used by child classes, not when BasicCrawler is instantiated directly.
127+
_additional_context_managers: Additional context managers to be used in the crawler lifecycle.
137128
"""
138129
self._router: Router[TCrawlingContext] | None = None
139130

@@ -186,10 +177,7 @@ def __init__(
186177
event_manager=self._event_manager,
187178
log_message=f'{logger.name} request statistics',
188179
)
189-
190-
self._use_browser_pool = use_browser_pool
191-
if self._use_browser_pool:
192-
self._browser_pool = browser_pool or BrowserPool()
180+
self._additional_context_managers = _additional_context_managers or []
193181

194182
self._running = False
195183
self._has_finished_before = False
@@ -304,8 +292,8 @@ async def run(self, requests: list[str | BaseRequestData] | None = None) -> Fina
304292
if self._use_session_pool:
305293
await exit_stack.enter_async_context(self._session_pool)
306294

307-
if self._use_browser_pool:
308-
await exit_stack.enter_async_context(self._browser_pool)
295+
for context_manager in self._additional_context_managers:
296+
await exit_stack.enter_async_context(context_manager)
309297

310298
await self._pool.run()
311299

‎src/crawlee/beautifulsoup_crawler/__init__.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,8 @@
1-
from .beautifulsoup_crawler import BeautifulSoupCrawler
2-
from .types import BeautifulSoupCrawlingContext
1+
try:
2+
from .beautifulsoup_crawler import BeautifulSoupCrawler
3+
from .types import BeautifulSoupCrawlingContext
4+
except ImportError as exc:
5+
raise ImportError(
6+
'To import anything from this subpacakge, you need to install the "beautifulsoup" extra. '
7+
'For example, if you use pip, run "pip install crawlee[beautifulsoup]".',
8+
) from exc

‎src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import asyncio
44
from typing import TYPE_CHECKING, Any, AsyncGenerator, Iterable, Literal
55

6+
from bs4 import BeautifulSoup, Tag
67
from typing_extensions import Unpack
78

89
from crawlee._utils.blocked import RETRY_CSS_SELECTORS
@@ -102,8 +103,6 @@ async def _handle_blocked_request(
102103
async def _parse_http_response(
103104
self, context: HttpCrawlingContext
104105
) -> AsyncGenerator[BeautifulSoupCrawlingContext, None]:
105-
from bs4 import BeautifulSoup, Tag
106-
107106
soup = await asyncio.to_thread(lambda: BeautifulSoup(context.http_response.read(), self._parser))
108107

109108
async def enqueue_links(

‎src/crawlee/browsers/__init__.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,8 @@
1-
from .browser_pool import BrowserPool
2-
from .playwright_browser_plugin import PlaywrightBrowserPlugin
1+
try:
2+
from .browser_pool import BrowserPool
3+
from .playwright_browser_plugin import PlaywrightBrowserPlugin
4+
except ImportError as exc:
5+
raise ImportError(
6+
'To import anything from this subpacakge, you need to install the "playwright" extra. '
7+
'For example, if you use pip, run "pip install crawlee[playwright]".',
8+
) from exc

‎src/crawlee/browsers/browser_pool.py‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,10 @@
44

55
import asyncio
66
import itertools
7+
from collections import defaultdict
78
from datetime import timedelta
89
from logging import getLogger
9-
from typing import TYPE_CHECKING
10+
from typing import TYPE_CHECKING, Any, Literal
1011
from weakref import WeakValueDictionary
1112

1213
from crawlee._utils.crypto import crypto_random_object_id
@@ -55,6 +56,32 @@ def __init__(
5556
self._pages = WeakValueDictionary[str, CrawleePage]() # Track the pages in the pool
5657
self._plugins_cycle = itertools.cycle(self._plugins) # Cycle through the plugins
5758

59+
@classmethod
60+
def with_default_plugin(
61+
cls,
62+
*,
63+
headless: bool | None = None,
64+
browser_type: Literal['chromium', 'firefox', 'webkit'] | None = None,
65+
**kwargs: Any,
66+
) -> BrowserPool:
67+
"""Create a new instance with a single `PlaywrightBrowserPlugin` configured with the provided options.
68+
69+
Args:
70+
headless: Whether to run the browser in headless mode.
71+
browser_type: The type of browser to launch ('chromium', 'firefox', or 'webkit').
72+
kwargs: Additional arguments for default constructor.
73+
"""
74+
plugin_options: dict = defaultdict(dict)
75+
76+
if headless is not None:
77+
plugin_options['browser_options']['headless'] = headless
78+
79+
if browser_type:
80+
plugin_options['browser_type'] = browser_type
81+
82+
plugin = PlaywrightBrowserPlugin(**plugin_options)
83+
return cls(plugins=[plugin], **kwargs)
84+
5885
@property
5986
def plugins(self) -> Sequence[BaseBrowserPlugin]:
6087
"""Return the browser plugins."""

‎src/crawlee/playwright_crawler/__init__.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,8 @@
1-
from .playwright_crawler import PlaywrightCrawler
2-
from .types import PlaywrightCrawlingContext
1+
try:
2+
from .playwright_crawler import PlaywrightCrawler
3+
from .types import PlaywrightCrawlingContext
4+
except ImportError as exc:
5+
raise ImportError(
6+
'To import anything from this subpacakge, you need to install the "playwright" extra. '
7+
'For example, if you use pip, run "pip install crawlee[playwright]".',
8+
) from exc

‎src/crawlee/playwright_crawler/playwright_crawler.py‎

Lines changed: 15 additions & 24 deletions

0 commit comments

Comments
 (0)