|
64 | 64 |
|
65 | 65 |
|
66 | 66 | class BasicCrawlerOptions(TypedDict, Generic[TCrawlingContext]): |
67 | | - """Copy of the parameter types of `BasicCrawler.__init__` meant for typing forwarded __init__ args in subclasses.""" |
| 67 | + """Arguments for the `BasicCrawler` constructor. |
| 68 | +
|
| 69 | + It is intended for typing forwarded `__init__` arguments in the subclasses. |
| 70 | + """ |
68 | 71 |
|
69 | 72 | request_provider: NotRequired[RequestProvider] |
| 73 | + """Provider for requests to be processed by the crawler.""" |
| 74 | + |
70 | 75 | request_handler: NotRequired[Callable[[TCrawlingContext], Awaitable[None]]] |
| 76 | + """A callable responsible for handling requests.""" |
| 77 | + |
71 | 78 | http_client: NotRequired[BaseHttpClient] |
| 79 | + """HTTP client used by `BasicCrawlingContext.send_request` and the HTTP-based crawling.""" |
| 80 | + |
72 | 81 | concurrency_settings: NotRequired[ConcurrencySettings] |
| 82 | + """Settings to fine-tune concurrency levels.""" |
| 83 | + |
73 | 84 | max_request_retries: NotRequired[int] |
| 85 | + """Maximum number of attempts to process a single request.""" |
| 86 | + |
74 | 87 | max_requests_per_crawl: NotRequired[int | None] |
| 88 | + """Maximum number of pages to open during a crawl. The crawl stops upon reaching this limit. |
| 89 | + Setting this value can help avoid infinite loops in misconfigured crawlers. `None` means no limit. |
| 90 | + Due to concurrency settings, the actual number of pages visited may slightly exceed this value.""" |
| 91 | + |
75 | 92 | max_session_rotations: NotRequired[int] |
| 93 | + """Maximum number of session rotations per request. The crawler rotates the session if a proxy error occurs |
| 94 | + or if the website blocks the request.""" |
| 95 | + |
76 | 96 | configuration: NotRequired[Configuration] |
| 97 | + """Crawler configuration.""" |
| 98 | + |
77 | 99 | request_handler_timeout: NotRequired[timedelta] |
78 | | - session_pool: NotRequired[SessionPool] |
| 100 | + """Maximum duration allowed for a single request handler to run.""" |
| 101 | + |
79 | 102 | use_session_pool: NotRequired[bool] |
| 103 | + """Enable the use of a session pool for managing sessions during crawling.""" |
| 104 | + |
| 105 | + session_pool: NotRequired[SessionPool] |
| 106 | + """A custom `SessionPool` instance, allowing the use of non-default configuration.""" |
| 107 | + |
80 | 108 | retry_on_blocked: NotRequired[bool] |
| 109 | + """If True, the crawler attempts to bypass bot protections automatically.""" |
| 110 | + |
81 | 111 | proxy_configuration: NotRequired[ProxyConfiguration] |
| 112 | + """HTTP proxy configuration used when making requests.""" |
| 113 | + |
82 | 114 | statistics: NotRequired[Statistics[StatisticsState]] |
| 115 | + """A custom `Statistics` instance, allowing the use of non-default configuration.""" |
| 116 | + |
83 | 117 | event_manager: NotRequired[EventManager] |
| 118 | + """A custom `EventManager` instance, allowing the use of non-default configuration.""" |
| 119 | + |
84 | 120 | configure_logging: NotRequired[bool] |
| 121 | + """If True, the crawler will set up logging infrastructure automatically.""" |
| 122 | + |
85 | 123 | _context_pipeline: NotRequired[ContextPipeline[TCrawlingContext]] |
| 124 | + """Enables extending the request lifecycle and modifying the crawling context. Intended for use by |
| 125 | + subclasses rather than direct instantiation of `BasicCrawler`.""" |
| 126 | + |
86 | 127 | _additional_context_managers: NotRequired[Sequence[AsyncContextManager]] |
| 128 | + """Additional context managers used throughout the crawler lifecycle.""" |
| 129 | + |
87 | 130 | _logger: NotRequired[logging.Logger] |
| 131 | + """A logger instance, typically provided by a subclass, for consistent logging labels.""" |
88 | 132 |
|
89 | 133 |
|
90 | 134 | class BasicCrawler(Generic[TCrawlingContext]): |
@@ -137,31 +181,30 @@ def __init__( |
137 | 181 | """A default constructor. |
138 | 182 |
|
139 | 183 | Args: |
140 | | - request_provider: Provides requests to be processed. |
141 | | - request_handler: A callable to which request handling is delegated. |
142 | | - http_client: HTTP client to be used for `BasicCrawlingContext.send_request` and HTTP-only crawling. |
143 | | - concurrency_settings: Allows fine-tuning concurrency levels. |
144 | | - max_request_retries: Maximum amount of attempts at processing a request. |
145 | | - max_requests_per_crawl: Maximum number of pages that the crawler will open. The crawl will stop when |
146 | | - the limit is reached. It is recommended to set this value in order to prevent infinite loops in |
147 | | - misconfigured crawlers. None means no limit. Due to concurrency_settings, the actual number of pages |
148 | | - visited may slightly exceed this value. |
149 | | - max_session_rotations: Maximum number of session rotations per request. |
150 | | - The crawler will automatically rotate the session in case of a proxy error or if it gets blocked by |
151 | | - the website. |
| 184 | + request_provider: Provider for requests to be processed by the crawler. |
| 185 | + request_handler: A callable responsible for handling requests. |
| 186 | + http_client: HTTP client used by `BasicCrawlingContext.send_request` and the HTTP-based crawling. |
| 187 | + concurrency_settings: Settings to fine-tune concurrency levels. |
| 188 | + max_request_retries: Maximum number of attempts to process a single request. |
| 189 | + max_requests_per_crawl: Maximum number of pages to open during a crawl. The crawl stops upon reaching |
| 190 | + this limit. Setting this value can help avoid infinite loops in misconfigured crawlers. `None` means |
| 191 | + no limit. Due to concurrency settings, the actual number of pages visited may slightly exceed |
| 192 | + this value. |
| 193 | + max_session_rotations: Maximum number of session rotations per request. The crawler rotates the session |
| 194 | + if a proxy error occurs or if the website blocks the request. |
152 | 195 | configuration: Crawler configuration. |
153 | | - request_handler_timeout: How long a single request handler is allowed to run. |
154 | | - use_session_pool: Enables using the session pool for crawling. |
155 | | - session_pool: A preconfigured `SessionPool` instance if you wish to use non-default configuration. |
156 | | - retry_on_blocked: If set to True, the crawler will try to automatically bypass any detected bot protection. |
157 | | - proxy_configuration: A HTTP proxy configuration to be used for making requests. |
158 | | - statistics: A preconfigured `Statistics` instance if you wish to use non-default configuration. |
159 | | - event_manager: A custom `EventManager` instance if you wish to use a non-default one. |
160 | | - configure_logging: If set to True, the crawler will configure the logging infrastructure. |
161 | | - _context_pipeline: Allows extending the request lifecycle and modifying the crawling context. |
162 | | - This parameter is meant to be used by child classes, not when BasicCrawler is instantiated directly. |
163 | | - _additional_context_managers: Additional context managers to be used in the crawler lifecycle. |
164 | | - _logger: A logger instance passed from a child class to ensure consistent labels. |
| 196 | + request_handler_timeout: Maximum duration allowed for a single request handler to run. |
| 197 | + use_session_pool: Enable the use of a session pool for managing sessions during crawling. |
| 198 | + session_pool: A custom `SessionPool` instance, allowing the use of non-default configuration. |
| 199 | + retry_on_blocked: If True, the crawler attempts to bypass bot protections automatically. |
| 200 | + proxy_configuration: HTTP proxy configuration used when making requests. |
| 201 | + statistics: A custom `Statistics` instance, allowing the use of non-default configuration. |
| 202 | + event_manager: A custom `EventManager` instance, allowing the use of non-default configuration. |
| 203 | + configure_logging: If True, the crawler will set up logging infrastructure automatically. |
| 204 | + _context_pipeline: Enables extending the request lifecycle and modifying the crawling context. |
| 205 | + Intended for use by subclasses rather than direct instantiation of `BasicCrawler`. |
| 206 | + _additional_context_managers: Additional context managers used throughout the crawler lifecycle. |
| 207 | + _logger: A logger instance, typically provided by a subclass, for consistent logging labels. |
165 | 208 | """ |
166 | 209 | self._router: Router[TCrawlingContext] | None = None |
167 | 210 |
|
|
0 commit comments