E2E_TESTS_CONCURRENCY = 1

clean:

rm -rf .mypy_cache .pytest_cache .ruff_cache build dist htmlcov .coverage

cd website && corepack enable && yarn && uv run yarn build

run-docs: build-api-reference

from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext

async def main() -> None:

crawler = PlaywrightCrawler(

max_requests_per_crawl=10, # Limit the max requests per crawl.

)

# Define the default request handler, which will be called for every request.

@crawler.router.default_handler

async def request_handler(context: PlaywrightCrawlingContext) -> None:

context.log.info(f'Processing {context.request.url} ...')

# Extract data from the page using Playwright API.

data = {

'url': context.request.url,

'title': await context.page.title(),

}

# Push the extracted data to the default dataset.

await context.push_data(data)

# Run the crawler with the initial list of URLs.

await crawler.run(['https://crawlee.dev'])

data = await crawler.get_data()

crawler.log.info(f'Extracted data: {data.items}')