FilesExpand file tree

master

/

pyproject.toml

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

240 lines (218 loc) · 8 KB

master

/

pyproject.toml

Copy path

Top

File metadata and controls

• Code
• Blame

240 lines (218 loc) · 8 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

[build-system]

requires = ["poetry-core"]

build-backend = "poetry.core.masonry.api"

[tool.poetry]

name = "crawlee"

version = "0.5.5"

description = "Crawlee for Python"

authors = ["Apify Technologies s.r.o. <support@apify.com>"]

license = "Apache-2.0"

readme = "README.md"

packages = [{ include = "crawlee", from = "src" }]

classifiers = [

"Development Status :: 4 - Beta",

"Intended Audience :: Developers",

"License :: OSI Approved :: Apache Software License",

"Operating System :: OS Independent",

"Programming Language :: Python :: 3.9",

"Programming Language :: Python :: 3.10",

"Programming Language :: Python :: 3.11",

"Programming Language :: Python :: 3.12",

"Topic :: Software Development :: Libraries",

]

keywords = [

"apify",

"automation",

"chrome",

"crawlee",

"crawler",

"headless",

"scraper",

"scraping",

]

[tool.poetry.urls]

"Homepage" = "https://crawlee.dev/python"

"Apify Homepage" = "https://apify.com"

"Changelog" = "https://crawlee.dev/python/docs/changelog"

"Documentation" = "https://crawlee.dev/python/docs/quick-start"

"Issue Tracker" = "https://github.com/apify/crawlee-python/issues"

"Repository" = "https://github.com/apify/crawlee-python"

[tool.poetry.dependencies]

python = "^3.9"

apify = { version = ">=2.0.0", optional = true }

beautifulsoup4 = { version = ">=4.12.0", optional = true }

browserforge = { version = ">=1.2.3", optional = true }

colorama = ">=0.4.0"

cookiecutter = ">=2.6.0"

curl-cffi = { version = ">=0.7.2", optional = true }

docutils = ">=0.21.0"

eval-type-backport = ">=0.2.0"

html5lib = { version = ">=1.0", optional = true }

httpx = {version = ">=0.27.0", extras = ["brotli", "http2", "zstd"]}

inquirer = ">=3.3.0"

jaro-winkler = { version = ">=2.0.3", optional = true }

lxml = { version = ">=5.2.0", optional = true }

more_itertools = ">=10.2.0"

parsel = { version = ">=1.9.0", optional = true }

playwright = { version = ">=1.27.0", optional = true }

psutil = ">=6.0.0"

pydantic = ">=2.8.1, !=2.10.0, !=2.10.1, !=2.10.2"

# TODO: relax the upper bound once the issue is resolved:

# https://github.com/apify/crawlee-python/issues/814

pydantic-settings = ">=2.2.0 <2.7.0"

pyee = ">=9.0.0"

scikit-learn = [

{version = ">=1.6.0", python = ">=3.10", optional = true},

{version = "==1.5.2", python = "==3.9", optional = true}

]

sortedcollections = ">=2.1.0"

tldextract = ">=5.1.0"

typer = ">=0.12.0"

typing-extensions = ">=4.1.0"

yarl = ">=1.18.0"

[tool.poetry.group.dev.dependencies]

build = "~1.2.0"

filelock = "~3.17.0"

ipdb = "~0.13.0"

mypy = "~1.15.0"

pre-commit = "~4.1.0"

proxy-py = "~2.4.0"

pydoc-markdown = "~4.8.0"

pytest = "~8.3.0"

pytest-asyncio = "~0.25.0"

pytest-cov = "~6.0.0"

pytest-only = "~2.1.0"

pytest-timeout = "~2.3.0"

pytest-xdist = "~3.6.0"

respx = "~0.22.0"

ruff = "~0.8.0"

setuptools = "~75.8.0" # setuptools are used by pytest, but not explicitly required

sortedcontainers-stubs = "^2.4.2"

types-beautifulsoup4 = "~4.12.0.20240229"

types-colorama = "~0.4.15.20240106"

types-psutil = "~5.9.5.20240205"

types-python-dateutil = "~2.9.0.20240316"

[tool.poetry.extras]

# Support for re-using groups in other groups https://peps.python.org/pep-0735/ in poetry:

# https://github.com/python-poetry/poetry/issues/9751

adaptive-playwright = ["jaro-winkler", "playwright", "scikit-learn"]

all = ["beautifulsoup4", "browserforge", "curl-cffi", "html5lib", "jaro-winkler", "lxml", "parsel", "playwright", "scikit-learn"]

beautifulsoup = ["beautifulsoup4", "lxml", "html5lib"]

curl-impersonate = ["curl-cffi"]

playwright = ["browserforge", "playwright"]

parsel = ["parsel"]

[tool.poetry.scripts]

crawlee = "crawlee._cli:cli"

[tool.ruff]

line-length = 120

include = ["src/**/*.py", "tests/**/*.py", "docs/**/*.py", "website/**/*.py"]

extend-exclude = ["project_template"]

[tool.ruff.lint]

select = ["ALL"]

ignore = [

"ANN401", # Dynamically typed expressions (typing.Any) are disallowed in {filename}

"ASYNC109", # Async function definition with a `timeout` parameter

"BLE001", # Do not catch blind exception

"C901", # `{name}` is too complex

"COM812", # This rule may cause conflicts when used with the formatter

"D100", # Missing docstring in public module

"D104", # Missing docstring in public package

"D107", # Missing docstring in `__init__`

"EM", # flake8-errmsg

"G004", # Logging statement uses f-string

"ISC001", # This rule may cause conflicts when used with the formatter

"FIX", # flake8-fixme

"PLR0911", # Too many return statements

"PLR0913", # Too many arguments in function definition

"PLR0915", # Too many statements

"PTH", # flake8-use-pathlib

"PYI034", # `__aenter__` methods in classes like `{name}` usually return `self` at runtime

"PYI036", # The second argument in `__aexit__` should be annotated with `object` or `BaseException | None`

"S102", # Use of `exec` detected

"S105", # Possible hardcoded password assigned to

"S106", # Possible hardcoded password assigned to argument: "{name}"

"S301", # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue

"S303", # Use of insecure MD2, MD4, MD5, or SHA1 hash function

"S311", # Standard pseudo-random generators are not suitable for cryptographic purposes

"TD002", # Missing author in TODO; try: `# TODO(<author_name>): ...` or `# TODO @<author_name>: ...

"TRY003", # Avoid specifying long messages outside the exception class

]

[tool.ruff.format]

quote-style = "single"

indent-style = "space"

[tool.ruff.lint.per-file-ignores]

"**/__init__.py" = [

"F401", # Unused imports

]

"**/{tests}/*" = [

"D", # Everything from the pydocstyle

"INP001", # File {filename} is part of an implicit namespace package, add an __init__.py

"PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable

"S101", # Use of assert detected

"SLF001", # Private member accessed: `{name}`

"T20", # flake8-print

"TRY301", # Abstract `raise` to an inner function

]

"**/{docs,website}/**" = [

"D", # Everything from the pydocstyle

"INP001", # File {filename} is part of an implicit namespace package, add an __init__.py

"F841", # Local variable {variable} is assigned to but never used

"N999", # Invalid module name

]

[tool.ruff.lint.flake8-quotes]

docstring-quotes = "double"

inline-quotes = "single"

[tool.ruff.lint.flake8-type-checking]

runtime-evaluated-base-classes = [

"pydantic.BaseModel",

"pydantic_settings.BaseSettings",

]

[tool.ruff.lint.flake8-builtins]

builtins-ignorelist = ["id"]

[tool.ruff.lint.pydocstyle]

convention = "google"

[tool.ruff.lint.isort]

known-first-party = ["crawlee"]

[tool.ruff.lint.pylint]

max-branches = 18

[tool.pytest.ini_options]

addopts = "-ra"

asyncio_default_fixture_loop_scope = "function"

asyncio_mode = "auto"

timeout = 1200

[tool.mypy]

python_version = "3.9"

plugins = ["pydantic.mypy"]

exclude = ["project_template"]

files = ["src", "tests", "docs", "website"]

check_untyped_defs = true

disallow_incomplete_defs = true

disallow_untyped_calls = true

disallow_untyped_decorators = true

disallow_untyped_defs = true

no_implicit_optional = true

warn_redundant_casts = true

warn_return_any = true

warn_unreachable = true

warn_unused_ignores = true

[[tool.mypy.overrides]]

# Example codes are sometimes showing integration of crawlee with external tool, which is not dependency of crawlee.

module = [

"apify", # Example code shows integration of apify and crawlee.

"camoufox", # Example code shows integration of camoufox and crawlee.

"jaro", # Untyped and stubs not available

"sklearn.linear_model", # Untyped and stubs not available

]

ignore_missing_imports = true

[tool.basedpyright]

pythonVersion = "3.9"

typeCheckingMode = "standard"

include = ["src", "tests", "docs", "website"]

[tool.coverage.report]

exclude_lines = [

"pragma: no cover",

"if TYPE_CHECKING:",

"assert_never()",

]

[tool.ipdb]

context = 7