FilesExpand file tree

main

/

pyproject.toml

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

284 lines (260 loc) · 8.36 KB

main

/

pyproject.toml

Copy path

Top

File metadata and controls

• Code
• Blame

284 lines (260 loc) · 8.36 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

# Licensed to the Apache Software Foundation (ASF) under one

# or more contributor license agreements. See the NOTICE file

# distributed with this work for additional information

# regarding copyright ownership. The ASF licenses this file

# to you under the Apache License, Version 2.0 (the

# "License"); you may not use this file except in compliance

# with the License. You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing,

# software distributed under the License is distributed on an

# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

# KIND, either express or implied. See the License for the

# specific language governing permissions and limitations

# under the License.

[project]

name = "pyiceberg"

version = "0.12.0"

description = "Apache Iceberg is an open table format for huge analytic datasets"

authors = [{ name = "Apache Software Foundation", email = "dev@iceberg.apache.org" }]

requires-python = ">=3.10.0,<4.0.0"

readme = "README.md"

license = "Apache-2.0"

license-files = ["LICENSE", "NOTICE"]

classifiers = [

"Operating System :: OS Independent",

"Programming Language :: Python :: 3.10",

"Programming Language :: Python :: 3.11",

"Programming Language :: Python :: 3.12",

"Programming Language :: Python :: 3.13",

"Programming Language :: Python :: 3.14",

]

dependencies = [

"mmh3>=4.0.0",

"requests>=2.20.0",

"click>=7.1.1",

"rich>=10.11.0",

"strictyaml>=1.7.0", # CVE-2020-14343 was fixed in 5.4.

"pydantic>=2.0,<3.0,!=2.4.0,!=2.4.1,!=2.12.0,!=2.12.1", # 2.4.0, 2.4.1, 2.12.0, 2.12.1 has a critical bug

"fsspec>=2023.1.0",

"pyparsing>=3.1.0",

"tenacity>=8.2.3",

"pyroaring>=1.0.0",

"cachetools>=5.5",

"zstandard>=0.13.0"

]

[project.urls]

Homepage = "https://py.iceberg.apache.org/"

Repository = "https://github.com/apache/iceberg-python"

[project.scripts]

pyiceberg = "pyiceberg.cli.console:run"

[project.optional-dependencies]

pyarrow = [

"pyarrow>=18.0.0",

]

pandas = [

"pandas>=1.0.0",

"pyarrow>=18.0.0",

]

duckdb = [

"duckdb>=0.5.0",

"pyarrow>=18.0.0",

]

ray = [

"ray>=2.10.0",

"pyarrow>=18.0.0",

"pandas>=1.0.0",

]

bodo = ["bodo>=2025.7.4"]

daft = ["daft>=0.7.10"]

polars = ["polars>=1.21.0"]

snappy = ["python-snappy>=0.6.0"]

hive = ["thrift>=0.13.0"]

hive-kerberos = [

"thrift>=0.13.0",

"thrift-sasl>=0.4.3",

"kerberos>=1.3.1",

]

s3fs = ["s3fs>=2023.1.0"]

glue = ["boto3>=1.24.59"]

adlfs = ["adlfs>=2024.7.0"]

dynamodb = ["boto3>=1.24.59"]

bigquery = ["google-cloud-bigquery>=3.33.0,<4"]

sql-postgres = [

"sqlalchemy>=2.0.18,<3",

"psycopg2-binary>=2.9.6",

]

sql-sqlite = ["sqlalchemy>=2.0.18,<3"]

gcsfs = ["gcsfs>=2023.1.0"]

rest-sigv4 = ["boto3>=1.24.59"]

hf = ["huggingface-hub>=0.24.0"]

# `pyiceberg-core` bundles a `datafusion-ffi` build, and the DataFusion FFI ABI is not stable

# across major versions (https://github.com/apache/datafusion/issues/17374). The two ranges below

# must name the same DataFusion major version: pyiceberg-core 0.10.x bundles datafusion-ffi 53.x.

pyiceberg-core = ["pyiceberg-core>=0.10.1,<0.11.0"]

datafusion = ["datafusion>=53,<54"]

gcp-auth = ["google-auth>=2.4.0"]

entra-auth = ["azure-identity>=1.25.1"]

geoarrow = ["geoarrow-pyarrow>=0.2.0"]

encryption = ["cryptography>=42.0.0"]

[dependency-groups]

dev = [

"pytest==9.1.1",

"pytest-checkdocs==2.14.0",

"setuptools>=83.0.0",

"prek>=0.2.1",

"pytest-lazy-fixtures==1.4.1",

"fastavro==1.12.2",

"coverage[toml]>=7.4.2",

"requests-mock==1.12.1",

"moto[server]>=5.0.2",

"typing-extensions==4.16.0",

"pytest-mock==3.15.1",

"pyspark[connect]==4.0.1",

"protobuf==7.36.2", # match Spark Connect's gencode

"cython>=3.0.0",

"deptry>=0.14",

"docutils!=0.21.post1",

"mypy-boto3-glue>=1.28.18",

"mypy-boto3-dynamodb>=1.28.18",

"google-cloud-bigquery>=3.33.0,<4",

"pyarrow-stubs>=20.0.0.20251107", # Remove when pyarrow >= 23.0.0 https://github.com/apache/arrow/pull/47609

"sqlalchemy>=2.0.18,<3",

"papermill>=2.6.0",

"nbformat>=5.10.0",

"ipykernel>=6.29.0",

"ty==0.0.84",

]

# for mkdocs

docs = [

"mkdocs==1.6.1",

"griffe==2.3.0",

"jinja2==3.1.6",

"mkdocstrings==1.0.6",

"mkdocstrings-python==2.0.9",

"mkdocs-literate-nav==0.6.3",

"mkdocs-autorefs==1.4.4",

"mkdocs-gen-files==0.6.1",

"mkdocs-material==9.7.7",

"mkdocs-material-extensions==1.3.1",

"mkdocs-section-index==0.3.12",

]

# for experimentation

notebook = [

"jupyterlab>=4.0.0",

]

[tool.uv]

default-groups = [

"dev",

]

[build-system]

requires = ["setuptools>=83.0.0", "wheel", "Cython>=3.0.0"]

build-backend = "setuptools.build_meta"

[tool.ruff]

src = ['pyiceberg','tests']

# Exclude a variety of commonly ignored directories.

exclude = [

".bzr",

".direnv",

".eggs",

".git",

".git-rewrite",

".hg",

".mypy_cache",

".nox",

".pants.d",

".pytype",

".ruff_cache",

".svn",

".tox",

".venv",

"__pypackages__",

"_build",

"buck-out",

"build",

"dist",

"node_modules",

"venv",

]

# Ignore _all_ violations.

# Same as Black.

line-length = 130

[tool.ruff.lint]

select = [

"E", # pycodestyle

"W", # pycodestyle

"F", # Pyflakes

"B", # flake8-bugbear

"PIE", # flake8-pie

"C4", # flake8-comprehensions

"I", # isort

"UP", # pyupgrade

"T100", # flake8-debugger

]

ignore = []

# Allow autofix for all enabled rules (when `--fix`) is provided.

fixable = ["ALL"]

unfixable = []

per-file-ignores = {}

# Allow unused variables when underscore-prefixed.

dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"

[tool.ruff.lint.pyupgrade]

# Preserve types, even if a file imports `from __future__ import annotations`.

keep-runtime-typing = true

[tool.ruff.lint.isort]

detect-same-package = true

lines-between-types = 0

known-first-party = ["pyiceberg", "tests"]

section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]

[tool.ruff.format]

quote-style = "double"

[tool.pytest.ini_options]

testpaths = ["tests"]

markers = [

"unit: marks a test as a unit test",

"s3: marks a test as requiring access to s3 compliant storage (use with --aws-access-key-id, --aws-secret-access-key, and --endpoint args)",

"adls: marks a test as requiring access to adls compliant storage (use with --adls.account-name, --adls.account-key, and --adls.endpoint args)",

"integration: marks integration tests against Apache Spark",

"gcs: marks a test as requiring access to gcs compliant storage (use with --gs.token, --gs.project, and --gs.endpoint)",

"benchmark: collection of tests to validate read/write performance before and after a change",

"notebook: marks tests that execute Jupyter notebooks via papermill",

]

# Turns a warning into an error

filterwarnings = [

"error",

# Ignore Python version deprecation, end-of-life and unsupported warnings from google.api_core while we still support 3.10

"ignore:You are using a (non-supported )?Python version:FutureWarning:google.api_core",

# Python 3.13 sqlite3 module ResourceWarnings for unclosed database connections

"ignore:unclosed database in <sqlite3.Connection object*:ResourceWarning",

# Ignore Ray subprocess cleanup warnings

"ignore:unclosed file:ResourceWarning",

"ignore:subprocess.*is still running:ResourceWarning",

# Ignore google-crc32c C extension missing warning (common on Python 3.14+)

"ignore:As the c extension couldn't be imported:RuntimeWarning:google_crc32c",

# Ignore Spark 4.0.1 pandas conversion warning under pandas 3.0

"ignore:The copy keyword is deprecated and will be removed in a future version.*",

]

[tool.mypy]

mypy_path = "python"

no_implicit_optional = true

namespace_packages = false

warn_redundant_casts = true

warn_unreachable = true

warn_unused_ignores = true

disallow_any_generics = true

disallow_untyped_defs = true

implicit_reexport = false

ignore_missing_imports = true

[tool.ty.src]

exclude = ["vendor/"]

[tool.ty.environment]

# The vendored Hive Metastore client is packaged from vendor/, so it must stay importable

extra-paths = ["vendor"]

[tool.ty.analysis]

# pyiceberg_core.datafusion is registered at runtime by its compiled extension; griffe and

# mkdocs_gen_files are only installed with the docs dependency group

allowed-unresolved-imports = ["pyiceberg_core.datafusion", "griffe", "mkdocs_gen_files"]

[tool.coverage.run]

source = ['pyiceberg/']