Read manifests with V3 projection (#3690) · bookingcom/iceberg-python@2dd080e · GitHub
Skip to content

Commit 2dd080e

Browse files
authored
Read manifests with V3 projection (apache#3690)
* Fix V3 manifest read projection Read manifest entries and manifest lists with the latest supported schema so V3-only fields are retained while older manifests resolve missing fields to null. * Fix legacy manifest test expectations Exclude V3-only properties from V1 and V2 fastavro comparisons and clarify the latest read-version constant name. * Fix V2 integration manifest expectation Exclude V3-only properties from the reflected V2 manifest dictionary used by the REST integration test. * Use V3 as default manifest read version
1 parent 7a6c2ec commit 2dd080e

4 files changed

Lines changed: 138 additions & 2 deletions

File tree

‎pyiceberg/manifest.py‎

Lines changed: 21 additions & 1 deletion

‎tests/avro/test_file.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,8 @@ def test_write_manifest_entry_with_iceberg_read_with_fastavro_v1() -> None:
164164
del v2_entry["file_sequence_number"]
165165
del v2_entry["data_file"]["content"]
166166
del v2_entry["data_file"]["equality_ids"]
167+
for field in ("first_row_id", "referenced_data_file", "content_offset", "content_size_in_bytes"):
168+
del v2_entry["data_file"][field]
167169

168170
# Required in V1
169171
v2_entry["data_file"]["block_size_in_bytes"] = DEFAULT_BLOCK_SIZE
@@ -222,7 +224,11 @@ def test_write_manifest_entry_with_iceberg_read_with_fastavro_v2() -> None:
222224

223225
fa_entry = next(it)
224226

225-
assert todict(entry) == fa_entry
227+
v2_entry = todict(entry)
228+
for field in ("first_row_id", "referenced_data_file", "content_offset", "content_size_in_bytes"):
229+
del v2_entry["data_file"][field]
230+
231+
assert v2_entry == fa_entry
226232

227233

228234
@pytest.mark.parametrize("format_version", [1, 2])

‎tests/integration/test_rest_manifest.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,8 @@ def test_write_sample_manifest(table_test_all_types: Table, compression: AvroCom
112112
wrapped_entry_v2 = copy(entry)
113113
wrapped_entry_v2.data_file = wrapped_data_file_v2_debug
114114
wrapped_entry_v2_dict = todict(wrapped_entry_v2, [field.name for field in test_spec.fields])
115+
for field in ("first_row_id", "referenced_data_file", "content_offset", "content_size_in_bytes"):
116+
del wrapped_entry_v2_dict["data_file"][field]
115117

116118
with TemporaryDirectory() as tmpdir:
117119
tmp_avro_file = tmpdir + "/test_write_manifest.avro"

‎tests/utils/test_manifest.py‎

Lines changed: 108 additions & 0 deletions

0 commit comments

Comments
 (0)