[python] Support encoding images as video (#10171) · apache/paimon@9e306a4 · GitHub
Skip to content

Commit 9e306a4

Browse files
authored
[python] Support encoding images as video (#10171)
1 parent 1cf42e2 commit 9e306a4

7 files changed

Lines changed: 368 additions & 2 deletions

File tree

‎.github/workflows/ci-python.yml‎

Lines changed: 1 addition & 1 deletion

‎docs/docs/pypaimon/video.md‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,33 @@ frames.add_videos([
7171
])
7272
```
7373

74+
To encode an ordered sequence of image bytes or Pillow images before storing
75+
it, use Python 3.10 or newer, install the optional video dependencies, and call
76+
`add_images_as_video`:
77+
78+
```shell
79+
pip install 'pypaimon[video]'
80+
```
81+
82+
```python
83+
frames.add_images_as_video(
84+
image_frames,
85+
episode_44_rows,
86+
fps=30,
87+
codec="libx264",
88+
pixel_format="yuv420p",
89+
gop_size=2,
90+
codec_options={"crf": "18", "preset": "fast"},
91+
)
92+
```
93+
94+
The image and row counts must match, and every image must have the same
95+
dimensions. The container is MP4; codec-specific options are passed to PyAV.
96+
Encoding may be lossy and the original image payloads are not retained. A
97+
smaller GOP improves random frame access but usually increases file size.
98+
After encoding, Paimon copies the MP4 bytes into `.video` storage without
99+
transcoding them again.
100+
74101
Each item may also be `(video, frame_rows, first_frame)`. Frame ordinals are
75102
generated consecutively from `first_frame`. If application semantics require
76103
PTS, wall-clock time, or a non-unit sampling map, retain that value in a normal

‎paimon-python/pypaimon/multimodal/table.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,54 @@ def add_video(
133133
[(video, frames, first_frame)], video_column=video_column
134134
)
135135

136+
def add_images_as_video(
137+
self,
138+
images,
139+
frames,
140+
*,
141+
fps,
142+
codec,
143+
pixel_format,
144+
gop_size,
145+
codec_options=None,
146+
video_column=None):
147+
"""Encode ordered images as one MP4 and append its logical frame rows.
148+
149+
Images may be encoded image bytes or Pillow images. ``frames`` supplies
150+
every table column except ``video_column`` and must contain one row per
151+
image. Encoding finishes before the Paimon write starts.
152+
"""
153+
import os
154+
import tempfile
155+
156+
from pypaimon.multimodal.video import _encode_images_to_video
157+
158+
column = self._resolve_video_frame_column(video_column)
159+
target_schema = _target_schema(self.raw_table)
160+
non_video_schema = pa.schema([
161+
field for field in target_schema if field.name != column
162+
])
163+
frame_table = _to_arrow_table(frames, non_video_schema)
164+
with tempfile.TemporaryDirectory(prefix="pypaimon_video_") as directory:
165+
video_path = os.path.join(directory, "video.mp4")
166+
image_count = _encode_images_to_video(
167+
images,
168+
video_path,
169+
fps=fps,
170+
codec=codec,
171+
pixel_format=pixel_format,
172+
gop_size=gop_size,
173+
codec_options=codec_options,
174+
)
175+
if image_count != frame_table.num_rows:
176+
raise ValueError(
177+
"Image count %d does not match frame row count %d."
178+
% (image_count, frame_table.num_rows)
179+
)
180+
return self.add_video(
181+
video_path, frame_table, video_column=column
182+
)
183+
136184
def add_videos(self, videos, *, video_column=None):
137185
"""Append several encoded videos with one writer and one commit.
138186

‎paimon-python/pypaimon/multimodal/video.py‎

Lines changed: 95 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,109 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
"""PyTorch DataLoader helpers for descriptor-backed video frame rows."""
18+
"""Helpers for descriptor-backed video frame rows."""
1919

20+
import io
2021
import os
2122
from collections import OrderedDict
2223
from collections.abc import Mapping
24+
from fractions import Fraction
25+
from itertools import chain
2326

2427
from pypaimon.table.row.blob import Blob, VideoFrameDescriptor
2528

2629

30+
def _encode_images_to_video(
31+
images,
32+
output_path,
33+
*,
34+
fps,
35+
codec,
36+
pixel_format,
37+
gop_size,
38+
codec_options=None):
39+
if isinstance(fps, bool) or not isinstance(fps, int) or fps <= 0:
40+
raise ValueError("fps must be a positive int.")
41+
if not isinstance(codec, str) or not codec:
42+
raise ValueError("codec must be a non-empty string.")
43+
if not isinstance(pixel_format, str) or not pixel_format:
44+
raise ValueError("pixel_format must be a non-empty string.")
45+
if isinstance(gop_size, bool) \
46+
or not isinstance(gop_size, int) or gop_size <= 0:
47+
raise ValueError("gop_size must be a positive int.")
48+
if codec_options is not None and not isinstance(codec_options, Mapping):
49+
raise ValueError("codec_options must be a mapping or None.")
50+
51+
try:
52+
import av
53+
from PIL import Image
54+
except ImportError as error:
55+
raise ImportError(
56+
"Image-to-video encoding requires PyAV and Pillow; install "
57+
"pypaimon[video]."
58+
) from error
59+
60+
iterator = iter(images)
61+
try:
62+
first = next(iterator)
63+
except StopIteration as error:
64+
raise ValueError("images must contain at least one frame.") from error
65+
66+
def open_image(value):
67+
if isinstance(value, (bytes, bytearray, memoryview)):
68+
image = Image.open(io.BytesIO(bytes(value)))
69+
elif isinstance(value, Image.Image):
70+
image = value
71+
else:
72+
raise TypeError(
73+
"images must contain encoded image bytes or Pillow images."
74+
)
75+
image.load()
76+
return image.convert("RGB")
77+
78+
first = open_image(first)
79+
width, height = first.size
80+
time_base = Fraction(1, fps)
81+
count = 0
82+
try:
83+
with av.open(output_path, mode="w", format="mp4") as container:
84+
stream = container.add_stream(
85+
codec,
86+
rate=fps,
87+
options={
88+
str(key): str(value)
89+
for key, value in (codec_options or {}).items()
90+
},
91+
)
92+
stream.width = width
93+
stream.height = height
94+
stream.pix_fmt = pixel_format
95+
stream.gop_size = gop_size
96+
stream.time_base = time_base
97+
decoded = chain(
98+
(first,), (open_image(value) for value in iterator)
99+
)
100+
for count, image in enumerate(decoded, start=1):
101+
if image.size != (width, height):
102+
raise ValueError(
103+
"All images must have the same dimensions."
104+
)
105+
frame = av.VideoFrame.from_image(image)
106+
frame.pts = count - 1
107+
frame.time_base = time_base
108+
for packet in stream.encode(frame):
109+
container.mux(packet)
110+
for packet in stream.encode():
111+
container.mux(packet)
112+
except BaseException:
113+
try:
114+
os.remove(output_path)
115+
except FileNotFoundError:
116+
pass
117+
raise
118+
return count
119+
120+
27121
class VideoFrameCollator:
28122
"""Decode frame rows in a DataLoader worker while reusing video sessions.
29123

‎paimon-python/pypaimon/tests/multimodal_table_test.py‎

Lines changed: 98 additions & 0 deletions

0 commit comments

Comments
 (0)