Fix: Cosmos2.5 Video2World frame extraction and add default negative … · zsc/diffusers@71a865b · GitHub
Skip to content

Commit 71a865b

Browse files
adi776borateyiyixuxugithub-actions[bot]
authored
Fix: Cosmos2.5 Video2World frame extraction and add default negative prompt (huggingface#13018)
* fix: Extract last frames for conditioning in Cosmos Video2World * Added default negative prompt * Apply style fixes * Added default negative prompt in cosmos2 text2image pipeline --------- Co-authored-by: YiYi Xu <yixu310@gmail.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
1 parent 53279ef commit 71a865b

5 files changed

Lines changed: 75 additions & 10 deletions

File tree

‎src/diffusers/pipelines/cosmos/pipeline_cosmos2_5_predict.py‎

Lines changed: 39 additions & 6 deletions

‎src/diffusers/pipelines/cosmos/pipeline_cosmos2_text2image.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,14 @@ def __init__(self, *args, **kwargs):
4949

5050
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
5151

52+
DEFAULT_NEGATIVE_PROMPT = (
53+
"The video captures a series of frames showing ugly scenes, static with no motion, motion blur, "
54+
"over-saturation, shaky footage, low resolution, grainy texture, pixelated images, poorly lit areas, "
55+
"underexposed and overexposed scenes, poor color balance, washed out colors, choppy sequences, "
56+
"jerky movements, low frame rate, artifacting, color banding, unnatural transitions, outdated special effects, "
57+
"fake elements, unconvincing visuals, poorly edited content, jump cuts, visual noise, and flickering. "
58+
"Overall, the video is of poor quality."
59+
)
5260

5361
EXAMPLE_DOC_STRING = """
5462
Examples:
@@ -300,7 +308,7 @@ def encode_prompt(
300308
prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
301309

302310
if do_classifier_free_guidance and negative_prompt_embeds is None:
303-
negative_prompt = negative_prompt or ""
311+
negative_prompt = negative_prompt if negative_prompt is not None else DEFAULT_NEGATIVE_PROMPT
304312
negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
305313

306314
if prompt is not None and type(prompt) is not type(negative_prompt):

‎src/diffusers/pipelines/cosmos/pipeline_cosmos2_video2world.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,14 @@ def __init__(self, *args, **kwargs):
5050

5151
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
5252

53+
DEFAULT_NEGATIVE_PROMPT = (
54+
"The video captures a series of frames showing ugly scenes, static with no motion, motion blur, "
55+
"over-saturation, shaky footage, low resolution, grainy texture, pixelated images, poorly lit areas, "
56+
"underexposed and overexposed scenes, poor color balance, washed out colors, choppy sequences, "
57+
"jerky movements, low frame rate, artifacting, color banding, unnatural transitions, outdated special effects, "
58+
"fake elements, unconvincing visuals, poorly edited content, jump cuts, visual noise, and flickering. "
59+
"Overall, the video is of poor quality."
60+
)
5361

5462
EXAMPLE_DOC_STRING = """
5563
Examples:
@@ -319,7 +327,7 @@ def encode_prompt(
319327
prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
320328

321329
if do_classifier_free_guidance and negative_prompt_embeds is None:
322-
negative_prompt = negative_prompt or ""
330+
negative_prompt = negative_prompt if negative_prompt is not None else DEFAULT_NEGATIVE_PROMPT
323331
negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
324332

325333
if prompt is not None and type(prompt) is not type(negative_prompt):

‎src/diffusers/pipelines/cosmos/pipeline_cosmos_text2world.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,14 @@ def __init__(self, *args, **kwargs):
4949

5050
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
5151

52+
DEFAULT_NEGATIVE_PROMPT = (
53+
"The video captures a series of frames showing ugly scenes, static with no motion, motion blur, "
54+
"over-saturation, shaky footage, low resolution, grainy texture, pixelated images, poorly lit areas, "
55+
"underexposed and overexposed scenes, poor color balance, washed out colors, choppy sequences, "
56+
"jerky movements, low frame rate, artifacting, color banding, unnatural transitions, outdated special effects, "
57+
"fake elements, unconvincing visuals, poorly edited content, jump cuts, visual noise, and flickering. "
58+
"Overall, the video is of poor quality."
59+
)
5260

5361
EXAMPLE_DOC_STRING = """
5462
Examples:
@@ -285,7 +293,7 @@ def encode_prompt(
285293
prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
286294

287295
if do_classifier_free_guidance and negative_prompt_embeds is None:
288-
negative_prompt = negative_prompt or ""
296+
negative_prompt = negative_prompt if negative_prompt is not None else DEFAULT_NEGATIVE_PROMPT
289297
negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
290298

291299
if prompt is not None and type(prompt) is not type(negative_prompt):

‎src/diffusers/pipelines/cosmos/pipeline_cosmos_video2world.py‎

Lines changed: 9 additions & 1 deletion

0 commit comments

Comments
 (0)