[Qwen] avoid creating attention masks when there is no padding (#12987) · huggingface/diffusers@d54669a · GitHub
Skip to content

Commit d54669a

Browse files
kashifsayakpaul
andauthored
[Qwen] avoid creating attention masks when there is no padding (#12987)
* avoid creating attention masks when there is no padding * make fix-copies * torch compile tests * set all ones mask to none * fix positional encoding from becoming > 4096 * fix from review * slice freqs_cis to match the input sequence length * keep only attenton masking change --------- Co-authored-by: Sayak Paul <spsayakpaul@gmail.com>
1 parent 22ac6fa commit d54669a

10 files changed

Lines changed: 98 additions & 0 deletions

‎src/diffusers/pipelines/qwenimage/pipeline_qwenimage.py‎

Lines changed: 3 additions & 0 deletions

‎src/diffusers/pipelines/qwenimage/pipeline_qwenimage_controlnet.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,9 @@ def encode_prompt(
324324
prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_images_per_prompt, 1)
325325
prompt_embeds_mask = prompt_embeds_mask.view(batch_size * num_images_per_prompt, seq_len)
326326

327+
if prompt_embeds_mask is not None and prompt_embeds_mask.all():
328+
prompt_embeds_mask = None
329+
327330
return prompt_embeds, prompt_embeds_mask
328331

329332
def check_inputs(

‎src/diffusers/pipelines/qwenimage/pipeline_qwenimage_controlnet_inpaint.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,9 @@ def encode_prompt(
305305
prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_images_per_prompt, 1)
306306
prompt_embeds_mask = prompt_embeds_mask.view(batch_size * num_images_per_prompt, seq_len)
307307

308+
if prompt_embeds_mask is not None and prompt_embeds_mask.all():
309+
prompt_embeds_mask = None
310+
308311
return prompt_embeds, prompt_embeds_mask
309312

310313
def check_inputs(

‎src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,9 @@ def encode_prompt(
309309
prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_images_per_prompt, 1)
310310
prompt_embeds_mask = prompt_embeds_mask.view(batch_size * num_images_per_prompt, seq_len)
311311

312+
if prompt_embeds_mask is not None and prompt_embeds_mask.all():
313+
prompt_embeds_mask = None
314+
312315
return prompt_embeds, prompt_embeds_mask
313316

314317
def check_inputs(

‎src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit_inpaint.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,9 @@ def encode_prompt(
321321
prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_images_per_prompt, 1)
322322
prompt_embeds_mask = prompt_embeds_mask.view(batch_size * num_images_per_prompt, seq_len)
323323

324+
if prompt_embeds_mask is not None and prompt_embeds_mask.all():
325+
prompt_embeds_mask = None
326+
324327
return prompt_embeds, prompt_embeds_mask
325328

326329
# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage_inpaint.QwenImageInpaintPipeline.check_inputs

‎src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit_plus.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,9 @@ def encode_prompt(
323323
prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_images_per_prompt, 1)
324324
prompt_embeds_mask = prompt_embeds_mask.view(batch_size * num_images_per_prompt, seq_len)
325325

326+
if prompt_embeds_mask is not None and prompt_embeds_mask.all():
327+
prompt_embeds_mask = None
328+
326329
return prompt_embeds, prompt_embeds_mask
327330

328331
# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage_edit.QwenImageEditPipeline.check_inputs

‎src/diffusers/pipelines/qwenimage/pipeline_qwenimage_img2img.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,9 @@ def encode_prompt(
305305
prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_images_per_prompt, 1)
306306
prompt_embeds_mask = prompt_embeds_mask.view(batch_size * num_images_per_prompt, seq_len)
307307

308+
if prompt_embeds_mask is not None and prompt_embeds_mask.all():
309+
prompt_embeds_mask = None
310+
308311
return prompt_embeds, prompt_embeds_mask
309312

310313
def check_inputs(

‎src/diffusers/pipelines/qwenimage/pipeline_qwenimage_inpaint.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,9 @@ def encode_prompt(
316316
prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_images_per_prompt, 1)
317317
prompt_embeds_mask = prompt_embeds_mask.view(batch_size * num_images_per_prompt, seq_len)
318318

319+
if prompt_embeds_mask is not None and prompt_embeds_mask.all():
320+
prompt_embeds_mask = None
321+
319322
return prompt_embeds, prompt_embeds_mask
320323

321324
def check_inputs(

‎src/diffusers/pipelines/qwenimage/pipeline_qwenimage_layered.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,9 @@ def encode_prompt(
328328
prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_images_per_prompt, 1)
329329
prompt_embeds_mask = prompt_embeds_mask.view(batch_size * num_images_per_prompt, seq_len)
330330

331+
if prompt_embeds_mask is not None and prompt_embeds_mask.all():
332+
prompt_embeds_mask = None
333+
331334
return prompt_embeds, prompt_embeds_mask
332335

333336
def get_image_caption(self, prompt_image, use_en_prompt=True, device=None):

‎tests/models/transformers/test_models_transformer_qwenimage.py‎

Lines changed: 71 additions & 0 deletions

0 commit comments

Comments
 (0)