[From Single File] support `from_single_file` method for `WanVACE3DTr… · huggingface/diffusers@0e95aa8 · GitHub
Skip to content

Commit 0e95aa8

Browse files
J4BEZDN6github-actions[bot]
authored
[From Single File] support from_single_file method for WanVACE3DTransformer (#11807)
* add `WandVACETransformer3DModel` in`SINGLE_FILE_LOADABLE_CLASSES` * add rename keys for `VACE` add rename keys for `VACE` * fix typo Sincere thanks to @nitinmukesh 🙇‍♂️ * support for `1.3B VACE` model Sincere thanks to @nitinmukesh again🙇‍♂️ * update * update * Apply style fixes --------- Co-authored-by: Dhruv Nair <dhruv.nair@gmail.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
1 parent 5ef74fd commit 0e95aa8

3 files changed

Lines changed: 87 additions & 1 deletion

File tree

‎src/diffusers/loaders/single_file_model.py‎

Lines changed: 4 additions & 0 deletions

‎src/diffusers/loaders/single_file_utils.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@
126126
],
127127
"wan": ["model.diffusion_model.head.modulation", "head.modulation"],
128128
"wan_vae": "decoder.middle.0.residual.0.gamma",
129+
"wan_vace": "vace_blocks.0.after_proj.bias",
129130
"hidream": "double_stream_blocks.0.block.adaLN_modulation.1.bias",
130131
"cosmos-1.0": [
131132
"net.x_embedder.proj.1.weight",
@@ -202,6 +203,8 @@
202203
"wan-t2v-1.3B": {"pretrained_model_name_or_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers"},
203204
"wan-t2v-14B": {"pretrained_model_name_or_path": "Wan-AI/Wan2.1-T2V-14B-Diffusers"},
204205
"wan-i2v-14B": {"pretrained_model_name_or_path": "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers"},
206+
"wan-vace-1.3B": {"pretrained_model_name_or_path": "Wan-AI/Wan2.1-VACE-1.3B-diffusers"},
207+
"wan-vace-14B": {"pretrained_model_name_or_path": "Wan-AI/Wan2.1-VACE-14B-diffusers"},
205208
"hidream": {"pretrained_model_name_or_path": "HiDream-ai/HiDream-I1-Dev"},
206209
"cosmos-1.0-t2w-7B": {"pretrained_model_name_or_path": "nvidia/Cosmos-1.0-Diffusion-7B-Text2World"},
207210
"cosmos-1.0-t2w-14B": {"pretrained_model_name_or_path": "nvidia/Cosmos-1.0-Diffusion-14B-Text2World"},
@@ -716,7 +719,13 @@ def infer_diffusers_model_type(checkpoint):
716719
else:
717720
target_key = "patch_embedding.weight"
718721

719-
if checkpoint[target_key].shape[0] == 1536:
722+
if CHECKPOINT_KEY_NAMES["wan_vace"] in checkpoint:
723+
if checkpoint[target_key].shape[0] == 1536:
724+
model_type = "wan-vace-1.3B"
725+
elif checkpoint[target_key].shape[0] == 5120:
726+
model_type = "wan-vace-14B"
727+
728+
elif checkpoint[target_key].shape[0] == 1536:
720729
model_type = "wan-t2v-1.3B"
721730
elif checkpoint[target_key].shape[0] == 5120 and checkpoint[target_key].shape[1] == 16:
722731
model_type = "wan-t2v-14B"
@@ -3132,6 +3141,9 @@ def convert_wan_transformer_to_diffusers(checkpoint, **kwargs):
31323141
"img_emb.proj.1": "condition_embedder.image_embedder.ff.net.0.proj",
31333142
"img_emb.proj.3": "condition_embedder.image_embedder.ff.net.2",
31343143
"img_emb.proj.4": "condition_embedder.image_embedder.norm2",
3144+
# For the VACE model
3145+
"before_proj": "proj_in",
3146+
"after_proj": "proj_out",
31353147
}
31363148

31373149
for key in list(checkpoint.keys()):

‎tests/quantization/gguf/test_gguf.py‎

Lines changed: 70 additions & 0 deletions

0 commit comments

Comments
 (0)