[refactor embeddings] gligen + ip-adapter (#6244) · fpgaminer/diffusers@4c483de · GitHub
Skip to content

Commit 4c483de

Browse files
yiyixuxuyiyixuxu
yiyixuxu
and
yiyixuxu
authored
[refactor embeddings] gligen + ip-adapter (huggingface#6244)
* refactor ip-adapter-imageproj, gligen --------- Co-authored-by: yiyixuxu <yixu310@gmail,com>
1 parent 1ac07d8 commit 4c483de

6 files changed

Lines changed: 32 additions & 27 deletions

File tree

‎src/diffusers/loaders/unet.py‎

Lines changed: 4 additions & 4 deletions

‎src/diffusers/models/embeddings.py‎

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -462,7 +462,7 @@ def forward(self, image_embeds: torch.FloatTensor):
462462
return image_embeds
463463

464464

465-
class MLPProjection(nn.Module):
465+
class IPAdapterFullImageProjection(nn.Module):
466466
def __init__(self, image_embed_dim=1024, cross_attention_dim=1024):
467467
super().__init__()
468468
from .attention import FeedForward
@@ -621,29 +621,34 @@ def shape(x):
621621
return a[:, 0, :] # cls_token
622622

623623

624-
class FourierEmbedder(nn.Module):
625-
def __init__(self, num_freqs=64, temperature=100):
626-
super().__init__()
624+
def get_fourier_embeds_from_boundingbox(embed_dim, box):
625+
"""
626+
Args:
627+
embed_dim: int
628+
box: a 3-D tensor [B x N x 4] representing the bounding boxes for GLIGEN pipeline
629+
Returns:
630+
[B x N x embed_dim] tensor of positional embeddings
631+
"""
632+
633+
batch_size, num_boxes = box.shape[:2]
627634

628-
self.num_freqs = num_freqs
629-
self.temperature = temperature
635+
emb = 100 ** (torch.arange(embed_dim) / embed_dim)
636+
emb = emb[None, None, None].to(device=box.device, dtype=box.dtype)
637+
emb = emb * box.unsqueeze(-1)
630638

631-
freq_bands = temperature ** (torch.arange(num_freqs) / num_freqs)
632-
freq_bands = freq_bands[None, None, None]
633-
self.register_buffer("freq_bands", freq_bands, persistent=False)
639+
emb = torch.stack((emb.sin(), emb.cos()), dim=-1)
640+
emb = emb.permute(0, 1, 3, 4, 2).reshape(batch_size, num_boxes, embed_dim * 2 * 4)
634641

635-
def __call__(self, x):
636-
x = self.freq_bands * x.unsqueeze(-1)
637-
return torch.stack((x.sin(), x.cos()), dim=-1).permute(0, 1, 3, 4, 2).reshape(*x.shape[:2], -1)
642+
return emb
638643

639644

640-
class PositionNet(nn.Module):
645+
class GLIGENTextBoundingboxProjection(nn.Module):
641646
def __init__(self, positive_len, out_dim, feature_type="text-only", fourier_freqs=8):
642647
super().__init__()
643648
self.positive_len = positive_len
644649
self.out_dim = out_dim
645650

646-
self.fourier_embedder = FourierEmbedder(num_freqs=fourier_freqs)
651+
self.fourier_embedder_dim = fourier_freqs
647652
self.position_dim = fourier_freqs * 2 * 4 # 2: sin/cos, 4: xyxy
648653

649654
if isinstance(out_dim, tuple):
@@ -692,7 +697,7 @@ def forward(
692697
masks = masks.unsqueeze(-1)
693698

694699
# embedding position (it may includes padding as placeholder)
695-
xyxy_embedding = self.fourier_embedder(boxes) # B*N*4 -> B*N*C
700+
xyxy_embedding = get_fourier_embeds_from_boundingbox(self.fourier_embedder_dim, boxes) # B*N*4 -> B*N*C
696701

697702
# learnable null embedding
698703
xyxy_null = self.null_position_feature.view(1, 1, -1)
@@ -787,7 +792,7 @@ def forward(self, caption):
787792
return hidden_states
788793

789794

790-
class Resampler(nn.Module):
795+
class IPAdapterPlusImageProjection(nn.Module):
791796
"""Resampler of IP-Adapter Plus.
792797
793798
Args:

‎src/diffusers/models/unet_2d_condition.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,10 @@
3232
)
3333
from .embeddings import (
3434
GaussianFourierProjection,
35+
GLIGENTextBoundingboxProjection,
3536
ImageHintTimeEmbedding,
3637
ImageProjection,
3738
ImageTimeEmbedding,
38-
PositionNet,
3939
TextImageProjection,
4040
TextImageTimeEmbedding,
4141
TextTimeEmbedding,
@@ -615,7 +615,7 @@ def __init__(
615615
positive_len = cross_attention_dim[0]
616616

617617
feature_type = "text-only" if attention_type == "gated" else "text-image"
618-
self.position_net = PositionNet(
618+
self.position_net = GLIGENTextBoundingboxProjection(
619619
positive_len=positive_len, out_dim=cross_attention_dim, feature_type=feature_type
620620
)
621621

‎src/diffusers/pipelines/deprecated/versatile_diffusion/modeling_text_unet.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ def __call__(self, x):
187187
return torch.stack((x.sin(), x.cos()), dim=-1).permute(0, 1, 3, 4, 2).reshape(*x.shape[:2], -1)
188188

189189

190-
class PositionNet(nn.Module):
190+
class GLIGENTextBoundingboxProjection(nn.Module):
191191
def __init__(self, positive_len, out_dim, feature_type, fourier_freqs=8):
192192
super().__init__()
193193
self.positive_len = positive_len
@@ -820,7 +820,7 @@ def __init__(
820820
positive_len = cross_attention_dim[0]
821821

822822
feature_type = "text-only" if attention_type == "gated" else "text-image"
823-
self.position_net = PositionNet(
823+
self.position_net = GLIGENTextBoundingboxProjection(
824824
positive_len=positive_len, out_dim=cross_attention_dim, feature_type=feature_type
825825
)
826826

‎src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -730,7 +730,7 @@ def __call__(
730730
)
731731
gligen_phrases = gligen_phrases[:max_objs]
732732
gligen_boxes = gligen_boxes[:max_objs]
733-
# prepare batched input to the PositionNet (boxes, phrases, mask)
733+
# prepare batched input to the GLIGENTextBoundingboxProjection (boxes, phrases, mask)
734734
# Get tokens for phrases from pre-trained CLIPTokenizer
735735
tokenizer_inputs = self.tokenizer(gligen_phrases, padding=True, return_tensors="pt").to(device)
736736
# For the token, we use the same pre-trained text encoder

‎tests/models/test_models_unet_2d_condition.py‎

Lines changed: 2 additions & 2 deletions

0 commit comments

Comments
 (0)