|
| 1 | +import os |
| 2 | +import torch |
| 3 | + |
| 4 | +from comfy import sd1_clip |
| 5 | +import comfy.model_management |
| 6 | +from .gemma import Gemma2Model |
| 7 | + |
| 8 | +from transformers import GemmaTokenizer as TFGemmaTokenizer |
| 9 | + |
| 10 | +class GemmaClipModel(sd1_clip.SDClipModel): |
| 11 | + def __init__(self, device="cpu", dtype=None, model_options={}, **kwargs): |
| 12 | + textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json") |
| 13 | + special_tokens = {"start": 2, "end": 1, "pad": 0} |
| 14 | + super().__init__(device=device, layer="last", layer_idx=None, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens=special_tokens, model_class=Gemma2Model, enable_attention_masks=True, return_attention_masks=False, model_options=model_options) |
| 15 | + |
| 16 | +class SanaClipModel(sd1_clip.SD1ClipModel): |
| 17 | + def __init__(self, device="cpu", dtype=None, model_options={}): |
| 18 | + super().__init__(device=device, dtype=dtype, name="gemma", clip_model=GemmaClipModel, model_options=model_options) |
| 19 | + |
| 20 | +class GemmaTokenizer(sd1_clip.SDTokenizer): |
| 21 | + def __init__(self, embedding_directory=None, tokenizer_data={}): |
| 22 | + tokenizer_path = os.path.join( |
| 23 | + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), |
| 24 | + "tokenizers", "gemma_tokenizer", |
| 25 | + ) |
| 26 | + # TODO: reenable proper logic here - needs comfy version 44db978 or higher |
| 27 | + super().__init__(tokenizer_path, embedding_directory=embedding_directory, pad_with_end=False, embedding_size=2304, embedding_key='gemma', tokenizer_class=TFGemmaTokenizer, has_start_token=False, pad_to_max_length=True, max_length=300, min_length=1) |
| 28 | + self.start_token = 2 |
| 29 | + self.end_token = 1 |
| 30 | + self.pad_token = 0 |
| 31 | + |
| 32 | + def tokenize_with_weights(self, text, return_word_ids=False): |
| 33 | + # TODO: see above, this is still just a wrapper for now |
| 34 | + tokens = self.tokenizer( |
| 35 | + text, |
| 36 | + max_length=300, |
| 37 | + padding="max_length", |
| 38 | + truncation=True, |
| 39 | + return_tensors="pt" |
| 40 | + ) |
| 41 | + batched_tokens = [(x.item(), 1.0) for x in tokens.input_ids[0]] |
| 42 | + return [batched_tokens] |
| 43 | + |
| 44 | +class SanaTokenizer(sd1_clip.SD1Tokenizer): |
| 45 | + def __init__(self, embedding_directory=None, tokenizer_data={}): |
| 46 | + super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, clip_name="gemma", tokenizer=GemmaTokenizer) |
0 commit comments