FilesExpand file tree

main

/

stable_diffusion_pipeline_custom_call.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

110 lines (95 loc) · 4.1 KB

main

/

stable_diffusion_pipeline_custom_call.py

Copy path

Top

File metadata and controls

• Code
• Blame

110 lines (95 loc) · 4.1 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

import torch

from typing import Any, Callable, Dict, List, Optional, Union

from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import rescale_noise_cfg, StableDiffusionPipelineOutput

@torch.no_grad()

def StableDiffusionPipeline__call__WithCustomDenoising(

self,

denoise: Callable,

prompt: Union[str, List[str]] = None,

height: Optional[int] = None,

width: Optional[int] = None,

num_inference_steps: int = 50,

guidance_scale: float = 7.5,

negative_prompt: Optional[Union[str, List[str]]] = None,

num_images_per_prompt: Optional[int] = 1,

eta: float = 0.75,

generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,

latents: Optional[torch.FloatTensor] = None,

prompt_embeds: Optional[torch.FloatTensor] = None,

negative_prompt_embeds: Optional[torch.FloatTensor] = None,

output_type: Optional[str] = "pil",

return_dict: bool = True,

callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,

callback_steps: int = 1,

cross_attention_kwargs: Optional[Dict[str, Any]] = None,

guidance_rescale: float = 0.0,

):

# 0. Default height and width to unet

height = height or self.unet.config.sample_size * self.vae_scale_factor

width = width or self.unet.config.sample_size * self.vae_scale_factor

# 1. Check inputs. Raise error if not correct

self.check_inputs(

prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds

)

# 2. Define call parameters

if prompt is not None and isinstance(prompt, str):

batch_size = 1

elif prompt is not None and isinstance(prompt, list):

batch_size = len(prompt)

else:

batch_size = prompt_embeds.shape[0]

device = self._execution_device

# here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)

# of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`

# corresponds to doing no classifier free guidance.

do_classifier_free_guidance = guidance_scale > 1.0

# 3. Encode input prompt

text_encoder_lora_scale = (

cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None

)

prompt_embeds = self._encode_prompt(

prompt,

device,

num_images_per_prompt,

do_classifier_free_guidance,

negative_prompt,

prompt_embeds=prompt_embeds,

negative_prompt_embeds=negative_prompt_embeds,

lora_scale=text_encoder_lora_scale,

)

# 4. Prepare timesteps

## self.scheduler.set_timesteps(num_inference_steps, device=device)

## timesteps = self.scheduler.timesteps

# 5. Prepare latent variables

num_channels_latents = self.unet.config.in_channels

latents = self.prepare_latents(

batch_size * num_images_per_prompt,

num_channels_latents,

height,

width,

prompt_embeds.dtype,

device,

generator,

latents,

)

# 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline

## extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)

# 7. Denoising loop

latents = denoise(latents, prompt_embeds, callback, callback_steps, width, height)

if not output_type == "latent":

image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]

image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)

else:

image = latents

has_nsfw_concept = None

if has_nsfw_concept is None:

do_denormalize = [True] * image.shape[0]

else:

do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept]

image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)

# Offload last model to CPU

if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:

self.final_offload_hook.offload()

if not return_dict:

return (image, has_nsfw_concept)

return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)