random.seed(seed)

np.random.seed(seed)

torch.manual_seed(seed)

torch.cuda.manual_seed_all(seed)

def decorator(func):

func.name = name

func.description = description

return func

new_size = new_image.size

old_size = old_image.size

easy_img = np.array(new_image)

gt_img_array = np.array(old_image)

pos_w = (new_size[0] - old_size[0]) // 2

pos_h = (new_size[1] - old_size[1]) // 2

kernel_h = cv2.getGaussianKernel(old_size[1], old_size[1] * sigma)

kernel_w = cv2.getGaussianKernel(old_size[0], old_size[0] * sigma)

kernel = np.multiply(kernel_h, np.transpose(kernel_w))

kernel[steps:-steps, steps:-steps] = 1

kernel[:steps, :steps] = kernel[:steps, :steps] / kernel[steps - 1, steps - 1]

kernel[:steps, -steps:] = kernel[:steps, -steps:] / kernel[steps - 1, -(steps)]

kernel[-steps:, :steps] = kernel[-steps:, :steps] / kernel[-steps, steps - 1]

kernel[-steps:, -steps:] = kernel[-steps:, -steps:] / kernel[-steps, -steps]

kernel = np.expand_dims(kernel, 2)

kernel = np.repeat(kernel, 3, 2)

weight = np.linspace(0, 1, steps)

top = np.expand_dims(weight, 1)

top = np.repeat(top, old_size[0] - 2 * steps, 1)

top = np.expand_dims(top, 2)

top = np.repeat(top, 3, 2)

weight = np.linspace(1, 0, steps)

down = np.expand_dims(weight, 1)

down = np.repeat(down, old_size[0] - 2 * steps, 1)

down = np.expand_dims(down, 2)

down = np.repeat(down, 3, 2)

weight = np.linspace(0, 1, steps)

left = np.expand_dims(weight, 0)

left = np.repeat(left, old_size[1] - 2 * steps, 0)

left = np.expand_dims(left, 2)

left = np.repeat(left, 3, 2)

weight = np.linspace(1, 0, steps)

right = np.expand_dims(weight, 0)

right = np.repeat(right, old_size[1] - 2 * steps, 0)

right = np.expand_dims(right, 2)

right = np.repeat(right, 3, 2)

kernel[:steps, steps:-steps] = top

kernel[-steps:, steps:-steps] = down

kernel[steps:-steps, :steps] = left

kernel[steps:-steps, -steps:] = right

pt_gt_img = easy_img[pos_h:pos_h + old_size[1], pos_w:pos_w + old_size[0]]

gaussian_gt_img = kernel * gt_img_array + (1 - kernel) * pt_gt_img # gt img with blur img

gaussian_gt_img = gaussian_gt_img.astype(np.int64)

easy_img[pos_h:pos_h + old_size[1], pos_w:pos_w + old_size[0]] = gaussian_gt_img

gaussian_img = Image.fromarray(easy_img)

if history_memory is None or len(history_memory) == 0:

return history_memory

tokens = history_memory.split()

n_tokens = len(tokens)

print(f"history_memory:{history_memory}, n_tokens: {n_tokens}")

if n_tokens < keep_last_n_words:

return history_memory

paragraphs = history_memory.split('\n')

last_n_tokens = n_tokens

while last_n_tokens >= keep_last_n_words:

last_n_tokens -= len(paragraphs[0].split(' '))

paragraphs = paragraphs[1:]

head_tail = os.path.split(org_img_name)

head = head_tail[0]

tail = head_tail[1]

name_split = tail.split('.')[0].split('_')

this_new_uuid = str(uuid.uuid4())[:4]

if len(name_split) == 1:

most_org_file_name = name_split[0]

else:

assert len(name_split) == 4

most_org_file_name = name_split[3]

recent_prev_file_name = name_split[0]

new_file_name = f'{this_new_uuid}_{func_name}_{recent_prev_file_name}_{most_org_file_name}.png'

def __init__(self, device):

print(f"Initializing InstructPix2Pix to {device}")

self.device = device

self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32

self.pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained("timbrooks/instruct-pix2pix",

safety_checker=StableDiffusionSafetyChecker.from_pretrained('CompVis/stable-diffusion-safety-checker'),

torch_dtype=self.torch_dtype).to(device)

self.pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(self.pipe.scheduler.config)

@prompts(name="Instruct Image Using Text",

def inference(self, inputs):

"""Change style of image."""

print("===>Starting InstructPix2Pix Inference")

image_path, text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])

original_image = Image.open(image_path)

image = self.pipe(text, image=original_image, num_inference_steps=40, image_guidance_scale=1.2).images[0]

updated_image_path = get_new_image_name(image_path, func_name="pix2pix")

image.save(updated_image_path)

print(f"\nProcessed InstructPix2Pix, Input Image: {image_path}, Instruct Text: {text}, "

f"Output Image: {updated_image_path}")

def __init__(self, device):

print(f"Initializing Text2Image to {device}")

self.device = device

self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32

self.pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5",

torch_dtype=self.torch_dtype)

self.pipe.to(device)

self.a_prompt = 'best quality, extremely detailed'

self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, ' \

'fewer digits, cropped, worst quality, low quality'

@prompts(name="Generate Image From User Input Text",

def inference(self, text):

image_filename = os.path.join('image', f"{str(uuid.uuid4())[:8]}.png")

prompt = text + ', ' + self.a_prompt

image = self.pipe(prompt, negative_prompt=self.n_prompt).images[0]

image.save(image_filename)

print(

f"\nProcessed Text2Image, Input Text: {text}, Output Image: {image_filename}")

def __init__(self, device):

print(f"Initializing ImageCaptioning to {device}")

self.device = device

self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32

self.processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")

self.model = BlipForConditionalGeneration.from_pretrained(

"Salesforce/blip-image-captioning-base", torch_dtype=self.torch_dtype).to(self.device)

@prompts(name="Get Photo Description",

def inference(self, image_path):

inputs = self.processor(Image.open(image_path), return_tensors="pt").to(self.device, self.torch_dtype)

out = self.model.generate(**inputs)

captions = self.processor.decode(out[0], skip_special_tokens=True)

print(f"\nProcessed ImageCaptioning, Input Image: {image_path}, Output Text: {captions}")

def __init__(self, device):

print("Initializing Image2Canny")

self.low_threshold = 100

self.high_threshold = 200

@prompts(name="Edge Detection On Image",

def inference(self, inputs):

image = Image.open(inputs)

image = np.array(image)

canny = cv2.Canny(image, self.low_threshold, self.high_threshold)

canny = canny[:, :, None]

canny = np.concatenate([canny, canny, canny], axis=2)

canny = Image.fromarray(canny)

updated_image_path = get_new_image_name(inputs, func_name="edge")

canny.save(updated_image_path)

print(f"\nProcessed Image2Canny, Input Image: {inputs}, Output Text: {updated_image_path}")

def __init__(self, device):

print(f"Initializing CannyText2Image to {device}")

self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32

self.controlnet = ControlNetModel.from_pretrained("fusing/stable-diffusion-v1-5-controlnet-canny",

torch_dtype=self.torch_dtype)

self.pipe = StableDiffusionControlNetPipeline.from_pretrained(

"runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=StableDiffusionSafetyChecker.from_pretrained('CompVis/stable-diffusion-safety-checker'),

torch_dtype=self.torch_dtype)

self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)

self.pipe.to(device)

self.seed = -1

self.a_prompt = 'best quality, extremely detailed'

self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, ' \

'fewer digits, cropped, worst quality, low quality'

@prompts(name="Generate Image Condition On Canny Image",

def inference(self, inputs):

image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])

image = Image.open(image_path)

self.seed = random.randint(0, 65535)

seed_everything(self.seed)

prompt = f'{instruct_text}, {self.a_prompt}'

image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,

guidance_scale=9.0).images[0]

updated_image_path = get_new_image_name(image_path, func_name="canny2image")

image.save(updated_image_path)

print(f"\nProcessed CannyText2Image, Input Canny: {image_path}, Input Text: {instruct_text}, "

f"Output Text: {updated_image_path}")

def __init__(self, device):

print("Initializing Image2Line")

self.detector = MLSDdetector.from_pretrained('lllyasviel/ControlNet')

@prompts(name="Line Detection On Image",

def inference(self, inputs):

image = Image.open(inputs)

mlsd = self.detector(image)

updated_image_path = get_new_image_name(inputs, func_name="line-of")

mlsd.save(updated_image_path)

print(f"\nProcessed Image2Line, Input Image: {inputs}, Output Line: {updated_image_path}")

def __init__(self, device):

print(f"Initializing LineText2Image to {device}")

self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32

self.controlnet = ControlNetModel.from_pretrained("fusing/stable-diffusion-v1-5-controlnet-mlsd",

torch_dtype=self.torch_dtype)

self.pipe = StableDiffusionControlNetPipeline.from_pretrained(

"runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=StableDiffusionSafetyChecker.from_pretrained('CompVis/stable-diffusion-safety-checker'),

torch_dtype=self.torch_dtype

)

self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)

self.pipe.to(device)

self.seed = -1

self.a_prompt = 'best quality, extremely detailed'

self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, ' \

'fewer digits, cropped, worst quality, low quality'

@prompts(name="Generate Image Condition On Line Image",

def inference(self, inputs):

image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])

image = Image.open(image_path)

self.seed = random.randint(0, 65535)

seed_everything(self.seed)

prompt = f'{instruct_text}, {self.a_prompt}'

image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,

guidance_scale=9.0).images[0]

updated_image_path = get_new_image_name(image_path, func_name="line2image")

image.save(updated_image_path)

print(f"\nProcessed LineText2Image, Input Line: {image_path}, Input Text: {instruct_text}, "

f"Output Text: {updated_image_path}")

def __init__(self, device):

print("Initializing Image2Hed")

self.detector = HEDdetector.from_pretrained('lllyasviel/ControlNet')

@prompts(name="Hed Detection On Image",

def inference(self, inputs):

image = Image.open(inputs)

hed = self.detector(image)

updated_image_path = get_new_image_name(inputs, func_name="hed-boundary")

hed.save(updated_image_path)

print(f"\nProcessed Image2Hed, Input Image: {inputs}, Output Hed: {updated_image_path}")

def __init__(self, device):

print(f"Initializing HedText2Image to {device}")

self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32

self.controlnet = ControlNetModel.from_pretrained("fusing/stable-diffusion-v1-5-controlnet-hed",

torch_dtype=self.torch_dtype)

self.pipe = StableDiffusionControlNetPipeline.from_pretrained(

"runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=StableDiffusionSafetyChecker.from_pretrained('CompVis/stable-diffusion-safety-checker'),

torch_dtype=self.torch_dtype

)

self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)

self.pipe.to(device)

self.seed = -1

self.a_prompt = 'best quality, extremely detailed'

self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, ' \

'fewer digits, cropped, worst quality, low quality'

@prompts(name="Generate Image Condition On Soft Hed Boundary Image",

def inference(self, inputs):

image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])

image = Image.open(image_path)

self.seed = random.randint(0, 65535)

seed_everything(self.seed)

prompt = f'{instruct_text}, {self.a_prompt}'

image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,

guidance_scale=9.0).images[0]

updated_image_path = get_new_image_name(image_path, func_name="hed2image")

image.save(updated_image_path)

print(f"\nProcessed HedText2Image, Input Hed: {image_path}, Input Text: {instruct_text}, "

f"Output Image: {updated_image_path}")

def __init__(self, device):

print("Initializing Image2Scribble")

self.detector = HEDdetector.from_pretrained('lllyasviel/ControlNet')

@prompts(name="Sketch Detection On Image",

def inference(self, inputs):

image = Image.open(inputs)

scribble = self.detector(image, scribble=True)

updated_image_path = get_new_image_name(inputs, func_name="scribble")

scribble.save(updated_image_path)

print(f"\nProcessed Image2Scribble, Input Image: {inputs}, Output Scribble: {updated_image_path}")

def __init__(self, device):

print(f"Initializing ScribbleText2Image to {device}")

self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32

self.controlnet = ControlNetModel.from_pretrained("fusing/stable-diffusion-v1-5-controlnet-scribble",

torch_dtype=self.torch_dtype)

self.pipe = StableDiffusionControlNetPipeline.from_pretrained(

"runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=StableDiffusionSafetyChecker.from_pretrained('CompVis/stable-diffusion-safety-checker'),

torch_dtype=self.torch_dtype

)

self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)

self.pipe.to(device)

self.seed = -1

self.a_prompt = 'best quality, extremely detailed'

self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, ' \

'fewer digits, cropped, worst quality, low quality'

@prompts(name="Generate Image Condition On Sketch Image",

def inference(self, inputs):

image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])

image = Image.open(image_path)

self.seed = random.randint(0, 65535)

seed_everything(self.seed)

prompt = f'{instruct_text}, {self.a_prompt}'

image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,

guidance_scale=9.0).images[0]

updated_image_path = get_new_image_name(image_path, func_name="scribble2image")

image.save(updated_image_path)

print(f"\nProcessed ScribbleText2Image, Input Scribble: {image_path}, Input Text: {instruct_text}, "

f"Output Image: {updated_image_path}")

def __init__(self, device):

print("Initializing Image2Pose")

self.detector = OpenposeDetector.from_pretrained('lllyasviel/ControlNet')

@prompts(name="Pose Detection On Image",

def inference(self, inputs):

image = Image.open(inputs)

pose = self.detector(image)

updated_image_path = get_new_image_name(inputs, func_name="human-pose")

pose.save(updated_image_path)

print(f"\nProcessed Image2Pose, Input Image: {inputs}, Output Pose: {updated_image_path}")

def __init__(self, device):

print(f"Initializing PoseText2Image to {device}")

self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32

self.controlnet = ControlNetModel.from_pretrained("fusing/stable-diffusion-v1-5-controlnet-openpose",

torch_dtype=self.torch_dtype)

self.pipe = StableDiffusionControlNetPipeline.from_pretrained(

"runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=StableDiffusionSafetyChecker.from_pretrained('CompVis/stable-diffusion-safety-checker'),

torch_dtype=self.torch_dtype)

self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)

self.pipe.to(device)

self.num_inference_steps = 20

self.seed = -1

self.unconditional_guidance_scale = 9.0

self.a_prompt = 'best quality, extremely detailed'

self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit,' \

' fewer digits, cropped, worst quality, low quality'

@prompts(name="Generate Image Condition On Pose Image",

def inference(self, inputs):

image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])

image = Image.open(image_path)

self.seed = random.randint(0, 65535)

seed_everything(self.seed)

prompt = f'{instruct_text}, {self.a_prompt}'

image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,

guidance_scale=9.0).images[0]

updated_image_path = get_new_image_name(image_path, func_name="pose2image")

image.save(updated_image_path)

print(f"\nProcessed PoseText2Image, Input Pose: {image_path}, Input Text: {instruct_text}, "

f"Output Image: {updated_image_path}")

def __init__(self, device):

print(f"Initializing SegText2Image to {device}")

self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32

self.controlnet = ControlNetModel.from_pretrained("fusing/stable-diffusion-v1-5-controlnet-seg",

torch_dtype=self.torch_dtype)

self.pipe = StableDiffusionControlNetPipeline.from_pretrained(

"runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=StableDiffusionSafetyChecker.from_pretrained('CompVis/stable-diffusion-safety-checker'),

torch_dtype=self.torch_dtype)

self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)

self.pipe.to(device)

self.seed = -1

self.a_prompt = 'best quality, extremely detailed'

self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit,' \

' fewer digits, cropped, worst quality, low quality'

@prompts(name="Generate Image Condition On Segmentations",

def inference(self, inputs):

image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])

image = Image.open(image_path)

self.seed = random.randint(0, 65535)

seed_everything(self.seed)

prompt = f'{instruct_text}, {self.a_prompt}'

image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,

guidance_scale=9.0).images[0]

updated_image_path = get_new_image_name(image_path, func_name="segment2image")

image.save(updated_image_path)

print(f"\nProcessed SegText2Image, Input Seg: {image_path}, Input Text: {instruct_text}, "

f"Output Image: {updated_image_path}")

def __init__(self, device):

print("Initializing Image2Depth")

self.depth_estimator = pipeline('depth-estimation')

@prompts(name="Predict Depth On Image",

def inference(self, inputs):

image = Image.open(inputs)

depth = self.depth_estimator(image)['depth']

depth = np.array(depth)

depth = depth[:, :, None]

depth = np.concatenate([depth, depth, depth], axis=2)

depth = Image.fromarray(depth)

updated_image_path = get_new_image_name(inputs, func_name="depth")

depth.save(updated_image_path)

print(f"\nProcessed Image2Depth, Input Image: {inputs}, Output Depth: {updated_image_path}")

def __init__(self, device):

print(f"Initializing DepthText2Image to {device}")

self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32

self.controlnet = ControlNetModel.from_pretrained(

"fusing/stable-diffusion-v1-5-controlnet-depth", torch_dtype=self.torch_dtype)

self.pipe = StableDiffusionControlNetPipeline.from_pretrained(

"runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=StableDiffusionSafetyChecker.from_pretrained('CompVis/stable-diffusion-safety-checker'),

torch_dtype=self.torch_dtype)

self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)

self.pipe.to(device)

self.seed = -1

self.a_prompt = 'best quality, extremely detailed'

self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit,' \

' fewer digits, cropped, worst quality, low quality'

@prompts(name="Generate Image Condition On Depth",

def inference(self, inputs):

image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])

image = Image.open(image_path)

self.seed = random.randint(0, 65535)

seed_everything(self.seed)

prompt = f'{instruct_text}, {self.a_prompt}'

image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,

guidance_scale=9.0).images[0]

updated_image_path = get_new_image_name(image_path, func_name="depth2image")

image.save(updated_image_path)

print(f"\nProcessed DepthText2Image, Input Depth: {image_path}, Input Text: {instruct_text}, "

f"Output Image: {updated_image_path}")

def __init__(self, device):

print("Initializing Image2Normal")

self.depth_estimator = pipeline("depth-estimation", model="Intel/dpt-hybrid-midas")

self.bg_threhold = 0.4

@prompts(name="Predict Normal Map On Image",

def inference(self, inputs):

image = Image.open(inputs)

original_size = image.size

image = self.depth_estimator(image)['predicted_depth'][0]

image = image.numpy()

image_depth = image.copy()

image_depth -= np.min(image_depth)

image_depth /= np.max(image_depth)

x = cv2.Sobel(image, cv2.CV_32F, 1, 0, ksize=3)

x[image_depth < self.bg_threhold] = 0

y = cv2.Sobel(image, cv2.CV_32F, 0, 1, ksize=3)

y[image_depth < self.bg_threhold] = 0

z = np.ones_like(x) * np.pi * 2.0

image = np.stack([x, y, z], axis=2)

image /= np.sum(image ** 2.0, axis=2, keepdims=True) ** 0.5

image = (image * 127.5 + 127.5).clip(0, 255).astype(np.uint8)

image = Image.fromarray(image)

image = image.resize(original_size)

updated_image_path = get_new_image_name(inputs, func_name="normal-map")

image.save(updated_image_path)

print(f"\nProcessed Image2Normal, Input Image: {inputs}, Output Depth: {updated_image_path}")

def __init__(self, device):

print(f"Initializing NormalText2Image to {device}")

self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32

self.controlnet = ControlNetModel.from_pretrained(

"fusing/stable-diffusion-v1-5-controlnet-normal", torch_dtype=self.torch_dtype)

self.pipe = StableDiffusionControlNetPipeline.from_pretrained(

"runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=StableDiffusionSafetyChecker.from_pretrained('CompVis/stable-diffusion-safety-checker'),

torch_dtype=self.torch_dtype)

self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)

self.pipe.to(device)

self.seed = -1

self.a_prompt = 'best quality, extremely detailed'

self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit,' \

' fewer digits, cropped, worst quality, low quality'

@prompts(name="Generate Image Condition On Normal Map",

def inference(self, inputs):

image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])

image = Image.open(image_path)

self.seed = random.randint(0, 65535)

seed_everything(self.seed)

prompt = f'{instruct_text}, {self.a_prompt}'

image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,

guidance_scale=9.0).images[0]

updated_image_path = get_new_image_name(image_path, func_name="normal2image")

image.save(updated_image_path)

print(f"\nProcessed NormalText2Image, Input Normal: {image_path}, Input Text: {instruct_text}, "

f"Output Image: {updated_image_path}")

def __init__(self, device):

print(f"Initializing VisualQuestionAnswering to {device}")

self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32

self.device = device

self.processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")

self.model = BlipForQuestionAnswering.from_pretrained(

"Salesforce/blip-vqa-base", torch_dtype=self.torch_dtype).to(self.device)

@prompts(name="Answer Question About The Image",

def inference(self, inputs):

image_path, question = inputs.split(",")[0], ','.join(inputs.split(',')[1:])

raw_image = Image.open(image_path).convert('RGB')

inputs = self.processor(raw_image, question, return_tensors="pt").to(self.device, self.torch_dtype)

out = self.model.generate(**inputs)

answer = self.processor.decode(out[0], skip_special_tokens=True)

print(f"\nProcessed VisualQuestionAnswering, Input Image: {image_path}, Input Question: {question}, "

f"Output Answer: {answer}")

def __init__(self, device):

print(f"Inintializing Segmentation to {device}")

self.device = device

self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32

self.model_checkpoint_path = os.path.join("checkpoints","sam")

self.download_parameters()

self.sam = build_sam(checkpoint=self.model_checkpoint_path).to(device)

self.sam_predictor = SamPredictor(self.sam)

self.mask_generator = SamAutomaticMaskGenerator(self.sam)

self.saved_points = []

self.saved_labels = []

def download_parameters(self):

url = "https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth"

if not os.path.exists(self.model_checkpoint_path):

wget.download(url,out=self.model_checkpoint_path)

def show_mask(self, mask: np.ndarray,image: np.ndarray,

random_color: bool = False, transparency=1) -> np.ndarray:

"""Visualize a mask on top of an image.

if random_color:

color = np.concatenate([np.random.random(3)], axis=0)

else:

color = np.array([30 / 255, 144 / 255, 255 / 255])

h, w = mask.shape[-2:]

mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1) * 255

image = cv2.addWeighted(image, 0.7, mask_image.astype('uint8'), transparency, 0)

return image

def show_box(self, box, ax, label):

x0, y0 = box[0], box[1]

w, h = box[2] - box[0], box[3] - box[1]

ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor='green', facecolor=(0,0,0,0), lw=2))

ax.text(x0, y0, label)

def get_mask_with_boxes(self, image_pil, image, boxes_filt):

size = image_pil.size

H, W = size[1], size[0]

for i in range(boxes_filt.size(0)):

boxes_filt[i] = boxes_filt[i] * torch.Tensor([W, H, W, H])

boxes_filt[i][:2] -= boxes_filt[i][2:] / 2

boxes_filt[i][2:] += boxes_filt[i][:2]

boxes_filt = boxes_filt.cpu()

transformed_boxes = self.sam_predictor.transform.apply_boxes_torch(boxes_filt, image.shape[:2]).to(self.device)

masks, _, _ = self.sam_predictor.predict_torch(

point_coords = None,

point_labels = None,

boxes = transformed_boxes.to(self.device),

multimask_output = False,

)

return masks

def segment_image_with_boxes(self, image_pil, image_path, boxes_filt, pred_phrases):