|
14 | 14 | # limitations under the License. |
15 | 15 |
|
16 | 16 | import gc |
17 | | -import random |
18 | | -import unittest |
19 | 17 |
|
20 | | -import numpy as np |
21 | 18 | import pytest |
22 | 19 | import torch |
23 | 20 | from transformers import XLMRobertaTokenizerFast |
24 | 21 |
|
25 | 22 | from diffusers import DDIMScheduler, KandinskyPipeline, KandinskyPriorPipeline, UNet2DConditionModel, VQModel |
26 | 23 | from diffusers.pipelines.kandinsky.text_encoder import MCLIPConfig, MultilingualCLIP |
27 | | -from diffusers.utils import is_transformers_version |
28 | 24 |
|
29 | 25 | from ...testing_utils import ( |
| 26 | + assert_tensors_close, |
30 | 27 | backend_empty_cache, |
31 | 28 | enable_full_determinism, |
32 | | - floats_tensor, |
33 | 29 | load_numpy, |
34 | 30 | require_torch_accelerator, |
35 | 31 | slow, |
36 | 32 | torch_device, |
37 | 33 | ) |
38 | | -from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference |
| 34 | +from ..test_pipelines_common import assert_mean_pixel_difference |
| 35 | +from ..testing_utils import ( |
| 36 | + BasePipelineTesterConfig, |
| 37 | + MemoryTesterMixin, |
| 38 | + PipelineTesterMixin, |
| 39 | +) |
39 | 40 |
|
40 | 41 |
|
41 | 42 | enable_full_determinism() |
42 | 43 |
|
43 | 44 |
|
44 | | -class Dummies: |
| 45 | +class KandinskyPipelineTesterConfig(BasePipelineTesterConfig): |
| 46 | + pipeline_class = KandinskyPipeline |
| 47 | + required_input_params_in_call_signature = frozenset(["prompt", "image_embeds", "negative_image_embeds"]) |
| 48 | + batch_input_params = frozenset(["prompt", "negative_prompt", "image_embeds", "negative_image_embeds"]) |
| 49 | + output_shape = (3, 64, 64) |
| 50 | + |
45 | 51 | @property |
46 | 52 | def text_embedder_hidden_size(self): |
47 | 53 | return 32 |
@@ -162,136 +168,67 @@ def get_dummy_components(self): |
162 | 168 | } |
163 | 169 | return components |
164 | 170 |
|
165 | | - def get_dummy_inputs(self, device, seed=0): |
166 | | - image_embeds = floats_tensor((1, self.cross_attention_dim), rng=random.Random(seed)).to(device) |
167 | | - negative_image_embeds = floats_tensor((1, self.cross_attention_dim), rng=random.Random(seed + 1)).to(device) |
168 | | - if str(device).startswith("mps"): |
169 | | - generator = torch.manual_seed(seed) |
170 | | - else: |
171 | | - generator = torch.Generator(device=device).manual_seed(seed) |
172 | | - inputs = { |
| 171 | + def get_dummy_inputs(self): |
| 172 | + image_embeds = torch.randn((1, self.cross_attention_dim), generator=self.get_generator(0)).to(torch_device) |
| 173 | + negative_image_embeds = torch.randn((1, self.cross_attention_dim), generator=self.get_generator(1)).to( |
| 174 | + torch_device |
| 175 | + ) |
| 176 | + return { |
173 | 177 | "prompt": "horse", |
174 | 178 | "image_embeds": image_embeds, |
175 | 179 | "negative_image_embeds": negative_image_embeds, |
176 | | - "generator": generator, |
| 180 | + "generator": self.get_generator(0), |
177 | 181 | "height": 64, |
178 | 182 | "width": 64, |
179 | 183 | "guidance_scale": 4.0, |
180 | 184 | "num_inference_steps": 2, |
181 | | - "output_type": "np", |
| 185 | + # Request torch outputs so tests compare torch tensors directly (see `BasePipelineTesterConfig`). |
| 186 | + # Note `"pt"` images are `(batch, channels, height, width)`, unlike `"np"` (`(batch, h, w, c)`). |
| 187 | + "output_type": "pt", |
182 | 188 | } |
183 | | - return inputs |
184 | 189 |
|
185 | 190 |
|
186 | | -class KandinskyPipelineFastTests(PipelineTesterMixin, unittest.TestCase): |
187 | | - pipeline_class = KandinskyPipeline |
188 | | - params = [ |
189 | | - "prompt", |
190 | | - "image_embeds", |
191 | | - "negative_image_embeds", |
192 | | - ] |
193 | | - batch_params = ["prompt", "negative_prompt", "image_embeds", "negative_image_embeds"] |
194 | | - required_optional_params = [ |
195 | | - "generator", |
196 | | - "height", |
197 | | - "width", |
198 | | - "latents", |
199 | | - "guidance_scale", |
200 | | - "negative_prompt", |
201 | | - "num_inference_steps", |
202 | | - "return_dict", |
203 | | - "guidance_scale", |
204 | | - "num_images_per_prompt", |
205 | | - "output_type", |
206 | | - "return_dict", |
207 | | - ] |
208 | | - test_xformers_attention = False |
| 191 | +class TestKandinskyPipeline(KandinskyPipelineTesterConfig, PipelineTesterMixin): |
| 192 | + def test_inference_batch_single_identical(self, batch_size=3, expected_max_diff=1e-1): |
| 193 | + # Batched inference is only approximately equal to single inference here: the batch pads to the longest |
| 194 | + # prompt and the tiny 2-step denoising loop amplifies the resulting attention differences. Tolerance set |
| 195 | + # from the measured drift. |
| 196 | + super().test_inference_batch_single_identical(batch_size=batch_size, expected_max_diff=expected_max_diff) |
209 | 197 |
|
210 | | - def get_dummy_components(self): |
211 | | - dummy = Dummies() |
212 | | - return dummy.get_dummy_components() |
213 | | - |
214 | | - def get_dummy_inputs(self, device, seed=0): |
215 | | - dummy = Dummies() |
216 | | - return dummy.get_dummy_inputs(device=device, seed=seed) |
217 | | - |
218 | | - @pytest.mark.xfail( |
219 | | - condition=is_transformers_version(">=", "4.56.2"), |
220 | | - reason="Latest transformers changes the slices", |
221 | | - strict=False, |
222 | | - ) |
223 | 198 | def test_kandinsky(self): |
224 | | - device = "cpu" |
225 | | - |
226 | | - components = self.get_dummy_components() |
| 199 | + # Run on CPU: the expected slice below is CPU-specific. |
| 200 | + pipe = self.get_pipeline() |
227 | 201 |
|
228 | | - pipe = self.pipeline_class(**components) |
229 | | - pipe = pipe.to(device) |
| 202 | + image = pipe(**self.get_dummy_inputs()).images |
| 203 | + image_from_tuple = pipe(**self.get_dummy_inputs(), return_dict=False)[0] |
230 | 204 |
|
231 | | - pipe.set_progress_bar_config(disable=None) |
| 205 | + assert image.shape == (1, *self.output_shape) |
232 | 206 |
|
233 | | - output = pipe(**self.get_dummy_inputs(device)) |
234 | | - image = output.images |
| 207 | + # This pipeline only denormalizes the decoded image for `output_type` "np"/"pil", so `"pt"` hands back the |
| 208 | + # raw decoder output. Map it into the [0, 1] range the expected slice below was recorded in. |
| 209 | + image = (image * 0.5 + 0.5).clamp(0, 1) |
| 210 | + image_from_tuple = (image_from_tuple * 0.5 + 0.5).clamp(0, 1) |
235 | 211 |
|
236 | | - image_from_tuple = pipe( |
237 | | - **self.get_dummy_inputs(device), |
238 | | - return_dict=False, |
239 | | - )[0] |
| 212 | + # fmt: off |
| 213 | + expected_slice = torch.tensor([0.4428, 0.7424, 0.3413, 1.0000, 0.7061, 0.3452, 0.5017, 0.3987, 0.5046]) |
| 214 | + # fmt: on |
| 215 | + assert_tensors_close(image[0, -1, -3:, -3:].flatten(), expected_slice, atol=1e-2) |
| 216 | + assert_tensors_close(image_from_tuple[0, -1, -3:, -3:].flatten(), expected_slice, atol=1e-2) |
240 | 217 |
|
241 | | - image_slice = image[0, -3:, -3:, -1] |
242 | | - image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] |
243 | 218 |
|
244 | | - assert image.shape == (1, 64, 64, 3) |
245 | | - |
246 | | - expected_slice = np.array([1.0000, 1.0000, 0.2766, 1.0000, 0.5447, 0.1737, 1.0000, 0.4316, 0.9024]) |
247 | | - |
248 | | - assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2, ( |
249 | | - f" expected_slice {expected_slice}, but got {image_slice.flatten()}" |
250 | | - ) |
251 | | - assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2, ( |
252 | | - f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" |
253 | | - ) |
254 | | - |
255 | | - @require_torch_accelerator |
256 | | - def test_offloads(self): |
257 | | - pipes = [] |
258 | | - components = self.get_dummy_components() |
259 | | - sd_pipe = self.pipeline_class(**components).to(torch_device) |
260 | | - pipes.append(sd_pipe) |
261 | | - |
262 | | - components = self.get_dummy_components() |
263 | | - sd_pipe = self.pipeline_class(**components) |
264 | | - sd_pipe.enable_model_cpu_offload(device=torch_device) |
265 | | - pipes.append(sd_pipe) |
266 | | - |
267 | | - components = self.get_dummy_components() |
268 | | - sd_pipe = self.pipeline_class(**components) |
269 | | - sd_pipe.enable_sequential_cpu_offload(device=torch_device) |
270 | | - pipes.append(sd_pipe) |
271 | | - |
272 | | - image_slices = [] |
273 | | - for pipe in pipes: |
274 | | - inputs = self.get_dummy_inputs(torch_device) |
275 | | - image = pipe(**inputs).images |
276 | | - |
277 | | - image_slices.append(image[0, -3:, -3:, -1].flatten()) |
278 | | - |
279 | | - assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 |
280 | | - assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 |
| 219 | +class TestKandinskyPipelineMemory(KandinskyPipelineTesterConfig, MemoryTesterMixin): |
| 220 | + """Memory optimization tests (CPU offload, group offload, layerwise casting) for the Kandinsky pipeline.""" |
281 | 221 |
|
282 | 222 |
|
283 | 223 | @slow |
284 | 224 | @require_torch_accelerator |
285 | | -class KandinskyPipelineIntegrationTests(unittest.TestCase): |
286 | | - def setUp(self): |
287 | | - # clean up the VRAM before each test |
288 | | - super().setUp() |
| 225 | +class TestKandinskyPipelineIntegration: |
| 226 | + @pytest.fixture(autouse=True) |
| 227 | + def cleanup(self): |
| 228 | + # clean up the VRAM before and after each test |
289 | 229 | gc.collect() |
290 | 230 | backend_empty_cache(torch_device) |
291 | | - |
292 | | - def tearDown(self): |
293 | | - # clean up the VRAM after each test |
294 | | - super().tearDown() |
| 231 | + yield |
295 | 232 | gc.collect() |
296 | 233 | backend_empty_cache(torch_device) |
297 | 234 |
|
|
0 commit comments