Layer split: size device ranges in proportion to capacity and keep co… · unslothai/stable-diffusion.cpp@11761ef · GitHub
Skip to content

Commit 11761ef

Browse files
Layer split: size device ranges in proportion to capacity and keep compute headroom under --max-vram (#32)
The partitioner filled the first device up to its weight capacity before using the next one, so a module whose weights fit the main device alone was never split, and the compute buffers then had no room under the budget. The 2 GiB compute headroom was also taken only from free memory, never from an explicit --max-vram budget, so a per-device budget was filled entirely with weights. Ranges are now sized in proportion to each device's capacity (the lower of free memory and its --max-vram budget, minus the headroom), as docs/backend.md already described, with the fill-first plan kept as a fallback when the proportional ranges do not fit. The planner is a header-only function with a standalone test (scripts/layer_split_ranges_test.cpp).
1 parent 6c29ce2 commit 11761ef

4 files changed

Lines changed: 250 additions & 23 deletions

File tree

‎docs/backend.md‎

Lines changed: 6 additions & 2 deletions
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
// Standalone check of the layer-split range planner: g++ -std=c++17 -I src scripts/layer_split_ranges_test.cpp && ./a.out
2+
#include <cstdio>
3+
#include <cstdlib>
4+
5+
#include "core/layer_split_ranges.h"
6+
7+
static int failures = 0;
8+
#define CHECK(cond) \
9+
do { \
10+
if (!(cond)) { \
11+
std::fprintf(stderr, "FAIL %s:%d %s\n", __FILE__, __LINE__, #cond); \
12+
failures++; \
13+
} \
14+
} while (0)
15+
16+
static bool contiguous(const sd::LayerSplitRangesResult& r) {
17+
for (size_t i = 1; i < r.device_by_segment.size(); i++) {
18+
if (r.device_by_segment[i] < r.device_by_segment[i - 1]) {
19+
return false;
20+
}
21+
}
22+
return true;
23+
}
24+
25+
int main() {
26+
const int64_t MB = 1024ll * 1024;
27+
// 54 equal segments, 10.98 GB total, two cards with 12.6 GB weight capacity each (16 GB card budget less headroom).
28+
std::vector<int64_t> segs(54, (int64_t)(10.98 * 1024 * MB / 54));
29+
std::vector<int64_t> caps = {12599 * MB, 12599 * MB};
30+
auto fill = sd::plan_layer_split_ranges_fill_first(segs, caps);
31+
CHECK(fill.ok);
32+
CHECK(fill.bytes_by_device[1] == 0); // the old plan: everything on card 0
33+
auto prop = sd::plan_layer_split_ranges(segs, caps);
34+
CHECK(prop.ok);
35+
CHECK(!prop.used_fill_first);
36+
CHECK(contiguous(prop));
37+
CHECK(prop.device_by_segment.front() == 0 && prop.device_by_segment.back() == 1);
38+
CHECK(std::llabs(prop.bytes_by_device[0] - prop.bytes_by_device[1]) <= segs[0]);
39+
40+
// Unequal capacities: shares follow the capacities.
41+
caps = {4000 * MB, 12000 * MB};
42+
prop = sd::plan_layer_split_ranges(segs, caps);
43+
CHECK(prop.ok && contiguous(prop));
44+
CHECK(prop.bytes_by_device[0] <= caps[0] && prop.bytes_by_device[1] <= caps[1]);
45+
CHECK(prop.bytes_by_device[0] < prop.bytes_by_device[1]);
46+
47+
// Exactly full: proportional must still fit (falls back to fill-first if rounding misses).
48+
std::vector<int64_t> four(4, 100);
49+
caps = {200, 200};
50+
prop = sd::plan_layer_split_ranges(four, caps);
51+
CHECK(prop.ok && prop.bytes_by_device[0] == 200 && prop.bytes_by_device[1] == 200);
52+
53+
// Does not fit at all: failure names the last device.
54+
caps = {150, 150};
55+
prop = sd::plan_layer_split_ranges(four, caps);
56+
CHECK(!prop.ok);
57+
CHECK(prop.failed_device == 1);
58+
59+
// Uneven segment sizes where the proportional plan overflows: fill-first rescues it.
60+
std::vector<int64_t> uneven = {10, 10, 10, 70};
61+
caps = {30, 70};
62+
prop = sd::plan_layer_split_ranges(uneven, caps);
63+
CHECK(prop.ok);
64+
CHECK(prop.bytes_by_device[0] == 30 && prop.bytes_by_device[1] == 70);
65+
66+
// Unknown capacity sentinel (no free-memory report): still splits evenly.
67+
caps = {INT64_MAX / 4, INT64_MAX / 4};
68+
prop = sd::plan_layer_split_ranges(segs, caps);
69+
CHECK(prop.ok && prop.bytes_by_device[1] > 0);
70+
71+
// Zero-byte segments inherit the current device and never trigger a move.
72+
std::vector<int64_t> zeros = {0, 50, 0, 50, 0};
73+
caps = {60, 60};
74+
prop = sd::plan_layer_split_ranges(zeros, caps);
75+
CHECK(prop.ok && contiguous(prop) && prop.bytes_by_device[0] == 50 && prop.bytes_by_device[1] == 50);
76+
77+
// Single device: same as fill-first.
78+
caps = {INT64_MAX / 4};
79+
prop = sd::plan_layer_split_ranges(segs, caps);
80+
CHECK(prop.ok && prop.bytes_by_device.size() == 1);
81+
82+
if (failures == 0) {
83+
std::printf("layer_split_ranges_test: all checks passed\n");
84+
}
85+
return failures == 0 ? 0 : 1;
86+
}

‎src/core/layer_split_partition.cpp‎

Lines changed: 34 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include <unordered_set>
88
#include <utility>
99

10+
#include "core/layer_split_ranges.h"
1011
#include "core/util.h"
1112

1213
namespace sd {
@@ -84,14 +85,20 @@ namespace sd {
8485
if (dev != nullptr) {
8586
ggml_backend_dev_memory(dev, &free_bytes, &total_bytes);
8687
}
87-
if (free_bytes > 0) {
88-
capacities[i] = std::max<int64_t>((int64_t)free_bytes - compute_headroom_bytes, 0);
89-
}
9088
size_t limit_bytes = graph_cut_layer_split_backend_vram_limit(backend_vram_limits,
9189
i,
9290
primary_backend_vram_limit);
91+
// The headroom comes off whichever bound applies: an explicit --max-vram budget also has to hold the
92+
// compute buffers of the segments this device runs, so weights alone may not fill it.
93+
int64_t bound = -1;
94+
if (free_bytes > 0) {
95+
bound = (int64_t)free_bytes;
96+
}
9397
if (limit_bytes > 0) {
94-
capacities[i] = std::min<int64_t>(capacities[i], (int64_t)limit_bytes);
98+
bound = bound < 0 ? (int64_t)limit_bytes : std::min<int64_t>(bound, (int64_t)limit_bytes);
99+
}
100+
if (bound >= 0) {
101+
capacities[i] = std::max<int64_t>(bound - compute_headroom_bytes, 0);
95102
}
96103
}
97104
return capacities;
@@ -151,27 +158,33 @@ namespace sd {
151158
return param_assignments.count(param) != 0;
152159
});
153160

154-
std::vector<ggml_backend_t> backend_by_segment(plan.segments.size(), split_backends[0]);
155-
size_t current_backend = 0;
156-
int64_t current_used = 0;
157-
for (size_t seg_idx = 0; seg_idx < plan.segments.size(); seg_idx++) {
158-
int64_t bytes = segment_param_bytes[seg_idx];
159-
while (!reuse_assignments && current_backend + 1 < split_backends.size() &&
160-
bytes > 0 &&
161-
current_used + bytes > backend_capacities[current_backend]) {
162-
current_backend++;
163-
current_used = 0;
164-
}
165-
if (!reuse_assignments && bytes > 0 && current_used + bytes > backend_capacities[current_backend]) {
161+
LayerSplitRangesResult ranges;
162+
if (!reuse_assignments) {
163+
ranges = plan_layer_split_ranges(segment_param_bytes, backend_capacities);
164+
if (!ranges.ok) {
166165
LOG_ERROR("%s graph-cut layer split: segment %zu needs %.1f MB on %s, but only %.1f MB is available under current VRAM limits",
167166
desc,
168-
seg_idx,
169-
(current_used + bytes) / (1024.0 * 1024.0),
170-
layer_split_backend_device_display_name(split_backends[current_backend]).c_str(),
171-
backend_capacities[current_backend] / (1024.0 * 1024.0));
167+
ranges.failed_segment,
168+
ranges.failed_need / (1024.0 * 1024.0),
169+
layer_split_backend_device_display_name(split_backends[ranges.failed_device]).c_str(),
170+
backend_capacities[ranges.failed_device] / (1024.0 * 1024.0));
172171
return false;
173172
}
174-
current_used += bytes;
173+
for (size_t i = 0; i < split_backends.size(); i++) {
174+
LOG_INFO("%s graph-cut layer split: %s weight capacity %.1f MB (%s)",
175+
desc,
176+
layer_split_backend_device_display_name(split_backends[i]).c_str(),
177+
backend_capacities[i] / (1024.0 * 1024.0),
178+
ranges.used_fill_first ? "fill-first plan, proportional plan did not fit" : "proportional plan");
179+
}
180+
}
181+
182+
std::vector<ggml_backend_t> backend_by_segment(plan.segments.size(), split_backends[0]);
183+
size_t current_backend = 0;
184+
for (size_t seg_idx = 0; seg_idx < plan.segments.size(); seg_idx++) {
185+
if (!reuse_assignments) {
186+
current_backend = ranges.device_by_segment[seg_idx];
187+
}
175188

176189
for (ggml_tensor* param : segment_params[seg_idx]) {
177190
ggml_backend_t target_backend = split_backends[current_backend];

‎src/core/layer_split_ranges.h‎

Lines changed: 124 additions & 0 deletions

0 commit comments

Comments
 (0)