{{ message }}
-
Notifications
You must be signed in to change notification settings - Fork 13
Expand file tree
/
Copy pathbmi_cache.cppm
More file actions
384 lines (343 loc) · 16.2 KB
/
Copy pathbmi_cache.cppm
File metadata and controls
384 lines (343 loc) · 16.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
// mcpp.bmi_cache — cross-project persistent cache of dependency build outputs.
//
// Layout:
// <cache root>/pkg/<index>/<pkg>@<version>/<key16>/
// (the cache root is $MCPP_HOME/build-cache/v1 — see mcpp.home::cache_root)
// entry.json sentinel + self-description + file list
// bmi/<module>.{gcm,pcm}
// obj/<package-internal path>.o
//
// The obj address is PACKAGE-INTERNAL: it mirrors the source's path relative to
// its own package root and contains nothing about the consuming project
// (mcpp#344). It used to be the consumer's build-dir path with `obj/` stripped,
// which made the layout depend on which OTHER packages the consumer happened to
// pull in — #233's basename disambiguation is triggered by a census over the
// whole build dir — while the key deliberately excludes the consumer. Two
// consumers then wrote and read incompatible layouts under one key and the
// second one died in ninja's graph phase.
//
// <key16> comes from mcpp.build.cache_key: a per-package Merkle key over the
// toolchain, the language/dialect settings, the resolved profile, the package's
// own identity and build config, and — recursively — the keys of its direct
// dependencies. See that module's header for why each axis is there.
//
// Two properties this layout has and the previous one did not:
//
// 1. The directory name is derived only from things that actually reach the
// package's compiler command lines. The old key was the whole-project
// fingerprint, so a consumer's own name and version were part of every
// dependency's cache path: `mcpp version bump` invalidated the entire cache
// and no two projects ever shared an entry.
//
// 2. An entry describes itself. `is_cached` compares the recorded inputs
// field by field against the inputs computed for this build, so a hit is
// evidence rather than the mere existence of a directory. The std module
// path has validated hits this way since it was written
// (mcpp.toolchain.stdmod's metadata_matches); the dependency path carried
// only a file list, which is why nothing could be audited when a wrong
// entry was suspected.
//
// populate_from holds an advisory exclusive lock on the entry directory so two
// concurrent builds racing to fill the same entry cannot interleave writes, and
// writes entry.json last so a crash mid-populate leaves a miss, not a
// half-populated hit.
module;
export module mcpp.bmi_cache;
import std;
import mcpp.libs.json;
import mcpp.platform;
export namespace mcpp::bmi_cache {
// Schema of entry.json. Bumped only if the file's own shape changes;
// cache-content compatibility is carried by cache_key::kCacheEpoch, which
// travels inside `inputs`.
inline constexpr int kEntrySchema = 1;
struct CacheKey {
// The resolved cache root (mcpp::home::cache_root()). Passed in rather than
// recomputed here: the layout root must have exactly ONE definition, and a
// second copy of "<home>/build-cache/v1" in this file is precisely the kind
// of cross-file invariant a comment cannot enforce. Tests supply a temp
// directory the same way.
std::filesystem::path cacheRoot;
std::string indexName; // "mcpplibs" / "compat" / ...
std::string packageName; // "compat.zlib"
std::string version; // "1.3.2"
std::string keyHex; // cache_key::key_hex(...)
// The full key inputs, recorded in entry.json and compared field by field
// on a hit. Never trust equal hashes alone.
nlohmann::json inputs;
std::string bmiDirName = "gcm.cache"; // consumer-side directory name
std::string manifestTag = "gcm"; // "gcm" | "pcm"
std::filesystem::path dir() const {
return cacheRoot / "pkg" / indexName
/ std::format("{}@{}", packageName, version) / keyHex;
}
std::filesystem::path entryFile() const { return dir() / "entry.json"; }
std::filesystem::path bmiDir() const { return dir() / "bmi"; }
std::filesystem::path objDir() const { return dir() / "obj"; }
};
// One cached object file. The two addresses are deliberately separate
// (mcpp#344):
//
// cacheRel — where it lives INSIDE the entry (`<entry>/obj/<cacheRel>`).
// Must be a pure function of the package, because the key
// deliberately excludes the consumer. plan.cppm derives it.
// buildRel — where THIS build produces it (`<buildDir>/<buildRel>`).
// Consumer-side and therefore not recordable: two consumers of
// one entry may legitimately place the same object at different
// build-dir paths.
//
// Collapsing the two — recording the consumer's path as the entry's address —
// is exactly what made the second consumer of an entry fail with ninja's
// "missing and no known rule to make it". Only `cacheRel` ever reaches
// entry.json; `buildRel` is populate-time input and is empty on read-back.
struct ObjArtifact {
std::string cacheRel;
std::filesystem::path buildRel;
};
// The artifacts belonging to one package's cache entry: BMI basenames plus the
// objects above.
struct DepArtifacts {
std::vector<std::string> bmiFiles;
std::vector<ObjArtifact> objFiles;
};
// Why an entry could not serve this build.
struct CacheProbe {
bool ok = false;
// Non-empty ONLY when the entry itself validated (schema, key, inputs) but
// does not carry the artifacts THIS build asked for. After mcpp#344 that
// shape should be unreachable, which is precisely why it must be reported
// rather than silently folded into "miss": a systematic recurrence would
// otherwise present as "the cache simply never hits", with no signal at
// all — the same failure mode as the fake `Cached` that went unnoticed for
// three months.
std::vector<std::string> layoutMismatch;
};
// Validate an entry AGAINST WHAT THIS BUILD WILL ACTUALLY READ.
//
// A hit requires all of: entry.json exists, its schema matches, its recorded
// key matches, its recorded inputs equal `key.inputs` field for field, and
// every artifact in `requested` is both listed by the entry and present on
// disk. Checking only the entry's OWN file list — which is what this used to
// do — validates a different question than the one the caller goes on to ask,
// and the two answers diverged the moment the object layout stopped being a
// function of the package alone.
//
// Anything short of a full match is a MISS. This function must never be the
// reason a build fails: an unusable entry costs a recompile, and the staging
// edges that would read it are never emitted.
CacheProbe probe_cached(const CacheKey& key, const DepArtifacts& requested);
// probe_cached(...).ok
bool is_cached(const CacheKey& key, const DepArtifacts& requested);
// The artifact list of a validated entry. Does NOT copy anything: the ninja
// backend stages cached files through its own `stage_file` edges, so that a
// staged file is the output of an edge ninja has a command-line record for.
// Copying them behind ninja's back — which this module used to do — left every
// staged output with no entry in .ninja_log, and ninja treats that as dirty
// ("command line not found in log"), so every cached dependency was recompiled
// anyway while the CLI reported it as cached.
std::expected<DepArtifacts, std::string> resolve_cached(const CacheKey& key);
// Refresh the entry's `accessed` stamp. Rewrites entry.json only — never the
// artifacts, whose mtimes must stay put (ninja's restat handling compares them).
// This is what makes `mcpp cache gc` a real LRU: pruning used to read the
// directory's mtime, which only ever recorded when the entry was WRITTEN, so a
// dependency that hit on every build looked stale.
void touch_accessed(const CacheKey& key);
// Copy fresh build outputs from projectTarget/{bmiDirName,obj} into the entry,
// then write entry.json last as the sentinel.
std::expected<void, std::string>
populate_from(const CacheKey& key,
const std::filesystem::path& projectTargetDir,
const DepArtifacts& artifacts);
// Absolute paths of an entry's artifacts, for the stage edges.
std::filesystem::path cached_bmi_path(const CacheKey& key, std::string_view basename);
std::filesystem::path cached_obj_path(const CacheKey& key, std::string_view rel);
} // namespace mcpp::bmi_cache
namespace mcpp::bmi_cache {
namespace {
bool copy_one(const std::filesystem::path& from,
const std::filesystem::path& to,
std::error_code& ec)
{
std::filesystem::create_directories(to.parent_path(), ec);
std::filesystem::copy_file(from, to,
std::filesystem::copy_options::overwrite_existing, ec);
return !ec;
}
std::optional<nlohmann::json> read_entry(const std::filesystem::path& p) {
std::ifstream is(p);
if (!is) return std::nullopt;
nlohmann::json j;
try { is >> j; } catch (...) { return std::nullopt; }
return j;
}
// Read-back fills `cacheRel` only: `buildRel` is consumer-side and is not — and
// must not be — recorded in the entry.
DepArtifacts artifacts_from(const nlohmann::json& j) {
DepArtifacts a;
if (auto it = j.find("bmi"); it != j.end() && it->is_array())
for (auto& v : *it) if (v.is_string()) a.bmiFiles.push_back(v.get<std::string>());
if (auto it = j.find("obj"); it != j.end() && it->is_array())
for (auto& v : *it)
if (v.is_string()) a.objFiles.push_back({v.get<std::string>(), {}});
return a;
}
// Field-by-field, not `==` on the whole object: an entry written by an older
// mcpp may legitimately carry extra keys, but every key the CURRENT build cares
// about has to be present and equal. A missing key is a mismatch, never a pass.
bool inputs_match(const nlohmann::json& recorded, const nlohmann::json& expected) {
if (!recorded.is_object() || !expected.is_object()) return false;
for (auto it = expected.begin(); it != expected.end(); ++it) {
auto found = recorded.find(it.key());
if (found == recorded.end()) return false;
if (*found != it.value()) return false;
}
return true;
}
std::string now_iso8601() {
// No <chrono> zoned formatting: libstdc++'s `import std` support for
// std::format on chrono types is partial (see fingerprint.cppm's
// hand-rolled hex for the same reason). Seconds since epoch is monotonic
// enough for an LRU stamp and needs no formatting support at all.
auto now = std::chrono::system_clock::now();
auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch()).count();
return std::to_string(secs);
}
std::expected<void, std::string>
write_entry(const std::filesystem::path& path, const nlohmann::json& j) {
auto tmp = path;
tmp += ".tmp";
{
std::ofstream os(tmp, std::ios::binary);
if (!os) return std::unexpected(std::format(
"cannot write cache entry '{}'", tmp.string()));
os << j.dump(2) << "\n";
if (!os) return std::unexpected(std::format(
"failed while writing cache entry '{}'", tmp.string()));
}
std::error_code ec;
std::filesystem::rename(tmp, path, ec);
if (ec) return std::unexpected(std::format(
"cache entry rename: {}", ec.message()));
return {};
}
} // namespace
std::filesystem::path cached_bmi_path(const CacheKey& key, std::string_view basename) {
return key.bmiDir() / std::string(basename);
}
std::filesystem::path cached_obj_path(const CacheKey& key, std::string_view rel) {
return key.objDir() / std::filesystem::path(std::string(rel));
}
CacheProbe probe_cached(const CacheKey& key, const DepArtifacts& requested) {
CacheProbe probe;
auto j = read_entry(key.entryFile());
if (!j) return probe;
if (j->value("schema", 0) != kEntrySchema) return probe;
if (j->value("key", std::string{}) != key.keyHex) return probe;
auto it = j->find("inputs");
if (it == j->end() || !inputs_match(*it, key.inputs)) return probe;
// The entry itself is valid. From here on, every remaining check is about
// whether it holds what THIS build is going to read.
auto recorded = artifacts_from(*j);
std::set<std::string> haveBmi(recorded.bmiFiles.begin(), recorded.bmiFiles.end());
std::set<std::string> haveObj;
for (auto& o : recorded.objFiles) haveObj.insert(o.cacheRel);
std::error_code ec;
for (auto& g : requested.bmiFiles) {
if (!haveBmi.contains(g)
|| !std::filesystem::exists(cached_bmi_path(key, g), ec))
probe.layoutMismatch.push_back(g);
}
for (auto& o : requested.objFiles) {
if (!haveObj.contains(o.cacheRel)
|| !std::filesystem::exists(cached_obj_path(key, o.cacheRel), ec))
probe.layoutMismatch.push_back(o.cacheRel);
}
probe.ok = probe.layoutMismatch.empty();
return probe;
}
bool is_cached(const CacheKey& key, const DepArtifacts& requested) {
return probe_cached(key, requested).ok;
}
std::expected<DepArtifacts, std::string> resolve_cached(const CacheKey& key) {
auto j = read_entry(key.entryFile());
if (!j) return std::unexpected(std::format(
"cannot read cache entry '{}'", key.entryFile().string()));
return artifacts_from(*j);
}
void touch_accessed(const CacheKey& key) {
auto j = read_entry(key.entryFile());
if (!j) return;
(*j)["accessed"] = now_iso8601();
(void)write_entry(key.entryFile(), *j);
}
std::expected<void, std::string>
populate_from(const CacheKey& key,
const std::filesystem::path& projectTargetDir,
const DepArtifacts& arts)
{
auto cacheDir = key.dir();
std::error_code ec;
std::filesystem::create_directories(cacheDir, ec);
auto lock = mcpp::platform::fs::FileLock::try_acquire(cacheDir);
if (!lock) {
// Another writer holds the lock; it will finish the entry.
return {};
}
auto cacheBmi = key.bmiDir();
auto cacheObj = key.objDir();
std::filesystem::create_directories(cacheBmi, ec);
std::filesystem::create_directories(cacheObj, ec);
auto projectBmi = projectTargetDir / key.bmiDirName;
for (auto& g : arts.bmiFiles) {
auto from = projectBmi / g;
if (!std::filesystem::exists(from)) {
return std::unexpected(std::format(
"expected build output missing: {}", from.string()));
}
if (!copy_one(from, cacheBmi / g, ec)) {
return std::unexpected(std::format(
"populate bmi '{}': {}", g, ec.message()));
}
}
// Read from `buildRel`, write at `cacheRel`. These are NOT the same path in
// general (mcpp#344): the build-dir layout partitions objects by package,
// the entry's layout is package-internal, and a source that sits outside its
// package root is re-anchored for the entry. Deriving one from the other
// here is what this split exists to prevent.
for (auto& o : arts.objFiles) {
auto from = projectTargetDir / o.buildRel;
if (o.buildRel.empty() || !std::filesystem::exists(from)) {
return std::unexpected(std::format(
"expected build output missing: {}", from.string()));
}
if (!copy_one(from, cached_obj_path(key, o.cacheRel), ec)) {
return std::unexpected(std::format(
"populate obj '{}': {}", o.cacheRel, ec.message()));
}
}
// entry.json LAST — it is the sentinel. Preserve `created` when refilling an
// existing entry so gc's age reporting stays meaningful.
nlohmann::json j;
if (auto prev = read_entry(key.entryFile()); prev && prev->contains("created"))
j["created"] = (*prev)["created"];
else
j["created"] = now_iso8601();
j["schema"] = kEntrySchema;
j["key"] = key.keyHex;
j["package"] = std::format("{}/{}@{}", key.indexName, key.packageName, key.version);
j["bmi_dir"] = key.bmiDirName;
j["tag"] = key.manifestTag;
j["inputs"] = key.inputs;
j["bmi"] = arts.bmiFiles;
// Only the entry-internal addresses. Recording the consumer's build path
// here is mcpp#344 in one line.
{
auto objs = nlohmann::json::array();
for (auto& o : arts.objFiles) objs.push_back(o.cacheRel);
j["obj"] = std::move(objs);
}
j["accessed"] = now_iso8601();
return write_entry(key.entryFile(), j);
}
} // namespace mcpp::bmi_cache
You can’t perform that action at this time.