{{ message }}
-
Notifications
You must be signed in to change notification settings - Fork 14
Expand file tree
/
Copy pathscanner.cppm
More file actions
1192 lines (1113 loc) · 54.4 KB
/
Copy pathscanner.cppm
File metadata and controls
1192 lines (1113 loc) · 54.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// mcpp.modgraph.scanner — regex-based scan of .cppm/.cpp for module statements.
//
// Hard constraints (per docs/01 §4.2):
// ✗ no #if/#ifdef-guarded import
// ✗ no header units (import "h" / import <h>)
// ✗ files outside [modules].sources glob
//
// Returns a Graph or a list of detailed errors.
export module mcpp.modgraph.scanner;
import std;
import mcpp.manifest;
import mcpp.modgraph.glob;
import mcpp.modgraph.graph;
import mcpp.modgraph.p1689;
import mcpp.source_kind;
import mcpp.toolchain.detect;
export namespace mcpp::modgraph {
struct ScanError {
std::filesystem::path path;
std::size_t line = 0;
std::string message;
std::string format() const {
if (line)
return std::format("{}:{}: {}", path.string(), line, message);
return std::format("{}: {}", path.string(), message);
}
};
// Expand a glob like "src/**/*.cppm" into a list of matching paths anchored at root.
std::vector<std::filesystem::path> expand_glob(const std::filesystem::path& root,
std::string_view glob);
// M6.x: same as expand_glob but matches DIRECTORIES (used for include_dirs
// like "*/include"). Always returns absolute paths under `root`.
std::vector<std::filesystem::path> expand_dir_glob(const std::filesystem::path& root,
std::string_view glob);
// mcpp#225 (test-exposed): pure literal (non-wildcard) directory-prefix
// derivation used to bound expand_glob/expand_dir_glob's walk start point.
// Exported (rather than kept file-local) solely so unit tests can assert its
// behavior directly and deterministically — see its definition below for
// the full contract.
std::filesystem::path glob_literal_prefix(std::string_view glob);
// mcpp#228: desugar brace alternation `{a,b}` into a cartesian product of
// plain globs, e.g. "a/{x,y}/**" -> ["a/x/**", "a/y/**"]. A glob with no `{`
// returns itself unchanged (the common case). Multiple and nested groups are
// supported ("a/{x,y}/{1,2}" -> 4 branches; "{a,{b,c}}" resolves the inner
// group too). Exported so unit tests can assert its behavior directly; also
// used internally at the entry of expand_glob and at the [build].flags
// per-glob-flag match point in scan_one_into, so every glob consumer sees
// alternation transparently.
std::vector<std::string> expand_braces(std::string_view glob);
// Scan a single source file. `extTable` is the OWNING PACKAGE's table — a
// dependency is classified by its own manifest, never by the consumer's.
std::expected<SourceUnit, ScanError> scan_file(const std::filesystem::path& file,
const std::string& packageName,
const mcpp::ExtensionTable& extTable);
// Scan the entire package: collects all sources via manifest globs and returns a Graph.
struct ScanResult {
Graph graph;
std::vector<ScanError> errors;
std::vector<ScanError> warnings;
};
ScanResult scan_package(const std::filesystem::path& root,
const mcpp::manifest::Manifest& manifest);
// Absolutize relative include/lib-search-path flags against the package root
// (G8b, generalized by #226). A manifest's relative include flag means
// root-relative, but ninja runs commands with cwd = the output dir, so a
// verbatim relative flag resolves against the wrong base. Recognizes the
// whole include-family prefix set — -I, -iquote, -isystem, -idirafter,
// -iprefix, -L — in BOTH the joined spelling (`-iquotehdr`) and the
// separated spelling (`-isystem` followed by a standalone `hdr` element).
// Called at every point where per-unit flag vectors are attached (the
// scanner here; plan.cppm for a target's entry unit; flags.cppm for the
// manifest-global [build] include_dirs).
void normalize_include_flags(const std::filesystem::path& root,
std::vector<std::string>& flags);
enum class DependencyVisibility {
Private,
Public,
Interface,
};
struct UsageRequirements {
std::vector<std::filesystem::path> includeDirs;
// #249: emitted as -idirafter (searched after system dirs); propagated
// along the same consumer edges as includeDirs, never upgraded to -I.
std::vector<std::filesystem::path> includeDirsAfter;
std::vector<std::string> cflags;
std::vector<std::string> cxxflags;
std::vector<std::string> ldflags;
std::vector<std::string> modules;
};
// Scan multiple packages (primary + path-based deps) into one combined Graph.
// Each SourceUnit retains its own packageName, so validate() applies the
// correct naming rules per-package.
struct PackageRoot {
std::filesystem::path root;
mcpp::manifest::Manifest manifest;
UsageRequirements privateBuild;
UsageRequirements publicUsage;
UsageRequirements linkUsage;
bool usageResolved = false;
};
ScanResult scan_packages(const std::vector<PackageRoot>& packages);
// Drop-in replacement that delegates per-file scanning to GCC's P1689r5
// (.ddi) output instead of regex parsing. Same ScanResult shape — used by
// cli when MCPP_SCANNER=p1689 (see docs/27).
ScanResult scan_packages_p1689(const std::vector<PackageRoot>& packages,
const mcpp::toolchain::Toolchain& tc,
const std::filesystem::path& tmpDir,
std::string_view cppStandardFlag);
} // namespace mcpp::modgraph
namespace mcpp::modgraph {
namespace {
// Trim leading/trailing whitespace.
std::string_view trim(std::string_view s) {
std::size_t i = 0, j = s.size();
while (i < j && std::isspace(static_cast<unsigned char>(s[i]))) ++i;
while (j > i && std::isspace(static_cast<unsigned char>(s[j-1]))) --j;
return s.substr(i, j - i);
}
// Strip a trailing line comment ("//...").
std::string_view strip_line_comment(std::string_view s) {
auto p = s.find("//");
if (p == std::string_view::npos) return s;
return s.substr(0, p);
}
// Remove C++ raw-string-literal bodies from a line, tracking multi-line raw
// strings across calls via (in_raw, raw_close). Returns the code-only portion
// with raw-string contents blanked out.
//
// Without this, a template that embeds source text — e.g. the `mcpp new
// --template gui` skeleton stored as R"GUI( ... import imgui.core; ... )GUI"
// in scaffold/create.cppm — has its inner `import` lines misdetected as real
// module imports, producing spurious "imported but not provided" warnings.
// Ordinary "..." strings are intentionally left as-is: the import/module
// matcher only fires on lines whose trimmed text *starts with* the keyword,
// which a string body can only do when it spans lines (i.e. a raw string).
std::string strip_raw_strings(std::string_view line, bool& in_raw,
std::string& raw_close) {
std::string out;
std::size_t i = 0;
while (i < line.size()) {
if (in_raw) {
auto p = line.find(raw_close, i);
if (p == std::string_view::npos) return out; // rest of line is raw body
i = p + raw_close.size();
in_raw = false;
raw_close.clear();
continue;
}
// Raw-string opener: R"delim( ... )delim" (delim is up to 16 chars,
// no '(' / whitespace per the standard). Optional u8/u/U/L prefixes
// precede the R; we only need to spot the R" boundary.
if (line[i] == 'R' && i + 1 < line.size() && line[i + 1] == '"') {
std::size_t d = i + 2;
std::string delim;
while (d < line.size() && line[d] != '(' && (d - (i + 2)) < 16) {
delim.push_back(line[d]);
++d;
}
if (d < line.size() && line[d] == '(') {
raw_close = ")" + delim + "\"";
in_raw = true;
i = d + 1;
continue;
}
}
out.push_back(line[i]);
++i;
}
return out;
}
bool is_module_name_char(char c) {
return std::isalnum(static_cast<unsigned char>(c)) || c == '_' || c == '.' || c == ':';
}
// mcpp#225: submodule paths registered in `<root>/.gitmodules` ("path = ..."
// entries), resolved to canonical absolute paths. is_excluded_walk_dir is
// called once per directory ENTRY seen during a walk, so this is parsed
// once per root and cached for the life of the process rather than
// re-reading .gitmodules on every call (that would defeat the point of
// bounding the walk).
const std::set<std::filesystem::path>&
submodule_paths(const std::filesystem::path& root) {
static std::map<std::filesystem::path, std::set<std::filesystem::path>> cache;
std::error_code kec;
auto key = std::filesystem::canonical(root, kec);
if (kec) key = root;
if (auto it = cache.find(key); it != cache.end()) return it->second;
std::set<std::filesystem::path> paths;
std::ifstream f(root / ".gitmodules");
std::string line;
while (f && std::getline(f, line)) {
auto eq = line.find('=');
if (eq == std::string::npos) continue;
std::string_view k = trim(std::string_view(line).substr(0, eq));
if (k != "path") continue;
std::string_view v = trim(std::string_view(line).substr(eq + 1));
if (v.empty()) continue;
std::error_code pec;
auto abs = std::filesystem::canonical(root / std::filesystem::path(std::string(v)), pec);
paths.insert(pec ? (root / std::filesystem::path(std::string(v))) : abs);
}
return cache.emplace(key, std::move(paths)).first->second;
}
// mcpp#225: directory names that never hold project sources — VCS metadata,
// mcpp's own build output, and mcpp's own project-metadata dir (mcpp#230:
// `.mcpp`'s xlings data tree holds a symlink back to each path-dep index
// root, so following it walks that entire checkout). A directory whose path
// matches a `.gitmodules`-registered submodule path under `root` is pruned
// too — submodules are foreign trees, often huge, and not part of this
// package's source glob.
bool is_excluded_walk_dir(const std::filesystem::path& dir,
const std::filesystem::path& root) {
// Compare as paths. Do NOT narrow.
//
// #516: `dir.filename().string()` went through MSVC's wide→ANSI
// conversion and threw std::system_error for any directory name the
// active code page cannot spell (`test/www/<CJK>Dir/` in cpp-httplib).
// This function is the FIRST line of the walk loop and runs once per
// directory entry, so it fires before `path_matches_glob`'s guard —
// hardening that one (#231) could never cover a directory name.
//
// The three literals are ASCII, so their conversion to the native
// representation is lossless, and `path::operator==` compares native
// strings case-sensitively — byte-for-byte the same decision the narrow
// comparison made. Static constants rather than temporaries per entry:
// #225 bounded this walk for a reason, and this is on its hot path.
static const std::filesystem::path kMcppDir{".mcpp"};
static const std::filesystem::path kGitDir{".git"};
static const std::filesystem::path kTargetDir{"target"};
const auto name = dir.filename();
if (name == kMcppDir || name == kGitDir || name == kTargetDir) return true;
auto const& submodules = submodule_paths(root);
if (submodules.empty()) return false;
std::error_code ec;
auto c = std::filesystem::canonical(dir, ec);
return submodules.contains(ec ? dir : c);
}
} // namespace
// mcpp#225: the literal (non-wildcard) directory prefix of a glob, e.g.
// "src/**/*.cppm" -> "src", "tests/**/*.cpp" -> "tests", "*/include" -> ""
// (wildcard already in the first segment). Used to bound the walk's START
// point in expand_glob/expand_dir_glob instead of always walking from root
// and filtering lexically afterward — path_matches_glob still does the full
// lexical match, so this only narrows WHERE the iterator begins, never
// which files can match. Truncates back to the last complete '/' so a
// partial segment (e.g. "src/pre*fix" -> "src/pre") is never mistaken for a
// real directory name. Conservative about '{' (and '?'/'['): brace-expansion
// globs are desugared into multiple plain globs before ever reaching here
// (mcpp#225 cluster E follow-up), so any of those chars seen here is just a
// segment boundary, never something to interpret. Declared in the exported
// namespace above (not kept file-local) purely so unit tests can assert its
// behavior directly, deterministically, and independently of filesystem
// enumeration order — see Scanner.GlobLiteralPrefixDerivation.
std::filesystem::path glob_literal_prefix(std::string_view glob) {
// Every current caller already strips a leading `!` (exclusion globs)
// before calling expand_glob/expand_dir_glob, but strip it here too —
// defense in depth, and a literal '!' is never a real path component.
if (!glob.empty() && glob.front() == '!') glob.remove_prefix(1);
auto wildcard = glob.find_first_of("*?{[");
std::string_view literal = wildcard == std::string_view::npos
? glob : glob.substr(0, wildcard);
auto slash = literal.find_last_of('/');
if (slash == std::string_view::npos) return {};
// Native separators, not the raw generic form: MSVC keeps the input's
// `/` verbatim, and `root / p` plus the directory walk then propagate a
// MIXED `root\generated/modules` into every downstream path — which is
// what `compile_commands.json`'s `file` field showed on Windows for
// multi-segment globs. See mcpp::modgraph::native_path_from_generic.
return native_path_from_generic(literal.substr(0, slash));
}
// mcpp#228: `{a,b}` alternation, recursively. Finds the first top-level `{`,
// its MATCHING `}` (brace-depth tracked, so a nested group's inner braces
// don't prematurely close the outer one), splits the interior on top-level
// commas (again depth-tracked, so a nested group's own commas don't split
// the outer one), then cartesian-products: each alternative is itself
// re-expanded (handles nesting, e.g. "{a,{b,c}}"), and everything AFTER the
// closing `}` is independently re-expanded and combined with every
// alternative (handles multiple groups, e.g. "a/{x,y}/{1,2}" -> 4
// branches). An unbalanced `{` (no matching `}`) is passed through as a
// literal — never a reason to fail the whole glob.
std::vector<std::string> expand_braces(std::string_view glob, int depthGuard);
std::vector<std::string> expand_braces(std::string_view glob) {
return expand_braces(glob, 0);
}
std::vector<std::string> expand_braces(std::string_view glob, int depthGuard) {
auto open = glob.find('{');
if (open == std::string_view::npos) return { std::string(glob) };
// Bound brace-nesting recursion depth: a pathological manifest with
// deeply nested `{` should not stack-overflow. Beyond the cap, treat the
// remainder as a literal passthrough rather than throwing — an
// unreasonably-nested glob simply won't desugar further, it still
// parses.
constexpr int kMaxBraceDepth = 32;
if (depthGuard >= kMaxBraceDepth) return { std::string(glob) };
int depth = 0;
std::size_t close = std::string_view::npos;
for (std::size_t i = open; i < glob.size(); ++i) {
if (glob[i] == '{') ++depth;
else if (glob[i] == '}') {
--depth;
if (depth == 0) { close = i; break; }
}
}
if (close == std::string_view::npos) return { std::string(glob) };
std::string_view prefix = glob.substr(0, open);
std::string_view inner = glob.substr(open + 1, close - open - 1);
std::string_view suffix = glob.substr(close + 1);
std::vector<std::string_view> alts;
int d = 0;
std::size_t start = 0;
for (std::size_t i = 0; i < inner.size(); ++i) {
if (inner[i] == '{') ++d;
else if (inner[i] == '}') --d;
else if (inner[i] == ',' && d == 0) {
alts.push_back(inner.substr(start, i - start));
start = i + 1;
}
}
alts.push_back(inner.substr(start));
auto suffixBranches = expand_braces(suffix, depthGuard + 1);
std::vector<std::string> out;
for (auto alt : alts) {
for (auto& altBranch : expand_braces(alt, depthGuard + 1)) {
for (auto& sufBranch : suffixBranches) {
out.push_back(std::string(prefix) + altBranch + sufBranch);
}
}
}
return out;
}
namespace {
// mcpp#225: the actual bounded recursive-directory walk for a SINGLE plain
// glob (no `{` — expand_glob below desugars brace alternation via
// expand_braces and calls this once per branch, unioning the results).
std::vector<std::filesystem::path> expand_glob_one(const std::filesystem::path& root,
std::string_view glob)
{
namespace fs = std::filesystem;
std::vector<fs::path> out;
if (!fs::exists(root)) return out;
// mcpp#225: bound the walk's start point to the glob's literal
// directory prefix instead of always walking the whole root. A prefix
// that doesn't exist means the glob can never match anything — return
// empty WITHOUT walking (not a full-tree fallback).
fs::path prefix = glob_literal_prefix(glob);
fs::path start = prefix.empty() ? root : root / prefix;
std::error_code startEc;
if (!fs::exists(start, startEc)) return out;
// Follow directory symlinks (vendored trees are often symlink farms).
// Cycle guard: a directory whose canonical path is already on the
// CURRENT recursion chain is a link loop — only that is pruned; the same
// real directory reached via a second lexical path (dir + link to it)
// still walks, because glob matching is lexical. Files reachable twice
// are deduped by canonical identity afterwards.
std::vector<fs::path> chain; // canonical dirs of the recursion stack
std::error_code ec, eec; // ec: iteration; eec: per-entry probes
{
auto c = fs::canonical(start, eec);
chain.push_back(eec ? start : c);
}
fs::recursive_directory_iterator it(
start, fs::directory_options::follow_directory_symlink, ec);
for (fs::recursive_directory_iterator end; !ec && it != end; it.increment(ec)) {
auto& e = *it;
if (e.is_directory(eec) && !eec) {
if (is_excluded_walk_dir(e.path(), root)) {
it.disable_recursion_pending();
continue;
}
auto depth = static_cast<std::size_t>(it.depth());
chain.resize(std::min(chain.size(), depth + 1));
auto c = fs::canonical(e.path(), eec);
if (!eec && std::find(chain.begin(), chain.end(), c) != chain.end()) {
it.disable_recursion_pending(); // link cycle
} else {
chain.push_back(eec ? e.path() : c);
}
continue;
}
if (!e.is_regular_file(eec) || eec) continue;
if (path_matches_glob(e.path(), root, glob)) out.push_back(e.path());
}
std::sort(out.begin(), out.end());
// Dedup files reachable through more than one directory link (first
// lexical occurrence wins).
std::set<fs::path> seenFiles;
out.erase(std::remove_if(out.begin(), out.end(), [&](const fs::path& p) {
auto c = fs::canonical(p, eec);
return !eec && !seenFiles.insert(c).second;
}), out.end());
return out;
}
} // namespace
std::vector<std::filesystem::path> expand_glob(const std::filesystem::path& root,
std::string_view glob)
{
// mcpp#228: desugar `{a,b}` alternation FIRST, then run the existing
// bounded walk per branch and union+dedup — each post-desugar branch has
// no `{`, so glob_literal_prefix's prefix narrowing still applies (and
// gets a LONGER, more specific prefix per branch than the pre-desugar
// glob would have yielded).
auto branches = expand_braces(glob);
if (branches.size() == 1) return expand_glob_one(root, branches.front());
namespace fs = std::filesystem;
std::set<fs::path> seen;
std::vector<fs::path> out;
for (auto const& b : branches) {
for (auto& p : expand_glob_one(root, b)) {
if (seen.insert(p).second) out.push_back(std::move(p));
}
}
std::sort(out.begin(), out.end());
return out;
}
std::vector<std::filesystem::path> expand_dir_glob(const std::filesystem::path& root,
std::string_view glob)
{
std::vector<std::filesystem::path> out;
std::error_code ec;
if (!std::filesystem::exists(root, ec)) return out;
// Fast path: glob with no wildcards → literal path under root. Brace
// alternation `{a,b}` is intentionally NOT desugared here (unlike
// expand_glob) — include_dirs entries are meant to name one literal
// directory each; a caller wanting alternatives lists multiple entries.
if (glob.find('*') == std::string_view::npos) {
// Native spelling (see native_path_from_generic — a raw `a/b` would
// come back mixed from .string() on MSVC).
auto p = root / native_path_from_generic(glob);
if (std::filesystem::is_directory(p, ec)) out.push_back(p);
return out;
}
// mcpp#225: bound the walk's start point the same way expand_glob does
// (see the comment there) — a prefix that doesn't exist means the glob
// can never match, so return empty without walking.
std::filesystem::path prefix = glob_literal_prefix(glob);
std::filesystem::path start = prefix.empty() ? root : root / prefix;
std::error_code startEc;
if (!std::filesystem::exists(start, startEc)) return out;
// Walk all directories under start, match each against the glob. Same
// follow-symlinks + recursion-chain cycle guard as expand_glob above.
out.push_back(root); // sentinel, always dropped below regardless of value
std::vector<std::filesystem::path> chain;
std::error_code eec; // per-entry probes; ec drives iteration
{
auto c = std::filesystem::canonical(start, eec);
chain.push_back(eec ? start : c);
}
std::filesystem::recursive_directory_iterator it(
start, std::filesystem::directory_options::follow_directory_symlink, ec);
for (std::filesystem::recursive_directory_iterator end;
!ec && it != end; it.increment(ec)) {
auto& e = *it;
if (!e.is_directory(eec) || eec) continue;
if (is_excluded_walk_dir(e.path(), root)) {
it.disable_recursion_pending();
continue;
}
auto depth = static_cast<std::size_t>(it.depth());
chain.resize(std::min(chain.size(), depth + 1));
auto c = std::filesystem::canonical(e.path(), eec);
if (!eec && std::find(chain.begin(), chain.end(), c) != chain.end()) {
it.disable_recursion_pending(); // link cycle
continue;
}
chain.push_back(eec ? e.path() : c);
if (path_matches_glob(e.path(), root, glob)) out.push_back(e.path());
}
out.erase(out.begin()); // drop root sentinel
std::sort(out.begin(), out.end());
out.erase(std::unique(out.begin(), out.end()), out.end());
return out;
}
namespace {
// has_root_path: leave absolute AND root-relative ("/x" on Windows)
// spellings alone — only genuinely root-less paths are project-relative.
// Both branches normalize to NATIVE separators: a `-Ithird_party/inc` cxxflag
// would otherwise come back as `C:\proj\third_party/inc` on MSVC (path keeps
// the input `/` verbatim) and reach the CDB's arguments via packageCxxflags.
std::string rewrite_rel_copy(const std::string& p, const std::filesystem::path& root) {
std::filesystem::path fp(p);
if (fp.has_root_path()) {
// Nothing to re-spell → hand back the ORIGINAL bytes rather than
// round-tripping them through path's narrow conversion, which throws
// std::system_error for names the ANSI codepage cannot express
// (mcpp#230 — see path_matches_glob). A rooted path with no '/' is
// already native on both platform families.
if (p.find('/') == std::string::npos) return p;
fp.make_preferred();
return fp.string();
}
auto joined = root / fp;
joined.make_preferred();
return joined.string();
}
void rewrite_rel(std::string& p, const std::filesystem::path& root) {
p = rewrite_rel_copy(p, root);
}
} // namespace
void normalize_include_flags(const std::filesystem::path& root,
std::vector<std::string>& flags)
{
// #226: the whole include/lib-search-path family, not just -I. Each
// prefix is checked in both spellings:
// joined: "-iquotehdr" (element starts_with prefix, has a tail)
// separated: "-isystem", "hdr" (element == bare prefix, rewrite next)
static constexpr std::string_view kIncPrefixes[] =
{"-I", "-iquote", "-isystem", "-idirafter", "-iprefix", "-L"};
for (std::size_t i = 0; i < flags.size(); ++i) {
for (auto pre : kIncPrefixes) {
if (flags[i] == pre && i + 1 < flags.size()) { // separated
rewrite_rel(flags[i + 1], root);
++i;
break;
}
if (flags[i].size() > pre.size() && flags[i].starts_with(pre)) { // joined
std::string tail = flags[i].substr(pre.size());
std::string abs = rewrite_rel_copy(tail, root);
if (abs != tail) flags[i] = std::string(pre) + abs;
break;
}
}
}
}
std::expected<SourceUnit, ScanError> scan_file(const std::filesystem::path& file,
const std::string& packageName,
const mcpp::ExtensionTable& extTable)
{
std::ifstream is(file);
if (!is) return std::unexpected(ScanError{file, 0, "cannot open"});
SourceUnit u;
// The module this TU belongs to (base name, no `:partition`). Set by the
// `module`/`export module` declaration and used to resolve `import :part;`.
std::string owningModule;
u.path = file;
u.packageName = packageName;
// The ONE place a source's role is decided. Everything downstream reads
// `u.kind`; nothing re-derives it from the extension.
u.kind = mcpp::classify(file, extTable);
// ⚠️ A file `sources` matched but the classifier cannot place.
//
// This used to be accepted, and what it produced was a compile edge whose
// object NOTHING LINKS. Measured with an `.ixx` that no
// `[build] module_extensions` declared:
//
// build obj/mathkit.ixx.o | gcm.cache/mathkit.gcm : cxx_object …
// bmi_out = gcm.cache/mathkit.gcm ← the BMI was produced
// build bin/app : cxx_link obj/main.o ← the object is not here
//
// ld: undefined reference to `mk::answer@mathkit()'
//
// Two answers to "is this a module interface" and only one of them read:
// the SCANNER sees `export module mathkit;` and records `provides`, which
// is why the edge got a `bmi_out`; the CLASSIFIER says `Other`, and the
// link set is built from the classifier. So the author is told about a
// symbol, when what happened is a missing line of configuration.
//
// Refused here rather than repaired downstream: the extension set is the
// project's to declare, and mcpp guessing which unknown extension "must
// have meant" a module interface would be a second answer to the same
// question — the very thing that produced this defect.
if (u.kind == mcpp::SourceKind::Other) {
// Narrow through try_narrow even here. Everything that reaches
// scan_file today came through a glob filter, so the conversion cannot
// actually fail — but a DIAGNOSTIC that throws the exception it is
// describing is the single most likely way this class of bug comes
// back, and it costs one `value_or` to make that impossible.
auto nameNarrow = try_narrow(file.filename()).value_or(
"(a name this code page cannot spell)");
auto extNarrow = try_narrow(file.extension()).value_or("");
return std::unexpected(ScanError{ file, 0, std::format(
"'{}' is listed in [build] sources, and mcpp has no role for the "
"extension '{}'.\n"
" Its object would be compiled and then linked by nothing, so this "
"is refused rather\n"
" than built. If it is a module interface, declare the extension:\n"
" [build]\n"
" module_extensions = [\"{}\"]\n"
" Otherwise remove it from `sources` — headers belong in "
"`include_dirs`, and\n"
" Windows resource scripts in `[resources]`.",
nameNarrow,
extNarrow.empty() ? "(none)" : extNarrow,
extNarrow.empty() ? ".ixx" : extNarrow) });
}
// C-like files are not C++ modules: they cannot legally contain `module` / `import`
// declarations, and we route them to the C-language compile rule (no
// P1689 scan, no BMI lookups). Skip the line-by-line module scan to
// avoid any chance of a benign identifier (`import_foo`, `module_t`, ...)
// being misparsed. Objective-C .m files use the same C-like path, and so
// does assembly (.S/.s via the C driver, .asm via NASM).
if (mcpp::is_scan_exempt(u.kind)) {
return u;
}
int if_depth = 0; // #if/#ifdef nesting
std::size_t lineno = 0;
bool in_raw = false; // inside a multi-line raw string
std::string raw_close; // active )delim" terminator
std::string line;
while (std::getline(is, line)) {
++lineno;
// Blank out raw-string-literal bodies first so embedded source text
// (e.g. scaffold templates) isn't misparsed as imports.
std::string code = strip_raw_strings(line, in_raw, raw_close);
std::string_view sv = strip_line_comment(code);
sv = trim(sv);
if (sv.empty()) continue;
// Track preprocessor depth (we only need to know if we're inside #if).
if (sv.size() > 0 && sv[0] == '#') {
std::string_view rest = trim(sv.substr(1));
if (rest.starts_with("if") || rest.starts_with("ifdef") || rest.starts_with("ifndef")) {
++if_depth;
} else if (rest.starts_with("endif")) {
if (if_depth > 0) --if_depth;
}
continue;
}
// Strip leading `export ` so we can handle uniformly:
// `export module foo;`
// `module foo;`
// `export import :part;` ← new: re-exported partitions / modules
// `import foo;`
std::string_view r = sv;
bool is_export = false;
if (r.starts_with("export") &&
(r.size() == 6 || r[6] == ' ' || r[6] == '\t')) {
is_export = true;
r = trim(r.substr(6));
}
// module name [: partition] ;
if (r.starts_with("module") &&
(r.size() == 6 || r[6] == ' ' || r[6] == '\t' || r[6] == ';')) {
r = trim(r.substr(6));
if (r.empty() || r == ";") {
continue; // global module fragment marker (`module;`)
}
std::string name;
std::size_t i = 0;
while (i < r.size() && is_module_name_char(r[i])) {
name.push_back(r[i]);
++i;
}
if (is_export) {
if (u.provides) {
return std::unexpected(ScanError{file, lineno,
std::format("file already exports module '{}'; cannot export '{}'",
u.provides->logicalName, name)});
}
u.provides = ModuleId{name};
u.providesInterface = true; // read from the keyword, not assumed
} else {
// A non-exporting `module …;` is TWO different declarations
// wearing one spelling, and they were treated as one:
//
// module M; a module IMPLEMENTATION UNIT. Belongs to
// M, provides nothing importable, needs M's
// interface.
// module M:part; an IMPLEMENTATION PARTITION. It IS
// importable — as `M:part`, by the other
// units of M — and it produces a BMI and an
// object exactly like an interface partition.
//
// Recording the second as `requires M:part` made every such
// file require its own name and provide nothing. So the graph
// held no edge from the unit that imports the partition to the
// unit that defines it, and the build order was unconstrained.
// Where the compiler's own scan recovers it (GCC, macOS clang)
// the build worked anyway; on Windows clang it failed with
// `failed to read compiled module` — an ordering bug wearing a
// scanner mistake's clothes, and invisible because the scanner
// ALSO warned "imported but not provided in this build" on
// every platform, which read like a note rather than a cause.
if (name.find(':') != std::string::npos) {
if (u.provides) {
return std::unexpected(ScanError{file, lineno,
std::format("file already provides module '{}'; "
"cannot also provide '{}'",
u.provides->logicalName, name)});
}
u.provides = ModuleId{name};
u.providesInterface = false;
} else if (!u.provides) {
u.requires_.push_back(ModuleId{name});
}
}
// The module this TU belongs to, for resolving `import :part;`
// below. Tracked separately from `u.provides` because an
// implementation unit (`module M;`) provides nothing and still has
// to be able to name its own partitions — before this, `import
// :secret;` inside one stayed the literal `:secret`, which no unit
// provides, which is the other half of the same warning.
owningModule = name.substr(0, name.find(':'));
continue;
}
// import [name | "h" | <h>] ;
if (r.starts_with("import") &&
(r.size() == 6 || r[6] == ' ' || r[6] == '\t' ||
r[6] == '<' || r[6] == '"' || r[6] == ':'))
{
if (if_depth > 0) {
return std::unexpected(ScanError{file, lineno,
"import statement inside conditional preprocessor block (forbidden in M1)"});
}
r = trim(r.substr(6));
if (r.empty()) continue;
if (r[0] == '<' || r[0] == '"') {
return std::unexpected(ScanError{file, lineno,
"header units (import \"h\" / import <h>) are forbidden in M1"});
}
std::string name;
std::size_t i = 0;
while (i < r.size() && is_module_name_char(r[i])) {
name.push_back(r[i]);
++i;
}
if (name.empty()) continue;
// Partition import within the same module: prepend the *base*
// module name. If the current TU itself owns a partition (e.g.
// its `export module foo:http;`), `u.provides->logicalName`
// already includes that suffix — concatenating naively would
// produce `foo:http:tls` instead of the intended `foo:tls`.
// Strip our own `:partition` first.
if (name.starts_with(":") && !owningModule.empty()) {
name = owningModule + name;
}
u.requires_.push_back(ModuleId{name});
continue;
}
}
return u;
}
namespace {
std::vector<std::filesystem::path>
local_include_dirs_for(const std::filesystem::path& root,
const mcpp::manifest::Manifest& manifest)
{
std::vector<std::filesystem::path> dirs;
for (auto const& inc : manifest.buildConfig.includeDirs) {
if (inc.is_absolute()) {
// A TOML value like `C:/SDL2/include` keeps its `/` on MSVC —
// normalize so the CDB's -I comes out native (mixed separators
// break CLion). Direct make_preferred, no generic_string round
// trip: the narrow conversion can throw for names the ANSI
// codepage cannot spell (mcpp#230).
auto n = inc;
n.make_preferred();
dirs.push_back(std::move(n));
continue;
}
for (auto& d : expand_dir_glob(root, inc.generic_string())) {
dirs.push_back(std::move(d));
}
}
return dirs;
}
// #249: same expansion for `include_dirs_after` entries (the -idirafter
// channel). Shares the `*` extracted-tarball-root glob convention.
std::vector<std::filesystem::path>
local_include_dirs_after_for(const std::filesystem::path& root,
const mcpp::manifest::Manifest& manifest)
{
std::vector<std::filesystem::path> dirs;
for (auto const& inc : manifest.buildConfig.includeDirsAfter) {
if (inc.is_absolute()) {
auto n = inc;
n.make_preferred();
dirs.push_back(std::move(n));
continue;
}
for (auto& d : expand_dir_glob(root, inc.generic_string())) {
dirs.push_back(std::move(d));
}
}
return dirs;
}
// Phase 1: scan a single package, append units to result.graph.units;
// errors go straight into result.errors. producerOf/edges are NOT built
// here — the caller does that after all packages are scanned.
void scan_one_into(ScanResult& result,
const std::filesystem::path& root,
const mcpp::manifest::Manifest& manifest,
const std::vector<std::filesystem::path>& localIncludeDirs,
const std::vector<std::filesystem::path>& localIncludeDirsAfter,
const std::vector<std::string>& packageCflags,
const std::vector<std::string>& packageCxxflags)
{
// This package's own extension table. Built once per package, not per
// file, and taken from THIS manifest — a dependency is classified by its
// own `[build] module_extensions`, never by the consumer's.
const auto extTable =
mcpp::extension_table_for(manifest.buildConfig.moduleExtensions);
// Glob exclusion: patterns starting with `!` remove files from the
// include set (like .gitignore).
// sources = ["src/**/*.cpp", "!src/**/*_test.cpp"]
// All positive patterns are expanded first, then all `!`-prefixed
// patterns are expanded and the resulting paths are removed.
std::set<std::filesystem::path> all_files;
std::set<std::filesystem::path> excluded;
for (auto const& g : manifest.modules.sources) {
if (!g.empty() && g[0] == '!') {
for (auto& p : expand_glob(root, g.substr(1))) {
excluded.insert(p);
}
} else {
// Literal absolute entry — e.g. a dependency build.mcpp's OUT_DIR
// generated source, which lives OUTSIDE the (possibly read-only)
// package root. No glob expansion; taken as-is when it exists.
// Native spelling: a raw `C:/abs/x.cppm` would stay mixed on MSVC
// (see native_path_from_generic) and leak into the CDB.
auto gp = native_path_from_generic(g);
if (gp.is_absolute()) {
std::error_code aec;
if (std::filesystem::is_regular_file(gp, aec)) all_files.insert(gp);
continue;
}
for (auto& p : expand_glob(root, g)) {
all_files.insert(p);
}
}
}
for (auto& p : excluded) all_files.erase(p);
// 0.0.6+: use qualified name (namespace.name) so the validator's
// "module must be prefixed by package name" check works when the
// manifest uses an explicit namespace field with a short name.
const std::string qualifiedName =
manifest.package.namespace_.empty()
? manifest.package.name
: manifest.package.namespace_ + "." + manifest.package.name;
// scan_overrides: author-asserted units bypass the text scan entirely.
// Every override glob must match at least one collected source file —
// an unmatched glob is a typo or a stale declaration, not a no-op.
std::map<std::string, int> overrideHits;
for (auto const& [glob, ov] : manifest.modules.scanOverrides)
overrideHits[glob] = 0;
// [build].flags per-glob entries (G4): append each matching entry's
// flags to the unit IN DECLARATION ORDER (later entries land later on
// the command line — "last flag wins" precedence). `defines` desugar to
// -D on every unit kind. Zero-hit globs warn after the walk (a typo'd
// glob silently doing nothing is the classic trap) — warn, not error:
// a cfg-gated source set can legitimately leave a glob empty on some
// targets.
std::vector<int> globFlagHits(manifest.buildConfig.globFlags.size(), 0);
// mcpp#228: pre-desugar each flag-entry's glob once (not per-file) —
// apply_glob_flags below matches via path_matches_glob directly (it
// isn't a filesystem walk like expand_glob, so there's no walk to bound
// a start point for), so a brace glob here needs its OR-of-branches
// matched explicitly rather than picking up desugaring for free.
std::vector<std::vector<std::string>> globFlagBranches;
globFlagBranches.reserve(manifest.buildConfig.globFlags.size());
for (auto const& gf : manifest.buildConfig.globFlags)
globFlagBranches.push_back(expand_braces(gf.glob));
auto apply_glob_flags = [&](SourceUnit& u) {
for (std::size_t i = 0; i < manifest.buildConfig.globFlags.size(); ++i) {
auto const& gf = manifest.buildConfig.globFlags[i];
bool matched = false;
for (auto const& branch : globFlagBranches[i]) {
if (path_matches_glob(u.path, root, branch)) { matched = true; break; }
}
if (!matched) continue;
++globFlagHits[i];
// defines reach asm units too — via the -D subset the backend
// filters out of packageCflags (no third copy needed here).
for (auto const& d : gf.defines) {
u.packageCflags.push_back("-D" + d);
u.packageCxxflags.push_back("-D" + d);
}
for (auto const& f : gf.cflags) u.packageCflags.push_back(f);
for (auto const& f : gf.cxxflags) u.packageCxxflags.push_back(f);
for (auto const& f : gf.asmflags) u.packageAsmflags.push_back(f);
}
};
for (auto const& f : all_files) {
const mcpp::manifest::ScanOverride* ov = nullptr;
for (auto const& [glob, o] : manifest.modules.scanOverrides) {
if (path_matches_glob(f, root, glob)) {
ov = &o;
++overrideHits[glob];
break;
}
}
if (ov) {
SourceUnit u;
u.path = f;
// mcpp#233: relative to the PACKAGE root (`root`), not the
// primary project root — a path dependency scans with its own
// root here.
u.relPath = std::filesystem::relative(f, root);
u.packageName = qualifiedName;
u.scanOverridden = true;
// A declared unit still gets its role from the same classifier —
// scan_overrides overrides what was SCANNED, not what the file is.
u.kind = mcpp::classify(f, extTable);
if (!ov->provides.empty()) {
u.provides = ModuleId{ov->provides.front()};
// `providesInterface` is deliberately left UNSET. The override
// says which module the file provides; there is nowhere in it
// to say whether the declaration carries `export`, and the two
// spellings decide whether `mcpp pack` may publish the source.
// It used to inherit a `true` default, which is the answer that
// produces no warning — so an implementation partition declared
// here was published in silence. Unknown is the truth, and
// mcpp.pack.interface reports it as such.
if (ov->provides.size() > 1) {
result.errors.push_back(ScanError{f, 0,
"scan_overrides: a unit may declare at most one "
"provided module"});
continue;
}
}
for (auto const& name : ov->imports)
u.requires_.push_back(ModuleId{name});
u.localIncludeDirs = localIncludeDirs;
u.localIncludeDirsAfter = localIncludeDirsAfter;
u.packageCflags = packageCflags;
u.packageCxxflags = packageCxxflags;
apply_glob_flags(u);
normalize_include_flags(root, u.packageCflags);
normalize_include_flags(root, u.packageCxxflags);
normalize_include_flags(root, u.packageAsmflags);
result.graph.units.push_back(std::move(u));
continue;
}
auto r = scan_file(f, qualifiedName, extTable);
if (!r) {
result.errors.push_back(r.error());
continue;
}
// mcpp#233: relative to the PACKAGE root (`root`), matching the
// scan_overrides branch above.
r->relPath = std::filesystem::relative(f, root);
r->localIncludeDirs = localIncludeDirs;
You can’t perform that action at this time.