{{ message }}
Repository navigation
Expand file tree
/
Copy pathaddNavHeaders.mjs
More file actions
338 lines (293 loc) · 13.7 KB
/
Copy pathaddNavHeaders.mjs
File metadata and controls
338 lines (293 loc) · 13.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
import fs from 'node:fs/promises';
import path from 'node:path';
// Prepend a Vercel-style nav header (frontmatter) to every per-page `.md` file
// generated by `@signalwire/docusaurus-plugin-llms-txt`. See issue #2557.
// Also prepends a Claude-style "documentation index" discovery blockquote to the
// page body, pointing agents at `llms.txt`. See issue #2822.
//
// Why post-build (not a remark/rehype plugin): the llms-txt plugin writes
// `llms-full.txt` from the SAME per-page content during its own `postBuild`.
// Editing the per-page `.md` files AFTER that — in npm `postbuild` — keeps the
// header AND the blockquote out of `llms-full.txt` for free, as required by
// both issues.
//
// Data sources (both generated by `docusaurus build`):
// - `.docusaurus/docusaurus-plugin-content-docs/<id>/p/*.json`
// The real left-menu tree (`version.docsSidebars`) — labels, order,
// nesting, hrefs. One file per docs instance (platform, academy, legal,
// openapi). Academy has TWO sidebars (`courses`, `tutorials`).
// - `.docusaurus/docusaurus-plugin-llms-txt/cache.json`
// Route → title fallback for pages absent from every sidebar
// (`/`, `/api`, `/open-source`, `/sdk`) and for section-landing titles
// where the sidebar label is "Home" but the page title is more descriptive.
const BUILD_DIR = path.resolve('build');
const DOCUSAURUS_DIR = path.resolve('.docusaurus');
const CONTENT_DOCS_DIR = path.join(DOCUSAURUS_DIR, 'docusaurus-plugin-content-docs');
const LLMS_CACHE_FILE = path.join(DOCUSAURUS_DIR, 'docusaurus-plugin-llms-txt', 'cache.json');
// Aligned with scripts/joinLlmsFiles.mjs + scripts/indentLlmsFile.mjs so
// preview/PR builds (which set APIFY_DOCS_ABSOLUTE_URL) get correct URLs.
const SITE_URL = process.env.APIFY_DOCS_ABSOLUTE_URL || 'https://docs.apify.com';
// --- URL / label helpers ---------------------------------------------------
function normalizeRoute(href) {
if (typeof href !== 'string' || href.length === 0) return null;
return href.length > 1 && href.endsWith('/') ? href.slice(0, -1) : href;
}
function routeToUrl(route) {
// Defensive: at least one source page (academy cloudflare-challenge) has a
// `slug:` that already ends in `.md`. Don't double up the suffix.
const stripped = route.endsWith('.md') ? route.slice(0, -3) : route;
return `${SITE_URL}${stripped}.md`;
}
// Escape characters that would break a `[Label](url)` markdown link parse.
// Labels are almost always clean; this is a safety net.
function escapeLinkLabel(label) {
return label.replace(/\\/g, '\\\\').replace(/([[\]])/g, '\\$1');
}
function mdLink(label, route) {
return `[${escapeLinkLabel(label)}](${routeToUrl(route)})`;
}
// Synthetic root parent prepended to every page — the agent's link back to the
// full index. It points at `llms.txt` (not a `.md`), so it carries an explicit
// `url` rather than a route. See issue #2557.
const ROOT_PARENT = { label: 'Apify documentation', url: `${SITE_URL}/llms.txt` };
// Agent-discovery blockquote, mirrored from code.claude.com/docs/*.md (issue
// #2822). Prepended to the body of every served per-page `.md`, below the
// nav-header frontmatter. Added here (after llms-full.txt is finalized) so it
// never leaks into llms-full.txt — same timing guarantee as the frontmatter.
const DOC_INDEX_BLOCKQUOTE =
`> ## Documentation index\n` +
`> Fetch the complete documentation index at: ${SITE_URL}/llms.txt\n` +
`> Use this file to discover all available pages before exploring further.\n`;
// A nav item is either {label, route} (a real per-page `.md`) or {label, url}
// (a literal link, e.g. the llms.txt root). Render whichever it carries.
function navLink(item) {
return item.url ? `[${escapeLinkLabel(item.label)}](${item.url})` : mdLink(item.label, item.route);
}
// The section landing (`/platform`, `/academy`, `/api`, ...) is modeled in the
// sidebar as a sibling *link* ("Home"), not an ancestor category, so `walk()`
// never records it as a parent. Derive it from the route's first path segment.
function sectionLandingRoute(route) {
const seg = route.split('/').find(Boolean);
return seg ? `/${seg}` : null;
}
// Full upward chain for a page: llms.txt root, then the section landing, then
// the sidebar-category ancestors from `walk()`. Bounded by site depth (~5), so
// unlike `children` there's no fan-out/size risk. See issue #2557.
function buildParents(route, sidebarParents, titleFor) {
const parents = [ROOT_PARENT];
const landing = sectionLandingRoute(route);
if (landing && landing !== route && !sidebarParents.some((p) => p.route === landing)) {
const label = titleFor(landing);
if (label) parents.push({ label, route: landing });
}
parents.push(...sidebarParents);
return parents;
}
// Quote the `title:` value if it contains characters that would derail a
// downstream YAML-ish parser reading the title line. We intentionally don't
// commit to strict YAML for the body (the `- [Label](url)` list items aren't
// valid YAML scalars), but the title line itself stays parseable.
const TITLE_NEEDS_QUOTE = /[:#&*!|>'"%@`[\]{},\n]|^\s|\s$|^[\d?-]/;
function quoteTitle(value) {
if (TITLE_NEEDS_QUOTE.test(value)) {
return `"${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
}
return value;
}
// --- Sidebar walking -------------------------------------------------------
// Push `{ label, route }` onto `list` if `node` is listed and has a real route.
// Returns whether a route was found, so the caller can decide whether to lift
// children up through an href-less grouping category.
function pushRoutedChild(list, node) {
if (node.unlisted === true) return false;
const route = normalizeRoute(node.href);
if (route) list.push({ label: node.label, route });
return route !== null;
}
/**
* Walk one sidebar tree, populating:
* - `indexMap`: normalizedRoute → { label, parents[], children[] }
* - `flat`: flat DFS list of {route, label} for prev/next within this sidebar
*
* Rules (per approved plan):
* - Skip `unlisted: true` nodes entirely (not in any list, no own entry).
* - Parents = sidebar category ancestors that have an `href`. Href-less
* categories ("Programming", API's "Actors") pass through but aren't
* recorded as parents — they're nav grouping labels, not real pages.
* - Children = the node's direct `items[]`. For href-less category children,
* we recurse ONE level to lift their href-bearing children up, so direct
* grouping is preserved even though the empty label itself is dropped.
* - First occurrence of a route wins (a route in two sidebars only ends up in
* the first-walked one's prev/next sequence).
*/
function walk(node, ancestors, indexMap, flat) {
const isUnlisted = node.unlisted === true;
const hasHref = typeof node.href === 'string' && node.href.length > 0;
const route = hasHref ? normalizeRoute(node.href) : null;
if (route && !isUnlisted && !indexMap.has(route)) {
const parents = ancestors
.filter((a) => a.route && !a.unlisted)
.map((a) => ({ label: a.label, route: a.route }));
const children = [];
if (Array.isArray(node.items)) {
for (const child of node.items) {
if (child.unlisted === true) continue;
// Lift one level through an href-less grouping category so its
// routed children stay grouped under this node.
if (!pushRoutedChild(children, child) && child.type === 'category' && Array.isArray(child.items)) {
for (const grand of child.items) pushRoutedChild(children, grand);
}
}
}
indexMap.set(route, { label: node.label, parents, children });
flat.push({ route, label: node.label });
}
if (Array.isArray(node.items)) {
const next = [...ancestors, { label: node.label, route, unlisted: isUnlisted }];
for (const child of node.items) walk(child, next, indexMap, flat);
}
}
async function loadSidebars() {
const indexMap = new Map();
const flatLists = []; // each is an array of {route, label} for one sidebar
let instances;
try {
instances = await fs.readdir(CONTENT_DOCS_DIR, { withFileTypes: true });
} catch {
console.warn(`addNavHeaders: ${CONTENT_DOCS_DIR} missing — no sidebars to load.`);
return { indexMap, flatLists };
}
for (const dirent of instances) {
if (!dirent.isDirectory()) continue;
const pDir = path.join(CONTENT_DOCS_DIR, dirent.name, 'p');
let propFiles;
try {
propFiles = await fs.readdir(pDir);
} catch {
continue;
}
for (const file of propFiles) {
if (!file.endsWith('.json')) continue;
const data = JSON.parse(await fs.readFile(path.join(pDir, file), 'utf8'));
const sidebars = data?.version?.docsSidebars;
if (!sidebars) continue;
for (const items of Object.values(sidebars)) {
if (!Array.isArray(items)) continue;
const flat = [];
for (const item of items) walk(item, [], indexMap, flat);
flatLists.push(flat);
}
}
}
return { indexMap, flatLists };
}
// --- Cache (title fallback) ------------------------------------------------
async function loadCacheTitles() {
try {
const data = JSON.parse(await fs.readFile(LLMS_CACHE_FILE, 'utf8'));
const map = new Map();
for (const r of data.routes || []) {
if (r.path && r.title) map.set(normalizeRoute(r.path), r.title);
}
return map;
} catch (err) {
console.warn('addNavHeaders: could not read llms-txt cache:', err.message);
return new Map();
}
}
// --- File processing -------------------------------------------------------
async function* walkMd(dir) {
const entries = await fs.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
const full = path.join(dir, entry.name);
if (entry.isDirectory()) {
yield* walkMd(full);
} else if (entry.isFile() && entry.name.endsWith('.md')) {
yield full;
}
}
}
function fileToRoute(filePath) {
const rel = path.relative(BUILD_DIR, filePath);
return `/${rel.replace(/\.md$/, '')}`;
}
function titleFromContent(content) {
const match = content.match(/^#\s+(.+?)\s*$/m);
return match ? match[1].trim() : null;
}
function renderHeader({ title, url, parents, children, previous, next }) {
const lines = ['---'];
lines.push(`title: ${quoteTitle(title)}`);
lines.push(`url: ${url}`);
if (parents && parents.length > 0) {
lines.push('parents:');
for (const p of parents) lines.push(` - ${navLink(p)}`);
}
if (children && children.length > 0) {
lines.push('children:');
for (const c of children) lines.push(` - ${mdLink(c.label, c.route)}`);
}
if (previous) lines.push(`previous: ${mdLink(previous.label, previous.route)}`);
if (next) lines.push(`next: ${mdLink(next.label, next.route)}`);
lines.push('---');
lines.push('');
return `${lines.join('\n')}\n`;
}
async function main() {
try {
await fs.access(BUILD_DIR);
} catch {
console.warn(`addNavHeaders: ${BUILD_DIR} not found — nothing to do.`);
return;
}
const [{ indexMap, flatLists }, cacheTitles] = await Promise.all([loadSidebars(), loadCacheTitles()]);
// Build prev/next from each sidebar's flat order independently. Doesn't
// cross sidebars (e.g. academy's `courses` vs `tutorials`).
const prevNext = new Map();
for (const flat of flatLists) {
for (let i = 0; i < flat.length; i++) {
const nav = {};
if (i > 0) nav.previous = flat[i - 1];
if (i < flat.length - 1) nav.next = flat[i + 1];
if (!prevNext.has(flat[i].route)) prevNext.set(flat[i].route, nav);
}
}
// Resolve a route to its best display title (cache title > sidebar label).
// Used for the section-landing parent, whose sidebar label is "Home".
const titleFor = (r) => cacheTitles.get(r) || indexMap.get(r)?.label || null;
let processed = 0;
let skipped = 0;
const unknownRoutes = [];
for await (const filePath of walkMd(BUILD_DIR)) {
const content = await fs.readFile(filePath, 'utf8');
if (content.startsWith('---\n')) {
skipped++;
continue;
}
const route = fileToRoute(filePath);
const sidebarEntry = indexMap.get(route);
const nav = prevNext.get(route) || {};
// Title preference: cache (true page title, e.g. "Apify platform
// documentation") > sidebar label (sometimes shorter, e.g. "Home") >
// first H1 in content > generic fallback.
const title = cacheTitles.get(route) || sidebarEntry?.label || titleFromContent(content) || 'Untitled';
if (!sidebarEntry && !cacheTitles.has(route)) unknownRoutes.push(route);
const header = renderHeader({
title,
url: routeToUrl(route),
parents: buildParents(route, sidebarEntry?.parents || [], titleFor),
children: sidebarEntry?.children || [],
previous: nav.previous,
next: nav.next,
});
await fs.writeFile(filePath, `${header}${DOC_INDEX_BLOCKQUOTE}\n${content}`, 'utf8');
processed++;
}
console.log(
`addNavHeaders: wrote headers to ${processed} files (${skipped} already had frontmatter, ${unknownRoutes.length} without sidebar or cache title)`,
);
if (unknownRoutes.length > 0) {
console.log(`addNavHeaders: routes without sidebar or cache title:\n ${unknownRoutes.sort().join('\n ')}`);
}
}
await main();
You can’t perform that action at this time.