{{ message }}
Repository navigation
Expand file tree
/
Copy pathcatalog-payload-assets.ts
More file actions
395 lines (363 loc) · 14.6 KB
/
Copy pathcatalog-payload-assets.ts
File metadata and controls
395 lines (363 loc) · 14.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
/**
* Asset handling for catalog preview payloads.
*
* Kept apart from the payload generator so the reference-matching rules can be
* tested without pulling in the renderer: everything here is pure string and
* file work, and the regex below has already been wrong twice in ways only a
* test catches.
*/
import { createHash } from "node:crypto";
import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
import { extname, join, resolve } from "node:path";
export const MIME_TYPES: Record<string, string> = {
".png": "image/png",
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".webp": "image/webp",
".gif": "image/gif",
".svg": "image/svg+xml",
".woff2": "font/woff2",
".woff": "font/woff",
".ttf": "font/ttf",
".otf": "font/otf",
".js": "text/javascript",
".mjs": "text/javascript",
".css": "text/css",
".json": "application/json",
".glb": "model/gltf-binary",
".gltf": "model/gltf+json",
".wav": "audio/wav",
".mp3": "audio/mpeg",
".mp4": "video/mp4",
".webm": "video/webm",
};
/**
* Extensions the docs host actually publishes out of `docs/public`, verified by
* fetching one file of each type from a deployed preview.
*
* Anything here is written once and linked. Anything else — `.glb`, `.js`,
* `.css` — is dropped from the deploy with no build error, so it has to travel
* inside the payload as a data URI instead. Getting this set wrong is not a
* build failure, it is a 404 nobody sees until a reader opens the page.
*/
export const HOSTED_EXTENSIONS = new Set([
".png",
".jpg",
".jpeg",
".webp",
".gif",
".svg",
".woff2",
".woff",
".ttf",
".otf",
".wav",
".mp3",
".mp4",
".webm",
]);
/** A reference with any query string or fragment removed. */
function pathPart(ref: string): string {
return ref.split(/[?#]/)[0] ?? ref;
}
/**
* Local files the composition loads from beside itself. A `srcdoc` iframe has
* no base URL of its own, so these would otherwise resolve against the docs
* page and 404.
*
* Two rules stop this over-matching. The attribute pattern requires a
* non-identifier character before `src`, or a shader assigned to `vertSrc`
* reads as a file reference. And a candidate only counts once it carries an
* extension we know, which drops `url(#noise)` filter references, `blob:`
* juggling, and bare CSS keywords.
*/
export function localReferences(html: string): string[] {
const found = new Set<string>();
const patterns = [
/(?<![\w$])(?:src|href)\s*=\s*["']([^"']+)["']/gi,
/url\(\s*["']?([^"')]+)["']?\s*\)/gi,
];
for (const pattern of patterns) {
for (const [, ref] of html.matchAll(pattern)) {
if (!ref) continue;
// `%23` is an encoded `#`: an in-document SVG filter reference, not a file.
if (/^(https?:|data:|blob:|mailto:|#|%23|\/\/)/i.test(ref)) continue;
if (ref.includes("\n")) continue;
if (!MIME_TYPES[extname(pathPart(ref)).toLowerCase()]) continue;
found.add(ref);
}
}
return [...found];
}
/**
* Files a script loads by name, such as `loader.load("models/iphone.glb")`.
*
* These cannot be told apart from ordinary strings by shape alone, so unlike
* the definite references above they are only acted on when the name resolves
* to a real file in the item's own directory, and a miss is ignored rather than
* failing the item. Without this pass a 3D model stayed a relative path, which
* resolves against the docs page inside a `srcdoc` iframe and 404s: the preview
* renders, just with nothing in it.
*/
export function probableReferences(html: string): string[] {
const definite = new Set(localReferences(html));
const found = new Set<string>();
for (const [, ref] of html.matchAll(/["']([^"'\s]+\.[a-z0-9]{2,5})["']/gi)) {
if (!ref || definite.has(ref)) continue;
if (/^(https?:|data:|blob:|mailto:|#|%23|\/\/)/i.test(ref)) continue;
if (!MIME_TYPES[extname(pathPart(ref)).toLowerCase()]) continue;
found.add(ref);
}
return [...found];
}
export interface AssetResult {
html: string;
/** Written once to the shared directory and linked. */
hosted: number;
/** Carried inside the payload because the host will not publish the type. */
inlined: number;
/** References left as they were, so the caller can refuse the payload. */
unresolved: string[];
}
export interface AssetTarget {
/** Directory shared by every item, so one font is stored once. */
dir: string;
/** URL the directory is served from. */
urlBase: string;
}
/**
* Point every local reference at something the browser can fetch.
*
* Assets are content-addressed and shared across items rather than inlined per
* item. The catalog's fonts are the reason: a handful of files were being
* base64'd into a hundred payloads apiece, which cost tens of megabytes in the
* repository to say the same thing over and over. Hashing also means a
* regenerated payload is byte-identical when nothing changed.
*
* Types the host will not publish still travel as data URIs, because a link to
* a file that 404s is worse than a larger payload.
*/
export function processAssets(html: string, projectDir: string, target: AssetTarget): AssetResult {
const root = resolve(projectDir);
let out = html;
let hosted = 0;
let inlined = 0;
const unresolved: string[] = [];
const definite = localReferences(html);
const candidates = [
...definite.map((ref) => ({ ref, strict: true })),
...probableReferences(html).map((ref) => ({ ref, strict: false })),
];
for (const { ref, strict } of candidates) {
const source = resolve(projectDir, pathPart(ref));
// A composition reaching outside its own directory would pull an arbitrary
// file from the build machine into a published payload.
const contained = source === root || source.startsWith(`${root}/`);
if (!contained || !existsSync(source) || !statSync(source).isFile()) {
// A name a script passed around that turned out not to be a file is just
// a string; only a reference we are sure about counts as a broken one.
if (strict) unresolved.push(ref);
continue;
}
const ext = extname(source).toLowerCase();
const mime = MIME_TYPES[ext];
if (!mime) {
unresolved.push(ref);
continue;
}
const bytes = readFileSync(source);
if (HOSTED_EXTENSIONS.has(ext)) {
const name = `${createHash("sha256").update(bytes).digest("hex").slice(0, 16)}${ext}`;
const dest = join(target.dir, name);
if (!existsSync(dest)) {
mkdirSync(target.dir, { recursive: true });
writeFileSync(dest, bytes);
}
out = out.split(ref).join(`${target.urlBase}/${name}`);
hosted += 1;
continue;
}
out = out.split(ref).join(`data:${mime};base64,${bytes.toString("base64")}`);
inlined += 1;
}
return { html: out, hosted, inlined, unresolved };
}
/** `image/png` -> `.png`, for naming a blob that arrives without a filename. */
const EXTENSION_FOR_MIME: Record<string, string> = Object.entries(MIME_TYPES).reduce(
(acc, [ext, mime]) => (acc[mime] ? acc : { ...acc, [mime]: ext }),
{} as Record<string, string>,
);
/**
* Below this, a data URI is cheaper than the request it would cost to fetch.
* Fonts, the reason this exists, are far above it.
*/
const EXTERNALIZE_MIN_BYTES = 4096;
/**
* Pull large data URIs already baked into the composition out into shared files.
*
* Compositions arrive with their fonts embedded, so `processAssets` never sees
* them as references and they survive into the payload untouched. Across the
* catalog that was 53.8 MB of base64, most of it the same few typefaces
* repeated. Hashing gives one copy per distinct file no matter how many items
* embed it.
*/
export function externalizeDataUris(
html: string,
target: AssetTarget,
): { html: string; externalized: number } {
let externalized = 0;
const out = html.replace(
/data:([a-z0-9.+-]+\/[a-z0-9.+-]+);base64,([A-Za-z0-9+/=]+)/gi,
(whole, mime: string, blob: string) => {
const ext = EXTENSION_FOR_MIME[mime.toLowerCase()];
if (!ext || !HOSTED_EXTENSIONS.has(ext)) return whole;
const bytes = Buffer.from(blob, "base64");
if (bytes.length < EXTERNALIZE_MIN_BYTES) return whole;
const name = `${createHash("sha256").update(bytes).digest("hex").slice(0, 16)}${ext}`;
const dest = join(target.dir, name);
if (!existsSync(dest)) {
mkdirSync(target.dir, { recursive: true });
writeFileSync(dest, bytes);
}
externalized += 1;
return `${target.urlBase}/${name}`;
},
);
return { html: out, externalized };
}
/** Copy a directory's publishable files into `destDir`, flattening one level. */
function walkInto(from: string, rel: string, destDir: string, onCopy: () => void): void {
for (const entry of readdirSync(from, { withFileTypes: true })) {
if (entry.isSymbolicLink()) continue;
const childRel = rel ? `${rel}/${entry.name}` : entry.name;
const childFrom = join(from, entry.name);
if (entry.isDirectory()) {
walkInto(childFrom, childRel, destDir, onCopy);
continue;
}
if (!HOSTED_EXTENSIONS.has(extname(entry.name).toLowerCase())) continue;
const to = join(destDir, childRel);
mkdirSync(join(to, ".."), { recursive: true });
writeFileSync(to, readFileSync(childFrom));
onCopy();
}
}
/**
* Publish the item's own directory and hand back a base URL for it.
*
* Some compositions build their paths at run time —
* `"compositions/components/" + texture + ".png"` for the texture masks, a
* downloaded font under `_remote_media/` — and no amount of scanning the markup
* can see a string that does not exist until a script concatenates it. Serving
* the directory and pointing `<base>` at it makes every relative path the
* composition can invent resolve, whether we predicted it or not.
*
* Only publishable types are copied; a composition needing something the host
* drops still falls back to inlining, which is handled by the caller.
*/
/** Publishable bytes an item would add, counted before anything is written. */
function directoryBytes(dir: string): number {
let total = 0;
for (const entry of readdirSync(dir, { withFileTypes: true })) {
if (entry.isSymbolicLink()) continue;
const child = join(dir, entry.name);
if (entry.isDirectory()) {
total += directoryBytes(child);
continue;
}
if (!HOSTED_EXTENSIONS.has(extname(entry.name).toLowerCase())) continue;
total += statSync(child).size;
}
return total;
}
/**
* What an item may add by publishing its own directory.
*
* The texture sheets are the reason: one ships 66 masks, twice over, for 12 MB
* — against a whole catalog that is otherwise around 30 MB. An item over budget
* keeps the recorded video it already had, which is no worse than before.
*/
export const MAX_HOSTED_DIRECTORY_BYTES = 2_000_000;
export function hostItemDirectory(projectDir: string, destDir: string, urlBase: string): string {
if (directoryBytes(projectDir) > MAX_HOSTED_DIRECTORY_BYTES) return "";
let copied = 0;
const walk = (from: string, rel: string): void => {
for (const entry of readdirSync(from, { withFileTypes: true })) {
// Nothing here should ever leave the prepared copy, and a symlink is the
// one entry that could point back out of it.
if (entry.isSymbolicLink()) continue;
const childRel = rel ? `${rel}/${entry.name}` : entry.name;
const childFrom = join(from, entry.name);
if (entry.isDirectory()) {
walk(childFrom, childRel);
continue;
}
if (!HOSTED_EXTENSIONS.has(extname(entry.name).toLowerCase())) continue;
const to = join(destDir, childRel);
mkdirSync(join(to, ".."), { recursive: true });
writeFileSync(to, readFileSync(childFrom));
copied += 1;
}
};
// Both layouts are published. The prepared copy holds each asset twice, once
// as the registry stores it and once at its install path, and which one a
// composition asks for differs per item: the texture masks use the registry
// spelling, the caption textures the install one. Guessing wrong is a 404 at
// run time, so the budget below is what keeps the cost in check instead.
walk(projectDir, "");
// The compiler pulls remote media into `_downloads/` but rewrites references
// as if the document sat inside it, so `_remote_media/x.woff2` has to resolve
// from the item root too. Mirroring rather than moving keeps both spellings
// working, and the files are content-identical either way.
const downloads = join(projectDir, "_downloads");
if (existsSync(downloads) && statSync(downloads).isDirectory()) {
walkInto(downloads, "", destDir, () => (copied += 1));
}
return copied > 0 ? urlBase : "";
}
/**
* Point the document at that base, ahead of anything that could resolve a URL.
*
* A `srcdoc` document has no base of its own, so relative paths resolve against
* the docs page and 404. The tag has to be the first thing in the head: a
* `<base>` only governs what follows it.
*/
export function withBaseHref(html: string, href: string): string {
if (!href) return html;
const tag = `<base href="${href}">`;
if (/<head[^>]*>/i.test(html)) return html.replace(/<head([^>]*)>/i, `<head$1>${tag}`);
if (/<html[^>]*>/i.test(html))
return html.replace(/<html([^>]*)>/i, `<html$1><head>${tag}</head>`);
return `${tag}${html}`;
}
/**
* Turn a mounted sub-composition into one the browser can fetch on its own.
*
* An interactive preview ships uncompiled so its values stay changeable, which
* leaves `data-composition-src` pointing at a sibling `.html`. That is the one
* type the docs host will not publish, so the file is carried inline as a data
* URI instead: the runtime still mounts it at run time, and the values on the
* host still govern it.
*/
export function inlineMountedComposition(html: string, projectDir: string): string {
return html.replace(
/data-composition-src=(["'])([^"']+)\1/gi,
(whole, quote: string, ref: string) => {
if (/^(https?:|data:)/i.test(ref)) return whole;
const source = resolve(projectDir, ref.replace(/^\.\//, "").split(/[?#]/)[0] ?? ref);
if (!source.startsWith(resolve(projectDir)) || !existsSync(source)) return whole;
const encoded = readFileSync(source).toString("base64");
return `data-composition-src=${quote}data:text/html;base64,${encoded}${quote}`;
},
);
}
/**
* Drop the values a demo pinned onto its own mount.
*
* A demo picks striking values to show itself off, and the runtime layers those
* over anything the reader chooses, so every control looked dead. Removing them
* leaves the declared defaults, which is the state the panel starts in.
*/
export function clearPinnedVariableValues(html: string): string {
return html.replace(/\sdata-variable-values=(?:"[^"]*"|'[^']*')/gi, "");
}
You can’t perform that action at this time.