{{ message }}
Repository navigation
337 lines (308 loc) · 16.4 KB
/
Copy pathoffline-suite.yml
File metadata and controls
337 lines (308 loc) · 16.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
name: offline-suite
# Everything here runs with NO credentials and no network egress to a vendor.
# That is the property worth protecting: a reviewer, or a fork, must be able to
# re-run every check without being given a key. The collector is the only part of
# this kit that spends anything, and it is deliberately not exercised here.
#
# Two platforms, because this kit's behaviour genuinely differs across them: the
# Firecrawl transport takes a `.cmd` route on Windows with its own argument guard,
# paths and line endings are handled explicitly, and git's executable bit does not
# exist on Windows at all. That last one is not hypothetical - `githooks/pre-commit`
# shipped as mode 100644, git silently skips a non-executable hook, and the machine
# it was authored on COULD NOT have detected it. The first Ubuntu run did, within
# minutes. One platform is one platform's worth of evidence.
on:
push:
branches: [main]
pull_request:
branches: [main]
workflow_dispatch:
# Third-party actions are pinned to COMMIT SHAs, not tags.
#
# A tag is mutable. `actions/checkout@v7` is whatever the v7 tag points at today, and
# whoever can move that tag can run code in this workflow with read access to the
# repository - on a project whose entire value is that its claims are checkable, trusting
# a moving reference for the environment those claims are produced in is the wrong trade.
#
# The version each SHA corresponds to is in the comment beside it. Updating is a reviewed
# change, not something that happens while nobody is looking. Re-resolve to the COMMIT a
# tag names, never the tag object - `gh api .../git/ref/tags/X --jq .object.sha`, the
# instruction this comment used to give, returns the tag object for an annotated tag:
# git ls-remote https://github.com/actions/checkout 'refs/tags/v7.0.1*'
# An annotated tag lists twice; the line ending in ^{} is the commit. Check `runs.using`
# in that commit's action.yml too. How the current pins were chosen, and why they are
# v7: docs/decisions/2026-09-26-actions-node24.
permissions:
contents: read
jobs:
platform:
name: platform
strategy:
# Never cancel the other legs. A Windows-only failure is the most valuable
# result this workflow can produce, and killing macOS to report it faster
# throws away the comparison that makes it diagnosable.
fail-fast: false
matrix:
# Windows and Linux. macOS is deliberately NOT here: it is not a platform this
# kit is maintained for, and a leg nobody will fix is a leg that teaches people
# to ignore red. Its one run was not wasted - it found the symlink containment
# defect in audit.mjs, which is fixed and affects Linux too wherever a project
# sits under a symlink. Add macos-latest here if that changes.
#
# ubuntu-26.04 is TEMPORARY (ADR-0043): ubuntu-latest moves from 24.04 to 26.04
# between 2026-10-19 and 2026-11-19, and GitHub's advice is to test on 26.04 before
# that. Linux is still the one policy; this checks it on the image it is about to
# become. Remove the leg once ubuntu-latest resolves to 26.04 - it is then a duplicate.
os: [ubuntu-latest, ubuntu-26.04, windows-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 30
defaults:
run:
# Windows runners default to pwsh. Every step here is POSIX shell, and
# `shell: bash` gets Git Bash on Windows so one script serves all three.
shell: bash
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1, resolved 2026-09-26
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0, resolved 2026-09-26
with:
node-version: '22'
# Three of the conformance runners ship twice, in Node and in Python, and
# must agree per vector. Without Python those tests report UNSUP and block.
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0, resolved 2026-09-26
with:
python-version: '3.11'
- name: versions
run: |
echo "os: ${{ matrix.os }} ($(uname -s 2>/dev/null || echo windows))"
node --version
"$(command -v python || command -v python3)" --version
git --version
- name: no credentials are present
run: |
for v in SERPAPI_API_KEY FIRECRAWL_API_KEY TAVILY_API_KEY; do
if [ -n "${!v:-}" ]; then echo "$v is set in CI; the offline suite must not need one"; exit 1; fi
done
echo "no vendor credentials in the environment"
# The defect that motivated this matrix. git tracks an executable bit that
# Windows does not have, so this is asserted only where it is meaningful -
# asserting it on Windows would fail for a reason unrelated to the bug.
- name: the commit hook is executable
if: runner.os != 'Windows'
run: |
mode=$(git ls-files -s fetchproof/githooks/pre-commit | awk '{print $1}')
if [ "$mode" != "100755" ]; then
echo "fetchproof/githooks/pre-commit is mode $mode, not 100755"
echo "git silently SKIPS a non-executable hook, which ADR-0001 reports as a clean commit"
echo "fix: git update-index --chmod=+x fetchproof/githooks/pre-commit"
exit 1
fi
echo "pre-commit is 100755"
# .gitattributes pins `* text=auto eol=lf`. If a checkout ever lands CRLF, the
# body hashes in the fetch ledger stop recomputing and the corpus fails its own
# gate. Windows is the only platform where this can regress, so it is the only
# platform where the check carries information.
- name: the checkout is LF, even on Windows
if: runner.os == 'Windows'
run: |
if git ls-files --eol fetchproof/conformance fetchproof/schemas | grep -v 'w/lf'; then
echo "the working tree has non-LF files despite .gitattributes eol=lf"
exit 1
fi
echo "working tree is LF"
- name: selftest
env:
# Structured result, so the summary below reads DATA rather than scraping prose.
FETCHPROOF_RESULT_FILE: ${{ runner.temp }}/suite-result.json
run: node fetchproof/bin/selftest.mjs
# Publish the count where a reviewer can read it WITHOUT running anything.
#
# An external audit of this repository could not confirm the claimed test count
# from the source alone: the number is only produced by executing the suite, and
# `selftest.mjs` enforcing it proves consistency, not the value. The job summary is
# the cheapest place a third party can read what actually ran, per platform, on a
# machine that is not the author's.
- name: publish the count
if: always()
run: |
result="$RUNNER_TEMP/suite-result.json"
if [ -f "$result" ]; then
line=$(node -e '
const r = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
const bits = [`${r.passed} passed`, `${r.failures} failed`];
if (r.unsupported) bits.push(`${r.unsupported} unsupported`);
process.stdout.write(`${bits.join(", ")} in ${r.seconds}s across ${r.files} files (node ${r.node})`);
' "$result")
else
# No file means the runner never reached its own reporting - a startup error,
# an import-time throw, an OOM kill. That is a DIFFERENT fact from a failing
# suite, and scraping stdout could not tell the two apart.
line='the runner produced no result file: it crashed before reporting'
fi
echo "| ${{ matrix.os }} | \`$line\` |" >> "$GITHUB_STEP_SUMMARY"
- name: preflight
run: node fetchproof/bin/preflight.mjs
- name: release evidence examples
run: node fetchproof/examples/release-evidence/run-example.mjs
- name: cross-language conformance
run: |
set -e
# RUNNER_TEMP, not /tmp: on a Windows runner /tmp is Git Bash's own mount and
# the two languages disagree about what that path means.
tmp="${RUNNER_TEMP:-/tmp}"
# `python` is what setup-python provides; a stock Ubuntu box has only `python3`,
# and this step should run there verbatim, as the suite does (2026-09-28).
py="$(command -v python || command -v python3)"
for pair in \
"ledger-conformance.mjs:ledger_conformance.py:qualification-ledger-vectors.json" \
"fi-sidecar-conformance.mjs:fi_sidecar_conformance.py:fi-sidecar-evidence-manifest-vectors.json" \
"property-vector-conformance.mjs:property_vector_conformance.py:property-graph-hash-vectors.json"
do
n="${pair%%:*}"; rest="${pair#*:}"; p="${rest%%:*}"; v="${rest##*:}"
node "fetchproof/bin/$n" --vectors "fetchproof/conformance/$v" --json > "$tmp/node.json"
"$py" "fetchproof/bin/$p" --vectors "fetchproof/conformance/$v" --json > "$tmp/py.json"
node -e '
const fs = require("fs");
const rows = (f) => {
const r = JSON.parse(fs.readFileSync(f, "utf8"));
return JSON.stringify((r.vectors ?? r.rows ?? []).map((x) => [x.vectorId ?? x.id, x.result]));
};
// Report hashes are NOT compared: reportSha256 covers implementation.runtime,
// so it differs between languages by design. Equivalence is per vector.
if (rows(process.argv[2] + "/node.json") !== rows(process.argv[2] + "/py.json")) {
console.error("node and python disagree for " + process.argv[1]);
process.exit(1);
}
console.log("agree: " + process.argv[1]);
' "$v" "$tmp"
done
- name: validator inputs are unchanged
run: |
if ! git diff --quiet -- fetchproof/conformance fetchproof/schemas; then
echo "a validator mutated its own inputs"
git --no-pager diff --stat -- fetchproof/conformance fetchproof/schemas
exit 1
fi
echo "conformance/ and schemas/ are byte-for-byte unchanged"
# The whole suite is read-only. A check that dirties the tree it is checking is a
# contradiction, and this repository has shipped that defect twice: a tracked
# .diagnostics.jsonl, and __pycache__ appearing the moment the Python runners began
# importing a shared module. Both were invisible until someone looked at git.
# Found 2026-09-29 (break-test PR #140): running the command research/TIMELINE.md names
# rewrote it with the machine's absolute path, and export-warc dropped an un-ignored
# 400 KB file in the root. Both are fixed; this proves it on every platform. The step
# below then fails if either changed a tracked file or left an untracked one.
- name: the corpus's derived files regenerate to their committed bytes
run: |
node fetchproof/bin/timeline.mjs
node fetchproof/bin/export-warc.mjs
- name: the suite left the working tree clean
run: |
if [ -n "$(git status --porcelain)" ]; then
echo "running the suite modified the working tree:"
git status --porcelain
exit 1
fi
echo "working tree clean"
# The single required status check.
#
# Branch protection names a check literally, and a matrix does NOT produce one: it
# produces `platform (ubuntu-latest)`, `platform (windows-latest)` and so on. If the
# matrix were named `suite`, the required check `suite` would never appear again and
# every pull request would wait forever for a job that cannot run. This job keeps that
# name attached to one thing that either passes or does not.
# The other Node lines the kit promises. The platform matrix above runs Node 22 on Linux and
# Windows; this runs 24 (Active LTS) and 26 (Current) on Linux, so every supported line is
# tested on every commit (docs/decisions/2026-09-27-node-support, ADR-0046). Linux only:
# the platform-specific checks above are about the OS, not the Node line.
node-lines:
name: node
strategy:
fail-fast: false
matrix:
node: ['24', '26']
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1, resolved 2026-09-26
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0, resolved 2026-09-26
with:
node-version: ${{ matrix.node }}
# The conformance runners ship in Node and Python and must agree; without Python
# those tests report UNSUP and block, exactly as on the platform legs.
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0, resolved 2026-09-26
with:
python-version: '3.11'
- name: versions
run: node --version
- name: selftest
run: node fetchproof/bin/selftest.mjs
# The suite as a "Download ZIP" user runs it: from a tree with no .git. Every other leg runs
# from a checkout, so a test that assumed one could not fail here - and one did (found
# 2026-09-30, break-test: warc.test.mjs ran `git check-ignore` in the repository and read
# git's exit 128 as "the ignore rule is missing"). `git archive` is what GitHub's ZIP is
# built from. Linux and Node 22 only: the property is the missing .git, not the platform.
#
# The folder's name has a SPACE in it, as "Downloads/Fetchproof main" and "C:\Users\John
# Smith" do. Every other leg runs from a path without one, so a test that assumed a bare
# kit path could not fail - and one did (found 2026-09-30, break-test: the audit test
# matched `audit.mjs --zip`, while the kit correctly prints `".../audit.mjs" --zip`).
archive-tree:
name: archive tree
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1, resolved 2026-09-26
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0, resolved 2026-09-26
with:
node-version: '22'
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0, resolved 2026-09-26
with:
python-version: '3.11'
- name: selftest from a git archive tree
run: |
set -e
tree="$RUNNER_TEMP/archive tree"
mkdir -p "$tree"
git archive HEAD | tar -x -C "$tree"
if [ -e "$tree/.git" ]; then echo "the exported tree has a .git; it is not what a ZIP user gets"; exit 1; fi
cd "$tree"
node fetchproof/bin/selftest.mjs
suite:
name: suite
needs: [platform, node-lines, archive-tree]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: every platform passed
run: |
result="${{ needs.platform.result }}"
if [ "$result" != "success" ]; then
echo "the platform matrix did not pass: $result"
echo "open the platform legs above; a single-platform failure is the point of this matrix"
exit 1
fi
lines="${{ needs.node-lines.result }}"
if [ "$lines" != "success" ]; then
echo "a Node line did not pass: $lines"
echo "open the node legs above; the kit promises every line it names in the README"
exit 1
fi
archive="${{ needs.archive-tree.result }}"
if [ "$archive" != "success" ]; then
echo "the suite did not pass from a git archive tree: $archive"
echo "open the archive tree job; a test there assumes a .git that a ZIP download lacks"
exit 1
fi
# Deliberately does NOT list the platforms.
#
# It used to, and the list went stale: it said "ubuntu, windows and macos all
# passed" for a while after the macOS leg was dropped, so a green run reported
# a result that did not exist. A test was added to keep the literal list in
# step with the matrix - but that only guards the drift, it does not remove it,
# because there were still two places saying which platforms run.
#
# Now there is one. The matrix above is the only statement of which platforms
# run, and this line cannot disagree with it because it says nothing about them.
# Which legs ran is on the job list, generated by Actions, which cannot be stale.
echo "every platform in the matrix passed"
You can’t perform that action at this time.