test_runner: add option to rerun only failed tests · nodejs/node@e076f78 · GitHub
Skip to content

Commit e076f78

Browse files
MoLowtargos
authored andcommitted
test_runner: add option to rerun only failed tests
PR-URL: #59443 Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com> Reviewed-By: Pietro Marchini <pietro.marchini94@gmail.com> Reviewed-By: Chemi Atlow <chemi@atlow.co.il>
1 parent 2b7a7a5 commit e076f78

13 files changed

Lines changed: 359 additions & 2 deletions

File tree

doc/api/cli.md

Lines changed: 16 additions & 0 deletions

doc/api/test.md

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,46 @@ test('skip() method with message', (t) => {
153153
});
154154
```
155155

156+
## Rerunning failed tests
157+
158+
The test runner supports persisting the state of the run to a file, allowing
159+
the test runner to rerun failed tests without having to re-run the entire test suite.
160+
Use the [`--test-rerun-failures`][] command-line option to specify a file path where the
161+
state of the run is stored. if the state file does not exist, the test runner will
162+
create it.
163+
the state file is a JSON file that contains an array of run attempts.
164+
Each run attempt is an object mapping successful tests to the attempt they have passed in.
165+
The key identifying a test in this map is the test file path, with the line and column where the test is defined.
166+
in a case where a test defined in a specific location is run multiple times,
167+
for example within a function or a loop,
168+
a counter will be appended to the key, to disambiguate the test runs.
169+
note changing the order of test execution or the location of a test can lead the test runner
170+
to consider tests as passed on a previous attempt,
171+
meaning `--test-rerun-failures` should be used when tests run in a deterministic order.
172+
173+
example of a state file:
174+
175+
```json
176+
[
177+
{
178+
"test.js:10:5": { "passed_on_attempt": 0, "name": "test 1" },
179+
},
180+
{
181+
"test.js:10:5": { "passed_on_attempt": 0, "name": "test 1" },
182+
"test.js:20:5": { "passed_on_attempt": 1, "name": "test 2" }
183+
}
184+
]
185+
```
186+
187+
in this example, there are two run attempts, with two tests defined in `test.js`,
188+
the first test succeeded on the first attempt, and the second test succeeded on the second attempt.
189+
190+
When the `--test-rerun-failures` option is used, the test runner will only run tests that have not yet passed.
191+
192+
```bash
193+
node --test-rerun-failures /path/to/state/file
194+
```
195+
156196
## TODO tests
157197

158198
Individual tests can be marked as flaky or incomplete by passing the `todo`
@@ -1342,6 +1382,9 @@ added:
13421382
- v18.9.0
13431383
- v16.19.0
13441384
changes:
1385+
- version: REPLACEME
1386+
pr-url: https://github.com/nodejs/node/pull/59443
1387+
description: Added a rerunFailuresFilePath option.
13451388
- version: v23.0.0
13461389
pr-url: https://github.com/nodejs/node/pull/54705
13471390
description: Added the `cwd` option.
@@ -1432,6 +1475,10 @@ changes:
14321475
that specifies the index of the shard to run. This option is _required_.
14331476
* `total` {number} is a positive integer that specifies the total number
14341477
of shards to split the test files to. This option is _required_.
1478+
* `rerunFailuresFilePath` {string} A file path where the test runner will
1479+
store the state of the tests to allow rerunning only the failed tests on a next run.
1480+
see \[Rerunning failed tests]\[] for more information.
1481+
**Default:** `undefined`.
14351482
* `coverage` {boolean} enable [code coverage][] collection.
14361483
**Default:** `false`.
14371484
* `coverageExcludeGlobs` {string|Array} Excludes specific files from code coverage
@@ -3219,6 +3266,8 @@ Emitted when a test is enqueued for execution.
32193266
* `cause` {Error} The actual error thrown by the test.
32203267
* `type` {string|undefined} The type of the test, used to denote whether
32213268
this is a suite.
3269+
* `attempt` {number|undefined} The attempt number of the test run,
3270+
present only when using the [`--test-rerun-failures`][] flag.
32223271
* `file` {string|undefined} The path of the test file,
32233272
`undefined` if test was run through the REPL.
32243273
* `line` {number|undefined} The line number where the test is defined, or
@@ -3243,6 +3292,10 @@ The corresponding execution ordered event is `'test:complete'`.
32433292
* `duration_ms` {number} The duration of the test in milliseconds.
32443293
* `type` {string|undefined} The type of the test, used to denote whether
32453294
this is a suite.
3295+
* `attempt` {number|undefined} The attempt number of the test run,
3296+
present only when using the [`--test-rerun-failures`][] flag.
3297+
* `passed_on_attempt` {number|undefined} The attempt number the test passed on,
3298+
present only when using the [`--test-rerun-failures`][] flag.
32463299
* `file` {string|undefined} The path of the test file,
32473300
`undefined` if test was run through the REPL.
32483301
* `line` {number|undefined} The line number where the test is defined, or
@@ -3946,6 +3999,7 @@ Can be used to abort test subtasks when the test has been aborted.
39463999
[`--test-only`]: cli.md#--test-only
39474000
[`--test-reporter-destination`]: cli.md#--test-reporter-destination
39484001
[`--test-reporter`]: cli.md#--test-reporter
4002+
[`--test-rerun-failures`]: cli.md#--test-rerun-failures
39494003
[`--test-skip-pattern`]: cli.md#--test-skip-pattern
39504004
[`--test-update-snapshots`]: cli.md#--test-update-snapshots
39514005
[`--test`]: cli.md#--test

doc/node-config-schema.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -443,6 +443,9 @@
443443
}
444444
]
445445
},
446+
"test-rerun-failures": {
447+
"type": "string"
448+
},
446449
"test-shard": {
447450
"type": "string"
448451
},
@@ -695,6 +698,9 @@
695698
}
696699
]
697700
},
701+
"test-rerun-failures": {
702+
"type": "string"
703+
},
698704
"test-shard": {
699705
"type": "string"
700706
},

doc/node.1

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -490,6 +490,10 @@ A test reporter to use when running tests.
490490
.It Fl -test-reporter-destination
491491
The destination for the corresponding test reporter.
492492
.
493+
.It Fl -test-rerun-failures
494+
Configures the tests runner to persist the state of tests to allow
495+
rerunning only failed tests.
496+
.
493497
.It Fl -test-only
494498
Configures the test runner to only execute top level tests that have the `only`
495499
option set.

lib/internal/test_runner/harness.js

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,10 @@ const {
2626
reporterScope,
2727
shouldColorizeTestFiles,
2828
setupGlobalSetupTeardownFunctions,
29+
parsePreviousRuns,
2930
} = require('internal/test_runner/utils');
31+
const { PassThrough, compose } = require('stream');
32+
const { reportReruns } = require('internal/test_runner/reporter/rerun');
3033
const { queueMicrotask } = require('internal/process/task_queues');
3134
const { TIMEOUT_MAX } = require('internal/timers');
3235
const { clearInterval, setInterval } = require('timers');
@@ -69,6 +72,7 @@ function createTestTree(rootTestOptions, globalOptions) {
6972
shouldColorizeTestFiles: shouldColorizeTestFiles(globalOptions.destinations),
7073
teardown: null,
7174
snapshotManager: null,
75+
previousRuns: null,
7276
isFilteringByName,
7377
isFilteringByOnly,
7478
async runBootstrap() {
@@ -203,6 +207,25 @@ function collectCoverage(rootTest, coverage) {
203207
return summary;
204208
}
205209

210+
function setupFailureStateFile(rootTest, globalOptions) {
211+
if (!globalOptions.rerunFailuresFilePath) {
212+
return;
213+
}
214+
rootTest.harness.previousRuns = parsePreviousRuns(globalOptions.rerunFailuresFilePath);
215+
if (rootTest.harness.previousRuns === null) {
216+
rootTest.diagnostic(`Warning: The rerun failures file at ` +
217+
`${globalOptions.rerunFailuresFilePath} is not a valid rerun file. ` +
218+
'The test runner will not be able to rerun failed tests.');
219+
rootTest.harness.success = false;
220+
process.exitCode = kGenericUserError;
221+
return;
222+
}
223+
if (!process.env.NODE_TEST_CONTEXT) {
224+
const reporter = reportReruns(rootTest.harness.previousRuns, globalOptions);
225+
compose(rootTest.reporter, reporter).pipe(new PassThrough());
226+
}
227+
}
228+
206229
function setupProcessState(root, globalOptions) {
207230
const hook = createHook({
208231
__proto__: null,
@@ -230,6 +253,9 @@ function setupProcessState(root, globalOptions) {
230253
const rejectionHandler =
231254
createProcessEventHandler('unhandledRejection', root);
232255
const coverage = configureCoverage(root, globalOptions);
256+
257+
setupFailureStateFile(root, globalOptions);
258+
233259
const exitHandler = async (kill) => {
234260
if (root.subtests.length === 0 && (root.hooks.before.length > 0 || root.hooks.after.length > 0)) {
235261
// Run global before/after hooks in case there are no tests
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
'use strict';
2+
3+
const {
4+
ArrayPrototypePush,
5+
JSONStringify,
6+
} = primordials;
7+
const { relative } = require('path');
8+
const { writeFileSync } = require('fs');
9+
10+
function reportReruns(previousRuns, globalOptions) {
11+
return async function reporter(source) {
12+
const obj = { __proto__: null };
13+
const disambiguator = { __proto__: null };
14+
15+
for await (const { type, data } of source) {
16+
if (type === 'test:pass') {
17+
let identifier = `${relative(globalOptions.cwd, data.file)}:${data.line}:${data.column}`;
18+
if (disambiguator[identifier] !== undefined) {
19+
identifier += `:(${disambiguator[identifier]})`;
20+
disambiguator[identifier] += 1;
21+
} else {
22+
disambiguator[identifier] = 1;
23+
}
24+
obj[identifier] = {
25+
__proto__: null,
26+
name: data.name,
27+
passed_on_attempt: data.details.passed_on_attempt ?? data.details.attempt,
28+
};
29+
}
30+
}
31+
32+
ArrayPrototypePush(previousRuns, obj);
33+
writeFileSync(globalOptions.rerunFailuresFilePath, JSONStringify(previousRuns, null, 2), 'utf8');
34+
};
35+
};
36+
37+
module.exports = {
38+
__proto__: null,
39+
reportReruns,
40+
};

lib/internal/test_runner/runner.js

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ function getRunArgs(path, { forceExit,
148148
only,
149149
argv: suppliedArgs,
150150
execArgv,
151+
rerunFailuresFilePath,
151152
root: { timeout },
152153
cwd }) {
153154
const processNodeOptions = getOptionsAsFlagsFromBinding();
@@ -170,6 +171,9 @@ function getRunArgs(path, { forceExit,
170171
if (timeout != null) {
171172
ArrayPrototypePush(runArgs, `--test-timeout=${timeout}`);
172173
}
174+
if (rerunFailuresFilePath) {
175+
ArrayPrototypePush(runArgs, `--test-rerun-failures=${rerunFailuresFilePath}`);
176+
}
173177

174178
ArrayPrototypePushApply(runArgs, execArgv);
175179

@@ -588,6 +592,7 @@ function run(options = kEmptyObject) {
588592
execArgv = [],
589593
argv = [],
590594
cwd = process.cwd(),
595+
rerunFailuresFilePath,
591596
} = options;
592597

593598
if (files != null) {
@@ -620,6 +625,10 @@ function run(options = kEmptyObject) {
620625
);
621626
}
622627

628+
if (rerunFailuresFilePath) {
629+
validatePath(rerunFailuresFilePath, 'options.rerunFailuresFilePath');
630+
}
631+
623632
if (shard != null) {
624633
validateObject(shard, 'options.shard');
625634
// Avoid re-evaluating the shard object in case it's a getter
@@ -702,6 +711,7 @@ function run(options = kEmptyObject) {
702711
coverage,
703712
coverageExcludeGlobs,
704713
coverageIncludeGlobs,
714+
rerunFailuresFilePath,
705715
lineCoverage: lineCoverage,
706716
branchCoverage: branchCoverage,
707717
functionCoverage: functionCoverage,
@@ -735,6 +745,7 @@ function run(options = kEmptyObject) {
735745
isolation,
736746
argv,
737747
execArgv,
748+
rerunFailuresFilePath,
738749
};
739750

740751
if (isolation === 'process') {

lib/internal/test_runner/test.js

Lines changed: 31 additions & 0 deletions

0 commit comments

Comments
 (0)