feat: first commit flaky

doc/api/test.md

@@ -351,6 +351,85 @@ it.todo('should do the thing', { expectFailure: true }, () => {
 });
 ```
 
+## Flaky tests
+
+This flag causes a test or suite to be re-run a number of times until it
+either passes or has not passed after the final re-try.
+
+When `flaky` is `true`, the test harness re-tries the test up to the default
+number of times (20), inclusive.
+
+When `flaky` is a positive integer, the test harness re-tries the test up to
+the specified number of times, inclusive.
+
+When `flaky` is falsy (the default), the test harness does not re-try the test.
+
+When both a suite and an included test specify the `flaky` flag, the
+test's `flaky` value wins.
+
+```js
+it.flaky('should do something', () => {
+  // This test will be retried up to 20 times if it fails
+});
+
+it('may take several times', { flaky: true }, () => {
+  // Also retries up to 20 times
+});
+
+it('may also take several times', { flaky: 5 }, () => {
+  // Retries up to 5 times
+});
+
+describe.flaky('flaky suite', () => {
+  it('inherits flaky from suite', () => {
+    // Retried up to 20 times (inherited from suite)
+  });
+
+  it('not flaky', { flaky: false }, () => {
+    // Not retried, overrides suite setting
+  });
+});
+```
+
+When a test marked `flaky` passes after retries, the number of re-tries taken
+is reported with that test.
+
+`skip` and `todo` take precedence over `flaky`.
+
+## `describe()` and `it()` aliases
+
+Suites and tests can also be written using the `describe()` and `it()`
+functions. [`describe()`][] is an alias for [`suite()`][], and [`it()`][] is an
+alias for [`test()`][].
+
+```js
+describe('A thing', () => {
+  it('should work', () => {
+    assert.strictEqual(1, 1);
+  });
+
+  it('should be ok', () => {
+    assert.strictEqual(2, 2);
+  });
+
+  describe('a nested thing', () => {
+    it('should work', () => {
+      assert.strictEqual(3, 3);
+    });
+  });
+});
+```
+
+`describe()` and `it()` are imported from the `node:test` module.
+
+```mjs
+import { describe, it } from 'node:test';
+```
+
+```cjs
+const { describe, it } = require('node:test');
+```
+
 ## `only` tests
 
 If Node.js is started with the [`--test-only`][] command-line option, or test
@@ -1793,6 +1872,16 @@ added:
 Shorthand for marking a suite as `only`. This is the same as
 [`suite([name], { only: true }[, fn])`][suite options].
 
+## `suite.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+ - REPLACEME
+-->
+
+Shorthand for marking a suite as flaky. This is the same as
+[`suite([name], { flaky: true }[, fn])`][suite options].
+
 ## `test([name][, options][, fn])`
 
 <!-- YAML
@@ -1828,6 +1917,10 @@ changes:
     thread. If `false`, only one test runs at a time.
     If unspecified, subtests inherit this value from their parent.
     **Default:** `false`.
+  * `flaky` {boolean|number} If truthy, the test is re-tried up to the
+    specified number of times (or `20` if `true`) until it passes. If the test
+    passes after retries, the number of retries taken is reported. When both a
+    suite and an included test specify the `flaky` flag, the test's value wins.
   * `expectFailure` {boolean|string|RegExp|Function|Object|Error} If truthy, the
     test is expected to fail. If a non-empty string is provided, that string is displayed
     in the test results as the reason why the test is expected to fail. If a {RegExp|Function|Object|Error}
@@ -1907,6 +2000,16 @@ same as [`test([name], { todo: true }[, fn])`][it options].
 Shorthand for marking a test as `only`,
 same as [`test([name], { only: true }[, fn])`][it options].
 
+## `test.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+ - REPLACEME
+-->
+
+Shorthand for marking a test as flaky,
+same as [`test([name], { flaky: true }[, fn])`][it options].
+
 ## `describe([name][, options][, fn])`
 
 Alias for [`suite()`][].
@@ -1934,6 +2037,16 @@ added:
 Shorthand for marking a suite as `only`. This is the same as
 [`describe([name], { only: true }[, fn])`][describe options].
 
+## `describe.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+ - REPLACEME
+-->
+
+Shorthand for marking a suite as flaky. This is the same as
+[`describe([name], { flaky: true }[, fn])`][describe options].
+
 ## `it([name][, options][, fn])`
 
 <!-- YAML
@@ -1973,6 +2086,16 @@ added:
 Shorthand for marking a test as `only`,
 same as [`it([name], { only: true }[, fn])`][it options].
 
+## `it.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+ - REPLACEME
+-->
+
+Shorthand for marking a test as flaky,
+same as [`it([name], { flaky: true }[, fn])`][it options].
+
 ## `before([fn][, options])`
 
 <!-- YAML
@@ -3527,6 +3650,8 @@ Emitted when a test is enqueued for execution.
   * `testNumber` {number} The ordinal number of the test.
   * `todo` {string|boolean|undefined} Present if [`context.todo`][] is called
   * `skip` {string|boolean|undefined} Present if [`context.skip`][] is called
+  * `retryCount` {number|undefined} The number of retries taken for a
+    flaky test. Present when a test is marked as flaky.
 
 Emitted when a test fails.
 This event is guaranteed to be emitted in the same order as the tests are
@@ -3586,6 +3711,8 @@ since the parent runner only knows about file-level tests. When using
   * `testNumber` {number} The ordinal number of the test.
   * `todo` {string|boolean|undefined} Present if [`context.todo`][] is called
   * `skip` {string|boolean|undefined} Present if [`context.skip`][] is called
+  * `retryCount` {number|undefined} The number of retries taken for a
+    flaky test. Present when a test is marked as flaky and passed after retries.
 
 Emitted when a test passes.
 This event is guaranteed to be emitted in the same order as the tests are
@@ -4362,6 +4489,9 @@ changes:
     If `false`, it would only run one test at a time.
     If unspecified, subtests inherit this value from their parent.
     **Default:** `null`.
+  * `flaky` {boolean|number} If truthy, the test is re-tried up to the
+    specified number of times (or `20` if `true`) until it passes.
+    **Default:** `false`.
   * `only` {boolean} If truthy, and the test context is configured to run
     `only` tests, then this test will be run. Otherwise, the test is skipped.
     **Default:** `false`.

lib/internal/test_runner/harness.js

@@ -62,6 +62,7 @@ function createTestTree(rootTestOptions, globalOptions) {
         failed: 0,
         passed: 0,
         cancelled: 0,
+        flaky: 0,
         skipped: 0,
         todo: 0,
         topLevel: 0,
@@ -404,7 +405,7 @@ function runInParentContext(Factory) {
 
     return run(name, options, fn, overrides);
   };
-  ArrayPrototypeForEach(['expectFailure', 'skip', 'todo', 'only'], (keyword) => {
+  ArrayPrototypeForEach(['expectFailure', 'flaky', 'skip', 'todo', 'only'], (keyword) => {
     test[keyword] = (name, options, fn) => {
       const overrides = {
         __proto__: null,

lib/internal/test_runner/reporter/dot.js

@@ -12,7 +12,11 @@ module.exports = async function* dot(source) {
   const failedTests = [];
   for await (const { type, data } of source) {
     if (type === 'test:pass') {
-      yield `${colors.green}.${colors.reset}`;
+      if (data.retryCount > 0) {
+        yield `${colors.yellow}F${colors.reset}`;
+      } else {
+        yield `${colors.green}.${colors.reset}`;
+      }
     }
     if (type === 'test:fail') {
       yield `${colors.red}X${colors.reset}`;

lib/internal/test_runner/reporter/junit.js

@@ -131,6 +131,26 @@ module.exports = async function* junitReporter(source) {
               attrs: { __proto__: null, type: 'todo', message: event.data.todo },
             });
           }
+          if (event.data.retryCount > 0) {
+            ArrayPrototypePush(currentTest.children, {
+              __proto__: null,
+              nesting: event.data.nesting + 1,
+              tag: 'properties',
+              attrs: { __proto__: null },
+              children: [
+                {
+                  __proto__: null,
+                  nesting: event.data.nesting + 2,
+                  tag: 'property',
+                  attrs: {
+                    __proto__: null,
+                    name: 'flaky',
+                    value: `${event.data.retryCount} retries`,
+                  },
+                },
+              ],
+            });
+          }
           if (event.type === 'test:fail') {
             const error = event.data.details?.error;
             currentTest.children.push({

lib/internal/test_runner/reporter/tap.js

@@ -33,12 +33,12 @@
   for await (const { type, data } of source) {
     switch (type) {
       case 'test:fail': {
-        yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure);
+        yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure, data.retryCount);
         const location = data.file ? `${data.file}:${data.line}:${data.column}` : null;
         yield reportDetails(data.nesting, data.details, location);
         break;
       } case 'test:pass':
-        yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure);
+        yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure, data.retryCount);
         yield reportDetails(data.nesting, data.details, null);
         break;
       case 'test:plan':
@@ -75,7 +75,7 @@
   }
 }
 
-function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure) {
+function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure, retryCount) {
   let line = `${indent(nesting)}${status} ${testNumber}`;
 
   if (name) {
@@ -88,6 +88,9 @@
     line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`;
   } else if (expectFailure !== undefined) {
     line += ` # EXPECTED FAILURE${typeof expectFailure === 'string' ? ` ${tapEscape(expectFailure)}` : ''}`;
+  } else if (retryCount !== undefined && retryCount > 0) {
+    const retryText = retryCount === 1 ? 're-try' : 're-tries';
+    line += ` # FLAKY ${retryCount} ${retryText}`;
   }
 
   line += '\n';

lib/internal/test_runner/reporter/utils.js

@@ -71,7 +71,7 @@ function formatError(error, indent) {
 function formatTestReport(type, data, showErrorDetails = true, prefix = '', indent = '') {
   let color = reporterColorMap[type] ?? colors.white;
   let symbol = reporterUnicodeSymbolMap[type] ?? ' ';
-  const { skip, todo, expectFailure } = data;
+  const { skip, todo, expectFailure, retryCount } = data;
   const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : '';
   let title = `${data.name}${duration_ms}`;
 
@@ -87,6 +87,9 @@ function formatTestReport(type, data, showErrorDetails = true, prefix = '', inde
     }
   } else if (expectFailure !== undefined) {
     title += ` # EXPECTED FAILURE`;
+  } else if (retryCount !== undefined && retryCount > 0) {
+    const retryText = retryCount === 1 ? 're-try' : 're-tries';
+    title += ` # FLAKY ${retryCount} ${retryText}`;
   }
 
   const err = showErrorDetails && data.details?.error ? formatError(data.details.error, indent) : '';