feat: first commit flaky

Author: vespa7

lib/internal/test_runner/harness.js

@@ -61,6 +61,7 @@ function createTestTree(rootTestOptions, globalOptions) {
         failed: 0,
         passed: 0,
         cancelled: 0,
+        flaky: 0,
         skipped: 0,
         todo: 0,
         topLevel: 0,
@@ -377,7 +378,7 @@ function runInParentContext(Factory) {
 
     return run(name, options, fn, overrides);
   };
-  ArrayPrototypeForEach(['expectFailure', 'skip', 'todo', 'only'], (keyword) => {
+  ArrayPrototypeForEach(['expectFailure', 'flaky', 'skip', 'todo', 'only'], (keyword) => {
     test[keyword] = (name, options, fn) => {
       const overrides = {
         __proto__: null,

lib/internal/test_runner/reporter/tap.js

@@ -33,12 +33,12 @@ async function * tapReporter(source) {
   for await (const { type, data } of source) {
     switch (type) {
       case 'test:fail': {
-        yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure);
+        yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure, data.flaky);
         const location = data.file ? `${data.file}:${data.line}:${data.column}` : null;
         yield reportDetails(data.nesting, data.details, location);
         break;
       } case 'test:pass':
-        yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure);
+        yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure, data.flaky);
         yield reportDetails(data.nesting, data.details, null);
         break;
       case 'test:plan':
@@ -65,7 +65,7 @@ async function * tapReporter(source) {
   }
 }
 
-function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure) {
+function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure, flaky) {
   let line = `${indent(nesting)}${status} ${testNumber}`;
 
   if (name) {
@@ -78,6 +78,10 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure
     line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`;
   } else if (expectFailure !== undefined) {
     line += ' # EXPECTED FAILURE';
+    //should we use flaky >=0 here? for always printing 0 retries
+  } else if (flaky !== undefined && flaky > 0) {
+    const retryText = flaky === 1 ? 're-try' : 're-tries';
+    line += ` # FLAKY ${flaky} ${retryText}`;
   }
 
   line += '\n';

lib/internal/test_runner/reporter/utils.js

@@ -71,7 +71,7 @@ function formatError(error, indent) {
 function formatTestReport(type, data, showErrorDetails = true, prefix = '', indent = '') {
   let color = reporterColorMap[type] ?? colors.white;
   let symbol = reporterUnicodeSymbolMap[type] ?? ' ';
-  const { skip, todo, expectFailure } = data;
+  const { skip, todo, expectFailure, flaky } = data;
   const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : '';
   let title = `${data.name}${duration_ms}`;
 
@@ -87,6 +87,9 @@ function formatTestReport(type, data, showErrorDetails = true, prefix = '', inde
     }
   } else if (expectFailure !== undefined) {
     title += ` # EXPECTED FAILURE`;
+  } else if (flaky !== undefined && flaky > 0) {
+    const retryText = flaky === 1 ? 're-try' : 're-tries';
+    title += ` # FLAKY ${flaky} ${retryText}`;
   }
 
   const err = showErrorDetails && data.details?.error ? formatError(data.details.error, indent) : '';

lib/internal/test_runner/test.js

@@ -85,6 +85,7 @@ const kTestTimeoutFailure = 'testTimeoutFailure';
 const kExpectedFailure = 'expectedFailure';
 const kHookFailure = 'hookFailed';
 const kDefaultTimeout = null;
+const kDefaultFlakyRetries = 20;
 const noop = FunctionPrototype;
 const kShouldAbort = Symbol('kShouldAbort');
 const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach']);
@@ -497,7 +498,8 @@ class Test extends AsyncResource {
     super('Test');
 
     let { fn, name, parent } = options;
-    const { concurrency, entryFile, expectFailure, loc, only, timeout, todo, skip, signal, plan } = options;
+
+    const { concurrency, entryFile, expectFailure, flaky, loc, only, timeout, todo, skip, signal, plan } = options;
 
     if (typeof fn !== 'function') {
       fn = noop;

lib/internal/test_runner/utils.js

@@ -401,6 +401,10 @@ function countCompletedTest(test, harness = test.root.harness) {
   } else {
     harness.counters.passed++;
   }
+
+  if (test.flakyRetries > 0) {
+    harness.counters.flaky++;
+  }
   harness.counters.tests++;
 }