test_runner: flatten TAP output when running using --test

Author: MoLow

lib/internal/test_runner/runner.js

@@ -8,9 +8,9 @@ const {
   ArrayPrototypeSlice,
   ArrayPrototypeSome,
   ArrayPrototypeSort,
+  Number,
   ObjectAssign,
   PromisePrototypeThen,
-  SafePromiseAll,
   SafePromiseAllReturnVoid,
   SafePromiseAllSettledReturnVoid,
   SafeMap,
@@ -33,7 +33,7 @@ const { validateArray, validateBoolean } = require('internal/validators');
 const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector');
 const { kEmptyObject } = require('internal/util');
 const { createTestTree } = require('internal/test_runner/harness');
-const { kSubtestsFailed, Test } = require('internal/test_runner/test');
+const { kSubtestsFailed, kTestCodeFailure, kCancelledByParent, Test } = require('internal/test_runner/test');
 const { TapParser } = require('internal/test_runner/tap_parser');
 const { YAMLToJs } = require('internal/test_runner/yaml_to_js');
 const { TokenKind } = require('internal/test_runner/tap_lexer');
@@ -130,27 +130,32 @@ function getRunArgs({ path, inspectPort }) {
 
 class FileTest extends Test {
   #buffer = [];
+  #hasSubtests = false;
+  get skipReporting() {
+    return this.#hasSubtests && (!this.error || this.error.failureType === kSubtestsFailed);
+  }
   #handleReportItem({ kind, node, nesting = 0 }) {
-    nesting += 1;
-
     switch (kind) {
       case TokenKind.TAP_VERSION:
         // TODO(manekinekko): handle TAP version coming from the parser.
         // this.reporter.version(node.version);
         break;
 
       case TokenKind.TAP_PLAN:
+        if (nesting === 0 && this.skipReporting) {
+          break;
+        }
         this.reporter.plan(nesting, this.name, node.end - node.start + 1);
         break;
 
       case TokenKind.TAP_SUBTEST_POINT:
         this.reporter.start(nesting, this.name, node.name);
         break;
 
-      case TokenKind.TAP_TEST_POINT:
-        // eslint-disable-next-line no-case-declarations
+      case TokenKind.TAP_TEST_POINT: {
+
         const { todo, skip, pass } = node.status;
-        // eslint-disable-next-line no-case-declarations
+
         let directive;
 
         if (skip) {
@@ -161,29 +166,20 @@ class FileTest extends Test {
           directive = kEmptyObject;
         }
 
-        if (pass) {
-          this.reporter.ok(
-            nesting,
-            this.name,
-            node.id,
-            node.description,
-            YAMLToJs(node.diagnostics),
-            directive
-          );
-        } else {
-          this.reporter.fail(
-            nesting,
-            this.name,
-            node.id,
-            node.description,
-            YAMLToJs(node.diagnostics),
-            directive
-          );
+        this.#hasSubtests = true;
+        const diagnostics = YAMLToJs(node.diagnostics);
+        const cancelled = diagnostics.error?.failureType === kCancelledByParent;
+        const testNumber = nesting === 0 ? (Number(node.id) + this.testNumber - 1) : node.id;
+        const method = pass ? 'ok' : 'fail';
+        this.reporter[method](nesting, this.name, testNumber, node.description, diagnostics, directive);
+        if (nesting === 0) {
+          this.parent.countSubtest({ finished: true, skipped: skip, isTodo: todo, passed: pass, cancelled });
+          this.countSubtest({ finished: true, skipped: skip, isTodo: todo, passed: pass, cancelled });
         }
         break;
 
-      case TokenKind.COMMENT:
-        if (nesting === 1) {
+      } case TokenKind.COMMENT:
+        if (nesting === 0) {
           // Ignore file top level diagnostics
           break;
         }
@@ -203,10 +199,15 @@ class FileTest extends Test {
     this.reportStarted();
     this.#handleReportItem(ast);
   }
+  reportStarted() {}
   report() {
-    this.reportStarted();
+    if (!this.skipReporting) {
+      super.reportStarted();
+    }
     ArrayPrototypeForEach(this.#buffer, (ast) => this.#handleReportItem(ast));
-    super.report();
+    if (!this.skipReporting) {
+      super.report();
+    }
   }
 }
 
@@ -261,16 +262,14 @@ function runTestFile(path, root, inspectPort, filesWatcher) {
       subtest.addToReport(ast);
     });
 
-    const { 0: { 0: code, 1: signal } } = await SafePromiseAll([
-      once(child, 'exit', { signal: t.signal }),
-      child.stdout.toArray({ signal: t.signal }),
-    ]);
+    const { 0: code, 1: signal } = await once(child, 'exit', { signal: t.signal });
 
     runningProcesses.delete(path);
     runningSubtests.delete(path);
     if (code !== 0 || signal !== null) {
       if (!err) {
-        err = ObjectAssign(new ERR_TEST_FAILURE('test failed', kSubtestsFailed), {
+        const failureType = subtest.completedCounters.totalFailed > 0 ? kSubtestsFailed : kTestCodeFailure;
+        err = ObjectAssign(new ERR_TEST_FAILURE('test failed', failureType), {
           __proto__: null,
           exitCode: code,
           signal: signal,

lib/internal/test_runner/test.js

@@ -286,6 +286,9 @@ class Test extends AsyncResource {
       beforeEach: [],
       afterEach: [],
     };
+    this.completedCounters = {
+      all: 0, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0
+    };
     this.waitingOn = 0;
     this.finished = false;
   }
@@ -577,9 +580,32 @@ class Test extends AsyncResource {
     this.postRun();
   }
 
-  postRun(pendingSubtestsError) {
-    const counters = { __proto__: null, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0 };
+  countSubtest(subtest) {
+    if (subtest.skipReporting || subtest.counted) {
+      return;
+    }
+    subtest.counted = true;
+    // Check SKIP and TODO tests first, as those should not be counted as
+    // failures.
+    if (subtest.skipped) {
+      this.completedCounters.skipped++;
+    } else if (subtest.isTodo) {
+      this.completedCounters.todo++;
+    } else if (subtest.cancelled) {
+      this.completedCounters.cancelled++;
+    } else if (!subtest.passed) {
+      this.completedCounters.failed++;
+    } else {
+      this.completedCounters.passed++;
+    }
+
+    if (!subtest.passed) {
+      this.completedCounters.totalFailed++;
+    }
+    this.completedCounters.all++;
+  }
 
+  postRun(pendingSubtestsError) {
     // If the test was failed before it even started, then the end time will
     // be earlier than the start time. Correct that here.
     if (this.endTime < this.startTime) {
@@ -592,34 +618,15 @@ class Test extends AsyncResource {
     this.pendingSubtests = [];
     for (let i = 0; i < this.subtests.length; i++) {
       const subtest = this.subtests[i];
-
       if (!subtest.finished) {
         subtest.cancel(pendingSubtestsError);
         subtest.postRun(pendingSubtestsError);
       }
-
-      // Check SKIP and TODO tests first, as those should not be counted as
-      // failures.
-      if (subtest.skipped) {
-        counters.skipped++;
-      } else if (subtest.isTodo) {
-        counters.todo++;
-      } else if (subtest.cancelled) {
-        counters.cancelled++;
-      } else if (!subtest.passed) {
-        counters.failed++;
-      } else {
-        counters.passed++;
-      }
-
-      if (!subtest.passed) {
-        counters.totalFailed++;
-      }
     }
 
-    if ((this.passed || this.parent === null) && counters.totalFailed > 0) {
-      const subtestString = `subtest${counters.totalFailed > 1 ? 's' : ''}`;
-      const msg = `${counters.totalFailed} ${subtestString} failed`;
+    if ((this.passed || this.parent === null) && this.completedCounters.totalFailed > 0) {
+      const subtestString = `subtest${this.completedCounters.totalFailed > 1 ? 's' : ''}`;
+      const msg = `${this.completedCounters.totalFailed} ${subtestString} failed`;
 
       this.fail(new ERR_TEST_FAILURE(msg, kSubtestsFailed));
     }
@@ -632,20 +639,21 @@ class Test extends AsyncResource {
       this.parent.addReadySubtest(this);
       this.parent.processReadySubtestRange(false);
       this.parent.processPendingSubtests();
+      this.parent.countSubtest(this);
     } else if (!this.reported) {
       this.reported = true;
-      this.reporter.plan(this.nesting, kFilename, this.subtests.length);
+      this.reporter.plan(this.nesting, kFilename, this.completedCounters.all);
 
       for (let i = 0; i < this.diagnostics.length; i++) {
         this.reporter.diagnostic(this.nesting, kFilename, this.diagnostics[i]);
       }
 
-      this.reporter.diagnostic(this.nesting, kFilename, `tests ${this.subtests.length}`);
-      this.reporter.diagnostic(this.nesting, kFilename, `pass ${counters.passed}`);
-      this.reporter.diagnostic(this.nesting, kFilename, `fail ${counters.failed}`);
-      this.reporter.diagnostic(this.nesting, kFilename, `cancelled ${counters.cancelled}`);
-      this.reporter.diagnostic(this.nesting, kFilename, `skipped ${counters.skipped}`);
-      this.reporter.diagnostic(this.nesting, kFilename, `todo ${counters.todo}`);
+      this.reporter.diagnostic(this.nesting, kFilename, `tests ${this.completedCounters.all}`);
+      this.reporter.diagnostic(this.nesting, kFilename, `pass ${this.completedCounters.passed}`);
+      this.reporter.diagnostic(this.nesting, kFilename, `fail ${this.completedCounters.failed}`);
+      this.reporter.diagnostic(this.nesting, kFilename, `cancelled ${this.completedCounters.cancelled}`);
+      this.reporter.diagnostic(this.nesting, kFilename, `skipped ${this.completedCounters.skipped}`);
+      this.reporter.diagnostic(this.nesting, kFilename, `todo ${this.completedCounters.todo}`);
       this.reporter.diagnostic(this.nesting, kFilename, `duration_ms ${this.#duration()}`);
 
       if (this.coverage) {