test_runner: add snapshot testing

This commit adds a t.assert.snapshot() method that implements
snapshot testing.

Author: cjihrig

lib/internal/test_runner/harness.js

@@ -210,6 +210,7 @@ function setup(root) {
     counters: null,
     shouldColorizeTestFiles: false,
     teardown: exitHandler,
+    snapshotManager: null,
   };
   root.harness.resetCounters();
   root.startTime = hrtime();

lib/internal/test_runner/snapshot.js

@@ -0,0 +1,234 @@
+'use strict';
+const {
+  ArrayPrototypeJoin,
+  ArrayPrototypeMap,
+  ArrayPrototypeSort,
+  JSONStringify,
+  ObjectKeys,
+  SafeMap,
+  String,
+  StringPrototypeReplaceAll,
+} = primordials;
+const {
+  codes: {
+    ERR_INVALID_STATE,
+  },
+} = require('internal/errors');
+const { emitExperimentalWarning, kEmptyObject } = require('internal/util');
+let debug = require('internal/util/debuglog').debuglog('test_runner', (fn) => {
+  debug = fn;
+});
+const {
+  validateArray,
+  validateFunction,
+  validateObject,
+} = require('internal/validators');
+const { strictEqual } = require('assert');
+const { mkdirSync, readFileSync, writeFileSync } = require('fs');
+const { dirname } = require('path');
+const { createContext, runInContext } = require('vm');
+const kExperimentalWarning = 'Snapshot testing';
+const kMissingSnapshotTip = 'Missing snapshots can be generated by rerunning ' +
+  'the command with the --test-update-snapshots flag.';
+const defaultSerializers = [
+  (value) => { return JSONStringify(value, null, 2); },
+];
+
+function defaultResolveSnapshotPath(testPath) {
+  if (typeof testPath !== 'string') {
+    return testPath;
+  }
+
+  return `${testPath}.snapshot`;
+}
+
+let resolveSnapshotPathFn = defaultResolveSnapshotPath;
+let serializerFns = defaultSerializers;
+
+function setResolveSnapshotPath(fn) {
+  emitExperimentalWarning(kExperimentalWarning);
+  validateFunction(fn, 'fn');
+  resolveSnapshotPathFn = fn;
+}
+
+function setDefaultSerializers(serializers) {
+  emitExperimentalWarning(kExperimentalWarning);
+  validateFunctionArray(serializers, 'serializers');
+  serializerFns = ArrayPrototypeMap(serializers, (fn) => {
+    return fn;
+  });
+}
+
+class SnapshotManager {
+  constructor(entryFile, updateSnapshots) {
+    this.entryFile = entryFile;
+    this.snapshotFile = undefined;
+    this.snapshots = { __proto__: null };
+    this.nameCounts = new SafeMap();
+    // A manager instance will only read or write snapshot files based on the
+    // updateSnapshots argument.
+    this.loaded = updateSnapshots;
+    this.updateSnapshots = updateSnapshots;
+  }
+
+  resolveSnapshotFile() {
+    if (this.snapshotFile === undefined) {
+      const resolved = resolveSnapshotPathFn(this.entryFile);
+
+      if (typeof resolved !== 'string') {
+        const err = new ERR_INVALID_STATE('Invalid snapshot filename.');
+        err.filename = resolved;
+        throw err;
+      }
+
+      this.snapshotFile = resolved;
+    }
+  }
+
+  serialize(input, serializers = serializerFns) {
+    try {
+      let value = input;
+
+      for (let i = 0; i < serializers.length; ++i) {
+        const fn = serializers[i];
+        value = fn(value);
+      }
+
+      return `\n${templateEscape(value)}\n`;
+    } catch (err) {
+      const error = new ERR_INVALID_STATE(
+        'The provided serializers did not generate a string.',
+      );
+      error.input = input;
+      error.cause = err;
+      throw error;
+    }
+  }
+
+  getSnapshot(id) {
+    if (!(id in this.snapshots)) {
+      const err = new ERR_INVALID_STATE(`Snapshot '${id}' not found in ` +
+        `'${this.snapshotFile}.' ${kMissingSnapshotTip}`);
+      err.snapshot = id;
+      err.filename = this.snapshotFile;
+      throw err;
+    }
+
+    return this.snapshots[id];
+  }
+
+  setSnapshot(id, value) {
+    this.snapshots[templateEscape(id)] = value;
+  }
+
+  nextId(name) {
+    const count = this.nameCounts.get(name) ?? 1;
+
+    this.nameCounts.set(name, count + 1);
+    return `${name} ${count}`;
+  }
+
+  readSnapshotFile() {
+    if (this.loaded) {
+      debug('skipping read of snapshot file');
+      return;
+    }
+
+    try {
+      const source = readFileSync(this.snapshotFile, 'utf8');
+      const context = { __proto__: null, exports: { __proto__: null } };
+
+      createContext(context);
+      runInContext(source, context);
+
+      if (context.exports === null || typeof context.exports !== 'object') {
+        throw new ERR_INVALID_STATE(
+          `Malformed snapshot file '${this.snapshotFile}'.`,
+        );
+      }
+
+      this.snapshots = context.exports;
+      this.loaded = true;
+    } catch (err) {
+      let msg = `Cannot read snapshot file '${this.snapshotFile}.'`;
+
+      if (err?.code === 'ENOENT') {
+        msg += ` ${kMissingSnapshotTip}`;
+      }
+
+      const error = new ERR_INVALID_STATE(msg);
+      error.cause = err;
+      error.filename = this.snapshotFile;
+      throw error;
+    }
+  }
+
+  writeSnapshotFile() {
+    if (!this.updateSnapshots) {
+      debug('skipping write of snapshot file');
+      return;
+    }
+
+    const keys = ArrayPrototypeSort(ObjectKeys(this.snapshots));
+    const snapshotStrings = ArrayPrototypeMap(keys, (key) => {
+      return `exports[\`${key}\`] = \`${this.snapshots[key]}\`;\n`;
+    });
+    const output = ArrayPrototypeJoin(snapshotStrings, '\n');
+    mkdirSync(dirname(this.snapshotFile), { __proto__: null, recursive: true });
+    writeFileSync(this.snapshotFile, output, 'utf8');
+  }
+
+  createAssert() {
+    const manager = this;
+
+    return function snapshotAssertion(actual, options = kEmptyObject) {
+      emitExperimentalWarning(kExperimentalWarning);
+      // Resolve the snapshot file here so that any resolution errors are
+      // surfaced as early as possible.
+      manager.resolveSnapshotFile();
+
+      const { fullName } = this;
+      const id = manager.nextId(fullName);
+
+      validateObject(options, 'options');
+
+      const {
+        serializers = serializerFns,
+      } = options;
+
+      validateFunctionArray(serializers, 'options.serializers');
+
+      const value = manager.serialize(actual, serializers);
+
+      if (manager.updateSnapshots) {
+        manager.setSnapshot(id, value);
+      } else {
+        manager.readSnapshotFile();
+        strictEqual(value, manager.getSnapshot(id));
+      }
+    };
+  }
+}
+
+function validateFunctionArray(fns, name) {
+  validateArray(fns, name);
+  for (let i = 0; i < fns.length; ++i) {
+    validateFunction(fns[i], `${name}[${i}]`);
+  }
+}
+
+function templateEscape(str) {
+  let result = String(str);
+  result = StringPrototypeReplaceAll(result, '\\', '\\\\');
+  result = StringPrototypeReplaceAll(result, '`', '\\`');
+  result = StringPrototypeReplaceAll(result, '${', '\\${');
+  return result;
+}
+
+module.exports = {
+  SnapshotManager,
+  defaultResolveSnapshotPath, // Exported for testing only.
+  defaultSerializers,         // Exported for testing only.
+  setDefaultSerializers,
+  setResolveSnapshotPath,
+};

lib/internal/test_runner/test.js

@@ -86,6 +86,7 @@ const {
   testNamePatterns,
   testSkipPatterns,
   testOnlyFlag,
+  updateSnapshots,
 } = parseCommandLine();
 let kResistStopPropagation;
 let assertObj;
@@ -102,11 +103,10 @@ function lazyFindSourceMap(file) {
   return findSourceMap(file);
 }
 
-function lazyAssertObject() {
+function lazyAssertObject(harness) {
   if (assertObj === undefined) {
     assertObj = new SafeMap();
     const assert = require('assert');
-
     const methodsToCopy = [
       'deepEqual',
       'deepStrictEqual',
@@ -129,6 +129,13 @@ function lazyAssertObject() {
     for (let i = 0; i < methodsToCopy.length; i++) {
       assertObj.set(methodsToCopy[i], assert[methodsToCopy[i]]);
     }
+
+    const { getOptionValue } = require('internal/options');
+    if (getOptionValue('--experimental-test-snapshots')) {
+      const { SnapshotManager } = require('internal/test_runner/snapshot');
+      harness.snapshotManager = new SnapshotManager(kFilename, updateSnapshots);
+      assertObj.set('snapshot', harness.snapshotManager.createAssert());
+    }
   }
   return assertObj;
 }
@@ -248,7 +255,7 @@ class TestContext {
   get assert() {
     if (this.#assert === undefined) {
       const { plan } = this.#test;
-      const map = lazyAssertObject();
+      const map = lazyAssertObject(this.#test.root.harness);
       const assert = { __proto__: null };
 
       this.#assert = assert;
@@ -257,7 +264,7 @@ class TestContext {
           if (plan !== null) {
             plan.actual++;
           }
-          return ReflectApply(method, assert, args);
+          return ReflectApply(method, this, args);
         };
       });
     }
@@ -960,6 +967,7 @@ class Test extends AsyncResource {
 
       // Call this harness.coverage() before collecting diagnostics, since failure to collect coverage is a diagnostic.
       const coverage = harness.coverage();
+      harness.snapshotManager?.writeSnapshotFile();
       for (let i = 0; i < diagnostics.length; i++) {
         reporter.diagnostic(nesting, loc, diagnostics[i]);
       }
@@ -980,6 +988,7 @@ class Test extends AsyncResource {
       if (harness.watching) {
         this.reported = false;
         harness.resetCounters();
+        assertObj = undefined;
       } else {
         reporter.end();
       }

lib/internal/test_runner/utils.js

@@ -195,6 +195,7 @@ function parseCommandLine() {
   const coverage = getOptionValue('--experimental-test-coverage');
   const forceExit = getOptionValue('--test-force-exit');
   const sourceMaps = getOptionValue('--enable-source-maps');
+  const updateSnapshots = getOptionValue('--test-update-snapshots');
   const isChildProcess = process.env.NODE_TEST_CONTEXT === 'child';
   const isChildProcessV8 = process.env.NODE_TEST_CONTEXT === 'child-v8';
   let destinations;
@@ -255,6 +256,7 @@ function parseCommandLine() {
     testOnlyFlag,
     testNamePatterns,
     testSkipPatterns,
+    updateSnapshots,
     reporters,
     destinations,
   };

lib/test.js

@@ -7,6 +7,7 @@ const {
 
 const { test, suite, before, after, beforeEach, afterEach } = require('internal/test_runner/harness');
 const { run } = require('internal/test_runner/runner');
+const { getOptionValue } = require('internal/options');
 
 module.exports = test;
 ObjectAssign(module.exports, {
@@ -37,3 +38,29 @@ ObjectDefineProperty(module.exports, 'mock', {
     return lazyMock;
   },
 });
+
+if (getOptionValue('--experimental-test-snapshots')) {
+  let lazySnapshot;
+
+  ObjectDefineProperty(module.exports, 'snapshot', {
+    __proto__: null,
+    configurable: true,
+    enumerable: true,
+    get() {
+      if (lazySnapshot === undefined) {
+        const {
+          setDefaultSerializers,
+          setResolveSnapshotPath,
+        } = require('internal/test_runner/snapshot');
+
+        lazySnapshot = {
+          __proto__: null,
+          setDefaultSerializers,
+          setResolveSnapshotPath,
+        };
+      }
+
+      return lazySnapshot;
+    },
+  });
+}

src/node_options.cc

@@ -623,12 +623,18 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
   AddOption("--test-timeout",
             "specify test runner timeout",
             &EnvironmentOptions::test_runner_timeout);
+  AddOption("--test-update-snapshots",
+            "regenerate test snapshots",
+            &EnvironmentOptions::test_runner_update_snapshots);
   AddOption("--experimental-test-coverage",
             "enable code coverage in the test runner",
             &EnvironmentOptions::test_runner_coverage);
   AddOption("--experimental-test-module-mocks",
             "enable module mocking in the test runner",
             &EnvironmentOptions::test_runner_module_mocks);
+  AddOption("--experimental-test-snapshots",
+            "enable snapshot testing in the test runner",
+            &EnvironmentOptions::test_runner_snapshots);
   AddOption("--test-name-pattern",
             "run tests whose name matches this regular expression",
             &EnvironmentOptions::test_name_pattern);

src/node_options.h

@@ -174,6 +174,8 @@ class EnvironmentOptions : public Options {
   bool test_runner_coverage = false;
   bool test_runner_force_exit = false;
   bool test_runner_module_mocks = false;
+  bool test_runner_snapshots = false;
+  bool test_runner_update_snapshots = false;
   std::vector<std::string> test_name_pattern;
   std::vector<std::string> test_reporter;
   std::vector<std::string> test_reporter_destination;

test/fixtures/test-runner/snapshots/malformed-exports.js.snapshot

@@ -0,0 +1 @@
+exports = null;