test_runner: add initial code coverage support

This commit adds code coverage functionality to the node:test
module. When node:test is used in conjunction with the new
--test-coverage CLI flag, a coverage report is created when
the test runner finishes. The coverage summary is forwarded to
any test runner reporters so that the display can be customized
as desired. This new functionality is compatible with the
existing NODE_V8_COVERAGE environment variable as well.

There are still several limitations, which will be addressed in
subsequent pull requests:

- Coverage is only reported for a single process. It is possible
  to merge coverage reports together. Once this is done, the
  --test flag will be supported as well.
- Source maps are not currently supported.
- Excluding specific files or directories from the coverage
  report is not currently supported. Node core modules and
  node_modules/ are excluded though.

PR-URL: nodejs#46017
Reviewed-By: Moshe Atlow <[email protected]>
Reviewed-By: Geoffrey Booth <[email protected]>

Author: cjihrig

doc/api/cli.md

@@ -1209,6 +1209,17 @@ Starts the Node.js command line test runner. This flag cannot be combined with
 See the documentation on [running tests from the command line][]
 for more details.
 
+### `--test-coverage`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+When used in conjunction with the `node:test` module, a code coverage report is
+generated as part of the test runner output. If no tests are run, a coverage
+report is not generated. See the documentation on
+[collecting code coverage from tests][] for more details.
+
 ### `--test-name-pattern`
 
 <!-- YAML
@@ -2313,6 +2324,7 @@ done
 [`unhandledRejection`]: process.md#event-unhandledrejection
 [`v8.startupSnapshot` API]: v8.md#startup-snapshot-api
 [`worker_threads.threadId`]: worker_threads.md#workerthreadid
+[collecting code coverage from tests]: test.md#collecting-code-coverage
 [conditional exports]: packages.md#conditional-exports
 [context-aware]: addons.md#context-aware-addons
 [customizing ESM specifier resolution]: esm.md#customizing-esm-specifier-resolution-algorithm

doc/api/test.md

@@ -368,6 +368,54 @@ Otherwise, the test is considered to be a failure. Test files must be
 executable by Node.js, but are not required to use the `node:test` module
 internally.
 
+## Collecting code coverage
+
+When Node.js is started with the [`--test-coverage`][] command-line flag, code
+coverage is collected and statistics are reported once all tests have completed.
+If the [`NODE_V8_COVERAGE`][] environment variable is used to specify a
+code coverage directory, the generated V8 coverage files are written to that
+directory. Node.js core modules and files within `node_modules/` directories
+are not included in the coverage report. If coverage is enabled, the coverage
+report is sent to any [test reporters][] via the `'test:coverage'` event.
+
+Coverage can be disabled on a series of lines using the following
+comment syntax:
+
+```js
+/* node:coverage disable */
+if (anAlwaysFalseCondition) {
+  // Code in this branch will never be executed, but the lines are ignored for
+  // coverage purposes. All lines following the 'disable' comment are ignored
+  // until a corresponding 'enable' comment is encountered.
+  console.log('this is never executed');
+}
+/* node:coverage enable */
+```
+
+Coverage can also be disabled for a specified number of lines. After the
+specified number of lines, coverage will be automatically reenabled. If the
+number of lines is not explicitly provided, a single line is ignored.
+
+```js
+/* node:coverage ignore next */
+if (anAlwaysFalseCondition) { console.log('this is never executed'); }
+
+/* node:coverage ignore next 3 */
+if (anAlwaysFalseCondition) {
+  console.log('this is never executed');
+}
+```
+
+The test runner's code coverage functionality has the following limitations,
+which will be addressed in a future Node.js release:
+
+* Although coverage data is collected for child processes, this information is
+  not included in the coverage report. Because the command line test runner uses
+  child processes to execute test files, it cannot be used with `--test-coverage`.
+* Source maps are not supported.
+* Excluding specific files or directories from the coverage report is not
+  supported.
+
 ## Mocking
 
 The `node:test` module supports mocking during testing via a top-level `mock`
@@ -1201,6 +1249,42 @@ A successful call to [`run()`][] method will return a new {TestsStream}
 object, streaming a series of events representing the execution of the tests.
 `TestsStream` will emit events, in the order of the tests definition
 
+### Event: `'test:coverage'`
+
+* `data` {Object}
+  * `summary` {Object} An object containing the coverage report.
+    * `files` {Array} An array of coverage reports for individual files. Each
+      report is an object with the following schema:
+      * `path` {string} The absolute path of the file.
+      * `totalLineCount` {number} The total number of lines.
+      * `totalBranchCount` {number} The total number of branches.
+      * `totalFunctionCount` {number} The total number of functions.
+      * `coveredLineCount` {number} The number of covered lines.
+      * `coveredBranchCount` {number} The number of covered branches.
+      * `coveredFunctionCount` {number} The number of covered functions.
+      * `coveredLinePercent` {number} The percentage of lines covered.
+      * `coveredBranchPercent` {number} The percentage of branches covered.
+      * `coveredFunctionPercent` {number} The percentage of functions covered.
+      * `uncoveredLineNumbers` {Array} An array of integers representing line
+        numbers that are uncovered.
+    * `totals` {Object} An object containing a summary of coverage for all
+      files.
+      * `totalLineCount` {number} The total number of lines.
+      * `totalBranchCount` {number} The total number of branches.
+      * `totalFunctionCount` {number} The total number of functions.
+      * `coveredLineCount` {number} The number of covered lines.
+      * `coveredBranchCount` {number} The number of covered branches.
+      * `coveredFunctionCount` {number} The number of covered functions.
+      * `coveredLinePercent` {number} The percentage of lines covered.
+      * `coveredBranchPercent` {number} The percentage of branches covered.
+      * `coveredFunctionPercent` {number} The percentage of functions covered.
+    * `workingDirectory` {string} The working directory when code coverage
+      began. This is useful for displaying relative path names in case the tests
+      changed the working directory of the Node.js process.
+  * `nesting` {number} The nesting level of the test.
+
+Emitted when code coverage is enabled and all tests have completed.
+
 ### Event: `'test:diagnostic'`
 
 * `data` {Object}
@@ -1550,6 +1634,7 @@ added: v18.7.0
   aborted.
 
 [TAP]: https://testanything.org/
+[`--test-coverage`]: cli.md#--test-coverage
 [`--test-name-pattern`]: cli.md#--test-name-pattern
 [`--test-only`]: cli.md#--test-only
 [`--test-reporter-destination`]: cli.md#--test-reporter-destination
@@ -1558,6 +1643,7 @@ added: v18.7.0
 [`MockFunctionContext`]: #class-mockfunctioncontext
 [`MockTracker.method`]: #mockmethodobject-methodname-implementation-options
 [`MockTracker`]: #class-mocktracker
+[`NODE_V8_COVERAGE`]: cli.md#node_v8_coveragedir
 [`SuiteContext`]: #class-suitecontext
 [`TestContext`]: #class-testcontext
 [`context.diagnostic`]: #contextdiagnosticmessage
@@ -1568,4 +1654,5 @@ added: v18.7.0
 [describe options]: #describename-options-fn
 [it options]: #testname-options-fn
 [stream.compose]: stream.md#streamcomposestreams
+[test reporters]: #test-reporters
 [test runner execution model]: #test-runner-execution-model

doc/node.1

@@ -391,6 +391,9 @@ Specify the minimum allocation from the OpenSSL secure heap. The default is 2. T
 .It Fl -test
 Starts the Node.js command line test runner.
 .
+.It Fl -test-coverage
+Enable code coverage in the test runner.
+.
 .It Fl -test-name-pattern
 A regular expression that configures the test runner to only execute tests
 whose name matches the provided pattern.

lib/internal/process/pre_execution.js

@@ -20,6 +20,7 @@ const { reconnectZeroFillToggle } = require('internal/buffer');
 const {
   defineOperation,
   exposeInterface,
+  setupCoverageHooks,
 } = require('internal/util');
 
 const {
@@ -63,15 +64,7 @@ function prepareExecution(options) {
   setupFetch();
   setupWebCrypto();
   setupCustomEvent();
-
-  // Resolve the coverage directory to an absolute path, and
-  // overwrite process.env so that the original path gets passed
-  // to child processes even when they switch cwd.
-  if (process.env.NODE_V8_COVERAGE) {
-    process.env.NODE_V8_COVERAGE =
-      setupCoverageHooks(process.env.NODE_V8_COVERAGE);
-  }
-
+  setupCodeCoverage();
   setupDebugEnv();
   // Process initial diagnostic reporting configuration, if present.
   initializeReport();
@@ -288,6 +281,17 @@ function setupWebCrypto() {
   }
 }
 
+function setupCodeCoverage() {
+  // Resolve the coverage directory to an absolute path, and
+  // overwrite process.env so that the original path gets passed
+  // to child processes even when they switch cwd. Don't do anything if the
+  // --test-coverage flag is present, as the test runner will handle coverage.
+  if (process.env.NODE_V8_COVERAGE && !getOptionValue('--test-coverage')) {
+    process.env.NODE_V8_COVERAGE =
+      setupCoverageHooks(process.env.NODE_V8_COVERAGE);
+  }
+}
+
 // TODO(daeyeon): move this to internal/bootstrap/browser when the CLI flag is
 //                removed.
 function setupCustomEvent() {
@@ -299,27 +303,6 @@ function setupCustomEvent() {
   exposeInterface(globalThis, 'CustomEvent', CustomEvent);
 }
 
-// Setup User-facing NODE_V8_COVERAGE environment variable that writes
-// ScriptCoverage to a specified file.
-function setupCoverageHooks(dir) {
-  const cwd = require('internal/process/execution').tryGetCwd();
-  const { resolve } = require('path');
-  const coverageDirectory = resolve(cwd, dir);
-  const { sourceMapCacheToObject } =
-    require('internal/source_map/source_map_cache');
-
-  if (process.features.inspector) {
-    internalBinding('profiler').setCoverageDirectory(coverageDirectory);
-    internalBinding('profiler').setSourceMapCacheGetter(sourceMapCacheToObject);
-  } else {
-    process.emitWarning('The inspector is disabled, ' +
-                        'coverage could not be collected',
-                        'Warning');
-    return '';
-  }
-  return coverageDirectory;
-}
-
 function setupStacktracePrinterOnSigint() {
   if (!getOptionValue('--trace-sigint')) {
     return;