benchmark: add initial support for benchmark coverage

benchmark/_cli.js

@@ -24,6 +24,7 @@ function CLI(usage, settings) {
   this.optional = {};
   this.items = [];
   this.test = false;
+  this.coverage = false;
 
   for (const argName of settings.arrayArgs) {
     this.optional[argName] = [];
@@ -66,6 +67,14 @@ function CLI(usage, settings) {
       mode = 'both';
     } else if (arg === 'test') {
       this.test = true;
+    } else if (arg === 'coverage') {
+      this.coverage = true;
+      // TODO: add support to those benchmarks
+      const excludedBenchmarks = ['napi', 'http'];
+      this.items = Object.keys(benchmarks)
+        .filter((b) => !excludedBenchmarks.includes(b));
+      // Run once
+      this.optional.set = ['n=1'];
     } else if (['both', 'item'].includes(mode)) {
       // item arguments
       this.items.push(arg);
@@ -140,8 +149,8 @@ CLI.prototype.getCpuCoreSetting = function() {
   const isValid = /^(\d+(-\d+)?)(,\d+(-\d+)?)*$/.test(value);
   if (!isValid) {
     throw new Error(`
-        Invalid CPUSET format: "${value}". Please use a single core number (e.g., "0"), 
-        a range of cores (e.g., "0-3"), or a list of cores/ranges 
+        Invalid CPUSET format: "${value}". Please use a single core number (e.g., "0"),
+        a range of cores (e.g., "0-3"), or a list of cores/ranges
         (e.g., "0,2,4" or "0-2,4").\n\n${this.usage}
     `);
   }

benchmark/coverage.js

@@ -0,0 +1,117 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const Mod = require('node:module');
+
+const benchmarkFolder = __dirname;
+const dir = fs.readdirSync(path.join(__dirname, '../lib'));
+
+const allModuleExports = {};
+
+function getCallSite() {
+  const originalStackFormatter = Error.prepareStackTrace;
+  Error.prepareStackTrace = (err, stack) => {
+    // Some benchmarks change the stackTraceLimit if so, get the last line
+    // TODO: check if it matches the benchmark folder
+    if (stack.length >= 2) {
+      return `${stack[2].getFileName()}:${stack[2].getLineNumber()}`;
+    }
+    return stack;
+  };
+
+  const err = new Error();
+  err.stack; // eslint-disable-line no-unused-expressions
+  Error.prepareStackTrace = originalStackFormatter;
+  return err.stack;
+}
+
+const skippedFunctionClasses = [
+  'EventEmitter',
+  'Worker',
+  'ClientRequest',
+  'Readable',
+  'StringDecoder',
+  'TLSSocket',
+  'MessageChannel',
+];
+
+const skippedModules = [
+  'node:cluster',
+  'node:trace_events',
+  'node:stream/promises',
+];
+
+function fetchModules(allModuleExports) {
+  for (const f of dir) {
+    if (f.endsWith('.js') && !f.startsWith('_')) {
+      const moduleName = `node:${f.slice(0, f.length - 3)}`;
+      if (skippedModules.includes(moduleName)) {
+        continue;
+      }
+      const exports = require(moduleName);
+      allModuleExports[moduleName] = Object.assign({}, exports);
+
+      for (const fnKey of Object.keys(exports)) {
+        if (typeof exports[fnKey] === 'function' && !fnKey.startsWith('_')) {
+          if (
+            exports[fnKey].toString().match(/^class/) ||
+            skippedFunctionClasses.includes(fnKey)
+          ) {
+            // Skip classes for now
+            continue;
+          }
+          const originalFn = exports[fnKey];
+          allModuleExports[moduleName][fnKey] = function() {
+            const callerStr = getCallSite();
+            if (typeof callerStr === 'string' && callerStr.startsWith(benchmarkFolder) &&
+              callerStr.replace(benchmarkFolder, '').match(/^\/.+\/.+/)) {
+              if (!allModuleExports[moduleName][fnKey]._called) {
+                allModuleExports[moduleName][fnKey]._called = 0;
+              }
+              allModuleExports[moduleName][fnKey]._called++;
+
+
+              if (!allModuleExports[moduleName][fnKey]._calls) {
+                allModuleExports[moduleName][fnKey]._calls = [];
+              }
+              allModuleExports[moduleName][fnKey]._calls.push(callerStr);
+            }
+            return originalFn.apply(exports, arguments);
+          };
+        }
+      }
+    }
+  }
+}
+
+fetchModules(allModuleExports);
+
+const req = Mod.prototype.require;
+Mod.prototype.require = function(id) {
+  let newId = id;
+  if (!id.startsWith('node:')) {
+    newId = `node:${id}`;
+  }
+  const data = allModuleExports[newId];
+  if (!data) {
+    return req.apply(this, arguments);
+  }
+  return data;
+};
+
+process.on('beforeExit', () => {
+  for (const module of Object.keys(allModuleExports)) {
+    for (const fn of Object.keys(allModuleExports[module])) {
+      if (allModuleExports[module][fn]?._called) {
+        const _fn = allModuleExports[module][fn];
+        process.send({
+          type: 'coverage',
+          module,
+          fn,
+          times: _fn._called,
+        });
+      }
+    }
+  }
+});

benchmark/run.js

@@ -1,7 +1,8 @@
 'use strict';
 
-const path = require('path');
+const path = require('node:path');
 const { spawn, fork } = require('node:child_process');
+const fs = require('node:fs');
 const CLI = require('./_cli.js');
 
 const cli = new CLI(`usage: ./node run.js [options] [--] <category> ...
@@ -16,6 +17,8 @@ const cli = new CLI(`usage: ./node run.js [options] [--] <category> ...
   --format [simple|csv]     optional value that specifies the output format
   test                      only run a single configuration from the options
                             matrix
+  coverage                  generate a coverage report for the nodejs
+                            benchmark suite
   all                       each benchmark category is run one after the other
 
   Examples:
@@ -41,15 +44,47 @@ if (!validFormats.includes(format)) {
   return;
 }
 
-if (format === 'csv') {
+if (format === 'csv' && !cli.coverage) {
   console.log('"filename", "configuration", "rate", "time"');
 }
 
+function fetchModules() {
+  const dir = fs.readdirSync(path.join(__dirname, '../lib'));
+  const allModuleExports = {};
+  for (const f of dir) {
+    if (f.endsWith('.js') && !f.startsWith('_')) {
+      const moduleName = `node:${f.slice(0, f.length - 3)}`;
+      const exports = require(moduleName);
+      allModuleExports[moduleName] = {};
+      for (const fnKey of Object.keys(exports)) {
+        if (typeof exports[fnKey] === 'function' && !fnKey.startsWith('_')) {
+          allModuleExports[moduleName] = {
+            ...allModuleExports[moduleName],
+            [fnKey]: 0,
+          };
+        }
+      }
+    }
+  }
+  return allModuleExports;
+}
+
+let allModuleExports = {};
+if (cli.coverage) {
+  allModuleExports = fetchModules();
+}
+
 (function recursive(i) {
   const filename = benchmarks[i];
   const scriptPath = path.resolve(__dirname, filename);
 
-  const args = cli.test ? ['--test'] : cli.optional.set;
+  const args = cli.test ? ['--test'] : [...cli.optional.set];
+
+  let execArgv = [];
+  if (cli.coverage) {
+    execArgv = ['-r', path.join(__dirname, './coverage.js'), '--experimental-sqlite', '--no-warnings'];
+  }
+
   const cpuCore = cli.getCpuCoreSetting();
   let child;
   if (cpuCore !== null) {
@@ -60,6 +95,7 @@ if (format === 'csv') {
     child = fork(
       scriptPath,
       args,
+      { execArgv },
     );
   }
 
@@ -69,6 +105,15 @@ if (format === 'csv') {
   }
 
   child.on('message', (data) => {
+    if (cli.coverage) {
+      if (data.type === 'coverage') {
+        if (allModuleExports[data.module][data.fn] !== undefined) {
+          delete allModuleExports[data.module][data.fn];
+        }
+      }
+      return;
+    }
+
     if (data.type !== 'report') {
       return;
     }
@@ -102,3 +147,44 @@ if (format === 'csv') {
     }
   });
 })(0);
+
+const skippedFunctionClasses = [
+  'EventEmitter',
+  'Worker',
+  'ClientRequest',
+  'Readable',
+  'StringDecoder',
+  'TLSSocket',
+  'MessageChannel',
+];
+
+const skippedModules = [
+  'node:cluster',
+  'node:trace_events',
+  'node:stream/promises',
+];
+
+if (cli.coverage) {
+  process.on('beforeExit', () => {
+    for (const key in allModuleExports) {
+      if (skippedModules.includes(key)) continue;
+      const tableData = [];
+      for (const innerKey in allModuleExports[key]) {
+        if (
+          allModuleExports[key][innerKey].toString().match(/^class/) ||
+            skippedFunctionClasses.includes(innerKey)
+        ) {
+          continue;
+        }
+
+        tableData.push({
+          [key]: innerKey,
+          Values: allModuleExports[key][innerKey],
+        });
+      }
+      if (tableData.length) {
+        console.table(tableData);
+      }
+    }
+  });
+}

benchmark/url/url-searchparams-update.js

@@ -17,7 +17,7 @@ function getMethod(url, property) {
 
 function main({ searchParams, property, n }) {
   const url = new URL('https://nodejs.org');
-  if (searchParams === 'true') assert(url.searchParams);
+  if (searchParams === 'true') assert.ok(url.searchParams);
 
   const method = getMethod(url, property);