core: convert to ES modules

.eslintrc.cjs

@@ -36,7 +36,7 @@ module.exports = {
     }],
     'no-floating-decimal': 2,
     'max-len': [2, 100, {
-      ignorePattern: 'readJson\\(',
+      ignorePattern: 'readJson\\(|^import ',
       ignoreComments: true,
       ignoreUrls: true,
       tabWidth: 2,

.github/workflows/unit.yml

@@ -11,6 +11,7 @@ jobs:
     strategy:
       matrix:
         node: ['14', '16', '17']
+      fail-fast: false
     runs-on: ubuntu-latest
     name: node ${{ matrix.node }}
     env:

build/build-bundle.js

@@ -16,11 +16,12 @@ import {createRequire} from 'module';
 
 import esMain from 'es-main';
 import {rollup} from 'rollup';
-// @ts-expect-error: plugin has no types.
-import PubAdsPlugin from 'lighthouse-plugin-publisher-ads/plugin.js';
+// TODO(esmodules): convert pubads to esm
+// // @ts-expect-error: plugin has no types.
+// import PubAdsPlugin from 'lighthouse-plugin-publisher-ads/plugin.js';
 
 import * as rollupPlugins from './rollup-plugins.js';
-import Runner from '../lighthouse-core/runner.js';
+import {Runner} from '../lighthouse-core/runner.js';
 import {LH_ROOT} from '../root.js';
 import {readJson} from '../lighthouse-core/test/test-utils.js';
 
@@ -31,8 +32,8 @@ const COMMIT_HASH = execSync('git rev-parse HEAD').toString().trim();
 
 // HACK: manually include the lighthouse-plugin-publisher-ads audits.
 /** @type {Array<string>} */
-// @ts-expect-error
-const pubAdsAudits = PubAdsPlugin.audits.map(a => a.path);
+// // @ts-expect-error
+// const pubAdsAudits = PubAdsPlugin.audits.map(a => a.path);
 
 /** @param {string} file */
 const isDevtools = file =>
@@ -83,20 +84,24 @@ async function buildBundle(entryPath, distPath, opts = {minify: true}) {
   ];
 
   // Include lighthouse-plugin-publisher-ads.
-  if (isDevtools(entryPath) || isLightrider(entryPath)) {
-    dynamicModulePaths.push('lighthouse-plugin-publisher-ads');
-    pubAdsAudits.forEach(pubAdAudit => {
-      dynamicModulePaths.push(pubAdAudit);
-    });
-  }
+  // if (isDevtools(entryPath) || isLightrider(entryPath)) {
+  //   dynamicModulePaths.push('lighthouse-plugin-publisher-ads');
+  //   pubAdsAudits.forEach(pubAdAudit => {
+  //     dynamicModulePaths.push(pubAdAudit);
+  //   });
+  // }
 
   const bundledMapEntriesCode = dynamicModulePaths.map(modulePath => {
     const pathNoExt = modulePath.replace('.js', '');
-    return `['${pathNoExt}', require('${modulePath}')]`;
+    return `['${pathNoExt}', import('${modulePath}')]`;
   }).join(',\n');
 
   /** @type {Record<string, string>} */
-  const shimsObj = {};
+  const shimsObj = {
+    [require.resolve('../lighthouse-core/gather/connections/cri.js')]:
+      'export const CriConnection = {}',
+    [require.resolve('../package.json')]: `export const version = '${pkg.version}';`,
+  };
 
   const modulesToIgnore = [
     'puppeteer-core',
@@ -106,7 +111,6 @@ async function buildBundle(entryPath, distPath, opts = {minify: true}) {
     '@sentry/node',
     'source-map',
     'ws',
-    require.resolve('../lighthouse-core/gather/connections/cri.js'),
   ];
 
   // Don't include the stringified report in DevTools - see devtools-report-assets.js
@@ -124,9 +128,6 @@ async function buildBundle(entryPath, distPath, opts = {minify: true}) {
     shimsObj[modulePath] = 'export default {}';
   }
 
-  shimsObj[require.resolve('../package.json')] =
-    `export const version = '${pkg.version}';`;
-
   const bundle = await rollup({
     input: entryPath,
     context: 'globalThis',
@@ -135,8 +136,6 @@ async function buildBundle(entryPath, distPath, opts = {minify: true}) {
         delimiters: ['', ''],
         values: {
           '/* BUILD_REPLACE_BUNDLED_MODULES */': `[\n${bundledMapEntriesCode},\n]`,
-          '__dirname': (id) => `'${path.relative(LH_ROOT, path.dirname(id))}'`,
-          '__filename': (id) => `'${path.relative(LH_ROOT, id)}'`,
           // This package exports to default in a way that causes Rollup to get confused,
           // resulting in MessageFormat being undefined.
           'require(\'intl-messageformat\').default': 'require(\'intl-messageformat\')',
@@ -148,6 +147,14 @@ async function buildBundle(entryPath, distPath, opts = {minify: true}) {
           // TODO: Use globalThis directly.
           'global.isLightrider': 'globalThis.isLightrider',
           'global.isDevtools': 'globalThis.isDevtools',
+          // For some reason, `shim` doesn't work to force this module to return false, so instead
+          // just replace usages of it with false.
+          'esMain(import.meta)': 'false',
+          'import esMain from \'es-main\'': '',
+          // By default Rollup converts `import.meta` to a big mess of `document.currentScript && ...`,
+          // and uses the output name as the url. Instead, do a simpler conversion and use the
+          // module path.
+          'import.meta': (id) => `{url: '${path.relative(LH_ROOT, id)}'}`,
         },
       }),
       rollupPlugins.alias({
@@ -160,14 +167,23 @@ async function buildBundle(entryPath, distPath, opts = {minify: true}) {
         ...shimsObj,
         // Allows for plugins to import lighthouse.
         'lighthouse': `
-          import Audit from '${require.resolve('../lighthouse-core/audits/audit.js')}';
+          import {Audit} from '${require.resolve('../lighthouse-core/audits/audit.js')}';
           export {Audit};
         `,
-        // Most node 'url' polyfills don't include the WHATWG `URL` property, but
-        // that's all that's needed, so make a mini-polyfill.
-        // @see https://github.com/GoogleChrome/lighthouse/issues/5273
-        // TODO: remove when not needed for pubads (https://github.com/googleads/publisher-ads-lighthouse-plugin/pull/325)
-        'url': 'export const URL = globalThis.URL;',
+        'url': `
+          export const URL = globalThis.URL;
+          export const fileURLToPath = url => url;
+          export default {URL, fileURLToPath};
+        `,
+        'module': `
+          export const createRequire = () => {
+            return {
+              resolve() {
+                throw new Error('createRequire.resolve is not supported in bundled Lighthouse');
+              },
+            };
+          };
+        `,
       }),
       rollupPlugins.json(),
       rollupPlugins.inlineFs({verbose: false}),
@@ -211,6 +227,8 @@ async function buildBundle(entryPath, distPath, opts = {minify: true}) {
     banner,
     format: 'iife',
     sourcemap: DEBUG,
+    // Suppress code splitting.
+    inlineDynamicImports: true,
   });
   await bundle.close();
 }
@@ -222,7 +240,7 @@ async function cli(argv) {
   // Take paths relative to cwd and build.
   const [entryPath, distPath] = argv.slice(2)
     .map(filePath => path.resolve(process.cwd(), filePath));
-  await buildBundle(entryPath, distPath);
+  await buildBundle(entryPath, distPath, {minify: !process.env.DEBUG});
 }
 
 // Test if called from the CLI or as a module.

build/build-lightrider-bundles.js

@@ -52,10 +52,6 @@ async function buildStaticServerBundle() {
   const bundle = await rollup({
     input: 'lighthouse-cli/test/fixtures/static-server.js',
     plugins: [
-      rollupPlugins.shim({
-        'es-main': 'export default function() { return false; }',
-      }),
-      rollupPlugins.commonjs(),
       rollupPlugins.nodeResolve(),
     ],
     external: ['mime-types', 'glob'],

build/build-smokehouse-bundle.js

@@ -22,6 +22,15 @@ async function main() {
       rollupPlugins.shim({
         [smokehouseCliFilename]:
           'export function runLighthouse() { throw new Error("not supported"); }',
+        'module': `
+          export const createRequire = () => {
+            return {
+              resolve() {
+                throw new Error('createRequire.resolve is not supported in bundled Lighthouse');
+              },
+            };
+          };
+        `,
       }),
       rollupPlugins.inlineFs({verbose: Boolean(process.env.DEBUG)}),
       rollupPlugins.commonjs(),

build/plugins/inline-fs.js

@@ -277,7 +277,7 @@ function collapseToStringLiteral(node, filepath) {
     }
 
     case 'Identifier': {
-      if (node.name === '__dirname') {
+      if (node.name === '__dirname' || node.name === 'moduleDir') {
         return path.dirname(filepath);
       } else if (node.name === '__filename') {
         return filepath;

clients/devtools/devtools-entry.js

@@ -9,13 +9,13 @@
 
 import {Buffer} from 'buffer';
 
-import lighthouse from '../../lighthouse-core/index.js';
+import lighthouse, {legacyNavigation} from '../../lighthouse-core/index.js';
 import {navigation, startTimespan, snapshot} from '../../lighthouse-core/fraggle-rock/api.js';
-import RawProtocol from '../../lighthouse-core/gather/connections/raw.js';
+import {RawConnection} from '../../lighthouse-core/gather/connections/raw.js';
 import log from 'lighthouse-logger';
 import {lookupLocale} from '../../lighthouse-core/lib/i18n/i18n.js';
 import {registerLocaleData, getCanonicalLocales} from '../../shared/localization/format.js';
-import constants from '../../lighthouse-core/config/constants.js';
+import * as constants from '../../lighthouse-core/config/constants.js';
 
 /** @typedef {import('../../lighthouse-core/gather/connections/connection.js')} Connection */
 
@@ -52,17 +52,18 @@ function createConfig(categoryIDs, device) {
 
   return {
     extends: 'lighthouse:default',
-    plugins: ['lighthouse-plugin-publisher-ads'],
+    // TODO(esmodules): re-enable when pubads works again
+    // plugins: ['lighthouse-plugin-publisher-ads'],
     settings,
   };
 }
 
 /**
- * @param {RawProtocol.Port} port
- * @return {RawProtocol}
+ * @param {import('../../lighthouse-core/gather/connections/raw.js').Port} port
+ * @return {RawConnection}
  */
 function setUpWorkerConnection(port) {
-  return new RawProtocol(port);
+  return new RawConnection(port);
 }
 
 /** @param {(status: [string, string, string]) => void} listenCallback */
@@ -90,7 +91,7 @@ if (typeof self !== 'undefined') {
   // @ts-expect-error
   self.setUpWorkerConnection = setUpWorkerConnection;
   // @ts-expect-error
-  self.runLighthouse = lighthouse.legacyNavigation;
+  self.runLighthouse = legacyNavigation;
   // @ts-expect-error
   self.runLighthouseNavigation = navigation;
   // @ts-expect-error
@@ -110,4 +111,6 @@ if (typeof self !== 'undefined') {
   // For the bundle smoke test.
   // @ts-expect-error
   global.runBundledLighthouse = lighthouse;
+  // @ts-expect-error
+  global.runBundledLighthouseLegacyNavigation = legacyNavigation;
 }

clients/lightrider/lightrider-entry.js

@@ -8,12 +8,12 @@
 /* global globalThis */
 
 import {Buffer} from 'buffer';
-
 import log from 'lighthouse-logger';
-import lighthouse from '../../lighthouse-core/index.js';
-import LighthouseError from '../../lighthouse-core/lib/lh-error.js';
-import preprocessor from '../../lighthouse-core/lib/proto-preprocessor.js';
-import assetSaver from '../../lighthouse-core/lib/asset-saver.js';
+
+import {legacyNavigation} from '../../lighthouse-core/index.js';
+import {LighthouseError} from '../../lighthouse-core/lib/lh-error.js';
+import {processForProto} from '../../lighthouse-core/lib/proto-preprocessor.js';
+import * as assetSaver from '../../lighthouse-core/lib/asset-saver.js';
 
 import mobileConfig from '../../lighthouse-core/config/lr-mobile-config.js';
 import desktopConfig from '../../lighthouse-core/config/lr-desktop-config.js';
@@ -24,7 +24,7 @@ const LR_PRESETS = {
   desktop: desktopConfig,
 };
 
-/** @typedef {import('../../lighthouse-core/gather/connections/connection.js')} Connection */
+/** @typedef {import('../../lighthouse-core/gather/connections/connection.js').Connection} Connection */
 
 // Rollup seems to overlook some references to `Buffer`, so it must be made explicit.
 // (`parseSourceMapFromDataUrl` breaks without this)
@@ -64,11 +64,11 @@ export async function runLighthouseInLR(connection, url, flags, lrOpts) {
   }
 
   try {
-    const runnerResult = await lighthouse.legacyNavigation(url, flags, config, connection);
+    const runnerResult = await legacyNavigation(url, flags, config, connection);
     if (!runnerResult) throw new Error('Lighthouse finished without a runnerResult');
 
     // pre process the LHR for proto
-    const preprocessedLhr = preprocessor.processForProto(runnerResult.lhr);
+    const preprocessedLhr = processForProto(runnerResult.lhr);
 
     // When LR is called with |internal: {keep_raw_response: true, save_lighthouse_assets: true}|,
     // we log artifacts to raw_response.artifacts.

clients/test/lightrider/lightrider-entry-test.js

@@ -8,8 +8,8 @@
 import jestMock from 'jest-mock';
 import {strict as assert} from 'assert';
 import {runLighthouseInLR} from '../../lightrider/lightrider-entry.js';
-import Runner from '../../../lighthouse-core/runner.js';
-import LighthouseError from '../../../lighthouse-core/lib/lh-error.js';
+import {Runner} from '../../../lighthouse-core/runner.js';
+import {LighthouseError} from '../../../lighthouse-core/lib/lh-error.js';
 
 describe('lightrider-entry', () => {
   describe('#runLighthouseInLR', () => {

docs/recipes/auth/example-lh-auth.js

@@ -10,8 +10,9 @@
  * See docs/recipes/auth/README.md for more.
  */
 
-const puppeteer = require('puppeteer');
-const lighthouse = require('lighthouse');
+import puppeteer from 'puppeteer';
+import lighthouse from 'lighthouse';
+import esMain from 'es-main';
 
 // This port will be used by Lighthouse later. The specific port is arbitrary.
 const PORT = 8041;
@@ -71,11 +72,11 @@ async function main() {
   console.log(JSON.stringify(result.lhr, null, 2));
 }
 
-if (require.main === module) {
+if (esMain(import.meta)) {
   main();
-} else {
-  module.exports = {
-    login,
-    logout,
-  };
 }
+
+export {
+  login,
+  logout,
+};

docs/recipes/auth/package.json

@@ -1,9 +1,11 @@
 {
   "private": true,
+  "type": "module",
   "scripts": {
     "start": "node server/server.js"
   },
   "dependencies": {
+    "es-main": "^1.2.0",
     "express": "^4.17.1",
     "express-session": "^1.16.2",
     "lighthouse": "file:../../../dist/lighthouse.tgz",

docs/recipes/auth/server/server.js

@@ -10,13 +10,18 @@
  * page. See docs/recipes/auth/README.md for more.
  */
 
-const createError = require('http-errors');
-const express = require('express');
-const morgan = require('morgan');
-const session = require('express-session');
-const http = require('http');
-const path = require('path');
-const PUBLIC_DIR = path.join(__dirname, 'public');
+import createError from 'http-errors';
+
+import express from 'express';
+import morgan from 'morgan';
+import session from 'express-session';
+import http from 'http';
+import path from 'path';
+import url from 'url';
+import esMain from 'es-main';
+
+const moduleDir = path.dirname(url.fileURLToPath(import.meta.url));
+const PUBLIC_DIR = path.join(moduleDir, 'public');
 
 const app = express();
 
@@ -78,8 +83,9 @@ app.use(function(err, req, res, next) {
 });
 
 const server = http.createServer(app);
-if (require.main === module) {
+
+if (esMain(import.meta)) {
   server.listen(10632);
-} else {
-  module.exports = server;
 }
+
+export default server;