Add profiling to webpack loaders (#20392)

Follow-up to #20357 with additional tracers.

.vscode/launch.json

@@ -22,7 +22,8 @@
       "runtimeExecutable": "yarn",
       "runtimeArgs": ["run", "debug", "build", "test/integration/basic"],
       "skipFiles": ["<node_internals>/**"],
-      "port": 9229
+      "port": 9229,
+      "outFiles": ["${workspaceFolder}/packages/next/dist/**/*"]
     },
     {
       "name": "Launch app production",

.vscode/settings.json

@@ -4,5 +4,6 @@
     "javascriptreact",
     { "language": "typescript", "autoFix": true },
     { "language": "typescriptreact", "autoFix": true }
-  ]
+  ],
+  "debug.javascript.unmapMissingSources": true
 }

packages/next/build/webpack/loaders/babel-loader/src/Error.js

@@ -0,0 +1,35 @@
+const STRIP_FILENAME_RE = /^[^:]+: /
+
+const format = (err) => {
+  if (err instanceof SyntaxError) {
+    err.name = 'SyntaxError'
+    err.message = err.message.replace(STRIP_FILENAME_RE, '')
+
+    err.hideStack = true
+  } else if (err instanceof TypeError) {
+    err.name = null
+    err.message = err.message.replace(STRIP_FILENAME_RE, '')
+
+    err.hideStack = true
+  }
+
+  return err
+}
+
+class LoaderError extends Error {
+  constructor(err) {
+    super()
+
+    const { name, message, codeFrame, hideStack } = format(err)
+
+    this.name = 'BabelLoaderError'
+
+    this.message = `${name ? `${name}: ` : ''}${message}\n\n${codeFrame}\n`
+
+    this.hideStack = hideStack
+
+    Error.captureStackTrace(this, this.constructor)
+  }
+}
+
+export default LoaderError

packages/next/build/webpack/loaders/babel-loader/src/cache.js

@@ -0,0 +1,57 @@
+import { createHash } from 'crypto'
+import { tracer, traceAsyncFn } from '../../../../tracer'
+import transform from './transform'
+import cacache from 'next/dist/compiled/cacache'
+
+async function read(cacheDirectory, etag) {
+  const cachedResult = await traceAsyncFn(
+    tracer.startSpan('read-cache-file'),
+    async () => await cacache.get(cacheDirectory, etag)
+  )
+
+  return JSON.parse(cachedResult.data)
+}
+
+function write(cacheDirectory, etag, data) {
+  return cacache.put(cacheDirectory, etag, JSON.stringify(data))
+}
+
+const etag = function (source, identifier, options) {
+  const hash = createHash('md4')
+
+  const contents = JSON.stringify({ source, options, identifier })
+
+  hash.update(contents)
+
+  return hash.digest('hex')
+}
+
+export default async function handleCache(params) {
+  const span = tracer.startSpan('handle-cache')
+  return traceAsyncFn(span, async () => {
+    const { source, options = {}, cacheIdentifier, cacheDirectory } = params
+
+    const file = etag(source, cacheIdentifier)
+
+    try {
+      // No errors mean that the file was previously cached
+      // we just need to return it
+      const res = await read(cacheDirectory, file)
+      span.setAttribute('cache', res ? 'HIT' : 'MISS')
+      return res
+    } catch (err) {}
+
+    // Otherwise just transform the file
+    // return it to the user asap and write it in cache
+    const result = await traceAsyncFn(
+      tracer.startSpan('transform'),
+      async () => {
+        return transform(source, options)
+      }
+    )
+
+    await write(cacheDirectory, file, result)
+
+    return result
+  })
+}

packages/next/build/webpack/loaders/babel-loader/src/index.js

@@ -0,0 +1,168 @@
+// import babel from 'next/dist/compiled/babel/core'
+import loaderUtils from 'loader-utils'
+import { tracer, traceAsyncFn, traceFn } from '../../../../tracer'
+import cache from './cache'
+import transform from './transform'
+
+// When using `import` Babel will be undefined
+const babel = require('next/dist/compiled/babel/core')
+
+export default function makeLoader(callback) {
+  const overrides = callback(babel)
+
+  return function (source, inputSourceMap) {
+    // Make the loader async
+    const cb = this.async()
+
+    loader.call(this, source, inputSourceMap, overrides).then(
+      (args) => cb(null, ...args),
+      (err) => cb(err)
+    )
+  }
+}
+
+async function loader(source, inputSourceMap, overrides) {
+  const span = tracer.startSpan('babel-loader')
+  return traceAsyncFn(span, async () => {
+    const filename = this.resourcePath
+    span.setAttribute('filename', filename)
+
+    let loaderOptions = loaderUtils.getOptions(this) || {}
+
+    let customOptions
+    if (overrides && overrides.customOptions) {
+      const result = await traceAsyncFn(
+        tracer.startSpan('loader-overrides-customoptions'),
+        async () =>
+          await overrides.customOptions.call(this, loaderOptions, {
+            source,
+            map: inputSourceMap,
+          })
+      )
+      customOptions = result.custom
+      loaderOptions = result.loader
+    }
+
+    // Standardize on 'sourceMaps' as the key passed through to Webpack, so that
+    // users may safely use either one alongside our default use of
+    // 'this.sourceMap' below without getting error about conflicting aliases.
+    if (
+      Object.prototype.hasOwnProperty.call(loaderOptions, 'sourceMap') &&
+      !Object.prototype.hasOwnProperty.call(loaderOptions, 'sourceMaps')
+    ) {
+      loaderOptions = Object.assign({}, loaderOptions, {
+        sourceMaps: loaderOptions.sourceMap,
+      })
+      delete loaderOptions.sourceMap
+    }
+
+    const programmaticOptions = Object.assign({}, loaderOptions, {
+      filename,
+      inputSourceMap: inputSourceMap || undefined,
+
+      // Set the default sourcemap behavior based on Webpack's mapping flag,
+      // but allow users to override if they want.
+      sourceMaps:
+        loaderOptions.sourceMaps === undefined
+          ? this.sourceMap
+          : loaderOptions.sourceMaps,
+
+      // Ensure that Webpack will get a full absolute path in the sourcemap
+      // so that it can properly map the module back to its internal cached
+      // modules.
+      sourceFileName: filename,
+      caller: {
+        name: 'babel-loader',
+
+        // Provide plugins with insight into webpack target.
+        // https://github.com/babel/babel-loader/issues/787
+        target: this.target,
+
+        // Webpack >= 2 supports ESM and dynamic import.
+        supportsStaticESM: true,
+        supportsDynamicImport: true,
+
+        // Webpack 5 supports TLA behind a flag. We enable it by default
+        // for Babel, and then webpack will throw an error if the experimental
+        // flag isn't enabled.
+        supportsTopLevelAwait: true,
+        ...loaderOptions.caller,
+      },
+    })
+    // Remove loader related options
+    delete programmaticOptions.cacheDirectory
+    delete programmaticOptions.cacheIdentifier
+
+    const config = traceFn(
+      tracer.startSpan('babel-load-partial-config-async'),
+      () => {
+        return babel.loadPartialConfig(programmaticOptions)
+      }
+    )
+
+    if (config) {
+      let options = config.options
+      if (overrides && overrides.config) {
+        options = await traceAsyncFn(
+          tracer.startSpan('loader-overrides-config'),
+          async () =>
+            await overrides.config.call(this, config, {
+              source,
+              map: inputSourceMap,
+              customOptions,
+            })
+        )
+      }
+
+      if (options.sourceMaps === 'inline') {
+        // Babel has this weird behavior where if you set "inline", we
+        // inline the sourcemap, and set 'result.map = null'. This results
+        // in bad behavior from Babel since the maps get put into the code,
+        // which Webpack does not expect, and because the map we return to
+        // Webpack is null, which is also bad. To avoid that, we override the
+        // behavior here so "inline" just behaves like 'true'.
+        options.sourceMaps = true
+      }
+
+      const { cacheDirectory, cacheIdentifier } = loaderOptions
+
+      let result
+      if (cacheDirectory) {
+        result = await cache({
+          source,
+          options,
+          cacheDirectory,
+          cacheIdentifier,
+          cacheCompression: false,
+        })
+      } else {
+        result = await traceAsyncFn(
+          tracer.startSpan('transform', {
+            attributes: {
+              filename,
+              cache: 'DISABLED',
+            },
+          }),
+          async () => {
+            return transform(source, options)
+          }
+        )
+      }
+
+      // TODO: Babel should really provide the full list of config files that
+      // were used so that this can also handle files loaded with 'extends'.
+      if (typeof config.babelrc === 'string') {
+        this.addDependency(config.babelrc)
+      }
+
+      if (result) {
+        const { code, map } = result
+
+        return [code, map]
+      }
+    }
+
+    // If the file was ignored, pass through the original content.
+    return [source, inputSourceMap]
+  })
+}

packages/next/build/webpack/loaders/babel-loader/src/transform.js

@@ -0,0 +1,29 @@
+import { transform as _transform } from 'next/dist/compiled/babel/core'
+import { promisify } from 'util'
+import LoaderError from './Error'
+
+const transform = promisify(_transform)
+
+export default async function (source, options) {
+  let result
+  try {
+    result = await transform(source, options)
+  } catch (err) {
+    throw err.message && err.codeFrame ? new LoaderError(err) : err
+  }
+
+  if (!result) return null
+
+  // We don't return the full result here because some entries are not
+  // really serializable. For a full list of properties see here:
+  // https://github.com/babel/babel/blob/main/packages/babel-core/src/transformation/index.js
+  // For discussion on this topic see here:
+  // https://github.com/babel/babel-loader/pull/629
+  const { ast, code, map, metadata, sourceType } = result
+
+  if (map && (!map.sourcesContent || !map.sourcesContent.length)) {
+    map.sourcesContent = [source]
+  }
+
+  return { ast, code, map, metadata, sourceType }
+}

packages/next/build/webpack/loaders/next-babel-loader.js

@@ -1,4 +1,4 @@
-import babelLoader from 'next/dist/compiled/babel-loader'
+import babelLoader from './babel-loader/src/index'
 import hash from 'next/dist/compiled/string-hash'
 import { basename, join } from 'path'
 import * as Log from '../../output/log'
@@ -8,7 +8,7 @@ import * as Log from '../../output/log'
 const cacheKey = 'babel-cache-' + 'o' + '-'
 const nextBabelPreset = require('../../babel/preset')
 
-module.exports = babelLoader.custom((babel) => {
+const customBabelLoader = babelLoader((babel) => {
   const presetItem = babel.createConfigItem(nextBabelPreset, {
     type: 'preset',
   })
@@ -37,7 +37,6 @@ module.exports = babelLoader.custom((babel) => {
       const loader = Object.assign(
         opts.cache
           ? {
-              cacheCompression: false,
               cacheDirectory: join(opts.distDir, 'cache', 'next-babel-loader'),
               cacheIdentifier:
                 cacheKey +
@@ -210,3 +209,5 @@ module.exports = babelLoader.custom((babel) => {
     },
   }
 })
+
+export default customBabelLoader

packages/next/build/webpack/loaders/next-client-pages-loader.ts

@@ -1,26 +1,33 @@
 import { loader } from 'webpack'
 import loaderUtils from 'loader-utils'
+import { tracer, traceFn } from '../../tracer'
 
 export type ClientPagesLoaderOptions = {
   absolutePagePath: string
   page: string
 }
 
 const nextClientPagesLoader: loader.Loader = function () {
-  const { absolutePagePath, page } = loaderUtils.getOptions(
-    this
-  ) as ClientPagesLoaderOptions
-  const stringifiedAbsolutePagePath = JSON.stringify(absolutePagePath)
-  const stringifiedPage = JSON.stringify(page)
+  const span = tracer.startSpan('next-client-pages-loader')
+  return traceFn(span, () => {
+    const { absolutePagePath, page } = loaderUtils.getOptions(
+      this
+    ) as ClientPagesLoaderOptions
 
-  return `
-    (window.__NEXT_P = window.__NEXT_P || []).push([
-      ${stringifiedPage},
-      function () {
-        return require(${stringifiedAbsolutePagePath});
-      }
-    ]);
-  `
+    span.setAttribute('absolutePagePath', absolutePagePath)
+
+    const stringifiedAbsolutePagePath = JSON.stringify(absolutePagePath)
+    const stringifiedPage = JSON.stringify(page)
+
+    return `
+      (window.__NEXT_P = window.__NEXT_P || []).push([
+        ${stringifiedPage},
+        function () {
+          return require(${stringifiedAbsolutePagePath});
+        }
+      ]);
+    `
+  })
 }
 
 export default nextClientPagesLoader