refactor(gatsby): load config and plugins in worker (#31773)

* refactor(gatsby): single function to load config and plugins

* expose loadConfigAndPlugins function in worker

* don't rely on cwd when loading plugins

* add test veryfing config can be loaded and APIs executed in worker

* adjust cli integration tests to cover for activity change (merging loading config and plugins

* fix functions (flag need to be handled before plugins loading)

* fix build error stack traces when ... gatsby-telemetry is imported in worker

* satisfy ts

* ensure double slashes (?)

integration-tests/gatsby-cli/__tests__/build.js

@@ -12,8 +12,9 @@ describe(`gatsby build`, () => {
   it(`creates a built gatsby site`, () => {
     const [code, logs] = GatsbyCLI.from(cwd).invoke(`build`)
 
-    logs.should.contain(`success open and validate gatsby-configs`)
-    logs.should.contain(`success load plugins`)
+    logs.should.contain(
+      `success open and validate gatsby-configs, load plugins`
+    )
     logs.should.contain(`success onPreInit`)
     logs.should.contain(`success initialize cache`)
     logs.should.contain(`success copy gatsby files`)

integration-tests/gatsby-cli/__tests__/develop.js

@@ -26,8 +26,9 @@ describe(`gatsby develop`, () => {
 
     // 3. Make sure logs for the user contain expected results
     const logs = getLogs()
-    logs.should.contain(`success open and validate gatsby-configs`)
-    logs.should.contain(`success load plugins`)
+    logs.should.contain(
+      `success open and validate gatsby-configs, load plugins`
+    )
     logs.should.contain(`success onPreInit`)
     logs.should.contain(`success initialize cache`)
     logs.should.contain(`success copy gatsby files`)

integration-tests/gatsby-cli/__tests__/repl.js

@@ -21,8 +21,9 @@ describe(`gatsby repl`, () => {
 
     // 3. Make assertions
     const logs = getLogs()
-    logs.should.contain(`success open and validate gatsby-configs`)
-    logs.should.contain(`success load plugins`)
+    logs.should.contain(
+      `success open and validate gatsby-configs, load plugins`
+    )
     logs.should.contain(`success onPreInit`)
     logs.should.contain(`success initialize cache`)
     logs.should.contain(`success copy gatsby files`)

packages/gatsby-cli/src/reporter/prepare-stack-trace.ts

@@ -109,6 +109,19 @@ function getPosition({
   map: BasicSourceMapConsumer | IndexedSourceMapConsumer
   frame: stackTrace.StackFrame
 }): NullableMappedPosition {
+  if (frame.getFileName().includes(`webpack:`)) {
+    // if source-map-register is initiated, stack traces would already be converted
+    return {
+      column: frame.getColumnNumber() - 1,
+      line: frame.getLineNumber(),
+      source: frame
+        .getFileName()
+        .substr(frame.getFileName().indexOf(`webpack:`))
+        .replace(/webpack:\/+/g, `webpack://`),
+      name: null,
+    }
+  }
+
   const line = frame.getLineNumber()
   const column = frame.getColumnNumber()
   return map.originalPositionFor({ line, column })

packages/gatsby/src/bootstrap/load-config-and-plugins.ts

@@ -0,0 +1,94 @@
+import reporter from "gatsby-cli/lib/reporter"
+import telemetry from "gatsby-telemetry"
+
+import { IFlattenedPlugin } from "./load-plugins/types"
+
+import { preferDefault } from "../bootstrap/prefer-default"
+import { getConfigFile } from "../bootstrap/get-config-file"
+import { loadPlugins } from "../bootstrap/load-plugins"
+import { internalActions } from "../redux/actions"
+import loadThemes from "../bootstrap/load-themes"
+import { store } from "../redux"
+import handleFlags from "../utils/handle-flags"
+import availableFlags from "../utils/flags"
+
+export async function loadConfigAndPlugins({
+  siteDirectory,
+  processFlags = false,
+}: {
+  siteDirectory: string
+  processFlags: boolean
+}): Promise<{
+  config: any
+  flattenedPlugins: Array<IFlattenedPlugin>
+}> {
+  // Try opening the site's gatsby-config.js file.
+  const { configModule, configFilePath } = await getConfigFile(
+    siteDirectory,
+    `gatsby-config`
+  )
+  let config = preferDefault(configModule)
+
+  // The root config cannot be exported as a function, only theme configs
+  if (typeof config === `function`) {
+    reporter.panic({
+      id: `10126`,
+      context: {
+        configName: `gatsby-config`,
+        siteDirectory,
+      },
+    })
+  }
+
+  if (config && processFlags) {
+    // Setup flags
+    if (config) {
+      // Get flags
+      const { enabledConfigFlags, unknownFlagMessage, message } = handleFlags(
+        availableFlags,
+        config.flags
+      )
+
+      if (unknownFlagMessage !== ``) {
+        reporter.warn(unknownFlagMessage)
+      }
+
+      //  set process.env for each flag
+      enabledConfigFlags.forEach(flag => {
+        process.env[flag.env] = `true`
+      })
+
+      // Print out message.
+      if (message !== ``) {
+        reporter.info(message)
+      }
+
+      //  track usage of feature
+      enabledConfigFlags.forEach(flag => {
+        if (flag.telemetryId) {
+          telemetry.trackFeatureIsUsed(flag.telemetryId)
+        }
+      })
+
+      // Track the usage of config.flags
+      if (config.flags) {
+        telemetry.trackFeatureIsUsed(`ConfigFlags`)
+      }
+    }
+  }
+
+  // theme gatsby configs can be functions or objects
+  if (config) {
+    const plugins = await loadThemes(config, {
+      configFilePath,
+      rootDir: siteDirectory,
+    })
+    config = plugins.config
+  }
+
+  store.dispatch(internalActions.setSiteConfig(config))
+
+  const flattenedPlugins = await loadPlugins(config, siteDirectory)
+
+  return { config, flattenedPlugins }
+}

packages/gatsby/src/bootstrap/load-plugins/load.ts

@@ -71,9 +71,9 @@ export function resolvePlugin(
   rootDir = (!_.isString(plugin) && plugin.parentDir) || rootDir
 
   // Only find plugins when we're not given an absolute path
-  if (!existsSync(pluginName)) {
+  if (!existsSync(pluginName) && rootDir) {
     // Find the plugin in the local plugins folder
-    const resolvedPath = slash(path.resolve(`./plugins/${pluginName}`))
+    const resolvedPath = slash(path.join(rootDir, `plugins/${pluginName}`))
 
     if (existsSync(resolvedPath)) {
       if (existsSync(`${resolvedPath}/package.json`)) {

packages/gatsby/src/bootstrap/load-themes/index.js

@@ -31,7 +31,7 @@ const resolveTheme = async (
     // local themes with same name that do different things and name being
     // main identifier that Gatsby uses right now, it's safer not to support it for now.
     if (isMainConfig) {
-      pathToLocalTheme = path.join(path.resolve(`.`), `plugins`, themeName)
+      pathToLocalTheme = path.join(rootDir, `plugins`, themeName)
       // is a local plugin OR it doesn't exist
       try {
         const { resolve } = resolvePlugin(themeName, rootDir)

packages/gatsby/src/services/initialize.ts

@@ -8,24 +8,18 @@ import path from "path"
 import telemetry from "gatsby-telemetry"
 
 import apiRunnerNode from "../utils/api-runner-node"
-import handleFlags from "../utils/handle-flags"
 import { getBrowsersList } from "../utils/browserslist"
 import { Store, AnyAction } from "redux"
-import { preferDefault } from "../bootstrap/prefer-default"
 import * as WorkerPool from "../utils/worker/pool"
 import { startPluginRunner } from "../redux/plugin-runner"
-import { loadPlugins } from "../bootstrap/load-plugins"
 import { store, emitter } from "../redux"
-import loadThemes from "../bootstrap/load-themes"
 import reporter from "gatsby-cli/lib/reporter"
-import { getConfigFile } from "../bootstrap/get-config-file"
 import { removeStaleJobs } from "../bootstrap/remove-stale-jobs"
 import { IPluginInfoOptions } from "../bootstrap/load-plugins/types"
-import { internalActions } from "../redux/actions"
 import { IGatsbyState } from "../redux/types"
 import { IBuildContext } from "./types"
-import availableFlags from "../utils/flags"
 import { detectLmdbStore } from "../datastore"
+import { loadConfigAndPlugins } from "../bootstrap/load-config-and-plugins"
 
 interface IPluginResolution {
   resolve: string
@@ -137,61 +131,18 @@ export async function initialize({
   emitter.on(`END_JOB`, onEndJob)
 
   // Try opening the site's gatsby-config.js file.
-  let activity = reporter.activityTimer(`open and validate gatsby-configs`, {
-    parentSpan,
-  })
-  activity.start()
-  const { configModule, configFilePath } = await getConfigFile(
-    program.directory,
-    `gatsby-config`
-  )
-  let config = preferDefault(configModule)
-
-  // The root config cannot be exported as a function, only theme configs
-  if (typeof config === `function`) {
-    reporter.panic({
-      id: `10126`,
-      context: {
-        configName: `gatsby-config`,
-        path: program.directory,
-      },
-    })
-  }
-
-  // Setup flags
-  if (config) {
-    // Get flags
-    const { enabledConfigFlags, unknownFlagMessage, message } = handleFlags(
-      availableFlags,
-      config.flags
-    )
-
-    if (unknownFlagMessage !== ``) {
-      reporter.warn(unknownFlagMessage)
+  let activity = reporter.activityTimer(
+    `open and validate gatsby-configs, load plugins`,
+    {
+      parentSpan,
     }
+  )
+  activity.start()
 
-    //  set process.env for each flag
-    enabledConfigFlags.forEach(flag => {
-      process.env[flag.env] = `true`
-    })
-
-    // Print out message.
-    if (message !== ``) {
-      reporter.info(message)
-    }
-
-    //  track usage of feature
-    enabledConfigFlags.forEach(flag => {
-      if (flag.telemetryId) {
-        telemetry.trackFeatureIsUsed(flag.telemetryId)
-      }
-    })
-
-    // Track the usage of config.flags
-    if (config.flags) {
-      telemetry.trackFeatureIsUsed(`ConfigFlags`)
-    }
-  }
+  const { config, flattenedPlugins } = await loadConfigAndPlugins({
+    siteDirectory: program.directory,
+    processFlags: true,
+  })
 
   // TODO: figure out proper way of disabling loading indicator
   // for now GATSBY_QUERY_ON_DEMAND_LOADING_INDICATOR=false gatsby develop
@@ -206,23 +157,12 @@ export async function initialize({
   }
   detectLmdbStore()
 
-  // theme gatsby configs can be functions or objects
-  if (config) {
-    const plugins = await loadThemes(config, {
-      configFilePath,
-      rootDir: program.directory,
-    })
-    config = plugins.config
-  }
-
   if (config && config.polyfill) {
     reporter.warn(
       `Support for custom Promise polyfills has been removed in Gatsby v2. We only support Babel 7's new automatic polyfilling behavior.`
     )
   }
 
-  store.dispatch(internalActions.setSiteConfig(config))
-
   activity.end()
 
   if (process.env.GATSBY_EXPERIMENTAL_QUERY_ON_DEMAND) {
@@ -241,13 +181,6 @@ export async function initialize({
   // run stale jobs
   store.dispatch(removeStaleJobs(store.getState()))
 
-  activity = reporter.activityTimer(`load plugins`, {
-    parentSpan,
-  })
-  activity.start()
-  const flattenedPlugins = await loadPlugins(config, program.directory)
-  activity.end()
-
   // Multiple occurrences of the same name-version-pair can occur,
   // so we report an array of unique pairs
   const pluginsStr = _.uniq(flattenedPlugins.map(p => `${p.name}@${p.version}`))

packages/gatsby/src/utils/worker/__tests__/config.ts

@@ -0,0 +1,31 @@
+import { createTestWorker, GatsbyTestWorkerPool } from "./test-helpers"
+import { store } from "../../../redux"
+import * as path from "path"
+
+let worker: GatsbyTestWorkerPool | undefined
+
+beforeEach(() => {
+  store.dispatch({ type: `DELETE_CACHE` })
+})
+
+afterEach(() => {
+  if (worker) {
+    worker.end()
+    worker = undefined
+  }
+})
+
+it(`can load config and execute node API in worker`, async () => {
+  worker = createTestWorker()
+
+  const siteDirectory = path.join(__dirname, `fixtures`, `sample-site`)
+
+  // plugin options for custom local plugin contains function (() => `foo`)
+  await worker.loadConfigAndPlugins({ siteDirectory })
+
+  // plugin API execute function from plugin options and store result in `global`
+  await worker.runAPI(`createSchemaCustomization`)
+
+  // getting result stored in `global`
+  expect(await worker.getAPIRunResult()).toEqual(`foo`)
+})

packages/gatsby/src/utils/worker/__tests__/fixtures/sample-site/gatsby-config.js

@@ -0,0 +1,10 @@
+module.exports = {
+  plugins: [
+    {
+      resolve: `gatsby-plugin-test`,
+      options: {
+        fn: () => `foo`
+      }
+    }
+  ],
+}

packages/gatsby/src/utils/worker/__tests__/fixtures/sample-site/plugins/gatsby-plugin-test/gatsby-node.js

@@ -0,0 +1,3 @@
+exports.createSchemaCustomization = (_, pluginOptions) => {
+  global.test = pluginOptions.fn()
+}

packages/gatsby/src/utils/worker/__tests__/fixtures/sample-site/plugins/gatsby-plugin-test/package.json

@@ -0,0 +1,3 @@
+{
+  "main": "gatsby-node.js"
+}

packages/gatsby/src/utils/worker/__tests__/test-helpers/child-for-tests.ts

@@ -1,5 +1,6 @@
 import { getNode } from "../../../../datastore"
 import reporter from "gatsby-cli/lib/reporter"
+import apiRunner from "../../../api-runner-node"
 
 // re-export all usual methods from production worker
 export * from "../../child"
@@ -16,3 +17,13 @@ export function log(message: string): boolean {
   reporter.log(message)
   return true
 }
+
+// test: config
+export async function runAPI(apiName: string): Promise<any> {
+  return await apiRunner(apiName)
+}
+
+// test: config
+export function getAPIRunResult(): string | undefined {
+  return (global as any).test
+}

packages/gatsby/src/utils/worker/child.ts → packages/gatsby/src/utils/worker/child/index.ts

@@ -1,2 +1,3 @@
 // Note: this doesn't check for conflicts between module exports
 export { renderHTMLProd, renderHTMLDev } from "./render-html"
+export { loadConfigAndPlugins } from "./load-config-and-plugins"

packages/gatsby/src/utils/worker/child/load-config-and-plugins.ts

@@ -0,0 +1 @@
+export { loadConfigAndPlugins } from "../../../bootstrap/load-config-and-plugins"