feat(gatsby): Schema rebuilding (#19092)

* Refactor example-value to support fast incremental rebuilding

* Detect node structure changes incrementally (to avoid expensive checks later)

* Track metadata for inference in the redux

* New `rebuildWithTypes` API for incremental schema rebuilding

* Schema hot reloading for `develop`

* Move things around for better cohesion

* Replace old `getExampleValue` with new APIs based on inference metadata

* Cleanup / rename things for consistency

* Proper handling of ADD_FIELD_TO_NODE action

* Make sure stale inferred fields are removed from the schema

* More tests to and related fixes to conflict reporting

* Clear derived TypeComposers and InputTypeComposers on type rebuild

* More tests for schema rebuilding

* Delete empty inferred type

* Refactor: use functions for field names generated out of a type name

* More tests + switched to inline snapshots for tests readability

* Support adding / deleting child convenience fields on parent type

* Added nested extension test

* Apply extensions and other special logic to all nested types

* Make sure all derived types are processed on rebuild (including recursive)

* Re-using common method in schema-hot-reloader

* Test conflicts during rebuild-schema

* Test incremental example value building

* Tests for compatibility with schema customization

* Parent type processing should not mess with child structure

* Fix typo in comments

Co-Authored-By: Michael <184316+muescha@users.noreply.github.com>

* Moved `createSchemaCustomization` API call before node sourcing

* Do not collect inference metadata for types with @dontInfer directive (or infer:false extension)

* Use constants vs literal strings for extension names when analyzing type defs

* Develop: reload eslint config on schema rebuild

* Re-run queries on schema rebuild

* Fix loki tests

* Fix eslint error

* Example value: do not return empty object

* Updating tests structure

* Split tests for sitepage and schema rebuild to separate files

* Tests for rebuilding types with existing relations

* Step back and use full schema rebuild (which is relatively fast with inference metadata)

* Fix eslint errors

* Fix invalid test

* Take a list of types for inference from metadata vs node store

* Handle DELETE_NODES action

* Cleaning up a little bit

* Fix loki reducer for DELETE_NODES action

* More descriptive naming for eslint graphql schema reload

* Fix invalid webpack compiler hook argument

* Better detection of changed types to avoid unnecessary rebuilds

* Add missing snapshot

* Add new tests to clarify updating of ___NODE fields

* Be a bit more defensive with haveEqualFields args

* Re-run schema customization on __refresh

* Rebuild schema on schema customization in develop (i.e. called via __refresh)

* Add support for node update

* Fix rebuildWithSitePage extensions

packages/gatsby-transformer-remark/src/__tests__/extend-node.js

@@ -50,17 +50,19 @@ async function queryResult(
     addInferredFields,
   } = require(`../../../gatsby/src/schema/infer/add-inferred-fields`)
   const {
-    getExampleValue,
-  } = require(`../../../gatsby/src/schema/infer/example-value`)
+    addNodes,
+    getExampleObject,
+  } = require(`../../../gatsby/src/schema/infer/inference-metadata`)
 
   const typeName = `MarkdownRemark`
   const sc = createSchemaComposer()
   const tc = sc.createObjectTC(typeName)
   sc.addTypeDefs(typeDefs)
+  const inferenceMetadata = addNodes({ typeName }, nodes)
   addInferredFields({
     schemaComposer: sc,
     typeComposer: tc,
-    exampleValue: getExampleValue({ nodes, typeName }),
+    exampleValue: getExampleObject(inferenceMetadata),
   })
   tc.addFields(extendNodeTypeFields)
   sc.Query.addFields({

packages/gatsby-transformer-remark/src/__tests__/on-node-create.js

@@ -127,16 +127,18 @@ yadda yadda
         addInferredFields,
       } = require(`../../../gatsby/src/schema/infer/add-inferred-fields`)
       const {
-        getExampleValue,
-      } = require(`../../../gatsby/src/schema/infer/example-value`)
+        addNodes,
+        getExampleObject,
+      } = require(`../../../gatsby/src/schema/infer/inference-metadata`)
 
       const sc = createSchemaComposer()
       const typeName = `MarkdownRemark`
       const tc = sc.createObjectTC(typeName)
+      const inferenceMetadata = addNodes({ typeName }, nodes)
       addInferredFields({
         schemaComposer: sc,
         typeComposer: tc,
-        exampleValue: getExampleValue({ nodes, typeName }),
+        exampleValue: getExampleObject(inferenceMetadata),
       })
       sc.Query.addFields({
         listNode: { type: [tc], resolve: () => nodes },

packages/gatsby/src/bootstrap/index.js

@@ -414,6 +414,16 @@ module.exports = async (args: BootstrapArgs) => {
   })
   activity.end()
 
+  // Prepare static schema types
+  activity = report.activityTimer(`createSchemaCustomization`, {
+    parentSpan: bootstrapSpan,
+  })
+  activity.start()
+  await require(`../utils/create-schema-customization`)({
+    parentSpan: bootstrapSpan,
+  })
+  activity.end()
+
   // Source nodes
   activity = report.activityTimer(`source and transform nodes`, {
     parentSpan: bootstrapSpan,

packages/gatsby/src/bootstrap/schema-hot-reloader.js

@@ -0,0 +1,49 @@
+const { debounce, cloneDeep } = require(`lodash`)
+const { emitter, store } = require(`../redux`)
+const { rebuild } = require(`../schema`)
+const { haveEqualFields } = require(`../schema/infer/inference-metadata`)
+const { updateStateAndRunQueries } = require(`../query/query-watcher`)
+const report = require(`gatsby-cli/lib/reporter`)
+
+const inferredTypesChanged = (inferenceMetadata, prevInferenceMetadata) =>
+  Object.keys(inferenceMetadata).filter(
+    type =>
+      inferenceMetadata[type].dirty &&
+      !haveEqualFields(inferenceMetadata[type], prevInferenceMetadata[type])
+  ).length > 0
+
+const schemaChanged = (schemaCustomization, lastSchemaCustomization) =>
+  [`fieldExtensions`, `printConfig`, `thirdPartySchemas`, `types`].some(
+    key => schemaCustomization[key] !== lastSchemaCustomization[key]
+  )
+
+let lastMetadata
+let lastSchemaCustomization
+
+// API_RUNNING_QUEUE_EMPTY could be emitted multiple types
+// in a short period of time, so debounce seems reasonable
+const maybeRebuildSchema = debounce(async () => {
+  const { inferenceMetadata, schemaCustomization } = store.getState()
+
+  if (
+    !inferredTypesChanged(inferenceMetadata, lastMetadata) &&
+    !schemaChanged(schemaCustomization, lastSchemaCustomization)
+  ) {
+    return
+  }
+
+  const activity = report.activityTimer(`rebuild schema`)
+  activity.start()
+  lastMetadata = cloneDeep(inferenceMetadata)
+  lastSchemaCustomization = schemaCustomization
+  await rebuild({ parentSpan: activity })
+  await updateStateAndRunQueries(false, { parentSpan: activity })
+  activity.end()
+}, 1000)
+
+module.exports = () => {
+  const { inferenceMetadata, schemaCustomization } = store.getState()
+  lastMetadata = cloneDeep(inferenceMetadata)
+  lastSchemaCustomization = schemaCustomization
+  emitter.on(`API_RUNNING_QUEUE_EMPTY`, maybeRebuildSchema)
+}

packages/gatsby/src/commands/develop.js

@@ -30,6 +30,7 @@ const WorkerPool = require(`../utils/worker/pool`)
 
 const withResolverContext = require(`../schema/context`)
 const sourceNodes = require(`../utils/source-nodes`)
+const createSchemaCustomization = require(`../utils/create-schema-customization`)
 const websocketManager = require(`../utils/websocket-manager`)
 const getSslCert = require(`../utils/get-ssl-cert`)
 const { slash } = require(`gatsby-core-utils`)
@@ -187,6 +188,20 @@ async function startServer(program) {
    * If no GATSBY_REFRESH_TOKEN env var is available, then no Authorization header is required
    **/
   const REFRESH_ENDPOINT = `/__refresh`
+  const refresh = async req => {
+    let activity = report.activityTimer(`createSchemaCustomization`, {})
+    activity.start()
+    await createSchemaCustomization({
+      refresh: true,
+    })
+    activity.end()
+    activity = report.activityTimer(`Refreshing source data`, {})
+    activity.start()
+    await sourceNodes({
+      webhookBody: req.body,
+    })
+    activity.end()
+  }
   app.use(REFRESH_ENDPOINT, express.json())
   app.post(REFRESH_ENDPOINT, (req, res) => {
     const enableRefresh = process.env.ENABLE_GATSBY_REFRESH_ENDPOINT
@@ -195,13 +210,7 @@ async function startServer(program) {
       !refreshToken || req.headers.authorization === refreshToken
 
     if (enableRefresh && authorizedRefresh) {
-      const activity = report.activityTimer(`Refreshing source data`, {})
-      activity.start()
-      sourceNodes({
-        webhookBody: req.body,
-      }).then(() => {
-        activity.end()
-      })
+      refresh(req)
     }
     res.end()
   })
@@ -374,6 +383,9 @@ module.exports = async (program: any) => {
   // Start the createPages hot reloader.
   require(`../bootstrap/page-hot-reloader`)(graphqlRunner)
 
+  // Start the schema hot reloader.
+  require(`../bootstrap/schema-hot-reloader`)()
+
   await queryUtil.initialProcessQueries()
 
   require(`../redux/actions`).boundActionCreators.setProgramStatus(

packages/gatsby/src/db/loki/nodes.js

@@ -2,9 +2,9 @@ const _ = require(`lodash`)
 const invariant = require(`invariant`)
 const { getDb, colls } = require(`./index`)
 
-/////////////////////////////////////////////////////////////////////
+// ///////////////////////////////////////////////////////////////////
 // Node collection metadata
-/////////////////////////////////////////////////////////////////////
+// ///////////////////////////////////////////////////////////////////
 
 function makeTypeCollName(type) {
   return `gatsby:nodeType:${type}`
@@ -39,7 +39,7 @@ function createNodeTypeCollection(type) {
 function getTypeCollName(type) {
   const nodeTypesColl = getDb().getCollection(colls.nodeTypes.name)
   invariant(nodeTypesColl, `Collection ${colls.nodeTypes.name} should exist`)
-  let nodeTypeInfo = nodeTypesColl.by(`type`, type)
+  const nodeTypeInfo = nodeTypesColl.by(`type`, type)
   return nodeTypeInfo ? nodeTypeInfo.collName : undefined
 }
 
@@ -72,7 +72,7 @@ function deleteNodeTypeCollections(force = false) {
   // find() returns all objects in collection
   const nodeTypes = nodeTypesColl.find()
   for (const nodeType of nodeTypes) {
-    let coll = getDb().getCollection(nodeType.collName)
+    const coll = getDb().getCollection(nodeType.collName)
     if (coll.count() === 0 || force) {
       getDb().removeCollection(coll.name)
       nodeTypesColl.remove(nodeType)
@@ -93,9 +93,9 @@ function deleteAll() {
   }
 }
 
-/////////////////////////////////////////////////////////////////////
+// ///////////////////////////////////////////////////////////////////
 // Queries
-/////////////////////////////////////////////////////////////////////
+// ///////////////////////////////////////////////////////////////////
 
 /**
  * Returns the node with `id` == id, or null if not found
@@ -191,9 +191,9 @@ function hasNodeChanged(id, digest) {
   }
 }
 
-/////////////////////////////////////////////////////////////////////
+// /////////////////////////////////////////////////////////////////
 // Create/Update/Delete
-/////////////////////////////////////////////////////////////////////
+// ///////////////////////////////////////////////////////////////////
 
 /**
  * Creates a node in the DB. Will create a collection for the node
@@ -242,11 +242,8 @@ function updateNode(node) {
   invariant(node.internal.type, `node has no "internal.type" field`)
   invariant(node.id, `node has no "id" field`)
 
-  const type = node.internal.type
-
-  let coll = getNodeTypeCollection(type)
-  invariant(coll, `${type} collection doesn't exist. When trying to update`)
-  coll.update(node)
+  const oldNode = getNode(node.id)
+  return createNode(node, oldNode)
 }
 
 /**
@@ -264,22 +261,23 @@ function deleteNode(node) {
 
   const type = node.internal.type
 
-  let nodeTypeColl = getNodeTypeCollection(type)
+  const nodeTypeColl = getNodeTypeCollection(type)
   if (!nodeTypeColl) {
     invariant(
       nodeTypeColl,
       `${type} collection doesn't exist. When trying to delete`
     )
   }
 
-  if (nodeTypeColl.by(`id`, node.id)) {
+  const obj = nodeTypeColl.by(`id`, node.id)
+  if (obj) {
     const nodeMetaColl = getDb().getCollection(colls.nodeMeta.name)
     invariant(nodeMetaColl, `Collection ${colls.nodeMeta.name} should exist`)
     nodeMetaColl.findAndRemove({ id: node.id })
     // TODO What if this `remove()` fails? We will have removed the id
     // -> collName mapping, but not the actual node in the
     // collection. Need to make this into a transaction
-    nodeTypeColl.remove(node)
+    nodeTypeColl.remove(obj)
   }
   // idempotent. Do nothing if node wasn't already in DB
 }
@@ -326,7 +324,7 @@ function ensureFieldIndexes(typeName, lokiArgs, sortArgs) {
     const { emitter } = require(`../../redux`)
 
     emitter.on(`DELETE_CACHE`, () => {
-      for (var field in fieldUsages) {
+      for (const field in fieldUsages) {
         delete fieldUsages[field]
       }
     })
@@ -350,9 +348,9 @@ function ensureFieldIndexes(typeName, lokiArgs, sortArgs) {
   })
 }
 
-/////////////////////////////////////////////////////////////////////
+// ///////////////////////////////////////////////////////////////////
 // Reducer
-/////////////////////////////////////////////////////////////////////
+// ///////////////////////////////////////////////////////////////////
 
 function reducer(state = new Map(), action) {
   switch (action.type) {
@@ -378,7 +376,7 @@ function reducer(state = new Map(), action) {
     }
 
     case `DELETE_NODES`: {
-      deleteNodes(action.payload)
+      deleteNodes(action.fullNodes)
       return null
     }
 
@@ -387,9 +385,9 @@ function reducer(state = new Map(), action) {
   }
 }
 
-/////////////////////////////////////////////////////////////////////
+// ///////////////////////////////////////////////////////////////////
 // Exports
-/////////////////////////////////////////////////////////////////////
+// ///////////////////////////////////////////////////////////////////
 
 module.exports = {
   getNodeTypeCollection,

packages/gatsby/src/query/query-watcher.js

@@ -265,3 +265,5 @@ exports.startWatchDeletePage = () => {
     }
   })
 }
+
+exports.updateStateAndRunQueries = updateStateAndRunQueries

packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap

@@ -16,6 +16,7 @@ Object {
       "query": "",
     },
   },
+  "inferenceMetadata": Object {},
   "staticQueryComponents": Map {},
   "status": Object {
     "plugins": Object {},

packages/gatsby/src/redux/actions/public.js

@@ -531,10 +531,15 @@ actions.deleteNodes = (nodes: any[], plugin: Plugin) => {
     nodes.map(n => findChildren(getNode(n).children))
   )
 
+  const nodeIds = [...nodes, ...descendantNodes]
+
   const deleteNodesAction = {
     type: `DELETE_NODES`,
     plugin,
-    payload: [...nodes, ...descendantNodes],
+    // Payload contains node IDs but inference-metadata and loki reducers require
+    // full node instances
+    payload: nodeIds,
+    fullNodes: nodeIds.map(getNode),
   }
   return deleteNodesAction
 }
@@ -961,6 +966,7 @@ actions.createNodeField = (
     type: `ADD_FIELD_TO_NODE`,
     plugin,
     payload: node,
+    addedField: name,
   }
 }
 

packages/gatsby/src/redux/actions/restricted.js

@@ -1,6 +1,7 @@
 // @flow
 const { camelCase } = require(`lodash`)
 const report = require(`gatsby-cli/lib/reporter`)
+const { parseTypeDef } = require(`../../schema/types/type-defs`)
 
 import type { Plugin } from "./types"
 
@@ -187,7 +188,9 @@ actions.createTypes = (
     type: `CREATE_TYPES`,
     plugin,
     traceId,
-    payload: types,
+    payload: Array.isArray(types)
+      ? types.map(parseTypeDef)
+      : parseTypeDef(types),
   }
 }
 

packages/gatsby/src/redux/index.js

@@ -60,6 +60,7 @@ const saveState = () => {
     `components`,
     `staticQueryComponents`,
     `webpackCompilationHash`,
+    `inferenceMetadata`,
   ])
 
   return writeToCache(pickedState)

packages/gatsby/src/redux/reducers/index.js

@@ -63,4 +63,5 @@ module.exports = {
   schemaCustomization: require(`./schema-customization`),
   themes: require(`./themes`),
   logs: require(`gatsby-cli/lib/reporter/redux/reducer`),
+  inferenceMetadata: require(`./inference-metadata`),
 }