feat(gatsby): selective cache invalidation

examples/using-gatsby-image/plugins/gatsby-source-remote-images/gatsby-node.js

@@ -14,6 +14,7 @@ exports.onCreateNode = async (
   if (filter(node)) {
     const fileNode = await createRemoteFileNode({
       url: node.url,
+      parentNodeId: node.id,
       store,
       cache,
       createNode,

packages/gatsby/src/bootstrap/cache/__tests__/plugin-hash.js

@@ -0,0 +1,143 @@
+jest.mock(`md5-file/promise`, () => {
+  const path = jest.requireActual(`path`)
+  return jest.fn(file => Promise.resolve(path.basename(file)))
+})
+const md5File = require(`md5-file/promise`)
+const getPluginHash = require(`../plugin-hash`)
+
+const getHash = ({
+  directory = __dirname,
+  plugins = [],
+  existing = {},
+  ...rest
+} = {}) => getPluginHash({ directory, plugins, existing, ...rest })
+
+beforeEach(() => {
+  md5File.mockClear()
+})
+
+describe(`basic functionality`, () => {
+  it(`invalidates gatsby-node.js if it has changed`, () => {
+    const additional = [`gatsby-node.js`]
+    const existing = {
+      "gatsby-node.js": ``,
+    }
+
+    expect(getHash({ additional, existing })).resolves.toEqual(
+      expect.objectContaining({
+        changes: [`gatsby-node.js`],
+      })
+    )
+  })
+})
+
+describe(`getting changed plugins`, () => {
+  it(`returns single plugin, if single plugin changed`, () => {
+    const plugins = [{ name: `gatsby-source-filesystem`, version: `2.0.0` }]
+    const existing = { "gatsby-source-filesystem": `1.0.0` }
+
+    expect(getHash({ plugins, existing })).resolves.toEqual(
+      expect.objectContaining({
+        changes: [`gatsby-source-filesystem`],
+      })
+    )
+  })
+
+  it(`returns multiple plugins, if multiple plugins changed`, () => {
+    const plugins = [
+      { name: `gatsby-source-filesystem`, version: `2.0.0` },
+      { name: `gatsby-remark-prismjs`, version: `1.0.0` },
+    ]
+    const existing = {
+      "gatsby-source-filesystem": `1.0.0`,
+      "gatsby-remark-prismjs": `0.9.9`,
+    }
+
+    expect(getHash({ plugins, existing })).resolves.toEqual(
+      expect.objectContaining({
+        changes: [`gatsby-source-filesystem`, `gatsby-remark-prismjs`],
+      })
+    )
+  })
+
+  it(`does not include plugin as changed if fresh install of plugin`, () => {
+    const plugins = [{ name: `gatsby-source-filesystem`, version: `1.0.0` }]
+    const existing = {}
+
+    expect(getHash({ plugins, existing })).resolves.toEqual(
+      expect.objectContaining({
+        changes: [],
+      })
+    )
+  })
+
+  it(`does not include plugin as changed if no changes in version`, () => {
+    const plugins = [{ name: `gatsby-source-filesystem`, version: `1.0.0` }]
+    const existing = { "gatsby-source-filesystem": plugins[0].version }
+
+    expect(getHash({ plugins, existing })).resolves.toEqual(
+      expect.objectContaining({
+        changes: [],
+      })
+    )
+  })
+
+  it(`does not include changed plugins if fresh cache`, () => {
+    const plugins = [
+      { name: `gatsby-source-filesystem`, version: `1.0.0` },
+      { name: `gatsby-remark-prismjs`, version: `1.0.0` },
+    ]
+    const existing = {}
+
+    expect(getHash({ plugins, existing })).resolves.toEqual(
+      expect.objectContaining({
+        changes: [],
+      })
+    )
+  })
+})
+
+describe(`getting hash`, () => {
+  it(`creates hash based on plugin versions`, () => {
+    const plugins = [
+      { name: `gatsby-source-filesystem`, version: `1.0.0` },
+      { name: `gatsby-remark-prismjs`, version: `1.0.0` },
+    ]
+
+    expect(getHash({ plugins })).resolves.toEqual(
+      expect.objectContaining({
+        hash: {
+          "gatsby-source-filesystem": `1.0.0`,
+          "gatsby-remark-prismjs": `1.0.0`,
+        },
+      })
+    )
+  })
+
+  it(`includes hash of additional file(s)`, () => {
+    const additional = [`gatsby-config.js`, `gatsby-node.js`, `package.json`]
+
+    expect(getHash({ additional })).resolves.toEqual(
+      expect.objectContaining({
+        hash: {
+          "gatsby-config.js": `gatsby-config.js`,
+          "gatsby-node.js": `gatsby-node.js`,
+          "package.json": `package.json`,
+        },
+      })
+    )
+  })
+
+  it(`uses empty string as key if file does not exist`, () => {
+    md5File.mockRejectedValueOnce(() => new Error(`File does not exist`))
+    const additional = [`gatsby-config.js`]
+
+    expect(getHash({ additional })).resolves.toEqual(
+      expect.objectContaining({
+        hash: {
+          "gatsby-config.js": ``,
+        },
+      })
+    )
+  })
+})

packages/gatsby/src/bootstrap/cache/plugin-hash.js

@@ -0,0 +1,18 @@
+const _ = require(`lodash`)
+
+module.exports = async function getPluginHash({ plugins, existing }) {
+  const pluginsHash = plugins.reduce((merged, plugin) => {
+    merged[plugin.name] = plugin.version
+    return merged
+  }, {})
+
+  return {
+    changes: _.uniq(
+      Object.keys({ ...pluginsHash, ...existing }).filter(name => {
+        const hash = pluginsHash[name]
+        return hash !== existing[name]
+      })
+    ),
+    hash: pluginsHash,
+  }
+}

packages/gatsby/src/bootstrap/cache/remove.js

@@ -0,0 +1,66 @@
+const path = require(`path`)
+const fs = require(`fs-extra`)
+const _ = require(`lodash`)
+
+const getPluginHash = require(`./plugin-hash`)
+
+const CACHE_FOLDER = `caches`
+const DONT_DELETE = [`redux.state`, `loki`]
+/*
+ * This function will first calculate a diff of plugin/file changes (e.g. if gatsby-node.js changes)
+ * From this, it will then selectively invalidate files in .cache/caches
+ * the location where Gatsby stores long-lived cache files that can be safely kept around
+ */
+module.exports = async function safeRemoveCache({
+  directory,
+  plugins,
+  existing,
+  report,
+}) {
+  const cacheDirectory = path.join(directory, `.cache`)
+
+  const { changes, hash } = await getPluginHash({
+    plugins,
+    existing,
+  })
+
+  const equalHashes = _.isEqual(existing, hash)
+  const hasExistingCache = Object.keys(existing).length > 0
+
+  await fs.ensureDir(cacheDirectory)
+
+  if (hasExistingCache && !equalHashes) {
+    report.info(report.stripIndent`
+      The following plugins were changed, added or removed since the last time you ran Gatsby:
+${changes.map(change => `        - ${change}`).join(`\n`)}
+      As a precaution, we're invalidating parts of your cache to ensure your data is sparkly fresh.
+    `)
+  }
+
+  if (!equalHashes) {
+    try {
+      const filesToRemove = await fs.readdir(cacheDirectory).then(allFiles =>
+        allFiles.reduce((merged, file) => {
+          if (file === CACHE_FOLDER) {
+            return merged.concat(
+              changes
+                .map(pluginName =>
+                  path.join(cacheDirectory, CACHE_FOLDER, pluginName)
+                )
+                .filter(absolutePath => fs.existsSync(absolutePath))
+            )
+          } else if (DONT_DELETE.includes(file)) {
+            return merged
+          }
+          return merged.concat(path.join(cacheDirectory, file))
+        }, [])
+      )
+
+      await Promise.all(filesToRemove.map(file => fs.remove(file)))
+    } catch (e) {
+      report.error(`Failed to remove .cache files.`, e)
+    }
+  }
+
+  return { cacheDirectory, changes, hash }
+}