From 2ed8c692b5311d7f49aa13149c8ee03cfea11779 Mon Sep 17 00:00:00 2001 From: Craigory Coppola Date: Mon, 2 Dec 2024 16:42:43 -0500 Subject: [PATCH 1/4] fix(core): provide a way to reuse cached graph in CI --- packages/nx/src/project-graph/project-graph.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/nx/src/project-graph/project-graph.ts b/packages/nx/src/project-graph/project-graph.ts index 30f2a1912db0c..2c91541a997d4 100644 --- a/packages/nx/src/project-graph/project-graph.ts +++ b/packages/nx/src/project-graph/project-graph.ts @@ -229,6 +229,13 @@ export async function createProjectGraphAsync( resetDaemonClient: false, } ): Promise { + if (process.env.NX_FORCE_REUSE_CACHED_GRAPH === 'true') { + try { + return readCachedProjectGraph(); + // If no cached graph is found, we will fall through to the normal flow + } catch {} + } + const projectGraphAndSourceMaps = await createProjectGraphAndSourceMapsAsync( opts ); From 4058191c70d92b0fbc161a1e0f63e9bac40c6e5a Mon Sep 17 00:00:00 2001 From: Craigory Coppola Date: Tue, 3 Dec 2024 10:12:45 -0500 Subject: [PATCH 2/4] fix(core): hydrate root map when using cached graph --- .../nx/src/project-graph/build-project-graph.ts | 10 ++++++++++ packages/nx/src/project-graph/project-graph.ts | 16 ++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/packages/nx/src/project-graph/build-project-graph.ts b/packages/nx/src/project-graph/build-project-graph.ts index 76b64136772d1..be61b9e227c78 100644 --- a/packages/nx/src/project-graph/build-project-graph.ts +++ b/packages/nx/src/project-graph/build-project-graph.ts @@ -72,6 +72,16 @@ export function getFileMap(): { } } +export function hydrateFileMap( + fileMap: FileMap, + allWorkspaceFiles: FileData[], + rustReferences: NxWorkspaceFilesExternals +) { + storedFileMap = fileMap; + storedAllWorkspaceFiles = allWorkspaceFiles; + storedRustReferences = rustReferences; +} + export async function buildProjectGraphUsingProjectFileMap( projectRootMap: Record, externalNodes: Record, diff --git a/packages/nx/src/project-graph/project-graph.ts b/packages/nx/src/project-graph/project-graph.ts index 2c91541a997d4..c7f9d2026e71c 100644 --- a/packages/nx/src/project-graph/project-graph.ts +++ b/packages/nx/src/project-graph/project-graph.ts @@ -12,7 +12,10 @@ import { fileExists } from '../utils/fileutils'; import { output } from '../utils/output'; import { stripIndents } from '../utils/strip-indents'; import { workspaceRoot } from '../utils/workspace-root'; -import { buildProjectGraphUsingProjectFileMap } from './build-project-graph'; +import { + buildProjectGraphUsingProjectFileMap, + hydrateFileMap, +} from './build-project-graph'; import { AggregateProjectGraphError, isAggregateProjectGraphError, @@ -231,7 +234,16 @@ export async function createProjectGraphAsync( ): Promise { if (process.env.NX_FORCE_REUSE_CACHED_GRAPH === 'true') { try { - return readCachedProjectGraph(); + const graph = readCachedProjectGraph(); + const projectRootMap = Object.fromEntries( + Object.entries(graph.nodes).map(([project, { data }]) => [ + data.root, + project, + ]) + ); + const { allWorkspaceFiles, fileMap, rustReferences } = + await retrieveWorkspaceFiles(workspaceRoot, projectRootMap); + hydrateFileMap(fileMap, allWorkspaceFiles, rustReferences); // If no cached graph is found, we will fall through to the normal flow } catch {} } From 0eeb22e4a432d86f2bf9e27f1fb3142c3b823791 Mon Sep 17 00:00:00 2001 From: Craigory Coppola Date: Tue, 3 Dec 2024 10:46:47 -0500 Subject: [PATCH 3/4] fix(core): actually return graph --- packages/nx/src/project-graph/project-graph.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/nx/src/project-graph/project-graph.ts b/packages/nx/src/project-graph/project-graph.ts index c7f9d2026e71c..2188c65a773c5 100644 --- a/packages/nx/src/project-graph/project-graph.ts +++ b/packages/nx/src/project-graph/project-graph.ts @@ -33,6 +33,7 @@ import { retrieveWorkspaceFiles, } from './utils/retrieve-workspace-files'; import { getPlugins } from './plugins/get-plugins'; +import { logger } from '../utils/logger'; /** * Synchronously reads the latest cached copy of the workspace's ProjectGraph. @@ -244,8 +245,11 @@ export async function createProjectGraphAsync( const { allWorkspaceFiles, fileMap, rustReferences } = await retrieveWorkspaceFiles(workspaceRoot, projectRootMap); hydrateFileMap(fileMap, allWorkspaceFiles, rustReferences); + return graph; // If no cached graph is found, we will fall through to the normal flow - } catch {} + } catch (e) { + logger.verbose('Unable to use cached project graph', e); + } } const projectGraphAndSourceMaps = await createProjectGraphAndSourceMapsAsync( From c171bae3f4387a99982b8ce1a18ffcf19c20b0c0 Mon Sep 17 00:00:00 2001 From: Craigory Coppola Date: Tue, 3 Dec 2024 11:22:19 -0500 Subject: [PATCH 4/4] chore(core): avoid loading plugins during affected calculation when reusing graph --- .../affected/locators/project-glob-changes.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts b/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts index aa1ab25d98a87..26293b8bc2855 100644 --- a/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts +++ b/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts @@ -9,8 +9,20 @@ import { getPlugins } from '../../plugins/get-plugins'; export const getTouchedProjectsFromProjectGlobChanges: TouchedProjectLocator = async (touchedFiles, projectGraphNodes): Promise => { - const plugins = await getPlugins(); - const globPattern = combineGlobPatterns(configurationGlobs(plugins)); + const globPattern = await (async () => { + // TODO: We need a quicker way to get patterns that should not + // require starting up plugin workers + if (process.env.NX_FORCE_REUSE_CACHED_GRAPH === 'true') { + return combineGlobPatterns([ + '**/package.json', + '**/project.json', + 'project.json', + 'package.json', + ]); + } + const plugins = await getPlugins(); + return combineGlobPatterns(configurationGlobs(plugins)); + })(); const touchedProjects = new Set(); for (const touchedFile of touchedFiles) {