Perf tests: Store raw test run results as artifact (#45747)

In this commit we're storing the raw results of all the performance test
runs as an artifact in GitHub Actions so that we can perform more extensive
analysis outside of workflow runs.

For every _round_ of testing we end up with a copy of the results in a JSON
file with the branch ref, the test suite name, and the test run index in its
name. This makes it possible to analyze each run separately without making
it hard to analyze everything together.

.github/workflows/performance.yml

@@ -44,6 +44,13 @@ jobs:
               if: github.event_name == 'pull_request'
               run: ./bin/plugin/cli.js perf $GITHUB_SHA trunk --tests-branch $GITHUB_SHA
 
+            - name: Store performance measurements
+              if: github.event_name == 'pull_request'
+              uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1
+              with:
+                  name: perf-test-results
+                  path: ./__test-results/*.json
+
             - name: Compare performance with current WordPress Core and previous Gutenberg versions
               if: github.event_name == 'release'
               env:

bin/plugin/commands/performance.js

@@ -186,14 +186,21 @@ function curateResults( testSuite, results ) {
  *
  * @param {string} testSuite                Name of the tests set.
  * @param {string} performanceTestDirectory Path to the performance tests' clone.
+ * @param {string} runKey                   Unique identifier for the test run, e.g. `branch-name_post-editor_run-3`.
  *
  * @return {Promise<WPPerformanceResults>} Performance results for the branch.
  */
-async function runTestSuite( testSuite, performanceTestDirectory ) {
+async function runTestSuite( testSuite, performanceTestDirectory, runKey ) {
 	await runShellScript(
 		`npm run test:performance -- packages/e2e-tests/specs/performance/${ testSuite }.test.js`,
 		performanceTestDirectory
 	);
+	const resultsFile = path.join(
+		performanceTestDirectory,
+		`packages/e2e-tests/specs/performance/${ testSuite }.test.results.json`
+	);
+	fs.mkdirSync( './__test-results', { recursive: true } );
+	fs.copyFileSync( resultsFile, `./__test-results/${ runKey }.results.json` );
 	const rawResults = await readJSONFile(
 		path.join(
 			performanceTestDirectory,
@@ -388,6 +395,7 @@ async function runPerformanceTests( branches, options ) {
 		for ( let i = 0; i < TEST_ROUNDS; i++ ) {
 			rawResults[ i ] = {};
 			for ( const branch of branches ) {
+				const runKey = `${ branch }_${ testSuite }_run-${ i }`;
 				// @ts-ignore
 				const environmentDirectory = branchDirectories[ branch ];
 				log( `    >> Branch: ${ branch }, Suite: ${ testSuite }` );
@@ -399,7 +407,8 @@ async function runPerformanceTests( branches, options ) {
 				log( '        >> Running the test.' );
 				rawResults[ i ][ branch ] = await runTestSuite(
 					testSuite,
-					performanceTestDirectory
+					performanceTestDirectory,
+					runKey
 				);
 				log( '        >> Stopping the environment' );
 				await runShellScript(