[buildkite] Improve failed test experience (#113483) (#113582)

Co-authored-by: Brian Seeders <brian.seeders@elastic.co>

.buildkite/pipelines/es_snapshots/verify.yml

@@ -91,13 +91,5 @@ steps:
   - wait: ~
     continue_on_failure: true
 
-  - plugins:
-      - junit-annotate#v1.9.0:
-          artifacts: target/junit/**/*.xml
-          job-uuid-file-pattern: '-bk__(.*).xml'
-
-  - wait: ~
-    continue_on_failure: true
-
   - command: .buildkite/scripts/lifecycle/post_build.sh
     label: Post-Build

.buildkite/pipelines/hourly.yml

@@ -159,13 +159,5 @@ steps:
   - wait: ~
     continue_on_failure: true
 
-  - plugins:
-      - junit-annotate#v1.9.0:
-          artifacts: target/junit/**/*.xml
-          job-uuid-file-pattern: '-bk__(.*).xml'
-
-  - wait: ~
-    continue_on_failure: true
-
   - command: .buildkite/scripts/lifecycle/post_build.sh
     label: Post-Build

.buildkite/scripts/lifecycle/annotate_test_failures.js

@@ -0,0 +1,14 @@
+const { TestFailures } = require('kibana-buildkite-library');
+
+(async () => {
+  try {
+    await TestFailures.annotateTestFailures();
+  } catch (ex) {
+    console.error('Annotate test failures error', ex.message);
+    if (ex.response) {
+      console.error('HTTP Error Response Status', ex.response.status);
+      console.error('HTTP Error Response Body', ex.response.data);
+    }
+    process.exit(1);
+  }
+})();

.buildkite/scripts/lifecycle/post_command.sh

@@ -23,4 +23,9 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload '.es/**/*.hprof'
 
   node scripts/report_failed_tests --build-url="${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" 'target/junit/**/*.xml'
+
+  if [[ -d 'target/test_failures' ]]; then
+    buildkite-agent artifact upload 'target/test_failures/**/*'
+    node .buildkite/scripts/lifecycle/annotate_test_failures.js
+  fi
 fi

packages/kbn-test/BUILD.bazel

@@ -50,6 +50,7 @@ RUNTIME_DEPS = [
     "@npm//exit-hook",
     "@npm//form-data",
     "@npm//globby",
+    "@npm//he",
     "@npm//history",
     "@npm//jest",
     "@npm//jest-cli",
@@ -86,6 +87,7 @@ TYPES_DEPS = [
   "@npm//xmlbuilder",
   "@npm//@types/chance",
   "@npm//@types/enzyme",
+  "@npm//@types/he",
   "@npm//@types/history",
   "@npm//@types/jest",
   "@npm//@types/joi",

packages/kbn-test/src/failed_tests_reporter/report_failures_to_file.ts

@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { createHash } from 'crypto';
+import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'fs';
+import { join, basename, resolve } from 'path';
+
+import { ToolingLog } from '@kbn/dev-utils';
+import { REPO_ROOT } from '@kbn/utils';
+import { escape } from 'he';
+
+import { TestFailure } from './get_failures';
+
+const findScreenshots = (dirPath: string, allScreenshots: string[] = []) => {
+  const files = readdirSync(dirPath);
+
+  for (const file of files) {
+    if (statSync(join(dirPath, file)).isDirectory()) {
+      if (file.match(/node_modules/)) {
+        continue;
+      }
+
+      allScreenshots = findScreenshots(join(dirPath, file), allScreenshots);
+    } else {
+      const fullPath = join(dirPath, file);
+      if (fullPath.match(/screenshots\/failure\/.+\.png$/)) {
+        allScreenshots.push(fullPath);
+      }
+    }
+  }
+
+  return allScreenshots;
+};
+
+export function reportFailuresToFile(log: ToolingLog, failures: TestFailure[]) {
+  if (!failures?.length) {
+    return;
+  }
+
+  let screenshots: string[];
+  try {
+    screenshots = [
+      ...findScreenshots(join(REPO_ROOT, 'test', 'functional')),
+      ...findScreenshots(join(REPO_ROOT, 'x-pack', 'test', 'functional')),
+    ];
+  } catch (e) {
+    log.error(e as Error);
+    screenshots = [];
+  }
+
+  const screenshotsByName: Record<string, string> = {};
+  for (const screenshot of screenshots) {
+    const [name] = basename(screenshot).split('.');
+    screenshotsByName[name] = screenshot;
+  }
+
+  // Jest could, in theory, fail 1000s of tests and write 1000s of failures
+  // So let's just write files for the first 20
+  for (const failure of failures.slice(0, 20)) {
+    const hash = createHash('md5').update(failure.name).digest('hex');
+    const filenameBase = `${
+      process.env.BUILDKITE_JOB_ID ? process.env.BUILDKITE_JOB_ID + '_' : ''
+    }${hash}`;
+    const dir = join('target', 'test_failures');
+
+    const failureLog = [
+      ['Test:', '-----', failure.classname, failure.name, ''],
+      ['Failure:', '--------', failure.failure],
+      failure['system-out'] ? ['', 'Standard Out:', '-------------', failure['system-out']] : [],
+    ]
+      .flat()
+      .join('\n');
+
+    const failureJSON = JSON.stringify(
+      {
+        ...failure,
+        hash,
+        buildId: process.env.BUJILDKITE_BUILD_ID || '',
+        jobId: process.env.BUILDKITE_JOB_ID || '',
+        url: process.env.BUILDKITE_BUILD_URL || '',
+        jobName: process.env.BUILDKITE_LABEL
+          ? `${process.env.BUILDKITE_LABEL}${
+              process.env.BUILDKITE_PARALLEL_JOB ? ` #${process.env.BUILDKITE_PARALLEL_JOB}` : ''
+            }`
+          : '',
+      },
+      null,
+      2
+    );
+
+    let screenshot = '';
+    const screenshotName = `${failure.name.replace(/([^ a-zA-Z0-9-]+)/g, '_')}`;
+    if (screenshotsByName[screenshotName]) {
+      try {
+        screenshot = readFileSync(screenshotsByName[screenshotName]).toString('base64');
+      } catch (e) {
+        log.error(e as Error);
+      }
+    }
+
+    const screenshotHtml = screenshot
+      ? `<img class="screenshot img-fluid img-thumbnail" src="data:image/png;base64,${screenshot}" />`
+      : '';
+
+    const failureHTML = readFileSync(
+      resolve(
+        REPO_ROOT,
+        'packages/kbn-test/src/failed_tests_reporter/report_failures_to_file_html_template.html'
+      )
+    )
+      .toString()
+      .replace('$TITLE', escape(failure.name))
+      .replace(
+        '$MAIN',
+        `
+        ${failure.classname
+          .split('.')
+          .map((part) => `<h5>${escape(part.replace('·', '.'))}</h5>`)
+          .join('')}
+        <hr />
+        <p><strong>${escape(failure.name)}</strong></p>
+        <pre>${escape(failure.failure)}</pre>
+        ${screenshotHtml}
+        <pre>${escape(failure['system-out'] || '')}</pre>
+      `
+      );
+
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(join(dir, `${filenameBase}.log`), failureLog, 'utf8');
+    writeFileSync(join(dir, `${filenameBase}.html`), failureHTML, 'utf8');
+    writeFileSync(join(dir, `${filenameBase}.json`), failureJSON, 'utf8');
+  }
+}

packages/kbn-test/src/failed_tests_reporter/report_failures_to_file_html_template.html

@@ -0,0 +1,41 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <link
+      href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.1/dist/css/bootstrap.min.css"
+      rel="stylesheet"
+      integrity="sha384-F3w7mX95PdgyTmZZMECAngseQB83DfGTowi0iMjiWaeVhAn4FJkqJByhZMI3AhiU"
+      crossorigin="anonymous"
+    />
+    <style type="text/css">
+      pre {
+        font-size: 0.75em !important;
+      }
+
+      img.screenshot {
+        cursor: pointer;
+        height: 200px;
+        margin: 5px 0;
+      }
+
+      img.screenshot.expanded {
+        height: auto;
+      }
+    </style>
+    <title>$TITLE</title>
+  </head>
+  <body>
+    <div class="col-lg-10 mx-auto p-3 py-md-5">
+      <main>$MAIN</main>
+    </div>
+    <script type="text/javascript">
+      for (const img of document.getElementsByTagName('img')) {
+        img.addEventListener('click', () => {
+          img.classList.toggle('expanded');
+        });
+      }
+    </script>
+  </body>
+</html>

packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts

@@ -21,6 +21,7 @@ import { readTestReport } from './test_report';
 import { addMessagesToReport } from './add_messages_to_report';
 import { getReportMessageIter } from './report_metadata';
 import { reportFailuresToEs } from './report_failures_to_es';
+import { reportFailuresToFile } from './report_failures_to_file';
 
 const DEFAULT_PATTERNS = [Path.resolve(REPO_ROOT, 'target/junit/**/*.xml')];
 
@@ -98,6 +99,8 @@ export function runFailedTestsReporterCli() {
         const messages = Array.from(getReportMessageIter(report));
         const failures = await getFailures(report);
 
+        reportFailuresToFile(log, failures);
+
         if (indexInEs) {
           await reportFailuresToEs(log, failures);
         }