chore: expose expect error details on TestError

docs/src/test-reporter-api/class-testerror.md

@@ -4,18 +4,54 @@
 
 Information about an error thrown during test execution.
 
+## property: TestError.expected
+* since: v1.49
+- type: ?<[string]>
+
+Expected value formatted as a human-readable string.
+
+## property: TestError.locator
+* since: v1.49
+- type: ?<[string]>
+
+Receiver's locator.
+
+## property: TestError.log
+* since: v1.49
+- type: ?<[Array]<[string]>>
+
+Call log.
+
+## property: TestError.matcherName
+* since: v1.49
+- type: ?<[string]>
+
+Expect matcher name.
+
 ## property: TestError.message
 * since: v1.10
 - type: ?<[string]>
 
 Error message. Set when [Error] (or its subclass) has been thrown.
 
+## property: TestError.received
+* since: v1.49
+- type: ?<[string]>
+
+Received value formatted as a human-readable string.
+
 ## property: TestError.stack
 * since: v1.10
 - type: ?<[string]>
 
 Error stack. Set when [Error] (or its subclass) has been thrown.
 
+## property: TestError.timeout
+* since: v1.49
+- type: ?<[int]>
+
+Timeout in milliseconds, if the error was caused by a timeout.
+
 ## property: TestError.value
 * since: v1.10
 - type: ?<[string]>

packages/playwright/src/matchers/matcherHint.ts

@@ -39,6 +39,10 @@ export type MatcherResult<E, A> = {
   actual?: A;
   log?: string[];
   timeout?: number;
+  locator?: string;
+  printedReceived?: string;
+  printedExpected?: string;
+  printedDiff?: string;
 };
 
 export class ExpectError extends Error {

packages/playwright/src/matchers/toBeTruthy.ts

@@ -39,22 +39,41 @@ export async function toBeTruthy(
   };
 
   const timeout = options.timeout ?? this.timeout;
-  const { matches, log, timedOut, received } = await query(!!this.isNot, timeout);
+  const { matches: pass, log, timedOut, received } = await query(!!this.isNot, timeout);
+  if (pass === !this.isNot) {
+    return {
+      name: matcherName,
+      message: () => '',
+      pass,
+      expected
+    };
+  }
+
   const notFound = received === kNoElementsFoundError ? received : undefined;
-  const actual = matches ? expected : unexpected;
+  const actual = pass ? expected : unexpected;
+  let printedReceived: string | undefined;
+  let printedExpected: string | undefined;
+  if (pass) {
+    printedExpected = `Expected: not ${expected}`;
+    printedReceived = `Received: ${notFound ? kNoElementsFoundError : expected}`;
+  } else {
+    printedExpected = `Expected: ${expected}`;
+    printedReceived = `Received: ${notFound ? kNoElementsFoundError : unexpected}`;
+  }
   const message = () => {
     const header = matcherHint(this, receiver, matcherName, 'locator', arg, matcherOptions, timedOut ? timeout : undefined);
     const logText = callLogText(log);
-    return matches ? `${header}Expected: not ${expected}\nReceived: ${notFound ? kNoElementsFoundError : expected}${logText}` :
-      `${header}Expected: ${expected}\nReceived: ${notFound ? kNoElementsFoundError : unexpected}${logText}`;
+    return `${header}${printedExpected}\n${printedReceived}${logText}`;
   };
   return {
     message,
-    pass: matches,
+    pass,
     actual,
     name: matcherName,
     expected,
     log,
     timeout: timedOut ? timeout : undefined,
+    ...(printedReceived ? { printedReceived } : {}),
+    ...(printedExpected ? { printedExpected } : {}),
   };
 }

packages/playwright/src/matchers/toEqual.ts

@@ -44,22 +44,35 @@ export async function toEqual<T>(
   const timeout = options.timeout ?? this.timeout;
 
   const { matches: pass, received, log, timedOut } = await query(!!this.isNot, timeout);
+  if (pass === !this.isNot) {
+    return {
+      name: matcherName,
+      message: () => '',
+      pass,
+      expected
+    };
+  }
 
-  const message = pass
-    ? () =>
-      matcherHint(this, receiver, matcherName, 'locator', undefined, matcherOptions, timedOut ? timeout : undefined) +
-      `Expected: not ${this.utils.printExpected(expected)}\n` +
-      `Received: ${this.utils.printReceived(received)}` + callLogText(log)
-    : () =>
-      matcherHint(this, receiver, matcherName, 'locator', undefined, matcherOptions, timedOut ? timeout : undefined) +
-      this.utils.printDiffOrStringify(
-          expected,
-          received,
-          EXPECTED_LABEL,
-          RECEIVED_LABEL,
-          false,
-      ) + callLogText(log);
-
+  let printedReceived: string | undefined;
+  let printedExpected: string | undefined;
+  let printedDiff: string | undefined;
+  if (pass) {
+    printedExpected = `Expected: not ${this.utils.printExpected(expected)}`;
+    printedReceived = `Received: ${this.utils.printReceived(received)}`;
+  } else {
+    printedDiff = this.utils.printDiffOrStringify(
+        expected,
+        received,
+        EXPECTED_LABEL,
+        RECEIVED_LABEL,
+        false,
+    );
+  }
+  const message = () => {
+    const header = matcherHint(this, receiver, matcherName, 'locator', undefined, matcherOptions, timedOut ? timeout : undefined);
+    const details = printedDiff || `${printedExpected}\n${printedReceived}`;
+    return `${header}${details}${callLogText(log)}`;
+  };
   // Passing the actual and expected objects so that a custom reporter
   // could access them, for example in order to display a custom visual diff,
   // or create a different error message
@@ -70,5 +83,8 @@ export async function toEqual<T>(
     pass,
     log,
     timeout: timedOut ? timeout : undefined,
+    ...(printedReceived ? { printedReceived } : {}),
+    ...(printedExpected ? { printedExpected } : {}),
+    ...(printedDiff ? { printedDiff } : {}),
   };
 }

packages/playwright/src/matchers/toMatchSnapshot.ts

@@ -194,6 +194,10 @@ class SnapshotHelper {
       pass,
       message: () => message,
       log,
+      // eslint-disable-next-line @typescript-eslint/no-base-to-string
+      ...(this.locator ? { locator: this.locator.toString() } : {}),
+      printedExpected: this.expectedPath,
+      printedReceived: this.actualPath,
     };
     return Object.fromEntries(Object.entries(unfiltered).filter(([_, v]) => v !== undefined)) as ImageMatcherResult;
   }

packages/playwright/src/matchers/toMatchText.ts

@@ -58,29 +58,56 @@ export async function toMatchText(
   const timeout = options.timeout ?? this.timeout;
 
   const { matches: pass, received, log, timedOut } = await query(!!this.isNot, timeout);
+  if (pass === !this.isNot) {
+    return {
+      name: matcherName,
+      message: () => '',
+      pass,
+      expected
+    };
+  }
+
   const stringSubstring = options.matchSubstring ? 'substring' : 'string';
   const receivedString = received || '';
   const messagePrefix = matcherHint(this, receiver, matcherName, 'locator', undefined, matcherOptions, timedOut ? timeout : undefined);
   const notFound = received === kNoElementsFoundError;
-  const message = () => {
-    if (pass) {
-      if (typeof expected === 'string') {
-        if (notFound)
-          return messagePrefix + `Expected ${stringSubstring}: not ${this.utils.printExpected(expected)}\nReceived: ${received}` + callLogText(log);
-        const printedReceived = printReceivedStringContainExpectedSubstring(receivedString, receivedString.indexOf(expected), expected.length);
-        return messagePrefix + `Expected ${stringSubstring}: not ${this.utils.printExpected(expected)}\nReceived string: ${printedReceived}` + callLogText(log);
+
+  let printedReceived: string | undefined;
+  let printedExpected: string | undefined;
+  let printedDiff: string | undefined;
+  if (pass) {
+    if (typeof expected === 'string') {
+      if (notFound) {
+        printedExpected = `Expected ${stringSubstring}: not ${this.utils.printExpected(expected)}`;
+        printedReceived = `Received: ${received}`;
       } else {
-        if (notFound)
-          return messagePrefix + `Expected pattern: not ${this.utils.printExpected(expected)}\nReceived: ${received}` + callLogText(log);
-        const printedReceived = printReceivedStringContainExpectedResult(receivedString, typeof expected.exec === 'function' ? expected.exec(receivedString) : null);
-        return messagePrefix + `Expected pattern: not ${this.utils.printExpected(expected)}\nReceived string: ${printedReceived}` + callLogText(log);
+        printedExpected = `Expected ${stringSubstring}: not ${this.utils.printExpected(expected)}`;
+        const formattedReceived = printReceivedStringContainExpectedSubstring(receivedString, receivedString.indexOf(expected), expected.length);
+        printedReceived = `Received string: ${formattedReceived}`;
       }
     } else {
-      const labelExpected = `Expected ${typeof expected === 'string' ? stringSubstring : 'pattern'}`;
-      if (notFound)
-        return messagePrefix + `${labelExpected}: ${this.utils.printExpected(expected)}\nReceived: ${received}` + callLogText(log);
-      return messagePrefix + this.utils.printDiffOrStringify(expected, receivedString, labelExpected, 'Received string', false) + callLogText(log);
+      if (notFound) {
+        printedExpected = `Expected pattern: not ${this.utils.printExpected(expected)}`;
+        printedReceived = `Received: ${received}`;
+      } else {
+        printedExpected = `Expected pattern: not ${this.utils.printExpected(expected)}`;
+        const formattedReceived = printReceivedStringContainExpectedResult(receivedString, typeof expected.exec === 'function' ? expected.exec(receivedString) : null);
+        printedReceived = `Received string: ${formattedReceived}`;
+      }
     }
+  } else {
+    const labelExpected = `Expected ${typeof expected === 'string' ? stringSubstring : 'pattern'}`;
+    if (notFound) {
+      printedExpected = `${labelExpected}: ${this.utils.printExpected(expected)}`;
+      printedReceived = `Received: ${received}`;
+    } else {
+      printedDiff = this.utils.printDiffOrStringify(expected, receivedString, labelExpected, 'Received string', false);
+    }
+  }
+
+  const message = () => {
+    const resultDetails = printedDiff ? printedDiff : printedExpected + '\n' + printedReceived;
+    return messagePrefix + resultDetails + callLogText(log);
   };
 
   return {
@@ -91,5 +118,10 @@ export async function toMatchText(
     actual: received,
     log,
     timeout: timedOut ? timeout : undefined,
+    // eslint-disable-next-line @typescript-eslint/no-base-to-string
+    locator: receiver.toString(),
+    ...(printedReceived ? { printedReceived } : {}),
+    ...(printedExpected ? { printedExpected } : {}),
+    ...(printedDiff ? { printedDiff } : {}),
   };
 }

packages/playwright/src/reporters/base.ts

@@ -32,6 +32,13 @@ type Annotation = {
 type ErrorDetails = {
   message: string;
   location?: Location;
+  timeout?: number;
+  matcherName?: string;
+  locator?: string;
+  expected?: string;
+  received?: string;
+  log?: string[];
+  snippet?: string;
 };
 
 type TestSummary = {
@@ -383,6 +390,13 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI
     errorDetails.push({
       message: indent(formattedError.message, initialIndent),
       location: formattedError.location,
+      timeout: error.timeout,
+      matcherName: error.matcherName,
+      locator: error.locator,
+      expected: error.expected,
+      received: error.received,
+      log: error.log,
+      snippet: error.snippet,
     });
   }
   return errorDetails;

packages/playwright/src/worker/testInfo.ts

@@ -24,10 +24,11 @@ import { TimeoutManager, TimeoutManagerError, kMaxDeadline } from './timeoutMana
 import type { RunnableDescription } from './timeoutManager';
 import type { Annotation, FullConfigInternal, FullProjectInternal } from '../common/config';
 import type { FullConfig, Location } from '../../types/testReporter';
-import { debugTest, filteredStackTrace, formatLocation, getContainedPath, normalizeAndSaveAttachment, serializeError, trimLongString, windowsFilesystemFriendlyLength } from '../util';
+import { debugTest, filteredStackTrace, formatLocation, getContainedPath, normalizeAndSaveAttachment, trimLongString, windowsFilesystemFriendlyLength } from '../util';
 import { TestTracing } from './testTracing';
 import type { Attachment } from './testTracing';
 import type { StackFrame } from '@protocol/channels';
+import { serializeWorkerError } from './util';
 
 export interface TestStepInternal {
   complete(result: { error?: Error | unknown, attachments?: Attachment[] }): void;
@@ -272,7 +273,7 @@ export class TestInfoImpl implements TestInfo {
         if (result.error) {
           if (typeof result.error === 'object' && !(result.error as any)?.[stepSymbol])
             (result.error as any)[stepSymbol] = step;
-          const error = serializeError(result.error);
+          const error = serializeWorkerError(result.error);
           if (data.boxedStack)
             error.stack = `${error.message}\n${stringifyStackFrames(data.boxedStack).join('\n')}`;
           step.error = error;
@@ -330,7 +331,7 @@ export class TestInfoImpl implements TestInfo {
   _failWithError(error: Error | unknown) {
     if (this.status === 'passed' || this.status === 'skipped')
       this.status = error instanceof TimeoutManagerError ? 'timedOut' : 'failed';
-    const serialized = serializeError(error);
+    const serialized = serializeWorkerError(error);
     const step: TestStepInternal | undefined = typeof error === 'object' ? (error as any)?.[stepSymbol] : undefined;
     if (step && step.boxedStack)
       serialized.stack = `${(error as Error).name}: ${(error as Error).message}\n${stringifyStackFrames(step.boxedStack).join('\n')}`;

packages/playwright/src/worker/util.ts

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { TestError } from '../../types/testReporter';
+import type { TestInfoError } from '../../types/test';
+import type { MatcherResult } from '../matchers/matcherHint';
+import { serializeError } from '../util';
+
+
+type MatcherResultDetails = Pick<TestError, 'timeout'|'matcherName'|'locator'|'expected'|'received'|'log'>;
+
+export function serializeWorkerError(error: Error | any): TestInfoError & MatcherResultDetails {
+  return {
+    ...serializeError(error),
+    ...serializeExpectDetails(error),
+  };
+}
+
+function serializeExpectDetails(e: Error): MatcherResultDetails {
+  const matcherResult = (e as any).matcherResult as MatcherResult<unknown, unknown>;
+  if (!matcherResult)
+    return {};
+  return {
+    timeout: matcherResult.timeout,
+    matcherName: matcherResult.name,
+    locator: matcherResult.locator,
+    expected: matcherResult.printedExpected,
+    received: matcherResult.printedReceived,
+    log: matcherResult.log,
+  };
+}
+