Testing: Fail E2E when page displays warning notices (#13452)

* Testing: Fail E2E when page displays warning notices

* Testing: Check markup for errors in visitAdminPage

* E2E Test Utils: Rename hasPHPError to hasPageError

* E2E Test Utils: Handle both plaintext and HTML errors

* E2E Test Utils: Refactor hasPageError as getPageError

Return error message so it can be used in failure output

* E2E Test Utils: Display page error message in failure output

packages/e2e-test-utils/CHANGELOG.md

@@ -1,3 +1,13 @@
+## Master
+
+### Enhancements
+
+- `visitAdminPage` will now throw an error (emit a test failure) when there are unexpected errors on hte page.
+
+### New Features
+
+- Added `getPageError` function, returning a promise which resolves to an error message present in the page, if any exists.
+
 ## 4.0.0 (2019-12-19)
 
 ### Breaking Changes

packages/e2e-test-utils/README.md

@@ -252,6 +252,21 @@ _Returns_
 
 -   `Promise`: Promise resolving with post content markup.
 
+<a name="getPageError" href="#getPageError">#</a> **getPageError**
+
+Returns a promise resolving to one of either a string or null. A string will
+be resolved if an error message is present in the contents of the page. If no
+error is present, a null value will be resolved instead. This requires the
+environment be configured to display errors.
+
+_Related_
+
+-   <http://php.net/manual/en/function.error-reporting.php>
+
+_Returns_
+
+-   `Promise<?string>`: Promise resolving to a string or null, depending whether a page error is present.
+
 <a name="hasBlockSwitcher" href="#hasBlockSwitcher">#</a> **hasBlockSwitcher**
 
 Returns a boolean indicating if the current selected block has a block switcher or not.

packages/e2e-test-utils/src/get-page-error.js

@@ -0,0 +1,25 @@
+/**
+ * Regular expression matching a displayed PHP error within a markup string.
+ *
+ * @see https://github.com/php/php-src/blob/598175e/main/main.c#L1257-L1297
+ *
+ * @type {RegExp}
+ */
+const REGEXP_PHP_ERROR = /(<b>)?(Fatal error|Recoverable fatal error|Warning|Parse error|Notice|Strict Standards|Deprecated|Unknown error)(<\/b>)?: (.*?) in (.*?) on line (<b>)?\d+(<\/b>)?/;
+
+/**
+ * Returns a promise resolving to one of either a string or null. A string will
+ * be resolved if an error message is present in the contents of the page. If no
+ * error is present, a null value will be resolved instead. This requires the
+ * environment be configured to display errors.
+ *
+ * @see http://php.net/manual/en/function.error-reporting.php
+ *
+ * @return {Promise<?string>} Promise resolving to a string or null, depending
+ *                            whether a page error is present.
+ */
+export async function getPageError() {
+	const content = await page.content();
+	const match = content.match( REGEXP_PHP_ERROR );
+	return match ? match[ 0 ] : null;
+}

packages/e2e-test-utils/src/index.js

@@ -22,6 +22,7 @@ export { getAvailableBlockTransforms } from './get-available-block-transforms';
 export { getBlockSetting } from './get-block-setting';
 export { getEditedPostContent } from './get-edited-post-content';
 export { hasBlockSwitcher } from './has-block-switcher';
+export { getPageError } from './get-page-error';
 export { insertBlock } from './insert-block';
 export { installPlugin } from './install-plugin';
 export { isCurrentURL } from './is-current-url';

packages/e2e-test-utils/src/test/get-page-error.js

@@ -0,0 +1,44 @@
+/**
+ * Internal dependencies
+ */
+import { getPageError } from '../get-page-error';
+
+describe( 'getPageError', () => {
+	let originalPage;
+
+	beforeEach( () => {
+		originalPage = global.page;
+	} );
+
+	afterEach( () => {
+		global.page = originalPage;
+	} );
+
+	it( 'resolves to null if there is no error', async () => {
+		global.page = {
+			content: () => 'Happy!',
+		};
+
+		expect( await getPageError() ).toBe( null );
+	} );
+
+	it.each( [
+		[
+			'PHP, HTML',
+			'<b>Notice</b>:  Undefined property: WP_Block_Type_Registry::$x in <b>/var/www/html/wp-content/plugins/gutenberg/lib/blocks.php</b> on line <b>47</b>',
+		],
+		[
+			'PHP, Plaintext',
+			'Notice: Undefined property: WP_Block_Type_Registry::$x in /var/www/html/wp-content/plugins/gutenberg/lib/blocks.php on line 47',
+		],
+	] )(
+		'resolves to the error message if there is an error (%s)',
+		async ( _variant, error ) => {
+			global.page = {
+				content: () => error,
+			};
+
+			expect( await getPageError() ).toBe( error );
+		}
+	);
+} );

packages/e2e-test-utils/src/visit-admin-page.js

@@ -9,6 +9,7 @@ import { join } from 'path';
 import { createURL } from './create-url';
 import { isCurrentURL } from './is-current-url';
 import { loginUser } from './login-user';
+import { getPageError } from './get-page-error';
 
 /**
  * Visits admin page; if user is not logged in then it logging in it first, then visits admin page.
@@ -23,4 +24,9 @@ export async function visitAdminPage( adminPath, query ) {
 		await loginUser();
 		await visitAdminPage( adminPath, query );
 	}
+
+	const error = await getPageError();
+	if ( error ) {
+		throw new Error( 'Unexpected error in page content: ' + error );
+	}
 }