Switch Accessibility tests to @axe-core/puppeteer

.github/workflows/tests.yml

@@ -178,6 +178,13 @@ jobs:
               .cache/jest
               .cache/puppeteer
 
+          - description: Accessibility tests
+            name: test-accessibility
+            run: npx jest --color --maxWorkers=2 --selectProjects "Accessibility tests"
+            cache: |
+              .cache/jest
+              .cache/puppeteer
+
     steps:
       - name: Checkout
         uses: actions/checkout@v3

app/src/views/all-components.njk

@@ -1,4 +1,4 @@
-{% extends "full-width.njk" %}
+{% extends "full-width-landmarks.njk" %}
 
 {% from "back-link/macro.njk" import govukBackLink %}
 {% from "macros/showExamples.njk" import showExamples %}

app/src/views/component.njk

@@ -1,7 +1,7 @@
 {% from "breadcrumbs/macro.njk" import govukBreadcrumbs %}
 {% from "macros/showExamples.njk" import showExamples %}
 
-{% extends "full-width.njk" %}
+{% extends "full-width-landmarks.njk" %}
 
 {% set bodyClasses %}
   language-markup

app/src/views/examples/footer-alignment/index.njk

@@ -1,7 +1,7 @@
 {% from "back-link/macro.njk" import govukBackLink %}
 {% from "footer/macro.njk" import govukFooter %}
 
-{% extends "full-width.njk" %}
+{% extends "full-width-landmarks.njk" %}
 
 {% block beforeContent %}
   {{ govukBackLink({

app/src/views/layouts/full-width-landmarks.njk

@@ -0,0 +1,10 @@
+{% extends "layout.njk" %}
+
+{% block main %}
+  <div class="govuk-width-container">
+    {% block beforeContent %}{% endblock %}
+  </div>
+  <main class="govuk-main-wrapper {{ mainClasses }}" id="main-content" role="main">
+    {% block content %}{% endblock %}
+  </main>
+{% endblock %}

app/src/views/layouts/full-width.njk

@@ -4,7 +4,5 @@
   <div class="govuk-width-container">
     {% block beforeContent %}{% endblock %}
   </div>
-  <main class="govuk-main-wrapper {{ mainClasses }}" id="main-content" role="main">
-    {% block content %}{% endblock %}
-  </main>
+  {% block content %}{% endblock %}
 {% endblock %}

docs/releasing/testing-and-linting.md

@@ -76,6 +76,12 @@ We write functional tests for every component to check the output of our Nunjuck
 
 If a component uses JavaScript, we also write functional tests in a `[component name].test.js` file, for example [checkboxes.test.js](../../src/govuk/components/checkboxes/checkboxes.test.js). These component tests check that interactions, such as a mouse click, have the expected result.
 
+If you want to inspect a test that's running in the browser, configure Jest Puppeteer in non-headless mode with the environment variable `HEADLESS=false` and then use [Jest Puppeteer's debug mode](https://github.com/argos-ci/jest-puppeteer/blob/main/README.md#debug-mode) to pause the test execution.
+
+```
+HEADLESS=false npx jest --watch src/govuk/components/tag/accessibility.test.mjs
+```
+
 You should also test component Javascript logic with unit tests, in a `[component name].unit.test.mjs` file. These tests are better suited for testing behind-the-scenes logic, or in cases where the final output of some logic is not a change to the component markup.
 
 ### Global tests

jest-puppeteer.config.js

@@ -30,6 +30,7 @@ module.exports = {
        */
       '--no-startup-window'
     ],
+    headless: process.env.HEADLESS !== 'false',
     waitForInitialPage: false
   },
 

jest.config.mjs

@@ -1,5 +1,12 @@
+import jestPuppeteerConfig from './jest-puppeteer.config.js'
+
+// Detect when browser has been launched headless
+const { headless = true } = jestPuppeteerConfig.launch
+
 /**
- * @type {import('jest').Config}
+ * Jest project config defaults
+ *
+ * @type {import('@jest/types').Config.ProjectConfig}
  */
 const config = {
   cacheDirectory: '<rootDir>/.cache/jest/',
@@ -37,10 +44,23 @@ const config = {
 }
 
 /**
- * @type {import('jest').Config}
+ * Jest config
+ *
+ * @type {import('@jest/types').Config.InitialOptions}
+ * @example
+ * ```console
+ * npx jest --selectProjects "Nunjucks macro tests"
+ * npx jest --selectProjects "JavaScript unit tests"
+ * ```
  */
 export default {
   collectCoverageFrom: ['./src/**/*.{js,mjs}'],
+
+  // Reduce CPU usage during project test runs
+  maxWorkers: headless
+    ? '50%' // Matches Jest default (50%) via `--watch`
+    : 1, // Use only 1x browser window when headless
+
   projects: [
     {
       ...config,
@@ -52,7 +72,6 @@ export default {
     {
       ...config,
       displayName: 'Nunjucks macro tests',
-      setupFilesAfterEnv: ['govuk-frontend-helpers/jest/matchers.js'],
       snapshotSerializers: [
         'jest-serializer-html'
       ],
@@ -97,11 +116,23 @@ export default {
         '**/components/globals.test.mjs',
         '**/components/*/*.test.{js,mjs}',
 
-        // Exclude macro/unit tests
+        // Exclude accessibility/macro/unit tests
+        '!**/*/accessibility.test.{js,mjs}',
         '!**/(*.)?template.test.{js,mjs}',
         '!**/*.unit.test.{js,mjs}'
       ],
 
+      // Web server and browser required
+      globalSetup: 'govuk-frontend-helpers/jest/browser/open.mjs',
+      globalTeardown: 'govuk-frontend-helpers/jest/browser/close.mjs'
+    },
+    {
+      ...config,
+      displayName: 'Accessibility tests',
+      setupFilesAfterEnv: ['govuk-frontend-helpers/jest/matchers.js'],
+      testEnvironment: 'govuk-frontend-helpers/jest/environment/puppeteer.mjs',
+      testMatch: ['**/*/accessibility.test.{js,mjs}'],
+
       // Web server and browser required
       globalSetup: 'govuk-frontend-helpers/jest/browser/open.mjs',
       globalTeardown: 'govuk-frontend-helpers/jest/browser/close.mjs'

puppeteer.config.js

@@ -4,5 +4,14 @@ const { join } = require('path')
  * @type {import('puppeteer').Configuration}
  */
 module.exports = {
-  cacheDirectory: join(__dirname, '.cache', 'puppeteer')
+  cacheDirectory: join(__dirname, '.cache', 'puppeteer'),
+  experiments: {
+    /**
+     * Use Chromium (for ARM) where supported to
+     * avoid 30-40% increase in total test time
+     *
+     * {@link https://pptr.dev/contributing#macos-arm-and-custom-executables}
+     */
+    macArmChromiumEnabled: true
+  }
 }

shared/helpers/package.json

@@ -14,6 +14,7 @@
   },
   "license": "MIT",
   "devDependencies": {
+    "@axe-core/puppeteer": "^4.6.1",
     "cheerio": "^1.0.0-rc.12",
     "govuk-frontend-config": "*",
     "govuk-frontend-lib": "*",

shared/helpers/puppeteer.js

@@ -1,8 +1,64 @@
+const { AxePuppeteer } = require('@axe-core/puppeteer')
 const { ports } = require('govuk-frontend-config')
 const { componentNameToClassName } = require('govuk-frontend-lib/names')
 
 const { renderHTML } = require('./nunjucks')
 
+/**
+ * Axe Puppeteer reporter
+ *
+ * @param {import('puppeteer').Page} page - Puppeteer page object
+ * @param {import('axe-core').RuleObject} [overrides] - Axe rule overrides
+ * @returns {Promise<import('axe-core').AxeResults>} Axe Puppeteer instance
+ */
+async function axe (page, overrides = {}) {
+  const reporter = new AxePuppeteer(page)
+    .setLegacyMode(true) // Share single page via iframe
+    .include('body')
+    .withRules([
+      'best-practice',
+
+      // WCAG 2.x
+      'wcag2a',
+      'wcag2aa',
+      'wcag2aaa',
+
+      // WCAG 2.1
+      'wcag21a',
+      'wcag21aa',
+
+      // WCAG 2.2
+      'wcag22aa'
+    ])
+
+  // Ignore colour contrast for 'inactive' components
+  if (page.url().includes('-disabled')) {
+    overrides['color-contrast'] = { enabled: false }
+  }
+
+  // Shared rules for GOV.UK Frontend
+  const rules = {
+    /**
+     * Ignore 'Some page content is not contained by landmarks'
+     * {@link https://github.com/alphagov/govuk-frontend/issues/1604}
+     */
+    region: { enabled: false },
+    ...overrides
+  }
+
+  // Create report
+  const report = await reporter
+    .options({ rules })
+    .analyze()
+
+  // Add preview URL to report violations
+  report.violations.forEach((violation) => {
+    violation.helpUrl = `${violation.helpUrl}\n${page.url()}`
+  })
+
+  return report
+}
+
 /**
  * Render and initialise a component within test boilerplate HTML
  *
@@ -153,6 +209,7 @@ async function isVisible ($element) {
 }
 
 module.exports = {
+  axe,
   renderAndInitialise,
   goTo,
   goToComponent,

shared/helpers/tests.js

@@ -1,7 +1,6 @@
 const { join } = require('path')
 
 const { paths } = require('govuk-frontend-config')
-const { configureAxe } = require('jest-axe')
 const { compileAsync, compileStringAsync, Logger } = require('sass-embedded')
 
 const sassPaths = [
@@ -60,19 +59,7 @@ function htmlWithClassName ($, className) {
   return $.html($component)
 }
 
-/**
- * As we're testing incomplete HTML fragments, we don't expect there to be a
- * skip link, or for them to be contained within landmarks.
- */
-const axe = configureAxe({
-  rules: {
-    'skip-link': { enabled: false },
-    region: { enabled: false }
-  }
-})
-
 module.exports = {
-  axe,
   htmlWithClassName,
   compileSassFile,
   compileSassString

src/.eslintrc.js

@@ -46,7 +46,7 @@ module.exports = {
     },
     {
       // Matches 'JavaScript component tests' in jest.config.mjs
-      // to ignore unknown 'page' and 'browser' Puppeteer globals
+      // to ignore unknown Jest Puppeteer globals
       files: [
         '**/components/globals.test.mjs',
         '**/components/*/*.test.{js,mjs}'
@@ -56,8 +56,9 @@ module.exports = {
         '**/*.unit.test.{js,mjs}'
       ],
       globals: {
-        page: true,
-        browser: true
+        page: 'readonly',
+        browser: 'readonly',
+        jestPuppeteer: 'readonly'
       }
     }
   ]

src/govuk/components/accordion/accessibility.test.mjs

@@ -0,0 +1,36 @@
+import { axe, goToComponent } from 'govuk-frontend-helpers/puppeteer'
+import { getExamples } from 'govuk-frontend-lib/files'
+
+describe('/components/accordion', () => {
+  let axeRules
+
+  beforeAll(() => {
+    axeRules = {
+      /**
+       * Ignore 'aria-labelledby attribute cannot be used on a div with
+       * no valid role attribute'
+       *
+       * {@link https://github.com/alphagov/govuk-frontend/issues/2472}
+       */
+      'aria-allowed-attr': { enabled: false }
+    }
+  })
+
+  describe('component examples', () => {
+    let exampleNames
+
+    beforeAll(async () => {
+      exampleNames = Object.keys(await getExamples('accordion'))
+    })
+
+    it('passes accessibility tests', async () => {
+      for (const name of exampleNames) {
+        const exampleName = name.replace(/ /g, '-')
+
+        // Navigation to example, create report
+        await goToComponent(page, 'accordion', { exampleName })
+        await expect(axe(page, axeRules)).resolves.toHaveNoViolations()
+      }
+    }, 60000)
+  })
+})