Add a11y tests infrastructure

.github/workflows/acceptance.yml

@@ -32,7 +32,7 @@ jobs:
           parallel: false
           browser: chrome
           working-directory: acceptance
-          spec: cypress/tests/*.js
+          spec: cypress/tests/main/**/*.js
           install: false
           start: |
             docker compose -f ci.yml --profile prod up
@@ -51,3 +51,48 @@ jobs:
         with:
           name: cypress-videos-acceptance
           path: acceptance/cypress/videos
+
+  acceptance-a11y:
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Install Cypress
+        run: |
+          cd acceptance
+          yarn
+
+      - name: "Cypress: Acceptance tests"
+        uses: cypress-io/github-action@v6
+        env:
+          BABEL_ENV: production
+          CYPRESS_RETRIES: 2
+          CYPRESS_a11y: 1
+        with:
+          parallel: false
+          browser: chrome
+          working-directory: acceptance
+          spec: cypress/tests/a11y/**/*.js
+          install: false
+          start: |
+            docker compose -f ci-a11y.yml --profile prod up
+          wait-on: 'npx wait-on --httpTimeout 20000 http-get://localhost:8080/Plone http://localhost:3000'
+
+      # Upload Cypress screenshots
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: cypress-screenshots-acceptance
+          path: acceptance/cypress/screenshots
+
+      # Upload Cypress videos
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: cypress-videos-acceptance
+          path: acceptance/cypress/videos

Makefile

@@ -28,11 +28,13 @@ ADDON_NAME='@kitconcept/volto-light-theme'
 ADDON_PATH='volto-light-theme'
 COMPOSE_FILE=dockerfiles/docker-compose.yml
 ACCEPTANCE_COMPOSE=acceptance/docker-compose.yml
+ACCEPTANCE_COMPOSE_A11Y=acceptance/docker-compose-a11y.yml
 CMD=CURRENT_DIR=${CURRENT_DIR} ADDON_NAME=${ADDON_NAME} ADDON_PATH=${ADDON_PATH} VOLTO_VERSION=${VOLTO_VERSION} PLONE_VERSION=${PLONE_VERSION} docker compose
 DOCKER_COMPOSE=${CMD} -p ${ADDON_PATH} -f ${COMPOSE_FILE}
 DEV_COMPOSE=COMPOSE_PROFILES=dev ${DOCKER_COMPOSE}
 LIVE_COMPOSE=COMPOSE_PROFILES=dev ${DOCKER_COMPOSE}
 ACCEPTANCE=${CMD} -p ${ADDON_PATH}-acceptance -f ${ACCEPTANCE_COMPOSE}
+ACCEPTANCE_A11Y=${CMD} -p ${ADDON_PATH}-acceptance -f ${ACCEPTANCE_COMPOSE_A11Y}
 
 .PHONY: build-backend
 build-backend: ## Build
@@ -124,7 +126,7 @@ start-test-acceptance-server-prod: ## Start acceptance server in prod (used by C
 
 .PHONY: test-acceptance
 test-acceptance: ## Start Cypress (for use it while developing)
-	(cd acceptance && ./node_modules/.bin/cypress open)
+	(cd acceptance && ./node_modules/.bin/cypress open --config specPattern='cypress/tests/main/**/*.{js,jsx,ts,tsx}')
 
 .PHONY: test-acceptance-headless
 test-acceptance-headless: ## Run cypress tests in CI
@@ -138,6 +140,18 @@ stop-test-acceptance-server: ## Stop acceptance server (for use it while finishe
 status-test-acceptance-server: ## Status of Acceptance Server (for use it while developing)
 	${ACCEPTANCE} ps
 
+.PHONY: start-test-acceptance-server-a11y
+start-test-acceptance-server-a11y: ## Start a11y acceptance server (for use it in while developing)
+	${ACCEPTANCE_A11Y} --profile dev up
+
+.PHONY: stop-test-acceptance-server-a11y
+stop-test-acceptance-server-a11y: ## Stop a11y acceptance server (for use it while finished developing)
+	${ACCEPTANCE_A11Y} --profile dev down
+
+.PHONY: test-acceptance-a11y
+test-acceptance-a11y: ## Start Cypress (for use it while developing)
+	(cd acceptance && CYPRESS_a11y=1 ./node_modules/.bin/cypress open --config specPattern='cypress/tests/a11y/**/*.{js,jsx,ts,tsx}')
+
 .PHONY: debug-frontend
 debug-frontend:  ## Run bash in the Frontend container (for debug infrastructure purposes)
 	${DEV_COMPOSE} run --entrypoint bash addon-dev

README.md

@@ -333,6 +333,38 @@ When finished, don't forget to shutdown the backend server.
 make stop-test-acceptance-server
 ```
 
+### Accessibility Acceptance tests
+
+Run once
+
+```shell
+make install-acceptance
+```
+
+For starting the servers
+
+Run
+
+```shell
+make start-test-acceptance-server-a11y
+```
+
+The frontend is run in dev mode, so development while writing tests is possible.
+
+Run
+
+```shell
+make test-acceptance-a11y
+```
+
+To run Cypress tests afterwards.
+
+When finished, don't forget to shutdown the backend server.
+
+```shell
+make stop-test-acceptance-server-a11y
+```
+
 ### Release
 
 Run

acceptance/ci-a11y.yml

@@ -0,0 +1,31 @@
+version: "3"
+
+services:
+
+  addon-acceptance:
+    build:
+      context: ../
+      dockerfile: ./dockerfiles/Dockerfile
+      args:
+        ADDON_NAME: "${ADDON_NAME}"
+        ADDON_PATH: "${ADDON_PATH}"
+        VOLTO_VERSION: ${VOLTO_VERSION:-17}
+    environment:
+      RAZZLE_INTERNAL_API_PATH: http://backend-acceptance-a11y:8080/Plone
+      RAZZLE_API_PATH: http://localhost:8080/Plone
+    ports:
+      - 3000:3000
+    depends_on:
+      - backend-acceptance-a11y
+    profiles:
+      - prod
+
+  backend-acceptance-a11y:
+    image: ghcr.io/kitconcept/voltolighttheme:latest
+    environment:
+      CORS_: true
+    ports:
+      - 8080:8080
+    profiles:
+      - dev
+      - prod

acceptance/cypress.config.js

@@ -4,6 +4,14 @@ module.exports = defineConfig({
   viewportWidth: 1280,
   e2e: {
     baseUrl: 'http://localhost:3000',
-    specPattern: 'cypress/tests/*.cy.{js,jsx}',
+    specPattern: 'cypress/tests/**/*.cy.{js,jsx,ts,tsx}',
+    setupNodeEvents(on, config) {
+      on('task', {
+        table(message) {
+          console.table(message);
+          return null;
+        },
+      });
+    },
   },
 });

acceptance/cypress/support/commands.js

@@ -1 +1,23 @@
 import '@plone/volto-testing/cypress/support/commands';
+
+// Print cypress-axe violations to the terminal
+function printAccessibilityViolations(violations) {
+  cy.task(
+    'table',
+    violations.map(({ id, impact, description, nodes }) => ({
+      impact,
+      description: `${description} (${id})`,
+      nodes: nodes.length,
+    })),
+  );
+}
+
+Cypress.Commands.add(
+  'checkAccessibility',
+  (subject, { skipFailures = false } = {}) => {
+    cy.checkA11y(subject, null, printAccessibilityViolations, skipFailures);
+  },
+  {
+    prevSubject: 'optional',
+  },
+);

acceptance/cypress/support/e2e.js

@@ -1,6 +1,7 @@
 import 'cypress-axe';
 import 'cypress-file-upload';
 import './commands';
+import 'cypress-axe';
 import { setup, teardown } from './reset-fixture';
 
 beforeEach(function () {
@@ -9,10 +10,14 @@ beforeEach(function () {
   cy.setCookie('confirm_facebook', '1');
   cy.setCookie('confirm_youtube', '1');
   cy.log('Setting up API fixture');
-  setup();
+  if (!Cypress.env('a11y')) {
+    setup();
+  }
 });
 
 afterEach(function () {
   cy.log('Tearing down API fixture');
-  teardown();
+  if (!Cypress.env('a11y')) {
+    teardown();
+  }
 });

acceptance/cypress/tests/a11y/accordionBlock.cy.js

@@ -0,0 +1,30 @@
+describe('a11y tests', () => {
+  beforeEach(() => {
+    // Cypress starts out with a blank slate for each test
+    // so we must tell it to visit our website with the `cy.visit()` command.
+    // Since we want to visit the same URL at the start of all our tests,
+    // we include it in our beforeEach function so that it runs before each test
+    cy.visit('/');
+  });
+
+  // Accordion Block
+  it('Accordion Block (/block/block-accordion)', () => {
+    cy.navigate('/block/block-accordion');
+    cy.wait(2000);
+    cy.injectAxe();
+    cy.configureAxe({
+      rules: [
+        // the example page intentionally omits the h1
+        {
+          id: 'page-has-heading-one',
+          enabled: false,
+        },
+        {
+          id: 'duplicate-id',
+          enabled: false,
+        },
+      ],
+    });
+    cy.checkAccessibility();
+  });
+});

acceptance/cypress/tests/a11y/basic.cy.js

@@ -0,0 +1,26 @@
+
+describe('a11y tests', () => {
+  beforeEach(() => {
+    // Cypress starts out with a blank slate for each test
+    // so we must tell it to visit our website with the `cy.visit()` command.
+    // Since we want to visit the same URL at the start of all our tests,
+    // we include it in our beforeEach function so that it runs before each test
+    cy.visit('/');
+  });
+
+  it('Home page (/)', () => {
+    cy.navigate('/');
+    cy.wait(2000);
+    cy.injectAxe();
+    cy.configureAxe({
+      rules: [
+        // the example home page intentionally omits the h1
+        {
+          id: 'page-has-heading-one',
+          enabled: false,
+        },
+      ],
+    });
+    cy.checkAccessibility();
+  });
+});

acceptance/cypress/tests/a11y/buttonBlock.cy.js

@@ -0,0 +1,18 @@
+describe('a11y tests', () => {
+  beforeEach(() => {
+    // Cypress starts out with a blank slate for each test
+    // so we must tell it to visit our website with the `cy.visit()` command.
+    // Since we want to visit the same URL at the start of all our tests,
+    // we include it in our beforeEach function so that it runs before each test
+    cy.visit('/');
+  });
+
+  //Button
+  it('Button-Block (/block/button-block)', () => {
+    cy.navigate('/block/button-block');
+    cy.wait(2000);
+    cy.injectAxe();
+    cy.configureAxe();
+    cy.checkAccessibility();
+  });
+});

acceptance/cypress/tests/a11y/eventContenttype.cy.js

@@ -0,0 +1,18 @@
+describe('a11y tests', () => {
+  beforeEach(() => {
+    // Cypress starts out with a blank slate for each test
+    // so we must tell it to visit our website with the `cy.visit()` command.
+    // Since we want to visit the same URL at the start of all our tests,
+    // we include it in our beforeEach function so that it runs before each test
+    cy.visit('/');
+  });
+
+  //Event
+  it('Event (/content-types/event)', () => {
+    cy.navigate('/content-types/event');
+    cy.wait(2000);
+    cy.injectAxe();
+    cy.configureAxe();
+    cy.checkAccessibility();
+  });
+});

acceptance/cypress/tests/a11y/fileContenttype.cy.js

@@ -0,0 +1,18 @@
+describe('a11y tests', () => {
+  beforeEach(() => {
+    // Cypress starts out with a blank slate for each test
+    // so we must tell it to visit our website with the `cy.visit()` command.
+    // Since we want to visit the same URL at the start of all our tests,
+    // we include it in our beforeEach function so that it runs before each test
+    cy.visit('/');
+  });
+
+  //File
+  it('File (/content-types/file)', () => {
+    cy.navigate('/content-types/file');
+    cy.wait(2000);
+    cy.injectAxe();
+    cy.configureAxe();
+    cy.checkAccessibility();
+  });
+});

acceptance/cypress/tests/a11y/gridBlock.cy.js

@@ -0,0 +1,18 @@
+describe('a11y tests', () => {
+  beforeEach(() => {
+    // Cypress starts out with a blank slate for each test
+    // so we must tell it to visit our website with the `cy.visit()` command.
+    // Since we want to visit the same URL at the start of all our tests,
+    // we include it in our beforeEach function so that it runs before each test
+    cy.visit('/');
+  });
+
+  // grid block
+  it('Grid-Block (/block/grid-block)', () => {
+    cy.navigate('/block/grid-block');
+    cy.wait(2000);
+    cy.injectAxe();
+    cy.configureAxe();
+    cy.checkAccessibility();
+  });
+});

acceptance/cypress/tests/a11y/gridImageBlock.cy.js

@@ -0,0 +1,18 @@
+describe('a11y tests', () => {
+  beforeEach(() => {
+    // Cypress starts out with a blank slate for each test
+    // so we must tell it to visit our website with the `cy.visit()` command.
+    // Since we want to visit the same URL at the start of all our tests,
+    // we include it in our beforeEach function so that it runs before each test
+    cy.visit('/');
+  });
+
+  // grid block Image
+  it('Grid-Block Image (/block/grid-block/image)', () => {
+    cy.navigate('/block/grid-block/image');
+    cy.wait(2000);
+    cy.injectAxe();
+    cy.configureAxe();
+    cy.checkAccessibility();
+  });
+});