Updating result schema to include more specific data

addon-test-support/index.ts

@@ -10,6 +10,7 @@ export { setCustomReporter } from './reporter';
 export {
   TEST_SUITE_RESULTS as _TEST_SUITE_RESULTS,
   middlewareReporter as _middlewareReporter,
+  pushTestResult as _pushTestResult,
   setupMiddlewareReporter,
 } from './setup-middleware-reporter';
 export { storeResults, printResults } from './logger';

addon-test-support/setup-middleware-reporter.ts

@@ -1,36 +1,79 @@
 import QUnit from 'qunit';
-import { getContext, getTestMetadata } from '@ember/test-helpers';
-import { AxeResults } from 'axe-core';
+import {
+  currentRouteName,
+  currentURL,
+  getContext,
+  getTestMetadata,
+} from '@ember/test-helpers';
+import { AxeResults, Result } from 'axe-core';
 import { setCustomReporter } from './reporter';
 import { DEBUG } from '@glimmer/env';
 
 interface AxeTestResult {
   moduleName: string;
   testName: string;
-  helperName: string;
+  urls: string[] | Set<string>;
+  routes: string[] | Set<string>;
+  helpers: string[];
   stack: string;
-  axeResults: AxeResults;
+  violations: Result[];
 }
 
 export const TEST_SUITE_RESULTS: AxeTestResult[] = [];
 
-export function buildResult(axeResults: AxeResults) {
-  let { module, testName } = QUnit.config.current;
-  let testMetaData = getTestMetadata(getContext());
+let currentTestResult: AxeTestResult | undefined = undefined;
+let currentViolationsMap: Map<string, Result> | undefined = undefined;
+let currentUrls = new Set<string>();
+let currentRouteNames = new Set<string>();
 
-  let stack = (!DEBUG && new Error().stack) || '';
+export async function middlewareReporter(axeResults: AxeResults) {
+  if (axeResults.violations.length === 0) {
+    return;
+  }
+
+  if (!currentTestResult) {
+    let { module, testName } = QUnit.config.current;
+    let testMetaData = getTestMetadata(getContext());
+
+    let stack = (!DEBUG && new Error().stack) || '';
+
+    currentTestResult = {
+      moduleName: module.name,
+      testName,
+      urls: [],
+      routes: [],
+      helpers: testMetaData.usedHelpers,
+      stack,
+      violations: [],
+    };
+
+    currentViolationsMap = new Map();
+  }
+
+  currentUrls.add(currentURL());
+  currentRouteNames.add(currentRouteName());
 
-  return {
-    moduleName: module.name,
-    testName,
-    helperName: testMetaData.usedHelpers.pop() || '',
-    stack,
-    axeResults,
-  };
+  axeResults.violations.forEach((violation) => {
+    let rule = currentViolationsMap!.get(violation.id);
+
+    if (rule === undefined) {
+      currentViolationsMap!.set(violation.id, violation);
+    } else {
+      rule.nodes.push(...violation.nodes);
+    }
+  });
 }
 
-export async function middlewareReporter(axeResults: AxeResults) {
-  TEST_SUITE_RESULTS.push(buildResult(axeResults));
+export function pushTestResult() {
+  if (typeof currentTestResult !== 'undefined') {
+    currentTestResult.violations = [...currentViolationsMap!.values()];
+    currentTestResult.urls = [...currentUrls.values()];
+    currentTestResult.routes = [...currentRouteNames.values()];
+    TEST_SUITE_RESULTS.push(currentTestResult);
+
+    currentTestResult = undefined;
+    currentViolationsMap = undefined;
+  }
 }
 
 /**
@@ -39,6 +82,8 @@ export async function middlewareReporter(axeResults: AxeResults) {
 export function setupMiddlewareReporter() {
   setCustomReporter(middlewareReporter);
 
+  QUnit.testDone(pushTestResult);
+
   QUnit.done(async function () {
     let response = await fetch('/report-violations', {
       method: 'POST',

setup-middleware.js

@@ -3,16 +3,16 @@
 const bodyParser = require('body-parser').json({ limit: '50mb' });
 const path = require('path');
 const date = require('date-and-time');
-const { ensureDirSync, writeJsonSync } = require('fs-extra');
+const { ensureDirSync, writeJsonSync, emptyDirSync } = require('fs-extra');
 
-function reportViolations(req, res, options) {
+let outputDir;
+
+function reportViolations(req, res) {
   const REPORT_TIMESTAMP = date.format(new Date(), 'YYYY-MM-DD-HH_mm_ss');
-  let outputDir = path.join(options.root, 'ember-a11y-report');
   let outputPath = path.resolve(
     path.join(outputDir, `${REPORT_TIMESTAMP}.json`)
   );
 
-  ensureDirSync(outputDir);
   writeJsonSync(outputPath, req.body);
 
   res.send({
@@ -26,11 +26,16 @@ function logError(err, req, res, next) {
 }
 
 function setupMiddleware(app, options) {
+  outputDir = path.join(options.root, 'ember-a11y-report');
+
+  ensureDirSync(outputDir);
+  emptyDirSync(outputDir);
+
   app.post(
     '/report-violations',
     bodyParser,
     (req, res) => {
-      reportViolations(req, res, options);
+      reportViolations(req, res);
     },
     logError
   );

tests/acceptance/setup-middleware-reporter-test.ts

@@ -7,6 +7,7 @@ import {
   setupGlobalA11yHooks,
   teardownGlobalA11yHooks,
   _middlewareReporter,
+  _pushTestResult,
   _TEST_SUITE_RESULTS,
 } from 'ember-a11y-testing/test-support';
 
@@ -34,6 +35,8 @@ module('setupMiddlewareReporter', function (hooks) {
 
     await visit('/');
 
-    assert.deepEqual(_TEST_SUITE_RESULTS[0].axeResults.violations.length, 3);
+    _pushTestResult();
+
+    assert.deepEqual(_TEST_SUITE_RESULTS[0].violations.length, 3);
   });
 });