diff --git a/accessibility-checker-engine/src/v4/api/IRule.ts b/accessibility-checker-engine/src/v4/api/IRule.ts index 19c1bdae8..32a0fd6f3 100644 --- a/accessibility-checker-engine/src/v4/api/IRule.ts +++ b/accessibility-checker-engine/src/v4/api/IRule.ts @@ -101,7 +101,8 @@ export type Issue = RuleResult & { ruleTime: number, message: string, bounds?: Bounds, - snippet: string + snippet: string, + level?: string } export type RuleContextHierarchy = { [namespace: string] : IMapResult[] }; diff --git a/accessibility-checker/src-ts/lib/ACHelper.ts b/accessibility-checker/src-ts/lib/ACHelper.ts index ab1cd9ce3..6f68a2f0c 100644 --- a/accessibility-checker/src-ts/lib/ACHelper.ts +++ b/accessibility-checker/src-ts/lib/ACHelper.ts @@ -6,7 +6,7 @@ import { IConfigInternal } from "./common/config/IConfig"; import { ReporterManager } from "./common/report/ReporterManager"; import { existsSync, mkdirSync, writeFileSync } from "fs"; import { IAbstractAPI } from "./common/api-ext/IAbstractAPI"; -import { IBaselineReport, IEngineReport } from "./common/engine/IReport"; +import { EngineSummaryCounts, IBaselineReport, IEngineReport } from "./common/engine/IReport"; import { dirname, join, resolve as pathResolve } from "path"; import { BaselineManager, RefactorMap } from "./common/report/BaselineManager"; @@ -229,6 +229,50 @@ async function getComplianceHelperSelenium(label, parsed, curPol) : Promise { + let reportLevel; + if (reportValue[1] === "PASS") { + reportLevel = "pass"; + } + else if ((reportValue[0] === "VIOLATION" || reportValue[0] === "RECOMMENDATION") && reportValue[1] === "MANUAL") { + reportLevel = "manual"; + } + else if (reportValue[0] === "VIOLATION") { + if (reportValue[1] === "FAIL") { + reportLevel = "violation"; + } + else if (reportValue[1] === "POTENTIAL") { + reportLevel = "potentialviolation"; + } + } + else if (reportValue[0] === "RECOMMENDATION") { + if (reportValue[1] === "FAIL") { + reportLevel = "recommendation"; + } + else if (reportValue[1] === "POTENTIAL") { + reportLevel = "potentialrecommendation"; + } + } + return reportLevel; +} + +const getCounts = (engineReport) => { + let counts = { + violation: 0, + potentialviolation: 0, + recommendation: 0, + potentialrecommendation: 0, + manual: 0, + pass: 0 + } + for (const issue of engineReport.results) { + ++counts[issue.level]; + } + return counts; +} + + let policies = ${JSON.stringify(Config.policies)}; let checker = new window.ace_ibma.Checker(); @@ -238,7 +282,14 @@ setTimeout(function() { checker.check(document, policies).then(function(report) { for (const result of report.results) { delete result.node; + result.level = valueToLevel(result.value) } + report.summary ||= {}; + report.summary.counts ||= getCounts(report); + let reportLevels = ${JSON.stringify((Config.reportLevels || []).concat(Config.failLevels || []).map(lvl => lvl.toString()))}; + // Filter out pass results unless they asked for them in reports + // We don't want to mess with baseline functions, but pass results can break the response object + report.results = report.results.filter(result => reportLevels.includes(result.level) || result.level !== "pass"); cb(report); }) },0) @@ -291,8 +342,50 @@ async function getComplianceHelperWebDriverIO(label, parsed, curPol) : Promise { + let report : IEngineReport = await page.executeAsync(({ policies, customRulesets, reportLevels }, done) => { + const valueToLevel = (reportValue) => { + let reportLevel; + if (reportValue[1] === "PASS") { + reportLevel = "pass"; + } + else if ((reportValue[0] === "VIOLATION" || reportValue[0] === "RECOMMENDATION") && reportValue[1] === "MANUAL") { + reportLevel = "manual"; + } + else if (reportValue[0] === "VIOLATION") { + if (reportValue[1] === "FAIL") { + reportLevel = "violation"; + } + else if (reportValue[1] === "POTENTIAL") { + reportLevel = "potentialviolation"; + } + } + else if (reportValue[0] === "RECOMMENDATION") { + if (reportValue[1] === "FAIL") { + reportLevel = "recommendation"; + } + else if (reportValue[1] === "POTENTIAL") { + reportLevel = "potentialrecommendation"; + } + } + return reportLevel; + } + + const getCounts = (engineReport) => { + let counts: EngineSummaryCounts = { + violation: 0, + potentialviolation: 0, + recommendation: 0, + potentialrecommendation: 0, + manual: 0, + pass: 0 + } + for (const issue of engineReport.results) { + ++counts[issue.level]; + } + return counts; + } + let checker = new (window as any).ace_ibma.Checker(); customRulesets.forEach((rs) => checker.addRuleset(rs)); return new Promise((resolve, reject) => { @@ -300,13 +393,19 @@ async function getComplianceHelperWebDriverIO(label, parsed, curPol) : Promise reportLevels.includes(result.level) || result.level !== "pass"); resolve(report); done(report); }) }, 0) }) - }, { policies: Config.policies, customRulesets: ACEngineManager.customRulesets }); + }, { policies: Config.policies, customRulesets: ACEngineManager.customRulesets, reportLevels: (Config.reportLevels || []).concat(Config.failLevels || []).map(lvl => lvl.toString()) }); if (curPol != null && !checkPolicy) { const valPolicies = ACEngineManager.customRulesets.map(rs => rs.id).concat(await page.execute(() => (new (window as any).ace_ibma.Checker().rulesetIds))); checkPolicy = true; @@ -347,8 +446,49 @@ async function getComplianceHelperPuppeteer(label, parsed, curPol) : Promise { - + let report : IEngineReport = await page.evaluate(({ policies, customRulesets, reportLevels }) => { + const valueToLevel = (reportValue) => { + let reportLevel; + if (reportValue[1] === "PASS") { + reportLevel = "pass"; + } + else if ((reportValue[0] === "VIOLATION" || reportValue[0] === "RECOMMENDATION") && reportValue[1] === "MANUAL") { + reportLevel = "manual"; + } + else if (reportValue[0] === "VIOLATION") { + if (reportValue[1] === "FAIL") { + reportLevel = "violation"; + } + else if (reportValue[1] === "POTENTIAL") { + reportLevel = "potentialviolation"; + } + } + else if (reportValue[0] === "RECOMMENDATION") { + if (reportValue[1] === "FAIL") { + reportLevel = "recommendation"; + } + else if (reportValue[1] === "POTENTIAL") { + reportLevel = "potentialrecommendation"; + } + } + return reportLevel; + } + + const getCounts = (engineReport) => { + let counts: EngineSummaryCounts = { + violation: 0, + potentialviolation: 0, + recommendation: 0, + potentialrecommendation: 0, + manual: 0, + pass: 0 + } + for (const issue of engineReport.results) { + ++counts[issue.level]; + } + return counts; + } + let checker = new (window as any).ace_ibma.Checker(); customRulesets.forEach((rs) => checker.addRuleset(rs)); return new Promise((resolve, reject) => { @@ -356,12 +496,18 @@ async function getComplianceHelperPuppeteer(label, parsed, curPol) : Promise reportLevels.includes(result.level) || result.level !== "pass"); resolve(report); }) }, 0) }) - }, { policies: Config.policies, customRulesets: ACEngineManager.customRulesets }); + }, { policies: Config.policies, customRulesets: ACEngineManager.customRulesets, reportLevels: (Config.reportLevels || []).concat(Config.failLevels || []).map(lvl => lvl.toString()) }); if (curPol != null && !checkPolicy) { const valPolicies = ACEngineManager.customRulesets.map(rs => rs.id).concat(await page.evaluate("new window.ace_ibma.Checker().rulesetIds")); checkPolicy = true; @@ -399,6 +545,48 @@ async function getComplianceHelperPuppeteer(label, parsed, curPol) : Promise { try { + const valueToLevel = (reportValue) => { + let reportLevel; + if (reportValue[1] === "PASS") { + reportLevel = "pass"; + } + else if ((reportValue[0] === "VIOLATION" || reportValue[0] === "RECOMMENDATION") && reportValue[1] === "MANUAL") { + reportLevel = "manual"; + } + else if (reportValue[0] === "VIOLATION") { + if (reportValue[1] === "FAIL") { + reportLevel = "violation"; + } + else if (reportValue[1] === "POTENTIAL") { + reportLevel = "potentialviolation"; + } + } + else if (reportValue[0] === "RECOMMENDATION") { + if (reportValue[1] === "FAIL") { + reportLevel = "recommendation"; + } + else if (reportValue[1] === "POTENTIAL") { + reportLevel = "potentialrecommendation"; + } + } + return reportLevel; + } + + const getCounts = (engineReport) => { + let counts: EngineSummaryCounts = { + violation: 0, + potentialviolation: 0, + recommendation: 0, + potentialrecommendation: 0, + manual: 0, + pass: 0 + } + for (const issue of engineReport.results) { + ++counts[issue.level]; + } + return counts; + } + let startScan = Date.now(); let checker = ACEngineManager.getChecker(); ACEngineManager.customRulesets.forEach((rs) => checker.addGuideline(rs)); @@ -406,7 +594,14 @@ async function getComplianceHelperLocal(label, parsed, curPol) : Promise lvl.toString()); + // Filter out pass results unless they asked for them in reports + // We don't want to mess with baseline functions, but pass results can break the response object + report.results = report.results.filter(result => reportLevels.includes(result.level) || result.level !== "pass"); return report; }) diff --git a/common/module/src/engine/IReport.ts b/common/module/src/engine/IReport.ts index 5c42187e4..b36abbbc3 100644 --- a/common/module/src/engine/IReport.ts +++ b/common/module/src/engine/IReport.ts @@ -65,6 +65,9 @@ export type IEngineReport = { [ruleId: string]: { [reasonId: string]: string } + }, + summary?: { + counts?: EngineSummaryCounts } } @@ -76,6 +79,28 @@ export type IBaselineResult = IEngineResult & { level: eRuleLevel } +export type EngineSummaryCounts = { + violation: number, + potentialviolation: number, + recommendation: number, + potentialrecommendation: number, + manual: number, + pass: number +} + +export type SummaryCounts = { + violation: number, + potentialviolation: number, + recommendation: number, + potentialrecommendation: number, + manual: number, + pass: number, + ignored: number, + elements: number, + elementsViolation: number, + elementsViolationReview: number +} + export type IBaselineReport = { results: IBaselineResult[] numExecuted: number, @@ -85,18 +110,7 @@ export type IBaselineReport = { } } summary: { - counts: { - violation: number, - potentialviolation: number, - recommendation: number, - potentialrecommendation: number, - manual: number, - pass: number, - ignored: number, - elements: number, - elementsViolation: number, - elementsViolationReview: number - } + counts: SummaryCounts, scanTime: number, ruleArchive: string policies: string[] @@ -120,7 +134,8 @@ export type CompressedReport = [ string, // ruleArchive string[], // policies string[], // reportLevels - CompressedIssue[] + CompressedIssue[], + SummaryCounts ] export type CompressedIssue = [ // results diff --git a/common/module/src/engine/IRule.ts b/common/module/src/engine/IRule.ts index f10a64d8c..dd65ba100 100644 --- a/common/module/src/engine/IRule.ts +++ b/common/module/src/engine/IRule.ts @@ -99,7 +99,8 @@ export type Issue = RuleResult & { ruleTime: number, message: string, bounds?: Bounds, - snippet: string + snippet: string, + level?: string } export type RuleContextHierarchy = { [namespace: string] : IMapResult[] }; diff --git a/common/module/src/report/ReporterManager.ts b/common/module/src/report/ReporterManager.ts index 605fc8a5c..d914e27f9 100644 --- a/common/module/src/report/ReporterManager.ts +++ b/common/module/src/report/ReporterManager.ts @@ -105,7 +105,8 @@ export class ReporterManager { report.engineReport.summary.ruleArchive, // 7 report.engineReport.summary.policies, // 8 report.engineReport.summary.reportLevels, // 9 - compressedResults // 10 + compressedResults, // 10 + report.engineReport.summary.counts // 11 ] for (let idx=0; idx 32000) { @@ -151,18 +152,7 @@ export class ReporterManager { numExecuted: report[5], nls, summary: { - counts: { - violation: 0, - potentialviolation: 0, - recommendation: 0, - potentialrecommendation: 0, - manual: 0, - pass: 0, - ignored: 0, - elements: 0, - elementsViolation: 0, - elementsViolationReview: 0 - }, + counts: report[11], scanTime: report[6], ruleArchive: report[7], policies: report[8], @@ -174,7 +164,6 @@ export class ReporterManager { toolID: ReporterManager.toolID, label: report[3] } - engineReport.summary.counts = ReporterManager.getCounts(engineReport); return { startScan: report[0], url: report[1], @@ -389,8 +378,10 @@ export class ReporterManager { } }); - (retVal as any).summary = {}; - retVal.summary.counts = ReporterManager.getCounts(retVal); + (retVal as any).summary = { + counts: engineResult.summary.counts + }; + retVal.summary.counts = ReporterManager.addCounts(retVal); retVal.results = retVal.results.filter(pageResult => { if (ReporterManager.config.reportLevels.includes(pageResult.level)) { @@ -427,18 +418,13 @@ export class ReporterManager { return retVal; } - private static getCounts(engineReport: IBaselineReport) { + private static addCounts(engineReport: IBaselineReport) { let counts = { - violation: 0, - potentialviolation: 0, - recommendation: 0, - potentialrecommendation: 0, - manual: 0, - pass: 0, ignored: 0, elements: 0, elementsViolation: 0, - elementsViolationReview: 0 + elementsViolationReview: 0, + ...engineReport.summary.counts } let elementSet = new Set(); let elementViolationSet = new Set(); @@ -447,8 +433,8 @@ export class ReporterManager { elementSet.add(issue.path.dom); if (issue.ignored) { ++counts.ignored; + --counts[issue.level.toString()]; } else { - ++counts[issue.level.toString()]; if (issue.level === eRuleLevel.violation) { elementViolationSet.add(issue.path.dom); elementViolationReviewSet.add(issue.path.dom); diff --git a/cypress-accessibility-checker/src/lib/ACCommands.js b/cypress-accessibility-checker/src/lib/ACCommands.js index 87a573493..ced1ae95c 100644 --- a/cypress-accessibility-checker/src/lib/ACCommands.js +++ b/cypress-accessibility-checker/src/lib/ACCommands.js @@ -16,6 +16,47 @@ let logger = { create: loggerCreate }; +function valueToLevel(reportValue) { + let reportLevel; + if (reportValue[1] === "PASS") { + reportLevel = "pass"; + } + else if ((reportValue[0] === "VIOLATION" || reportValue[0] === "RECOMMENDATION") && reportValue[1] === "MANUAL") { + reportLevel = "manual"; + } + else if (reportValue[0] === "VIOLATION") { + if (reportValue[1] === "FAIL") { + reportLevel = "violation"; + } + else if (reportValue[1] === "POTENTIAL") { + reportLevel = "potentialviolation"; + } + } + else if (reportValue[0] === "RECOMMENDATION") { + if (reportValue[1] === "FAIL") { + reportLevel = "recommendation"; + } + else if (reportValue[1] === "POTENTIAL") { + reportLevel = "potentialrecommendation"; + } + } + return reportLevel; +} + +function getCounts(engineReport) { + let counts = { + violation: 0, + potentialviolation: 0, + recommendation: 0, + potentialrecommendation: 0, + manual: 0 + } + for (const issue of engineReport.results) { + ++counts[issue.level]; + } + return counts; +} + let ACCommands = module.exports = { DEBUG: false, initialize: (win, fileConfig) => { @@ -107,7 +148,14 @@ let ACCommands = module.exports = { .then(function (report) { for (const result of report.results) { delete result.node; + result.level = valueToLevel(result.value) } + report.summary ||= {}; + report.summary.counts ||= getCounts(report); + let reportLevels = (ACCommands.Config.reportLevels || []).concat(ACCommands.Config.failLevels || []); + // Filter out pass results unless they asked for them in reports + // We don't want to mess with baseline functions, but pass results can break the response object + report.results = report.results.filter(result => reportLevels.includes(result.level) || result.level !== "pass"); return report; }) } catch (err) { diff --git a/karma-accessibility-checker/src/lib/ACHelper.js b/karma-accessibility-checker/src/lib/ACHelper.js index 20f5cda98..b49117054 100644 --- a/karma-accessibility-checker/src/lib/ACHelper.js +++ b/karma-accessibility-checker/src/lib/ACHelper.js @@ -235,7 +235,50 @@ let aChecker = { * @memberOf this */ aChecker.runScan = async function (content, policies, url, pageTitle, label, iframeWindow) { - try { + try { + const valueToLevel = (reportValue) => { + let reportLevel; + if (reportValue[1] === "PASS") { + reportLevel = "pass"; + } + else if ((reportValue[0] === "VIOLATION" || reportValue[0] === "RECOMMENDATION") && reportValue[1] === "MANUAL") { + reportLevel = "manual"; + } + else if (reportValue[0] === "VIOLATION") { + if (reportValue[1] === "FAIL") { + reportLevel = "violation"; + } + else if (reportValue[1] === "POTENTIAL") { + reportLevel = "potentialviolation"; + } + } + else if (reportValue[0] === "RECOMMENDATION") { + if (reportValue[1] === "FAIL") { + reportLevel = "recommendation"; + } + else if (reportValue[1] === "POTENTIAL") { + reportLevel = "potentialrecommendation"; + } + } + return reportLevel; + } + + const getCounts = (engineReport) => { + let counts = { + violation: 0, + potentialviolation: 0, + recommendation: 0, + potentialrecommendation: 0, + manual: 0, + pass: 0 + } + for (const issue of engineReport.results) { + ++counts[issue.level]; + } + return counts; + } + + // Get the Data when the scan is started // Start time will be in milliseconds elapsed since 1 January 1970 00:00:00 UTC up until now. const startScan = Date.now(); @@ -247,7 +290,14 @@ let aChecker = { let engineReport = await checker.check(content, policies); for (const result of engineReport.results) { delete result.node; + result.level = valueToLevel(result.value) } + let reportLevels = (aChecker.Config.reportLevels || []).concat(aChecker.Config.failLevels || []).map(lvl => lvl.toString()); + engineReport.summary ||= {}; + engineReport.summary.counts ||= getCounts(engineReport); + // Filter out pass results unless they asked for them in reports + // We don't want to mess with baseline functions, but pass results can break the response object + engineReport.results = engineReport.results.filter(result => reportLevels.includes(result.level) || result.level !== "pass"); ReporterManager.config = BaselineManager.config = aChecker.Config; diff --git a/karma-accessibility-checker/src/lib/ReporterManager.js b/karma-accessibility-checker/src/lib/ReporterManager.js index ce00870e2..14daa7235 100644 --- a/karma-accessibility-checker/src/lib/ReporterManager.js +++ b/karma-accessibility-checker/src/lib/ReporterManager.js @@ -141,8 +141,10 @@ class ReporterManager { } }); - retVal.summary = {}; - retVal.summary.counts = ReporterManager.getCounts(retVal); + retVal.summary = { + counts: retVal.summary.counts + }; + retVal.summary.counts = ReporterManager.addCounts(retVal); retVal.results = retVal.results.filter(pageResult => { if (ReporterManager.config.reportLevels.includes(pageResult.level)) { @@ -179,18 +181,13 @@ class ReporterManager { return retVal; } - static getCounts(engineReport) { + static addCounts(engineReport) { let counts = { - violation: 0, - potentialviolation: 0, - recommendation: 0, - potentialrecommendation: 0, - manual: 0, - pass: 0, ignored: 0, elements: 0, elementsViolation: 0, - elementsViolationReview: 0 + elementsViolationReview: 0, + ...engineReport.summary.counts } let elementSet = new Set(); let elementViolationSet = new Set(); @@ -199,8 +196,8 @@ class ReporterManager { elementSet.add(issue.path.dom); if (issue.ignored) { ++counts.ignored; + --counts[issue.level.toString()]; } else { - ++counts[issue.level.toString()]; if (issue.level === eRuleLevel.violation) { elementViolationSet.add(issue.path.dom); elementViolationReviewSet.add(issue.path.dom);