From 0ce25e6722c3fb8869d1864e0acbeb35a0ed3ec8 Mon Sep 17 00:00:00 2001
From: tonyfast <tony.fast@gmail.com>
Date: Sat, 2 Dec 2023 19:31:56 -0800
Subject: [PATCH 1/8] add trusted tester and more complete accessibility tests

---
 tests/test_axe.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tests/test_axe.py b/tests/test_axe.py
index fb26fcbc..41be4ed5 100644
--- a/tests/test_axe.py
+++ b/tests/test_axe.py
@@ -60,18 +60,23 @@
 
 axe_config_aaa = {
     "runOnly": [
-        "act",
+        "ACT",
         "best-practice",
         "experimental",
+        "wcag2a",
+        "wcag2aa",
+        "wcag2aaa",
         "wcag21a",
         "wcag21aa",
         "wcag22aa",
-        "wcag2aaa",
+        "TTv5"
     ],
     "allowedOrigins": ["<same_origin>"],
 }
 
 
+
+
 @config_notebooks_aa
 def test_axe_aa(axe, config, notebook):
     target = get_target_html(config, notebook)

From cf21b937546362e39e44aeeb8c695d6e0939ef70 Mon Sep 17 00:00:00 2001
From: tonyfast <tony.fast@gmail.com>
Date: Mon, 4 Dec 2023 10:22:39 -0800
Subject: [PATCH 2/8] being work on axe exception

---
 nbconvert_a11y/pytest_axe.py | 60 +++++++++++++++++++++++++++++++++---
 1 file changed, 56 insertions(+), 4 deletions(-)

diff --git a/nbconvert_a11y/pytest_axe.py b/nbconvert_a11y/pytest_axe.py
index 84c65cd4..337c0c97 100644
--- a/nbconvert_a11y/pytest_axe.py
+++ b/nbconvert_a11y/pytest_axe.py
@@ -1,7 +1,8 @@
 # requires node and axe
 # requires playwright
+from collections import defaultdict
 import dataclasses
-from functools import lru_cache
+from functools import lru_cache, partial
 from json import dumps, loads
 from pathlib import Path
 from shlex import quote, split
@@ -63,6 +64,58 @@ def dump(self, file: Path):
         return self
 
 
+@dataclasses.dataclass
+class Violation(Exception):
+    id: str
+    impact: str | None
+    tags: list = dataclasses.field(default=None, repr=False)
+    description: str = ""
+    help: str = ""
+    helpUrl: str = ""
+    nodes: list = dataclasses.field(default=None, repr=False)
+    elements: dict = dataclasses.field(default_factory=partial(defaultdict, list))
+    map = {}
+
+    def __class_getitem__(cls, id):
+        if id in cls.map:
+            return cls.map[id]
+        return cls.map.setdefault(id, type(id, (Violation,), {}))
+
+    def __new__(cls, **kwargs):
+        if cls is Violation:
+            target = cls.cast(kwargs)
+            self = Exception.__new__(target)
+            self.__init__(**kwargs)
+            return self
+        self = super().__new__(cls, **kwargs)
+        self.__init__(**kwargs)
+        return self
+
+    @classmethod
+    def cast(cls, data):
+        object = {"__doc__": f"""{data.get("help")} {data.get("helpUrl")}"""}
+        if data["id"] in cls.map:
+            return cls.map.get(data["id"])(**data)
+        bases = ()
+        # these generate types primitves
+        if data["impact"]:
+            bases += (Violation[data["impact"]],)
+        for tag in data["tags"]:
+            bases += (Violation[tag],)
+        return cls.map.setdefault(("-".join(data["impact"], data["id"])), type(data["id"], bases, object))
+
+    def get_elements(self, N=150):
+        for node in self.nodes:
+            key = node["html"]
+            if len(key) > N:
+                key = key[:N] + "..."
+            self.elements[key].extend(node["target"])
+
+    def __str__(self):
+        self.get_elements()
+        return repr(self)
+
+
 @dataclasses.dataclass
 class AxeException(Exception):
     message: str
@@ -90,9 +143,8 @@ def new(cls, id, impact, message, data, target, **kwargs):
     def from_violations(cls, data):
         out = []
         for violation in (violations := data.get("violations")):
-            for node in violation["nodes"]:
-                for exc in node["any"]:
-                    out.append(cls.new(**exc, target=node["target"]))
+            out.append(Violation(**violation))
+
         return exceptiongroup.ExceptionGroup(f"{len(violations)} accessibility violations", out)
 
 

From 5915b191cc568644e1825dfdd7992a4b38a24a07 Mon Sep 17 00:00:00 2001
From: tonyfast <tony.fast@gmail.com>
Date: Mon, 4 Dec 2023 10:52:03 -0800
Subject: [PATCH 3/8] clean up axe fixture

---
 nbconvert_a11y/pytest_axe.py | 42 ++++++++++++------------------------
 1 file changed, 14 insertions(+), 28 deletions(-)

diff --git a/nbconvert_a11y/pytest_axe.py b/nbconvert_a11y/pytest_axe.py
index 337c0c97..baa04f84 100644
--- a/nbconvert_a11y/pytest_axe.py
+++ b/nbconvert_a11y/pytest_axe.py
@@ -1,3 +1,10 @@
+"""report axe violations in html content
+
+* an axe fixture to use in pytest
+* a command line application for auditing files.
+
+"""
+
 # requires node and axe
 # requires playwright
 from collections import defaultdict
@@ -38,6 +45,7 @@
 
 
 def get_npm_directory(package, data=False):
+    """get the path of an npm package in the environment"""
     try:
         info = loads(check_output(split(f"npm ls --long --depth 0 --json {quote(package)}")))
     except CalledProcessError:
@@ -53,7 +61,7 @@ class AxeResults:
 
     def raises(self):
         if self.data["violations"]:
-            raise AxeException.from_violations(self.data)
+            raise Violation.from_violations(self.data)
         return self
 
     def dump(self, file: Path):
@@ -66,8 +74,8 @@ def dump(self, file: Path):
 
 @dataclasses.dataclass
 class Violation(Exception):
-    id: str
-    impact: str | None
+    id: str = dataclasses.field(repr=False)
+    impact: str | None = dataclasses.field(repr=False)
     tags: list = dataclasses.field(default=None, repr=False)
     description: str = ""
     help: str = ""
@@ -102,7 +110,9 @@ def cast(cls, data):
             bases += (Violation[data["impact"]],)
         for tag in data["tags"]:
             bases += (Violation[tag],)
-        return cls.map.setdefault(("-".join(data["impact"], data["id"])), type(data["id"], bases, object))
+        return cls.map.setdefault(
+            data["id"], type(("-".join((data["impact"], data["id"]))), bases, object)
+        )
 
     def get_elements(self, N=150):
         for node in self.nodes:
@@ -115,30 +125,6 @@ def __str__(self):
         self.get_elements()
         return repr(self)
 
-
-@dataclasses.dataclass
-class AxeException(Exception):
-    message: str
-    target: list
-    data: dict = dataclasses.field(repr=False)
-
-    types = {}
-
-    @classmethod
-    def new(cls, id, impact, message, data, target, **kwargs):
-        if id in cls.types:
-            cls = cls.types.get(id)
-        else:
-            cls = cls.types.setdefault(
-                id,
-                type(
-                    f"{impact.capitalize()}{''.join(map(str.capitalize, id.split('-')))}Exception",
-                    (cls,),
-                    {},
-                ),
-            )
-        return cls(message, target, data)
-
     @classmethod
     def from_violations(cls, data):
         out = []

From 285895165c5ba26fe2b5f9f49366255b25ca5f6f Mon Sep 17 00:00:00 2001
From: tonyfast <tony.fast@gmail.com>
Date: Mon, 4 Dec 2023 21:17:08 -0800
Subject: [PATCH 4/8] update new tests to use updated axe api

---
 nbconvert_a11y/pytest_axe.py                  | 224 +++++++++++++-----
 tests/test_a11y_baseline.py                   |  57 +++++
 tests/{test_a11y.py => test_a11y_settings.py} |  37 ++-
 tests/test_axe.py                             |  96 --------
 tests/test_third.py                           |  65 +++++
 5 files changed, 306 insertions(+), 173 deletions(-)
 create mode 100644 tests/test_a11y_baseline.py
 rename tests/{test_a11y.py => test_a11y_settings.py} (56%)
 delete mode 100644 tests/test_axe.py
 create mode 100644 tests/test_third.py

diff --git a/nbconvert_a11y/pytest_axe.py b/nbconvert_a11y/pytest_axe.py
index baa04f84..e1a6180d 100644
--- a/nbconvert_a11y/pytest_axe.py
+++ b/nbconvert_a11y/pytest_axe.py
@@ -7,6 +7,7 @@
 
 # requires node and axe
 # requires playwright
+from ast import Not
 from collections import defaultdict
 import dataclasses
 from functools import lru_cache, partial
@@ -17,31 +18,79 @@
 from typing import Any
 
 import exceptiongroup
-from attr import dataclass
+from numpy import isin
 from pytest import fixture, mark, param
 
-nbconvert_a11y_DYNAMIC_TEST = "nbconvert_a11y_DYNAMIC_TEST"
-
-axe_config_aa = {
-    "runOnly": ["act", "best-practice", "experimental", "wcag21a", "wcag21aa", "wcag22aa"],
-    "allowedOrigins": ["<same_origin>"],
-}
-
-axe_config_aaa = {
-    "runOnly": [
-        "act",
-        "best-practice",
-        "experimental",
-        "wcag21a",
-        "wcag21aa",
-        "wcag22aa",
-        "wcag2aaa",
-    ],
-    "allowedOrigins": ["<same_origin>"],
-}
-
+# selectors for regions of the notebook
 MATHJAX = "[id^=MathJax]"
-tests_axe = {"exclude": [MATHJAX]}
+JUPYTER_WIDGETS = ".jupyter-widgets"
+OUTPUTS = ".jp-OutputArea-output"
+NO_ALT = "img:not([alt])"
+PYGMENTS = ".highlight"
+
+# axe test tags
+# https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#axe-core-tags
+TEST_TAGS = [
+    "ACT",
+    "best-practice",
+    "experimental",
+    "wcag2a",
+    "wcag2aa",
+    "wcag2aaa",
+    "wcag21a",
+    "wcag21aa",
+    "wcag22aa",
+    "TTv5",
+]
+
+
+class Base:
+    """base class for exceptions and models"""
+
+    def __init_subclass__(cls) -> None:
+        dataclasses.dataclass(cls)
+
+    def dict(self):
+        return {k: v for k, v in dataclasses.asdict(self).items() if v is not None}
+
+    def dump(self):
+        return dumps(self.dict())
+
+
+# https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#api-name-axeconfigure
+class AxeConfigure(Base):
+    """axe configuration model"""
+
+    branding: str = None
+    reporter: str = None
+    checks: list = None
+    rules: list = None
+    standards: list = None
+    disableOtherRules: bool = None
+    local: str = None
+    axeVersion: str = None
+    noHtml: bool = False
+    allowedOrigins: list = dataclasses.field(default_factory=["<same_origin>"].copy)
+
+
+# https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#options-parameter
+class AxeOptions(Base):
+    """axe options model"""
+
+    runOnly: list = dataclasses.field(default_factory=TEST_TAGS.copy)
+    rules: list = None
+    reporter: str = None
+    resultTypes: Any = None
+    selectors: bool = None
+    ancestry: bool = None
+    xpath: bool = None
+    absolutePaths: bool = None
+    iframes: bool = True
+    elementRef: bool = None
+    frameWaitTime: int = None
+    preload: bool = None
+    performanceTimer: bool = None
+    pingWaitTime: int = None
 
 
 def get_npm_directory(package, data=False):
@@ -55,14 +104,17 @@ def get_npm_directory(package, data=False):
     return Path(info.get("dependencies").get(package).get("path"))
 
 
-@dataclass
-class AxeResults:
+class AxeResults(Base):
     data: Any
 
-    def raises(self):
+    def exception(self):
         if self.data["violations"]:
-            raise Violation.from_violations(self.data)
-        return self
+            return Violation.from_violations(self.data)
+
+    def raises(self):
+        exc = self.exception()
+        if exc:
+            raise exc
 
     def dump(self, file: Path):
         if file.is_dir():
@@ -72,8 +124,82 @@ def dump(self, file: Path):
         return self
 
 
+class NotAllOf(Exception):
+    ...
+
+
+class AllOf(Exception):
+    ...
+
+
+class NoAllOfMember(Exception):
+    ...
+
+
 @dataclasses.dataclass
-class Violation(Exception):
+class Axe(Base):
+    page: Any = None
+    url: str = None
+    results: Any = None
+
+    def __post_init__(self):
+        self.page.goto(self.url)
+
+    def inject(self):
+        self.page.evaluate(get_axe())
+        return self
+
+    def configure(self, **config):
+        self.page.evaluate(f"window.axe.configure({AxeConfigure(**config).dump()})")
+        return self
+
+    def reset(self):
+        self.page.evaluate(f"""window.axe.reset()""")
+        return self
+
+    def __enter__(self):
+        self.reset()
+
+    def __exit__(self, *e):
+        None
+
+    def run(self, test=None, options=None):
+        self.results = AxeResults(
+            self.page.evaluate(
+                f"""window.axe.run({test and dumps(test) or "document"}, {AxeOptions(**options or {}).dump()})"""
+            )
+        )
+        return self
+
+    def raises(self, allof=None):
+        if allof:
+            self.raises_allof(allof)
+        else:
+            self.results.raises()
+
+    def raises_allof(self, *types, extra=False):
+        found = set()
+        allof = set()
+        exc = self.results.exception()
+        if exc:
+            for t in list(types):
+                for e in exc.exceptions:
+                    allof.add(type(e))
+                    if isinstance(e, t):
+                        found.add(t)
+        not_found = set(types).difference(found)
+        if not_found:
+            raise NotAllOf(f"""{",".join(map(str, not_found))} not raised""")
+        elif not extra:
+            excess = allof.difference(found)
+            if excess:
+                raise NoAllOfMember(f"""{",".join(map(str, excess))} """)
+        result = AllOf(f"""{",".join(map(str, allof))} exceptions raised""")
+        result.__cause__ = exc
+        return result
+
+
+class Violation(Exception, Base):
     id: str = dataclasses.field(repr=False)
     impact: str | None = dataclasses.field(repr=False)
     tags: list = dataclasses.field(default=None, repr=False)
@@ -92,9 +218,7 @@ def __class_getitem__(cls, id):
     def __new__(cls, **kwargs):
         if cls is Violation:
             target = cls.cast(kwargs)
-            self = Exception.__new__(target)
-            self.__init__(**kwargs)
-            return self
+            return target(**kwargs)
         self = super().__new__(cls, **kwargs)
         self.__init__(**kwargs)
         return self
@@ -102,17 +226,16 @@ def __new__(cls, **kwargs):
     @classmethod
     def cast(cls, data):
         object = {"__doc__": f"""{data.get("help")} {data.get("helpUrl")}"""}
-        if data["id"] in cls.map:
-            return cls.map.get(data["id"])(**data)
+        name = "-".join((data["impact"], data["id"]))
+        if name in cls.map:
+            return cls.map.get(name)
         bases = ()
         # these generate types primitves
         if data["impact"]:
             bases += (Violation[data["impact"]],)
         for tag in data["tags"]:
             bases += (Violation[tag],)
-        return cls.map.setdefault(
-            data["id"], type(("-".join((data["impact"], data["id"]))), bases, object)
-        )
+        return cls.map.setdefault(name, type(name, bases, object))
 
     def get_elements(self, N=150):
         for node in self.nodes:
@@ -122,8 +245,12 @@ def get_elements(self, N=150):
             self.elements[key].extend(node["target"])
 
     def __str__(self):
-        self.get_elements()
-        return repr(self)
+        try:
+            self.get_elements()
+            return repr(self)
+        except BaseException as e:
+            print(e)
+            raise e
 
     @classmethod
     def from_violations(cls, data):
@@ -144,23 +271,12 @@ def get_axe():
     return (get_npm_directory("axe-core") / "axe.js").read_text()
 
 
-def inject_axe(page):
-    page.evaluate(get_axe())
-
-
-def run_axe_test(page, tests_config=None, axe_config=None):
-    return AxeResults(
-        page.evaluate(
-            f"window.axe.run({tests_config and dumps(tests_config) or 'document'}, {dumps(axe_config or {})})"
-        )
-    )
-
-
 @fixture()
 def axe(page):
-    def go(url, tests=tests_axe, axe_config=axe_config_aa):
-        page.goto(url)
-        inject_axe(page)
-        return run_axe_test(page, tests, axe_config)
+    def go(url, tests="document", axe_config=AxeConfigure().dict()):
+        axe = Axe(page=page, url=url)
+        axe.inject()
+        axe.configure(**axe_config)
+        return axe
 
     return go
diff --git a/tests/test_a11y_baseline.py b/tests/test_a11y_baseline.py
new file mode 100644
index 00000000..2a6b3401
--- /dev/null
+++ b/tests/test_a11y_baseline.py
@@ -0,0 +1,57 @@
+"""axe accessibility testing on exported nbconvert scripts.
+
+* test the accessibility of exported notebooks
+* test the accessibility of nbconvert-a11y dialogs
+"""
+
+from json import dumps
+from pathlib import Path
+
+from pytest import mark, param
+from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, MATHJAX
+
+from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, SKIPCI, get_target_html
+
+TPL_NOT_ACCESSIBLE = mark.xfail(reason="template is not accessible")
+HERE = Path(__file__).parent
+EXPORTS = HERE / "exports"
+HTML = EXPORTS / "html"
+AUDIT = EXPORTS / "audit"
+TREE = AUDIT / "tree"
+
+# ignore mathjax at the moment. we might be able to turne mathjax to have better
+# accessibility. https://github.com/Iota-School/notebooks-for-all/issues/81
+
+
+@mark.parametrize(
+    "config,notebook",
+    [
+        param(
+            CONFIGURATIONS / "a11y.py",
+            NOTEBOOKS / "lorenz-executed.ipynb",
+            id="lorenz-executed-a11y",
+        ),
+        param(
+            CONFIGURATIONS / "section.py",
+            NOTEBOOKS / "lorenz-executed.ipynb",
+            id="lorenz-executed-section",
+        ),
+    ],
+)
+def test_axe(axe, config, notebook):
+    """verify the baseline templates satisify all rules update AAA.
+
+    any modifications to the template can only degrade accessibility.
+    this baseline is critical for adding more features. all testing piles
+    up without automation. these surface protections allow more manual testing
+    or verified conformations of html.
+    """
+    target = get_target_html(config, notebook)
+    audit = AUDIT / target.with_suffix(".json").name
+
+    test = axe(Path.as_uri(target))
+    test.run({"exclude": [JUPYTER_WIDGETS, MATHJAX]})
+    # this is not a good place to export an audit except to
+    # verify what tests apply and what tests don't
+    # this could be a good time to export the accessibility tree.
+    test.raises()
diff --git a/tests/test_a11y.py b/tests/test_a11y_settings.py
similarity index 56%
rename from tests/test_a11y.py
rename to tests/test_a11y_settings.py
index 82a97d4d..58930c12 100644
--- a/tests/test_a11y.py
+++ b/tests/test_a11y_settings.py
@@ -3,26 +3,17 @@
 
 from pytest import fixture, mark, param
 
-from nbconvert_a11y.pytest_axe import inject_axe, run_axe_test
+from nbconvert_a11y.pytest_axe import Axe
 from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, get_target_html
 
 NEEDS_WORK = "state needs work"
 
+LORENZ_EXECUTED = get_target_html(CONFIGURATIONS / "a11y.py", NOTEBOOKS / "lorenz-executed.ipynb")
 
-@fixture(
-    params=[
-        param(
-            get_target_html(CONFIGURATIONS / "a11y.py", NOTEBOOKS / "lorenz-executed.ipynb"),
-            id="executed-a11y",
-        )
-    ],
-)
-def test_page(request, page):
-    # https://github.com/microsoft/playwright-pytest/issues/73
-    page.goto(request.param.absolute().as_uri())
-    inject_axe(page)
-    yield page
-    page.close()
+@fixture
+def lorenz(page):
+    axe = Axe(page=page, url=LORENZ_EXECUTED.as_uri())
+    yield axe.configure()
 
 
 @mark.parametrize(
@@ -38,12 +29,12 @@ def test_page(request, page):
         # failing selectors timeout and slow down tests.
     ],
 )
-def test_dialogs(test_page, dialog):
+def test_dialogs(lorenz, dialog):
     """Test the controls in dialogs."""
     # dialogs are not tested in the baseline axe test. they need to be active to test.
     # these tests activate the dialogs to assess their accessibility with the active dialogs.
-    test_page.click(dialog)
-    run_axe_test(test_page).raises()
+    lorenz.page.click(dialog)
+    lorenz.run().raises()
 
 
 SNIPPET_FONT_SIZE = (
@@ -51,9 +42,9 @@ def test_dialogs(test_page, dialog):
 )
 
 
-def test_settings_font_size(test_page):
+def test_settings_font_size(lorenz):
     """Test that the settings make their expected changes."""
-    assert test_page.evaluate(SNIPPET_FONT_SIZE) == "16px", "the default font size is unexpected"
-    test_page.click("[aria-controls=nb-settings]")
-    test_page.locator("[name=font-size]").select_option("xx-large")
-    assert test_page.evaluate(SNIPPET_FONT_SIZE) == "32px", "font size not changed"
+    assert lorenz.page.evaluate(SNIPPET_FONT_SIZE) == "16px", "the default font size is unexpected"
+    lorenz.page.click("[aria-controls=nb-settings]")
+    lorenz.page.locator("[name=font-size]").select_option("xx-large")
+    assert lorenz.page.evaluate(SNIPPET_FONT_SIZE) == "32px", "font size not changed"
diff --git a/tests/test_axe.py b/tests/test_axe.py
deleted file mode 100644
index 41be4ed5..00000000
--- a/tests/test_axe.py
+++ /dev/null
@@ -1,96 +0,0 @@
-"""axe accessibility testing on exported nbconvert scripts.
-
-* test the accessibility of exported notebooks
-* test the accessibility of nbconvert-a11y dialogs
-"""
-
-from json import dumps
-from logging import getLogger
-from pathlib import Path
-
-from pytest import mark, param
-
-from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, SKIPCI, get_target_html
-
-TPL_NOT_ACCESSIBLE = mark.xfail(reason="template is not accessible")
-HERE = Path(__file__).parent
-EXPORTS = HERE / "exports"
-HTML = EXPORTS / "html"
-LOGGER = getLogger(__name__)
-AUDIT = EXPORTS / "audit"
-TREE = AUDIT / "tree"
-
-# ignore mathjax at the moment. we might be able to turne mathjax to have better
-# accessibility. https://github.com/Iota-School/notebooks-for-all/issues/81
-MATHJAX = "[id^=MathJax]"
-
-
-config_notebooks_aa = mark.parametrize(
-    "config,notebook",
-    [
-        param(
-            (CONFIGURATIONS / (a := "default")).with_suffix(".py"),
-            (NOTEBOOKS / (b := "lorenz-executed")).with_suffix(".ipynb"),
-            marks=[SKIPCI, TPL_NOT_ACCESSIBLE],
-            id="-".join((b, a)),
-        )
-    ],
-)
-
-config_notebooks_aaa = mark.parametrize(
-    "config,notebook",
-    [
-        param(
-            (CONFIGURATIONS / (a := "a11y")).with_suffix(".py"),
-            (NOTEBOOKS / (b := "lorenz-executed")).with_suffix(".ipynb"),
-            id="-".join(
-                (b, a),
-            ),
-        ),
-        param(
-            (CONFIGURATIONS / (a := "section")).with_suffix(".py"),
-            (NOTEBOOKS / (b := "lorenz-executed")).with_suffix(".ipynb"),
-            id="-".join(
-                (b, a),
-            ),
-        ),
-    ],
-)
-
-
-axe_config_aaa = {
-    "runOnly": [
-        "ACT",
-        "best-practice",
-        "experimental",
-        "wcag2a",
-        "wcag2aa",
-        "wcag2aaa",
-        "wcag21a",
-        "wcag21aa",
-        "wcag22aa",
-        "TTv5"
-    ],
-    "allowedOrigins": ["<same_origin>"],
-}
-
-
-
-
-@config_notebooks_aa
-def test_axe_aa(axe, config, notebook):
-    target = get_target_html(config, notebook)
-    audit = AUDIT / target.with_suffix(".json").name
-
-    axe(Path.as_uri(target)).dump(audit).raises()
-
-
-@config_notebooks_aaa
-def test_axe_aaa(axe, page, config, notebook):
-    target = get_target_html(config, notebook)
-    audit = AUDIT / target.with_suffix(".json").name
-    tree = TREE / audit.name
-    test = axe(Path.as_uri(target), axe_config=axe_config_aaa).dump(audit)
-    tree.parent.mkdir(parents=True, exist_ok=True)
-    tree.write_text(dumps(page.accessibility.snapshot()))
-    test.raises()
diff --git a/tests/test_third.py b/tests/test_third.py
new file mode 100644
index 00000000..1a994624
--- /dev/null
+++ b/tests/test_third.py
@@ -0,0 +1,65 @@
+"""these tests verify how third party tools effect the accessibility of rendered notebook components.
+
+these tests allow us to track ongoing community progress and record inaccessibilities
+upstream of our control.
+"""
+
+from functools import partial
+from os import environ
+from pathlib import Path
+from unittest import TestCase
+
+from pytest import fixture, skip, mark
+from nbconvert_a11y.pytest_axe import JUPYTER_WIDGETS, MATHJAX, NO_ALT, PYGMENTS, AllOf, Violation
+from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, get_target_html
+
+# only run these tests when the CI environment variables are defined.
+environ.get("CI") or skip(allow_module_level=True)
+xfail = partial(mark.xfail, raises=AllOf, strict=True)
+
+
+class DefaultTemplate(TestCase):
+    """automated accessibility testing of the default nbconvert light theme."""
+
+    @xfail(reason="the default pygments theme has priority AA and AAA color contrast issues.")
+    def test_highlight_pygments(self):
+        """the default template has two serious color contrast violations.
+
+        an issue needs to be opened or referenced.
+        """
+        # further verification would testing the nbviewer layer.
+        raise self.axe.run({"include": [PYGMENTS]}).raises_allof(
+            Violation["serious-color-contrast-enhanced"],
+            Violation["serious-color-contrast"],
+        )
+
+    @xfail(reason="widgets have not recieved a concerted effort.")
+    def test_widget_display(self):
+        """the simple lorenz widget generates one minor and one serious accessibility violation."""
+        raise self.axe.run({"include": [JUPYTER_WIDGETS], "exclude": [NO_ALT]}).raises_allof(
+            Violation["minor-focus-order-semantics"],
+            Violation["serious-aria-input-field-name"],
+        )
+
+    @xfail(reason="mathjax provides accessibility through tabindex")
+    def test_mathjax(self):
+        """mathjax has accessibility features. one of them make equation tabbable.
+        this raises a minor axe violation. this errors indicates that we need to consider
+        configuring accessible mathjax experiences."""
+        # https://github.com/Iota-School/notebooks-for-all/issues/81
+        raise self.axe.run({"include": [MATHJAX]}).raises_allof(
+            Violation["minor-focus-order-semantics"],
+        )
+
+    # todo test mermaid
+    # test widgets kitchen sink
+    # test pandas
+
+    @fixture(autouse=True)
+    def lorenz(
+        self,
+        axe,
+        config=(CONFIGURATIONS / (a := "default")).with_suffix(".py"),
+        notebook=(NOTEBOOKS / (b := "lorenz-executed")).with_suffix(".ipynb"),
+    ):
+        self.axe = axe(Path.as_uri(get_target_html(config, notebook)))

From ce51108de30336d5479a92c1457707fe06c4f42d Mon Sep 17 00:00:00 2001
From: tonyfast <tony.fast@gmail.com>
Date: Mon, 4 Dec 2023 21:31:03 -0800
Subject: [PATCH 5/8] update fixture

---
 nbconvert_a11y/pytest_axe.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/nbconvert_a11y/pytest_axe.py b/nbconvert_a11y/pytest_axe.py
index e1a6180d..f073f862 100644
--- a/nbconvert_a11y/pytest_axe.py
+++ b/nbconvert_a11y/pytest_axe.py
@@ -138,16 +138,15 @@ class NoAllOfMember(Exception):
 
 @dataclasses.dataclass
 class Axe(Base):
+    """the Axe class is a fluent api for configuring and running accessibility tests."""
+
     page: Any = None
     url: str = None
     results: Any = None
 
     def __post_init__(self):
         self.page.goto(self.url)
-
-    def inject(self):
         self.page.evaluate(get_axe())
-        return self
 
     def configure(self, **config):
         self.page.evaluate(f"window.axe.configure({AxeConfigure(**config).dump()})")
@@ -273,9 +272,8 @@ def get_axe():
 
 @fixture()
 def axe(page):
-    def go(url, tests="document", axe_config=AxeConfigure().dict()):
+    def go(url, **axe_config):
         axe = Axe(page=page, url=url)
-        axe.inject()
         axe.configure(**axe_config)
         return axe
 

From 881cc5d7bcb17f2ff0395ed5ed3fab0fed7d3bb3 Mon Sep 17 00:00:00 2001
From: tonyfast <tony.fast@gmail.com>
Date: Mon, 4 Dec 2023 21:44:44 -0800
Subject: [PATCH 6/8] add a full accessibility test and measure expected
 failures.

---
 tests/test_third.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/tests/test_third.py b/tests/test_third.py
index 1a994624..aec34de9 100644
--- a/tests/test_third.py
+++ b/tests/test_third.py
@@ -21,6 +21,16 @@
 class DefaultTemplate(TestCase):
     """automated accessibility testing of the default nbconvert light theme."""
 
+    @xfail(reason="there is a lot of complexity in ammending accessibility in many projects")
+    def test_all(self):
+        raise self.axe.run().raises_allof(
+            Violation["serious-color-contrast-enhanced"],
+            Violation["serious-aria-input-field-name"],
+            Violation["serious-color-contrast"],
+            Violation["minor-focus-order-semantics"],
+            Violation["critical-image-alt"],
+        )
+
     @xfail(reason="the default pygments theme has priority AA and AAA color contrast issues.")
     def test_highlight_pygments(self):
         """the default template has two serious color contrast violations.
@@ -41,16 +51,6 @@ def test_widget_display(self):
             Violation["serious-aria-input-field-name"],
         )
 
-    @xfail(reason="mathjax provides accessibility through tabindex")
-    def test_mathjax(self):
-        """mathjax has accessibility features. one of them make equation tabbable.
-        this raises a minor axe violation. this errors indicates that we need to consider
-        configuring accessible mathjax experiences."""
-        # https://github.com/Iota-School/notebooks-for-all/issues/81
-        raise self.axe.run({"include": [MATHJAX]}).raises_allof(
-            Violation["minor-focus-order-semantics"],
-        )
-
     # todo test mermaid
     # test widgets kitchen sink
     # test pandas
@@ -62,4 +62,4 @@ def lorenz(
         config=(CONFIGURATIONS / (a := "default")).with_suffix(".py"),
         notebook=(NOTEBOOKS / (b := "lorenz-executed")).with_suffix(".ipynb"),
     ):
-        self.axe = axe(Path.as_uri(get_target_html(config, notebook)))
+        self.axe = axe(Path.as_uri(get_target_html(config, notebook))).configure()

From c15e282880b31b545fc149b08b0839b195d949fa Mon Sep 17 00:00:00 2001
From: tonyfast <tony.fast@gmail.com>
Date: Mon, 4 Dec 2023 21:45:40 -0800
Subject: [PATCH 7/8] comments

---
 tests/test_third.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_third.py b/tests/test_third.py
index aec34de9..32df9728 100644
--- a/tests/test_third.py
+++ b/tests/test_third.py
@@ -21,6 +21,8 @@
 class DefaultTemplate(TestCase):
     """automated accessibility testing of the default nbconvert light theme."""
 
+    # test all of the accessibility violations
+    # then incrementally explain them in smaller tests.
     @xfail(reason="there is a lot of complexity in ammending accessibility in many projects")
     def test_all(self):
         raise self.axe.run().raises_allof(

From 3e6c30d1ba4cfa799bdf865faa8cd90e75267747 Mon Sep 17 00:00:00 2001
From: tonyfast <tony.fast@gmail.com>
Date: Mon, 4 Dec 2023 21:49:09 -0800
Subject: [PATCH 8/8] critical test first

---
 tests/test_third.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_third.py b/tests/test_third.py
index 32df9728..1b7f0e00 100644
--- a/tests/test_third.py
+++ b/tests/test_third.py
@@ -26,11 +26,11 @@ class DefaultTemplate(TestCase):
     @xfail(reason="there is a lot of complexity in ammending accessibility in many projects")
     def test_all(self):
         raise self.axe.run().raises_allof(
+            Violation["critical-image-alt"],
             Violation["serious-color-contrast-enhanced"],
             Violation["serious-aria-input-field-name"],
             Violation["serious-color-contrast"],
             Violation["minor-focus-order-semantics"],
-            Violation["critical-image-alt"],
         )
 
     @xfail(reason="the default pygments theme has priority AA and AAA color contrast issues.")