Merge pull request #1564 from nationalarchives/FCL-391/axe-automated-…

…accessibility-checking

FCL-391 | add axe accessibility checking assersions

compose/local/e2e_tests/Dockerfile

@@ -2,7 +2,7 @@ FROM python:3.12
 
 WORKDIR /app
 
-RUN pip install pytest pytest-playwright
+RUN pip install pytest pytest-playwright axe-playwright-python
 RUN playwright install-deps
 RUN playwright install
 COPY . .

e2e_tests/test_advanced_search_page.py

@@ -0,0 +1,13 @@
+from playwright.sync_api import Page, expect
+
+from .utils.assertions import assert_is_accessible
+
+
+def test_advanced_search_page(page: Page):
+    page.goto("/advanced_search")
+
+    expect(page).to_have_title("Advanced search - Find Case Law - The National Archives")
+
+    expect(page.locator("h1")).to_have_text("Advanced search")
+
+    assert_is_accessible(page)

e2e_tests/test_home_page.py

@@ -1,7 +1,11 @@
 from playwright.sync_api import Page, expect
 
+from .utils.assertions import assert_is_accessible
+
 
 def test_home_page(page: Page):
     page.goto("/")
 
     expect(page).to_have_title("Find Case Law - The National Archives")
+
+    assert_is_accessible(page)

e2e_tests/test_static_pages.py

@@ -0,0 +1,51 @@
+import pytest
+from playwright.sync_api import Page, expect
+
+from .utils.assertions import assert_is_accessible
+
+static_pages = [
+    {"url": "/about-this-service", "title": "About Find Case Law", "heading": "About Find Case Law"},
+    {"url": "/accessibility-statement", "title": "Accessibility statement", "heading": "Accessibility statement"},
+    {"url": "/contact-us", "title": "Contact Us", "heading": "Contact us"},
+    {
+        "url": "/courts-and-tribunals",
+        "title": "Judgments and decisions by court or tribunal",
+        "heading": "Judgments and decisions by court or tribunal",
+    },
+    {"url": "/help-and-guidance", "title": "Help and guidance", "heading": "Help and guidance"},
+    {
+        "url": "/how-to-search-find-case-law",
+        "title": "How to search Find Case Law",
+        "heading": "How to search Find Case Law",
+    },
+    {
+        "url": "/how-to-use-this-service",
+        "title": "How to use the Find Case Law service",
+        "heading": "How to use the Find Case Law service",
+    },
+    {"url": "/open-justice-licence", "title": "Open Justice Licence", "heading": "Open Justice Licence"},
+    {"url": "/privacy-notice", "title": "Privacy Notice", "heading": "Privacy notice"},
+    {"url": "/publishing-policy", "title": "Publishing policy", "heading": "Publishing policy"},
+    {"url": "/terms-and-policies", "title": "Terms and policies", "heading": "Terms and policies"},
+    {"url": "/terms-of-use", "title": "Terms of use", "heading": "Terms of use"},
+    {
+        "url": "/understanding-judgments-and-decisions",
+        "title": "Understanding judgments and decisions",
+        "heading": "Understanding judgments and decisions",
+    },
+]
+
+
+@pytest.mark.parametrize("static_page", static_pages)
+def test_static_pages(page: Page, static_page):
+    url = static_page.get("url")
+    title = static_page.get("title")
+    heading = static_page.get("heading")
+
+    page.goto(url)
+
+    expect(page).to_have_title(f"{title} - Find Case Law - The National Archives")
+
+    expect(page.locator("h1:visible")).to_have_text(heading)
+
+    assert_is_accessible(page)

e2e_tests/test_transactional_licence_form.py

@@ -2,6 +2,8 @@
 
 from playwright.sync_api import Page, expect
 
+from .utils.assertions import assert_is_accessible
+
 
 def test_transactional_licence_form(page: Page):
     """
@@ -12,12 +14,14 @@ def test_transactional_licence_form(page: Page):
 
     # Preamble pages
     page.goto("/re-use-find-case-law-records/")
+    assert_is_accessible(page)
     page.get_by_text("I want to perform computational analysis").click()
     page.get_by_text("What you need to apply for a licence").click()
     page.get_by_role("button", name="Apply for a licence").click()
     page.get_by_text("Apply now").click()
 
     # Contact details
+    assert_is_accessible(page)
     page.get_by_label("Contact Full Name").fill("Full Name")
     page.get_by_label("Contact Email address").fill("[email protected]")
     page.get_by_label("This is a different person (please enter their details below)").click()
@@ -26,6 +30,7 @@ def test_transactional_licence_form(page: Page):
     page.get_by_text("Next").click()
 
     # Organization details
+    assert_is_accessible(page)
     page.get_by_label("What is the full legal name of your organisation?").fill("Organisation name")
     page.get_by_label("Please enter any other names your organisation is known by").fill("Organisation alias")
     page.get_by_label("Which country is your organisation registered in?").fill("Chi")
@@ -40,6 +45,7 @@ def test_transactional_licence_form(page: Page):
     page.get_by_text("Next").click()
 
     # Purpose and Activities
+    assert_is_accessible(page)
     page.get_by_label("Please give any project or product name associated with this work").fill("Project name")
     page.get_by_label("Please share a link to the project or product site").fill("https://example.com")
     page.get_by_label("Publish legal information").click()
@@ -54,10 +60,12 @@ def test_transactional_licence_form(page: Page):
     page.get_by_text("Next").click()
 
     # Public Sttatement
+    assert_is_accessible(page)
     page.get_by_label("Please provide a public statement").fill("Public statement")
     page.get_by_text("Next").click()
 
     # Working practices
+    assert_is_accessible(page)
     page.get_by_role(
         "group",
         name="Will the computational analysis focus on specific individuals or specific groups of people?",
@@ -74,6 +82,7 @@ def test_transactional_licence_form(page: Page):
     page.get_by_text("Next").click()
 
     # Working practices - 2
+    assert_is_accessible(page)
     page.get_by_role("group", name="Will you make the entire record available online?").get_by_label("No").click()
     page.get_by_role("group", name="Will data extracted from these records be published online?").get_by_label(
         "Yes"
@@ -91,20 +100,25 @@ def test_transactional_licence_form(page: Page):
     page.get_by_text("Next").click()
 
     # Nine principles - 1
+    assert_is_accessible(page)
     page.get_by_label("Yes").click()
     page.get_by_text("Next").click()
 
     # Nine principles - 2
+    assert_is_accessible(page)
     page.get_by_label("Please describe how you will meet the 9 principles as terms.").fill("Nine principles statement")
     page.get_by_text("Next").click()
 
     # Additional comments
+    assert_is_accessible(page)
     page.get_by_label(
         "Are there any additional comments you would like us to consider as part of your application?"
     ).fill("Additional comments")
     page.get_by_text("Next").click()
 
     # Review
+    assert_is_accessible(page)
+
     def get_review_row(page, label):
         return page.locator("dt", has_text=label).locator("..").locator("dd.govuk-summary-list__value")
 
@@ -188,6 +202,7 @@ def get_review_row(page, label):
     ).to_have_text("Additional comments")
 
     # Editing responses
+    assert_is_accessible(page)
     page.locator("dt", has_text="Contact Full Name").locator("..").get_by_text("Change").click()
     page.get_by_label("Contact Full Name").fill("New Full Name")
     page.get_by_text("Next").click()

e2e_tests/utils/assertions.py

@@ -0,0 +1,68 @@
+import warnings
+
+from axe_playwright_python.sync_playwright import Axe
+
+axe = Axe()
+
+
+class AccessibilityWarning(UserWarning):
+    pass
+
+
+def generate_accessibility_report(violations):
+    return "\n".join(format_violation(v) for v in violations)
+
+
+def assert_critical_violations(violations, page_url):
+    critical_or_serious_violations = []
+
+    for violation in violations:
+        if violation["impact"] in ["critical", "serious"]:
+            critical_or_serious_violations.append(violation)
+
+    if critical_or_serious_violations:
+        report = generate_accessibility_report(critical_or_serious_violations)
+        # TODO: Once we have fixed all our critical/serious accessibility violations, make this fail the test
+        warnings.warn(
+            f"\nCritical/Serious Accessibility Violations Detected ({page_url}):\n{report}", AccessibilityWarning
+        )
+
+
+def check_other_violations(violations, page_url):
+    other_violations = []
+
+    for violation in violations:
+        if violation["impact"] not in ["critical", "serious"]:
+            other_violations.append(violation)
+
+    if other_violations:
+        report = generate_accessibility_report(other_violations)
+        warnings.warn(f"\nAccessibility Violations (Minor/Moderate) ({page_url}):\n{report}", AccessibilityWarning)
+
+
+def assert_is_accessible(page):
+    results = axe.run(page)
+    violations = results.response.get("violations", [])
+
+    if not violations:
+        return
+
+    assert_critical_violations(violations, page.url)
+    check_other_violations(violations, page.url)
+
+
+def format_violation(violation):
+    violation_str = f"Violation: {violation['description']}\n"
+    violation_str += f"Impact: {violation['impact'].capitalize()}\n"
+    violation_str += f"Help: {violation['help']}\n"
+    violation_str += f"URL: {violation['helpUrl']}\n"
+
+    for node in violation["nodes"]:
+        violation_str += f"HTML Element: {node['html']}\n"
+        violation_str += f"Target: {', '.join(node['target'])}\n"
+        violation_str += "Issues:\n"
+        for issue in node["any"]:
+            violation_str += f"  - {issue['message']}\n"
+        violation_str += "\n"
+
+    return violation_str