chore: add accessibility test suite documentation and baseline expiration dates

This commit is contained in:
GitHub Actions
2026-04-21 00:24:52 +00:00
parent b59a788101
commit 5db3f7046c
3 changed files with 112 additions and 0 deletions

99
tests/a11y/README.md Normal file
View File

@@ -0,0 +1,99 @@
## Accessibility Test Suite (`tests/a11y`)
### Purpose and Scope
This suite checks key Charon pages for accessibility issues using Playwright and axe.
It is focused on page-level smoke coverage so we can catch major accessibility regressions early.
### Run Locally
Run a quick single-browser check:
```bash
npx playwright test tests/a11y/ --project=firefox
```
Run the full cross-browser matrix:
```bash
npx playwright test tests/a11y/ --project=chromium --project=firefox --project=webkit
```
### CI Execution
In CI, this suite runs in the non-security shard jobs of the E2E split workflow:
- Workflow: `.github/workflows/e2e-tests-split.yml`
- Jobs: non-security shard jobs for Chromium, Firefox, and WebKit
- Behavior: `tests/a11y` is included in the Playwright test paths and distributed by `--shard`
### Add a New Page Accessibility Test
1. Create or update a spec in `tests/a11y/`.
2. Import the accessibility fixture from `../fixtures/a11y`.
3. Use wait helpers (for example from `../utils/wait-helpers`) before running axe so page state is stable.
4. Attach scan results with `test.info().attach(...)` for report debugging.
5. Filter known accepted baseline items using `getBaselinedRuleIds('<page-path>')`.
6. Assert with `expectNoA11yViolations`.
Minimal pattern:
```ts
import { test } from '../fixtures/a11y';
import { waitForLoadingComplete } from '../utils/wait-helpers';
import { expectNoA11yViolations } from '../utils/a11y-helpers';
import { getBaselinedRuleIds } from './a11y-baseline';
test('example page has no critical a11y violations', async ({ page, makeAxeBuilder }) => {
await page.goto('/example');
await waitForLoadingComplete(page);
const results = await makeAxeBuilder().analyze();
test.info().attach('a11y-results', {
body: JSON.stringify(results.violations, null, 2),
contentType: 'application/json',
});
expectNoA11yViolations(results, {
knownViolations: getBaselinedRuleIds('/example'),
});
});
```
### Baseline Policy
Baseline entries are allowed only for known and accepted issues with clear rationale and a tracking ticket.
- Add a clear `reason` and a `ticket` reference.
- Add `expiresAt` so each baseline is reviewed periodically.
- Remove the baseline entry as soon as the underlying issue is fixed.
### Failure Semantics
- `critical` and `serious` violations fail the test.
- `moderate` and `minor` violations are reported in attached output and do not fail by default.
### Troubleshooting Timeout Flakes
Intermittent timeout flakes can happen, especially on Firefox.
Recommended rerun strategy:
1. Rerun the same failed spec once in Firefox.
2. If it passes on rerun, treat it as a transient flake and continue.
3. If it fails again, run the full a11y suite in Firefox.
4. If still failing, run all three browsers and inspect `a11y-results` attachments.
Useful commands:
```bash
# Rerun one spec in Firefox
npx playwright test tests/a11y/<spec-file>.spec.ts --project=firefox
# Rerun full a11y suite in Firefox
npx playwright test tests/a11y/ --project=firefox
# Rerun full a11y suite in all browsers
npx playwright test tests/a11y/ --project=chromium --project=firefox --project=webkit
```

View File

@@ -12,30 +12,35 @@ export const A11Y_BASELINE: BaselineEntry[] = [
pages: ['/'],
reason: 'Tailwind blue-500 buttons (#3b82f6) have 3.67:1 contrast with white text; requires design system update',
ticket: '#929',
expiresAt: '2026-07-31',
},
{
ruleId: 'label',
pages: ['/settings/users', '/security', '/tasks/backups', '/tasks/import/caddyfile', '/tasks/import/crowdsec'],
reason: 'Form inputs missing associated labels; requires frontend component fixes',
ticket: '#929',
expiresAt: '2026-07-31',
},
{
ruleId: 'button-name',
pages: ['/settings', '/security/headers'],
reason: 'Icon-only buttons missing accessible names; requires aria-label additions',
ticket: '#929',
expiresAt: '2026-07-31',
},
{
ruleId: 'select-name',
pages: ['/tasks/logs'],
reason: 'Select element missing associated label',
ticket: '#929',
expiresAt: '2026-07-31',
},
{
ruleId: 'scrollable-region-focusable',
pages: ['/tasks/logs'],
reason: 'Log output container is scrollable but not keyboard-focusable',
ticket: '#929',
expiresAt: '2026-07-31',
},
];

View File

@@ -10,6 +10,14 @@ test.describe('Accessibility: DNS Providers', () => {
await test.step('Navigate to DNS providers', async () => {
await page.goto('/dns/providers');
await waitForLoadingComplete(page);
await page.getByRole('heading', { name: 'DNS Management', level: 1 }).waitFor({
state: 'visible',
timeout: 10000,
});
await page.getByRole('button', { name: 'Add DNS Provider' }).waitFor({
state: 'visible',
timeout: 10000,
});
});
await test.step('Run axe accessibility scan', async () => {