Fix a11y CI workflow

.github/workflows/ci.yml

@@ -88,9 +88,9 @@ jobs:
       - name: Type check packages
         run: pnpm typecheck
 
-  pa11y:
+  a11y:
     name: Check for accessibility issues
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -107,15 +107,9 @@ jobs:
       - name: Install Dependencies
         run: pnpm i
 
-      - name: Build docs site
-        working-directory: ./docs
-        run: pnpm build
-        env:
-          NO_GRADIENTS: true
-
       - name: Run accessibility audit
-        working-directory: ./docs
-        run: pnpm t
+        working-directory: ./packages/starlight
+        run: pnpm test:a11y
 
   windows-smoke:
     name: Docs site builds on Windows

CONTRIBUTING.md

@@ -251,7 +251,7 @@ To add a language, you will need its BCP-47 tag and a label. See [“Adding a ne
 - Add your language to the `locales` config in `docs/astro.config.mjs`
 - Add your language to the `locales` config in `docs/lunaria.config.json`
 - Add your language’s subtag to the i18n label config in `.github/labeler.yml`
-- Add your language to the `pa11y` script’s `--sitemap-exclude` flag in `package.json`
+- Add your language to the `config.sitemap.exclude` option in `packages/starlight/__a11y__/test-utils.ts`
 - Create the first translated page for your language.
   This must be the Starlight landing page: `docs/src/content/docs/{language}/index.mdx`.
 - Open a pull request on GitHub to add your changes to Starlight!

docs/astro.config.mjs

@@ -65,7 +65,7 @@ export default defineConfig({
 					attrs: { property: 'og:image:alt', content: ogImageAlt },
 				},
 			],
-			customCss: process.env.NO_GRADIENTS ? [] : ['./src/assets/landing.css'],
+			customCss: ['./src/assets/landing.css'],
 			locales,
 			sidebar: [
 				{

docs/package.json

@@ -4,8 +4,6 @@
   "type": "module",
   "version": "0.0.1",
   "scripts": {
-    "test": "start-server-and-test 'pnpm preview' http://localhost:4321 'pnpm pa11y'",
-    "pa11y": "pa11y-ci --sitemap 'http://localhost:4321/sitemap-0.xml' --sitemap-find 'https://starlight.astro.build' --sitemap-replace 'http://localhost:4321' --sitemap-exclude '/(de|zh-cn|fr|es|pt-br|pt-pt|it|id|ko|ru|tr|hi|da|uk)/.*'",
     "dev": "astro dev",
     "start": "astro dev",
     "build": "astro build",
@@ -25,8 +23,6 @@
     "sharp": "^0.32.5"
   },
   "devDependencies": {
-    "pa11y-ci": "^3.0.1",
-    "starlight-links-validator": "^0.12.1",
-    "start-server-and-test": "^2.0.4"
+    "starlight-links-validator": "^0.12.1"
   }
 }

package.json

@@ -47,6 +47,7 @@
     "peerDependencyRules": {
       "ignoreMissing": [
         "@algolia/client-search",
+        "playwright",
         "search-insights"
       ]
     }

packages/starlight/__a11y__/docs.test.ts

@@ -0,0 +1,21 @@
+import { expect, test } from './test-utils';
+
+test('does not report accessibility violations on the docs site', async ({ docsSite }) => {
+	let violationsCount = 0;
+
+	const urls = await docsSite.getAllUrls();
+
+	for (const url of urls) {
+		const violations = await docsSite.testPage(url);
+
+		if (violations.length > 0) {
+			violationsCount += violations.length;
+			await docsSite.reportPageViolations(violations);
+		}
+	}
+
+	expect(
+		violationsCount,
+		`Found ${violationsCount} accessibility violations. Check the errors above for more details.`
+	).toBe(0);
+});

packages/starlight/__a11y__/test-utils.ts

@@ -0,0 +1,82 @@
+import { test as baseTest, type Page } from '@playwright/test';
+import {
+	DefaultTerminalReporter,
+	getViolations,
+	injectAxe,
+	reportViolations,
+} from 'axe-playwright';
+import Sitemapper from 'sitemapper';
+
+export { expect, type Locator } from '@playwright/test';
+
+const config: Config = {
+	axe: {
+		// https://www.deque.com/axe/core-documentation/api-documentation/#axecore-tags
+		runOnly: {
+			type: 'tag',
+			values: ['wcag2a', 'wcag21a', 'wcag2aa', 'wcag21aa', 'wcag22aa', 'best-practice'],
+		},
+	},
+	sitemap: {
+		url: 'http://localhost:4321/sitemap-index.xml',
+		exclude: /\/(de|zh-cn|fr|es|pt-br|pt-pt|it|id|ko|ru|tr|hi|da|uk)\/.*/,
+		replace: {
+			query: 'https://starlight.astro.build',
+			value: 'http://localhost:4321',
+		},
+	},
+};
+
+process.env.ASTRO_TELEMETRY_DISABLED = 'true';
+process.env.ASTRO_DISABLE_UPDATE_CHECK = 'true';
+
+export const test = baseTest.extend<{
+	docsSite: DocsSite;
+}>({
+	docsSite: async ({ page }, use) => use(new DocsSite(page)),
+});
+
+// A Playwright test fixture accessible from within all tests.
+class DocsSite {
+	constructor(private readonly page: Page) {}
+
+	async getAllUrls() {
+		const sitemap = new Sitemapper({ url: config.sitemap.url });
+		const { sites } = await sitemap.fetch();
+
+		if (sites.length === 0) {
+			throw new Error('No URLs found in sitemap.');
+		}
+
+		return sites
+			.map((url) => url.replace(config.sitemap.replace.query, config.sitemap.replace.value))
+			.filter((url) => !config.sitemap.exclude.test(url));
+	}
+
+	async testPage(url: string) {
+		await this.page.goto(url);
+		await injectAxe(this.page);
+		await this.page.waitForLoadState('networkidle');
+		return getViolations(this.page, undefined, config.axe);
+	}
+
+	async reportPageViolations(violations: Awaited<ReturnType<typeof this.testPage>>) {
+		console.error(
+			`> Found ${violations.length} violations on ${this.page.url().replace(config.sitemap.replace.value, '')}\n`
+		);
+		await reportViolations(violations, new DefaultTerminalReporter(true, true, false));
+		console.error('\n');
+	}
+}
+
+interface Config {
+	axe: Parameters<typeof getViolations>[2];
+	sitemap: {
+		url: string;
+		exclude: RegExp;
+		replace: {
+			query: string;
+			value: string;
+		};
+	};
+}

packages/starlight/package.json

@@ -5,7 +5,8 @@
   "scripts": {
     "test": "vitest",
     "test:coverage": "vitest run --coverage",
-    "test:e2e": "playwright install --with-deps chromium && playwright test"
+    "test:e2e": "playwright install --with-deps chromium && playwright test",
+    "test:a11y": "playwright install --with-deps chromium && playwright test -c playwright.config.a11y.ts"
   },
   "keywords": [
     "docs",
@@ -180,6 +181,8 @@
     "@types/node": "^18.16.19",
     "@vitest/coverage-v8": "^1.6.0",
     "astro": "^4.15.3",
+    "axe-playwright": "^2.0.3",
+    "sitemapper": "^3.2.12",
     "vitest": "^1.6.0",
     "linkedom": "^0.18.4"
   },

packages/starlight/playwright.config.a11y.ts

@@ -0,0 +1,20 @@
+import { defineConfig } from '@playwright/test';
+import config from './playwright.config';
+
+export default defineConfig({
+	...config,
+	testMatch: '__a11y__/*.test.ts',
+	// The timeout for the accessibility tests only.
+	timeout: 180 * 1_000,
+	webServer: [
+		{
+			command: 'pnpm run build && pnpm run preview',
+			cwd: '../../docs',
+			reuseExistingServer: !process.env['CI'],
+			stdout: 'pipe',
+			// The timeout of the single build step ran before the accessibility tests.
+			timeout: 120 * 1_000,
+			url: 'http://localhost:4321',
+		},
+	],
+});

packages/starlight/vitest.config.ts

@@ -7,6 +7,7 @@ export default defineConfig({
 			reportsDirectory: './__coverage__',
 			exclude: [
 				...coverageConfigDefaults.exclude,
+				'**/__a11y__/**',
 				'**/__e2e__/**',
 				'playwright.config.*',
 				'**/vitest.*',