Tests: Improve visual regression workflow for admin reskin iteration

Add impact summary reporter, fast re-run mode, and auto-open HTML report
to streamline CSS iteration during the admin reskin effort.

Author: josephfusco

package.json

@@ -130,6 +130,7 @@
 		"test:coverage": "npm run test:php -- --coverage-html ./coverage/html/ --coverage-php ./coverage/php/report.php --coverage-text=./coverage/text/report.txt",
 		"test:e2e": "wp-scripts test-playwright --config tests/e2e/playwright.config.js",
 		"test:visual": "bash tests/visual-regression/compare-branches.sh",
+		"test:visual:quick": "bash tests/visual-regression/compare-branches.sh --skip-baselines",
 		"gutenberg:checkout": "node tools/gutenberg/checkout-gutenberg.js",
 		"gutenberg:build": "node tools/gutenberg/build-gutenberg.js",
 		"gutenberg:copy": "node tools/gutenberg/copy-gutenberg-build.js",

tests/visual-regression/README.md

@@ -2,6 +2,12 @@
 
 These tests make use of Playwright, with a setup very similar to that of the e2e tests.
 
+## Prerequisites
+
+- Node.js >= 20.10.0
+- A running WordPress environment (`npm run env:start`)
+- Playwright browsers installed (`npx playwright install chromium`)
+
 ## How to Run the Tests Locally
 
 From a feature branch with a clean working tree, run:
@@ -14,8 +20,27 @@ This will automatically:
 1. Check out trunk to generate baseline snapshots.
 2. Return to your feature branch.
 3. Compare the current branch against those baselines.
+4. Print a visual impact summary in the terminal.
+5. Open the HTML report with side-by-side visual diffs.
+
+### Quick Re-run (Skip Baseline Generation)
 
-If any tests fail, diff images can be found in `artifacts/`.
+After the first run, baselines from trunk are already saved. To re-compare without regenerating them:
+
+```bash
+npm run test:visual:quick
+```
+
+This is useful when iterating on CSS — no need to commit changes or wait for trunk baselines to regenerate each time.
+
+**Typical workflow:**
+```bash
+npm run test:visual          # First run: generates baselines from trunk
+# ... tweak CSS ...
+npm run test:visual:quick    # Fast: reuses existing baselines
+# ... tweak more CSS ...
+npm run test:visual:quick    # Fast again
+```
 
 ### Specifying a Base Branch
 
@@ -33,3 +58,24 @@ You can also generate baselines and compare manually:
 2. Run `npx playwright test --config tests/visual-regression/playwright.config.js --update-snapshots`
 3. Check out the feature branch to be tested.
 4. Run `npx playwright test --config tests/visual-regression/playwright.config.js`
+
+## Impact Summary
+
+After each run, a summary is printed to the terminal showing which pages changed and which remained unchanged:
+
+```
+────────────────────────────────────────────────
+Visual Impact Summary
+────────────────────────────────────────────────
+Changed:   27 of 35 pages
+Unchanged: 8 of 35 pages
+
+Changed:
+  Dashboard, All Posts, Categories, Tags, ...
+
+Unchanged:
+  Login, Media Settings, ...
+────────────────────────────────────────────────
+```
+
+The HTML report also opens automatically with side-by-side visual diffs for detailed inspection.

tests/visual-regression/compare-branches.sh

@@ -1,8 +1,22 @@
 #!/bin/bash
 set -e
 
+SKIP_BASELINES=false
+BASELINE_BRANCH="trunk"
+
+# Parse arguments.
+for arg in "$@"; do
+	case "$arg" in
+		--skip-baselines)
+			SKIP_BASELINES=true
+			;;
+		*)
+			BASELINE_BRANCH="$arg"
+			;;
+	esac
+done
+
 CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
-BASELINE_BRANCH="${1:-trunk}"
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 CONFIG="$SCRIPT_DIR/playwright.config.js"
 
@@ -11,15 +25,34 @@ if [ "$CURRENT_BRANCH" = "$BASELINE_BRANCH" ]; then
 	exit 1
 fi
 
-if ! git diff-index --quiet HEAD --; then
-	echo "Error: Uncommitted changes detected. Please commit or stash before running."
-	exit 1
-fi
+if [ "$SKIP_BASELINES" = true ]; then
+	# Verify baselines exist before skipping regeneration.
+	SNAPSHOT_DIR="$SCRIPT_DIR/specs/__snapshots__"
+	if [ ! -d "$SNAPSHOT_DIR" ] || [ -z "$(ls -A "$SNAPSHOT_DIR" 2>/dev/null)" ]; then
+		echo "Error: No baselines found. Run 'npm run test:visual' first to generate them."
+		exit 1
+	fi
 
-echo "Generating baselines from $BASELINE_BRANCH..."
-git checkout "$BASELINE_BRANCH"
-npx playwright test --config "$CONFIG" --update-snapshots
+	echo "Skipping baseline generation (reusing existing baselines)..."
+	echo "Ensuring Playwright browsers are installed..."
+	npx playwright install --with-deps chromium
 
-echo "Comparing against $CURRENT_BRANCH..."
-git checkout "$CURRENT_BRANCH"
-npx playwright test --config "$CONFIG"
+	echo "Comparing against existing baselines..."
+	npx playwright test --config "$CONFIG" || true
+else
+	if ! git diff-index --quiet HEAD --; then
+		echo "Error: Uncommitted changes detected. Please commit or stash before running."
+		exit 1
+	fi
+
+	echo "Ensuring Playwright browsers are installed..."
+	npx playwright install --with-deps chromium
+
+	echo "Generating baselines from $BASELINE_BRANCH..."
+	git checkout "$BASELINE_BRANCH"
+	npx playwright test --config "$CONFIG" --update-snapshots
+
+	echo "Comparing against $CURRENT_BRANCH..."
+	git checkout "$CURRENT_BRANCH"
+	npx playwright test --config "$CONFIG" || true
+fi

tests/visual-regression/playwright.config.js

@@ -20,13 +20,14 @@ const reporter = [
 	[
 		'html',
 		{
-			open: 'on-failure',
+			open: process.env.CI ? 'never' : 'always',
 			outputFolder: path.join(
 				process.env.WP_ARTIFACTS_PATH,
 				'visual-report'
 			),
 		},
 	],
+	[ './reporters/impact-summary.js' ],
 ];
 
 const config = defineConfig( {

tests/visual-regression/reporters/impact-summary.js

@@ -0,0 +1,61 @@
+/**
+ * Impact Summary Reporter
+ *
+ * A custom Playwright reporter that prints a visual impact summary
+ * at the end of the test run, showing which pages changed and which
+ * remained unchanged.
+ */
+class ImpactSummaryReporter {
+	constructor() {
+		this.changed = [];
+		this.unchanged = [];
+	}
+
+	onTestEnd( test, result ) {
+		const name = test.title;
+
+		if ( result.status === 'passed' ) {
+			this.unchanged.push( name );
+		} else {
+			this.changed.push( name );
+		}
+	}
+
+	onEnd() {
+		const total = this.changed.length + this.unchanged.length;
+
+		if ( total === 0 ) {
+			return;
+		}
+
+		const separator = '─'.repeat( 48 );
+
+		console.log( '' );
+		console.log( separator );
+		console.log( 'Visual Impact Summary' );
+		console.log( separator );
+		console.log(
+			`Changed:   ${ this.changed.length } of ${ total } pages`
+		);
+		console.log(
+			`Unchanged: ${ this.unchanged.length } of ${ total } pages`
+		);
+
+		if ( this.changed.length > 0 ) {
+			console.log( '' );
+			console.log( 'Changed:' );
+			console.log( `  ${ this.changed.join( ', ' ) }` );
+		}
+
+		if ( this.unchanged.length > 0 ) {
+			console.log( '' );
+			console.log( 'Unchanged:' );
+			console.log( `  ${ this.unchanged.join( ', ' ) }` );
+		}
+
+		console.log( separator );
+		console.log( '' );
+	}
+}
+
+module.exports = ImpactSummaryReporter;

tests/visual-regression/specs/visual-snapshots.test.js

@@ -231,4 +231,34 @@ test.describe( 'Admin Visual Snapshots', () => {
 			].map( ( selector ) => page.locator( selector ) ),
 		});
 	} );
+
+	test( 'Site Health', async ({ admin, page }) => {
+		await admin.visitAdminPage( '/site-health.php' );
+		await expect( page ).toHaveScreenshot( 'Site Health.png', {
+			mask: [
+				...elementsToHide,
+				'.site-health-issues .health-check-accordion',
+			].map( ( selector ) => page.locator( selector ) ),
+		});
+	} );
+
+	test( 'Updates', async ({ admin, page }) => {
+		await admin.visitAdminPage( '/update-core.php' );
+		await expect( page ).toHaveScreenshot( 'Updates.png', {
+			mask: [
+				...elementsToHide,
+				'form.upgrade',
+				'.last-checked',
+			].map( ( selector ) => page.locator( selector ) ),
+		});
+	} );
+} );
+
+test.describe( 'Unauthenticated Visual Snapshots', () => {
+	test.use( { storageState: {} } );
+
+	test( 'Login', async ({ page }) => {
+		await page.goto( '/wp-login.php' );
+		await expect( page ).toHaveScreenshot( 'Login.png' );
+	} );
 } );