Skip to content

Commit b741ba1

Browse files
josephfuscojosephfusco
committed
Tests: Add visual regression testing for WordPress admin pages.
1 parent dbf8614 commit b741ba1

8 files changed

Lines changed: 549 additions & 162 deletions

File tree

‎.github/workflows/visual-regression-tests.yml‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
name: Visual Regression Tests
2+
3+
on:
4+
workflow_dispatch:
5+
6+
concurrency:
7+
group: ${{ github.workflow }}-${{ github.ref }}
8+
cancel-in-progress: true
9+
10+
# Disable permissions for all available scopes by default.
11+
# Any needed permissions should be configured at the job level.
12+
permissions: {}
13+
14+
env:
15+
LOCAL_DIR: build
16+
PUPPETEER_SKIP_DOWNLOAD: ${{ true }}
17+
18+
jobs:
19+
visual-regression:
20+
name: Visual Regression
21+
runs-on: ubuntu-24.04
22+
timeout-minutes: 30
23+
permissions:
24+
contents: read
25+
26+
steps:
27+
- name: Configure environment variables
28+
run: |
29+
echo "PHP_FPM_UID=$(id -u)" >> "$GITHUB_ENV"
30+
echo "PHP_FPM_GID=$(id -g)" >> "$GITHUB_ENV"
31+
32+
- name: Checkout repository
33+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
34+
with:
35+
show-progress: ${{ runner.debug == '1' && 'true' || 'false' }}
36+
fetch-depth: 0
37+
persist-credentials: false
38+
39+
- name: Determine baseline commit
40+
id: baseline
41+
run: |
42+
echo "current_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
43+
echo "base_sha=$(git merge-base HEAD origin/trunk)" >> "$GITHUB_OUTPUT"
44+
45+
- name: Set up Node.js
46+
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
47+
with:
48+
node-version-file: '.nvmrc'
49+
cache: npm
50+
51+
- name: Install npm dependencies
52+
run: npm ci
53+
54+
- name: Install Playwright browsers
55+
run: npx playwright install --with-deps chromium
56+
57+
- name: Checkout merge base
58+
run: git checkout ${{ steps.baseline.outputs.base_sha }}
59+
60+
- name: Build WordPress (baseline)
61+
run: npm run build
62+
63+
- name: Start Docker environment
64+
run: npm run env:start
65+
66+
- name: Install WordPress
67+
run: npm run env:install
68+
69+
- name: Generate baseline snapshots
70+
run: npx wp-scripts test-playwright --config tests/visual-regression/playwright.config.js --update-snapshots
71+
72+
- name: Checkout current commit
73+
run: git checkout ${{ steps.baseline.outputs.current_sha }}
74+
75+
- name: Rebuild WordPress
76+
run: npm run build
77+
78+
- name: Restart Docker environment
79+
run: npm run env:stop && npm run env:start
80+
81+
- name: Reinstall WordPress
82+
run: npm run env:install
83+
84+
- name: Run visual comparison
85+
id: visual-tests
86+
continue-on-error: true
87+
run: npx wp-scripts test-playwright --config tests/visual-regression/playwright.config.js
88+
89+
- name: Upload report artifacts
90+
if: always()
91+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
92+
with:
93+
name: visual-regression-report-${{ github.run_id }}
94+
path: |
95+
artifacts/visual-report
96+
artifacts/visual-results.json
97+
artifacts/test-results
98+
if-no-files-found: ignore
99+
include-hidden-files: true
100+
101+
- name: Generate summary
102+
if: always()
103+
env:
104+
WORKFLOW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
105+
run: node tests/visual-regression/generate-summary.js "${RUNNER_TEMP}/visual-summary.md"
106+
107+
- name: Write step summary
108+
if: always()
109+
run: cat "${RUNNER_TEMP}/visual-summary.md" >> "$GITHUB_STEP_SUMMARY"
110+
111+
- name: Fail if visual changes detected
112+
if: steps.visual-tests.outcome == 'failure'
113+
run: |
114+
echo "Visual regressions were detected. See the workflow summary and artifacts for details."
115+
exit 1

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@
129129
"test:php": "node ./tools/local-env/scripts/docker.js run --rm php ./vendor/bin/phpunit",
130130
"test:coverage": "npm run test:php -- --coverage-html ./coverage/html/ --coverage-php ./coverage/php/report.php --coverage-text=./coverage/text/report.txt",
131131
"test:e2e": "wp-scripts test-playwright --config tests/e2e/playwright.config.js",
132-
"test:visual": "wp-scripts test-playwright --config tests/visual-regression/playwright.config.js",
132+
"test:visual": "bash tests/visual-regression/compare-branches.sh",
133133
"typecheck:php": "node ./tools/local-env/scripts/docker.js run --rm php composer phpstan",
134134
"gutenberg:checkout": "node tools/gutenberg/checkout-gutenberg.js",
135135
"gutenberg:build": "node tools/gutenberg/build-gutenberg.js",

‎tests/visual-regression/README.md‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,34 @@
22

33
These tests make use of Playwright, with a setup very similar to that of the e2e tests.
44

5+
## Prerequisites
6+
7+
- Node.js >= 20.10.0
8+
- A running WordPress environment (`npm run env:start`)
9+
- Playwright browsers installed (`npx playwright install chromium`)
10+
511
## How to Run the Tests Locally
612

7-
1. Check out trunk.
8-
2. Run `npm run test:visual` to generate some base snapshots.
9-
3. Check out the feature branch to be tested.
10-
4. Run `npm run test:visual` again. If any tests fail, the diff images can be found in `artifacts/`
13+
### Full branch comparison
14+
15+
From a feature branch with a clean working tree, run:
16+
17+
```bash
18+
npm run test:visual
19+
```
20+
21+
This will automatically:
22+
1. Check out trunk to generate baseline snapshots.
23+
2. Build WordPress and start the environment for the baseline.
24+
3. Return to your feature branch.
25+
4. Rebuild and compare the current branch against those baselines.
26+
5. Generate and open the HTML report with side-by-side visual diffs.
27+
28+
### Direct Playwright execution
29+
30+
To run the tests directly against existing snapshots (useful for quick iterations or updating baselines manually):
1131

32+
```bash
33+
npx wp-scripts test-playwright --config tests/visual-regression/playwright.config.js
34+
npx wp-scripts test-playwright --config tests/visual-regression/playwright.config.js --update-snapshots
35+
```

‎tests/visual-regression/compare-branches.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/bash
2+
set -e
3+
4+
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
5+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
6+
CONFIG="$SCRIPT_DIR/playwright.config.js"
7+
8+
# Always restore the original branch on exit, even on failure.
9+
trap 'git checkout "$CURRENT_BRANCH" 2>/dev/null' EXIT
10+
11+
if [ "$CURRENT_BRANCH" = "trunk" ]; then
12+
echo "Error: Already on trunk. Checkout a feature branch first."
13+
exit 1
14+
fi
15+
16+
if ! git diff-index --quiet HEAD --; then
17+
echo "Error: Uncommitted changes detected. Please commit or stash before running."
18+
exit 1
19+
fi
20+
21+
echo "Ensuring Playwright browsers are installed..."
22+
npx playwright install chromium
23+
24+
echo "Generating baselines from trunk..."
25+
git checkout trunk
26+
npm run build
27+
npm run env:start
28+
npm run env:install
29+
npx wp-scripts test-playwright --config "$CONFIG" --update-snapshots
30+
31+
echo "Comparing against $CURRENT_BRANCH..."
32+
git checkout "$CURRENT_BRANCH"
33+
npm run build
34+
npm run env:stop && npm run env:start
35+
npm run env:install
36+
npx wp-scripts test-playwright --config "$CONFIG" || true

‎tests/visual-regression/config/screenshot.css‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Global stylesheet applied during screenshot capture.
3+
*
4+
* Hides volatile elements that change between environments or runs,
5+
* preventing false positives in visual regression comparisons.
6+
* Applied via Playwright's stylePath config option.
7+
*
8+
* See: https://playwright.dev/docs/test-snapshots#stylepath
9+
*/
10+
11+
/* WordPress version/update nag in the admin footer. */
12+
#footer-upgrade {
13+
visibility: hidden !important;
14+
}
15+
16+
/* Admin bar user-specific content (Howdy, gravatar). */
17+
#wp-admin-bar-root-default {
18+
visibility: hidden !important;
19+
}
20+
21+
/* Gutenberg plugin menu item — not present in all environments. */
22+
#toplevel_page_gutenberg {
23+
visibility: hidden !important;
24+
}
25+
26+
/* Gravatar images — external service, different per environment. */
27+
.avatar {
28+
visibility: hidden !important;
29+
}
30+
31+
/* Date columns in list tables — relative timestamps shift between runs. */
32+
.column-date {
33+
visibility: hidden !important;
34+
}
35+
36+
/* Dashboard widgets with dynamic counts and activity. */
37+
#dashboard_right_now .inside,
38+
#dashboard_activity .inside {
39+
visibility: hidden !important;
40+
}
41+
42+
/* Update-related timestamps. */
43+
.update-last-checked {
44+
visibility: hidden !important;
45+
}
46+
47+
/* Admin notices — various nags (PHP deprecation, updates, etc.). */
48+
.notice,
49+
.update-nag,
50+
.updated,
51+
.error:not(#error),
52+
#message {
53+
visibility: hidden !important;
54+
}
55+
56+
/* General Settings — live date/time preview changes on every run. */
57+
#local-time,
58+
.example {
59+
visibility: hidden !important;
60+
}
61+
62+
/* Users list table — post counts vary based on test data. */
63+
.column-posts {
64+
visibility: hidden !important;
65+
}

‎tests/visual-regression/generate-summary.js‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
#!/usr/bin/env node
2+
3+
/**
4+
* External dependencies.
5+
*/
6+
const { readFileSync, writeFileSync, existsSync } = require( 'node:fs' );
7+
const { join } = require( 'node:path' );
8+
9+
process.env.WP_ARTIFACTS_PATH ??= join( process.cwd(), 'artifacts' );
10+
11+
const args = process.argv.slice( 2 );
12+
const summaryFile = args[ 0 ];
13+
const workflowRunUrl = process.env.WORKFLOW_RUN_URL || '';
14+
15+
const resultsFile = join(
16+
process.env.WP_ARTIFACTS_PATH,
17+
'visual-results.json'
18+
);
19+
20+
if ( ! existsSync( resultsFile ) ) {
21+
const errorMarkdown =
22+
'<!-- visual-regression-report -->\n## Visual Regression Report\n\n' +
23+
'**Error:** Could not find visual regression results file.\n';
24+
console.log( errorMarkdown );
25+
if ( summaryFile ) {
26+
writeFileSync( summaryFile, errorMarkdown );
27+
}
28+
process.exit( 1 );
29+
}
30+
31+
const results = JSON.parse( readFileSync( resultsFile, 'utf8' ) );
32+
33+
/**
34+
* Recursively flattens all suites to collect specs.
35+
*
36+
* @param {Object} suite A Playwright JSON reporter suite object.
37+
* @return {Array} Flat array of spec objects.
38+
*/
39+
function flattenSpecs( suite ) {
40+
const specs = [ ...( suite.specs || [] ) ];
41+
for ( const child of suite.suites || [] ) {
42+
specs.push( ...flattenSpecs( child ) );
43+
}
44+
return specs;
45+
}
46+
47+
const allSpecs = [];
48+
for ( const suite of results.suites || [] ) {
49+
allSpecs.push( ...flattenSpecs( suite ) );
50+
}
51+
52+
const changed = [];
53+
const unchanged = [];
54+
const errored = [];
55+
56+
for ( const spec of allSpecs ) {
57+
const status = spec.ok ? 'passed' : 'failed';
58+
const hasError = spec.tests?.some( ( t ) =>
59+
[ 'timedOut', 'interrupted' ].includes( t.status )
60+
);
61+
62+
if ( hasError ) {
63+
errored.push( spec.title );
64+
} else if ( status === 'failed' ) {
65+
changed.push( spec.title );
66+
} else {
67+
unchanged.push( spec.title );
68+
}
69+
}
70+
71+
const total = changed.length + unchanged.length + errored.length;
72+
73+
let markdown = '<!-- visual-regression-report -->\n## Visual Regression Report\n\n';
74+
75+
if ( changed.length === 0 && errored.length === 0 ) {
76+
markdown += `**No visual changes detected** across all ${ total } pages.\n`;
77+
} else {
78+
const runLink = workflowRunUrl
79+
? ` | [View workflow run](${ workflowRunUrl })`
80+
: '';
81+
markdown += `**${ changed.length } of ${ total } pages changed**${ runLink }\n`;
82+
}
83+
84+
if ( changed.length > 0 ) {
85+
markdown += `\n<details>\n<summary>Changed pages (${ changed.length })</summary>\n\n`;
86+
for ( const name of changed ) {
87+
markdown += `- ${ name }\n`;
88+
}
89+
markdown += '\n</details>\n';
90+
}
91+
92+
if ( unchanged.length > 0 ) {
93+
markdown += `\n<details>\n<summary>Unchanged pages (${ unchanged.length })</summary>\n\n`;
94+
markdown += unchanged.join( ', ' ) + '\n';
95+
markdown += '\n</details>\n';
96+
}
97+
98+
if ( errored.length > 0 ) {
99+
markdown += `\n<details>\n<summary>Errored pages (${ errored.length })</summary>\n\n`;
100+
for ( const name of errored ) {
101+
markdown += `- ${ name }\n`;
102+
}
103+
markdown += '\n</details>\n';
104+
}
105+
106+
markdown +=
107+
'\n---\nDownload the **visual-regression-report** artifact from the workflow run for an HTML report with side-by-side image diffs.\n';
108+
109+
console.log( markdown );
110+
111+
if ( summaryFile ) {
112+
writeFileSync( summaryFile, markdown );
113+
}

0 commit comments

Comments
 (0)