Skip to content

Commit fff0a92

Browse files
josephfuscojosephfusco
committed
Tests: Add visual regression testing for WordPress admin pages.
1 parent dbf8614 commit fff0a92

7 files changed

Lines changed: 408 additions & 162 deletions

File tree

‎.github/workflows/visual-regression-tests.yml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
name: Visual Regression Tests
2+
3+
on:
4+
workflow_dispatch:
5+
6+
concurrency:
7+
group: ${{ github.workflow }}-${{ github.ref }}
8+
cancel-in-progress: true
9+
10+
# Disable permissions for all available scopes by default.
11+
# Any needed permissions should be configured at the job level.
12+
permissions: {}
13+
14+
env:
15+
LOCAL_DIR: build
16+
PUPPETEER_SKIP_DOWNLOAD: ${{ true }}
17+
18+
jobs:
19+
visual-regression:
20+
name: Visual Regression
21+
runs-on: ubuntu-24.04
22+
timeout-minutes: 30
23+
permissions:
24+
contents: read
25+
26+
steps:
27+
- name: Configure environment variables
28+
run: |
29+
echo "PHP_FPM_UID=$(id -u)" >> "$GITHUB_ENV"
30+
echo "PHP_FPM_GID=$(id -g)" >> "$GITHUB_ENV"
31+
32+
- name: Checkout repository
33+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
34+
with:
35+
show-progress: ${{ runner.debug == '1' && 'true' || 'false' }}
36+
fetch-depth: 0
37+
persist-credentials: false
38+
39+
- name: Determine baseline commit
40+
id: baseline
41+
run: |
42+
echo "current_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
43+
echo "base_sha=$(git merge-base HEAD origin/trunk)" >> "$GITHUB_OUTPUT"
44+
45+
- name: Set up Node.js
46+
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
47+
with:
48+
node-version-file: '.nvmrc'
49+
cache: npm
50+
51+
- name: Install npm dependencies
52+
run: npm ci
53+
54+
- name: Install Playwright browsers
55+
run: npx playwright install --with-deps chromium
56+
57+
- name: Checkout merge base
58+
run: git checkout ${{ steps.baseline.outputs.base_sha }}
59+
60+
- name: Build WordPress (baseline)
61+
run: npm run build
62+
63+
- name: Start Docker environment
64+
run: npm run env:start
65+
66+
- name: Install WordPress
67+
run: npm run env:install
68+
69+
- name: Generate baseline snapshots
70+
run: npx wp-scripts test-playwright --config tests/visual-regression/playwright.config.js --update-snapshots
71+
72+
- name: Checkout current commit
73+
run: git checkout ${{ steps.baseline.outputs.current_sha }}
74+
75+
- name: Rebuild WordPress
76+
run: npm run build
77+
78+
- name: Restart Docker environment
79+
run: npm run env:stop && npm run env:start
80+
81+
- name: Reinstall WordPress
82+
run: npm run env:install
83+
84+
- name: Run visual comparison
85+
run: npx wp-scripts test-playwright --config tests/visual-regression/playwright.config.js
86+
87+
- name: Upload report artifacts
88+
if: always()
89+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
90+
with:
91+
name: visual-regression-report-${{ github.run_id }}
92+
path: |
93+
artifacts/visual-report
94+
artifacts/test-results
95+
if-no-files-found: ignore
96+
include-hidden-files: true
97+

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@
129129
"test:php": "node ./tools/local-env/scripts/docker.js run --rm php ./vendor/bin/phpunit",
130130
"test:coverage": "npm run test:php -- --coverage-html ./coverage/html/ --coverage-php ./coverage/php/report.php --coverage-text=./coverage/text/report.txt",
131131
"test:e2e": "wp-scripts test-playwright --config tests/e2e/playwright.config.js",
132-
"test:visual": "wp-scripts test-playwright --config tests/visual-regression/playwright.config.js",
132+
"test:visual": "bash tests/visual-regression/compare-branches.sh",
133133
"typecheck:php": "node ./tools/local-env/scripts/docker.js run --rm php composer phpstan",
134134
"gutenberg:checkout": "node tools/gutenberg/checkout-gutenberg.js",
135135
"gutenberg:build": "node tools/gutenberg/build-gutenberg.js",

‎tests/visual-regression/README.md‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,34 @@
22

33
These tests make use of Playwright, with a setup very similar to that of the e2e tests.
44

5+
## Prerequisites
6+
7+
- Node.js >= 20.10.0
8+
- A running WordPress environment (`npm run env:start`)
9+
- Playwright browsers installed (`npx playwright install chromium`)
10+
511
## How to Run the Tests Locally
612

7-
1. Check out trunk.
8-
2. Run `npm run test:visual` to generate some base snapshots.
9-
3. Check out the feature branch to be tested.
10-
4. Run `npm run test:visual` again. If any tests fail, the diff images can be found in `artifacts/`
13+
### Full branch comparison
14+
15+
From a feature branch with a clean working tree, run:
16+
17+
```bash
18+
npm run test:visual
19+
```
20+
21+
This will automatically:
22+
1. Check out trunk to generate baseline snapshots.
23+
2. Build WordPress and start the environment for the baseline.
24+
3. Return to your feature branch.
25+
4. Rebuild and compare the current branch against those baselines.
26+
5. Generate and open the HTML report with side-by-side visual diffs.
27+
28+
### Direct Playwright execution
29+
30+
To run the tests directly against existing snapshots (useful for quick iterations or updating baselines manually):
1131

32+
```bash
33+
npx wp-scripts test-playwright --config tests/visual-regression/playwright.config.js
34+
npx wp-scripts test-playwright --config tests/visual-regression/playwright.config.js --update-snapshots
35+
```

‎tests/visual-regression/compare-branches.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/bash
2+
set -e
3+
4+
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
5+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
6+
CONFIG="$SCRIPT_DIR/playwright.config.js"
7+
8+
# Always restore the original branch on exit, even on failure.
9+
trap 'git checkout "$CURRENT_BRANCH" 2>/dev/null' EXIT
10+
11+
if [ "$CURRENT_BRANCH" = "trunk" ]; then
12+
echo "Error: Already on trunk. Checkout a feature branch first."
13+
exit 1
14+
fi
15+
16+
if ! git diff-index --quiet HEAD --; then
17+
echo "Error: Uncommitted changes detected. Please commit or stash before running."
18+
exit 1
19+
fi
20+
21+
echo "Ensuring Playwright browsers are installed..."
22+
npx playwright install chromium
23+
24+
echo "Generating baselines from trunk..."
25+
git checkout trunk
26+
npm run build
27+
npm run env:start
28+
npm run env:install
29+
npx wp-scripts test-playwright --config "$CONFIG" --update-snapshots
30+
31+
echo "Comparing against $CURRENT_BRANCH..."
32+
git checkout "$CURRENT_BRANCH"
33+
npm run build
34+
npm run env:stop && npm run env:start
35+
npm run env:install
36+
npx wp-scripts test-playwright --config "$CONFIG" || true

‎tests/visual-regression/config/screenshot.css‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Global stylesheet applied during screenshot capture.
3+
*
4+
* Hides volatile elements that change between environments or runs,
5+
* preventing false positives in visual regression comparisons.
6+
* Applied via Playwright's stylePath config option.
7+
*
8+
* See: https://playwright.dev/docs/test-snapshots#stylepath
9+
*/
10+
11+
/* WordPress version/update nag in the admin footer. */
12+
#footer-upgrade {
13+
visibility: hidden !important;
14+
}
15+
16+
/* Admin bar user-specific content (Howdy, gravatar). */
17+
#wp-admin-bar-root-default {
18+
visibility: hidden !important;
19+
}
20+
21+
/* Gutenberg plugin menu item — not present in all environments. */
22+
#toplevel_page_gutenberg {
23+
visibility: hidden !important;
24+
}
25+
26+
/* Gravatar images — external service, different per environment. */
27+
.avatar {
28+
visibility: hidden !important;
29+
}
30+
31+
/* Date columns in list tables — relative timestamps shift between runs. */
32+
.column-date {
33+
visibility: hidden !important;
34+
}
35+
36+
/* Dashboard widgets with dynamic counts and activity. */
37+
#dashboard_right_now .inside,
38+
#dashboard_activity .inside {
39+
visibility: hidden !important;
40+
}
41+
42+
/* Update-related timestamps. */
43+
.update-last-checked {
44+
visibility: hidden !important;
45+
}
46+
47+
/* Admin notices — various nags (PHP deprecation, updates, etc.). */
48+
.notice,
49+
.update-nag,
50+
.updated,
51+
.error:not(#error),
52+
#message {
53+
visibility: hidden !important;
54+
}
55+
56+
/* General Settings — live date/time preview changes on every run. */
57+
#local-time,
58+
.example {
59+
visibility: hidden !important;
60+
}
61+
62+
/* Users list table — post counts vary based on test data. */
63+
.column-posts {
64+
visibility: hidden !important;
65+
}

‎tests/visual-regression/playwright.config.js‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,39 @@ process.env.STORAGE_STATE_PATH ??= path.join(
1515
'storage-states/admin.json'
1616
);
1717

18+
const reporter = [
19+
[ 'list' ],
20+
...( process.env.CI ? [ [ 'github' ] ] : [] ),
21+
[
22+
'html',
23+
{
24+
open: process.env.CI ? 'never' : 'always',
25+
outputFolder: path.join(
26+
process.env.WP_ARTIFACTS_PATH,
27+
'visual-report'
28+
),
29+
},
30+
],
31+
];
32+
1833
const config = defineConfig( {
1934
...baseConfig,
20-
globalSetup: undefined,
35+
fullyParallel: true,
36+
retries: 0,
37+
workers: process.env.CI ? 1 : undefined,
38+
reporter,
39+
use: {
40+
...baseConfig.use,
41+
viewport: { width: 1280, height: 720 },
42+
},
43+
expect: {
44+
toHaveScreenshot: {
45+
animations: 'disabled',
46+
fullPage: true,
47+
maxDiffPixelRatio: 0.01,
48+
stylePath: path.join( __dirname, 'config', 'screenshot.css' ),
49+
},
50+
},
2151
webServer: {
2252
...baseConfig.webServer,
2353
command: 'npm run env:start',

0 commit comments

Comments
 (0)