fix: pass matcher threshold to core as saveAboveTolerance (#1113)

* fix: pass matcher threshold to core as saveAboveTolerance

When using visual matchers with a threshold (e.g., toMatchScreenSnapshot('tag', 0.9))
and alwaysSaveActualImage: false, images were still saved even when the comparison
passed within the threshold.

The matcher now passes the expected threshold to the core as saveAboveTolerance,
ensuring images are only saved when mismatch exceeds the user's acceptable threshold.

Fixes #1111

Co-authored-by: Cursor <[email protected]>

* chore: add extra UI test

* chore: update changelog

---------

Co-authored-by: Cursor <[email protected]>

Author: wswebcreation

.changeset/fix-matcher-threshold-save.md

@@ -0,0 +1,18 @@
+---
+"@wdio/visual-service": patch
+---
+
+# 🐛 Bugfixes
+
+## #1111 Pass matcher threshold to core as saveAboveTolerance
+
+When using visual matchers like `toMatchScreenSnapshot('tag', 0.9)` with `alwaysSaveActualImage: false`, the actual image was still being saved even when the comparison passed within the threshold.
+
+The root cause was that the matcher's expected threshold was not being passed to the core comparison logic. The core used `saveAboveTolerance` (defaulting to 0) to decide whether to save images, while the matcher used the user-provided threshold to determine pass/fail - these were disconnected.
+
+This fix ensures the matcher passes the expected threshold to the core as `saveAboveTolerance`, so images are only saved when the mismatch actually exceeds the user's acceptable threshold.
+
+
+# Committers: 1
+
+- Wim Selles ([@wswebcreation](https://github.com/wswebcreation))

packages/visual-service/src/matcher.ts

@@ -119,6 +119,25 @@ function parseMatcherParams (
      */
     options.returnAllCompareData = true
 
+    /**
+     * Pass the expected threshold to the core as `saveAboveTolerance` so it knows
+     * when to save actual images (only when mismatch exceeds the threshold).
+     * This ensures that when `alwaysSaveActualImage: false`, images are not saved
+     * if the comparison passes within the user's acceptable threshold.
+     * Only set if user hasn't explicitly set saveAboveTolerance.
+     * For numeric thresholds, use that value; otherwise default to 0 (same as comparison default).
+     * @see https://github.com/webdriverio/visual-testing/issues/1111
+     */
+    if (options.saveAboveTolerance === undefined) {
+        // Only set saveAboveTolerance for numeric thresholds (including undefined which defaults to 0)
+        // Asymmetric matchers can't be converted to a numeric tolerance
+        if (typeof expectedResult === 'number') {
+            options.saveAboveTolerance = expectedResult
+        } else if (expectedResult === undefined) {
+            options.saveAboveTolerance = DEFAULT_EXPECTED_RESULT
+        }
+    }
+
     return { expectedResult, options }
 }
 

packages/visual-service/tests/matcher.test.ts

@@ -145,4 +145,72 @@ describe('custom visual matcher', () => {
         expect(results.pass).toBe(false)
         expect(results.message()).toMatchSnapshot()
     })
+
+    describe('saveAboveTolerance threshold passthrough (#1111)', () => {
+        it('should pass numeric expectedResult as saveAboveTolerance to checkScreen', async () => {
+            await toMatchScreenSnapshot(browser, 'foo', 0.9, {})
+            expect(browser.checkScreen).toHaveBeenCalledWith('foo', {
+                returnAllCompareData: true,
+                saveAboveTolerance: 0.9
+            })
+        })
+
+        it('should pass numeric expectedResult as saveAboveTolerance to checkFullPageScreen', async () => {
+            await toMatchFullPageSnapshot(browser, 'foo', 0.5, {})
+            expect(browser.checkFullPageScreen).toHaveBeenCalledWith('foo', {
+                returnAllCompareData: true,
+                saveAboveTolerance: 0.5
+            })
+        })
+
+        it('should pass numeric expectedResult as saveAboveTolerance to checkElement', async () => {
+            await toMatchElementSnapshot(browser as any as WebdriverIO.Element, 'foo', 1.5, {})
+            expect(browser.checkElement).toHaveBeenCalledWith(browser, 'foo', {
+                returnAllCompareData: true,
+                saveAboveTolerance: 1.5
+            })
+        })
+
+        it('should pass numeric expectedResult as saveAboveTolerance to checkTabbablePage', async () => {
+            await toMatchTabbablePageSnapshot(browser, 'foo', 2.0, {})
+            expect(browser.checkTabbablePage).toHaveBeenCalledWith('foo', {
+                returnAllCompareData: true,
+                saveAboveTolerance: 2.0
+            })
+        })
+
+        it('should use default saveAboveTolerance of 0 when no threshold is provided', async () => {
+            await toMatchScreenSnapshot(browser, 'foo')
+            expect(browser.checkScreen).toHaveBeenCalledWith('foo', {
+                returnAllCompareData: true,
+                saveAboveTolerance: 0
+            })
+        })
+
+        it('should not override user-provided saveAboveTolerance', async () => {
+            await toMatchScreenSnapshot(browser, 'foo', 0.9, { saveAboveTolerance: 0.1 })
+            expect(browser.checkScreen).toHaveBeenCalledWith('foo', {
+                returnAllCompareData: true,
+                saveAboveTolerance: 0.1  // User's explicit value is preserved
+            })
+        })
+
+        it('should not set saveAboveTolerance for asymmetric matchers', async () => {
+            await toMatchScreenSnapshot(browser, 'foo', expect.any(Number))
+            expect(browser.checkScreen).toHaveBeenCalledWith('foo', {
+                returnAllCompareData: true
+                // No saveAboveTolerance - can't convert asymmetric matcher to number
+            })
+        })
+
+        it('should set saveAboveTolerance to 0 when options object is passed without threshold', async () => {
+            // When only options are passed (no expectedResult), threshold defaults to 0
+            await toMatchScreenSnapshot(browser, 'foo', { hideScrollBars: true })
+            expect(browser.checkScreen).toHaveBeenCalledWith('foo', {
+                hideScrollBars: true,
+                returnAllCompareData: true,
+                saveAboveTolerance: 0
+            })
+        })
+    })
 })

tests/specs/matcher.spec.ts

@@ -1,4 +1,5 @@
-/// <reference types="@wdio/visual-service" />
+import { readdirSync } from 'node:fs'
+import { join } from 'node:path'
 import { browser, expect } from '@wdio/globals'
 
 describe('@wdio/visual-service matcher', () => {
@@ -36,4 +37,36 @@ describe('@wdio/visual-service matcher', () => {
             ]
         })
     })
+
+    it(`should NOT save actual image when mismatch is within threshold (#1111) for '${browserName}'`, async () => {
+        const tag = 'threshold-test-1111'
+        const actualFolder = join(process.cwd(), '.tmp/actual')
+        const subtitle = await $('.hero__subtitle')
+        const getActualImageCount = () => {
+            try {
+                return readdirSync(actualFolder).filter(f => f.includes(tag)).length
+            } catch {
+                return 0
+            }
+        }
+
+        // 1. Save the original subtitle as baseline
+        await browser.saveElement(subtitle, tag)
+
+        // 2. Manipulate the subtitle to create a small text difference
+        await browser.execute(
+            'arguments[0].innerHTML = "Test Demo Page";',
+            subtitle
+        )
+
+        const beforeCount = getActualImageCount()
+
+        // 3. Run the matcher with a threshold (90%) higher than the expected mismatch
+        await expect(subtitle).toMatchElementSnapshot(tag, 90)
+
+        const afterCount = getActualImageCount()
+
+        // 4. With the fix: no new actual image should be saved when mismatch is within threshold
+        expect(afterCount).toBe(beforeCount)
+    })
 })