fix: fix 1115

Author: wswebcreation

.changeset/fix-always-save-actual-image.md

@@ -0,0 +1,18 @@
+---
+"@wdio/image-comparison-core": patch
+"@wdio/visual-service": patch
+---
+
+## #1115 Respect `alwaysSaveActualImage: false` for `checkScreen` methods
+
+When using visual matchers like `toMatchScreenSnapshot('tag', 0.9)` with `alwaysSaveActualImage: false`, the actual image was still being saved even when the comparison passed within the threshold.
+
+The root cause was that the matcher's expected threshold was not being passed to the core comparison logic. The core used `saveAboveTolerance` (defaulting to 0) to decide whether to save images, while the matcher used the user-provided threshold to determine pass/fail - these were disconnected.
+
+This fix ensures:
+- When `alwaysSaveActualImage: false` and `saveAboveTolerance` is not explicitly set, actual images are never saved (respecting the literal meaning of the option)
+- When `saveAboveTolerance` is explicitly set (like matchers do internally), actual images are saved only when the mismatch exceeds that threshold
+
+# Committers: 1
+
+-   Wim Selles ([@wswebcreation](https://github.com/wswebcreation))

package.json

@@ -19,7 +19,7 @@
     "test:unit": "vitest --coverage --run",
     "test:unit:ui": "vitest --coverage --ui",
     "test:unit:watch": "vitest --coverage --watch",
-    "test.local.init": "rimraf localBaseline && wdio ./tests/configs/wdio.local.init.conf.ts",
+    "test.local.init": "rimraf localBaseline && SAVE_ACTUAL=true wdio ./tests/configs/wdio.local.init.conf.ts",
     "test.local.desktop": "wdio tests/configs/wdio.local.desktop.conf.ts",
     "test.local.emus.app": "wdio tests/configs/wdio.local.android.emus.app.conf.ts",
     "test.local.emus.web": "wdio tests/configs/wdio.local.android.emus.web.conf.ts",

packages/image-comparison-core/src/__snapshots__/base.test.ts.snap

@@ -21,7 +21,6 @@ exports[`BaseClass > initializes default options correctly 1`] = `
     "ignoreNothing": false,
     "rawMisMatchPercentage": false,
     "returnAllCompareData": false,
-    "saveAboveTolerance": 0,
     "scaleImagesToSameSize": false,
   },
   "disableBlinkingCursor": false,

packages/image-comparison-core/src/helpers/__snapshots__/options.test.ts.snap

@@ -97,7 +97,6 @@ exports[`options > defaultOptions > should return the default options when no op
     "ignoreNothing": false,
     "rawMisMatchPercentage": false,
     "returnAllCompareData": false,
-    "saveAboveTolerance": 0,
     "scaleImagesToSameSize": false,
   },
   "disableBlinkingCursor": false,

packages/image-comparison-core/src/helpers/constants.ts

@@ -16,7 +16,6 @@ export const DEFAULT_COMPARE_OPTIONS = {
     ignoreNothing: false,
     rawMisMatchPercentage: false,
     returnAllCompareData: false,
-    saveAboveTolerance: 0,
     scaleImagesToSameSize: false,
 }
 export const DEFAULT_FORMAT_STRING = '{tag}-{browserName}-{width}x{height}-dpr-{dpr}'

packages/image-comparison-core/src/methods/images.executeImageCompare.test.ts

@@ -693,14 +693,60 @@ describe('executeImageCompare', () => {
         expect(fsPromises.writeFile).toHaveBeenCalledWith('/mock/actual/test.png', Buffer.from(base64Image, 'base64'))
     })
 
-    it('should save base64 actual on diff when not always saving', async () => {
+    it('should NOT save base64 actual on diff when alwaysSaveActualImage is false and saveAboveTolerance is not set (#1115)', async () => {
+        // When alwaysSaveActualImage: false and saveAboveTolerance is not explicitly set,
+        // actual images should never be saved - respecting the literal meaning of the option
         const base64Image = Buffer.from('base64-image').toString('base64')
         const optionsWithDiff = {
             ...mockOptions,
             folderOptions: {
                 ...mockOptions.folderOptions,
                 alwaysSaveActualImage: false,
-            }
+            },
+            compareOptions: {
+                ...mockOptions.compareOptions,
+                wic: {
+                    ...mockOptions.compareOptions.wic,
+                    saveAboveTolerance: undefined,
+                },
+            },
+        }
+        vi.mocked(compareImages.default).mockResolvedValue({
+            rawMisMatchPercentage: 0.5,
+            misMatchPercentage: 0.5,
+            getBuffer: vi.fn().mockResolvedValue(Buffer.from('diff-image-data')),
+            diffBounds: { left: 0, top: 0, right: 0, bottom: 0 },
+            analysisTime: 10,
+            diffPixels: []
+        })
+
+        await executeImageCompare({
+            isViewPortScreenshot: true,
+            isNativeContext: false,
+            options: optionsWithDiff,
+            testContext: mockTestContext,
+            actualBase64Image: base64Image,
+        })
+
+        expect(fsPromises.writeFile).not.toHaveBeenCalled()
+    })
+
+    it('should save base64 actual on diff when saveAboveTolerance is explicitly set to 0', async () => {
+        // When saveAboveTolerance is explicitly set (even to 0), save actual images when diff exceeds it
+        const base64Image = Buffer.from('base64-image').toString('base64')
+        const optionsWithDiff = {
+            ...mockOptions,
+            folderOptions: {
+                ...mockOptions.folderOptions,
+                alwaysSaveActualImage: false,
+            },
+            compareOptions: {
+                ...mockOptions.compareOptions,
+                wic: {
+                    ...mockOptions.compareOptions.wic,
+                    saveAboveTolerance: 0,
+                },
+            },
         }
         vi.mocked(compareImages.default).mockResolvedValue({
             rawMisMatchPercentage: 0.5,

packages/image-comparison-core/src/methods/images.ts

@@ -450,10 +450,15 @@ export async function executeImageCompare(
     )
 
     // 6a. Save actual image on failure if alwaysSaveActualImage is false
-    const saveAboveTolerance = imageCompareOptions.saveAboveTolerance ?? 0
-    const hasFailure = rawMisMatchPercentage > saveAboveTolerance
-    if (useBase64Image && hasFailure && actualBase64Image) {
-        // Save the actual image only when comparison fails
+    // Only save when saveAboveTolerance is explicitly set (e.g., by matchers).
+    // When using checkScreen directly without saveAboveTolerance, respect
+    // alwaysSaveActualImage: false literally and don't save actual images.
+    // @see https://github.com/webdriverio/visual-testing/issues/1115
+    const saveAboveTolerance = imageCompareOptions.saveAboveTolerance
+    const shouldSaveOnFailure = saveAboveTolerance !== undefined
+    const hasFailure = rawMisMatchPercentage > (saveAboveTolerance ?? 0)
+    if (useBase64Image && shouldSaveOnFailure && hasFailure && actualBase64Image) {
+        // Save the actual image only when comparison fails and threshold was explicitly set
         await saveBase64Image(actualBase64Image, actualFilePath)
     }
 

tests/configs/wdio.local.desktop.conf.ts

@@ -72,7 +72,7 @@ export const config: WebdriverIO.Config = {
                 createJsonReportFiles: true,
                 clearRuntimeFolder: true,
                 enableLayoutTesting: true,
-                alwaysSaveActualImage: false,
+                alwaysSaveActualImage: process.env.SAVE_ACTUAL === 'true',
             },
         ]
     ],

tests/specs/desktop.spec.ts

@@ -1,4 +1,6 @@
+import type { ImageCompareResult } from '@wdio/image-comparison-core'
 import { browser, expect } from '@wdio/globals'
+import { fileExists } from '../helpers/fileExists.ts'
 
 describe('@wdio/visual-service desktop', () => {
     // @TODO
@@ -39,4 +41,13 @@ describe('@wdio/visual-service desktop', () => {
             ],
         })
     })
+
+    it(`should not store an actual image for '${browserName}' when the diff is below the threshold`, async function() {
+        const result = await browser.checkScreen('examplePageFail', {
+            returnAllCompareData: true,
+        }) as ImageCompareResult
+
+        expect(result.misMatchPercentage).toBeLessThanOrEqual(50)
+        expect(fileExists(result.folders.actual)).toBe(false)
+    })
 })