feat: add soft assertions feature

Author: JustasMonkev

docs/API.md

@@ -2,6 +2,85 @@
 
 When you're writing tests, you often need to check that values meet certain conditions. `expect` gives you access to a number of "matchers" that let you validate different things on the `browser`, an `element` or `mock` object.
 
+## Soft Assertions
+
+Soft assertions allow you to continue test execution even when an assertion fails. This is useful when you want to check multiple conditions in a test and collect all failures rather than stopping at the first failure. Failures are collected and reported at the end of the test.
+
+### Usage
+
+```js
+// Mocha example
+it('product page smoke', async () => {
+  // These won't throw immediately if they fail
+  await expect.soft(await $('h1').getText()).toEqual('Basketball Shoes');
+  await expect.soft(await $('#price').getText()).toMatch(/€\d+/);
+
+  // Regular assertions still throw immediately
+  await expect(await $('.add-to-cart').isClickable()).toBe(true);
+});
+
+// At the end of the test, all soft assertion failures
+// will be reported together with their details
+```
+
+### Soft Assertion API
+
+#### expect.soft()
+
+Creates a soft assertion that collects failures instead of immediately throwing errors.
+
+```js
+await expect.soft(actual).toBeDisplayed();
+await expect.soft(actual).not.toHaveText('Wrong text');
+```
+
+#### expect.getSoftFailures()
+
+Get all collected soft assertion failures for the current test.
+
+```js
+const failures = expect.getSoftFailures();
+console.log(`There are ${failures.length} soft assertion failures`);
+```
+
+#### expect.assertSoftFailures()
+
+Manually assert all collected soft failures. This will throw an aggregated error if any soft assertions have failed.
+
+```js
+// Manually throw if any soft assertions have failed
+expect.assertSoftFailures();
+```
+
+#### expect.clearSoftFailures()
+
+Clear all collected soft assertion failures for the current test.
+
+```js
+// Clear all collected failures
+expect.clearSoftFailures();
+```
+
+### Integration with Test Frameworks
+
+The soft assertions feature integrates with WebdriverIO's test runner automatically. It will report all soft assertion failures at the end of each test (Mocha/Jasmine) or step (Cucumber).
+
+To use with WebdriverIO, add the SoftAssertionService to your services list:
+
+```js
+// wdio.conf.js
+import { SoftAssertionService } from 'expect-webdriverio'
+
+export const config = {
+  // ...
+  services: [
+    // ...other services
+    [SoftAssertionService]
+  ],
+  // ...
+}
+```
+
 ## Default Options
 
 These default options below are connected to the [`waitforTimeout`](https://webdriver.io/docs/options#waitfortimeout) and [`waitforInterval`](https://webdriver.io/docs/options#waitforinterval) options set in the config.

src/index.ts

@@ -4,6 +4,9 @@ import type { RawMatcherFn } from './types.js'
 
 import wdioMatchers from './matchers.js'
 import { DEFAULT_OPTIONS } from './constants.js'
+import createSoftExpect from './softExpect.js'
+import { SoftAssertService } from './softAssert.js'
+import { SoftAssertionService } from './softAssertService.js'
 
 export const matchers = new Map<string, RawMatcherFn>()
 
@@ -20,7 +23,28 @@ expectLib.extend = (m) => {
 type MatchersObject = Parameters<typeof expectLib.extend>[0]
 
 expectLib.extend(wdioMatchers as MatchersObject)
-export const expect = expectLib as unknown as ExpectWebdriverIO.Expect
+
+// Extend the expect object with soft assertions
+const expectWithSoft = expectLib as unknown as ExpectWebdriverIO.Expect
+Object.defineProperty(expectWithSoft, 'soft', {
+    value: <T = unknown>(actual: T) => createSoftExpect(actual)
+})
+
+// Add soft assertions utility methods
+Object.defineProperty(expectWithSoft, 'getSoftFailures', {
+    value: (testId?: string) => SoftAssertService.getInstance().getFailures(testId)
+})
+
+Object.defineProperty(expectWithSoft, 'assertSoftFailures', {
+    value: (testId?: string) => SoftAssertService.getInstance().assertNoFailures(testId)
+})
+
+Object.defineProperty(expectWithSoft, 'clearSoftFailures', {
+    value: (testId?: string) => SoftAssertService.getInstance().clearFailures(testId)
+})
+
+export const expect = expectWithSoft
+
 export const getConfig = (): ExpectWebdriverIO.DefaultOptions => DEFAULT_OPTIONS
 export const setDefaultOptions = (options = {}): void => {
     Object.entries(options).forEach(([key, value]) => {
@@ -37,6 +61,11 @@ export const setOptions = setDefaultOptions
  */
 export { SnapshotService } from './snapshot.js'
 
+/**
+ * export soft assertion utilities
+ */
+export { SoftAssertService, SoftAssertionService }
+
 /**
  * export utils
  */

src/softAssert.ts

@@ -0,0 +1,139 @@
+import type { AssertionError } from 'node:assert'
+
+interface SoftFailure {
+    error: AssertionError | Error;
+    matcherName: string;
+    location?: string;
+}
+
+interface TestIdentifier {
+    id: string;
+    name?: string;
+    file?: string;
+}
+
+/**
+ * Soft assertion service to collect failures without stopping test execution
+ */
+export class SoftAssertService {
+    private static instance: SoftAssertService
+    private failureMap: Map<string, SoftFailure[]> = new Map()
+    private currentTest: TestIdentifier | null = null
+
+    private constructor() { }
+
+    /**
+     * Get singleton instance
+     */
+    public static getInstance(): SoftAssertService {
+        if (!SoftAssertService.instance) {
+            SoftAssertService.instance = new SoftAssertService()
+        }
+        return SoftAssertService.instance
+    }
+
+    /**
+     * Set the current test context
+     */
+    public setCurrentTest(testId: string, testName?: string, testFile?: string): void {
+        this.currentTest = { id: testId, name: testName, file: testFile }
+        if (!this.failureMap.has(testId)) {
+            this.failureMap.set(testId, [])
+        }
+    }
+
+    /**
+     * Clear the current test context
+     */
+    public clearCurrentTest(): void {
+        this.currentTest = null
+    }
+
+    /**
+     * Get current test ID
+     */
+    public getCurrentTestId(): string | null {
+        return this.currentTest?.id || null
+    }
+
+    /**
+     * Add a soft failure for the current test
+     */
+    public addFailure(error: Error, matcherName: string): void {
+        const testId = this.getCurrentTestId()
+        if (!testId) {
+            throw error // If no test context, throw the error immediately
+        }
+
+        // Extract stack information to get file and line number
+        const stackLines = error.stack?.split('\n') || []
+        let location = ''
+
+        // Find the first non-expect-webdriverio line in the stack
+        for (const line of stackLines) {
+            if (line && !line.includes('expect-webdriverio') && !line.includes('node_modules')) {
+                location = line.trim()
+                break
+            }
+        }
+
+        const failures = this.failureMap.get(testId) || []
+        failures.push({ error, matcherName, location })
+        this.failureMap.set(testId, failures)
+    }
+
+    /**
+     * Get all failures for a specific test
+     */
+    public getFailures(testId?: string): SoftFailure[] {
+        const id = testId || this.getCurrentTestId()
+        if (!id) {
+            return []
+        }
+        return this.failureMap.get(id) || []
+    }
+
+    /**
+     * Clear failures for a specific test
+     */
+    public clearFailures(testId?: string): void {
+        const id = testId || this.getCurrentTestId()
+        if (id) {
+            this.failureMap.delete(id)
+        }
+    }
+
+    /**
+     * Throw an aggregated error if there are failures for the current test
+     */
+    public assertNoFailures(testId?: string): void {
+        const id = testId || this.getCurrentTestId()
+        if (!id) {
+            return
+        }
+
+        const failures = this.getFailures(id)
+        if (failures.length === 0) {
+            return
+        }
+
+        // Create a formatted error message with all failures
+        let message = `${failures.length} soft assertion failure${failures.length > 1 ? 's' : ''}:\n\n`
+
+        failures.forEach((failure, index) => {
+            message += `${index + 1}) ${failure.matcherName}: ${failure.error.message}\n`
+            if (failure.location) {
+                message += `   at ${failure.location}\n`
+            }
+            message += '\n'
+        })
+
+        // Clear failures for this test to prevent duplicate reporting
+        this.clearFailures(id)
+
+        // Throw an aggregated error
+        const error = new Error(message)
+        error.name = 'SoftAssertionsError'
+        throw error
+    }
+}

src/softAssertService.ts

@@ -0,0 +1,73 @@
+import type { Services } from '@wdio/types'
+import type { Frameworks } from '@wdio/types'
+import { SoftAssertService } from './softAssert'
+
+/**
+ * WebdriverIO service to integrate soft assertions into the test lifecycle
+ */
+export class SoftAssertionService implements Services.ServiceInstance {
+    private softAssertService: SoftAssertService
+
+    constructor() {
+        this.softAssertService = SoftAssertService.getInstance()
+    }
+
+    /**
+     * Hook before a test starts
+     */
+    beforeTest(test: Frameworks.Test) {
+        const testId = this.getTestId(test)
+        this.softAssertService.setCurrentTest(testId, test.title, test.file)
+    }
+
+    /**
+     * Hook before a Cucumber step starts
+     */
+    beforeStep(step: Frameworks.PickleStep, scenario: Frameworks.Scenario) {
+        const stepId = `${scenario.uri || ''}:${scenario.name || ''}:${step.text || ''}`
+        this.softAssertService.setCurrentTest(stepId, step.text, scenario.uri)
+    }
+
+    /**
+     * Hook after a test completes
+     */
+    afterTest(test: Frameworks.Test, _: any, result: Frameworks.TestResult) {
+        // Only assert failures if the test hasn't already failed for another reason
+        if (!result.error) {
+            try {
+                const testId = this.getTestId(test)
+                this.softAssertService.assertNoFailures(testId)
+            } catch (error) {
+                // Update the test result with our aggregated error
+                result.error = error
+                result.passed = false
+            }
+        }
+        this.softAssertService.clearCurrentTest()
+    }
+
+    /**
+     * Hook after a Cucumber step completes
+     */
+    afterStep(step: Frameworks.PickleStep, scenario: Frameworks.Scenario, result: { passed: boolean, error?: Error }) {
+        // Only assert failures if the step hasn't already failed for another reason
+        if (result.passed) {
+            try {
+                const stepId = `${scenario.uri || ''}:${scenario.name || ''}:${step.text || ''}`
+                this.softAssertService.assertNoFailures(stepId)
+            } catch (error) {
+                // Update the step result with our aggregated error
+                result.error = error as Error
+                result.passed = false
+            }
+        }
+        this.softAssertService.clearCurrentTest()
+    }
+
+    /**
+     * Generate a unique test ID from a test object
+     */
+    private getTestId(test: Frameworks.Test): string {
+        return `${test.file || ''}:${test.parent || ''}:${test.title || ''}`
+    }
+}