feat: Standardization of test execution methods in Windows

.github/workflows/ci-smoke.yml

@@ -5,6 +5,10 @@ on:
     # Make this a reusable workflow, no value needed
     # https://docs.github.com/en/actions/using-workflows/reusing-workflows
     inputs:
+      os:
+        description: 'OS for run'
+        type: string
+        default: "['ubuntu-latest', 'windows-latest', 'macos-latest']"
       scenario:
         description: 'Smoke scenario'
         type: string
@@ -19,7 +23,7 @@ jobs:
       fail-fast: false
       matrix:
         node-version: ['20']
-        os: ['ubuntu-latest', 'windows-latest', 'macos-latest']
+        os: ${{ fromJSON(inputs.os) }}
     runs-on: ${{ matrix.os }}
     steps:
       - name: 👷 Checkout

.github/workflows/ci.yml

@@ -74,3 +74,11 @@ jobs:
     uses: ./.github/workflows/ci-smoke.yml
     with:
       scenario: 'env'
+
+  smoke-retro-wdio:
+    name: Smoke - Run test with temporary configuration file when lower version of WDIO
+    needs: [lint, build, e2e]
+    uses: ./.github/workflows/ci-smoke.yml
+    with:
+      os: "['windows-latest']"
+      scenario: 'retro-wdio-win'

e2e/package.json

@@ -19,22 +19,27 @@
     "test:smoke": "run-s test:smoke:*",
     "test:smoke:config": "cross-env VSCODE_WDIO_E2E_SCENARIO=config xvfb-maybe wdio run ./wdioSmoke.conf.ts",
     "test:smoke:timeout": "cross-env VSCODE_WDIO_E2E_SCENARIO=timeout xvfb-maybe wdio run ./wdioSmoke.conf.ts",
+    "test:smoke:retro-wdio-win": "run-s test:smoke:retro-wdio-win:*",
+    "test:smoke:retro-wdio-win:run-latest": "cross-env VSCODE_WDIO_E2E_SCENARIO=mocha VSCODE_WDIO_SMOKE_RETRO_WIN=no xvfb-maybe pnpm run wdio",
+    "test:smoke:retro-wdio-win:prepare": "tsx ./scripts/retro-wdio-win.ts",
+    "test:smoke:retro-wdio-win:run": "cross-env VSCODE_WDIO_E2E_SCENARIO=mocha VSCODE_WDIO_SMOKE_RETRO_WIN=yes xvfb-maybe pnpm run wdio",
+    "test:smoke:retro-wdio-win:cleanup": "git checkout ../samples/e2e/mocha/package.json ../pnpm-lock.yaml && pnpm --filter @vscode-wdio/e2e-mocha install",
     "test:smoke:env": "cross-env VSCODE_WDIO_E2E_SCENARIO=env xvfb-maybe wdio run ./wdioSmoke.conf.ts",
     "wdio": "wdio run ./wdio.conf.ts"
   },
   "devDependencies": {
     "@types/semver": "^7.7.0",
-    "@wdio/cli": "^9.13.0",
-    "@wdio/globals": "^9.12.6",
-    "@wdio/local-runner": "^9.13.0",
-    "@wdio/mocha-framework": "^9.13.0",
-    "@wdio/spec-reporter": "^9.13.0",
-    "@wdio/types": "^9.15.0",
+    "@wdio/cli": "^9.16.2",
+    "@wdio/globals": "^9.16.2",
+    "@wdio/local-runner": "^9.16.2",
+    "@wdio/mocha-framework": "^9.16.2",
+    "@wdio/spec-reporter": "^9.16.2",
+    "@wdio/types": "^9.16.2",
     "chai": "^5.2.0",
     "expect": "^30.0.0",
     "semver": "^7.7.2",
     "wdio-vscode-service": "^6.1.3",
-    "webdriver": "^9.13.0",
-    "webdriverio": "^9.13.0"
+    "webdriver": "^9.16.2",
+    "webdriverio": "^9.16.2"
   }
 }

e2e/scripts/retro-wdio-win.ts

@@ -0,0 +1,17 @@
+import shell from 'shelljs'
+
+import pkg from '../../samples/e2e/mocha/package.json' with { type: 'json' }
+
+const targetPkg = '@vscode-wdio/e2e-mocha'
+const targetWdioVersion = '9.15.0'
+
+const wdioPkgs = Object.keys(pkg.devDependencies).filter((pkg) => pkg.startsWith('@wdio/'))
+
+const targetWdioPkgs = wdioPkgs.map((pkg) => `${pkg}@${targetWdioVersion}`)
+
+const cmd = `pnpm --filter ${targetPkg} install -D ${targetWdioPkgs.join(' ')}`
+
+console.log(`\n>${cmd}\n`)
+
+const result = shell.exec(cmd)
+process.exit(result.code)

e2e/tests/basic.spec.ts

@@ -85,4 +85,21 @@ describe(`VS Code Extension Testing with ${targetFramework}`, function () {
 
         await expect(items).toMatchTreeStructure(expected.runPartially)
     })
+
+    // eslint-disable-next-line mocha/no-setup-in-describe
+    if (process.env.VSCODE_WDIO_SMOKE_RETRO_WIN === 'yes') {
+
+        it('should use temporally configuration file', async function () {
+            await expect(workbench).hasExpectedLog(
+                'Use temporary configuration files to run WDIO: @wdio/[email protected] < 9.16.0'
+            )
+        })
+
+        // eslint-disable-next-line mocha/no-setup-in-describe
+    } else if (process.env.VSCODE_WDIO_SMOKE_RETRO_WIN === 'no') {
+
+        it('should not use temporally configuration file', async function () {
+            await expect(workbench).hasExpectedLog(/Use cli argument to run WDIO: @wdio\/utils/)
+        })
+    }
 })

e2e/wdio.conf.ts

@@ -20,6 +20,8 @@ const version = isCompatibilityMode ? minimumVersion : 'stable'
 const outputDir = path.join(__dirname, 'logs', [isCompatibilityMode ? 'compatibility' : 'e2e', target].join('-'))
 process.env.VSCODE_WDIO_TRACE_LOG_PATH = outputDir
 
+const loglevel = process.env.VSCODE_WDIO_SMOKE_RETRO_WIN ? 'debug' : 'trace'
+
 function defineSpecs(target: TestTargets) {
     switch (target) {
         case 'cucumber':
@@ -38,7 +40,7 @@ export function createBaseConfig(workspacePath: string, userSettings = {}): Webd
     const resolvedUserSettings = Object.assign(
         {},
         {
-            'webdriverio.logLevel': 'trace',
+            'webdriverio.logLevel': loglevel,
         },
         userSettings
     )

packages/vscode-wdio-reporter/package.json

@@ -16,10 +16,10 @@
   },
   "dependencies": {
     "@vscode-wdio/constants": "workspace:*",
-    "@wdio/reporter": "^9.12.3"
+    "@wdio/reporter": "^9.16.2"
   },
   "devDependencies": {
     "@vscode-wdio/types": "workspace:*",
-    "@wdio/types": "^9.13.0"
+    "@wdio/types": "^9.16.2"
   }
 }

packages/vscode-wdio-types/package.json

@@ -47,7 +47,7 @@
   },
   "devDependencies": {
     "@vscode-wdio/constants": "workspace:*",
-    "@wdio/types": "^9.13.0"
+    "@wdio/types": "^9.16.2"
   },
   "dependencies": {
     "@types/ws": "^8.18.1"

packages/vscode-wdio-worker/package.json

@@ -31,12 +31,13 @@
     "@vscode-wdio/utils": "workspace:*",
     "birpc": "^2.3.0",
     "dotenv": "^16.5.0",
+    "import-meta-resolve": "^4.1.0",
     "recast": "^0.23.11",
     "ws": "^8.18.1"
   },
   "devDependencies": {
     "@types/ws": "^8.18.1",
     "@vscode-wdio/types": "workspace:*",
-    "@wdio/cli": "^9.12.4"
+    "@wdio/cli": "^9.16.2"
   }
 }

packages/vscode-wdio-worker/src/test.ts

@@ -3,7 +3,7 @@ import * as os from 'node:os'
 import { dirname, isAbsolute, join, resolve } from 'node:path'
 
 import { getLauncherInstance } from './cli.js'
-import { getTempConfigCreator, isWindows } from './utils.js'
+import { getTempConfigCreator, isFixedWdio, isWindows } from './utils.js'
 import type { ResultSet } from '@vscode-wdio/types/reporter'
 import type { RunTestOptions, TestResultData } from '@vscode-wdio/types/server'
 import type { ILogger } from '@vscode-wdio/types/utils'
@@ -15,6 +15,13 @@ const VSCODE_REPORTER_PATH = resolve(__dirname, 'reporter.cjs')
 export async function runTest(this: WorkerMetaContext, options: RunTestOptions): Promise<TestResultData> {
     const outputDir = await getOutputDir.call(this)
     let configFile: string | undefined = undefined
+
+    // To avoid this issue, We use temporary configuration files
+    // only on Windows platforms and for versions of @wdio/utils prior to 9.15.0.
+    // https://github.com/webdriverio/webdriverio/issues/14532
+    // This issue was fixed by this PR.
+    // https://github.com/webdriverio/webdriverio/pull/14565
+    const useTempConfigFile = isWindows() && !(await isFixedWdio.call(this, options.configPath))
     try {
         // Prepare launcher options
         const wdioArgs: RunCommandArguments = {
@@ -42,16 +49,14 @@ export async function runTest(this: WorkerMetaContext, options: RunTestOptions):
             await fs.mkdir(logDir, { recursive: true })
         }
 
-        if (isWindows()) {
+        if (useTempConfigFile) {
             const creator = await getTempConfigCreator(this)
             configFile = await creator(options.configPath, outputDir.json!)
             options.configPath = configFile
             wdioArgs.configPath = configFile
-        }
-
-        // The `stdout` must be true because the name of the logger is
-        // the name of the file and the initialization of Write Stream will fail.
-        if (!isWindows()) {
+        } else {
+            // The `stdout` must be true because the name of the logger is
+            // the name of the file and the initialization of Write Stream will fail.
             wdioArgs.reporters = [[VSCODE_REPORTER_PATH, { stdout: true, outputDir: outputDir.json }]]
         }
 
@@ -122,7 +127,7 @@ export async function runTest(this: WorkerMetaContext, options: RunTestOptions):
         if (outputDir.json) {
             await removeResultDir(this.log, outputDir.json)
         }
-        if (isWindows() && configFile) {
+        if (useTempConfigFile && configFile) {
             try {
                 this.log.debug(`Remove temp config file...: ${configFile}`)
                 await fs.rm(configFile, { recursive: true, force: true })

packages/vscode-wdio-worker/src/utils.ts

@@ -1,5 +1,8 @@
+import fs from 'node:fs/promises'
 import path from 'node:path'
-import { pathToFileURL } from 'node:url'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+
+import { resolve } from 'import-meta-resolve'
 
 import type { WorkerMetaContext } from '@vscode-wdio/types'
 import type { createTempConfigFile } from './config.js'
@@ -23,6 +26,54 @@ export function isWindows() {
     return process.platform === 'win32'
 }
 
+type PackageJson = { name: string; version: string }
+
+export async function isFixedWdio(this: WorkerMetaContext, configPath: string) {
+    try {
+        const pkgName = '@wdio/utils'
+        this.log.debug(`Try to detect the version of ${pkgName}`)
+        const utilEntryPoint = resolve(`${pkgName}`, resolve('@wdio/cli', pathToFileURL(configPath).href))
+        const utilPkg = await findPackageJson(fileURLToPath(utilEntryPoint))
+        if (!utilPkg) {
+            throw new Error(`Could not detect the entry point of ${pkgName}`)
+        }
+        const pkg = JSON.parse(await fs.readFile(utilPkg, { encoding: 'utf-8' })) as PackageJson
+        if (pkg.name !== pkgName) {
+            throw new Error(`Could not detect the version of ${pkgName}`)
+        }
+        this.log.debug(`Detected version of ${pkgName}@${pkg.version}`)
+        const versions = pkg.version.split('.')
+
+        if (Number(versions[0]) >= 10 || (Number(versions[0]) >= 9 && Number(versions[1]) >= 16)) {
+            this.log.debug(`Use cli argument to run WDIO: ${pkgName}@${pkg.version} >= 9.16.0`)
+            return true
+        }
+        this.log.debug(`Use temporary configuration files to run WDIO: ${pkgName}@${pkg.version} < 9.16.0`)
+        return false
+    } catch (error) {
+        const msg = error instanceof Error ? error.message : String(error)
+        this.log.debug(`Use temporary configuration files because ${msg}`)
+        return false
+    }
+}
+
+async function findPackageJson(startPath: string) {
+    let dir = path.dirname(startPath)
+    const root = path.parse(dir).root
+
+    while (dir !== root) {
+        const pkgPath = path.join(dir, 'package.json')
+        try {
+            await fs.access(pkgPath)
+            return pkgPath
+        } catch {
+            dir = path.dirname(dir)
+        }
+    }
+
+    return undefined
+}
+
 export async function dynamicLoader(
     context: WorkerMetaContext,
     libModule: unknown,

packages/vscode-wdio-worker/tests/test.test.ts

@@ -8,6 +8,7 @@ import { getLauncherInstance } from '../src/cli.js'
 import { runTest } from '../src/test.js'
 import { getTempConfigCreator, isWindows } from '../src/utils.js'
 import type { Dirent } from 'node:fs'
+import type { RunTestOptions } from '@vscode-wdio/types'
 import type { WorkerMetaContext } from '@vscode-wdio/types/worker'
 
 // Mock dependencies
@@ -25,6 +26,7 @@ vi.mock('../src/cli.js', () => {
 vi.mock('../src/utils.js', () => {
     return {
         isWindows: vi.fn(() => false),
+        isFixedWdio: vi.fn(() => false),
         getTempConfigCreator: vi.fn(),
     }
 })
@@ -42,11 +44,15 @@ describe('runTest', () => {
     } as unknown as WorkerMetaContext
 
     const mockConfigFile = '/path/to/wdio.conf.js'
-    const mockOptions = {
+    const mockEnv = {
+        paths: [],
+        override: false,
+    }
+    const mockOptions: RunTestOptions = {
         configPath: mockConfigFile,
         specs: ['test.spec.js'],
         grep: 'test pattern',
-        env: { paths: [], override: false },
+        env: mockEnv,
     }
 
     // Mock temporary directories and files
@@ -142,7 +148,7 @@ describe('runTest', () => {
         // Arrange
         const optionsNoSpecs = {
             configPath: '/path/to/wdio.conf.js',
-            env: { paths: [], override: false },
+            env: mockEnv,
         }
 
         // Act
@@ -157,7 +163,7 @@ describe('runTest', () => {
         const optionsNoGrep = {
             configPath: '/path/to/wdio.conf.js',
             specs: ['test.spec.js'],
-            env: { paths: [], override: false },
+            env: mockEnv,
         }
 
         // Act