ci: add smoke test for worker idle timeout

Author: mato533

e2e/assertions/index.ts

@@ -1,5 +1,5 @@
 import type { MatcherContext } from 'expect'
-import type { TreeItem } from 'wdio-vscode-service'
+import type { TreeItem, Workbench } from 'wdio-vscode-service'
 import type { STATUS } from '../helpers/index.ts'
 
 export interface ExpectedTreeItem {
@@ -92,6 +92,18 @@ try {
             async toMatchTreeStructure(tree: TreeItem[], expectedStructure: ExpectedTreeItem[]) {
                 return await expectTreeToMatchStructure.call(this as unknown as MatcherContext, tree, expectedStructure)
             },
+            async hasExpectedLog(workbench: Workbench, expectedLog: RegExp | string) {
+                const bottomBar = workbench.getBottomBar()
+                const outputView = await bottomBar.openOutputView()
+                await outputView.selectChannel('WebdriverIO')
+                const logs = await outputView.getText()
+
+                const regexp = typeof expectedLog === 'string' ? new RegExp(expectedLog) : expectedLog
+
+                const pass = logs.some((log) => regexp.test(log))
+                const message = pass ? 'The log outputs include expected text.' : 'The expected text is not included'
+                return { pass, message }
+            },
         })
     }
 } catch (error) {

e2e/assertions/wdio.shim.d.ts

@@ -4,6 +4,7 @@ declare global {
     namespace ExpectWebdriverIO {
         interface Matchers<R, T> {
             toMatchTreeStructure(expectedStructure: ExpectedTreeItem[]): R
+            hasExpectedLog(expectedLog: RegExp | string): R
         }
     }
 }

e2e/package.json

@@ -17,7 +17,8 @@
     "test:e2e:basic:cucumber": "cross-env VSCODE_WDIO_E2E_SCENARIO=cucumber xvfb-maybe pnpm run wdio",
     "test:e2e:workspace": "cross-env VSCODE_WDIO_E2E_SCENARIO=workspace xvfb-maybe pnpm run wdio",
     "test:smoke": "run-s test:smoke:*",
-    "test:smoke:update-config": "xvfb-maybe wdio run ./wdioSmoke.conf.ts",
+    "test:smoke:update-config": "cross-env VSCODE_WDIO_E2E_SCENARIO=config xvfb-maybe wdio run ./wdioSmoke.conf.ts",
+    "test:smoke:timeout": "cross-env VSCODE_WDIO_E2E_SCENARIO=timeout xvfb-maybe wdio run ./wdioSmoke.conf.ts",
     "wdio": "wdio run ./wdio.conf.ts"
   },
   "devDependencies": {

e2e/tests/workerIdleTimeout.spec.ts

@@ -0,0 +1,83 @@
+import { browser, expect } from '@wdio/globals'
+
+import { createCucumberExpected } from '../helpers/cucumber.ts'
+import {
+    STATUS,
+    clearAllTestResults,
+    clickTitleActionButton,
+    collapseAllTests,
+    getTestingSection,
+    openTestingView,
+    waitForResolved,
+    waitForTestStatus,
+} from '../helpers/index.ts'
+
+import type { SideBarView, ViewControl, Workbench } from 'wdio-vscode-service'
+
+const targetFramework = process.env.VSCODE_WDIO_E2E_SCENARIO || 'mocha'
+
+const expected = createCucumberExpected()
+
+describe(`VS Code Extension Testing with ${targetFramework}`, function () {
+    this.retries(3)
+    let workbench: Workbench
+    let testingViewControl: ViewControl
+    let sideBarView: SideBarView<any>
+
+    beforeEach(async function () {
+        workbench = await browser.getWorkbench()
+        testingViewControl = await openTestingView(workbench)
+        sideBarView = workbench.getSideBar()
+
+        const testingSection = await getTestingSection(sideBarView.getContent())
+        await collapseAllTests(testingSection)
+
+        await browser.waitUntil(async () => (await testingSection.getVisibleItems()).length === 1)
+    })
+
+    afterEach(async function () {
+        await clearAllTestResults(workbench)
+    })
+
+    it('should be displayed the testing screen at the sideBar', async function () {
+        expect(await testingViewControl.getTitle()).toBe('Testing')
+        expect(await sideBarView.getTitlePart().getTitle()).toBe('TESTING')
+    })
+
+    it('should resolve defined tests correctly', async function () {
+        const testingSection = await getTestingSection(sideBarView.getContent())
+        const items = await testingSection.getVisibleItems()
+
+        await waitForResolved(browser, items[0])
+
+        await expect(items).toMatchTreeStructure(expected.notRun)
+    })
+
+    it('should shutdown the work process after idle timeout was reached', async function () {
+        await new Promise((resolve) => setTimeout(resolve, 2000))
+
+        await expect(workbench).hasExpectedLog(/Worker#0 process shutdown gracefully/)
+
+        const bottomBar = workbench.getBottomBar()
+        const outputView = await bottomBar.openOutputView()
+        await outputView.selectChannel('WebdriverIO')
+        await outputView.clearText()
+    })
+
+    it('should start work process and run test successfully', async function () {
+        const testingSection = await getTestingSection(sideBarView.getContent())
+        const items = await testingSection.getVisibleItems()
+
+        await waitForResolved(browser, items[0])
+
+        await clickTitleActionButton(sideBarView.getTitlePart(), 'Run Tests')
+
+        await waitForTestStatus(browser, items[0], STATUS.PASSED)
+
+        // assert that start work process
+        await expect(workbench).hasExpectedLog(/\[#1\] Worker process started successfully/)
+
+        // assert that run test successfully
+        await expect(items).toMatchTreeStructure(expected.runAll)
+    })
+})

e2e/wdioSmoke.conf.ts

@@ -1,14 +1,29 @@
 import { createBaseConfig } from './wdio.conf.ts'
 
-const specs = [
-    './tests/updateConfig.spec.ts',
-    './tests/updateSpec.spec.ts',
-    './tests/updateErrorSpec.spec.ts',
-    './tests/updateErrorConfig.spec.ts',
-]
+type TestTargets = 'config' | 'timeout'
+
+const target = (process.env.VSCODE_WDIO_E2E_SCENARIO || 'config') as TestTargets
+
+const workspace = target === 'config' ? '../samples/smoke/update-config' : '../samples/e2e/cucumber'
+
+function defineSpecs(target: TestTargets) {
+    switch (target) {
+        case 'config':
+            return [
+                './tests/updateConfig.spec.ts',
+                './tests/updateSpec.spec.ts',
+                './tests/updateErrorSpec.spec.ts',
+                './tests/updateErrorConfig.spec.ts',
+            ]
+        default:
+            return ['./tests/workerIdleTimeout.spec.ts']
+    }
+}
+
+const specs = defineSpecs(target)
 
 export const config: WebdriverIO.Config = {
-    ...createBaseConfig('../samples/smoke/update-config'),
+    ...createBaseConfig(workspace, { 'webdriverio.logLevel': 'debug', 'webdriverio.workerIdleTimeout': 2 }),
     specs,
     maxInstances: 1,
 }

packages/vscode-wdio-server/src/worker.ts

@@ -206,7 +206,7 @@ export class WdioExtensionWorker extends TypedEventEmitter<WdioExtensionWorkerEv
             // Start idle monitoring after successful connection
             this.idleMonitor.start()
             this.startHealthCheck()
-            log.debug('Worker process started successfully')
+            log.debug(`[${this.cid}] Worker process started successfully`)
         })
     }