Skip to content

Commit 748190e

Browse files
authored
fix: fix not refrecting configuration change due to drive letter inconsistencies (#36)
* chore: update changelog when there are no updates [skip ci] * test: update retry * test: add sample for smoke tests * test: add test case for updating the wdio configuration * test: update assertion * ci: add smoke test to ci workflow * chore: update lockfile * fix: fix failed of the smoke test on linux * fix: fix due to drive letter inconsistencies
1 parent bd63cef commit 748190e

26 files changed

Lines changed: 438 additions & 50 deletions

File tree

.github/workflows/ci-smoke.yml

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
name: Smoke tests
2+
3+
on:
4+
workflow_call:
5+
# Make this a reusable workflow, no value needed
6+
# https://docs.github.com/en/actions/using-workflows/reusing-workflows
7+
8+
env:
9+
TURBO_TELEMETRY_DISABLED: 1
10+
11+
jobs:
12+
unit:
13+
name: Smoke Tests (${{ matrix.os }}.${{ matrix.node-version }})
14+
strategy:
15+
fail-fast: false
16+
matrix:
17+
node-version: ['20']
18+
os: ['ubuntu-latest', 'windows-latest', 'macos-latest']
19+
runs-on: ${{ matrix.os }}
20+
steps:
21+
- name: 👷 Checkout
22+
uses: actions/checkout@v4
23+
with:
24+
ssh-key: ${{ secrets.DEPLOY_KEY }}
25+
26+
- name: 🛠️ Setup workspace
27+
uses: ./.github/workflows/actions/setup-workspace
28+
with:
29+
node-version: ${{ matrix.node-version }}
30+
31+
- name: ⬇️ Download Build Archive
32+
uses: ./.github/workflows/actions/download-archive
33+
with:
34+
name: vscode-webdriverio
35+
path: .
36+
filename: vscode-webdriverio-build.zip
37+
38+
- name: 🚂 Run the smoke test
39+
run: pnpm run test:smoke
40+
shell: bash
41+
42+
- name: 🐛 Debug Build
43+
uses: stateful/[email protected]
44+
if: failure()
45+
with:
46+
timeout: '180000'

.github/workflows/ci.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,3 +39,8 @@ jobs:
3939
name: E2E
4040
needs: [build]
4141
uses: ./.github/workflows/ci-e2e.yml
42+
43+
smoke:
44+
name: Smoke
45+
needs: [build]
46+
uses: ./.github/workflows/ci-smoke.yml

e2e/assertions/index.ts

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import type { TreeItem } from 'wdio-vscode-service'
33
import type { STATUS } from '../helpers.js'
44

55
export interface ExpectedTreeItem {
6-
label: string
6+
text: string
77
status: StatusStrings
88
children?: ExpectedTreeItem[]
99
}
@@ -37,12 +37,15 @@ async function expectTreeToMatchStructure(
3737
const rootLabel = await item.getLabel()
3838

3939
const expectItem = expectedTree[Number(index)]
40-
const labelRegex = new RegExp(expectItem.label)
40+
if (typeof expectItem.text === 'undefined' || typeof expectItem.status === 'undefined') {
41+
throw new MismatchTreeStructureError(() => `The expected values are not set (${index} : ${level})`)
42+
}
43+
const labelRegex = new RegExp(expectItem.text)
4144
if (!labelRegex.test(rootLabel)) {
4245
throw new MismatchTreeStructureError(
4346
() =>
4447
`Mismatch the label of items at (${index} : ${level})` +
45-
` Expected: ${this.utils.printExpected(expectItem.label)}` +
48+
` Expected: ${this.utils.printExpected(expectItem.text)}` +
4649
` Received: ${this.utils.printReceived(rootLabel)}`
4750
)
4851
}

e2e/assertions/wdio.shim.d.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ import 'expect-webdriverio'
33
declare global {
44
namespace ExpectWebdriverIO {
55
interface Matchers<R, T> {
6-
toBeWithinRange(floor: number, ceiling: number): R
76
toMatchTreeStructure(expectedStructure: ExpectedTreeItem[]): R
87
}
98
}

e2e/package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
"test:e2e:mocha": "cross-env VSCODE_WDIO_E2E_FRAMEWORK=mocha xvfb-maybe pnpm run wdio",
1515
"test:e2e:jasmine": "cross-env VSCODE_WDIO_E2E_FRAMEWORK=jasmine xvfb-maybe pnpm run wdio",
1616
"test:e2e:cucumber": "cross-env VSCODE_WDIO_E2E_FRAMEWORK=cucumber xvfb-maybe pnpm run wdio",
17+
"test:smoke": "run-s test:smoke:*",
18+
"test:smoke:update-config": "xvfb-maybe wdio run ./wdioSmoke.conf.ts",
1719
"wdio": "wdio run ./wdio.conf.ts"
1820
},
1921
"devDependencies": {

e2e/tests/basic.spec.ts

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ import type { SideBarView, ViewControl, Workbench } from 'wdio-vscode-service'
1313

1414
const targetFramework = process.env.VSCODE_WDIO_E2E_FRAMEWORK || 'mocha'
1515

16-
describe(`VS Code Extension Testing with ${targetFramework}`, () => {
16+
describe(`VS Code Extension Testing with ${targetFramework}`, function () {
17+
this.retries(3)
1718
let workbench: Workbench
1819
let testingViewControl: ViewControl
1920
let sideBarView: SideBarView<any>
@@ -63,7 +64,10 @@ describe(`VS Code Extension Testing with ${targetFramework}`, () => {
6364
children: [
6465
{ text: 'should be a pending test', status: STATUS.NOT_YET_RUN },
6566
{ text: 'should have the right title - WebdriverIO', status: STATUS.NOT_YET_RUN },
66-
{ text: 'should have the right title - Github', status: STATUS.NOT_YET_RUN },
67+
{
68+
text: 'should have the right title - WebdriverJS Testpage',
69+
status: STATUS.NOT_YET_RUN,
70+
},
6771
{ text: 'should fail', status: STATUS.NOT_YET_RUN },
6872
],
6973
},

e2e/tests/basicCucumber.spec.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ import type { SideBarView, ViewControl, Workbench } from 'wdio-vscode-service'
1313

1414
const targetFramework = process.env.VSCODE_WDIO_E2E_FRAMEWORK || 'mocha'
1515

16-
describe(`VS Code Extension Testing with ${targetFramework}`, () => {
16+
describe(`VS Code Extension Testing with ${targetFramework}`, function () {
17+
this.retries(3)
1718
let workbench: Workbench
1819
let testingViewControl: ViewControl
1920
let sideBarView: SideBarView<any>

e2e/tests/updateConfig.spec.ts

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
// /Users/abetaiki/Documents/50_workspace/30_js/wdio-vscode/samples/smoke/mocha/wdio.conf.ts.tmplate
2+
import path from 'node:path'
3+
import url from 'node:url'
4+
5+
import { browser, expect } from '@wdio/globals'
6+
import shell from 'shelljs'
7+
8+
import {
9+
STATUS,
10+
clearAllTestResults,
11+
getTestingSection,
12+
openTestingView,
13+
waitForResolved,
14+
waitForTestStatus,
15+
} from '../helpers.ts'
16+
17+
import type { SideBarView, Workbench } from 'wdio-vscode-service'
18+
19+
const __dirname = path.dirname(url.fileURLToPath(import.meta.url))
20+
const rootDir = path.resolve(__dirname, '..', '..')
21+
const workspacePath = path.resolve(rootDir, 'samples/smoke/update-config/')
22+
const beforeConfig = path.resolve(workspacePath, 'wdio.conf.ts')
23+
const afterConfig = path.resolve(workspacePath, 'wdio.conf.ts.template')
24+
25+
describe('VS Code Extension Testing (Update config)', function () {
26+
let workbench: Workbench
27+
let sideBarView: SideBarView<any>
28+
29+
beforeEach(async () => {
30+
workbench = await browser.getWorkbench()
31+
await openTestingView(workbench)
32+
sideBarView = workbench.getSideBar()
33+
34+
const testingSection = await getTestingSection(sideBarView.getContent())
35+
const items = (await testingSection.getVisibleItems()).reverse()
36+
for (const item of items) {
37+
if ((await item.isExpandable()) && (await item.isExpanded())) {
38+
await item.collapse()
39+
}
40+
}
41+
await browser.waitUntil(async () => (await testingSection.getVisibleItems()).length === 1)
42+
})
43+
44+
afterEach(async () => {
45+
await clearAllTestResults(workbench)
46+
})
47+
this.afterAll(() => {
48+
shell.exec(`git checkout ${beforeConfig}`)
49+
})
50+
51+
it('should be resolved the defined tests after configuration changed', async () => {
52+
const testingSection = await getTestingSection(sideBarView.getContent())
53+
const items = await testingSection.getVisibleItems()
54+
55+
await waitForResolved(browser, items[0])
56+
57+
await expect(items).toMatchTreeStructure([
58+
{
59+
text: 'wdio.conf.ts',
60+
status: STATUS.NOT_YET_RUN,
61+
children: [
62+
{
63+
text: 'before.spec.ts',
64+
status: STATUS.NOT_YET_RUN,
65+
children: [
66+
{
67+
text: 'Before Tests',
68+
status: STATUS.NOT_YET_RUN,
69+
children: [{ text: 'TEST BEFORE 1', status: STATUS.NOT_YET_RUN }],
70+
},
71+
],
72+
},
73+
],
74+
},
75+
])
76+
77+
// Emulate the changing configuration
78+
shell.cp('-f', afterConfig, beforeConfig)
79+
await new Promise((resolve) => setTimeout(resolve, 3000))
80+
await browser.waitUntil(
81+
async () => {
82+
if (!(await items[0].isExpanded())) {
83+
await items[0].expand()
84+
}
85+
86+
const children = await items[0].getChildren()
87+
const target = children[0]
88+
if (!target) {
89+
return false
90+
}
91+
const regex = new RegExp('after.test.ts')
92+
return regex.test(await target.getLabel())
93+
},
94+
{
95+
timeoutMsg: 'The label "after.test.ts" is not found.',
96+
}
97+
)
98+
99+
await expect(items).toMatchTreeStructure([
100+
{
101+
text: 'wdio.conf.ts',
102+
status: STATUS.NOT_YET_RUN,
103+
children: [
104+
{
105+
text: 'after.test.ts',
106+
status: STATUS.NOT_YET_RUN,
107+
children: [
108+
{
109+
text: 'After Tests',
110+
status: STATUS.NOT_YET_RUN,
111+
children: [{ text: 'TEST AFTER 1', status: STATUS.NOT_YET_RUN }],
112+
},
113+
],
114+
},
115+
],
116+
},
117+
])
118+
})
119+
120+
it('should run tests successfully after changing the configuration', async () => {
121+
const testingSection = await getTestingSection(sideBarView.getContent())
122+
const items = await testingSection.getVisibleItems()
123+
124+
await waitForResolved(browser, items[0])
125+
126+
const actionRunTest = await items[0].getActionButton('Run Test')
127+
await (actionRunTest!.elem as WebdriverIO.Element).click()
128+
129+
await waitForTestStatus(browser, items[0], STATUS.PASSED)
130+
131+
await expect(items).toMatchTreeStructure([
132+
{
133+
text: 'wdio.conf.ts',
134+
status: STATUS.PASSED,
135+
children: [
136+
{
137+
text: 'after.test.ts',
138+
status: STATUS.PASSED,
139+
children: [
140+
{
141+
text: 'After Tests',
142+
status: STATUS.PASSED,
143+
children: [{ text: 'TEST AFTER 1', status: STATUS.PASSED }],
144+
},
145+
],
146+
},
147+
],
148+
},
149+
])
150+
})
151+
})

e2e/wdioSmoke.conf.ts

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import * as path from 'node:path'
2+
3+
const specs = ['./tests/updateConfig.spec.ts']
4+
5+
export const config: WebdriverIO.Config = {
6+
runner: 'local',
7+
tsConfigPath: './tsconfig.json',
8+
specs,
9+
maxInstances: 10,
10+
capabilities: [
11+
{
12+
browserName: 'vscode',
13+
browserVersion: 'stable', // also possible: "insiders" or a specific version e.g. "1.80.0"
14+
'wdio:vscodeOptions': {
15+
// points to directory where extension package.json is located
16+
extensionPath: path.resolve('../packages/vscode-webdriverio'),
17+
// optional VS Code settings
18+
userSettings: {
19+
'webdriverio.logLevel': 'trace',
20+
},
21+
workspacePath: path.resolve('../samples/smoke/update-config'),
22+
},
23+
'wdio:enforceWebDriverClassic': true,
24+
},
25+
],
26+
27+
logLevel: 'debug',
28+
outputDir: 'logs',
29+
bail: 0,
30+
waitforTimeout: 120000,
31+
connectionRetryTimeout: 120000,
32+
connectionRetryCount: 3,
33+
services: ['vscode'],
34+
framework: 'mocha',
35+
reporters: ['spec'],
36+
mochaOpts: {
37+
ui: 'bdd',
38+
timeout: 6000000,
39+
require: ['assertions/index.ts'],
40+
},
41+
}

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
"test": "run-s test:*",
2929
"test:unit": "vitest --run",
3030
"test:e2e": "pnpm --filter @vscode-wdio/e2e run test:e2e",
31+
"test:smoke": "pnpm --filter @vscode-wdio/e2e run test:smoke",
3132
"coverage": "vitest --run --coverage",
3233
"graph": "pnpm run build --graph assets/build.png",
3334
"version": "pnpm --filter @vscode-wdio/release run changelog && git add CHANGELOG.md",

0 commit comments

Comments
 (0)