Skip to content

Commit f70c14a

Browse files
cwalthercwalther
authored
Make tests easier to run (#4620)
* Skip server mode tests on CMake >= 3.20 Server Mode was removed in favor of the File API in CMake 3.20, so skip the respective tests when it is not supported. Whether server mode is supported was tracked already, but only with a lower version bound. Availability of server mode is explicitly named in the `-E capabilities` response, so additionally take that into account, effectively adding an upper bound. * Fix sysroot access test Skip configure-time compiler checks when passing a fake sysroot to the compiler, they would fail because it is unable to link an executable that way. I am not sure under what circumstances this has ever worked. * Clarify that fakebin must be built before running tests Make the tests that rely on executables in test/fakebin/ check whether they exist, and if not give a helpful error message, instead of failing in unclear ways when they find things in the system $PATH instead. Add a command `yarn pretest-buildfakebin` that documents how to build these executables, which was otherwise only documented in .github/workflows/ci-main*.yml and .vscode/settings.json. * Formatting in unrelated parts of touched files
1 parent 350e56a commit f70c14a

9 files changed

Lines changed: 143 additions & 57 deletions

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ Improvements:
4848
- Add `cmake.removeStaleKitsOnScan` setting to optionally remove stale compiler kits from the kit picker after a "Scan for Kits" when they are no longer rediscovered. This is useful after compiler upgrades that leave older versions outside `PATH`. Set `"keep": true` in a kit entry to prevent automatic removal. [#3852](https://github.com/microsoft/vscode-cmake-tools/issues/3852)
4949
- Add `pr-readiness` Copilot skill to verify PRs have a descriptive title, meaningful description, and a properly formatted CHANGELOG entry. [#4862](https://github.com/microsoft/vscode-cmake-tools/pull/4862)
5050
- Updated IntelliSense tooltips with changes from CMake 4.3.1. [#4872](https://github.com/microsoft/vscode-cmake-tools/pull/4872)
51+
- Make it easier for a new developer of CMake Tools to run tests. [#4620](https://github.com/microsoft/vscode-cmake-tools/pull/4620) [@cwalther](https://github.com/cwalther)
5152

5253
Bug Fixes:
5354
- Fix stale C/C++ custom-configuration entries persisting after reconfigure/preset switches, which could cause Go to Definition/IntelliSense to surface symbols from inactive sources in the same folder. [#4472](https://github.com/microsoft/vscode-cmake-tools/issues/4472)

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4436,6 +4436,7 @@
44364436
"translations-import": "gulp translations-import",
44374437
"package": "vsce package --yarn -o cmake-tools.vsix",
44384438
"pretest": "tsc -p test.tsconfig.json",
4439+
"pretest-buildfakebin": "cmake -DCMAKE_INSTALL_PREFIX=test/fakebin -DCMAKE_BUILD_TYPE=Release -S test/fakeOutputGenerator -B test/fakeOutputGenerator/build && cmake --build test/fakeOutputGenerator/build && cmake --install test/fakeOutputGenerator/build && cmake -E rm -rf test/fakeOutputGenerator/build",
44394440
"lint": "gulp lint",
44404441
"smokeTests": "yarn run pretest && node ./out/test/smoke/badProject/runTest.js && node ./out/test/smoke/goodProject/runTest.js && node ./out/test/smoke/noCtest/runTest.js",
44414442
"unitTests": "yarn run pretest && node ./out/test/unit-tests/runTest.js",

‎src/cmakeExecutable.ts‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import * as proc from '@cmt/proc';
22
import * as util from '@cmt/util';
3-
import {setContextAndStore} from '@cmt/extension';
3+
import { setContextAndStore } from '@cmt/extension';
44
import * as logging from '@cmt/logging';
55
import * as nls from 'vscode-nls';
66
import { ConfigurationReader } from './config';
@@ -69,10 +69,13 @@ export async function getCMakeExecutableInformation(path: string, config?: Confi
6969
// Support for CMake using an internal default generator when one isn't provided
7070
cmake.isDefaultGeneratorSupported = util.versionGreaterOrEquals(cmake.version, cmake.minimalDefaultGeneratorVersion);
7171
}
72-
const debuggerPresent = await proc.execute(path, ['-E', 'capabilities'], null, execOpt).result;
73-
if (debuggerPresent.retc === 0 && debuggerPresent.stdout) {
74-
console.assert(debuggerPresent.stdout);
75-
const stdoutJson = JSON.parse(debuggerPresent.stdout);
72+
const capabilities = await proc.execute(path, ['-E', 'capabilities'], null, execOpt).result;
73+
if (capabilities.retc === 0 && capabilities.stdout) {
74+
console.assert(capabilities.stdout);
75+
const stdoutJson = JSON.parse(capabilities.stdout);
76+
if (cmake.isServerModeSupported && !stdoutJson["serverMode"]) {
77+
cmake.isServerModeSupported = false;
78+
}
7679
cmake.isDebuggerSupported = stdoutJson["debugger"];
7780
await setCMakeDebuggerAvailableContext(
7881
cmake.isDebuggerSupported?.valueOf() ?? false

‎test/unit-tests/driver/cmakeFileApiDriver.test.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,9 @@ async function cmakeFileApiDriverFactory(cmake: CMakeExecutable, config: Configu
1111
return d;
1212
}
1313

14-
makeDriverTestsuite('FileAPI', cmakeFileApiDriverFactory);
15-
makeCodeModelDriverTestsuite('FileAPI', cmakeFileApiDriverFactory);
14+
function driverSupportsCMake(cmake: CMakeExecutable) {
15+
return cmake.isFileApiModeSupported ?? false;
16+
}
17+
18+
makeDriverTestsuite('FileAPI', cmakeFileApiDriverFactory, driverSupportsCMake);
19+
makeCodeModelDriverTestsuite('FileAPI', cmakeFileApiDriverFactory, driverSupportsCMake);

‎test/unit-tests/driver/cmakeServerDriver.test.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,16 @@ async function cmakeServerDriverFactory(cmake: CMakeExecutable, config: Configur
1010
return d;
1111
}
1212

13+
function driverSupportsCMake(cmake: CMakeExecutable) {
14+
return cmake.isServerModeSupported ?? false;
15+
}
16+
1317
// CMake 3.18.3 has problems on macOS, but we don't have an action to install 3.18.2 right now.
1418
// CMake Server is deprecated and unavailable after 3.20 so we will just skip the tests on macOS.
1519
// We still have coverage on other platforms.
1620
// Also removing support for CMake Server on Windows as we switch to windows-2022 and beyond. CMake version 3.20 and below
1721
// don't have VS 2022 support, and CMake 3.20 and beyond don't have server support, so we will disable the tests on Windows as well.
1822
if (process.platform !== 'darwin' && process.platform !== 'win32') {
19-
makeDriverTestsuite('Server', cmakeServerDriverFactory);
20-
makeCodeModelDriverTestsuite('Server', cmakeServerDriverFactory);
23+
makeDriverTestsuite('Server', cmakeServerDriverFactory, driverSupportsCMake);
24+
makeCodeModelDriverTestsuite('Server', cmakeServerDriverFactory, driverSupportsCMake);
2125
}

‎test/unit-tests/driver/driver-codemodel-tests.ts‎

Lines changed: 25 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@ function getTestRootFilePath(filename: string): string {
2121
}
2222

2323
function cleanupBuildDir(build_dir: string): boolean {
24-
fs.rmSync(build_dir, {recursive: true, force: true});
24+
fs.rmSync(build_dir, { recursive: true, force: true });
2525
return !fs.existsSync(build_dir);
2626
}
2727

2828
let driver: CMakeDriver | null = null;
2929

30-
export function makeCodeModelDriverTestsuite(driverName: string, driver_generator: (cmake: CMakeExecutable, config: ConfigurationReader, kit: Kit | null, workspaceFolder: string, preconditionHandler: CMakePreconditionProblemSolver, preferredGenerators: CMakeGenerator[]) => Promise<CMakeDriver>) {
30+
export function makeCodeModelDriverTestsuite(driverName: string, driver_generator: (cmake: CMakeExecutable, config: ConfigurationReader, kit: Kit | null, workspaceFolder: string, preconditionHandler: CMakePreconditionProblemSolver, preferredGenerators: CMakeGenerator[]) => Promise<CMakeDriver>, driverSupportsCMake: (cmake: CMakeExecutable) => boolean) {
3131
suite(`CMake CodeModel ${driverName} Driver tests`, () => {
3232
const cmakePath: string = process.env.CMAKE_EXECUTABLE ? process.env.CMAKE_EXECUTABLE : 'cmake';
3333
const workspacePath: string = 'test/unit-tests/driver/workspace';
@@ -43,7 +43,7 @@ export function makeCodeModelDriverTestsuite(driverName: string, driver_generato
4343
name: 'Visual Studio Community 2022',
4444
visualStudio: 'VisualStudio.17.0',
4545
visualStudioArchitecture: 'x64',
46-
preferredGenerator: {name: 'Visual Studio 17 2022', platform: 'x64', toolset: 'host=x64'}
46+
preferredGenerator: { name: 'Visual Studio 17 2022', platform: 'x64', toolset: 'host=x64' }
4747
} as Kit;
4848
} else {
4949
kitDefault = { name: 'GCC', compilers: { C: 'gcc', CXX: 'g++' }, preferredGenerator: { name: 'Unix Makefiles' }, isTrusted: true } as Kit;
@@ -74,11 +74,14 @@ export function makeCodeModelDriverTestsuite(driverName: string, driver_generato
7474
}
7575
});
7676

77-
async function generateCodeModelForConfiguredDriver(args: string[] = [],
77+
async function generateCodeModelForConfiguredDriver(testContext: Mocha.Context, args: string[] = [],
7878
workspaceFolder: string = defaultWorkspaceFolder):
7979
Promise<null | CodeModelContent> {
8080
const config = ConfigurationReader.create();
8181
const executable = await getCMakeExecutableInformation(cmakePath);
82+
if (!driverSupportsCMake(executable)) {
83+
testContext.skip();
84+
}
8285

8386
driver = await driver_generator(executable, config, kitDefault, workspaceFolder, async () => true, []);
8487
let code_model: null | CodeModelContent = null;
@@ -91,28 +94,28 @@ export function makeCodeModelDriverTestsuite(driverName: string, driver_generato
9194
return code_model;
9295
}
9396

94-
test('Test generation of code model with multi configuration like VS', async () => {
97+
test('Test generation of code model with multi configuration like VS', async function () {
9598
if (process.platform !== 'win32') {
9699
return;
97100
}
98101

99-
const codemodel_data = await generateCodeModelForConfiguredDriver();
102+
const codemodel_data = await generateCodeModelForConfiguredDriver(this);
100103
expect(codemodel_data).to.be.not.null;
101104
expect(codemodel_data!.configurations.length).to.be.eql(4);
102105
}).timeout(90000);
103106

104-
test('Test generation of code model with one configuration like make on linux', async () => {
107+
test('Test generation of code model with one configuration like make on linux', async function () {
105108
if (process.platform === 'win32') {
106109
return;
107110
}
108111

109-
const codemodel_data = await generateCodeModelForConfiguredDriver();
112+
const codemodel_data = await generateCodeModelForConfiguredDriver(this);
110113
expect(codemodel_data).to.be.not.null;
111114
expect(codemodel_data!.configurations.length).to.be.eql(1);
112115
}).timeout(90000);
113116

114-
test('Test project information', async () => {
115-
const codemodel_data = await generateCodeModelForConfiguredDriver();
117+
test('Test project information', async function () {
118+
const codemodel_data = await generateCodeModelForConfiguredDriver(this);
116119
expect(codemodel_data).to.be.not.null;
117120

118121
const project = codemodel_data!.configurations[0].projects[0];
@@ -126,8 +129,8 @@ export function makeCodeModelDriverTestsuite(driverName: string, driver_generato
126129
.to.eq(path.normalize(path.join(root, 'test_project')).toLowerCase());
127130
}).timeout(90000);
128131

129-
test('Test executable target information', async () => {
130-
const codemodel_data = await generateCodeModelForConfiguredDriver();
132+
test('Test executable target information', async function () {
133+
const codemodel_data = await generateCodeModelForConfiguredDriver(this);
131134
expect(codemodel_data).to.be.not.null;
132135

133136
const target = codemodel_data!.configurations[0].projects[0].targets.find(t => t.type === 'EXECUTABLE' && t.name === 'TestBuildProcess');
@@ -152,8 +155,8 @@ export function makeCodeModelDriverTestsuite(driverName: string, driver_generato
152155
expect(compile_information!.sources).to.include('main.cpp');
153156
}).timeout(90000);
154157

155-
test('Test first static library target directory', async () => {
156-
const codemodel_data = await generateCodeModelForConfiguredDriver();
158+
test('Test first static library target directory', async function () {
159+
const codemodel_data = await generateCodeModelForConfiguredDriver(this);
157160
expect(codemodel_data).to.be.not.null;
158161

159162
const target = codemodel_data!.configurations[0].projects[0].targets.find((t: any) => t.type === 'STATIC_LIBRARY');
@@ -185,8 +188,8 @@ export function makeCodeModelDriverTestsuite(driverName: string, driver_generato
185188
}
186189
}).timeout(90000);
187190

188-
test('Test first shared library target directory', async () => {
189-
const codemodel_data = await generateCodeModelForConfiguredDriver();
191+
test('Test first shared library target directory', async function () {
192+
const codemodel_data = await generateCodeModelForConfiguredDriver(this);
190193
expect(codemodel_data).to.be.not.null;
191194

192195
const target = codemodel_data!.configurations[0].projects[0].targets.find((t: any) => t.type === 'SHARED_LIBRARY');
@@ -222,8 +225,8 @@ export function makeCodeModelDriverTestsuite(driverName: string, driver_generato
222225

223226
}).timeout(90000);
224227

225-
test('Test cache access', async () => {
226-
const codemodel_data = await generateCodeModelForConfiguredDriver();
228+
test('Test cache access', async function () {
229+
const codemodel_data = await generateCodeModelForConfiguredDriver(this);
227230
expect(codemodel_data).to.be.not.null;
228231

229232
const target = codemodel_data!.configurations[0].projects[0].targets.find((t: any) => t.type === 'UTILITY'
@@ -235,25 +238,25 @@ export function makeCodeModelDriverTestsuite(driverName: string, driver_generato
235238
expect(target!.fileGroups![last].isGenerated).to.be.true;
236239
}).timeout(90000);
237240

238-
test('Test sysroot access', async () => {
241+
test('Test sysroot access', async function () {
239242
// This test does not work with VisualStudio.
240243
// VisualStudio generator does not provide the sysroot in the code model.
241244
// macOS has separate sysroot variable (see CMAKE_OSX_SYSROOT); this build fails.
242245
if (process.platform === 'win32' || process.platform === 'darwin') {
243246
return;
244247
}
245248

246-
const codemodel_data = await generateCodeModelForConfiguredDriver(['-DCMAKE_SYSROOT=/tmp']);
249+
const codemodel_data = await generateCodeModelForConfiguredDriver(this, ['-DCMAKE_SYSROOT=/tmp', '-DCMAKE_C_COMPILER_WORKS=1', '-DCMAKE_CXX_COMPILER_WORKS=1']);
247250
expect(codemodel_data).to.be.not.null;
248251

249252
const target = codemodel_data!.configurations[0].projects[0].targets.find((t: any) => t.type === 'EXECUTABLE');
250253
expect(target).to.be.not.undefined;
251254
expect(target!.sysroot).to.be.eq('/tmp');
252255
}).timeout(90000);
253256

254-
test('Test source files outside of workspace root', async () => {
257+
test('Test source files outside of workspace root', async function () {
255258
const project_name: string = 'source_outside_of_workspace';
256-
const codemodel_data = await generateCodeModelForConfiguredDriver([], sourceOutsideOfWorkspace);
259+
const codemodel_data = await generateCodeModelForConfiguredDriver(this, [], sourceOutsideOfWorkspace);
257260
expect(codemodel_data).to.be.not.null;
258261

259262
for (const [target_name, target_subdir, sourcefile_name] of [['root_target', '', '../main.cpp'], ['subdir_target', 'subdir', '../../main.cpp']] as const) {

0 commit comments

Comments
 (0)