Skip to content

Commit 1caac56

Browse files
axelonetaxelonet
committed
fix(NODE-6795): support build-from-source installs
1 parent 0c7efad commit 1caac56

4 files changed

Lines changed: 329 additions & 4 deletions

File tree

‎etc/get-commit-from-ref.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/usr/bin/env bash
2+
set -o errexit
3+
4+
git clone https://github.com/mongodb/libmongocrypt.git _libmongocrypt
5+
cd _libmongocrypt
6+
git fetch --tags
7+
8+
if [[ "${REF}" != "latest" ]]; then
9+
git checkout "${REF}"
10+
fi
11+
12+
echo "COMMIT_HASH=$(git rev-parse HEAD)"
13+
14+
cd ..
15+
rm -rf _libmongocrypt

‎etc/libmongocrypt.mjs‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
// @ts-check
2+
3+
import util from 'node:util';
4+
import process from 'node:process';
5+
import fs, { readFile } from 'node:fs/promises';
6+
import child_process from 'node:child_process';
7+
import events from 'node:events';
8+
import path from 'node:path';
9+
import https from 'node:https';
10+
import stream from 'node:stream/promises';
11+
import {
12+
buildLibmongocryptDownloadUrl,
13+
getLibmongocryptPrebuildName,
14+
resolveRoot,
15+
run
16+
} from './utils.mjs';
17+
18+
async function parseArguments() {
19+
const pkg = JSON.parse(await fs.readFile(resolveRoot('package.json'), 'utf8'));
20+
21+
const options = {
22+
gitURL: { short: 'u', type: 'string', default: 'https://github.com/mongodb/libmongocrypt.git' },
23+
libVersion: { short: 'l', type: 'string', default: pkg['mongodb:libmongocrypt'] },
24+
clean: { short: 'c', type: 'boolean', default: false },
25+
build: { short: 'b', type: 'boolean', default: false },
26+
dynamic: { type: 'boolean', default: false },
27+
'skip-bindings': { type: 'boolean', default: false },
28+
help: { short: 'h', type: 'boolean', default: false }
29+
};
30+
31+
const args = util.parseArgs({ args: process.argv.slice(2), options, allowPositionals: false });
32+
33+
if (args.values.help) {
34+
console.log(
35+
`${path.basename(process.argv[1])} ${[...Object.keys(options)]
36+
.filter(k => k !== 'help')
37+
.map(k => `[--${k}=${options[k].type}]`)
38+
.join(' ')}`
39+
);
40+
process.exit(0);
41+
}
42+
43+
return {
44+
url: args.values.gitURL,
45+
ref: args.values.libVersion,
46+
clean: args.values.clean,
47+
build: args.values.build,
48+
dynamic: args.values.dynamic,
49+
skipBindings: args.values['skip-bindings'],
50+
pkg
51+
};
52+
}
53+
54+
export async function cloneLibMongoCrypt(libmongocryptRoot, { url, ref }) {
55+
console.error('fetching libmongocrypt...', { url, ref });
56+
await fs.rm(libmongocryptRoot, { recursive: true, force: true });
57+
await run('git', ['clone', url, libmongocryptRoot]);
58+
if (ref !== 'latest') {
59+
await run('git', ['fetch', '--tags'], { cwd: libmongocryptRoot });
60+
await run('git', ['checkout', ref, '-b', `r-${ref}`], { cwd: libmongocryptRoot });
61+
}
62+
}
63+
64+
export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, options) {
65+
function toCLIFlags(object) {
66+
return Array.from(Object.entries(object)).map(([k, v]) => `-${k}=${v}`);
67+
}
68+
69+
console.error('building libmongocrypt...');
70+
71+
const nodeBuildRoot = resolveRoot(nodeDepsRoot, 'tmp', 'libmongocrypt-build');
72+
73+
await fs.rm(nodeBuildRoot, { recursive: true, force: true });
74+
await fs.mkdir(nodeBuildRoot, { recursive: true });
75+
76+
const CMAKE_FLAGS = toCLIFlags({
77+
DDISABLE_NATIVE_CRYPTO: '1',
78+
DCMAKE_INSTALL_LIBDIR: 'lib',
79+
DENABLE_MORE_WARNINGS_AS_ERRORS: 'ON',
80+
DCMAKE_PREFIX_PATH: nodeDepsRoot,
81+
DCMAKE_INSTALL_PREFIX: nodeDepsRoot
82+
});
83+
84+
const WINDOWS_CMAKE_FLAGS =
85+
process.platform === 'win32'
86+
? toCLIFlags({
87+
DCMAKE_MSVC_RUNTIME_LIBRARY: 'MultiThreaded',
88+
T: 'host=x64',
89+
A: 'x64'
90+
})
91+
: [];
92+
93+
const DARWIN_CMAKE_FLAGS =
94+
process.platform === 'darwin' ? toCLIFlags({ DCMAKE_OSX_DEPLOYMENT_TARGET: '10.12' }) : [];
95+
96+
const cmakeProgram = process.platform === 'win32' ? 'cmake.exe' : 'cmake';
97+
98+
await run(
99+
cmakeProgram,
100+
[...CMAKE_FLAGS, ...WINDOWS_CMAKE_FLAGS, ...DARWIN_CMAKE_FLAGS, libmongocryptRoot],
101+
{ cwd: nodeBuildRoot, shell: process.platform === 'win32' }
102+
);
103+
104+
await run(cmakeProgram, ['--build', '.', '--target', 'install', '--config', 'RelWithDebInfo'], {
105+
cwd: nodeBuildRoot,
106+
shell: process.platform === 'win32'
107+
});
108+
}
109+
110+
export async function downloadLibMongoCrypt(nodeDepsRoot, { ref }) {
111+
const prebuild = getLibmongocryptPrebuildName();
112+
const downloadURL = buildLibmongocryptDownloadUrl(ref, prebuild);
113+
114+
console.error('downloading libmongocrypt...', downloadURL);
115+
const destination = resolveRoot(`_libmongocrypt-${ref}`);
116+
117+
await fs.rm(destination, { recursive: true, force: true });
118+
await fs.mkdir(destination);
119+
120+
const downloadDestination = 'nocrypto';
121+
const unzipArgs = ['-xzv', '-C', `_libmongocrypt-${ref}`, downloadDestination];
122+
console.error(`+ tar ${unzipArgs.join(' ')}`);
123+
const unzip = child_process.spawn('tar', unzipArgs, {
124+
stdio: ['pipe', 'inherit', 'pipe'],
125+
cwd: resolveRoot('.')
126+
});
127+
if (unzip.stdin == null) throw new Error('Tar process must have piped stdin');
128+
129+
const [response] = await events.once(https.get(downloadURL), 'response');
130+
const start = performance.now();
131+
132+
try {
133+
await stream.pipeline(response, unzip.stdin);
134+
} catch {
135+
await fs.access(path.join(`_libmongocrypt-${ref}`, downloadDestination));
136+
}
137+
138+
const end = performance.now();
139+
console.error(`downloaded libmongocrypt in ${(end - start) / 1000} secs...`);
140+
141+
await fs.rm(nodeDepsRoot, { recursive: true, force: true });
142+
await fs.cp(resolveRoot(destination, 'nocrypto'), nodeDepsRoot, { recursive: true });
143+
const potentialLib64Path = path.join(nodeDepsRoot, 'lib64');
144+
try {
145+
await fs.rename(potentialLib64Path, path.join(nodeDepsRoot, 'lib'));
146+
} catch {
147+
await fs.access(path.join(nodeDepsRoot, 'lib'));
148+
}
149+
}
150+
151+
async function buildBindings(args, pkg) {
152+
await fs.rm(resolveRoot('build'), { force: true, recursive: true });
153+
await fs.rm(resolveRoot('prebuilds'), { force: true, recursive: true });
154+
155+
await run('npm', ['install', '--ignore-scripts']);
156+
157+
let gypDefines = process.env.GYP_DEFINES ?? '';
158+
if (args.dynamic) {
159+
gypDefines += ' libmongocrypt_link_type=dynamic';
160+
}
161+
162+
gypDefines = gypDefines.trim();
163+
const prebuildOptions =
164+
gypDefines.length > 0 ? { env: { ...process.env, GYP_DEFINES: gypDefines } } : undefined;
165+
166+
await run('npm', ['run', 'prebuild'], prebuildOptions);
167+
await run('npm', ['run', 'prepare']);
168+
169+
if (process.platform === 'darwin' && process.arch === 'arm64') {
170+
const {
171+
binary: {
172+
napi_versions: [napiVersion]
173+
}
174+
} = JSON.parse(await readFile(resolveRoot('package.json'), 'utf-8'));
175+
const armTar = `mongodb-client-encryption-v${pkg.version}-napi-v${napiVersion}-darwin-arm64.tar.gz`;
176+
const x64Tar = `mongodb-client-encryption-v${pkg.version}-napi-v${napiVersion}-darwin-x64.tar.gz`;
177+
await fs.copyFile(resolveRoot('prebuilds', armTar), resolveRoot('prebuilds', x64Tar));
178+
}
179+
}
180+
181+
async function main() {
182+
const { pkg, ...args } = await parseArguments();
183+
console.log(args);
184+
185+
const nodeDepsDir = resolveRoot('deps');
186+
187+
if (args.build && !args.dynamic) {
188+
const libmongocryptCloneDir = resolveRoot('_libmongocrypt');
189+
190+
const currentLibMongoCryptBranch = await fs
191+
.readFile(path.join(libmongocryptCloneDir, '.git', 'HEAD'), 'utf8')
192+
.catch(() => '');
193+
const isClonedAndCheckedOut = currentLibMongoCryptBranch.trim().endsWith(`r-${args.ref}`);
194+
195+
if (args.clean || !isClonedAndCheckedOut) {
196+
await cloneLibMongoCrypt(libmongocryptCloneDir, args);
197+
}
198+
199+
const libmongocryptBuiltVersion = await fs
200+
.readFile(path.join(libmongocryptCloneDir, 'VERSION_CURRENT'), 'utf8')
201+
.catch(() => '');
202+
const isBuilt = libmongocryptBuiltVersion.trim() === args.ref;
203+
204+
if (args.clean || !isBuilt) {
205+
await buildLibMongoCrypt(libmongocryptCloneDir, nodeDepsDir, args);
206+
}
207+
} else if (!args.dynamic) {
208+
await downloadLibMongoCrypt(nodeDepsDir, args);
209+
}
210+
211+
if (!args.skipBindings) {
212+
await buildBindings(args, pkg);
213+
}
214+
}
215+
216+
await main();

‎etc/utils.mjs‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
// @ts-check
2+
3+
import path from 'node:path';
4+
import url from 'node:url';
5+
import { spawn, execSync } from 'node:child_process';
6+
import { once } from 'node:events';
7+
8+
const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
9+
10+
export function resolveRoot(...paths) {
11+
return path.resolve(__dirname, '..', ...paths);
12+
}
13+
14+
export function getCommitFromRef(ref) {
15+
console.error(`resolving ref: ${ref}`);
16+
const script = resolveRoot('etc', 'get-commit-from-ref.sh');
17+
const output = execSync(`bash ${script}`, { env: { REF: ref }, encoding: 'utf-8' });
18+
19+
const regex = /COMMIT_HASH=(?<hash>[a-zA-Z0-9]+)/;
20+
const result = regex.exec(output);
21+
22+
if (!result?.groups) throw new Error('unable to parse ref.');
23+
24+
const { hash } = result.groups;
25+
console.error(`resolved to: ${hash}`);
26+
return hash;
27+
}
28+
29+
export function buildLibmongocryptDownloadUrl(ref, platform) {
30+
const hash = getCommitFromRef(ref);
31+
32+
if (ref.includes('.')) {
33+
const [major, minor] = ref.split('.');
34+
const branch = `r${major}.${minor}`;
35+
return `https://mciuploads.s3.amazonaws.com/libmongocrypt-release/${platform}/${branch}/${hash}/libmongocrypt.tar.gz`;
36+
}
37+
38+
return `https://mciuploads.s3.amazonaws.com/libmongocrypt/${platform}/master/${hash}/libmongocrypt.tar.gz`;
39+
}
40+
41+
export function getLibmongocryptPrebuildName() {
42+
const prebuildIdentifierFactory = {
43+
darwin: () => 'macos',
44+
win32: () => 'windows-test',
45+
linux: () => {
46+
const key = `${getLibc()}-${process.arch}`;
47+
return {
48+
'musl-x64': 'alpine-amd64-earthly',
49+
'musl-arm64': 'alpine-arm64-earthly',
50+
'glibc-ppc64': 'rhel-71-ppc64el',
51+
'glibc-s390x': 'rhel72-zseries-test',
52+
'glibc-arm64': 'ubuntu1804-arm64',
53+
'glibc-x64': 'rhel-70-64-bit'
54+
}[key];
55+
}
56+
}[process.platform] ?? (() => {
57+
throw new Error('Unsupported platform');
58+
});
59+
60+
return prebuildIdentifierFactory();
61+
}
62+
63+
export async function run(command, args = [], options = {}) {
64+
const commandDetails = `+ ${command} ${args.join(' ')}${options.cwd ? ` (in: ${options.cwd})` : ''}`;
65+
console.error(commandDetails);
66+
const proc = spawn(command, args, {
67+
shell: process.platform === 'win32',
68+
stdio: 'inherit',
69+
cwd: resolveRoot('.'),
70+
...options
71+
});
72+
await once(proc, 'exit');
73+
74+
if (proc.exitCode != 0) throw new Error(`CRASH(${proc.exitCode}): ${commandDetails}`);
75+
}
76+
77+
function getLibc() {
78+
if (process.platform !== 'linux') return null;
79+
80+
function lddVersion() {
81+
try {
82+
return execSync('ldd --version', { encoding: 'utf-8' });
83+
} catch (error) {
84+
return error.stderr;
85+
}
86+
}
87+
88+
console.error({ ldd: lddVersion() });
89+
return lddVersion().includes('musl') ? 'musl' : 'glibc';
90+
}

‎package.json‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,18 @@
1010
"addon",
1111
"lib",
1212
"src",
13-
"binding.gyp"
13+
"binding.gyp",
14+
"etc/libmongocrypt.mjs",
15+
"etc/utils.mjs",
16+
"etc/get-commit-from-ref.sh"
1417
],
1518
"directories": {
1619
"lib": "lib"
1720
},
1821
"scripts": {
19-
"install:libmongocrypt": "node .github/scripts/libmongocrypt.mjs",
20-
"install": "prebuild-install --runtime napi || node-gyp rebuild",
22+
"install:libmongocrypt": "node etc/libmongocrypt.mjs",
23+
"clean-install": "node etc/libmongocrypt.mjs --skip-bindings && node-gyp rebuild",
24+
"install": "prebuild-install --runtime napi || npm run clean-install",
2125
"clang-format": "clang-format --style=file:.clang-format --Werror -i addon/*",
2226
"check:eslint": "ESLINT_USE_FLAT_CONFIG=false eslint src test",
2327
"check:clang-format": "clang-format --style=file:.clang-format --dry-run --Werror addon/*",
@@ -91,4 +95,4 @@
9195
"moduleResolution": "node"
9296
}
9397
}
94-
}
98+
}

0 commit comments

Comments
 (0)