Merge branch 'main' of github.com:devcontainers/cli into fea/pkg

Author: trxcllnt

CHANGELOG.md

@@ -4,6 +4,16 @@ Notable changes.
 
 ## January 2023
 
+### [0.29.0]
+
+- Add `set-up` command. (https://github.com/microsoft/vscode-remote-release/issues/7872)
+
+### [0.28.0]
+
+- Features preamble: Add warnings for Feature renames & deprecation. (https://github.com/devcontainers/cli/pull/366)
+- Add dotfiles functionallity. (https://github.com/devcontainers/cli/pull/362)
+- Cache user env for performance improvement. (https://github.com/devcontainers/cli/pull/374)
+
 ### [0.27.1]
 
 - Fix: Modify argument regex to only allow certain set of values (https://github.com/devcontainers/cli/pull/361)

package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@devcontainers/cli",
 	"description": "Dev Containers CLI",
-	"version": "0.27.1",
+	"version": "0.29.0",
 	"bin": {
 		"devcontainer": "devcontainer.js"
 	},

src/spec-common/commonUtils.ts

@@ -346,11 +346,12 @@ async function findLocalWindowsExecutable(command: string, cwd = process.cwd(),
 			}
 		}
 	}
-	// No PATH environment. Make path absolute to the cwd.
+	// No PATH environment. Bail out.
 	if (paths === void 0 || paths.length === 0) {
-		output.write(`findLocalWindowsExecutable: No PATH to look up exectuable '${command}'.`);
-		const fullPath = path.join(cwd, command);
-		return await findLocalWindowsExecutableWithExtension(fullPath) || fullPath;
+		output.write(`findLocalWindowsExecutable: No PATH to look up executable '${command}'.`);
+		const err = new Error(`No PATH to look up executable '${command}'.`);
+		(err as any).code = 'ENOENT';
+		throw err;
 	}
 	// We have a simple file name. We get the path variable from the env
 	// and try to find the executable on the path.
@@ -367,9 +368,11 @@ async function findLocalWindowsExecutable(command: string, cwd = process.cwd(),
 			return withExtension;
 		}
 	}
+	// Not found in PATH. Bail out.
 	output.write(`findLocalWindowsExecutable: Exectuable '${command}' not found on PATH '${pathValue}'.`);
-	const fullPath = path.join(cwd, command);
-	return await findLocalWindowsExecutableWithExtension(fullPath) || fullPath;
+	const err = new Error(`Exectuable '${command}' not found on PATH '${pathValue}'.`);
+	(err as any).code = 'ENOENT';
+	throw err;
 }
 
 const pathext = process.env.PATHEXT;

src/spec-common/dotfiles.ts

@@ -0,0 +1,112 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as path from 'path';
+import { LogLevel } from '../spec-utils/log';
+
+import { ResolverParameters, ContainerProperties, createFileCommand } from './injectHeadless';
+
+const installCommands = [
+	'install.sh',
+	'install',
+	'bootstrap.sh',
+	'bootstrap',
+	'script/bootstrap',
+	'setup.sh',
+	'setup',
+	'script/setup',
+];
+
+export async function installDotfiles(params: ResolverParameters, properties: ContainerProperties, dockerEnvP: Promise<Record<string, string>>) {
+	let { repository, installCommand, targetPath } = params.dotfilesConfiguration;
+	if (!repository) {
+		return;
+	}
+	const dockerEnv = await dockerEnvP;
+	if (repository.indexOf(':') === -1 && !/^\.{0,2}\//.test(repository)) {
+		repository = `https://github.com/${repository}.git`;
+	}
+	const shellServer = properties.shellServer;
+	const markerFile = getDotfilesMarkerFile(properties);
+	const env = Object.keys(dockerEnv)
+		.filter(key => !(key.startsWith('BASH_FUNC_') && key.endsWith('%%')))
+		.reduce((env, key) => `${env}${key}=${quoteValue(dockerEnv[key])} `, '');
+	try {
+		params.output.event({
+			type: 'progress',
+			name: 'Installing Dotfiles',
+			status: 'running',
+		});
+		if (installCommand) {
+			await shellServer.exec(`# Clone & install dotfiles
+${createFileCommand(markerFile)} || (echo dotfiles marker found && exit 1) || exit 0
+command -v git >/dev/null 2>&1 || (echo git not found && exit 1) || exit 0
+[ -e ${targetPath} ] || ${env}git clone ${repository} ${targetPath} || exit $?
+if [ -x ${installCommand} ]
+then
+	echo Executing script ${installCommand}
+	cd ${targetPath} && ${env}${installCommand}
+else
+	echo Error: ${installCommand} not executable
+	exit 126
+fi
+`, { logOutput: 'continuous', logLevel: LogLevel.Info });
+		} else {
+			await shellServer.exec(`# Clone & install dotfiles
+${createFileCommand(markerFile)} || (echo dotfiles marker found && exit 1) || exit 0
+command -v git >/dev/null 2>&1 || (echo git not found && exit 1) || exit 0
+[ -e ${targetPath} ] || ${env}git clone ${repository} ${targetPath} || exit $?
+cd ${targetPath}
+for f in ${installCommands.join(' ')}
+do
+	if [ -e $f ]
+	then
+		installCommand=$f
+		break
+	fi
+done
+if [ -z "$installCommand" ]
+then
+	dotfiles=$(ls -d ${targetPath}/.* 2>/dev/null | grep -v -E '/(.|..|.git)$')
+	if [ ! -z "$dotfiles" ]
+	then
+		echo Linking dotfiles: $dotfiles
+		ln -sf $dotfiles ~ 2>/dev/null
+	else
+		echo No dotfiles found.
+	fi
+else
+	if [ -x "$installCommand" ]
+	then
+		echo Executing script '${targetPath}'/"$installCommand"
+		${env}./"$installCommand"
+	else
+		echo Error: '${targetPath}'/"$installCommand" not executable
+		exit 126
+	fi
+fi
+`, { logOutput: 'continuous', logLevel: LogLevel.Info });
+		}
+		params.output.event({
+			type: 'progress',
+			name: 'Installing Dotfiles',
+			status: 'succeeded',
+		});
+	} catch (err) {
+		params.output.event({
+			type: 'progress',
+			name: 'Installing Dotfiles',
+			status: 'failed',
+		});
+	}
+}
+
+function quoteValue(value: string | undefined) {
+	return `'${(value || '').replace(/'+/g, '\'"$&"\'')}'`;
+}
+
+function getDotfilesMarkerFile(properties: ContainerProperties) {
+	return path.posix.join(properties.userDataFolder, '.dotfilesMarker');
+}

src/spec-common/injectHeadless.ts

@@ -19,6 +19,7 @@ import { containerSubstitute } from './variableSubstitution';
 import { delay } from './async';
 import { Log, LogEvent, LogLevel, makeLog, nullLog } from '../spec-utils/log';
 import { buildProcessTrees, findProcesses, Process, processTreeToString } from './proc';
+import { installDotfiles } from './dotfiles';
 
 export enum ResolverProgress {
 	Begin,
@@ -44,6 +45,7 @@ export interface ResolverParameters {
 	env: NodeJS.ProcessEnv;
 	cwd: string;
 	isLocalContainer: boolean;
+	dotfilesConfiguration: DotfilesConfiguration;
 	progress: (current: ResolverProgress) => void;
 	output: Log;
 	allowSystemConfigChange: boolean;
@@ -62,6 +64,7 @@ export interface ResolverParameters {
 	skipFeatureAutoMapping: boolean;
 	skipPostAttach: boolean;
 	experimentalImageMetadata: boolean;
+	containerSessionDataFolder?: string;
 	skipPersistingCustomizationsFromFeatures: boolean;
 }
 
@@ -185,6 +188,12 @@ export interface ContainerProperties {
 	launchRootShellServer?: () => Promise<ShellServer>;
 }
 
+export interface DotfilesConfiguration {
+	repository: string | undefined;
+	installCommand: string | undefined;
+	targetPath: string;
+}
+
 export async function getContainerProperties(options: {
 	params: ResolverParameters;
 	createdAt: string | undefined;
@@ -347,6 +356,10 @@ export async function runPostCreateCommands(params: ResolverParameters, containe
 		return 'skipNonBlocking';
 	}
 
+	if (params.dotfilesConfiguration) {
+		await installDotfiles(params, containerProperties, remoteEnv);
+	}
+
 	if (stopForPersonalization) {
 		return 'stopForPersonalization';
 	}
@@ -513,7 +526,7 @@ async function createFile(shellServer: ShellServer, location: string) {
 	}
 }
 
-function createFileCommand(location: string) {
+export function createFileCommand(location: string) {
 	return `test ! -f '${location}' && set -o noclobber && mkdir -p '${path.posix.dirname(location)}' && { > '${location}' ; } 2> /dev/null`;
 }
 
@@ -655,22 +668,72 @@ async function patchEtcProfile(params: ResolverParameters, containerProperties:
 	}
 }
 
-async function probeUserEnv(params: { defaultUserEnvProbe: UserEnvProbe; allowSystemConfigChange: boolean; output: Log }, containerProperties: { shell: string; remoteExec: ExecFunction; installFolder?: string; env?: NodeJS.ProcessEnv; shellServer?: ShellServer; launchRootShellServer?: (() => Promise<ShellServer>); user?: string }, config?: CommonMergedDevContainerConfig) {
-	const env = await runUserEnvProbe(params, containerProperties, config, 'cat /proc/self/environ', '\0');
+async function probeUserEnv(params: { defaultUserEnvProbe: UserEnvProbe; allowSystemConfigChange: boolean; output: Log; containerSessionDataFolder?: string }, containerProperties: { shell: string; remoteExec: ExecFunction; installFolder?: string; env?: NodeJS.ProcessEnv; shellServer?: ShellServer; launchRootShellServer?: (() => Promise<ShellServer>); user?: string }, config?: CommonMergedDevContainerConfig) {
+	let userEnvProbe = getUserEnvProb(config, params);
+	if (!userEnvProbe || userEnvProbe === 'none') {
+		return {};
+	}
+
+	let env = await readUserEnvFromCache(userEnvProbe, params, containerProperties.shellServer);
 	if (env) {
 		return env;
 	}
-	params.output.write('userEnvProbe: falling back to printenv');
-	const env2 = await runUserEnvProbe(params, containerProperties, config, 'printenv', '\n');
-	return env2 || {};
+
+	params.output.write('userEnvProbe: not found in cache');
+	env = await runUserEnvProbe(userEnvProbe, params, containerProperties, 'cat /proc/self/environ', '\0');
+	if (!env) {
+		params.output.write('userEnvProbe: falling back to printenv');
+		env = await runUserEnvProbe(userEnvProbe, params, containerProperties, 'printenv', '\n');
+	}
+
+	if (env) {
+		await updateUserEnvCache(env, userEnvProbe, params, containerProperties.shellServer);
+	}
+
+	return env || {};
 }
 
-async function runUserEnvProbe(params: { defaultUserEnvProbe: UserEnvProbe; allowSystemConfigChange: boolean; output: Log }, containerProperties: { shell: string; remoteExec: ExecFunction; installFolder?: string; env?: NodeJS.ProcessEnv; shellServer?: ShellServer; launchRootShellServer?: (() => Promise<ShellServer>); user?: string }, config: CommonMergedDevContainerConfig | undefined, cmd: string, sep: string) {
-	let userEnvProbe = config?.userEnvProbe;
-	params.output.write(`userEnvProbe: ${userEnvProbe || params.defaultUserEnvProbe}${userEnvProbe ? '' : ' (default)'}`);
-	if (!userEnvProbe) {
-		userEnvProbe = params.defaultUserEnvProbe;
+async function readUserEnvFromCache(userEnvProbe: UserEnvProbe, params: { output: Log; containerSessionDataFolder?: string }, shellServer?: ShellServer) {
+	if (!shellServer || !params.containerSessionDataFolder) {
+		return undefined;
+	}
+
+	const cacheFile = getUserEnvCacheFilePath(userEnvProbe, params.containerSessionDataFolder);
+	try {
+		if (await isFile(shellServer, cacheFile)) {
+			const { stdout } = await shellServer.exec(`cat '${cacheFile}'`);
+			return JSON.parse(stdout);
+		}
 	}
+	catch (e) {
+		params.output.write(`Failed to read/parse user env cache: ${e}`, LogLevel.Error);
+	}
+
+	return undefined;
+}
+
+async function updateUserEnvCache(env: Record<string, string>, userEnvProbe: UserEnvProbe, params: { output: Log; containerSessionDataFolder?: string }, shellServer?: ShellServer) {
+	if (!shellServer || !params.containerSessionDataFolder) {
+		return;
+	}
+
+	const cacheFile = getUserEnvCacheFilePath(userEnvProbe, params.containerSessionDataFolder);
+	try {
+		await shellServer.exec(`mkdir -p '${path.posix.dirname(cacheFile)}' && cat > '${cacheFile}' << 'envJSON'
+${JSON.stringify(env, null, '\t')}
+envJSON
+`);
+	}
+	catch (e) {
+		params.output.write(`Failed to cache user env: ${e}`, LogLevel.Error);
+	}
+}
+
+function getUserEnvCacheFilePath(userEnvProbe: UserEnvProbe, cacheFolder: string): string {
+	return path.posix.join(cacheFolder, `env-${userEnvProbe}.json`);
+}
+
+async function runUserEnvProbe(userEnvProbe: UserEnvProbe, params: { allowSystemConfigChange: boolean; output: Log }, containerProperties: { shell: string; remoteExec: ExecFunction; installFolder?: string; env?: NodeJS.ProcessEnv; shellServer?: ShellServer; launchRootShellServer?: (() => Promise<ShellServer>); user?: string }, cmd: string, sep: string) {
 	if (userEnvProbe === 'none') {
 		return {};
 	}
@@ -767,6 +830,15 @@ Merged:    ${typeof env.PATH === 'string' ? `'${env.PATH}'` : 'None'}` : ''}`);
 	}
 }
 
+function getUserEnvProb(config: CommonMergedDevContainerConfig | undefined, params: { defaultUserEnvProbe: UserEnvProbe; allowSystemConfigChange: boolean; output: Log }) {
+	let userEnvProbe = config?.userEnvProbe;
+	params.output.write(`userEnvProbe: ${userEnvProbe || params.defaultUserEnvProbe}${userEnvProbe ? '' : ' (default)'}`);
+	if (!userEnvProbe) {
+		userEnvProbe = params.defaultUserEnvProbe;
+	}
+	return userEnvProbe;
+}
+
 function mergePaths(shellPath: string, containerPath: string, rootUser: boolean) {
 	const result = shellPath.split(':');
 	let insertAt = 0;

src/spec-common/variableSubstitution.ts

@@ -11,9 +11,9 @@ import { URI } from 'vscode-uri';
 
 export interface SubstitutionContext {
 	platform: NodeJS.Platform;
-	configFile: URI;
-	localWorkspaceFolder: string | undefined;
-	containerWorkspaceFolder: string | undefined;
+	configFile?: URI;
+	localWorkspaceFolder?: string;
+	containerWorkspaceFolder?: string;
 	env: NodeJS.ProcessEnv;
 }
 
@@ -33,9 +33,9 @@ export function substitute<T extends object>(context: SubstitutionContext, value
 	return substitute0(replace, value);
 }
 
-export function beforeContainerSubstitute<T extends object>(idLabels: Record<string, string>, value: T): T {
+export function beforeContainerSubstitute<T extends object>(idLabels: Record<string, string> | undefined, value: T): T {
 	let devcontainerId: string | undefined;
-	return substitute0(replaceDevContainerId.bind(undefined, () => devcontainerId || (devcontainerId = devcontainerIdForLabels(idLabels))), value);
+	return substitute0(replaceDevContainerId.bind(undefined, () => devcontainerId || (idLabels && (devcontainerId = devcontainerIdForLabels(idLabels)))), value);
 }
 
 export function containerSubstitute<T extends object>(platform: NodeJS.Platform, configFile: URI | undefined, containerEnv: NodeJS.ProcessEnv, value: T): T {
@@ -124,10 +124,10 @@ function replaceContainerEnv(isWindows: boolean, configFile: URI | undefined, co
 	}
 }
 
-function replaceDevContainerId(getDevContainerId: () => string, match: string, variable: string) {
+function replaceDevContainerId(getDevContainerId: () => string | undefined, match: string, variable: string) {
 	switch (variable) {
 		case 'devcontainerId':
-			return getDevContainerId();
+			return getDevContainerId() || match;
 
 		default:
 			return match;

src/spec-configuration/configuration.ts

@@ -39,8 +39,8 @@ export interface DevContainerFeature {
 }
 
 export interface DevContainerFromImageConfig {
-	configFilePath: URI;
-	image: string;
+	configFilePath?: URI;
+	image?: string; // Only optional when setting up an existing container as a dev container.
 	name?: string;
 	forwardPorts?: (number | string)[];
 	appPort?: number | string | (number | string)[];