feat: support build secrets in features

This adds support for using Docker build secrets in devcontainer
features, allowing sensitive information to be used during the build
process without being included in the final image.

Author: KarstenB

src/spec-configuration/containerFeaturesConfiguration.ts

@@ -290,7 +290,17 @@ function escapeQuotesForShell(input: string) {
 	return input.replace(new RegExp(`'`, 'g'), `'\\''`);
 }
 
-export function getFeatureLayers(featuresConfig: FeaturesConfig, containerUser: string, remoteUser: string, useBuildKitBuildContexts = false, contentSourceRootPath = '/tmp/build-features') {
+export interface BuildSecret {
+	id: string;
+	file?: string;
+	env?: string;
+}
+
+function getSecretMounts(buildSecrets: BuildSecret[]): string {
+	return buildSecrets.map(secret => `--mount=type=secret,id=${secret.id}`).join(' ');
+}
+
+export function getFeatureLayers(featuresConfig: FeaturesConfig, containerUser: string, remoteUser: string, useBuildKitBuildContexts = false, contentSourceRootPath = '/tmp/build-features', buildSecrets: BuildSecret[] = []) {
 
 	const builtinsEnvFile = `${path.posix.join(FEATURES_CONTAINER_TEMP_DEST_FOLDER, 'devcontainer-features.builtin.env')}`;
 	let result = `RUN \\
@@ -313,8 +323,10 @@ RUN chmod -R 0755 ${dest} \\
 
 `;
 		} else {
-			result += `RUN --mount=type=bind,from=dev_containers_feature_content_source,source=${source},target=/tmp/build-features-src/${folder} \\
-    cp -ar /tmp/build-features-src/${folder} ${FEATURES_CONTAINER_TEMP_DEST_FOLDER} \\
+			const secretMounts = getSecretMounts(buildSecrets);
+			const runPrefix = secretMounts ? `RUN ${secretMounts} ` : 'RUN ';
+			result += `${runPrefix}--mount=type=bind,from=dev_containers_feature_content_source,source=${source},target=/tmp/build-features-src/${folder} \\
+	cp -ar /tmp/build-features-src/${folder} ${FEATURES_CONTAINER_TEMP_DEST_FOLDER} \\
  && chmod -R 0755 ${dest} \\
  && cd ${dest} \\
  && chmod +x ./install.sh \\
@@ -340,9 +352,11 @@ RUN chmod -R 0755 ${dest} \\
 
 `;
 			} else {
+				const secretMounts = getSecretMounts(buildSecrets);
+				const runPrefix = secretMounts ? `RUN ${secretMounts} ` : 'RUN ';
 				result += `
-RUN --mount=type=bind,from=dev_containers_feature_content_source,source=${source},target=/tmp/build-features-src/${feature.consecutiveId} \\
-    cp -ar /tmp/build-features-src/${feature.consecutiveId} ${FEATURES_CONTAINER_TEMP_DEST_FOLDER} \\
+${runPrefix}--mount=type=bind,from=dev_containers_feature_content_source,source=${source},target=/tmp/build-features-src/${feature.consecutiveId} \\
+	cp -ar /tmp/build-features-src/${feature.consecutiveId} ${FEATURES_CONTAINER_TEMP_DEST_FOLDER} \\
  && chmod -R 0755 ${dest} \\
  && cd ${dest} \\
  && chmod +x ./devcontainer-features-install.sh \\

src/spec-node/containerFeatures.ts

@@ -99,6 +99,14 @@ export async function extendImage(params: DockerResolverParameters, config: Subs
 		for (const securityOpt of featureBuildInfo.securityOpts) {
 			args.push('--security-opt', securityOpt);
 		}
+
+		for (const secret of params.buildSecrets) {
+			if (secret.file) {
+				args.push('--secret', `id=${secret.id},src=${secret.file}`);
+			} else if (secret.env) {
+				args.push('--secret', `id=${secret.id},env=${secret.env}`);
+			}
+		}
 	} else {
 		// Not using buildx
 		args.push(
@@ -263,7 +271,7 @@ async function getFeaturesBuildOptions(params: DockerResolverParameters, devCont
 	const contentSourceRootPath = useBuildKitBuildContexts ? '.' : '/tmp/build-features/';
 	const dockerfile = getContainerFeaturesBaseDockerFile(contentSourceRootPath)
 		.replace('#{nonBuildKitFeatureContentFallback}', useBuildKitBuildContexts ? '' : `FROM ${buildContentImageName} as dev_containers_feature_content_source`)
-		.replace('#{featureLayer}', getFeatureLayers(featuresConfig, containerUser, remoteUser, useBuildKitBuildContexts, contentSourceRootPath))
+		.replace('#{featureLayer}', getFeatureLayers(featuresConfig, containerUser, remoteUser, useBuildKitBuildContexts, contentSourceRootPath, params.buildSecrets))
 		.replace('#{containerEnv}', generateContainerEnvsV1(featuresConfig))
 		.replace('#{devcontainerMetadata}', getDevcontainerMetadataLabel(imageMetadata))
 		.replace('#{containerEnvMetadata}', generateContainerEnvs(devContainerConfig.config.containerEnv, true))

src/spec-node/devContainers.ts

@@ -8,7 +8,7 @@ import * as crypto from 'crypto';
 import * as os from 'os';
 
 import { mapNodeOSToGOOS, mapNodeArchitectureToGOARCH } from '../spec-configuration/containerCollectionsOCI';
-import { DockerResolverParameters, DevContainerAuthority, UpdateRemoteUserUIDDefault, BindMountConsistency, getCacheFolder, GPUAvailability } from './utils';
+import { DockerResolverParameters, DevContainerAuthority, UpdateRemoteUserUIDDefault, BindMountConsistency, getCacheFolder, GPUAvailability, BuildSecret } from './utils';
 import { createNullLifecycleHook, finishBackgroundTasks, ResolverParameters, UserEnvProbe } from '../spec-common/injectHeadless';
 import { GoARCH, GoOS, getCLIHost, loadNativeModule } from '../spec-common/commonUtils';
 import { resolve } from './configContainer';
@@ -74,6 +74,7 @@ export interface ProvisionOptions {
 	omitSyntaxDirective?: boolean;
 	includeConfig?: boolean;
 	includeMergedConfig?: boolean;
+	buildSecrets: BuildSecret[];
 }
 
 export async function launch(options: ProvisionOptions, providedIdLabels: string[] | undefined, disposables: (() => Promise<unknown> | undefined)[]) {
@@ -246,7 +247,8 @@ export async function createDockerParams(options: ProvisionOptions, disposables:
 		additionalLabels: options.additionalLabels,
 		buildxOutput: common.buildxOutput,
 		buildxCacheTo: common.buildxCacheTo,
-		platformInfo
+		platformInfo,
+		buildSecrets: options.buildSecrets
 	};
 }
 

src/spec-node/devContainersSpecCLI.ts

@@ -10,7 +10,7 @@ import textTable from 'text-table';
 import * as jsonc from 'jsonc-parser';
 
 import { createDockerParams, createLog, launch, ProvisionOptions } from './devContainers';
-import { SubstitutedConfig, createContainerProperties, envListToObj, inspectDockerImage, isDockerFileConfig, SubstituteConfig, addSubstitution, findContainerAndIdLabels, getCacheFolder, runAsyncHandler } from './utils';
+import { SubstitutedConfig, createContainerProperties, envListToObj, inspectDockerImage, isDockerFileConfig, SubstituteConfig, addSubstitution, findContainerAndIdLabels, getCacheFolder, runAsyncHandler, BuildSecret } from './utils';
 import { URI } from 'vscode-uri';
 import { ContainerError } from '../spec-common/errors';
 import { Log, LogDimensions, LogLevel, makeLog, mapLogLevel } from '../spec-utils/log';
@@ -50,6 +50,43 @@ const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell';
 
 const mountRegex = /^type=(bind|volume),source=([^,]+),target=([^,]+)(?:,external=(true|false))?$/;
 
+function parseBuildSecrets(buildSecretsArg: string[] | undefined): BuildSecret[] {
+	if (!buildSecretsArg) {
+		return [];
+	}
+	const secrets = Array.isArray(buildSecretsArg) ? buildSecretsArg : [buildSecretsArg];
+	return secrets.map(secret => {
+		// Support shorthand: id=name (assumes env=name)
+		const shorthandMatch = secret.match(/^id=([^,]+)$/);
+		if (shorthandMatch) {
+			return {
+				id: shorthandMatch[1],
+				env: shorthandMatch[1].toUpperCase()
+			};
+		}
+
+		// Support file format: id=name,src=path
+		const fileMatch = secret.match(/^id=([^,]+),src=(.+)$/);
+		if (fileMatch) {
+			return {
+				id: fileMatch[1],
+				file: path.resolve(process.cwd(), fileMatch[2])
+			};
+		}
+
+		// Support env format: id=name,env=VAR
+		const envMatch = secret.match(/^id=([^,]+),env=(.+)$/);
+		if (envMatch) {
+			return {
+				id: envMatch[1],
+				env: envMatch[2]
+			};
+		}
+
+		throw new Error(`Invalid build-secret format: ${secret}. Supported formats are: "id=<id>,src=<path>", "id=<id>,env=<var>", or "id=<id>" (which assumes env=<ID>).`);
+	});
+}
+
 (async () => {
 
 	const packageFolder = path.join(__dirname, '..', '..');
@@ -138,6 +175,7 @@ function provisionOptions(y: Argv) {
 		'container-session-data-folder': { type: 'string', description: 'Folder to cache CLI data, for example userEnvProbe results' },
 		'omit-config-remote-env-from-metadata': { type: 'boolean', default: false, hidden: true, description: 'Omit remoteEnv from devcontainer.json for container metadata label' },
 		'secrets-file': { type: 'string', description: 'Path to a json file containing secret environment variables as key-value pairs.' },
+		'build-secret': { type: 'string', description: 'Build secrets in the format id=<id>,src=<path>, id=<id>,env=<var>, or id=<id> (assumes env=<ID>). These will be passed as Docker build secrets to feature installation steps.' },
 		'experimental-lockfile': { type: 'boolean', default: false, hidden: true, description: 'Write lockfile' },
 		'experimental-frozen-lockfile': { type: 'boolean', default: false, hidden: true, description: 'Ensure lockfile remains unchanged' },
 		'omit-syntax-directive': { type: 'boolean', default: false, hidden: true, description: 'Omit Dockerfile syntax directives' },
@@ -161,6 +199,10 @@ function provisionOptions(y: Argv) {
 			if (remoteEnvs?.some(remoteEnv => !/.+=.*/.test(remoteEnv))) {
 				throw new Error('Unmatched argument format: remote-env must match <name>=<value>');
 			}
+			const buildSecrets = (argv['build-secret'] && (Array.isArray(argv['build-secret']) ? argv['build-secret'] : [argv['build-secret']])) as string[] | undefined;
+			if (buildSecrets?.some(buildSecret => !/^id=[^,]+(,src=.+|,env=.+)?$/.test(buildSecret))) {
+				throw new Error('Unmatched argument format: build-secret must match id=<id>,src=<path>, id=<id>,env=<var>, or id=<id>');
+			}
 			return true;
 		});
 }
@@ -211,6 +253,7 @@ async function provision({
 	'container-session-data-folder': containerSessionDataFolder,
 	'omit-config-remote-env-from-metadata': omitConfigRemotEnvFromMetadata,
 	'secrets-file': secretsFile,
+	'build-secret': buildSecret,
 	'experimental-lockfile': experimentalLockfile,
 	'experimental-frozen-lockfile': experimentalFrozenLockfile,
 	'omit-syntax-directive': omitSyntaxDirective,
@@ -223,6 +266,7 @@ async function provision({
 	const addCacheFroms = addCacheFrom ? (Array.isArray(addCacheFrom) ? addCacheFrom as string[] : [addCacheFrom]) : [];
 	const additionalFeatures = additionalFeaturesJson ? jsonc.parse(additionalFeaturesJson) as Record<string, string | boolean | Record<string, string | boolean>> : {};
 	const providedIdLabels = idLabel ? Array.isArray(idLabel) ? idLabel as string[] : [idLabel] : undefined;
+	const buildSecrets = parseBuildSecrets(buildSecret as string[] | undefined);
 
 	const cwd = workspaceFolder || process.cwd();
 	const cliHost = await getCLIHost(cwd, loadNativeModule, logFormat === 'text');
@@ -287,6 +331,7 @@ async function provision({
 		omitSyntaxDirective,
 		includeConfig,
 		includeMergedConfig,
+		buildSecrets,
 	};
 
 	const result = await doProvision(options, providedIdLabels);
@@ -454,6 +499,7 @@ async function doSetUp({
 				installCommand: dotfilesInstallCommand,
 				targetPath: dotfilesTargetPath,
 			},
+			buildSecrets: [],
 		}, disposables);
 
 		const { common } = params;
@@ -525,6 +571,7 @@ function buildOptions(y: Argv) {
 		'additional-features': { type: 'string', description: 'Additional features to apply to the dev container (JSON as per "features" section in devcontainer.json)' },
 		'skip-feature-auto-mapping': { type: 'boolean', default: false, hidden: true, description: 'Temporary option for testing.' },
 		'skip-persisting-customizations-from-features': { type: 'boolean', default: false, hidden: true, description: 'Do not save customizations from referenced Features as image metadata' },
+		'build-secret': { type: 'string', description: 'Build secrets in the format id=<id>,src=<path>, id=<id>,env=<var>, or id=<id> (assumes env=<ID>). These will be passed as Docker build secrets to feature installation steps.' },
 		'experimental-lockfile': { type: 'boolean', default: false, hidden: true, description: 'Write lockfile' },
 		'experimental-frozen-lockfile': { type: 'boolean', default: false, hidden: true, description: 'Ensure lockfile remains unchanged' },
 		'omit-syntax-directive': { type: 'boolean', default: false, hidden: true, description: 'Omit Dockerfile syntax directives' },
@@ -570,6 +617,7 @@ async function doBuild({
 	'experimental-lockfile': experimentalLockfile,
 	'experimental-frozen-lockfile': experimentalFrozenLockfile,
 	'omit-syntax-directive': omitSyntaxDirective,
+	'build-secret': buildSecret,
 }: BuildArgs) {
 	const disposables: (() => Promise<unknown> | undefined)[] = [];
 	const dispose = async () => {
@@ -581,6 +629,7 @@ async function doBuild({
 		const overrideConfigFile: URI | undefined = /* overrideConfig ? URI.file(path.resolve(process.cwd(), overrideConfig)) : */ undefined;
 		const addCacheFroms = addCacheFrom ? (Array.isArray(addCacheFrom) ? addCacheFrom as string[] : [addCacheFrom]) : [];
 		const additionalFeatures = additionalFeaturesJson ? jsonc.parse(additionalFeaturesJson) as Record<string, string | boolean | Record<string, string | boolean>> : {};
+		const buildSecrets = parseBuildSecrets(buildSecret as string[] | undefined);
 		const params = await createDockerParams({
 			dockerPath,
 			dockerComposePath,
@@ -620,6 +669,7 @@ async function doBuild({
 			experimentalLockfile,
 			experimentalFrozenLockfile,
 			omitSyntaxDirective,
+			buildSecrets,
 		}, disposables);
 
 		const { common, dockerComposeCLI } = params;
@@ -854,6 +904,7 @@ async function doRunUserCommands({
 		const cwd = workspaceFolder || process.cwd();
 		const cliHost = await getCLIHost(cwd, loadNativeModule, logFormat === 'text');
 		const secretsP = readSecretsFromFile({ secretsFile, cliHost });
+		const buildSecrets: BuildSecret[] = [];
 
 		const params = await createDockerParams({
 			dockerPath,
@@ -897,6 +948,7 @@ async function doRunUserCommands({
 			},
 			containerSessionDataFolder,
 			secretsP,
+			buildSecrets,
 		}, disposables);
 
 		const { common } = params;
@@ -1344,7 +1396,8 @@ export async function doExec({
 			buildxOutput: undefined,
 			skipPostAttach: false,
 			skipPersistingCustomizationsFromFeatures: false,
-			dotfiles: {}
+			dotfiles: {},
+			buildSecrets: []
 		}, disposables);
 
 		const { common } = params;

src/spec-node/dockerCompose.ts

@@ -6,7 +6,7 @@
 import * as yaml from 'js-yaml';
 import * as shellQuote from 'shell-quote';
 
-import { createContainerProperties, startEventSeen, ResolverResult, getTunnelInformation, DockerResolverParameters, inspectDockerImage, getEmptyContextFolder, getFolderImageName, SubstitutedConfig, checkDockerSupportForGPU, isBuildKitImagePolicyError } from './utils';
+import { createContainerProperties, startEventSeen, ResolverResult, getTunnelInformation, DockerResolverParameters, inspectDockerImage, getEmptyContextFolder, getFolderImageName, SubstitutedConfig, checkDockerSupportForGPU, isBuildKitImagePolicyError, BuildSecret } from './utils';
 import { ContainerProperties, setupInContainer, ResolverProgress } from '../spec-common/injectHeadless';
 import { ContainerError } from '../spec-common/errors';
 import { Workspace } from '../spec-utils/workspaces';
@@ -243,6 +243,13 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf
 				buildOverrideContent += `        - ${buildKitContext}=${featureBuildInfo.buildKitContexts[buildKitContext]}\n`;
 			}
 		}
+
+		if (params.buildSecrets.length > 0) {
+			buildOverrideContent += '      secrets:\n';
+			for (const secret of params.buildSecrets) {
+				buildOverrideContent += `        - ${secret.id}\n`;
+			}
+		}
 	}
 
 	// Generate the docker-compose override and build
@@ -253,10 +260,27 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf
 		await cliHost.mkdirp(composeFolder);
 		const composeOverrideFile = cliHost.path.join(composeFolder, `${overrideFilePrefix}-${Date.now()}.yml`);
 		const cacheFromOverrideContent = (additionalCacheFroms && additionalCacheFroms.length > 0) ? `      cache_from:\n${additionalCacheFroms.map(cacheFrom => `        - ${cacheFrom}\n`).join('\n')}` : '';
+		const secretsOverrideContent = generateSecretsOverrideContent(params.buildSecrets);
+
+		function generateSecretsOverrideContent(buildSecrets: BuildSecret[]): string {
+			if (!buildSecrets || buildSecrets.length === 0) {
+				return '';
+			}
+			let content = 'secrets:\n';
+			for (const secret of buildSecrets) {
+				if (secret.file) {
+					content += `  ${secret.id}:\n    file: ${secret.file}\n`;
+				} else if (secret.env) {
+					content += `  ${secret.id}:\n    environment: ${secret.env}\n`;
+				}
+			}
+			return content;
+		}
 		const composeOverrideContent = `${versionPrefix}services:
   ${config.service}:
 ${buildOverrideContent?.trimEnd()}
 ${cacheFromOverrideContent}
+${secretsOverrideContent}
 `;
 		output.write(`Docker Compose override file for building image:\n${composeOverrideContent}`);
 		await cliHost.writeFile(composeOverrideFile, Buffer.from(composeOverrideContent));

src/spec-node/singleContainer.ts

@@ -200,7 +200,7 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config
 		if (buildParams.buildxPush) {
 			args.push('--push');
 		} else {
-			if (buildParams.buildxOutput) { 
+			if (buildParams.buildxOutput) {
 				args.push('--output', buildParams.buildxOutput);
 			} else {
 				args.push('--load'); // (short for --output=docker, i.e. load into normal 'docker images' collection)
@@ -212,6 +212,13 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config
 		if (!isBuildxCacheToInline(buildParams.buildxCacheTo)) {
 			args.push('--build-arg', 'BUILDKIT_INLINE_CACHE=1');
 		}
+		for (const secret of buildParams.buildSecrets) {
+			if (secret.file) {
+				args.push('--secret', `id=${secret.id},src=${secret.file}`);
+			} else if (secret.env) {
+				args.push('--secret', `id=${secret.id},env=${secret.env}`);
+			}
+		}
 	} else {
 		args.push('build');
 	}

src/spec-node/utils.ts

@@ -36,6 +36,12 @@ export type BindMountConsistency = 'consistent' | 'cached' | 'delegated' | undef
 
 export type GPUAvailability = 'all' | 'detect' | 'none';
 
+export interface BuildSecret {
+	id: string;
+	file?: string;
+	env?: string;
+}
+
 // Generic retry function
 export async function retry<T>(fn: () => Promise<T>, options: { retryIntervalMilliseconds: number; maxRetries: number; output: Log }): Promise<T> {
 	const { retryIntervalMilliseconds, maxRetries, output } = options;
@@ -135,6 +141,7 @@ export interface DockerResolverParameters {
 	buildxOutput: string | undefined;
 	buildxCacheTo: string | undefined;
 	platformInfo: PlatformInfo;
+	buildSecrets: BuildSecret[];
 }
 
 export interface ResolverResult {