Merge branch 'main' into poetry-error-message

CHANGELOG.md

@@ -4,6 +4,18 @@ Notable changes.
 
 ## August 2024
 
+### [0.69.0]
+- Enhance Template metadata (https://github.com/devcontainers/cli/pull/875)
+    - Caches additional Template metadata (such as `files`) onto the manifest
+	- Resolves full file paths for `optionalPaths` directories that only contain one file (for better usability in upstream tools)
+	- Fixes bugs
+
+### [0.68.0]
+- Supporting changes for [Template `optionalPaths` specification](https://github.com/devcontainers/spec/blob/main/docs/specs/devcontainer-templates.md#the-optionalpaths-property) (https://github.com/microsoft/vscode-remote-release/issues/10095)
+	- Publish metadata on Template OCI manifests (https://github.com/devcontainers/cli/pull/865)
+	- Add `--omit-paths` option to `templates apply` command (https://github.com/devcontainers/cli/pull/868)
+	- Add `templates metadata` command (https://github.com/devcontainers/cli/pull/866)
+
 ### [0.67.0]
 - Fix containerEnv substitution. (https://github.com/microsoft/vscode-remote-release/issues/10033)
 

package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@devcontainers/cli",
 	"description": "Dev Containers CLI",
-	"version": "0.67.0",
+	"version": "0.69.0",
 	"bin": {
 		"devcontainer": "devcontainer.js"
 	},

src/spec-configuration/containerCollectionsOCI.ts

@@ -504,7 +504,7 @@ export async function getPublishedTags(params: CommonParams, ref: OCIRef): Promi
 	}
 }
 
-export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, expectedDigest: string, ignoredFilesDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> {
+export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, expectedDigest: string, omitDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> {
 	// TODO: Parallelize if multiple layers (not likely).
 	// TODO: Seeking might be needed if the size is too large.
 
@@ -543,24 +543,37 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st
 		await mkdirpLocal(destCachePath);
 		await writeLocalFile(tempTarballPath, resBody);
 
+		// https://github.com/devcontainers/spec/blob/main/docs/specs/devcontainer-templates.md#the-optionalpaths-property
+		const directoriesToOmit = omitDuringExtraction.filter(f => f.endsWith('/*')).map(f => f.slice(0, -1));
+		const filesToOmit = omitDuringExtraction.filter(f => !f.endsWith('/*'));
+
+		output.write(`omitDuringExtraction: '${omitDuringExtraction.join(', ')}`, LogLevel.Trace);
+		output.write(`Files to omit: '${filesToOmit.join(', ')}'`, LogLevel.Info);
+		if (directoriesToOmit.length) {
+			output.write(`Dirs to omit : '${directoriesToOmit.join(', ')}'`, LogLevel.Info);
+		}
+
 		const files: string[] = [];
 		await tar.x(
 			{
 				file: tempTarballPath,
 				cwd: destCachePath,
-				filter: (path: string, stat: tar.FileStat) => {
-					// Skip files that are in the ignore list
-					if (ignoredFilesDuringExtraction.some(f => path.indexOf(f) !== -1)) {
-						// Skip.
-						output.write(`Skipping file '${path}' during blob extraction`, LogLevel.Trace);
-						return false;
+				filter: (tPath: string, stat: tar.FileStat) => {
+					output.write(`Testing '${tPath}'(${stat.type})`, LogLevel.Trace);
+					const cleanedPath = tPath
+						.replace(/\\/g, '/')
+						.replace(/^\.\//, '');
+
+					if (filesToOmit.includes(cleanedPath) || directoriesToOmit.some(d => cleanedPath.startsWith(d))) {
+						output.write(`  Omitting '${tPath}'`, LogLevel.Trace);
+						return false; // Skip
 					}
-					// Keep track of all files extracted, in case the caller is interested.
-					output.write(`${path} : ${stat.type}`, LogLevel.Trace);
-					if ((stat.type.toString() === 'File')) {
-						files.push(path);
+
+					if (stat.type.toString() === 'File') {
+						files.push(tPath);
 					}
-					return true;
+
+					return true; // Keep
 				}
 			}
 		);
@@ -576,8 +589,8 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st
 			{
 				file: tempTarballPath,
 				cwd: ociCacheDir,
-				filter: (path: string, _: tar.FileStat) => {
-					return path === `./${metadataFile}`;
+				filter: (tPath: string, _: tar.FileStat) => {
+					return tPath === `./${metadataFile}`;
 				}
 			});
 		const pathToMetadataFile = path.join(ociCacheDir, metadataFile);

src/spec-configuration/containerCollectionsOCIPush.ts

@@ -11,7 +11,7 @@ import { requestEnsureAuthenticated } from './httpOCIRegistry';
 //     Devcontainer Spec (features) : https://containers.dev/implementors/features-distribution/#oci-registry
 //     Devcontainer Spec (templates): https://github.com/devcontainers/spec/blob/main/proposals/devcontainer-templates-distribution.md#oci-registry
 //     OCI Spec                     : https://github.com/opencontainers/distribution-spec/blob/main/spec.md#push
-export async function pushOCIFeatureOrTemplate(params: CommonParams, ociRef: OCIRef, pathToTgz: string, tags: string[], collectionType: string, featureAnnotations = {}): Promise<string | undefined> {
+export async function pushOCIFeatureOrTemplate(params: CommonParams, ociRef: OCIRef, pathToTgz: string, tags: string[], collectionType: string, annotations: { [key: string]: string } = {}): Promise<string | undefined> {
 	const { output } = params;
 
 	output.write(`-- Starting push of ${collectionType} '${ociRef.id}' to '${ociRef.resource}' with tags '${tags.join(', ')}'`);
@@ -25,7 +25,7 @@ export async function pushOCIFeatureOrTemplate(params: CommonParams, ociRef: OCI
 	const dataBytes = fs.readFileSync(pathToTgz);
 
 	// Generate Manifest for given feature/template artifact.
-	const manifest = await generateCompleteManifestForIndividualFeatureOrTemplate(output, dataBytes, pathToTgz, ociRef, collectionType, featureAnnotations);
+	const manifest = await generateCompleteManifestForIndividualFeatureOrTemplate(output, dataBytes, pathToTgz, ociRef, collectionType, annotations);
 	if (!manifest) {
 		output.write(`Failed to generate manifest for ${ociRef.id}`, LogLevel.Error);
 		return;
@@ -268,14 +268,13 @@ async function putBlob(params: CommonParams, blobPutLocationUriPath: string, oci
 // Generate a layer that follows the `application/vnd.devcontainers.layer.v1+tar` mediaType as defined in
 //     Devcontainer Spec (features) : https://containers.dev/implementors/features-distribution/#oci-registry
 //     Devcontainer Spec (templates): https://github.com/devcontainers/spec/blob/main/proposals/devcontainer-templates-distribution.md#oci-registry
-async function generateCompleteManifestForIndividualFeatureOrTemplate(output: Log, dataBytes: Buffer, pathToTgz: string, ociRef: OCIRef, collectionType: string, featureAnnotations = {}): Promise<ManifestContainer | undefined> {
+async function generateCompleteManifestForIndividualFeatureOrTemplate(output: Log, dataBytes: Buffer, pathToTgz: string, ociRef: OCIRef, collectionType: string, annotations: { [key: string]: string } = {}): Promise<ManifestContainer | undefined> {
 	const tgzLayer = await calculateDataLayer(output, dataBytes, path.basename(pathToTgz), DEVCONTAINER_TAR_LAYER_MEDIATYPE);
 	if (!tgzLayer) {
 		output.write(`Failed to calculate tgz layer.`, LogLevel.Error);
 		return undefined;
 	}
 
-	let annotations: { [key: string]: string } = featureAnnotations;
 	// Specific registries look for certain optional metadata 
 	// in the manifest, in this case for UI presentation.
 	if (ociRef.registry === 'ghcr.io') {

src/spec-configuration/containerTemplatesConfiguration.ts

@@ -5,13 +5,15 @@ export interface Template {
 	description?: string;
 	documentationURL?: string;
 	licenseURL?: string;
-	type?: string;
-	fileCount?: number;
+	type?: string;             // Added programatically during packaging
+	fileCount?: number;        // Added programatically during packaging
 	featureIds?: string[];
 	options?: Record<string, TemplateOption>;
 	platforms?: string[];
 	publisher?: string;
 	keywords?: string[];
+	optionalPaths?: string[];
+	files: string[];           // Added programatically during packaging
 }
 
 export type TemplateOption = {

src/spec-configuration/containerTemplatesOCI.ts

@@ -19,12 +19,13 @@ export interface SelectedTemplate {
 	id: string;
 	options: TemplateOptions;
 	features: TemplateFeatureOption[];
+	omitPaths: string[];
 }
 
 export async function fetchTemplate(params: CommonParams, selectedTemplate: SelectedTemplate, templateDestPath: string, userProvidedTmpDir?: string): Promise<string[] | undefined> {
 	const { output } = params;
 
-	let { id: userSelectedId, options: userSelectedOptions } = selectedTemplate;
+	let { id: userSelectedId, options: userSelectedOptions, omitPaths } = selectedTemplate;
 	const templateRef = getRef(output, userSelectedId);
 	if (!templateRef) {
 		output.write(`Failed to parse template ref for ${userSelectedId}`, LogLevel.Error);
@@ -46,7 +47,7 @@ export async function fetchTemplate(params: CommonParams, selectedTemplate: Sele
 	output.write(`blob url: ${blobUrl}`, LogLevel.Trace);
 
 	const tmpDir = userProvidedTmpDir || path.join(os.tmpdir(), 'vsch-template-temp', `${Date.now()}`);
-	const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, blobDigest, ['devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json');
+	const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, blobDigest, [...omitPaths, 'devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json');
 
 	if (!blobResult) {
 		throw new Error(`Failed to download package for ${templateRef.resource}`);

src/spec-node/collectionCommonUtils/packageCommandImpl.ts

@@ -9,6 +9,7 @@ import path from 'path';
 import { DevContainerConfig, isDockerFileConfig } from '../../spec-configuration/configuration';
 import { Template } from '../../spec-configuration/containerTemplatesConfiguration';
 import { Feature } from '../../spec-configuration/containerFeaturesConfiguration';
+import { getRef } from '../../spec-configuration/containerCollectionsOCI';
 
 export interface SourceInformation {
 	source: string;
@@ -133,9 +134,45 @@ async function addsAdditionalTemplateProps(srcFolder: string, devcontainerTempla
 		return false;
 	}
 
+	const fileNames = (await recursiveDirReader.default(srcFolder))?.map((f) => path.relative(srcFolder, f)) ?? [];
+
 	templateData.type = type;
-	templateData.fileCount = (await recursiveDirReader.default(srcFolder)).length;
-	templateData.featureIds = config.features ? Object.keys(config.features).map((k) => k.split(':')[0]) : [];
+	templateData.files = fileNames;
+	templateData.fileCount = fileNames.length;
+	templateData.featureIds =
+		config.features
+			? Object.keys(config.features)
+				.map((f) => getRef(output, f)?.resource)
+				.filter((f) => f !== undefined) as string[]
+			: [];
+
+	// If the Template is omitting a folder and that folder contains just a single file, 
+	// replace the entry in the metadata with the full file name,
+	// as that provides a better user experience when tools consume the metadata.
+	// Eg: If the template is omitting ".github/*" and the Template source contains just a single file
+	//     "workflow.yml", replace ".github/*" with ".github/workflow.yml"
+	if (templateData.optionalPaths && templateData.optionalPaths?.length) {
+		const optionalPaths = templateData.optionalPaths;
+		for (const optPath of optionalPaths) {
+			// Skip if not a directory
+			if (!optPath.endsWith('/*') || optPath.length < 3) {
+				continue;
+			}
+			const dirPath = optPath.slice(0, -2);
+			const dirFiles = fileNames.filter((f) => f.startsWith(dirPath));
+			output.write(`Given optionalPath starting with '${dirPath}' has ${dirFiles.length} files`, LogLevel.Trace);
+			if (dirFiles.length === 1) {
+				// If that one item is a file and not a directory
+				const f = dirFiles[0];
+				output.write(`Checking if optionalPath '${optPath}' with lone contents '${f}' is a file `, LogLevel.Trace);
+				const localPath = path.join(srcFolder, f);
+				if (await isLocalFile(localPath)) {
+					output.write(`Checked path '${localPath}' on disk is a file. Replacing optionalPaths entry '${optPath}' with '${f}'`, LogLevel.Trace);
+					templateData.optionalPaths[optionalPaths.indexOf(optPath)] = f;
+				}
+			}
+		}
+	}
 
 	await writeLocalFile(devcontainerTemplateJsonPath, JSON.stringify(templateData, null, 4));
 

src/spec-node/collectionCommonUtils/publishCommandImpl.ts

@@ -39,7 +39,7 @@ export function getSemanticTags(version: string, tags: string[], output: Log) {
 	return semanticVersions;
 }
 
-export async function doPublishCommand(params: CommonParams, version: string, ociRef: OCIRef, outputDir: string, collectionType: string, archiveName: string, featureAnnotations = {}) {
+export async function doPublishCommand(params: CommonParams, version: string, ociRef: OCIRef, outputDir: string, collectionType: string, archiveName: string, annotations: { [key: string]: string } = {}) {
 	const { output } = params;
 
 	output.write(`Fetching published versions...`, LogLevel.Info);
@@ -54,7 +54,7 @@ export async function doPublishCommand(params: CommonParams, version: string, oc
 	if (!!semanticTags) {
 		output.write(`Publishing tags: ${semanticTags.toString()}...`, LogLevel.Info);
 		const pathToTgz = path.join(outputDir, archiveName);
-		const digest = await pushOCIFeatureOrTemplate(params, ociRef, pathToTgz, semanticTags, collectionType, featureAnnotations);
+		const digest = await pushOCIFeatureOrTemplate(params, ociRef, pathToTgz, semanticTags, collectionType, annotations);
 		if (!digest) {
 			output.write(`(!) ERR: Failed to publish ${collectionType}: '${ociRef.resource}'`, LogLevel.Error);
 			return;

src/spec-node/devContainersSpecCLI.ts

@@ -44,6 +44,7 @@ import { readFeaturesConfig } from './featureUtils';
 import { featuresGenerateDocsHandler, featuresGenerateDocsOptions } from './featuresCLI/generateDocs';
 import { templatesGenerateDocsHandler, templatesGenerateDocsOptions } from './templatesCLI/generateDocs';
 import { mapNodeOSToGOOS, mapNodeArchitectureToGOARCH } from '../spec-configuration/containerCollectionsOCI';
+import { templateMetadataHandler, templateMetadataOptions } from './templatesCLI/metadata';
 
 const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell';
 
@@ -85,6 +86,7 @@ const mountRegex = /^type=(bind|volume),source=([^,]+),target=([^,]+)(?:,externa
 	y.command('templates', 'Templates commands', (y: Argv) => {
 		y.command('apply', 'Apply a template to the project', templateApplyOptions, templateApplyHandler);
 		y.command('publish <target>', 'Package and publish templates', templatesPublishOptions, templatesPublishHandler);
+		y.command('metadata <templateId>', 'Fetch a published Template\'s metadata', templateMetadataOptions, templateMetadataHandler);
 		y.command('generate-docs', 'Generate documentation', templatesGenerateDocsOptions, templatesGenerateDocsHandler);
 	});
 	y.command(restArgs ? ['exec', '*'] : ['exec <cmd> [args..]'], 'Execute a command on a running dev container', execOptions, execHandler);

src/spec-node/templatesCLI/apply.ts

@@ -15,6 +15,7 @@ export function templateApplyOptions(y: Argv) {
 			'features': { type: 'string', alias: 'f', default: '[]', description: 'Features to add to the provided Template, provided as JSON.' },
 			'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' },
 			'tmp-dir': { type: 'string', description: 'Directory to use for temporary files. If not provided, the system default will be inferred.' },
+			'omit-paths': { type: 'string', default: '[]', description: 'List of paths within the Template to omit applying, provided as JSON.  To ignore a directory append \'/*\'. Eg: \'[".github/*", "dir/a/*", "file.ts"]\'' },
 		})
 		.check(_argv => {
 			return true;
@@ -34,6 +35,7 @@ async function templateApply({
 	'features': featuresArgs,
 	'log-level': inputLogLevel,
 	'tmp-dir': userProvidedTmpDir,
+	'omit-paths': omitPathsArg,
 }: TemplateApplyArgs) {
 	const disposables: (() => Promise<unknown> | undefined)[] = [];
 	const dispose = async () => {
@@ -65,13 +67,23 @@ async function templateApply({
 		process.exit(1);
 	}
 
+	let omitPaths: string[] = [];
+	if (omitPathsArg) {
+		let omitPathsErrors: jsonc.ParseError[] = [];
+		omitPaths = jsonc.parse(omitPathsArg, omitPathsErrors);
+		if (!Array.isArray(omitPaths)) {
+			output.write('Invalid \'--omitPaths\' argument provided. Provide as a JSON array, eg: \'[".github/*", "dir/a/*", "file.ts"]\'', LogLevel.Error);
+			process.exit(1);
+		}
+	}
+
 	const selectedTemplate: SelectedTemplate = {
 		id: templateId,
 		options,
-		features
+		features,
+		omitPaths,
 	};
 
-
 	const files = await fetchTemplate({ output, env: process.env }, selectedTemplate, workspaceFolder, userProvidedTmpDir);
 	if (!files) {
 		output.write(`Failed to fetch template '${id}'.`, LogLevel.Error);

src/spec-node/templatesCLI/metadata.ts

@@ -0,0 +1,74 @@
+import { Argv } from 'yargs';
+import { LogLevel, mapLogLevel } from '../../spec-utils/log';
+import { getPackageConfig } from '../../spec-utils/product';
+import { createLog } from '../devContainers';
+import { fetchOCIManifestIfExists, getRef } from '../../spec-configuration/containerCollectionsOCI';
+
+import { UnpackArgv } from '../devContainersSpecCLI';
+
+export function templateMetadataOptions(y: Argv) {
+	return y
+		.options({
+			'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' },
+		})
+		.positional('templateId', { type: 'string', demandOption: true, description: 'Template Identifier' });
+}
+
+export type TemplateMetadataArgs = UnpackArgv<ReturnType<typeof templateMetadataOptions>>;
+
+export function templateMetadataHandler(args: TemplateMetadataArgs) {
+	(async () => await templateMetadata(args))().catch(console.error);
+}
+
+async function templateMetadata({
+	'log-level': inputLogLevel,
+	'templateId': templateId,
+}: TemplateMetadataArgs) {
+	const disposables: (() => Promise<unknown> | undefined)[] = [];
+	const dispose = async () => {
+		await Promise.all(disposables.map(d => d()));
+	};
+
+	const pkg = getPackageConfig();
+
+	const output = createLog({
+		logLevel: mapLogLevel(inputLogLevel),
+		logFormat: 'text',
+		log: (str) => process.stderr.write(str),
+		terminalDimensions: undefined,
+	}, pkg, new Date(), disposables);
+
+	const params = { output, env: process.env };
+	output.write(`Fetching metadata for ${templateId}`, LogLevel.Trace);
+
+	const templateRef = getRef(output, templateId);
+	if (!templateRef) {
+		console.log(JSON.stringify({}));
+		process.exit(1);
+	}
+
+	const manifestContainer = await fetchOCIManifestIfExists(params, templateRef, undefined);
+	if (!manifestContainer) {
+		console.log(JSON.stringify({}));
+		process.exit(1);
+	}
+
+	const { manifestObj, canonicalId } = manifestContainer;
+	output.write(`Template '${templateId}' resolved to '${canonicalId}'`, LogLevel.Trace);
+
+	// Templates must have been published with a CLI post commit
+	// https://github.com/devcontainers/cli/commit/6c6aebfa7b74aea9d67760fd1e74b09573d31536
+	// in order to contain attached metadata.
+	const metadata = manifestObj.annotations?.['dev.containers.metadata'];
+	if (!metadata) {
+		output.write(`Template resolved to '${canonicalId}' but does not contain metadata on its manifest.`, LogLevel.Warning);
+		output.write(`Ask the Template owner to republish this Template to populate the manifest.`, LogLevel.Warning);
+		console.log(JSON.stringify({}));
+		process.exit(1);
+	}
+
+	const unescaped = JSON.parse(metadata);
+	console.log(JSON.stringify(unescaped));
+	await dispose();
+	process.exit();
+}

src/spec-node/templatesCLI/publish.ts

@@ -88,7 +88,14 @@ async function templatesPublish({
         }
 
         const archiveName = getArchiveName(t.id, collectionType);
-        const publishResult = await doPublishCommand(params, t.version, templateRef, outputDir, collectionType, archiveName);
+
+        // Properties here are available on the manifest without needing to download the full Template archive.
+        const templateAnnotations = {
+            'dev.containers.metadata': JSON.stringify(t),
+        };
+        output.write(`Template Annotations: ${JSON.stringify(templateAnnotations)}`, LogLevel.Debug);
+
+        const publishResult = await doPublishCommand(params, t.version, templateRef, outputDir, collectionType, archiveName, templateAnnotations);
         if (!publishResult) {
             output.write(`(!) ERR: Failed to publish '${resource}'`, LogLevel.Error);
             process.exit(1);