pw_ide: Natively process compDBs in VS Code

Change-Id: I48b8983851361a28008a72a95cf0bb4f62222a09
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/267215
Commit-Queue: Auto-Submit <[email protected]>
Docs-Not-Needed: Chad Norvell <[email protected]>
Reviewed-by: Asad Memon <[email protected]>
Lint: Lint 🤖 <[email protected]>
Pigweed-Auto-Submit: Chad Norvell <[email protected]>

pw_ide/ts/pigweed-vscode/package.json

@@ -100,6 +100,25 @@
           "type": "string",
           "description": "The compile commands directory for the selected build target to use for editor code intelligence"
         },
+        "pigweed.compDbSearchPaths": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "pathGlob": {
+                "type": "string",
+                "description": "..."
+              },
+              "targetInferencePattern": {
+                "type": "string",
+                "default": "?",
+                "description": "..."
+              }
+            }
+          },
+          "default": [],
+          "description": "..."
+        },
         "pigweed.disableBazelSettingsRecommendations": {
           "type": "boolean",
           "default": "false",

pw_ide/ts/pigweed-vscode/src/clangd/parser.ts

@@ -17,6 +17,11 @@ import * as fs_p from 'fs/promises';
 import * as path from 'path';
 
 import { z } from 'zod';
+import { loadLegacySettings, loadLegacySettingsFile } from '../settings/legacy';
+import { settings, workingDir } from '../settings/vscode';
+import { globStream } from 'glob';
+import { CDB_FILE_DIR, CDB_FILE_NAME } from './paths';
+import logger from '../logging';
 
 /**
  * Given a target inference glob, infer which path positions contain tokens that
@@ -62,7 +67,10 @@ export function inferTarget(
     ? path.relative(outputPath, rootPath).split(path.sep)
     : outputPath.split(path.sep);
 
-  return targetPositions.map((pos) => pathParts[pos]).join('_');
+  return targetPositions
+    .map((pos) => pathParts[pos])
+    .join('_')
+    .replace('.', '_');
 }
 
 /** Supported compile command wrapper executables. */
@@ -201,7 +209,7 @@ const UNSUPPORTED_EXECUTABLES = ['_pw_invalid', 'python'];
  * See: https://clang.llvm.org/docs/JSONCompilationDatabase.html
  */
 class CompileCommand {
-  private readonly data: CompileCommandData;
+  readonly data: CompileCommandData;
   private readonly commandParts: CommandParts;
 
   constructor(data: CompileCommandData) {
@@ -217,7 +225,7 @@ class CompileCommand {
     this.commandParts = this.data.arguments
       ? parseCommandParts(this.data.arguments)
       : // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-        parseCommandParts(this.data.command!.split(/(\s+)/));
+        parseCommandParts(this.data.command!.split(/\s+/));
   }
 
   /**
@@ -313,13 +321,29 @@ export class CompilationDatabase {
    * Instantiate a compilation database from a file.
    *
    * @param path A path to a compilation database file
+   * @param onFailure An optional callback to run when the file can't be loaded
+   *   or parsed.
    * @returns A `CompilationDatabase` with the contents of the file
-   * @throws On any schema or invariant violations in the source data
+   * @throws On any schema or invariant violations in the source data, unless
+   *   `onFailure` is provided instead.
    */
-  static fromFile(path: string): CompilationDatabase {
+  static async fromFile(
+    path: string,
+    onFailure?: () => void,
+  ): Promise<CompilationDatabase | null> {
     const compDb = new CompilationDatabase();
-    compDb.loadFromFile(path);
-    return compDb;
+
+    try {
+      await compDb.loadFromFile(path);
+      return compDb;
+    } catch (e) {
+      if (onFailure) {
+        onFailure();
+        return null;
+      } else {
+        throw e;
+      }
+    }
   }
 
   /** Add a compile command to the database. */
@@ -364,16 +388,19 @@ export class CompilationDatabase {
    * processed files to the compile commands directory. This case applies to
    * files generated by GN.
    */
-  process(): CompilationDatabaseMap | null {
+  process(targetInference: string): CompilationDatabaseMap | null {
     const cleanCompilationDatabases = new CompilationDatabaseMap();
 
     for (const compileCommand of this.db) {
       const processedCommand = compileCommand.process();
 
       if (processedCommand !== null) {
-        // TODO(chadnorvell): Get inference from settings
-        const target = inferTarget('?', '?', processedCommand.outputPath);
-        const db = cleanCompilationDatabases.get(target);
+        const targetName = inferTarget(
+          targetInference,
+          processedCommand.outputPath,
+        );
+
+        const db = cleanCompilationDatabases.get(targetName);
         db.add(processedCommand);
         db.sourceFilePath = this.filePath;
         db.fileHash = this.fileHash;
@@ -397,6 +424,12 @@ export class CompilationDatabase {
 
     return cleanCompilationDatabases;
   }
+
+  async write(filePath: string) {
+    const fileData = JSON.stringify(this.db.map((c) => c.data));
+    await fs_p.mkdir(path.dirname(filePath), { recursive: true });
+    await fs_p.writeFile(filePath, fileData);
+  }
 }
 
 class CompilationDatabaseMap extends Map<string, CompilationDatabase> {
@@ -417,4 +450,178 @@ class CompilationDatabaseMap extends Map<string, CompilationDatabase> {
 
     return undefined;
   }
+
+  async writeAll(): Promise<void> {
+    const workingDirPath = workingDir.get();
+    const promises: Promise<void>[] = [];
+
+    for (const [targetName, compDb] of this.entries()) {
+      const filePath = path.join(
+        workingDirPath,
+        CDB_FILE_DIR,
+        targetName,
+        CDB_FILE_NAME,
+      );
+      promises.push(compDb.write(filePath));
+    }
+
+    await Promise.all(promises);
+  }
+
+  static merge(
+    ...compDbMaps: CompilationDatabaseMap[]
+  ): CompilationDatabaseMap {
+    const merged = new CompilationDatabaseMap();
+
+    for (const compDbMap of compDbMaps) {
+      for (const [targetName, compDb] of compDbMap) {
+        merged.set(targetName, compDb);
+      }
+    }
+
+    return merged;
+  }
+}
+
+interface CompDbProcessingSettings {
+  compDbSearchPaths: string[][];
+  workingDir: string;
+}
+
+/**
+ * Get user settings related to compilation database processing, from the
+ * VS Code settings (canonical source) or the legacy `.pw_ide.yaml`.
+ */
+async function getCompDbProcessingSettings(): Promise<CompDbProcessingSettings> {
+  // If this returns null, we assume there is no legacy settings file.
+  const legacySettingsData = await loadLegacySettingsFile();
+
+  // If there is a legacy settings file, assume all relevant config is there.
+  if (legacySettingsData !== null) {
+    logger.info('Using legacy settings file: .pw_ide.yaml');
+
+    const legacySettings = await loadLegacySettings(
+      legacySettingsData ?? '',
+      true,
+    );
+
+    return {
+      compDbSearchPaths: legacySettings.compdb_search_paths,
+      workingDir: legacySettings.working_dir,
+    };
+  }
+
+  // Otherwise use the values from the VS Code config or the defaults.
+  return {
+    compDbSearchPaths: settings
+      .compDbSearchPaths()
+      .map(({ pathGlob, targetInferencePattern }) => [
+        pathGlob,
+        targetInferencePattern,
+      ]),
+    workingDir: workingDir.get(),
+  };
+}
+
+/**
+ * An async generator that finds compilation database files based on provided
+ * search path globs.
+ *
+ * For example, a common search path glob for GN projects would be `out/*`.
+ * That would expand to every top-level directory in `out`. Then, each of those
+ * directories will be recursively searched for compilation databases.
+ *
+ * The return value is a tuple of the specific (expanded) search path that
+ * yielded the compilation database, the path to the compilation database,
+ * and the target inference pattern that was associated with the original
+ * search path glob.
+ */
+async function* assembleCompDbFileData(
+  compDbSearchPaths: string[][],
+  workingDir: string,
+) {
+  for (const [
+    baseSearchPathGlob,
+    searchPathTargetInference,
+  ] of compDbSearchPaths) {
+    const searchPathGlob = path.join(workingDir, baseSearchPathGlob);
+
+    // For each search path glob, get an array of concrete directory paths.
+    for await (const searchPath of globStream(searchPathGlob)) {
+      // For each directory path, get an array of compDb file paths.
+      for await (const compDbFilePath of globStream(
+        `${searchPath}/${CDB_FILE_NAME}`,
+      )) {
+        // Associate each compDb file path with its root directory and target
+        // inference pattern.
+        yield [searchPath, compDbFilePath, searchPathTargetInference];
+      }
+    }
+  }
+}
+
+/**
+ * Process compilation databases found in the search path globs in settings.
+ *
+ * This returns two things:
+ *
+ * 1. A compilation database map that associates target names (generated from
+ * target inference) with compilation database objects containing compile
+ * commands only for that target. These should be written to disk in the
+ * canonical compile commands directory.
+ *
+ * 2. A collection of compilation databases that don't require processing,
+ * because they already contain compile commands only for a single target. These
+ * include only the target name and the path to the compilation database, so
+ * the IDE can be configured to point directly at the source files.
+ */
+export async function processCompDbs() {
+  logger.info('Processing compilation databases...');
+
+  const { compDbSearchPaths, workingDir } = await getCompDbProcessingSettings();
+
+  const unprocessedCompDbs: [string, string][] = [];
+  const processedCompDbMaps: CompilationDatabaseMap[] = [];
+  let fileCount = 0;
+
+  for await (const [
+    searchPath,
+    compDbFilePath,
+    searchPathTargetInference,
+  ] of assembleCompDbFileData(compDbSearchPaths, workingDir)) {
+    const compDb = await CompilationDatabase.fromFile(compDbFilePath, () =>
+      logger.error('bad file'),
+    );
+
+    if (!compDb) continue;
+
+    const processed = compDb.process(searchPathTargetInference);
+    fileCount++;
+
+    if (!processed) {
+      const outputPath = path.dirname(
+        path.relative(searchPath, compDbFilePath),
+      );
+      const targetName = inferTarget(searchPathTargetInference, outputPath);
+      unprocessedCompDbs.push([targetName, compDbFilePath]);
+    } else {
+      processedCompDbMaps.push(processed);
+    }
+  }
+
+  const processedCompDbs = CompilationDatabaseMap.merge(...processedCompDbMaps);
+
+  logger.info(`↳ Processed ${fileCount} files`);
+  logger.info(
+    `↳ Found ${unprocessedCompDbs.length} compilation databases that can be used in place`,
+  );
+  logger.info(
+    `↳ Produced ${processedCompDbs.size} clean compilation databases`,
+  );
+  logger.info('');
+
+  return {
+    processedCompDbs,
+    unprocessedCompDbs,
+  };
 }

pw_ide/ts/pigweed-vscode/src/clangd/paths.ts

@@ -18,10 +18,12 @@ import { glob } from 'glob';
 
 import { settings, workingDir } from '../settings/vscode';
 
-const CDB_FILE_NAME = 'compile_commands.json' as const;
+export const CDB_FILE_NAME = 'compile_commands.json' as const;
+
+export const CDB_FILE_DIR = '.compile_commands';
 
 const CDB_FILE_DIRS = [
-  '.compile_commands',
+  CDB_FILE_DIR,
   '.pw_ide', // The legacy pw_ide directory
 ];
 

pw_ide/ts/pigweed-vscode/src/extension.ts

@@ -72,6 +72,7 @@ import {
 import { commandRegisterer, VscCommandCallback } from './utils';
 import { shouldSupportGn } from './gn';
 import { shouldSupportCmake } from './cmake';
+import { processCompDbs } from './clangd/parser';
 
 interface CommandEntry {
   name: string;
@@ -91,13 +92,17 @@ const disposer = new Disposer();
 
 type ProjectType = 'bazel' | 'bootstrap' | 'both';
 
-function registerCommands(
+async function registerCommands(
   projectType: ProjectType,
   context: ExtensionContext,
   refreshManager: RefreshManager<any>,
   clangdActiveFilesCache: ClangdActiveFilesCache,
   bazelCompileCommandsWatcher?: BazelRefreshCompileCommandsWatcher | undefined,
-): void {
+): Promise<void> {
+  const useBazel = await shouldSupportBazel();
+  const useCmake = await shouldSupportCmake();
+  const useGn = await shouldSupportGn();
+
   const commands: CommandEntry[] = [
     {
       name: 'pigweed.open-output-panel',
@@ -206,9 +211,16 @@ function registerCommands(
     },
     {
       name: 'pigweed.refresh-compile-commands',
-      callback: () => {
-        bazelCompileCommandsWatcher!.refresh();
-        showProgressDuringRefresh(refreshManager);
+      callback: async () => {
+        if (useGn || useCmake) {
+          const { processedCompDbs } = await processCompDbs();
+          await processedCompDbs.writeAll();
+        }
+
+        if (useBazel) {
+          bazelCompileCommandsWatcher!.refresh();
+          showProgressDuringRefresh(refreshManager);
+        }
       },
       projectType: ['bazel', 'both'],
     },