Add CTU On-Demand analysis support

Add an option to enable on-demand parsing of needed ASTs during CTU
analysis, and another option to specify the compilation database used.
The option CTUOnDemandParsing is a boolean flag, which enables the new
AST-loading mode. Option CTUOnDemandParsingDatabase is a string, which
should be the path of the compilation database used for on-demand parsing.
The compilation database is needed for on-demand mode, because it has all the
necessary information to generate the ASTs.

Please enter the commit message for your changes. Lines starting

include/clang/CrossTU/CrossTranslationUnit.h

@@ -32,6 +32,10 @@ class FunctionDecl;
 class NamedDecl;
 class TranslationUnitDecl;
 
+namespace tooling {
+class JSONCompilationDatabase;
+}
+
 namespace cross_tu {
 
 enum class index_error_code {
@@ -41,12 +45,14 @@ enum class index_error_code {
   multiple_definitions,
   missing_definition,
   failed_import,
+  failed_to_load_compilation_database,
   failed_to_get_external_ast,
   failed_to_generate_usr,
   triple_mismatch,
   lang_mismatch,
   lang_dialect_mismatch,
-  load_threshold_reached
+  load_threshold_reached,
+  ambiguous_compile_commands_database
 };
 
 class IndexError : public llvm::ErrorInfo<IndexError> {
@@ -85,7 +91,8 @@ class IndexError : public llvm::ErrorInfo<IndexError> {
 /// \return Returns a map where the USR is the key and the filepath is the value
 ///         or an error.
 llvm::Expected<llvm::StringMap<std::string>>
-parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir);
+parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir,
+                  llvm::Optional<StringRef> OnDemandParsingDatabase);
 
 std::string createCrossTUIndexString(const llvm::StringMap<std::string> &Index);
 
@@ -125,7 +132,8 @@ class CrossTranslationUnitContext {
   llvm::Expected<const FunctionDecl *>
   getCrossTUDefinition(const FunctionDecl *FD, StringRef CrossTUDir,
                        StringRef IndexName, bool DisplayCTUProgress,
-                       unsigned CTULoadThreshold);
+                       unsigned CTULoadThreshold,
+                       llvm::Optional<StringRef> OnDemandParsingDatabase);
 
   /// This function loads a function definition from an external AST
   ///        file.
@@ -141,11 +149,11 @@ class CrossTranslationUnitContext {
   /// The returned pointer is never a nullptr.
   ///
   /// Note that the AST files should also be in the \p CrossTUDir.
-  llvm::Expected<ASTUnit *> loadExternalAST(StringRef LookupName,
-                                            StringRef CrossTUDir,
-                                            StringRef IndexName,
-                                            bool DisplayCTUProgress,
-                                            unsigned CTULoadThreshold);
+  llvm::Expected<ASTUnit *>
+  loadExternalAST(StringRef LookupName, StringRef CrossTUDir,
+                  StringRef IndexName, bool DisplayCTUProgress,
+                  unsigned CTULoadThreshold,
+                  llvm::Optional<StringRef> OnDemandParsingDatabase);
 
   /// This function merges a definition from a separate AST Unit into
   ///        the current one which was created by the compiler instance that
@@ -172,6 +180,11 @@ class CrossTranslationUnitContext {
   GetImportedFromSourceLocation(const clang::SourceLocation &ToLoc) const;
 
 private:
+  llvm::Expected<std::unique_ptr<ASTUnit>>
+  loadASTFromDump(StringRef ASTSourcePath) const;
+  llvm::Expected<std::unique_ptr<ASTUnit>>
+  loadASTOnDemand(StringRef ASTSourcePath) const;
+  llvm::Error lazyInitCompileCommands(StringRef CompileCommandsFile);
   void lazyInitImporterSharedSt(TranslationUnitDecl *ToTU);
   ASTImporter &getOrCreateASTImporter(ASTUnit *Unit);
   const FunctionDecl *findFunctionInDeclContext(const DeclContext *DC,
@@ -190,6 +203,9 @@ class CrossTranslationUnitContext {
   ASTContext &Context;
   std::shared_ptr<ASTImporterSharedState> ImporterSharedSt;
   unsigned NumASTLoaded{0u};
+  /// In case of on-demand parsing, the compilation database is parsed and
+  /// stored.
+  std::unique_ptr<tooling::JSONCompilationDatabase> CompileCommands;
 };
 
 } // namespace cross_tu

include/clang/StaticAnalyzer/Core/AnalyzerOptions.def

@@ -355,6 +355,21 @@ ANALYZER_OPTION(StringRef, CTUIndexName, "ctu-index-name",
                 "the name of the file containing the CTU index of definitions.",
                 "externalDefMap.txt")
 
+ANALYZER_OPTION(bool, CTUOnDemandParsing, "ctu-on-demand-parsing",
+                "Whether to parse function definitions from external TUs in "
+                "an on-demand manner during analysis. When using on-demand "
+                "parsing there is no need for pre-dumping ASTs. External "
+                "definition mapping is still needed, and a valid compilation "
+                "database with compile commands for the external TUs is also "
+                "necessary. Disabled by default.",
+                false)
+
+ANALYZER_OPTION(StringRef, CTUOnDemandParsingDatabase,
+                "ctu-on-demand-parsing-database",
+                "The path to the compilation database used for on-demand "
+                "parsing of ASTs during CTU analysis.",
+                "compile_commands.json")
+
 ANALYZER_OPTION(
     StringRef, ModelPath, "model-path",
     "The analyzer can inline an alternative implementation written in C at the "

lib/CrossTU/CMakeLists.txt

@@ -10,4 +10,5 @@ add_clang_library(clangCrossTU
   clangBasic
   clangFrontend
   clangIndex
+  clangTooling
   )