Add On-the-fly analysis support

Add an option to enable on-the-fly parsing of needed ASTs during CTU
analysis. The option CTUCompilationDatabase should be a path to a
compilation database, which has all the necessary information to
generate the ASTs. In case an empty string is given, on-the-fly parsing
is disabled.

Author: gamesh411

include/clang/CrossTU/CrossTranslationUnit.h

@@ -32,6 +32,10 @@ class FunctionDecl;
 class NamedDecl;
 class TranslationUnitDecl;
 
+namespace tooling {
+class JSONCompilationDatabase;
+}
+
 namespace cross_tu {
 
 enum class index_error_code {
@@ -41,6 +45,7 @@ enum class index_error_code {
   multiple_definitions,
   missing_definition,
   failed_import,
+  failed_to_load_compilation_database,
   failed_to_get_external_ast,
   failed_to_generate_usr,
   triple_mismatch,
@@ -85,7 +90,8 @@ class IndexError : public llvm::ErrorInfo<IndexError> {
 /// \return Returns a map where the USR is the key and the filepath is the value
 ///         or an error.
 llvm::Expected<llvm::StringMap<std::string>>
-parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir);
+parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir,
+                  llvm::Optional<StringRef> OnDemandParsingDatabase);
 
 std::string createCrossTUIndexString(const llvm::StringMap<std::string> &Index);
 
@@ -125,7 +131,8 @@ class CrossTranslationUnitContext {
   llvm::Expected<const FunctionDecl *>
   getCrossTUDefinition(const FunctionDecl *FD, StringRef CrossTUDir,
                        StringRef IndexName, bool DisplayCTUProgress,
-                       unsigned CTULoadThreshold);
+                       unsigned CTULoadThreshold,
+                       llvm::Optional<StringRef> OnDemandParsingDatabase);
 
   /// This function loads a function definition from an external AST
   ///        file.
@@ -141,11 +148,11 @@ class CrossTranslationUnitContext {
   /// The returned pointer is never a nullptr.
   ///
   /// Note that the AST files should also be in the \p CrossTUDir.
-  llvm::Expected<ASTUnit *> loadExternalAST(StringRef LookupName,
-                                            StringRef CrossTUDir,
-                                            StringRef IndexName,
-                                            bool DisplayCTUProgress,
-                                            unsigned CTULoadThreshold);
+  llvm::Expected<ASTUnit *>
+  loadExternalAST(StringRef LookupName, StringRef CrossTUDir,
+                  StringRef IndexName, bool DisplayCTUProgress,
+                  unsigned CTULoadThreshold,
+                  llvm::Optional<StringRef> OnDemandParsingDatabase);
 
   /// This function merges a definition from a separate AST Unit into
   ///        the current one which was created by the compiler instance that
@@ -172,6 +179,7 @@ class CrossTranslationUnitContext {
   GetImportedFromSourceLocation(const clang::SourceLocation &ToLoc) const;
 
 private:
+  std::unique_ptr<ASTUnit> loadASTOnDemand(StringRef ASTFileName) const;
   void lazyInitImporterSharedSt(TranslationUnitDecl *ToTU);
   ASTImporter &getOrCreateASTImporter(ASTUnit *Unit);
   const FunctionDecl *findFunctionInDeclContext(const DeclContext *DC,
@@ -190,6 +198,9 @@ class CrossTranslationUnitContext {
   ASTContext &Context;
   std::shared_ptr<ASTImporterSharedState> ImporterSharedSt;
   unsigned NumASTLoaded{0u};
+  /// In case of on-demand parsing, the compilation database is parsed and
+  /// stored.
+  std::unique_ptr<tooling::JSONCompilationDatabase> CompileCommands;
 };
 
 } // namespace cross_tu

include/clang/StaticAnalyzer/Core/AnalyzerOptions.def

@@ -355,6 +355,21 @@ ANALYZER_OPTION(StringRef, CTUIndexName, "ctu-index-name",
                 "the name of the file containing the CTU index of definitions.",
                 "externalDefMap.txt")
 
+ANALYZER_OPTION(bool, CTUOnDemandParsing, "ctu-on-demand-parsing",
+                "Whether to parse function definitions from external TUs in "
+                "an on-demand manner during analysis. When using on-demand "
+                "parsing there is no need for pre-dumping ASTs. External "
+                "definition mapping is still needed, and a valid compilation "
+                "database with compile commands for the external TUs is also "
+                "necessary. Disabled by default.",
+                false)
+
+ANALYZER_OPTION(StringRef, CTUOnDemandParsingDatabase,
+                "ctu-on-demand-parsing-database",
+                "The path to the compilation database used for on-demand "
+                "parsing of ASTs during CTU analysis.",
+                "compile_commands.json")
+
 ANALYZER_OPTION(
     StringRef, ModelPath, "model-path",
     "The analyzer can inline an alternative implementation written in C at the "

lib/CrossTU/CMakeLists.txt

@@ -10,4 +10,5 @@ add_clang_library(clangCrossTU
   clangBasic
   clangFrontend
   clangIndex
+  clangTooling
   )

lib/CrossTU/CrossTranslationUnit.cpp

@@ -19,12 +19,15 @@
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/TextDiagnosticPrinter.h"
 #include "clang/Index/USRGeneration.h"
-#include "llvm/ADT/Triple.h"
+#include "clang/Tooling/JSONCompilationDatabase.h"
+#include "clang/Tooling/Tooling.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/Triple.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
+#include <algorithm>
 #include <fstream>
 #include <sstream>
 
@@ -101,6 +104,8 @@ class IndexErrorCategory : public std::error_category {
       return "Failed to import the definition.";
     case index_error_code::failed_to_get_external_ast:
       return "Failed to load external AST source.";
+    case index_error_code::failed_to_load_compilation_database:
+      return "Failed to load compilation database.";
     case index_error_code::failed_to_generate_usr:
       return "Failed to generate USR.";
     case index_error_code::triple_mismatch:
@@ -130,7 +135,8 @@ std::error_code IndexError::convertToErrorCode() const {
 }
 
 llvm::Expected<llvm::StringMap<std::string>>
-parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir) {
+parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir,
+                  llvm::Optional<StringRef> OnDemandParsingDatabase) {
   std::ifstream ExternalMapFile(IndexPath);
   if (!ExternalMapFile)
     return llvm::make_error<IndexError>(index_error_code::missing_index_file,
@@ -148,9 +154,14 @@ parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir) {
         return llvm::make_error<IndexError>(
             index_error_code::multiple_definitions, IndexPath.str(), LineNo);
       StringRef FileName = LineRef.substr(Pos + 1);
-      SmallString<256> FilePath = CrossTUDir;
-      llvm::sys::path::append(FilePath, FileName);
-      Result[LookupName] = FilePath.str().str();
+      // AST-dump based analysis requires a prefixed path.
+      if (!OnDemandParsingDatabase) {
+        SmallString<256> FilePath = CrossTUDir;
+        llvm::sys::path::append(FilePath, FileName);
+        Result[LookupName] = FilePath.str().str();
+      } else {
+        Result[LookupName] = FileName.str();
+      }
     } else
       return llvm::make_error<IndexError>(
           index_error_code::invalid_index_format, IndexPath.str(), LineNo);
@@ -203,11 +214,10 @@ CrossTranslationUnitContext::findFunctionInDeclContext(const DeclContext *DC,
 }
 
 llvm::Expected<const FunctionDecl *>
-CrossTranslationUnitContext::getCrossTUDefinition(const FunctionDecl *FD,
-                                                  StringRef CrossTUDir,
-                                                  StringRef IndexName,
-                                                  bool DisplayCTUProgress,
-                                                  unsigned CTULoadThreshold) {
+CrossTranslationUnitContext::getCrossTUDefinition(
+    const FunctionDecl *FD, StringRef CrossTUDir, StringRef IndexName,
+    bool DisplayCTUProgress, unsigned CTULoadThreshold,
+    llvm::Optional<StringRef> OnDemandParsingDatabase) {
   assert(FD && "FD is missing, bad call to this function!");
   assert(!FD->hasBody() && "FD has a definition in current translation unit!");
   ++NumGetCTUCalled;
@@ -217,7 +227,7 @@ CrossTranslationUnitContext::getCrossTUDefinition(const FunctionDecl *FD,
         index_error_code::failed_to_generate_usr);
   llvm::Expected<ASTUnit *> ASTUnitOrError =
       loadExternalAST(LookupFnName, CrossTUDir, IndexName, DisplayCTUProgress,
-                      CTULoadThreshold);
+                      CTULoadThreshold, OnDemandParsingDatabase);
   if (!ASTUnitOrError)
     return ASTUnitOrError.takeError();
   ASTUnit *Unit = *ASTUnitOrError;
@@ -302,9 +312,93 @@ void CrossTranslationUnitContext::emitCrossTUDiagnostics(const IndexError &IE) {
   }
 }
 
+/// Load the AST from a source-file, which is supposed to be located inside the
+/// compilation database \p OnDemandParsingCommands. The compilation database
+/// can contain the path of the file under the key "file" as an absolute path,
+/// or as a relative path. When emitting diagnostics, plist files may contain
+/// references to a location in a TU, that is different from the main TU. In
+/// such cases, the file path emitted by the DiagnosticEngine is based on how
+/// the exact invocation is assembled inside the ClangTool, which performs the
+/// building of the ASTs. In order ensure absolute paths inside the diagnostics,
+/// we use the ArgumentsAdjuster API of ClangTool to make sure that the
+/// invocation inside ClangTool is always made with an absolute path. \p
+/// ASTSourcePath is assumed to be the lookup-name of the file, which comes from
+/// the Index. The Index is built by the \p clang-extdef-mapping tool, which is
+/// supposed to generate absolute paths.
+///
+/// We must have absolute paths inside the plist, because otherwise we would
+/// not be able to parse the bug, because we could not find the files with
+/// relative paths. The directory of one entry in the compilation db may be
+/// different from the directory where the plist is interpreted.
+///
+/// Note that as the ClangTool is instantiated with a lookup-vector, which
+/// contains a single entry; the supposedly absolute path of the source file.
+/// So, the ArgumentAdjuster will only be used on the single corresponding
+/// invocation. This garantees that even if two files match in name, but
+/// differ in location, only the correct one's invocation will be handled. This
+/// is due to the fact that the lookup is done correctly inside the
+/// OnDemandParsingDatabase, so it works for already absolute paths given under
+/// the "file" entry of the compilation database, but also if a relative path is
+/// given. In such a case, the lookup uses the "directory" entry as well to
+/// identify the correct file.
+std::unique_ptr<ASTUnit>
+CrossTranslationUnitContext::loadASTOnDemand(StringRef ASTSourcePath) const {
+
+  using namespace tooling;
+
+  SmallVector<std::string, 1> Files;
+  Files.push_back(ASTSourcePath);
+  ClangTool Tool(*CompileCommands, Files, CI.getPCHContainerOperations());
+
+  /// Lambda filter designed to find the source file argument inside an
+  /// invocation used to build the ASTs, and replace it with its absolute path
+  /// equivalent.
+  auto SourcePathNormalizer = [ASTSourcePath](const CommandLineArguments &Args,
+                                              StringRef FileName) {
+    /// Match the argument to the absolute path by checking whether it is a
+    /// postfix.
+    auto IsPostfixOfLookup = [ASTSourcePath](const std::string &Arg) {
+      return ASTSourcePath.rfind(Arg) != llvm::StringRef::npos;
+    };
+
+    /// Commandline arguments are modified, and the API dictates the return a
+    /// new instance, so copy the original.
+    CommandLineArguments Result{Args};
+
+    /// Search for the source file argument. Start from the end as a heuristic,
+    /// as most invocations tend to contain the source file argument in their
+    /// the latter half. Only the first match is replaced.
+    auto SourceFilePath =
+        std::find_if(Result.rbegin(), Result.rend(), IsPostfixOfLookup);
+
+    /// If source file argument could not been found, return the original
+    /// CommandlineArgumentsInstance.
+    if (SourceFilePath == Result.rend())
+      return Result;
+
+    /// Overwrite the argument with the \p ASTSourcePath, as it is assumed to be
+    /// the absolute path of the file.
+    *SourceFilePath = ASTSourcePath.str();
+
+    return Result;
+  };
+
+  Tool.appendArgumentsAdjuster(std::move(SourcePathNormalizer));
+
+  std::vector<std::unique_ptr<ASTUnit>> ASTs;
+  Tool.buildASTs(ASTs);
+
+  if (ASTs.size() > 0) {
+    ASTs[0]->enableSourceFileDiagnostics();
+    return std::move(ASTs[0]);
+  } else
+    return nullptr;
+}
+
 llvm::Expected<ASTUnit *> CrossTranslationUnitContext::loadExternalAST(
     StringRef LookupName, StringRef CrossTUDir, StringRef IndexName,
-    bool DisplayCTUProgress, unsigned CTULoadThreshold) {
+    bool DisplayCTUProgress, unsigned CTULoadThreshold,
+    llvm::Optional<StringRef> OnDemandParsingDatabase) {
   // FIXME: The current implementation only supports loading functions with
   //        a lookup name from a single translation unit. If multiple
   //        translation units contains functions with the same lookup name an
@@ -326,7 +420,7 @@ llvm::Expected<ASTUnit *> CrossTranslationUnitContext::loadExternalAST(
       else
         llvm::sys::path::append(IndexFile, IndexName);
       llvm::Expected<llvm::StringMap<std::string>> IndexOrErr =
-          parseCrossTUIndex(IndexFile, CrossTUDir);
+          parseCrossTUIndex(IndexFile, CrossTUDir, OnDemandParsingDatabase);
       if (IndexOrErr)
         FunctionFileMap = *IndexOrErr;
       else
@@ -338,33 +432,63 @@ llvm::Expected<ASTUnit *> CrossTranslationUnitContext::loadExternalAST(
       ++NumNotInOtherTU;
       return llvm::make_error<IndexError>(index_error_code::missing_definition);
     }
-    StringRef ASTFileName = It->second;
-    auto ASTCacheEntry = FileASTUnitMap.find(ASTFileName);
+    StringRef ASTSource = It->second;
+    auto ASTCacheEntry = FileASTUnitMap.find(ASTSource);
     if (ASTCacheEntry == FileASTUnitMap.end()) {
-      IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
-      TextDiagnosticPrinter *DiagClient =
-          new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
-      IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-      IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-          new DiagnosticsEngine(DiagID, &*DiagOpts, DiagClient));
-
-      std::unique_ptr<ASTUnit> LoadedUnit(ASTUnit::LoadFromASTFile(
-          ASTFileName, CI.getPCHContainerOperations()->getRawReader(),
-          ASTUnit::LoadEverything, Diags, CI.getFileSystemOpts()));
-      Unit = LoadedUnit.get();
-      FileASTUnitMap[ASTFileName] = std::move(LoadedUnit);
+      if (!OnDemandParsingDatabase) {
+        // If no \p OnDemandParsingDatabase is given, try to load from AST dump
+        // file, as on-demand parsing is disabled.
+        IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts =
+            new DiagnosticOptions();
+        TextDiagnosticPrinter *DiagClient =
+            new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
+        IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
+        IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
+            new DiagnosticsEngine(DiagID, &*DiagOpts, DiagClient));
+
+        std::unique_ptr<ASTUnit> LoadedUnit(ASTUnit::LoadFromASTFile(
+            ASTSource, CI.getPCHContainerOperations()->getRawReader(),
+            ASTUnit::LoadEverything, Diags, CI.getFileSystemOpts()));
+        Unit = LoadedUnit.get();
+
+        // Cache the resulting ASTUnit.
+        if (Unit)
+          FileASTUnitMap[ASTSource] = std::move(LoadedUnit);
+      } else {
+
+        // Lazily initialize the compilation database.
+        if (!CompileCommands) {
+          std::string LoadError;
+          CompileCommands = tooling::JSONCompilationDatabase::loadFromFile(
+              OnDemandParsingDatabase.getValue(), LoadError,
+              tooling::JSONCommandLineSyntax::AutoDetect);
+          if (!CompileCommands)
+            return llvm::make_error<IndexError>(
+                index_error_code::failed_to_get_external_ast);
+        }
+
+        // Try loading on-demand.
+        std::unique_ptr<ASTUnit> LoadedUnit = loadASTOnDemand(ASTSource);
+        Unit = LoadedUnit.get();
+
+        // Cache the sulting ASTUnit.
+        if (Unit)
+          FileASTUnitMap[ASTSource] = std::move(LoadedUnit);
+      }
       ++NumASTLoaded;
       if (DisplayCTUProgress) {
-        llvm::errs() << "CTU loaded AST file: "
-                     << ASTFileName << "\n";
+        llvm::errs() << "CTU loaded AST file: " << ASTSource << "\n";
       }
     } else {
       Unit = ASTCacheEntry->second.get();
     }
-    FunctionASTUnitMap[LookupName] = Unit;
+    // Fill the cache for the lookup name as well.
+    if (Unit)
+      FunctionASTUnitMap[LookupName] = Unit;
   } else {
     Unit = FnUnitCacheEntry->second;
   }
+
   if (!Unit)
     return llvm::make_error<IndexError>(
         index_error_code::failed_to_get_external_ast);
@@ -397,7 +521,7 @@ CrossTranslationUnitContext::importDefinition(const FunctionDecl *FD,
                     });
     return llvm::make_error<IndexError>(index_error_code::failed_import);
   }
-  auto *ToDecl = cast<FunctionDecl>(*ToDeclOrError);  
+  auto *ToDecl = cast<FunctionDecl>(*ToDeclOrError);
   assert(ToDecl->hasBody() && "Imported function should have body.");
   ++NumGetCTUSuccess;
 

lib/Frontend/CompilerInvocation.cpp

@@ -460,6 +460,12 @@ static void parseAnalyzerConfigs(AnalyzerOptions &AnOpts,
     Diags->Report(diag::err_analyzer_config_invalid_input) << "ctu-dir"
                                                            << "a filename";
 
+  if (AnOpts.CTUOnDemandParsing &&
+      !llvm::sys::fs::exists(AnOpts.CTUOnDemandParsingDatabase))
+    Diags->Report(diag::err_analyzer_config_invalid_input)
+        << "ctu-on-demand-parsing-database"
+        << "a filename";
+
   if (!AnOpts.ModelPath.empty() &&
       !llvm::sys::fs::is_directory(AnOpts.ModelPath))
     Diags->Report(diag::err_analyzer_config_invalid_input) << "model-path"