On demand parsing for clang-11

clang/include/clang/CrossTU/CrossTranslationUnit.h

@@ -20,6 +20,8 @@
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/OperandTraits.h"
 #include "llvm/Support/Error.h"
 
 namespace clang {
@@ -33,6 +35,10 @@ class VarDecl;
 class NamedDecl;
 class TranslationUnitDecl;
 
+namespace tooling {
+class JSONCompilationDatabase;
+}
+
 namespace cross_tu {
 
 enum class index_error_code {
@@ -42,12 +48,14 @@ enum class index_error_code {
   multiple_definitions,
   missing_definition,
   failed_import,
+  failed_to_load_compilation_database,
   failed_to_get_external_ast,
   failed_to_generate_usr,
   triple_mismatch,
   lang_mismatch,
   lang_dialect_mismatch,
-  load_threshold_reached
+  load_threshold_reached,
+  ambiguous_compile_commands_database
 };
 
 class IndexError : public llvm::ErrorInfo<IndexError> {
@@ -86,7 +94,7 @@ class IndexError : public llvm::ErrorInfo<IndexError> {
 /// \return Returns a map where the USR is the key and the filepath is the value
 ///         or an error.
 llvm::Expected<llvm::StringMap<std::string>>
-parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir);
+parseCrossTUIndex(StringRef IndexPath);
 
 std::string createCrossTUIndexString(const llvm::StringMap<std::string> &Index);
 
@@ -180,6 +188,7 @@ class CrossTranslationUnitContext {
   using ImportedFileIDMap =
       llvm::DenseMap<FileID, std::pair<FileID, ASTUnit *>>;
 
+
   void lazyInitImporterSharedSt(TranslationUnitDecl *ToTU);
   ASTImporter &getOrCreateASTImporter(ASTUnit *Unit);
   template <typename T>
@@ -209,16 +218,47 @@ class CrossTranslationUnitContext {
   /// imported the FileID.
   ImportedFileIDMap ImportedFileIDs;
 
-  /// Functor for loading ASTUnits from AST-dump files.
-  class ASTFileLoader {
+  using LoadResultTy = llvm::Expected<std::unique_ptr<ASTUnit>>;
+
+  struct ASTLoader {
+    /// Load the ASTUnit by an identifier. Subclasses should determine what this would be.
+    virtual LoadResultTy load(StringRef Identifier) = 0;
+    virtual ~ASTLoader() = default;
+  };
+
+  /// Implementation for loading ASTUnits from AST-dump files.
+  class ASTFileLoader: public ASTLoader {
+  public:
+    explicit ASTFileLoader(CompilerInstance &CI, StringRef CTUDir);
+
+    /// ASTFileLoader uses a the path of the dump file as Identifier.
+    LoadResultTy load(StringRef Identifier) override;
+
+  private:
+    CompilerInstance &CI;
+    StringRef CTUDir;
+  };
+
+  /// Implementation for loading ASTUnits by parsing them on-demand.
+  class ASTOnDemandLoader: public ASTLoader {
   public:
-    ASTFileLoader(const CompilerInstance &CI);
-    std::unique_ptr<ASTUnit> operator()(StringRef ASTFilePath);
+    ASTOnDemandLoader(CompilerInstance &CI, StringRef OnDemandParsingDatabase);
+
+    /// ASTOnDemandLoader uses the path of the source file to be parsed as Identifier.
+    LoadResultTy load(StringRef Identifier) override;
+
+    llvm::Error lazyInitCompileCommands();
 
   private:
-    const CompilerInstance &CI;
+    CompilerInstance &CI;
+    StringRef OnDemandParsingDatabase;
+    /// In case of on-demand parsing, the compilation database is parsed and
+    /// stored.
+    std::unique_ptr<tooling::JSONCompilationDatabase> CompileCommands;
   };
 
+
+
   /// Maintain number of AST loads and check for reaching the load limit.
   class ASTLoadGuard {
   public:
@@ -242,7 +282,7 @@ class CrossTranslationUnitContext {
   /// are the concerns of ASTUnitStorage class.
   class ASTUnitStorage {
   public:
-    ASTUnitStorage(const CompilerInstance &CI);
+    ASTUnitStorage(CompilerInstance &CI);
     /// Loads an ASTUnit for a function.
     ///
     /// \param FunctionName USR name of the function.
@@ -287,18 +327,17 @@ class CrossTranslationUnitContext {
     using IndexMapTy = BaseMapTy<std::string>;
     IndexMapTy NameFileMap;
 
-    ASTFileLoader FileAccessor;
+    std::unique_ptr<ASTLoader> Loader;
 
-    /// Limit the number of loaded ASTs. Used to limit the  memory usage of the
+    /// Limit the number of loaded ASTs. It is used to limit the  memory usage of the
     /// CrossTranslationUnitContext.
-    /// The ASTUnitStorage has the knowledge about if the AST to load is
+    /// The ASTUnitStorage has the information whether the AST to load is
     /// actually loaded or returned from cache. This information is needed to
     /// maintain the counter.
     ASTLoadGuard LoadGuard;
   };
 
   ASTUnitStorage ASTStorage;
-
 };
 
 } // namespace cross_tu

clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def

@@ -377,6 +377,21 @@ ANALYZER_OPTION(StringRef, CTUIndexName, "ctu-index-name",
                 "the name of the file containing the CTU index of definitions.",
                 "externalDefMap.txt")
 
+ANALYZER_OPTION(bool, CTUOnDemandParsing, "ctu-on-demand-parsing",
+                "Whether to parse function definitions from external TUs in "
+                "an on-demand manner during analysis. When using on-demand "
+                "parsing there is no need for pre-dumping ASTs. External "
+                "definition mapping is still needed, and a valid compilation "
+                "database with compile commands for the external TUs is also "
+                "necessary. Disabled by default.",
+                false)
+
+ANALYZER_OPTION(StringRef, CTUOnDemandParsingDatabase,
+                "ctu-on-demand-parsing-database",
+                "The path to the compilation database used for on-demand "
+                "parsing of ASTs during CTU analysis.",
+                "compile_commands.json")
+
 ANALYZER_OPTION(
     StringRef, ModelPath, "model-path",
     "The analyzer can inline an alternative implementation written in C at the "

clang/lib/CrossTU/CMakeLists.txt

@@ -10,4 +10,5 @@ add_clang_library(clangCrossTU
   clangBasic
   clangFrontend
   clangIndex
+  clangTooling
   )