Faster incremental imports

index-import.cpp

@@ -1,12 +1,15 @@
 #include "clang/Basic/FileManager.h"
 #include "clang/Index/IndexUnitReader.h"
 #include "clang/Index/IndexUnitWriter.h"
+#include "llvm/ADT/APInt.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Errc.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 
 #include <cstdlib>
+#include <fstream>
 #include <regex>
 #include <set>
 #include <string>
@@ -27,7 +30,17 @@ static cl::list<std::string> PathRemaps("remap", cl::OneOrMore,
                                         cl::value_desc("regex=replacement"));
 static cl::alias PathRemapsAlias("r", cl::aliasopt(PathRemaps));
 
-static cl::list<std::string> InputIndexPaths(cl::Positional, cl::OneOrMore,
+static cl::opt<std::string> UnitPathsFile(
+    "unit-paths-file",
+    cl::desc("Path to a file containing paths to unit files. If specified "
+             "record-paths-file must also be specified"));
+
+static cl::opt<std::string> RecordPathsFile(
+    "record-paths-file",
+    cl::desc("Path to a file containing paths to record files. If specified "
+             "unit-paths-file must also be specified"));
+
+static cl::list<std::string> InputIndexPaths(cl::Positional, cl::ZeroOrMore,
                                              cl::desc("<input-indexstores>"));
 
 static cl::opt<std::string> OutputIndexPath(cl::Positional, cl::Required,
@@ -102,12 +115,96 @@ static const FileEntry *getFileEntry(FileManager &fileMgr, StringRef path) {
   return fileMgr.getVirtualFile(path, /*size*/ 0, /*modtime*/ 0);
 }
 
-static IndexUnitWriter remapUnit(const std::unique_ptr<IndexUnitReader> &reader,
-                                 const Remapper &remapper, FileManager &fileMgr,
-                                 ModuleNameScope &moduleNames) {
+void getUnitPathForOutputFile(StringRef unitsPath, StringRef filePath,
+                              SmallVectorImpl<char> &str,
+                              FileManager &fileMgr) {
+  str.append(unitsPath.begin(), unitsPath.end());
+  str.push_back('/');
+  SmallString<256> absPath(filePath);
+  fileMgr.makeAbsolutePath(absPath);
+  StringRef fname = sys::path::filename(absPath);
+  str.append(fname.begin(), fname.end());
+  str.push_back('-');
+  llvm::hash_code pathHashVal = llvm::hash_value(absPath);
+  llvm::APInt(64, pathHashVal).toString(str, 36, /*Signed=*/false);
+}
+
+Optional<bool>
+isUnitUpToDateForOutputFile(StringRef unitsPath, StringRef filePath,
+                            Optional<StringRef> timeCompareFilePath,
+                            FileManager &fileMgr, std::string &error) {
+  SmallString<256> unitPath;
+  getUnitPathForOutputFile(unitsPath, filePath, unitPath, fileMgr);
+
+  llvm::sys::fs::file_status unitStat;
+  if (std::error_code ec = llvm::sys::fs::status(unitPath.c_str(), unitStat)) {
+    if (ec != llvm::errc::no_such_file_or_directory) {
+      llvm::raw_string_ostream err(error);
+      err << "could not access path '" << unitPath << "': " << ec.message();
+      return None;
+    }
+    return false;
+  }
+
+  if (!timeCompareFilePath.hasValue())
+    return true;
+
+  llvm::sys::fs::file_status compareStat;
+  if (std::error_code ec =
+          llvm::sys::fs::status(*timeCompareFilePath, compareStat)) {
+    if (ec != llvm::errc::no_such_file_or_directory) {
+      llvm::raw_string_ostream err(error);
+      err << "could not access path '" << *timeCompareFilePath
+          << "': " << ec.message();
+      return None;
+    }
+    return true;
+  }
+
+  // Return true (unit is up-to-date) if the file to compare is older than the
+  // unit file.
+  return compareStat.getLastModificationTime() <=
+         unitStat.getLastModificationTime();
+}
+
+// Returns true if the Unit file of given output file already exists and is
+// older than the input file.
+static bool isUnitUpToDate(StringRef unitsPath, StringRef outputFile,
+                           StringRef inputFile, FileManager &fileMgr) {
+  std::string error;
+  auto isUpToDateOpt = isUnitUpToDateForOutputFile(unitsPath, outputFile,
+                                                   inputFile, fileMgr, error);
+  if (!isUpToDateOpt.hasValue()) {
+    errs() << "error: failed file status check:\n" << error << "\n";
+    return false;
+  }
+
+  return *isUpToDateOpt;
+}
+
+// Returns None if the Unit file is already up to date
+static Optional<IndexUnitWriter>
+remapUnit(StringRef outputUnitsPath, StringRef inputUnitPath,
+          const std::unique_ptr<IndexUnitReader> &reader,
+          const Remapper &remapper, FileManager &fileMgr,
+          ModuleNameScope &moduleNames) {
   // The set of remapped paths.
   auto workingDir = remapper.remap(reader->getWorkingDirectory());
   auto outputFile = remapper.remap(reader->getOutputFile());
+
+  // Check if the unit file is already up to date
+  SmallString<256> remappedOutputFilePath;
+  if (outputFile[0] != '/') {
+    // Convert outputFile to absolute path
+    path::append(remappedOutputFilePath, workingDir, outputFile);
+  } else {
+    remappedOutputFilePath = outputFile;
+  }
+  if (isUnitUpToDate(outputUnitsPath, remappedOutputFilePath, inputUnitPath,
+                     fileMgr)) {
+    return None;
+  }
+
   auto mainFilePath = remapper.remap(reader->getMainFilePath());
   auto sysrootPath = remapper.remap(reader->getSysrootPath());
 
@@ -184,6 +281,17 @@ static bool cloneRecord(StringRef from, StringRef to) {
   return (failed == 0 || errno == EEXIST);
 }
 
+static bool createDirectories(StringRef path) {
+  std::error_code failed =
+      fs::create_directories(path, /*IgnoreExisting=*/true);
+  if (failed) {
+    errs() << "Could not create directory `" << path
+           << "`: " << failed.message() << "\n";
+    return false;
+  }
+  return true;
+}
+
 static bool cloneRecords(StringRef recordsDirectory,
                          const std::string &inputIndexPath,
                          const std::string &outputIndexPath) {
@@ -206,11 +314,8 @@ static bool cloneRecords(StringRef recordsDirectory,
     path::replace_path_prefix(outputPath, inputIndexPath, outputIndexPath);
 
     if (status->type() == fs::file_type::directory_file) {
-      std::error_code failed = fs::create_directory(outputPath);
-      if (failed && failed != std::errc::file_exists) {
+      if (not createDirectories(outputPath)) {
         success = false;
-        errs() << "Could not create directory `" << outputPath
-               << "`: " << failed.message() << "\n";
       }
     } else if (status->type() == fs::file_type::regular_file) {
       if (not cloneRecord(inputPath, outputPath)) {
@@ -230,6 +335,39 @@ static bool cloneRecords(StringRef recordsDirectory,
   return success;
 }
 
+static bool cloneRecords(const std::vector<std::string> &recordPaths,
+                         const std::string &outputIndexPath) {
+  bool success = true;
+
+  for (auto inputPath : recordPaths) {
+    auto inputIndexPath = path::parent_path(
+        path::parent_path(path::parent_path(path::parent_path(inputPath))));
+
+    SmallString<128> outputPath{inputPath};
+    path::replace_path_prefix(outputPath, inputIndexPath, outputIndexPath);
+
+    if (not cloneRecord(inputPath, outputPath)) {
+      bool innerSuccess = true;
+      if (errno == ENOENT) {
+        // We need to create the directories below it
+        if (not createDirectories(path::parent_path(outputPath))) {
+          innerSuccess = false;
+        } else if (not cloneRecord(inputPath, outputPath)) {
+          innerSuccess = false;
+        }
+      }
+
+      if (!innerSuccess) {
+        success = false;
+        errs() << "Could not copy record file from `" << inputPath << "` to `"
+               << outputPath << "`: " << strerror(errno) << "\n";
+      }
+    }
+  }
+
+  return success;
+}
+
 // Normalize a path by removing /./ or // from it.
 static std::string normalizePath(StringRef Path) {
   SmallString<128> NormalizedPath;
@@ -247,6 +385,8 @@ static bool remapIndex(const Remapper &remapper,
   path::append(unitDirectory, InputIndexPath, "v5", "units");
   SmallString<256> recordsDirectory;
   path::append(recordsDirectory, InputIndexPath, "v5", "records");
+  SmallString<256> outputUnitDirectory;
+  path::append(outputUnitDirectory, OutputIndexPath, "v5", "units");
 
   if (not fs::is_directory(unitDirectory) ||
       not fs::is_directory(recordsDirectory)) {
@@ -284,13 +424,16 @@ static bool remapIndex(const Remapper &remapper,
     }
 
     ModuleNameScope moduleNames;
-    auto writer = remapUnit(reader, remapper, fileMgr, moduleNames);
-
-    std::string unitWriteError;
-    if (writer.write(unitWriteError)) {
-      errs() << "error: failed to write index store; " << unitWriteError
-             << "\n";
-      success = false;
+    auto writer = remapUnit(outputUnitDirectory, unitPath, reader, remapper,
+                            fileMgr, moduleNames);
+
+    if (writer.hasValue()) {
+      std::string unitWriteError;
+      if (writer->write(unitWriteError)) {
+        errs() << "error: failed to write index store; " << unitWriteError
+               << "\n";
+        success = false;
+      }
     }
   }
 
@@ -302,6 +445,79 @@ static bool remapIndex(const Remapper &remapper,
   return success;
 }
 
+static bool remapUnits(const Remapper &remapper,
+                       const std::vector<std::string> &unitPaths,
+                       const SmallString<256> &outputUnitDirectory) {
+  FileSystemOptions fsOpts;
+  FileManager fileMgr{fsOpts};
+
+  bool success = true;
+
+  for (const auto unitPath : unitPaths) {
+    std::string unitReadError;
+    auto reader = IndexUnitReader::createWithFilePath(unitPath, unitReadError);
+    if (not reader) {
+      errs() << "error: failed to read unit file " << unitPath << "\n"
+             << unitReadError;
+      success = false;
+      continue;
+    }
+
+    ModuleNameScope moduleNames;
+    auto writer = remapUnit(outputUnitDirectory, unitPath, reader, remapper,
+                            fileMgr, moduleNames);
+
+    if (writer.hasValue()) {
+      std::string unitWriteError;
+      if (writer->write(unitWriteError)) {
+        errs() << "error: failed to write index store; " << unitWriteError
+               << "\n";
+        success = false;
+      }
+    }
+  }
+
+  return success;
+}
+
+static bool remapIndex(const Remapper &remapper,
+                       const std::vector<std::string> &unitPaths,
+                       const std::vector<std::string> &recordPaths,
+                       const std::string &outputIndexPath) {
+  SmallString<256> outputUnitDirectory;
+  path::append(outputUnitDirectory, OutputIndexPath, "v5", "units");
+
+  __block bool success = true;
+
+  if (not cloneRecords(recordPaths, outputIndexPath)) {
+    success = false;
+  }
+
+  if (ParallelStride == 0 || ParallelStride >= unitPaths.size()) {
+    if (not remapUnits(remapper, unitPaths, outputUnitDirectory)) {
+      success = false;
+    }
+    return success;
+  }
+
+  // Process the units in groups according to the parallel stride.
+  const size_t stride = static_cast<size_t>(ParallelStride);
+  const size_t length = unitPaths.size();
+  const size_t numStrides = ((length - 1) / stride) + 1;
+
+  dispatch_apply(numStrides, DISPATCH_APPLY_AUTO, ^(size_t strideIndex) {
+    const size_t offset = strideIndex * stride;
+    auto start = unitPaths.begin() + offset;
+    auto end = unitPaths.begin() + std::min(offset + stride, length);
+    std::vector<std::string> unitPathsSlice(start, end);
+    if (not remapUnits(remapper, unitPathsSlice, outputUnitDirectory)) {
+      success = false;
+    }
+  });
+
+  return success;
+}
+
 int main(int argc, char **argv) {
   cl::ParseCommandLineOptions(argc, argv);
 
@@ -340,6 +556,41 @@ int main(int argc, char **argv) {
     return EXIT_FAILURE;
   }
 
+  // If given the paths to individual unit paths and records, use those instead
+  // of enumerating directories
+  if (!UnitPathsFile.empty() || !RecordPathsFile.empty()) {
+    if (UnitPathsFile.empty()) {
+      errs() << "record-paths-file set but not unit-paths-file.\n";
+      return EXIT_FAILURE;
+    }
+    if (RecordPathsFile.empty()) {
+      errs() << "unit-paths-file set but not record-paths-file.\n";
+      return EXIT_FAILURE;
+    }
+
+    std::ifstream unitPathsStream(UnitPathsFile);
+    std::vector<std::string> unitPaths;
+    std::string unitPath;
+    while (unitPathsStream >> unitPath) {
+      unitPaths.push_back(unitPath);
+    }
+
+    std::ifstream recordPathsStream(RecordPathsFile);
+    std::vector<std::string> recordPaths;
+    std::string recordPath;
+    while (recordPathsStream >> recordPath) {
+      recordPaths.push_back(recordPath);
+    }
+
+    return remapIndex(remapper, unitPaths, recordPaths, OutputIndexPath)
+               ? EXIT_SUCCESS
+               : EXIT_FAILURE;
+  } else if (InputIndexPaths.empty()) {
+    errs() << "input-indexstores must be set if unit-paths-file and "
+              "record-paths-file are not.\n";
+    return EXIT_FAILURE;
+  }
+
   if (ParallelStride == 0 || ParallelStride >= InputIndexPaths.size()) {
     bool success = true;
     for (auto &InputIndexPath : InputIndexPaths) {