Merge from 'master' to 'sycl-web' (#89)

Author: Valery N Dmitriev

clang-tools-extra/clang-change-namespace/ChangeNamespace.cpp

@@ -19,14 +19,8 @@ namespace change_namespace {
 
 namespace {
 
-inline std::string
-joinNamespaces(const llvm::SmallVectorImpl<StringRef> &Namespaces) {
-  if (Namespaces.empty())
-    return "";
-  std::string Result(Namespaces.front());
-  for (auto I = Namespaces.begin() + 1, E = Namespaces.end(); I != E; ++I)
-    Result += ("::" + *I).str();
-  return Result;
+inline std::string joinNamespaces(ArrayRef<StringRef> Namespaces) {
+  return llvm::join(Namespaces, "::");
 }
 
 // Given "a::b::c", returns {"a", "b", "c"}.

clang-tools-extra/clangd/ClangdServer.h

@@ -100,7 +100,7 @@ class ClangdServer {
     bool StorePreamblesInMemory = true;
     /// Reuse even stale preambles, and rebuild them in the background.
     /// This improves latency at the cost of accuracy.
-    bool AsyncPreambleBuilds = false;
+    bool AsyncPreambleBuilds = true;
 
     /// If true, ClangdServer builds a dynamic in-memory index for symbols in
     /// opened files and uses the index to augment code completion results.

clang-tools-extra/clangd/ConfigYAML.cpp

@@ -26,49 +26,47 @@ using llvm::yaml::SequenceNode;
 
 class Parser {
   llvm::SourceMgr &SM;
+  bool HadError = false;
 
 public:
   Parser(llvm::SourceMgr &SM) : SM(SM) {}
 
   // Tries to parse N into F, returning false if it failed and we couldn't
-  // meaningfully recover (e.g. YAML syntax error broke the stream).
-  // The private parse() helpers follow the same pattern.
+  // meaningfully recover (YAML syntax error, or hard semantic error).
   bool parse(Fragment &F, Node &N) {
     DictParser Dict("Config", this);
-    Dict.handle("If", [&](Node &N) { return parse(F.If, N); });
-    Dict.handle("CompileFlags",
-                [&](Node &N) { return parse(F.CompileFlags, N); });
-    return Dict.parse(N);
+    Dict.handle("If", [&](Node &N) { parse(F.If, N); });
+    Dict.handle("CompileFlags", [&](Node &N) { parse(F.CompileFlags, N); });
+    Dict.parse(N);
+    return !(N.failed() || HadError);
   }
 
 private:
-  bool parse(Fragment::IfBlock &F, Node &N) {
+  void parse(Fragment::IfBlock &F, Node &N) {
     DictParser Dict("If", this);
     Dict.unrecognized(
         [&](llvm::StringRef) { F.HasUnrecognizedCondition = true; });
     Dict.handle("PathMatch", [&](Node &N) {
       if (auto Values = scalarValues(N))
         F.PathMatch = std::move(*Values);
-      return !N.failed();
     });
-    return Dict.parse(N);
+    Dict.parse(N);
   }
 
-  bool parse(Fragment::CompileFlagsBlock &F, Node &N) {
+  void parse(Fragment::CompileFlagsBlock &F, Node &N) {
     DictParser Dict("CompileFlags", this);
     Dict.handle("Add", [&](Node &N) {
       if (auto Values = scalarValues(N))
         F.Add = std::move(*Values);
-      return !N.failed();
     });
-    return Dict.parse(N);
+    Dict.parse(N);
   }
 
   // Helper for parsing mapping nodes (dictionaries).
   // We don't use YamlIO as we want to control over unknown keys.
   class DictParser {
     llvm::StringRef Description;
-    std::vector<std::pair<llvm::StringRef, std::function<bool(Node &)>>> Keys;
+    std::vector<std::pair<llvm::StringRef, std::function<void(Node &)>>> Keys;
     std::function<void(llvm::StringRef)> Unknown;
     Parser *Outer;
 
@@ -79,7 +77,7 @@ class Parser {
     // Parse is called when Key is encountered, and passed the associated value.
     // It should emit diagnostics if the value is invalid (e.g. wrong type).
     // If Key is seen twice, Parse runs only once and an error is reported.
-    void handle(llvm::StringLiteral Key, std::function<bool(Node &)> Parse) {
+    void handle(llvm::StringLiteral Key, std::function<void(Node &)> Parse) {
       for (const auto &Entry : Keys) {
         (void) Entry;
         assert(Entry.first != Key && "duplicate key handler");
@@ -94,16 +92,17 @@ class Parser {
     }
 
     // Process a mapping node and call handlers for each key/value pair.
-    bool parse(Node &N) const {
+    void parse(Node &N) const {
       if (N.getType() != Node::NK_Mapping) {
         Outer->error(Description + " should be a dictionary", N);
-        return false;
+        return;
       }
       llvm::SmallSet<std::string, 8> Seen;
+      // We *must* consume all items, even on error, or the parser will assert.
       for (auto &KV : llvm::cast<MappingNode>(N)) {
         auto *K = KV.getKey();
         if (!K) // YAMLParser emitted an error.
-          return false;
+          continue;
         auto Key = Outer->scalarValue(*K, "Dictionary key");
         if (!Key)
           continue;
@@ -113,13 +112,12 @@ class Parser {
         }
         auto *Value = KV.getValue();
         if (!Value) // YAMLParser emitted an error.
-          return false;
+          continue;
         bool Matched = false;
         for (const auto &Handler : Keys) {
           if (Handler.first == **Key) {
-            if (!Handler.second(*Value))
-              return false;
             Matched = true;
+            Handler.second(*Value);
             break;
           }
         }
@@ -129,7 +127,6 @@ class Parser {
             Unknown(**Key);
         }
       }
-      return true;
     }
   };
 
@@ -154,6 +151,7 @@ class Parser {
     } else if (auto *S = llvm::dyn_cast<BlockScalarNode>(&N)) {
       Result.emplace_back(S->getValue().str(), N.getSourceRange());
     } else if (auto *S = llvm::dyn_cast<SequenceNode>(&N)) {
+      // We *must* consume all items, even on error, or the parser will assert.
       for (auto &Child : *S) {
         if (auto Value = scalarValue(Child, "List item"))
           Result.push_back(std::move(*Value));
@@ -167,6 +165,7 @@ class Parser {
 
   // Report a "hard" error, reflecting a config file that can never be valid.
   void error(const llvm::Twine &Msg, const Node &N) {
+    HadError = true;
     SM.PrintMessage(N.getSourceRange().Start, llvm::SourceMgr::DK_Error, Msg,
                     N.getSourceRange());
   }
@@ -196,7 +195,7 @@ std::vector<Fragment> Fragment::parseYAML(llvm::StringRef YAML,
       &Diags);
   std::vector<Fragment> Result;
   for (auto &Doc : llvm::yaml::Stream(*Buf, *SM)) {
-    if (Node *N = Doc.parseBlockNode()) {
+    if (Node *N = Doc.getRoot()) {
       Fragment Fragment;
       Fragment.Source.Manager = SM;
       Fragment.Source.Location = N->getSourceRange().Start;

clang-tools-extra/clangd/TUScheduler.h

@@ -194,7 +194,7 @@ class TUScheduler {
 
     /// Whether to run PreamblePeer asynchronously.
     /// No-op if AsyncThreadsCount is 0.
-    bool AsyncPreambleBuilds = false;
+    bool AsyncPreambleBuilds = true;
 
     /// Used to create a context that wraps each single operation.
     /// Typically to inject per-file configuration.

clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp

@@ -32,6 +32,9 @@ llvm::cl::opt<std::string> IndexLocation(
 llvm::cl::opt<std::string>
     ExecCommand("c", llvm::cl::desc("Command to execute and then exit"));
 
+llvm::cl::opt<std::string> ProjectRoot("project-root",
+                                       llvm::cl::desc("Path to the project"));
+
 static constexpr char Overview[] = R"(
 This is an **experimental** interactive tool to process user-provided search
 queries over given symbol collection obtained via clangd-indexer. The
@@ -326,7 +329,8 @@ struct {
 
 std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) {
   return Index.startswith("remote:")
-             ? remote::getClient(Index.drop_front(strlen("remote:")))
+             ? remote::getClient(Index.drop_front(strlen("remote:")),
+                                 ProjectRoot)
              : loadIndex(Index, /*UseDex=*/true);
 }
 

clang-tools-extra/clangd/index/remote/Client.cpp

@@ -10,10 +10,12 @@
 
 #include "Client.h"
 #include "Index.grpc.pb.h"
+#include "index/Index.h"
 #include "index/Serialization.h"
 #include "marshalling/Marshalling.h"
 #include "support/Logger.h"
 #include "support/Trace.h"
+#include "llvm/ADT/StringRef.h"
 
 #include <chrono>
 
@@ -27,14 +29,24 @@ class IndexClient : public clangd::SymbolIndex {
   using StreamingCall = std::unique_ptr<grpc::ClientReader<ReplyT>> (
       remote::SymbolIndex::Stub::*)(grpc::ClientContext *, const RequestT &);
 
+  template <typename ClangdRequestT, typename RequestT>
+  RequestT serializeRequest(ClangdRequestT Request) const {
+    return toProtobuf(Request);
+  }
+
+  template <>
+  FuzzyFindRequest serializeRequest(clangd::FuzzyFindRequest Request) const {
+    return toProtobuf(Request, ProjectRoot);
+  }
+
   template <typename RequestT, typename ReplyT, typename ClangdRequestT,
             typename CallbackT>
   bool streamRPC(ClangdRequestT Request,
                  StreamingCall<RequestT, ReplyT> RPCCall,
                  CallbackT Callback) const {
     bool FinalResult = false;
     trace::Span Tracer(RequestT::descriptor()->name());
-    const auto RPCRequest = toProtobuf(Request);
+    const auto RPCRequest = serializeRequest<ClangdRequestT, RequestT>(Request);
     grpc::ClientContext Context;
     std::chrono::system_clock::time_point Deadline =
         std::chrono::system_clock::now() + DeadlineWaitingTime;
@@ -48,20 +60,21 @@ class IndexClient : public clangd::SymbolIndex {
         FinalResult = Reply.final_result();
         continue;
       }
-      auto Sym = fromProtobuf(Reply.stream_result(), &Strings);
-      if (!Sym)
+      auto Response =
+          fromProtobuf(Reply.stream_result(), &Strings, ProjectRoot);
+      if (!Response)
         elog("Received invalid {0}", ReplyT::descriptor()->name());
-      Callback(*Sym);
+      Callback(*Response);
     }
     SPAN_ATTACH(Tracer, "status", Reader->Finish().ok());
     return FinalResult;
   }
 
 public:
   IndexClient(
-      std::shared_ptr<grpc::Channel> Channel,
+      std::shared_ptr<grpc::Channel> Channel, llvm::StringRef ProjectRoot,
       std::chrono::milliseconds DeadlineTime = std::chrono::milliseconds(1000))
-      : Stub(remote::SymbolIndex::NewStub(Channel)),
+      : Stub(remote::SymbolIndex::NewStub(Channel)), ProjectRoot(ProjectRoot),
         DeadlineWaitingTime(DeadlineTime) {}
 
   void lookup(const clangd::LookupRequest &Request,
@@ -86,22 +99,26 @@ class IndexClient : public clangd::SymbolIndex {
             llvm::function_ref<void(const SymbolID &, const clangd::Symbol &)>)
       const {}
 
-  // IndexClient does not take any space since the data is stored on the server.
+  // IndexClient does not take any space since the data is stored on the
+  // server.
   size_t estimateMemoryUsage() const { return 0; }
 
 private:
   std::unique_ptr<remote::SymbolIndex::Stub> Stub;
+  std::string ProjectRoot;
   // Each request will be terminated if it takes too long.
   std::chrono::milliseconds DeadlineWaitingTime;
 };
 
 } // namespace
 
-std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address) {
+std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address,
+                                               llvm::StringRef ProjectRoot) {
   const auto Channel =
       grpc::CreateChannel(Address.str(), grpc::InsecureChannelCredentials());
   Channel->GetState(true);
-  return std::unique_ptr<clangd::SymbolIndex>(new IndexClient(Channel));
+  return std::unique_ptr<clangd::SymbolIndex>(
+      new IndexClient(Channel, ProjectRoot));
 }
 
 } // namespace remote

clang-tools-extra/clangd/index/remote/Client.h

@@ -10,19 +10,24 @@
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_REMOTE_INDEX_H
 
 #include "index/Index.h"
+#include "llvm/ADT/StringRef.h"
 
 namespace clang {
 namespace clangd {
 namespace remote {
 
 /// Returns an SymbolIndex client that passes requests to remote index located
 /// at \p Address. The client allows synchronous RPC calls.
+/// \p IndexRoot is an absolute path on the local machine to the source tree
+/// described by the remote index. Paths returned by the index will be treated
+/// as relative to this directory.
 ///
 /// This method attempts to resolve the address and establish the connection.
 ///
 /// \returns nullptr if the address is not resolved during the function call or
 /// if the project was compiled without Remote Index support.
-std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address);
+std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address,
+                                               llvm::StringRef IndexRoot);
 
 } // namespace remote
 } // namespace clangd

clang-tools-extra/clangd/index/remote/Index.proto

@@ -71,7 +71,7 @@ message Symbol {
   string name = 3;
   SymbolLocation definition = 4;
   string scope = 5;
-  SymbolLocation canonical_declarattion = 6;
+  SymbolLocation canonical_declaration = 6;
   int32 references = 7;
   uint32 origin = 8;
   string signature = 9;
@@ -99,7 +99,10 @@ message SymbolInfo {
 message SymbolLocation {
   Position start = 1;
   Position end = 2;
-  string file_uri = 3;
+  // clangd::SymbolLocation stores FileURI, but the protocol transmits a the
+  // relative path. Because paths are different on the remote and local machines
+  // they will be translated in the marshalling layer.
+  string file_path = 3;
 }
 
 message Position {