Merge pull request #29869 from davidungar/unit-test-users-v4

[Incremental] Reorganization of SourceFileDepGraph building and mocking.

Author: David Ungar

include/swift/AST/FineGrainedDependencies.h

@@ -93,14 +93,20 @@ template <typename KeyT, typename ValueT> class Memoizer {
 public:
   Memoizer() = default;
 
+  Optional<ValueT> findExisting(KeyT key) {
+    auto iter = memos.find(key);
+    if (iter != memos.end())
+      return iter->second;
+    return None;
+  }
+
   /// \p createFn must create a \ref ValueT that corresponds to the \ref KeyT
   /// passed into it.
   ValueT
   findExistingOrCreateIfNew(KeyT key,
                             function_ref<ValueT(const KeyT &)> createFn) {
-    auto iter = memos.find(key);
-    if (iter != memos.end())
-      return iter->second;
+    if (auto existing = findExisting(key))
+      return existing.getValue();
     ValueT v = createFn(key);
     (void)insert(key, v);
     return v;
@@ -372,6 +378,7 @@ const std::string NodeKindNames[]{
     "topLevel",      "nominal",        "potentialMember",  "member",
     "dynamicLookup", "externalDepend", "sourceFileProvide"};
 
+
 /// Instead of the status quo scheme of two kinds of "Depends", cascading and
 /// non-cascading this code represents each entity ("Provides" in the status
 /// quo), by a pair of nodes. One node represents the "implementation." If the
@@ -391,6 +398,7 @@ template <typename FnT> void forEachAspect(FnT fn) {
     fn(DeclAspect(i));
 }
 
+
 /// A pair of nodes that represent the two aspects of a given entity.
 /// Templated in order to serve for either SourceFileDepGraphNodes or
 /// ModuleDepGraphNodes.
@@ -499,10 +507,23 @@ class DependencyKey {
   }
   bool isInterface() const { return getAspect() == DeclAspect::interface; }
 
+  /// Create just the interface half of the keys for a provided Decl or Decl
+  /// pair
+  template <NodeKind kind, typename Entity>
+  static DependencyKey createForProvidedEntityInterface(Entity);
+
   /// Given some type of provided entity compute the context field of the key.
   template <NodeKind kind, typename Entity>
   static std::string computeContextForProvidedEntity(Entity);
 
+  DependencyKey correspondingImplementation() const {
+    return withAspect(DeclAspect::implementation);
+  }
+
+  DependencyKey withAspect(DeclAspect aspect) const {
+    return DependencyKey(kind, aspect, context, name);
+  }
+
   /// Given some type of provided entity compute the name field of the key.
   template <NodeKind kind, typename Entity>
   static std::string computeNameForProvidedEntity(Entity);
@@ -514,7 +535,8 @@ class DependencyKey {
   template <NodeKind kind>
   static DependencyKey createDependedUponKey(StringRef);
 
-  static DependencyKey createKeyForWholeSourceFile(StringRef swiftDeps);
+  static DependencyKey createKeyForWholeSourceFile(DeclAspect,
+                                                   StringRef swiftDeps);
 
   std::string humanReadableName() const;
 
@@ -555,6 +577,14 @@ struct std::hash<typename swift::fine_grained_dependencies::DeclAspect> {
     return size_t(aspect);
   }
 };
+template <>
+struct std::hash<typename swift::fine_grained_dependencies::NodeKind> {
+  size_t
+  operator()(const swift::fine_grained_dependencies::NodeKind kind) const {
+    return size_t(kind);
+  }
+};
+
 
 namespace swift {
 namespace fine_grained_dependencies {
@@ -616,6 +646,10 @@ class DepGraphNode {
   /// See SourceFileDepGraphNode::SourceFileDepGraphNode(...) and
   /// ModuleDepGraphNode::ModuleDepGraphNode(...) Don't set swiftDeps on
   /// creation because this field can change if a node is moved.
+  DepGraphNode(DependencyKey key, Optional<StringRef> fingerprint)
+      : DepGraphNode(key, fingerprint ? fingerprint->str()
+                                      : Optional<std::string>()) {}
+
   DepGraphNode(DependencyKey key, Optional<std::string> fingerprint)
       : key(key), fingerprint(fingerprint) {}
   DepGraphNode(const DepGraphNode &other) = default;
@@ -627,8 +661,12 @@ class DepGraphNode {
 
   const DependencyKey &getKey() const { return key; }
 
-  const Optional<std::string> &getFingerprint() const { return fingerprint; }
-
+  const Optional<StringRef> getFingerprint() const {
+    if (fingerprint) {
+      return StringRef(fingerprint.getValue());
+    }
+    return None;
+  }
   /// When driver reads a SourceFileDepGraphNode, it may be a node that was
   /// created to represent a name-lookup (a.k.a a "depend") in the frontend. In
   /// that case, the node represents an entity that resides in some other file
@@ -637,7 +675,9 @@ class DepGraphNode {
   /// (someday) have a fingerprint. In order to preserve the
   /// ModuleDepGraphNode's identity but bring its fingerprint up to date, it
   /// needs to set the fingerprint *after* the node has been created.
-  void setFingerprint(Optional<std::string> fp) { fingerprint = fp; }
+  void setFingerprint(Optional<StringRef> fp) {
+    fingerprint = fp ? fp->str() : Optional<std::string>();
+  }
 
   SWIFT_DEBUG_DUMP;
   void dump(llvm::raw_ostream &os) const;
@@ -684,7 +724,7 @@ class SourceFileDepGraphNode : public DepGraphNode {
   SourceFileDepGraphNode() : DepGraphNode(), sequenceNumber(~0) {}
 
   /// Used by the frontend to build nodes.
-  SourceFileDepGraphNode(DependencyKey key, Optional<std::string> fingerprint,
+  SourceFileDepGraphNode(DependencyKey key, Optional<StringRef> fingerprint,
                          bool isProvides)
       : DepGraphNode(key, fingerprint), isProvides(isProvides) {
     assert(key.verify());
@@ -780,34 +820,6 @@ class SourceFileDepGraph {
   SourceFileDepGraph(const SourceFileDepGraph &g) = delete;
   SourceFileDepGraph(SourceFileDepGraph &&g) = default;
 
-  /// Simulate loading for unit testing:
-  /// \param swiftDepsFileName The name of the swiftdeps file of the phony job
-  /// \param includePrivateDeps Whether the graph includes intra-file arcs
-  /// \param hadCompilationError Simulate a compilation error
-  /// \param interfaceHash The interface hash of the simulated graph
-  /// \param simpleNamesByRDK A map of vectors of names keyed by reference
-  /// dependency key \param compoundNamesByRDK A map of (mangledHolder,
-  /// baseName) pairs keyed by reference dependency key. For single-name
-  /// dependencies, an initial underscore indicates that the name does not
-  /// cascade. For compound names, it is the first name, the holder which
-  /// indicates non-cascading. For member names, an initial underscore indicates
-  /// file-privacy.
-  static SourceFileDepGraph
-  simulateLoad(std::string swiftDepsFileName, const bool includePrivateDeps,
-               const bool hadCompilationError, std::string interfaceHash,
-               llvm::StringMap<std::vector<std::string>> simpleNamesByRDK,
-               llvm::StringMap<std::vector<std::pair<std::string, std::string>>>
-                   compoundNamesByRDK);
-
-  static constexpr char noncascadingOrPrivatePrefix = '#';
-  static constexpr char nameFingerprintSeparator = ',';
-
-  static std::string noncascading(std::string name);
-
-  LLVM_ATTRIBUTE_UNUSED
-  static std::string privatize(std::string name);
-
-
   /// Nodes are owned by the graph.
   ~SourceFileDepGraph() {
     forEachNode([&](SourceFileDepGraphNode *n) { delete n; });
@@ -851,12 +863,15 @@ class SourceFileDepGraph {
   /// The frontend creates a pair of nodes for every tracked Decl and the source
   /// file itself.
   InterfaceAndImplementationPair<SourceFileDepGraphNode>
-  findExistingNodePairOrCreateAndAddIfNew(
-      NodeKind k, const ContextNameFingerprint &contextNameFingerprint);
+  findExistingNodePairOrCreateAndAddIfNew(const DependencyKey &interfaceKey,
+                                          Optional<StringRef> fingerprint);
+
+  NullablePtr<SourceFileDepGraphNode>
+  findExistingNode(const DependencyKey &key);
 
   SourceFileDepGraphNode *
-  findExistingNodeOrCreateIfNew(DependencyKey key,
-                                const Optional<std::string> &fingerprint,
+  findExistingNodeOrCreateIfNew(const DependencyKey &key,
+                                const Optional<StringRef> fingerprint,
                                 bool isProvides);
 
   /// \p Use is the Node that must be rebuilt when \p def changes.

include/swift/AST/SourceFileDepGraphConstructor.h

@@ -0,0 +1,172 @@
+//===----- SourceFileDepGraphConstructor.h ----------------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_AST_SOURCE_FILE_DEP_GRAPH_CONSTRUCTOR_H
+#define SWIFT_AST_SOURCE_FILE_DEP_GRAPH_CONSTRUCTOR_H
+
+#include <swift/AST/DeclContext.h>
+
+namespace swift {
+namespace fine_grained_dependencies {
+/// Abstract class for building a \c SourceFileDepGraph from either a real
+/// \c SourceFile or a unit test
+class SourceFileDepGraphConstructor {
+protected:
+  /// To match the existing system, set this to false.
+  /// To include even private entities and get intra-file info, set to true.
+  const bool includePrivateDeps;
+
+  /// If there was an error, cannot get accurate info.
+  const bool hadCompilationError;
+
+  /// The name of the swiftDeps file.
+  const std::string swiftDeps;
+
+  /// The fingerprint of the whole file
+  const std::string fileFingerprint;
+
+  /// For debugging
+  const bool emitDotFileAfterConstruction;
+
+  DiagnosticEngine &diags;
+
+  /// Graph under construction
+  SourceFileDepGraph g;
+
+public:
+  /// Expose this layer to enable faking up a constructor for testing.
+  /// See the instance variable comments for explanation.
+  SourceFileDepGraphConstructor(bool includePrivateDeps,
+                                bool hadCompilationError, StringRef swiftDeps,
+                                StringRef fileFingerprint,
+                                bool emitDotFileAfterConstruction,
+                                DiagnosticEngine &diags);
+
+  virtual ~SourceFileDepGraphConstructor() = default;
+
+  /// Create a SourceFileDepGraph.
+  SourceFileDepGraph construct();
+
+private:
+  void addSourceFileNodesToGraph();
+
+  /// Add the "provides" nodes when mocking up a graph
+  virtual void addAllDefinedDecls() = 0;
+
+  /// Add the "depends" nodes and arcs when mocking a graph
+  virtual void addAllUsedDecls() = 0;
+
+protected:
+  /// Add an pair of interface, implementation nodes to the graph, which
+  /// represent some \c Decl defined in this source file. \param key the
+  /// interface key of the pair
+  void addADefinedDecl(const DependencyKey &key,
+                       Optional<StringRef> fingerprint);
+
+  void addAUsedDecl(const DependencyKey &def, const DependencyKey &use);
+};
+
+/// Constructs a SourceFileDepGraph from a *real* \c SourceFile
+/// Reads the information provided by the frontend and builds the
+/// SourceFileDepGraph
+
+class RealSourceFileDepGraphConstructor : public SourceFileDepGraphConstructor {
+  SourceFile *const SF;
+  const DependencyTracker &depTracker;
+
+public:
+  RealSourceFileDepGraphConstructor(SourceFile *SF, StringRef outputPath,
+                                    const DependencyTracker &depTracker,
+                                    bool alsoEmitDotFile);
+
+  ~RealSourceFileDepGraphConstructor() override = default;
+
+private:
+  static std::string getFingerprint(SourceFile *SF);
+
+  static bool computeIncludePrivateDeps(SourceFile *SF);
+  static std::string getInterfaceHash(SourceFile *SF);
+
+  void addAllDefinedDecls() override;
+  void addAllUsedDecls() override;
+
+  /// Given an array of Decls or pairs of them in \p declsOrPairs
+  /// create node pairs for context and name
+  template <NodeKind kind, typename ContentsT>
+  void addAllDefinedDeclsOfAGivenType(std::vector<ContentsT> &contentsVec);
+
+  /// At present, only nominals, protocols, and extensions have (body)
+  /// fingerprints
+  static Optional<std::string>
+  getFingerprintIfAny(std::pair<const NominalTypeDecl *, const ValueDecl *>);
+  static Optional<std::string> getFingerprintIfAny(const Decl *d);
+};
+
+using DependencyDescriptions =
+    std::unordered_multimap<NodeKind, std::vector<std::string>>;
+
+class MockSourceFileDepGraphConstructor : public SourceFileDepGraphConstructor {
+  const DependencyDescriptions dependencyDescriptions;
+
+public:
+  MockSourceFileDepGraphConstructor(
+      bool includePrivateDeps, bool hadCompilationError, StringRef swiftDeps,
+      StringRef fileFingerprint, bool emitDotFileAfterConstruction,
+      const DependencyDescriptions &dependencyDescriptions,
+      DiagnosticEngine &diags)
+      : SourceFileDepGraphConstructor(includePrivateDeps, hadCompilationError,
+                                      swiftDeps, fileFingerprint,
+                                      emitDotFileAfterConstruction, diags),
+        dependencyDescriptions(dependencyDescriptions) {}
+
+  ~MockSourceFileDepGraphConstructor() override = default;
+
+private:
+  void addAllDefinedDecls() override;
+  void addAllUsedDecls() override;
+
+  /// For brevity, unit tests specify dependencies by NodeKind,
+  /// but for processing, the kind is needed for each entry.
+  void forEachEntry(function_ref<void(NodeKind kind, StringRef entry)> fn);
+
+  static const char *defUseSeparator;
+  static bool isADefinedDecl(StringRef s);
+
+  void addADefinedDecl(StringRef s, NodeKind kind);
+  void addAUsedDecl(StringRef s, NodeKind kind);
+
+  Optional<std::pair<DependencyKey, DependencyKey>> parseAUsedDecl(StringRef s,
+                                                                   NodeKind);
+
+  /// Parse and return an interface \c DependencyKey
+  Optional<DependencyKey> parseADefinedDecl(StringRef s, NodeKind, DeclAspect);
+
+  DependencyKey computeUseKey(StringRef s, bool isCascadingUse);
+
+  /// Return true if when the name appears in a unit test, it represents a
+  /// context, not a baseName. Return false if a single name is a baseName,
+  /// without context Return None if there shoud be two names
+  static Optional<bool> singleNameIsContext(NodeKind kind);
+
+  static constexpr char nameContextSeparator = ',';
+
+  static constexpr char fingerprintSeparator = '@';
+
+  static std::string parseContext(const StringRef s, const NodeKind kind);
+
+  static std::string parseName(const StringRef s, const NodeKind kind);
+};
+
+} // namespace fine_grained_dependencies
+} // namespace swift
+
+#endif // SWIFT_AST_SOURCE_FILE_DEP_GRAPH_CONSTRUCTOR_H