Merge pull request #32355 from nkcsgexi/json-for-explicit-swift-modules

ExplicitModuleLoader: take a JSON file that specifies details of explicit Swift modules

Author: nkcsgexi

include/swift/AST/DiagnosticsFrontend.def

@@ -251,6 +251,14 @@ ERROR(previous_installname_map_corrupted,none,
       "previous install name map from %0 is malformed",
       (StringRef))
 
+ERROR(explicit_swift_module_map_missing,none,
+      "cannot open explicit Swift module map from %0",
+      (StringRef))
+
+ERROR(explicit_swift_module_map_corrupted,none,
+      "explicit Swift module map from %0 is malformed",
+      (StringRef))
+
 REMARK(default_previous_install_name, none,
       "default previous install name for %0 is %1", (StringRef, StringRef))
 

include/swift/AST/SearchPathOptions.h

@@ -85,6 +85,8 @@ class SearchPathOptions {
   /// The paths to a set of explicitly built modules from interfaces.
   std::vector<std::string> ExplicitSwiftModules;
 
+  /// A map of explict Swift module information.
+  std::string ExplicitSwiftModuleMap;
 private:
   static StringRef
   pathStringFromFrameworkSearchPath(const FrameworkSearchPath &next) {

include/swift/Frontend/ModuleInterfaceLoader.h

@@ -150,6 +150,7 @@ class ExplicitSwiftModuleLoader: public SerializedModuleLoaderBase {
   create(ASTContext &ctx,
          DependencyTracker *tracker, ModuleLoadingMode loadMode,
          ArrayRef<std::string> ExplicitModulePaths,
+         StringRef ExplicitSwiftModuleMap,
          bool IgnoreSwiftSourceInfoFile);
 
   /// Append visible module names to \p names. Note that names are possibly

include/swift/Option/FrontendOptions.td

@@ -220,6 +220,10 @@ def disable_implicit_swift_modules: Flag<["-"], "disable-implicit-swift-modules"
 def swift_module_file
   : Separate<["-"], "swift-module-file">, MetaVarName<"<path>">,
     HelpText<"Specify Swift module explicitly built from textual interface">;
+
+def explict_swift_module_map
+  : Separate<["-"], "explicit-swift-module-map-file">, MetaVarName<"<path>">,
+    HelpText<"Specify a JSON file containing information of explict Swift modules">;
 }
 
 

lib/Frontend/CompilerInvocation.cpp

@@ -872,6 +872,8 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts,
   for (auto A: Args.filtered(OPT_swift_module_file)) {
     Opts.ExplicitSwiftModules.push_back(resolveSearchPath(A->getValue()));
   }
+  if (const Arg *A = Args.getLastArg(OPT_explict_swift_module_map))
+    Opts.ExplicitSwiftModuleMap = A->getValue();
   // Opts.RuntimeIncludePath is set by calls to
   // setRuntimeIncludePath() or setMainExecutablePath().
   // Opts.RuntimeImportPath is set by calls to

lib/Frontend/Frontend.cpp

@@ -457,6 +457,7 @@ bool CompilerInstance::setUpModuleLoaders() {
         *Context,
         getDependencyTracker(), MLM,
         Invocation.getSearchPathOptions().ExplicitSwiftModules,
+        Invocation.getSearchPathOptions().ExplicitSwiftModuleMap,
         IgnoreSourceInfoFile);
     Context->addModuleLoader(std::move(ESML));
   }

lib/Frontend/ModuleInterfaceLoader.cpp

@@ -36,6 +36,7 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/YAMLParser.h"
 #include "ModuleInterfaceBuilder.h"
 
 using namespace swift;
@@ -1218,6 +1219,15 @@ InterfaceSubContextDelegateImpl::InterfaceSubContextDelegateImpl(
     GenericArgs.push_back("-swift-module-file");
     GenericArgs.push_back(ArgSaver.save(EM));
   }
+  // Pass down -explicit-swift-module-map-file
+  // FIXME: we shouldn't need this. Remove it?
+  StringRef explictSwiftModuleMap = searchPathOpts.ExplicitSwiftModuleMap;
+  subInvocation.getSearchPathOptions().ExplicitSwiftModuleMap =
+    explictSwiftModuleMap;
+  if (!explictSwiftModuleMap.empty()) {
+    GenericArgs.push_back("-explicit-swift-module-map-file");
+    GenericArgs.push_back(explictSwiftModuleMap);
+  }
   if (clangImporter) {
     // We need to add these extra clang flags because explict module building
     // related flags are all there: -fno-implicit-modules, -fmodule-map-file=,
@@ -1418,14 +1428,99 @@ bool InterfaceSubContextDelegateImpl::runInSubCompilerInstance(StringRef moduleN
 }
 
 struct ExplicitSwiftModuleLoader::Implementation {
+
   // Information about explicitly specified Swift module files.
   struct ExplicitModuleInfo {
-    // Path of the module file.
-    StringRef path;
-    // Buffer of the module content.
+    // Path of the .swiftmodule file.
+    StringRef modulePath;
+    // Path of the .swiftmoduledoc file.
+    StringRef moduleDocPath;
+    // Path of the .swiftsourceinfo file.
+    StringRef moduleSourceInfoPath;
+    // Opened buffer for the .swiftmodule file.
     std::unique_ptr<llvm::MemoryBuffer> moduleBuffer;
   };
+  ASTContext &Ctx;
+  llvm::BumpPtrAllocator Allocator;
+  llvm::StringSaver Saver;
   llvm::StringMap<ExplicitModuleInfo> ExplicitModuleMap;
+  Implementation(ASTContext &Ctx): Ctx(Ctx), Saver(Allocator) {}
+
+  StringRef getScalaNodeText(llvm::yaml::Node *N) {
+    SmallString<32> Buffer;
+    return Saver.save(cast<llvm::yaml::ScalarNode>(N)->getValue(Buffer));
+  }
+
+  bool parseSingleModuleEntry(llvm::yaml::KeyValueNode &node) {
+    using namespace llvm::yaml;
+    auto moduleName = getScalaNodeText(node.getKey());
+    auto insertRes = ExplicitModuleMap.insert({moduleName,
+      ExplicitModuleInfo()});
+    if (!insertRes.second) {
+      return true;
+    }
+    auto moduleDetails = dyn_cast<MappingNode>(node.getValue());
+    if (!moduleDetails)
+      return true;
+    for (auto &entry: *moduleDetails) {
+      auto key = getScalaNodeText(entry.getKey());
+      auto val = getScalaNodeText(entry.getValue());
+      if (key == "SwiftModulePath") {
+        insertRes.first->second.modulePath = val;
+      } else if (key == "SwiftDocPath") {
+        insertRes.first->second.moduleDocPath = val;
+      } else if (key == "SwiftSourceInfoPath") {
+        insertRes.first->second.moduleSourceInfoPath = val;
+      } else {
+        return true;
+      }
+    }
+    return false;
+  }
+  //  {
+  //    "A": {
+  //      "SwiftModulePath": "A.swiftmodule",
+  //      "SwiftDocPath": "A.swiftdoc",
+  //      "SwiftSourceInfoPath": "A.swiftsourceinfo"
+  //    },
+  //    "B": {
+  //      "SwiftModulePath": "B.swiftmodule",
+  //      "SwiftDocPath": "B.swiftdoc",
+  //      "SwiftSourceInfoPath": "B.swiftsourceinfo"
+  //    }
+  //  }
+  void parseSwiftExplicitModuleMap(StringRef fileName) {
+    using namespace llvm::yaml;
+    // Load the input file.
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileBufOrErr =
+      llvm::MemoryBuffer::getFile(fileName);
+    if (!fileBufOrErr) {
+      Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_missing,
+                         fileName);
+      return;
+    }
+    StringRef Buffer = fileBufOrErr->get()->getBuffer();
+    Stream Stream(llvm::MemoryBufferRef(Buffer, fileName),
+                        Ctx.SourceMgr.getLLVMSourceMgr());
+    for (auto DI = Stream.begin(); DI != Stream.end(); ++ DI) {
+      assert(DI != Stream.end() && "Failed to read a document");
+      if (auto *MN = dyn_cast_or_null<MappingNode>(DI->getRoot())) {
+        for (auto &entry: *MN) {
+          if (parseSingleModuleEntry(entry)) {
+            Ctx.Diags.diagnose(SourceLoc(),
+                               diag::explicit_swift_module_map_corrupted,
+                               fileName);
+            return;
+          }
+        }
+      } else {
+        Ctx.Diags.diagnose(SourceLoc(),
+                           diag::explicit_swift_module_map_corrupted,
+                           fileName);
+        return;
+      }
+    }
+  }
 };
 
 ExplicitSwiftModuleLoader::ExplicitSwiftModuleLoader(
@@ -1435,7 +1530,7 @@ ExplicitSwiftModuleLoader::ExplicitSwiftModuleLoader(
       bool IgnoreSwiftSourceInfoFile):
         SerializedModuleLoaderBase(ctx, tracker, loadMode,
                                    IgnoreSwiftSourceInfoFile),
-        Impl(*new Implementation()) {}
+        Impl(*new Implementation(ctx)) {}
 
 ExplicitSwiftModuleLoader::~ExplicitSwiftModuleLoader() { delete &Impl; }
 
@@ -1453,9 +1548,33 @@ std::error_code ExplicitSwiftModuleLoader::findModuleFilesInDirectory(
   if (it == Impl.ExplicitModuleMap.end()) {
     return std::make_error_code(std::errc::not_supported);
   }
-  // We found an explicit module matches the given name, give the buffer
-  // back to the caller side.
-  *ModuleBuffer = std::move(it->getValue().moduleBuffer);
+  auto &moduleInfo = it->getValue();
+  if (moduleInfo.moduleBuffer) {
+    // We found an explicit module matches the given name, give the buffer
+    // back to the caller side.
+    *ModuleBuffer = std::move(moduleInfo.moduleBuffer);
+    return std::error_code();
+  }
+
+  auto &fs = *Ctx.SourceMgr.getFileSystem();
+  // Open .swiftmodule file
+  auto moduleBuf = fs.getBufferForFile(moduleInfo.modulePath);
+  if (!moduleBuf)
+    return moduleBuf.getError();
+  *ModuleBuffer = std::move(moduleBuf.get());
+
+  // Open .swiftdoc file
+  if (!moduleInfo.moduleDocPath.empty()) {
+    auto moduleDocBuf = fs.getBufferForFile(moduleInfo.moduleDocPath);
+    if (moduleBuf)
+      *ModuleDocBuffer = std::move(moduleDocBuf.get());
+  }
+  // Open .swiftsourceinfo file
+  if (!moduleInfo.moduleSourceInfoPath.empty()) {
+    auto moduleSourceInfoBuf = fs.getBufferForFile(moduleInfo.moduleSourceInfoPath);
+    if (moduleSourceInfoBuf)
+      *ModuleSourceInfoBuffer = std::move(moduleSourceInfoBuf.get());
+  }
   return std::error_code();
 }
 
@@ -1470,19 +1589,24 @@ std::unique_ptr<ExplicitSwiftModuleLoader>
 ExplicitSwiftModuleLoader::create(ASTContext &ctx,
     DependencyTracker *tracker, ModuleLoadingMode loadMode,
     ArrayRef<std::string> ExplicitModulePaths,
+    StringRef ExplicitSwiftModuleMap,
     bool IgnoreSwiftSourceInfoFile) {
   auto result = std::unique_ptr<ExplicitSwiftModuleLoader>(
     new ExplicitSwiftModuleLoader(ctx, tracker, loadMode,
                                   IgnoreSwiftSourceInfoFile));
   auto &Impl = result->Impl;
+  // Parse a JSON file to collect explicitly built modules.
+  Impl.parseSwiftExplicitModuleMap(ExplicitSwiftModuleMap);
+  // Collect .swiftmodule paths from -swift-module-path
+  // FIXME: remove these.
   for (auto path: ExplicitModulePaths) {
     std::string name;
     // Load the explicit module into a buffer and get its name.
     std::unique_ptr<llvm::MemoryBuffer> buffer = getModuleName(ctx, path, name);
     if (buffer) {
       // Register this module for future loading.
       auto &entry = Impl.ExplicitModuleMap[name];
-      entry.path = path;
+      entry.modulePath = path;
       entry.moduleBuffer = std::move(buffer);
     } else {
       // We cannot read the module content, diagnose.

test/ScanDependencies/explicit-module-map.swift

@@ -0,0 +1,18 @@
+// RUN: %empty-directory(%t)
+// RUN: mkdir -p %t/clang-module-cache
+// RUN: mkdir -p %t/inputs
+// RUN: echo "/// Some cool comments" > %t/foo.swift
+// RUN: echo "public func foo() {}" >> %t/foo.swift
+// RUN: %target-swift-frontend -emit-module -emit-module-path %t/inputs/Foo.swiftmodule -emit-module-doc-path %t/inputs/Foo.swiftdoc -emit-module-source-info -emit-module-source-info-path %t/inputs/Foo.swiftsourceinfo -module-cache-path %t.module-cache %t/foo.swift -module-name Foo
+
+// RUN: echo "{" > %t/inputs/map.json
+// RUN: echo "\"Foo\": {" >> %t/inputs/map.json
+// RUN: echo "\"SwiftModulePath\": \"%t/inputs/Foo.swiftmodule\"," >> %t/inputs/map.json
+// RUN: echo "\"SwiftDocPath\": \"%t/inputs/Foo.swiftdoc\"," >> %t/inputs/map.json
+// RUN: echo "\"SwiftSourceInfoPath\": \"%t/inputs/Foo.swiftsourceinfo\"" >> %t/inputs/map.json
+// RUN: echo "}" >> %t/inputs/map.json
+// RUN: echo "}" >> %t/inputs/map.json
+
+// RUN: %target-swift-ide-test -print-module-comments -module-to-print=Foo -enable-swiftsourceinfo -source-filename %s -explicit-swift-module-map-file %t/inputs/map.json | %FileCheck %s
+
+// CHECK: foo.swift:2:13: Func/foo RawComment=[/// Some cool comments

tools/swift-ide-test/swift-ide-test.cpp

@@ -723,6 +723,10 @@ EnableSwiftSourceInfo("enable-swiftsourceinfo",
                  llvm::cl::cat(Category),
                  llvm::cl::init(false));
 
+static llvm::cl::opt<std::string>
+ExplicitSwiftModuleMap("explicit-swift-module-map-file",
+                       llvm::cl::desc("JSON file to include explicit Swift modules"),
+                       llvm::cl::cat(Category));
 } // namespace options
 
 static std::unique_ptr<llvm::MemoryBuffer>
@@ -3462,6 +3466,11 @@ int main(int argc, char *argv[]) {
   for (auto ConfigName : options::BuildConfigs)
     InitInvok.getLangOptions().addCustomConditionalCompilationFlag(ConfigName);
 
+  if (!options::ExplicitSwiftModuleMap.empty()) {
+    InitInvok.getSearchPathOptions().ExplicitSwiftModuleMap =
+      options::ExplicitSwiftModuleMap;
+    InitInvok.getFrontendOptions().DisableImplicitModules = true;
+  }
   // Process the clang arguments last and allow them to override previously
   // set options.
   if (!CCArgs.empty()) {