diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h index 6cce6c7cf386a..016b16706ab3b 100644 --- a/include/swift/ClangImporter/ClangImporter.h +++ b/include/swift/ClangImporter/ClangImporter.h @@ -43,6 +43,12 @@ namespace clang { class Type; class VisibleDeclConsumer; class DeclarationName; + class CompilerInvocation; +namespace tooling { +namespace dependencies { + struct FullDependenciesResult; +} +} } namespace swift { @@ -148,6 +154,14 @@ class ClangImporter final : public ClangModuleLoader { std::string swiftPCHHash = "", DependencyTracker *tracker = nullptr, DWARFImporterDelegate *dwarfImporterDelegate = nullptr); + static std::vector + getClangArguments(ASTContext &ctx, const ClangImporterOptions &importerOpts); + + static std::unique_ptr + createClangInvocation(ClangImporter *importer, + const ClangImporterOptions &importerOpts, + ArrayRef invocationArgStrs, + std::vector *CC1Args = nullptr); ClangImporter(const ClangImporter &) = delete; ClangImporter(ClangImporter &&) = delete; ClangImporter &operator=(const ClangImporter &) = delete; @@ -369,6 +383,10 @@ class ClangImporter final : public ClangModuleLoader { void verifyAllModules() override; + void recordModuleDependencies( + ModuleDependenciesCache &cache, + const clang::tooling::dependencies::FullDependenciesResult &clangDependencies); + Optional getModuleDependencies( StringRef moduleName, ModuleDependenciesCache &cache, InterfaceSubContextDelegate &delegate) override; @@ -460,6 +478,12 @@ class ClangImporter final : public ClangModuleLoader { ImportDecl *createImportDecl(ASTContext &Ctx, DeclContext *DC, ClangNode ClangN, ArrayRef Exported); +/// Extract the specified-or-defaulted -module-cache-path that winds up in +/// the clang importer, for reuse as the .swiftmodule cache path when +/// building a ModuleInterfaceLoader. +std::string +getModuleCachePathFromClang(const clang::CompilerInstance &Instance); + } // end namespace swift #endif diff --git a/include/swift/Frontend/ModuleInterfaceLoader.h b/include/swift/Frontend/ModuleInterfaceLoader.h index 2a793922a1605..61d265d3a775b 100644 --- a/include/swift/Frontend/ModuleInterfaceLoader.h +++ b/include/swift/Frontend/ModuleInterfaceLoader.h @@ -198,12 +198,6 @@ class ModuleInterfaceLoader : public SerializedModuleLoaderBase { bool RemarkOnRebuildFromInterface, bool DisableInterfaceFileLock); }; -/// Extract the specified-or-defaulted -module-cache-path that winds up in -/// the clang importer, for reuse as the .swiftmodule cache path when -/// building a ModuleInterfaceLoader. -std::string -getModuleCachePathFromClang(const clang::CompilerInstance &Instance); - struct InterfaceSubContextDelegateImpl: InterfaceSubContextDelegate { private: SourceManager &SM; diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index 5d5fdb30326e4..2d2d229c7bcd0 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -905,15 +905,10 @@ ClangImporter::getOrCreatePCH(const ClangImporterOptions &ImporterOptions, return PCHFilename.getValue(); } -std::unique_ptr -ClangImporter::create(ASTContext &ctx, const ClangImporterOptions &importerOpts, - std::string swiftPCHHash, DependencyTracker *tracker, - DWARFImporterDelegate *dwarfImporterDelegate) { - std::unique_ptr importer{ - new ClangImporter(ctx, importerOpts, tracker, dwarfImporterDelegate)}; - +std::vector +ClangImporter::getClangArguments(ASTContext &ctx, + const ClangImporterOptions &importerOpts) { std::vector invocationArgStrs; - // Clang expects this to be like an actual command line. So we need to pass in // "clang" for argv[0] invocationArgStrs.push_back("clang"); @@ -927,6 +922,49 @@ ClangImporter::create(ASTContext &ctx, const ClangImporterOptions &importerOpts, break; } addCommonInvocationArguments(invocationArgStrs, ctx, importerOpts); + return invocationArgStrs; +} + +std::unique_ptr +ClangImporter::createClangInvocation(ClangImporter *importer, + const ClangImporterOptions &importerOpts, + ArrayRef invocationArgStrs, + std::vector *CC1Args) { + std::vector invocationArgs; + invocationArgs.reserve(invocationArgStrs.size()); + for (auto &argStr : invocationArgStrs) + invocationArgs.push_back(argStr.c_str()); + // Set up a temporary diagnostic client to report errors from parsing the + // command line, which may be important for Swift clients if, for example, + // they're using -Xcc options. Unfortunately this diagnostic engine has to + // use the default options because the /actual/ options haven't been parsed + // yet. + // + // The long-term client for Clang diagnostics is set up below, after the + // clang::CompilerInstance is created. + llvm::IntrusiveRefCntPtr tempDiagOpts{ + new clang::DiagnosticOptions + }; + + ClangDiagnosticConsumer tempDiagClient{importer->Impl, *tempDiagOpts, + importerOpts.DumpClangDiagnostics}; + llvm::IntrusiveRefCntPtr tempClangDiags = + clang::CompilerInstance::createDiagnostics(tempDiagOpts.get(), + &tempDiagClient, + /*owned*/false); + + return clang::createInvocationFromCommandLine(invocationArgs, tempClangDiags, + nullptr, false, CC1Args); +} + +std::unique_ptr +ClangImporter::create(ASTContext &ctx, const ClangImporterOptions &importerOpts, + std::string swiftPCHHash, DependencyTracker *tracker, + DWARFImporterDelegate *dwarfImporterDelegate) { + std::unique_ptr importer{ + new ClangImporter(ctx, importerOpts, tracker, dwarfImporterDelegate)}; + importer->Impl.ClangArgs = getClangArguments(ctx, importerOpts); + ArrayRef invocationArgStrs = importer->Impl.ClangArgs; if (importerOpts.DumpClangDiagnostics) { llvm::errs() << "'"; @@ -936,10 +974,7 @@ ClangImporter::create(ASTContext &ctx, const ClangImporterOptions &importerOpts, llvm::errs() << "'\n"; } - std::vector invocationArgs; - invocationArgs.reserve(invocationArgStrs.size()); - for (auto &argStr : invocationArgStrs) - invocationArgs.push_back(argStr.c_str()); + if (llvm::sys::path::extension(importerOpts.BridgingHeader) .endswith(file_types::getExtension(file_types::TY_PCH))) { @@ -957,27 +992,9 @@ ClangImporter::create(ASTContext &ctx, const ClangImporterOptions &importerOpts, // Create a new Clang compiler invocation. { - // Set up a temporary diagnostic client to report errors from parsing the - // command line, which may be important for Swift clients if, for example, - // they're using -Xcc options. Unfortunately this diagnostic engine has to - // use the default options because the /actual/ options haven't been parsed - // yet. - // - // The long-term client for Clang diagnostics is set up below, after the - // clang::CompilerInstance is created. - llvm::IntrusiveRefCntPtr tempDiagOpts{ - new clang::DiagnosticOptions - }; - - ClangDiagnosticConsumer tempDiagClient{importer->Impl, *tempDiagOpts, - importerOpts.DumpClangDiagnostics}; - llvm::IntrusiveRefCntPtr tempClangDiags = - clang::CompilerInstance::createDiagnostics(tempDiagOpts.get(), - &tempDiagClient, - /*owned*/false); - - importer->Impl.Invocation = - clang::createInvocationFromCommandLine(invocationArgs, tempClangDiags); + importer->Impl.Invocation = createClangInvocation(importer.get(), + importerOpts, + invocationArgStrs); if (!importer->Impl.Invocation) return nullptr; } @@ -4002,3 +4019,20 @@ bool ClangImporter::isInOverlayModuleForImportedModule( return !clangModule->ExportAsModule.empty() && clangModule->ExportAsModule == overlayModule->getName().str(); } + +/// Extract the specified-or-defaulted -module-cache-path that winds up in +/// the clang importer, for reuse as the .swiftmodule cache path when +/// building a ModuleInterfaceLoader. +std::string +swift::getModuleCachePathFromClang(const clang::CompilerInstance &Clang) { + if (!Clang.hasPreprocessor()) + return ""; + std::string SpecificModuleCachePath = + Clang.getPreprocessor().getHeaderSearchInfo().getModuleCachePath().str(); + + // The returned-from-clang module cache path includes a suffix directory + // that is specific to the clang version and invocation; we want the + // directory above that. + return llvm::sys::path::parent_path(SpecificModuleCachePath).str(); +} + diff --git a/lib/ClangImporter/ClangModuleDependencyScanner.cpp b/lib/ClangImporter/ClangModuleDependencyScanner.cpp index d333d0e966ffb..d63601776f078 100644 --- a/lib/ClangImporter/ClangModuleDependencyScanner.cpp +++ b/lib/ClangImporter/ClangModuleDependencyScanner.cpp @@ -188,12 +188,48 @@ static ClangModuleDependenciesCacheImpl *getOrCreateClangImpl( return clangImpl; } +static std::string getModuleFilePath(StringRef moduleCacheDir, + StringRef moduleName, + StringRef contextHash) { + SmallString<128> outputPath(moduleCacheDir); + llvm::sys::path::append(outputPath, (llvm::Twine(moduleName) + + "-" + contextHash + ".pcm").str()); + return outputPath.str().str(); +} + +static std::string getModuleFilePath(StringRef moduleCacheDir, + const ModuleDeps &dep) { + return getModuleFilePath(moduleCacheDir, dep.ModuleName, dep.ContextHash); +} + /// Record the module dependencies we found by scanning Clang modules into /// the module dependencies cache. -static void recordModuleDependencies( +void ClangImporter::recordModuleDependencies( ModuleDependenciesCache &cache, const FullDependenciesResult &clangDependencies) { + struct ModuleInfo { + std::string PCMPath; + std::string ModuleMapPath; + }; + auto ModuleCacheDir = swift::getModuleCachePathFromClang(getClangInstance()); + + // A map keyed by module name and context hash. + llvm::StringMap> moduleInfoMap; + + // Traverse all Clang modules to populate moduleInfoMap for cross + // referencing later. + for (const auto &clangModuleDep : clangDependencies.DiscoveredModules) { + moduleInfoMap[clangModuleDep.ModuleName][clangModuleDep.ContextHash] = + { + // Keep track of pcm path for output. + getModuleFilePath(ModuleCacheDir, clangModuleDep), + // Keep track of modulemap file for input. + clangModuleDep.ClangModuleMapFile + }; + } for (const auto &clangModuleDep : clangDependencies.DiscoveredModules) { + assert(moduleInfoMap[clangModuleDep.ModuleName] + .count(clangModuleDep.ContextHash)); // If we've already cached this information, we're done. if (cache.hasDependencies(clangModuleDep.ModuleName, ModuleDependenciesKind::Clang)) @@ -204,14 +240,58 @@ static void recordModuleDependencies( for (const auto &fileDep : clangModuleDep.FileDeps) { fileDeps.push_back(fileDep.getKey().str()); } + // Inherit all Clang driver args when creating the clang importer. + std::vector allArgs = Impl.ClangArgs; + ClangImporterOptions Opts; + std::vector cc1Args; + + // Calling this to convert driver args to CC1 args. + createClangInvocation(this, Opts, allArgs, &cc1Args); + std::vector swiftArgs; + // We are using Swift frontend mode. + swiftArgs.push_back("-frontend"); + auto addClangArg = [&](StringRef arg) { + swiftArgs.push_back("-Xcc"); + swiftArgs.push_back("-Xclang"); + swiftArgs.push_back("-Xcc"); + swiftArgs.push_back(arg); + }; + // Add all args inheritted from creating the importer. + for (auto arg: cc1Args) { + addClangArg(arg); + } + // Add all args reported from the Clang dependencies scanner. + for(auto arg: clangModuleDep.NonPathCommandLine) { + addClangArg(arg); + } + // Add -fmodule-map-file and -fmodule-file for direct dependencies. + for (auto &dep: clangModuleDep.ClangModuleDeps) { + assert(moduleInfoMap[dep.ModuleName].count(dep.ContextHash)); + addClangArg((llvm::Twine("-fmodule-map-file=") + + moduleInfoMap[dep.ModuleName][dep.ContextHash].ModuleMapPath).str()); + addClangArg((llvm::Twine("-fmodule-file=") + + moduleInfoMap[dep.ModuleName][dep.ContextHash].PCMPath).str()); + } + // Swift frontend action: -emit-pcm + swiftArgs.push_back("-emit-pcm"); + swiftArgs.push_back("-module-name"); + swiftArgs.push_back(clangModuleDep.ModuleName); + + // Swift frontend option for output file path (Foo.pcm). + swiftArgs.push_back("-o"); + swiftArgs.push_back(moduleInfoMap[clangModuleDep.ModuleName] + [clangModuleDep.ContextHash].PCMPath); + + // Swift frontend option for input file path (Foo.modulemap). + swiftArgs.push_back(clangModuleDep.ClangModuleMapFile); // Module-level dependencies. llvm::StringSet<> alreadyAddedModules; auto dependencies = ModuleDependencies::forClangModule( clangModuleDep.ImplicitModulePCMPath, clangModuleDep.ClangModuleMapFile, clangModuleDep.ContextHash, - clangModuleDep.NonPathCommandLine, + swiftArgs, fileDeps); for (const auto &moduleName : clangModuleDep.ClangModuleDeps) { dependencies.addModuleDependency(moduleName.ModuleName, alreadyAddedModules); diff --git a/lib/ClangImporter/ImporterImpl.h b/lib/ClangImporter/ImporterImpl.h index 72de3a47bcde7..182556bca571f 100644 --- a/lib/ClangImporter/ImporterImpl.h +++ b/lib/ClangImporter/ImporterImpl.h @@ -390,6 +390,8 @@ class LLVM_LIBRARY_VISIBILITY ClangImporter::Implementation /// Clang parser, which is used to load textual headers. std::unique_ptr Mangler; + /// Clang arguments used to create the Clang invocation. + std::vector ClangArgs; public: /// Mapping of already-imported declarations. llvm::DenseMap, Decl *> ImportedDecls; diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index 6419b805edb90..884c5fa9f47bf 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -41,22 +41,6 @@ using namespace swift; using FileDependency = SerializationOptions::FileDependency; -/// Extract the specified-or-defaulted -module-cache-path that winds up in -/// the clang importer, for reuse as the .swiftmodule cache path when -/// building a ModuleInterfaceLoader. -std::string -swift::getModuleCachePathFromClang(const clang::CompilerInstance &Clang) { - if (!Clang.hasPreprocessor()) - return ""; - std::string SpecificModuleCachePath = - Clang.getPreprocessor().getHeaderSearchInfo().getModuleCachePath().str(); - - // The returned-from-clang module cache path includes a suffix directory - // that is specific to the clang version and invocation; we want the - // directory above that. - return llvm::sys::path::parent_path(SpecificModuleCachePath).str(); -} - #pragma mark - Forwarding Modules namespace { @@ -1048,6 +1032,7 @@ void ModuleInterfaceLoader::collectVisibleTopLevelModuleNames( void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface( const SearchPathOptions &SearchPathOpts, const LangOptions &LangOpts) { + GenericArgs.push_back("-frontend"); // Start with a SubInvocation that copies various state from our // invoking ASTContext. GenericArgs.push_back("-compile-module-from-interface"); diff --git a/test/ScanDependencies/Inputs/BuildModulesFromGraph.swift b/test/ScanDependencies/Inputs/BuildModulesFromGraph.swift new file mode 100644 index 0000000000000..bd6e5057b9a47 --- /dev/null +++ b/test/ScanDependencies/Inputs/BuildModulesFromGraph.swift @@ -0,0 +1,46 @@ +//===--------------- BuildModulesFromGraph.swift --------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +import Foundation + +let fileName = CommandLine.arguments[1] +let swiftPath = CommandLine.arguments[2] +let moduleName = CommandLine.arguments[3] +let data = try! Data(contentsOf: URL(fileURLWithPath: fileName)) + +let decoder = JSONDecoder() +let moduleDependencyGraph = try! decoder.decode( + ModuleDependencyGraph.self, from: data) + +func findModuleBuildingCommand(_ moduleName: String) -> [String]? { + for (_, dep) in moduleDependencyGraph.modules { + if dep.modulePath.hasSuffix(moduleName) { + switch dep.details { + case .swift(let details): + return details.commandLine + case .clang(let details): + return details.commandLine + } + } else { + continue + } + } + return nil +} + +if let command = findModuleBuildingCommand(moduleName) { + var result = swiftPath + command.forEach { result += " \($0)"} + print(result) + exit(0) +} else { + fatalError("cannot find module building commands for \(moduleName)") +} diff --git a/test/ScanDependencies/Inputs/CHeaders/B.h b/test/ScanDependencies/Inputs/CHeaders/B.h index dedb1b5524f7f..f065892fde57a 100644 --- a/test/ScanDependencies/Inputs/CHeaders/B.h +++ b/test/ScanDependencies/Inputs/CHeaders/B.h @@ -1,4 +1,4 @@ -#include +#include "A.h" void funcB(void); diff --git a/test/ScanDependencies/Inputs/CHeaders/C.h b/test/ScanDependencies/Inputs/CHeaders/C.h index 8a5a9b119ec9d..ec830425f36c5 100644 --- a/test/ScanDependencies/Inputs/CHeaders/C.h +++ b/test/ScanDependencies/Inputs/CHeaders/C.h @@ -1,3 +1,3 @@ -#include +#include "B.h" void funcC(void); diff --git a/test/ScanDependencies/Inputs/CommandRunner.py b/test/ScanDependencies/Inputs/CommandRunner.py new file mode 100755 index 0000000000000..c54adba3fce2c --- /dev/null +++ b/test/ScanDependencies/Inputs/CommandRunner.py @@ -0,0 +1,7 @@ +#!/usr/bin/python + +import subprocess +import sys + +for line in sys.stdin: + subprocess.check_call(line.split()) diff --git a/test/ScanDependencies/Inputs/ModuleDependencyGraph.swift b/test/ScanDependencies/Inputs/ModuleDependencyGraph.swift index 127126b58aeae..bfcb553fabb03 100644 --- a/test/ScanDependencies/Inputs/ModuleDependencyGraph.swift +++ b/test/ScanDependencies/Inputs/ModuleDependencyGraph.swift @@ -65,6 +65,10 @@ struct SwiftModuleDetails: Codable { /// The bridging header, if any. var bridgingHeader: BridgingHeader? + + /// The Swift command line arguments that need to be passed through + /// to the -compile-module-from-interface action to build this module. + var commandLine: [String]? } /// Details specific to Clang modules. @@ -144,11 +148,3 @@ struct ModuleDependencyGraph: Codable { /// Information about the main module. var mainModule: ModuleDependencies { modules[.swift(mainModuleName)]! } } - -let fileName = CommandLine.arguments[1] -let data = try! Data(contentsOf: URL(fileURLWithPath: fileName)) - -let decoder = JSONDecoder() -let moduleDependencyGraph = try! decoder.decode( - ModuleDependencyGraph.self, from: data) -print(moduleDependencyGraph) diff --git a/test/ScanDependencies/Inputs/PrintGraph.swift b/test/ScanDependencies/Inputs/PrintGraph.swift new file mode 100644 index 0000000000000..98ecf6acb5881 --- /dev/null +++ b/test/ScanDependencies/Inputs/PrintGraph.swift @@ -0,0 +1,9 @@ +import Foundation + +let fileName = CommandLine.arguments[1] +let data = try! Data(contentsOf: URL(fileURLWithPath: fileName)) + +let decoder = JSONDecoder() +let moduleDependencyGraph = try! decoder.decode( + ModuleDependencyGraph.self, from: data) +print(moduleDependencyGraph) diff --git a/test/ScanDependencies/module_deps.swift b/test/ScanDependencies/module_deps.swift index 641f622efcc16..84ff06c2b4c58 100644 --- a/test/ScanDependencies/module_deps.swift +++ b/test/ScanDependencies/module_deps.swift @@ -11,10 +11,32 @@ // Check that the JSON parses correctly into the canonical Swift data // structures. -// RUN: %target-build-swift %S/Inputs/ModuleDependencyGraph.swift -o %t/main +// RUN: mkdir -p %t/PrintGraph +// RUN: cp %S/Inputs/PrintGraph.swift %t/main.swift +// RUN: %target-build-swift %S/Inputs/ModuleDependencyGraph.swift %t/main.swift -o %t/main // RUN: %target-codesign %t/main // RUN: %target-run %t/main %t/deps.json +// RUN: mkdir -p %t/BuildModules +// RUN: cp %S/Inputs/BuildModulesFromGraph.swift %t/BuildModules/main.swift +// RUN: %target-build-swift %S/Inputs/ModuleDependencyGraph.swift %t/BuildModules/main.swift -o %t/ModuleBuilder +// RUN: %target-codesign %t/ModuleBuilder + +// RUN: %target-run %t/ModuleBuilder %t/deps.json %swift-path A.pcm | %S/Inputs/CommandRunner.py +// RUN: ls %t/clang-module-cache/A-*.pcm +// RUN: %target-run %t/ModuleBuilder %t/deps.json %swift-path B.pcm | %S/Inputs/CommandRunner.py +// RUN: ls %t/clang-module-cache/B-*.pcm +// RUN: %target-run %t/ModuleBuilder %t/deps.json %swift-path C.pcm | %S/Inputs/CommandRunner.py +// RUN: ls %t/clang-module-cache/C-*.pcm +// RUN: %target-run %t/ModuleBuilder %t/deps.json %swift-path A.swiftmodule | %S/Inputs/CommandRunner.py +// RUN: ls %t/clang-module-cache/A-*.swiftmodule +// RUN: %target-run %t/ModuleBuilder %t/deps.json %swift-path E.swiftmodule | %S/Inputs/CommandRunner.py +// RUN: ls %t/clang-module-cache/E-*.swiftmodule +// RUN: %target-run %t/ModuleBuilder %t/deps.json %swift-path F.swiftmodule | %S/Inputs/CommandRunner.py +// RUN: ls %t/clang-module-cache/F-*.swiftmodule +// RUN: %target-run %t/ModuleBuilder %t/deps.json %swift-path G.swiftmodule | %S/Inputs/CommandRunner.py +// RUN: ls %t/clang-module-cache/G-*.swiftmodule + // REQUIRES: executable_test // REQUIRES: objc_interop @@ -82,7 +104,12 @@ import G // CHECK-SAME: "{{.*}}" // CHECK: "commandLine": [ -// CHECK-NEXT: "-remove-preceeding-explicit-module-build-incompatible-options" +// CHECK-NEXT: "-frontend" +// CHECK-NEXT: "-Xcc" +// CHECK-NEXT: "-Xclang" +// CHECK-NEXT: "-Xcc" +// CHECK-NEXT: "-cc1" +// CHECK: "-remove-preceeding-explicit-module-build-incompatible-options" /// --------Swift module E // CHECK: "swift": "E" diff --git a/test/lit.cfg b/test/lit.cfg index e95e0cd8d2905..e4055228e7632 100644 --- a/test/lit.cfg +++ b/test/lit.cfg @@ -409,6 +409,7 @@ config.substitutions.append( ('%Benchmark_O', config.benchmark_o) ) config.substitutions.append( ('%Benchmark_Driver', config.benchmark_driver) ) config.substitutions.append( ('%llvm-strings', config.llvm_strings) ) config.substitutions.append( ('%target-ptrauth', run_ptrauth ) ) +config.substitutions.append( ('%swift-path', config.swift) ) # This must come after all substitutions containing "%swift". config.substitutions.append(