Skip to content

[clang][Dependency Scanning] Report What a Module Exports during Scanning (#137421) #10604

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: stable/20240723
Choose a base branch
from
Open

[clang][Dependency Scanning] Report What a Module Exports during Scanning (#137421) #10604

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 22 additions & 8 deletions clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,12 +142,25 @@ struct ModuleDeps {
/// on, not including transitive dependencies.
std::vector<PrebuiltModuleDep> PrebuiltModuleDeps;

/// A list of module identifiers this module directly depends on, not
/// including transitive dependencies.
/// This struct contains information about a single dependency.
struct DepInfo {
/// Identifies the dependency.
ModuleID ID;

/// Indicates if the module that has this dependency exports it or not.
bool Exported = false;

bool operator<(const DepInfo &Other) const {
return std::tie(ID, Exported) < std::tie(Other.ID, Other.Exported);
}
};

/// A list of DepsInfo containing information about modules this module
/// directly depends on, not including transitive dependencies.
///
/// This may include modules with a different context hash when it can be
/// determined that the differences are benign for this compilation.
std::vector<ModuleID> ClangModuleDeps;
std::vector<ModuleDeps::DepInfo> ClangModuleDeps;

/// The CASID for the module input dependency tree, if any.
std::optional<llvm::cas::CASID> CASFileSystemRootID;
Expand Down Expand Up @@ -245,7 +258,8 @@ class ModuleDepCollectorPP final : public PPCallbacks {
llvm::DenseSet<const Module *> &AddedModules);

/// Add discovered module dependency for the given module.
void addOneModuleDep(const Module *M, const ModuleID ID, ModuleDeps &MD);
void addOneModuleDep(const Module *M, bool Exported, const ModuleID ID,
ModuleDeps &MD);
};

/// Collects modular and non-modular dependencies of the main file by attaching
Expand Down Expand Up @@ -322,16 +336,16 @@ class ModuleDepCollector final : public DependencyCollector {

/// Collect module map files for given modules.
llvm::DenseSet<const FileEntry *>
collectModuleMapFiles(ArrayRef<ModuleID> ClangModuleDeps) const;
collectModuleMapFiles(ArrayRef<ModuleDeps::DepInfo> ClangModuleDeps) const;

/// Add module map files to the invocation, if needed.
void addModuleMapFiles(CompilerInvocation &CI,
ArrayRef<ModuleID> ClangModuleDeps) const;
ArrayRef<ModuleDeps::DepInfo> ClangModuleDeps) const;
/// Add module files (pcm) to the invocation, if needed.
void addModuleFiles(CompilerInvocation &CI,
ArrayRef<ModuleID> ClangModuleDeps) const;
ArrayRef<ModuleDeps::DepInfo> ClangModuleDeps) const;
void addModuleFiles(CowCompilerInvocation &CI,
ArrayRef<ModuleID> ClangModuleDeps) const;
ArrayRef<ModuleDeps::DepInfo> ClangModuleDeps) const;

/// Add paths that require looking up outputs to the given dependencies.
void addOutputPaths(CowCompilerInvocation &CI, ModuleDeps &Deps);
Expand Down
65 changes: 38 additions & 27 deletions clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,10 +393,10 @@ ModuleDepCollector::getInvocationAdjustedForModuleBuildWithoutOutputs(
}

llvm::DenseSet<const FileEntry *> ModuleDepCollector::collectModuleMapFiles(
ArrayRef<ModuleID> ClangModuleDeps) const {
ArrayRef<ModuleDeps::DepInfo> ClangModuleDeps) const {
llvm::DenseSet<const FileEntry *> ModuleMapFiles;
for (const ModuleID &MID : ClangModuleDeps) {
ModuleDeps *MD = ModuleDepsByID.lookup(MID);
for (const auto &Info : ClangModuleDeps) {
ModuleDeps *MD = ModuleDepsByID.lookup(Info.ID);
assert(MD && "Inconsistent dependency info");
// TODO: Track ClangModuleMapFile as `FileEntryRef`.
auto FE = ScanInstance.getFileManager().getFile(MD->ClangModuleMapFile);
Expand All @@ -407,21 +407,23 @@ llvm::DenseSet<const FileEntry *> ModuleDepCollector::collectModuleMapFiles(
}

void ModuleDepCollector::addModuleMapFiles(
CompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {
CompilerInvocation &CI,
ArrayRef<ModuleDeps::DepInfo> ClangModuleDeps) const {
if (Service.shouldEagerLoadModules())
return; // Only pcm is needed for eager load.

for (const ModuleID &MID : ClangModuleDeps) {
ModuleDeps *MD = ModuleDepsByID.lookup(MID);
for (const auto &Info : ClangModuleDeps) {
ModuleDeps *MD = ModuleDepsByID.lookup(Info.ID);
assert(MD && "Inconsistent dependency info");
CI.getFrontendOpts().ModuleMapFiles.push_back(MD->ClangModuleMapFile);
}
}

void ModuleDepCollector::addModuleFiles(
CompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {
for (const ModuleID &MID : ClangModuleDeps) {
ModuleDeps *MD = ModuleDepsByID.lookup(MID);
CompilerInvocation &CI,
ArrayRef<ModuleDeps::DepInfo> ClangModuleDeps) const {
for (const auto &Info : ClangModuleDeps) {
ModuleDeps *MD = ModuleDepsByID.lookup(Info.ID);
std::string PCMPath =
Controller.lookupModuleOutput(*MD, ModuleOutputKind::ModuleFile);

Expand All @@ -434,14 +436,15 @@ void ModuleDepCollector::addModuleFiles(
CI.getFrontendOpts().ModuleFiles.push_back(std::move(PCMPath));
else
CI.getHeaderSearchOpts().PrebuiltModuleFiles.insert(
{MID.ModuleName, std::move(PCMPath)});
{Info.ID.ModuleName, std::move(PCMPath)});
}
}

void ModuleDepCollector::addModuleFiles(
CowCompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {
for (const ModuleID &MID : ClangModuleDeps) {
ModuleDeps *MD = ModuleDepsByID.lookup(MID);
CowCompilerInvocation &CI,
ArrayRef<ModuleDeps::DepInfo> ClangModuleDeps) const {
for (const auto &Info : ClangModuleDeps) {
ModuleDeps *MD = ModuleDepsByID.lookup(Info.ID);
std::string PCMPath =
Controller.lookupModuleOutput(*MD, ModuleOutputKind::ModuleFile);

Expand All @@ -454,7 +457,7 @@ void ModuleDepCollector::addModuleFiles(
CI.getMutFrontendOpts().ModuleFiles.push_back(std::move(PCMPath));
else
CI.getMutHeaderSearchOpts().PrebuiltModuleFiles.insert(
{MID.ModuleName, std::move(PCMPath)});
{Info.ID.ModuleName, std::move(PCMPath)});
}
}

Expand Down Expand Up @@ -484,10 +487,10 @@ void ModuleDepCollector::applyDiscoveredDependencies(CompilerInvocation &CI) {
CI.getFrontendOpts().ModuleMapFiles.emplace_back(
CurrentModuleMap->getNameAsRequested());

SmallVector<ModuleID> DirectDeps;
SmallVector<ModuleDeps::DepInfo> DirectDeps;
for (const auto &KV : ModularDeps)
if (DirectModularDeps.contains(KV.first))
DirectDeps.push_back(KV.second->ID);
DirectDeps.push_back({KV.second->ID, /* Exported = */ false});

// TODO: Report module maps the same way it's done for modular dependencies.
addModuleMapFiles(CI, DirectDeps);
Expand Down Expand Up @@ -636,9 +639,9 @@ static std::string getModuleContextHash(const ModuleDeps &MD,
// example, case-insensitive paths to modulemap files. Usually such a case
// would indicate a missed optimization to canonicalize, but it may be
// difficult to canonicalize all cases when there is a VFS.
for (const auto &ID : MD.ClangModuleDeps) {
HashBuilder.add(ID.ModuleName);
HashBuilder.add(ID.ContextHash);
for (const auto &Info : MD.ClangModuleDeps) {
HashBuilder.add(Info.ID.ModuleName);
HashBuilder.add(Info.ID.ContextHash);
}

HashBuilder.add(EagerLoadModules);
Expand Down Expand Up @@ -1017,22 +1020,30 @@ void ModuleDepCollectorPP::addAllSubmoduleDeps(
});
}

void ModuleDepCollectorPP::addOneModuleDep(const Module *M, const ModuleID ID,
ModuleDeps &MD) {
MD.ClangModuleDeps.push_back(ID);
void ModuleDepCollectorPP::addOneModuleDep(const Module *M, bool Exported,
const ModuleID ID, ModuleDeps &MD) {
MD.ClangModuleDeps.push_back({ID, Exported});

if (MD.IsInStableDirectories)
MD.IsInStableDirectories = MDC.ModularDeps[M]->IsInStableDirectories;
}

void ModuleDepCollectorPP::addModuleDep(
const Module *M, ModuleDeps &MD,
llvm::DenseSet<const Module *> &AddedModules) {
SmallVector<Module *> ExportedModulesVector;
M->getExportedModules(ExportedModulesVector);
llvm::DenseSet<const Module *> ExportedModulesSet(
ExportedModulesVector.begin(), ExportedModulesVector.end());
for (const Module *Import : M->Imports) {
if (Import->getTopLevelModule() != M->getTopLevelModule() &&
const Module *ImportedTopLevelModule = Import->getTopLevelModule();
if (ImportedTopLevelModule != M->getTopLevelModule() &&
!MDC.isPrebuiltModule(Import)) {
if (auto ImportID = handleTopLevelModule(Import->getTopLevelModule()))
if (AddedModules.insert(Import->getTopLevelModule()).second)
addOneModuleDep(Import->getTopLevelModule(), *ImportID, MD);
if (auto ImportID = handleTopLevelModule(ImportedTopLevelModule))
if (AddedModules.insert(ImportedTopLevelModule).second) {
bool Exported = ExportedModulesSet.contains(ImportedTopLevelModule);
addOneModuleDep(ImportedTopLevelModule, Exported, *ImportID, MD);
}
}
}
}
Expand All @@ -1056,7 +1067,7 @@ void ModuleDepCollectorPP::addAffectingClangModule(
!MDC.isPrebuiltModule(Affecting)) {
if (auto ImportID = handleTopLevelModule(Affecting))
if (AddedModules.insert(Affecting).second)
addOneModuleDep(Affecting, *ImportID, MD);
addOneModuleDep(Affecting, /* Exported = */ false, *ImportID, MD);
}
}
}
Expand Down
133 changes: 133 additions & 0 deletions clang/test/ClangScanDeps/export.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
// Test correctly reporting what a module exports during dependency scanning.
// Module A depends on modules B, C and D, but only exports B and C.
// Module E depends on modules B, C and D, and exports all of them.

// RUN: rm -rf %t
// RUN: split-file %s %t
// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
// RUN: clang-scan-deps -compilation-database \
// RUN: %t/cdb.json -format experimental-full > %t/deps.db
// RUN: cat %t/deps.db | sed 's:\\\\\?:/:g' | FileCheck %s

//--- cdb.json.template
[
{
"directory": "DIR",
"command": "clang -c DIR/test.c -I DIR/AH -I DIR/BH -I DIR/CH -I DIR/DH -I DIR/EH -fmodules -fmodules-cache-path=DIR/cache",
"file": "DIR/test.c"
},
]

//--- AH/A.h
#include "B.h"
#include "C.h"
#include "D.h"

int funcA();

//--- AH/module.modulemap
module A {
header "A.h"

export B
export C
}

//--- BH/B.h
//--- BH/module.modulemap
module B {
header "B.h"
}

//--- CH/C.h
//--- CH/module.modulemap
module C {
header "C.h"
}

//--- DH/D.h
//--- DH/module.modulemap
module D {
header "D.h"
}

//--- EH/E.h
#include "B.h"
#include "C.h"
#include "D.h"

//--- EH/module.modulemap
module E {
header "E.h"
export *
}

//--- test.c
#include "A.h"
#include "E.h"

int test1() {
return funcA();
}

// CHECK: {
// CHECK-NEXT: "modules": [
// CHECK-NEXT: {
// CHECK-NEXT: "clang-module-deps": [
// CHECK-NEXT: {
// CHECK-NEXT: "context-hash": "[[HASH_MOD_B:.*]]",
// CHECK-NEXT: "module-name": "B",
// CHECK-NEXT: "exported": "true"
// CHECK-NEXT: },
// CHECK-NEXT: {
// CHECK-NEXT: "context-hash": "[[HASH_MOD_C:.*]]",
// CHECK-NEXT: "module-name": "C",
// CHECK-NEXT: "exported": "true"
// CHECK-NEXT: },
// CHECK-NEXT: {
// CHECK-NEXT: "context-hash": "[[HASH_MOD_D:.*]]",
// CHECK-NEXT: "module-name": "D"
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "clang-modulemap-file":{{.*}},
// CHECK-NEXT: "command-line": [
// CHECK: ],
// CHECK: "name": "A"
// CHECK-NEXT: }
// CHECK: {
// CHECK: "name": "B"
// CHECK: }
// CHECK: {
// CHECK: "name": "C"
// CHECK: }
// CHECK: {
// CHECK: "name": "D"
// CHECK: }
// CHECK: {
// CHECK-NEXT: "clang-module-deps": [
// CHECK-NEXT: {
// CHECK-NEXT: "context-hash": "[[HASH_MOD_B]]",
// CHECK-NEXT: "module-name": "B",
// CHECK-NEXT: "exported": "true"
// CHECK-NEXT: },
// CHECK-NEXT: {
// CHECK-NEXT: "context-hash": "[[HASH_MOD_C]]",
// CHECK-NEXT: "module-name": "C",
// CHECK-NEXT: "exported": "true"
// CHECK-NEXT: },
// CHECK-NEXT: {
// CHECK-NEXT: "context-hash": "[[HASH_MOD_D]]",
// CHECK-NEXT: "module-name": "D",
// CHECK-NEXT: "exported": "true"
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "clang-modulemap-file":{{.*}},
// CHECK-NEXT: "command-line": [
// CHECK: ],
// CHECK: "name": "E"
// CHECK-NEXT: }
// CHECK: ]
// CHECK: }



1 change: 1 addition & 0 deletions clang/test/ClangScanDeps/optimize-vfs-pch-tree.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
// CHECK-NEXT: {
// CHECK-NEXT: "context-hash": "{{.*}}",
// CHECK-NEXT: "module-name": "E"
// CHECK-NEXT: "exported": "true"
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/modules/D/module.modulemap",
Expand Down
3 changes: 2 additions & 1 deletion clang/test/ClangScanDeps/optimize-vfs-pch.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@
// CHECK-NEXT: "clang-module-deps": [
// CHECK-NEXT: {
// CHECK-NEXT: "context-hash": "{{.*}}",
// CHECK-NEXT: "module-name": "E"
// CHECK-NEXT: "module-name": "E",
// CHECK-NEXT: "exported": "true"
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/modules/D/module.modulemap",
Expand Down
26 changes: 21 additions & 5 deletions clang/tools/clang-scan-deps/ClangScanDeps.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -601,16 +601,32 @@ static auto toJSONStrings(llvm::json::OStream &JOS, Container &&Strings) {
};
}

static auto toJSONModuleID(llvm::json::OStream &JOS, StringRef ContextHash,
StringRef ModuleName, bool Exported) {
return JOS.object([&] {
JOS.attribute("context-hash", StringRef(ContextHash));
JOS.attribute("module-name", StringRef(ModuleName));
if (Exported)
JOS.attribute("exported", StringRef("true"));
});
}

// Technically, we don't need to sort the dependency list to get determinism.
// Leaving these be will simply preserve the import order.
static auto toJSONSorted(llvm::json::OStream &JOS, std::vector<ModuleID> V) {
llvm::sort(V);
return [&JOS, V = std::move(V)] {
for (const ModuleID &MID : V)
JOS.object([&] {
JOS.attribute("context-hash", StringRef(MID.ContextHash));
JOS.attribute("module-name", StringRef(MID.ModuleName));
});
for (const auto &MID : V)
toJSONModuleID(JOS, MID.ContextHash, MID.ModuleName, false);
};
}

static auto toJSONSorted(llvm::json::OStream &JOS,
std::vector<ModuleDeps::DepInfo> V) {
llvm::sort(V);
return [&JOS, V = std::move(V)] {
for (const ModuleDeps::DepInfo &MID : V)
toJSONModuleID(JOS, MID.ID.ContextHash, MID.ID.ModuleName, MID.Exported);
};
}

Expand Down
Loading