diff --git a/include/eld/Config/GeneralOptions.h b/include/eld/Config/GeneralOptions.h index 49818606c..ec6b48871 100644 --- a/include/eld/Config/GeneralOptions.h +++ b/include/eld/Config/GeneralOptions.h @@ -18,6 +18,7 @@ #include "eld/Support/FileSystem.h" #include "eld/Support/Memory.h" #include "eld/Support/MsgHandling.h" +#include "eld/Support/RegisterTimer.h" #include "eld/SymbolResolver/ResolveInfo.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" @@ -844,6 +845,19 @@ class GeneralOptions { void setTimingStatsFile(std::string StatsFile) { TimingStatsFile = StatsFile; } + + // --emit-memory-stats + void setMemoryStatsFile(std::string statsFile); + + bool hasMemoryStatsFile() const { + return m_MemoryStatsFile.has_value(); + } + + std::string getMemoryStatsFile() const { + ASSERT(m_MemoryStatsFile.has_value(), "memory stats file not available!"); + return m_MemoryStatsFile.value(); + } + //--------------------Plugin Config-------------------------------- void addPluginConfig(const std::string &Config) { PluginConfig.push_back(Config); @@ -1352,6 +1366,7 @@ class GeneralOptions { std::string LinkLaunchDirectory; bool ShowRMSectNameInDiag = false; bool UseDefaultPlugins = true; + std::optional m_MemoryStatsFile; }; } // namespace eld diff --git a/include/eld/Driver/GnuLinkerOptions.td b/include/eld/Driver/GnuLinkerOptions.td index 4151c3a0a..90994fafd 100644 --- a/include/eld/Driver/GnuLinkerOptions.td +++ b/include/eld/Driver/GnuLinkerOptions.td @@ -833,6 +833,11 @@ def W : Joined<["-"], "W">, "with different values for OS/ABI\n" >, Group; +defm emit_memory_stats : mDashEq<"emit-memory-stats", "emit_memory_stats", + "Emit memory statistics of various linker " + "operations to the specified file">, + MetaVarName<"">, + Group; //===----------------------------------------------------------------------===// /// Optimization options diff --git a/include/eld/Support/RegisterTimer.h b/include/eld/Support/RegisterTimer.h index cde523fa0..a339ab871 100644 --- a/include/eld/Support/RegisterTimer.h +++ b/include/eld/Support/RegisterTimer.h @@ -7,7 +7,11 @@ #ifndef ELD_SUPPORT_REGISTERTIMER_H #define ELD_SUPPORT_REGISTERTIMER_H +#include "llvm/ADT/MapVector.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" +#include namespace eld { @@ -19,12 +23,100 @@ class RegisterTimer : public llvm::NamedRegionTimer { // Params: Name -> Stats Description, Group -> Name of Sub-section in Linker // timing stats and string to "Group-by", // Enable -> Turn Timer On/Off. - RegisterTimer(llvm::StringRef Name, llvm::StringRef Group, bool Enable) - : NamedRegionTimer(Name, Name, Group, Group, Enable) {} + RegisterTimer(llvm::StringRef name, llvm::StringRef group, bool enable); - ~RegisterTimer() {} + ~RegisterTimer(); + +#ifdef __linux__ + static void setShouldRecordMemoryStats(bool shouldRecordMemoryStats) { + ShouldRecordMemoryStats = shouldRecordMemoryStats; + } + + /// A simple data structure to track memory usage information. + struct MemoryUsageInfo { + /// Current resident set size in kilobytes. Resident set size is the amount of virtual + /// memory that is actually residing in memory. + int64_t RSSCur = 0; + /// Peak resident set size in kilobytes seen so far + int64_t RSSPeak = 0; + + std::string getHumanReadableRSSCur() const { + if (RSSCur > 1024) + return std::to_string(RSSCur / 1024) + "MB"; + else + return std::to_string(RSSCur) + "KB"; + } + + std::string getHumanReadableRSSPeak() const { + if (RSSPeak > 1024) + return std::to_string(RSSPeak / 1024) + "MB"; + else + return std::to_string(RSSPeak) + "KB"; + } + + MemoryUsageInfo operator-(const MemoryUsageInfo &rhs) const { + MemoryUsageInfo res; + res.RSSCur = RSSCur - rhs.RSSCur; + res.RSSPeak = RSSPeak - rhs.RSSPeak; + return res; + } + + MemoryUsageInfo &operator+=(const MemoryUsageInfo &rhs) { + RSSCur += rhs.RSSCur; + RSSPeak += rhs.RSSPeak; + return *this; + } + + MemoryUsageInfo operator+(const MemoryUsageInfo &rhs) { + MemoryUsageInfo res = *this; + res += rhs; + return res; + } + }; + + static void emitMemoryStats(llvm::raw_ostream &OS); +#endif +private: + llvm::StringRef Name; + llvm::StringRef Group; + +#ifdef __linux__ + static bool ShouldRecordMemoryStats; + /// Stores the memory usage information for each timer. + /// + /// MapVector is used here to preserve the insertion order. The order is + /// important for the memory report to be easy to read and understand. + static llvm::MapVector> + AbsMemoryInfo; + /// Stores the diff of memory usage information at the start and end timer for + /// each timer + static llvm::MapVector> + DiffMemoryInfo; + + /// Stores the memory usage information at the start of the timer. + MemoryUsageInfo StartMemInfo; + + /// Computes MemoryUsageInfo by parsing the virtual file /proc/self/status + static MemoryUsageInfo getMemoryUsageInfo(); + + /// Computes diff MemoryUsageInfo for the group by summing up DiffMemoryInfo + /// for each member of the group. It is used to determine total amount of + /// memory used by the group. + static MemoryUsageInfo getGroupMemoryUsageInfo(llvm::StringRef groupName); + + /// Returns true if we should record memory for this timer. + /// It returns false for all timers if link time memory report is not + /// requested. + /// It also returns false for timers that are called large number of times. + /// This is because reading the virtual file /proc/self/status large number of + /// times can be time-consuming. + bool shouldRecordMemory() const; +#endif }; + class Timer { public: // Generic timer. diff --git a/lib/Config/GeneralOptions.cpp b/lib/Config/GeneralOptions.cpp index a47593870..22bdb6860 100644 --- a/lib/Config/GeneralOptions.cpp +++ b/lib/Config/GeneralOptions.cpp @@ -676,3 +676,8 @@ bool GeneralOptions::traceSymbol(const ResolveInfo &RI) const { } return false; } + +void GeneralOptions::setMemoryStatsFile(std::string statsFile) { + m_MemoryStatsFile = statsFile; + RegisterTimer::setShouldRecordMemoryStats(true); +} \ No newline at end of file diff --git a/lib/LinkerWrapper/GnuLdDriver.cpp b/lib/LinkerWrapper/GnuLdDriver.cpp index 26625d5d3..c74dc6f0b 100644 --- a/lib/LinkerWrapper/GnuLdDriver.cpp +++ b/lib/LinkerWrapper/GnuLdDriver.cpp @@ -8,6 +8,7 @@ #include "eld/Diagnostics/DiagnosticEngine.h" #include "eld/Diagnostics/DiagnosticPrinter.h" #include "eld/Input/InputAction.h" +#include "eld/Support/RegisterTimer.h" #if defined(ELD_ENABLE_TARGET_ARM) || defined(ELD_ENABLE_TARGET_AARCH64) #include "eld/Driver/ARMLinkDriver.h" #endif @@ -130,7 +131,8 @@ GnuLdDriver *GnuLdDriver::Create(LinkerConfig &C, DriverFlavor F, } bool GnuLdDriver::emitStats(eld::Module &M) const { - std::string File = Config.options().timingStatsFile(); + const GeneralOptions &genOptions = Config.options(); + std::string File = genOptions.timingStatsFile(); std::error_code error; llvm::raw_fd_ostream *StatsFile = nullptr; if (!File.empty()) { @@ -147,6 +149,16 @@ bool GnuLdDriver::emitStats(eld::Module &M) const { llvm::TimerGroup::clearAll(); M.getLinkerScript().printPluginTimers(*OutStream); delete StatsFile; + if (genOptions.hasMemoryStatsFile()) { + std::string memStatsFile = genOptions.getMemoryStatsFile(); + llvm::raw_fd_ostream memStatsFileStream(memStatsFile, error, + llvm::sys::fs::OF_None); + if (error) { + Config.raise(Diag::fatal_unwritable_output) << File << error.message(); + return false; + } + RegisterTimer::emitMemoryStats(memStatsFileStream); + } return true; } @@ -392,6 +404,10 @@ bool GnuLdDriver::processOptions(llvm::opt::InputArgList &Args) { Config.options().setTimingStatsFile(arg->getValue()); } + if (llvm::opt::Arg *arg = Args.getLastArg(T::emit_memory_stats)) { + Config.options().setMemoryStatsFile(arg->getValue()); + } + // --time-region if (llvm::opt::Arg *arg = Args.getLastArg(T::time_region)) { Config.options().setPrintTimingStats(); diff --git a/lib/Support/RegisterTimer.cpp b/lib/Support/RegisterTimer.cpp index 5791d8338..6b143e12e 100644 --- a/lib/Support/RegisterTimer.cpp +++ b/lib/Support/RegisterTimer.cpp @@ -9,6 +9,7 @@ #include "eld/Support/MsgHandling.h" #include "llvm/Support/Format.h" #include "llvm/Support/raw_ostream.h" +#include using namespace eld; @@ -66,3 +67,117 @@ void Timer::print(llvm::raw_ostream &Os) { void Timer::printVal(double Val, llvm::raw_ostream &Os) { Os << llvm::format(" %7.4f ", Val); } + +llvm::MapVector< + llvm::StringRef, + llvm::MapVector> + RegisterTimer::DiffMemoryInfo; +llvm::MapVector< + llvm::StringRef, + llvm::MapVector> + RegisterTimer::AbsMemoryInfo; + +RegisterTimer::RegisterTimer(llvm::StringRef name, llvm::StringRef group, + bool enable) + : NamedRegionTimer(name, name, group, group, enable), Name(name), + Group(group) { + if (!shouldRecordMemory()) + return; + StartMemInfo = getMemoryUsageInfo(); + // Required to preserve the order! + DiffMemoryInfo[Group]; +} + +RegisterTimer::~RegisterTimer() { + if (!shouldRecordMemory()) + return; + MemoryUsageInfo curMemInfo = getMemoryUsageInfo(); + DiffMemoryInfo[Group][Name] += curMemInfo - StartMemInfo; + AbsMemoryInfo[Group][Name] = curMemInfo; + // llvm::errs() << "Storing memory info for " << Group << ":" << Name << "\n"; +} + +#ifdef __linux__ +bool RegisterTimer::ShouldRecordMemoryStats = false; + +RegisterTimer::MemoryUsageInfo RegisterTimer::getMemoryUsageInfo() { + std::ifstream statusFile("/proc/self/status"); + std::stringstream content; + content << statusFile.rdbuf(); + std::string line; + MemoryUsageInfo memInfo; + while (std::getline(content, line)) { + std::size_t pos = line.find(":"); + if (pos == std::string::npos) + continue; + std::string key = line.substr(0, pos); + std::string valueStr = line.substr(pos + 1); + llvm::StringRef valueStrRef = valueStr; + valueStrRef = valueStrRef.trim(); + valueStrRef.consume_back_insensitive(" kb"); + if (key == "VmHWM") { + valueStrRef.getAsInteger(/*Radix=*/10, memInfo.RSSPeak); + } else if (key == "VmRSS") { + valueStrRef.getAsInteger(/*Radix=*/10, memInfo.RSSCur); + } + } + return memInfo; +} + +void RegisterTimer::emitMemoryStats(llvm::raw_ostream &OS) { + for (auto &group : DiffMemoryInfo) { + llvm::StringRef groupName = group.first; + // llvm::errs() << "Writing memory stats for group: " << groupName << "\n"; + const auto &groupMembersMemInfo = group.second; + OS << "===" << std::string(73, '-') << "===" << "\n"; + unsigned padding = (80 - groupName.size()) / 2; + if (padding > 80) + padding = 0; + OS.indent(padding); + OS << groupName << "\n"; + OS << "===" << std::string(73, '-') << "===" << "\n"; + MemoryUsageInfo groupMemInfo = getGroupMemoryUsageInfo(groupName); + OS << "Total resident set size change: " + << groupMemInfo.getHumanReadableRSSCur() << "\n"; + OS << "\n"; + OS << " ------RSS------"; + OS << " -RSS peak-"; + OS << " ---Name---"; + OS << "\n"; + for (const auto &elem : groupMembersMemInfo) { + llvm::StringRef name = elem.first; + const auto &memInfo = elem.second; + const auto &absMemInfo = AbsMemoryInfo[groupName][name]; + OS << " " + << llvm::format("%6s(+%6s)", + absMemInfo.getHumanReadableRSSCur().c_str(), + memInfo.getHumanReadableRSSCur().c_str()); + OS << std::string(3, ' ') + << llvm::format("%6s", absMemInfo.getHumanReadableRSSPeak().c_str()); + OS << std::string(7, ' ') << name << "\n"; + } + OS << "\n\n"; + } +} + +bool RegisterTimer::shouldRecordMemory() const { + if (!ShouldRecordMemoryStats) + return false; + // All these timers are not suitable for memory tracking because they are + // triggered large number of times. + if (Group == "Symbol Resolution" || Name == "VisitSymbol" || + Name == "VisitSections" || Name == "Sort Sections" || + Name == "Evaluate Expressions") + return false; + return true; +} + +RegisterTimer::MemoryUsageInfo +RegisterTimer::getGroupMemoryUsageInfo(llvm::StringRef groupName) { + MemoryUsageInfo memInfo; + for (const auto &elem : DiffMemoryInfo[groupName]) { + memInfo += elem.second; + } + return memInfo; +} +#endif \ No newline at end of file diff --git a/test/Common/standalone/EmitMemoryStats/EmitMemoryStats.test b/test/Common/standalone/EmitMemoryStats/EmitMemoryStats.test new file mode 100644 index 000000000..adac11333 --- /dev/null +++ b/test/Common/standalone/EmitMemoryStats/EmitMemoryStats.test @@ -0,0 +1,15 @@ +#---EmitMemoryStats.test--------------------------- Executable --------------------# +#BEGIN_COMMENT +# This test checks that --emit-memory-stats option works as expected. +#END_COMMENT +#START_TEST +RUN: %clang %clangopts -o %t1.1.o %p/Inputs/1.c -c -ffunction-sections +RUN: %link %linkopts -o %t1.1.out %t1.1.o --emit-memory-stats %t1.1.memory.stats +RUN: %filecheck %s < %t1.1.memory.stats +#END_TEST +CHECK: ===-------------------------------------------------------------------------=== +CHECK: Link Summary +CHECK: ===-------------------------------------------------------------------------=== +CHECK: Total resident set size change: {{.*}} +CHECK: ------RSS------ -RSS peak- ---Name--- + diff --git a/test/Common/standalone/EmitMemoryStats/Inputs/1.c b/test/Common/standalone/EmitMemoryStats/Inputs/1.c new file mode 100644 index 000000000..b79e0f77b --- /dev/null +++ b/test/Common/standalone/EmitMemoryStats/Inputs/1.c @@ -0,0 +1 @@ +int foo() { return 1; } \ No newline at end of file