Add primitive support for link-time memory reports

include/eld/Config/GeneralOptions.h

@@ -18,6 +18,7 @@
 #include "eld/Support/FileSystem.h"
 #include "eld/Support/Memory.h"
 #include "eld/Support/MsgHandling.h"
+#include "eld/Support/RegisterTimer.h"
 #include "eld/SymbolResolver/ResolveInfo.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
@@ -844,6 +845,19 @@ class GeneralOptions {
   void setTimingStatsFile(std::string StatsFile) {
     TimingStatsFile = StatsFile;
   }
+
+  // --emit-memory-stats <file>
+  void setMemoryStatsFile(std::string statsFile);
+
+  bool hasMemoryStatsFile() const {
+    return m_MemoryStatsFile.has_value();
+  }
+
+  std::string getMemoryStatsFile() const {
+    ASSERT(m_MemoryStatsFile.has_value(), "memory stats file not available!");
+    return m_MemoryStatsFile.value();
+  }
+
   //--------------------Plugin Config--------------------------------
   void addPluginConfig(const std::string &Config) {
     PluginConfig.push_back(Config);
@@ -1352,6 +1366,7 @@ class GeneralOptions {
   std::string LinkLaunchDirectory;
   bool ShowRMSectNameInDiag = false;
   bool UseDefaultPlugins = true;
+  std::optional<std::string> m_MemoryStatsFile;
 };
 
 } // namespace eld

include/eld/Driver/GnuLinkerOptions.td

@@ -833,6 +833,11 @@ def W : Joined<["-"], "W">,
              "with different values for OS/ABI\n"
             >,
         Group<grp_diagopts>;
+defm emit_memory_stats : mDashEq<"emit-memory-stats", "emit_memory_stats",
+                                 "Emit memory statistics of various linker "
+                                 "operations to the specified file">,
+                         MetaVarName<"<filename>">,
+                         Group<grp_diagopts>;
 
 //===----------------------------------------------------------------------===//
 /// Optimization options

include/eld/Support/RegisterTimer.h

@@ -7,7 +7,11 @@
 #ifndef ELD_SUPPORT_REGISTERTIMER_H
 #define ELD_SUPPORT_REGISTERTIMER_H
 
+#include "llvm/ADT/MapVector.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Timer.h"
+#include "llvm/Support/raw_ostream.h"
+#include <iostream>
 
 namespace eld {
 
@@ -19,12 +23,100 @@ class RegisterTimer : public llvm::NamedRegionTimer {
   // Params: Name -> Stats Description, Group -> Name of Sub-section in Linker
   // timing stats and string to "Group-by",
   // Enable -> Turn Timer On/Off.
-  RegisterTimer(llvm::StringRef Name, llvm::StringRef Group, bool Enable)
-      : NamedRegionTimer(Name, Name, Group, Group, Enable) {}
+  RegisterTimer(llvm::StringRef name, llvm::StringRef group, bool enable);
 
-  ~RegisterTimer() {}
+  ~RegisterTimer();
+
+#ifdef __linux__
+  static void setShouldRecordMemoryStats(bool shouldRecordMemoryStats) {
+    ShouldRecordMemoryStats = shouldRecordMemoryStats;
+  }
+
+  /// A simple data structure to track memory usage information.
+  struct MemoryUsageInfo {
+    /// Current resident set size in kilobytes. Resident set size is the amount of virtual
+    /// memory that is actually residing in memory.
+    int64_t RSSCur = 0;
+    /// Peak resident set size in kilobytes seen so far
+    int64_t RSSPeak = 0;
+
+    std::string getHumanReadableRSSCur() const {
+      if (RSSCur > 1024)
+        return std::to_string(RSSCur / 1024) + "MB";
+      else
+        return std::to_string(RSSCur) + "KB";
+    }
+
+    std::string getHumanReadableRSSPeak() const {
+      if (RSSPeak > 1024)
+        return std::to_string(RSSPeak / 1024) + "MB";
+      else
+        return std::to_string(RSSPeak) + "KB";
+    }
+
+    MemoryUsageInfo operator-(const MemoryUsageInfo &rhs) const {
+      MemoryUsageInfo res;
+      res.RSSCur = RSSCur - rhs.RSSCur;
+      res.RSSPeak = RSSPeak - rhs.RSSPeak;
+      return res;
+    }
+
+    MemoryUsageInfo &operator+=(const MemoryUsageInfo &rhs) {
+      RSSCur += rhs.RSSCur;
+      RSSPeak += rhs.RSSPeak;
+      return *this;
+    }
+
+    MemoryUsageInfo operator+(const MemoryUsageInfo &rhs) {
+      MemoryUsageInfo res = *this;
+      res += rhs;
+      return res;
+    }
+  };
+
+  static void emitMemoryStats(llvm::raw_ostream &OS);
+#endif
+private:
+  llvm::StringRef Name;
+  llvm::StringRef Group;
+
+#ifdef __linux__
+  static bool ShouldRecordMemoryStats;
+  /// Stores the memory usage information for each timer.
+  ///
+  /// MapVector is used here to preserve the insertion order. The order is
+  /// important for the memory report to be easy to read and understand.
+  static llvm::MapVector<llvm::StringRef,
+                         llvm::MapVector<llvm::StringRef, MemoryUsageInfo>>
+      AbsMemoryInfo;
+  /// Stores the diff of memory usage information at the start and end timer for
+  /// each timer
+  static llvm::MapVector<llvm::StringRef,
+                         llvm::MapVector<llvm::StringRef, MemoryUsageInfo>>
+      DiffMemoryInfo;
+
+  /// Stores the memory usage information at the start of the timer.
+  MemoryUsageInfo StartMemInfo;
+
+  /// Computes MemoryUsageInfo by parsing the virtual file /proc/self/status
+  static MemoryUsageInfo getMemoryUsageInfo();
+
+  /// Computes diff MemoryUsageInfo for the group by summing up DiffMemoryInfo
+  /// for each member of the group. It is used to determine total amount of
+  /// memory used by the group.
+  static MemoryUsageInfo getGroupMemoryUsageInfo(llvm::StringRef groupName);
+
+  /// Returns true if we should record memory for this timer.
+  /// It returns false for all timers if link time memory report is not
+  /// requested.
+  /// It also returns false for timers that are called large number of times.
+  /// This is because reading the virtual file /proc/self/status large number of
+  /// times can be time-consuming.
+  bool shouldRecordMemory() const;
+#endif
 };
 
+
 class Timer {
 public:
   // Generic timer.

lib/Config/GeneralOptions.cpp

@@ -676,3 +676,8 @@ bool GeneralOptions::traceSymbol(const ResolveInfo &RI) const {
   }
   return false;
 }
+
+void GeneralOptions::setMemoryStatsFile(std::string statsFile) {
+  m_MemoryStatsFile = statsFile;
+  RegisterTimer::setShouldRecordMemoryStats(true);
+}

lib/LinkerWrapper/GnuLdDriver.cpp

@@ -8,6 +8,7 @@
 #include "eld/Diagnostics/DiagnosticEngine.h"
 #include "eld/Diagnostics/DiagnosticPrinter.h"
 #include "eld/Input/InputAction.h"
+#include "eld/Support/RegisterTimer.h"
 #if defined(ELD_ENABLE_TARGET_ARM) || defined(ELD_ENABLE_TARGET_AARCH64)
 #include "eld/Driver/ARMLinkDriver.h"
 #endif
@@ -130,7 +131,8 @@ GnuLdDriver *GnuLdDriver::Create(LinkerConfig &C, DriverFlavor F,
 }
 
 bool GnuLdDriver::emitStats(eld::Module &M) const {
-  std::string File = Config.options().timingStatsFile();
+  const GeneralOptions &genOptions = Config.options();
+  std::string File = genOptions.timingStatsFile();
   std::error_code error;
   llvm::raw_fd_ostream *StatsFile = nullptr;
   if (!File.empty()) {
@@ -147,6 +149,16 @@ bool GnuLdDriver::emitStats(eld::Module &M) const {
   llvm::TimerGroup::clearAll();
   M.getLinkerScript().printPluginTimers(*OutStream);
   delete StatsFile;
+  if (genOptions.hasMemoryStatsFile()) {
+    std::string memStatsFile = genOptions.getMemoryStatsFile();
+    llvm::raw_fd_ostream memStatsFileStream(memStatsFile, error,
+                                            llvm::sys::fs::OF_None);
+    if (error) {
+      Config.raise(Diag::fatal_unwritable_output) << File << error.message();
+      return false;
+    }
+    RegisterTimer::emitMemoryStats(memStatsFileStream);
+  }
   return true;
 }
 
@@ -392,6 +404,10 @@ bool GnuLdDriver::processOptions(llvm::opt::InputArgList &Args) {
     Config.options().setTimingStatsFile(arg->getValue());
   }
 
+  if (llvm::opt::Arg *arg = Args.getLastArg(T::emit_memory_stats)) {
+    Config.options().setMemoryStatsFile(arg->getValue());
+  }
+
   // --time-region
   if (llvm::opt::Arg *arg = Args.getLastArg(T::time_region)) {
     Config.options().setPrintTimingStats();

lib/Support/RegisterTimer.cpp

@@ -9,6 +9,7 @@
 #include "eld/Support/MsgHandling.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
+#include <fstream>
 
 using namespace eld;
 
@@ -66,3 +67,117 @@ void Timer::print(llvm::raw_ostream &Os) {
 void Timer::printVal(double Val, llvm::raw_ostream &Os) {
   Os << llvm::format("  %7.4f ", Val);
 }
+
+llvm::MapVector<
+    llvm::StringRef,
+    llvm::MapVector<llvm::StringRef, RegisterTimer::MemoryUsageInfo>>
+    RegisterTimer::DiffMemoryInfo;
+llvm::MapVector<
+    llvm::StringRef,
+    llvm::MapVector<llvm::StringRef, RegisterTimer::MemoryUsageInfo>>
+    RegisterTimer::AbsMemoryInfo;
+
+RegisterTimer::RegisterTimer(llvm::StringRef name, llvm::StringRef group,
+                             bool enable)
+    : NamedRegionTimer(name, name, group, group, enable), Name(name),
+      Group(group) {
+  if (!shouldRecordMemory())
+    return;
+  StartMemInfo = getMemoryUsageInfo();
+  // Required to preserve the order!
+  DiffMemoryInfo[Group];
+}
+
+RegisterTimer::~RegisterTimer() {
+  if (!shouldRecordMemory())
+    return;
+  MemoryUsageInfo curMemInfo = getMemoryUsageInfo();
+  DiffMemoryInfo[Group][Name] += curMemInfo - StartMemInfo;
+  AbsMemoryInfo[Group][Name] = curMemInfo;
+  // llvm::errs() << "Storing memory info for " << Group << ":" << Name << "\n";
+}
+
+#ifdef __linux__
+bool RegisterTimer::ShouldRecordMemoryStats = false;
+
+RegisterTimer::MemoryUsageInfo RegisterTimer::getMemoryUsageInfo() {
+  std::ifstream statusFile("/proc/self/status");
+  std::stringstream content;
+  content << statusFile.rdbuf();
+  std::string line;
+  MemoryUsageInfo memInfo;
+  while (std::getline(content, line)) {
+    std::size_t pos = line.find(":");
+    if (pos == std::string::npos)
+      continue;
+    std::string key = line.substr(0, pos);
+    std::string valueStr = line.substr(pos + 1);
+    llvm::StringRef valueStrRef = valueStr;
+    valueStrRef = valueStrRef.trim();
+    valueStrRef.consume_back_insensitive(" kb");
+    if (key == "VmHWM") {
+      valueStrRef.getAsInteger(/*Radix=*/10, memInfo.RSSPeak);
+    } else if (key == "VmRSS") {
+      valueStrRef.getAsInteger(/*Radix=*/10, memInfo.RSSCur);
+    }
+  }
+  return memInfo;
+}
+
+void RegisterTimer::emitMemoryStats(llvm::raw_ostream &OS) {
+  for (auto &group : DiffMemoryInfo) {
+    llvm::StringRef groupName = group.first;
+    // llvm::errs() << "Writing memory stats for group: " << groupName << "\n";
+    const auto &groupMembersMemInfo = group.second;
+    OS << "===" << std::string(73, '-') << "===" << "\n";
+    unsigned padding = (80 - groupName.size()) / 2;
+    if (padding > 80)
+      padding = 0;
+    OS.indent(padding);
+    OS << groupName << "\n";
+    OS << "===" << std::string(73, '-') << "===" << "\n";
+    MemoryUsageInfo groupMemInfo = getGroupMemoryUsageInfo(groupName);
+    OS << "Total resident set size change: "
+       << groupMemInfo.getHumanReadableRSSCur() << "\n";
+    OS << "\n";
+    OS << "   ------RSS------";
+    OS << "   -RSS peak-";
+    OS << "   ---Name---";
+    OS << "\n";
+    for (const auto &elem : groupMembersMemInfo) {
+      llvm::StringRef name = elem.first;
+      const auto &memInfo = elem.second;
+      const auto &absMemInfo = AbsMemoryInfo[groupName][name];
+      OS << "   "
+         << llvm::format("%6s(+%6s)",
+                         absMemInfo.getHumanReadableRSSCur().c_str(),
+                         memInfo.getHumanReadableRSSCur().c_str());
+      OS << std::string(3, ' ')
+         << llvm::format("%6s", absMemInfo.getHumanReadableRSSPeak().c_str());
+      OS << std::string(7, ' ') << name << "\n";
+    }
+    OS << "\n\n";
+  }
+}
+
+bool RegisterTimer::shouldRecordMemory() const {
+  if (!ShouldRecordMemoryStats)
+    return false;
+  // All these timers are not suitable for memory tracking because they are
+  // triggered large number of times.
+  if (Group == "Symbol Resolution" || Name == "VisitSymbol" ||
+      Name == "VisitSections" || Name == "Sort Sections" ||
+      Name == "Evaluate Expressions")
+    return false;
+  return true;
+}
+
+RegisterTimer::MemoryUsageInfo
+RegisterTimer::getGroupMemoryUsageInfo(llvm::StringRef groupName) {
+  MemoryUsageInfo memInfo;
+  for (const auto &elem : DiffMemoryInfo[groupName]) {
+    memInfo += elem.second;
+  }
+  return memInfo;
+}
+#endif

test/Common/standalone/EmitMemoryStats/EmitMemoryStats.test

@@ -0,0 +1,15 @@
+#---EmitMemoryStats.test--------------------------- Executable --------------------#
+#BEGIN_COMMENT
+# This test checks that --emit-memory-stats option works as expected.
+#END_COMMENT
+#START_TEST
+RUN: %clang %clangopts -o %t1.1.o %p/Inputs/1.c -c -ffunction-sections
+RUN: %link %linkopts -o %t1.1.out %t1.1.o --emit-memory-stats %t1.1.memory.stats
+RUN: %filecheck %s < %t1.1.memory.stats
+#END_TEST
+CHECK: ===-------------------------------------------------------------------------===
+CHECK:                                   Link Summary
+CHECK: ===-------------------------------------------------------------------------===
+CHECK: Total resident set size change: {{.*}}
+CHECK:    ------RSS------   -RSS peak-   ---Name---
+

test/Common/standalone/EmitMemoryStats/Inputs/1.c

@@ -0,0 +1 @@
+int foo() { return 1; }