Skip to content

Commit

Permalink
Expand PerfMap format to support metadata for symbol indexation (dotn…
Browse files Browse the repository at this point in the history
…et#53792)

I have expanded the PerfMap format produced by Crossgen2 and
R2RDump to produce metadata in form of pseudo-symbol records with
high addresses. In this version I have implemented four metadata
entries - output GUID, target OS, target architecture and perfmap
format version number.  I have verified for System.Private.CoreLib
and for the composite framework that Crossgen2 and R2RDump
produce identical metadata.

To facilitate a smooth transition to the new perfmap format, in
accordance with Juan's suggestion I have introduced a new command-line
option to explicitly specify the perfmap format revision. As of today,
0 corresponds to the legacy Crossgen1-style output where the
perfmap file name includes the {MVID} section, perfmap format #1
corresponds to current Crossgen2 with its new naming scheme.
As of today there are no differences in the file content.

Thanks

Tomas
  • Loading branch information
trylek authored Jul 1, 2021
1 parent 08a7b23 commit e3dd985
Show file tree
Hide file tree
Showing 17 changed files with 321 additions and 47 deletions.
19 changes: 19 additions & 0 deletions src/coreclr/tools/aot/ILCompiler.Diagnostics/AssemblyInfo.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;

namespace ILCompiler.Diagnostics
{
public struct AssemblyInfo
{
public readonly string Name;
public readonly Guid Mvid;

public AssemblyInfo(string name, Guid mvid)
{
Name = name;
Mvid = mvid;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,8 @@
<Configurations>Debug;Release;Checked</Configurations>
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="..\ILCompiler.TypeSystem.ReadyToRun\ILCompiler.TypeSystem.ReadyToRun.csproj" />
</ItemGroup>

</Project>
39 changes: 38 additions & 1 deletion src/coreclr/tools/aot/ILCompiler.Diagnostics/PerfMapWriter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,60 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;

using Internal.TypeSystem;

namespace ILCompiler.Diagnostics
{
public class PerfMapWriter
{
public const int LegacyCrossgen1FormatVersion = 0;

public const int CurrentFormatVersion = 1;

public enum PseudoRVA : uint
{
OutputGuid = 0xFFFFFFFF,
TargetOS = 0xFFFFFFFE,
TargetArchitecture = 0xFFFFFFFD,
FormatVersion = 0xFFFFFFFC,
}

private TextWriter _writer;

private PerfMapWriter(TextWriter writer)
{
_writer = writer;
}

public static void Write(string perfMapFileName, IEnumerable<MethodInfo> methods)
public static void Write(string perfMapFileName, int perfMapFormatVersion, IEnumerable<MethodInfo> methods, IEnumerable<AssemblyInfo> inputAssemblies, TargetOS targetOS, TargetArchitecture targetArch)
{
if (perfMapFormatVersion > CurrentFormatVersion)
{
throw new NotSupportedException(perfMapFormatVersion.ToString());
}

using (TextWriter writer = new StreamWriter(perfMapFileName))
{
IEnumerable<AssemblyInfo> orderedInputs = inputAssemblies.OrderBy(asm => asm.Name, StringComparer.OrdinalIgnoreCase);

PerfMapWriter perfMapWriter = new PerfMapWriter(writer);

List<byte> inputHash = new List<byte>();
foreach (AssemblyInfo inputAssembly in orderedInputs)
{
inputHash.AddRange(inputAssembly.Mvid.ToByteArray());
}
inputHash.Add((byte)targetOS);
inputHash.Add((byte)targetArch);
Guid outputGuid = new Guid(MD5.HashData(inputHash.ToArray()));
perfMapWriter.WriteLine(outputGuid.ToString(), (uint)PseudoRVA.OutputGuid, 0);
perfMapWriter.WriteLine(targetOS.ToString(), (uint)PseudoRVA.TargetOS, 0);
perfMapWriter.WriteLine(targetArch.ToString(), (uint)PseudoRVA.TargetArchitecture, 0);
perfMapWriter.WriteLine(CurrentFormatVersion.ToString(), (uint)PseudoRVA.FormatVersion, 0);

foreach (MethodInfo methodInfo in methods)
{
if (methodInfo.HotRVA != 0 && methodInfo.HotLength != 0)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ internal class ReadyToRunObjectWriter
/// </summary>
private readonly EcmaModule _componentModule;

/// <summary>
/// Compilation input files. Input files are emitted as perfmap entries and used
/// to calculate the output GUID of the ReadyToRun executable for symbol indexation.
/// </summary>
private readonly IEnumerable<string> _inputFiles;

/// <summary>
/// Nodes to emit into the output executable as collected by the dependency analysis.
/// </summary>
Expand Down Expand Up @@ -101,9 +107,9 @@ internal class ReadyToRunObjectWriter
private string _perfMapPath;

/// <summary>
/// MVID of the input managed module to embed in the perfmap file name.
/// Requested version of the perfmap file format
/// </summary>
private Guid? _perfMapMvid;
private int _perfMapFormatVersion;

/// <summary>
/// If non-zero, the PE file will be laid out such that it can naturally be mapped with a higher alignment than 4KB.
Expand Down Expand Up @@ -132,6 +138,7 @@ public NodeInfo(ISymbolNode node, int nodeIndex, int symbolIndex)
public ReadyToRunObjectWriter(
string objectFilePath,
EcmaModule componentModule,
IEnumerable<string> inputFiles,
IEnumerable<DependencyNode> nodes,
NodeFactory factory,
bool generateMapFile,
Expand All @@ -140,13 +147,14 @@ public ReadyToRunObjectWriter(
string pdbPath,
bool generatePerfMapFile,
string perfMapPath,
Guid? perfMapMvid,
int perfMapFormatVersion,
bool generateProfileFile,
CallChainProfile callChainProfile,
int customPESectionAlignment)
{
_objectFilePath = objectFilePath;
_componentModule = componentModule;
_inputFiles = inputFiles;
_nodes = nodes;
_nodeFactory = factory;
_customPESectionAlignment = customPESectionAlignment;
Expand All @@ -156,7 +164,7 @@ public ReadyToRunObjectWriter(
_pdbPath = pdbPath;
_generatePerfMapFile = generatePerfMapFile;
_perfMapPath = perfMapPath;
_perfMapMvid = perfMapMvid;
_perfMapFormatVersion = perfMapFormatVersion;

bool generateMap = (generateMapFile || generateMapCsvFile);
bool generateSymbols = (generatePdbFile || generatePerfMapFile);
Expand Down Expand Up @@ -329,6 +337,11 @@ public void EmitPortableExecutable()

if (_outputInfoBuilder != null)
{
foreach (string inputFile in _inputFiles)
{
_outputInfoBuilder.AddInputModule(_nodeFactory.TypeSystemContext.GetModuleFromPath(inputFile));
}

r2rPeBuilder.AddSections(_outputInfoBuilder);

if (_generateMapFile)
Expand Down Expand Up @@ -361,7 +374,7 @@ public void EmitPortableExecutable()
{
path = Path.GetDirectoryName(_objectFilePath);
}
_symbolFileBuilder.SavePerfMap(path, _objectFilePath, _perfMapMvid);
_symbolFileBuilder.SavePerfMap(path, _perfMapFormatVersion, _objectFilePath, _nodeFactory.Target.OperatingSystem, _nodeFactory.Target.Architecture);
}

if (_profileFileBuilder != null)
Expand Down Expand Up @@ -430,6 +443,7 @@ private void EmitObjectData(R2RPEBuilder r2rPeBuilder, ObjectData data, int node
public static void EmitObject(
string objectFilePath,
EcmaModule componentModule,
IEnumerable<string> inputFiles,
IEnumerable<DependencyNode> nodes,
NodeFactory factory,
bool generateMapFile,
Expand All @@ -438,7 +452,7 @@ public static void EmitObject(
string pdbPath,
bool generatePerfMapFile,
string perfMapPath,
Guid? perfMapMvid,
int perfMapFormatVersion,
bool generateProfileFile,
CallChainProfile callChainProfile,
int customPESectionAlignment)
Expand All @@ -447,6 +461,7 @@ public static void EmitObject(
ReadyToRunObjectWriter objectWriter = new ReadyToRunObjectWriter(
objectFilePath,
componentModule,
inputFiles,
nodes,
factory,
generateMapFile: generateMapFile,
Expand All @@ -455,7 +470,7 @@ public static void EmitObject(
pdbPath: pdbPath,
generatePerfMapFile: generatePerfMapFile,
perfMapPath: perfMapPath,
perfMapMvid: perfMapMvid,
perfMapFormatVersion: perfMapFormatVersion,
generateProfileFile: generateProfileFile,
callChainProfile,
customPESectionAlignment);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ public sealed class ReadyToRunCodegenCompilation : Compilation
private readonly string _pdbPath;
private readonly bool _generatePerfMapFile;
private readonly string _perfMapPath;
private readonly Guid? _perfMapMvid;
private readonly int _perfMapFormatVersion;
private readonly bool _generateProfileFile;
private readonly Func<MethodDesc, string> _printReproInstructions;

Expand Down Expand Up @@ -283,7 +283,7 @@ internal ReadyToRunCodegenCompilation(
string pdbPath,
bool generatePerfMapFile,
string perfMapPath,
Guid? perfMapMvid,
int perfMapFormatVersion,
bool generateProfileFile,
int parallelism,
ProfileDataManager profileData,
Expand All @@ -309,7 +309,7 @@ internal ReadyToRunCodegenCompilation(
_pdbPath = pdbPath;
_generatePerfMapFile = generatePerfMapFile;
_perfMapPath = perfMapPath;
_perfMapMvid = perfMapMvid;
_perfMapFormatVersion = perfMapFormatVersion;
_generateProfileFile = generateProfileFile;
_customPESectionAlignment = customPESectionAlignment;
SymbolNodeFactory = new ReadyToRunSymbolNodeFactory(nodeFactory, verifyTypeAndFieldLayout);
Expand Down Expand Up @@ -347,6 +347,7 @@ public override void Compile(string outputFile)
ReadyToRunObjectWriter.EmitObject(
outputFile,
componentModule: null,
inputFiles: _inputFiles,
nodes,
NodeFactory,
generateMapFile: _generateMapFile,
Expand All @@ -355,7 +356,7 @@ public override void Compile(string outputFile)
pdbPath: _pdbPath,
generatePerfMapFile: _generatePerfMapFile,
perfMapPath: _perfMapPath,
perfMapMvid: _perfMapMvid,
perfMapFormatVersion: _perfMapFormatVersion,
generateProfileFile: _generateProfileFile,
callChainProfile: _profileData.CallChainProfile,
_customPESectionAlignment);
Expand Down Expand Up @@ -427,6 +428,7 @@ private void RewriteComponentFile(string inputFile, string outputFile, string ow
ReadyToRunObjectWriter.EmitObject(
outputFile,
componentModule: inputModule,
inputFiles: new string[] { inputFile },
componentGraph.MarkedNodeList,
componentFactory,
generateMapFile: false,
Expand All @@ -435,7 +437,7 @@ private void RewriteComponentFile(string inputFile, string outputFile, string ow
pdbPath: null,
generatePerfMapFile: false,
perfMapPath: null,
perfMapMvid: null,
perfMapFormatVersion: _perfMapFormatVersion,
generateProfileFile: false,
_profileData.CallChainProfile,
customPESectionAlignment: 0);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public sealed class ReadyToRunCodegenCompilationBuilder : CompilationBuilder
private string _pdbPath;
private bool _generatePerfMapFile;
private string _perfMapPath;
private Guid? _perfMapMvid;
private int _perfMapFormatVersion;
private bool _generateProfileFile;
private int _parallelism;
Func<MethodDesc, string> _printReproInstructions;
Expand Down Expand Up @@ -156,11 +156,11 @@ public ReadyToRunCodegenCompilationBuilder UsePdbFile(bool generatePdbFile, stri
return this;
}

public ReadyToRunCodegenCompilationBuilder UsePerfMapFile(bool generatePerfMapFile, string perfMapPath, Guid? inputModuleMvid)
public ReadyToRunCodegenCompilationBuilder UsePerfMapFile(bool generatePerfMapFile, string perfMapPath, int perfMapFormatVersion)
{
_generatePerfMapFile = generatePerfMapFile;
_perfMapPath = perfMapPath;
_perfMapMvid = inputModuleMvid;
_perfMapFormatVersion = perfMapFormatVersion;
return this;
}

Expand Down Expand Up @@ -312,7 +312,7 @@ public override ICompilation ToCompilation()
pdbPath: _pdbPath,
generatePerfMapFile: _generatePerfMapFile,
perfMapPath: _perfMapPath,
perfMapMvid: _perfMapMvid,
perfMapFormatVersion: _perfMapFormatVersion,
generateProfileFile: _generateProfileFile,
_parallelism,
_profileData,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ public OutputSymbol(int sectionIndex, int offset, string name)
/// </summary>
public class OutputInfoBuilder
{
private readonly List<EcmaModule> _inputModules;
private readonly List<OutputNode> _nodes;
private readonly List<OutputSymbol> _symbols;
private readonly List<Section> _sections;
Expand All @@ -117,6 +118,7 @@ public class OutputInfoBuilder

public OutputInfoBuilder()
{
_inputModules = new List<EcmaModule>();
_nodes = new List<OutputNode>();
_symbols = new List<OutputSymbol>();
_sections = new List<Section>();
Expand All @@ -127,6 +129,11 @@ public OutputInfoBuilder()
_relocCounts = new Dictionary<RelocType, int>();
}

public void AddInputModule(EcmaModule module)
{
_inputModules.Add(module);
}

public void AddNode(OutputNode node, ISymbolDefinitionNode symbol)
{
_nodes.Add(node);
Expand Down Expand Up @@ -197,6 +204,16 @@ public IEnumerable<MethodInfo> EnumerateMethods()
}
}

public IEnumerable<AssemblyInfo> EnumerateInputAssemblies()
{
foreach (EcmaModule inputModule in _inputModules)
{
yield return new AssemblyInfo(
inputModule.Assembly.GetName().Name,
inputModule.MetadataReader.GetGuid(inputModule.MetadataReader.GetModuleDefinition().Mvid));
}
}

private string FormatMethodName(MethodDesc method, TypeNameFormatter typeNameFormatter)
{
StringBuilder output = new StringBuilder();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using System.Text;
using System.Threading.Tasks;

using Internal.TypeSystem;
using ILCompiler.Diagnostics;

namespace ILCompiler.PEWriter
Expand All @@ -28,12 +29,34 @@ public void SavePdb(string pdbPath, string dllFileName)
new PdbWriter(pdbPath, PDBExtraData.None).WritePDBData(dllFileName, _outputInfoBuilder.EnumerateMethods());
}

public void SavePerfMap(string perfMapPath, string dllFileName, Guid? perfMapMvid)
public void SavePerfMap(string perfMapPath, int perfMapFormatVersion, string dllFileName, TargetOS targetOS, TargetArchitecture targetArch)
{
string mvidComponent = (perfMapMvid.HasValue ? perfMapMvid.Value.ToString() : "composite");
string perfMapFileName = Path.Combine(perfMapPath, Path.GetFileNameWithoutExtension(dllFileName) + ".ni.{" + mvidComponent + "}.map");
string perfMapExtension;
if (perfMapFormatVersion == PerfMapWriter.LegacyCrossgen1FormatVersion)
{
string mvidComponent = null;
foreach (AssemblyInfo inputAssembly in _outputInfoBuilder.EnumerateInputAssemblies())
{
if (mvidComponent == null)
{
mvidComponent = inputAssembly.Mvid.ToString();
}
else
{
mvidComponent = "composite";
break;
}
}
perfMapExtension = ".ni.{" + mvidComponent + "}.map";
}
else
{
perfMapExtension = ".ni.r2rmap";
}

string perfMapFileName = Path.Combine(perfMapPath, Path.GetFileNameWithoutExtension(dllFileName) + perfMapExtension);
Console.WriteLine("Emitting PerfMap file: {0}", perfMapFileName);
PerfMapWriter.Write(perfMapFileName, _outputInfoBuilder.EnumerateMethods());
PerfMapWriter.Write(perfMapFileName, perfMapFormatVersion, _outputInfoBuilder.EnumerateMethods(), _outputInfoBuilder.EnumerateInputAssemblies(), targetOS, targetArch);
}
}
}
Loading

0 comments on commit e3dd985

Please sign in to comment.