diff --git a/src/mono/mono/mini/aot-compiler.c b/src/mono/mono/mini/aot-compiler.c index 52147fcbec951..60d68481cc417 100644 --- a/src/mono/mono/mini/aot-compiler.c +++ b/src/mono/mono/mini/aot-compiler.c @@ -9828,8 +9828,9 @@ compile_method (MonoAotCompile *acfg, MonoMethod *method) mono_atomic_inc_i32 (&acfg->stats.ccount); if (acfg->aot_opts.compiled_methods_outfile && acfg->compiled_methods_outfile != NULL) { - if (!mono_method_is_generic_impl (method) && method->token != 0) + if (!mono_method_is_generic_impl (method) && method->token != 0) { fprintf (acfg->compiled_methods_outfile, "%x\n", method->token); + } } } @@ -14832,6 +14833,10 @@ aot_assembly (MonoAssembly *ass, guint32 jit_opts, MonoAotOptions *aot_options) acfg->compiled_methods_outfile = fopen (acfg->aot_opts.compiled_methods_outfile, "w+"); if (!acfg->compiled_methods_outfile) aot_printerrf (acfg, "Unable to open compiled-methods-outfile specified file %s\n", acfg->aot_opts.compiled_methods_outfile); + else { + fprintf(acfg->compiled_methods_outfile, "%s\n", ass->image->filename); + fprintf(acfg->compiled_methods_outfile, "%s\n", ass->image->guid); + } } if (acfg->aot_opts.data_outfile) { diff --git a/src/mono/sample/HelloWorld/HelloWorld.csproj b/src/mono/sample/HelloWorld/HelloWorld.csproj index b4e843e802673..bc6db3770c040 100644 --- a/src/mono/sample/HelloWorld/HelloWorld.csproj +++ b/src/mono/sample/HelloWorld/HelloWorld.csproj @@ -25,6 +25,8 @@ LibraryFormat="$(_AotLibraryFormat)" Assemblies="@(AotInputAssemblies)" OutputDir="$(PublishDir)" + CollectCompiledMethods="$(StripILCode)" + CompiledMethodsOutputDirectory="$(CompiledMethodsOutputDirectory)" IntermediateOutputPath="$(IntermediateOutputPath)" UseAotDataFile="$(UseAotDataFile)" CacheFilePath="$(IntermediateOutputPath)aot_compiler_cache.json" @@ -35,4 +37,26 @@ + + + + + + true + + + + + + + + + diff --git a/src/mono/sample/HelloWorld/Makefile b/src/mono/sample/HelloWorld/Makefile index 8b18dc45c65a0..8441e565d29c5 100644 --- a/src/mono/sample/HelloWorld/Makefile +++ b/src/mono/sample/HelloWorld/Makefile @@ -6,6 +6,8 @@ MONO_CONFIG?=Debug MONO_ARCH?=$(shell . $(TOP)eng/native/init-os-and-arch.sh && echo $${arch}) TARGET_OS?=$(shell . $(TOP)eng/native/init-os-and-arch.sh && echo $${os}) AOT?=false +StripILCode?=false +CompiledMethodsOutputDirectory?= # #NET_TRACE_PATH= #PGO_BINARY_PATH= @@ -18,6 +20,8 @@ publish: -c $(MONO_CONFIG) \ -r $(TARGET_OS)-$(MONO_ARCH) \ /p:RunAOTCompilation=$(AOT) \ + /p:StripILCode=$(StripILCode) \ + /p:CompiledMethodsOutputDirectory=$(CompiledMethodsOutputDirectory) \ '/p:NetTracePath="$(NET_TRACE_PATH)"' \ '/p:PgoBinaryPath="$(PGO_BINARY_PATH)"' \ '/p:MibcProfilePath="$(MIBC_PROFILE_PATH)"' diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 553d9dc5a63a3..a762eb8a3266b 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -67,6 +67,7 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task /// - LlvmObjectFile (if using LLVM) /// - LlvmBitcodeFile (if using LLVM-only) /// - ExportsFile (used in LibraryMode only) + /// - MethodTokenFile (when using CollectCompiledMethods=true) /// [Output] public ITaskItem[]? CompiledAssemblies { get; set; } @@ -152,6 +153,16 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task /// public bool UseDwarfDebug { get; set; } + /// + /// Instructs the AOT compiler to print the list of aot compiled methods + /// + public bool CollectCompiledMethods { get; set; } + + /// + /// Directory to store the aot output when using switch compiled-methods-outfile + /// + public string? CompiledMethodsOutputDirectory { get; set; } + /// /// File to use for profile-guided optimization, *only* the methods described in the file will be AOT compiled. /// @@ -436,6 +447,17 @@ private bool ProcessAndValidateArguments() throw new LogAsErrorException($"Could not find {fullPath} to AOT"); } + if (CollectCompiledMethods) + { + if (string.IsNullOrEmpty(CompiledMethodsOutputDirectory)) + throw new LogAsErrorException($"{nameof(CompiledMethodsOutputDirectory)} is empty. When {nameof(CollectCompiledMethods)} is set to true, the user needs to provide a directory for {nameof(CompiledMethodsOutputDirectory)}."); + + if (!Directory.Exists(CompiledMethodsOutputDirectory)) + { + Directory.CreateDirectory(CompiledMethodsOutputDirectory); + } + } + return !Log.HasLoggedErrors; } @@ -711,6 +733,23 @@ private PrecompileArguments GetPrecompileArgumentsFor(ITaskItem assemblyItem, st aotArgs.Add("dedup-skip"); } + if (CollectCompiledMethods) + { + string assemblyName = assemblyFilename.Replace(".", "_"); + string outputFileName = assemblyName + "_compiled_methods.txt"; + string outputFilePath; + if (string.IsNullOrEmpty(CompiledMethodsOutputDirectory)) + { + outputFilePath = outputFileName; + } + else + { + outputFilePath = Path.Combine(CompiledMethodsOutputDirectory, outputFileName); + } + aotArgs.Add($"compiled-methods-outfile={outputFilePath}"); + aotAssembly.SetMetadata("MethodTokenFile", outputFilePath); + } + // compute output mode and file names if (parsedAotMode == MonoAotMode.LLVMOnly || parsedAotMode == MonoAotMode.LLVMOnlyInterp) { diff --git a/src/tasks/MonoTargetsTasks/ILStrip/ILStrip.cs b/src/tasks/MonoTargetsTasks/ILStrip/ILStrip.cs index 1f80296537177..34c81b67f54e4 100644 --- a/src/tasks/MonoTargetsTasks/ILStrip/ILStrip.cs +++ b/src/tasks/MonoTargetsTasks/ILStrip/ILStrip.cs @@ -12,14 +12,18 @@ using CilStrip.Mono.Cecil.Binary; using CilStrip.Mono.Cecil.Cil; using CilStrip.Mono.Cecil.Metadata; +using System.Reflection.Metadata; +using System.Reflection.Metadata.Ecma335; +using System.Reflection.PortableExecutable; +using System.Buffers; public class ILStrip : Microsoft.Build.Utilities.Task { + [Required] /// /// Assemblies to be stripped. /// The assemblies will be modified in place if OutputPath metadata is not set. /// - [Required] public ITaskItem[] Assemblies { get; set; } = Array.Empty(); /// @@ -27,6 +31,22 @@ public class ILStrip : Microsoft.Build.Utilities.Task /// public bool DisableParallelStripping { get; set; } + /// + /// Enable the feature of trimming indiviual methods + /// + public bool TrimIndividualMethods { get; set; } + + /// + /// Assembilies got trimmed successfully. + /// + /// Successful trimming will set the following metadata on the items: + /// - TrimmedAssemblyFileName + /// + [Output] + public ITaskItem[]? TrimmedAssemblies { get; set; } + + private readonly List _trimmedAssemblies = new(); + public override bool Execute() { if (Assemblies.Length == 0) @@ -38,12 +58,25 @@ public override bool Execute() if (BuildEngine is IBuildEngine9 be9) allowedParallelism = be9.RequestCores(allowedParallelism); ParallelLoopResult result = Parallel.ForEach(Assemblies, - new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, - (assemblyItem, state) => - { - if (!StripAssembly(assemblyItem)) - state.Stop(); - }); + new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, + (assemblyItem, state) => + { + if (!TrimIndividualMethods) + { + if (!StripAssembly(assemblyItem)) + state.Stop(); + } + else + { + if (!TrimMethods(assemblyItem)) + state.Stop(); + } + }); + + if (TrimIndividualMethods) + { + TrimmedAssemblies = _trimmedAssemblies.ToArray(); + } if (!result.IsCompleted && !Log.HasLoggedErrors) { @@ -81,4 +114,189 @@ private bool StripAssembly(ITaskItem assemblyItem) return true; } + private bool TrimMethods(ITaskItem assemblyItem) + { + string methodTokenFile = assemblyItem.GetMetadata("MethodTokenFile"); + if (string.IsNullOrEmpty(methodTokenFile)) + { + Log.LogError($"Metadata MethodTokenFile of {assemblyItem.ItemSpec} is empty"); + return true; + } + if (!File.Exists(methodTokenFile)) + { + Log.LogError($"{methodTokenFile} doesn't exist."); + return true; + } + + using StreamReader sr = new(methodTokenFile); + string? assemblyFilePath = sr.ReadLine(); + if (string.IsNullOrEmpty(assemblyFilePath)) + { + Log.LogError($"The first line of {assemblyFilePath} is empty."); + return true; + } + + if (!File.Exists(assemblyFilePath)) + { + Log.LogError($"{assemblyFilePath} read from {methodTokenFile} doesn't exist."); + return true; + } + + string trimmedAssemblyFilePath = ComputeTrimmedAssemblyPath(assemblyFilePath); + bool isTrimmed = false; + using FileStream fs = File.Open(assemblyFilePath, FileMode.Open); + using PEReader peReader = new(fs, PEStreamOptions.LeaveOpen); + MetadataReader mr = peReader.GetMetadataReader(); + string actualGuidValue = ComputeGuid(mr); + string? expectedGuidValue = sr.ReadLine(); + if (!string.Equals(actualGuidValue, expectedGuidValue, StringComparison.OrdinalIgnoreCase)) + { + Log.LogError($"[ILStrip] GUID value of {assemblyFilePath} doesn't match the value listed in {methodTokenFile}."); + return true; + } + + string? line = sr.ReadLine(); + if (!string.IsNullOrEmpty(line)) + { + isTrimmed = true; + Dictionary methodBodyUses = ComputeMethodBodyUsage(mr, sr, line, methodTokenFile); + CreateTrimmedAssembly(peReader, trimmedAssemblyFilePath, fs, methodBodyUses); + } + + if (isTrimmed) + { + AddItemToTrimmedList(assemblyFilePath, trimmedAssemblyFilePath); + } + + return true; + } + + private static string ComputeTrimmedAssemblyPath(string assemblyFilePath) + { + string? assemblyPath = Path.GetDirectoryName(assemblyFilePath); + string? assemblyName = Path.GetFileNameWithoutExtension(assemblyFilePath); + if (string.IsNullOrEmpty(assemblyPath)) + { + return (assemblyName + "_trimmed.dll"); + } + else + { + return Path.Combine(assemblyPath, (assemblyName + "_trimmed.dll")); + } + } + + private static string ComputeGuid(MetadataReader mr) + { + GuidHandle mvidHandle = mr.GetModuleDefinition().Mvid; + Guid mvid = mr.GetGuid(mvidHandle); + return mvid.ToString(); + } + + private Dictionary ComputeMethodBodyUsage(MetadataReader mr, StreamReader sr, string? line, string methodTokenFile) + { + Dictionary tokenToRva = new(); + Dictionary methodBodyUses = new(); + + foreach (MethodDefinitionHandle mdefh in mr.MethodDefinitions) + { + int methodToken = MetadataTokens.GetToken(mr, mdefh); + MethodDefinition mdef = mr.GetMethodDefinition(mdefh); + int rva = mdef.RelativeVirtualAddress; + + tokenToRva.Add(methodToken, rva); + + if (methodBodyUses.TryGetValue(rva, out var _)) + { + methodBodyUses[rva]++; + } + else + { + methodBodyUses.Add(rva, 1); + } + } + + do + { + int methodToken2Trim = Convert.ToInt32(line, 16); + if (methodToken2Trim <= 0) + { + Log.LogError($"Method token: {line} in {methodTokenFile} is not a valid hex value."); + } + if (tokenToRva.TryGetValue(methodToken2Trim, out int rva2Trim)) + { + methodBodyUses[rva2Trim]--; + } + else + { + Log.LogError($"Method token: {line} in {methodTokenFile} can't be found within the assembly."); + } + } while ((line = sr.ReadLine()) != null); + + return methodBodyUses; + } + + private void CreateTrimmedAssembly(PEReader peReader, string trimmedAssemblyFilePath, FileStream fs, Dictionary methodBodyUses) + { + using FileStream os = File.Open(trimmedAssemblyFilePath, FileMode.Create); + { + fs.Position = 0; + MemoryStream memStream = new MemoryStream((int)fs.Length); + fs.CopyTo(memStream); + + foreach (var kvp in methodBodyUses) + { + int rva = kvp.Key; + int count = kvp.Value; + if (count == 0) + { + int methodSize = ComputeMethodSize(peReader, rva); + int actualLoc = ComputeMethodHash(peReader, rva); + int headerSize = ComputeMethodHeaderSize(memStream, actualLoc); + ZeroOutMethodBody(ref memStream, methodSize, actualLoc, headerSize); + } + else if (count < 0) + { + Log.LogError($"Method usage count is less than zero for rva: {rva}."); + } + } + + memStream.Position = 0; + memStream.CopyTo(os); + } + } + + private static int ComputeMethodSize(PEReader peReader, int rva) => peReader.GetMethodBody(rva).Size; + + private static int ComputeMethodHash(PEReader peReader, int rva) + { + int sectionIndex = peReader.PEHeaders.GetContainingSectionIndex(rva); + int relativeOffset = rva - peReader.PEHeaders.SectionHeaders[sectionIndex].VirtualAddress; + return (peReader.PEHeaders.SectionHeaders[sectionIndex].PointerToRawData + relativeOffset); + } + + private static int ComputeMethodHeaderSize(MemoryStream memStream, int actualLoc) + { + memStream.Position = actualLoc; + int firstbyte = memStream.ReadByte(); + int headerFlag = firstbyte & 0b11; + return (headerFlag == 2 ? 1 : 4); + } + + private static void ZeroOutMethodBody(ref MemoryStream memStream, int methodSize, int actualLoc, int headerSize) + { + memStream.Position = actualLoc + headerSize; + + byte[] zeroBuffer; + zeroBuffer = ArrayPool.Shared.Rent(methodSize); + Array.Clear(zeroBuffer, 0, zeroBuffer.Length); + memStream.Write(zeroBuffer, 0, methodSize - headerSize); + ArrayPool.Shared.Return(zeroBuffer); + } + + private void AddItemToTrimmedList(string assemblyFilePath, string trimmedAssemblyFilePath) + { + var trimmedAssemblyItem = new TaskItem(assemblyFilePath); + trimmedAssemblyItem.SetMetadata("TrimmedAssemblyFileName", trimmedAssemblyFilePath); + _trimmedAssemblies.Add(trimmedAssemblyItem); + } }