Wasm dedup

src/mono/mono/mini/aot-runtime.c

@@ -4337,9 +4337,12 @@ mono_aot_can_dedup (MonoMethod *method)
 			info->subtype == WRAPPER_SUBTYPE_INTERP_LMF ||
 			info->subtype == WRAPPER_SUBTYPE_AOT_INIT)
 			return FALSE;
+#if 0
+		// See is_linkonce_method () in mini-llvm.c
 		if (info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_IN_SIG || info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_OUT_SIG)
 			/* Handled using linkonce */
 			return FALSE;
+#endif
 		return TRUE;
 	}
 	default:

src/mono/mono/mini/llvm-runtime.cpp

@@ -3,13 +3,25 @@
 
 #include <glib.h>
 
+#ifdef HOST_WASM
+#include <emscripten.h>
+#endif
+
 extern "C" {
 
 void
 mono_llvm_cpp_throw_exception (void)
 {
 	gint32 *ex = NULL;
 
+#ifdef HOST_WASM
+		EM_ASM(
+			   var err = new Error();
+			   console.log ("Throw stacktrace: \n");
+			   console.log (err.stack);
+			   );
+#endif
+
 	/* The generated code catches an int32* */
 	throw ex;
 }

src/mono/mono/mini/mini-llvm.c

@@ -9521,11 +9521,14 @@ is_linkonce_method (MonoMethod *method)
 	 * FIXME: Fails System.Core tests
 	 * -> amodule->sorted_methods contains duplicates, screwing up jit tables.
 	 */
+	// FIXME: This works, but the aot data for the methods is still kept, so size still increases
+#if 0
 	if (method->wrapper_type == MONO_WRAPPER_OTHER) {
 		WrapperInfo *info = mono_marshal_get_wrapper_info (method);
 		if (info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_IN_SIG || info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_OUT_SIG)
 			return TRUE;
 	}
+#endif
 #endif
 	return FALSE;
 }

src/mono/wasm/build/WasmApp.targets

@@ -24,6 +24,7 @@
       - $(WasmDebugLevel)
       - $(WasmNativeDebugSymbols) - Build with native debug symbols, useful only with `$(RunAOTCompilation)`, or `$(WasmBuildNative)`
                                     Defaults to true.
+      - $(WasmDedup)         - Whenever to dedup generic instances when using AOT. Defaults to true.
 
       - $(WasmProfilers)     - Profilers to use
       - $(AOTMode)           - Defaults to `AotInterp`
@@ -64,6 +65,7 @@
 
   <PropertyGroup>
     <WasmStripAOTAssemblies>false</WasmStripAOTAssemblies>
+    <WasmDedup Condition="'$(WasmDedup)' == ''">$(RunAOTCompilation)</WasmDedup>
 
     <!--<WasmStripAOTAssemblies Condition="'$(AOTMode)' == 'AotInterp'">false</WasmStripAOTAssemblies>-->
     <!--<WasmStripAOTAssemblies Condition="'$(WasmStripAOTAssemblies)' == ''">$(RunAOTCompilation)</WasmStripAOTAssemblies>-->
@@ -106,6 +108,27 @@
            Text="Builing in AOTMode=LLVMonly, but found some assemblies marked as _InternalForceInterpret: @(_AOT_InternalForceInterpretAssemblies)" />
 
     <Message Text="AOT'ing @(_AotInputAssemblies->Count()) assemblies" Importance="High" />
+
+    <!-- Dedup -->
+    <PropertyGroup Condition="'$(WasmDedup)' == 'true'">
+      <_WasmDedupAssembly>$(_WasmIntermediateOutputPath)\aot-instances.dll</_WasmDedupAssembly>
+    </PropertyGroup>
+    <WriteLinesToFile Condition="'$(WasmDedup)' == 'true'" File="$(_WasmIntermediateOutputPath)/aot-instances.cs" Overwrite="true" Lines="" />
+    <Csc
+      Condition="'$(WasmDedup)' == 'true'"
+      Sources="$(_WasmIntermediateOutputPath)\aot-instances.cs"
+      OutputAssembly="$(_WasmDedupAssembly)"
+      TargetType="library"
+      References="@(ReferencePath)"
+      ToolExe="$(CscToolExe)"
+      ToolPath="$(CscToolPath)" />
+    <ItemGroup Condition="'$(WasmDedup)' == 'true'">
+      <_AotInputAssemblies Include="$(_WasmDedupAssembly)">
+        <AotArguments>@(MonoAOTCompilerDefaultAotArguments, ';')</AotArguments>
+        <ProcessArguments>@(MonoAOTCompilerDefaultProcessArguments, ';')</ProcessArguments>
+      </_AotInputAssemblies>
+    </ItemGroup>
+
     <MonoAOTCompiler
       CompilerBinaryPath="$(MonoAotCrossCompilerPath)"
       OutputDir="$(_WasmIntermediateOutputPath)"
@@ -119,6 +142,7 @@
       AotModulesTablePath="$(_WasmIntermediateOutputPath)driver-gen.c"
       UseLLVM="true"
       DisableParallelAot="true"
+      DedupAssembly="$(_WasmDedupAssembly)"
       LLVMPath="$(EMSDK_PATH)\upstream\bin">
       <Output TaskParameter="CompiledAssemblies" ItemName="_WasmAssembliesInternal" />
       <Output TaskParameter="FileWrites" ItemName="FileWrites" />
@@ -407,7 +431,7 @@ EMSCRIPTEN_KEEPALIVE void mono_wasm_load_profiler_aot (const char *desc) { mono_
     </ItemGroup>
 
     <!-- $(WasmResolveAssembliesBeforeBuild) can add more assemblies, so no point checking the count in that case -->
-    <Error Condition="'$(WasmResolveAssembliesBeforeBuild)' != 'true' and @(WasmAssembliesFinal->Distinct()->Count()) != @(WasmAssembliesToBundle->Distinct()->Count())"
+    <Error Condition="'$(WasmResolveAssembliesBeforeBuild)' != 'true' and @(WasmAssembliesFinal->Distinct()->Count()) != @(WasmAssembliesToBundle->Distinct()->Count()) and '$(WasmDedup)' != 'true'"
       Text="Bug: The wasm build started with @(WasmAssembliesToBundle->Count()) assemblies, but completed with @(WasmAssembliesFinal->Count()).
       WasmAssembliesToBundle: @(WasmAssembliesToBundle)
       WasmAssembliesFinal: @(WasmAssembliesFinal)" />

src/tasks/AotCompilerTask/MonoAOTCompiler.cs

@@ -121,6 +121,11 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task
     /// </summary>
     public string? MsymPath { get; set; }
 
+    /// <summary>
+    /// The assembly whose AOT image will contained dedup-ed generic instances
+    /// </summary>
+    public string? DedupAssembly { get; set; }
+
     [Output]
     public string[]? FileWrites { get; private set; }
 
@@ -242,6 +247,7 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths)
         var aotAssembly = new TaskItem(assembly);
         var aotArgs = new List<string>();
         var processArgs = new List<string>();
+        bool isDedup = assembly == DedupAssembly;
 
         var a = assemblyItem.GetMetadata("AotArguments");
         if (a != null)
@@ -274,6 +280,15 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths)
 
         string assemblyFilename = Path.GetFileName(assembly);
 
+        if (isDedup)
+        {
+            aotArgs.Add($"dedup-include={assemblyFilename}");
+        }
+        else if (!string.IsNullOrEmpty (DedupAssembly))
+        {
+            aotArgs.Add("dedup-skip");
+        }
+
         // compute output mode and file names
         if (parsedAotMode == MonoAotMode.LLVMOnly || parsedAotMode == MonoAotMode.AotInterp)
         {
@@ -350,14 +365,43 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths)
         // values, which wont work.
         processArgs.Add($"\"--aot={string.Join(",", aotArgs)}\"");
 
-        processArgs.Add($"\"{assemblyFilename}\"");
+        string paths = "";
+        if (isDedup)
+        {
+            StringBuilder sb = new StringBuilder();
+            HashSet<string> allPaths = new HashSet<string>();
+            foreach (var aItem in Assemblies)
+            {
+                string filename = aItem.ItemSpec;
+                processArgs.Add(filename);
+                string dir = Path.GetDirectoryName(filename)!;
+                if (!allPaths.Contains(dir))
+                {
+                    allPaths.Add(dir);
+                    if (sb.Length > 0)
+                        sb.Append(Path.PathSeparator);
+                    sb.Append(dir);
+                }
+            }
+            if (sb.Length > 0)
+                sb.Append(Path.PathSeparator);
+            sb.Append(monoPaths);
+            paths = sb.ToString();
+        }
+        else
+        {
+            paths = $"{assemblyDir}{Path.PathSeparator}{monoPaths}";
+            processArgs.Add(assemblyFilename);
+        }
 
         var envVariables = new Dictionary<string, string>
         {
-            {"MONO_PATH", $"{assemblyDir}{Path.PathSeparator}{monoPaths}"},
+            {"MONO_PATH", paths},
             {"MONO_ENV_OPTIONS", string.Empty} // we do not want options to be provided out of band to the cross compilers
         };
 
+        Utils.LogInfo ($"MONO_PATH={paths}");
+
         try
         {
             // run the AOT compiler