[browser][coreclr] ICU download

src/coreclr/vm/wasm/callhelpers-reverse.cpp

@@ -46,10 +46,25 @@ extern "C" void SystemJS_ExecuteTimerCallback()
     Call_System_Private_CoreLib_System_Threading_TimerQueue_TimerHandler();
 }
 
+static MethodDesc* MD_CalendarData_EnumCalendarInfoCallback = nullptr;
+extern "C" void Call_CalendarData_EnumCalendarInfoCallback(void* arg0, void* arg1)
+{
+    // Lazy lookup of MethodDesc for the function export scenario.
+    if (!MD_CalendarData_EnumCalendarInfoCallback)
+    {
+        LookupMethodByName("System.Globalization.CalendarData, System.Private.CoreLib", "EnumCalendarInfoCallback", &MD_CalendarData_EnumCalendarInfoCallback);
+    }
+
+    int64_t args[2] = { (int64_t)arg0, (int64_t)arg1 };
+
+    ExecuteInterpretedMethodFromUnmanaged(MD_CalendarData_EnumCalendarInfoCallback, (int8_t*)args, sizeof(args), nullptr, (PCODE)&Call_CalendarData_EnumCalendarInfoCallback);
+}
+
 extern const ReverseThunkMapEntry g_ReverseThunks[] =
 {
     { 100678287, { &MD_System_Private_CoreLib_System_Threading_ThreadPool_BackgroundJobHandler_Void_RetVoid, (void*)&Call_System_Private_CoreLib_System_Threading_ThreadPool_BackgroundJobHandler } },
     { 100678363, { &MD_System_Private_CoreLib_System_Threading_TimerQueue_TimerHandler_Void_RetVoid, (void*)&Call_System_Private_CoreLib_System_Threading_TimerQueue_TimerHandler } },
+    { 2638826848, { &MD_CalendarData_EnumCalendarInfoCallback, (void*)&Call_CalendarData_EnumCalendarInfoCallback } },
 };
 
 const size_t g_ReverseThunksCount = sizeof(g_ReverseThunks) / sizeof(g_ReverseThunks[0]);

src/libraries/Common/src/Interop/Browser/Interop.Locale.CoreCLR.cs

@@ -0,0 +1,14 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Runtime.InteropServices;
+
+internal static partial class Interop
+{
+    internal static unsafe partial class JsGlobalization
+    {
+        [LibraryImport(Libraries.JavaScriptNative, EntryPoint = "SystemJS_GetLocaleInfo")]
+        public static unsafe partial nint GetLocaleInfo(char* locale, int localeLength, char* culture, int cultureLength, char* buffer, int bufferLength, out int resultLength);
+    }
+}

src/libraries/Common/src/Interop/Unix/Interop.Libraries.cs

@@ -15,5 +15,6 @@ internal static partial class Libraries
         internal const string GlobalizationNative = "libSystem.Globalization.Native";
         internal const string IOPortsNative = "libSystem.IO.Ports.Native";
         internal const string HostPolicy = "libhostpolicy";
+        internal const string JavaScriptNative = "libSystem.JavaScript.Native";
     }
 }

src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems

@@ -1375,7 +1375,10 @@
     <Compile Include="$(MSBuildThisFileDirectory)System\WeakReference.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\WeakReference.T.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\ComAwareWeakReference.cs" />
-    <Compile Include="$(CommonPath)Interop\Browser\Interop.Locale.cs" Condition="'$(TargetsBrowser)' == 'true'">
+    <Compile Include="$(CommonPath)Interop\Browser\Interop.Locale.CoreCLR.cs" Condition="'$(TargetsBrowser)' == 'true' and '$(RuntimeFlavor)' == 'CoreCLR'">
+      <Link>Common\Interop\Browser\Interop.Locale.CoreCLR.cs</Link>
+    </Compile>
+    <Compile Include="$(CommonPath)Interop\Browser\Interop.Locale.cs" Condition="'$(TargetsBrowser)' == 'true' and '$(RuntimeFlavor)' == 'Mono'">
       <Link>Common\Interop\Browser\Interop.Locale.cs</Link>
     </Compile>
     <Compile Include="$(CommonPath)Interop\Interop.Calendar.cs">

src/mono/browser/runtime/assets.ts

@@ -4,7 +4,7 @@
 import type { AssetEntryInternal } from "./types/internal";
 
 import cwraps from "./cwraps";
-import { mono_wasm_load_icu_data } from "./icu";
+import { wasm_load_icu_data } from "./icu";
 import { Module, loaderHelpers, mono_assert, runtimeHelpers } from "./globals";
 import { mono_log_info, mono_log_debug, parseSymbolMapFile } from "./logging";
 import { mono_wasm_load_bytes_into_heap_persistent } from "./memory";
@@ -87,7 +87,7 @@ export function instantiate_asset (asset: AssetEntry, url: string, bytes: Uint8A
     } else if (asset.behavior === "pdb") {
         cwraps.mono_wasm_add_assembly(virtualName, offset!, bytes.length);
     } else if (asset.behavior === "icu") {
-        mono_wasm_load_icu_data(offset!);
+        wasm_load_icu_data(offset!);
     } else if (asset.behavior === "resource") {
         cwraps.mono_wasm_add_satellite_assembly(virtualName, asset.culture || "", offset!, bytes.length);
     }

src/mono/browser/runtime/cwraps.ts

@@ -43,7 +43,7 @@ const fn_signatures: SigLine[] = [
     [true, "mono_background_exec", null, []],
     [true, "mono_wasm_ds_exec", null, []],
     [true, "mono_wasm_execute_timer", null, []],
-    [true, "mono_wasm_load_icu_data", "number", ["number"]],
+    [true, "wasm_load_icu_data", "number", ["number"]],
     [false, "mono_wasm_add_assembly", "number", ["string", "number", "number"]],
     [true, "mono_wasm_add_satellite_assembly", "void", ["string", "string", "number", "number"]],
     [false, "mono_wasm_load_runtime", null, ["number", "number", "number", "number"]],
@@ -170,7 +170,7 @@ export interface t_Cwraps {
     mono_background_exec(): void;
     mono_wasm_ds_exec(): void;
     mono_wasm_execute_timer(): void;
-    mono_wasm_load_icu_data(offset: VoidPtr): number;
+    wasm_load_icu_data(offset: VoidPtr): number;
     mono_wasm_add_assembly(name: string, data: VoidPtr, size: number): number;
     mono_wasm_add_satellite_assembly(name: string, culture: string, data: VoidPtr, size: number): void;
     mono_wasm_load_runtime(debugLevel: number, propertyCount:number, propertyKeys:CharPtrPtr, propertyValues:CharPtrPtr): void;

src/mono/browser/runtime/icu.ts

@@ -4,8 +4,8 @@
 import cwraps from "./cwraps";
 import { VoidPtr } from "./types/emscripten";
 
-export function mono_wasm_load_icu_data (offset: VoidPtr) {
-    if (!cwraps.mono_wasm_load_icu_data(offset)) {
+export function wasm_load_icu_data (offset: VoidPtr) {
+    if (!cwraps.wasm_load_icu_data(offset)) {
         throw new Error("Failed to load ICU data");
     }
 }

src/native/corehost/browserhost/host/cross-linked.ts

@@ -2,8 +2,10 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 import { } from "../../../libs/Common/JavaScript/cross-linked";
+import type { VoidPtr } from "./types";
 
 declare global {
     export const BROWSER_HOST: any;
     export function _BrowserHost_ExecuteAssembly(mainAssemblyNamePtr: number, argsLength: number, argsPtr: number): number;
+    export function _wasm_load_icu_data(dataPtr: VoidPtr): number;
 }

src/native/corehost/browserhost/host/host.ts

@@ -24,6 +24,27 @@ export function registerDllBytes(bytes: Uint8Array, asset: { name: string, virtu
     }
 }
 
+export function loadIcuData(bytes: Uint8Array) {
+    const sp = Module.stackSave();
+    try {
+        const sizeOfPtr = 4;
+        const ptrPtr = Module.stackAlloc(sizeOfPtr);
+        if (Module._posix_memalign(ptrPtr as any, 16, bytes.length)) {
+            throw new Error("posix_memalign failed for ICU data");
+        }
+
+        const ptr = Module.HEAPU32[ptrPtr as any >>> 2];
+        Module.HEAPU8.set(bytes, ptr >>> 0);
+
+        const result = _wasm_load_icu_data(ptr as unknown as VoidPtr);
+        if (!result) {
+            throw new Error("Failed to initialize ICU data");
+        }
+    } finally {
+        Module.stackRestore(sp);
+    }
+}
+
 export function installVfsFile(bytes: Uint8Array, asset: VfsAsset) {
     const virtualName: string = typeof (asset.virtualPath) === "string"
         ? asset.virtualPath

src/native/corehost/browserhost/host/index.ts

@@ -5,7 +5,7 @@ import type { InternalExchange, BrowserHostExports, RuntimeAPI, BrowserHostExpor
 import { InternalExchangeIndex } from "./types";
 import { } from "./cross-linked"; // ensure ambient symbols are declared
 
-import { runMain, runMainAndExit, registerDllBytes, installVfsFile } from "./host";
+import { runMain, runMainAndExit, registerDllBytes, installVfsFile, loadIcuData } from "./host";
 
 export function dotnetInitializeModule(internals: InternalExchange): void {
     if (!Array.isArray(internals)) throw new Error("Expected internals to be an array");
@@ -20,13 +20,15 @@ export function dotnetInitializeModule(internals: InternalExchange): void {
     internals[InternalExchangeIndex.BrowserHostExportsTable] = browserHostExportsToTable({
         registerDllBytes,
         installVfsFile,
+        loadIcuData
     });
     dotnetUpdateInternals(internals, dotnetUpdateInternalsSubscriber);
     function browserHostExportsToTable(map: BrowserHostExports): BrowserHostExportsTable {
         // keep in sync with browserHostExportsFromTable()
         return [
             map.registerDllBytes,
             map.installVfsFile,
+            map.loadIcuData,
         ];
     }
 }

src/native/corehost/browserhost/libBrowserHost.footer.js

@@ -19,7 +19,7 @@
         const exports = {};
         libBrowserHost(exports);
 
-        let commonDeps = ["$libBrowserHostFn", "$DOTNET", "$DOTNET_INTEROP", "$ENV", "$FS", "$NODEFS"];
+        let commonDeps = ["$libBrowserHostFn", "$DOTNET", "$DOTNET_INTEROP", "$ENV", "$FS", "$NODEFS", "wasm_load_icu_data"];
         const lib = {
             $BROWSER_HOST: {
                 selfInitialize: () => {
@@ -52,9 +52,6 @@
                         ENV[HOST_PROPERTY_APP_PATHS] = config.environmentVariables[HOST_PROPERTY_APP_PATHS] = config.virtualWorkingDirectory;
                         ENV[HOST_PROPERTY_ENTRY_ASSEMBLY_NAME] = config.environmentVariables[HOST_PROPERTY_ENTRY_ASSEMBLY_NAME] = config.mainAssemblyName;
 
-                        // WASM-TODO: remove once globalization is loaded via ICU
-                        ENV["DOTNET_SYSTEM_GLOBALIZATION_INVARIANT"] = "true";
-
                         if (ENVIRONMENT_IS_NODE) {
                             Module.preInit = [() => {
                                 FS.mkdir("/managed");

src/native/corehost/browserhost/loader/bootstrap.ts

@@ -2,9 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 import type { LoadBootResourceCallback, JsModuleExports, JsAsset, AssemblyAsset, PdbAsset, WasmAsset, IcuAsset, EmscriptenModuleInternal, LoaderConfig, DotnetHostBuilder } from "./types";
-
 import { dotnetAssert, dotnetGetInternals, dotnetBrowserHostExports, dotnetUpdateInternals } from "./cross-module";
 import { ENVIRONMENT_IS_NODE, ENVIRONMENT_IS_SHELL } from "./per-module";
+import { getIcuResourceName } from "./icu";
 import { getLoaderConfig } from "./config";
 import { BrowserHost_InitializeCoreCLR } from "./run";
 import { createPromiseCompletionSource } from "./promise-completion-source";
@@ -22,7 +22,6 @@ const nativeModulePromiseController = createPromiseCompletionSource<EmscriptenMo
 
 // WASM-TODO: retry logic
 // WASM-TODO: throttling logic
-// WASM-TODO: load icu data
 // WASM-TODO: invokeLibraryInitializers
 // WASM-TODO: webCIL
 // WASM-TODO: downloadOnly - blazor render mode auto pre-download. Really no start.
@@ -37,6 +36,8 @@ export async function createRuntime(downloadOnly: boolean, loadBootResource?: Lo
     const coreVfsPromise = Promise.all((config.resources.coreVfs || []).map(fetchVfs));
     const assembliesPromise = Promise.all(config.resources.assembly.map(fetchDll));
     const vfsPromise = Promise.all((config.resources.vfs || []).map(fetchVfs));
+    const icuResourceName = getIcuResourceName(config);
+    const icuDataPromise = icuResourceName ? Promise.all((config.resources.icu || []).filter(asset => asset.name === icuResourceName).map(fetchIcu)) : Promise.resolve([]);
     // WASM-TODO fetchWasm(config.resources.wasmNative[0]);// start loading early, no await
 
     const nativeModule = await nativeModulePromise;
@@ -50,7 +51,7 @@ export async function createRuntime(downloadOnly: boolean, loadBootResource?: Lo
     await coreAssembliesPromise;
     await coreVfsPromise;
     await vfsPromise;
-
+    await icuDataPromise;
     if (!downloadOnly) {
         BrowserHost_InitializeCoreCLR();
     }
@@ -68,6 +69,15 @@ async function loadJSModule(asset: JsAsset, loadBootResource?: LoadBootResourceC
     return await import(/* webpackIgnore: true */ asset.resolvedUrl);
 }
 
+async function fetchIcu(asset: IcuAsset): Promise<void> {
+    if (asset.name && !asset.resolvedUrl) {
+        asset.resolvedUrl = locateFile(asset.name);
+    }
+    const bytes = await fetchBytes(asset);
+    await nativeModulePromiseController.promise;
+    dotnetBrowserHostExports.loadIcuData(bytes);
+}
+
 async function fetchDll(asset: AssemblyAsset): Promise<void> {
     if (asset.name && !asset.resolvedUrl) {
         asset.resolvedUrl = locateFile(asset.name);

src/native/corehost/browserhost/loader/icu.ts

@@ -0,0 +1,50 @@
+import type { LoaderConfig } from "./types";
+import { GlobalizationMode } from "./types";
+import { ENVIRONMENT_IS_WEB } from "./per-module";
+
+export function getIcuResourceName(config: LoaderConfig): string | null {
+    if (config.resources?.icu && config.globalizationMode != GlobalizationMode.Invariant) {
+        const culture = config.applicationCulture || (ENVIRONMENT_IS_WEB ? (globalThis.navigator && globalThis.navigator.languages && globalThis.navigator.languages[0]) : Intl.DateTimeFormat().resolvedOptions().locale);
+        if (!config.applicationCulture) {
+            config.applicationCulture = culture;
+        }
+
+        const icuFiles = config.resources.icu;
+        let icuFile = null;
+        if (config.globalizationMode === GlobalizationMode.Custom) {
+            // custom ICU file is saved in the resources with fingerprinting and does not require mapping
+            if (icuFiles.length >= 1) {
+                return icuFiles[0].name;
+            }
+        } else if (!culture || config.globalizationMode === GlobalizationMode.All) {
+            icuFile = "icudt.dat";
+        } else if (config.globalizationMode === GlobalizationMode.Sharded) {
+            icuFile = getShardedIcuResourceName(culture);
+        }
+
+        if (icuFile) {
+            for (let i = 0; i < icuFiles.length; i++) {
+                const asset = icuFiles[i];
+                if (asset.virtualPath === icuFile) {
+                    return asset.name;
+                }
+            }
+        }
+    }
+
+    config.globalizationMode = GlobalizationMode.Invariant;
+    return null;
+}
+
+function getShardedIcuResourceName(culture: string): string {
+    const prefix = culture.split("-")[0];
+    if (prefix === "en" || ["fr", "fr-FR", "it", "it-IT", "de", "de-DE", "es", "es-ES"].includes(culture)) {
+        return "icudt_EFIGS.dat";
+    }
+
+    if (["zh", "ko", "ja"].includes(prefix)) {
+        return "icudt_CJK.dat";
+    }
+
+    return "icudt_no_CJK.dat";
+}

src/native/libs/Common/JavaScript/cross-module/index.ts

@@ -129,6 +129,7 @@ export function dotnetUpdateInternalsSubscriber() {
         const nativeLocal: BrowserHostExports = {
             registerDllBytes: table[0],
             installVfsFile: table[1],
+            loadIcuData: table[2],
         };
         Object.assign(native, nativeLocal);
     }

src/native/libs/Common/JavaScript/types/exchange.ts

@@ -1,7 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-import type { installVfsFile, registerDllBytes } from "../../../../corehost/browserhost/host/host";
+import type { installVfsFile, registerDllBytes, loadIcuData } from "../../../../corehost/browserhost/host/host";
 import type { check, error, info, warn, debug } from "../../../../corehost/browserhost/loader/logging";
 import type { createPromiseCompletionSource, getPromiseCompletionSource, isControllablePromise } from "../../../../corehost/browserhost/loader/promise-completion-source";
 import type { resolveRunMainPromise, rejectRunMainPromise, getRunMainPromise } from "../../../../corehost/browserhost/loader/run";
@@ -51,11 +51,13 @@ export type LoaderExportsTable = [
 export type BrowserHostExports = {
     registerDllBytes: typeof registerDllBytes
     installVfsFile: typeof installVfsFile
+    loadIcuData: typeof loadIcuData
 }
 
 export type BrowserHostExportsTable = [
     typeof registerDllBytes,
     typeof installVfsFile,
+    typeof loadIcuData,
 ]
 
 export type InteropJavaScriptExports = {

src/native/libs/System.Globalization.Native/pal_icushim_static.c

@@ -55,13 +55,12 @@ static int32_t load_icu_data(const void* pData);
 #ifdef __EMSCRIPTEN__
 #include <emscripten.h>
 
-EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(const void* pData);
+EMSCRIPTEN_KEEPALIVE int32_t wasm_load_icu_data(const void* pData);
 
-EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(const void* pData)
+EMSCRIPTEN_KEEPALIVE int32_t wasm_load_icu_data(const void* pData)
 {
     return load_icu_data(pData);
 }
-
 /*
  * driver.c calls this to make sure this file is linked, otherwise
  * its not, meaning the EMSCRIPTEN_KEEPALIVE functions above
@@ -72,7 +71,6 @@ void mono_wasm_link_icu_shim(void);
 void mono_wasm_link_icu_shim(void)
 {
 }
-
 #endif
 
 int32_t mono_wasi_load_icu_data(const void* pData);
@@ -210,7 +208,7 @@ int32_t GlobalizationNative_LoadICU(void)
 // GlobalizationNative_LoadICUData() as entrypoint
     if (!isDataSet)
     {
-        // don't try to locate icudt.dat automatically if mono_wasm_load_icu_data wasn't called
+        // don't try to locate icudt.dat automatically if wasm_load_icu_data wasn't called
         // and fallback to invariant mode
         return 0;
     }