[wasm] Enable ICU sharding

src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props

@@ -185,8 +185,24 @@
  <PlatformManifestFileEntry Include="icudt_no_CJK.dat" IsNative="true" />
  <PlatformManifestFileEntry Include="icudt_CJK.dat" IsNative="true" />
  <PlatformManifestFileEntry Include="icudt_EFIGS.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icudt_efigs_locales.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icudt_cjk_locales.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icudt_no_cjk_locales.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icudt_normalization.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icudt_base.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icudt_locales.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icudt_currency.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icudt_coll.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icudt_efigs_coll.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icudt_cjk_coll.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icudt_no_cjk_coll.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icudt_zones.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icudt_coll.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icudt_cjk_zones.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icudt_efigs_zones.dat" IsNative="true" />
  <PlatformManifestFileEntry Include="icudt_optimal.dat" IsNative="true" />
  <PlatformManifestFileEntry Include="icudt_optimal_no_CJK.dat" IsNative="true" />
+ <PlatformManifestFileEntry Include="icu_dictionary.json" IsNative="true" />
  <PlatformManifestFileEntry Include="binding_support.js" IsNative="true" />
  <PlatformManifestFileEntry Include="dotnet_support.js" IsNative="true" />
  <PlatformManifestFileEntry Include="library_mono.js" IsNative="true" />

src/libraries/Native/Unix/System.Globalization.Native/pal_icushim_static.c

@@ -47,7 +47,7 @@ static void U_CALLCONV icu_trace_data(const void* context, int32_t fnNumber, int
 #ifdef __EMSCRIPTEN__
 #include <emscripten.h>
 
-static int32_t load_icu_data(void* pData);
+static int32_t load_icu_data(void* pData, int32_t type);
 
 EMSCRIPTEN_KEEPALIVE const char* mono_wasm_get_icudt_name(const char* culture);
 
@@ -56,11 +56,11 @@ EMSCRIPTEN_KEEPALIVE const char* mono_wasm_get_icudt_name(const char* culture)
  return GlobalizationNative_GetICUDTName(culture);
 }
 
-EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void* pData);
+EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void* pData, int32_t type);
 
-EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void* pData)
+EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void* pData, int32_t type)
 {
- return load_icu_data(pData);
+ return load_icu_data(pData, type);
 }
 
 
@@ -77,14 +77,22 @@ void mono_wasm_link_icu_shim(void)
 
 #endif
 
-static int32_t load_icu_data(void* pData)
+static int32_t load_icu_data(void* pData, int32_t type)
 {
 
  UErrorCode status = 0;
- udata_setCommonData(pData, &status);
+ if (type == 0) {
+ udata_setAppData(NULL, pData, &status);
+ }
+ if (type == 1) {
+ udata_setCommonData(pData, &status);
+ }
 
  if (U_FAILURE(status)) {
- log_icu_error("udata_setCommonData", status);
+ if (type)
+ log_icu_error("udata_setCommonData", status);
+ else
+ log_icu_error("udata_setAppData", status);
  return 0;
  } else {
 
@@ -146,7 +154,7 @@ int32_t GlobalizationNative_LoadICUData(const char* path)
 
  fclose(fp);
 
- if (load_icu_data(icu_data) == 0) {
+ if (load_icu_data(icu_data, strcasecmp("icudt.dat", path)) == 0) {
  log_shim_error("ICU BAD EXIT %d.", ret);
  return ret;
  }

src/mono/sample/wasm/wasm.mk

@@ -23,8 +23,8 @@ run-browser:
  echo "The tool dotnet-serve could not be found. Install with: $(DOTNET) tool install --global dotnet-serve"; \
  exit 1; \
  else \
- $(DOTNET) serve -d bin/$(CONFIG)/AppBundle -p 8000; \
+ $(DOTNET) serve --directory ./bin/$(CONFIG)/AppBundle -p 8000; \
  fi
 
 run-console:
- cd bin/$(CONFIG)/AppBundle && ~/.jsvu/v8 --expose_wasm runtime.js -- $(DOTNET_MONO_LOG_LEVEL) --run Wasm.Console.Sample.dll
+ cd bin/$(CONFIG)/AppBundle && ~/.jsvu/v8 --expose_wasm runtime.js -- $(DOTNET_MONO_LOG_LEVEL) --enable-sharding=true --run Wasm.Console.Sample.dll

src/mono/wasm/build/WasmApp.targets

@@ -360,18 +360,17 @@
  <Error Condition="'$(WasmMainJSPath)' == ''" Text="%24(WasmMainJSPath) property needs to be set" />
 
  <PropertyGroup>
- <WasmIcuDataFileName Condition="'$(InvariantGlobalization)' != 'true'">icudt.dat</WasmIcuDataFileName>
-
  <_HasDotnetWasm Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.wasm'">true</_HasDotnetWasm>
  <_HasDotnetJs Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.js'">true</_HasDotnetJs>
+ <IcuDictionaryPath Condition="'$(InvariantGlobalization)' != 'true'">$(MicrosoftNetCoreAppRuntimePackRidNativeDir)icu_dictionary.json</IcuDictionaryPath>
  </PropertyGroup>
 
  <ItemGroup>
  <!-- If dotnet.{wasm,js} weren't added already (eg. AOT can add them), then add the default ones -->
  <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)dotnet.wasm" Condition="'$(_HasDotnetWasm)' != 'true'" />
  <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)dotnet.js" Condition="'$(_HasDotnetJs)' != 'true'" />
 
- <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)$(WasmIcuDataFileName)" Condition="'$(InvariantGlobalization)' != 'true'" />
+ <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)*.dat" Condition="'$(InvariantGlobalization)' != 'true'" />
  <WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)dotnet.timezones.blat" />
  </ItemGroup>
 
@@ -382,7 +381,7 @@
  InvariantGlobalization="$(InvariantGlobalization)"
  SatelliteAssemblies="@(WasmSatelliteAssemblies)"
  FilesToIncludeInFileSystem="@(WasmFilesToIncludeInFileSystem)"
- IcuDataFileName="$(WasmIcuDataFileName)"
+ IcuDictionary="$(IcuDictionaryPath)"
  RemoteSources="@(WasmRemoteSources)"
  ExtraFilesToDeploy="@(WasmExtraFilesToDeploy)"
  ExtraConfig="@(WasmExtraConfig)"

src/mono/wasm/runtime-test.js

@@ -2,8 +2,8 @@
 //
 // Run runtime tests under a JS shell or a browser
 //
-
 //glue code to deal with the differences between chrome, ch, d8, jsc and sm.
+
 var is_browser = typeof window != "undefined";
 
 // if the engine doesn't provide a console
@@ -151,6 +151,7 @@ setenv = {};
 runtime_args = [];
 enable_gc = true;
 enable_zoneinfo = false;
+enable_sharding = false;
 working_dir='/';
 while (args !== undefined && args.length > 0) {
  if (args [0].startsWith ("--profile=")) {
@@ -177,6 +178,10 @@ while (args !== undefined && args.length > 0) {
  var arg = args [0].substring ("--working-dir=".length);
  working_dir = arg;
  args = args.slice (1);
+ } else if (args [0].startsWith ("--enable-sharding=")) {
+ var arg = args [0].substring ("--enable-sharding=".length);
+ enable_sharding = arg;
+ args = args.slice (1);
  } else {
  break;
  }
@@ -279,6 +284,8 @@ var Module = {
  })
  }
  };
+ if (enable_sharding)
+ config.application_culture = Intl.DateTimeFormat().resolvedOptions().locale;
 
  MONO.mono_load_runtime_and_bcl_args (config);
  },

src/mono/wasm/runtime/library_mono.js

@@ -1135,6 +1135,38 @@ var MonoSupportLib = {
  return res;
  },
 
+ _get_shard_name: function (fileList) {
+ var name_lengths = fileList.map(x => x.split("_").length);
+ var shard_ix = name_lengths.indexOf(Math.max(...name_lengths));
+ return fileList[shard_ix];
+ },
+
+ _get_list_of_icu_files: function (dictionary, culture, feature_shards=true) {
+ var icu_files = []
+ if (culture == null || culture.length < 2) {
+ icu_files = [dictionary.complete];
+ } else {
+ var parent_culture = culture.split('-')[0];
+ var files = dictionary[parent_culture];
+ if (!feature_shards) {
+ icu_files = [files.full];
+ } else {
+ icu_files = files.essentials;
+ icu_files.push(this._get_shard_name(files.coll));
+ icu_files.push(this._get_shard_name(files.locales));
+ }
+ }
+
+ icu_assets = [];
+ icu_files.forEach(file => icu_assets.push({
+ "behavior": "icu",
+ "name": file,
+ "load_remote": false,
+ "data_type": "common"
+ }));
+ return icu_assets;
+ },
+
  mono_wasm_get_details: function (objectId, args={}) {
  let id = this._parse_object_id (objectId, true);
 
@@ -1581,7 +1613,7 @@ var MonoSupportLib = {
  }
  }
  else if (asset.behavior === "icu") {
- if (this.mono_wasm_load_icu_data (offset))
+ if (this.mono_wasm_load_icu_data (offset, +(asset.data_type == "common")))
  ctx.num_icu_assets_loaded_successfully += 1;
  else
  console.error ("Error loading ICU asset", asset.name);
@@ -1660,6 +1692,7 @@ var MonoSupportLib = {
  // "icu": load ICU globalization data from any runtime assets with behavior "icu".
  // "invariant": operate in invariant globalization mode.
  // "auto" (default): if "icu" behavior assets are present, use ICU, otherwise invariant.
+ // application_culture: (optional) current browser culture
  // diagnostic_tracing: (optional) enables diagnostic log messages during startup
  mono_load_runtime_and_bcl_args: function (args) {
  try {
@@ -1683,9 +1716,9 @@ var MonoSupportLib = {
 
  // @offset must be the address of an ICU data archive in the native heap.
  // returns true on success.
- mono_wasm_load_icu_data: function (offset) {
- var fn = Module.cwrap ('mono_wasm_load_icu_data', 'number', ['number']);
- var ok = (fn (offset)) === 1;
+ mono_wasm_load_icu_data: function (offset, type) {
+ var fn = Module.cwrap ('mono_wasm_load_icu_data', 'number', ['number', 'number']);
+ var ok = (fn (offset, type)) === 1;
  if (ok)
  this.num_icu_assets_loaded_successfully++;
  return ok;
@@ -1751,7 +1784,8 @@ var MonoSupportLib = {
  throw new Error ("Invalid args (runtime_asset_sources was replaced by remote_sources)");
  if (!args.loaded_cb)
  throw new Error ("loaded_cb not provided");
-
+
+ args.assets = args.assets.concat(this._get_list_of_icu_files(args.icu_dictionary, args.application_culture));
  var ctx = {
  tracing: args.diagnostic_tracing || false,
  pending_count: args.assets.length,
@@ -1799,7 +1833,6 @@ var MonoSupportLib = {
  var attemptNextSource;
  var sourceIndex = 0;
  var sourcesList = asset.load_remote ? args.remote_sources : [""];
-
  var handleFetchResponse = function (response) {
  if (!response.ok) {
  try {

src/tasks/WasmAppBuilder/WasmAppBuilder.cs

@@ -41,7 +41,7 @@ public class WasmAppBuilder : Task
 
  // full list of ICU data files we produce can be found here:
  // https://github.com/dotnet/icu/tree/maint/maint-67/icu-filters
- public string? IcuDataFileName { get; set; }
+ public string? IcuDictionary { get; set; }
 
  public int DebugLevel { get; set; }
  public ITaskItem[]? SatelliteAssemblies { get; set; }
@@ -119,14 +119,16 @@ private class IcuData : AssetEntry
  public IcuData(string name) : base(name, "icu") {}
  [JsonPropertyName("load_remote")]
  public bool LoadRemote { get; set; }
+ [JsonPropertyName("data_type")]
+ public string? DataType { get; set; }
  }
 
  public override bool Execute ()
  {
  if (!File.Exists(MainJS))
  throw new ArgumentException($"File MainJS='{MainJS}' doesn't exist.");
- if (!InvariantGlobalization && string.IsNullOrEmpty(IcuDataFileName))
- throw new ArgumentException("IcuDataFileName property shouldn't be empty if InvariantGlobalization=false");
+ if (!InvariantGlobalization && string.IsNullOrEmpty(IcuDictionary))
+ throw new ArgumentException("IcuDictionary property shouldn't be empty if InvariantGlobalization=false");
 
  if (Assemblies?.Length == 0)
  {
@@ -225,9 +227,6 @@ public override bool Execute ()
  }
  }
 
- if (!InvariantGlobalization)
- config.Assets.Add(new IcuData(IcuDataFileName!) { LoadRemote = RemoteSources?.Length > 0 });
-
  config.Assets.Add(new VfsEntry ("dotnet.timezones.blat") { VirtualPath = "/usr/share/zoneinfo/"});
 
  if (RemoteSources?.Length > 0)
@@ -246,6 +245,12 @@ public override bool Execute ()
  config.Extra[name] = valueObject;
  }
 
+ if (!InvariantGlobalization)
+ {
+ string? icuDictionary = File.ReadAllText(IcuDictionary!);
+ config.Extra["icu_dictionary"] = JsonSerializer.Deserialize<Dictionary<string, object>>(icuDictionary!);
+ }
+
  string monoConfigPath = Path.Join(AppDir, "mono-config.js");
  using (var sw = File.CreateText(monoConfigPath))
  {