From 310aa3e266d61bed8d1ea6c8c56a1d13cf9f9fe7 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 4 May 2023 16:08:22 -0400 Subject: [PATCH 01/30] Add option to emit webcil inside a wasm module wrapper --- .../src/Webcil/WebcilConverter.cs | 23 ++- .../src/Webcil/WebcilWasmWrapper.cs | 166 ++++++++++++++++++ 2 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs index 28a8d0e8a35b8..f2fb9ae130a49 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs @@ -42,6 +42,8 @@ FilePosition SectionStart private string InputPath => _inputPath; + public bool WrapInWebAssembly { get; set; } = true; + private WebcilConverter(string inputPath, string outputPath) { _inputPath = inputPath; @@ -62,6 +64,25 @@ public void ConvertToWebcil() } using var outputStream = File.Open(_outputPath, FileMode.Create, FileAccess.Write); + if (!WrapInWebAssembly) + { + WriteConversionTo(outputStream, inputStream, peInfo, wcInfo); + } + else + { + // if wrapping in WASM, write the webcil payload to memory because we need to discover the length + + // webcil is about the same size as the PE file + using var memoryStream = new MemoryStream (checked((int)inputStream.Length)); + WriteConversionTo(memoryStream, inputStream, peInfo, wcInfo); + var wrapper = new WebcilWasmWrapper (memoryStream); + memoryStream.Seek (0, SeekOrigin.Begin); + wrapper.WriteWasmWrappedWebcil(outputStream); + } + } + + public void WriteConversionTo(Stream outputStream, FileStream inputStream, PEFileInfo peInfo, WCFileInfo wcInfo) + { WriteHeader(outputStream, wcInfo.Header); WriteSectionHeaders(outputStream, wcInfo.SectionHeaders); CopySections(outputStream, inputStream, peInfo.SectionHeaders); @@ -210,7 +231,7 @@ private static void WriteStructure(Stream s, T structure) } #endif - private static void CopySections(FileStream outStream, FileStream inputStream, ImmutableArray peSections) + private static void CopySections(Stream outStream, FileStream inputStream, ImmutableArray peSections) { // endianness: ok, we're just copying from one stream to another foreach (var peHeader in peSections) diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs new file mode 100644 index 0000000000000..a0e23b44025db --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs @@ -0,0 +1,166 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Collections.Immutable; +using System.Reflection.PortableExecutable; +using System.Runtime.InteropServices; + +namespace Microsoft.NET.WebAssembly.Webcil; + +// +// This is a pretty dumb wrapper. It assumes that the entire wasm module is going to be unchanging, +// except for the data section which has 2 passive segments. segment 0 is 4 bytes and contains the +// length of the webcil payload. segment 1 is of a variable size and contains the webcil payload. +// +// the unchanging parts are stored as a "prefix" and "suffix" which contain the bytes for the following +// WAT program, split into the parts that come before the data section, and the bytes that come after: +// +// (module +// (data "\0f\00\00\00") ;; data segment 0: payload size as a 4 byte LE uint32 +// (data "webcil Payload\cc") ;; data segment 1: webcil payload +// (memory (import "webcil" "memory") 1) +// (global (export "webcilVersion") i32 (i32.const 0)) +// (func (export "getWebcilSize") (param $destPtr i32) (result) +// local.get $destPtr +// i32.const 0 +// i32.const 4 +// memory.init 0) +// (func (export "getWebcilPayload") (param $d i32) (param $n i32) (result) +// local.get $d +// i32.const 0 +// local.get $n +// memory.init 1)) +public class WebcilWasmWrapper +{ + private readonly Stream _webcilPayloadStream; + private readonly uint _webcilPayloadSize; + + public WebcilWasmWrapper (Stream webcilPayloadStream) + { + _webcilPayloadStream = webcilPayloadStream; + long len = webcilPayloadStream.Length; + if (len > (long)uint.MaxValue) + throw new InvalidOperationException("webcil payload too large"); + _webcilPayloadSize = (uint)len; + } + + public void WriteWasmWrappedWebcil(Stream outputStream) + { + WriteWasmHeader(outputStream); + using (var writer = new BinaryWriter(outputStream, System.Text.Encoding.UTF8, leaveOpen: true)) + { + WriteDataSection (writer); + } + WriteWasmSuffix(outputStream); + } + + // + // Everything from the above wat module before the data section + // + // extracted by wasm-reader -s wrapper.wasm + private static +#if NET7_0_OR_GREATER + ReadOnlyMemory +#else + byte[] +#endif + s_wasmWrapperPrefix = new byte[] { + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x0a, 0x02, 0x60, 0x01, 0x7f, 0x00, 0x60, 0x02, 0x7f, 0x7f, 0x00, 0x02, 0x12, 0x01, 0x06, 0x77, 0x65, 0x62, 0x63, 0x69, 0x6c, 0x06, 0x6d, + 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x02, 0x00, 0x01, 0x03, 0x03, 0x02, 0x00, 0x01, 0x06, 0x0b, 0x02, 0x7f, 0x00, 0x41, 0x00, 0x0b, 0x7f, 0x00, 0x41, 0x00, 0x0b, 0x07, 0x41, 0x04, 0x0d, 0x77, 0x65, + 0x62, 0x63, 0x69, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x03, 0x00, 0x0a, 0x77, 0x65, 0x62, 0x63, 0x69, 0x6c, 0x53, 0x69, 0x7a, 0x65, 0x03, 0x01, 0x0d, 0x67, 0x65, 0x74, 0x57, 0x65, + 0x62, 0x63, 0x69, 0x6c, 0x53, 0x69, 0x7a, 0x65, 0x00, 0x00, 0x10, 0x67, 0x65, 0x74, 0x57, 0x65, 0x62, 0x63, 0x69, 0x6c, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x00, 0x01, 0x0c, 0x01, 0x02, + 0x0a, 0x1b, 0x02, 0x0c, 0x00, 0x20, 0x00, 0x41, 0x00, 0x41, 0x04, 0xfc, 0x08, 0x00, 0x00, 0x0b, 0x0c, 0x00, 0x20, 0x00, 0x41, 0x00, 0x20, 0x01, 0xfc, 0x08, 0x01, 0x00, 0x0b, 0x0b, 0x1b, + }; + // + // Everything from the above wat module after the data section + // + // extracted by wasm-reader -s wrapper.wasm + private static +#if NET7_0_OR_GREATER + ReadOnlyMemory +#else + byte[] +#endif + s_wasmWrapperSuffix = new byte[] { + 0x00, 0x1b, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x02, 0x14, 0x02, 0x00, 0x01, 0x00, 0x07, 0x64, 0x65, 0x73, 0x74, 0x50, 0x74, 0x72, 0x01, 0x02, 0x00, 0x01, 0x64, 0x01, 0x01, 0x6e, + }; + + private static void WriteWasmHeader(Stream outputStream) + { +#if NET7_0_OR_GREATER + outputStream.Write(s_wasmWrapperPrefix.Span); +#else + outputStream.Write(s_wasmWrapperPrefix, 0, s_wasmWrapperPrefix.Length); +#endif + } + + private static void WriteWasmSuffix(Stream outputStream) + { +#if NET7_0_OR_GREATER + outputStream.Write(s_wasmWrapperSuffix.Span); +#else + outputStream.Write(s_wasmWrapperSuffix, 0, s_wasmWrapperSuffix.Length); +#endif + } + + // 1 byte to encode "passive" data segment + private const uint SegmentCodeSize = 1; + + private void WriteDataSection(BinaryWriter writer) + { + uint dataSectionSize = 0; + // compute the segment 0 size: + // segment 0 has 1 byte segment code, 1 byte of size and 4 bytes of payload + dataSectionSize += SegmentCodeSize + 1 + 4; + + // encode webcil size as a uleb128 + byte[] ulebSegmentSize = ULEB128Encode(_webcilPayloadSize); + + // compute the segment 1 size: + // segment 1 has 1 byte segment code, a uleb128 encoding of the webcilPayloadSize, and the payload + checked + { + dataSectionSize += SegmentCodeSize + (uint)ulebSegmentSize.Length + _webcilPayloadSize; + } + + byte[] ulebSectionSize = ULEB128Encode(dataSectionSize); + + writer.Write((byte)11); // section Data + writer.Write(ulebSectionSize, 0, ulebSectionSize.Length); + + // write segment 0 + writer.Write((byte)1); // passive segment + writer.Write((byte)4); // segment size: 4 + writer.Write((uint)_webcilPayloadSize); // payload is an unsigned 32 bit number + + // write segment 1 + writer.Write((byte)1); // passive segment + writer.Write(ulebSegmentSize, 0, ulebSegmentSize.Length); // segment size: _webcilPayloadSize + _webcilPayloadStream.CopyTo(writer.BaseStream); // payload is the entire webcil content + } + + private static byte[] ULEB128Encode(uint value) + { + uint n = value; + int len = 0; + do + { + n >>= 7; + len++; + } while (n != 0); + byte[] arr = new byte[len]; + int i = 0; + n = value; + do + { + byte b = (byte)(n & 0x7f); + n >>= 7; + if (n != 0) + b |= 0x80; + arr[i++] = b; + } while (n != 0); + return arr; + } +} From 2f2459d6ab07df8c6e37ec7aa7cc57163779ea4d Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 8 May 2023 14:57:48 -0400 Subject: [PATCH 02/30] [mono][loader] implement a webcil-in-wasm reader --- src/mono/mono/metadata/webcil-loader.c | 75 +++++++++++- src/mono/mono/utils/CMakeLists.txt | 5 +- src/mono/mono/utils/wasm-module-reader.c | 140 +++++++++++++++++++++++ src/mono/mono/utils/wasm-module-reader.h | 35 ++++++ src/mono/mono/utils/wasm-sections.def | 21 ++++ 5 files changed, 273 insertions(+), 3 deletions(-) create mode 100644 src/mono/mono/utils/wasm-module-reader.c create mode 100644 src/mono/mono/utils/wasm-module-reader.h create mode 100644 src/mono/mono/utils/wasm-sections.def diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c index 1323c0f3ff137..aea8aaa753442 100644 --- a/src/mono/mono/metadata/webcil-loader.c +++ b/src/mono/mono/metadata/webcil-loader.c @@ -8,6 +8,7 @@ #include "mono/metadata/metadata-internals.h" #include "mono/metadata/webcil-loader.h" +#include "mono/utils/wasm-module-reader.h" /* keep in sync with webcil-writer */ enum { @@ -34,13 +35,24 @@ typedef struct MonoWebCilHeader { // 28 bytes } MonoWebCilHeader; +static gboolean +find_webcil_in_wasm (const uint8_t *ptr, const uint8_t *boundp, const uint8_t **webcil_payload_start); + static gboolean webcil_image_match (MonoImage *image) { + gboolean success = FALSE; if (image->raw_data_len >= sizeof (MonoWebCilHeader)) { - return image->raw_data[0] == 'W' && image->raw_data[1] == 'b' && image->raw_data[2] == 'I' && image->raw_data[3] == 'L'; + success = image->raw_data[0] == 'W' && image->raw_data[1] == 'b' && image->raw_data[2] == 'I' && image->raw_data[3] == 'L'; + + if (!success && mono_wasm_module_is_wasm ((const uint8_t*)image->raw_data, (const uint8_t*)image->raw_data + image->raw_data_len)) { + /* if it's a WebAssembly module, assume it's webcil-in-wasm and + * optimistically return TRUE + */ + success = TRUE; + } } - return FALSE; + return success; } /* @@ -52,6 +64,16 @@ static int32_t do_load_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header) { MonoWebCilHeader wcheader; + const uint8_t *raw_data_bound = (const uint8_t*)raw_data + raw_data_len; + if (mono_wasm_module_is_wasm ((const uint8_t*)raw_data, raw_data_bound)) { + /* assume it's webcil wrapped in wasm */ + const uint8_t *webcil_segment_start = NULL; + if (!find_webcil_in_wasm ((const uint8_t*)raw_data, raw_data_bound, &webcil_segment_start)) + return -1; + // skip to the beginning of the webcil payload + offset += (int32_t)(webcil_segment_start - (const uint8_t*)raw_data); + } + if (offset + sizeof (MonoWebCilHeader) > raw_data_len) return -1; memcpy (&wcheader, raw_data + offset, sizeof (wcheader)); @@ -168,3 +190,52 @@ mono_webcil_load_cli_header (const char *raw_data, uint32_t raw_data_len, int32_ { return do_load_header (raw_data, raw_data_len, offset, header); } + +struct webcil_in_wasm_ud +{ + const uint8_t *data_segment_1_start; +}; + +static gboolean +webcil_in_wasm_section_visitor (uint8_t sec_code, const uint8_t *sec_content, uint32_t sec_length, gpointer user_data, gboolean *should_stop) +{ + *should_stop = FALSE; + if (sec_code != MONO_WASM_MODULE_DATA_SECTION) + return TRUE; + struct webcil_in_wasm_ud *data = (struct webcil_in_wasm_ud *)user_data; + + *should_stop = TRUE; // we don't care about the sections after the data section + const uint8_t *ptr = sec_content; + const uint8_t *boundp = sec_content + sec_length; + + uint32_t num_segments = 0; + if (!mono_wasm_module_decode_uleb128 (ptr, boundp, &ptr, &num_segments)) + return FALSE; + + if (num_segments != 2) + return FALSE; + + // skip over data segment 0, it's the webcil payload length as a u32 - we don't care about it + uint32_t passive_segment_len = 0; + const uint8_t *passive_segment_start = NULL; + if (!mono_wasm_module_decode_passive_data_segment (ptr, boundp, &ptr, &passive_segment_len, &passive_segment_start)) + return FALSE; + // data segment 1 is the actual webcil payload. + if (!mono_wasm_module_decode_passive_data_segment (ptr, boundp, &ptr, &passive_segment_len, &passive_segment_start)) + return FALSE; + data->data_segment_1_start = passive_segment_start; + return TRUE; +} + +static gboolean +find_webcil_in_wasm (const uint8_t *ptr, const uint8_t *boundp, const uint8_t **webcil_payload_start) +{ + struct webcil_in_wasm_ud user_data = {0,}; + MonoWasmModuleVisitor visitor = {0,}; + visitor.section_visitor = &webcil_in_wasm_section_visitor; + if (!mono_wasm_module_visit(ptr, boundp, &visitor, &user_data)) + return FALSE; + + *webcil_payload_start = user_data.data_segment_1_start; + return TRUE; +} diff --git a/src/mono/mono/utils/CMakeLists.txt b/src/mono/mono/utils/CMakeLists.txt index e26ec96b784f8..efbfa3c119cff 100644 --- a/src/mono/mono/utils/CMakeLists.txt +++ b/src/mono/mono/utils/CMakeLists.txt @@ -184,7 +184,10 @@ set(utils_common_sources options.h options-def.h options.c - ftnptr.h) + ftnptr.h + wasm-module-reader.h + wasm-module-reader.c + ) if(MONO_CROSS_COMPILE) set(utils_arch_sources mach-support-unknown.c) diff --git a/src/mono/mono/utils/wasm-module-reader.c b/src/mono/mono/utils/wasm-module-reader.c new file mode 100644 index 0000000000000..d70e21d6df52c --- /dev/null +++ b/src/mono/mono/utils/wasm-module-reader.c @@ -0,0 +1,140 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include + +#include "mono/metadata/mono-endian.h" + +#include "wasm-module-reader.h" + +#define WASM_MODULE_SECTION(ident,str) str, +static const char * +wasm_module_section_names[] = { +#include "wasm-sections.def" +#undef WASM_MODULE_SECTION +}; + +static const char * +mono_wasm_module_section_get_name (int section) +{ + g_assert (section > 0 && section < MONO_WASM_MODULE_NUM_SECTIONS); + return wasm_module_section_names[section]; +} + +static gboolean +bc_read8 (const uint8_t *ptr, const uint8_t *boundp, const uint8_t **endp, uint8_t *out) +{ + if (ptr < boundp) { + *out = *ptr; + *endp = ptr + 1; + return TRUE; + } + return FALSE; +} + +static gboolean +bc_read32 (const uint8_t *ptr, const uint8_t *boundp, const uint8_t **endp, uint32_t *out) +{ + if (ptr + 3 < boundp) { + *out = read32 (ptr); + *endp = ptr + 4; + return TRUE; + } + return FALSE; +} + +static gboolean +bc_read_uleb128 (const uint8_t *ptr, const uint8_t *boundp, const uint8_t **endp, uint32_t *out) +{ + uint32_t val = 0; + unsigned int shift = 0; + while (1) { + uint8_t b; + if (!bc_read8 (ptr, boundp, &ptr, &b)) + return FALSE; + val |= (b & 0x7f) << shift; + if ((b & 0x80) == 0) break; + shift += 7; + g_assertf (shift < 35, "expected uleb128 encoded u32, got extra bytes\n"); + } + *out = val; + *endp = ptr; + + return TRUE; +} + +gboolean +mono_wasm_module_decode_uleb128 (const uint8_t *ptr, const uint8_t *boundp, const uint8_t **endp, uint32_t *out) +{ + return bc_read_uleb128 (ptr, boundp, endp, out); +} + +static gboolean +visit_section (const uint8_t *ptr, const uint8_t *boundp, const uint8_t **endp, MonoWasmModuleVisitor *visitor, gpointer user_data, gboolean *should_stop) +{ + uint8_t code = 0; + uint32_t sec_size = 0; + if (!bc_read8 (ptr, boundp, &ptr, &code)) + return FALSE; + if (!bc_read_uleb128 (ptr, boundp, &ptr, &sec_size)) + return FALSE; + + *should_stop = FALSE; + gboolean success = visitor->section_visitor (code, ptr, sec_size, user_data, should_stop); + *endp = ptr; + return success; +} + +/* + * return TRUE if successfully visited, FALSE if there was a problem + */ +gboolean +mono_wasm_module_visit (const uint8_t *ptr, const uint8_t *boundp, MonoWasmModuleVisitor *visitor, gpointer user_data) +{ + if (!mono_wasm_module_is_wasm (ptr, boundp)) + return FALSE; + + ptr += 4; + + uint32_t version = 0; + if (!bc_read32 (ptr, boundp, &ptr, &version)) + return FALSE; + if (version != 1) + return FALSE; + + gboolean success = TRUE; + + gboolean stop = FALSE; + while (success && !stop && ptr < boundp) { + success = visit_section (ptr, boundp, &ptr, visitor, user_data, &stop); + } + + return success; +} + +gboolean +mono_wasm_module_is_wasm (const uint8_t *ptr, const uint8_t *boundp) +{ + const uint32_t wasm_magic = 0x6d736100u; // "\0asm" + uint32_t magic = 0; + if (!bc_read32 (ptr, boundp, &ptr, &magic)) + return FALSE; + return magic == wasm_magic; +} + +gboolean +mono_wasm_module_decode_passive_data_segment (const uint8_t *ptr, const uint8_t *boundp, const uint8_t **endp, uint32_t *data_len, const uint8_t **data_start) +{ + uint8_t code = 0; + if (!bc_read8 (ptr, boundp, &ptr, &code)) + return FALSE; + if (code != 1) + return FALSE; // not a passive segment + uint32_t len = 0; + if (!bc_read_uleb128 (ptr, boundp, &ptr, &len)) + return FALSE; + *data_start = ptr; + *data_len = len; + *endp = ptr; + return TRUE; +} diff --git a/src/mono/mono/utils/wasm-module-reader.h b/src/mono/mono/utils/wasm-module-reader.h new file mode 100644 index 0000000000000..1bf7fa65c0385 --- /dev/null +++ b/src/mono/mono/utils/wasm-module-reader.h @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#ifndef __MONO_WASM_MODULE_READER_H__ +#define __MONO_WASM_MODULE_READER_H__ + +#include + +typedef struct MonoWasmModuleVisitor +{ + /* return TRUE for success, set *should_stop to stop visitation */ + gboolean (*section_visitor) (uint8_t sec_code, const uint8_t *sec_content, uint32_t sec_length, gpointer user_data, gboolean *should_stop); +} MonoWasmModuleVisitor; + +#define WASM_MODULE_SECTION(ident,str) MONO_WASM_MODULE_ ## ident ## _SECTION, +enum { +#include "wasm-sections.def" + MONO_WASM_MODULE_NUM_SECTIONS, +}; +#undef WASM_MODULE_SECTION + +gboolean +mono_wasm_module_decode_uleb128 (const uint8_t *ptr, const uint8_t *boundp, const uint8_t **endp, uint32_t *out); + +gboolean +mono_wasm_module_is_wasm (const uint8_t *ptr, const uint8_t *boundp); + +gboolean +mono_wasm_module_visit (const uint8_t *ptr, const uint8_t *boundp, MonoWasmModuleVisitor *visitor, gpointer user_data); + +/* returns FALSE if the data segment is not passive */ +gboolean +mono_wasm_module_decode_passive_data_segment (const uint8_t *ptr, const uint8_t *boundp, const uint8_t **endp, uint32_t *data_len, const uint8_t **data_start); + +#endif /* __MONO_WASM_MODULE_READER_H__*/ diff --git a/src/mono/mono/utils/wasm-sections.def b/src/mono/mono/utils/wasm-sections.def new file mode 100644 index 0000000000000..a09b65a8e88c2 --- /dev/null +++ b/src/mono/mono/utils/wasm-sections.def @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#ifndef WASM_MODULE_SECTION +#error "define WASM_MODULE_SECTION(ident,str) before including this heaer" +#endif + +// order matters and must match the section codes from the WebAssembly spec +WASM_MODULE_SECTION(CUSTOM, "custom") +WASM_MODULE_SECTION(TYPE, "type") +WASM_MODULE_SECTION(IMPORT, "import") +WASM_MODULE_SECTION(FUNCTION, "function") +WASM_MODULE_SECTION(TABLE, "table") +WASM_MODULE_SECTION(MEMORY, "memory") +WASM_MODULE_SECTION(GLOBAL, "global") +WASM_MODULE_SECTION(EXPORT, "export") +WASM_MODULE_SECTION(START, "start") +WASM_MODULE_SECTION(ELEMENT, "element") +WASM_MODULE_SECTION(CODE, "code") +WASM_MODULE_SECTION(DATA, "data") +WASM_MODULE_SECTION(DATACOUNT, "data count") \ No newline at end of file From 7be51f84c30a3e559d7bea2f9ec5a72ed22370d8 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 8 May 2023 15:23:29 -0400 Subject: [PATCH 03/30] reword WebcilWasmWrapper summary comment --- .../src/Webcil/WebcilWasmWrapper.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs index a0e23b44025db..90b5afb46ae65 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs @@ -10,12 +10,14 @@ namespace Microsoft.NET.WebAssembly.Webcil; // -// This is a pretty dumb wrapper. It assumes that the entire wasm module is going to be unchanging, -// except for the data section which has 2 passive segments. segment 0 is 4 bytes and contains the -// length of the webcil payload. segment 1 is of a variable size and contains the webcil payload. +// Emits a simple WebAssembly wrapper module around a given webcil payload. // -// the unchanging parts are stored as a "prefix" and "suffix" which contain the bytes for the following -// WAT program, split into the parts that come before the data section, and the bytes that come after: +// The entire wasm module is going to be unchanging, except for the data section which has 2 passive +// segments. segment 0 is 4 bytes and contains the length of the webcil payload. segment 1 is of a +// variable size and contains the webcil payload. +// +// The unchanging parts are stored as a "prefix" and "suffix" which contain the bytes for the following +// WAT module, split into the parts that come before the data section, and the bytes that come after: // // (module // (data "\0f\00\00\00") ;; data segment 0: payload size as a 4 byte LE uint32 From 2e58062aeeff1d113547e83d0f8d387fc63a1b09 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 8 May 2023 15:31:36 -0400 Subject: [PATCH 04/30] fix whitespace --- .../src/Webcil/WebcilConverter.cs | 6 +++--- .../src/Webcil/WebcilWasmWrapper.cs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs index f2fb9ae130a49..dfec29cf6a01a 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs @@ -73,10 +73,10 @@ public void ConvertToWebcil() // if wrapping in WASM, write the webcil payload to memory because we need to discover the length // webcil is about the same size as the PE file - using var memoryStream = new MemoryStream (checked((int)inputStream.Length)); + using var memoryStream = new MemoryStream(checked((int)inputStream.Length)); WriteConversionTo(memoryStream, inputStream, peInfo, wcInfo); - var wrapper = new WebcilWasmWrapper (memoryStream); - memoryStream.Seek (0, SeekOrigin.Begin); + var wrapper = new WebcilWasmWrapper(memoryStream); + memoryStream.Seek(0, SeekOrigin.Begin); wrapper.WriteWasmWrappedWebcil(outputStream); } } diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs index 90b5afb46ae65..b57f1127297cf 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs @@ -39,7 +39,7 @@ public class WebcilWasmWrapper private readonly Stream _webcilPayloadStream; private readonly uint _webcilPayloadSize; - public WebcilWasmWrapper (Stream webcilPayloadStream) + public WebcilWasmWrapper(Stream webcilPayloadStream) { _webcilPayloadStream = webcilPayloadStream; long len = webcilPayloadStream.Length; @@ -53,7 +53,7 @@ public void WriteWasmWrappedWebcil(Stream outputStream) WriteWasmHeader(outputStream); using (var writer = new BinaryWriter(outputStream, System.Text.Encoding.UTF8, leaveOpen: true)) { - WriteDataSection (writer); + WriteDataSection(writer); } WriteWasmSuffix(outputStream); } From 056b9acce750f864573abffd119c3d4a2d1f2ed2 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 8 May 2023 15:31:49 -0400 Subject: [PATCH 05/30] fix typos --- src/mono/mono/utils/wasm-sections.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/mono/utils/wasm-sections.def b/src/mono/mono/utils/wasm-sections.def index a09b65a8e88c2..dd55c718c9b7f 100644 --- a/src/mono/mono/utils/wasm-sections.def +++ b/src/mono/mono/utils/wasm-sections.def @@ -2,7 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. #ifndef WASM_MODULE_SECTION -#error "define WASM_MODULE_SECTION(ident,str) before including this heaer" +#error "define WASM_MODULE_SECTION(ident,str) before including this header" #endif // order matters and must match the section codes from the WebAssembly spec From 897df892e9bfccc3cc414bad4dbba3e4ff8ca1d6 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 8 May 2023 15:39:39 -0400 Subject: [PATCH 06/30] visit_section should bump the ptr after traversal --- src/mono/mono/utils/wasm-module-reader.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/mono/utils/wasm-module-reader.c b/src/mono/mono/utils/wasm-module-reader.c index d70e21d6df52c..eb3c19c25fe94 100644 --- a/src/mono/mono/utils/wasm-module-reader.c +++ b/src/mono/mono/utils/wasm-module-reader.c @@ -81,7 +81,7 @@ visit_section (const uint8_t *ptr, const uint8_t *boundp, const uint8_t **endp, *should_stop = FALSE; gboolean success = visitor->section_visitor (code, ptr, sec_size, user_data, should_stop); - *endp = ptr; + *endp = ptr + sec_size; // advance past the section payload return success; } From 15a1894e25f45088879fe1a3a39ac5b3cea8fb52 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 9 May 2023 10:06:53 -0400 Subject: [PATCH 07/30] remove extra bytes from wasm webcil prefix the split utility was including the data header prefix in the output --- .../src/Webcil/WebcilWasmWrapper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs index b57f1127297cf..816fb1c3940d3 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs @@ -73,7 +73,7 @@ private static 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x02, 0x00, 0x01, 0x03, 0x03, 0x02, 0x00, 0x01, 0x06, 0x0b, 0x02, 0x7f, 0x00, 0x41, 0x00, 0x0b, 0x7f, 0x00, 0x41, 0x00, 0x0b, 0x07, 0x41, 0x04, 0x0d, 0x77, 0x65, 0x62, 0x63, 0x69, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x03, 0x00, 0x0a, 0x77, 0x65, 0x62, 0x63, 0x69, 0x6c, 0x53, 0x69, 0x7a, 0x65, 0x03, 0x01, 0x0d, 0x67, 0x65, 0x74, 0x57, 0x65, 0x62, 0x63, 0x69, 0x6c, 0x53, 0x69, 0x7a, 0x65, 0x00, 0x00, 0x10, 0x67, 0x65, 0x74, 0x57, 0x65, 0x62, 0x63, 0x69, 0x6c, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x00, 0x01, 0x0c, 0x01, 0x02, - 0x0a, 0x1b, 0x02, 0x0c, 0x00, 0x20, 0x00, 0x41, 0x00, 0x41, 0x04, 0xfc, 0x08, 0x00, 0x00, 0x0b, 0x0c, 0x00, 0x20, 0x00, 0x41, 0x00, 0x20, 0x01, 0xfc, 0x08, 0x01, 0x00, 0x0b, 0x0b, 0x1b, + 0x0a, 0x1b, 0x02, 0x0c, 0x00, 0x20, 0x00, 0x41, 0x00, 0x41, 0x04, 0xfc, 0x08, 0x00, 0x00, 0x0b, 0x0c, 0x00, 0x20, 0x00, 0x41, 0x00, 0x20, 0x01, 0xfc, 0x08, 0x01, 0x00, 0x0b, }; // // Everything from the above wat module after the data section From 1688dece08997c763988ea99c35aa3a27e059d67 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 9 May 2023 10:47:24 -0400 Subject: [PATCH 08/30] don't forget to include number of segments in the data section --- .../src/Webcil/WebcilWasmWrapper.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs index 816fb1c3940d3..172dc87940130 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs @@ -113,6 +113,8 @@ private static void WriteWasmSuffix(Stream outputStream) private void WriteDataSection(BinaryWriter writer) { uint dataSectionSize = 0; + // uleb128 encoding of number of segments + dataSectionSize += 1; // there's always 2 segments which encodes to 1 byte // compute the segment 0 size: // segment 0 has 1 byte segment code, 1 byte of size and 4 bytes of payload dataSectionSize += SegmentCodeSize + 1 + 4; @@ -132,6 +134,8 @@ private void WriteDataSection(BinaryWriter writer) writer.Write((byte)11); // section Data writer.Write(ulebSectionSize, 0, ulebSectionSize.Length); + writer.Write((byte)2); // number of segments + // write segment 0 writer.Write((byte)1); // passive segment writer.Write((byte)4); // segment size: 4 From d8638f755455bbf6e9a840cb6abaed68fe33f670 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 9 May 2023 11:11:17 -0400 Subject: [PATCH 09/30] update the Webcil spec to include the WebAssembly wrapper module --- docs/design/mono/webcil.md | 71 +++++++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/docs/design/mono/webcil.md b/docs/design/mono/webcil.md index 3bcb7d6365353..59a19912947c4 100644 --- a/docs/design/mono/webcil.md +++ b/docs/design/mono/webcil.md @@ -2,7 +2,8 @@ ## Version -This is version 0.0 of the Webcil format. +This is version 0.0 of the Webcil payload format. +This is version 0 of the WebAssembly module Webcil wrapper. ## Motivation @@ -10,13 +11,65 @@ When deploying the .NET runtime to the browser using WebAssembly, we have receiv customers that certain users are unable to use their apps because firewalls and anti-virus software may prevent browsers from downloading or caching assemblies with a .DLL extension and PE contents. -This document defines a new container format for ECMA-335 assemblies -that uses the `.webcil` extension and uses a new WebCIL container -format. +This document defines a new container format for ECMA-335 assemblies that uses the `.wasm` extension +and uses a new WebCIL metadata payload format wrapped in a WebAssembly module. ## Specification +### Webcil WebAssembly module + +Webcil consists of a standard [binary WebAssembly version 0 module](https://webassembly.github.io/spec/core/binary/index.html) containing the following WAT module: + +``` wat +(module + (data "\0f\00\00\00") ;; data segment 0: payload size as a 4 byte LE uint32 + (data "webcil Payload\cc") ;; data segment 1: webcil payload + (memory (import "webcil" "memory") 1) + (global (export "webcilVersion") i32 (i32.const 0)) + (func (export "getWebcilSize") (param $destPtr i32) (result) + local.get $destPtr + i32.const 0 + i32.const 4 + memory.init 0) + (func (export "getWebcilPayload") (param $d i32) (param $n i32) (result) + local.get $d + i32.const 0 + local.get $n + memory.init 1)) +``` + +That is, the module imports linear memory 0 and exports: +* a global `i32` `webcilVersion` encoding the version of the WebAssembly wrapper (currently 0), +* a function `getWebcilSize : i32 -> ()` that writes the size of the Webcil payload to the specified + address in linear memory as a `u32` (that is: 4 LE bytes). +* a function `getWebcilPayload : i32 i32 -> ()` that writes `$n` bytes of the content of the Webcil + payload at the spcified address `$d` in linear memory. + +The Webcil payload size and payload content are stored in the data section of the WebAssembly module +as passive data segments 0 and 1, respectively. The module must not contain additional data +segments. The module must store the payload size in data segment 0, and the payload content in data +segment 1. + +(**Rationale**: With this wrapper it is possible to split the WebAssembly module into a *prefix* +consisting of everything before the data section, the data section, and a *suffix* that consists of +everything after the data section. The prefix and suffix do not depend on the contents of the +Webcil payload and a tool that generates Webcil files could simply emit the prefix and suffix from +constant data. The data section is the only variable content between different Webcil-encoded .NET +assemblies) + +(**Rational**: Encoding the payload in the data section in passive data segments with known indices +allows a runtime that does not include a WebAssembly host or a runtime that does not wish to +instantiate the WebAssembly module to extract the payload by traversing the WebAssembly module and +locating the Webcil payload in the data section at segment 1.) + +(**Note**: the wrapper may be versioned independently of the payload.) + + +### Webcil payload + +The webcil payload contains the ECMA-335 metadata, IL and resources comprising a .NET assembly. + As our starting point we take section II.25.1 "Structure of the runtime file format" from ECMA-335 6th Edition. @@ -40,12 +93,12 @@ A Webcil file follows a similar structure | CLI Data | | | -## Webcil Headers +### Webcil Headers The Webcil headers consist of a Webcil header followed by a sequence of section headers. (All multi-byte integers are in little endian format). -### Webcil Header +#### Webcil Header ``` c struct WebcilHeader { @@ -75,7 +128,7 @@ The next pairs of integers are a subset of the PE Header data directory specifyi of the CLI header, as well as the directory entry for the PE debug directory. -### Section header table +#### Section header table Immediately following the Webcil header is a sequence (whose length is given by `coff_sections` above) of section headers giving their virtual address and virtual size, as well as the offset in @@ -92,11 +145,11 @@ struct SectionHeader { }; ``` -### Sections +#### Sections Immediately following the section table are the sections. These are copied verbatim from the PE file. -## Rationale +### Rationale The intention is to include only the information necessary for the runtime to locate the metadata root, and to resolve the RVA references in the metadata (for locating data declarations and method IL). From e8f52bbca688cf15b3d080dd8ae8a04cf87b4ec9 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 9 May 2023 11:19:18 -0400 Subject: [PATCH 10/30] fix typos and whitespace --- docs/design/mono/webcil.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/design/mono/webcil.md b/docs/design/mono/webcil.md index 59a19912947c4..f41f4caeef618 100644 --- a/docs/design/mono/webcil.md +++ b/docs/design/mono/webcil.md @@ -40,7 +40,7 @@ Webcil consists of a standard [binary WebAssembly version 0 module](https://weba ``` That is, the module imports linear memory 0 and exports: -* a global `i32` `webcilVersion` encoding the version of the WebAssembly wrapper (currently 0), +* a global `i32` `webcilVersion` encoding the version of the WebAssembly wrapper (currently 0), * a function `getWebcilSize : i32 -> ()` that writes the size of the Webcil payload to the specified address in linear memory as a `u32` (that is: 4 LE bytes). * a function `getWebcilPayload : i32 i32 -> ()` that writes `$n` bytes of the content of the Webcil @@ -58,7 +58,7 @@ Webcil payload and a tool that generates Webcil files could simply emit the pref constant data. The data section is the only variable content between different Webcil-encoded .NET assemblies) -(**Rational**: Encoding the payload in the data section in passive data segments with known indices +(**Rationale**: Encoding the payload in the data section in passive data segments with known indices allows a runtime that does not include a WebAssembly host or a runtime that does not wish to instantiate the WebAssembly module to extract the payload by traversing the WebAssembly module and locating the Webcil payload in the data section at segment 1.) From f06938f0345ba9f199f34f64f94c49521c0aeb38 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 9 May 2023 12:27:05 -0400 Subject: [PATCH 11/30] advance endp past the data segment payload --- src/mono/mono/utils/wasm-module-reader.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/mono/utils/wasm-module-reader.c b/src/mono/mono/utils/wasm-module-reader.c index eb3c19c25fe94..0f078f92ed478 100644 --- a/src/mono/mono/utils/wasm-module-reader.c +++ b/src/mono/mono/utils/wasm-module-reader.c @@ -135,6 +135,6 @@ mono_wasm_module_decode_passive_data_segment (const uint8_t *ptr, const uint8_t return FALSE; *data_start = ptr; *data_len = len; - *endp = ptr; + *endp = ptr + len; return TRUE; } From 0f5e02f441cc544bb1e5dac0818cb6a25111142b Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 9 May 2023 13:19:47 -0400 Subject: [PATCH 12/30] Adjust RVA map offsets to account for wasm prefix MonoImage:raw_data is used as a base when applying the RVA map to map virtual addresses to physical offsets in the assembly. With webcil-in-wasm there's an extra wasm prefix before the webcil payload starts, so we need to account for this extra data when creating the mapping. An alternative is to compute the correct offsets as part of generating the webcil, but that would entangle the wasm module and the webcil payload. The current (somewhat hacky approach) keeps them logically separate. --- src/mono/mono/metadata/image.c | 5 +++-- src/mono/mono/metadata/webcil-loader.c | 21 +++++++++++++-------- src/mono/mono/metadata/webcil-loader.h | 4 ++-- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/mono/mono/metadata/image.c b/src/mono/mono/metadata/image.c index 2eef0230f2a71..0eea10e3cdc18 100644 --- a/src/mono/mono/metadata/image.c +++ b/src/mono/mono/metadata/image.c @@ -958,8 +958,9 @@ mono_has_pdb_checksum (char *raw_data, uint32_t raw_data_len) int32_t ret = try_load_pe_cli_header (raw_data, raw_data_len, &cli_header); #ifdef ENABLE_WEBCIL + int32_t webcil_section_adjustment = 0; if (ret == -1) { - ret = mono_webcil_load_cli_header (raw_data, raw_data_len, 0, &cli_header); + ret = mono_webcil_load_cli_header (raw_data, raw_data_len, 0, &cli_header, &webcil_section_adjustment); is_pe = FALSE; } #endif @@ -992,7 +993,7 @@ mono_has_pdb_checksum (char *raw_data, uint32_t raw_data_len) } #ifdef ENABLE_WEBCIL else { - ret = mono_webcil_load_section_table (raw_data, raw_data_len, ret, &t); + ret = mono_webcil_load_section_table (raw_data, raw_data_len, ret, webcil_section_adjustment, &t); if (ret == -1) return FALSE; } diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c index aea8aaa753442..81e790c0d95a2 100644 --- a/src/mono/mono/metadata/webcil-loader.c +++ b/src/mono/mono/metadata/webcil-loader.c @@ -61,17 +61,21 @@ webcil_image_match (MonoImage *image) * most of MonoDotNetHeader is unused and left uninitialized (assumed zero); */ static int32_t -do_load_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header) +do_load_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header, int32_t *raw_data_rva_map_wasm_bump) { MonoWebCilHeader wcheader; const uint8_t *raw_data_bound = (const uint8_t*)raw_data + raw_data_len; + *raw_data_rva_map_wasm_bump = 0; if (mono_wasm_module_is_wasm ((const uint8_t*)raw_data, raw_data_bound)) { /* assume it's webcil wrapped in wasm */ const uint8_t *webcil_segment_start = NULL; if (!find_webcil_in_wasm ((const uint8_t*)raw_data, raw_data_bound, &webcil_segment_start)) return -1; + // HACK: adjust all the rva physical offsets by this amount + int32_t offset_adjustment = (int32_t)(webcil_segment_start - (const uint8_t*)raw_data); + *raw_data_rva_map_wasm_bump = offset_adjustment; // skip to the beginning of the webcil payload - offset += (int32_t)(webcil_segment_start - (const uint8_t*)raw_data); + offset += offset_adjustment; } if (offset + sizeof (MonoWebCilHeader) > raw_data_len) @@ -94,7 +98,7 @@ do_load_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, Mon } int32_t -mono_webcil_load_section_table (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoSectionTable *t) +mono_webcil_load_section_table (const char *raw_data, uint32_t raw_data_len, int32_t offset, int32_t webcil_section_adjustment, MonoSectionTable *t) { /* WebCIL section table entries are a subset of a PE section * header. Initialize just the parts we have. @@ -109,7 +113,7 @@ mono_webcil_load_section_table (const char *raw_data, uint32_t raw_data_len, int t->st_virtual_size = GUINT32_FROM_LE (st [0]); t->st_virtual_address = GUINT32_FROM_LE (st [1]); t->st_raw_data_size = GUINT32_FROM_LE (st [2]); - t->st_raw_data_ptr = GUINT32_FROM_LE (st [3]); + t->st_raw_data_ptr = GUINT32_FROM_LE (st [3]) + (uint32_t)webcil_section_adjustment; offset += sizeof(st); return offset; } @@ -121,12 +125,13 @@ webcil_image_load_pe_data (MonoImage *image) MonoCLIImageInfo *iinfo; MonoDotNetHeader *header; int32_t offset = 0; + int32_t webcil_section_adjustment = 0; int top; iinfo = image->image_info; header = &iinfo->cli_header; - offset = do_load_header (image->raw_data, image->raw_data_len, offset, header); + offset = do_load_header (image->raw_data, image->raw_data_len, offset, header, &webcil_section_adjustment); if (offset == -1) goto invalid_image; @@ -138,7 +143,7 @@ webcil_image_load_pe_data (MonoImage *image) for (int i = 0; i < top; i++) { MonoSectionTable *t = &iinfo->cli_section_tables [i]; - offset = mono_webcil_load_section_table (image->raw_data, image->raw_data_len, offset, t); + offset = mono_webcil_load_section_table (image->raw_data, image->raw_data_len, offset, webcil_section_adjustment, t); if (offset == -1) goto invalid_image; } @@ -186,9 +191,9 @@ mono_webcil_loader_install (void) } int32_t -mono_webcil_load_cli_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header) +mono_webcil_load_cli_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header, int32_t *webcil_section_adjustment) { - return do_load_header (raw_data, raw_data_len, offset, header); + return do_load_header (raw_data, raw_data_len, offset, header, webcil_section_adjustment); } struct webcil_in_wasm_ud diff --git a/src/mono/mono/metadata/webcil-loader.h b/src/mono/mono/metadata/webcil-loader.h index c95c2c5690409..e0ae1d47fc3c2 100644 --- a/src/mono/mono/metadata/webcil-loader.h +++ b/src/mono/mono/metadata/webcil-loader.h @@ -9,9 +9,9 @@ void mono_webcil_loader_install (void); int32_t -mono_webcil_load_cli_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header); +mono_webcil_load_cli_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header, int32_t *webcil_section_adjustment); int32_t -mono_webcil_load_section_table (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoSectionTable *t); +mono_webcil_load_section_table (const char *raw_data, uint32_t raw_data_len, int32_t offset, int32_t webcil_section_adjustment, MonoSectionTable *t); #endif /*_MONO_METADATA_WEBCIL_LOADER_H*/ From 4271d8f9b3ceea401c0d1a6a35d33eab8373e7a2 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 9 May 2023 13:27:52 -0400 Subject: [PATCH 13/30] Add a note about the rva mapping to the spec --- docs/design/mono/webcil.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/design/mono/webcil.md b/docs/design/mono/webcil.md index f41f4caeef618..1f3f21fdcb010 100644 --- a/docs/design/mono/webcil.md +++ b/docs/design/mono/webcil.md @@ -132,7 +132,7 @@ of the CLI header, as well as the directory entry for the PE debug directory. Immediately following the Webcil header is a sequence (whose length is given by `coff_sections` above) of section headers giving their virtual address and virtual size, as well as the offset in -the Webcil file and the size in the file. This is a subset of the PE section header that includes +the Webcil payload and the size in the file. This is a subset of the PE section header that includes enough information to correctly interpret the RVAs from the webcil header and from the .NET metadata. Other information (such as the section names) are not included. @@ -145,6 +145,8 @@ struct SectionHeader { }; ``` +(**Note**: the `st_raw_data_ptr` member is an offset from the beginning of the Webcil payload, not from the beginning of the WebAssembly wrapper module.) + #### Sections Immediately following the section table are the sections. These are copied verbatim from the PE file. From 2c90fb5f9e0b8ceafe7aab349b8052f596eb5b96 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 9 May 2023 13:54:00 -0400 Subject: [PATCH 14/30] Serve webcil-in-wasm as .wasm --- src/mono/mono/metadata/assembly.c | 14 ++++++++++++++ src/mono/mono/metadata/mono-debug.c | 8 ++++++++ src/mono/mono/metadata/webcil-loader.h | 2 ++ src/mono/mono/mini/monovm.c | 13 ++++++++++++- src/mono/sample/wasm/browser-advanced/index.html | 4 ++-- .../Wasm.Build.Tests/Blazor/BuildPublishTests.cs | 2 +- src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs | 6 ++++-- src/mono/wasm/build/WasmApp.targets | 2 +- .../debugger/BrowserDebugProxy/MonoSDBHelper.cs | 7 +++++++ .../debugger/DebuggerTestSuite/DebuggerTestBase.cs | 2 ++ .../wasm/debugger/DebuggerTestSuite/MonoJsTests.cs | 2 +- src/tasks/Common/Utils.cs | 2 ++ .../ComputeWasmPublishAssets.cs | 2 +- .../ConvertDllsToWebCil.cs | 6 +++--- .../GenerateWasmBootJson.cs | 2 +- src/tasks/WasmAppBuilder/WasmAppBuilder.cs | 6 +++--- 16 files changed, 64 insertions(+), 16 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index c2caedec37c95..6758877040c01 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -44,6 +44,7 @@ #include #include #include +#include #ifndef HOST_WIN32 #include @@ -1465,6 +1466,13 @@ bundled_assembly_match (const char *bundled_name, const char *name) if (bprefix == nprefix && strncmp (bundled_name, name, bprefix) == 0) return TRUE; } + /* if they want a .dll and we have the matching .wasm webcil-in-wasm, return it */ + if (g_str_has_suffix (bundled_name, MONO_WEBCIL_IN_WASM_EXTENSION) && g_str_has_suffix (name, ".dll")) { + size_t bprefix = strlen (bundled_name) - strlen (MONO_WEBCIL_IN_WASM_EXTENSION); + size_t nprefix = strlen (name) - strlen (".dll"); + if (bprefix == nprefix && strncmp (bundled_name, name, bprefix) == 0) + return TRUE; + } return FALSE; #endif } @@ -2737,6 +2745,12 @@ mono_assembly_load_corlib (void) corlib = mono_assembly_request_open (corlib_name, &req, &status); g_free (corlib_name); } + if (!corlib) { + /* Maybe its in a bundle */ + char *corlib_name = g_strdup_printf ("%s%s", MONO_ASSEMBLY_CORLIB_NAME, MONO_WEBCIL_IN_WASM_EXTENSION); + corlib = mono_assembly_request_open (corlib_name, &req, &status); + g_free (corlib_name); + } #endif g_assert (corlib); diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index 993a6fd1edfbe..c415e3b4ce640 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #if NO_UNALIGNED_ACCESS @@ -1111,6 +1112,13 @@ bsymfile_match (BundledSymfile *bsymfile, const char *assembly_name) && !strcmp (bsymfile->aname + n, ".dll")) return TRUE; } + p = strstr (assembly_name, MONO_WEBCIL_IN_WASM_EXTENSION); + if (p && *(p + strlen(MONO_WEBCIL_IN_WASM_EXTENSION)) == 0) { + size_t n = p - assembly_name; + if (!strncmp (bsymfile->aname, assembly_name, n) + && !strcmp (bsymfile->aname + n, ".dll")) + return TRUE; + } #endif return FALSE; } diff --git a/src/mono/mono/metadata/webcil-loader.h b/src/mono/mono/metadata/webcil-loader.h index e0ae1d47fc3c2..daf6217150483 100644 --- a/src/mono/mono/metadata/webcil-loader.h +++ b/src/mono/mono/metadata/webcil-loader.h @@ -5,6 +5,8 @@ #ifndef _MONO_METADATA_WEBCIL_LOADER_H #define _MONO_METADATA_WEBCIL_LOADER_H +#define MONO_WEBCIL_IN_WASM_EXTENSION ".wasm" + void mono_webcil_loader_install (void); diff --git a/src/mono/mono/mini/monovm.c b/src/mono/mono/mini/monovm.c index 3730d6b256da5..6c55982732a6b 100644 --- a/src/mono/mono/mini/monovm.c +++ b/src/mono/mono/mini/monovm.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -140,7 +141,7 @@ mono_core_preload_hook (MonoAssemblyLoadContext *alc, MonoAssemblyName *aname, c n -= strlen(".dll"); char *fullpath2 = g_malloc (n + strlen(".webcil") + 1); g_strlcpy (fullpath2, fullpath, n + 1); - g_strlcpy (fullpath2 + n, ".webcil", 8); + g_strlcpy (fullpath2 + n, ".webcil", strlen(".webcil") + 1); if (g_file_test (fullpath2, G_FILE_TEST_IS_REGULAR)) { MonoImageOpenStatus status; result = mono_assembly_request_open (fullpath2, &req, &status); @@ -148,6 +149,16 @@ mono_core_preload_hook (MonoAssemblyLoadContext *alc, MonoAssemblyName *aname, c g_free (fullpath2); if (result) break; + char *fullpath3 = g_malloc (n + strlen(MONO_WEBCIL_IN_WASM_EXTENSION) + 1); + g_strlcpy (fullpath3, fullpath, n + 1); + g_strlcpy (fullpath3 + n, MONO_WEBCIL_IN_WASM_EXTENSION, strlen(MONO_WEBCIL_IN_WASM_EXTENSION) + 1); + if (g_file_test (fullpath3, G_FILE_TEST_IS_REGULAR)) { + MonoImageOpenStatus status; + result = mono_assembly_request_open (fullpath3, &req, &status); + } + g_free (fullpath3); + if (result) + break; } #endif } diff --git a/src/mono/sample/wasm/browser-advanced/index.html b/src/mono/sample/wasm/browser-advanced/index.html index 0532ef8fcef32..3ec9f7934e8dc 100644 --- a/src/mono/sample/wasm/browser-advanced/index.html +++ b/src/mono/sample/wasm/browser-advanced/index.html @@ -13,7 +13,7 @@ - + @@ -21,4 +21,4 @@ Answer to the Ultimate Question of Life, the Universe, and Everything is : - \ No newline at end of file + diff --git a/src/mono/wasm/Wasm.Build.Tests/Blazor/BuildPublishTests.cs b/src/mono/wasm/Wasm.Build.Tests/Blazor/BuildPublishTests.cs index 3e2c0b2175a0a..d6fee05cc1068 100644 --- a/src/mono/wasm/Wasm.Build.Tests/Blazor/BuildPublishTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/Blazor/BuildPublishTests.cs @@ -197,7 +197,7 @@ public void BugRegression_60479_WithRazorClassLib() .ExecuteWithCapturedOutput("new razorclasslib") .EnsureSuccessful(); - string razorClassLibraryFileName = UseWebcil ? "RazorClassLibrary.webcil" : "RazorClassLibrary.dll"; + string razorClassLibraryFileName = UseWebcil ? $"RazorClassLibrary{WebcilInWasmExtension}" : "RazorClassLibrary.dll"; AddItemsPropertiesToProject(wasmProjectFile, extraItems: @$" diff --git a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs index 6dc3e5e525468..bf1d6f33c1168 100644 --- a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs +++ b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs @@ -62,6 +62,8 @@ public static string GetNuGetConfigPathFor(string targetFramework) => Path.Combine(BuildEnvironment.TestDataPath, "nuget8.config"); // for now - we are still using net7, but with // targetFramework == "net7.0" ? "nuget7.config" : "nuget8.config"); + public const string WebcilInWasmExtension = ".wasm"; + static BuildTestBase() { try @@ -80,7 +82,7 @@ static BuildTestBase() Console.WriteLine ($"=============================================================================================="); Console.WriteLine ($"=============== Running with {(s_buildEnv.IsWorkload ? "Workloads" : "No workloads")} ==============="); if (UseWebcil) - Console.WriteLine($"=============== Using .webcil ==============="); + Console.WriteLine($"=============== Using webcil-in-wasm ==============="); Console.WriteLine ($"=============================================================================================="); Console.WriteLine (""); } @@ -682,7 +684,7 @@ protected static void AssertBasicAppBundle(string bundleDir, string managedDir = Path.Combine(bundleDir, "managed"); string bundledMainAppAssembly = - useWebcil ? $"{projectName}.webcil" : $"{projectName}.dll"; + useWebcil ? $"{projectName}${WebcilInWasmExtension}" : $"{projectName}.dll"; AssertFilesExist(managedDir, new[] { bundledMainAppAssembly }); bool is_debug = config == "Debug"; diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets index 0d50ada7ad3a2..9a153cabb1797 100644 --- a/src/mono/wasm/build/WasmApp.targets +++ b/src/mono/wasm/build/WasmApp.targets @@ -71,7 +71,7 @@ - $(WasmEnableLegacyJsInterop) - Include support for legacy JS interop. Defaults to true. - $(WasmEnableExceptionHandling) - Enable support for the WASM post MVP Exception Handling runtime extension. - $(WasmEnableSIMD) - Enable support for the WASM post MVP SIMD runtime extension. - - $(WasmEnableWebcil) - Enable conversion of assembly .dlls to .webcil + - $(WasmEnableWebcil) - Enable conversion of assembly .dlls to Webcil wrapped in .wasm - $(WasmIncludeFullIcuData) - Loads full ICU data (icudt.dat). Defaults to false. Only applicable when InvariantGlobalization=false. - $(WasmIcuDataFileName) - Name/path of ICU globalization file loaded to app. Only when InvariantGloblization=false and WasmIncludeFullIcuData=false. - $(WasmAllowUndefinedSymbols) - Controls whether undefined symbols are allowed or not, diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs b/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs index 9e4538ff4d7c1..4278f98b26113 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs +++ b/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs @@ -787,6 +787,8 @@ public async Task GetValue(MonoSDBHelper sdbHelper, CancellationToken t } internal sealed partial class MonoSDBHelper { + public const string WebcilInWasmExtension = ".wasm"; + private static int debuggerObjectId; private static int cmdId = 1; //cmdId == 0 is used by events which come from runtime private const int MINOR_VERSION = 61; @@ -1219,6 +1221,11 @@ public async Task GetAssemblyName(int assembly_id, CancellationToken tok string baseName = result.Substring(0, result.Length - 7); result = baseName + ".dll"; } + if (result.EndsWith(WebcilInWasmExtension)) { + /* don't leak webcil .wasm names to the debugger - work in terms of the original .dlls */ + string baseName = result.Substring(0, result.Length - WebcilInWasmExtension.Length); + result = baseName + ".dll"; + } return result; } diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs b/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs index c5119a28d2ce7..59f890e1f7bcb 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs @@ -24,6 +24,8 @@ public class DebuggerTests : DebuggerTestFirefox #endif { + public const string WebcilInWasmExtension = ".wasm"; + public DebuggerTests(ITestOutputHelper testOutput, string locale = "en-US", string driver = "debugger-driver.html") : base(testOutput, locale, driver) {} diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs b/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs index 139daae4ae326..46e054e8d85c2 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs @@ -136,7 +136,7 @@ async Task AssemblyLoadedEventTest(string asm_name, string asm_path, string pdb_ : await Task.FromResult(ProtocolEventHandlerReturn.KeepHandler); }); - byte[] bytes = File.Exists(asm_path) ? File.ReadAllBytes(asm_path) : File.ReadAllBytes(Path.ChangeExtension(asm_path, ".webcil")); // hack! + byte[] bytes = File.Exists(asm_path) ? File.ReadAllBytes(asm_path) : File.ReadAllBytes(Path.ChangeExtension(asm_path, WebcilInWasmExtension)); // hack! string asm_base64 = Convert.ToBase64String(bytes); string pdb_base64 = String.Empty; diff --git a/src/tasks/Common/Utils.cs b/src/tasks/Common/Utils.cs index 89f2bfe122491..2f1cbd98fb3b6 100644 --- a/src/tasks/Common/Utils.cs +++ b/src/tasks/Common/Utils.cs @@ -16,6 +16,8 @@ internal static class Utils { + public static string WebcilInWasmExtension = ".wasm"; + private static readonly object s_SyncObj = new object(); public static string GetEmbeddedResource(string file) diff --git a/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/ComputeWasmPublishAssets.cs b/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/ComputeWasmPublishAssets.cs index 0eddaf4d4aed9..b6dfd3392984f 100644 --- a/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/ComputeWasmPublishAssets.cs +++ b/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/ComputeWasmPublishAssets.cs @@ -576,7 +576,7 @@ private void GroupResolvedFilesToPublish( } var extension = candidate.GetMetadata("Extension"); - if (string.Equals(extension, ".dll", StringComparison.Ordinal) || string.Equals(extension, ".webcil", StringComparison.Ordinal)) + if (string.Equals(extension, ".dll", StringComparison.Ordinal) || string.Equals(extension, ".webcil", StringComparison.Ordinal) || string.Equals (extension, Utils.WebcilInWasmExtension, StringComparison.Ordinal)) { var culture = candidate.GetMetadata("Culture"); var inferredCulture = candidate.GetMetadata("DestinationSubDirectory").Replace("\\", "/").Trim('/'); diff --git a/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/ConvertDllsToWebCil.cs b/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/ConvertDllsToWebCil.cs index 17cc80053284d..413fcdff3da76 100644 --- a/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/ConvertDllsToWebCil.cs +++ b/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/ConvertDllsToWebCil.cs @@ -66,7 +66,7 @@ public override bool Execute() if (!Directory.Exists(candicatePath)) Directory.CreateDirectory(candicatePath); - var finalWebcil = Path.Combine(candicatePath, Path.GetFileNameWithoutExtension(filePath) + ".webcil"); + var finalWebcil = Path.Combine(candicatePath, Path.GetFileNameWithoutExtension(filePath) + Utils.WebcilInWasmExtension); if (Utils.CopyIfDifferent(tmpWebcil, finalWebcil, useHash: true)) Log.LogMessage(MessageImportance.Low, $"Generated {finalWebcil} ."); else @@ -75,13 +75,13 @@ public override bool Execute() _fileWrites.Add(finalWebcil); var webcilItem = new TaskItem(finalWebcil, candidate.CloneCustomMetadata()); - webcilItem.SetMetadata("RelativePath", Path.ChangeExtension(candidate.GetMetadata("RelativePath"), ".webcil")); + webcilItem.SetMetadata("RelativePath", Path.ChangeExtension(candidate.GetMetadata("RelativePath"), Utils.WebcilInWasmExtension)); webcilItem.SetMetadata("OriginalItemSpec", finalWebcil); if (webcilItem.GetMetadata("AssetTraitName") == "Culture") { string relatedAsset = webcilItem.GetMetadata("RelatedAsset"); - relatedAsset = Path.ChangeExtension(relatedAsset, ".webcil"); + relatedAsset = Path.ChangeExtension(relatedAsset, Utils.WebcilInWasmExtension); webcilItem.SetMetadata("RelatedAsset", relatedAsset); Log.LogMessage(MessageImportance.Low, $"Changing related asset of {webcilItem} to {relatedAsset}."); } diff --git a/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/GenerateWasmBootJson.cs b/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/GenerateWasmBootJson.cs index 31ed093e9a202..a3760029e9257 100644 --- a/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/GenerateWasmBootJson.cs +++ b/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/GenerateWasmBootJson.cs @@ -174,7 +174,7 @@ public void WriteBootJson(Stream output, string entryAssemblyName) } else if (string.Equals("symbol", assetTraitValue, StringComparison.OrdinalIgnoreCase)) { - if (TryGetLazyLoadedAssembly($"{fileName}.dll", out _) || TryGetLazyLoadedAssembly($"{fileName}.webcil", out _)) + if (TryGetLazyLoadedAssembly($"{fileName}.dll", out _) || TryGetLazyLoadedAssembly($"{fileName}.webcil", out _) || TryGetLazyLoadedAssembly($"{fileName}{Utils.WebcilInWasmExtension}", out _)) { Log.LogMessage(MessageImportance.Low, "Candidate '{0}' is defined as a lazy loaded symbols file.", resource.ItemSpec); resourceData.lazyAssembly ??= new ResourceHashesByNameDictionary(); diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index 439ebf64e7ed5..ffa0c513aa00a 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -172,7 +172,7 @@ protected override bool ExecuteInternal() var tmpWebcil = Path.GetTempFileName(); var webcilWriter = Microsoft.WebAssembly.Build.Tasks.WebcilConverter.FromPortableExecutable(inputPath: assembly, outputPath: tmpWebcil, logger: Log); webcilWriter.ConvertToWebcil(); - var finalWebcil = Path.Combine(asmRootPath, Path.ChangeExtension(Path.GetFileName(assembly), ".webcil")); + var finalWebcil = Path.Combine(asmRootPath, Path.ChangeExtension(Path.GetFileName(assembly), Utils.WebcilInWasmExtension)); if (Utils.CopyIfDifferent(tmpWebcil, finalWebcil, useHash: true)) Log.LogMessage(MessageImportance.Low, $"Generated {finalWebcil} ."); else @@ -232,7 +232,7 @@ protected override bool ExecuteInternal() { if (UseWebcil) { - assemblyPath = Path.Combine(asmRootPath, Path.ChangeExtension(Path.GetFileName(assembly), ".webcil")); + assemblyPath = Path.Combine(asmRootPath, Path.ChangeExtension(Path.GetFileName(assembly), Utils.WebcilInWasmExtension)); // For the hash, read the bytes from the webcil file, not the dll file. bytes = File.ReadAllBytes(assemblyPath); } @@ -260,7 +260,7 @@ protected override bool ExecuteInternal() var tmpWebcil = Path.GetTempFileName(); var webcilWriter = Microsoft.WebAssembly.Build.Tasks.WebcilConverter.FromPortableExecutable(inputPath: args.fullPath, outputPath: tmpWebcil, logger: Log); webcilWriter.ConvertToWebcil(); - var finalWebcil = Path.Combine(directory, Path.ChangeExtension(name, ".webcil")); + var finalWebcil = Path.Combine(directory, Path.ChangeExtension(name, Utils.WebcilInWasmExtension)); if (Utils.CopyIfDifferent(tmpWebcil, finalWebcil, useHash: true)) Log.LogMessage(MessageImportance.Low, $"Generated {finalWebcil} ."); else From 307ae9acf9c597a973720867809372aceec4153d Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Wed, 10 May 2023 21:29:55 -0400 Subject: [PATCH 15/30] fix wbt --- src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs index bf1d6f33c1168..bbcc94a608503 100644 --- a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs +++ b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs @@ -684,7 +684,7 @@ protected static void AssertBasicAppBundle(string bundleDir, string managedDir = Path.Combine(bundleDir, "managed"); string bundledMainAppAssembly = - useWebcil ? $"{projectName}${WebcilInWasmExtension}" : $"{projectName}.dll"; + useWebcil ? $"{projectName}{WebcilInWasmExtension}" : $"{projectName}.dll"; AssertFilesExist(managedDir, new[] { bundledMainAppAssembly }); bool is_debug = config == "Debug"; From 930cfeb052ebb1829e33a309514c8adb6e6f8c83 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 11 May 2023 11:28:09 -0400 Subject: [PATCH 16/30] remove old .webcil support from Sdk Pack Tasks --- .../ComputeWasmPublishAssets.cs | 2 +- .../GenerateWasmBootJson.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/ComputeWasmPublishAssets.cs b/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/ComputeWasmPublishAssets.cs index b6dfd3392984f..ef59c7ca0481e 100644 --- a/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/ComputeWasmPublishAssets.cs +++ b/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/ComputeWasmPublishAssets.cs @@ -576,7 +576,7 @@ private void GroupResolvedFilesToPublish( } var extension = candidate.GetMetadata("Extension"); - if (string.Equals(extension, ".dll", StringComparison.Ordinal) || string.Equals(extension, ".webcil", StringComparison.Ordinal) || string.Equals (extension, Utils.WebcilInWasmExtension, StringComparison.Ordinal)) + if (string.Equals(extension, ".dll", StringComparison.Ordinal) || string.Equals (extension, Utils.WebcilInWasmExtension, StringComparison.Ordinal)) { var culture = candidate.GetMetadata("Culture"); var inferredCulture = candidate.GetMetadata("DestinationSubDirectory").Replace("\\", "/").Trim('/'); diff --git a/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/GenerateWasmBootJson.cs b/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/GenerateWasmBootJson.cs index a3760029e9257..d4a51186f7b4f 100644 --- a/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/GenerateWasmBootJson.cs +++ b/src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/GenerateWasmBootJson.cs @@ -174,7 +174,7 @@ public void WriteBootJson(Stream output, string entryAssemblyName) } else if (string.Equals("symbol", assetTraitValue, StringComparison.OrdinalIgnoreCase)) { - if (TryGetLazyLoadedAssembly($"{fileName}.dll", out _) || TryGetLazyLoadedAssembly($"{fileName}.webcil", out _) || TryGetLazyLoadedAssembly($"{fileName}{Utils.WebcilInWasmExtension}", out _)) + if (TryGetLazyLoadedAssembly($"{fileName}.dll", out _) || TryGetLazyLoadedAssembly($"{fileName}{Utils.WebcilInWasmExtension}", out _)) { Log.LogMessage(MessageImportance.Low, "Candidate '{0}' is defined as a lazy loaded symbols file.", resource.ItemSpec); resourceData.lazyAssembly ??= new ResourceHashesByNameDictionary(); From 7670b3cb1002d86575e01362722390271ea07ba1 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Thu, 11 May 2023 12:02:01 -0500 Subject: [PATCH 17/30] Set SelfContained=true for browser-wasm runtimes (#86102) --- .../WorkloadManifest.targets.in | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in index e5c4a1e176e31..00d3e8b023f1d 100644 --- a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in +++ b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in @@ -30,6 +30,7 @@ + true $(WasmNativeWorkload7) $(WasmNativeWorkload8) $(WasmNativeWorkload) From d17b8ddd725999eab748c7df7466d78a2bae9868 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 11 May 2023 16:25:40 -0400 Subject: [PATCH 18/30] Implement support for webcil in wasm in the managed WebcilReader --- .../src/Webcil/WasmModuleReader.cs | 148 ++++++++++++++++++ .../src/Webcil/WebcilReader.cs | 62 +++++++- 2 files changed, 205 insertions(+), 5 deletions(-) create mode 100644 src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WasmModuleReader.cs diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WasmModuleReader.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WasmModuleReader.cs new file mode 100644 index 0000000000000..a0744c33a8376 --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WasmModuleReader.cs @@ -0,0 +1,148 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Immutable; +using System.IO; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text; + +namespace Microsoft.NET.WebAssembly.Webcil; + +internal class WasmModuleReader : IDisposable +{ + public enum Section : byte + { + // order matters: enum values must match the WebAssembly spec + Custom, + Type, + Import, + Function, + Table, + Memory, + Global, + Export, + Start, + Element, + Code, + Data, + DataCount, + } + + private readonly BinaryReader _reader; + + private readonly Lazy _isWasmModule; + + public bool IsWasmModule => _isWasmModule.Value; + + public WasmModuleReader(Stream stream) + { + _reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: true); + _isWasmModule = new Lazy(this.GetIsWasmModule); + } + + + public void Dispose() + { + Dispose(true); + } + + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + _reader.Dispose(); + } + } + + protected virtual bool VisitSection (Section sec, out bool shouldStop) + { + shouldStop = false; + return true; + } + + private const uint WASM_MAGIC = 0x6d736100u; // "\0asm" + + private bool GetIsWasmModule() + { + _reader.BaseStream.Seek(0, SeekOrigin.Begin); + try + { + uint magic = _reader.ReadUInt32(); + if (magic == WASM_MAGIC) + return true; + } catch (EndOfStreamException) {} + return false; + } + + public bool Visit() + { + if (!IsWasmModule) + return false; + _reader.BaseStream.Seek(4L, SeekOrigin.Begin); // skip magic + + uint version = _reader.ReadUInt32(); + if (version != 1) + return false; + + bool success = true; + while (success) { + success = DoVisitSection (out bool shouldStop); + if (shouldStop) + break; + } + return success; + } + + private bool DoVisitSection(out bool shouldStop) + { + shouldStop = false; + byte code = _reader.ReadByte(); + Section section = (Section)code; + if (!Enum.IsDefined(typeof(Section), section)) + return false; + uint sectionSize = ReadULEB128(); + + long savedPos = _reader.BaseStream.Position; + try + { + return VisitSection(section, out shouldStop); + } + finally + { + _reader.BaseStream.Seek(savedPos + (long)sectionSize, SeekOrigin.Begin); + } + } + + protected uint ReadULEB128() + { + uint val = 0; + int shift = 0; + while (true) + { + byte b = _reader.ReadByte(); + val |= (b & 0x7fu) << shift; + if ((b & 0x80u) == 0) break; + shift += 7; + if (shift >= 35) + throw new OverflowException(); + } + return val; + } + + protected bool TryReadPassiveDataSegment (out long segmentLength, out long segmentStart) + { + segmentLength = 0; + segmentStart = 0; + byte code = _reader.ReadByte(); + if (code != 1) + return false; // not passive + segmentLength = ReadULEB128(); + segmentStart = _reader.BaseStream.Position; + // skip over the data + _reader.BaseStream.Seek (segmentLength, SeekOrigin.Current); + return true; + } +} diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs index 39d58134ce6b4..c6e24931834d3 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs @@ -27,6 +27,8 @@ public sealed partial class WebcilReader : IDisposable private string? InputPath { get; } + private readonly long _webcilInWasmOffset; + public WebcilReader(Stream stream) { this._stream = stream; @@ -34,6 +36,10 @@ public WebcilReader(Stream stream) { throw new ArgumentException("Stream must be readable and seekable", nameof(stream)); } + if (TryReadWasmWrapper(out var webcilInWasmOffset)) { + _webcilInWasmOffset = webcilInWasmOffset; + _stream.Seek(_webcilInWasmOffset, SeekOrigin.Begin); + } if (!ReadHeader()) { throw new BadImageFormatException("Stream does not contain a valid Webcil file", nameof(stream)); @@ -181,7 +187,7 @@ private static ImmutableArray ReadDebugDirectoryEntries(Blo public CodeViewDebugDirectoryData ReadCodeViewDebugDirectoryData(DebugDirectoryEntry entry) { - var pos = entry.DataPointer; + var pos = entry.DataPointer + _webcilInWasmOffset; var buffer = new byte[entry.DataSize]; if (_stream.Seek(pos, SeekOrigin.Begin) != pos) { @@ -227,7 +233,7 @@ private static CodeViewDebugDirectoryData DecodeCodeViewDebugDirectoryData(BlobR public MetadataReaderProvider ReadEmbeddedPortablePdbDebugDirectoryData(DebugDirectoryEntry entry) { - var pos = entry.DataPointer; + var pos = entry.DataPointer + _webcilInWasmOffset; var buffer = new byte[entry.DataSize]; if (_stream.Seek(pos, SeekOrigin.Begin) != pos) { @@ -289,7 +295,7 @@ public PdbChecksumDebugDirectoryData ReadPdbChecksumDebugDirectoryData(DebugDire throw new ArgumentException($"expected debug directory entry type {nameof(DebugDirectoryEntryType.PdbChecksum)}", nameof(entry)); } - var pos = entry.DataPointer; + var pos = entry.DataPointer + _webcilInWasmOffset; var buffer = new byte[entry.DataSize]; if (_stream.Seek(pos, SeekOrigin.Begin) != pos) { @@ -330,7 +336,7 @@ private long TranslateRVA(uint rva) { if (rva >= section.VirtualAddress && rva < section.VirtualAddress + section.VirtualSize) { - return section.PointerToRawData + (rva - section.VirtualAddress); + return section.PointerToRawData + (rva - section.VirtualAddress) + _webcilInWasmOffset; } } throw new BadImageFormatException("RVA not found in any section", nameof(_stream)); @@ -342,7 +348,7 @@ private unsafe ImmutableArray ReadSections() { var sections = ImmutableArray.CreateBuilder(_header.coff_sections); var buffer = new byte[Marshal.SizeOf()]; - _stream.Seek(SectionDirectoryOffset, SeekOrigin.Begin); + _stream.Seek(SectionDirectoryOffset + _webcilInWasmOffset, SeekOrigin.Begin); for (int i = 0; i < _header.coff_sections; i++) { if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) @@ -362,4 +368,50 @@ public void Dispose() { _stream.Dispose(); } + + private bool TryReadWasmWrapper(out long webcilInWasmOffset) + { + webcilInWasmOffset = 0; + using var reader = new WasmWrapperModuleReader(_stream); + if (!reader.IsWasmModule) + return false; + if (!reader.Visit()) + return false; + if (!reader.HasWebcil) + return false; + webcilInWasmOffset = reader.WebcilPayloadOffset; + return true; + } + + private sealed class WasmWrapperModuleReader : WasmModuleReader + { + internal bool HasWebcil {get; private set;} + internal long WebcilPayloadOffset {get; private set; } + public WasmWrapperModuleReader(Stream stream) : base (stream) + { + } + + protected override bool VisitSection (WasmModuleReader.Section sec, out bool shouldStop) + { + shouldStop = false; + if (sec != WasmModuleReader.Section.Data) + return true; + shouldStop = true; + + uint numSegments = ReadULEB128(); + if (numSegments != 2) + return false; + + // skip the first segment + if (!TryReadPassiveDataSegment (out long _, out long _)) + return false; + + if (!TryReadPassiveDataSegment (out long _, out long segmentStart)) + return false; + + HasWebcil = true; + WebcilPayloadOffset = segmentStart; + return true; + } + } } From 61afae7da8b8c8940dd46b49162e8077d1539321 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 11 May 2023 21:03:12 -0400 Subject: [PATCH 19/30] why fail? --- src/mono/wasm/Wasm.Build.Tests/IcuShardingTests.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/mono/wasm/Wasm.Build.Tests/IcuShardingTests.cs b/src/mono/wasm/Wasm.Build.Tests/IcuShardingTests.cs index 19a4ac6343468..e794f0fd880e1 100644 --- a/src/mono/wasm/Wasm.Build.Tests/IcuShardingTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/IcuShardingTests.cs @@ -128,10 +128,18 @@ private static string GetProgramText(string testedLocales, bool onlyPredefinedCu continue; }} }} - catch(CultureNotFoundException cnfe) when (ctorShouldFail && cnfe.Message.Contains($""{{testLocale.Code}} is an invalid culture identifier."")) + catch(CultureNotFoundException cnfe) {{ - Console.WriteLine($""{{testLocale.Code}}: Success. .ctor failed as expected.""); - continue; + if (ctorShouldFail && cnfe.Message.Contains($""{{testLocale.Code}} is an invalid culture identifier."")) {{ + Console.WriteLine($""{{testLocale.Code}}: Success. .ctor failed as expected.""); + continue; + }} else {{ + if (ctorShouldFail) + Console.WriteLine($""{{testLocale.Code}}: Failed with the wrong message: {{cnfe.Message}}""); + else + Console.WriteLine($""{{testLocale.Code}}: Should not have failed, but did.""); + throw; + }} }} string expectedSundayName = (expectMissing && !onlyPredefinedCultures) From 8a42b7031b16216dcbeef93c43d3d4b4914170c6 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 11 May 2023 22:40:46 -0400 Subject: [PATCH 20/30] did we load the same asm twice?? --- src/mono/wasm/Wasm.Build.Tests/WasmTemplateTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mono/wasm/Wasm.Build.Tests/WasmTemplateTests.cs b/src/mono/wasm/Wasm.Build.Tests/WasmTemplateTests.cs index 2d9100b6832d5..8adb64aea645b 100644 --- a/src/mono/wasm/Wasm.Build.Tests/WasmTemplateTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/WasmTemplateTests.cs @@ -367,10 +367,12 @@ public void ConsolePublishAndRun(string config, bool aot, bool relinking) UpdateProgramCS(); UpdateConsoleMainJs(); + if (!aot) + UpdateMainJsEnvironmentVariables(("MONO_LOG_MASK", "asm"), ("MONO_LOG_LEVEL", "debug")); if (aot) { // FIXME: pass envvars via the environment, once that is supported - UpdateMainJsEnvironmentVariables(("MONO_LOG_MASK", "aot"), ("MONO_LOG_LEVEL", "debug")); + UpdateMainJsEnvironmentVariables(("MONO_LOG_MASK", "aot,asm"), ("MONO_LOG_LEVEL", "debug")); AddItemsPropertiesToProject(projectFile, "true"); } else if (relinking) From 6166a675ebff74633642ff768926641d1c04b45b Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 12 May 2023 16:41:04 -0400 Subject: [PATCH 21/30] checkpoint. things are broken. but I adjusted MonoImage:raw_data --- .../src/Webcil/WebcilConverter.cs | 1 + src/mono/llvm/llvm-init.proj | 2 +- src/mono/mono/metadata/image.c | 7 ++++++- src/mono/mono/metadata/object.c | 3 +++ src/mono/mono/metadata/webcil-loader.c | 14 ++++++++++++-- src/mono/wasm/build/WasmApp.LocalBuild.props | 6 ++++++ 6 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs index dfec29cf6a01a..a38af7270a2da 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs @@ -75,6 +75,7 @@ public void ConvertToWebcil() // webcil is about the same size as the PE file using var memoryStream = new MemoryStream(checked((int)inputStream.Length)); WriteConversionTo(memoryStream, inputStream, peInfo, wcInfo); + memoryStream.Flush(); var wrapper = new WebcilWasmWrapper(memoryStream); memoryStream.Seek(0, SeekOrigin.Begin); wrapper.WriteWasmWrappedWebcil(outputStream); diff --git a/src/mono/llvm/llvm-init.proj b/src/mono/llvm/llvm-init.proj index c7351f4f8d055..ee4c0b4446a07 100644 --- a/src/mono/llvm/llvm-init.proj +++ b/src/mono/llvm/llvm-init.proj @@ -86,7 +86,7 @@ - + diff --git a/src/mono/mono/metadata/image.c b/src/mono/mono/metadata/image.c index 0eea10e3cdc18..b4faf3a2e139a 100644 --- a/src/mono/mono/metadata/image.c +++ b/src/mono/mono/metadata/image.c @@ -1349,7 +1349,7 @@ mono_image_storage_dtor (gpointer self) } } if (storage->raw_data_allocated) { - g_free (storage->raw_data); + g_free (storage->raw_data_handle); } g_free (storage->key); @@ -1430,6 +1430,7 @@ mono_image_storage_new_raw_data (char *datac, guint32 data_len, gboolean raw_dat storage->raw_data = datac; storage->raw_data_len = data_len; storage->raw_data_allocated = !!raw_data_allocated; + storage->raw_data_handle = datac; storage->key = key; MonoImageStorage *other_storage = NULL; @@ -2546,16 +2547,20 @@ mono_image_get_resource (MonoImage *image, guint32 offset, guint32 *size) MonoCLIHeader *ch = &iinfo->cli_cli_header; const char* data; + mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_ASSEMBLY, "getting resource from offset 0x%x in image %s [%p].", offset, image->name, image); + if (!ch->ch_resources.rva || offset + 4 > ch->ch_resources.size) return NULL; data = mono_image_rva_map (image, ch->ch_resources.rva); + mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_ASSEMBLY, "getting resource from offset 0x%x in image %s [%p]: resource rva is %p mapped to %p", offset, image->name, image, (void*)ch->ch_resources.rva, (void*)data); if (!data) return NULL; data += offset; if (size) *size = read32 (data); data += 4; + mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_ASSEMBLY, "getting resource from offset 0x%x in image %s [%p]: resource size is 0x%08x, first word is 0x%08x", offset, image->name, image, *size, *(int32_t*)data); return data; } diff --git a/src/mono/mono/metadata/object.c b/src/mono/mono/metadata/object.c index c8fb3f7913578..1a70fbcaa49b3 100644 --- a/src/mono/mono/metadata/object.c +++ b/src/mono/mono/metadata/object.c @@ -4532,6 +4532,9 @@ mono_unhandled_exception_checked (MonoObjectHandle exc, MonoError *error) if (!field) goto leave; + MonoClass *k = MONO_HANDLE_RAW(exc)->vtable->klass; + g_warning ("unhandled execption '%s.%s' from image %s (%p)\n", m_class_get_name_space (k), m_class_get_name (k), m_class_get_image (k)->name, (void*)m_class_get_image (k)); + MonoObject *delegate = NULL; MonoObjectHandle delegate_handle; MonoVTable *vt = mono_class_vtable_checked (mono_defaults.appcontext_class, error); diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c index 81e790c0d95a2..22ffdb5c27fd9 100644 --- a/src/mono/mono/metadata/webcil-loader.c +++ b/src/mono/mono/metadata/webcil-loader.c @@ -8,6 +8,7 @@ #include "mono/metadata/metadata-internals.h" #include "mono/metadata/webcil-loader.h" +#include "mono/utils/mono-logger-internals.h" #include "mono/utils/wasm-module-reader.h" /* keep in sync with webcil-writer */ @@ -134,7 +135,16 @@ webcil_image_load_pe_data (MonoImage *image) offset = do_load_header (image->raw_data, image->raw_data_len, offset, header, &webcil_section_adjustment); if (offset == -1) goto invalid_image; - + /* HACK! RVAs and debug table entry pointers are from the beginning of the webcil payload. adjust MonoImage:raw_data to point to it */ + g_assert (image->ref_count == 1); + g_assert (image->storage->ref.ref == 1); + mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_ASSEMBLY, "Adjusting offset image %s [%p].", image->name, image); + image->storage->raw_data += webcil_section_adjustment; + image->storage->raw_data_len -= webcil_section_adjustment; + image->raw_data = image->storage->raw_data; + image->raw_data_len = image->storage->raw_data_len; + offset -= webcil_section_adjustment; + top = iinfo->cli_header.coff.coff_sections; iinfo->cli_section_count = top; @@ -143,7 +153,7 @@ webcil_image_load_pe_data (MonoImage *image) for (int i = 0; i < top; i++) { MonoSectionTable *t = &iinfo->cli_section_tables [i]; - offset = mono_webcil_load_section_table (image->raw_data, image->raw_data_len, offset, webcil_section_adjustment, t); + offset = mono_webcil_load_section_table (image->raw_data, image->raw_data_len, offset, /*webcil_section_adjustment*/ 0, t); if (offset == -1) goto invalid_image; } diff --git a/src/mono/wasm/build/WasmApp.LocalBuild.props b/src/mono/wasm/build/WasmApp.LocalBuild.props index 63f06f51252e0..31b94d4d425dc 100644 --- a/src/mono/wasm/build/WasmApp.LocalBuild.props +++ b/src/mono/wasm/build/WasmApp.LocalBuild.props @@ -68,4 +68,10 @@ $([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'MonoTargetsTasks.dll')) + + + + + From eedf44779ba8ca709e8d683c4a2bcec2ff83d712 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 15 May 2023 16:22:47 -0400 Subject: [PATCH 22/30] align webcil payload to a 4-byte boundary within the wasm module Add padding to data segment 0 to ensure that data segment 1's payload (ie the webcil content itself) is 4-byte aligned --- .../src/Webcil/WebcilWasmWrapper.cs | 66 +++++++++++++++++-- src/mono/mono/metadata/webcil-loader.c | 5 +- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs index 172dc87940130..3f8560446306f 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilWasmWrapper.cs @@ -110,27 +110,62 @@ private static void WriteWasmSuffix(Stream outputStream) // 1 byte to encode "passive" data segment private const uint SegmentCodeSize = 1; + // Align the payload start to a 4-byte boundary within the wrapper. If the runtime reads the + // payload directly, instead of by instantiatng the wasm module, we don't want the WebAssembly + // prefix to push some of the values inside the image to odd byte offsets as the runtime assumes + // the image will be aligned. + // + // There are requirements in ECMA-335 (Section II.25.4) that fat method headers and method data + // sections be 4-byte aligned. + private const uint WebcilPayloadInternalAlignment = 4; + private void WriteDataSection(BinaryWriter writer) { + uint dataSectionSize = 0; // uleb128 encoding of number of segments dataSectionSize += 1; // there's always 2 segments which encodes to 1 byte // compute the segment 0 size: - // segment 0 has 1 byte segment code, 1 byte of size and 4 bytes of payload - dataSectionSize += SegmentCodeSize + 1 + 4; + // segment 0 has 1 byte segment code, 1 byte of size and at least 4 bytes of payload + uint segment0MinimumSize = SegmentCodeSize + 1 + 4; + dataSectionSize += segment0MinimumSize; // encode webcil size as a uleb128 - byte[] ulebSegmentSize = ULEB128Encode(_webcilPayloadSize); + byte[] ulebWebcilPayloadSize = ULEB128Encode(_webcilPayloadSize); // compute the segment 1 size: // segment 1 has 1 byte segment code, a uleb128 encoding of the webcilPayloadSize, and the payload + // don't count the size of the payload yet + checked + { + dataSectionSize += SegmentCodeSize + (uint)ulebWebcilPayloadSize.Length; + } + + // at this point the data section size includes everything except the data section code, the data section size and the webcil payload itself + // and any extra padding that we may want to add to segment 0. + // So we can compute the offset of the payload within the wasm module. + byte[] putativeULEBDataSectionSize = ULEB128Encode(dataSectionSize + _webcilPayloadSize); + uint payloadOffset = (uint)s_wasmWrapperPrefix.Length + 1 + (uint)putativeULEBDataSectionSize.Length + dataSectionSize ; + + uint paddingSize = PadTo(payloadOffset, WebcilPayloadInternalAlignment); + + if (paddingSize > 0) + { + checked + { + dataSectionSize += paddingSize; + } + } + checked { - dataSectionSize += SegmentCodeSize + (uint)ulebSegmentSize.Length + _webcilPayloadSize; + dataSectionSize += _webcilPayloadSize; } byte[] ulebSectionSize = ULEB128Encode(dataSectionSize); + if (putativeULEBDataSectionSize.Length != ulebSectionSize.Length) + throw new InvalidOperationException ("adding padding would cause data section's encoded length to chane"); // TODO: fixme: there's upto one extra byte to encode the section length - take away a padding byte. writer.Write((byte)11); // section Data writer.Write(ulebSectionSize, 0, ulebSectionSize.Length); @@ -138,12 +173,20 @@ private void WriteDataSection(BinaryWriter writer) // write segment 0 writer.Write((byte)1); // passive segment - writer.Write((byte)4); // segment size: 4 + if (paddingSize + 4 > 127) { + throw new InvalidOperationException ("padding would cause segment 0 to need a multi-byte ULEB128 size encoding"); + } + writer.Write((byte)(4 + paddingSize)); // segment size: 4 plus any padding writer.Write((uint)_webcilPayloadSize); // payload is an unsigned 32 bit number + for (int i = 0; i < paddingSize; i++) + writer.Write((byte)0); // write segment 1 writer.Write((byte)1); // passive segment - writer.Write(ulebSegmentSize, 0, ulebSegmentSize.Length); // segment size: _webcilPayloadSize + writer.Write(ulebWebcilPayloadSize, 0, ulebWebcilPayloadSize.Length); // segment size: _webcilPayloadSize + if (writer.BaseStream.Position % WebcilPayloadInternalAlignment != 0) { + throw new Exception ($"predited offset {payloadOffset}, actual position {writer.BaseStream.Position}"); + } _webcilPayloadStream.CopyTo(writer.BaseStream); // payload is the entire webcil content } @@ -169,4 +212,15 @@ private static byte[] ULEB128Encode(uint value) } while (n != 0); return arr; } + + private static uint PadTo (uint value, uint align) + { + uint newValue = AlignTo(value, align); + return newValue - value; + } + + private static uint AlignTo (uint value, uint align) + { + return (value + (align - 1)) & ~(align - 1); + } } diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c index 22ffdb5c27fd9..a391c6a7d3a0d 100644 --- a/src/mono/mono/metadata/webcil-loader.c +++ b/src/mono/mono/metadata/webcil-loader.c @@ -144,6 +144,9 @@ webcil_image_load_pe_data (MonoImage *image) image->raw_data = image->storage->raw_data; image->raw_data_len = image->storage->raw_data_len; offset -= webcil_section_adjustment; + if (((intptr_t)image->raw_data) % 4 != 0) { + g_warning ("webcil image %s [%p] raw data %p not 4 byte aligned\n", image->name, image, image->raw_data); + } top = iinfo->cli_header.coff.coff_sections; @@ -230,7 +233,7 @@ webcil_in_wasm_section_visitor (uint8_t sec_code, const uint8_t *sec_content, ui if (num_segments != 2) return FALSE; - // skip over data segment 0, it's the webcil payload length as a u32 - we don't care about it + // skip over data segment 0, it's the webcil payload length as a u32 plus padding - we don't care about it uint32_t passive_segment_len = 0; const uint8_t *passive_segment_start = NULL; if (!mono_wasm_module_decode_passive_data_segment (ptr, boundp, &ptr, &passive_segment_len, &passive_segment_start)) From d9b396e9af81cecd75216a7b3e3ab9e43d09f11d Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 15 May 2023 22:05:22 -0400 Subject: [PATCH 23/30] remove WIP tracing --- src/mono/mono/metadata/image.c | 4 ---- src/mono/mono/metadata/object.c | 3 --- 2 files changed, 7 deletions(-) diff --git a/src/mono/mono/metadata/image.c b/src/mono/mono/metadata/image.c index b4faf3a2e139a..6e07a5da28d48 100644 --- a/src/mono/mono/metadata/image.c +++ b/src/mono/mono/metadata/image.c @@ -2547,20 +2547,16 @@ mono_image_get_resource (MonoImage *image, guint32 offset, guint32 *size) MonoCLIHeader *ch = &iinfo->cli_cli_header; const char* data; - mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_ASSEMBLY, "getting resource from offset 0x%x in image %s [%p].", offset, image->name, image); - if (!ch->ch_resources.rva || offset + 4 > ch->ch_resources.size) return NULL; data = mono_image_rva_map (image, ch->ch_resources.rva); - mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_ASSEMBLY, "getting resource from offset 0x%x in image %s [%p]: resource rva is %p mapped to %p", offset, image->name, image, (void*)ch->ch_resources.rva, (void*)data); if (!data) return NULL; data += offset; if (size) *size = read32 (data); data += 4; - mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_ASSEMBLY, "getting resource from offset 0x%x in image %s [%p]: resource size is 0x%08x, first word is 0x%08x", offset, image->name, image, *size, *(int32_t*)data); return data; } diff --git a/src/mono/mono/metadata/object.c b/src/mono/mono/metadata/object.c index 1a70fbcaa49b3..c8fb3f7913578 100644 --- a/src/mono/mono/metadata/object.c +++ b/src/mono/mono/metadata/object.c @@ -4532,9 +4532,6 @@ mono_unhandled_exception_checked (MonoObjectHandle exc, MonoError *error) if (!field) goto leave; - MonoClass *k = MONO_HANDLE_RAW(exc)->vtable->klass; - g_warning ("unhandled execption '%s.%s' from image %s (%p)\n", m_class_get_name_space (k), m_class_get_name (k), m_class_get_image (k)->name, (void*)m_class_get_image (k)); - MonoObject *delegate = NULL; MonoObjectHandle delegate_handle; MonoVTable *vt = mono_class_vtable_checked (mono_defaults.appcontext_class, error); From 89ef589517fc24d048ff84a2a7454d50cda1d70b Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 15 May 2023 22:09:30 -0400 Subject: [PATCH 24/30] assert that webcil raw data is 4-byte aligned --- src/mono/mono/metadata/webcil-loader.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c index a391c6a7d3a0d..fbc08d256a357 100644 --- a/src/mono/mono/metadata/webcil-loader.c +++ b/src/mono/mono/metadata/webcil-loader.c @@ -144,10 +144,9 @@ webcil_image_load_pe_data (MonoImage *image) image->raw_data = image->storage->raw_data; image->raw_data_len = image->storage->raw_data_len; offset -= webcil_section_adjustment; - if (((intptr_t)image->raw_data) % 4 != 0) { - g_warning ("webcil image %s [%p] raw data %p not 4 byte aligned\n", image->name, image, image->raw_data); - } - + // parts of ecma-335 loading depend on 4-byte alignment of the image + g_assertf (((intptr_t)image->raw_data) % 4 == 0, "webcil image %s [%p] raw data %p not 4 byte aligned\n", image->name, image, image->raw_data); + top = iinfo->cli_header.coff.coff_sections; iinfo->cli_section_count = top; From d9a3fea159fd41a1d0942997f5a88099f0cc68c3 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 15 May 2023 22:09:43 -0400 Subject: [PATCH 25/30] revert unrelated build change --- src/mono/wasm/build/WasmApp.LocalBuild.props | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/mono/wasm/build/WasmApp.LocalBuild.props b/src/mono/wasm/build/WasmApp.LocalBuild.props index 31b94d4d425dc..63f06f51252e0 100644 --- a/src/mono/wasm/build/WasmApp.LocalBuild.props +++ b/src/mono/wasm/build/WasmApp.LocalBuild.props @@ -68,10 +68,4 @@ $([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'MonoTargetsTasks.dll')) - - - - - From 6de7deef39fbb759dc2eea1c788e54694269357f Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 15 May 2023 22:13:46 -0400 Subject: [PATCH 26/30] revert unrelated change --- src/mono/llvm/llvm-init.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/llvm/llvm-init.proj b/src/mono/llvm/llvm-init.proj index ee4c0b4446a07..c7351f4f8d055 100644 --- a/src/mono/llvm/llvm-init.proj +++ b/src/mono/llvm/llvm-init.proj @@ -86,7 +86,7 @@ - + From 88f956e2e581387556a01e94bf9d710049b7ec48 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 15 May 2023 22:13:55 -0400 Subject: [PATCH 27/30] revert whitespace --- src/mono/mono/metadata/webcil-loader.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c index fbc08d256a357..00d4987c6e84f 100644 --- a/src/mono/mono/metadata/webcil-loader.c +++ b/src/mono/mono/metadata/webcil-loader.c @@ -252,7 +252,6 @@ find_webcil_in_wasm (const uint8_t *ptr, const uint8_t *boundp, const uint8_t ** visitor.section_visitor = &webcil_in_wasm_section_visitor; if (!mono_wasm_module_visit(ptr, boundp, &visitor, &user_data)) return FALSE; - *webcil_payload_start = user_data.data_segment_1_start; return TRUE; } From 0542c51f4af1a521430c2d136ea1fef6b173ad12 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 15 May 2023 22:14:05 -0400 Subject: [PATCH 28/30] revert WBT debugging output changes --- src/mono/wasm/Wasm.Build.Tests/IcuShardingTests.cs | 14 +++----------- .../wasm/Wasm.Build.Tests/WasmTemplateTests.cs | 4 +--- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/mono/wasm/Wasm.Build.Tests/IcuShardingTests.cs b/src/mono/wasm/Wasm.Build.Tests/IcuShardingTests.cs index e794f0fd880e1..19a4ac6343468 100644 --- a/src/mono/wasm/Wasm.Build.Tests/IcuShardingTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/IcuShardingTests.cs @@ -128,18 +128,10 @@ private static string GetProgramText(string testedLocales, bool onlyPredefinedCu continue; }} }} - catch(CultureNotFoundException cnfe) + catch(CultureNotFoundException cnfe) when (ctorShouldFail && cnfe.Message.Contains($""{{testLocale.Code}} is an invalid culture identifier."")) {{ - if (ctorShouldFail && cnfe.Message.Contains($""{{testLocale.Code}} is an invalid culture identifier."")) {{ - Console.WriteLine($""{{testLocale.Code}}: Success. .ctor failed as expected.""); - continue; - }} else {{ - if (ctorShouldFail) - Console.WriteLine($""{{testLocale.Code}}: Failed with the wrong message: {{cnfe.Message}}""); - else - Console.WriteLine($""{{testLocale.Code}}: Should not have failed, but did.""); - throw; - }} + Console.WriteLine($""{{testLocale.Code}}: Success. .ctor failed as expected.""); + continue; }} string expectedSundayName = (expectMissing && !onlyPredefinedCultures) diff --git a/src/mono/wasm/Wasm.Build.Tests/WasmTemplateTests.cs b/src/mono/wasm/Wasm.Build.Tests/WasmTemplateTests.cs index 04e917ad40a55..ea57866fc86fa 100644 --- a/src/mono/wasm/Wasm.Build.Tests/WasmTemplateTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/WasmTemplateTests.cs @@ -367,12 +367,10 @@ public void ConsolePublishAndRun(string config, bool aot, bool relinking) UpdateProgramCS(); UpdateConsoleMainJs(); - if (!aot) - UpdateMainJsEnvironmentVariables(("MONO_LOG_MASK", "asm"), ("MONO_LOG_LEVEL", "debug")); if (aot) { // FIXME: pass envvars via the environment, once that is supported - UpdateMainJsEnvironmentVariables(("MONO_LOG_MASK", "aot,asm"), ("MONO_LOG_LEVEL", "debug")); + UpdateMainJsEnvironmentVariables(("MONO_LOG_MASK", "aot"), ("MONO_LOG_LEVEL", "debug")); AddItemsPropertiesToProject(projectFile, "true"); } else if (relinking) From 7c0643d555a80f9a2ff3b4d0aa89db64e92419ba Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 15 May 2023 22:21:05 -0400 Subject: [PATCH 29/30] add 4-byte alignment requirement to the webcil spec --- docs/design/mono/webcil.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/design/mono/webcil.md b/docs/design/mono/webcil.md index 1f3f21fdcb010..346d7633f6a94 100644 --- a/docs/design/mono/webcil.md +++ b/docs/design/mono/webcil.md @@ -51,6 +51,10 @@ as passive data segments 0 and 1, respectively. The module must not contain add segments. The module must store the payload size in data segment 0, and the payload content in data segment 1. +The payload content in data segment 1 must be aligned on a 4-byte boundary within the web assembly +module. Additional trailing padding may be added to the data segment 0 content to correctly align +data segment 1's content. + (**Rationale**: With this wrapper it is possible to split the WebAssembly module into a *prefix* consisting of everything before the data section, the data section, and a *suffix* that consists of everything after the data section. The prefix and suffix do not depend on the contents of the @@ -63,6 +67,11 @@ allows a runtime that does not include a WebAssembly host or a runtime that does instantiate the WebAssembly module to extract the payload by traversing the WebAssembly module and locating the Webcil payload in the data section at segment 1.) +(**Rationale**: The alignment requirement is due to ECMA-335 metadata requiring certain portions of +the physical layout to be 4-byte aligned, for example ECMA-335 Section II.25.4 and II.25.4.5. +Aligning the Webcil content within the wasm module allows tools that directly examine the wasm +module without instantiating it to properly parse the ECMA-335 metadata in the Webcil payload.) + (**Note**: the wrapper may be versioned independently of the payload.) From e187b00be3ad3b84e598983363efea1a3e152d64 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 16 May 2023 09:52:27 -0400 Subject: [PATCH 30/30] Don't modify MonoImageStorage:raw_data instead just keep track of the webcil offset in the MonoImageStorage. This introduces a situation where MonoImage:raw_data is different from MonoImageStorage:raw_data. The one to use for accessing IL and metadata is MonoImage:raw_data. The storage pointer is just used by the image loading machinery --- src/mono/mono/metadata/metadata-internals.h | 8 +++++++- src/mono/mono/metadata/webcil-loader.c | 14 +++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/mono/mono/metadata/metadata-internals.h b/src/mono/mono/metadata/metadata-internals.h index 009861f03dfe4..e073ab2ea4831 100644 --- a/src/mono/mono/metadata/metadata-internals.h +++ b/src/mono/mono/metadata/metadata-internals.h @@ -282,6 +282,12 @@ typedef struct { /* Module entry point is _CorDllMain. */ guint8 has_entry_point : 1; #endif +#ifdef ENABLE_WEBCIL + /* set to a non-zero value when we load a webcil-in-wasm image. + * Note that in that case MonoImage:raw_data is not equal to MonoImageStorage:raw_data + */ + int32_t webcil_section_adjustment; +#endif } MonoImageStorage; struct _MonoImage { @@ -297,7 +303,7 @@ struct _MonoImage { MonoImageStorage *storage; - /* Aliases storage->raw_data when storage is non-NULL. Otherwise NULL. */ + /* Points into storage->raw_data when storage is non-NULL. Otherwise NULL. */ char *raw_data; guint32 raw_data_len; diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c index 00d4987c6e84f..c96523af29373 100644 --- a/src/mono/mono/metadata/webcil-loader.c +++ b/src/mono/mono/metadata/webcil-loader.c @@ -137,12 +137,16 @@ webcil_image_load_pe_data (MonoImage *image) goto invalid_image; /* HACK! RVAs and debug table entry pointers are from the beginning of the webcil payload. adjust MonoImage:raw_data to point to it */ g_assert (image->ref_count == 1); - g_assert (image->storage->ref.ref == 1); + // NOTE: image->storage->raw_data could be shared if we loaded this image multiple times (for different ALCs, for example) + // Do not adjust image->storage->raw_data. +#ifdef ENABLE_WEBCIL + int32_t old_adjustment; + old_adjustment = mono_atomic_cas_i32 ((volatile gint32*)&image->storage->webcil_section_adjustment, webcil_section_adjustment, 0); + g_assert (old_adjustment == 0 || old_adjustment == webcil_section_adjustment); +#endif mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_ASSEMBLY, "Adjusting offset image %s [%p].", image->name, image); - image->storage->raw_data += webcil_section_adjustment; - image->storage->raw_data_len -= webcil_section_adjustment; - image->raw_data = image->storage->raw_data; - image->raw_data_len = image->storage->raw_data_len; + image->raw_data += webcil_section_adjustment; + image->raw_data_len -= webcil_section_adjustment; offset -= webcil_section_adjustment; // parts of ecma-335 loading depend on 4-byte alignment of the image g_assertf (((intptr_t)image->raw_data) % 4 == 0, "webcil image %s [%p] raw data %p not 4 byte aligned\n", image->name, image, image->raw_data);