diff --git a/.circleci/config.yml b/.circleci/config.yml index ccb5451981f6d..ce3e2fa2e76cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -420,6 +420,16 @@ jobs: apt-get update -q apt-get install -q -y python3 cmake - checkout + - run: + name: get wasmer + command: | + curl https://get.wasmer.io -sSfL | sh + - run: + name: get wasmtime + command: | + wget https://github.com/CraneStation/wasmtime/releases/download/dev/wasmtime-dev-x86_64-linux.tar.xz + tar -xf wasmtime-dev-x86_64-linux.tar.xz + cp wasmtime-dev-x86_64-linux/wasmtime ~/ - build-upstream test-upstream-wasm0: executor: bionic diff --git a/ChangeLog.md b/ChangeLog.md index 867137e2a27be..4226fcbefb4e4 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -21,6 +21,10 @@ Current Trunk - Module.abort is no longer exported by default. It can be exported in the normal way using `EXTRA_EXPORTED_RUNTIME_METHODS`, and as with other such changes in the past, forgetting to export it with show a clear error in `ASSERTIONS` mode. + - Remove `EMITTING_JS` flag, and replace it with `STANDALONE_WASM`. That flag indicates + that we want the wasm to be as standalone as possible. We may still emit JS in + that case, but the JS would just be a convenient way to run the wasm on the Web + or in Node.js. v.1.38.44: 09/11/2019 --------------------- diff --git a/emcc.py b/emcc.py index 0ed9f9895738a..ee62c3c048631 100755 --- a/emcc.py +++ b/emcc.py @@ -1697,9 +1697,14 @@ def check_human_readable_list(items): if use_source_map(options): exit_with_error('wasm2js does not support source maps yet (debug in wasm for now)') - # wasm outputs are only possible with a side wasm if target.endswith(WASM_ENDINGS): - shared.Settings.EMITTING_JS = 0 + # if the output is just a wasm file, it will normally be a standalone one, + # as there is no JS. an exception are side modules, as we can't tell at + # compile time whether JS will be involved or not - the main module may + # have JS, and the side module is expected to link against that. + # we also do not support standalone mode in fastcomp. + if shared.Settings.WASM_BACKEND and not shared.Settings.SIDE_MODULE: + shared.Settings.STANDALONE_WASM = 1 js_target = misc_temp_files.get(suffix='.js').name if shared.Settings.EVAL_CTORS: @@ -1764,6 +1769,19 @@ def check_human_readable_list(items): if shared.Settings.MINIMAL_RUNTIME and not shared.Settings.WASM: options.separate_asm = True + if shared.Settings.STANDALONE_WASM: + if not shared.Settings.WASM_BACKEND: + exit_with_error('STANDALONE_WASM is only available in the upstream wasm backend path') + if shared.Settings.USE_PTHREADS: + exit_with_error('STANDALONE_WASM does not support pthreads yet') + if shared.Settings.SIMD: + exit_with_error('STANDALONE_WASM does not support simd yet') + if shared.Settings.ALLOW_MEMORY_GROWTH: + exit_with_error('STANDALONE_WASM does not support memory growth yet') + # the wasm must be runnable without the JS, so there cannot be anything that + # requires JS legalization + shared.Settings.LEGALIZE_JS_FFI = 0 + if shared.Settings.WASM_BACKEND: if shared.Settings.SIMD: newargs.append('-msimd128') @@ -3012,7 +3030,8 @@ def do_binaryen(target, asm_target, options, memfile, wasm_binary_target, wasm_file=wasm_binary_target, expensive_optimizations=will_metadce(options), minify_whitespace=optimizer.minify_whitespace, - debug_info=intermediate_debug_info) + debug_info=intermediate_debug_info, + emitting_js=not target.endswith(WASM_ENDINGS)) save_intermediate_with_wasm('postclean', wasm_binary_target) def run_closure_compiler(final): diff --git a/emscripten.py b/emscripten.py index 1ccfd76df9e2d..380da4d831850 100644 --- a/emscripten.py +++ b/emscripten.py @@ -2321,6 +2321,8 @@ def debug_copy(src, dst): cmd.append('--global-base=%s' % shared.Settings.GLOBAL_BASE) if shared.Settings.SAFE_STACK: cmd.append('--check-stack-overflow') + if shared.Settings.STANDALONE_WASM: + cmd.append('--standalone-wasm') shared.print_compiler_stage(cmd) stdout = shared.check_call(cmd, stdout=subprocess.PIPE).stdout if write_source_map: diff --git a/site/source/docs/tools_reference/emcc.rst b/site/source/docs/tools_reference/emcc.rst index 3439e226dd565..d36115a29c13e 100644 --- a/site/source/docs/tools_reference/emcc.rst +++ b/site/source/docs/tools_reference/emcc.rst @@ -443,7 +443,7 @@ Options that are modified or new in *emcc* are listed below: - **.html** : HTML + separate JavaScript file (**.js**; + separate **.wasm** file if emitting WebAssembly). - **.bc** : LLVM bitcode. - **.o** : LLVM bitcode (same as .bc), unless in `WASM_OBJECT_FILES` mode, in which case it will contain a WebAssembly object. - - **.wasm** : WebAssembly without JavaScript support code ("standalone wasm"). + - **.wasm** : WebAssembly without JavaScript support code ("standalone wasm"; this enables ``STANDALONE_WASM``). .. note:: If ``--memory-init-file`` is used, a **.mem** file will be created in addition to the generated **.js** and/or **.html** file. diff --git a/src/library_wasi.js b/src/library_wasi.js new file mode 100644 index 0000000000000..8f082e0fbd9a0 --- /dev/null +++ b/src/library_wasi.js @@ -0,0 +1,14 @@ +/* + * Copyright 2019 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +mergeInto(LibraryManager.library, { + proc_exit__deps: ['exit'], + proc_exit: function(code) { + return _exit(code); + }, +}); + diff --git a/src/modules.js b/src/modules.js index 1f03a3fa30e5c..85e281fe4fbed 100644 --- a/src/modules.js +++ b/src/modules.js @@ -152,6 +152,10 @@ var LibraryManager = { libraries.push('library_glemu.js'); } + if (STANDALONE_WASM) { + libraries.push('library_wasi.js'); + } + libraries = libraries.concat(additionalLibraries); if (BOOTSTRAPPING_STRUCT_INFO) libraries = ['library_bootstrap_structInfo.js', 'library_formatString.js']; diff --git a/src/preamble.js b/src/preamble.js index 06e421e66a770..d7fa22ba71179 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -989,6 +989,12 @@ function createWasm() { exports = Asyncify.instrumentWasmExports(exports); #endif Module['asm'] = exports; +#if STANDALONE_WASM + // In pure wasm mode the memory is created in the wasm (not imported), and + // then exported. + // TODO: do not create a Memory earlier in JS + updateGlobalBufferAndViews(exports['memory'].buffer); +#endif #if USE_PTHREADS // Keep a reference to the compiled module so we can post it to the workers. wasmModule = module; diff --git a/src/settings.js b/src/settings.js index ecb9def92316e..c0313efffcd4b 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1049,9 +1049,6 @@ var EMTERPRETIFY_SYNCLIST = []; // whether js opts will be run, after the main compiler var RUNNING_JS_OPTS = 0; -// whether we are emitting JS glue code -var EMITTING_JS = 1; - // whether we are in the generate struct_info bootstrap phase var BOOTSTRAPPING_STRUCT_INFO = 0; @@ -1074,6 +1071,31 @@ var USE_GLFW = 2; // still make sense there, see that option for more details. var WASM = 1; +// STANDALONE_WASM indicates that we want to emit a wasm file that can run without +// JavaScript. The file will use standard APIs such as wasi as much as possible +// to achieve that. +// +// This option does not guarantee that the wasm can be used by itself - if you +// use APIs with no non-JS alternative, we will still use those (e.g., OpenGL +// at the time of writing this). This gives you the option to see which APIs +// are missing, and if you are compiling for a custom wasi embedding, to add +// those to your embedding. +// +// We may still emit JS with this flag, but the JS should only be a convenient +// way to run the wasm on the Web or in Node.js, and you can run the wasm by +// itself without that JS (again, unless you use APIs for which there is no +// non-JS alternative) in a wasm runtime like wasmer or wasmtime. +// +// Note that even without this option we try to use wasi etc. syscalls as much +// as possible. What this option changes is that we do so even when it means +// a tradeoff with JS size. For example, when this option is set we do not +// import the Memory - importing it is useful for JS, so that JS can start to +// use it before the wasm is even loaded, but in wasi and other wasm-only +// environments the expectation is to create the memory in the wasm itself. +// Doing so prevents some possible JS optimizations, so we only do it behind +// this flag. +var STANDALONE_WASM = 0; + // Whether to use the WebAssembly backend that is in development in LLVM. You // should not set this yourself, instead set EMCC_WASM_BACKEND=1 in the // environment. @@ -1638,4 +1660,5 @@ var LEGACY_SETTINGS = [ ['PRECISE_I64_MATH', [1, 2], 'Starting from Emscripten 1.38.26, PRECISE_I64_MATH is always enabled (https://github.com/emscripten-core/emscripten/pull/7935)'], ['MEMFS_APPEND_TO_TYPED_ARRAYS', [1], 'Starting from Emscripten 1.38.26, MEMFS_APPEND_TO_TYPED_ARRAYS=0 is no longer supported. MEMFS no longer supports using JS arrays for file data (https://github.com/emscripten-core/emscripten/pull/7918)'], ['ERROR_ON_MISSING_LIBRARIES', [1], 'missing libraries are always an error now'], + ['EMITTING_JS', [1], 'The new STANDALONE_WASM flag replaces this (replace EMITTING_JS=0 with STANDALONE_WASM=1)'], ]; diff --git a/system/lib/standalone_wasm.c b/system/lib/standalone_wasm.c new file mode 100644 index 0000000000000..10895ed288f33 --- /dev/null +++ b/system/lib/standalone_wasm.c @@ -0,0 +1,67 @@ +/* + * Copyright 2019 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#include +#include +#include +#include + +#include + +/* + * WASI support code. These are compiled with the program, and call out + * using wasi APIs, which can be provided either by a wasi VM or by our + * emitted JS. + */ + +// libc + +void exit(int status) { + __wasi_proc_exit(status); + __builtin_unreachable(); +} + +void abort() { + exit(1); +} + +// Musl lock internals. As we assume wasi is single-threaded for now, these +// are no-ops. + +void __lock(void* ptr) {} +void __unlock(void* ptr) {} + +// Emscripten additions + +void *emscripten_memcpy_big(void *restrict dest, const void *restrict src, size_t n) { + // This normally calls out into JS which can do a single fast operation, + // but with wasi we can't do that. As this is called when n >= 8192, we + // can just split into smaller calls. + // TODO optimize, maybe build our memcpy with a wasi variant, maybe have + // a SIMD variant, etc. + const int CHUNK = 8192; + unsigned char* d = (unsigned char*)dest; + unsigned char* s = (unsigned char*)src; + while (n > 0) { + size_t curr_n = n; + if (curr_n > CHUNK) curr_n = CHUNK; + memcpy(d, s, curr_n); + d += CHUNK; + s += CHUNK; + n -= curr_n; + } + return dest; +} + +static const int WASM_PAGE_SIZE = 65536; + +// Note that this does not support memory growth in JS because we don't update the JS +// heaps. Wasm and wasi lack a good API for that. +int emscripten_resize_heap(size_t size) { + size_t result = __builtin_wasm_memory_grow(0, (size + WASM_PAGE_SIZE - 1) / WASM_PAGE_SIZE); + return result != (size_t)-1; +} diff --git a/tests/test_other.py b/tests/test_other.py index 10416141f974f..6e851dfef898a 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -30,7 +30,7 @@ raise Exception('do not run this file directly; do something like: tests/runner.py other') from tools.shared import Building, PIPE, run_js, run_process, STDOUT, try_delete, listify -from tools.shared import EMCC, EMXX, EMAR, EMRANLIB, PYTHON, FILE_PACKAGER, WINDOWS, MACOS, LLVM_ROOT, EMCONFIG, EM_BUILD_VERBOSE +from tools.shared import EMCC, EMXX, EMAR, EMRANLIB, PYTHON, FILE_PACKAGER, WINDOWS, MACOS, LINUX, LLVM_ROOT, EMCONFIG, EM_BUILD_VERBOSE from tools.shared import CLANG, CLANG_CC, CLANG_CPP, LLVM_AR from tools.shared import COMPILER_ENGINE, NODE_JS, SPIDERMONKEY_ENGINE, JS_ENGINES, V8_ENGINE from tools.shared import WebAssembly @@ -8287,16 +8287,22 @@ def run(args, expected): run(['-s', 'TOTAL_MEMORY=32MB', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'BINARYEN=1'], (2 * 1024 * 1024 * 1024 - 65536) // 16384) run(['-s', 'TOTAL_MEMORY=32MB', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'BINARYEN=1', '-s', 'WASM_MEM_MAX=128MB'], 2048 * 4) - def test_wasm_targets(self): + def test_wasm_target_and_STANDALONE_WASM(self): + # STANDALONE_WASM means we never minify imports and exports. for opts, potentially_expect_minified_exports_and_imports in ( - ([], False), - (['-O2'], False), - (['-O3'], True), - (['-Os'], True), + ([], False), + (['-O2'], False), + (['-O3'], True), + (['-O3', '-s', 'STANDALONE_WASM'], False), + (['-Os'], True), ): + if 'STANDALONE_WASM' in opts and not self.is_wasm_backend(): + continue + # targeting .wasm (without .js) means we enable STANDALONE_WASM automatically, and don't minify imports/exports for target in ('out.js', 'out.wasm'): expect_minified_exports_and_imports = potentially_expect_minified_exports_and_imports and target.endswith('.js') - print(opts, potentially_expect_minified_exports_and_imports, target, ' => ', expect_minified_exports_and_imports) + standalone = target.endswith('.wasm') or 'STANDALONE_WASM' in opts + print(opts, potentially_expect_minified_exports_and_imports, target, ' => ', expect_minified_exports_and_imports, standalone) self.clear() run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-o', target] + opts) @@ -8308,13 +8314,33 @@ def test_wasm_targets(self): exports = [line.strip().split(' ')[1].replace('"', '') for line in wast_lines if "(export " in line] imports = [line.strip().split(' ')[2].replace('"', '') for line in wast_lines if "(import " in line] exports_and_imports = exports + imports - print(exports) - print(imports) + print(' exports', exports) + print(' imports', imports) if expect_minified_exports_and_imports: assert 'a' in exports_and_imports else: assert 'a' not in exports_and_imports - assert 'memory' in exports_and_imports, 'some things are not minified anyhow' + assert 'memory' in exports_and_imports or 'fd_write' in exports_and_imports, 'some things are not minified anyhow' + # verify the wasm runs with the JS + if target.endswith('.js'): + self.assertContained('hello, world!', run_js('out.js')) + # verify the wasm runs in a wasm VM, without the JS + # TODO: more platforms than linux + if LINUX and standalone and self.is_wasm_backend(): + WASMER = os.path.expanduser(os.path.join('~', '.wasmer', 'bin', 'wasmer')) + if os.path.isfile(WASMER): + print(' running in wasmer') + out = run_process([WASMER, 'run', 'out.wasm'], stdout=PIPE).stdout + self.assertContained('hello, world!', out) + else: + print('[WARNING - no wasmer]') + WASMTIME = os.path.expanduser(os.path.join('~', 'wasmtime')) + if os.path.isfile(WASMTIME): + print(' running in wasmtime') + out = run_process([WASMTIME, 'out.wasm'], stdout=PIPE).stdout + self.assertContained('hello, world!', out) + else: + print('[WARNING - no wasmtime]') def test_wasm_targets_side_module(self): # side modules do allow a wasm target diff --git a/tools/shared.py b/tools/shared.py index dcc4cef23252b..ab0760c1ecbe8 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -500,7 +500,7 @@ def get_emscripten_version(path): # NB: major version 0 implies no compatibility # NB: when changing the metadata format, we should only append new fields, not # reorder, modify, or remove existing ones. -(EMSCRIPTEN_METADATA_MAJOR, EMSCRIPTEN_METADATA_MINOR) = (0, 2) +(EMSCRIPTEN_METADATA_MAJOR, EMSCRIPTEN_METADATA_MINOR) = (0, 3) # For the JS/WASM ABI, specifies the minimum ABI version required of # the WASM runtime implementation by the generated WASM binary. It follows # semver and changes whenever C types change size/signedness or @@ -509,7 +509,7 @@ def get_emscripten_version(path): # change, increment EMSCRIPTEN_ABI_MINOR if EMSCRIPTEN_ABI_MAJOR == 0 # or the ABI change is backwards compatible, otherwise increment # EMSCRIPTEN_ABI_MAJOR and set EMSCRIPTEN_ABI_MINOR = 0. -(EMSCRIPTEN_ABI_MAJOR, EMSCRIPTEN_ABI_MINOR) = (0, 6) +(EMSCRIPTEN_ABI_MAJOR, EMSCRIPTEN_ABI_MINOR) = (0, 7) def generate_sanity(): @@ -1822,11 +1822,16 @@ def link_lld(args, target, opts=[], lto_level=0): '-o', target, '--allow-undefined', - '--import-memory', - '--import-table', '--lto-O%d' % lto_level, ] + args + # wasi does not import the memory (but for JS it is efficient to do so, + # as it allows us to set up memory, preload files, etc. even before the + # wasm module arrives) + if not Settings.STANDALONE_WASM: + cmd.append('--import-memory') + cmd.append('--import-table') + if Settings.USE_PTHREADS: cmd.append('--shared-memory') @@ -2449,7 +2454,7 @@ def closure_compiler(filename, pretty=True, advanced=True, extra_closure_args=[] # minify the final wasm+JS combination. this is done after all the JS # and wasm optimizations; here we do the very final optimizations on them @staticmethod - def minify_wasm_js(js_file, wasm_file, expensive_optimizations, minify_whitespace, debug_info): + def minify_wasm_js(js_file, wasm_file, expensive_optimizations, minify_whitespace, debug_info, emitting_js): # start with JSDCE, to clean up obvious JS garbage. When optimizing for size, # use AJSDCE (aggressive JS DCE, performs multiple iterations). Clean up # whitespace if necessary too. @@ -2475,11 +2480,11 @@ def minify_wasm_js(js_file, wasm_file, expensive_optimizations, minify_whitespac passes.append('minifyWhitespace') logger.debug('running post-meta-DCE cleanup on shell code: ' + ' '.join(passes)) js_file = Building.acorn_optimizer(js_file, passes) - # also minify the names used between js and wasm, if we emitting JS (then the JS knows how to load the minified names) + # Also minify the names used between js and wasm, if we are emitting an optimized JS+wasm combo (then the JS knows how to load the minified names). # If we are building with DECLARE_ASM_MODULE_EXPORTS=0, we must *not* minify the exports from the wasm module, since in DECLARE_ASM_MODULE_EXPORTS=0 mode, the code that # reads out the exports is compacted by design that it does not have a chance to unminify the functions. If we are building with DECLARE_ASM_MODULE_EXPORTS=1, we might # as well minify wasm exports to regain some of the code size loss that setting DECLARE_ASM_MODULE_EXPORTS=1 caused. - if Settings.EMITTING_JS and not Settings.AUTODEBUG and not Settings.ASSERTIONS: + if not Settings.STANDALONE_WASM and not Settings.AUTODEBUG and not Settings.ASSERTIONS and not Settings.SIDE_MODULE and emitting_js: js_file = Building.minify_wasm_imports_and_exports(js_file, wasm_file, minify_whitespace=minify_whitespace, minify_exports=Settings.DECLARE_ASM_MODULE_EXPORTS, debug_info=debug_info) return js_file @@ -2514,8 +2519,19 @@ def metadce(js_file, wasm_file, minify_whitespace, debug_info): export = '_' + export if export in Building.user_requested_exports or Settings.EXPORT_ALL: item['root'] = True + # in standalone wasm, always export the memory + if Settings.STANDALONE_WASM: + graph.append({ + 'export': 'memory', + 'name': 'emcc$export$memory', + 'reaches': [], + 'root': True + }) # fix wasi imports TODO: support wasm stable with an option? - WASI_IMPORTS = set(['fd_write']) + WASI_IMPORTS = set([ + 'fd_write', + 'proc_exit', + ]) for item in graph: if 'import' in item and item['import'][1][1:] in WASI_IMPORTS: item['import'][0] = 'wasi_unstable' @@ -3078,7 +3094,8 @@ def add_emscripten_metadata(js_file, wasm_file): WebAssembly.lebify(global_base) + WebAssembly.lebify(dynamic_base) + WebAssembly.lebify(dynamictop_ptr) + - WebAssembly.lebify(tempdouble_ptr) + WebAssembly.lebify(tempdouble_ptr) + + WebAssembly.lebify(int(Settings.STANDALONE_WASM)) # NB: more data can be appended here as long as you increase # the EMSCRIPTEN_METADATA_MINOR diff --git a/tools/system_libs.py b/tools/system_libs.py index 5637a133a4eae..83ec69df84e09 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1140,6 +1140,17 @@ class libasan_rt_wasm(SanitizerLibrary): src_dir = ['system', 'lib', 'compiler-rt', 'lib', 'asan'] +class libstandalonewasm(Library): + name = 'libstandalonewasm' + + cflags = ['-Os'] + src_dir = ['system', 'lib'] + src_files = ['standalone_wasm.c'] + + def can_build(self): + return shared.Settings.WASM_BACKEND + + # If main() is not in EXPORTED_FUNCTIONS, it may be dce'd out. This can be # confusing, so issue a warning. def warn_on_unexported_main(symbolses): @@ -1316,6 +1327,9 @@ def add_library(lib): force_include.add('libasan_rt_wasm') add_library(system_libs_map['libasan_rt_wasm']) + if shared.Settings.STANDALONE_WASM: + add_library(system_libs_map['libstandalonewasm']) + libs_to_link.sort(key=lambda x: x[0].endswith('.a')) # make sure to put .a files at the end. # libc++abi and libc++ *static* linking is tricky. e.g. cxa_demangle.cpp disables c++