From 53552f274d91b3e7171fe96862a68f5e60d15579 Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Sat, 14 Jan 2023 21:53:50 -0800 Subject: [PATCH] Process --pre-js and --post-js files in jsifier.js This teats pre and post JS files more like normal runtime and library files which simplifies the code and avoids the special case processing in emcc.py. This also means that pre and post JS also now go through macro preprocessor which should be useful in some cases. --- ChangeLog.md | 3 +++ emcc.py | 44 +++++++++++++------------------ src/jsifier.js | 4 +++ src/parseTools.js | 8 ++++++ src/settings_internal.js | 4 +++ src/shell.js | 2 +- src/shell_minimal.js | 3 +-- test/return64bit/testbindstart.js | 4 +-- test/test_other.py | 15 +++++++++++ tools/js_manipulation.py | 21 ++++++++++----- tools/settings.py | 4 +-- 11 files changed, 72 insertions(+), 40 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 5b9bef68768f1..4af620225c20c 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -23,6 +23,9 @@ See docs/process.md for more on how version tagging works. - The `STACK_SIZE`, `STACK_ALIGN`, `POINTER_SIZE`, and `ASSERTIONS` JavaScript globals were removed by default. In debug builds a clear error is shown if you try to use these. (#18503) +- --pre-js and --post-js files are now fed through the JS preprocesor, just + like JS library files and the core runtime JS files. This means they can + now contain #if/#else/#endif blocks and {{{ }}} macro blocks. (#18525) 3.1.30 - 01/11/23 ----------------- diff --git a/emcc.py b/emcc.py index 0d698d3c46e8a..2f0adcd63a64c 100755 --- a/emcc.py +++ b/emcc.py @@ -524,7 +524,11 @@ def get_all_js_syms(): # We define a cache hit as when the settings and `--js-library` contents are # identical. - input_files = [json.dumps(settings.external_dict(), sort_keys=True, indent=2)] + # Ignore certain settings that can are no relevant to library deps. Here we + # skip PRE_JS_FILES/POST_JS_FILES which don't effect the library symbol list + # and can contain full paths to temporary files. + skip_settings = {'PRE_JS_FILES', 'POST_JS_FILES'} + input_files = [json.dumps(settings.external_dict(skip_keys=skip_settings), sort_keys=True, indent=2)] for jslib in sorted(glob.glob(utils.path_from_root('src') + '/library*.js')): input_files.append(read_file(jslib)) for jslib in settings.JS_LIBRARIES: @@ -1119,7 +1123,7 @@ def package_files(options, target): if options.preload_files: # Preloading files uses --pre-js code that runs before the module is loaded. file_code = shared.check_call(cmd, stdout=PIPE).stdout - options.pre_js = js_manipulation.add_files_pre_js(options.pre_js, file_code) + js_manipulation.add_files_pre_js(options.pre_js, file_code) else: # Otherwise, we are embedding files, which does not require --pre-js code, # and instead relies on a static constrcutor to populate the filesystem. @@ -1293,6 +1297,9 @@ def run(args): if len(options.preload_files) or len(options.embed_files): linker_arguments += package_files(options, target) + settings.PRE_JS_FILES = [os.path.abspath(f) for f in options.pre_js] + settings.POST_JS_FILES = [os.path.abspath(f) for f in options.post_js] + if options.oformat == OFormat.OBJECT: logger.debug(f'link_to_object: {linker_arguments} -> {target}') building.link_to_object(linker_arguments, target) @@ -1750,8 +1757,6 @@ def phase_linker_setup(options, state, newargs): exit_with_error('PTHREADS_PROFILING only works with ASSERTIONS enabled') options.post_js.append(utils.path_from_root('src/threadprofiler.js')) - options.pre_js = read_js_files(options.pre_js) - options.post_js = read_js_files(options.post_js) options.extern_pre_js = read_js_files(options.extern_pre_js) options.extern_post_js = read_js_files(options.extern_post_js) @@ -3021,7 +3026,8 @@ def phase_post_link(options, state, in_wasm, wasm_target, target): phase_emscript(options, in_wasm, wasm_target, memfile) - phase_source_transforms(options) + if options.js_transform: + phase_source_transforms(options) if memfile and not settings.MINIMAL_RUNTIME: # MINIMAL_RUNTIME doesn't use `var memoryInitializer` but instead expects Module['mem'] to @@ -3055,28 +3061,14 @@ def phase_emscript(options, in_wasm, wasm_target, memfile): @ToolchainProfiler.profile_block('source transforms') def phase_source_transforms(options): - global final_js - - # Apply pre and postjs files - if final_js and (options.pre_js or options.post_js): - logger.debug('applying pre/postjses') - src = read_file(final_js) - final_js += '.pp.js' - with open(final_js, 'w', encoding='utf-8') as f: - # pre-js code goes right after the Module integration code (so it - # can use Module), we have a marker for it - f.write(do_replace(src, '// {{PRE_JSES}}', options.pre_js)) - f.write(options.post_js) - save_intermediate('pre-post') - # Apply a source code transformation, if requested - if options.js_transform: - safe_copy(final_js, final_js + '.tr.js') - final_js += '.tr.js' - posix = not shared.WINDOWS - logger.debug('applying transform: %s', options.js_transform) - shared.check_call(building.remove_quotes(shlex.split(options.js_transform, posix=posix) + [os.path.abspath(final_js)])) - save_intermediate('transformed') + global final_js + safe_copy(final_js, final_js + '.tr.js') + final_js += '.tr.js' + posix = not shared.WINDOWS + logger.debug('applying transform: %s', options.js_transform) + shared.check_call(building.remove_quotes(shlex.split(options.js_transform, posix=posix) + [os.path.abspath(final_js)])) + save_intermediate('transformed') @ToolchainProfiler.profile_block('memory initializer') diff --git a/src/jsifier.js b/src/jsifier.js index ebcbbb7b45e51..e79a432cd6c9b 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -526,6 +526,10 @@ function ${name}(${args}) { const postFile = MINIMAL_RUNTIME ? 'postamble_minimal.js' : 'postamble.js'; includeFile(postFile); + for (const fileName of POST_JS_FILES) { + includeFile(fileName); + } + print('//FORWARDED_DATA:' + JSON.stringify({ librarySymbols: librarySymbols, warnings: warnings, diff --git a/src/parseTools.js b/src/parseTools.js index 5d5e8fefd1241..b6a917ec17aa6 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -1104,3 +1104,11 @@ function getUnsharedTextDecoderView(heap, start, end) { // or can use .subarray() otherwise. return `${heap}.buffer instanceof SharedArrayBuffer ? ${shared} : ${unshared}`; } + +function preJS() { + let result = ''; + for (const fileName of PRE_JS_FILES) { + result += preprocess(fileName); + } + return result; +} diff --git a/src/settings_internal.js b/src/settings_internal.js index 60e4cc541c390..5e83803ca0c57 100644 --- a/src/settings_internal.js +++ b/src/settings_internal.js @@ -243,3 +243,7 @@ var WEAK_IMPORTS = []; var STACK_FIRST = false; var HAVE_EM_ASM = true; + +var PRE_JS_FILES = []; + +var POST_JS_FILES = []; diff --git a/src/shell.js b/src/shell.js index 887e3e3806af6..97b597bda7d7d 100644 --- a/src/shell.js +++ b/src/shell.js @@ -72,7 +72,7 @@ Module['ready'] = new Promise(function(resolve, reject) { // --pre-jses are emitted after the Module integration code, so that they can // refer to Module (if they choose; they can also define Module) -// {{PRE_JSES}} +{{{ preJS() }}} // Sometimes an existing Module object exists with properties // meant to overwrite the default module functionality. Here diff --git a/src/shell_minimal.js b/src/shell_minimal.js index c1e9b92e71e42..f1a14cef39fa7 100644 --- a/src/shell_minimal.js +++ b/src/shell_minimal.js @@ -136,8 +136,7 @@ function ready() { // --pre-jses are emitted after the Module integration code, so that they can // refer to Module (if they choose; they can also define Module) - -// {{PRE_JSES}} +{{{ preJS() }}} #if USE_PTHREADS diff --git a/test/return64bit/testbindstart.js b/test/return64bit/testbindstart.js index 4956806bd8f24..9efa56558001a 100644 --- a/test/return64bit/testbindstart.js +++ b/test/return64bit/testbindstart.js @@ -1,3 +1 @@ - -(function() { // Start of self-calling lambda used to avoid polluting global namespace. - +(function() { // Start of IIFE used to avoid polluting global namespace. diff --git a/test/test_other.py b/test/test_other.py index d7017fd59d5d7..9d2c7cd125f07 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -9041,6 +9041,21 @@ def test_js_preprocess(self): self.assertContained('JSLIB: EXIT_RUNTIME', err) self.assertNotContained('JSLIB: MAIN_MODULE', err) + def test_js_preprocess_pre_post(self): + create_file('pre.js', ''' + #if ASSERTIONS + console.log('assertions enabled') + #else + console.log('assertions disabled') + #endif + ''') + create_file('post.js', ''' + console.log({{{ POINTER_SIZE }}}); + ''') + self.emcc_args += ['--pre-js', 'pre.js', '--post-js', 'post.js'] + self.do_runf(test_file('hello_world.c'), 'assertions enabled\n4') + self.do_runf(test_file('hello_world.c'), 'assertions disabled\n4', emcc_args=['-sASSERTIONS=0']) + def test_html_preprocess(self): src_file = test_file('module/test_stdin.c') output_file = 'test_stdin.html' diff --git a/tools/js_manipulation.py b/tools/js_manipulation.py index d96969a2e55cf..cf9fc90d0abb6 100644 --- a/tools/js_manipulation.py +++ b/tools/js_manipulation.py @@ -6,7 +6,7 @@ import re from .settings import settings -from . import utils +from . import utils, shared emscripten_license = '''\ /** @@ -28,25 +28,34 @@ emscripten_license_regex = r'\/\*\*?(\s*\*?\s*@license)?(\s*\*?\s*Copyright \d+ The Emscripten Authors\s*\*?\s*SPDX-License-Identifier: MIT)+\s*\*\/\s*' -def add_files_pre_js(user_pre_js, files_pre_js): +def add_files_pre_js(pre_js_list, files_pre_js): # the normal thing is to just combine the pre-js content + filename = shared.get_temp_files().get('.js').name + utils.write_file(filename, files_pre_js) + pre_js_list.insert(0, filename) if not settings.ASSERTIONS: - return files_pre_js + user_pre_js + return # if a user pre-js tramples the file code's changes to Module.preRun # that could be confusing. show a clear error at runtime if assertions are # enabled - return files_pre_js + ''' + pre = shared.get_temp_files().get('.js').name + post = shared.get_temp_files().get('.js').name + utils.write_file(pre, ''' // All the pre-js content up to here must remain later on, we need to run // it. if (Module['ENVIRONMENT_IS_PTHREAD']) Module['preRun'] = []; var necessaryPreJSTasks = Module['preRun'].slice(); - ''' + user_pre_js + ''' + ''') + utils.write_file(post, ''' if (!Module['preRun']) throw 'Module.preRun should exist because file support used it; did a pre-js delete it?'; necessaryPreJSTasks.forEach(function(task) { if (Module['preRun'].indexOf(task) < 0) throw 'All preRun tasks that exist before user pre-js code should remain after; did you replace Module or modify Module.preRun?'; }); - ''' + ''') + + pre_js_list.insert(1, pre) + pre_js_list.append(post) def handle_license(js_target): diff --git a/tools/settings.py b/tools/settings.py index 01d0d5b535315..fd41dde2410d5 100644 --- a/tools/settings.py +++ b/tools/settings.py @@ -156,8 +156,8 @@ def infer_types(self): def dict(self): return self.attrs - def external_dict(self): - external_settings = {k: v for k, v in self.dict().items() if k not in INTERNAL_SETTINGS} + def external_dict(self, skip_keys={}): # noqa + external_settings = {k: v for k, v in self.dict().items() if k not in INTERNAL_SETTINGS and k not in skip_keys} # Only the names of the legacy settings are used by the JS compiler # so we can reduce the size of serialized json by simplifying this # otherwise complex value.