From a03c205c44a96e16445975b87e2bbd5c3bfb84b0 Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Fri, 10 Mar 2023 15:12:42 -0800 Subject: [PATCH] Move wasm module preload plugin out of library_browser.js. NFC Also test the preload plugin system under node. It was working previously but not covered by any of the test we run in CI. I'm hoping to unify this preloading system with the one planned as part of #18552. --- src/compiler.js | 6 ++ src/library_browser.js | 59 +++---------------- src/library_dylink.js | 42 ++++++++++++- src/library_fs.js | 4 -- src/library_fs_shared.js | 34 ++++++++++- src/library_lz4.js | 4 +- src/library_wasmfs.js | 4 -- .../metadce/test_metadce_hello_O0.jssize | 2 +- .../metadce/test_metadce_hello_dylink.jssize | 2 +- .../metadce/test_metadce_minimal_O0.jssize | 2 +- test/other/test_unoptimized_code_size.js.size | 2 +- .../test_unoptimized_code_size_strict.js.size | 2 +- test/test_other.py | 32 ++++++++++ 13 files changed, 124 insertions(+), 71 deletions(-) diff --git a/src/compiler.js b/src/compiler.js index 6176da18a09e6..976cd55527cc8 100755 --- a/src/compiler.js +++ b/src/compiler.js @@ -64,6 +64,12 @@ Object.assign(global, settings); global.symbolsOnly = symbolsOnlyArg != -1; +// In case compiler.js is run directly (as in gen_sig_info) +// ALL_INCOMING_MODULE_JS_API might not be populated yet. +if (!ALL_INCOMING_MODULE_JS_API.length) { + ALL_INCOMING_MODULE_JS_API = INCOMING_MODULE_JS_API +} + EXPORTED_FUNCTIONS = new Set(EXPORTED_FUNCTIONS); WASM_EXPORTS = new Set(WASM_EXPORTS); SIDE_MODULE_EXPORTS = new Set(SIDE_MODULE_EXPORTS); diff --git a/src/library_browser.js b/src/library_browser.js index 5a4d234070547..df6b6a9ee41cf 100644 --- a/src/library_browser.js +++ b/src/library_browser.js @@ -12,6 +12,10 @@ var LibraryBrowser = { '$safeSetTimeout', '$warnOnce', 'emscripten_set_main_loop_timing', + '$preloadPlugins', +#if MAIN_MODULE + '$preloadedWasm', +#endif ], $Browser__postset: ` // exports @@ -25,9 +29,6 @@ var LibraryBrowser = { Module["resumeMainLoop"] = function Module_resumeMainLoop() { Browser.mainLoop.resume() }; Module["getUserMedia"] = function Module_getUserMedia() { Browser.getUserMedia() }; Module["createContext"] = function Module_createContext(canvas, useWebGL, setInModule, webGLContextAttributes) { return Browser.createContext(canvas, useWebGL, setInModule, webGLContextAttributes) }; -#if MAIN_MODULE - var preloadedWasm = {}; -#endif var preloadedImages = {}; var preloadedAudios = {};`, @@ -101,13 +102,11 @@ var LibraryBrowser = { workers: [], init: function() { - if (!Module["preloadPlugins"]) Module["preloadPlugins"] = []; // needs to exist even in workers - if (Browser.initted) return; Browser.initted = true; // Support for plugins that can process preloaded files. You can add more of these to - // your app by creating and appending to Module.preloadPlugins. + // your app by creating and appending to preloadPlugins. // // Each plugin is asked if it can handle a file based on the file's name. If it can, // it is given the file's raw data. When it is done, it calls a callback with the file's @@ -146,7 +145,7 @@ var LibraryBrowser = { }; img.src = url; }; - Module['preloadPlugins'].push(imagePlugin); + preloadPlugins.push(imagePlugin); var audioPlugin = {}; audioPlugin['canHandle'] = function audioPlugin_canHandle(name) { @@ -209,34 +208,7 @@ var LibraryBrowser = { finish(audio); // try to use it even though it is not necessarily ready to play }, 10000); }; - Module['preloadPlugins'].push(audioPlugin); - -#if MAIN_MODULE - // Use string keys here to avoid minification since the plugin consumer - // also uses string keys. - var wasmPlugin = { - 'promiseChainEnd': Promise.resolve(), - 'canHandle': function(name) { - return !Module.noWasmDecoding && name.endsWith('.so') - }, - 'handle': function(byteArray, name, onload, onerror) { - // loadWebAssemblyModule can not load modules out-of-order, so rather - // than just running the promises in parallel, this makes a chain of - // promises to run in series. - wasmPlugin['promiseChainEnd'] = wasmPlugin['promiseChainEnd'].then( - () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true})).then( - (module) => { - preloadedWasm[name] = module; - onload(); - }, - (err) => { - console.warn("Couldn't instantiate wasm: " + name + " '" + err + "'"); - onerror(); - }); - } - }; - Module['preloadPlugins'].push(wasmPlugin); -#endif // MAIN_MODULE + preloadPlugins.push(audioPlugin); // Canvas event setup @@ -279,23 +251,6 @@ var LibraryBrowser = { } }, - // Tries to handle an input byteArray using preload plugins. Returns true if - // it was handled. - handledByPreloadPlugin: function(byteArray, fullname, finish, onerror) { - // Ensure plugins are ready. - Browser.init(); - - var handled = false; - Module['preloadPlugins'].forEach((plugin) => { - if (handled) return; - if (plugin['canHandle'](fullname)) { - plugin['handle'](byteArray, fullname, finish, onerror); - handled = true; - } - }); - return handled; - }, - createContext: function(/** @type {HTMLCanvasElement} */ canvas, useWebGL, setInModule, webGLContextAttributes) { if (useWebGL && Module.ctx && canvas == Module.canvas) return Module.ctx; // no need to recreate GL context if it's already been created for this canvas. diff --git a/src/library_dylink.js b/src/library_dylink.js index 91691a4303c99..471f73cf84ab4 100644 --- a/src/library_dylink.js +++ b/src/library_dylink.js @@ -10,6 +10,39 @@ var dlopenMissingError = "'To use dlopen, you need enable dynamic linking, see h var LibraryDylink = { #if RELOCATABLE + $registerWasmPlugin__deps: ['$preloadPlugins'], + $registerWasmPlugin: function() { + // Use string keys here to avoid minification since the plugin consumer + // also uses string keys. + var wasmPlugin = { + 'promiseChainEnd': Promise.resolve(), + 'canHandle': function(name) { + return !Module.noWasmDecoding && name.endsWith('.so') + }, + 'handle': function(byteArray, name, onload, onerror) { + // loadWebAssemblyModule can not load modules out-of-order, so rather + // than just running the promises in parallel, this makes a chain of + // promises to run in series. + wasmPlugin['promiseChainEnd'] = wasmPlugin['promiseChainEnd'].then( + () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true})).then( + (module) => { + preloadedWasm[name] = module; + onload(); + }, + (error) => { + err('failed to instantiate wasm: ' + name + ': ' + error); + onerror(); + }); + } + }; + preloadPlugins.push(wasmPlugin); + }, + + $preloadedWasm__deps: ['$registerWasmPlugin'], + $preloadedWasm__postset: ` + registerWasmPlugin(); + `, + $preloadedWasm: {}, $isSymbolDefined: function(symName) { // Ignore 'stub' symbols that are auto-generated as part of the original @@ -884,7 +917,9 @@ var LibraryDylink = { // If a library was already loaded, it is not loaded a second time. However // flags.global and flags.nodelete are handled every time a load request is made. // Once a library becomes "global" or "nodelete", it cannot be removed or unloaded. - $loadDynamicLibrary__deps: ['$LDSO', '$loadWebAssemblyModule', '$isInternalSym', '$mergeLibSymbols', '$newDSO', '$asyncLoad'], + $loadDynamicLibrary__deps: ['$LDSO', '$loadWebAssemblyModule', + '$isInternalSym', '$mergeLibSymbols', '$newDSO', + '$asyncLoad', '$preloadedWasm'], $loadDynamicLibrary__docs: ` /** * @param {number=} handle @@ -955,7 +990,10 @@ var LibraryDylink = { // libName -> exports function getExports() { // lookup preloaded cache first - if (typeof preloadedWasm != 'undefined' && preloadedWasm[libName]) { + if (preloadedWasm[libName]) { +#if DYLINK_DEBUG + err('using preloaded module for: ' + libName); +#endif var libModule = preloadedWasm[libName]; return flags.loadAsync ? Promise.resolve(libModule) : libModule; } diff --git a/src/library_fs.js b/src/library_fs.js index ce9c739f59f1a..bd432c991f25c 100644 --- a/src/library_fs.js +++ b/src/library_fs.js @@ -93,10 +93,6 @@ Object.defineProperties(FSNode.prototype, { FS.FSNode = FSNode; FS.createPreloadedFile = FS_createPreloadedFile; FS.staticInit();` + -#if USE_CLOSURE_COMPILER - // Declare variable for Closure, FS.createPreloadedFile() below calls Browser.handledByPreloadPlugin() - '/**@suppress {duplicate, undefinedVars}*/var Browser;' + -#endif // Get module methods from settings '{{{ EXPORTED_RUNTIME_METHODS.filter(function(func) { return func.substr(0, 3) === 'FS_' }).map(function(func){return 'Module["' + func + '"] = FS.' + func.substr(3) + ";"}).reduce(function(str, func){return str + func;}, '') }}}'; }, diff --git a/src/library_fs_shared.js b/src/library_fs_shared.js index 1d7336d64b6c1..e595fb3fe2c7a 100644 --- a/src/library_fs_shared.js +++ b/src/library_fs_shared.js @@ -5,6 +5,31 @@ */ mergeInto(LibraryManager.library, { + $preloadPlugins: "{{{ makeModuleReceiveExpr('preloadPlugins', '[]') }}}", + +#if !MINIMAL_RUNTIME + // Tries to handle an input byteArray using preload plugins. Returns true if + // it was handled. + $FS_handledByPreloadPlugin__internal: true, + $FS_handledByPreloadPlugin__deps: ['$preloadPlugins'], + $FS_handledByPreloadPlugin: function(byteArray, fullname, finish, onerror) { +#if LibraryManager.has('library_browser.js') + // Ensure plugins are ready. + if (typeof Browser != 'undefined') Browser.init(); +#endif + + var handled = false; + preloadPlugins.forEach(function(plugin) { + if (handled) return; + if (plugin['canHandle'](fullname)) { + plugin['handle'](byteArray, fullname, finish, onerror); + handled = true; + } + }); + return handled; + }, +#endif + // Preloads a file asynchronously. You can call this before run, for example in // preRun. run will be delayed until this file arrives and is set up. // If you call it after run(), you may want to pause the main loop until it @@ -17,7 +42,11 @@ mergeInto(LibraryManager.library, { // You can also call this with a typed array instead of a url. It will then // do preloading for the Image/Audio part, as if the typed array were the // result of an XHR that you did manually. - $FS_createPreloadedFile__deps: ['$asyncLoad'], + $FS_createPreloadedFile__deps: ['$asyncLoad', +#if !MINIMAL_RUNTIME + '$FS_handledByPreloadPlugin', +#endif + ], $FS_createPreloadedFile: function(parent, name, url, canRead, canWrite, onload, onerror, dontCreateFile, canOwn, preFinish) { #if WASMFS // TODO: use WasmFS code to resolve and join the path here? @@ -38,7 +67,7 @@ mergeInto(LibraryManager.library, { removeRunDependency(dep); } #if !MINIMAL_RUNTIME - if (Browser.handledByPreloadPlugin(byteArray, fullname, finish, () => { + if (FS_handledByPreloadPlugin(byteArray, fullname, finish, () => { if (onerror) onerror(); removeRunDependency(dep); })) { @@ -76,4 +105,5 @@ mergeInto(LibraryManager.library, { if (canWrite) mode |= {{{ cDefs.S_IWUGO }}}; return mode; }, + }); diff --git a/src/library_lz4.js b/src/library_lz4.js index 60fe36ff84374..94d565c667931 100644 --- a/src/library_lz4.js +++ b/src/library_lz4.js @@ -6,7 +6,7 @@ #if LZ4 mergeInto(LibraryManager.library, { - $LZ4__deps: ['$FS'], + $LZ4__deps: ['$FS', '$preloadPlugins'], $LZ4: { DIR_MODE: {{{ cDefs.S_IFDIR }}} | 511 /* 0777 */, FILE_MODE: {{{ cDefs.S_IFREG }}} | 511 /* 0777 */, @@ -52,7 +52,7 @@ mergeInto(LibraryManager.library, { pack['metadata'].files.forEach(function(file) { var handled = false; var fullname = file.filename; - Module['preloadPlugins'].forEach(function(plugin) { + preloadPlugins.forEach(function(plugin) { if (handled) return; if (plugin['canHandle'](fullname)) { var dep = getUniqueRunDependency('fp ' + fullname); diff --git a/src/library_wasmfs.js b/src/library_wasmfs.js index 3e39b150ac2c3..fc05172dd8d5b 100644 --- a/src/library_wasmfs.js +++ b/src/library_wasmfs.js @@ -7,11 +7,7 @@ mergeInto(LibraryManager.library, { $wasmFSPreloadedFiles: [], $wasmFSPreloadedDirs: [], - // Declare variable for Closure, FS.createPreloadedFile() below calls Browser.handledByPreloadPlugin() $FS__postset: ` -#if USE_CLOSURE_COMPILER -/**@suppress {duplicate, undefinedVars}*/var Browser; -#endif FS.createPreloadedFile = FS_createPreloadedFile; `, $FS__deps: [ diff --git a/test/other/metadce/test_metadce_hello_O0.jssize b/test/other/metadce/test_metadce_hello_O0.jssize index 24a44d0d1ceae..a17534e412b0e 100644 --- a/test/other/metadce/test_metadce_hello_O0.jssize +++ b/test/other/metadce/test_metadce_hello_O0.jssize @@ -1 +1 @@ -23827 +23842 diff --git a/test/other/metadce/test_metadce_hello_dylink.jssize b/test/other/metadce/test_metadce_hello_dylink.jssize index 005f3bc6c8629..bc6d98594e845 100644 --- a/test/other/metadce/test_metadce_hello_dylink.jssize +++ b/test/other/metadce/test_metadce_hello_dylink.jssize @@ -1 +1 @@ -27887 +28153 diff --git a/test/other/metadce/test_metadce_minimal_O0.jssize b/test/other/metadce/test_metadce_minimal_O0.jssize index 55ec08fe69674..ee2c9295201b2 100644 --- a/test/other/metadce/test_metadce_minimal_O0.jssize +++ b/test/other/metadce/test_metadce_minimal_O0.jssize @@ -1 +1 @@ -20076 +20091 diff --git a/test/other/test_unoptimized_code_size.js.size b/test/other/test_unoptimized_code_size.js.size index 77888a8cbbf48..10afb236d5a46 100644 --- a/test/other/test_unoptimized_code_size.js.size +++ b/test/other/test_unoptimized_code_size.js.size @@ -1 +1 @@ -59626 +59646 diff --git a/test/other/test_unoptimized_code_size_strict.js.size b/test/other/test_unoptimized_code_size_strict.js.size index ea43e56b33d17..0049768cac040 100644 --- a/test/other/test_unoptimized_code_size_strict.js.size +++ b/test/other/test_unoptimized_code_size_strict.js.size @@ -1 +1 @@ -58568 +58588 diff --git a/test/test_other.py b/test/test_other.py index d8e1916d2e203..ae17d6a0d3612 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -13390,3 +13390,35 @@ def test_node_pthreads_err_out(self): def test_windows_batch_file_dp0_expansion_bug(self): create_file('build_with_quotes.bat', f'@"emcc" {test_file("hello_world.c")}') self.run_process(['build_with_quotes.bat']) + + def test_preload_module(self): + # TODO(sbc): This test is copyied from test_browser.py. Perhaps find a better way to + # share code between them. + create_file('library.c', r''' + #include + int library_func() { + return 42; + } + ''') + self.run_process([EMCC, 'library.c', '-sSIDE_MODULE', '-o', 'library.so']) + create_file('main.c', r''' + #include + #include + #include + #include + int main() { + int found = EM_ASM_INT( + return preloadedWasm['/library.so'] !== undefined; + ); + assert(found); + void *lib_handle = dlopen("/library.so", RTLD_NOW); + assert(lib_handle); + typedef int (*voidfunc)(); + voidfunc x = (voidfunc)dlsym(lib_handle, "library_func"); + assert(x); + assert(x() == 42); + printf("done\n"); + return 0; + } + ''') + self.do_runf('main.c', 'done\n', emcc_args=['-sMAIN_MODULE=2', '--preload-file', '.@/', '--use-preload-plugins'])