Custom bundle loaders + WASM support -> master (#565)

* add support to rust/wasm * use child-process-promise instead of async-child-process * use parent extension instead of fromHtml flag * create WasmAsset * RustAsset refactor get rid of wargo use wasm32-unknown-unknown * fix lint errors, remove toml package and delete jsconfig.json * Inline wasm into JS, instantiate, and return exports * Split bundle loaders into separate modules Register custom loaders with `bundler.addBundleLoader` * Split out build queue logic into its own class Enables `bundler.getAsset` method to resolve and process an asset at any time, outside of the normal asset tree construction. * Only add a single asset to bundle if raw packager is used This gives us a separate bundle for each output file. * Include used bundle loaders, and preload external modules Allows synchronous import to preload modules in external files, e.g. .wasm file, prior to execution of the JS bundle. Also processes the HMR runtime like other assets. * Register wasm loader * Replace dedicated WASMAsset with RawAsset No longer returns a URL to the JS bundle if there is a bundle loader defined. This will cause the asset to be preloaded prior to JS bundle execution. * Update tests * Clean up bundling code * Hopefully fix test in travis * wasm tests * Define correct wasm mime type Until pillarjs/send#154 is merged. * Use WebAssembly.instantiate instead of constructor Should fix error “WebAssembly.Instance is disallowed on the main thread, if the buffer size is larger than 4KB” * Fix test * Fix PromiseQueue bug * Remove rust for now. Will be added in a separate PR.
parcel-bundler · Jan 16, 2018 · 1a8ceb7 · 1a8ceb7
1 parent b80e523
commit 1a8ceb7
Show file tree

Hide file tree

Showing 33 changed files with 577 additions and 160 deletions.
diff --git a/.eslintignore b/.eslintignore
@@ -10,6 +10,8 @@
 /test/integration/dynamic-references-raw/index.js
 /test/integration/dynamic-references-raw/local.js
 /test/integration/hmr-dynamic/index.js
+/test/integration/wasm-async/index.js
+/test/integration/wasm-dynamic/index.js
 
 # Generated by the build
 lib

diff --git a/packages/core/parcel/package.json b/packages/core/parcel/package.json
@@ -15,6 +15,7 @@
     "babylon-walk": "^1.0.2",
     "browser-resolve": "^1.11.2",
     "chalk": "^2.1.0",
+    "child-process-promise": "^2.2.1",
     "chokidar": "^1.7.0",
     "commander": "^2.11.0",
     "cross-spawn": "^5.1.0",

diff --git a/packages/core/parcel/src/Bundle.js b/packages/core/parcel/src/Bundle.js
@@ -15,7 +15,20 @@ class Bundle {
     this.entryAsset = null;
     this.assets = new Set();
     this.childBundles = new Set();
-    this.siblingBundles = new Map();
+    this.siblingBundles = new Set;
+    this.siblingBundlesMap = new Map();
+  }
+
+  static createWithAsset(asset, parentBundle) {
+    let bundle = new Bundle(
+      asset.type,
+      Path.join(asset.options.outDir, asset.generateBundleName()),
+      parentBundle
+    );
+
+    bundle.entryAsset = asset;
+    bundle.addAsset(asset);
+    return bundle;
   }
 
   addAsset(asset) {
@@ -33,26 +46,36 @@ class Bundle {
       return this;
     }
 
-    if (!this.siblingBundles.has(type)) {
-      let bundle = this.createChildBundle(
+    if (!this.siblingBundlesMap.has(type)) {
+      let bundle = new Bundle(
         type,
         Path.join(
           Path.dirname(this.name),
           Path.basename(this.name, Path.extname(this.name)) + '.' + type
-        )
+        ),
+        this
       );
-      this.siblingBundles.set(type, bundle);
+
+      this.childBundles.add(bundle);
+      this.siblingBundles.add(bundle);
+      this.siblingBundlesMap.set(type, bundle);
     }
 
-    return this.siblingBundles.get(type);
+    return this.siblingBundlesMap.get(type);
   }
 
-  createChildBundle(type, name) {
-    let bundle = new Bundle(type, name, this);
+  createChildBundle(entryAsset) {
+    let bundle = Bundle.createWithAsset(entryAsset, this);
     this.childBundles.add(bundle);
     return bundle;
   }
 
+  createSiblingBundle(entryAsset) {
+    let bundle = this.createChildBundle(entryAsset);
+    this.siblingBundles.add(bundle);
+    return bundle;
+  }
+
   get isEmpty() {
     return this.assets.size === 0;
   }

diff --git a/packages/core/parcel/src/Bundler.js b/packages/core/parcel/src/Bundler.js
@@ -15,6 +15,7 @@ const localRequire = require('./utils/localRequire');
 const config = require('./utils/config');
 const emoji = require('./utils/emoji');
 const loadEnv = require('./utils/env');
+const PromiseQueue = require('./utils/PromiseQueue');
 
 /**
  * The Bundler is the main entry point. It resolves and loads assets,
@@ -32,6 +33,17 @@ class Bundler extends EventEmitter {
     this.cache = this.options.cache ? new FSCache(this.options) : null;
     this.logger = new Logger(this.options);
     this.delegate = options.delegate || {};
+    this.bundleLoaders = {};
+
+    this.addBundleLoader(
+      'wasm',
+      require.resolve('./builtins/loaders/wasm-loader')
+    );
+    this.addBundleLoader(
+      'css',
+      require.resolve('./builtins/loaders/css-loader')
+    );
+    this.addBundleLoader('js', require.resolve('./builtins/loaders/js-loader'));
 
     this.pending = false;
     this.loadedAssets = new Map();
@@ -41,7 +53,7 @@ class Bundler extends EventEmitter {
     this.hmr = null;
     this.bundleHashes = null;
     this.errored = false;
-    this.buildQueue = new Set();
+    this.buildQueue = new PromiseQueue(this.processAsset.bind(this));
     this.rebuildTimeout = null;
   }
 
@@ -92,6 +104,18 @@ class Bundler extends EventEmitter {
     this.packagers.add(type, packager);
   }
 
+  addBundleLoader(type, path) {
+    if (typeof path !== 'string') {
+      throw new Error('Bundle loader should be a module path.');
+    }
+
+    if (this.farm) {
+      throw new Error('Bundle loaders must be added before bundling.');
+    }
+
+    this.bundleLoaders[type] = path;
+  }
+
   async loadPlugins() {
     let pkg = await config.load(this.mainFile, ['package.json']);
     if (!pkg) {
@@ -141,8 +165,26 @@ class Bundler extends EventEmitter {
         this.buildQueue.add(this.mainAsset);
       }
 
-      // Build the queued assets, and produce a bundle tree.
-      let bundle = await this.buildQueuedAssets(isInitialBundle);
+      // Build the queued assets.
+      let loadedAssets = await this.buildQueue.run();
+
+      // Emit an HMR update for any new assets (that don't have a parent bundle yet)
+      // plus the asset that actually changed.
+      if (this.hmr && !isInitialBundle) {
+        this.hmr.emitUpdate([...this.findOrphanAssets(), ...loadedAssets]);
+      }
+
+      // Invalidate bundles
+      for (let asset of this.loadedAssets.values()) {
+        asset.invalidateBundle();
+      }
+
+      // Create a new bundle tree and package everything up.
+      let bundle = this.createBundleTree(this.mainAsset);
+      this.bundleHashes = await bundle.package(this, this.bundleHashes);
+
+      // Unload any orphaned assets
+      this.unloadOrphanedAssets();
 
       let buildTime = Date.now() - startTime;
       let time =
@@ -151,6 +193,7 @@ class Bundler extends EventEmitter {
           : `${(buildTime / 1000).toFixed(2)}s`;
       this.logger.status(emoji.success, `Built in ${time}.`, 'green');
 
+      this.emit('bundled', bundle);
       return bundle;
     } catch (err) {
       this.errored = true;
@@ -182,8 +225,8 @@ class Bundler extends EventEmitter {
     await loadEnv(this.mainFile);
 
     this.options.extensions = Object.assign({}, this.parser.extensions);
+    this.options.bundleLoaders = this.bundleLoaders;
     this.options.env = process.env;
-    this.farm = WorkerFarm.getShared(this.options);
 
     if (this.options.watch) {
       // FS events on macOS are flakey in the tests, which write lots of files very quickly
@@ -199,6 +242,8 @@ class Bundler extends EventEmitter {
       this.hmr = new HMRServer();
       this.options.hmrPort = await this.hmr.start(this.options.hmrPort);
     }
+
+    this.farm = WorkerFarm.getShared(this.options);
   }
 
   stop() {
@@ -215,49 +260,11 @@ class Bundler extends EventEmitter {
     }
   }
 
-  async buildQueuedAssets(isInitialBundle = false) {
-    // Consume the rebuild queue until it is empty.
-    let loadedAssets = new Set();
-    while (this.buildQueue.size > 0) {
-      let promises = [];
-      for (let asset of this.buildQueue) {
-        // Invalidate the asset, unless this is the initial bundle
-        if (!isInitialBundle) {
-          asset.invalidate();
-          if (this.cache) {
-            this.cache.invalidate(asset.name);
-          }
-        }
-
-        promises.push(this.loadAsset(asset));
-        loadedAssets.add(asset);
-      }
-
-      // Wait for all assets to load. If there are more added while
-      // these are processing, they'll be loaded in the next batch.
-      await Promise.all(promises);
-    }
-
-    // Emit an HMR update for any new assets (that don't have a parent bundle yet)
-    // plus the asset that actually changed.
-    if (this.hmr && !isInitialBundle) {
-      this.hmr.emitUpdate([...this.findOrphanAssets(), ...loadedAssets]);
-    }
-
-    // Invalidate bundles
-    for (let asset of this.loadedAssets.values()) {
-      asset.invalidateBundle();
-    }
-
-    // Create a new bundle tree and package everything up.
-    let bundle = this.createBundleTree(this.mainAsset);
-    this.bundleHashes = await bundle.package(this, this.bundleHashes);
-
-    // Unload any orphaned assets
-    this.unloadOrphanedAssets();
-
-    this.emit('bundled', bundle);
-    return bundle;
+  async getAsset(name, parent) {
+    let asset = await this.resolveAsset(name, parent);
+    this.buildQueue.add(asset);
+    await this.buildQueue.run();
+    return asset;
   }
 
   async resolveAsset(name, parent) {
@@ -328,9 +335,19 @@ class Bundler extends EventEmitter {
     }
   }
 
+  async processAsset(asset, isRebuild) {
+    if (isRebuild) {
+      asset.invalidate();
+      if (this.cache) {
+        this.cache.invalidate(asset.name);
+      }
+    }
+
+    await this.loadAsset(asset);
+  }
+
   async loadAsset(asset) {
     if (asset.processed) {
-      this.buildQueue.delete(asset);
       return;
     }
 
@@ -386,8 +403,6 @@ class Bundler extends EventEmitter {
         asset.depAssets.set(dep, assetDep);
       }
     });
-
-    this.buildQueue.delete(asset);
   }
 
   createBundleTree(asset, dep, bundle, parentBundles = new Set()) {
@@ -416,27 +431,20 @@ class Bundler extends EventEmitter {
       }
     }
 
-    // Create the root bundle if it doesn't exist
     if (!bundle) {
-      bundle = new Bundle(
-        asset.type,
-        Path.join(this.options.outDir, asset.generateBundleName())
-      );
-      bundle.entryAsset = asset;
+      // Create the root bundle if it doesn't exist
+      bundle = Bundle.createWithAsset(asset);
+    } else if (dep && dep.dynamic) {
+      // Create a new bundle for dynamic imports
+      bundle = bundle.createChildBundle(asset);
+    } else if (asset.type && !this.packagers.has(asset.type)) {
+      // No packager is available for this asset type. Create a new bundle with only this asset.
+      bundle.createSiblingBundle(asset);
+    } else {
+      // Add the asset to the common bundle of the asset's type
+      bundle.getSiblingBundle(asset.type).addAsset(asset);
     }
 
-    // Create a new bundle for dynamic imports
-    if (dep && dep.dynamic) {
-      bundle = bundle.createChildBundle(
-        asset.type,
-        Path.join(this.options.outDir, asset.generateBundleName())
-      );
-      bundle.entryAsset = asset;
-    }
-
-    // Add the asset to the bundle of the asset's type
-    bundle.getSiblingBundle(asset.type).addAsset(asset);
-
     // If the asset generated a representation for the parent bundle type, also add it there
     if (asset.generated[bundle.type] != null) {
       bundle.addAsset(asset);
@@ -523,7 +531,7 @@ class Bundler extends EventEmitter {
 
     // Add the asset to the rebuild queue, and reset the timeout.
     for (let asset of assets) {
-      this.buildQueue.add(asset);
+      this.buildQueue.add(asset, true);
     }
 
     clearTimeout(this.rebuildTimeout);

diff --git a/packages/core/parcel/src/Server.js b/packages/core/parcel/src/Server.js
@@ -5,6 +5,10 @@ const getPort = require('get-port');
 const serverErrors = require('./utils/customErrors').serverErrors;
 const generateCertificate = require('./utils/generateCertificate');
 
+serveStatic.mime.define({
+  'application/wasm': ['wasm']
+});
+
 function middleware(bundler) {
   const serve = serveStatic(bundler.options.outDir, {index: false});
 

diff --git a/packages/core/parcel/src/assets/HTMLAsset.js b/packages/core/parcel/src/assets/HTMLAsset.js
@@ -47,7 +47,9 @@ class HTMLAsset extends Asset {
             continue;
           }
           if (elements && elements.includes(node.tag)) {
-            let assetPath = this.addURLDependency(decodeURIComponent(node.attrs[attr]));
+            let assetPath = this.addURLDependency(
+              decodeURIComponent(node.attrs[attr])
+            );
             if (!isURL(assetPath)) {
               assetPath = urlJoin(this.options.publicURL, assetPath);
             }

diff --git a/packages/core/parcel/src/assets/RawAsset.js b/packages/core/parcel/src/assets/RawAsset.js
@@ -6,6 +6,12 @@ class RawAsset extends Asset {
   load() {}
 
   generate() {
+    // Don't return a URL to the JS bundle if there is a bundle loader defined for this asset type.
+    // This will cause the actual asset to be automatically preloaded prior to the JS bundle running.
+    if (this.options.bundleLoaders[this.type]) {
+      return {};
+    }
+
     const pathToAsset = urlJoin(
       this.options.publicURL,
       this.generateBundleName()