Merge pull request #3726 from parcel-bundler/wbinnssmith/inline-bundl…

…e-urls

Encoded inline bundles

flow-libs/isbinaryfile.js.flow

@@ -0,0 +1,16 @@
+// @flow
+
+// Derived from the README for isbinaryfile, available at:
+// https://github.com/gjtorikian/isBinaryFile#readme
+// Which is licensed MIT.
+
+declare module 'isbinaryfile' {
+ declare export function isBinaryFile(
+ buffer: Buffer,
+ bytesToRead?: number
+ ): Promise<boolean>;
+ declare export function isBinaryFileSync(
+ buffer: Buffer,
+ bytesToRead?: number
+ ): boolean;
+}

packages/bundlers/default/src/DefaultBundler.js

@@ -40,9 +40,10 @@ export default new Bundler({
  let resolution = bundleGraph.getDependencyResolution(dependency);
 
  if (
- dependency.isEntry ||
- dependency.isAsync ||
- resolution?.isIsolated
+ (dependency.isEntry && resolution) ||
+ (dependency.isAsync && resolution) ||
+ resolution?.isIsolated ||
+ resolution?.isInline
  ) {
  let bundleGroup = bundleGraph.createBundleGroup(
  dependency,

packages/configs/default/index.json

@@ -2,6 +2,8 @@
  "bundler": "@parcel/bundler-default",
  "transforms": {
  "types:*.{ts,tsx}": ["@parcel/transformer-typescript-types"],
+ "bundle-text:*": ["@parcel/transformer-inline-string", "..."],
+ "data-url:*": ["@parcel/transformer-inline-string", "..."],
  "*.{js,mjs,jsm,jsx,es6,ts,tsx}": [
  "@parcel/transformer-babel",
  "@parcel/transformer-js"
@@ -29,6 +31,7 @@
  "node": ["@parcel/runtime-js"]
  },
  "optimizers": {
+ "data-url:*": ["...", "@parcel/optimizer-data-url"],
  "*.css": ["@parcel/optimizer-cssnano"],
  "*.js": ["@parcel/optimizer-terser"],
  "*.html": ["@parcel/optimizer-htmlnano"]

packages/configs/default/package.json

@@ -17,6 +17,7 @@
  "@parcel/bundler-default": "^2.0.0-alpha.2.1",
  "@parcel/namer-default": "^2.0.0-alpha.2.1",
  "@parcel/optimizer-cssnano": "^2.0.0-alpha.2.1",
+ "@parcel/optimizer-data-url": "^2.0.0-alpha.2.1",
  "@parcel/optimizer-terser": "^2.0.0-alpha.2.1",
  "@parcel/optimizer-htmlnano": "^2.0.0-alpha.2.1",
  "@parcel/packager-css": "^2.0.0-alpha.2.1",
@@ -34,6 +35,7 @@
  "@parcel/transformer-coffeescript": "^2.0.0-alpha.2.1",
  "@parcel/transformer-css": "^2.0.0-alpha.2.1",
  "@parcel/transformer-html": "^2.0.0-alpha.2.1",
+ "@parcel/transformer-inline-string": "^2.0.0-alpha.2.1",
  "@parcel/transformer-js": "^2.0.0-alpha.2.1",
  "@parcel/transformer-json": "^2.0.0-alpha.2.1",
  "@parcel/transformer-less": "^2.0.0-alpha.2.1",

packages/configs/default/test/config.test.js

@@ -53,6 +53,10 @@ function collectConfigPackageReferences(
 
  for (let value of Object.values(configSection)) {
  if (typeof value === 'string') {
+ if (value === '...') {
+ continue;
+ }
+
  references.add(value);
  } else if (configSection != null && typeof configSection === 'object') {
  collectConfigPackageReferences(value, references);

packages/core/core/src/InternalAsset.js

@@ -45,6 +45,7 @@ type AssetOptions = {|
  outputHash?: string,
  env: Environment,
  meta?: Meta,
+ pipeline?: ?string,
  stats: Stats,
  symbols?: Map<Symbol, Symbol>,
  sideEffects?: boolean,
@@ -72,6 +73,7 @@ export function createAsset(options: AssetOptions): Asset {
  includedFiles: options.includedFiles || new Map(),
  isSource: options.isSource,
  outputHash: options.outputHash || '',
+ pipeline: options.pipeline,
  env: options.env,
  meta: options.meta || {},
  stats: options.stats,
@@ -287,6 +289,9 @@ export default class InternalAsset {
  : new Map(),
  includedFiles: new Map(this.value.includedFiles),
  meta: {...this.value.meta, ...result.meta},
+ pipeline:
+ result.pipeline ??
+ (this.value.type === result.type ? this.value.pipeline : null),
  stats: {
  time: 0,
  size

packages/core/core/src/PackagerRunner.js

@@ -15,16 +15,10 @@ import type ParcelConfig from './ParcelConfig';
 import type InternalBundleGraph from './BundleGraph';
 import type {FileSystem, FileOptions} from '@parcel/fs';
 
-import {
- urlJoin,
- md5FromObject,
- md5FromString,
- blobToStream
-} from '@parcel/utils';
+import {md5FromObject, md5FromString, blobToStream} from '@parcel/utils';
 import {PluginLogger} from '@parcel/logger';
 import ThrowableDiagnostic, {errorToDiagnostic} from '@parcel/diagnostic';
 import {Readable} from 'stream';
-import invariant from 'assert';
 import nullthrows from 'nullthrows';
 import path from 'path';
 import url from 'url';
@@ -233,7 +227,7 @@ export default class PackagerRunner {
 
  let packager = await this.config.getPackager(bundle.filePath);
  try {
- let packaged = await packager.plugin.package({
+ return await packager.plugin.package({
  bundle,
  bundleGraph: new BundleGraph(bundleGraph, this.options),
  getSourceMapReference: map => {
@@ -260,17 +254,6 @@ export default class PackagerRunner {
  );
  }
  });
-
- return {
- contents:
- typeof packaged.contents === 'string'
- ? replaceReferences(
- packaged.contents,
- generateDepToBundlePath(internalBundle, bundleGraph)
- )
- : packaged.contents,
- map: packaged.map
- };
  } catch (e) {
  throw new ThrowableDiagnostic({
  diagnostic: errorToDiagnostic(e, packager.name)
@@ -285,7 +268,10 @@ export default class PackagerRunner {
  map?: ?SourceMap
  ): Promise<BundleResult> {
  let bundle = new NamedBundle(internalBundle, bundleGraph, this.options);
- let optimizers = await this.config.getOptimizers(bundle.filePath);
+ let optimizers = await this.config.getOptimizers(
+ bundle.filePath,
+ internalBundle.pipeline
+ );
  if (!optimizers.length) {
  return {contents, map};
  }
@@ -490,72 +476,6 @@ function writeFileStream(
  });
 }
 
-/*
- * Build a mapping from async, url dependency ids to web-friendly relative paths
- * to their bundles. These will be relative to the current bundle if `publicUrl`
- * is not provided. If `publicUrl` is provided, the paths will be joined to it.
- *
- * These are used to translate any placeholder dependency ids written during
- * transformation back to a path that can be loaded in a browser (such as
- * in a "raw" loader or any transformed dependencies referred to by url).
- */
-function generateDepToBundlePath(
- bundle: InternalBundle,
- bundleGraph: InternalBundleGraph
-): Map<string, FilePath> {
- let depToBundlePath: Map<string, FilePath> = new Map();
- bundleGraph.traverseBundle(bundle, node => {
- if (node.type !== 'dependency') {
- return;
- }
-
- let dep = node.value;
- if (!dep.isURL || !dep.isAsync) {
- return;
- }
-
- let [bundleGroupNode] = bundleGraph._graph.getNodesConnectedFrom(node);
- invariant(bundleGroupNode && bundleGroupNode.type === 'bundle_group');
-
- let [entryBundleNode] = bundleGraph._graph.getNodesConnectedFrom(
- bundleGroupNode,
- 'bundle'
- );
- invariant(entryBundleNode && entryBundleNode.type === 'bundle');
-
- let entryBundle = entryBundleNode.value;
- depToBundlePath.set(
- dep.id,
- urlJoin(
- nullthrows(entryBundle.target).publicUrl ?? '/',
- nullthrows(entryBundle.name)
- )
- );
- });
-
- return depToBundlePath;
-}
-
-// replace references to url dependencies with relative paths to their
-// corresponding bundles.
-// TODO: This likely alters the length of the column in the source text.
-// Update any sourcemaps accordingly.
-function replaceReferences(
- code: string,
- depToBundlePath: Map<string, FilePath>
-): string {
- let output = code;
- for (let [depId, replacement] of depToBundlePath) {
- let split = output.split(depId);
- if (split.length > 1) {
- // the dependency id was found in the text. replace it.
- output = split.join(replacement);
- }
- }
-
- return output;
-}
-
 function getContentKey(cacheKey: string) {
  return md5FromString(`${cacheKey}:content`);
 }

packages/core/core/src/ParcelConfig.js

@@ -153,6 +153,12 @@ export default class ParcelConfig {
  return this.loadPlugins<Validator>(names);
  }
 
+ getNamedPipelines(): $ReadOnlyArray<string> {
+ return Object.keys(this.transforms)
+ .filter(glob => glob.includes(':'))
+ .map(glob => glob.split(':')[0]);
+ }
+
  getTransformers(filePath: FilePath, pipeline?: ?string) {
  return this.loadPlugins<Transformer>(
  this.getTransformerNames(filePath, pipeline)
@@ -215,26 +221,22 @@ export default class ParcelConfig {
  };
  }
 
- getOptimizerNames(filePath: FilePath): Array<string> {
- let optimizers: ?Pipeline = this.matchGlobMapPipelines(
- filePath,
- this.optimizers
+ getOptimizerNames(filePath: FilePath, pipeline: ?string): Array<string> {
+ return (
+ this.matchGlobMapPipelines(filePath, this.optimizers, pipeline) ?? []
  );
- if (!optimizers) {
- return [];
- }
- return optimizers;
  }
 
  getOptimizers(
- filePath: FilePath
+ filePath: FilePath,
+ pipeline: ?string
  ): Promise<
  Array<{|
  name: string,
  plugin: Optimizer
  |}>
  > {
- let optimizers = this.getOptimizerNames(filePath);
+ let optimizers = this.getOptimizerNames(filePath, pipeline);
  if (optimizers.length === 0) {
  return Promise.resolve([]);
  }

packages/core/core/src/ResolverRunner.js

@@ -1,11 +1,13 @@
 // @flow
 
 import type {AssetRequestDesc, Dependency, ParcelOptions} from './types';
+import type ParcelConfig from './ParcelConfig';
 
 import {PluginLogger} from '@parcel/logger';
 import ThrowableDiagnostic, {errorToDiagnostic} from '@parcel/diagnostic';
 import path from 'path';
-import type ParcelConfig from './ParcelConfig';
+import URL from 'url';
+
 import {report} from './ReporterRunner';
 import PublicDependency from './public/Dependency';
 import PluginOptions from './public/PluginOptions';
@@ -36,9 +38,41 @@ export default class ResolverRunner {
 
  let resolvers = await this.config.getResolvers();
 
+ let pipeline;
+ let filePath;
+ let validPipelines = new Set(this.config.getNamedPipelines());
+ if (
+ // Don't consider absolute paths. Absolute paths are only supported for entries,
+ // and include e.g. `C:\` on Windows, conflicting with pipelines.
+ !path.isAbsolute(dependency.moduleSpecifier) &&
+ dependency.moduleSpecifier.includes(':')
+ ) {
+ [pipeline, filePath] = dependency.moduleSpecifier.split(':');
+ if (!validPipelines.has(pipeline)) {
+ if (dep.isURL) {
+ // This may be a url protocol or scheme rather than a pipeline, such as
+ // `url('http://example.com/foo.png')`
+ return null;
+ } else {
+ throw new Error(`Unknown pipeline ${pipeline}.`);
+ }
+ }
+ } else {
+ filePath = dependency.moduleSpecifier;
+ }
+
+ if (dependency.isURL) {
+ let parsed = URL.parse(filePath);
+ if (typeof parsed.pathname !== 'string') {
+ throw new Error('Received URL without a pathname.');
+ }
+ filePath = decodeURIComponent(parsed.pathname);
+ }
+
  for (let resolver of resolvers) {
  try {
  let result = await resolver.plugin.resolve({
+ filePath,
  dependency: dep,
  options: this.pluginOptions,
  logger: new PluginLogger({origin: resolver.name})
@@ -54,7 +88,7 @@ export default class ResolverRunner {
  sideEffects: result.sideEffects,
  code: result.code,
  env: dependency.env,
- pipeline: dependency.pipeline
+ pipeline: pipeline ?? dependency.pipeline
  };
  }
  } catch (e) {

packages/core/core/src/Transformation.js

@@ -111,7 +111,7 @@ export default class Transformation {
  }
 
  async loadAsset(): Promise<InternalAsset> {
- let {filePath, env, code, sideEffects} = this.request;
+ let {filePath, env, code, pipeline, sideEffects} = this.request;
  let {content, size, hash, isSource} = await summarizeRequest(
  this.options.inputFS,
  this.request
@@ -128,6 +128,7 @@ export default class Transformation {
  isSource,
  type: path.extname(filePath).slice(1),
  hash,
+ pipeline,
  env,
  stats: {
  time: 0,
@@ -594,6 +595,7 @@ function normalizeAssets(
  env: result.env,
  isIsolated: result.isIsolated,
  isInline: result.isInline,
+ pipeline: internalAsset.value.pipeline,
  meta: result.meta,
  uniqueKey: internalAsset.value.uniqueKey
  };