Custom messengers and RPC in web-worker

packages/react-web-worker/CHANGELOG.md

@@ -7,6 +7,10 @@ and adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 <!-- ## [Unreleased] -->
 
+## [1.1.0] - 2019-11-08
+
+- You can now pass options as the second argument to `useWorker`. These options are forwarded as the [options to the worker creator](../web-worker#customizing-worker-creation).
+
 ## [1.0.4] - 2019-11-07
 
 ### Fixed

packages/react-web-worker/README.md

@@ -35,18 +35,17 @@ function Home() {
   const worker = useWorker(createWorker);
   const [message, setMessage] = React.useState(null);
 
-  useEffect(
-    () => {
-      (async () => {
-        // Note: in your actual app code, make sure to check if Home
-        // is still mounted before setting state asynchronously!
-        const webWorkerMessage = await worker.hello('Tobi');
-        setMessage(webWorkerMessage);
-      })();
-    },
-    [worker],
-  );
+  useEffect(() => {
+    (async () => {
+      // Note: in your actual app code, make sure to check if Home
+      // is still mounted before setting state asynchronously!
+      const webWorkerMessage = await worker.hello('Tobi');
+      setMessage(webWorkerMessage);
+    })();
+  }, [worker]);
 
   return <Page title="Home"> {message} </Page>;
 }
 ```
+
+You can optionally pass a second argument to `useWorker`, which will be used as the [options to the worker creator function](../web-worker#customizing-worker-creation).

packages/react-web-worker/src/hooks.ts

@@ -1,9 +1,12 @@
 import {useEffect} from 'react';
-import {terminate} from '@shopify/web-worker';
+import {terminate, CreateWorkerOptions} from '@shopify/web-worker';
 import {useLazyRef} from '@shopify/react-hooks';
 
-export function useWorker<T>(creator: () => T) {
-  const {current: worker} = useLazyRef(creator);
+export function useWorker<T>(
+  creator: (options?: CreateWorkerOptions) => T,
+  options?: CreateWorkerOptions,
+) {
+  const {current: worker} = useLazyRef(() => creator(options));
 
   useEffect(() => {
     return () => {

packages/rpc/CHANGELOG.md

@@ -7,6 +7,10 @@ and adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Fixed
+
+- Made the default function strategy use the messenger, which makes is more broadly useful across different `postMessage` interfaces.
+
 ### Added
 
 - `@shopify/rpc` package

packages/rpc/src/endpoint.ts

@@ -5,7 +5,7 @@ import {
   FunctionStrategyOptions,
 } from './types';
 import {Retainer, StackFrame} from './memory';
-import {createChannelFunctionStrategy} from './strategies';
+import {createMessengerFunctionStrategy} from './strategies';
 
 const APPLY = 0;
 const RESULT = 1;
@@ -32,7 +32,7 @@ export function createEndpoint<T>(
   initialMessenger: MessageEndpoint,
   {
     uuid = defaultUuid,
-    createFunctionStrategy = createChannelFunctionStrategy,
+    createFunctionStrategy = createMessengerFunctionStrategy,
   }: Options = {},
 ): Endpoint<T> {
   let terminated = false;

packages/rpc/src/index.ts

@@ -19,4 +19,5 @@ export {
   FunctionStrategyOptions,
   RemoteCallable,
   SafeRpcArgument,
+  MessageEndpoint,
 } from './types';

packages/web-worker/CHANGELOG.md

@@ -7,6 +7,17 @@ and adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 <!-- ## [Unreleased] -->
 
+## [1.2.0] - 2019-11-08
+
+### Changed
+
+- Uses the new `@shopify/rpc` library for communication with the worker.
+
+### Added
+
+- You can now supply an optional options object to the `createWorkerFactory` functions. One option is currently supported: `createMessenger`, which allows you to customize the message channel for the worker.
+- To support creating workers that are not treated as same-origin, the library now provides a `createIframeWorkerMessenger` function. This function is passed to the new `createMessenger` API, and works by creating a message channel directly from the host page to a worker in a sandboxed `iframe`.
+
 ## [1.1.0] - 2019-10-21
 
 ### Added

packages/web-worker/README.md

@@ -47,7 +47,7 @@ const result = await worker.hello('world'); // 'Hello, world'
 Note that more complex workers are allowed; it can export multiple functions, including default exports, and it can accept complex argument types [with some restrictions](#limitations):
 
 ```ts
-const worker = makeWorker();
+const worker = createWorker();
 
 // Assuming worker was:
 // export default function hello(name) {
@@ -82,119 +82,86 @@ const worker = createWorker();
 terminate(worker);
 ```
 
-##### Naming the worker file
-
-By default, worker files created using `createWorkerFactory` are given incrementing IDs as the file name. This strategy is generally less than ideal for long-term caching, as the name of the file depends on the order in which it was encountered during the build. For long-term caching, it is better to provide a static name for the worker file. This can be done by supplying the [`webpackChunkName` comment](https://webpack.js.org/api/module-methods/#magic-comments) before your import:
-
-```tsx
-import {createWorkerFactory, terminate} from '@shopify/web-worker;
-
-// Note: only webpackChunkName is currently supported. Don’t try to use
-// other magic webpack comments.
-const createWorker = createWorkerFactory(() => import(/* webpackChunkName: 'myWorker' */ './worker'));
-```
+##### Customizing worker creation
 
-This name will be used as the prefix for the worker file. The worker will always end in `.worker.js`, and may also include additional hashes or other content (this library re-uses your `output.filename` and `output.chunkFilename` webpack options).
+By default, this library will create a worker by calling `new Worker` with a blob URL for the worker script. This is generally all you need, but some use cases may want to construct the worker differently. For example, you might want to construct a worker in a sandboxed iframe to ensure the worker is not treated as same-origin, or create a worker farm instead of a worker per script. To do so, you can supply the `createMessenger` option to the function provided by `createWorkerFactory`. This option should be a function that accepts a `URL` object for the location of the worker script, and return a `MessageEndpoint` compatible with being passed to `@shopify/rpc`’s `createEndpoint` API.
 
-#### Worker
+```ts
+import {fromMessagePort} from '@shopify/rpc';
+import {createWorkerFactory} from '@shopify/web-worker';
 
-Your worker can be written almost indistinguishably from a "normal" module. It can import other modules (including async `import()` statements), use modern JavaScript features, and more. The exported functions from your module form its public API, which the main thread code will call into as shown above. Note that only functions can be exported; this library is primarily meant to be used to create an imperative API for offloading work to a worker, for which only function exports are needed.
+/* imaginary abstraction that vends workers */
+const workerFarm = {};
 
-As noted in the browser section, worker code should be mindful of the [limitations](#limitations) of what can be passed into and out of a worker function.
+const createWorker = createWorkerFactory(() => import('./worker'));
+const worker = createWorker({
+  createMessenger(url) {
+    return workerFarm.workerWithUrl(url);
+  },
+});
+```
 
-Your worker functions should be careful to note that, if they accept any arguments that include functions, those functions should at least optionally return a promise. This is because, when this argument is passed from the main thread to the worker, it can only pass a function that returns a promise. To help you make sure you are respecting this condition, we provide a `SafeWorkerArgument` helper type you can use for all arguments that your worker accepts.
+An optimization many uses of `createMessenger` will want to make is to use a `MessageChannel` to directly connect the worker and the main page, even if the worker itself is constructed unconventionally (e.g., inside an iframe). As a convenience, the worker that is constructed by this library supports `postMessage`ing a special `{__replace: MessagePort}` object. When sent, the `MessagePort` will be used as an argument to the worker’s [`Endpoing#replace` method](../rpc#endpoint-replace), making it the communication channel for all messages between the parent page and worker.
 
 ```ts
-import {SafeWorkerArgument} from '@shopify/web-worker';
-
-export function greet(name: SafeWorkerArgument<string | () => string>) {
-  // name is `string | (() => string | Promise<string>)` because a worker
-  // can synchronously pass a `string` argument, but can only provide a
-  // `() => Promise<string>` function, since it will have to proxy over
-  // message passing. Note that `() => string` is still allowed because
-  // it could still be valid for another function in the worker to call
-  // with a function of that type.
-  return (
-    typeof name === 'string'
-      ? `Hello, ${name}`
-      : Promise.resolve(name()).then((name) => `Hello, ${name}`)
-  );
-}
-```
+import {fromMessagePort} from '@shopify/rpc';
+import {createWorkerFactory} from '@shopify/web-worker';
 
-The same [memory management concerns](#memory) apply to the worker as they do on the main thread.
+/* imaginary abstraction that creates workers in an iframe */
+const iframe = {};
 
-#### Limitations
+const createWorker = createWorkerFactory(() => import('./worker'));
+const worker = createWorker({
+  createMessenger(url) {
+    const {port1, port2} = new MessageChannel();
+    iframe.createWorker(url).postMessage({__replace: port2}, [port2]);
+
+    // In a real example, you'd want to also clean up the worker
+    // you've created in the iframe as part of the `terminate()` method.
+    return fromMessagePort(port1);
+  },
+});
+```
 
-There are two key limitations to be aware of when calling functions in a worker with the help of this library:
+###### `createIframeWorkerMessenger()`
 
-1. Only basic data structures are supported. Any function involved in a call across the main thread/ worker boundary can accept and return primitive types, objects, arrays, and functions. You can’t pass other data structures (like `Map`s or `Set`s), and if you pass class instances back and forth, only their own properties will be available on the other side.
-2. The worker can only export functions. Because the main thread can’t access the worker thread values synchronously, there is no way to implement arbitrary access to non-function exports without awkward use of promises.
+The `createIframeWorkerMessenger` is provided to make it easy to create a worker that is not treated as same-origin to the parent page. This function will, for each worker, create an iframe with a restrictive `sandbox` attribute and an anonymous origin, and will force that iframe to create a worker. It then passes a `MessagePort` through to the worker as the `postMessage` interface to use so that messages go directly between the worker and the original page.
 
-Additionally, when passing functions to and from the worker, developers may occasionally need to manually manage memory. This is detailed in the next section.
+```ts
+import {
+  createWorkerFactory,
+  createIframeWorkerMessenger,
+} from '@shopify/web-worker';
 
-#### Memory
+const createWorker = createWorkerFactory(() => import('./worker'));
+const worker = createWorker({
+  createMessenger: createIframeWorkerMessenger,
+});
+```
 
-Web worker’s can’t share memory with their parent, and functions can’t be serialized for `postMessage`. The implementation of passing functions between worker and parent is therefore implemented very differently from other data types: the worker and parent side keep references to functions that have been passed between the two, and they have a shared strategy for proxying calls from the "target" side back to the original source function.
+##### Naming the worker file
 
-This strategy is effective, but without extra intervention it will leak memory. Even if the parent and worker no longer have references to that function, it must still be retained because the parent can’t know that the worker no longer needs to call that function.
+By default, worker files created using `createWorkerFactory` are given incrementing IDs as the file name. This strategy is generally less than ideal for long-term caching, as the name of the file depends on the order in which it was encountered during the build. For long-term caching, it is better to provide a static name for the worker file. This can be done by supplying the [`webpackChunkName` comment](https://webpack.js.org/api/module-methods/#magic-comments) before your import:
 
-This library automatically implements some memory management for you. A function passed between the worker and parent is automatically retained for the lifetime of the original function call, and is subsequently released.
+```tsx
+import {createWorkerFactory, terminate} from '@shopify/web-worker;
 
-```ts
-// PARENT
-
-const worker = createWorker(/* ... */)();
-const funcForWorker = () => 'Tobi';
-
-worker
-  // funcForWorker is retained on the main thread here...
-  .greet(funcForWorker)
-  // And is automatically released here because the worker
-  // signals it no longer needs the function
-  .then(result => console.log(result));
-
-// WORKER
-
-export async function greet(getName: () => Promise<string>) {
-  // Worker signals that it needs to retain `getName`, which
-  // was passed from the parent.
-
-  try {
-    return `Hello, ${await getName()}`;
-  } finally {
-    // Once this function exits, the library defaults to releasing
-    // `getName`, which signals to the main thread it can release
-    // the original function.
-  }
-}
+// Note: only webpackChunkName is currently supported. Don’t try to use
+// other magic webpack comments.
+const createWorker = createWorkerFactory(() => import(/* webpackChunkName: 'myWorker' */ './worker'));
 ```
 
-This covers most common memory management cases, but one important exception remains: if you save the function on to an object in context, it will be still be accessible to your program, but the source of the function will be told to release the reference to that function. In this case, if you try to call the function from the worker at a later time, you will receive an error indicating that the value has been released.
-
-To resolve this problem, this library provides `retain` and `release` functions. Calling these on an object will increment the number of "retainers", allowing the source function to be retained. Any time you call `retain`, you must eventually call `release`, when you know you will no longer call that function.
-
-```ts
-// WORKER
-
-import {retain, release} from '@shopify/web-worker';
+This name will be used as the prefix for the worker file. The worker will always end in `.worker.js`, and may also include additional hashes or other content (this library re-uses your `output.filename` and `output.chunkFilename` webpack options).
 
-export function setNameGetter(getName: () => Promise<string>) {
-  retain(getName);
+#### Worker
 
-  if (self.getName) {
-    release(self.getName);
-  }
+Your worker can be written almost indistinguishably from a "normal" module. It can import other modules (including async `import()` statements), use modern JavaScript features, and more. The exported functions from your module form its public API, which the main thread code will call into as shown above. Note that only functions can be exported; this library is primarily meant to be used to create an imperative API for offloading work to a worker, for which only function exports are needed.
 
-  self.getName = getName;
-}
+As noted in the browser section, worker code should be mindful of the [limitations](#limitations) of what can be passed into and out of a worker function.
 
-export async function greet() {
-  return `Hello, ${self.getName ? await self.getName() : 'friend'}!`;
-}
-```
+#### Limitations
 
-Remember that any function passed between the worker and its parent, including functions attached as properties of objects, must be retained manually if you intend to call them outside the scope of the first function where they were passed over the bridge. To help make this easier, `release` and `retain` will automatically deeply release/ retain all functions when they are called with objects or arrays.
+This library implements the calling of functions on a worker using [`@shopify/rpc`](../rpc). As such, all the limitations and additional considerations in that library must be considered with the functions you expose from the worker. In particular, note the [memory management concerns](../rpc#memory) when passing functions to and from the worker. For convenience, the `release` and `retain` methods from `@shopify/rpc` are re-exported from this library.
 
 ### Tooling
 

packages/web-worker/package.json

@@ -24,6 +24,7 @@
   },
   "homepage": "https://github.com/Shopify/quilt/blob/master/packages/web-workers/README.md",
   "dependencies": {
+    "@shopify/rpc": "^1.0.0",
     "@types/webpack": "^4.0.0",
     "loader-utils": "^1.0.0",
     "webpack-virtual-modules": "^0.1.12"

packages/web-worker/src/create.ts

@@ -1,4 +1,6 @@
-import {createEndpoint, Endpoint} from './endpoint';
+import {createEndpoint, Endpoint, MessageEndpoint} from '@shopify/rpc';
+
+import {createWorkerMessenger} from './messenger';
 
 const workerEndpointCache = new WeakMap<Endpoint<any>['call'], Endpoint<any>>();
 
@@ -23,8 +25,14 @@ export function getEndpoint(caller: any) {
   return workerEndpointCache.get(caller);
 }
 
+export interface CreateWorkerOptions {
+  createMessenger?(url: URL): MessageEndpoint;
+}
+
 export function createWorkerFactory<T>(script: () => Promise<T>) {
-  return function createWorker(): Endpoint<T>['call'] {
+  return function createWorker({
+    createMessenger = createWorkerMessenger,
+  }: CreateWorkerOptions = {}): Endpoint<T>['call'] {
     // The babel plugin that comes with this package actually turns the argument
     // into a string (the public path of the worker script). If it’s a function,
     // it’s because we’re in an environment where we didn’t transform it into a
@@ -46,7 +54,7 @@ export function createWorkerFactory<T>(script: () => Promise<T>) {
 
     // If we aren’t in an environment that supports Workers, just bail out
     // with a dummy worker that throws for every method call.
-    if (typeof Worker === 'undefined') {
+    if (typeof window === 'undefined') {
       return new Proxy(
         {},
         {
@@ -61,21 +69,8 @@ export function createWorkerFactory<T>(script: () => Promise<T>) {
       ) as any;
     }
 
-    const absoluteScript = new URL(script, window.location.href).href;
-
-    const workerScript = URL.createObjectURL(
-      new Blob([`importScripts(${JSON.stringify(absoluteScript)})`]),
-    );
-
-    const worker = new Worker(workerScript);
-
-    const originalTerminate = worker.terminate.bind(worker);
-    worker.terminate = () => {
-      URL.revokeObjectURL(workerScript);
-      originalTerminate();
-    };
-
-    const endpoint = createEndpoint(worker);
+    const scriptUrl = new URL(script, window.location.href);
+    const endpoint = createEndpoint(createMessenger(scriptUrl));
     const {call: caller} = endpoint;
 
     workerEndpointCache.set(caller, endpoint);