feat: handle Promise symbols added by node async_hooks

.github/workflows/ci.yml

@@ -108,6 +108,61 @@ jobs:
     - name: Run yarn test
       run: yarn test
 
+  test-async-hooks:
+    name: test-async-hooks
+
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version:
+          - '14.2' # last version before promise fast-path without destroy
+          - '14.17' # last version before unconditional promise fast-path
+          - '14.18' # first version after unconditional promise fast-path
+          - '16.1' # last version before some significant promise hooks changes
+          - '16.5' # last version before unconditional promise fast-path
+          - '16.6' # first version after unconditional promise fast-path
+        platform:
+          - ubuntu-latest
+
+# begin macro
+
+    steps:
+
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    - name: Echo node version
+      run: node --version
+
+    - name: Get yarn cache directory path
+      id: yarn-cache-dir-path
+      run: echo "::set-output name=dir::$(yarn cache dir)"
+
+    - name: Cache npm modules
+      uses: actions/cache@v1
+      id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+      with:
+        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+
+    - name: Install dependencies
+      run: yarn --frozen-lockfile
+
+# end macro
+
+    - name: Run yarn build
+      run: yarn build
+
+    - name: Run yarn test (@endo/init)
+      working-directory: packages/init
+      run: yarn test
+
   cover:
     name: cover
 

packages/init/node-async_hooks-patch.js

@@ -0,0 +1,3 @@
+import { setup } from './node-async_hooks.js';
+
+setup(true);

packages/init/node-async_hooks-symbols.d.ts

@@ -0,0 +1,11 @@
+// @ts-check
+
+export {};
+
+declare global {
+  interface SymbolConstructor {
+    readonly nodeAsyncHooksAsyncId: unique symbol;
+    readonly nodeAsyncHooksTriggerAsyncId: unique symbol;
+    readonly nodeAsyncHooksDestroyed: unique symbol;
+  }
+}

packages/init/node-async_hooks.js

@@ -0,0 +1,226 @@
+// @ts-check
+
+import { createHook, AsyncResource } from 'async_hooks';
+
+/// <reference path="./node-async_hooks-symbols.d.ts" />
+
+const asyncHooksWellKnownNameFromDescription = {
+  async_id_symbol: 'nodeAsyncHooksAsyncId',
+  trigger_async_id_symbol: 'nodeAsyncHooksTriggerAsyncId',
+  destroyed: 'nodeAsyncHooksDestroyed',
+};
+
+const promiseAsyncHookFallbackStates = new WeakMap();
+
+const setAsyncSymbol = (description, symbol) => {
+  const wellKnownName = asyncHooksWellKnownNameFromDescription[description];
+  if (!wellKnownName) {
+    throw new Error('Unknown symbol');
+  } else if (!Symbol[wellKnownName]) {
+    Symbol[wellKnownName] = symbol;
+    return true;
+  } else if (Symbol[wellKnownName] !== symbol) {
+    // console.warn(
+    //   `Found duplicate ${description}:`,
+    //   symbol,
+    //   Symbol[wellKnownName],
+    // );
+    return false;
+  } else {
+    return true;
+  }
+};
+
+// We can get the `async_id_symbol` and `trigger_async_id_symbol` through a
+// simple instantiation of async_hook.AsyncResource, which causes little side
+// effects. These are the 2 symbols that may be late bound, aka after the promise
+// is returned to the program and would normally be frozen.
+const findAsyncSymbolsFromAsyncResource = () => {
+  let found = 0;
+  Object.getOwnPropertySymbols(new AsyncResource('Bootstrap')).forEach(sym => {
+    const { description } = sym;
+    if (description in asyncHooksWellKnownNameFromDescription) {
+      if (setAsyncSymbol(description, sym)) {
+        found += 1;
+      }
+    }
+  });
+  return found;
+};
+
+// To get the `destroyed` symbol installed on promises by async_hooks,
+// the only option is to create and enable an AsyncHook.
+// Different versions of Node handle this in various ways.
+const findAsyncSymbolsFromPromiseCreateHook = () => {
+  const bootstrapData = [];
+
+  {
+    const bootstrapHook = createHook({
+      init(asyncId, type, triggerAsyncId, resource) {
+        if (type !== 'PROMISE') return;
+        // console.log('Bootstrap', asyncId, triggerAsyncId, resource);
+        bootstrapData.push({ asyncId, triggerAsyncId, resource });
+      },
+      destroy(_asyncId) {
+        // Needs to be present to trigger the addition of the destroyed symbol
+      },
+    });
+
+    bootstrapHook.enable();
+    // Use a never resolving promise to avoid triggering settlement hooks
+    const trigger = new Promise(() => {});
+    bootstrapHook.disable();
+
+    // In some versions of Node, async_hooks don't give access to the resource
+    // itself, but to a "wrapper" which is basically hooks metadata for the promise
+    const promisesData = bootstrapData.filter(
+      ({ resource }) => Promise.resolve(resource) === resource,
+    );
+    bootstrapData.length = 0;
+    const { length } = promisesData;
+    if (length > 1) {
+      // console.warn('Found multiple potential candidates');
+    }
+
+    const promiseData = promisesData.find(
+      ({ resource }) => resource === trigger,
+    );
+    if (promiseData) {
+      bootstrapData.push(promiseData);
+    } else if (length) {
+      // console.warn('No candidates matched');
+    }
+  }
+
+  if (bootstrapData.length) {
+    // Normally all promise hooks are disabled in a subsequent microtask
+    // That means Node versions that modify promises at init will still
+    // trigger our proto hooks for promises created in this turn
+    // The following trick will disable the internal promise init hook
+    // However, only do this for destroy modifying versions, since some versions
+    // only modify promises if no destroy hook is requested, and do not correctly
+    // reset the internal init promise hook in those case. (e.g. v14.16.2)
+    const resetHook = createHook({});
+    resetHook.enable();
+    resetHook.disable();
+
+    const { asyncId, triggerAsyncId, resource } = bootstrapData.pop();
+    const symbols = Object.getOwnPropertySymbols(resource);
+    // const { length } = symbols;
+    let found = 0;
+    // if (length !== 3) {
+    //   console.error(`Found ${length} symbols on promise:`, ...symbols);
+    // }
+    symbols.forEach(symbol => {
+      const value = resource[symbol];
+      let type;
+      if (value === asyncId) {
+        type = 'async_id_symbol';
+      } else if (value === triggerAsyncId) {
+        type = 'trigger_async_id_symbol';
+      } else if (typeof value === 'object' && 'destroyed' in value) {
+        type = 'destroyed';
+      } else {
+        // console.error(`Unexpected symbol`, symbol);
+        return;
+      }
+
+      if (setAsyncSymbol(type, symbol)) {
+        found += 1;
+      }
+    });
+    return found;
+  } else {
+    // This node version is not mutating promises
+    return -2;
+  }
+};
+
+const getAsyncHookFallbackState = (promise, create) => {
+  let state = promiseAsyncHookFallbackStates.get(promise);
+  if (!state && create) {
+    state = {
+      [Symbol.nodeAsyncHooksAsyncId]: undefined,
+      [Symbol.nodeAsyncHooksTriggerAsyncId]: undefined,
+    };
+    if (Symbol.nodeAsyncHooksDestroyed) {
+      state[Symbol.nodeAsyncHooksDestroyed] = undefined;
+    }
+    promiseAsyncHookFallbackStates.set(promise, state);
+  }
+  return state;
+};
+
+const setAsyncIdFallback = (promise, symbol, value) => {
+  const state = getAsyncHookFallbackState(promise, true);
+
+  if (state[symbol]) {
+    if (state[symbol] !== value) {
+      // This can happen if a frozen promise created before hooks were enabled
+      // is used multiple times as a parent promise
+      // It's safe to ignore subsequent values
+    }
+  } else {
+    state[symbol] = value;
+  }
+};
+
+const getAsyncHookSymbolPromiseProtoDesc = (symbol, disallowGet) => ({
+  set(value) {
+    if (Object.isExtensible(this)) {
+      Object.defineProperty(this, symbol, {
+        value,
+        writable: false,
+        configurable: false,
+        enumerable: false,
+      });
+    } else {
+      // console.log('fallback set of async id', symbol, value, new Error().stack);
+      setAsyncIdFallback(this, symbol, value);
+    }
+  },
+  get() {
+    if (disallowGet) {
+      return undefined;
+    }
+    const state = getAsyncHookFallbackState(this, false);
+    return state && state[symbol];
+  },
+  enumerable: false,
+  configurable: true,
+});
+
+export const setup = (withDestroy = true) => {
+  if (withDestroy) {
+    findAsyncSymbolsFromPromiseCreateHook();
+  } else {
+    findAsyncSymbolsFromAsyncResource();
+  }
+
+  if (!Symbol.nodeAsyncHooksAsyncId || !Symbol.nodeAsyncHooksTriggerAsyncId) {
+    // console.log(`Async symbols not found, moving on`);
+    return;
+  }
+
+  const PromiseProto = Promise.prototype;
+  Object.defineProperty(
+    PromiseProto,
+    Symbol.nodeAsyncHooksAsyncId,
+    getAsyncHookSymbolPromiseProtoDesc(Symbol.nodeAsyncHooksAsyncId),
+  );
+  Object.defineProperty(
+    PromiseProto,
+    Symbol.nodeAsyncHooksTriggerAsyncId,
+    getAsyncHookSymbolPromiseProtoDesc(Symbol.nodeAsyncHooksTriggerAsyncId),
+  );
+
+  if (Symbol.nodeAsyncHooksDestroyed) {
+    Object.defineProperty(
+      PromiseProto,
+      Symbol.nodeAsyncHooksDestroyed,
+      getAsyncHookSymbolPromiseProtoDesc(Symbol.nodeAsyncHooksDestroyed, true),
+    );
+  } else if (withDestroy) {
+    // console.warn(`Couldn't find destroyed symbol to setup trap`);
+  }
+};

packages/init/pre.js

@@ -1,2 +1,4 @@
 // Generic preamble for all shims.
+
+import './node-async_hooks-patch.js';
 import '@endo/lockdown';

packages/init/test/test-async_hooks.js

@@ -0,0 +1,79 @@
+/* global globalThis, $262 */
+
+import '../index.js';
+import test from 'ava';
+import { createHook } from 'async_hooks';
+import { setTimeout } from 'timers';
+
+const gcP = (async () => {
+  let gc = globalThis.gc || (typeof $262 !== 'undefined' ? $262.gc : null);
+  if (!gc) {
+    gc = () => {
+      Array.from({ length: 2 ** 24 }, () => Math.random());
+    };
+  }
+  return gc;
+})();
+
+test('async_hooks Promise patch', async t => {
+  const hasSymbols =
+    Symbol.nodeAsyncHooksAsyncId && Symbol.nodeAsyncHooksTriggerAsyncId;
+  let resolve;
+  const q = (() => {
+    const p1 = new Promise(r => (resolve = r));
+    t.deepEqual(
+      Reflect.ownKeys(p1),
+      [],
+      `Promise instances don't start with any own keys`,
+    );
+    harden(p1);
+
+    // The `.then()` fulfillment triggers the "before" hook for `p2`,
+    // which enforces that `p2` is a tracked promise by installing async id symbols
+    const p2 = Promise.resolve().then(() => {});
+    t.deepEqual(
+      Reflect.ownKeys(p2),
+      [],
+      `Promise instances don't start with any own keys`,
+    );
+    harden(p2);
+
+    const testHooks = createHook({
+      init() {},
+      before() {},
+      // after() {},
+      destroy() {},
+    });
+    testHooks.enable();
+
+    // Create a promise with symbols attached
+    const p3 = Promise.resolve();
+    if (hasSymbols) {
+      t.truthy(Reflect.ownKeys(p3));
+    }
+
+    return Promise.resolve().then(() => {
+      resolve(8);
+      // ret is a tracked promise created from parent `p1`
+      // async_hooks will attempt to get the asyncId from `p1`
+      // which was created and frozen before the symbols were installed
+      const ret = p1.then(() => {});
+      // Trigger attempting to get asyncId of `p1` again, which in current
+      // node versions will fail and generate a new one because of an own check
+      p1.then(() => {});
+
+      if (hasSymbols) {
+        t.truthy(Reflect.ownKeys(ret));
+      }
+
+      // testHooks.disable();
+
+      return ret;
+    });
+  })();
+
+  return q
+    .then(() => new Promise(r => setTimeout(r, 0, gcP)))
+    .then(gc => gc())
+    .then(() => new Promise(r => setTimeout(r)));
+});