Skip to content

Commit

Permalink
src: compile native modules and their code cache in C++
Browse files Browse the repository at this point in the history 
This patch refactors out a part of NativeModule.prototype.compile
(in JS land) into a C++ NativeModule class, this enables a
couple of possibilities:

1. By moving the code to the C++ land, we have more opportunity
  to specialize the compilation process of the native modules
  (e.g. compilation options, code cache) that is orthogonal to
  how user land modules are compiled
2. We can reuse the code to compile bootstrappers and context
  fixers and enable them to be compiled with the code cache later,
  since they are not loaded by NativeModule in the JS land their
  caching must be done in C++.
3. Since there is no need to pass the static data to JS for
  compilation anymore, this enables us to use
  (std::map<std::string, const char*>) in the generated
  node_code_cache.cc and node_javascript.cc later, and scope
  every actual access to the source of native modules to a
  std::map lookup instead of a lookup on a v8::Object in
  dictionary mode.

This patch also refactor the code cache generator and tests
a bit and trace the `withCodeCache` and `withoutCodeCache`
in a Set instead of an Array, and makes sure that all the cachable
builtins are tested.

PR-URL: nodejs#24221
Reviewed-By: Refael Ackermann <refack@gmail.com>
Reviewed-By: Anna Henningsen <anna@addaleax.net>
  • Loading branch information
@joyeecheung
joyeecheung committed Nov 14, 2018
1 parent 3856d8a commit bd765d6
Show file tree
Hide file tree
Showing 12 changed files with 680 additions and 364 deletions.
30 changes: 8 additions & 22 deletions lib/internal/bootstrap/cache.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,13 @@
const {
NativeModule
} = require('internal/bootstrap/loaders');
const {
source,
compileCodeCache
} = internalBinding('native_module');
const { hasTracing } = process.binding('config');

function getCodeCache(id) {
const cached = NativeModule.getCached(id);
if (cached && (cached.loaded || cached.loading)) {
return cached.script.createCachedData();
}

// The script has not been compiled and run
NativeModule.require(id);
return getCodeCache(id);
}

const depsModule = Object.keys(NativeModule._source).filter(
const depsModule = Object.keys(source).filter(
(key) => NativeModule.isDepsModule(key) || key.startsWith('internal/deps')
);

Expand Down Expand Up @@ -75,17 +68,10 @@ if (!process.versions.openssl) {
}

module.exports = {
cachableBuiltins: Object.keys(NativeModule._source).filter(
cachableBuiltins: Object.keys(source).filter(
(key) => !cannotUseCache.includes(key)
),
builtinSource: Object.assign({}, NativeModule._source),
getCodeCache,
getSource: NativeModule.getSource,
codeCache: internalBinding('code_cache'),
compiledWithoutCache: NativeModule.compiledWithoutCache,
compiledWithCache: NativeModule.compiledWithCache,
nativeModuleWrap(script) {
return NativeModule.wrap(script);
},
getSource(id) { return source[id]; },
getCodeCache: compileCodeCache,
cannotUseCache
};
63 changes: 3 additions & 60 deletions lib/internal/bootstrap/loaders.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,6 @@

// Create this WeakMap in js-land because V8 has no C++ API for WeakMap
internalBinding('module_wrap').callbackMap = new WeakMap();
const { ContextifyScript } = internalBinding('contextify');

// Set up NativeModule
function NativeModule(id) {
Expand All @@ -160,20 +159,13 @@
this.exportKeys = undefined;
this.loaded = false;
this.loading = false;
this.script = null; // The ContextifyScript of the module
}

NativeModule._source = getInternalBinding('natives');
NativeModule._cache = {};

const config = getBinding('config');

const codeCache = getInternalBinding('code_cache');
const codeCacheHash = getInternalBinding('code_cache_hash');
const sourceHash = getInternalBinding('natives_hash');
const compiledWithoutCache = NativeModule.compiledWithoutCache = [];
const compiledWithCache = NativeModule.compiledWithCache = [];

// Think of this as module.exports in this file even though it is not
// written in CommonJS style.
const loaderExports = { internalBinding, NativeModule };
Expand Down Expand Up @@ -332,67 +324,18 @@
this.exports = new Proxy(this.exports, handler);
};

const { compileFunction } = getInternalBinding('native_module');
NativeModule.prototype.compile = function() {
const id = this.id;
let source = NativeModule.getSource(id);
source = NativeModule.wrap(source);

this.loading = true;

try {
// Currently V8 only checks that the length of the source code is the
// same as the code used to generate the hash, so we add an additional
// check here:
// 1. During compile time, when generating node_javascript.cc and
// node_code_cache.cc, we compute and include the hash of the
// (unwrapped) JavaScript source in both.
// 2. At runtime, we check that the hash of the code being compiled
// and the hash of the code used to generate the cache
// (inside the wrapper) is the same.
// This is based on the assumptions:
// 1. `internalBinding('code_cache_hash')` must be in sync with
// `internalBinding('code_cache')` (same C++ file)
// 2. `internalBinding('natives_hash')` must be in sync with
// `internalBinding('natives')` (same C++ file)
// 3. If `internalBinding('natives_hash')` is in sync with
// `internalBinding('natives_hash')`, then the (unwrapped)
// code used to generate `internalBinding('code_cache')`
// should be in sync with the (unwrapped) code in
// `internalBinding('natives')`
// There will be, however, false positives if the wrapper used
// to generate the cache is different from the one used at run time,
// and the length of the wrapper somehow stays the same.
// But that should be rare and can be eased once we make the
// two bootstrappers cached and checked as well.
const cache = codeCacheHash[id] &&
(codeCacheHash[id] === sourceHash[id]) ? codeCache[id] : undefined;

// (code, filename, lineOffset, columnOffset
// cachedData, produceCachedData, parsingContext)
const script = new ContextifyScript(
source, this.filename, 0, 0,
cache, false, undefined
);

// This will be used to create code cache in tools/generate_code_cache.js
this.script = script;

// One of these conditions may be false when any of the inputs
// of the `node_js2c` target in node.gyp is modified.
// FIXME(joyeecheung): Figure out how to resolve the dependency issue.
// When the code cache was introduced we were at a point where refactoring
// node.gyp may not be worth the effort.
if (!cache || script.cachedDataRejected) {
compiledWithoutCache.push(this.id);
} else {
compiledWithCache.push(this.id);
}

// Arguments: timeout, displayErrors, breakOnSigint
const fn = script.runInThisContext(-1, true, false);
const requireFn = this.id.startsWith('internal/deps/') ?
NativeModule.requireForDeps :
NativeModule.require;

const fn = compileFunction(id);
fn(this.exports, requireFn, this, process, internalBinding);

if (config.experimentalModules && !NativeModule.isInternal(this.id)) {
Expand Down
2 changes: 2 additions & 0 deletions node.gyp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,8 @@
'src/node_javascript.h',
'src/node_messaging.h',
'src/node_mutex.h',
'src/node_native_module.h',
'src/node_native_module.cc',
'src/node_options.h',
'src/node_options-inl.h',
'src/node_perf.h',
Expand Down
Loading

0 comments on commit bd765d6

Please sign in to comment.