From 42ab2d3c6116736ee5c2f44103fe8ff877a3cb52 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Mon, 3 Dec 2018 00:49:12 +0800 Subject: [PATCH] process: specialize building and storage of process.config Instead of treating config.gypi as a JavaScript file, specialize the processing in js2c and make the serialized result a real JSON string (with 'true' and 'false' converted to boolean values) so we don't have to use a custom deserializer during bootstrap. In addition, store the JSON string separately in NativeModuleLoader, and keep it separate from the map of the builtin source code, so we don't have to put it onto `NativeModule._source` and delete it later, though we still preserve it in `process.binding('natives')`, which we don't use anymore. This patch also makes the map of builtin source code and the config.gypi string available through side-effect-free getters in C++. --- lib/internal/bootstrap/cache.js | 4 +-- lib/internal/bootstrap/loaders.js | 10 ++++-- lib/internal/bootstrap/node.js | 2 +- lib/internal/process/per_thread.js | 14 ++------ src/env.h | 3 +- src/node_binding.cc | 7 ++++ src/node_native_module.cc | 48 +++++++++++++++++++++---- src/node_native_module.h | 18 ++++++++-- src/node_union_bytes.h | 10 ++++++ test/parallel/test-bootstrap-modules.js | 2 +- tools/js2c.py | 38 +++++++++++++------- 11 files changed, 114 insertions(+), 42 deletions(-) diff --git a/lib/internal/bootstrap/cache.js b/lib/internal/bootstrap/cache.js index 97a587b8a02b03..bc0a01cf6770c5 100644 --- a/lib/internal/bootstrap/cache.js +++ b/lib/internal/bootstrap/cache.js @@ -6,10 +6,9 @@ // cannot be tampered with even with --expose-internals. const { NativeModule } = require('internal/bootstrap/loaders'); -const { getSource, compileCodeCache } = internalBinding('native_module'); +const { source, compileCodeCache } = internalBinding('native_module'); const { hasTracing } = process.binding('config'); -const source = getSource(); const depsModule = Object.keys(source).filter( (key) => NativeModule.isDepsModule(key) || key.startsWith('internal/deps') ); @@ -17,7 +16,6 @@ const depsModule = Object.keys(source).filter( // Modules with source code compiled in js2c that // cannot be compiled with the code cache. const cannotUseCache = [ - 'config', 'sys', // Deprecated. 'internal/v8_prof_polyfill', 'internal/v8_prof_processor', diff --git a/lib/internal/bootstrap/loaders.js b/lib/internal/bootstrap/loaders.js index dfe904857fa3e5..c6764cfe5a5041 100644 --- a/lib/internal/bootstrap/loaders.js +++ b/lib/internal/bootstrap/loaders.js @@ -167,10 +167,15 @@ function NativeModule(id) { this.loading = false; } -NativeModule._source = getInternalBinding('natives'); +const { + source, + compileFunction +} = internalBinding('native_module'); + +NativeModule._source = source; NativeModule._cache = {}; -const config = getInternalBinding('config'); +const config = internalBinding('config'); // Think of this as module.exports in this file even though it is not // written in CommonJS style. @@ -330,7 +335,6 @@ NativeModule.prototype.proxifyExports = function() { this.exports = new Proxy(this.exports, handler); }; -const { compileFunction } = getInternalBinding('native_module'); NativeModule.prototype.compile = function() { const id = this.id; diff --git a/lib/internal/bootstrap/node.js b/lib/internal/bootstrap/node.js index 51e27afecaea3c..816957ae59b022 100644 --- a/lib/internal/bootstrap/node.js +++ b/lib/internal/bootstrap/node.js @@ -59,7 +59,7 @@ function startup() { } perThreadSetup.setupAssert(); - perThreadSetup.setupConfig(NativeModule._source); + perThreadSetup.setupConfig(); if (isMainThread) { mainThreadSetup.setupSignalHandlers(internalBinding); diff --git a/lib/internal/process/per_thread.js b/lib/internal/process/per_thread.js index 952abda8b67147..9db039af5ab1eb 100644 --- a/lib/internal/process/per_thread.js +++ b/lib/internal/process/per_thread.js @@ -143,17 +143,9 @@ function setupMemoryUsage(_memoryUsage) { }; } -function setupConfig(_source) { - // NativeModule._source - // used for `process.config`, but not a real module - const config = _source.config; - delete _source.config; - - process.config = JSON.parse(config, function(key, value) { - if (value === 'true') return true; - if (value === 'false') return false; - return value; - }); +function setupConfig() { + // Serialized config.gypi + process.config = JSON.parse(internalBinding('native_module').config); } diff --git a/src/env.h b/src/env.h index 64593a0025c3b6..b94d31386ddbb1 100644 --- a/src/env.h +++ b/src/env.h @@ -138,6 +138,7 @@ constexpr size_t kFsStatsBufferLength = kFsStatsFieldsNumber * 2; V(channel_string, "channel") \ V(chunks_sent_since_last_write_string, "chunksSentSinceLastWrite") \ V(code_string, "code") \ + V(config_string, "config") \ V(constants_string, "constants") \ V(cwd_string, "cwd") \ V(dest_string, "dest") \ @@ -307,7 +308,7 @@ constexpr size_t kFsStatsBufferLength = kFsStatsFieldsNumber * 2; V(write_host_object_string, "_writeHostObject") \ V(write_queue_size_string, "writeQueueSize") \ V(x_forwarded_string, "x-forwarded-for") \ - V(zero_return_string, "ZERO_RETURN") \ + V(zero_return_string, "ZERO_RETURN") #define ENVIRONMENT_STRONG_PERSISTENT_PROPERTIES(V) \ V(as_external, v8::External) \ diff --git a/src/node_binding.cc b/src/node_binding.cc index 6d70d0a501e335..bdfb74e2744845 100644 --- a/src/node_binding.cc +++ b/src/node_binding.cc @@ -412,6 +412,13 @@ void GetInternalBinding(const FunctionCallbackInfo& args) { DefineConstants(env->isolate(), exports); } else if (!strcmp(*module_v, "natives")) { exports = per_process_loader.GetSourceObject(env->context()); + // Legacy feature: process.binding('natives').config contains stringified + // config.gypi + CHECK(exports + ->Set(env->context(), + env->config_string(), + per_process_loader.GetConfigString(env->isolate())) + .FromJust()); } else { return ThrowIfNoSuchModule(env, *module_v); } diff --git a/src/node_native_module.cc b/src/node_native_module.cc index 6e5f08e35451f0..85f8d83d63a162 100644 --- a/src/node_native_module.cc +++ b/src/node_native_module.cc @@ -9,6 +9,7 @@ using v8::Array; using v8::ArrayBuffer; using v8::ArrayBufferCreationMode; using v8::Context; +using v8::DEFAULT; using v8::EscapableHandleScope; using v8::Function; using v8::FunctionCallbackInfo; @@ -19,11 +20,15 @@ using v8::Isolate; using v8::Local; using v8::Maybe; using v8::MaybeLocal; +using v8::Name; +using v8::None; using v8::Object; +using v8::PropertyCallbackInfo; using v8::Script; using v8::ScriptCompiler; using v8::ScriptOrigin; using v8::Set; +using v8::SideEffectType; using v8::String; using v8::Uint8Array; using v8::Value; @@ -70,10 +75,16 @@ void NativeModuleLoader::GetCacheUsage( args.GetReturnValue().Set(result); } -void NativeModuleLoader::GetSourceObject( - const FunctionCallbackInfo& args) { - Environment* env = Environment::GetCurrent(args); - args.GetReturnValue().Set(per_process_loader.GetSourceObject(env->context())); +void NativeModuleLoader::SourceObjectGetter( + Local property, const PropertyCallbackInfo& info) { + Local context = info.GetIsolate()->GetCurrentContext(); + info.GetReturnValue().Set(per_process_loader.GetSourceObject(context)); +} + +void NativeModuleLoader::ConfigStringGetter( + Local property, const PropertyCallbackInfo& info) { + info.GetReturnValue().Set( + per_process_loader.GetConfigString(info.GetIsolate())); } Local NativeModuleLoader::GetSourceObject( @@ -81,6 +92,10 @@ Local NativeModuleLoader::GetSourceObject( return MapToObject(context, source_); } +Local NativeModuleLoader::GetConfigString(Isolate* isolate) const { + return config_.ToStringChecked(isolate); +} + Local NativeModuleLoader::GetSource(Isolate* isolate, const char* id) const { const auto it = source_.find(id); @@ -88,7 +103,7 @@ Local NativeModuleLoader::GetSource(Isolate* isolate, return it->second.ToStringChecked(isolate); } -NativeModuleLoader::NativeModuleLoader() { +NativeModuleLoader::NativeModuleLoader() : config_(GetConfig()) { LoadJavaScriptSource(); LoadJavaScriptHash(); LoadCodeCache(); @@ -321,8 +336,27 @@ void NativeModuleLoader::Initialize(Local target, void* priv) { Environment* env = Environment::GetCurrent(context); - env->SetMethod( - target, "getSource", NativeModuleLoader::GetSourceObject); + CHECK(target + ->SetAccessor(env->context(), + env->config_string(), + ConfigStringGetter, + nullptr, + MaybeLocal(), + DEFAULT, + None, + SideEffectType::kHasNoSideEffect) + .FromJust()); + CHECK(target + ->SetAccessor(env->context(), + env->source_string(), + SourceObjectGetter, + nullptr, + MaybeLocal(), + DEFAULT, + None, + SideEffectType::kHasNoSideEffect) + .FromJust()); + env->SetMethod( target, "getCacheUsage", NativeModuleLoader::GetCacheUsage); env->SetMethod( diff --git a/src/node_native_module.h b/src/node_native_module.h index fc0e33c73500d1..02c769ef023907 100644 --- a/src/node_native_module.h +++ b/src/node_native_module.h @@ -40,6 +40,9 @@ class NativeModuleLoader { v8::Local context, void* priv); v8::Local GetSourceObject(v8::Local context) const; + // Returns config.gypi as a JSON string + v8::Local GetConfigString(v8::Isolate* isolate) const; + v8::Local GetSource(v8::Isolate* isolate, const char* id) const; // Run a script with JS source bundled inside the binary as if it's wrapped @@ -56,9 +59,15 @@ class NativeModuleLoader { private: static void GetCacheUsage(const v8::FunctionCallbackInfo& args); - // For legacy process.binding('natives') which is mutable, and for - // internalBinding('native_module').source for internal use - static void GetSourceObject(const v8::FunctionCallbackInfo& args); + // Passing map of builtin module source code into JS land as + // internalBinding('native_module').source + static void SourceObjectGetter( + v8::Local property, + const v8::PropertyCallbackInfo& info); + // Passing config.gypi into JS land as internalBinding('native_module').config + static void ConfigStringGetter( + v8::Local property, + const v8::PropertyCallbackInfo& info); // Compile code cache for a specific native module static void CompileCodeCache(const v8::FunctionCallbackInfo& args); // Compile a specific native module as a function @@ -67,6 +76,7 @@ class NativeModuleLoader { // Generated by tools/js2c.py as node_javascript.cc void LoadJavaScriptSource(); // Loads data into source_ void LoadJavaScriptHash(); // Loads data into source_hash_ + UnionBytes GetConfig(); // Return data for config.gypi // Generated by tools/generate_code_cache.js as node_code_cache.cc when // the build is configured with --code-cache-path=.... They are noops @@ -94,6 +104,8 @@ class NativeModuleLoader { bool has_code_cache_ = false; NativeModuleRecordMap source_; NativeModuleRecordMap code_cache_; + UnionBytes config_; + NativeModuleHashMap source_hash_; NativeModuleHashMap code_cache_hash_; }; diff --git a/src/node_union_bytes.h b/src/node_union_bytes.h index 128bb11ed0d0ab..66d8509beaf188 100644 --- a/src/node_union_bytes.h +++ b/src/node_union_bytes.h @@ -55,21 +55,31 @@ class UnionBytes { : is_one_byte_(false), two_bytes_(data), length_(length) {} UnionBytes(const uint8_t* data, size_t length) : is_one_byte_(true), one_bytes_(data), length_(length) {} + + UnionBytes(const UnionBytes&) = default; + UnionBytes& operator=(const UnionBytes&) = default; + UnionBytes(UnionBytes&&) = default; + UnionBytes& operator=(UnionBytes&&) = default; + bool is_one_byte() const { return is_one_byte_; } const uint16_t* two_bytes_data() const { CHECK(!is_one_byte_); + CHECK_NE(two_bytes_, nullptr); return two_bytes_; } const uint8_t* one_bytes_data() const { CHECK(is_one_byte_); + CHECK_NE(one_bytes_, nullptr); return one_bytes_; } v8::Local ToStringChecked(v8::Isolate* isolate) const { if (is_one_byte_) { + CHECK_NE(one_bytes_, nullptr); NonOwningExternalOneByteResource* source = new NonOwningExternalOneByteResource(one_bytes_, length_); return v8::String::NewExternalOneByte(isolate, source).ToLocalChecked(); } else { + CHECK_NE(two_bytes_, nullptr); NonOwningExternalTwoByteResource* source = new NonOwningExternalTwoByteResource(two_bytes_, length_); return v8::String::NewExternalTwoByte(isolate, source).ToLocalChecked(); diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js index 8f83194cbb1a52..5c1693ca89c5db 100644 --- a/test/parallel/test-bootstrap-modules.js +++ b/test/parallel/test-bootstrap-modules.js @@ -9,7 +9,7 @@ const common = require('../common'); const assert = require('assert'); const isMainThread = common.isMainThread; -const kMaxModuleCount = isMainThread ? 58 : 80; +const kMaxModuleCount = isMainThread ? 60 : 82; assert(list.length <= kMaxModuleCount, `Total length: ${list.length}\n` + list.join('\n') diff --git a/tools/js2c.py b/tools/js2c.py index e01e385afcf75c..ab27c9e894db24 100755 --- a/tools/js2c.py +++ b/tools/js2c.py @@ -188,6 +188,10 @@ def ReadMacros(lines): {hash_initializers} }} +UnionBytes NativeModuleLoader::GetConfig() {{ + return UnionBytes(config_raw, arraysize(config_raw)); // config.gypi +}} + }} // namespace native_module }} // namespace node @@ -243,21 +247,23 @@ def JS2C(source, target): definitions = [] initializers = [] hash_initializers = [] + config_initializers = [] - def AddModule(module, source): - var = '%s_raw' % (module.replace('-', '_').replace('/', '_')) - source_hash = hashlib.sha256(source).hexdigest() - + def GetDefinition(var, source): # Treat non-ASCII as UTF-8 and convert it to UTF-16. if any(ord(c) > 127 for c in source): source = map(ord, source.decode('utf-8').encode('utf-16be')) source = [source[i] * 256 + source[i+1] for i in xrange(0, len(source), 2)] source = ToCArray(source) - definition = TWO_BYTE_STRING.format(var=var, data=source) + return TWO_BYTE_STRING.format(var=var, data=source) else: source = ToCArray(map(ord, source), step=20) - definition = ONE_BYTE_STRING.format(var=var, data=source) + return ONE_BYTE_STRING.format(var=var, data=source) + def AddModule(module, source): + var = '%s_raw' % (module.replace('-', '_').replace('/', '_')) + source_hash = hashlib.sha256(source).hexdigest() + definition = GetDefinition(var, source) initializer = INITIALIZER.format(module=module, var=var) hash_initializer = HASH_INITIALIZER.format(module=module, @@ -287,11 +293,17 @@ def AddModule(module, source): # if its a gypi file we're going to want it as json # later on anyway, so get it out of the way now - if name.endswith(".gypi"): + if name.endswith('.gypi'): + # Currently only config.gypi is allowed + assert name == 'config.gypi' + lines = re.sub(r'\'true\'', 'true', lines) + lines = re.sub(r'\'false\'', 'false', lines) lines = re.sub(r'#.*?\n', '', lines) lines = re.sub(r'\'', '"', lines) - - AddModule(name.split('.', 1)[0], lines) + definition = GetDefinition('config_raw', lines) + definitions.append(definition) + else: + AddModule(name.split('.', 1)[0], lines) # Add deprecated aliases for deps without 'deps/' if deprecated_deps is not None: @@ -301,9 +313,11 @@ def AddModule(module, source): # Emit result output = open(str(target[0]), "w") - output.write(TEMPLATE.format(definitions=''.join(definitions), - initializers=''.join(initializers), - hash_initializers=''.join(hash_initializers))) + output.write( + TEMPLATE.format(definitions=''.join(definitions), + initializers=''.join(initializers), + hash_initializers=''.join(hash_initializers), + config_initializers=''.join(config_initializers))) output.close() def main():