process: specialize building and storage of process.config

joyeecheung · addaleax · commit 0e2fbe4ff4c1 · 2019-01-14T15:35:04.000+01:00
Instead of treating config.gypi as a JavaScript file, specialize the processing in js2c and make the serialized result a real JSON string (with 'true' and 'false' converted to boolean values) so we don't have to use a custom deserializer during bootstrap. In addition, store the JSON string separately in NativeModuleLoader, and keep it separate from the map of the builtin source code, so we don't have to put it onto `NativeModule._source` and delete it later, though we still preserve it in `process.binding('natives')`, which we don't use anymore. This patch also makes the map of builtin source code and the config.gypi string available through side-effect-free getters in C++. PR-URL: #24816 Reviewed-By: Gus Caplan <me@gus.host>
diff --git a/lib/internal/bootstrap/cache.js b/lib/internal/bootstrap/cache.js
@@ -6,18 +6,16 @@
 // cannot be tampered with even with --expose-internals.
 
 const { NativeModule } = require('internal/bootstrap/loaders');
-const { getSource, compileCodeCache } = internalBinding('native_module');
+const { source, compileCodeCache } = internalBinding('native_module');
 const { hasTracing } = process.binding('config');
 
-const source = getSource();
 const depsModule = Object.keys(source).filter(
   (key) => NativeModule.isDepsModule(key) || key.startsWith('internal/deps')
 );
 
 // Modules with source code compiled in js2c that
 // cannot be compiled with the code cache.
 const cannotUseCache = [
-  'config',
   'sys',  // Deprecated.
   'internal/v8_prof_polyfill',
   'internal/v8_prof_processor',
diff --git a/lib/internal/bootstrap/loaders.js b/lib/internal/bootstrap/loaders.js
@@ -169,10 +169,15 @@ function NativeModule(id) {
   this.loading = false;
 }
 
-NativeModule._source = getInternalBinding('natives');
+const {
+  source,
+  compileFunction
+} = internalBinding('native_module');
+
+NativeModule._source = source;
 NativeModule._cache = {};
 
-const config = getInternalBinding('config');
+const config = internalBinding('config');
 
 // Think of this as module.exports in this file even though it is not
 // written in CommonJS style.
@@ -323,7 +328,6 @@ NativeModule.prototype.proxifyExports = function() {
   this.exports = new Proxy(this.exports, handler);
 };
 
-const { compileFunction } = getInternalBinding('native_module');
 NativeModule.prototype.compile = function() {
   const id = this.id;
 
diff --git a/lib/internal/bootstrap/node.js b/lib/internal/bootstrap/node.js
@@ -59,7 +59,7 @@ function startup() {
   }
 
   perThreadSetup.setupAssert();
-  perThreadSetup.setupConfig(NativeModule._source);
+  perThreadSetup.setupConfig();
 
   if (isMainThread) {
     mainThreadSetup.setupSignalHandlers(internalBinding);
diff --git a/lib/internal/process/per_thread.js b/lib/internal/process/per_thread.js
@@ -143,17 +143,9 @@ function setupMemoryUsage(_memoryUsage) {
   };
 }
 
-function setupConfig(_source) {
-  // NativeModule._source
-  // used for `process.config`, but not a real module
-  const config = _source.config;
-  delete _source.config;
-
-  process.config = JSON.parse(config, function(key, value) {
-    if (value === 'true') return true;
-    if (value === 'false') return false;
-    return value;
-  });
+function setupConfig() {
+  // Serialized config.gypi
+  process.config = JSON.parse(internalBinding('native_module').config);
 }
 
 
diff --git a/src/env.h b/src/env.h
@@ -140,6 +140,7 @@ constexpr size_t kFsStatsBufferLength = kFsStatsFieldsNumber * 2;
   V(channel_string, "channel")                                                 \
   V(chunks_sent_since_last_write_string, "chunksSentSinceLastWrite")           \
   V(code_string, "code")                                                       \
+  V(config_string, "config")                                                   \
   V(constants_string, "constants")                                             \
   V(crypto_dsa_string, "dsa")                                                  \
   V(crypto_ec_string, "ec")                                                    \
@@ -312,7 +313,7 @@ constexpr size_t kFsStatsBufferLength = kFsStatsFieldsNumber * 2;
   V(write_host_object_string, "_writeHostObject")                              \
   V(write_queue_size_string, "writeQueueSize")                                 \
   V(x_forwarded_string, "x-forwarded-for")                                     \
-  V(zero_return_string, "ZERO_RETURN")                                         \
+  V(zero_return_string, "ZERO_RETURN")
 
 #define ENVIRONMENT_STRONG_PERSISTENT_PROPERTIES(V)                            \
   V(as_external, v8::External)                                                 \
diff --git a/src/node_binding.cc b/src/node_binding.cc
@@ -390,6 +390,13 @@ void GetInternalBinding(const FunctionCallbackInfo<Value>& args) {
     DefineConstants(env->isolate(), exports);
   } else if (!strcmp(*module_v, "natives")) {
     exports = per_process_loader.GetSourceObject(env->context());
+    // Legacy feature: process.binding('natives').config contains stringified
+    // config.gypi
+    CHECK(exports
+              ->Set(env->context(),
+                    env->config_string(),
+                    per_process_loader.GetConfigString(env->isolate()))
+              .FromJust());
   } else {
     return ThrowIfNoSuchModule(env, *module_v);
   }
diff --git a/src/node_native_module.cc b/src/node_native_module.cc
@@ -9,6 +9,7 @@ using v8::Array;
 using v8::ArrayBuffer;
 using v8::ArrayBufferCreationMode;
 using v8::Context;
+using v8::DEFAULT;
 using v8::EscapableHandleScope;
 using v8::Function;
 using v8::FunctionCallbackInfo;
@@ -19,11 +20,15 @@ using v8::Isolate;
 using v8::Local;
 using v8::Maybe;
 using v8::MaybeLocal;
+using v8::Name;
+using v8::None;
 using v8::Object;
+using v8::PropertyCallbackInfo;
 using v8::Script;
 using v8::ScriptCompiler;
 using v8::ScriptOrigin;
 using v8::Set;
+using v8::SideEffectType;
 using v8::String;
 using v8::Uint8Array;
 using v8::Value;
@@ -70,25 +75,35 @@ void NativeModuleLoader::GetCacheUsage(
   args.GetReturnValue().Set(result);
 }
 
-void NativeModuleLoader::GetSourceObject(
-    const FunctionCallbackInfo<Value>& args) {
-  Environment* env = Environment::GetCurrent(args);
-  args.GetReturnValue().Set(per_process_loader.GetSourceObject(env->context()));
+void NativeModuleLoader::SourceObjectGetter(
+    Local<Name> property, const PropertyCallbackInfo<Value>& info) {
+  Local<Context> context = info.GetIsolate()->GetCurrentContext();
+  info.GetReturnValue().Set(per_process_loader.GetSourceObject(context));
+}
+
+void NativeModuleLoader::ConfigStringGetter(
+    Local<Name> property, const PropertyCallbackInfo<Value>& info) {
+  info.GetReturnValue().Set(
+      per_process_loader.GetConfigString(info.GetIsolate()));
 }
 
 Local<Object> NativeModuleLoader::GetSourceObject(
     Local<Context> context) const {
   return MapToObject(context, source_);
 }
 
+Local<String> NativeModuleLoader::GetConfigString(Isolate* isolate) const {
+  return config_.ToStringChecked(isolate);
+}
+
 Local<String> NativeModuleLoader::GetSource(Isolate* isolate,
                                             const char* id) const {
   const auto it = source_.find(id);
   CHECK_NE(it, source_.end());
   return it->second.ToStringChecked(isolate);
 }
 
-NativeModuleLoader::NativeModuleLoader() {
+NativeModuleLoader::NativeModuleLoader() : config_(GetConfig()) {
   LoadJavaScriptSource();
   LoadJavaScriptHash();
   LoadCodeCache();
@@ -321,8 +336,27 @@ void NativeModuleLoader::Initialize(Local<Object> target,
                                     void* priv) {
   Environment* env = Environment::GetCurrent(context);
 
-  env->SetMethod(
-      target, "getSource", NativeModuleLoader::GetSourceObject);
+  CHECK(target
+            ->SetAccessor(env->context(),
+                          env->config_string(),
+                          ConfigStringGetter,
+                          nullptr,
+                          MaybeLocal<Value>(),
+                          DEFAULT,
+                          None,
+                          SideEffectType::kHasNoSideEffect)
+            .FromJust());
+  CHECK(target
+            ->SetAccessor(env->context(),
+                          env->source_string(),
+                          SourceObjectGetter,
+                          nullptr,
+                          MaybeLocal<Value>(),
+                          DEFAULT,
+                          None,
+                          SideEffectType::kHasNoSideEffect)
+            .FromJust());
+
   env->SetMethod(
       target, "getCacheUsage", NativeModuleLoader::GetCacheUsage);
   env->SetMethod(
diff --git a/src/node_native_module.h b/src/node_native_module.h
@@ -40,6 +40,9 @@ class NativeModuleLoader {
                          v8::Local<v8::Context> context,
                          void* priv);
   v8::Local<v8::Object> GetSourceObject(v8::Local<v8::Context> context) const;
+  // Returns config.gypi as a JSON string
+  v8::Local<v8::String> GetConfigString(v8::Isolate* isolate) const;
+
   v8::Local<v8::String> GetSource(v8::Isolate* isolate, const char* id) const;
 
   // Run a script with JS source bundled inside the binary as if it's wrapped
@@ -56,9 +59,15 @@ class NativeModuleLoader {
 
  private:
   static void GetCacheUsage(const v8::FunctionCallbackInfo<v8::Value>& args);
-  // For legacy process.binding('natives') which is mutable, and for
-  // internalBinding('native_module').source for internal use
-  static void GetSourceObject(const v8::FunctionCallbackInfo<v8::Value>& args);
+  // Passing map of builtin module source code into JS land as
+  // internalBinding('native_module').source
+  static void SourceObjectGetter(
+      v8::Local<v8::Name> property,
+      const v8::PropertyCallbackInfo<v8::Value>& info);
+  // Passing config.gypi into JS land as internalBinding('native_module').config
+  static void ConfigStringGetter(
+      v8::Local<v8::Name> property,
+      const v8::PropertyCallbackInfo<v8::Value>& info);
   // Compile code cache for a specific native module
   static void CompileCodeCache(const v8::FunctionCallbackInfo<v8::Value>& args);
   // Compile a specific native module as a function
@@ -67,6 +76,7 @@ class NativeModuleLoader {
   // Generated by tools/js2c.py as node_javascript.cc
   void LoadJavaScriptSource();  // Loads data into source_
   void LoadJavaScriptHash();    // Loads data into source_hash_
+  UnionBytes GetConfig();       // Return data for config.gypi
 
   // Generated by tools/generate_code_cache.js as node_code_cache.cc when
   // the build is configured with --code-cache-path=.... They are noops
@@ -94,6 +104,8 @@ class NativeModuleLoader {
   bool has_code_cache_ = false;
   NativeModuleRecordMap source_;
   NativeModuleRecordMap code_cache_;
+  UnionBytes config_;
+
   NativeModuleHashMap source_hash_;
   NativeModuleHashMap code_cache_hash_;
 };
diff --git a/src/node_union_bytes.h b/src/node_union_bytes.h
@@ -55,21 +55,31 @@ class UnionBytes {
       : is_one_byte_(false), two_bytes_(data), length_(length) {}
   UnionBytes(const uint8_t* data, size_t length)
       : is_one_byte_(true), one_bytes_(data), length_(length) {}
+
+  UnionBytes(const UnionBytes&) = default;
+  UnionBytes& operator=(const UnionBytes&) = default;
+  UnionBytes(UnionBytes&&) = default;
+  UnionBytes& operator=(UnionBytes&&) = default;
+
   bool is_one_byte() const { return is_one_byte_; }
   const uint16_t* two_bytes_data() const {
     CHECK(!is_one_byte_);
+    CHECK_NE(two_bytes_, nullptr);
     return two_bytes_;
   }
   const uint8_t* one_bytes_data() const {
     CHECK(is_one_byte_);
+    CHECK_NE(one_bytes_, nullptr);
     return one_bytes_;
   }
   v8::Local<v8::String> ToStringChecked(v8::Isolate* isolate) const {
     if (is_one_byte_) {
+      CHECK_NE(one_bytes_, nullptr);
       NonOwningExternalOneByteResource* source =
           new NonOwningExternalOneByteResource(one_bytes_, length_);
       return v8::String::NewExternalOneByte(isolate, source).ToLocalChecked();
     } else {
+      CHECK_NE(two_bytes_, nullptr);
       NonOwningExternalTwoByteResource* source =
           new NonOwningExternalTwoByteResource(two_bytes_, length_);
       return v8::String::NewExternalTwoByte(isolate, source).ToLocalChecked();
diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js
@@ -9,7 +9,7 @@ const common = require('../common');
 const assert = require('assert');
 
 const isMainThread = common.isMainThread;
-const kMaxModuleCount = isMainThread ? 59 : 80;
+const kMaxModuleCount = isMainThread ? 61 : 82;
 
 assert(list.length <= kMaxModuleCount,
        `Total length: ${list.length}\n` + list.join('\n')
diff --git a/tools/js2c.py b/tools/js2c.py
@@ -193,6 +193,10 @@ def ReadMacros(lines):
   {hash_initializers}
 }}
 
+UnionBytes NativeModuleLoader::GetConfig() {{
+  return UnionBytes(config_raw, arraysize(config_raw));  // config.gypi
+}}
+
 }}  // namespace native_module
 
 }}  // namespace node
@@ -248,21 +252,23 @@ def JS2C(source, target):
   definitions = []
   initializers = []
   hash_initializers = []
+  config_initializers = []
 
-  def AddModule(module, source):
-    var = '%s_raw' % (module.replace('-', '_').replace('/', '_'))
-    source_hash = hashlib.sha256(source).hexdigest()
-
+  def GetDefinition(var, source):
     # Treat non-ASCII as UTF-8 and convert it to UTF-16.
     if any(ord(c) > 127 for c in source):
       source = map(ord, source.decode('utf-8').encode('utf-16be'))
       source = [source[i] * 256 + source[i+1] for i in xrange(0, len(source), 2)]
       source = ToCArray(source)
-      definition = TWO_BYTE_STRING.format(var=var, data=source)
+      return TWO_BYTE_STRING.format(var=var, data=source)
     else:
       source = ToCArray(map(ord, source), step=20)
-      definition = ONE_BYTE_STRING.format(var=var, data=source)
+      return ONE_BYTE_STRING.format(var=var, data=source)
 
+  def AddModule(module, source):
+    var = '%s_raw' % (module.replace('-', '_').replace('/', '_'))
+    source_hash = hashlib.sha256(source).hexdigest()
+    definition = GetDefinition(var, source)
     initializer = INITIALIZER.format(module=module,
                                      var=var)
     hash_initializer = HASH_INITIALIZER.format(module=module,
@@ -292,11 +298,17 @@ def AddModule(module, source):
 
     # if its a gypi file we're going to want it as json
     # later on anyway, so get it out of the way now
-    if name.endswith(".gypi"):
+    if name.endswith('.gypi'):
+      # Currently only config.gypi is allowed
+      assert name == 'config.gypi'
+      lines = re.sub(r'\'true\'', 'true', lines)
+      lines = re.sub(r'\'false\'', 'false', lines)
       lines = re.sub(r'#.*?\n', '', lines)
       lines = re.sub(r'\'', '"', lines)
-
-    AddModule(name.split('.', 1)[0], lines)
+      definition = GetDefinition('config_raw', lines)
+      definitions.append(definition)
+    else:
+      AddModule(name.split('.', 1)[0], lines)
 
     # Add deprecated aliases for deps without 'deps/'
     if deprecated_deps is not None:
@@ -306,9 +318,11 @@ def AddModule(module, source):
 
   # Emit result
   output = open(str(target[0]), "w")
-  output.write(TEMPLATE.format(definitions=''.join(definitions),
-                               initializers=''.join(initializers),
-                               hash_initializers=''.join(hash_initializers)))
+  output.write(
+    TEMPLATE.format(definitions=''.join(definitions),
+                    initializers=''.join(initializers),
+                    hash_initializers=''.join(hash_initializers),
+                    config_initializers=''.join(config_initializers)))
   output.close()
 
 def main():

Original file line number	Diff line number	Diff line change
`@@ -59,7 +59,7 @@ function startup() {`
`59`	`59`	`}`
`60`	`60`
`61`	`61`	`perThreadSetup.setupAssert();`
`62`		`- perThreadSetup.setupConfig(NativeModule._source);`
	`62`	`+ perThreadSetup.setupConfig();`
`63`	`63`
`64`	`64`	`if (isMainThread) {`
`65`	`65`	`mainThreadSetup.setupSignalHandlers(internalBinding);`