src: port bootstrap/cache.js to C++

This allows us to query the categories of modules in C++ so we can implement the code cache generator in C++ that does not depend on a Node.js binary. PR-URL: #27046 Refs: #21563 Reviewed-By: Anna Henningsen <anna@addaleax.net> Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
nodejs · Apr 4, 2019 · e1d55a0 · e1d55a0
1 parent f59ec2a
commit e1d55a0
Show file tree

Hide file tree

Showing 7 changed files with 140 additions and 107 deletions.
diff --git a/benchmark/fixtures/require-cachable.js b/benchmark/fixtures/require-cachable.js
@@ -1,13 +1,10 @@
 'use strict';
 
-const list = require('internal/bootstrap/cache');
+const { internalBinding } = require('internal/test/binding');
 const {
- isMainThread
-} = require('worker_threads');
+ moduleCategories: { canBeRequired }
+} = internalBinding('native_module');
 
-for (const key of list.cachableBuiltins) {
- if (!isMainThread && key === 'trace_events') {
- continue;
- }
+for (const key of canBeRequired) {
  require(key);
 }
diff --git a/lib/internal/bootstrap/cache.js b/lib/internal/bootstrap/cache.js
diff --git a/node.gyp b/node.gyp
@@ -27,7 +27,6 @@
  'node_intermediate_lib_type%': 'static_library',
  'library_files': [
  'lib/internal/bootstrap/primordials.js',
- 'lib/internal/bootstrap/cache.js',
  'lib/internal/bootstrap/environment.js',
  'lib/internal/bootstrap/loaders.js',
  'lib/internal/bootstrap/node.js',

diff --git a/src/node_native_module.cc b/src/node_native_module.cc
@@ -36,6 +36,75 @@ using v8::String;
 using v8::Uint8Array;
 using v8::Value;
 
+void NativeModuleLoader::InitializeModuleCategories() {
+ if (module_categories_.is_initialized) {
+ DCHECK(!module_categories_.can_be_required.empty());
+ return;
+ }
+
+ std::vector<std::string> prefixes = {
+#if !HAVE_OPENSSL
+ "internal/crypto/",
+#endif // !HAVE_OPENSSL
+
+ "internal/bootstrap/",
+ "internal/per_context/",
+ "internal/deps/",
+ "internal/main/"
+ };
+
+ module_categories_.cannot_be_required = std::set<std::string> {
+#if !HAVE_INSPECTOR
+ "inspector",
+ "internal/util/inspector",
+#endif // !HAVE_INSPECTOR
+
+#if !NODE_USE_V8_PLATFORM || !defined(NODE_HAVE_I18N_SUPPORT)
+ "trace_events",
+#endif // !NODE_USE_V8_PLATFORM
+
+#if !HAVE_OPENSSL
+ "crypto",
+ "https",
+ "http2",
+ "tls",
+ "_tls_common",
+ "_tls_wrap",
+ "internal/http2/core",
+ "internal/http2/compat",
+ "internal/policy/manifest",
+ "internal/process/policy",
+ "internal/streams/lazy_transform",
+#endif // !HAVE_OPENSSL
+
+ "sys", // Deprecated.
+ "internal/test/binding",
+ "internal/v8_prof_polyfill",
+ "internal/v8_prof_processor",
+ };
+
+ for (auto const& x : source_) {
+ const std::string& id = x.first;
+ for (auto const& prefix : prefixes) {
+ if (prefix.length() > id.length()) {
+ continue;
+ }
+ if (id.find(prefix) == 0) {
+ module_categories_.cannot_be_required.emplace(id);
+ }
+ }
+ }
+
+ for (auto const& x : source_) {
+ const std::string& id = x.first;
+ if (0 == module_categories_.cannot_be_required.count(id)) {
+ module_categories_.can_be_required.emplace(id);
+ }
+ }
+
+ module_categories_.is_initialized = true;
+}
+
 // TODO(joyeecheung): make these more general and put them into util.h
 Local<Object> MapToObject(Local<Context> context,
  const NativeModuleRecordMap& in) {
@@ -63,6 +132,39 @@ bool NativeModuleLoader::Exists(const char* id) {
  return source_.find(id) != source_.end();
 }
 
+void NativeModuleLoader::GetModuleCategories(
+ Local<Name> property, const PropertyCallbackInfo<Value>& info) {
+ per_process::native_module_loader.InitializeModuleCategories();
+
+ Environment* env = Environment::GetCurrent(info);
+ Isolate* isolate = env->isolate();
+ Local<Context> context = env->context();
+ Local<Object> result = Object::New(isolate);
+
+ // Copy from the per-process categories
+ std::set<std::string> cannot_be_required =
+ per_process::native_module_loader.module_categories_.cannot_be_required;
+ std::set<std::string> can_be_required =
+ per_process::native_module_loader.module_categories_.can_be_required;
+
+ if (!env->owns_process_state()) {
+ can_be_required.erase("trace_events");
+ cannot_be_required.insert("trace_events");
+ }
+
+ result
+ ->Set(context,
+ OneByteString(isolate, "cannotBeRequired"),
+ ToJsSet(context, cannot_be_required))
+ .FromJust();
+ result
+ ->Set(context,
+ OneByteString(isolate, "canBeRequired"),
+ ToJsSet(context, can_be_required))
+ .FromJust();
+ info.GetReturnValue().Set(result);
+}
+
 void NativeModuleLoader::GetCacheUsage(
  const FunctionCallbackInfo<Value>& args) {
  Environment* env = Environment::GetCurrent(args);
@@ -298,6 +400,18 @@ void NativeModuleLoader::Initialize(Local<Object> target,
  SideEffectType::kHasNoSideEffect)
  .FromJust());
 
+ CHECK(target
+ ->SetAccessor(
+ env->context(),
+ FIXED_ONE_BYTE_STRING(env->isolate(), "moduleCategories"),
+ GetModuleCategories,
+ nullptr,
+ env->as_callback_data(),
+ DEFAULT,
+ None,
+ SideEffectType::kHasNoSideEffect)
+ .FromJust());
+
  env->SetMethod(
  target, "getCacheUsage", NativeModuleLoader::GetCacheUsage);
  env->SetMethod(

diff --git a/src/node_native_module.h b/src/node_native_module.h
@@ -54,6 +54,9 @@ class NativeModuleLoader {
  Environment* optional_env);
 
  private:
+ static void GetModuleCategories(
+ v8::Local<v8::Name> property,
+ const v8::PropertyCallbackInfo<v8::Value>& info);
  static void GetCacheUsage(const v8::FunctionCallbackInfo<v8::Value>& args);
  // Passing ids of builtin module source code into JS land as
  // internalBinding('native_module').moduleIds
@@ -84,6 +87,15 @@ class NativeModuleLoader {
  static v8::MaybeLocal<v8::Function> CompileAsModule(Environment* env,
  const char* id);
 
+ void InitializeModuleCategories();
+ struct ModuleCategories {
+ bool is_initialized = false;
+ std::set<std::string> can_be_required;
+ std::set<std::string> cannot_be_required;
+ };
+
+ ModuleCategories module_categories_;
+
  NativeModuleRecordMap source_;
  NativeModuleCacheMap code_cache_;
  UnionBytes config_;

diff --git a/test/code-cache/test-code-cache.js b/test/code-cache/test-code-cache.js
@@ -7,22 +7,15 @@
 
 const { isMainThread } = require('../common');
 const assert = require('assert');
-const {
- cachableBuiltins,
- cannotBeRequired
-} = require('internal/bootstrap/cache');
-
 const {
  internalBinding
 } = require('internal/test/binding');
 const {
- getCacheUsage
+ getCacheUsage,
+ moduleCategories: { canBeRequired, cannotBeRequired }
 } = internalBinding('native_module');
 
-for (const key of cachableBuiltins) {
- if (!isMainThread && key === 'trace_events') {
- continue; // Cannot load trace_events in workers
- }
+for (const key of canBeRequired) {
  require(key);
 }
 
@@ -60,7 +53,7 @@ if (process.config.variables.node_code_cache_path === undefined) {
  );
 
  for (const key of loadedModules) {
- if (cannotBeRequired.includes(key)) {
+ if (cannotBeRequired.has(key)) {
  assert(compiledWithoutCache.has(key),
  `"${key}" should've been compiled without code cache`);
  } else {

diff --git a/tools/generate_code_cache.js b/tools/generate_code_cache.js
@@ -7,11 +7,12 @@
 // compiled into the binary using the `--code-cache-path` option
 // of `configure`.
 
+const { internalBinding } = require('internal/test/binding');
 const {
+ moduleCategories: { canBeRequired },
  getCodeCache,
  compileFunction,
- cachableBuiltins
-} = require('internal/bootstrap/cache');
+} = internalBinding('native_module');
 
 const {
  types: {
@@ -85,7 +86,9 @@ function lexical(a, b) {
  return 0;
 }
 
-for (const key of cachableBuiltins.sort(lexical)) {
+// TODO(joyeecheung): support non-modules that require different
+// parameters in the wrapper.
+for (const key of [...canBeRequired].sort(lexical)) {
  compileFunction(key); // compile it
  const cachedData = getCodeCache(key);
  if (!isUint8Array(cachedData)) {