build,tools: use mkcodecache to add code cache to the binaries

Makefile

@@ -103,25 +103,17 @@ $(NODE_G_EXE): config.gypi out/Makefile
 	$(MAKE) -C out BUILDTYPE=Debug V=$(V)
 	if [ ! -r $@ -o ! -L $@ ]; then ln -fs out/Debug/$(NODE_EXE) $@; fi
 
-CODE_CACHE_DIR ?= out/$(BUILDTYPE)/obj/gen
-CODE_CACHE_FILE ?= $(CODE_CACHE_DIR)/node_code_cache.cc
-
 ifeq ($(BUILDTYPE),Debug)
 CONFIG_FLAGS += --debug
 endif
+
 .PHONY: with-code-cache
 with-code-cache:
-	@echo $(CONFIG_FLAGS)
-	$(PYTHON) ./configure $(CONFIG_FLAGS)
-	$(MAKE)
-	mkdir -p $(CODE_CACHE_DIR)
-	out/$(BUILDTYPE)/$(NODE_EXE) --expose-internals tools/generate_code_cache.js $(CODE_CACHE_FILE)
-	$(PYTHON) ./configure --code-cache-path $(CODE_CACHE_FILE) $(CONFIG_FLAGS)
-	$(MAKE)
+	echo "'with-code-cache' target is a noop"
 
 .PHONY: test-code-cache
 test-code-cache: with-code-cache
-	$(PYTHON) tools/test.py $(PARALLEL_ARGS) --mode=$(BUILDTYPE_LOWER) code-cache
+	echo "'test-code-cache' target is a noop"
 
 out/Makefile: config.gypi common.gypi node.gyp \
   deps/uv/uv.gyp deps/http_parser/http_parser.gyp deps/zlib/zlib.gyp \
@@ -1232,6 +1224,8 @@ LINT_CPP_FILES = $(filter-out $(LINT_CPP_EXCLUDE), $(wildcard \
 	test/node-api/*/*.h \
 	tools/icu/*.cc \
 	tools/icu/*.h \
+	tools/code_cache/*.cc \
+	tools/code_cache/*.h \
 	))
 
 # Code blocks don't have newline at the end,

configure.py

@@ -485,8 +485,9 @@
 parser.add_option('--code-cache-path',
     action='store',
     dest='code_cache_path',
-    help='Use a file generated by tools/generate_code_cache.js to compile the'
-         ' code cache for builtin modules into the binary')
+    help='optparse.SUPPRESS_HELP')
+
+# End dummy list.
 
 parser.add_option('--without-ssl',
     action='store_true',
@@ -1050,8 +1051,7 @@ def configure_node(o):
     o['variables']['debug_nghttp2'] = 'false'
 
   o['variables']['node_no_browser_globals'] = b(options.no_browser_globals)
-  if options.code_cache_path:
-    o['variables']['node_code_cache_path'] = options.code_cache_path
+  o['variables']['node_code_cache_path'] = 'yes'
   o['variables']['node_shared'] = b(options.shared)
   node_module_version = getmoduleversion.get_version()
 

node.gyp

@@ -6,7 +6,6 @@
     'node_use_dtrace%': 'false',
     'node_use_etw%': 'false',
     'node_no_browser_globals%': 'false',
-    'node_code_cache_path%': '',
     'node_use_v8_platform%': 'true',
     'node_use_bundled_v8%': 'true',
     'node_shared%': 'false',
@@ -224,6 +223,7 @@
       'deps/acorn/acorn/dist/acorn.js',
       'deps/acorn/acorn-walk/dist/walk.js',
     ],
+    'mkcodecache_exec': '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)mkcodecache<(EXECUTABLE_SUFFIX)',
     'conditions': [
       [ 'node_shared=="true"', {
         'node_target_type%': 'shared_library',
@@ -280,16 +280,24 @@
     {
       'target_name': '<(node_core_target_name)',
       'type': 'executable',
-      'sources': [
-        'src/node_main.cc'
+
+      'defines': [
+        'NODE_WANT_INTERNALS=1',
       ],
+
       'includes': [
         'node.gypi'
       ],
+
       'include_dirs': [
         'src',
         'deps/v8/include'
       ],
+
+      'sources': [
+        'src/node_main.cc'
+      ],
+
       'dependencies': [ 'deps/histogram/histogram.gyp:histogram' ],
 
       'msvs_settings': {
@@ -402,7 +410,32 @@
               'LinkIncremental': 2                 # enable incremental linking
             },
           },
-        }]
+         }],
+        ['want_separate_host_toolset==0', {
+          'dependencies': [
+            'mkcodecache',
+          ],
+          'actions': [
+            {
+              'action_name': 'run_mkcodecache',
+              'process_outputs_as_sources': 1,
+              'inputs': [
+                '<(mkcodecache_exec)',
+              ],
+              'outputs': [
+                '<(SHARED_INTERMEDIATE_DIR)/node_code_cache.cc',
+              ],
+              'action': [
+                '<@(_inputs)',
+                '<@(_outputs)',
+              ],
+            },
+          ],
+        }, {
+          'sources': [
+            'src/node_code_cache_stub.cc'
+          ],
+        }],
       ],
     }, # node_core_target_name
     {
@@ -615,11 +648,6 @@
       'msvs_disabled_warnings!': [4244],
 
       'conditions': [
-        [ 'node_code_cache_path!=""', {
-          'sources': [ '<(node_code_cache_path)' ]
-        }, {
-          'sources': [ 'src/node_code_cache_stub.cc' ]
-        }],
         [ 'node_shared=="true" and node_module_version!="" and OS!="win"', {
           'product_extension': '<(shlib_suffix)',
           'xcode_settings': {
@@ -1047,6 +1075,7 @@
       'defines': [ 'NODE_WANT_INTERNALS=1' ],
 
       'sources': [
+        'src/node_code_cache_stub.cc',
         'test/cctest/node_test_fixture.cc',
         'test/cctest/test_aliased_buffer.cc',
         'test/cctest/test_base64.cc',
@@ -1075,7 +1104,9 @@
             'HAVE_INSPECTOR=1',
           ],
         }, {
-          'defines': [ 'HAVE_INSPECTOR=0' ]
+           'defines': [
+             'HAVE_INSPECTOR=0',
+           ]
         }],
         ['OS=="solaris"', {
           'ldflags': [ '-I<(SHARED_INTERMEDIATE_DIR)' ]
@@ -1103,21 +1134,77 @@
         }],
       ],
     }, # cctest
-  ], # end targets
+    # TODO(joyeecheung): do not depend on node_lib,
+    # instead create a smaller static library node_lib_base that does
+    # just enough for node_native_module.cc and the cache builder to
+    # compile without compiling the generated code cache C++ file.
+    # So generate_code_cache -> mkcodecache -> node_lib_base,
+    #    node_lib -> node_lib_base & generate_code_cache
+    {
+      'target_name': 'mkcodecache',
+      'type': 'executable',
+
+      'dependencies': [
+        '<(node_lib_target_name)',
+        'deps/histogram/histogram.gyp:histogram',
+      ],
+
+      'includes': [
+        'node.gypi'
+      ],
+
+      'include_dirs': [
+        'src',
+        'tools/msvs/genfiles',
+        'deps/v8/include',
+        'deps/cares/include',
+        'deps/uv/include',
+      ],
+
+      'defines': [
+        'NODE_WANT_INTERNALS=1'
+      ],
+      'sources': [
+        'src/node_code_cache_stub.cc',
+        'tools/code_cache/mkcodecache.cc',
+        'tools/code_cache/cache_builder.cc',
+        'tools/code_cache/cache_builder.h',
+      ],
+
+      'conditions': [
+        [ 'node_report=="true"', {
+          'conditions': [
+            ['OS=="win"', {
+              'libraries': [
+                'dbghelp.lib',
+                'PsApi.lib',
+                'Ws2_32.lib',
+              ],
+              'dll_files': [
+                'dbghelp.dll',
+                'PsApi.dll',
+                'Ws2_32.dll',
+              ],
+            }],
+          ],
+        }],
+      ],
+    }, # mkcodecache
+  ],  # end targets
 
   'conditions': [
-    [ 'OS=="aix" and node_shared=="true"', {
+    ['OS=="aix" and node_shared=="true"', {
       'targets': [
         {
           'target_name': 'node_aix_shared',
           'type': 'shared_library',
           'product_name': '<(node_core_target_name)',
-          'ldflags': [ '--shared' ],
+          'ldflags': ['--shared'],
           'product_extension': '<(shlib_suffix)',
           'includes': [
             'node.gypi'
           ],
-          'dependencies': [ '<(node_lib_target_name)' ],
+          'dependencies': ['<(node_lib_target_name)'],
           'include_dirs': [
             'src',
             'deps/v8/include',

node.gypi

@@ -143,8 +143,8 @@
           'conditions': [
             ['OS!="aix" and node_shared=="false"', {
               'ldflags': [
-                '-Wl,--whole-archive,<(obj_dir)/deps/zlib/<(STATIC_LIB_PREFIX)'
-                    'zlib<(STATIC_LIB_SUFFIX)',
+                '-Wl,--whole-archive,'
+                '<(obj_dir)/deps/zlib/<(STATIC_LIB_PREFIX)zlib<(STATIC_LIB_SUFFIX)',
                 '-Wl,--no-whole-archive',
               ],
             }],
@@ -184,8 +184,8 @@
           'conditions': [
             ['OS!="aix" and node_shared=="false"', {
               'ldflags': [
-                '-Wl,--whole-archive,<(obj_dir)/deps/uv/<(STATIC_LIB_PREFIX)'
-                    'uv<(STATIC_LIB_SUFFIX)',
+                '-Wl,--whole-archive,'
+                '<(obj_dir)/deps/uv/<(STATIC_LIB_PREFIX)uv<(STATIC_LIB_SUFFIX)',
                 '-Wl,--no-whole-archive',
               ],
             }],
@@ -223,27 +223,33 @@
     [ 'OS=="aix"', {
       'defines': [
         '_LINUX_SOURCE_COMPAT',
-        '__STDC_FORMAT_MACROS'
+        '__STDC_FORMAT_MACROS',
       ],
       'conditions': [
         [ 'force_load=="true"', {
-
+          'variables': {
+            'exp_filename': '<(PRODUCT_DIR)/<(_target_name).exp',
+          },
           'actions': [
             {
               'action_name': 'expfile',
               'inputs': [
-                '<(obj_dir)'
+                '<(obj_dir)',
               ],
               'outputs': [
-                '<(PRODUCT_DIR)/node.exp'
+                '<(exp_filename)',
               ],
               'action': [
                 'sh', 'tools/create_expfile.sh',
-                      '<@(_inputs)', '<@(_outputs)'
+                '<@(_inputs)',
+                '<@(_outputs)',
               ],
             }
           ],
-          'ldflags': ['-Wl,-bE:<(PRODUCT_DIR)/node.exp', '-Wl,-brtl'],
+          'ldflags': [
+            '-Wl,-bE:<(exp_filename)',
+            '-Wl,-brtl',
+          ],
         }],
       ],
     }],

src/node_native_module_env.cc

@@ -4,7 +4,6 @@
 namespace node {
 namespace native_module {
 
-using v8::ArrayBuffer;
 using v8::Context;
 using v8::DEFAULT;
 using v8::Function;
@@ -18,11 +17,9 @@ using v8::Name;
 using v8::None;
 using v8::Object;
 using v8::PropertyCallbackInfo;
-using v8::ScriptCompiler;
 using v8::Set;
 using v8::SideEffectType;
 using v8::String;
-using v8::Uint8Array;
 using v8::Value;
 
 // TODO(joyeecheung): make these more general and put them into util.h
@@ -154,26 +151,6 @@ MaybeLocal<Function> NativeModuleEnv::LookupAndCompile(
   return maybe;
 }
 
-// This is supposed to be run only by the main thread in
-// tools/generate_code_cache.js
-void NativeModuleEnv::GetCodeCache(const FunctionCallbackInfo<Value>& args) {
-  Environment* env = Environment::GetCurrent(args);
-  Isolate* isolate = env->isolate();
-  CHECK(env->is_main_thread());
-
-  CHECK(args[0]->IsString());
-  node::Utf8Value id_v(isolate, args[0].As<String>());
-  const char* id = *id_v;
-
-  ScriptCompiler::CachedData* cached_data =
-      NativeModuleLoader::GetInstance()->GetCodeCache(id);
-  if (cached_data != nullptr) {
-    Local<ArrayBuffer> buf = ArrayBuffer::New(isolate, cached_data->length);
-    memcpy(buf->GetContents().Data(), cached_data->data, cached_data->length);
-    args.GetReturnValue().Set(Uint8Array::New(buf, 0, cached_data->length));
-  }
-}
-
 // TODO(joyeecheung): It is somewhat confusing that Class::Initialize
 // is used to initilaize to the binding, but it is the current convention.
 // Rename this across the code base to something that makes more sense.
@@ -216,7 +193,6 @@ void NativeModuleEnv::Initialize(Local<Object> target,
       .Check();
 
   env->SetMethod(target, "getCacheUsage", NativeModuleEnv::GetCacheUsage);
-  env->SetMethod(target, "getCodeCache", NativeModuleEnv::GetCodeCache);
   env->SetMethod(target, "compileFunction", NativeModuleEnv::CompileFunction);
   // internalBinding('native_module') should be frozen
   target->SetIntegrityLevel(context, IntegrityLevel::kFrozen).FromJust();

src/node_native_module_env.h

@@ -52,7 +52,6 @@ class NativeModuleEnv {
       const v8::PropertyCallbackInfo<v8::Value>& info);
   // Compile a specific native module as a function
   static void CompileFunction(const v8::FunctionCallbackInfo<v8::Value>& args);
-  static void GetCodeCache(const v8::FunctionCallbackInfo<v8::Value>& args);
 };
 
 }  // namespace native_module