refactor: linker uses rlocation

internal/linker/link_node_modules.js

@@ -6,78 +6,151 @@
 const fs = require('fs');
 const path = require('path');
 
-function debug(...m) {
-  if (!!process.env['VERBOSE_LOGS']) console.error('[link_node_modules.js]', ...m);
+const VERBOSE_LOGS = !!process.env['VERBOSE_LOGS'];
+
+function log_verbose(...m) {
+  // This is a template file so we use __filename to output the actual filename
+  if (VERBOSE_LOGS) console.error('[link_node_modules.js]', ...m);
+}
+
+function symlink(target, path) {
+  if (fs.existsSync(path)) {
+    // We assume here that the path is already linked to the correct target.
+    // Could add some logic that asserts it here, but we want to avoid an extra
+    // filesystem access so we should only do it under some kind of strict mode.
+    return;
+  }
+  log_verbose(`symlink( ${path} -> ${target} )`);
+  // Use junction on Windows since symlinks require elevated permissions
+  // we only link to directories so junctions work for us.
+  fs.symlinkSync(target, path, 'junction');
 }
 
-function ls_r(d) {
-  for (const p of fs.readdirSync(d)) {
-    const f = path.join(d, p);
-    debug('  ', f);
-    if (fs.statSync(f).isDirectory()) {
-      ls_r(f);
+/**
+ * The runfiles manifest maps from short_path
+ * https://docs.bazel.build/versions/master/skylark/lib/File.html#short_path
+ * to the actual location on disk where the file can be read.
+ *
+ * In a sandboxed execution, it does not exist. In that case, runfiles must be
+ * resolved from a symlink tree under the runfiles dir.
+ * See https://github.com/bazelbuild/bazel/issues/3726
+ */
+function loadRunfilesManifest(manifestPath) {
+  log_verbose(`using runfiles manifest ${manifestPath}`);
+
+  // Create the manifest and reverse manifest maps.
+  const runfilesEntries = new Map();
+  const input = fs.readFileSync(manifestPath, {encoding: 'utf-8'});
+
+  for (const line of input.split('\n')) {
+    if (!line) continue;
+    const [runfilesPath, realPath] = line.split(' ');
+    runfilesEntries.set(runfilesPath, realPath);
+  }
+
+  return runfilesEntries;
+}
+
+function lookupDirectory(dir, runfilesManifest) {
+  for (const [k, v] of runfilesManifest) {
+    // Entry looks like
+    // k: npm/node_modules/semver/LICENSE
+    // v: /path/to/external/npm/node_modules/semver/LICENSE
+    // calculate l = length(`/semver/LICENSE`)
+    if (k.startsWith(dir)) {
+      const l = k.length - dir.length;
+      return v.substring(0, v.length - l);
     }
   }
+  throw new Error(`Internal failure, please report an issue.
+  RunfilesManifest has no key for ${dir}
+  `);
 }
 
-function symlink(target, path) {
-  debug(`symlink( ${path} -> ${target} )`);
-  fs.symlinkSync(target, path);
+/**
+ * Resolve a root directory string to the actual location on disk
+ * where node_modules was installed
+ * @param root a string like 'npm/node_modules'
+ */
+function resolveRoot(root, runfilesManifest) {
+  // create a node_modules directory if no root
+  // this will be the case if only first-party modules are installed
+  if (!root) {
+    log_verbose('no third-party packages; mkdir node_modules in ', process.cwd);
+    fs.mkdirSync('node_modules');
+    return 'node_modules';
+  }
+
+  // If we got a runfilesManifest map, look through it for a resolution
+  if (runfilesManifest) {
+    return lookupDirectory(root, runfilesManifest);
+  }
+
+  // Account for Bazel --legacy_external_runfiles
+  // which look like 'my_wksp/external/npm/node_modules'
+  if (fs.existsSync(path.join('external', root))) {
+    log_verbose('Found legacy_external_runfiles, switching root to', path.join('external', root));
+    return path.join('external', root);
+  }
+
+  // The repository should be layed out in the parent directory
+  // since bazel sets our working directory to the repository where the build is happening
+  return path.join('..', root);
 }
 
-function main(args) {
+function main(args, runfilesManifestPath) {
   if (!args || args.length < 1)
     throw new Error('link_node_modules.js requires one argument: modulesManifest path');
 
   const [modulesManifest] = args;
   let {root, modules} = JSON.parse(fs.readFileSync(modulesManifest));
   modules = modules || {};
-  debug('read module manifest, node_modules root is', root, 'with first-party packages', modules);
-
-  // First, symlink from a local `$PWD/node_modules` directory to the root of the third-party
-  // installed node_modules (or create a node_modules directory if no root)
-
-  if (!!root) {
-    // Account for Bazel --legacy_external_runfiles
-    if (fs.existsSync(path.join('external', root))) {
-      debug('Found legacy_external_runfiles, switching root to', path.join('external', root));
-      root = path.join('external', root);
-    } else {
-      // The repository should be layed out in the parent directory
-      // since bazel sets our working directory to the repository where the build is happening
-      root = path.join('..', root);
-    }
-    if (!fs.existsSync('node_modules')) {
-      // Bazel starts our process in the working directory of the current workspace
-      // Since our mappings contain the workspace name, we want to be relative to the parent
-      // directory
-      symlink(root, 'node_modules');
-    }
-  } else {
-    fs.mkdirSync('node_modules');
-    console.error('mkdir node_modules in ', process.cwd)
-    root = 'node_modules';
-  }
+  log_verbose(
+      'read module manifest, node_modules root is', root, 'with first-party packages', modules);
+
+  const runfilesManifest =
+      runfilesManifestPath ? loadRunfilesManifest(runfilesManifestPath) : undefined;
+  const rootDir = resolveRoot(root, runfilesManifest);
+  log_verbose('resolved root', root, 'to', rootDir);
+
+  // Create the execroot/my_wksp/node_modules directory that node will resolve from
+  symlink(rootDir, 'node_modules');
 
   // Typically, cwd=foo, root=external/npm/node_modules, so we want links to be
   // ../../../../foo/path/to/package
-  const symlinkRelativeTarget = path.relative(root, '..');
-  process.chdir(root);
-
+  const symlinkRelativeTarget = path.relative(rootDir, '..');
+  process.chdir(rootDir);
 
   // Now add symlinks to each of our first-party packages so they appear under the node_modules tree
   for (const m of Object.keys(modules)) {
-    if (fs.existsSync(m)) continue;
-
-    let target = path.join(symlinkRelativeTarget, modules[m]);
+    const target = runfilesManifest ? lookupDirectory(modules[m], runfilesManifest) :
+                                      path.join(symlinkRelativeTarget, modules[m]);
     symlink(target, m);
   }
-  debug('at the end cwd', process.cwd())
-  ls_r('.');
+
+  return 0;
 }
 
 exports.main = main;
 
 if (require.main === module) {
-  process.exitCode = main(process.argv.slice(2));
+  // If Bazel sets a variable pointing to a runfiles manifest,
+  // we'll always use it.
+  // Note that this has a slight performance implication on Mac/Linux
+  // where we could use the runfiles tree already laid out on disk
+  // but this just costs one file read for the external npm/node_modules
+  // and one for each first-party module, not one per file.
+  const runfilesManifestPath = process.env['RUNFILES_MANIFEST_FILE'];
+  // Under --noenable_runfiles (in particular on Windows)
+  // Bazel sets RUNFILES_MANIFEST_ONLY=1.
+  // When this happens, we need to read the manifest file to locate
+  // inputs
+  if (process.env['RUNFILES_MANIFEST_ONLY'] === '1' && !runfilesManifestPath) {
+    log_verbose(`Workaround https://github.com/bazelbuild/bazel/issues/7994
+         RUNFILES_MANIFEST_FILE should have been set but wasn't.
+         falling back to using runfiles symlinks.
+         If you want to test runfiles manifest behavior, add
+         --spawn_strategy=standalone to the command line.`);
+  }
+  process.exitCode = main(process.argv.slice(2), runfilesManifestPath);
 }

internal/linker/test/integration/BUILD.bazel

@@ -13,7 +13,7 @@ genrule(
     name = "replace_node_path",
     srcs = [":test.sh"],
     outs = ["test_with_node.sh"],
-    cmd = "sed s#node#../$(NODE_PATH)# $< > $@",
+    cmd = "sed s#NODE_PATH#$(NODE_PATH)# $< > $@",
     toolchains = ["@build_bazel_rules_nodejs//toolchains/node:toolchain"],
 )
 
@@ -24,6 +24,7 @@ sh_test(
         ":example",
         ":program.js",
         "//internal/linker:link_node_modules.js",
+        "@bazel_tools//tools/bash/runfiles",
         "@build_bazel_rules_nodejs//toolchains/node:node_bin",
         # TODO: we shouldn't need to repeat this here. There's a bug somewhere
         "@npm//semver",

internal/linker/test/integration/test.sh

@@ -1,17 +1,49 @@
 #!/usr/bin/env bash
+# Copyright 2019 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Immediately exit if any command fails.
+set -e
 
 # Turn on extra logging so that test failures are easier to debug
 export VERBOSE_LOGS=1
 export NODE_DEBUG=module
 
-ls -R ..
-# Assume node is on the machine
-# This needs to be changed to use bazel toolchains before merging
-node internal/linker/link_node_modules.js internal/linker/test/integration/_example.module_mappings.json
-node --preserve-symlinks-main internal/linker/test/integration/program.js > $TEST_TMPDIR/out
+# --- begin runfiles.bash initialization v2 ---
+# Copy-pasted from the Bazel Bash runfiles library v2.
+set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$0.runfiles/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v2 ---
+
+readonly DIR="${TEST_WORKSPACE}/internal/linker"
+
+$(rlocation NODE_PATH) \
+  $(rlocation $DIR/link_node_modules.js)\
+  $(rlocation $DIR/test/integration/_example.module_mappings.json)
+
+readonly ACTUAL=$(
+  $(rlocation NODE_PATH) \
+  --preserve-symlinks-main \
+  $(rlocation $DIR/test/integration/program.js)
+)
 
-out=`cat $TEST_TMPDIR/out`
-if [[ "$out" != "1.2.3_a" ]]; then
+if [[ "$ACTUAL" != "1.2.3_a" ]]; then
   echo "expected 1.2.3_a but was ${out}" >&2
   exit 1
 fi

internal/node/node_launcher.sh

@@ -133,7 +133,7 @@ for ARG in "${ALL_ARGS[@]}"; do
 done
 
 # Link the first-party modules into node_modules directory before running the actual program
-if [[ -n "$MODULES_MANIFEST" ]]; then
+if [[ -n "${MODULES_MANIFEST:-}" ]]; then
   "${node}" "${link_modules_script}" "${MODULES_MANIFEST}"
 fi