diff --git a/internal/linker/link_node_modules.js b/internal/linker/link_node_modules.js index 88176369f7..97ffad6e3d 100644 --- a/internal/linker/link_node_modules.js +++ b/internal/linker/link_node_modules.js @@ -6,78 +6,147 @@ const fs = require('fs'); const path = require('path'); -function debug(...m) { - if (!!process.env['VERBOSE_LOGS']) console.error('[link_node_modules.js]', ...m); +const VERBOSE_LOGS = !!process.env['VERBOSE_LOGS']; + +function log_verbose(...m) { + // This is a template file so we use __filename to output the actual filename + if (VERBOSE_LOGS) console.error('[link_node_modules.js]', ...m); +} + +function symlink(target, path) { + log_verbose(`symlink( ${path} -> ${target} )`); + // Use junction on Windows since symlinks require elevated permissions + // we only link to directories so junctions work for us. + fs.symlinkSync(target, path, 'junction'); } -function ls_r(d) { - for (const p of fs.readdirSync(d)) { - const f = path.join(d, p); - debug(' ', f); - if (fs.statSync(f).isDirectory()) { - ls_r(f); +/** + * The runfiles manifest maps from short_path + * https://docs.bazel.build/versions/master/skylark/lib/File.html#short_path + * to the actual location on disk where the file can be read. + * + * In a sandboxed execution, it does not exist. In that case, runfiles must be + * resolved from a symlink tree under the runfiles dir. + * See https://github.com/bazelbuild/bazel/issues/3726 + */ +function loadRunfilesManifest(manifestPath) { + log_verbose(`using runfiles manifest ${manifestPath}`); + + // Create the manifest and reverse manifest maps. + const runfilesEntries = new Map(); + const input = fs.readFileSync(manifestPath, {encoding: 'utf-8'}); + + for (const line of input.split('\n')) { + if (!line) continue; + const [runfilesPath, realPath] = line.split(' '); + runfilesEntries.set(runfilesPath, realPath); + } + + return runfilesEntries; +} + +function lookupDirectory(dir, runfilesManifest) { + for (const [k, v] of runfilesManifest) { + // Entry looks like + // k: npm/node_modules/semver/LICENSE + // v: /path/to/external/npm/node_modules/semver/LICENSE + // calculate l = length(`/semver/LICENSE`) + if (k.startsWith(dir)) { + const l = k.length - dir.length; + return v.substring(0, v.length - l); } } + throw new Error(`Internal failure, please report an issue. + RunfilesManifest has no key for ${dir} + `); } -function symlink(target, path) { - debug(`symlink( ${path} -> ${target} )`); - fs.symlinkSync(target, path); +/** + * Resolve a root directory string to the actual location on disk + * where node_modules was installed + * @param root a string like 'npm/node_modules' + */ +function resolveRoot(root, runfilesManifest) { + // create a node_modules directory if no root + // this will be the case if only first-party modules are installed + if (!root) { + log_verbose('no third-party packages; mkdir node_modules in ', process.cwd); + fs.mkdirSync('node_modules'); + return 'node_modules'; + } + + // If we got a runfilesManifest map, look through it for a resolution + if (runfilesManifest) { + return lookupDirectory(root, runfilesManifest); + } + + // Account for Bazel --legacy_external_runfiles + // which look like 'my_wksp/external/npm/node_modules' + if (fs.existsSync(path.join('external', root))) { + log_verbose('Found legacy_external_runfiles, switching root to', path.join('external', root)); + return path.join('external', root); + } + + // The repository should be layed out in the parent directory + // since bazel sets our working directory to the repository where the build is happening + return path.join('..', root); } -function main(args) { +function main(args, runfilesManifestPath) { if (!args || args.length < 1) throw new Error('link_node_modules.js requires one argument: modulesManifest path'); const [modulesManifest] = args; let {root, modules} = JSON.parse(fs.readFileSync(modulesManifest)); modules = modules || {}; - debug('read module manifest, node_modules root is', root, 'with first-party packages', modules); - - // First, symlink from a local `$PWD/node_modules` directory to the root of the third-party - // installed node_modules (or create a node_modules directory if no root) - - if (!!root) { - // Account for Bazel --legacy_external_runfiles - if (fs.existsSync(path.join('external', root))) { - debug('Found legacy_external_runfiles, switching root to', path.join('external', root)); - root = path.join('external', root); - } else { - // The repository should be layed out in the parent directory - // since bazel sets our working directory to the repository where the build is happening - root = path.join('..', root); - } - if (!fs.existsSync('node_modules')) { - // Bazel starts our process in the working directory of the current workspace - // Since our mappings contain the workspace name, we want to be relative to the parent - // directory - symlink(root, 'node_modules'); - } - } else { - fs.mkdirSync('node_modules'); - console.error('mkdir node_modules in ', process.cwd) - root = 'node_modules'; + log_verbose( + 'read module manifest, node_modules root is', root, 'with first-party packages', modules); + + const runfilesManifest = + runfilesManifestPath ? loadRunfilesManifest(runfilesManifestPath) : undefined; + const rootDir = resolveRoot(root, runfilesManifest); + log_verbose('resolved root', root, 'to', rootDir); + + // Create the execroot/my_wksp/node_modules directory that node will resolve from + if (!fs.existsSync('node_modules')) { + symlink(rootDir, 'node_modules'); } // Typically, cwd=foo, root=external/npm/node_modules, so we want links to be // ../../../../foo/path/to/package - const symlinkRelativeTarget = path.relative(root, '..'); - process.chdir(root); - + const symlinkRelativeTarget = path.relative(rootDir, '..'); + process.chdir(rootDir); // Now add symlinks to each of our first-party packages so they appear under the node_modules tree for (const m of Object.keys(modules)) { if (fs.existsSync(m)) continue; - let target = path.join(symlinkRelativeTarget, modules[m]); + const target = runfilesManifest ? lookupDirectory(modules[m], runfilesManifest) : + path.join(symlinkRelativeTarget, modules[m]); symlink(target, m); } - debug('at the end cwd', process.cwd()) - ls_r('.'); } exports.main = main; if (require.main === module) { - process.exitCode = main(process.argv.slice(2)); + // If Bazel sets a variable pointing to a runfiles manifest, + // we'll always use it. + // Note that this has a slight performance implication on Mac/Linux + // where we could use the runfiles tree already laid out on disk + // but this just costs one file read for the external npm/node_modules + // and one for each first-party module, not one per file. + const runfilesManifestPath = process.env['RUNFILES_MANIFEST_FILE']; + // Under --noenable_runfiles (in particular on Windows) + // Bazel sets RUNFILES_MANIFEST_ONLY=1. + // When this happens, we need to read the manifest file to locate + // inputs + if (process.env['RUNFILES_MANIFEST_ONLY'] === '1' && !runfilesManifestPath) { + log_verbose(`Workaround https://github.com/bazelbuild/bazel/issues/7994 + RUNFILES_MANIFEST_FILE should have been set but wasn't. + falling back to using runfiles symlinks. + If you want to test runfiles manifest behavior, add + --spawn_strategy=standalone to the command line.`); + } + process.exitCode = main(process.argv.slice(2), runfilesManifestPath); } \ No newline at end of file diff --git a/internal/linker/test/integration/BUILD.bazel b/internal/linker/test/integration/BUILD.bazel index d6f5e0c684..1221fd1edc 100644 --- a/internal/linker/test/integration/BUILD.bazel +++ b/internal/linker/test/integration/BUILD.bazel @@ -13,7 +13,7 @@ genrule( name = "replace_node_path", srcs = [":test.sh"], outs = ["test_with_node.sh"], - cmd = "sed s#node#../$(NODE_PATH)# $< > $@", + cmd = "sed s#NODE_PATH#$(NODE_PATH)# $< > $@", toolchains = ["@build_bazel_rules_nodejs//toolchains/node:toolchain"], ) @@ -24,6 +24,7 @@ sh_test( ":example", ":program.js", "//internal/linker:link_node_modules.js", + "@bazel_tools//tools/bash/runfiles", "@build_bazel_rules_nodejs//toolchains/node:node_bin", # TODO: we shouldn't need to repeat this here. There's a bug somewhere "@npm//semver", diff --git a/internal/linker/test/integration/test.sh b/internal/linker/test/integration/test.sh index 638fe48523..fd2a61c348 100755 --- a/internal/linker/test/integration/test.sh +++ b/internal/linker/test/integration/test.sh @@ -1,17 +1,49 @@ #!/usr/bin/env bash +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Immediately exit if any command fails. +set -e # Turn on extra logging so that test failures are easier to debug export VERBOSE_LOGS=1 export NODE_DEBUG=module -ls -R .. -# Assume node is on the machine -# This needs to be changed to use bazel toolchains before merging -node internal/linker/link_node_modules.js internal/linker/test/integration/_example.module_mappings.json -node --preserve-symlinks-main internal/linker/test/integration/program.js > $TEST_TMPDIR/out +# --- begin runfiles.bash initialization v2 --- +# Copy-pasted from the Bazel Bash runfiles library v2. +set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v2 --- + +readonly DIR="${TEST_WORKSPACE}/internal/linker" + +$(rlocation NODE_PATH) \ + $(rlocation $DIR/link_node_modules.js)\ + $(rlocation $DIR/test/integration/_example.module_mappings.json) + +readonly ACTUAL=$( + $(rlocation NODE_PATH) \ + --preserve-symlinks-main \ + $(rlocation $DIR/test/integration/program.js) +) -out=`cat $TEST_TMPDIR/out` -if [[ "$out" != "1.2.3_a" ]]; then +if [[ "$ACTUAL" != "1.2.3_a" ]]; then echo "expected 1.2.3_a but was ${out}" >&2 exit 1 fi diff --git a/internal/node/node_launcher.sh b/internal/node/node_launcher.sh index e70f580d67..813df9bc7d 100644 --- a/internal/node/node_launcher.sh +++ b/internal/node/node_launcher.sh @@ -133,7 +133,7 @@ for ARG in "${ALL_ARGS[@]}"; do done # Link the first-party modules into node_modules directory before running the actual program -if [[ -n "$MODULES_MANIFEST" ]]; then +if [[ -n "${MODULES_MANIFEST:-}" ]]; then "${node}" "${link_modules_script}" "${MODULES_MANIFEST}" fi