Skip to content

Commit

Permalink
fix: don't symlink execroot node_modules when under bazel run
Browse files Browse the repository at this point in the history
  • Loading branch information
@gregmagolan @alexeagle
gregmagolan authored and alexeagle committed Dec 3, 2021
1 parent 33665eb commit d19e20b
Show file tree
Hide file tree
Showing 8 changed files with 207 additions and 146 deletions.
3 changes: 3 additions & 0 deletions .circleci/bazel.rc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,6 @@ test --flaky_test_attempts=2
# Expose the SSH_AUTH_SOCK variable to integration tests that need to run git_repository repository rules
# This is needed by the CircleCI git-over-ssh environment
test --test_env=SSH_AUTH_SOCK

# Keep going after first failure
build --keep_going
54 changes: 36 additions & 18 deletions internal/linker/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,8 @@ function main(args, runfiles) {
log_verbose('execroot:', execroot ? execroot : 'not found');
const isExecroot = startCwd == execroot;
log_verbose('isExecroot:', isExecroot.toString());
const isBazelRun = !!process.env['BUILD_WORKSPACE_DIRECTORY'];
log_verbose('isBazelRun:', isBazelRun.toString());
if (!isExecroot && execroot) {
process.chdir(execroot);
log_verbose('changed directory to execroot', execroot);
Expand Down Expand Up @@ -327,32 +329,46 @@ function main(args, runfiles) {
else {
workspaceNodeModules = undefined;
}
let primaryNodeModules;
if (packagePath) {
yield mkdirp(packagePath);
}
const execrootNodeModules = path.posix.join(packagePath, `node_modules`);
if (workspaceNodeModules) {
yield symlinkWithUnlink(workspaceNodeModules, execrootNodeModules);
const binNodeModules = path.posix.join(bin, packagePath, 'node_modules');
yield mkdirp(path.dirname(binNodeModules));
if (workspaceNodeModules) {
yield symlinkWithUnlink(workspaceNodeModules, binNodeModules);
primaryNodeModules = workspaceNodeModules;
}
else {
yield mkdirp(binNodeModules);
primaryNodeModules = binNodeModules;
}
if (!isBazelRun) {
const execrootNodeModules = path.posix.join(packagePath, 'node_modules');
yield mkdirp(path.dirname(execrootNodeModules));
yield symlinkWithUnlink(primaryNodeModules, execrootNodeModules);
}
}
else {
yield mkdirp(execrootNodeModules);
}
if (packagePath) {
const packagePathBin = path.posix.join(bin, packagePath);
yield mkdirp(`${packagePathBin}`);
yield symlinkWithUnlink(execrootNodeModules, `${packagePathBin}/node_modules`);
const execrootNodeModules = 'node_modules';
if (workspaceNodeModules) {
yield symlinkWithUnlink(workspaceNodeModules, execrootNodeModules);
primaryNodeModules = workspaceNodeModules;
}
else {
yield mkdirp(execrootNodeModules);
primaryNodeModules = execrootNodeModules;
}
}
if (!isExecroot) {
const runfilesPackagePath = path.posix.join(startCwd, packagePath);
yield mkdirp(`${runfilesPackagePath}`);
yield symlinkWithUnlink(!packagePath && workspaceNodeModules ? workspaceNodeModules : execrootNodeModules, `${runfilesPackagePath}/node_modules`);
const runfilesNodeModules = path.posix.join(startCwd, packagePath, 'node_modules');
yield mkdirp(path.dirname(runfilesNodeModules));
yield symlinkWithUnlink(primaryNodeModules, runfilesNodeModules);
}
if (process.env['RUNFILES']) {
const stat = yield gracefulLstat(process.env['RUNFILES']);
if (stat && stat.isDirectory()) {
const runfilesPackagePath = path.posix.join(process.env['RUNFILES'], workspace, packagePath);
yield mkdirp(`${runfilesPackagePath}`);
yield symlinkWithUnlink(!packagePath && workspaceNodeModules ? workspaceNodeModules : execrootNodeModules, `${runfilesPackagePath}/node_modules`);
const runfilesNodeModules = path.posix.join(process.env['RUNFILES'], workspace, 'node_modules');
yield mkdirp(path.dirname(runfilesNodeModules));
yield symlinkWithUnlink(primaryNodeModules, runfilesNodeModules);
}
}
}
Expand Down Expand Up @@ -393,7 +409,9 @@ function main(args, runfiles) {
}
function linkModules(package_path, m) {
return __awaiter(this, void 0, void 0, function* () {
const symlinkIn = package_path ? `${package_path}/node_modules` : 'node_modules';
const symlinkIn = package_path ?
path.posix.join(bin, package_path, 'node_modules') :
'node_modules';
if (path.dirname(m.name)) {
yield mkdirp(`${symlinkIn}/${path.dirname(m.name)}`);
}
Expand Down
82 changes: 49 additions & 33 deletions internal/linker/link_node_modules.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,9 @@ export async function main(args: string[], runfiles: Runfiles) {
const isExecroot = startCwd == execroot;
log_verbose('isExecroot:', isExecroot.toString());

const isBazelRun = !!process.env['BUILD_WORKSPACE_DIRECTORY']
log_verbose('isBazelRun:', isBazelRun.toString());

if (!isExecroot && execroot) {
// If we're not in the execroot and we've found one then change to the execroot
// directory to create the node_modules symlinks
Expand Down Expand Up @@ -481,55 +484,66 @@ export async function main(args: string[], runfiles: Runfiles) {
workspaceNodeModules = undefined;
}

let primaryNodeModules;
if (packagePath) {
// In some cases packagePath may not exist in sandbox if there are no source deps
// and only generated file deps. we create it so that we that we can link to
// packagePath/node_modules since packagePathBin/node_modules is a symlink to
// packagePath/node_modules and is unguarded in launcher.sh as we allow symlinks to fall
// through to from output tree to source tree to prevent resolving the same npm package to
// multiple locations on disk
await mkdirp(packagePath);
}

// There are third party modules at this package path
const execrootNodeModules = path.posix.join(packagePath, `node_modules`);
const binNodeModules = path.posix.join(bin, packagePath, 'node_modules');
await mkdirp(path.dirname(binNodeModules));

// Create bin/<package_path>/node_modules symlink
// (or empty directory if there are no 3rd party deps to symlink to)
if (workspaceNodeModules) {
await symlinkWithUnlink(workspaceNodeModules, binNodeModules);
primaryNodeModules = workspaceNodeModules;
} else {
await mkdirp(binNodeModules);
primaryNodeModules = binNodeModules;
}

if (workspaceNodeModules) {
// Execroot symlink -> external workspace node_modules
await symlinkWithUnlink(workspaceNodeModules, execrootNodeModules);
if (!isBazelRun) {
// Special case under bazel run where we don't want to create node_modules
// in an execroot under a package path as this will end up in the user's
// workspace via the package path folder symlink
const execrootNodeModules = path.posix.join(packagePath, 'node_modules');
await mkdirp(path.dirname(execrootNodeModules));
await symlinkWithUnlink(primaryNodeModules, execrootNodeModules);
}
} else {
// Create an execroot node_modules directory since there are no third party node_modules to symlink to
await mkdirp(execrootNodeModules);
}
const execrootNodeModules = 'node_modules';

if (packagePath) {
// Bin symlink -> execroot node_modules
// NB: don't do this for the root of the bin tree since standard node_modules resolution
// will fall back to the execroot node_modules naturally
// See https://github.com/bazelbuild/rules_nodejs/issues/3054
const packagePathBin = path.posix.join(bin, packagePath);
await mkdirp(`${packagePathBin}`);
await symlinkWithUnlink(execrootNodeModules, `${packagePathBin}/node_modules`);
// Create execroot/node_modules symlink (or empty directory if there are
// no 3rd party deps to symlink to)
if (workspaceNodeModules) {
await symlinkWithUnlink(workspaceNodeModules, execrootNodeModules);
primaryNodeModules = workspaceNodeModules;
} else {
await mkdirp(execrootNodeModules);
primaryNodeModules = execrootNodeModules;
}

// NB: Don't create a bin/node_modules since standard node_modules
// resolution will fall back to the execroot node_modules naturally. See
// https://github.com/bazelbuild/rules_nodejs/issues/3054
}

// Start CWD symlink -> execroot node_modules
// If start cwd was in runfiles then create
// start/cwd
if (!isExecroot) {
const runfilesPackagePath = path.posix.join(startCwd, packagePath);
await mkdirp(`${runfilesPackagePath}`);
const runfilesNodeModules = path.posix.join(startCwd, packagePath, 'node_modules');
await mkdirp(path.dirname(runfilesNodeModules));
// Don't link to the root execroot node_modules if there is a workspace node_modules.
// Bazel will delete that symlink on rebuild in the ibazel run context.
await symlinkWithUnlink(!packagePath && workspaceNodeModules ? workspaceNodeModules : execrootNodeModules, `${runfilesPackagePath}/node_modules`);
await symlinkWithUnlink(primaryNodeModules, runfilesNodeModules);
}

// RUNFILES symlink -> execroot node_modules
if (process.env['RUNFILES']) {
const stat = await gracefulLstat(process.env['RUNFILES']);
if (stat && stat.isDirectory()) {
const runfilesPackagePath = path.posix.join(process.env['RUNFILES'], workspace, packagePath);
await mkdirp(`${runfilesPackagePath}`);
const runfilesNodeModules = path.posix.join(process.env['RUNFILES'], workspace, 'node_modules');
await mkdirp(path.dirname(runfilesNodeModules));
// Don't link to the root execroot node_modules if there is a workspace node_modules.
// Bazel will delete that symlink on rebuild in the ibazel run context.
await symlinkWithUnlink(!packagePath && workspaceNodeModules ? workspaceNodeModules : execrootNodeModules, `${runfilesPackagePath}/node_modules`);
await symlinkWithUnlink(primaryNodeModules, runfilesNodeModules);
}
}
}
Expand Down Expand Up @@ -593,7 +607,9 @@ export async function main(args: string[], runfiles: Runfiles) {
}

async function linkModules(package_path: string, m: LinkerTreeElement) {
const symlinkIn = package_path ? `${package_path}/node_modules` : 'node_modules';
const symlinkIn = package_path ?
path.posix.join(bin, package_path, 'node_modules') :
'node_modules';

// ensure the parent directory exist
if (path.dirname(m.name)) {
Expand Down
62 changes: 25 additions & 37 deletions internal/node/launcher.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,29 +271,24 @@ else
RUNFILES_ROOT=
fi

# Tell the node_patches_script that programs should not escape the execroot
export BAZEL_PATCH_ROOTS="${EXECROOT}"
# Set all bazel managed node_modules directories as guarded so no symlinks may
# escape and no symlinks may enter.
# We always guard against the root node_modules where 1st party deps go.
# (e.g., /private/.../execroot/build_bazel_rules_nodejs/node_modules)
export BAZEL_PATCH_ROOTS="${BAZEL_PATCH_ROOTS},${EXECROOT}/node_modules"
# Tell the node_patches_script that programs should not escape the execroot or
# root node_modules
# For example,
# .../execroot/build_bazel_rules_nodejs
# .../execroot/build_bazel_rules_nodejs/node_modules
export BAZEL_PATCH_ROOTS="${EXECROOT},${EXECROOT}/node_modules"
if [[ "${RUNFILES_ROOT}" ]]; then
# If in runfiles, guard the runfiles root itself
export BAZEL_PATCH_ROOTS="${BAZEL_PATCH_ROOTS},${RUNFILES_ROOT}"
# If in runfiles guard the node_modules location in runfiles as well
# (e.g., /private/.../execroot/build_bazel_rules_nodejs/bazel-out/darwin-fastbuild/bin/internal/linker/test/multi_linker/test.sh.runfiles/build_bazel_rules_nodejs/node_modules)
export BAZEL_PATCH_ROOTS="${BAZEL_PATCH_ROOTS},${RUNFILES_ROOT}/${BAZEL_WORKSPACE}/node_modules"
# Guard the RUNFILES_ROOT & RUNFILES_ROOT/workspace/node_modules
export BAZEL_PATCH_ROOTS="${BAZEL_PATCH_ROOTS},${RUNFILES_ROOT},${RUNFILES_ROOT}/${BAZEL_WORKSPACE}/node_modules"
fi
if [[ "${RUNFILES}" ]]; then
# If in runfiles, guard the RUNFILES root itself
export BAZEL_PATCH_ROOTS="${BAZEL_PATCH_ROOTS},${RUNFILES}"
# If RUNFILES is set, guard the RUNFILES node_modules as well
export BAZEL_PATCH_ROOTS="${BAZEL_PATCH_ROOTS},${RUNFILES}/${BAZEL_WORKSPACE}/node_modules"
if [[ "${RUNFILES}" ]] && [[ "${RUNFILES}" != "${RUNFILES_ROOT}" ]]; then
# Same as above but for RUNFILES is not equal to RUNFILES_ROOT
export BAZEL_PATCH_ROOTS="${BAZEL_PATCH_ROOTS},${RUNFILES},${RUNFILES}/${BAZEL_WORKSPACE}/node_modules"
fi
if [[ -n "${BAZEL_NODE_MODULES_ROOTS:-}" ]]; then
# BAZEL_NODE_MODULES_ROOTS is in the format "<path>:<workspace>,<path>:<workspace>"
# (e.g., "internal/linker/test:npm_internal_linker_test,:npm")
# BAZEL_NODE_MODULES_ROOTS is in the format "<path>:<workspace>,<path>:<workspace>".
# For example,
# "internal/linker/test:npm_internal_linker_test,:npm"
if [[ -n "${VERBOSE_LOGS:-}" ]]; then
echo "BAZEL_NODE_MODULES_ROOTS=${BAZEL_NODE_MODULES_ROOTS}" >&2
fi
Expand All @@ -310,27 +305,20 @@ if [[ -n "${BAZEL_NODE_MODULES_ROOTS:-}" ]]; then
root_path="${root_pair[0]}"
root_workspace="${root_pair[1]:-}"
if [[ "${root_path}" ]]; then
# Guard non-root node_modules as well
# (e.g., /private/.../execroot/build_bazel_rules_nodejs/internal/linker/test/node_modules)
export BAZEL_PATCH_ROOTS="${BAZEL_PATCH_ROOTS},${EXECROOT}/${root_path}/node_modules"
# Guard non-root execroot & bazel-out node_modules as well.
# For example,
# .../execroot/build_bazel_rules_nodejs/internal/linker/test/node_modules
# .../execroot/build_bazel_rules_nodejs/bazel-out/*/bin/internal/linker/test/node_modules
export BAZEL_PATCH_ROOTS="${BAZEL_PATCH_ROOTS},${EXECROOT}/${root_path}/node_modules,${EXECROOT}/bazel-out/*/bin/${root_path}/node_modules"
if [[ "${RUNFILES_ROOT}" ]]; then
# If in runfiles guard the node_modules location in runfiles as well
# (e.g., /private/.../execroot/build_bazel_rules_nodejs/bazel-out/darwin-fastbuild/bin/internal/linker/test/multi_linker/test.sh.runfiles/build_bazel_rules_nodejs/internal/linker/test/node_modules)
# If in runfiles guard the node_modules location in runfiles as well.
# For example,
# .../execroot/build_bazel_rules_nodejs/bazel-out/darwin-fastbuild/bin/internal/linker/test/multi_linker/test.sh.runfiles/build_bazel_rules_nodejs/internal/linker/test/node_modules
export BAZEL_PATCH_ROOTS="${BAZEL_PATCH_ROOTS},${RUNFILES_ROOT}/${BAZEL_WORKSPACE}/${root_path}/node_modules"
fi
fi
# TODO: the following guards on the external workspaces may not be necessary and could be removed in the future with care
if [[ "${root_workspace}" ]] && [[ "${root_workspace}" != "${BAZEL_WORKSPACE}" ]]; then
# Guard the external workspaces if they are not the user workspace
# (e.g., /private/.../execroot/build_bazel_rules_nodejs/external/npm_internal_linker_test/node_modules)
export BAZEL_PATCH_ROOTS="${BAZEL_PATCH_ROOTS},${EXECROOT}/external/${root_workspace}/node_modules"
if [[ "${RUNFILES_ROOT}" ]]; then
# If in runfiles guard the external workspace location in runfiles as well
# (e.g., /private/.../execroot/build_bazel_rules_nodejs/bazel-out/darwin-fastbuild/bin/internal/linker/test/multi_linker/test.sh.runfiles/npm_internal_linker_test/node_modules)
export BAZEL_PATCH_ROOTS="${BAZEL_PATCH_ROOTS},${RUNFILES_ROOT}/${root_workspace}/node_modules"
# and include the legacy runfiles location incase legacy runfiles are enabled
# (e.g., /private/.../bazel-out/darwin-fastbuild/bin/internal/linker/test/multi_linker/test.sh.runfiles/build_bazel_rules_nodejs/external/npm_internal_linker_test/node_modules)
export BAZEL_PATCH_ROOTS="${BAZEL_PATCH_ROOTS},${RUNFILES_ROOT}/${BAZEL_WORKSPACE}/external/${root_workspace}/node_modules"
if [[ "${RUNFILES}" ]] && [[ "${RUNFILES}" != "${RUNFILES_ROOT}" ]]; then
# Same as above but for RUNFILES is not equal to RUNFILES_ROOT
export BAZEL_PATCH_ROOTS="${BAZEL_PATCH_ROOTS},${RUNFILES}/${BAZEL_WORKSPACE}/${root_path}/node_modules"
fi
fi
fi
Expand Down
13 changes: 12 additions & 1 deletion internal/node/node_patches.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -486,7 +486,18 @@ const patcher = (fs = fs__default['default'], roots) => {
};
exports.patcher = patcher;
function isOutPath(root, str) {
return !root || (!str.startsWith(root + path__default['default'].sep) && str !== root);
if (!root)
return true;
let strParts = str.split(path__default['default'].sep);
let rootParts = root.split(path__default['default'].sep);
let i = 0;
for (; i < rootParts.length && i < strParts.length; i++) {
if (rootParts[i] === strParts[i] || rootParts[i] === '*') {
continue;
}
break;
}
return i < rootParts.length;
}
exports.isOutPath = isOutPath;
const escapeFunction = (roots) => {
Expand Down
Loading

0 comments on commit d19e20b

Please sign in to comment.