From 8f74acef5f394e2b4cbcbed9ea73ad2b4fbaede7 Mon Sep 17 00:00:00 2001 From: Cameron Martin Date: Tue, 27 Feb 2024 01:12:22 -0800 Subject: [PATCH] Exclude convenience symlinks after changing the output base When the output base is changed after the convenience symlinks have been created, queries such as `//...` try to traverse into them and load them as packages since they are only excluded based on whether the resolved location is in the output base. This adds some heuristics to determine if a symlink is a convenience symlink: 1. The symlink name has an appropriate suffix. 2. An ancestor of the symlink target at the appropriate level is called `execroot`. 3. The `execroot` directory contains a file called `DO_NOT_BUILD_HERE`. These heuristics should work if both the output base and the symlink prefix change while being quite robust to false positives. This is important for IDE integration, where the [output base is often changed](https://bazel.build/run/scripts#output-base-option) for queries, to prevent concurrent builds from blocking them. An example of this is the vscode-bazel extension. Fixes #10653 Fixes #13951 Closes #21005. PiperOrigin-RevId: 610667735 Change-Id: I1869c9a2063f7f526950e48c0b1ee6efa89fd202 --- .../lib/skyframe/ProcessPackageDirectory.java | 92 +++++++++++++++++-- src/test/py/bazel/query_test.py | 16 ++++ 2 files changed, 98 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ProcessPackageDirectory.java b/src/main/java/com/google/devtools/build/lib/skyframe/ProcessPackageDirectory.java index cd434404d964b4..e0f300fdce81c2 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ProcessPackageDirectory.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ProcessPackageDirectory.java @@ -123,19 +123,14 @@ public ProcessPackageDirectoryResult getPackageExistenceAndSubdirDeps( PackageIdentifier packageId = PackageIdentifier.create(repositoryName, rootRelativePath); - if (packageId.getRepository().isMain() - && fileValue.isSymlink() - && fileValue - .getUnresolvedLinkTarget() - .startsWith(directories.getExecRootBase().asFragment())) { - // Symlinks back to the execroot are not traversed so that we avoid convenience symlinks. - // Note that it's not enough to just check for the convenience symlinks themselves, - // because if the value of --symlink_prefix changes, the old symlinks are left in place. This - // algorithm also covers more creative use cases where people create convenience symlinks - // somewhere in the directory tree manually. + if (packageId.getRepository().isMain() && isConvenienceSymlink(fileValue, rootedPath, env)) { return ProcessPackageDirectoryResult.EMPTY_RESULT; } + if (env.valuesMissing()) { + return null; + } + SkyKey pkgLookupKey = PackageLookupValue.key(packageId); SkyKey dirListingKey = DirectoryListingValue.key(rootedPath); SkyframeLookupResult pkgLookupAndDirectoryListingDeps = @@ -187,6 +182,83 @@ public ProcessPackageDirectoryResult getPackageExistenceAndSubdirDeps( /*additionalValuesToAggregate=*/ ImmutableMap.of()); } + // Note that it's not enough to just check for the convenience symlinks themselves, + // because if the value of --symlink_prefix changes, the old symlinks are left in place. It + // is also not sufficient to check whether the symlink points to a directory in the current + // exec root, since this can change between bazel invocations. Therefore we check if the + // suffix of the symlink source suggests it is a convenience symlink, then see if the symlink + // target is in a directory that looks like an execroot. This algorithm also covers more + // creative use cases where people create convenience symlinks somewhere in the directory + // tree manually. + private boolean isConvenienceSymlink( + FileValue fileValue, RootedPath rootedPath, SkyFunction.Environment env) + throws InterruptedException { + if (!fileValue.isSymlink()) { + return false; + } + + PathFragment linkTarget = fileValue.getUnresolvedLinkTarget(); + + if (linkTarget.startsWith(directories.getExecRootBase().asFragment())) { + return true; + } + + PathFragment rootRelativePath = rootedPath.getRootRelativePath(); + Root root = rootedPath.getRoot(); + + if (rootRelativePath.getBaseName().endsWith("-bin") && isInExecRoot(linkTarget, root, 4, env)) { + return true; + } + + if (rootRelativePath.getBaseName().endsWith("-genfiles") + && isInExecRoot(linkTarget, root, 4, env)) { + return true; + } + + if (rootRelativePath.getBaseName().endsWith("-out") && isInExecRoot(linkTarget, root, 2, env)) { + return true; + } + + if (rootRelativePath.getBaseName().endsWith("-testlogs") + && isInExecRoot(linkTarget, root, 4, env)) { + return true; + } + + if (rootRelativePath + .getBaseName() + .endsWith("-" + directories.getWorkingDirectory().getBaseName()) + && isInExecRoot(linkTarget, root, 1, env)) { + return true; + } + + return false; + } + + private boolean isInExecRoot(PathFragment path, Root root, int depth, SkyFunction.Environment env) + throws InterruptedException { + int segmentCount = path.segmentCount(); + + if (segmentCount <= depth) { + return false; + } + + PathFragment candidateExecRoot = path.subFragment(0, segmentCount - depth); + + if (!candidateExecRoot.getBaseName().equals("execroot")) { + return false; + } + + Root absoluteRoot = Root.absoluteRoot(root.getFileSystem()); + RootedPath doNotBuildPath = + RootedPath.toRootedPath(absoluteRoot, candidateExecRoot.getChild("DO_NOT_BUILD_HERE")); + FileValue doNotBuildValue = (FileValue) env.getValue(FileValue.key(doNotBuildPath)); + if (doNotBuildValue == null) { + return false; + } + + return doNotBuildValue.exists(); + } + private Iterable getSubdirDeps( DirectoryListingValue dirListingValue, RootedPath rootedPath, diff --git a/src/test/py/bazel/query_test.py b/src/test/py/bazel/query_test.py index 0af495e1281fb5..d651300217cb48 100644 --- a/src/test/py/bazel/query_test.py +++ b/src/test/py/bazel/query_test.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import tempfile from absl.testing import absltest from src.test.py.bazel import test_base @@ -129,6 +131,20 @@ def testBuildFilesForExternalRepos_NoDuplicates(self): self.assertNotIn(item, result) result.add(item) + def testQueryWithDifferentOutputBaseAfterBuilding(self): + output_base = tempfile.mkdtemp(dir=os.getenv('TEST_TMPDIR')) + + self.ScratchFile('MODULE.bazel') + self.ScratchFile( + 'BUILD', + [ + 'py_binary(name="a", srcs=["a.py"])', + ], + ) + self.ScratchFile('a.py') + self.RunBazel(['build', '//...']) + self.RunBazel([f'--output_base={output_base}', 'query', '//...']) + def _AssertQueryOutput(self, query_expr, *expected_results): _, stdout, _ = self.RunBazel(['query', query_expr])