Open-source LocalDiffAwarenessIntegrationTest.

PiperOrigin-RevId: 420127324

src/test/java/com/google/devtools/build/lib/buildtool/util/BUILD

@@ -98,6 +98,7 @@ java_library(
  "//src/test/java/com/google/devtools/build/lib/testutil:TestUtils",
  "//src/test/java/com/google/devtools/build/lib/vfs/util",
  "//third_party:guava",
+ "//third_party:guava-testlib",
  "//third_party:jsr305",
  "//third_party:junit4",
  "//third_party:truth",

src/test/java/com/google/devtools/build/lib/buildtool/util/SkyframeIntegrationTestBase.java

@@ -0,0 +1,88 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.buildtool.util;
+
+import static com.google.common.truth.Truth.assertThat;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.testing.GcFinalization;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/** Infrastructure to support Skyframe integration tests. */
+public abstract class SkyframeIntegrationTestBase extends BuildIntegrationTestCase {
+
+ protected SkyframeExecutor skyframeExecutor() {
+ return runtimeWrapper.getSkyframeExecutor();
+ }
+
+ protected static List<WeakReference<?>> weakRefs(Object... strongRefs) throws Exception {
+ List<WeakReference<?>> result = new ArrayList<>();
+ for (Object ref : strongRefs) {
+ result.add(new WeakReference<>(ref));
+ }
+ return result;
+ }
+
+ protected static void assertAllReleased(Iterable<WeakReference<?>> refs) {
+ for (WeakReference<?> ref : refs) {
+ GcFinalization.awaitClear(ref);
+ }
+ }
+
+ private String makeGenruleContents(String value) {
+ return String.format(
+ "genrule(name='target', outs=['out'], cmd='/bin/echo %s > $(location out)')", value);
+ }
+
+ protected void writeGenrule(String filename, String value) throws Exception {
+ write(filename, makeGenruleContents(value));
+ }
+
+ protected void writeGenruleAbsolute(Path file, String value) throws Exception {
+ writeAbsolute(file, makeGenruleContents(value));
+ }
+
+ protected void assertCharContentsIgnoringOrderAndWhitespace(
+ String expectedCharContents, String target) throws Exception {
+ Path path = Iterables.getOnlyElement(getArtifacts(target)).getPath();
+ char[] actualChars = FileSystemUtils.readContentAsLatin1(path);
+ char[] expectedChars = expectedCharContents.toCharArray();
+ Arrays.sort(actualChars);
+ Arrays.sort(expectedChars);
+ assertThat(new String(actualChars).trim()).isEqualTo(new String(expectedChars).trim());
+ }
+
+ protected void assertContents(String expectedContents, String target) throws Exception {
+ assertContents(expectedContents, Iterables.getOnlyElement(getArtifacts(target)).getPath());
+ }
+
+ protected void assertContents(String expectedContents, Path path) throws Exception {
+ String actualContents = new String(FileSystemUtils.readContentAsLatin1(path));
+ assertThat(actualContents.trim()).isEqualTo(expectedContents);
+ }
+
+ protected ImmutableList<String> getOnlyOutputContentAsLines(String target) throws Exception {
+ return FileSystemUtils.readLines(
+ Iterables.getOnlyElement(getArtifacts(target)).getPath(), UTF_8);
+ }
+}

src/test/java/com/google/devtools/build/lib/skyframe/BUILD

@@ -20,6 +20,7 @@ CROSS_PLATFORM_WINDOWS_TESTS = [
 
 # Tests that are broken out from the SkyframeTests target into separate targets.
 EXCLUDED_FROM_SKYFRAME_TESTS = [
+ "LocalDiffAwarenessIntegrationTest.java",
  "PrepareDepsOfTargetsUnderDirectoryFunctionTest.java", # b/179148968
 ] + CROSS_PLATFORM_WINDOWS_TESTS
 
@@ -378,3 +379,28 @@ java_test(
  "//third_party:truth",
  ],
 )
+
+java_test(
+ name = "LocalDiffAwarenessIntegrationTest",
+ srcs = ["LocalDiffAwarenessIntegrationTest.java"],
+ # TODO(pcloudy): Even with --experimental_windows_watchfs, there's an extra
+ # getValues() on the second build in
+ # externalSymlink_doesNotTriggerFullGraphTraversal with Windows, and
+ # non-deterministic failure to detect changes (watchfs bug?).
+ tags = ["no_windows"],
+ deps = [
+ "//src/main/java/com/google/devtools/build/lib:runtime",
+ "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",
+ "//src/main/java/com/google/devtools/build/lib/skyframe:local_diff_awareness",
+ "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception",
+ "//src/main/java/com/google/devtools/build/lib/util:os",
+ "//src/main/java/com/google/devtools/build/lib/vfs",
+ "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
+ "//src/main/java/com/google/devtools/common/options",
+ "//src/test/java/com/google/devtools/build/lib/buildtool/util",
+ "//src/test/java/com/google/devtools/build/skyframe:testutil",
+ "//third_party:guava",
+ "//third_party:junit4",
+ "//third_party:truth",
+ ],
+)

src/test/java/com/google/devtools/build/lib/skyframe/LocalDiffAwarenessIntegrationTest.java

@@ -0,0 +1,180 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+import static org.junit.Assert.assertThrows;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.buildtool.util.SkyframeIntegrationTestBase;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.runtime.WorkspaceBuilder;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.vfs.DelegateFileSystem;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.NotifyingHelper;
+import com.google.devtools.common.options.OptionsBase;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.After;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for local diff awareness. A good place for general tests of Bazel's interactions with
+ * "smart" filesystems, so that open-source changes don't break Google-internal features around
+ * smart filesystems.
+ */
+@RunWith(JUnit4.class)
+public class LocalDiffAwarenessIntegrationTest extends SkyframeIntegrationTestBase {
+ private final Map<PathFragment, IOException> throwOnNextStatIfFound = new HashMap<>();
+
+ @Override
+ protected BlazeRuntime.Builder getRuntimeBuilder() throws Exception {
+ return super.getRuntimeBuilder()
+ .addBlazeModule(
+ new BlazeModule() {
+ @Override
+ public void workspaceInit(
+ BlazeRuntime runtime, BlazeDirectories directories, WorkspaceBuilder builder) {
+ builder.addDiffAwarenessFactory(new LocalDiffAwareness.Factory(ImmutableList.of()));
+ }
+
+ @Override
+ public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
+ return ImmutableList.of(LocalDiffAwareness.Options.class);
+ }
+ });
+ }
+
+ @Override
+ public FileSystem createFileSystem() throws Exception {
+ return new DelegateFileSystem(super.createFileSystem()) {
+ @Override
+ protected FileStatus statIfFound(PathFragment path, boolean followSymlinks)
+ throws IOException {
+ IOException e = throwOnNextStatIfFound.remove(path);
+ if (e != null) {
+ throw e;
+ }
+ return super.statIfFound(path, followSymlinks);
+ }
+ };
+ }
+
+ @Before
+ public void addOptions() {
+ addOptions("--watchfs", "--experimental_windows_watchfs");
+ }
+
+ @After
+ public void checkExceptionsThrown() {
+ assertWithMessage("Injected exception(s) not thrown").that(throwOnNextStatIfFound).isEmpty();
+ }
+
+ @Test
+ public void changedFile_detectsChange() throws Exception {
+ // TODO(bazel-team): Understand why these tests are flaky on Mac. Probably real watchfs bug?
+ Assume.assumeFalse(OS.DARWIN.equals(OS.getCurrent()));
+ write("foo/BUILD", "genrule(name='foo', outs=['out'], cmd='echo hello > $@')");
+ buildTarget("//foo");
+ assertContents("hello", "//foo");
+ write("foo/BUILD", "genrule(name='foo', outs=['out'], cmd='echo there > $@')");
+
+ buildTarget("//foo");
+
+ assertContents("there", "//foo");
+ }
+
+ @Test
+ public void changedFile_statFails_throwsError() throws Exception {
+ Assume.assumeFalse(OS.DARWIN.equals(OS.getCurrent()));
+ write("foo/BUILD", "genrule(name='foo', outs=['out'], cmd='echo hello > $@')");
+ buildTarget("//foo");
+ assertContents("hello", "//foo");
+ Path buildFile = write("foo/BUILD", "genrule(name='foo', outs=['out'], cmd='echo there > $@')");
+ IOException injectedException = new IOException("oh no!");
+ throwOnNextStatIfFound.put(buildFile.asFragment(), injectedException);
+
+ AbruptExitException e = assertThrows(AbruptExitException.class, () -> buildTarget("//foo"));
+
+ assertThat(e.getCause()).hasCauseThat().hasCauseThat().isSameInstanceAs(injectedException);
+ }
+
+ @Test
+ public void externalSymlink_doesNotTriggerFullGraphTraversal() throws Exception {
+ addOptions("--symlink_prefix=/");
+ AtomicInteger calledGetValues = new AtomicInteger(0);
+ skyframeExecutor()
+ .getEvaluator()
+ .injectGraphTransformerForTesting(
+ NotifyingHelper.makeNotifyingTransformer(
+ (key, type, order, context) -> {
+ if (type == NotifyingHelper.EventType.GET_VALUES) {
+ calledGetValues.incrementAndGet();
+ }
+ }));
+ write(
+ "hello/BUILD",
+ "genrule(name='target', srcs = ['external'], outs=['out'], cmd='/bin/cat $(SRCS) > $@')");
+ String externalLink = System.getenv("TEST_TMPDIR") + "/target";
+ write(externalLink, "one");
+ createSymlink(externalLink, "hello/external");
+
+ // Trivial build: external symlink is not seen, so normal diff awareness is in play.
+ buildTarget("//hello:BUILD");
+ // New package path on first build triggers full-graph work.
+ calledGetValues.set(0);
+ // getValues() called during output file checking (although if an output service is able to
+ // report modified files in practice there is no iteration).
+ // If external repositories are being used, getValues called because of that too.
+ // TODO(bazel-team): get rid of this when we can disable checks for external repositories.
+ int numGetValuesInFullDiffAwarenessBuild =
+ 1 + ("bazel".equals(this.getRuntime().getProductName()) ? 1 : 0);
+
+ buildTarget("//hello:BUILD");
+ assertThat(calledGetValues.getAndSet(0)).isEqualTo(numGetValuesInFullDiffAwarenessBuild);
+
+ // Now bring the external symlink into Bazel's awareness.
+ buildTarget("//hello:target");
+ assertContents("one", "//hello:target");
+ assertThat(calledGetValues.getAndSet(0)).isEqualTo(numGetValuesInFullDiffAwarenessBuild);
+
+ // Builds that follow a build containing an external file don't trigger a traversal.
+ buildTarget("//hello:target");
+ assertContents("one", "//hello:target");
+ assertThat(calledGetValues.getAndSet(0)).isEqualTo(numGetValuesInFullDiffAwarenessBuild);
+
+ write(externalLink, "two");
+
+ buildTarget("//hello:target");
+ // External file changes are tracked.
+ assertContents("two", "//hello:target");
+ assertThat(calledGetValues.getAndSet(0)).isEqualTo(numGetValuesInFullDiffAwarenessBuild);
+ }
+}