Implement Hermetic sandbox with support for hardlinks

Adds linux-sandbox flag:
--experimental_use_hermetic_linux_sandbox - Configure linux-sandbox
 to run in a chroot environment to prevent access to files not
 mentioned in the bazel rules unless they can be found via
 explicitly whitelisted directories using --sandbox_add_mount_pair
 create hardlinks instead of symlinks, and fallback to copying.
 In case of writes to input files, the build will be aborted.

src/main/java/com/google/devtools/build/lib/actions/FileContentsProxy.java

@@ -20,27 +20,33 @@
 
 /**
  * In case we can't get a fast digest from the filesystem, we store this metadata as a proxy to the
- * file contents. Currently it is a pair of a relevant timestamp and a "node id". On Linux the
- * former is the ctime and the latter is the inode number. We might want to add the device number in
+ * file contents. Currently it is two timestamps and a "node id". On Linux we
+ * use both ctime and mtime and inode number. We might want to add the device number in
  * the future.
  *
- * <p>For a Linux example of why mtime alone is insufficient, note that 'mv' preserves timestamps.
+ * <p>For a Linux example of why mtime alone is insufficient, note that 'mv' preserves mtime.
  * So if files 'a' and 'b' initially have the same timestamp, then we would think 'b' is unchanged
  * after the user executes `mv a b` between two builds.
+ *
+ * <p>On Linux we also need mtime for hardlinking sandbox, since updating the inode reference counter
+ * preserves mtime, but updates ctime. isModified() call can be used to compare two FileContentsProxys
+ * of hardlinked files.
  */
 public final class FileContentsProxy {
  private final long ctime;
+ private final long mtime;
  private final long nodeId;
 
- private FileContentsProxy(long ctime, long nodeId) {
+ public FileContentsProxy(long ctime, long mtime, long nodeId) {
  this.ctime = ctime;
+ this.mtime = mtime;
  this.nodeId = nodeId;
  }
 
  public static FileContentsProxy create(FileStatus stat) throws IOException {
  // Note: there are file systems that return mtime for this call instead of ctime, such as the
  // WindowsFileSystem.
- return new FileContentsProxy(stat.getLastChangeTime(), stat.getNodeId());
+ return new FileContentsProxy(stat.getLastChangeTime(), stat.getLastModifiedTime(), stat.getNodeId());
  }
 
  @Override
@@ -54,16 +60,31 @@ public boolean equals(Object other) {
  }
 
  FileContentsProxy that = (FileContentsProxy) other;
- return ctime == that.ctime && nodeId == that.nodeId;
+ return ctime == that.ctime && mtime == that.mtime && nodeId == that.nodeId;
+ }
+
+ /**
+ * Can be used when hardlink reference counter changes
+ * should not be considered a file modification.
+ * Is only comparing mtime and not ctime and is therefore
+ * not detecting changed metadata like permission.
+ */
+ public boolean isModified(FileContentsProxy other) {
+ if (other == this) {
+ return false;
+ }
+ // true if nodeId are different or inode has a new mtime
+ return nodeId != other.nodeId || mtime != other.mtime;
  }
 
  @Override
  public int hashCode() {
- return Objects.hash(ctime, nodeId);
+ return Objects.hash(ctime, mtime, nodeId);
  }
 
  void addToFingerprint(Fingerprint fp) {
  fp.addLong(ctime);
+ fp.addLong(mtime);
  fp.addLong(nodeId);
  }
 
@@ -73,6 +94,6 @@ public String toString() {
  }
 
  public String prettyPrint() {
- return String.format("ctime of %d and nodeId of %d", ctime, nodeId);
+ return String.format("ctime of %d and mtime of %d and nodeId of %d", ctime, mtime, nodeId);
  }
 }

src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java

@@ -131,6 +131,9 @@ private SpawnResult runSpawn(
  try (SilentCloseable c = Profiler.instance().profile("subprocess.run")) {
  result = run(originalSpawn, sandbox, context.getTimeout(), outErr);
  }
+ try (SilentCloseable c = Profiler.instance().profile("sandbox.verifyPostCondition")) {
+ verifyPostCondition(originalSpawn, sandbox, context);
+ }
 
  context.lockOutputFiles();
  try (SilentCloseable c = Profiler.instance().profile("sandbox.copyOutputs")) {
@@ -148,6 +151,11 @@ private SpawnResult runSpawn(
  }
  }
  }
+ /**
+ * Override this method if you need to run a post condition after the action has executed
+ */
+ public void verifyPostCondition(Spawn originalSpawn, SandboxedSpawn sandbox,
+ SpawnExecutionContext context) throws IOException, ForbiddenActionInputException {}
 
  private String makeFailureMessage(Spawn originalSpawn, SandboxedSpawn sandbox) {
  if (sandboxOptions.sandboxDebug) {

src/main/java/com/google/devtools/build/lib/sandbox/BUILD

@@ -18,6 +18,7 @@ java_library(
  "//src/main/java/com/google/devtools/build/lib:runtime",
  "//src/main/java/com/google/devtools/build/lib/actions",
  "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
+ "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
  "//src/main/java/com/google/devtools/build/lib/actions:execution_requirements",
  "//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
  "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",

src/main/java/com/google/devtools/build/lib/sandbox/HardlinkedSandboxedSpawn.java

@@ -0,0 +1,100 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.sandbox;
+
+import com.google.common.flogger.GoogleLogger;
+import com.google.devtools.build.lib.exec.TreeDeleter;
+import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxInputs;
+import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxOutputs;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.Symlinks;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Creates an execRoot for a Spawn that contains input files as hardlinks to their original
+ * destination.
+ */
+public class HardlinkedSandboxedSpawn extends AbstractContainerizingSandboxedSpawn {
+ private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
+ private boolean sandboxDebug = false;
+ public HardlinkedSandboxedSpawn(
+ Path sandboxPath,
+ Path sandboxExecRoot,
+ List<String> arguments,
+ Map<String, String> environment,
+ SandboxInputs inputs,
+ SandboxOutputs outputs,
+ Set<Path> writableDirs,
+ TreeDeleter treeDeleter,
+ @Nullable Path statisticsPath,
+ boolean sandboxDebug) {
+ super(
+ sandboxPath,
+ sandboxExecRoot,
+ arguments,
+ environment,
+ inputs,
+ outputs,
+ writableDirs,
+ treeDeleter,
+ statisticsPath);
+ this.sandboxDebug = sandboxDebug;
+ }
+
+ @Override
+ protected void copyFile(Path source, Path target) throws IOException {
+ hardLinkRecursive(source, target);
+ }
+
+ /**
+ * Recursively creates hardlinks for all files in @param source path, in @param target path.
+ * Symlinks are resolved. If files is located on another disk, hardlink will fail
+ * and a copy will be made instead.
+ * Throws IllegalArgumentException if source path is a subdirectory of target path.
+ */
+ private void hardLinkRecursive(Path source, Path target) throws IOException {
+ if (source.isSymbolicLink()) {
+ source = source.resolveSymbolicLinks();
+ }
+
+ if (source.isFile(Symlinks.NOFOLLOW)) {
+ try {
+ source.createHardLink(target);
+ } catch (IOException e) {
+ if (sandboxDebug) {
+ logger.atInfo().log("File %s could not be hardlinked, file will be copied instead.", source);
+ }
+ FileSystemUtils.copyFile(source, target);
+ }
+ } else if (source.isDirectory()) {
+ if (source.startsWith(target)) {
+ throw new IllegalArgumentException(source + " is a subdirectory of " + target);
+ }
+ target.createDirectory();
+ Collection<Path> entries = source.getDirectoryEntries();
+ for (Path entry : entries) {
+ Path toPath = target.getChild(entry.getBaseName());
+ hardLinkRecursive(entry, toPath);
+ }
+ }
+ }
+}

src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxUtil.java

@@ -57,7 +57,7 @@ public static CommandLineBuilder commandLineBuilder(
  public static class CommandLineBuilder {
  private final Path linuxSandboxPath;
  private final List<String> commandArguments;
-
+ private Path hermeticSandboxPath;
  private Path workingDirectory;
  private Duration timeout;
  private Duration killDelay;
@@ -79,6 +79,13 @@ private CommandLineBuilder(Path linuxSandboxPath, List<String> commandArguments)
  this.commandArguments = commandArguments;
  }
 
+ /** Sets the sandbox path to chroot to, required for the hermetic linux sandbox to figure out
+ where the working directory is. */
+ public CommandLineBuilder setHermeticSandboxPath(Path sandboxPath) {
+ this.hermeticSandboxPath = sandboxPath;
+ return this;
+ }
+
  /** Sets the working directory to use, if any. */
  public CommandLineBuilder setWorkingDirectory(Path workingDirectory) {
  this.workingDirectory = workingDirectory;
@@ -221,6 +228,9 @@ public ImmutableList<String> build() {
  if (statisticsPath != null) {
  commandLineBuilder.add("-S", statisticsPath.getPathString());
  }
+ if (hermeticSandboxPath != null) {
+ commandLineBuilder.add("-h", hermeticSandboxPath.getPathString());
+ }
  if (useFakeHostname) {
  commandLineBuilder.add("-H");
  }

src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java

@@ -19,12 +19,16 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
 import com.google.common.io.ByteStreams;
+import com.google.devtools.build.lib.actions.ActionInput;
 import com.google.devtools.build.lib.actions.ExecException;
 import com.google.devtools.build.lib.actions.ExecutionRequirements;
+import com.google.devtools.build.lib.actions.FileArtifactValue;
+import com.google.devtools.build.lib.actions.FileContentsProxy;
 import com.google.devtools.build.lib.actions.ForbiddenActionInputException;
 import com.google.devtools.build.lib.actions.Spawn;
 import com.google.devtools.build.lib.actions.Spawns;
 import com.google.devtools.build.lib.actions.UserExecException;
+import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
 import com.google.devtools.build.lib.analysis.BlazeDirectories;
 import com.google.devtools.build.lib.exec.TreeDeleter;
 import com.google.devtools.build.lib.exec.local.LocalEnvProvider;
@@ -38,17 +42,19 @@
 import com.google.devtools.build.lib.shell.Command;
 import com.google.devtools.build.lib.shell.CommandException;
 import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.vfs.FileStatus;
 import com.google.devtools.build.lib.vfs.FileSystem;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.devtools.build.lib.vfs.Symlinks;
+
+import javax.annotation.Nullable;
 import java.io.File;
 import java.io.IOException;
 import java.time.Duration;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.SortedMap;
-import javax.annotation.Nullable;
 
 /** Spawn runner that uses linux sandboxing APIs to execute a local subprocess. */
 final class LinuxSandboxedSpawnRunner extends AbstractSandboxSpawnRunner {
@@ -229,6 +235,19 @@ spawn, getSandboxOptions().defaultSandboxAllowNetwork)))
  sandboxfsMapSymlinkTargets,
  treeDeleter,
  statisticsPath);
+ } else if (getSandboxOptions().useHermetic) {
+ commandLineBuilder.setHermeticSandboxPath(sandboxPath);
+ return new HardlinkedSandboxedSpawn(
+ sandboxPath,
+ sandboxExecRoot,
+ commandLineBuilder.build(),
+ environment,
+ inputs,
+ outputs,
+ writableDirs,
+ treeDeleter,
+ statisticsPath,
+ getSandboxOptions().sandboxDebug);
  } else {
  return new SymlinkedSandboxedSpawn(
  sandboxPath,
@@ -352,6 +371,42 @@ private void validateBindMounts(SortedMap<Path, Path> bindMounts) throws UserExe
  }
  }
  }
+ @Override
+ public void verifyPostCondition(
+ Spawn originalSpawn, SandboxedSpawn sandbox, SpawnExecutionContext context) throws IOException, ForbiddenActionInputException {
+ if(getSandboxOptions().useHermetic){
+ checkForConcurrentModifications(context);
+ }
+ }
+
+ private void checkForConcurrentModifications(SpawnExecutionContext context) throws IOException, ForbiddenActionInputException {
+ for (ActionInput input : (context.getInputMapping(PathFragment.EMPTY_FRAGMENT).values())) {
+ if (input instanceof VirtualActionInput) {
+ continue;
+ }
+
+ FileArtifactValue metadata = context.getMetadataProvider().getMetadata(input);
+ Path path = execRoot.getRelative(input.getExecPath());
+
+ try {
+ if (wasModifiedSinceDigest(metadata.getContentsProxy(), path)) {
+ throw new IOException("input dependency " + path + " was modified during execution.");
+ }
+ } catch (UnsupportedOperationException e) {
+ throw new IOException(
+ "input dependency " + path + " could not be checked for modifications during execution.",
+ e);
+ }
+ }
+ }
+
+ private boolean wasModifiedSinceDigest(FileContentsProxy proxy, Path path) throws IOException {
+ if (proxy == null) {
+ return false;
+ }
+ FileStatus stat = path.statIfFound(Symlinks.FOLLOW);
+ return stat == null || !stat.isFile() || proxy.isModified(FileContentsProxy.create(stat));
+ }
 
  @Override
  public void cleanupSandboxBase(Path sandboxBase, TreeDeleter treeDeleter) throws IOException {

src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java

@@ -361,6 +361,19 @@ public ImmutableSet<Path> getInaccessiblePaths(FileSystem fs) {
  + " instead.")
  public boolean legacyLocalFallback;
 
+ @Option(
+ name = "experimental_use_hermetic_linux_sandbox",
+ defaultValue = "false",
+ documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
+ effectTags = {OptionEffectTag.EXECUTION},
+ help =
+ "If set to true, do not mount root, only mount whats provided with "
+ + "sandbox_add_mount_pair. Input files will be hardlinked to the sandbox instead of "
+ + "symlinked to from the sandbox. "
+ + "If action input files are located on a filesystem different from the sandbox, "
+ + "then the input files will be copied instead.")
+ public boolean useHermetic;
+
  /** Converter for the number of threads used for asynchronous tree deletion. */
  public static final class AsyncTreeDeletesConverter extends ResourceConverter {
  public AsyncTreeDeletesConverter() {