Skip to content

Commit

Permalink
Implement Hermetic sandbox with support for hardlinks
Browse files Browse the repository at this point in the history
Adds linux-sandbox flag:
--experimental_use_hermetic_linux_sandbox - Configure linux-sandbox
 to run in a chroot environment to prevent access to files not
 mentioned in the bazel rules unless they can be found via
 explicitly whitelisted directories using --sandbox_add_mount_pair
 create hardlinks instead of symlinks, and fallback to copying.
 In case of writes to input files, the build will be aborted.
  • Loading branch information
frazze-jobb committed Aug 10, 2021
1 parent 238367a commit ba775b2
Show file tree
Hide file tree
Showing 14 changed files with 683 additions and 49 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -20,27 +20,33 @@

/**
* In case we can't get a fast digest from the filesystem, we store this metadata as a proxy to the
* file contents. Currently it is a pair of a relevant timestamp and a "node id". On Linux the
* former is the ctime and the latter is the inode number. We might want to add the device number in
* file contents. Currently it is two timestamps and a "node id". On Linux we
* use both ctime and mtime and inode number. We might want to add the device number in
* the future.
*
* <p>For a Linux example of why mtime alone is insufficient, note that 'mv' preserves timestamps.
* <p>For a Linux example of why mtime alone is insufficient, note that 'mv' preserves mtime.
* So if files 'a' and 'b' initially have the same timestamp, then we would think 'b' is unchanged
* after the user executes `mv a b` between two builds.
*
* <p>On Linux we also need mtime for hardlinking sandbox, since updating the inode reference counter
* preserves mtime, but updates ctime. isModified() call can be used to compare two FileContentsProxys
* of hardlinked files.
*/
public final class FileContentsProxy {
private final long ctime;
private final long mtime;
private final long nodeId;

private FileContentsProxy(long ctime, long nodeId) {
public FileContentsProxy(long ctime, long mtime, long nodeId) {
this.ctime = ctime;
this.mtime = mtime;
this.nodeId = nodeId;
}

public static FileContentsProxy create(FileStatus stat) throws IOException {
// Note: there are file systems that return mtime for this call instead of ctime, such as the
// WindowsFileSystem.
return new FileContentsProxy(stat.getLastChangeTime(), stat.getNodeId());
return new FileContentsProxy(stat.getLastChangeTime(), stat.getLastModifiedTime(), stat.getNodeId());
}

@Override
Expand All @@ -54,16 +60,31 @@ public boolean equals(Object other) {
}

FileContentsProxy that = (FileContentsProxy) other;
return ctime == that.ctime && nodeId == that.nodeId;
return ctime == that.ctime && mtime == that.mtime && nodeId == that.nodeId;
}

/**
* Can be used when hardlink reference counter changes
* should not be considered a file modification.
* Is only comparing mtime and not ctime and is therefore
* not detecting changed metadata like permission.
*/
public boolean isModified(FileContentsProxy other) {
if (other == this) {
return false;
}
// true if nodeId are different or inode has a new mtime
return nodeId != other.nodeId || mtime != other.mtime;
}

@Override
public int hashCode() {
return Objects.hash(ctime, nodeId);
return Objects.hash(ctime, mtime, nodeId);
}

void addToFingerprint(Fingerprint fp) {
fp.addLong(ctime);
fp.addLong(mtime);
fp.addLong(nodeId);
}

Expand All @@ -73,6 +94,6 @@ public String toString() {
}

public String prettyPrint() {
return String.format("ctime of %d and nodeId of %d", ctime, nodeId);
return String.format("ctime of %d and mtime of %d and nodeId of %d", ctime, mtime, nodeId);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,9 @@ private SpawnResult runSpawn(
try (SilentCloseable c = Profiler.instance().profile("subprocess.run")) {
result = run(originalSpawn, sandbox, context.getTimeout(), outErr);
}
try (SilentCloseable c = Profiler.instance().profile("sandbox.verifyPostCondition")) {
verifyPostCondition(originalSpawn, sandbox, context);
}

context.lockOutputFiles();
try (SilentCloseable c = Profiler.instance().profile("sandbox.copyOutputs")) {
Expand All @@ -148,6 +151,11 @@ private SpawnResult runSpawn(
}
}
}
/**
* Override this method if you need to run a post condition after the action has executed
*/
public void verifyPostCondition(Spawn originalSpawn, SandboxedSpawn sandbox,
SpawnExecutionContext context) throws IOException, ForbiddenActionInputException {}

private String makeFailureMessage(Spawn originalSpawn, SandboxedSpawn sandbox) {
if (sandboxOptions.sandboxDebug) {
Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/google/devtools/build/lib/sandbox/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/actions:artifacts",
"//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
"//src/main/java/com/google/devtools/build/lib/actions:execution_requirements",
"//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
"//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.sandbox;

import com.google.common.flogger.GoogleLogger;
import com.google.devtools.build.lib.exec.TreeDeleter;
import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxInputs;
import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxOutputs;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.Symlinks;

import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
* Creates an execRoot for a Spawn that contains input files as hardlinks to their original
* destination.
*/
public class HardlinkedSandboxedSpawn extends AbstractContainerizingSandboxedSpawn {
private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
private boolean sandboxDebug = false;
public HardlinkedSandboxedSpawn(
Path sandboxPath,
Path sandboxExecRoot,
List<String> arguments,
Map<String, String> environment,
SandboxInputs inputs,
SandboxOutputs outputs,
Set<Path> writableDirs,
TreeDeleter treeDeleter,
@Nullable Path statisticsPath,
boolean sandboxDebug) {
super(
sandboxPath,
sandboxExecRoot,
arguments,
environment,
inputs,
outputs,
writableDirs,
treeDeleter,
statisticsPath);
this.sandboxDebug = sandboxDebug;
}

@Override
protected void copyFile(Path source, Path target) throws IOException {
hardLinkRecursive(source, target);
}

/**
* Recursively creates hardlinks for all files in @param source path, in @param target path.
* Symlinks are resolved. If files is located on another disk, hardlink will fail
* and a copy will be made instead.
* Throws IllegalArgumentException if source path is a subdirectory of target path.
*/
private void hardLinkRecursive(Path source, Path target) throws IOException {
if (source.isSymbolicLink()) {
source = source.resolveSymbolicLinks();
}

if (source.isFile(Symlinks.NOFOLLOW)) {
try {
source.createHardLink(target);
} catch (IOException e) {
if (sandboxDebug) {
logger.atInfo().log("File %s could not be hardlinked, file will be copied instead.", source);
}
FileSystemUtils.copyFile(source, target);
}
} else if (source.isDirectory()) {
if (source.startsWith(target)) {
throw new IllegalArgumentException(source + " is a subdirectory of " + target);
}
target.createDirectory();
Collection<Path> entries = source.getDirectoryEntries();
for (Path entry : entries) {
Path toPath = target.getChild(entry.getBaseName());
hardLinkRecursive(entry, toPath);
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public static CommandLineBuilder commandLineBuilder(
public static class CommandLineBuilder {
private final Path linuxSandboxPath;
private final List<String> commandArguments;

private Path hermeticSandboxPath;
private Path workingDirectory;
private Duration timeout;
private Duration killDelay;
Expand All @@ -79,6 +79,13 @@ private CommandLineBuilder(Path linuxSandboxPath, List<String> commandArguments)
this.commandArguments = commandArguments;
}

/** Sets the sandbox path to chroot to, required for the hermetic linux sandbox to figure out
where the working directory is. */
public CommandLineBuilder setHermeticSandboxPath(Path sandboxPath) {
this.hermeticSandboxPath = sandboxPath;
return this;
}

/** Sets the working directory to use, if any. */
public CommandLineBuilder setWorkingDirectory(Path workingDirectory) {
this.workingDirectory = workingDirectory;
Expand Down Expand Up @@ -221,6 +228,9 @@ public ImmutableList<String> build() {
if (statisticsPath != null) {
commandLineBuilder.add("-S", statisticsPath.getPathString());
}
if (hermeticSandboxPath != null) {
commandLineBuilder.add("-h", hermeticSandboxPath.getPathString());
}
if (useFakeHostname) {
commandLineBuilder.add("-H");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,16 @@
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.io.ByteStreams;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.ExecutionRequirements;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.FileContentsProxy;
import com.google.devtools.build.lib.actions.ForbiddenActionInputException;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.Spawns;
import com.google.devtools.build.lib.actions.UserExecException;
import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.exec.TreeDeleter;
import com.google.devtools.build.lib.exec.local.LocalEnvProvider;
Expand All @@ -38,17 +42,19 @@
import com.google.devtools.build.lib.shell.Command;
import com.google.devtools.build.lib.shell.CommandException;
import com.google.devtools.build.lib.util.OS;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Symlinks;

import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
import java.util.SortedMap;
import javax.annotation.Nullable;

/** Spawn runner that uses linux sandboxing APIs to execute a local subprocess. */
final class LinuxSandboxedSpawnRunner extends AbstractSandboxSpawnRunner {
Expand Down Expand Up @@ -229,6 +235,19 @@ spawn, getSandboxOptions().defaultSandboxAllowNetwork)))
sandboxfsMapSymlinkTargets,
treeDeleter,
statisticsPath);
} else if (getSandboxOptions().useHermetic) {
commandLineBuilder.setHermeticSandboxPath(sandboxPath);
return new HardlinkedSandboxedSpawn(
sandboxPath,
sandboxExecRoot,
commandLineBuilder.build(),
environment,
inputs,
outputs,
writableDirs,
treeDeleter,
statisticsPath,
getSandboxOptions().sandboxDebug);
} else {
return new SymlinkedSandboxedSpawn(
sandboxPath,
Expand Down Expand Up @@ -355,6 +374,42 @@ private void validateBindMounts(SortedMap<Path, Path> bindMounts) throws UserExe
}
}
}
@Override
public void verifyPostCondition(
Spawn originalSpawn, SandboxedSpawn sandbox, SpawnExecutionContext context) throws IOException, ForbiddenActionInputException {
if(getSandboxOptions().useHermetic){
checkForConcurrentModifications(context);
}
}

private void checkForConcurrentModifications(SpawnExecutionContext context) throws IOException, ForbiddenActionInputException {
for (ActionInput input : (context.getInputMapping(PathFragment.EMPTY_FRAGMENT).values())) {
if (input instanceof VirtualActionInput) {
continue;
}

FileArtifactValue metadata = context.getMetadataProvider().getMetadata(input);
Path path = execRoot.getRelative(input.getExecPath());

try {
if (wasModifiedSinceDigest(metadata.getContentsProxy(), path)) {
throw new IOException("input dependency " + path + " was modified during execution.");
}
} catch (UnsupportedOperationException e) {
throw new IOException(
"input dependency " + path + " could not be checked for modifications during execution.",
e);
}
}
}

private boolean wasModifiedSinceDigest(FileContentsProxy proxy, Path path) throws IOException {
if (proxy == null) {
return false;
}
FileStatus stat = path.statIfFound(Symlinks.FOLLOW);
return stat == null || !stat.isFile() || proxy.isModified(FileContentsProxy.create(stat));
}

@Override
public void cleanupSandboxBase(Path sandboxBase, TreeDeleter treeDeleter) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,19 @@ public ImmutableSet<Path> getInaccessiblePaths(FileSystem fs) {
+ " avoid unnecessary setup costs.")
public boolean reuseSandboxDirectories;

@Option(
name = "experimental_use_hermetic_linux_sandbox",
defaultValue = "false",
documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
effectTags = {OptionEffectTag.EXECUTION},
help =
"If set to true, do not mount root, only mount whats provided with "
+ "sandbox_add_mount_pair. Input files will be hardlinked to the sandbox instead of "
+ "symlinked to from the sandbox. "
+ "If action input files are located on a filesystem different from the sandbox, "
+ "then the input files will be copied instead.")
public boolean useHermetic;

/** Converter for the number of threads used for asynchronous tree deletion. */
public static final class AsyncTreeDeletesConverter extends ResourceConverter {
public AsyncTreeDeletesConverter() {
Expand Down
Loading

0 comments on commit ba775b2

Please sign in to comment.