Vendor registry files

site/en/external/vendor.md

@@ -67,6 +67,23 @@ Under the hood, it's doing a `bazel build --nobuild` command to analyze the
 target patterns, therefore build flags could be applied to this command and
 affect the result.
 
+### Build the target offline {:#build-the-target-offline}
+
+With the external dependencies vendored, you can build the target offline by
+
+```none
+bazel build --vendor_dir=vendor_src //src/main:hello-world //src/test/...
+```
+
+The build should work in a clean build environment without network access and
+repository cache.
+
+Therefore, you should be able to check in the vendored source and build the same
+targets offline on another machine.
+
+Note: If you build different targets or change the external dependencies, build
+configuration, or Bazel version, you may need to re-vendor.
+
 ## Vendor all external dependencies {:#vendor-all-dependencies}
 
 To vendor all repos in your transitive external dependencies graph, you can

src/main/java/com/google/devtools/build/lib/bazel/BUILD

@@ -34,6 +34,7 @@ java_library(
  "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution",
  "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution_impl",
  "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:tidy_impl",
+ "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:vendor",
  "//src/main/java/com/google/devtools/build/lib/bazel/commands",
  "//src/main/java/com/google/devtools/build/lib/bazel/repository",
  "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options",

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -165,7 +165,7 @@ public class BazelRepositoryModule extends BlazeModule {
  private Clock clock;
  private Instant lastRegistryInvalidation = Instant.EPOCH;
 
- private Optional<Path> vendorDirectory;
+ private Optional<Path> vendorDirectory = Optional.empty();
  private List<String> allowedYankedVersions = ImmutableList.of();
  private boolean disableNativeRepoRules;
  private SingleExtensionEvalFunction singleExtensionEvalFunction;
@@ -224,7 +224,7 @@ public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builde
  builder.addCommands(new FetchCommand());
  builder.addCommands(new ModCommand());
  builder.addCommands(new SyncCommand());
- builder.addCommands(new VendorCommand());
+ builder.addCommands(new VendorCommand(downloadManager, clientEnvironmentSupplier));
  builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache));
  }
 
@@ -495,15 +495,9 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
  bazelCompatibilityMode = repoOptions.bazelCompatibilityMode;
  bazelLockfileMode = repoOptions.lockfileMode;
  allowedYankedVersions = repoOptions.allowedYankedVersions;
-
- if (repoOptions.vendorDirectory != null) {
- vendorDirectory =
- Optional.of(
- repoOptions.vendorDirectory.isAbsolute()
- ? filesystem.getPath(repoOptions.vendorDirectory)
- : env.getWorkspace().getRelative(repoOptions.vendorDirectory));
- } else {
- vendorDirectory = Optional.empty();
+ if (env.getWorkspace() != null) {
+ vendorDirectory = Optional.ofNullable(repoOptions.vendorDirectory)
+ .map(vendorDirectory -> env.getWorkspace().getRelative(vendorDirectory));
  }
 
  if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) {

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD

@@ -46,6 +46,17 @@ java_library(
  ],
 )
 
+java_library(
+ name = "vendor",
+ srcs = ["VendorManager.java"],
+ deps = [
+ "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
+ "//src/main/java/com/google/devtools/build/lib/cmdline",
+ "//src/main/java/com/google/devtools/build/lib/vfs",
+ "//third_party:guava",
+ ],
+)
+
 java_library(
  name = "module_extension",
  srcs = [
@@ -80,13 +91,15 @@ java_library(
  ],
  deps = [
  ":common",
+ ":vendor",
  ":yanked_versions_value",
  "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options",
  "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",
  "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
  "//src/main/java/com/google/devtools/build/lib/events",
  "//src/main/java/com/google/devtools/build/lib/profiler",
  "//src/main/java/com/google/devtools/build/lib/util:os",
+ "//src/main/java/com/google/devtools/build/lib/vfs",
  "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
  "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
  "//third_party:gson",

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java

@@ -48,7 +48,7 @@ public abstract class BazelModuleResolutionValue implements SkyValue {
  /**
  * Hashes of files obtained (or known to be missing) from registries while performing resolution.
  */
- abstract ImmutableMap<String, Optional<Checksum>> getRegistryFileHashes();
+ public abstract ImmutableMap<String, Optional<Checksum>> getRegistryFileHashes();
 
  /**
  * Selected module versions that are known to be yanked (and hence must have been explicitly

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java

@@ -32,6 +32,7 @@
 import com.google.devtools.build.lib.profiler.ProfilerTask;
 import com.google.devtools.build.lib.profiler.SilentCloseable;
 import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.gson.FieldNamingPolicy;
 import com.google.gson.Gson;
@@ -46,6 +47,8 @@
 import java.util.Map.Entry;
 import java.util.Optional;
 
+import javax.annotation.Nullable;
+
 /**
  * Represents a Bazel module registry that serves a list of module metadata from a static HTTP
  * server or a local file path.
@@ -88,6 +91,8 @@ public enum KnownFileHashesMode {
  private final Gson gson;
  private final ImmutableMap<String, Optional<Checksum>> knownFileHashes;
  private final ImmutableMap<ModuleKey, String> previouslySelectedYankedVersions;
+ @Nullable
+ private final VendorManager vendorUtil;
  private final KnownFileHashesMode knownFileHashesMode;
  private volatile Optional<BazelRegistryJson> bazelRegistryJson;
  private volatile StoredEventHandler bazelRegistryJsonEvents;
@@ -100,7 +105,8 @@ public IndexRegistry(
  Map<String, String> clientEnv,
  ImmutableMap<String, Optional<Checksum>> knownFileHashes,
  KnownFileHashesMode knownFileHashesMode,
- ImmutableMap<ModuleKey, String> previouslySelectedYankedVersions) {
+ ImmutableMap<ModuleKey, String> previouslySelectedYankedVersions,
+ Optional<Path> vendorDir) {
  this.uri = uri;
  this.downloadManager = downloadManager;
  this.clientEnv = clientEnv;
@@ -111,6 +117,7 @@ public IndexRegistry(
  this.knownFileHashes = knownFileHashes;
  this.knownFileHashesMode = knownFileHashesMode;
  this.previouslySelectedYankedVersions = previouslySelectedYankedVersions;
+ this.vendorUtil = vendorDir.map(VendorManager::new).orElse(null);
  }
 
  @Override
@@ -143,19 +150,19 @@ private Optional<byte[]> grabFile(
  }
 
  private Optional<byte[]> doGrabFile(
- String url, ExtendedEventHandler eventHandler, boolean useChecksum)
+ String rawUrl, ExtendedEventHandler eventHandler, boolean useChecksum)
  throws IOException, InterruptedException {
  Optional<Checksum> checksum;
  if (knownFileHashesMode != KnownFileHashesMode.IGNORE && useChecksum) {
- Optional<Checksum> knownChecksum = knownFileHashes.get(url);
+ Optional<Checksum> knownChecksum = knownFileHashes.get(rawUrl);
  if (knownChecksum == null) {
  if (knownFileHashesMode == KnownFileHashesMode.ENFORCE) {
  throw new MissingChecksumException(
  String.format(
  "Missing checksum for registry file %s not permitted with --lockfile_mode=error."
  + " Please run `bazel mod deps --lockfile_mode=update` to update your"
  + " lockfile.",
- url));
+ rawUrl));
  }
  // This is a new file, download without providing a checksum.
  checksum = Optional.empty();
@@ -182,17 +189,37 @@ private Optional<byte[]> doGrabFile(
  "Cannot fetch a file without a checksum in ENFORCE mode. This is a bug in Bazel, please "
  + "report at https://github.com/bazelbuild/bazel/issues/new/choose.");
  }
+
+ URL url = URI.create(rawUrl).toURL();
+ // Don't read the registry URL from the vendor directory in the following cases:
+ // 1. vendorUtil is null, which means vendor mode is disabled.
+ // 2. The checksum is not present, which means the URL is not vendored or the vendored content
+ // is out-dated.
+ // 3. The URL starts with "file:", which means it's a local file and isn't vendored.
+ // 4. The vendor path doesn't exist, which means the URL is not vendored.
+ if (vendorUtil != null && checksum.isPresent() && !url.getProtocol().equals("file") && vendorUtil.isUrlVendored(url)) {
+ try {
+ return Optional.of(vendorUtil.readRegistryUrl(url, checksum.get()));
+ } catch (IOException e) {
+ throw new IOException(
+ String.format(
+ "Failed to read vendored registry file %s at %s: %s. Please rerun the bazel"
+ + " vendor command.",
+ rawUrl, vendorUtil.getVendorPathForUrl(url), e.getMessage()),
+ e);
+ }
+ }
+
  try (SilentCloseable c =
- Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + url)) {
+ Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + rawUrl)) {
  return Optional.of(
- downloadManager.downloadAndReadOneUrlForBzlmod(
- new URL(url), eventHandler, clientEnv, checksum));
+ downloadManager.downloadAndReadOneUrlForBzlmod(url, eventHandler, clientEnv, checksum));
  } catch (FileNotFoundException e) {
  return Optional.empty();
  } catch (IOException e) {
  // Include the URL in the exception message for easier debugging.
  throw new IOException(
- "Failed to fetch registry file %s: %s".formatted(url, e.getMessage()), e);
+ "Failed to fetch registry file %s: %s".formatted(rawUrl, e.getMessage()), e);
  }
  }
 

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java

@@ -18,6 +18,7 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions;
 import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
+import com.google.devtools.build.lib.vfs.Path;
 import java.net.URISyntaxException;
 import java.util.Optional;
 
@@ -33,6 +34,7 @@ Registry createRegistry(
  String url,
  RepositoryOptions.LockfileMode lockfileMode,
  ImmutableMap<String, Optional<Checksum>> fileHashes,
- ImmutableMap<ModuleKey, String> previouslySelectedYankedVersions)
+ ImmutableMap<ModuleKey, String> previouslySelectedYankedVersions,
+ Optional<Path> vendorDir)
  throws URISyntaxException;
 }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java

@@ -20,6 +20,7 @@
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode;
 import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
 import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager;
+import com.google.devtools.build.lib.vfs.Path;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Map;
@@ -42,7 +43,8 @@ public Registry createRegistry(
  String url,
  LockfileMode lockfileMode,
  ImmutableMap<String, Optional<Checksum>> knownFileHashes,
- ImmutableMap<ModuleKey, String> previouslySelectedYankedVersions)
+ ImmutableMap<ModuleKey, String> previouslySelectedYankedVersions,
+ Optional<Path> vendorDir)
  throws URISyntaxException {
  URI uri = new URI(url);
  if (uri.getScheme() == null) {
@@ -75,6 +77,7 @@ public Registry createRegistry(
  clientEnvironmentSupplier.get(),
  knownFileHashes,
  knownFileHashesMode,
- previouslySelectedYankedVersions);
+ previouslySelectedYankedVersions,
+ vendorDir);
  }
 }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java

@@ -16,6 +16,7 @@
 package com.google.devtools.build.lib.bazel.bzlmod;
 
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode;
+import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction;
 import com.google.devtools.build.lib.server.FailureDetails;
 import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed;
 import com.google.devtools.build.lib.vfs.Path;
@@ -26,6 +27,7 @@
 import java.net.URISyntaxException;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.Optional;
 import javax.annotation.Nullable;
 
 /** A simple SkyFunction that creates a {@link Registry} with a given URL. */
@@ -56,6 +58,7 @@ public RegistryFunction(RegistryFactory registryFactory, Path workspaceRoot) {
  public SkyValue compute(SkyKey skyKey, Environment env)
  throws InterruptedException, RegistryException {
  LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env);
+ Optional<Path> vendorDir = RepositoryDelegatorFunction.VENDOR_DIRECTORY.get(env);
 
  if (lockfileMode == LockfileMode.REFRESH) {
  RegistryFunction.LAST_INVALIDATION.get(env);
@@ -72,7 +75,8 @@ public SkyValue compute(SkyKey skyKey, Environment env)
  key.getUrl().replace("%workspace%", workspaceRoot.getPathString()),
  lockfileMode,
  lockfile.getRegistryFileHashes(),
- lockfile.getSelectedYankedVersions());
+ lockfile.getSelectedYankedVersions(),
+ vendorDir);
  } catch (URISyntaxException e) {
  throw new RegistryException(
  ExternalDepsException.withCauseAndMessage(