From 9f00d63fe295448dc8d27560ae76a0c3163ce691 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Mon, 27 May 2024 17:30:33 +0200 Subject: [PATCH 01/14] Vendor registry files - Vendor registry files needed for Bazel module resolution to achieve offline build with vendor mode. - Also refactored bazel_vendor_test to avoid vendoring dependencies of bazel_tools, which speeds up the test significantly. Fixes https://github.com/bazelbuild/bazel/issues/22554 --- .../lib/bazel/BazelRepositoryModule.java | 2 +- .../devtools/build/lib/bazel/bzlmod/BUILD | 14 ++ .../bzlmod/BazelModuleResolutionValue.java | 2 +- .../build/lib/bazel/bzlmod/IndexRegistry.java | 38 +++- .../lib/bazel/bzlmod/RegistryFactory.java | 5 +- .../lib/bazel/bzlmod/RegistryFactoryImpl.java | 8 +- .../lib/bazel/bzlmod/RegistryFunction.java | 7 +- .../build/lib/bazel/bzlmod/VendorUtil.java | 162 ++++++++++++++++++ .../devtools/build/lib/bazel/commands/BUILD | 2 + .../lib/bazel/commands/VendorCommand.java | 130 ++++++++------ .../RepositoryDelegatorFunction.java | 6 +- .../devtools/build/lib/bazel/bzlmod/BUILD | 1 + .../bzlmod/BzlmodRepoRuleFunctionTest.java | 4 + .../build/lib/bazel/bzlmod/FakeRegistry.java | 4 +- .../lib/bazel/bzlmod/IndexRegistryTest.java | 33 ++-- .../lib/bazel/bzlmod/RegistryFactoryTest.java | 9 +- src/test/py/bazel/bzlmod/bazel_vendor_test.py | 116 +++++++------ 17 files changed, 405 insertions(+), 138 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 81bbf1c016ce30..5d3dd5eef632e9 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -224,7 +224,7 @@ public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builde builder.addCommands(new FetchCommand()); builder.addCommands(new ModCommand()); builder.addCommands(new SyncCommand()); - builder.addCommands(new VendorCommand()); + builder.addCommands(new VendorCommand(downloadManager, clientEnvironmentSupplier)); builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache)); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index cd9c2b40c6a673..f9a6bd9b6c4564 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -46,6 +46,18 @@ java_library( ], ) +java_library( + name = "vendor", + srcs = ["VendorUtil.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", + "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//third_party:guava", + ], +) + java_library( name = "module_extension", srcs = [ @@ -80,6 +92,7 @@ java_library( ], deps = [ ":common", + ":vendor", ":yanked_versions_value", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", @@ -87,6 +100,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/profiler", "//src/main/java/com/google/devtools/build/lib/util:os", + "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//third_party:gson", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java index 3dc6849db48f4b..e776d19d317811 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java @@ -48,7 +48,7 @@ public abstract class BazelModuleResolutionValue implements SkyValue { /** * Hashes of files obtained (or known to be missing) from registries while performing resolution. */ - abstract ImmutableMap> getRegistryFileHashes(); + public abstract ImmutableMap> getRegistryFileHashes(); /** * Selected module versions that are known to be yanked (and hence must have been explicitly diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index d0cef909bc9834..ef0fc56ddf6e9d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -32,11 +32,13 @@ import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.profiler.SilentCloseable; import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.gson.FieldNamingPolicy; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonParseException; + import java.io.FileNotFoundException; import java.io.IOException; import java.net.MalformedURLException; @@ -88,6 +90,7 @@ public enum KnownFileHashesMode { private final Gson gson; private final ImmutableMap> knownFileHashes; private final ImmutableMap previouslySelectedYankedVersions; + private final Optional vendorDir; private final KnownFileHashesMode knownFileHashesMode; private volatile Optional bazelRegistryJson; private volatile StoredEventHandler bazelRegistryJsonEvents; @@ -100,7 +103,8 @@ public IndexRegistry( Map clientEnv, ImmutableMap> knownFileHashes, KnownFileHashesMode knownFileHashesMode, - ImmutableMap previouslySelectedYankedVersions) { + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) { this.uri = uri; this.downloadManager = downloadManager; this.clientEnv = clientEnv; @@ -111,6 +115,7 @@ public IndexRegistry( this.knownFileHashes = knownFileHashes; this.knownFileHashesMode = knownFileHashesMode; this.previouslySelectedYankedVersions = previouslySelectedYankedVersions; + this.vendorDir = vendorDir; } @Override @@ -143,11 +148,11 @@ private Optional grabFile( } private Optional doGrabFile( - String url, ExtendedEventHandler eventHandler, boolean useChecksum) + String rawURL, ExtendedEventHandler eventHandler, boolean useChecksum) throws IOException, InterruptedException { Optional checksum; if (knownFileHashesMode != KnownFileHashesMode.IGNORE && useChecksum) { - Optional knownChecksum = knownFileHashes.get(url); + Optional knownChecksum = knownFileHashes.get(rawURL); if (knownChecksum == null) { if (knownFileHashesMode == KnownFileHashesMode.ENFORCE) { throw new MissingChecksumException( @@ -155,7 +160,7 @@ private Optional doGrabFile( "Missing checksum for registry file %s not permitted with --lockfile_mode=error." + " Please run `bazel mod deps --lockfile_mode=update` to update your" + " lockfile.", - url)); + rawURL)); } // This is a new file, download without providing a checksum. checksum = Optional.empty(); @@ -182,17 +187,36 @@ private Optional doGrabFile( "Cannot fetch a file without a checksum in ENFORCE mode. This is a bug in Bazel, please " + "report at https://github.com/bazelbuild/bazel/issues/new/choose."); } + + URL url = new URL(rawURL); + // Don't try to read the registry URL from the vendor directory in the following cases: + // 1. The vendor directory is not set, which means vendor mode is disabled. + // 2. The checksum is not present, which means the URL is not vendored or the vendored content is out-dated. + // 3. The URL starts with "file:", which means it's a local file and isn't vendored. + // Otherwise, check if the URL is vendored and read the registry file from the vendor directory. + if (vendorDir.isPresent() && checksum.isPresent() && !url.getProtocol().equals("file")) { + VendorUtil vendorUtil = new VendorUtil(vendorDir.get()); + if (vendorUtil.isUrlVendored(url)) { + try { + return Optional.of(vendorUtil.readRegistryURL(url, checksum.get())); + } catch (IOException e) { + throw new IOException( + "Failed to read vendored registry file %s at %s: %s. Please rerun the bazel vendor command.".formatted(rawURL, vendorUtil.getVendorPathForURL(url), e.getMessage()), e); + } + } + } + try (SilentCloseable c = - Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + url)) { + Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + rawURL)) { return Optional.of( downloadManager.downloadAndReadOneUrlForBzlmod( - new URL(url), eventHandler, clientEnv, checksum)); + url, eventHandler, clientEnv, checksum)); } catch (FileNotFoundException e) { return Optional.empty(); } catch (IOException e) { // Include the URL in the exception message for easier debugging. throw new IOException( - "Failed to fetch registry file %s: %s".formatted(url, e.getMessage()), e); + "Failed to fetch registry file %s: %s".formatted(rawURL, e.getMessage()), e); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java index d910c3d2d75c33..19c1ed70ebf77c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java @@ -18,6 +18,8 @@ import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; +import com.google.devtools.build.lib.vfs.Path; + import java.net.URISyntaxException; import java.util.Optional; @@ -33,6 +35,7 @@ Registry createRegistry( String url, RepositoryOptions.LockfileMode lockfileMode, ImmutableMap> fileHashes, - ImmutableMap previouslySelectedYankedVersions) + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) throws URISyntaxException; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java index 32c34daa0c4d4a..3e422fa315f589 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java @@ -20,6 +20,8 @@ import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; +import com.google.devtools.build.lib.vfs.Path; + import java.net.URI; import java.net.URISyntaxException; import java.util.Map; @@ -42,7 +44,8 @@ public Registry createRegistry( String url, LockfileMode lockfileMode, ImmutableMap> knownFileHashes, - ImmutableMap previouslySelectedYankedVersions) + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) throws URISyntaxException { URI uri = new URI(url); if (uri.getScheme() == null) { @@ -75,6 +78,7 @@ public Registry createRegistry( clientEnvironmentSupplier.get(), knownFileHashes, knownFileHashesMode, - previouslySelectedYankedVersions); + previouslySelectedYankedVersions, + vendorDir); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java index 4e72d565bca8fe..d90033498cb43e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.bazel.bzlmod; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.server.FailureDetails; import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; import com.google.devtools.build.lib.vfs.Path; @@ -26,6 +27,8 @@ import java.net.URISyntaxException; import java.time.Duration; import java.time.Instant; +import java.util.Optional; + import javax.annotation.Nullable; /** A simple SkyFunction that creates a {@link Registry} with a given URL. */ @@ -56,6 +59,7 @@ public RegistryFunction(RegistryFactory registryFactory, Path workspaceRoot) { public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException, RegistryException { LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env); + Optional vendorDir = RepositoryDelegatorFunction.VENDOR_DIRECTORY.get(env); if (lockfileMode == LockfileMode.REFRESH) { RegistryFunction.LAST_INVALIDATION.get(env); @@ -72,7 +76,8 @@ public SkyValue compute(SkyKey skyKey, Environment env) key.getUrl().replace("%workspace%", workspaceRoot.getPathString()), lockfileMode, lockfile.getRegistryFileHashes(), - lockfile.getSelectedYankedVersions()); + lockfile.getSelectedYankedVersions(), + vendorDir); } catch (URISyntaxException e) { throw new RegistryException( ExternalDepsException.withCauseAndMessage( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java new file mode 100644 index 00000000000000..1bb3a53e45ce73 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java @@ -0,0 +1,162 @@ +package com.google.devtools.build.lib.bazel.bzlmod; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import com.google.common.hash.HashCode; +import com.google.common.hash.Hasher; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.Symlinks; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URL; +import java.net.URLDecoder; +import java.util.Objects; +import java.util.Optional; + + +/** + * Utility class for vendoring external repositories. + */ +public class VendorUtil { + + private final Path vendorDirectory; + + public VendorUtil(Path vendorDirectory) { + this.vendorDirectory = vendorDirectory; + } + + /** + * Vendors the specified repositories under the vendor directory. + * + * @param externalRepoRoot The root directory of the external repositories. + * @param reposToVendor The list of repositories to vendor. + * @throws IOException if an I/O error occurs. + */ + public void vendorRepos( + Path externalRepoRoot, + ImmutableList reposToVendor) + throws IOException { + if (!vendorDirectory.exists()) { + vendorDirectory.createDirectoryAndParents(); + } + + for (RepositoryName repo : reposToVendor) { + // Only re-vendor the repository if it is not up-to-date. + if (!isRepoUpToDate(repo.getName(), externalRepoRoot)) { + Path repoUnderVendor = vendorDirectory.getRelative(repo.getName()); + if (!repoUnderVendor.exists()) { + repoUnderVendor.createDirectory(); + } + FileSystemUtils.copyTreesBelow( + externalRepoRoot.getRelative(repo.getName()), repoUnderVendor, Symlinks.NOFOLLOW); + FileSystemUtils.copyFile( + externalRepoRoot.getChild("@" + repo.getName() + ".marker"), + vendorDirectory.getChild("@" + repo.getName() + ".marker")); + } + } + } + + /** + * Checks if the given URL is vendored. + * + * @param url The URL to check. + * @return true if the URL is vendored, false otherwise. + * @throws UnsupportedEncodingException if the URL decoding fails. + */ + public boolean isUrlVendored(URL url) throws UnsupportedEncodingException { + return getVendorPathForURL(url).isFile(); + } + + /** + * Vendors the registry URL with the specified content. + * + * @param url The registry URL to vendor. + * @param content The content to write. + * @throws IOException if an I/O error occurs. + */ + public void vendorRegistryURL(URL url, byte[] content) + throws IOException { + Path outputPath = getVendorPathForURL(url); + Objects.requireNonNull(outputPath.getParentDirectory()).createDirectoryAndParents(); + FileSystemUtils.writeContent(outputPath, content); + } + + /** + * Reads the content of the registry URL and verifies its checksum. + * + * @param url The registry URL to read. + * @param checksum The checksum to verify. + * @return The content of the registry URL. + * @throws IOException if an I/O error occurs or the checksum verification fails. + */ + public byte[] readRegistryURL(URL url, Checksum checksum) + throws IOException { + byte[] content = FileSystemUtils.readContent(getVendorPathForURL(url)); + Hasher hasher = checksum.getKeyType().newHasher(); + hasher.putBytes(content); + HashCode actual = hasher.hash(); + if (!checksum.getHashCode().equals(actual)) { + throw new IOException( + String.format( + "Checksum was %s but wanted %s", + checksum.emitOtherHashInSameFormat(actual), + checksum.emitOtherHashInSameFormat(checksum.getHashCode()))); + } + return content; + } + + /** + * Checks if the repository under vendor dir needs to be updated by comparing its marker file with the + * one under /external. This function assumes the marker file under /external exists + * and is up-to-date. + * + * @param repoName The name of the repository. + * @param externalPath The root directory of the external repositories. + * @return true if the repository is up-to-date, false otherwise. + * @throws IOException if an I/O error occurs. + */ + private boolean isRepoUpToDate(String repoName, Path externalPath) + throws IOException { + Path vendorMarkerFile = vendorDirectory.getChild("@" + repoName + ".marker"); + if (!vendorMarkerFile.exists()) { + return false; + } + + Path externalMarkerFile = externalPath.getChild("@" + repoName + ".marker"); + String vendorMarkerContent = FileSystemUtils.readContent(vendorMarkerFile, UTF_8); + String externalMarkerContent = FileSystemUtils.readContent(externalMarkerFile, UTF_8); + return Objects.equals(vendorMarkerContent, externalMarkerContent); + } + + /** + * Returns the vendor path for the given URL. + * + * The vendor path is constructed as follows: + * /registry_cache// + * + * The host name is case-insensitive, so it is converted to lowercase. + * The path is case-sensitive, so it is left as is. + * The port number is not included in the vendor path. + * + * Note that the vendor path may conflicts if two URLs only differ by the case or port number. + * But this is unlikely to happen in practice, and conflicts are checked in VendorCommand.java. + * + * @param url The URL to get the vendor path for. + * @return The vendor path. + * @throws UnsupportedEncodingException if the URL decoding fails. + */ + public Path getVendorPathForURL(URL url) throws UnsupportedEncodingException { + String host = url.getHost().toLowerCase(); // Host names are case-insensitive + String path = url.getPath(); + path = URLDecoder.decode(path, "UTF-8"); + if (path.startsWith("/")) { + path = path.substring(1); + } + return vendorDirectory.getRelative("registry_cache").getRelative(host).getRelative(path); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD index 34b0022e442af0..35ec3ac15cc6dd 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD @@ -39,9 +39,11 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution_impl", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:root_module_file_fixup", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:tidy", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:vendor", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand", "//src/main/java/com/google/devtools/build/lib/bazel/repository", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java index fd6e4411815b49..79938efb57938b 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java @@ -14,7 +14,6 @@ package com.google.devtools.build.lib.bazel.commands; import static com.google.common.collect.ImmutableList.toImmutableList; -import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -22,9 +21,13 @@ import com.google.devtools.build.lib.analysis.NoBuildEvent; import com.google.devtools.build.lib.analysis.NoBuildRequestFinishedEvent; import com.google.devtools.build.lib.bazel.bzlmod.BazelFetchAllValue; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionValue; +import com.google.devtools.build.lib.bazel.bzlmod.VendorUtil; import com.google.devtools.build.lib.bazel.commands.RepositoryFetcher.RepositoryFetcherException; import com.google.devtools.build.lib.bazel.commands.TargetFetcher.TargetFetcherException; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; +import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.buildtool.BuildResult; import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.cmdline.RepositoryName; @@ -49,10 +52,8 @@ import com.google.devtools.build.lib.skyframe.RepositoryMappingValue.RepositoryMappingResolutionException; import com.google.devtools.build.lib.util.DetailedExitCode; import com.google.devtools.build.lib.util.InterruptedFailureDetails; -import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; -import com.google.devtools.build.lib.vfs.Symlinks; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; import com.google.devtools.build.skyframe.InMemoryGraph; @@ -62,15 +63,22 @@ import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.common.options.OptionsParser; import com.google.devtools.common.options.OptionsParsingResult; + import java.io.IOException; +import java.net.URL; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Optional; import java.util.Queue; import java.util.Set; +import java.util.function.Supplier; + import javax.annotation.Nullable; /** @@ -102,6 +110,14 @@ public final class VendorCommand implements BlazeCommand { public static final String NAME = "vendor"; + private final DownloadManager downloadManager; + private final Supplier> clientEnvironmentSupplier; + + public VendorCommand(DownloadManager downloadManager, Supplier> clientEnvironmentSupplier) { + this.downloadManager = downloadManager; + this.clientEnvironmentSupplier = clientEnvironmentSupplier; + } + @Override public void editOptions(OptionsParser optionsParser) { // We only need to inject these options with fetch target (when there is a residue) @@ -135,9 +151,10 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti BlazeCommandResult result; VendorOptions vendorOptions = options.getOptions(VendorOptions.class); - PathFragment vendorDirectory = options.getOptions(RepositoryOptions.class).vendorDirectory; + Path vendorDirectory = getVendorPath(env, options.getOptions(RepositoryOptions.class).vendorDirectory); LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); try { + env.getReporter().handle(Event.info("Vendoring ...")); if (!options.getResidue().isEmpty()) { result = vendorTargets(env, options, options.getResidue(), vendorDirectory); } else if (!vendorOptions.repos.isEmpty()) { @@ -181,7 +198,7 @@ private BlazeCommandResult validateOptions(CommandEnvironment env, OptionsParsin } private BlazeCommandResult vendorAll( - CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, PathFragment vendorDirectory) + CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, Path vendorDirectory) throws InterruptedException, IOException { EvaluationContext evaluationContext = EvaluationContext.newBuilder() @@ -209,7 +226,7 @@ private BlazeCommandResult vendorRepos( CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, List repos, - PathFragment vendorDirectory) + Path vendorDirectory) throws InterruptedException, IOException { ImmutableMap repositoryNamesAndValues; try { @@ -248,7 +265,7 @@ private BlazeCommandResult vendorTargets( CommandEnvironment env, OptionsParsingResult options, List targets, - PathFragment vendorDirectory) + Path vendorDirectory) throws InterruptedException, IOException { // Call fetch which runs build to have the targets graph and configuration set BuildResult buildResult; @@ -309,58 +326,71 @@ private ImmutableSet collectReposFromTargets( */ private void vendor( CommandEnvironment env, - PathFragment vendorDirectory, + Path vendorDirectory, ImmutableList reposToVendor) - throws IOException { - Path vendorPath = - vendorDirectory.isAbsolute() - ? env.getRuntime().getFileSystem().getPath(vendorDirectory) - : env.getWorkspace().getRelative(vendorDirectory); - Path externalPath = - env.getDirectories() - .getOutputBase() - .getRelative(LabelConstants.EXTERNAL_REPOSITORY_LOCATION); + throws IOException, InterruptedException { + VendorUtil vendorUtil = new VendorUtil(vendorDirectory); - if (!vendorPath.exists()) { - vendorPath.createDirectory(); - } + // 1. Vendor registry files + BazelModuleResolutionValue moduleResolutionValue = (BazelModuleResolutionValue) env.getSkyframeExecutor().getEvaluator().getExistingValue(BazelModuleResolutionValue.KEY); + ImmutableMap> registryFiles = Objects.requireNonNull( + moduleResolutionValue).getRegistryFileHashes(); - env.getReporter().handle(Event.info("Vendoring ...")); + // vendorPathToURL is a map of + // key: a vendor path string converted to lower case + // value: a URL string + // This map is for detecting potential rare vendor path conflicts, such as: + // http://foo.bar.com/BCR vs http://foo.bar.com/bcr => conflict vendor paths on case-insensitive system + // http://foo.bar.com/bcr vs http://foo.bar.com:8081/bcr => conflict vendor path because port number is ignored in vendor path + // The user has to update the Bazel registries this if such conflicts occur. + Map vendorPathToURL = new HashMap<>(); + for (Entry> entry : registryFiles.entrySet()) { + URL url = new URL(entry.getKey()); + if (url.getProtocol().equals("file")) { + continue; + } - // Update "out-of-date" repos under the vendor directory - for (RepositoryName repo : reposToVendor) { - if (!isRepoUpToDate(repo.getName(), vendorPath, externalPath)) { - Path repoUnderVendor = vendorPath.getRelative(repo.getName()); - if (!repoUnderVendor.exists()) { - repoUnderVendor.createDirectory(); + String outputPath = vendorUtil.getVendorPathForURL(url).getPathString(); + if (vendorPathToURL.containsKey(outputPath.toLowerCase())) { + String previousURL = vendorPathToURL.get(outputPath.toLowerCase()); + throw new IOException(String.format( + "Vendor paths conflict detected for registry URLs:\n %s => %s\n %s => %s\nTheir output paths are either the same or only differ by case, which will cause conflict on case insensitive file systems, please fix by changing the registry URLs!", + previousURL, + vendorUtil.getVendorPathForURL(new URL(previousURL)).getPathString(), + entry.getKey(), + outputPath + )); + } + + Optional checksum = entry.getValue(); + if (!vendorUtil.isUrlVendored(url) + // Only vendor a registry URL when its checksum exists, otherwise the URL should be + // recorded as "not found" in moduleResolutionValue.getRegistryFileHashes() + && checksum.isPresent()) { + try { + vendorUtil.vendorRegistryURL(url, downloadManager.downloadAndReadOneUrlForBzlmod(url, env.getReporter(), clientEnvironmentSupplier.get(), checksum)); + } catch (IOException e) { + throw new IOException(String.format("Failed to vendor registry URL %s at %s: %s", url, outputPath, e.getMessage()), e.getCause()); } - FileSystemUtils.copyTreesBelow( - externalPath.getRelative(repo.getName()), repoUnderVendor, Symlinks.NOFOLLOW); - FileSystemUtils.copyFile( - externalPath.getChild("@" + repo.getName() + ".marker"), - vendorPath.getChild("@" + repo.getName() + ".marker")); } - } - } - /** - * Returns whether the repo under vendor needs to be updated by comparing its marker file with the - * one under /external - */ - private boolean isRepoUpToDate(String repoName, Path vendorPath, Path externalPath) - throws IOException { - Path vendorMarkerFile = vendorPath.getChild("@" + repoName + ".marker"); - if (!vendorMarkerFile.exists()) { - return false; + vendorPathToURL.put(outputPath.toLowerCase(), entry.getKey()); } - // Since this runs after fetching repos, its guaranteed that the marker files - // under $OUTPUT_BASE/external are up-to-date. We just need to compare it against the marker - // under vendor. - Path externalMarkerFile = externalPath.getChild("@" + repoName + ".marker"); - String vendorMarkerContent = FileSystemUtils.readContent(vendorMarkerFile, UTF_8); - String externalMarkerContent = FileSystemUtils.readContent(externalMarkerFile, UTF_8); - return Objects.equals(vendorMarkerContent, externalMarkerContent); + // 2. Vendor repos + Path externalPath = + env.getDirectories() + .getOutputBase() + .getRelative(LabelConstants.EXTERNAL_REPOSITORY_LOCATION); + vendorUtil.vendorRepos(externalPath, reposToVendor); + } + + private static Path getVendorPath(CommandEnvironment env, PathFragment vendorDirectory) { + Path vendorPath = + vendorDirectory.isAbsolute() + ? env.getRuntime().getFileSystem().getPath(vendorDirectory) + : env.getWorkspace().getRelative(vendorDirectory); + return vendorPath; } private static BlazeCommandResult createFailedBlazeCommandResult( diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java index ecebc7f072da1d..72bf3268d15a9d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java @@ -303,7 +303,7 @@ private RepositoryDirectoryValue tryGettingValueUsingVendoredRepo( String.format( "Vendored repository '%s' is out-of-date and fetching is disabled." + " Run build without the '--nofetch' option or run" - + " `bazel vendor` to update it", + + " the bazel vendor command to update it", rule.getName()))); } return setupOverride(vendorRepoPath.asFragment(), env, repoRoot, repositoryName); @@ -316,7 +316,7 @@ private RepositoryDirectoryValue tryGettingValueUsingVendoredRepo( String.format( "Vendored repository '%s' is out-of-date. The up-to-date version will" + " be fetched into the external cache and used. To update the repo" - + " in the vendor directory, run 'bazel vendor'", + + " in the vendor directory, run the bazel vendor command", rule.getName()))); } } else if (vendorFile.getPinnedRepos().contains(repositoryName)) { @@ -332,7 +332,7 @@ private RepositoryDirectoryValue tryGettingValueUsingVendoredRepo( "Vendored repository " + repositoryName.getName() + " not found under the vendor directory and fetching is disabled." - + " To fix run 'bazel vendor' or build without the '--nofetch'"), + + " To fix, run the bazel vendor command or build without the '--nofetch'"), Transience.TRANSIENT); } return null; diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index d783e91e4edd3c..3566f8a1456407 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -120,6 +120,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/packages", + "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/net/starlark/java/eval", "//src/main/java/net/starlark/java/syntax", "//third_party:guava", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index 9c98532eec437b..8806f9abc028da 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -37,6 +37,7 @@ import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.Type; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; import com.google.devtools.build.lib.skyframe.BzlmodRepoRuleFunction; import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; @@ -62,6 +63,8 @@ import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionName; + +import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import net.starlark.java.eval.StarlarkSemantics; import org.junit.Before; @@ -158,6 +161,7 @@ public void setup() throws Exception { BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); + RepositoryDelegatorFunction.VENDOR_DIRECTORY.set(differencer, Optional.empty()); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java index e093f5f326a807..0b9b5be3c23e0d 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java @@ -23,6 +23,7 @@ import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.vfs.Path; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.HashMap; import java.util.Map; @@ -136,7 +137,8 @@ public Registry createRegistry( String url, LockfileMode lockfileMode, ImmutableMap> fileHashes, - ImmutableMap previouslySelectedYankedVersions) { + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) { return Preconditions.checkNotNull(registries.get(url), "unknown registry url: %s", url); } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java index b4d259c398aee9..9fff42177d0ded 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java @@ -97,7 +97,7 @@ public void testHttpUrl() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue( ModuleFile.create( @@ -115,7 +115,7 @@ public void testHttpUrlWithNetrcCreds() throws Exception { "machine [::1] login rinne password rinnepass\n".getBytes(UTF_8))); Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); var e = assertThrows( @@ -148,7 +148,8 @@ public void testFileUrl() throws Exception { new File(tempFolder.getRoot(), "fakereg").toURI().toString(), LockfileMode.UPDATE, ImmutableMap.of(), - ImmutableMap.of()); + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue(ModuleFile.create("lol".getBytes(UTF_8), file.toURI().toString())); assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty(); @@ -197,7 +198,7 @@ public void testGetArchiveRepoSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -265,7 +266,7 @@ public void testGetLocalPathRepoSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( RepoSpec.builder() @@ -289,7 +290,7 @@ public void testGetRepoInvalidRegistryJsonSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -323,7 +324,7 @@ public void testGetRepoInvalidModuleJsonSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); } @@ -352,7 +353,7 @@ public void testGetYankedVersion() throws Exception { server.start(); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); Optional> yankedVersion = registry.getYankedVersions("red-pill", reporter); assertThat(yankedVersion) @@ -375,7 +376,7 @@ public void testArchiveWithExplicitType() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("archive_type", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -405,7 +406,7 @@ public void testGetModuleFileChecksums() throws Exception { Optional.of(sha256("unused"))); Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue( ModuleFile.create( @@ -431,7 +432,7 @@ public void testGetModuleFileChecksums() throws Exception { registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of()); + server.getUrl() + "/myreg", LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of(), Optional.empty()); // Test that the recorded hashes are used for repo cache hits even when the server content // changes. server.unserve("/myreg/modules/foo/1.0/MODULE.bazel"); @@ -461,7 +462,7 @@ public void testGetModuleFileChecksumMismatch() throws Exception { Optional.of(sha256("original"))); Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); var e = assertThrows( IOException.class, @@ -502,7 +503,7 @@ public void testGetRepoSpecChecksum() throws Exception { server.getUrl() + "/modules/foo/2.0/source.json", Optional.of(sha256("unused"))); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( RepoSpec.builder() @@ -522,7 +523,7 @@ public void testGetRepoSpecChecksum() throws Exception { registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of(), Optional.empty()); // Test that the recorded hashes are used for repo cache hits even when the server content // changes. server.unserve("/bazel_registry.json"); @@ -566,7 +567,7 @@ public void testGetRepoSpecChecksumMismatch() throws Exception { Optional.of(sha256(sourceJson))); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); var e = assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); @@ -610,7 +611,7 @@ public void testBazelRegistryChecksumMismatch() throws Exception { Optional.of(sha256(sourceJson))); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); var e = assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java index cc963f233e5e3e..f29d6c5a704cb4 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java @@ -25,6 +25,8 @@ import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; import java.net.URISyntaxException; +import java.util.Optional; + import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -44,14 +46,15 @@ public void badSchemes() { URISyntaxException.class, () -> registryFactory.createRegistry( - "/home/www", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of())); + "/home/www", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), + Optional.empty())); assertThat(exception).hasMessageThat().contains("Registry URL has no scheme"); exception = assertThrows( URISyntaxException.class, () -> registryFactory.createRegistry( - "foo://bar", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of())); + "foo://bar", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty())); assertThat(exception).hasMessageThat().contains("Unrecognized registry URL protocol"); } @@ -69,7 +72,7 @@ public void badPath() { "file:c:/path/to/workspace/registry", LockfileMode.UPDATE, ImmutableMap.of(), - ImmutableMap.of())); + ImmutableMap.of(), Optional.empty())); assertThat(exception).hasMessageThat().contains("Registry URL path is not valid"); } } diff --git a/src/test/py/bazel/bzlmod/bazel_vendor_test.py b/src/test/py/bazel/bzlmod/bazel_vendor_test.py index 0ac2b7cad18294..0b4479bc52e5a8 100644 --- a/src/test/py/bazel/bzlmod/bazel_vendor_test.py +++ b/src/test/py/bazel/bzlmod/bazel_vendor_test.py @@ -258,35 +258,6 @@ def testVendorDirIsNotCheckedForWorkspaceRepos(self): "Vendored repository 'dummyRepo' is out-of-date.", '\n'.join(stderr) ) - def testBuildingWithVendoredRepos(self): - self.main_registry.createCcModule('aaa', '1.0') - self.ScratchFile( - 'MODULE.bazel', - [ - 'bazel_dep(name = "aaa", version = "1.0")', - ], - ) - self.ScratchFile('BUILD') - self.RunBazel(['vendor', '--vendor_dir=vendor']) - self.assertIn('aaa~', os.listdir(self._test_cwd + '/vendor')) - - # Empty external & build with vendor - self.RunBazel(['clean', '--expunge']) - _, _, stderr = self.RunBazel(['build', '@aaa//:all', '--vendor_dir=vendor']) - self.assertNotIn( - "Vendored repository '_main~ext~justRepo' is out-of-date.", - '\n'.join(stderr), - ) - - # Assert repo aaa in {OUTPUT_BASE}/external is a symlink (junction on - # windows, this validates it was created from vendor and not fetched)= - _, stdout, _ = self.RunBazel(['info', 'output_base']) - repo_path = stdout[0] + '/external/aaa~' - if self.IsWindows(): - self.assertTrue(self.IsJunction(repo_path)) - else: - self.assertTrue(os.path.islink(repo_path)) - def testIgnoreFromVendoring(self): # Repos should be excluded from vendoring: # 1.Local Repos, 2.Config Repos, 3.Repos declared in VENDOR.bazel file @@ -372,7 +343,7 @@ def testBuildingWithPinnedRepo(self): ) self.ScratchFile('BUILD') - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@venRepo']) self.assertIn('_main~ext~venRepo', os.listdir(self._test_cwd + '/vendor')) self.ScratchFile( 'extension.bzl', @@ -421,7 +392,7 @@ def testBuildingWithPinnedRepo(self): ) # Re-vendor & build make sure the repo is successfully updated - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@venRepo']) _, _, stderr = self.RunBazel( ['build', '@venRepo//:all', '--vendor_dir=vendor'], ) @@ -454,7 +425,7 @@ def testBuildingOutOfDateVendoredRepo(self): ) # Vendor, assert and build with no problems - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@justRepo']) self.assertIn('_main~ext~justRepo', os.listdir(self._test_cwd + '/vendor')) _, _, stderr = self.RunBazel( ['build', '@justRepo//:all', '--vendor_dir=vendor'] @@ -462,8 +433,8 @@ def testBuildingOutOfDateVendoredRepo(self): self.assertNotIn( "WARNING: : Vendored repository '_main~ext~justRepo' is" ' out-of-date. The up-to-date version will be fetched into the external' - ' cache and used. To update the repo in the vendor directory, run' - " 'bazel vendor'", + ' cache and used. To update the repo in the vendor directory, run' + " the bazel vendor command", stderr, ) @@ -492,15 +463,15 @@ def testBuildingOutOfDateVendoredRepo(self): self.assertIn( "WARNING: : Vendored repository '_main~ext~justRepo' is" ' out-of-date. The up-to-date version will be fetched into the external' - ' cache and used. To update the repo in the vendor directory, run' - " 'bazel vendor'", + ' cache and used. To update the repo in the vendor directory, run' + " the bazel vendor command", stderr, ) _, stdout, _ = self.RunBazel(['info', 'output_base']) self.assertFalse(os.path.islink(stdout[0] + '/external/bbb~')) # Assert vendoring again solves the problem - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@justRepo']) self.RunBazel(['clean', '--expunge']) _, _, stderr = self.RunBazel( ['build', '@justRepo//:all', '--vendor_dir=vendor'] @@ -508,12 +479,12 @@ def testBuildingOutOfDateVendoredRepo(self): self.assertNotIn( "WARNING: : Vendored repository '_main~ext~justRepo' is" ' out-of-date. The up-to-date version will be fetched into the external' - ' cache and used. To update the repo in the vendor directory, run' - " 'bazel vendor'", + ' cache and used. To update the repo in the vendor directory, run' + " the bazel vendor command", stderr, ) - def testBuildingVendoredRepoInOfflineMode(self): + def testBuildingVendoredRepoWithNoFetch(self): self.ScratchFile( 'MODULE.bazel', [ @@ -537,7 +508,7 @@ def testBuildingVendoredRepoInOfflineMode(self): self.ScratchFile('BUILD') # Vendor, assert and build with no problems - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '@venRepo//:all']) self.assertIn('_main~ext~venRepo', os.listdir(self._test_cwd + '/vendor')) # Make updates in repo definition @@ -571,8 +542,8 @@ def testBuildingVendoredRepoInOfflineMode(self): ) self.assertIn( 'ERROR: Vendored repository _main~ext~noVenRepo not found under the' - " vendor directory and fetching is disabled. To fix run 'bazel" - " vendor' or build without the '--nofetch'", + " vendor directory and fetching is disabled. To fix, run the bazel" + " vendor command or build without the '--nofetch'", stderr, ) @@ -584,7 +555,7 @@ def testBuildingVendoredRepoInOfflineMode(self): self.assertIn( "WARNING: : Vendored repository '_main~ext~venRepo' is" ' out-of-date and fetching is disabled. Run build without the' - " '--nofetch' option or run `bazel vendor` to update it", + " '--nofetch' option or run the bazel vendor command to update it", stderr, ) # Assert the out-dated repo is the one built with @@ -615,7 +586,7 @@ def testBasicVendorTarget(self): self.assertIn('bbb~', os.listdir(self._test_cwd + '/vendor')) self.assertNotIn('ccc~', os.listdir(self._test_cwd + '/vendor')) - def testVendorTarget(self): + def testBuildVendoredTargetOffline(self): self.main_registry.createCcModule('aaa', '1.0').createCcModule( 'bbb', '1.0', {'aaa': '1.0'} ) @@ -640,22 +611,63 @@ def testVendorTarget(self): self.ScratchFile( 'main.cc', [ - '#include "aaa.h"', + '#include "bbb.h"', 'int main() {', - ' hello_aaa("Hello there!");', + ' hello_bbb("Hello there!");', '}', ], ) self.RunBazel(['vendor', '//:main', '--vendor_dir=vendor']) - # Run the vendored target with --nofetch should only use what is under - # vendor to build, meaning we have vendored everything we need to build/run - # this target + # Build and run the target in a clean build with internet blocked and make sure it works + _, _, _ = self.RunBazel(['clean', '--expunge']) _, stdout, _ = self.RunBazel( - ['run', '//:main', '--vendor_dir=vendor', '--nofetch'] + ['run', '//:main', '--vendor_dir=vendor', '--repository_cache='], + env_add= { + "HTTP_PROXY": "internet_blocked", + "HTTPS_PROXY": "internet_blocked", + }, + ) + self.assertIn('Hello there! => bbb@1.0', stdout) + + # Assert repos in {OUTPUT_BASE}/external are symlinks (junction on + # windows, this validates it was created from vendor and not fetched) + _, stdout, _ = self.RunBazel(['info', 'output_base']) + for repo in ["aaa~", "bbb~"]: + repo_path = stdout[0] + '/external/' + repo + if self.IsWindows(): + self.assertTrue(self.IsJunction(repo_path)) + else: + self.assertTrue(os.path.islink(repo_path)) + + def testVendorConflictRegistryFile(self): + self.main_registry.createCcModule('aaa', '1.0').createCcModule( + 'bbb', '1.0', {'aaa': '1.0'} + ) + # The registry URLs of main_registry and another_registry only differ by the port number + another_registry = BazelRegistry( + os.path.join(self.registries_work_dir, 'MAIN'), ) - self.assertIn('Hello there! => aaa@1.0', stdout) + another_registry.start() + another_registry.createCcModule('aaa', '1.0') + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "bbb", version = "1.0")', + 'local_path_override(module_name="bazel_tools", path="tools_mock")', + 'local_path_override(module_name="local_config_platform", ', + 'path="platforms_mock")', + 'single_version_override(', + ' module_name = "aaa",', + ' registry = "%s",' % another_registry.getURL(), + ')', + ], + ) + self.ScratchFile('BUILD') + exit_code, _, stderr = self.RunBazel(['vendor', '--vendor_dir=vendor'], allow_failure=True) + self.AssertExitCode(exit_code, 8, stderr) + self.assertIn('ERROR: Error while vendoring repos: Vendor paths conflict detected for registry URLs:', stderr) if __name__ == '__main__': From dfba2d97413e793c36ff41045fa9948e2dcfa62a Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Mon, 3 Jun 2024 17:47:01 +0200 Subject: [PATCH 02/14] Add doc about offline --- site/en/external/vendor.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/site/en/external/vendor.md b/site/en/external/vendor.md index 2d8685604e7b0d..d2ad7f939d79b4 100644 --- a/site/en/external/vendor.md +++ b/site/en/external/vendor.md @@ -67,6 +67,20 @@ Under the hood, it's doing a `bazel build --nobuild` command to analyze the target patterns, therefore build flags could be applied to this command and affect the result. +### Build the target offline {:#build-the-target-offline} + +With the external dependencies vendored, you can build the target offline by + +```none +bazel build --vendor_dir=vendor_src //src/main:hello-world //src/test/... +``` + +The build should work in a clean build environment without network access and repository cache. + +Therefore, you should be able to check in the vendored source and build the same targets offline on another machine. + +Note: If you build different targets or change the external dependencies, build configuration, or Bazel version, you may need to re-vendor. + ## Vendor all external dependencies {:#vendor-all-dependencies} To vendor all repos in your transitive external dependencies graph, you can From 70eb000285dce7518d2e7630daef9aeb20240721 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Tue, 4 Jun 2024 13:38:52 +0200 Subject: [PATCH 03/14] internal cleanups --- site/en/external/vendor.md | 9 ++- .../devtools/build/lib/bazel/bzlmod/BUILD | 1 - .../build/lib/bazel/bzlmod/IndexRegistry.java | 27 ++++--- .../lib/bazel/bzlmod/RegistryFactory.java | 1 - .../lib/bazel/bzlmod/RegistryFactoryImpl.java | 1 - .../lib/bazel/bzlmod/RegistryFunction.java | 1 - .../build/lib/bazel/bzlmod/VendorUtil.java | 60 +++++++------- .../devtools/build/lib/bazel/commands/BUILD | 2 - .../lib/bazel/commands/VendorCommand.java | 80 +++++++++++-------- .../bzlmod/BzlmodRepoRuleFunctionTest.java | 1 - .../lib/bazel/bzlmod/IndexRegistryTest.java | 72 ++++++++++++++--- .../lib/bazel/bzlmod/RegistryFactoryTest.java | 15 +++- src/test/py/bazel/bzlmod/bazel_vendor_test.py | 42 ++++++---- 13 files changed, 196 insertions(+), 116 deletions(-) diff --git a/site/en/external/vendor.md b/site/en/external/vendor.md index d2ad7f939d79b4..41911be394e3f2 100644 --- a/site/en/external/vendor.md +++ b/site/en/external/vendor.md @@ -75,11 +75,14 @@ With the external dependencies vendored, you can build the target offline by bazel build --vendor_dir=vendor_src //src/main:hello-world //src/test/... ``` -The build should work in a clean build environment without network access and repository cache. +The build should work in a clean build environment without network access and +repository cache. -Therefore, you should be able to check in the vendored source and build the same targets offline on another machine. +Therefore, you should be able to check in the vendored source and build the same +targets offline on another machine. -Note: If you build different targets or change the external dependencies, build configuration, or Bazel version, you may need to re-vendor. +Note: If you build different targets or change the external dependencies, build +configuration, or Bazel version, you may need to re-vendor. ## Vendor all external dependencies {:#vendor-all-dependencies} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index f9a6bd9b6c4564..5e761993a4a95f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -53,7 +53,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/vfs", - "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//third_party:guava", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index ef0fc56ddf6e9d..f7e2ced1c5babd 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -38,7 +38,6 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonParseException; - import java.io.FileNotFoundException; import java.io.IOException; import java.net.MalformedURLException; @@ -148,11 +147,11 @@ private Optional grabFile( } private Optional doGrabFile( - String rawURL, ExtendedEventHandler eventHandler, boolean useChecksum) + String rawUrl, ExtendedEventHandler eventHandler, boolean useChecksum) throws IOException, InterruptedException { Optional checksum; if (knownFileHashesMode != KnownFileHashesMode.IGNORE && useChecksum) { - Optional knownChecksum = knownFileHashes.get(rawURL); + Optional knownChecksum = knownFileHashes.get(rawUrl); if (knownChecksum == null) { if (knownFileHashesMode == KnownFileHashesMode.ENFORCE) { throw new MissingChecksumException( @@ -160,7 +159,7 @@ private Optional doGrabFile( "Missing checksum for registry file %s not permitted with --lockfile_mode=error." + " Please run `bazel mod deps --lockfile_mode=update` to update your" + " lockfile.", - rawURL)); + rawUrl)); } // This is a new file, download without providing a checksum. checksum = Optional.empty(); @@ -188,35 +187,39 @@ private Optional doGrabFile( + "report at https://github.com/bazelbuild/bazel/issues/new/choose."); } - URL url = new URL(rawURL); + URL url = URI.create(rawUrl).toURL(); // Don't try to read the registry URL from the vendor directory in the following cases: // 1. The vendor directory is not set, which means vendor mode is disabled. - // 2. The checksum is not present, which means the URL is not vendored or the vendored content is out-dated. + // 2. The checksum is not present, which means the URL is not vendored or the vendored content + // is out-dated. // 3. The URL starts with "file:", which means it's a local file and isn't vendored. // Otherwise, check if the URL is vendored and read the registry file from the vendor directory. if (vendorDir.isPresent() && checksum.isPresent() && !url.getProtocol().equals("file")) { VendorUtil vendorUtil = new VendorUtil(vendorDir.get()); if (vendorUtil.isUrlVendored(url)) { try { - return Optional.of(vendorUtil.readRegistryURL(url, checksum.get())); + return Optional.of(vendorUtil.readRegistryUrl(url, checksum.get())); } catch (IOException e) { throw new IOException( - "Failed to read vendored registry file %s at %s: %s. Please rerun the bazel vendor command.".formatted(rawURL, vendorUtil.getVendorPathForURL(url), e.getMessage()), e); + String.format( + "Failed to read vendored registry file %s at %s: %s. Please rerun the bazel" + + " vendor command.", + rawUrl, vendorUtil.getVendorPathForUrl(url), e.getMessage()), + e); } } } try (SilentCloseable c = - Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + rawURL)) { + Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + rawUrl)) { return Optional.of( - downloadManager.downloadAndReadOneUrlForBzlmod( - url, eventHandler, clientEnv, checksum)); + downloadManager.downloadAndReadOneUrlForBzlmod(url, eventHandler, clientEnv, checksum)); } catch (FileNotFoundException e) { return Optional.empty(); } catch (IOException e) { // Include the URL in the exception message for easier debugging. throw new IOException( - "Failed to fetch registry file %s: %s".formatted(rawURL, e.getMessage()), e); + "Failed to fetch registry file %s: %s".formatted(rawUrl, e.getMessage()), e); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java index 19c1ed70ebf77c..47d0bb77e9579b 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java @@ -19,7 +19,6 @@ import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.vfs.Path; - import java.net.URISyntaxException; import java.util.Optional; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java index 3e422fa315f589..e26468f4890528 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java @@ -21,7 +21,6 @@ import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.vfs.Path; - import java.net.URI; import java.net.URISyntaxException; import java.util.Map; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java index d90033498cb43e..836c19e90bdb84 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java @@ -28,7 +28,6 @@ import java.time.Duration; import java.time.Instant; import java.util.Optional; - import javax.annotation.Nullable; /** A simple SkyFunction that creates a {@link Registry} with a given URL. */ diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java index 1bb3a53e45ce73..748ef17ef5c478 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java @@ -1,3 +1,16 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package com.google.devtools.build.lib.bazel.bzlmod; import static java.nio.charset.StandardCharsets.UTF_8; @@ -10,18 +23,14 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.Symlinks; - import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URL; import java.net.URLDecoder; +import java.util.Locale; import java.util.Objects; -import java.util.Optional; - -/** - * Utility class for vendoring external repositories. - */ +/** Utility class for vendoring external repositories. */ public class VendorUtil { private final Path vendorDirectory; @@ -37,9 +46,7 @@ public VendorUtil(Path vendorDirectory) { * @param reposToVendor The list of repositories to vendor. * @throws IOException if an I/O error occurs. */ - public void vendorRepos( - Path externalRepoRoot, - ImmutableList reposToVendor) + public void vendorRepos(Path externalRepoRoot, ImmutableList reposToVendor) throws IOException { if (!vendorDirectory.exists()) { vendorDirectory.createDirectoryAndParents(); @@ -69,7 +76,7 @@ public void vendorRepos( * @throws UnsupportedEncodingException if the URL decoding fails. */ public boolean isUrlVendored(URL url) throws UnsupportedEncodingException { - return getVendorPathForURL(url).isFile(); + return getVendorPathForUrl(url).isFile(); } /** @@ -79,9 +86,8 @@ public boolean isUrlVendored(URL url) throws UnsupportedEncodingException { * @param content The content to write. * @throws IOException if an I/O error occurs. */ - public void vendorRegistryURL(URL url, byte[] content) - throws IOException { - Path outputPath = getVendorPathForURL(url); + public void vendorRegistryUrl(URL url, byte[] content) throws IOException { + Path outputPath = getVendorPathForUrl(url); Objects.requireNonNull(outputPath.getParentDirectory()).createDirectoryAndParents(); FileSystemUtils.writeContent(outputPath, content); } @@ -94,9 +100,8 @@ public void vendorRegistryURL(URL url, byte[] content) * @return The content of the registry URL. * @throws IOException if an I/O error occurs or the checksum verification fails. */ - public byte[] readRegistryURL(URL url, Checksum checksum) - throws IOException { - byte[] content = FileSystemUtils.readContent(getVendorPathForURL(url)); + public byte[] readRegistryUrl(URL url, Checksum checksum) throws IOException { + byte[] content = FileSystemUtils.readContent(getVendorPathForUrl(url)); Hasher hasher = checksum.getKeyType().newHasher(); hasher.putBytes(content); HashCode actual = hasher.hash(); @@ -111,17 +116,16 @@ public byte[] readRegistryURL(URL url, Checksum checksum) } /** - * Checks if the repository under vendor dir needs to be updated by comparing its marker file with the - * one under /external. This function assumes the marker file under /external exists - * and is up-to-date. + * Checks if the repository under vendor dir needs to be updated by comparing its marker file with + * the one under /external. This function assumes the marker file under + * /external exists and is up-to-date. * * @param repoName The name of the repository. * @param externalPath The root directory of the external repositories. * @return true if the repository is up-to-date, false otherwise. * @throws IOException if an I/O error occurs. */ - private boolean isRepoUpToDate(String repoName, Path externalPath) - throws IOException { + private boolean isRepoUpToDate(String repoName, Path externalPath) throws IOException { Path vendorMarkerFile = vendorDirectory.getChild("@" + repoName + ".marker"); if (!vendorMarkerFile.exists()) { return false; @@ -136,22 +140,20 @@ private boolean isRepoUpToDate(String repoName, Path externalPath) /** * Returns the vendor path for the given URL. * - * The vendor path is constructed as follows: - * /registry_cache// + *

The vendor path is constructed as follows: /registry_cache// * - * The host name is case-insensitive, so it is converted to lowercase. - * The path is case-sensitive, so it is left as is. - * The port number is not included in the vendor path. + *

The host name is case-insensitive, so it is converted to lowercase. The path is + * case-sensitive, so it is left as is. The port number is not included in the vendor path. * - * Note that the vendor path may conflicts if two URLs only differ by the case or port number. + *

Note that the vendor path may conflicts if two URLs only differ by the case or port number. * But this is unlikely to happen in practice, and conflicts are checked in VendorCommand.java. * * @param url The URL to get the vendor path for. * @return The vendor path. * @throws UnsupportedEncodingException if the URL decoding fails. */ - public Path getVendorPathForURL(URL url) throws UnsupportedEncodingException { - String host = url.getHost().toLowerCase(); // Host names are case-insensitive + public Path getVendorPathForUrl(URL url) throws UnsupportedEncodingException { + String host = url.getHost().toLowerCase(Locale.ROOT); // Host names are case-insensitive String path = url.getPath(); path = URLDecoder.decode(path, "UTF-8"); if (path.startsWith("/")) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD index 35ec3ac15cc6dd..37fcb87144414c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD @@ -31,12 +31,10 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:no_build_request_finished_event", "//src/main/java/com/google/devtools/build/lib/bazel:resolved_event", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", - "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:exception", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:repo_rule_value", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution", - "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution_impl", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:root_module_file_fixup", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:tidy", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:vendor", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java index 79938efb57938b..e86aadf472d45e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java @@ -63,14 +63,15 @@ import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.common.options.OptionsParser; import com.google.devtools.common.options.OptionsParsingResult; - import java.io.IOException; +import java.net.URI; import java.net.URL; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; @@ -78,7 +79,6 @@ import java.util.Queue; import java.util.Set; import java.util.function.Supplier; - import javax.annotation.Nullable; /** @@ -113,7 +113,8 @@ public final class VendorCommand implements BlazeCommand { private final DownloadManager downloadManager; private final Supplier> clientEnvironmentSupplier; - public VendorCommand(DownloadManager downloadManager, Supplier> clientEnvironmentSupplier) { + public VendorCommand( + DownloadManager downloadManager, Supplier> clientEnvironmentSupplier) { this.downloadManager = downloadManager; this.clientEnvironmentSupplier = clientEnvironmentSupplier; } @@ -151,7 +152,8 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti BlazeCommandResult result; VendorOptions vendorOptions = options.getOptions(VendorOptions.class); - Path vendorDirectory = getVendorPath(env, options.getOptions(RepositoryOptions.class).vendorDirectory); + Path vendorDirectory = + getVendorPath(env, options.getOptions(RepositoryOptions.class).vendorDirectory); LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); try { env.getReporter().handle(Event.info("Vendoring ...")); @@ -325,41 +327,51 @@ private ImmutableSet collectReposFromTargets( * ignored or was already vendored and up-to-date */ private void vendor( - CommandEnvironment env, - Path vendorDirectory, - ImmutableList reposToVendor) + CommandEnvironment env, Path vendorDirectory, ImmutableList reposToVendor) throws IOException, InterruptedException { VendorUtil vendorUtil = new VendorUtil(vendorDirectory); // 1. Vendor registry files - BazelModuleResolutionValue moduleResolutionValue = (BazelModuleResolutionValue) env.getSkyframeExecutor().getEvaluator().getExistingValue(BazelModuleResolutionValue.KEY); - ImmutableMap> registryFiles = Objects.requireNonNull( - moduleResolutionValue).getRegistryFileHashes(); + BazelModuleResolutionValue moduleResolutionValue = + (BazelModuleResolutionValue) + env.getSkyframeExecutor() + .getEvaluator() + .getExistingValue(BazelModuleResolutionValue.KEY); + ImmutableMap> registryFiles = + Objects.requireNonNull(moduleResolutionValue).getRegistryFileHashes(); // vendorPathToURL is a map of // key: a vendor path string converted to lower case // value: a URL string // This map is for detecting potential rare vendor path conflicts, such as: - // http://foo.bar.com/BCR vs http://foo.bar.com/bcr => conflict vendor paths on case-insensitive system - // http://foo.bar.com/bcr vs http://foo.bar.com:8081/bcr => conflict vendor path because port number is ignored in vendor path + // http://foo.bar.com/BCR vs http://foo.bar.com/bcr => conflict vendor paths on + // case-insensitive system + // http://foo.bar.com/bcr vs http://foo.bar.com:8081/bcr => conflict vendor path because port + // number is ignored in vendor path // The user has to update the Bazel registries this if such conflicts occur. - Map vendorPathToURL = new HashMap<>(); + Map vendorPathToUrl = new HashMap<>(); for (Entry> entry : registryFiles.entrySet()) { - URL url = new URL(entry.getKey()); + URL url = URI.create(entry.getKey()).toURL(); if (url.getProtocol().equals("file")) { continue; } - String outputPath = vendorUtil.getVendorPathForURL(url).getPathString(); - if (vendorPathToURL.containsKey(outputPath.toLowerCase())) { - String previousURL = vendorPathToURL.get(outputPath.toLowerCase()); - throw new IOException(String.format( - "Vendor paths conflict detected for registry URLs:\n %s => %s\n %s => %s\nTheir output paths are either the same or only differ by case, which will cause conflict on case insensitive file systems, please fix by changing the registry URLs!", - previousURL, - vendorUtil.getVendorPathForURL(new URL(previousURL)).getPathString(), - entry.getKey(), - outputPath - )); + String outputPath = vendorUtil.getVendorPathForUrl(url).getPathString(); + String outputPathLowerCase = outputPath.toLowerCase(Locale.ROOT); + if (vendorPathToUrl.containsKey(outputPathLowerCase)) { + String previousUrl = vendorPathToUrl.get(outputPathLowerCase); + throw new IOException( + String.format( + "Vendor paths conflict detected for registry URLs:\n" + + " %s => %s\n" + + " %s => %s\n" + + "Their output paths are either the same or only differ by case, which will" + + " cause conflict on case insensitive file systems, please fix by changing the" + + " registry URLs!", + previousUrl, + vendorUtil.getVendorPathForUrl(URI.create(previousUrl).toURL()).getPathString(), + entry.getKey(), + outputPath)); } Optional checksum = entry.getValue(); @@ -368,13 +380,19 @@ private void vendor( // recorded as "not found" in moduleResolutionValue.getRegistryFileHashes() && checksum.isPresent()) { try { - vendorUtil.vendorRegistryURL(url, downloadManager.downloadAndReadOneUrlForBzlmod(url, env.getReporter(), clientEnvironmentSupplier.get(), checksum)); + vendorUtil.vendorRegistryUrl( + url, + downloadManager.downloadAndReadOneUrlForBzlmod( + url, env.getReporter(), clientEnvironmentSupplier.get(), checksum)); } catch (IOException e) { - throw new IOException(String.format("Failed to vendor registry URL %s at %s: %s", url, outputPath, e.getMessage()), e.getCause()); + throw new IOException( + String.format( + "Failed to vendor registry URL %s at %s: %s", url, outputPath, e.getMessage()), + e.getCause()); } } - vendorPathToURL.put(outputPath.toLowerCase(), entry.getKey()); + vendorPathToUrl.put(outputPathLowerCase, entry.getKey()); } // 2. Vendor repos @@ -386,11 +404,9 @@ private void vendor( } private static Path getVendorPath(CommandEnvironment env, PathFragment vendorDirectory) { - Path vendorPath = - vendorDirectory.isAbsolute() - ? env.getRuntime().getFileSystem().getPath(vendorDirectory) - : env.getWorkspace().getRelative(vendorDirectory); - return vendorPath; + return vendorDirectory.isAbsolute() + ? env.getRuntime().getFileSystem().getPath(vendorDirectory) + : env.getWorkspace().getRelative(vendorDirectory); } private static BlazeCommandResult createFailedBlazeCommandResult( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index 8806f9abc028da..763eeec7b74044 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -63,7 +63,6 @@ import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionName; - import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import net.starlark.java.eval.StarlarkSemantics; diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java index 9fff42177d0ded..333aa82d197fe1 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java @@ -97,7 +97,11 @@ public void testHttpUrl() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); + server.getUrl() + "/myreg", + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue( ModuleFile.create( @@ -115,7 +119,11 @@ public void testHttpUrlWithNetrcCreds() throws Exception { "machine [::1] login rinne password rinnepass\n".getBytes(UTF_8))); Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); + server.getUrl() + "/myreg", + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); var e = assertThrows( @@ -198,7 +206,11 @@ public void testGetArchiveRepoSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -266,7 +278,11 @@ public void testGetLocalPathRepoSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( RepoSpec.builder() @@ -290,7 +306,11 @@ public void testGetRepoInvalidRegistryJsonSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -324,7 +344,11 @@ public void testGetRepoInvalidModuleJsonSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); } @@ -353,7 +377,11 @@ public void testGetYankedVersion() throws Exception { server.start(); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); Optional> yankedVersion = registry.getYankedVersions("red-pill", reporter); assertThat(yankedVersion) @@ -376,7 +404,11 @@ public void testArchiveWithExplicitType() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("archive_type", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -406,7 +438,11 @@ public void testGetModuleFileChecksums() throws Exception { Optional.of(sha256("unused"))); Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); + server.getUrl() + "/myreg", + LockfileMode.UPDATE, + knownFiles, + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue( ModuleFile.create( @@ -432,7 +468,11 @@ public void testGetModuleFileChecksums() throws Exception { registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of(), Optional.empty()); + server.getUrl() + "/myreg", + LockfileMode.UPDATE, + recordedChecksums, + ImmutableMap.of(), + Optional.empty()); // Test that the recorded hashes are used for repo cache hits even when the server content // changes. server.unserve("/myreg/modules/foo/1.0/MODULE.bazel"); @@ -462,7 +502,11 @@ public void testGetModuleFileChecksumMismatch() throws Exception { Optional.of(sha256("original"))); Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); + server.getUrl() + "/myreg", + LockfileMode.UPDATE, + knownFiles, + ImmutableMap.of(), + Optional.empty()); var e = assertThrows( IOException.class, @@ -523,7 +567,11 @@ public void testGetRepoSpecChecksum() throws Exception { registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of(), Optional.empty()); + server.getUrl(), + LockfileMode.UPDATE, + recordedChecksums, + ImmutableMap.of(), + Optional.empty()); // Test that the recorded hashes are used for repo cache hits even when the server content // changes. server.unserve("/bazel_registry.json"); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java index f29d6c5a704cb4..fb52c7fe216416 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java @@ -26,7 +26,6 @@ import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; import java.net.URISyntaxException; import java.util.Optional; - import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -46,7 +45,10 @@ public void badSchemes() { URISyntaxException.class, () -> registryFactory.createRegistry( - "/home/www", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), + "/home/www", + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), Optional.empty())); assertThat(exception).hasMessageThat().contains("Registry URL has no scheme"); exception = @@ -54,7 +56,11 @@ public void badSchemes() { URISyntaxException.class, () -> registryFactory.createRegistry( - "foo://bar", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty())); + "foo://bar", + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty())); assertThat(exception).hasMessageThat().contains("Unrecognized registry URL protocol"); } @@ -72,7 +78,8 @@ public void badPath() { "file:c:/path/to/workspace/registry", LockfileMode.UPDATE, ImmutableMap.of(), - ImmutableMap.of(), Optional.empty())); + ImmutableMap.of(), + Optional.empty())); assertThat(exception).hasMessageThat().contains("Registry URL path is not valid"); } } diff --git a/src/test/py/bazel/bzlmod/bazel_vendor_test.py b/src/test/py/bazel/bzlmod/bazel_vendor_test.py index 0b4479bc52e5a8..0034f05f163cd3 100644 --- a/src/test/py/bazel/bzlmod/bazel_vendor_test.py +++ b/src/test/py/bazel/bzlmod/bazel_vendor_test.py @@ -434,7 +434,7 @@ def testBuildingOutOfDateVendoredRepo(self): "WARNING: : Vendored repository '_main~ext~justRepo' is" ' out-of-date. The up-to-date version will be fetched into the external' ' cache and used. To update the repo in the vendor directory, run' - " the bazel vendor command", + ' the bazel vendor command', stderr, ) @@ -464,7 +464,7 @@ def testBuildingOutOfDateVendoredRepo(self): "WARNING: : Vendored repository '_main~ext~justRepo' is" ' out-of-date. The up-to-date version will be fetched into the external' ' cache and used. To update the repo in the vendor directory, run' - " the bazel vendor command", + ' the bazel vendor command', stderr, ) _, stdout, _ = self.RunBazel(['info', 'output_base']) @@ -480,7 +480,7 @@ def testBuildingOutOfDateVendoredRepo(self): "WARNING: : Vendored repository '_main~ext~justRepo' is" ' out-of-date. The up-to-date version will be fetched into the external' ' cache and used. To update the repo in the vendor directory, run' - " the bazel vendor command", + ' the bazel vendor command', stderr, ) @@ -542,7 +542,7 @@ def testBuildingVendoredRepoWithNoFetch(self): ) self.assertIn( 'ERROR: Vendored repository _main~ext~noVenRepo not found under the' - " vendor directory and fetching is disabled. To fix, run the bazel" + ' vendor directory and fetching is disabled. To fix, run the bazel' " vendor command or build without the '--nofetch'", stderr, ) @@ -620,13 +620,14 @@ def testBuildVendoredTargetOffline(self): self.RunBazel(['vendor', '//:main', '--vendor_dir=vendor']) - # Build and run the target in a clean build with internet blocked and make sure it works + # Build and run the target in a clean build with internet blocked and make + # sure it works _, _, _ = self.RunBazel(['clean', '--expunge']) _, stdout, _ = self.RunBazel( ['run', '//:main', '--vendor_dir=vendor', '--repository_cache='], - env_add= { - "HTTP_PROXY": "internet_blocked", - "HTTPS_PROXY": "internet_blocked", + env_add={ + 'HTTP_PROXY': 'internet_blocked', + 'HTTPS_PROXY': 'internet_blocked', }, ) self.assertIn('Hello there! => bbb@1.0', stdout) @@ -634,18 +635,19 @@ def testBuildVendoredTargetOffline(self): # Assert repos in {OUTPUT_BASE}/external are symlinks (junction on # windows, this validates it was created from vendor and not fetched) _, stdout, _ = self.RunBazel(['info', 'output_base']) - for repo in ["aaa~", "bbb~"]: - repo_path = stdout[0] + '/external/' + repo - if self.IsWindows(): - self.assertTrue(self.IsJunction(repo_path)) - else: - self.assertTrue(os.path.islink(repo_path)) + for repo in ['aaa~', 'bbb~']: + repo_path = stdout[0] + '/external/' + repo + if self.IsWindows(): + self.assertTrue(self.IsJunction(repo_path)) + else: + self.assertTrue(os.path.islink(repo_path)) def testVendorConflictRegistryFile(self): self.main_registry.createCcModule('aaa', '1.0').createCcModule( 'bbb', '1.0', {'aaa': '1.0'} ) - # The registry URLs of main_registry and another_registry only differ by the port number + # The registry URLs of main_registry and another_registry only differ by the + # port number another_registry = BazelRegistry( os.path.join(self.registries_work_dir, 'MAIN'), ) @@ -665,9 +667,15 @@ def testVendorConflictRegistryFile(self): ], ) self.ScratchFile('BUILD') - exit_code, _, stderr = self.RunBazel(['vendor', '--vendor_dir=vendor'], allow_failure=True) + exit_code, _, stderr = self.RunBazel( + ['vendor', '--vendor_dir=vendor'], allow_failure=True + ) self.AssertExitCode(exit_code, 8, stderr) - self.assertIn('ERROR: Error while vendoring repos: Vendor paths conflict detected for registry URLs:', stderr) + self.assertIn( + 'ERROR: Error while vendoring repos: Vendor paths conflict detected for' + ' registry URLs:', + stderr, + ) if __name__ == '__main__': From 205a1701ec0a4dbd66d2c081a692d3390ef50803 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Tue, 4 Jun 2024 13:47:09 +0200 Subject: [PATCH 04/14] Polish messages --- .../devtools/build/lib/bazel/commands/VendorCommand.java | 8 +++++--- src/test/py/bazel/bzlmod/bazel_vendor_test.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java index e86aadf472d45e..d2f1d6bee9eb5a 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java @@ -156,7 +156,6 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti getVendorPath(env, options.getOptions(RepositoryOptions.class).vendorDirectory); LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); try { - env.getReporter().handle(Event.info("Vendoring ...")); if (!options.getResidue().isEmpty()) { result = vendorTargets(env, options, options.getResidue(), vendorDirectory); } else if (!vendorOptions.repos.isEmpty()) { @@ -190,11 +189,11 @@ private BlazeCommandResult validateOptions(CommandEnvironment env, OptionsParsin return createFailedBlazeCommandResult( env.getReporter(), Code.OPTIONS_INVALID, - "You cannot run vendor without specifying --vendor_dir"); + "You cannot run the vendor command without specifying --vendor_dir"); } if (!options.getOptions(PackageOptions.class).fetch) { return createFailedBlazeCommandResult( - env.getReporter(), Code.OPTIONS_INVALID, "You cannot run vendor with --nofetch"); + env.getReporter(), Code.OPTIONS_INVALID, "You cannot run the vendor command with --nofetch"); } return null; } @@ -219,6 +218,7 @@ private BlazeCommandResult vendorAll( } BazelFetchAllValue fetchAllValue = (BazelFetchAllValue) evaluationResult.get(fetchKey); + env.getReporter().handle(Event.info("Vendoring all external repositories...")); vendor(env, vendorDirectory, fetchAllValue.getReposToVendor()); env.getReporter().handle(Event.info("All external dependencies vendored successfully.")); return BlazeCommandResult.success(); @@ -254,6 +254,7 @@ private BlazeCommandResult vendorRepos( } } + env.getReporter().handle(Event.info("Vendoring repositories...")); vendor(env, vendorDirectory, reposToVendor.build()); if (!notFoundRepoErrors.isEmpty()) { return createFailedBlazeCommandResult( @@ -291,6 +292,7 @@ private BlazeCommandResult vendorTargets( InMemoryGraph inMemoryGraph = env.getSkyframeExecutor().getEvaluator().getInMemoryGraph(); ImmutableSet reposToVendor = collectReposFromTargets(inMemoryGraph, targetKeys); + env.getReporter().handle(Event.info("Vendoring dependencies for targets...")); vendor(env, vendorDirectory, reposToVendor.asList()); env.getReporter() .handle( diff --git a/src/test/py/bazel/bzlmod/bazel_vendor_test.py b/src/test/py/bazel/bzlmod/bazel_vendor_test.py index 0034f05f163cd3..322d5a5a7c6574 100644 --- a/src/test/py/bazel/bzlmod/bazel_vendor_test.py +++ b/src/test/py/bazel/bzlmod/bazel_vendor_test.py @@ -117,7 +117,7 @@ def testVendorFailsWithNofetch(self): _, _, stderr = self.RunBazel( ['vendor', '--vendor_dir=vendor', '--nofetch'], allow_failure=True ) - self.assertIn('ERROR: You cannot run vendor with --nofetch', stderr) + self.assertIn('ERROR: You cannot run the vendor command with --nofetch', stderr) def testVendoringMultipleTimes(self): self.main_registry.createCcModule('aaa', '1.0') From a7882a103926275365edeccbf817b1d0f0aaa51a Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Tue, 4 Jun 2024 14:23:23 +0200 Subject: [PATCH 05/14] Refactor VendorUtil --- .../com/google/devtools/build/lib/bazel/BUILD | 1 + .../lib/bazel/BazelRepositoryModule.java | 3 +- .../build/lib/bazel/bzlmod/IndexRegistry.java | 36 ++++++++--------- .../build/lib/bazel/bzlmod/VendorUtil.java | 1 + .../lib/bazel/commands/VendorCommand.java | 39 ++++++++----------- 5 files changed, 39 insertions(+), 41 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD index 1ff603849fa87a..da461723be39cb 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD @@ -34,6 +34,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution_impl", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:tidy_impl", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:vendor", "//src/main/java/com/google/devtools/build/lib/bazel/commands", "//src/main/java/com/google/devtools/build/lib/bazel/repository", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 5d3dd5eef632e9..84158ce8a04eb9 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -55,6 +55,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionFunction; import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionUsagesFunction; import com.google.devtools.build.lib.bazel.bzlmod.VendorFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.VendorUtil; import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsFunction; import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.commands.FetchCommand; @@ -224,7 +225,7 @@ public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builde builder.addCommands(new FetchCommand()); builder.addCommands(new ModCommand()); builder.addCommands(new SyncCommand()); - builder.addCommands(new VendorCommand(downloadManager, clientEnvironmentSupplier)); + builder.addCommands(new VendorCommand(vendorDirectory, downloadManager, clientEnvironmentSupplier)); builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache)); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index f7e2ced1c5babd..5b944e23f029a0 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -47,6 +47,8 @@ import java.util.Map.Entry; import java.util.Optional; +import javax.annotation.Nullable; + /** * Represents a Bazel module registry that serves a list of module metadata from a static HTTP * server or a local file path. @@ -89,7 +91,8 @@ public enum KnownFileHashesMode { private final Gson gson; private final ImmutableMap> knownFileHashes; private final ImmutableMap previouslySelectedYankedVersions; - private final Optional vendorDir; + @Nullable + private final VendorUtil vendorUtil; private final KnownFileHashesMode knownFileHashesMode; private volatile Optional bazelRegistryJson; private volatile StoredEventHandler bazelRegistryJsonEvents; @@ -114,7 +117,7 @@ public IndexRegistry( this.knownFileHashes = knownFileHashes; this.knownFileHashesMode = knownFileHashesMode; this.previouslySelectedYankedVersions = previouslySelectedYankedVersions; - this.vendorDir = vendorDir; + this.vendorUtil = vendorDir.map(VendorUtil::new).orElse(null); } @Override @@ -188,25 +191,22 @@ private Optional doGrabFile( } URL url = URI.create(rawUrl).toURL(); - // Don't try to read the registry URL from the vendor directory in the following cases: - // 1. The vendor directory is not set, which means vendor mode is disabled. + // Don't read the registry URL from the vendor directory in the following cases: + // 1. vendorUtil is null, which means vendor mode is disabled. // 2. The checksum is not present, which means the URL is not vendored or the vendored content // is out-dated. // 3. The URL starts with "file:", which means it's a local file and isn't vendored. - // Otherwise, check if the URL is vendored and read the registry file from the vendor directory. - if (vendorDir.isPresent() && checksum.isPresent() && !url.getProtocol().equals("file")) { - VendorUtil vendorUtil = new VendorUtil(vendorDir.get()); - if (vendorUtil.isUrlVendored(url)) { - try { - return Optional.of(vendorUtil.readRegistryUrl(url, checksum.get())); - } catch (IOException e) { - throw new IOException( - String.format( - "Failed to read vendored registry file %s at %s: %s. Please rerun the bazel" - + " vendor command.", - rawUrl, vendorUtil.getVendorPathForUrl(url), e.getMessage()), - e); - } + // 4. The vendor path doesn't exist, which means the URL is not vendored. + if (vendorUtil != null && checksum.isPresent() && !url.getProtocol().equals("file") && vendorUtil.isUrlVendored(url)) { + try { + return Optional.of(vendorUtil.readRegistryUrl(url, checksum.get())); + } catch (IOException e) { + throw new IOException( + String.format( + "Failed to read vendored registry file %s at %s: %s. Please rerun the bazel" + + " vendor command.", + rawUrl, vendorUtil.getVendorPathForUrl(url), e.getMessage()), + e); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java index 748ef17ef5c478..d88a39d3933ad8 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java @@ -23,6 +23,7 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.Symlinks; + import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URL; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java index d2f1d6bee9eb5a..b778a0ad8e7d15 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java @@ -53,7 +53,6 @@ import com.google.devtools.build.lib.util.DetailedExitCode; import com.google.devtools.build.lib.util.InterruptedFailureDetails; import com.google.devtools.build.lib.vfs.Path; -import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; import com.google.devtools.build.skyframe.InMemoryGraph; @@ -63,6 +62,7 @@ import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.common.options.OptionsParser; import com.google.devtools.common.options.OptionsParsingResult; + import java.io.IOException; import java.net.URI; import java.net.URL; @@ -79,6 +79,7 @@ import java.util.Queue; import java.util.Set; import java.util.function.Supplier; + import javax.annotation.Nullable; /** @@ -112,9 +113,13 @@ public final class VendorCommand implements BlazeCommand { private final DownloadManager downloadManager; private final Supplier> clientEnvironmentSupplier; + @Nullable + private final VendorUtil vendorUtil; public VendorCommand( + Optional vendorDir, DownloadManager downloadManager, Supplier> clientEnvironmentSupplier) { + this.vendorUtil = vendorDir.map(VendorUtil::new).orElse(null); this.downloadManager = downloadManager; this.clientEnvironmentSupplier = clientEnvironmentSupplier; } @@ -152,16 +157,14 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti BlazeCommandResult result; VendorOptions vendorOptions = options.getOptions(VendorOptions.class); - Path vendorDirectory = - getVendorPath(env, options.getOptions(RepositoryOptions.class).vendorDirectory); LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); try { if (!options.getResidue().isEmpty()) { - result = vendorTargets(env, options, options.getResidue(), vendorDirectory); + result = vendorTargets(env, options, options.getResidue()); } else if (!vendorOptions.repos.isEmpty()) { - result = vendorRepos(env, threadsOption, vendorOptions.repos, vendorDirectory); + result = vendorRepos(env, threadsOption, vendorOptions.repos); } else { - result = vendorAll(env, threadsOption, vendorDirectory); + result = vendorAll(env, threadsOption); } } catch (InterruptedException e) { return createFailedBlazeCommandResult( @@ -199,7 +202,7 @@ private BlazeCommandResult validateOptions(CommandEnvironment env, OptionsParsin } private BlazeCommandResult vendorAll( - CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, Path vendorDirectory) + CommandEnvironment env, LoadingPhaseThreadsOption threadsOption) throws InterruptedException, IOException { EvaluationContext evaluationContext = EvaluationContext.newBuilder() @@ -219,7 +222,7 @@ private BlazeCommandResult vendorAll( BazelFetchAllValue fetchAllValue = (BazelFetchAllValue) evaluationResult.get(fetchKey); env.getReporter().handle(Event.info("Vendoring all external repositories...")); - vendor(env, vendorDirectory, fetchAllValue.getReposToVendor()); + vendor(env, fetchAllValue.getReposToVendor()); env.getReporter().handle(Event.info("All external dependencies vendored successfully.")); return BlazeCommandResult.success(); } @@ -227,8 +230,7 @@ private BlazeCommandResult vendorAll( private BlazeCommandResult vendorRepos( CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, - List repos, - Path vendorDirectory) + List repos) throws InterruptedException, IOException { ImmutableMap repositoryNamesAndValues; try { @@ -255,7 +257,7 @@ private BlazeCommandResult vendorRepos( } env.getReporter().handle(Event.info("Vendoring repositories...")); - vendor(env, vendorDirectory, reposToVendor.build()); + vendor(env, reposToVendor.build()); if (!notFoundRepoErrors.isEmpty()) { return createFailedBlazeCommandResult( env.getReporter(), "Vendoring some repos failed with errors: " + notFoundRepoErrors); @@ -267,8 +269,7 @@ private BlazeCommandResult vendorRepos( private BlazeCommandResult vendorTargets( CommandEnvironment env, OptionsParsingResult options, - List targets, - Path vendorDirectory) + List targets) throws InterruptedException, IOException { // Call fetch which runs build to have the targets graph and configuration set BuildResult buildResult; @@ -293,7 +294,7 @@ private BlazeCommandResult vendorTargets( ImmutableSet reposToVendor = collectReposFromTargets(inMemoryGraph, targetKeys); env.getReporter().handle(Event.info("Vendoring dependencies for targets...")); - vendor(env, vendorDirectory, reposToVendor.asList()); + vendor(env, reposToVendor.asList()); env.getReporter() .handle( Event.info( @@ -329,9 +330,9 @@ private ImmutableSet collectReposFromTargets( * ignored or was already vendored and up-to-date */ private void vendor( - CommandEnvironment env, Path vendorDirectory, ImmutableList reposToVendor) + CommandEnvironment env, ImmutableList reposToVendor) throws IOException, InterruptedException { - VendorUtil vendorUtil = new VendorUtil(vendorDirectory); + Objects.requireNonNull(vendorUtil); // 1. Vendor registry files BazelModuleResolutionValue moduleResolutionValue = @@ -405,12 +406,6 @@ private void vendor( vendorUtil.vendorRepos(externalPath, reposToVendor); } - private static Path getVendorPath(CommandEnvironment env, PathFragment vendorDirectory) { - return vendorDirectory.isAbsolute() - ? env.getRuntime().getFileSystem().getPath(vendorDirectory) - : env.getWorkspace().getRelative(vendorDirectory); - } - private static BlazeCommandResult createFailedBlazeCommandResult( Reporter reporter, Code fetchCommandCode, String message) { return createFailedBlazeCommandResult( From 7ae8e0a5538763bd4cda8c553f1ba37c3cbf6433 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Tue, 4 Jun 2024 14:41:32 +0200 Subject: [PATCH 06/14] Add comment for parallelizing vendoring --- .../com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java index d88a39d3933ad8..f5ee31bf0bfe00 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java @@ -43,6 +43,8 @@ public VendorUtil(Path vendorDirectory) { /** * Vendors the specified repositories under the vendor directory. * + * TODO(pcloudy): Parallelize vendoring repos + * * @param externalRepoRoot The root directory of the external repositories. * @param reposToVendor The list of repositories to vendor. * @throws IOException if an I/O error occurs. From bc00d2b8cf27971991e29c71bd6b1e3a6f151ac0 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Tue, 4 Jun 2024 15:10:25 +0200 Subject: [PATCH 07/14] Refactor marker file name --- .../build/lib/bazel/bzlmod/VendorUtil.java | 14 +++++++------- .../devtools/build/lib/cmdline/RepositoryName.java | 5 +++++ .../repository/RepositoryDelegatorFunction.java | 12 ++++++------ 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java index f5ee31bf0bfe00..1f19b719b3df97 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java @@ -57,7 +57,7 @@ public void vendorRepos(Path externalRepoRoot, ImmutableList rep for (RepositoryName repo : reposToVendor) { // Only re-vendor the repository if it is not up-to-date. - if (!isRepoUpToDate(repo.getName(), externalRepoRoot)) { + if (!isRepoUpToDate(repo, externalRepoRoot)) { Path repoUnderVendor = vendorDirectory.getRelative(repo.getName()); if (!repoUnderVendor.exists()) { repoUnderVendor.createDirectory(); @@ -65,8 +65,8 @@ public void vendorRepos(Path externalRepoRoot, ImmutableList rep FileSystemUtils.copyTreesBelow( externalRepoRoot.getRelative(repo.getName()), repoUnderVendor, Symlinks.NOFOLLOW); FileSystemUtils.copyFile( - externalRepoRoot.getChild("@" + repo.getName() + ".marker"), - vendorDirectory.getChild("@" + repo.getName() + ".marker")); + externalRepoRoot.getChild(repo.getMarkerFileName()), + vendorDirectory.getChild(repo.getMarkerFileName())); } } } @@ -128,13 +128,13 @@ public byte[] readRegistryUrl(URL url, Checksum checksum) throws IOException { * @return true if the repository is up-to-date, false otherwise. * @throws IOException if an I/O error occurs. */ - private boolean isRepoUpToDate(String repoName, Path externalPath) throws IOException { - Path vendorMarkerFile = vendorDirectory.getChild("@" + repoName + ".marker"); + private boolean isRepoUpToDate(RepositoryName repo, Path externalPath) throws IOException { + Path vendorMarkerFile = vendorDirectory.getChild(repo.getMarkerFileName()); if (!vendorMarkerFile.exists()) { return false; } - Path externalMarkerFile = externalPath.getChild("@" + repoName + ".marker"); + Path externalMarkerFile = externalPath.getChild(repo.getMarkerFileName()); String vendorMarkerContent = FileSystemUtils.readContent(vendorMarkerFile, UTF_8); String externalMarkerContent = FileSystemUtils.readContent(externalMarkerFile, UTF_8); return Objects.equals(vendorMarkerContent, externalMarkerContent); @@ -148,7 +148,7 @@ private boolean isRepoUpToDate(String repoName, Path externalPath) throws IOExce *

The host name is case-insensitive, so it is converted to lowercase. The path is * case-sensitive, so it is left as is. The port number is not included in the vendor path. * - *

Note that the vendor path may conflicts if two URLs only differ by the case or port number. + *

Note that the vendor path may conflict if two URLs only differ by the case or port number. * But this is unlikely to happen in practice, and conflicts are checked in VendorCommand.java. * * @param url The URL to get the vendor path for. diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryName.java b/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryName.java index b2f98e579a368f..588833aa670ad2 100644 --- a/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryName.java +++ b/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryName.java @@ -192,6 +192,11 @@ public String getName() { return name; } + /** Returns the marker file name for this repository. */ + public String getMarkerFileName() { + return "@" + name + ".marker"; + } + /** * Create a {@link RepositoryName} instance that indicates the requested repository name is * actually not visible from the owner repository and should fail in {@code diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java index 72bf3268d15a9d..7dd616f0577741 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java @@ -274,7 +274,7 @@ private RepositoryDirectoryValue tryGettingValueUsingVendoredRepo( Path vendorPath = VENDOR_DIRECTORY.get(env).get(); Path vendorRepoPath = vendorPath.getRelative(repositoryName.getName()); if (vendorRepoPath.exists()) { - Path vendorMarker = vendorPath.getChild("@" + repositoryName.getName() + ".marker"); + Path vendorMarker = vendorPath.getChild(repositoryName.getMarkerFileName()); if (vendorFile.getPinnedRepos().contains(repositoryName)) { // pinned repos are used as they are without checking their marker file try { @@ -621,7 +621,7 @@ private static class DigestWriter { StarlarkSemantics starlarkSemantics) { this.directories = directories; ruleKey = computeRuleKey(rule, starlarkSemantics); - markerPath = getMarkerPath(directories, repositoryName.getName()); + markerPath = getMarkerPath(directories, repositoryName); this.rule = rule; recordedInputValues = Maps.newTreeMap(); } @@ -731,15 +731,15 @@ private String computeRuleKey(Rule rule, StarlarkSemantics starlarkSemantics) { .hexDigestAndReset(); } - private static Path getMarkerPath(BlazeDirectories directories, String ruleName) { + private static Path getMarkerPath(BlazeDirectories directories, RepositoryName repo) { return RepositoryFunction.getExternalRepositoryDirectory(directories) - .getChild("@" + ruleName + ".marker"); + .getChild(repo.getMarkerFileName()); } - static void clearMarkerFile(BlazeDirectories directories, RepositoryName repoName) + static void clearMarkerFile(BlazeDirectories directories, RepositoryName repo) throws RepositoryFunctionException { try { - getMarkerPath(directories, repoName.getName()).delete(); + getMarkerPath(directories, repo).delete(); } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } From 2a01092f0693853158c12e3993c39786e5fb7466 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Tue, 4 Jun 2024 15:34:19 +0200 Subject: [PATCH 08/14] Fix vendorDirectory init --- .../devtools/build/lib/bazel/BazelRepositoryModule.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 84158ce8a04eb9..08a42acd14cc22 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -166,7 +166,7 @@ public class BazelRepositoryModule extends BlazeModule { private Clock clock; private Instant lastRegistryInvalidation = Instant.EPOCH; - private Optional vendorDirectory; + private Optional vendorDirectory = Optional.empty(); private List allowedYankedVersions = ImmutableList.of(); private boolean disableNativeRepoRules; private SingleExtensionEvalFunction singleExtensionEvalFunction; @@ -503,8 +503,6 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { repoOptions.vendorDirectory.isAbsolute() ? filesystem.getPath(repoOptions.vendorDirectory) : env.getWorkspace().getRelative(repoOptions.vendorDirectory)); - } else { - vendorDirectory = Optional.empty(); } if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) { From 25fb936fbc9492e613e35793f0b024ca9fd2df77 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Tue, 4 Jun 2024 15:59:48 +0200 Subject: [PATCH 09/14] Fix init VendorCommand.java --- .../build/lib/bazel/BazelRepositoryModule.java | 2 +- .../build/lib/bazel/bzlmod/VendorUtil.java | 2 +- .../build/lib/bazel/commands/VendorCommand.java | 14 +++++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 08a42acd14cc22..e42f63255bb5be 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -225,7 +225,7 @@ public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builde builder.addCommands(new FetchCommand()); builder.addCommands(new ModCommand()); builder.addCommands(new SyncCommand()); - builder.addCommands(new VendorCommand(vendorDirectory, downloadManager, clientEnvironmentSupplier)); + builder.addCommands(new VendorCommand(downloadManager, clientEnvironmentSupplier)); builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache)); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java index 1f19b719b3df97..e4fd75d28102c3 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java @@ -123,7 +123,7 @@ public byte[] readRegistryUrl(URL url, Checksum checksum) throws IOException { * the one under /external. This function assumes the marker file under * /external exists and is up-to-date. * - * @param repoName The name of the repository. + * @param repo The name of the repository. * @param externalPath The root directory of the external repositories. * @return true if the repository is up-to-date, false otherwise. * @throws IOException if an I/O error occurs. diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java index b778a0ad8e7d15..eb06b1f59b4708 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java @@ -53,6 +53,7 @@ import com.google.devtools.build.lib.util.DetailedExitCode; import com.google.devtools.build.lib.util.InterruptedFailureDetails; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; import com.google.devtools.build.skyframe.InMemoryGraph; @@ -114,12 +115,10 @@ public final class VendorCommand implements BlazeCommand { private final DownloadManager downloadManager; private final Supplier> clientEnvironmentSupplier; @Nullable - private final VendorUtil vendorUtil; + private VendorUtil vendorUtil = null; public VendorCommand( - Optional vendorDir, DownloadManager downloadManager, Supplier> clientEnvironmentSupplier) { - this.vendorUtil = vendorDir.map(VendorUtil::new).orElse(null); this.downloadManager = downloadManager; this.clientEnvironmentSupplier = clientEnvironmentSupplier; } @@ -158,6 +157,9 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti BlazeCommandResult result; VendorOptions vendorOptions = options.getOptions(VendorOptions.class); LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); + Path vendorDirectory = + getVendorPath(env, options.getOptions(RepositoryOptions.class).vendorDirectory); + this.vendorUtil = new VendorUtil(vendorDirectory); try { if (!options.getResidue().isEmpty()) { result = vendorTargets(env, options, options.getResidue()); @@ -406,6 +408,12 @@ private void vendor( vendorUtil.vendorRepos(externalPath, reposToVendor); } + private static Path getVendorPath(CommandEnvironment env, PathFragment vendorDirectory) { + return vendorDirectory.isAbsolute() + ? env.getRuntime().getFileSystem().getPath(vendorDirectory) + : env.getWorkspace().getRelative(vendorDirectory); + } + private static BlazeCommandResult createFailedBlazeCommandResult( Reporter reporter, Code fetchCommandCode, String message) { return createFailedBlazeCommandResult( From 766a6fea49f47006be688aec7f1a7e8dc38523c8 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Tue, 4 Jun 2024 16:05:32 +0200 Subject: [PATCH 10/14] Make marker file copy atomic --- .../google/devtools/build/lib/bazel/bzlmod/VendorUtil.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java index e4fd75d28102c3..23caf636db0d1f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java @@ -64,9 +64,11 @@ public void vendorRepos(Path externalRepoRoot, ImmutableList rep } FileSystemUtils.copyTreesBelow( externalRepoRoot.getRelative(repo.getName()), repoUnderVendor, Symlinks.NOFOLLOW); + Path tmpMarkerFile = vendorDirectory.getChild(repo.getMarkerFileName() + ".tmp"); FileSystemUtils.copyFile( externalRepoRoot.getChild(repo.getMarkerFileName()), - vendorDirectory.getChild(repo.getMarkerFileName())); + tmpMarkerFile); + tmpMarkerFile.renameTo(vendorDirectory.getChild(repo.getMarkerFileName())); } } } From 5ab21440b0ef22313a44ab9c31efbe30888fa201 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Wed, 5 Jun 2024 14:26:20 +0200 Subject: [PATCH 11/14] Address reviewer comments --- .../build/lib/bazel/BazelRepositoryModule.java | 10 +++------- .../devtools/build/lib/bazel/bzlmod/VendorUtil.java | 6 ++++-- .../build/lib/bazel/commands/VendorCommand.java | 9 +-------- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index e42f63255bb5be..0eea3e3858ab91 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -496,13 +496,9 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { bazelCompatibilityMode = repoOptions.bazelCompatibilityMode; bazelLockfileMode = repoOptions.lockfileMode; allowedYankedVersions = repoOptions.allowedYankedVersions; - - if (repoOptions.vendorDirectory != null) { - vendorDirectory = - Optional.of( - repoOptions.vendorDirectory.isAbsolute() - ? filesystem.getPath(repoOptions.vendorDirectory) - : env.getWorkspace().getRelative(repoOptions.vendorDirectory)); + if (env.getWorkspace() != null) { + vendorDirectory = Optional.ofNullable(repoOptions.vendorDirectory) + .map(vendorDirectory -> env.getWorkspace().getRelative(vendorDirectory)); } if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java index 23caf636db0d1f..3d2d0783e5a848 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java @@ -34,6 +34,8 @@ /** Utility class for vendoring external repositories. */ public class VendorUtil { + private final static String REGISTRIES_DIR = "_registries"; + private final Path vendorDirectory; public VendorUtil(Path vendorDirectory) { @@ -121,7 +123,7 @@ public byte[] readRegistryUrl(URL url, Checksum checksum) throws IOException { } /** - * Checks if the repository under vendor dir needs to be updated by comparing its marker file with + * Checks if the repository under vendor dir is up-to-date by comparing its marker file with * the one under /external. This function assumes the marker file under * /external exists and is up-to-date. * @@ -164,6 +166,6 @@ public Path getVendorPathForUrl(URL url) throws UnsupportedEncodingException { if (path.startsWith("/")) { path = path.substring(1); } - return vendorDirectory.getRelative("registry_cache").getRelative(host).getRelative(path); + return vendorDirectory.getRelative(REGISTRIES_DIR).getRelative(host).getRelative(path); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java index eb06b1f59b4708..c3782245d71305 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java @@ -157,8 +157,7 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti BlazeCommandResult result; VendorOptions vendorOptions = options.getOptions(VendorOptions.class); LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); - Path vendorDirectory = - getVendorPath(env, options.getOptions(RepositoryOptions.class).vendorDirectory); + Path vendorDirectory = env.getWorkspace().getRelative(options.getOptions(RepositoryOptions.class).vendorDirectory); this.vendorUtil = new VendorUtil(vendorDirectory); try { if (!options.getResidue().isEmpty()) { @@ -408,12 +407,6 @@ private void vendor( vendorUtil.vendorRepos(externalPath, reposToVendor); } - private static Path getVendorPath(CommandEnvironment env, PathFragment vendorDirectory) { - return vendorDirectory.isAbsolute() - ? env.getRuntime().getFileSystem().getPath(vendorDirectory) - : env.getWorkspace().getRelative(vendorDirectory); - } - private static BlazeCommandResult createFailedBlazeCommandResult( Reporter reporter, Code fetchCommandCode, String message) { return createFailedBlazeCommandResult( From f205946817dd6903d3a5c21c9d1b9de62a275fc7 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Wed, 5 Jun 2024 14:43:55 +0200 Subject: [PATCH 12/14] clean up before vendoring --- .../build/lib/bazel/bzlmod/VendorUtil.java | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java index 3d2d0783e5a848..506b20f4fae850 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java @@ -60,17 +60,22 @@ public void vendorRepos(Path externalRepoRoot, ImmutableList rep for (RepositoryName repo : reposToVendor) { // Only re-vendor the repository if it is not up-to-date. if (!isRepoUpToDate(repo, externalRepoRoot)) { + Path markerUnderVendor = vendorDirectory.getChild(repo.getMarkerFileName()); Path repoUnderVendor = vendorDirectory.getRelative(repo.getName()); - if (!repoUnderVendor.exists()) { - repoUnderVendor.createDirectory(); - } + + // 1. Clean up existing marker file and repo vendor directory + markerUnderVendor.delete(); + repoUnderVendor.deleteTree(); + repoUnderVendor.createDirectory(); + + // 2. Copy over the repo source. FileSystemUtils.copyTreesBelow( externalRepoRoot.getRelative(repo.getName()), repoUnderVendor, Symlinks.NOFOLLOW); - Path tmpMarkerFile = vendorDirectory.getChild(repo.getMarkerFileName() + ".tmp"); - FileSystemUtils.copyFile( - externalRepoRoot.getChild(repo.getMarkerFileName()), - tmpMarkerFile); - tmpMarkerFile.renameTo(vendorDirectory.getChild(repo.getMarkerFileName())); + + // 3. Copy the marker file atomically + Path tmpMarker = vendorDirectory.getChild(repo.getMarkerFileName() + ".tmp"); + FileSystemUtils.copyFile(externalRepoRoot.getChild(repo.getMarkerFileName()), tmpMarker); + tmpMarker.renameTo(markerUnderVendor); } } } From dd3e0a55cf710c1e45294375417e519c35c6b25b Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Wed, 5 Jun 2024 15:20:49 +0200 Subject: [PATCH 13/14] Add test case to check re-vendor --- src/test/py/bazel/bzlmod/bazel_vendor_test.py | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/test/py/bazel/bzlmod/bazel_vendor_test.py b/src/test/py/bazel/bzlmod/bazel_vendor_test.py index 322d5a5a7c6574..d1ca73787b5a1f 100644 --- a/src/test/py/bazel/bzlmod/bazel_vendor_test.py +++ b/src/test/py/bazel/bzlmod/bazel_vendor_test.py @@ -79,7 +79,7 @@ def generateBuiltinModules(self): def testBasicVendoring(self): self.main_registry.createCcModule('aaa', '1.0').createCcModule( 'bbb', '1.0', {'aaa': '1.0'} - ) + ).createCcModule('bbb', '2.0') self.ScratchFile( 'MODULE.bazel', [ @@ -94,13 +94,31 @@ def testBasicVendoring(self): self.RunBazel(['vendor', '--vendor_dir=vendor']) # Assert repos are vendored with marker files and VENDOR.bazel is created - repos_vendored = os.listdir(self._test_cwd + '/vendor') + vendor_dir = self._test_cwd + '/vendor' + repos_vendored = os.listdir(vendor_dir) self.assertIn('aaa~', repos_vendored) self.assertIn('bbb~', repos_vendored) self.assertIn('@aaa~.marker', repos_vendored) self.assertIn('@bbb~.marker', repos_vendored) self.assertIn('VENDOR.bazel', repos_vendored) + # Update bbb to 2.0 and re-vendor + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "bbb", version = "2.0")', + 'local_path_override(module_name="bazel_tools", path="tools_mock")', + 'local_path_override(module_name="local_config_platform", ', + 'path="platforms_mock")', + ], + ) + self.ScratchFile("vendor/bbb~/foo") + self.RunBazel(['vendor', '--vendor_dir=vendor']) + bbb_module_bazel = os.path.join(vendor_dir, 'bbb~/MODULE.bazel') + self.AssertFileContentContains(bbb_module_bazel, 'version = "2.0"') + foo = os.path.join(vendor_dir, 'bbb~/foo') + self.assertFalse(os.path.exists(foo)) # foo should be removed due to re-vendor + def testVendorFailsWithNofetch(self): self.ScratchFile( 'MODULE.bazel', From 076de9f75990a1f934cd52d07eeab12c7ef5ade5 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Thu, 6 Jun 2024 18:03:39 +0200 Subject: [PATCH 14/14] Rename VendorUtil to VendorManager --- .../lib/bazel/BazelRepositoryModule.java | 1 - .../devtools/build/lib/bazel/bzlmod/BUILD | 2 +- .../build/lib/bazel/bzlmod/IndexRegistry.java | 4 ++-- .../{VendorUtil.java => VendorManager.java} | 4 ++-- .../lib/bazel/commands/VendorCommand.java | 22 +++++++++---------- 5 files changed, 15 insertions(+), 18 deletions(-) rename src/main/java/com/google/devtools/build/lib/bazel/bzlmod/{VendorUtil.java => VendorManager.java} (98%) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 0eea3e3858ab91..64b05810dfe81f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -55,7 +55,6 @@ import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionFunction; import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionUsagesFunction; import com.google.devtools.build.lib.bazel.bzlmod.VendorFileFunction; -import com.google.devtools.build.lib.bazel.bzlmod.VendorUtil; import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsFunction; import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.commands.FetchCommand; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 5e761993a4a95f..191dd183841d13 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -48,7 +48,7 @@ java_library( java_library( name = "vendor", - srcs = ["VendorUtil.java"], + srcs = ["VendorManager.java"], deps = [ "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/cmdline", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index 5b944e23f029a0..984be513760466 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -92,7 +92,7 @@ public enum KnownFileHashesMode { private final ImmutableMap> knownFileHashes; private final ImmutableMap previouslySelectedYankedVersions; @Nullable - private final VendorUtil vendorUtil; + private final VendorManager vendorUtil; private final KnownFileHashesMode knownFileHashesMode; private volatile Optional bazelRegistryJson; private volatile StoredEventHandler bazelRegistryJsonEvents; @@ -117,7 +117,7 @@ public IndexRegistry( this.knownFileHashes = knownFileHashes; this.knownFileHashesMode = knownFileHashesMode; this.previouslySelectedYankedVersions = previouslySelectedYankedVersions; - this.vendorUtil = vendorDir.map(VendorUtil::new).orElse(null); + this.vendorUtil = vendorDir.map(VendorManager::new).orElse(null); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorManager.java similarity index 98% rename from src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java rename to src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorManager.java index 506b20f4fae850..67acb0fe159715 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorManager.java @@ -32,13 +32,13 @@ import java.util.Objects; /** Utility class for vendoring external repositories. */ -public class VendorUtil { +public class VendorManager { private final static String REGISTRIES_DIR = "_registries"; private final Path vendorDirectory; - public VendorUtil(Path vendorDirectory) { + public VendorManager(Path vendorDirectory) { this.vendorDirectory = vendorDirectory; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java index c3782245d71305..7082496087e627 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java @@ -22,7 +22,7 @@ import com.google.devtools.build.lib.analysis.NoBuildRequestFinishedEvent; import com.google.devtools.build.lib.bazel.bzlmod.BazelFetchAllValue; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionValue; -import com.google.devtools.build.lib.bazel.bzlmod.VendorUtil; +import com.google.devtools.build.lib.bazel.bzlmod.VendorManager; import com.google.devtools.build.lib.bazel.commands.RepositoryFetcher.RepositoryFetcherException; import com.google.devtools.build.lib.bazel.commands.TargetFetcher.TargetFetcherException; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; @@ -53,7 +53,6 @@ import com.google.devtools.build.lib.util.DetailedExitCode; import com.google.devtools.build.lib.util.InterruptedFailureDetails; import com.google.devtools.build.lib.vfs.Path; -import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; import com.google.devtools.build.skyframe.InMemoryGraph; @@ -107,15 +106,14 @@ usesConfigurationOptions = true, help = "resource:vendor.txt", shortDescription = - "Fetches external repositories into a specific folder specified by the flag " - + "--vendor_dir.") + "Fetches external repositories into a folder specified by the flag --vendor_dir.") public final class VendorCommand implements BlazeCommand { public static final String NAME = "vendor"; private final DownloadManager downloadManager; private final Supplier> clientEnvironmentSupplier; @Nullable - private VendorUtil vendorUtil = null; + private VendorManager vendorManager = null; public VendorCommand( DownloadManager downloadManager, Supplier> clientEnvironmentSupplier) { @@ -158,7 +156,7 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti VendorOptions vendorOptions = options.getOptions(VendorOptions.class); LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); Path vendorDirectory = env.getWorkspace().getRelative(options.getOptions(RepositoryOptions.class).vendorDirectory); - this.vendorUtil = new VendorUtil(vendorDirectory); + this.vendorManager = new VendorManager(vendorDirectory); try { if (!options.getResidue().isEmpty()) { result = vendorTargets(env, options, options.getResidue()); @@ -333,7 +331,7 @@ private ImmutableSet collectReposFromTargets( private void vendor( CommandEnvironment env, ImmutableList reposToVendor) throws IOException, InterruptedException { - Objects.requireNonNull(vendorUtil); + Objects.requireNonNull(vendorManager); // 1. Vendor registry files BazelModuleResolutionValue moduleResolutionValue = @@ -360,7 +358,7 @@ private void vendor( continue; } - String outputPath = vendorUtil.getVendorPathForUrl(url).getPathString(); + String outputPath = vendorManager.getVendorPathForUrl(url).getPathString(); String outputPathLowerCase = outputPath.toLowerCase(Locale.ROOT); if (vendorPathToUrl.containsKey(outputPathLowerCase)) { String previousUrl = vendorPathToUrl.get(outputPathLowerCase); @@ -373,18 +371,18 @@ private void vendor( + " cause conflict on case insensitive file systems, please fix by changing the" + " registry URLs!", previousUrl, - vendorUtil.getVendorPathForUrl(URI.create(previousUrl).toURL()).getPathString(), + vendorManager.getVendorPathForUrl(URI.create(previousUrl).toURL()).getPathString(), entry.getKey(), outputPath)); } Optional checksum = entry.getValue(); - if (!vendorUtil.isUrlVendored(url) + if (!vendorManager.isUrlVendored(url) // Only vendor a registry URL when its checksum exists, otherwise the URL should be // recorded as "not found" in moduleResolutionValue.getRegistryFileHashes() && checksum.isPresent()) { try { - vendorUtil.vendorRegistryUrl( + vendorManager.vendorRegistryUrl( url, downloadManager.downloadAndReadOneUrlForBzlmod( url, env.getReporter(), clientEnvironmentSupplier.get(), checksum)); @@ -404,7 +402,7 @@ private void vendor( env.getDirectories() .getOutputBase() .getRelative(LabelConstants.EXTERNAL_REPOSITORY_LOCATION); - vendorUtil.vendorRepos(externalPath, reposToVendor); + vendorManager.vendorRepos(externalPath, reposToVendor); } private static BlazeCommandResult createFailedBlazeCommandResult(