From 0747210dad33cb3fb98ad511c6cab309d49a33a8 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Mon, 27 May 2024 17:30:33 +0200 Subject: [PATCH] Vendor registry files - Vendor registry files needed for Bazel module resolution to achieve offline build with vendor mode. - Also refactored bazel_vendor_test to avoid vendoring dependencies of bazel_tools, which speeds up the test significantly. Fixes https://github.com/bazelbuild/bazel/issues/22554 --- .../lib/bazel/BazelRepositoryModule.java | 2 +- .../devtools/build/lib/bazel/bzlmod/BUILD | 14 ++ .../bzlmod/BazelModuleResolutionValue.java | 2 +- .../build/lib/bazel/bzlmod/IndexRegistry.java | 38 +++- .../lib/bazel/bzlmod/RegistryFactory.java | 5 +- .../lib/bazel/bzlmod/RegistryFactoryImpl.java | 8 +- .../lib/bazel/bzlmod/RegistryFunction.java | 7 +- .../build/lib/bazel/bzlmod/VendorUtil.java | 162 ++++++++++++++++++ .../devtools/build/lib/bazel/commands/BUILD | 2 + .../lib/bazel/commands/VendorCommand.java | 130 ++++++++------ .../RepositoryDelegatorFunction.java | 6 +- .../devtools/build/lib/bazel/bzlmod/BUILD | 1 + .../bzlmod/BzlmodRepoRuleFunctionTest.java | 4 + .../build/lib/bazel/bzlmod/FakeRegistry.java | 4 +- .../lib/bazel/bzlmod/IndexRegistryTest.java | 33 ++-- .../lib/bazel/bzlmod/RegistryFactoryTest.java | 9 +- src/test/py/bazel/bzlmod/bazel_vendor_test.py | 116 +++++++------ 17 files changed, 405 insertions(+), 138 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 81bbf1c016ce30..5d3dd5eef632e9 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -224,7 +224,7 @@ public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builde builder.addCommands(new FetchCommand()); builder.addCommands(new ModCommand()); builder.addCommands(new SyncCommand()); - builder.addCommands(new VendorCommand()); + builder.addCommands(new VendorCommand(downloadManager, clientEnvironmentSupplier)); builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache)); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index cd9c2b40c6a673..f9a6bd9b6c4564 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -46,6 +46,18 @@ java_library( ], ) +java_library( + name = "vendor", + srcs = ["VendorUtil.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", + "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//third_party:guava", + ], +) + java_library( name = "module_extension", srcs = [ @@ -80,6 +92,7 @@ java_library( ], deps = [ ":common", + ":vendor", ":yanked_versions_value", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", @@ -87,6 +100,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/profiler", "//src/main/java/com/google/devtools/build/lib/util:os", + "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//third_party:gson", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java index 3dc6849db48f4b..e776d19d317811 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java @@ -48,7 +48,7 @@ public abstract class BazelModuleResolutionValue implements SkyValue { /** * Hashes of files obtained (or known to be missing) from registries while performing resolution. */ - abstract ImmutableMap> getRegistryFileHashes(); + public abstract ImmutableMap> getRegistryFileHashes(); /** * Selected module versions that are known to be yanked (and hence must have been explicitly diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index 57881065649dd6..eb065eac900d27 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -32,11 +32,13 @@ import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.profiler.SilentCloseable; import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.gson.FieldNamingPolicy; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonParseException; + import java.io.FileNotFoundException; import java.io.IOException; import java.net.MalformedURLException; @@ -88,6 +90,7 @@ public enum KnownFileHashesMode { private final Gson gson; private final ImmutableMap> knownFileHashes; private final ImmutableMap previouslySelectedYankedVersions; + private final Optional vendorDir; private final KnownFileHashesMode knownFileHashesMode; private volatile Optional bazelRegistryJson; private volatile StoredEventHandler bazelRegistryJsonEvents; @@ -100,7 +103,8 @@ public IndexRegistry( Map clientEnv, ImmutableMap> knownFileHashes, KnownFileHashesMode knownFileHashesMode, - ImmutableMap previouslySelectedYankedVersions) { + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) { this.uri = uri; this.downloadManager = downloadManager; this.clientEnv = clientEnv; @@ -111,6 +115,7 @@ public IndexRegistry( this.knownFileHashes = knownFileHashes; this.knownFileHashesMode = knownFileHashesMode; this.previouslySelectedYankedVersions = previouslySelectedYankedVersions; + this.vendorDir = vendorDir; } @Override @@ -143,11 +148,11 @@ private Optional grabFile( } private Optional doGrabFile( - String url, ExtendedEventHandler eventHandler, boolean useChecksum) + String rawURL, ExtendedEventHandler eventHandler, boolean useChecksum) throws IOException, InterruptedException { Optional checksum; if (knownFileHashesMode != KnownFileHashesMode.IGNORE && useChecksum) { - Optional knownChecksum = knownFileHashes.get(url); + Optional knownChecksum = knownFileHashes.get(rawURL); if (knownChecksum == null) { if (knownFileHashesMode == KnownFileHashesMode.ENFORCE) { throw new MissingChecksumException( @@ -155,7 +160,7 @@ private Optional doGrabFile( "Missing checksum for registry file %s not permitted with --lockfile_mode=error." + " Please run `bazel mod deps --lockfile_mode=update` to update your" + " lockfile.", - url)); + rawURL)); } // This is a new file, download without providing a checksum. checksum = Optional.empty(); @@ -182,17 +187,36 @@ private Optional doGrabFile( "Cannot fetch a file without a checksum in ENFORCE mode. This is a bug in Bazel, please " + "report at https://github.com/bazelbuild/bazel/issues/new/choose."); } + + URL url = new URL(rawURL); + // Don't try to read the registry URL from the vendor directory in the following cases: + // 1. The vendor directory is not set, which means vendor mode is disabled. + // 2. The checksum is not present, which means the URL is not vendored or the vendored content is out-dated. + // 3. The URL starts with "file:", which means it's a local file and isn't vendored. + // Otherwise, check if the URL is vendored and read the registry file from the vendor directory. + if (vendorDir.isPresent() && checksum.isPresent() && !url.getProtocol().equals("file")) { + VendorUtil vendorUtil = new VendorUtil(vendorDir.get()); + if (vendorUtil.isUrlVendored(url)) { + try { + return Optional.of(vendorUtil.readRegistryURL(url, checksum.get())); + } catch (IOException e) { + throw new IOException( + "Failed to read vendored registry file %s at %s: %s. Please rerun the bazel vendor command.".formatted(rawURL, vendorUtil.getVendorPathForURL(url), e.getMessage()), e); + } + } + } + try (SilentCloseable c = - Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + url)) { + Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + rawURL)) { return Optional.of( downloadManager.downloadAndReadOneUrlForBzlmod( - new URL(url), eventHandler, clientEnv, checksum)); + url, eventHandler, clientEnv, checksum)); } catch (FileNotFoundException e) { return Optional.empty(); } catch (IOException e) { // Include the URL in the exception message for easier debugging. throw new IOException( - "Failed to fetch registry file %s: %s".formatted(url, e.getMessage()), e); + "Failed to fetch registry file %s: %s".formatted(rawURL, e.getMessage()), e); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java index d910c3d2d75c33..19c1ed70ebf77c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java @@ -18,6 +18,8 @@ import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; +import com.google.devtools.build.lib.vfs.Path; + import java.net.URISyntaxException; import java.util.Optional; @@ -33,6 +35,7 @@ Registry createRegistry( String url, RepositoryOptions.LockfileMode lockfileMode, ImmutableMap> fileHashes, - ImmutableMap previouslySelectedYankedVersions) + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) throws URISyntaxException; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java index 32c34daa0c4d4a..3e422fa315f589 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java @@ -20,6 +20,8 @@ import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; +import com.google.devtools.build.lib.vfs.Path; + import java.net.URI; import java.net.URISyntaxException; import java.util.Map; @@ -42,7 +44,8 @@ public Registry createRegistry( String url, LockfileMode lockfileMode, ImmutableMap> knownFileHashes, - ImmutableMap previouslySelectedYankedVersions) + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) throws URISyntaxException { URI uri = new URI(url); if (uri.getScheme() == null) { @@ -75,6 +78,7 @@ public Registry createRegistry( clientEnvironmentSupplier.get(), knownFileHashes, knownFileHashesMode, - previouslySelectedYankedVersions); + previouslySelectedYankedVersions, + vendorDir); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java index 4e72d565bca8fe..d90033498cb43e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.bazel.bzlmod; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.server.FailureDetails; import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; import com.google.devtools.build.lib.vfs.Path; @@ -26,6 +27,8 @@ import java.net.URISyntaxException; import java.time.Duration; import java.time.Instant; +import java.util.Optional; + import javax.annotation.Nullable; /** A simple SkyFunction that creates a {@link Registry} with a given URL. */ @@ -56,6 +59,7 @@ public RegistryFunction(RegistryFactory registryFactory, Path workspaceRoot) { public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException, RegistryException { LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env); + Optional vendorDir = RepositoryDelegatorFunction.VENDOR_DIRECTORY.get(env); if (lockfileMode == LockfileMode.REFRESH) { RegistryFunction.LAST_INVALIDATION.get(env); @@ -72,7 +76,8 @@ public SkyValue compute(SkyKey skyKey, Environment env) key.getUrl().replace("%workspace%", workspaceRoot.getPathString()), lockfileMode, lockfile.getRegistryFileHashes(), - lockfile.getSelectedYankedVersions()); + lockfile.getSelectedYankedVersions(), + vendorDir); } catch (URISyntaxException e) { throw new RegistryException( ExternalDepsException.withCauseAndMessage( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java new file mode 100644 index 00000000000000..1bb3a53e45ce73 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorUtil.java @@ -0,0 +1,162 @@ +package com.google.devtools.build.lib.bazel.bzlmod; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import com.google.common.hash.HashCode; +import com.google.common.hash.Hasher; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.Symlinks; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URL; +import java.net.URLDecoder; +import java.util.Objects; +import java.util.Optional; + + +/** + * Utility class for vendoring external repositories. + */ +public class VendorUtil { + + private final Path vendorDirectory; + + public VendorUtil(Path vendorDirectory) { + this.vendorDirectory = vendorDirectory; + } + + /** + * Vendors the specified repositories under the vendor directory. + * + * @param externalRepoRoot The root directory of the external repositories. + * @param reposToVendor The list of repositories to vendor. + * @throws IOException if an I/O error occurs. + */ + public void vendorRepos( + Path externalRepoRoot, + ImmutableList reposToVendor) + throws IOException { + if (!vendorDirectory.exists()) { + vendorDirectory.createDirectoryAndParents(); + } + + for (RepositoryName repo : reposToVendor) { + // Only re-vendor the repository if it is not up-to-date. + if (!isRepoUpToDate(repo.getName(), externalRepoRoot)) { + Path repoUnderVendor = vendorDirectory.getRelative(repo.getName()); + if (!repoUnderVendor.exists()) { + repoUnderVendor.createDirectory(); + } + FileSystemUtils.copyTreesBelow( + externalRepoRoot.getRelative(repo.getName()), repoUnderVendor, Symlinks.NOFOLLOW); + FileSystemUtils.copyFile( + externalRepoRoot.getChild("@" + repo.getName() + ".marker"), + vendorDirectory.getChild("@" + repo.getName() + ".marker")); + } + } + } + + /** + * Checks if the given URL is vendored. + * + * @param url The URL to check. + * @return true if the URL is vendored, false otherwise. + * @throws UnsupportedEncodingException if the URL decoding fails. + */ + public boolean isUrlVendored(URL url) throws UnsupportedEncodingException { + return getVendorPathForURL(url).isFile(); + } + + /** + * Vendors the registry URL with the specified content. + * + * @param url The registry URL to vendor. + * @param content The content to write. + * @throws IOException if an I/O error occurs. + */ + public void vendorRegistryURL(URL url, byte[] content) + throws IOException { + Path outputPath = getVendorPathForURL(url); + Objects.requireNonNull(outputPath.getParentDirectory()).createDirectoryAndParents(); + FileSystemUtils.writeContent(outputPath, content); + } + + /** + * Reads the content of the registry URL and verifies its checksum. + * + * @param url The registry URL to read. + * @param checksum The checksum to verify. + * @return The content of the registry URL. + * @throws IOException if an I/O error occurs or the checksum verification fails. + */ + public byte[] readRegistryURL(URL url, Checksum checksum) + throws IOException { + byte[] content = FileSystemUtils.readContent(getVendorPathForURL(url)); + Hasher hasher = checksum.getKeyType().newHasher(); + hasher.putBytes(content); + HashCode actual = hasher.hash(); + if (!checksum.getHashCode().equals(actual)) { + throw new IOException( + String.format( + "Checksum was %s but wanted %s", + checksum.emitOtherHashInSameFormat(actual), + checksum.emitOtherHashInSameFormat(checksum.getHashCode()))); + } + return content; + } + + /** + * Checks if the repository under vendor dir needs to be updated by comparing its marker file with the + * one under /external. This function assumes the marker file under /external exists + * and is up-to-date. + * + * @param repoName The name of the repository. + * @param externalPath The root directory of the external repositories. + * @return true if the repository is up-to-date, false otherwise. + * @throws IOException if an I/O error occurs. + */ + private boolean isRepoUpToDate(String repoName, Path externalPath) + throws IOException { + Path vendorMarkerFile = vendorDirectory.getChild("@" + repoName + ".marker"); + if (!vendorMarkerFile.exists()) { + return false; + } + + Path externalMarkerFile = externalPath.getChild("@" + repoName + ".marker"); + String vendorMarkerContent = FileSystemUtils.readContent(vendorMarkerFile, UTF_8); + String externalMarkerContent = FileSystemUtils.readContent(externalMarkerFile, UTF_8); + return Objects.equals(vendorMarkerContent, externalMarkerContent); + } + + /** + * Returns the vendor path for the given URL. + * + * The vendor path is constructed as follows: + * /registry_cache// + * + * The host name is case-insensitive, so it is converted to lowercase. + * The path is case-sensitive, so it is left as is. + * The port number is not included in the vendor path. + * + * Note that the vendor path may conflicts if two URLs only differ by the case or port number. + * But this is unlikely to happen in practice, and conflicts are checked in VendorCommand.java. + * + * @param url The URL to get the vendor path for. + * @return The vendor path. + * @throws UnsupportedEncodingException if the URL decoding fails. + */ + public Path getVendorPathForURL(URL url) throws UnsupportedEncodingException { + String host = url.getHost().toLowerCase(); // Host names are case-insensitive + String path = url.getPath(); + path = URLDecoder.decode(path, "UTF-8"); + if (path.startsWith("/")) { + path = path.substring(1); + } + return vendorDirectory.getRelative("registry_cache").getRelative(host).getRelative(path); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD index 34b0022e442af0..35ec3ac15cc6dd 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD @@ -39,9 +39,11 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution_impl", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:root_module_file_fixup", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:tidy", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:vendor", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand", "//src/main/java/com/google/devtools/build/lib/bazel/repository", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java index fd6e4411815b49..79938efb57938b 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java @@ -14,7 +14,6 @@ package com.google.devtools.build.lib.bazel.commands; import static com.google.common.collect.ImmutableList.toImmutableList; -import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -22,9 +21,13 @@ import com.google.devtools.build.lib.analysis.NoBuildEvent; import com.google.devtools.build.lib.analysis.NoBuildRequestFinishedEvent; import com.google.devtools.build.lib.bazel.bzlmod.BazelFetchAllValue; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionValue; +import com.google.devtools.build.lib.bazel.bzlmod.VendorUtil; import com.google.devtools.build.lib.bazel.commands.RepositoryFetcher.RepositoryFetcherException; import com.google.devtools.build.lib.bazel.commands.TargetFetcher.TargetFetcherException; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; +import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.buildtool.BuildResult; import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.cmdline.RepositoryName; @@ -49,10 +52,8 @@ import com.google.devtools.build.lib.skyframe.RepositoryMappingValue.RepositoryMappingResolutionException; import com.google.devtools.build.lib.util.DetailedExitCode; import com.google.devtools.build.lib.util.InterruptedFailureDetails; -import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; -import com.google.devtools.build.lib.vfs.Symlinks; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; import com.google.devtools.build.skyframe.InMemoryGraph; @@ -62,15 +63,22 @@ import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.common.options.OptionsParser; import com.google.devtools.common.options.OptionsParsingResult; + import java.io.IOException; +import java.net.URL; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Optional; import java.util.Queue; import java.util.Set; +import java.util.function.Supplier; + import javax.annotation.Nullable; /** @@ -102,6 +110,14 @@ public final class VendorCommand implements BlazeCommand { public static final String NAME = "vendor"; + private final DownloadManager downloadManager; + private final Supplier> clientEnvironmentSupplier; + + public VendorCommand(DownloadManager downloadManager, Supplier> clientEnvironmentSupplier) { + this.downloadManager = downloadManager; + this.clientEnvironmentSupplier = clientEnvironmentSupplier; + } + @Override public void editOptions(OptionsParser optionsParser) { // We only need to inject these options with fetch target (when there is a residue) @@ -135,9 +151,10 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti BlazeCommandResult result; VendorOptions vendorOptions = options.getOptions(VendorOptions.class); - PathFragment vendorDirectory = options.getOptions(RepositoryOptions.class).vendorDirectory; + Path vendorDirectory = getVendorPath(env, options.getOptions(RepositoryOptions.class).vendorDirectory); LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); try { + env.getReporter().handle(Event.info("Vendoring ...")); if (!options.getResidue().isEmpty()) { result = vendorTargets(env, options, options.getResidue(), vendorDirectory); } else if (!vendorOptions.repos.isEmpty()) { @@ -181,7 +198,7 @@ private BlazeCommandResult validateOptions(CommandEnvironment env, OptionsParsin } private BlazeCommandResult vendorAll( - CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, PathFragment vendorDirectory) + CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, Path vendorDirectory) throws InterruptedException, IOException { EvaluationContext evaluationContext = EvaluationContext.newBuilder() @@ -209,7 +226,7 @@ private BlazeCommandResult vendorRepos( CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, List repos, - PathFragment vendorDirectory) + Path vendorDirectory) throws InterruptedException, IOException { ImmutableMap repositoryNamesAndValues; try { @@ -248,7 +265,7 @@ private BlazeCommandResult vendorTargets( CommandEnvironment env, OptionsParsingResult options, List targets, - PathFragment vendorDirectory) + Path vendorDirectory) throws InterruptedException, IOException { // Call fetch which runs build to have the targets graph and configuration set BuildResult buildResult; @@ -309,58 +326,71 @@ private ImmutableSet collectReposFromTargets( */ private void vendor( CommandEnvironment env, - PathFragment vendorDirectory, + Path vendorDirectory, ImmutableList reposToVendor) - throws IOException { - Path vendorPath = - vendorDirectory.isAbsolute() - ? env.getRuntime().getFileSystem().getPath(vendorDirectory) - : env.getWorkspace().getRelative(vendorDirectory); - Path externalPath = - env.getDirectories() - .getOutputBase() - .getRelative(LabelConstants.EXTERNAL_REPOSITORY_LOCATION); + throws IOException, InterruptedException { + VendorUtil vendorUtil = new VendorUtil(vendorDirectory); - if (!vendorPath.exists()) { - vendorPath.createDirectory(); - } + // 1. Vendor registry files + BazelModuleResolutionValue moduleResolutionValue = (BazelModuleResolutionValue) env.getSkyframeExecutor().getEvaluator().getExistingValue(BazelModuleResolutionValue.KEY); + ImmutableMap> registryFiles = Objects.requireNonNull( + moduleResolutionValue).getRegistryFileHashes(); - env.getReporter().handle(Event.info("Vendoring ...")); + // vendorPathToURL is a map of + // key: a vendor path string converted to lower case + // value: a URL string + // This map is for detecting potential rare vendor path conflicts, such as: + // http://foo.bar.com/BCR vs http://foo.bar.com/bcr => conflict vendor paths on case-insensitive system + // http://foo.bar.com/bcr vs http://foo.bar.com:8081/bcr => conflict vendor path because port number is ignored in vendor path + // The user has to update the Bazel registries this if such conflicts occur. + Map vendorPathToURL = new HashMap<>(); + for (Entry> entry : registryFiles.entrySet()) { + URL url = new URL(entry.getKey()); + if (url.getProtocol().equals("file")) { + continue; + } - // Update "out-of-date" repos under the vendor directory - for (RepositoryName repo : reposToVendor) { - if (!isRepoUpToDate(repo.getName(), vendorPath, externalPath)) { - Path repoUnderVendor = vendorPath.getRelative(repo.getName()); - if (!repoUnderVendor.exists()) { - repoUnderVendor.createDirectory(); + String outputPath = vendorUtil.getVendorPathForURL(url).getPathString(); + if (vendorPathToURL.containsKey(outputPath.toLowerCase())) { + String previousURL = vendorPathToURL.get(outputPath.toLowerCase()); + throw new IOException(String.format( + "Vendor paths conflict detected for registry URLs:\n %s => %s\n %s => %s\nTheir output paths are either the same or only differ by case, which will cause conflict on case insensitive file systems, please fix by changing the registry URLs!", + previousURL, + vendorUtil.getVendorPathForURL(new URL(previousURL)).getPathString(), + entry.getKey(), + outputPath + )); + } + + Optional checksum = entry.getValue(); + if (!vendorUtil.isUrlVendored(url) + // Only vendor a registry URL when its checksum exists, otherwise the URL should be + // recorded as "not found" in moduleResolutionValue.getRegistryFileHashes() + && checksum.isPresent()) { + try { + vendorUtil.vendorRegistryURL(url, downloadManager.downloadAndReadOneUrlForBzlmod(url, env.getReporter(), clientEnvironmentSupplier.get(), checksum)); + } catch (IOException e) { + throw new IOException(String.format("Failed to vendor registry URL %s at %s: %s", url, outputPath, e.getMessage()), e.getCause()); } - FileSystemUtils.copyTreesBelow( - externalPath.getRelative(repo.getName()), repoUnderVendor, Symlinks.NOFOLLOW); - FileSystemUtils.copyFile( - externalPath.getChild("@" + repo.getName() + ".marker"), - vendorPath.getChild("@" + repo.getName() + ".marker")); } - } - } - /** - * Returns whether the repo under vendor needs to be updated by comparing its marker file with the - * one under /external - */ - private boolean isRepoUpToDate(String repoName, Path vendorPath, Path externalPath) - throws IOException { - Path vendorMarkerFile = vendorPath.getChild("@" + repoName + ".marker"); - if (!vendorMarkerFile.exists()) { - return false; + vendorPathToURL.put(outputPath.toLowerCase(), entry.getKey()); } - // Since this runs after fetching repos, its guaranteed that the marker files - // under $OUTPUT_BASE/external are up-to-date. We just need to compare it against the marker - // under vendor. - Path externalMarkerFile = externalPath.getChild("@" + repoName + ".marker"); - String vendorMarkerContent = FileSystemUtils.readContent(vendorMarkerFile, UTF_8); - String externalMarkerContent = FileSystemUtils.readContent(externalMarkerFile, UTF_8); - return Objects.equals(vendorMarkerContent, externalMarkerContent); + // 2. Vendor repos + Path externalPath = + env.getDirectories() + .getOutputBase() + .getRelative(LabelConstants.EXTERNAL_REPOSITORY_LOCATION); + vendorUtil.vendorRepos(externalPath, reposToVendor); + } + + private static Path getVendorPath(CommandEnvironment env, PathFragment vendorDirectory) { + Path vendorPath = + vendorDirectory.isAbsolute() + ? env.getRuntime().getFileSystem().getPath(vendorDirectory) + : env.getWorkspace().getRelative(vendorDirectory); + return vendorPath; } private static BlazeCommandResult createFailedBlazeCommandResult( diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java index ecebc7f072da1d..72bf3268d15a9d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java @@ -303,7 +303,7 @@ private RepositoryDirectoryValue tryGettingValueUsingVendoredRepo( String.format( "Vendored repository '%s' is out-of-date and fetching is disabled." + " Run build without the '--nofetch' option or run" - + " `bazel vendor` to update it", + + " the bazel vendor command to update it", rule.getName()))); } return setupOverride(vendorRepoPath.asFragment(), env, repoRoot, repositoryName); @@ -316,7 +316,7 @@ private RepositoryDirectoryValue tryGettingValueUsingVendoredRepo( String.format( "Vendored repository '%s' is out-of-date. The up-to-date version will" + " be fetched into the external cache and used. To update the repo" - + " in the vendor directory, run 'bazel vendor'", + + " in the vendor directory, run the bazel vendor command", rule.getName()))); } } else if (vendorFile.getPinnedRepos().contains(repositoryName)) { @@ -332,7 +332,7 @@ private RepositoryDirectoryValue tryGettingValueUsingVendoredRepo( "Vendored repository " + repositoryName.getName() + " not found under the vendor directory and fetching is disabled." - + " To fix run 'bazel vendor' or build without the '--nofetch'"), + + " To fix, run the bazel vendor command or build without the '--nofetch'"), Transience.TRANSIENT); } return null; diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index d783e91e4edd3c..3566f8a1456407 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -120,6 +120,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/packages", + "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/net/starlark/java/eval", "//src/main/java/net/starlark/java/syntax", "//third_party:guava", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index 9c98532eec437b..8806f9abc028da 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -37,6 +37,7 @@ import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.Type; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; import com.google.devtools.build.lib.skyframe.BzlmodRepoRuleFunction; import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; @@ -62,6 +63,8 @@ import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionName; + +import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import net.starlark.java.eval.StarlarkSemantics; import org.junit.Before; @@ -158,6 +161,7 @@ public void setup() throws Exception { BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); + RepositoryDelegatorFunction.VENDOR_DIRECTORY.set(differencer, Optional.empty()); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java index e093f5f326a807..0b9b5be3c23e0d 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java @@ -23,6 +23,7 @@ import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.vfs.Path; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.HashMap; import java.util.Map; @@ -136,7 +137,8 @@ public Registry createRegistry( String url, LockfileMode lockfileMode, ImmutableMap> fileHashes, - ImmutableMap previouslySelectedYankedVersions) { + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) { return Preconditions.checkNotNull(registries.get(url), "unknown registry url: %s", url); } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java index b4d259c398aee9..9fff42177d0ded 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java @@ -97,7 +97,7 @@ public void testHttpUrl() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue( ModuleFile.create( @@ -115,7 +115,7 @@ public void testHttpUrlWithNetrcCreds() throws Exception { "machine [::1] login rinne password rinnepass\n".getBytes(UTF_8))); Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); var e = assertThrows( @@ -148,7 +148,8 @@ public void testFileUrl() throws Exception { new File(tempFolder.getRoot(), "fakereg").toURI().toString(), LockfileMode.UPDATE, ImmutableMap.of(), - ImmutableMap.of()); + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue(ModuleFile.create("lol".getBytes(UTF_8), file.toURI().toString())); assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty(); @@ -197,7 +198,7 @@ public void testGetArchiveRepoSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -265,7 +266,7 @@ public void testGetLocalPathRepoSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( RepoSpec.builder() @@ -289,7 +290,7 @@ public void testGetRepoInvalidRegistryJsonSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -323,7 +324,7 @@ public void testGetRepoInvalidModuleJsonSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); } @@ -352,7 +353,7 @@ public void testGetYankedVersion() throws Exception { server.start(); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); Optional> yankedVersion = registry.getYankedVersions("red-pill", reporter); assertThat(yankedVersion) @@ -375,7 +376,7 @@ public void testArchiveWithExplicitType() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("archive_type", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -405,7 +406,7 @@ public void testGetModuleFileChecksums() throws Exception { Optional.of(sha256("unused"))); Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue( ModuleFile.create( @@ -431,7 +432,7 @@ public void testGetModuleFileChecksums() throws Exception { registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of()); + server.getUrl() + "/myreg", LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of(), Optional.empty()); // Test that the recorded hashes are used for repo cache hits even when the server content // changes. server.unserve("/myreg/modules/foo/1.0/MODULE.bazel"); @@ -461,7 +462,7 @@ public void testGetModuleFileChecksumMismatch() throws Exception { Optional.of(sha256("original"))); Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); var e = assertThrows( IOException.class, @@ -502,7 +503,7 @@ public void testGetRepoSpecChecksum() throws Exception { server.getUrl() + "/modules/foo/2.0/source.json", Optional.of(sha256("unused"))); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( RepoSpec.builder() @@ -522,7 +523,7 @@ public void testGetRepoSpecChecksum() throws Exception { registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of(), Optional.empty()); // Test that the recorded hashes are used for repo cache hits even when the server content // changes. server.unserve("/bazel_registry.json"); @@ -566,7 +567,7 @@ public void testGetRepoSpecChecksumMismatch() throws Exception { Optional.of(sha256(sourceJson))); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); var e = assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); @@ -610,7 +611,7 @@ public void testBazelRegistryChecksumMismatch() throws Exception { Optional.of(sha256(sourceJson))); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); var e = assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java index cc963f233e5e3e..f29d6c5a704cb4 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java @@ -25,6 +25,8 @@ import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; import java.net.URISyntaxException; +import java.util.Optional; + import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -44,14 +46,15 @@ public void badSchemes() { URISyntaxException.class, () -> registryFactory.createRegistry( - "/home/www", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of())); + "/home/www", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), + Optional.empty())); assertThat(exception).hasMessageThat().contains("Registry URL has no scheme"); exception = assertThrows( URISyntaxException.class, () -> registryFactory.createRegistry( - "foo://bar", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of())); + "foo://bar", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of(), Optional.empty())); assertThat(exception).hasMessageThat().contains("Unrecognized registry URL protocol"); } @@ -69,7 +72,7 @@ public void badPath() { "file:c:/path/to/workspace/registry", LockfileMode.UPDATE, ImmutableMap.of(), - ImmutableMap.of())); + ImmutableMap.of(), Optional.empty())); assertThat(exception).hasMessageThat().contains("Registry URL path is not valid"); } } diff --git a/src/test/py/bazel/bzlmod/bazel_vendor_test.py b/src/test/py/bazel/bzlmod/bazel_vendor_test.py index 0ac2b7cad18294..0b4479bc52e5a8 100644 --- a/src/test/py/bazel/bzlmod/bazel_vendor_test.py +++ b/src/test/py/bazel/bzlmod/bazel_vendor_test.py @@ -258,35 +258,6 @@ def testVendorDirIsNotCheckedForWorkspaceRepos(self): "Vendored repository 'dummyRepo' is out-of-date.", '\n'.join(stderr) ) - def testBuildingWithVendoredRepos(self): - self.main_registry.createCcModule('aaa', '1.0') - self.ScratchFile( - 'MODULE.bazel', - [ - 'bazel_dep(name = "aaa", version = "1.0")', - ], - ) - self.ScratchFile('BUILD') - self.RunBazel(['vendor', '--vendor_dir=vendor']) - self.assertIn('aaa~', os.listdir(self._test_cwd + '/vendor')) - - # Empty external & build with vendor - self.RunBazel(['clean', '--expunge']) - _, _, stderr = self.RunBazel(['build', '@aaa//:all', '--vendor_dir=vendor']) - self.assertNotIn( - "Vendored repository '_main~ext~justRepo' is out-of-date.", - '\n'.join(stderr), - ) - - # Assert repo aaa in {OUTPUT_BASE}/external is a symlink (junction on - # windows, this validates it was created from vendor and not fetched)= - _, stdout, _ = self.RunBazel(['info', 'output_base']) - repo_path = stdout[0] + '/external/aaa~' - if self.IsWindows(): - self.assertTrue(self.IsJunction(repo_path)) - else: - self.assertTrue(os.path.islink(repo_path)) - def testIgnoreFromVendoring(self): # Repos should be excluded from vendoring: # 1.Local Repos, 2.Config Repos, 3.Repos declared in VENDOR.bazel file @@ -372,7 +343,7 @@ def testBuildingWithPinnedRepo(self): ) self.ScratchFile('BUILD') - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@venRepo']) self.assertIn('_main~ext~venRepo', os.listdir(self._test_cwd + '/vendor')) self.ScratchFile( 'extension.bzl', @@ -421,7 +392,7 @@ def testBuildingWithPinnedRepo(self): ) # Re-vendor & build make sure the repo is successfully updated - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@venRepo']) _, _, stderr = self.RunBazel( ['build', '@venRepo//:all', '--vendor_dir=vendor'], ) @@ -454,7 +425,7 @@ def testBuildingOutOfDateVendoredRepo(self): ) # Vendor, assert and build with no problems - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@justRepo']) self.assertIn('_main~ext~justRepo', os.listdir(self._test_cwd + '/vendor')) _, _, stderr = self.RunBazel( ['build', '@justRepo//:all', '--vendor_dir=vendor'] @@ -462,8 +433,8 @@ def testBuildingOutOfDateVendoredRepo(self): self.assertNotIn( "WARNING: : Vendored repository '_main~ext~justRepo' is" ' out-of-date. The up-to-date version will be fetched into the external' - ' cache and used. To update the repo in the vendor directory, run' - " 'bazel vendor'", + ' cache and used. To update the repo in the vendor directory, run' + " the bazel vendor command", stderr, ) @@ -492,15 +463,15 @@ def testBuildingOutOfDateVendoredRepo(self): self.assertIn( "WARNING: : Vendored repository '_main~ext~justRepo' is" ' out-of-date. The up-to-date version will be fetched into the external' - ' cache and used. To update the repo in the vendor directory, run' - " 'bazel vendor'", + ' cache and used. To update the repo in the vendor directory, run' + " the bazel vendor command", stderr, ) _, stdout, _ = self.RunBazel(['info', 'output_base']) self.assertFalse(os.path.islink(stdout[0] + '/external/bbb~')) # Assert vendoring again solves the problem - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@justRepo']) self.RunBazel(['clean', '--expunge']) _, _, stderr = self.RunBazel( ['build', '@justRepo//:all', '--vendor_dir=vendor'] @@ -508,12 +479,12 @@ def testBuildingOutOfDateVendoredRepo(self): self.assertNotIn( "WARNING: : Vendored repository '_main~ext~justRepo' is" ' out-of-date. The up-to-date version will be fetched into the external' - ' cache and used. To update the repo in the vendor directory, run' - " 'bazel vendor'", + ' cache and used. To update the repo in the vendor directory, run' + " the bazel vendor command", stderr, ) - def testBuildingVendoredRepoInOfflineMode(self): + def testBuildingVendoredRepoWithNoFetch(self): self.ScratchFile( 'MODULE.bazel', [ @@ -537,7 +508,7 @@ def testBuildingVendoredRepoInOfflineMode(self): self.ScratchFile('BUILD') # Vendor, assert and build with no problems - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '@venRepo//:all']) self.assertIn('_main~ext~venRepo', os.listdir(self._test_cwd + '/vendor')) # Make updates in repo definition @@ -571,8 +542,8 @@ def testBuildingVendoredRepoInOfflineMode(self): ) self.assertIn( 'ERROR: Vendored repository _main~ext~noVenRepo not found under the' - " vendor directory and fetching is disabled. To fix run 'bazel" - " vendor' or build without the '--nofetch'", + " vendor directory and fetching is disabled. To fix, run the bazel" + " vendor command or build without the '--nofetch'", stderr, ) @@ -584,7 +555,7 @@ def testBuildingVendoredRepoInOfflineMode(self): self.assertIn( "WARNING: : Vendored repository '_main~ext~venRepo' is" ' out-of-date and fetching is disabled. Run build without the' - " '--nofetch' option or run `bazel vendor` to update it", + " '--nofetch' option or run the bazel vendor command to update it", stderr, ) # Assert the out-dated repo is the one built with @@ -615,7 +586,7 @@ def testBasicVendorTarget(self): self.assertIn('bbb~', os.listdir(self._test_cwd + '/vendor')) self.assertNotIn('ccc~', os.listdir(self._test_cwd + '/vendor')) - def testVendorTarget(self): + def testBuildVendoredTargetOffline(self): self.main_registry.createCcModule('aaa', '1.0').createCcModule( 'bbb', '1.0', {'aaa': '1.0'} ) @@ -640,22 +611,63 @@ def testVendorTarget(self): self.ScratchFile( 'main.cc', [ - '#include "aaa.h"', + '#include "bbb.h"', 'int main() {', - ' hello_aaa("Hello there!");', + ' hello_bbb("Hello there!");', '}', ], ) self.RunBazel(['vendor', '//:main', '--vendor_dir=vendor']) - # Run the vendored target with --nofetch should only use what is under - # vendor to build, meaning we have vendored everything we need to build/run - # this target + # Build and run the target in a clean build with internet blocked and make sure it works + _, _, _ = self.RunBazel(['clean', '--expunge']) _, stdout, _ = self.RunBazel( - ['run', '//:main', '--vendor_dir=vendor', '--nofetch'] + ['run', '//:main', '--vendor_dir=vendor', '--repository_cache='], + env_add= { + "HTTP_PROXY": "internet_blocked", + "HTTPS_PROXY": "internet_blocked", + }, + ) + self.assertIn('Hello there! => bbb@1.0', stdout) + + # Assert repos in {OUTPUT_BASE}/external are symlinks (junction on + # windows, this validates it was created from vendor and not fetched) + _, stdout, _ = self.RunBazel(['info', 'output_base']) + for repo in ["aaa~", "bbb~"]: + repo_path = stdout[0] + '/external/' + repo + if self.IsWindows(): + self.assertTrue(self.IsJunction(repo_path)) + else: + self.assertTrue(os.path.islink(repo_path)) + + def testVendorConflictRegistryFile(self): + self.main_registry.createCcModule('aaa', '1.0').createCcModule( + 'bbb', '1.0', {'aaa': '1.0'} + ) + # The registry URLs of main_registry and another_registry only differ by the port number + another_registry = BazelRegistry( + os.path.join(self.registries_work_dir, 'MAIN'), ) - self.assertIn('Hello there! => aaa@1.0', stdout) + another_registry.start() + another_registry.createCcModule('aaa', '1.0') + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "bbb", version = "1.0")', + 'local_path_override(module_name="bazel_tools", path="tools_mock")', + 'local_path_override(module_name="local_config_platform", ', + 'path="platforms_mock")', + 'single_version_override(', + ' module_name = "aaa",', + ' registry = "%s",' % another_registry.getURL(), + ')', + ], + ) + self.ScratchFile('BUILD') + exit_code, _, stderr = self.RunBazel(['vendor', '--vendor_dir=vendor'], allow_failure=True) + self.AssertExitCode(exit_code, 8, stderr) + self.assertIn('ERROR: Error while vendoring repos: Vendor paths conflict detected for registry URLs:', stderr) if __name__ == '__main__':