diff --git a/site/en/external/vendor.md b/site/en/external/vendor.md index 2d8685604e7b0d..41911be394e3f2 100644 --- a/site/en/external/vendor.md +++ b/site/en/external/vendor.md @@ -67,6 +67,23 @@ Under the hood, it's doing a `bazel build --nobuild` command to analyze the target patterns, therefore build flags could be applied to this command and affect the result. +### Build the target offline {:#build-the-target-offline} + +With the external dependencies vendored, you can build the target offline by + +```none +bazel build --vendor_dir=vendor_src //src/main:hello-world //src/test/... +``` + +The build should work in a clean build environment without network access and +repository cache. + +Therefore, you should be able to check in the vendored source and build the same +targets offline on another machine. + +Note: If you build different targets or change the external dependencies, build +configuration, or Bazel version, you may need to re-vendor. + ## Vendor all external dependencies {:#vendor-all-dependencies} To vendor all repos in your transitive external dependencies graph, you can diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD index 1ff603849fa87a..da461723be39cb 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD @@ -34,6 +34,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution_impl", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:tidy_impl", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:vendor", "//src/main/java/com/google/devtools/build/lib/bazel/commands", "//src/main/java/com/google/devtools/build/lib/bazel/repository", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 81bbf1c016ce30..64b05810dfe81f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -165,7 +165,7 @@ public class BazelRepositoryModule extends BlazeModule { private Clock clock; private Instant lastRegistryInvalidation = Instant.EPOCH; - private Optional vendorDirectory; + private Optional vendorDirectory = Optional.empty(); private List allowedYankedVersions = ImmutableList.of(); private boolean disableNativeRepoRules; private SingleExtensionEvalFunction singleExtensionEvalFunction; @@ -224,7 +224,7 @@ public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builde builder.addCommands(new FetchCommand()); builder.addCommands(new ModCommand()); builder.addCommands(new SyncCommand()); - builder.addCommands(new VendorCommand()); + builder.addCommands(new VendorCommand(downloadManager, clientEnvironmentSupplier)); builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache)); } @@ -495,15 +495,9 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { bazelCompatibilityMode = repoOptions.bazelCompatibilityMode; bazelLockfileMode = repoOptions.lockfileMode; allowedYankedVersions = repoOptions.allowedYankedVersions; - - if (repoOptions.vendorDirectory != null) { - vendorDirectory = - Optional.of( - repoOptions.vendorDirectory.isAbsolute() - ? filesystem.getPath(repoOptions.vendorDirectory) - : env.getWorkspace().getRelative(repoOptions.vendorDirectory)); - } else { - vendorDirectory = Optional.empty(); + if (env.getWorkspace() != null) { + vendorDirectory = Optional.ofNullable(repoOptions.vendorDirectory) + .map(vendorDirectory -> env.getWorkspace().getRelative(vendorDirectory)); } if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index cd9c2b40c6a673..191dd183841d13 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -46,6 +46,17 @@ java_library( ], ) +java_library( + name = "vendor", + srcs = ["VendorManager.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", + "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//third_party:guava", + ], +) + java_library( name = "module_extension", srcs = [ @@ -80,6 +91,7 @@ java_library( ], deps = [ ":common", + ":vendor", ":yanked_versions_value", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", @@ -87,6 +99,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/profiler", "//src/main/java/com/google/devtools/build/lib/util:os", + "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//third_party:gson", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java index 3dc6849db48f4b..e776d19d317811 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java @@ -48,7 +48,7 @@ public abstract class BazelModuleResolutionValue implements SkyValue { /** * Hashes of files obtained (or known to be missing) from registries while performing resolution. */ - abstract ImmutableMap> getRegistryFileHashes(); + public abstract ImmutableMap> getRegistryFileHashes(); /** * Selected module versions that are known to be yanked (and hence must have been explicitly diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index d0cef909bc9834..984be513760466 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -32,6 +32,7 @@ import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.profiler.SilentCloseable; import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.gson.FieldNamingPolicy; import com.google.gson.Gson; @@ -46,6 +47,8 @@ import java.util.Map.Entry; import java.util.Optional; +import javax.annotation.Nullable; + /** * Represents a Bazel module registry that serves a list of module metadata from a static HTTP * server or a local file path. @@ -88,6 +91,8 @@ public enum KnownFileHashesMode { private final Gson gson; private final ImmutableMap> knownFileHashes; private final ImmutableMap previouslySelectedYankedVersions; + @Nullable + private final VendorManager vendorUtil; private final KnownFileHashesMode knownFileHashesMode; private volatile Optional bazelRegistryJson; private volatile StoredEventHandler bazelRegistryJsonEvents; @@ -100,7 +105,8 @@ public IndexRegistry( Map clientEnv, ImmutableMap> knownFileHashes, KnownFileHashesMode knownFileHashesMode, - ImmutableMap previouslySelectedYankedVersions) { + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) { this.uri = uri; this.downloadManager = downloadManager; this.clientEnv = clientEnv; @@ -111,6 +117,7 @@ public IndexRegistry( this.knownFileHashes = knownFileHashes; this.knownFileHashesMode = knownFileHashesMode; this.previouslySelectedYankedVersions = previouslySelectedYankedVersions; + this.vendorUtil = vendorDir.map(VendorManager::new).orElse(null); } @Override @@ -143,11 +150,11 @@ private Optional grabFile( } private Optional doGrabFile( - String url, ExtendedEventHandler eventHandler, boolean useChecksum) + String rawUrl, ExtendedEventHandler eventHandler, boolean useChecksum) throws IOException, InterruptedException { Optional checksum; if (knownFileHashesMode != KnownFileHashesMode.IGNORE && useChecksum) { - Optional knownChecksum = knownFileHashes.get(url); + Optional knownChecksum = knownFileHashes.get(rawUrl); if (knownChecksum == null) { if (knownFileHashesMode == KnownFileHashesMode.ENFORCE) { throw new MissingChecksumException( @@ -155,7 +162,7 @@ private Optional doGrabFile( "Missing checksum for registry file %s not permitted with --lockfile_mode=error." + " Please run `bazel mod deps --lockfile_mode=update` to update your" + " lockfile.", - url)); + rawUrl)); } // This is a new file, download without providing a checksum. checksum = Optional.empty(); @@ -182,17 +189,37 @@ private Optional doGrabFile( "Cannot fetch a file without a checksum in ENFORCE mode. This is a bug in Bazel, please " + "report at https://github.com/bazelbuild/bazel/issues/new/choose."); } + + URL url = URI.create(rawUrl).toURL(); + // Don't read the registry URL from the vendor directory in the following cases: + // 1. vendorUtil is null, which means vendor mode is disabled. + // 2. The checksum is not present, which means the URL is not vendored or the vendored content + // is out-dated. + // 3. The URL starts with "file:", which means it's a local file and isn't vendored. + // 4. The vendor path doesn't exist, which means the URL is not vendored. + if (vendorUtil != null && checksum.isPresent() && !url.getProtocol().equals("file") && vendorUtil.isUrlVendored(url)) { + try { + return Optional.of(vendorUtil.readRegistryUrl(url, checksum.get())); + } catch (IOException e) { + throw new IOException( + String.format( + "Failed to read vendored registry file %s at %s: %s. Please rerun the bazel" + + " vendor command.", + rawUrl, vendorUtil.getVendorPathForUrl(url), e.getMessage()), + e); + } + } + try (SilentCloseable c = - Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + url)) { + Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + rawUrl)) { return Optional.of( - downloadManager.downloadAndReadOneUrlForBzlmod( - new URL(url), eventHandler, clientEnv, checksum)); + downloadManager.downloadAndReadOneUrlForBzlmod(url, eventHandler, clientEnv, checksum)); } catch (FileNotFoundException e) { return Optional.empty(); } catch (IOException e) { // Include the URL in the exception message for easier debugging. throw new IOException( - "Failed to fetch registry file %s: %s".formatted(url, e.getMessage()), e); + "Failed to fetch registry file %s: %s".formatted(rawUrl, e.getMessage()), e); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java index d910c3d2d75c33..47d0bb77e9579b 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; +import com.google.devtools.build.lib.vfs.Path; import java.net.URISyntaxException; import java.util.Optional; @@ -33,6 +34,7 @@ Registry createRegistry( String url, RepositoryOptions.LockfileMode lockfileMode, ImmutableMap> fileHashes, - ImmutableMap previouslySelectedYankedVersions) + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) throws URISyntaxException; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java index 32c34daa0c4d4a..e26468f4890528 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java @@ -20,6 +20,7 @@ import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; +import com.google.devtools.build.lib.vfs.Path; import java.net.URI; import java.net.URISyntaxException; import java.util.Map; @@ -42,7 +43,8 @@ public Registry createRegistry( String url, LockfileMode lockfileMode, ImmutableMap> knownFileHashes, - ImmutableMap previouslySelectedYankedVersions) + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) throws URISyntaxException { URI uri = new URI(url); if (uri.getScheme() == null) { @@ -75,6 +77,7 @@ public Registry createRegistry( clientEnvironmentSupplier.get(), knownFileHashes, knownFileHashesMode, - previouslySelectedYankedVersions); + previouslySelectedYankedVersions, + vendorDir); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java index 4e72d565bca8fe..836c19e90bdb84 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.bazel.bzlmod; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.server.FailureDetails; import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; import com.google.devtools.build.lib.vfs.Path; @@ -26,6 +27,7 @@ import java.net.URISyntaxException; import java.time.Duration; import java.time.Instant; +import java.util.Optional; import javax.annotation.Nullable; /** A simple SkyFunction that creates a {@link Registry} with a given URL. */ @@ -56,6 +58,7 @@ public RegistryFunction(RegistryFactory registryFactory, Path workspaceRoot) { public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException, RegistryException { LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env); + Optional vendorDir = RepositoryDelegatorFunction.VENDOR_DIRECTORY.get(env); if (lockfileMode == LockfileMode.REFRESH) { RegistryFunction.LAST_INVALIDATION.get(env); @@ -72,7 +75,8 @@ public SkyValue compute(SkyKey skyKey, Environment env) key.getUrl().replace("%workspace%", workspaceRoot.getPathString()), lockfileMode, lockfile.getRegistryFileHashes(), - lockfile.getSelectedYankedVersions()); + lockfile.getSelectedYankedVersions(), + vendorDir); } catch (URISyntaxException e) { throw new RegistryException( ExternalDepsException.withCauseAndMessage( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorManager.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorManager.java new file mode 100644 index 00000000000000..67acb0fe159715 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorManager.java @@ -0,0 +1,176 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.bzlmod; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import com.google.common.hash.HashCode; +import com.google.common.hash.Hasher; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.Symlinks; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URL; +import java.net.URLDecoder; +import java.util.Locale; +import java.util.Objects; + +/** Utility class for vendoring external repositories. */ +public class VendorManager { + + private final static String REGISTRIES_DIR = "_registries"; + + private final Path vendorDirectory; + + public VendorManager(Path vendorDirectory) { + this.vendorDirectory = vendorDirectory; + } + + /** + * Vendors the specified repositories under the vendor directory. + * + * TODO(pcloudy): Parallelize vendoring repos + * + * @param externalRepoRoot The root directory of the external repositories. + * @param reposToVendor The list of repositories to vendor. + * @throws IOException if an I/O error occurs. + */ + public void vendorRepos(Path externalRepoRoot, ImmutableList reposToVendor) + throws IOException { + if (!vendorDirectory.exists()) { + vendorDirectory.createDirectoryAndParents(); + } + + for (RepositoryName repo : reposToVendor) { + // Only re-vendor the repository if it is not up-to-date. + if (!isRepoUpToDate(repo, externalRepoRoot)) { + Path markerUnderVendor = vendorDirectory.getChild(repo.getMarkerFileName()); + Path repoUnderVendor = vendorDirectory.getRelative(repo.getName()); + + // 1. Clean up existing marker file and repo vendor directory + markerUnderVendor.delete(); + repoUnderVendor.deleteTree(); + repoUnderVendor.createDirectory(); + + // 2. Copy over the repo source. + FileSystemUtils.copyTreesBelow( + externalRepoRoot.getRelative(repo.getName()), repoUnderVendor, Symlinks.NOFOLLOW); + + // 3. Copy the marker file atomically + Path tmpMarker = vendorDirectory.getChild(repo.getMarkerFileName() + ".tmp"); + FileSystemUtils.copyFile(externalRepoRoot.getChild(repo.getMarkerFileName()), tmpMarker); + tmpMarker.renameTo(markerUnderVendor); + } + } + } + + /** + * Checks if the given URL is vendored. + * + * @param url The URL to check. + * @return true if the URL is vendored, false otherwise. + * @throws UnsupportedEncodingException if the URL decoding fails. + */ + public boolean isUrlVendored(URL url) throws UnsupportedEncodingException { + return getVendorPathForUrl(url).isFile(); + } + + /** + * Vendors the registry URL with the specified content. + * + * @param url The registry URL to vendor. + * @param content The content to write. + * @throws IOException if an I/O error occurs. + */ + public void vendorRegistryUrl(URL url, byte[] content) throws IOException { + Path outputPath = getVendorPathForUrl(url); + Objects.requireNonNull(outputPath.getParentDirectory()).createDirectoryAndParents(); + FileSystemUtils.writeContent(outputPath, content); + } + + /** + * Reads the content of the registry URL and verifies its checksum. + * + * @param url The registry URL to read. + * @param checksum The checksum to verify. + * @return The content of the registry URL. + * @throws IOException if an I/O error occurs or the checksum verification fails. + */ + public byte[] readRegistryUrl(URL url, Checksum checksum) throws IOException { + byte[] content = FileSystemUtils.readContent(getVendorPathForUrl(url)); + Hasher hasher = checksum.getKeyType().newHasher(); + hasher.putBytes(content); + HashCode actual = hasher.hash(); + if (!checksum.getHashCode().equals(actual)) { + throw new IOException( + String.format( + "Checksum was %s but wanted %s", + checksum.emitOtherHashInSameFormat(actual), + checksum.emitOtherHashInSameFormat(checksum.getHashCode()))); + } + return content; + } + + /** + * Checks if the repository under vendor dir is up-to-date by comparing its marker file with + * the one under /external. This function assumes the marker file under + * /external exists and is up-to-date. + * + * @param repo The name of the repository. + * @param externalPath The root directory of the external repositories. + * @return true if the repository is up-to-date, false otherwise. + * @throws IOException if an I/O error occurs. + */ + private boolean isRepoUpToDate(RepositoryName repo, Path externalPath) throws IOException { + Path vendorMarkerFile = vendorDirectory.getChild(repo.getMarkerFileName()); + if (!vendorMarkerFile.exists()) { + return false; + } + + Path externalMarkerFile = externalPath.getChild(repo.getMarkerFileName()); + String vendorMarkerContent = FileSystemUtils.readContent(vendorMarkerFile, UTF_8); + String externalMarkerContent = FileSystemUtils.readContent(externalMarkerFile, UTF_8); + return Objects.equals(vendorMarkerContent, externalMarkerContent); + } + + /** + * Returns the vendor path for the given URL. + * + *

The vendor path is constructed as follows: /registry_cache// + * + *

The host name is case-insensitive, so it is converted to lowercase. The path is + * case-sensitive, so it is left as is. The port number is not included in the vendor path. + * + *

Note that the vendor path may conflict if two URLs only differ by the case or port number. + * But this is unlikely to happen in practice, and conflicts are checked in VendorCommand.java. + * + * @param url The URL to get the vendor path for. + * @return The vendor path. + * @throws UnsupportedEncodingException if the URL decoding fails. + */ + public Path getVendorPathForUrl(URL url) throws UnsupportedEncodingException { + String host = url.getHost().toLowerCase(Locale.ROOT); // Host names are case-insensitive + String path = url.getPath(); + path = URLDecoder.decode(path, "UTF-8"); + if (path.startsWith("/")) { + path = path.substring(1); + } + return vendorDirectory.getRelative(REGISTRIES_DIR).getRelative(host).getRelative(path); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD index 34b0022e442af0..37fcb87144414c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD @@ -31,17 +31,17 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:no_build_request_finished_event", "//src/main/java/com/google/devtools/build/lib/bazel:resolved_event", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", - "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:exception", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:repo_rule_value", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution", - "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution_impl", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:root_module_file_fixup", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:tidy", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:vendor", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand", "//src/main/java/com/google/devtools/build/lib/bazel/repository", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java index fd6e4411815b49..7082496087e627 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java @@ -14,7 +14,6 @@ package com.google.devtools.build.lib.bazel.commands; import static com.google.common.collect.ImmutableList.toImmutableList; -import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -22,9 +21,13 @@ import com.google.devtools.build.lib.analysis.NoBuildEvent; import com.google.devtools.build.lib.analysis.NoBuildRequestFinishedEvent; import com.google.devtools.build.lib.bazel.bzlmod.BazelFetchAllValue; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionValue; +import com.google.devtools.build.lib.bazel.bzlmod.VendorManager; import com.google.devtools.build.lib.bazel.commands.RepositoryFetcher.RepositoryFetcherException; import com.google.devtools.build.lib.bazel.commands.TargetFetcher.TargetFetcherException; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; +import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.buildtool.BuildResult; import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.cmdline.RepositoryName; @@ -49,10 +52,7 @@ import com.google.devtools.build.lib.skyframe.RepositoryMappingValue.RepositoryMappingResolutionException; import com.google.devtools.build.lib.util.DetailedExitCode; import com.google.devtools.build.lib.util.InterruptedFailureDetails; -import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; -import com.google.devtools.build.lib.vfs.PathFragment; -import com.google.devtools.build.lib.vfs.Symlinks; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; import com.google.devtools.build.skyframe.InMemoryGraph; @@ -62,15 +62,24 @@ import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.common.options.OptionsParser; import com.google.devtools.common.options.OptionsParsingResult; + import java.io.IOException; +import java.net.URI; +import java.net.URL; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Optional; import java.util.Queue; import java.util.Set; +import java.util.function.Supplier; + import javax.annotation.Nullable; /** @@ -97,11 +106,21 @@ usesConfigurationOptions = true, help = "resource:vendor.txt", shortDescription = - "Fetches external repositories into a specific folder specified by the flag " - + "--vendor_dir.") + "Fetches external repositories into a folder specified by the flag --vendor_dir.") public final class VendorCommand implements BlazeCommand { public static final String NAME = "vendor"; + private final DownloadManager downloadManager; + private final Supplier> clientEnvironmentSupplier; + @Nullable + private VendorManager vendorManager = null; + + public VendorCommand( + DownloadManager downloadManager, Supplier> clientEnvironmentSupplier) { + this.downloadManager = downloadManager; + this.clientEnvironmentSupplier = clientEnvironmentSupplier; + } + @Override public void editOptions(OptionsParser optionsParser) { // We only need to inject these options with fetch target (when there is a residue) @@ -135,15 +154,16 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti BlazeCommandResult result; VendorOptions vendorOptions = options.getOptions(VendorOptions.class); - PathFragment vendorDirectory = options.getOptions(RepositoryOptions.class).vendorDirectory; LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); + Path vendorDirectory = env.getWorkspace().getRelative(options.getOptions(RepositoryOptions.class).vendorDirectory); + this.vendorManager = new VendorManager(vendorDirectory); try { if (!options.getResidue().isEmpty()) { - result = vendorTargets(env, options, options.getResidue(), vendorDirectory); + result = vendorTargets(env, options, options.getResidue()); } else if (!vendorOptions.repos.isEmpty()) { - result = vendorRepos(env, threadsOption, vendorOptions.repos, vendorDirectory); + result = vendorRepos(env, threadsOption, vendorOptions.repos); } else { - result = vendorAll(env, threadsOption, vendorDirectory); + result = vendorAll(env, threadsOption); } } catch (InterruptedException e) { return createFailedBlazeCommandResult( @@ -171,17 +191,17 @@ private BlazeCommandResult validateOptions(CommandEnvironment env, OptionsParsin return createFailedBlazeCommandResult( env.getReporter(), Code.OPTIONS_INVALID, - "You cannot run vendor without specifying --vendor_dir"); + "You cannot run the vendor command without specifying --vendor_dir"); } if (!options.getOptions(PackageOptions.class).fetch) { return createFailedBlazeCommandResult( - env.getReporter(), Code.OPTIONS_INVALID, "You cannot run vendor with --nofetch"); + env.getReporter(), Code.OPTIONS_INVALID, "You cannot run the vendor command with --nofetch"); } return null; } private BlazeCommandResult vendorAll( - CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, PathFragment vendorDirectory) + CommandEnvironment env, LoadingPhaseThreadsOption threadsOption) throws InterruptedException, IOException { EvaluationContext evaluationContext = EvaluationContext.newBuilder() @@ -200,7 +220,8 @@ private BlazeCommandResult vendorAll( } BazelFetchAllValue fetchAllValue = (BazelFetchAllValue) evaluationResult.get(fetchKey); - vendor(env, vendorDirectory, fetchAllValue.getReposToVendor()); + env.getReporter().handle(Event.info("Vendoring all external repositories...")); + vendor(env, fetchAllValue.getReposToVendor()); env.getReporter().handle(Event.info("All external dependencies vendored successfully.")); return BlazeCommandResult.success(); } @@ -208,8 +229,7 @@ private BlazeCommandResult vendorAll( private BlazeCommandResult vendorRepos( CommandEnvironment env, LoadingPhaseThreadsOption threadsOption, - List repos, - PathFragment vendorDirectory) + List repos) throws InterruptedException, IOException { ImmutableMap repositoryNamesAndValues; try { @@ -235,7 +255,8 @@ private BlazeCommandResult vendorRepos( } } - vendor(env, vendorDirectory, reposToVendor.build()); + env.getReporter().handle(Event.info("Vendoring repositories...")); + vendor(env, reposToVendor.build()); if (!notFoundRepoErrors.isEmpty()) { return createFailedBlazeCommandResult( env.getReporter(), "Vendoring some repos failed with errors: " + notFoundRepoErrors); @@ -247,8 +268,7 @@ private BlazeCommandResult vendorRepos( private BlazeCommandResult vendorTargets( CommandEnvironment env, OptionsParsingResult options, - List targets, - PathFragment vendorDirectory) + List targets) throws InterruptedException, IOException { // Call fetch which runs build to have the targets graph and configuration set BuildResult buildResult; @@ -272,7 +292,8 @@ private BlazeCommandResult vendorTargets( InMemoryGraph inMemoryGraph = env.getSkyframeExecutor().getEvaluator().getInMemoryGraph(); ImmutableSet reposToVendor = collectReposFromTargets(inMemoryGraph, targetKeys); - vendor(env, vendorDirectory, reposToVendor.asList()); + env.getReporter().handle(Event.info("Vendoring dependencies for targets...")); + vendor(env, reposToVendor.asList()); env.getReporter() .handle( Event.info( @@ -308,59 +329,80 @@ private ImmutableSet collectReposFromTargets( * ignored or was already vendored and up-to-date */ private void vendor( - CommandEnvironment env, - PathFragment vendorDirectory, - ImmutableList reposToVendor) - throws IOException { - Path vendorPath = - vendorDirectory.isAbsolute() - ? env.getRuntime().getFileSystem().getPath(vendorDirectory) - : env.getWorkspace().getRelative(vendorDirectory); - Path externalPath = - env.getDirectories() - .getOutputBase() - .getRelative(LabelConstants.EXTERNAL_REPOSITORY_LOCATION); + CommandEnvironment env, ImmutableList reposToVendor) + throws IOException, InterruptedException { + Objects.requireNonNull(vendorManager); - if (!vendorPath.exists()) { - vendorPath.createDirectory(); - } + // 1. Vendor registry files + BazelModuleResolutionValue moduleResolutionValue = + (BazelModuleResolutionValue) + env.getSkyframeExecutor() + .getEvaluator() + .getExistingValue(BazelModuleResolutionValue.KEY); + ImmutableMap> registryFiles = + Objects.requireNonNull(moduleResolutionValue).getRegistryFileHashes(); - env.getReporter().handle(Event.info("Vendoring ...")); + // vendorPathToURL is a map of + // key: a vendor path string converted to lower case + // value: a URL string + // This map is for detecting potential rare vendor path conflicts, such as: + // http://foo.bar.com/BCR vs http://foo.bar.com/bcr => conflict vendor paths on + // case-insensitive system + // http://foo.bar.com/bcr vs http://foo.bar.com:8081/bcr => conflict vendor path because port + // number is ignored in vendor path + // The user has to update the Bazel registries this if such conflicts occur. + Map vendorPathToUrl = new HashMap<>(); + for (Entry> entry : registryFiles.entrySet()) { + URL url = URI.create(entry.getKey()).toURL(); + if (url.getProtocol().equals("file")) { + continue; + } + + String outputPath = vendorManager.getVendorPathForUrl(url).getPathString(); + String outputPathLowerCase = outputPath.toLowerCase(Locale.ROOT); + if (vendorPathToUrl.containsKey(outputPathLowerCase)) { + String previousUrl = vendorPathToUrl.get(outputPathLowerCase); + throw new IOException( + String.format( + "Vendor paths conflict detected for registry URLs:\n" + + " %s => %s\n" + + " %s => %s\n" + + "Their output paths are either the same or only differ by case, which will" + + " cause conflict on case insensitive file systems, please fix by changing the" + + " registry URLs!", + previousUrl, + vendorManager.getVendorPathForUrl(URI.create(previousUrl).toURL()).getPathString(), + entry.getKey(), + outputPath)); + } - // Update "out-of-date" repos under the vendor directory - for (RepositoryName repo : reposToVendor) { - if (!isRepoUpToDate(repo.getName(), vendorPath, externalPath)) { - Path repoUnderVendor = vendorPath.getRelative(repo.getName()); - if (!repoUnderVendor.exists()) { - repoUnderVendor.createDirectory(); + Optional checksum = entry.getValue(); + if (!vendorManager.isUrlVendored(url) + // Only vendor a registry URL when its checksum exists, otherwise the URL should be + // recorded as "not found" in moduleResolutionValue.getRegistryFileHashes() + && checksum.isPresent()) { + try { + vendorManager.vendorRegistryUrl( + url, + downloadManager.downloadAndReadOneUrlForBzlmod( + url, env.getReporter(), clientEnvironmentSupplier.get(), checksum)); + } catch (IOException e) { + throw new IOException( + String.format( + "Failed to vendor registry URL %s at %s: %s", url, outputPath, e.getMessage()), + e.getCause()); } - FileSystemUtils.copyTreesBelow( - externalPath.getRelative(repo.getName()), repoUnderVendor, Symlinks.NOFOLLOW); - FileSystemUtils.copyFile( - externalPath.getChild("@" + repo.getName() + ".marker"), - vendorPath.getChild("@" + repo.getName() + ".marker")); } - } - } - /** - * Returns whether the repo under vendor needs to be updated by comparing its marker file with the - * one under /external - */ - private boolean isRepoUpToDate(String repoName, Path vendorPath, Path externalPath) - throws IOException { - Path vendorMarkerFile = vendorPath.getChild("@" + repoName + ".marker"); - if (!vendorMarkerFile.exists()) { - return false; + vendorPathToUrl.put(outputPathLowerCase, entry.getKey()); } - // Since this runs after fetching repos, its guaranteed that the marker files - // under $OUTPUT_BASE/external are up-to-date. We just need to compare it against the marker - // under vendor. - Path externalMarkerFile = externalPath.getChild("@" + repoName + ".marker"); - String vendorMarkerContent = FileSystemUtils.readContent(vendorMarkerFile, UTF_8); - String externalMarkerContent = FileSystemUtils.readContent(externalMarkerFile, UTF_8); - return Objects.equals(vendorMarkerContent, externalMarkerContent); + // 2. Vendor repos + Path externalPath = + env.getDirectories() + .getOutputBase() + .getRelative(LabelConstants.EXTERNAL_REPOSITORY_LOCATION); + vendorManager.vendorRepos(externalPath, reposToVendor); } private static BlazeCommandResult createFailedBlazeCommandResult( diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryName.java b/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryName.java index b2f98e579a368f..588833aa670ad2 100644 --- a/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryName.java +++ b/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryName.java @@ -192,6 +192,11 @@ public String getName() { return name; } + /** Returns the marker file name for this repository. */ + public String getMarkerFileName() { + return "@" + name + ".marker"; + } + /** * Create a {@link RepositoryName} instance that indicates the requested repository name is * actually not visible from the owner repository and should fail in {@code diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java index ecebc7f072da1d..7dd616f0577741 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java @@ -274,7 +274,7 @@ private RepositoryDirectoryValue tryGettingValueUsingVendoredRepo( Path vendorPath = VENDOR_DIRECTORY.get(env).get(); Path vendorRepoPath = vendorPath.getRelative(repositoryName.getName()); if (vendorRepoPath.exists()) { - Path vendorMarker = vendorPath.getChild("@" + repositoryName.getName() + ".marker"); + Path vendorMarker = vendorPath.getChild(repositoryName.getMarkerFileName()); if (vendorFile.getPinnedRepos().contains(repositoryName)) { // pinned repos are used as they are without checking their marker file try { @@ -303,7 +303,7 @@ private RepositoryDirectoryValue tryGettingValueUsingVendoredRepo( String.format( "Vendored repository '%s' is out-of-date and fetching is disabled." + " Run build without the '--nofetch' option or run" - + " `bazel vendor` to update it", + + " the bazel vendor command to update it", rule.getName()))); } return setupOverride(vendorRepoPath.asFragment(), env, repoRoot, repositoryName); @@ -316,7 +316,7 @@ private RepositoryDirectoryValue tryGettingValueUsingVendoredRepo( String.format( "Vendored repository '%s' is out-of-date. The up-to-date version will" + " be fetched into the external cache and used. To update the repo" - + " in the vendor directory, run 'bazel vendor'", + + " in the vendor directory, run the bazel vendor command", rule.getName()))); } } else if (vendorFile.getPinnedRepos().contains(repositoryName)) { @@ -332,7 +332,7 @@ private RepositoryDirectoryValue tryGettingValueUsingVendoredRepo( "Vendored repository " + repositoryName.getName() + " not found under the vendor directory and fetching is disabled." - + " To fix run 'bazel vendor' or build without the '--nofetch'"), + + " To fix, run the bazel vendor command or build without the '--nofetch'"), Transience.TRANSIENT); } return null; @@ -621,7 +621,7 @@ private static class DigestWriter { StarlarkSemantics starlarkSemantics) { this.directories = directories; ruleKey = computeRuleKey(rule, starlarkSemantics); - markerPath = getMarkerPath(directories, repositoryName.getName()); + markerPath = getMarkerPath(directories, repositoryName); this.rule = rule; recordedInputValues = Maps.newTreeMap(); } @@ -731,15 +731,15 @@ private String computeRuleKey(Rule rule, StarlarkSemantics starlarkSemantics) { .hexDigestAndReset(); } - private static Path getMarkerPath(BlazeDirectories directories, String ruleName) { + private static Path getMarkerPath(BlazeDirectories directories, RepositoryName repo) { return RepositoryFunction.getExternalRepositoryDirectory(directories) - .getChild("@" + ruleName + ".marker"); + .getChild(repo.getMarkerFileName()); } - static void clearMarkerFile(BlazeDirectories directories, RepositoryName repoName) + static void clearMarkerFile(BlazeDirectories directories, RepositoryName repo) throws RepositoryFunctionException { try { - getMarkerPath(directories, repoName.getName()).delete(); + getMarkerPath(directories, repo).delete(); } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index d783e91e4edd3c..3566f8a1456407 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -120,6 +120,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/packages", + "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/net/starlark/java/eval", "//src/main/java/net/starlark/java/syntax", "//third_party:guava", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index 9c98532eec437b..763eeec7b74044 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -37,6 +37,7 @@ import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.Type; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; import com.google.devtools.build.lib.skyframe.BzlmodRepoRuleFunction; import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; @@ -62,6 +63,7 @@ import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionName; +import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import net.starlark.java.eval.StarlarkSemantics; import org.junit.Before; @@ -158,6 +160,7 @@ public void setup() throws Exception { BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); + RepositoryDelegatorFunction.VENDOR_DIRECTORY.set(differencer, Optional.empty()); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java index e093f5f326a807..0b9b5be3c23e0d 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java @@ -23,6 +23,7 @@ import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.vfs.Path; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.HashMap; import java.util.Map; @@ -136,7 +137,8 @@ public Registry createRegistry( String url, LockfileMode lockfileMode, ImmutableMap> fileHashes, - ImmutableMap previouslySelectedYankedVersions) { + ImmutableMap previouslySelectedYankedVersions, + Optional vendorDir) { return Preconditions.checkNotNull(registries.get(url), "unknown registry url: %s", url); } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java index b4d259c398aee9..333aa82d197fe1 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java @@ -97,7 +97,11 @@ public void testHttpUrl() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl() + "/myreg", + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue( ModuleFile.create( @@ -115,7 +119,11 @@ public void testHttpUrlWithNetrcCreds() throws Exception { "machine [::1] login rinne password rinnepass\n".getBytes(UTF_8))); Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl() + "/myreg", + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); var e = assertThrows( @@ -148,7 +156,8 @@ public void testFileUrl() throws Exception { new File(tempFolder.getRoot(), "fakereg").toURI().toString(), LockfileMode.UPDATE, ImmutableMap.of(), - ImmutableMap.of()); + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue(ModuleFile.create("lol".getBytes(UTF_8), file.toURI().toString())); assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty(); @@ -197,7 +206,11 @@ public void testGetArchiveRepoSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -265,7 +278,11 @@ public void testGetLocalPathRepoSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( RepoSpec.builder() @@ -289,7 +306,11 @@ public void testGetRepoInvalidRegistryJsonSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -323,7 +344,11 @@ public void testGetRepoInvalidModuleJsonSpec() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); } @@ -352,7 +377,11 @@ public void testGetYankedVersion() throws Exception { server.start(); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); Optional> yankedVersion = registry.getYankedVersions("red-pill", reporter); assertThat(yankedVersion) @@ -375,7 +404,11 @@ public void testArchiveWithExplicitType() throws Exception { Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); + server.getUrl(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("archive_type", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -405,7 +438,11 @@ public void testGetModuleFileChecksums() throws Exception { Optional.of(sha256("unused"))); Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl() + "/myreg", + LockfileMode.UPDATE, + knownFiles, + ImmutableMap.of(), + Optional.empty()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue( ModuleFile.create( @@ -431,7 +468,11 @@ public void testGetModuleFileChecksums() throws Exception { registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of()); + server.getUrl() + "/myreg", + LockfileMode.UPDATE, + recordedChecksums, + ImmutableMap.of(), + Optional.empty()); // Test that the recorded hashes are used for repo cache hits even when the server content // changes. server.unserve("/myreg/modules/foo/1.0/MODULE.bazel"); @@ -461,7 +502,11 @@ public void testGetModuleFileChecksumMismatch() throws Exception { Optional.of(sha256("original"))); Registry registry = registryFactory.createRegistry( - server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl() + "/myreg", + LockfileMode.UPDATE, + knownFiles, + ImmutableMap.of(), + Optional.empty()); var e = assertThrows( IOException.class, @@ -502,7 +547,7 @@ public void testGetRepoSpecChecksum() throws Exception { server.getUrl() + "/modules/foo/2.0/source.json", Optional.of(sha256("unused"))); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( RepoSpec.builder() @@ -522,7 +567,11 @@ public void testGetRepoSpecChecksum() throws Exception { registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of()); + server.getUrl(), + LockfileMode.UPDATE, + recordedChecksums, + ImmutableMap.of(), + Optional.empty()); // Test that the recorded hashes are used for repo cache hits even when the server content // changes. server.unserve("/bazel_registry.json"); @@ -566,7 +615,7 @@ public void testGetRepoSpecChecksumMismatch() throws Exception { Optional.of(sha256(sourceJson))); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); var e = assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); @@ -610,7 +659,7 @@ public void testBazelRegistryChecksumMismatch() throws Exception { Optional.of(sha256(sourceJson))); Registry registry = registryFactory.createRegistry( - server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); + server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of(), Optional.empty()); var e = assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java index cc963f233e5e3e..fb52c7fe216416 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java @@ -25,6 +25,7 @@ import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; import java.net.URISyntaxException; +import java.util.Optional; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -44,14 +45,22 @@ public void badSchemes() { URISyntaxException.class, () -> registryFactory.createRegistry( - "/home/www", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of())); + "/home/www", + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty())); assertThat(exception).hasMessageThat().contains("Registry URL has no scheme"); exception = assertThrows( URISyntaxException.class, () -> registryFactory.createRegistry( - "foo://bar", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of())); + "foo://bar", + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty())); assertThat(exception).hasMessageThat().contains("Unrecognized registry URL protocol"); } @@ -69,7 +78,8 @@ public void badPath() { "file:c:/path/to/workspace/registry", LockfileMode.UPDATE, ImmutableMap.of(), - ImmutableMap.of())); + ImmutableMap.of(), + Optional.empty())); assertThat(exception).hasMessageThat().contains("Registry URL path is not valid"); } } diff --git a/src/test/py/bazel/bzlmod/bazel_vendor_test.py b/src/test/py/bazel/bzlmod/bazel_vendor_test.py index 0ac2b7cad18294..d1ca73787b5a1f 100644 --- a/src/test/py/bazel/bzlmod/bazel_vendor_test.py +++ b/src/test/py/bazel/bzlmod/bazel_vendor_test.py @@ -79,7 +79,7 @@ def generateBuiltinModules(self): def testBasicVendoring(self): self.main_registry.createCcModule('aaa', '1.0').createCcModule( 'bbb', '1.0', {'aaa': '1.0'} - ) + ).createCcModule('bbb', '2.0') self.ScratchFile( 'MODULE.bazel', [ @@ -94,13 +94,31 @@ def testBasicVendoring(self): self.RunBazel(['vendor', '--vendor_dir=vendor']) # Assert repos are vendored with marker files and VENDOR.bazel is created - repos_vendored = os.listdir(self._test_cwd + '/vendor') + vendor_dir = self._test_cwd + '/vendor' + repos_vendored = os.listdir(vendor_dir) self.assertIn('aaa~', repos_vendored) self.assertIn('bbb~', repos_vendored) self.assertIn('@aaa~.marker', repos_vendored) self.assertIn('@bbb~.marker', repos_vendored) self.assertIn('VENDOR.bazel', repos_vendored) + # Update bbb to 2.0 and re-vendor + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "bbb", version = "2.0")', + 'local_path_override(module_name="bazel_tools", path="tools_mock")', + 'local_path_override(module_name="local_config_platform", ', + 'path="platforms_mock")', + ], + ) + self.ScratchFile("vendor/bbb~/foo") + self.RunBazel(['vendor', '--vendor_dir=vendor']) + bbb_module_bazel = os.path.join(vendor_dir, 'bbb~/MODULE.bazel') + self.AssertFileContentContains(bbb_module_bazel, 'version = "2.0"') + foo = os.path.join(vendor_dir, 'bbb~/foo') + self.assertFalse(os.path.exists(foo)) # foo should be removed due to re-vendor + def testVendorFailsWithNofetch(self): self.ScratchFile( 'MODULE.bazel', @@ -117,7 +135,7 @@ def testVendorFailsWithNofetch(self): _, _, stderr = self.RunBazel( ['vendor', '--vendor_dir=vendor', '--nofetch'], allow_failure=True ) - self.assertIn('ERROR: You cannot run vendor with --nofetch', stderr) + self.assertIn('ERROR: You cannot run the vendor command with --nofetch', stderr) def testVendoringMultipleTimes(self): self.main_registry.createCcModule('aaa', '1.0') @@ -258,35 +276,6 @@ def testVendorDirIsNotCheckedForWorkspaceRepos(self): "Vendored repository 'dummyRepo' is out-of-date.", '\n'.join(stderr) ) - def testBuildingWithVendoredRepos(self): - self.main_registry.createCcModule('aaa', '1.0') - self.ScratchFile( - 'MODULE.bazel', - [ - 'bazel_dep(name = "aaa", version = "1.0")', - ], - ) - self.ScratchFile('BUILD') - self.RunBazel(['vendor', '--vendor_dir=vendor']) - self.assertIn('aaa~', os.listdir(self._test_cwd + '/vendor')) - - # Empty external & build with vendor - self.RunBazel(['clean', '--expunge']) - _, _, stderr = self.RunBazel(['build', '@aaa//:all', '--vendor_dir=vendor']) - self.assertNotIn( - "Vendored repository '_main~ext~justRepo' is out-of-date.", - '\n'.join(stderr), - ) - - # Assert repo aaa in {OUTPUT_BASE}/external is a symlink (junction on - # windows, this validates it was created from vendor and not fetched)= - _, stdout, _ = self.RunBazel(['info', 'output_base']) - repo_path = stdout[0] + '/external/aaa~' - if self.IsWindows(): - self.assertTrue(self.IsJunction(repo_path)) - else: - self.assertTrue(os.path.islink(repo_path)) - def testIgnoreFromVendoring(self): # Repos should be excluded from vendoring: # 1.Local Repos, 2.Config Repos, 3.Repos declared in VENDOR.bazel file @@ -372,7 +361,7 @@ def testBuildingWithPinnedRepo(self): ) self.ScratchFile('BUILD') - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@venRepo']) self.assertIn('_main~ext~venRepo', os.listdir(self._test_cwd + '/vendor')) self.ScratchFile( 'extension.bzl', @@ -421,7 +410,7 @@ def testBuildingWithPinnedRepo(self): ) # Re-vendor & build make sure the repo is successfully updated - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@venRepo']) _, _, stderr = self.RunBazel( ['build', '@venRepo//:all', '--vendor_dir=vendor'], ) @@ -454,7 +443,7 @@ def testBuildingOutOfDateVendoredRepo(self): ) # Vendor, assert and build with no problems - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@justRepo']) self.assertIn('_main~ext~justRepo', os.listdir(self._test_cwd + '/vendor')) _, _, stderr = self.RunBazel( ['build', '@justRepo//:all', '--vendor_dir=vendor'] @@ -462,8 +451,8 @@ def testBuildingOutOfDateVendoredRepo(self): self.assertNotIn( "WARNING: : Vendored repository '_main~ext~justRepo' is" ' out-of-date. The up-to-date version will be fetched into the external' - ' cache and used. To update the repo in the vendor directory, run' - " 'bazel vendor'", + ' cache and used. To update the repo in the vendor directory, run' + ' the bazel vendor command', stderr, ) @@ -492,15 +481,15 @@ def testBuildingOutOfDateVendoredRepo(self): self.assertIn( "WARNING: : Vendored repository '_main~ext~justRepo' is" ' out-of-date. The up-to-date version will be fetched into the external' - ' cache and used. To update the repo in the vendor directory, run' - " 'bazel vendor'", + ' cache and used. To update the repo in the vendor directory, run' + ' the bazel vendor command', stderr, ) _, stdout, _ = self.RunBazel(['info', 'output_base']) self.assertFalse(os.path.islink(stdout[0] + '/external/bbb~')) # Assert vendoring again solves the problem - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '--repo=@justRepo']) self.RunBazel(['clean', '--expunge']) _, _, stderr = self.RunBazel( ['build', '@justRepo//:all', '--vendor_dir=vendor'] @@ -508,12 +497,12 @@ def testBuildingOutOfDateVendoredRepo(self): self.assertNotIn( "WARNING: : Vendored repository '_main~ext~justRepo' is" ' out-of-date. The up-to-date version will be fetched into the external' - ' cache and used. To update the repo in the vendor directory, run' - " 'bazel vendor'", + ' cache and used. To update the repo in the vendor directory, run' + ' the bazel vendor command', stderr, ) - def testBuildingVendoredRepoInOfflineMode(self): + def testBuildingVendoredRepoWithNoFetch(self): self.ScratchFile( 'MODULE.bazel', [ @@ -537,7 +526,7 @@ def testBuildingVendoredRepoInOfflineMode(self): self.ScratchFile('BUILD') # Vendor, assert and build with no problems - self.RunBazel(['vendor', '--vendor_dir=vendor']) + self.RunBazel(['vendor', '--vendor_dir=vendor', '@venRepo//:all']) self.assertIn('_main~ext~venRepo', os.listdir(self._test_cwd + '/vendor')) # Make updates in repo definition @@ -571,8 +560,8 @@ def testBuildingVendoredRepoInOfflineMode(self): ) self.assertIn( 'ERROR: Vendored repository _main~ext~noVenRepo not found under the' - " vendor directory and fetching is disabled. To fix run 'bazel" - " vendor' or build without the '--nofetch'", + ' vendor directory and fetching is disabled. To fix, run the bazel' + " vendor command or build without the '--nofetch'", stderr, ) @@ -584,7 +573,7 @@ def testBuildingVendoredRepoInOfflineMode(self): self.assertIn( "WARNING: : Vendored repository '_main~ext~venRepo' is" ' out-of-date and fetching is disabled. Run build without the' - " '--nofetch' option or run `bazel vendor` to update it", + " '--nofetch' option or run the bazel vendor command to update it", stderr, ) # Assert the out-dated repo is the one built with @@ -615,7 +604,7 @@ def testBasicVendorTarget(self): self.assertIn('bbb~', os.listdir(self._test_cwd + '/vendor')) self.assertNotIn('ccc~', os.listdir(self._test_cwd + '/vendor')) - def testVendorTarget(self): + def testBuildVendoredTargetOffline(self): self.main_registry.createCcModule('aaa', '1.0').createCcModule( 'bbb', '1.0', {'aaa': '1.0'} ) @@ -640,22 +629,71 @@ def testVendorTarget(self): self.ScratchFile( 'main.cc', [ - '#include "aaa.h"', + '#include "bbb.h"', 'int main() {', - ' hello_aaa("Hello there!");', + ' hello_bbb("Hello there!");', '}', ], ) self.RunBazel(['vendor', '//:main', '--vendor_dir=vendor']) - # Run the vendored target with --nofetch should only use what is under - # vendor to build, meaning we have vendored everything we need to build/run - # this target + # Build and run the target in a clean build with internet blocked and make + # sure it works + _, _, _ = self.RunBazel(['clean', '--expunge']) _, stdout, _ = self.RunBazel( - ['run', '//:main', '--vendor_dir=vendor', '--nofetch'] + ['run', '//:main', '--vendor_dir=vendor', '--repository_cache='], + env_add={ + 'HTTP_PROXY': 'internet_blocked', + 'HTTPS_PROXY': 'internet_blocked', + }, + ) + self.assertIn('Hello there! => bbb@1.0', stdout) + + # Assert repos in {OUTPUT_BASE}/external are symlinks (junction on + # windows, this validates it was created from vendor and not fetched) + _, stdout, _ = self.RunBazel(['info', 'output_base']) + for repo in ['aaa~', 'bbb~']: + repo_path = stdout[0] + '/external/' + repo + if self.IsWindows(): + self.assertTrue(self.IsJunction(repo_path)) + else: + self.assertTrue(os.path.islink(repo_path)) + + def testVendorConflictRegistryFile(self): + self.main_registry.createCcModule('aaa', '1.0').createCcModule( + 'bbb', '1.0', {'aaa': '1.0'} + ) + # The registry URLs of main_registry and another_registry only differ by the + # port number + another_registry = BazelRegistry( + os.path.join(self.registries_work_dir, 'MAIN'), + ) + another_registry.start() + another_registry.createCcModule('aaa', '1.0') + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "bbb", version = "1.0")', + 'local_path_override(module_name="bazel_tools", path="tools_mock")', + 'local_path_override(module_name="local_config_platform", ', + 'path="platforms_mock")', + 'single_version_override(', + ' module_name = "aaa",', + ' registry = "%s",' % another_registry.getURL(), + ')', + ], + ) + self.ScratchFile('BUILD') + exit_code, _, stderr = self.RunBazel( + ['vendor', '--vendor_dir=vendor'], allow_failure=True + ) + self.AssertExitCode(exit_code, 8, stderr) + self.assertIn( + 'ERROR: Error while vendoring repos: Vendor paths conflict detected for' + ' registry URLs:', + stderr, ) - self.assertIn('Hello there! => aaa@1.0', stdout) if __name__ == '__main__':