diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 93b58dd58c431f..344180e200cef3 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -2917,7 +2917,7 @@ "bzlTransitiveDigest": "tunTSmgwd2uvTzkCLtdbuCp0AI+WR+ftiPNqZ0rmcZk=", "recordedFileInputs": { "@@//MODULE.bazel": "eba5503742af5785c2d0d81d88e7407c7f23494b5162c055227435549b8774d1", - "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "36d43264878997e1a777c4af38df4d4f17b0d7dcb37559e39bfb574ec2e33f42" + "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "547b1ca7af37ca0b4e7c7de36093d66b81d46440b58b41c76fe9d6df3af9ea52" }, "recordedDirentsInputs": {}, "envVariables": {}, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index ef04d6c6d86361..cd407673755a47 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -15,12 +15,16 @@ package com.google.devtools.build.lib.bazel; +import static com.google.common.collect.ImmutableSet.toImmutableSet; + import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.CharMatcher; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.devtools.build.lib.analysis.BlazeDirectories; @@ -134,8 +138,8 @@ public class BazelRepositoryModule extends BlazeModule { public static final String DEFAULT_CACHE_LOCATION = "cache/repos/v1"; // Default list of registries. - public static final ImmutableList DEFAULT_REGISTRIES = - ImmutableList.of("https://bcr.bazel.build/"); + public static final ImmutableSet DEFAULT_REGISTRIES = + ImmutableSet.of("https://bcr.bazel.build/"); // A map of repository handlers that can be looked up by rule class name. private final ImmutableMap repositoryHandlers; @@ -153,7 +157,7 @@ public class BazelRepositoryModule extends BlazeModule { private ImmutableMap moduleOverrides = ImmutableMap.of(); private Optional resolvedFileReplacingWorkspace = Optional.empty(); private FileSystem filesystem; - private List registries; + private ImmutableSet registries; private final AtomicBoolean ignoreDevDeps = new AtomicBoolean(false); private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING; private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR; @@ -530,7 +534,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { } if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) { - registries = repoOptions.registries; + registries = normalizeRegistries(repoOptions.registries); } else { registries = DEFAULT_REGISTRIES; } @@ -560,6 +564,14 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { } } + private static ImmutableSet normalizeRegistries(List registries) { + // Ensure that registries aren't duplicated even after `/modules/...` paths are appended to + // them. + return registries.stream() + .map(url -> CharMatcher.is('/').trimTrailingFrom(url)) + .collect(toImmutableSet()); + } + /** * If the given path is absolute path, leave it as it is. If the given path is a relative path, it * is relative to the current working directory. If the given path starts with '%workspace%, it is diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 6df8365918bc28..20b00ede7f6655 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -77,9 +77,11 @@ java_library( "Registry.java", "RegistryFactory.java", "RegistryFactoryImpl.java", + "RegistryFileDownloadEvent.java", ], deps = [ ":common", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", @@ -98,11 +100,13 @@ java_library( srcs = ["BazelLockFileModule.java"], deps = [ ":exception", + ":registry", ":resolution", ":resolution_impl", "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_cluster", "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception", @@ -144,6 +148,7 @@ java_library( "RegistryKey.java", "RegistryOverride.java", "RepoSpecKey.java", + "RepoSpecValue.java", "SingleExtensionUsagesValue.java", "SingleExtensionValue.java", "SingleVersionOverride.java", @@ -160,6 +165,7 @@ java_library( ":root_module_file_fixup", "//src/main/java/com/google/devtools/build/docgen/annot", "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/packages", @@ -226,6 +232,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info", "//src/main/java/com/google/devtools/build/lib/bazel:bazel_version", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark", "//src/main/java/com/google/devtools/build/lib/cmdline", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java index 21dce5d1e59c03..6bdf154a116ecb 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java @@ -30,6 +30,7 @@ import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableTable; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; @@ -186,7 +187,7 @@ static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env) throws Interrup return null; } - ImmutableList registries = ImmutableList.copyOf(ModuleFileFunction.REGISTRIES.get(env)); + ImmutableSet registries = ImmutableSet.copyOf(ModuleFileFunction.REGISTRIES.get(env)); ImmutableMap moduleOverrides = ModuleFileFunction.MODULE_OVERRIDES.get(env).entrySet().stream() .collect( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java index 500a5775bd8051..20206b8d30ab69 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.cmdline.LabelConstants; @@ -55,7 +56,7 @@ public class BazelLockFileFunction implements SkyFunction { private static final BzlmodFlagsAndEnvVars EMPTY_FLAGS = BzlmodFlagsAndEnvVars.create( - ImmutableList.of(), ImmutableMap.of(), ImmutableList.of(), "", false, "", ""); + ImmutableSet.of(), ImmutableMap.of(), ImmutableList.of(), "", false, "", ""); private static final BazelLockFileValue EMPTY_LOCKFILE = BazelLockFileValue.builder() @@ -65,6 +66,7 @@ public class BazelLockFileFunction implements SkyFunction { .setLocalOverrideHashes(ImmutableMap.of()) .setModuleDepGraph(ImmutableMap.of()) .setModuleExtensions(ImmutableMap.of()) + .setRegistryFileHashes(ImmutableMap.of()) .build(); public BazelLockFileFunction(Path rootDirectory) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java index 6116bd5b7171dd..4456e09f9f8c43 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java @@ -22,11 +22,11 @@ import com.google.common.flogger.GoogleLogger; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.runtime.BlazeModule; import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.skyframe.SkyframeExecutor; -import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.Root; @@ -34,6 +34,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nullable; @@ -45,6 +46,7 @@ public class BazelLockFileModule extends BlazeModule { private SkyframeExecutor executor; private Path workspaceRoot; + private boolean enabled; @Nullable private BazelModuleResolutionEvent moduleResolutionEvent; private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); @@ -53,20 +55,44 @@ public class BazelLockFileModule extends BlazeModule { public void beforeCommand(CommandEnvironment env) { executor = env.getSkyframeExecutor(); workspaceRoot = env.getWorkspace(); - RepositoryOptions options = env.getOptions().getOptions(RepositoryOptions.class); - if (options.lockfileMode.equals(LockfileMode.UPDATE)) { - env.getEventBus().register(this); - } + + enabled = + env.getOptions().getOptions(RepositoryOptions.class).lockfileMode == LockfileMode.UPDATE; + moduleResolutionEvent = null; + env.getEventBus().register(this); } @Override - public void afterCommand() throws AbruptExitException { - if (moduleResolutionEvent == null) { + public void afterCommand() { + if (!enabled || moduleResolutionEvent == null) { // Command does not use Bazel modules or the lockfile mode is not update. // Since Skyframe caches events, they are replayed even when nothing has changed. return; } + BazelDepGraphValue depGraphValue; + BazelModuleResolutionValue moduleResolutionValue; + try { + depGraphValue = + (BazelDepGraphValue) executor.getEvaluator().getExistingValue(BazelDepGraphValue.KEY); + moduleResolutionValue = + (BazelModuleResolutionValue) + executor.getEvaluator().getExistingValue(BazelModuleResolutionValue.KEY); + } catch (InterruptedException e) { + // Not thrown in Bazel. + throw new IllegalStateException(e); + } + + BazelLockFileValue oldLockfile = moduleResolutionEvent.getOnDiskLockfileValue(); + ImmutableMap> fileHashes; + if (moduleResolutionValue == null) { + // BazelDepGraphFunction took the dep graph from the lockfile and didn't cause evaluation of + // BazelModuleResolutionFunction. The file hashes in the lockfile are still up-to-date. + fileHashes = oldLockfile.getRegistryFileHashes(); + } else { + fileHashes = ImmutableSortedMap.copyOf(moduleResolutionValue.getRegistryFileHashes()); + } + // All nodes corresponding to module extensions that have been evaluated in the current build // are done at this point. Look up entries by eval keys to record results even if validation // later fails due to invalid imports. @@ -88,24 +114,16 @@ public void afterCommand() throws AbruptExitException { newExtensionInfos.put(key.argument(), value.getLockFileInfo().get()); } }); + var combinedExtensionInfos = + combineModuleExtensions( + oldLockfile.getModuleExtensions(), newExtensionInfos, depGraphValue); - BazelDepGraphValue depGraphValue; - try { - depGraphValue = - (BazelDepGraphValue) executor.getEvaluator().getExistingValue(BazelDepGraphValue.KEY); - } catch (InterruptedException e) { - // Not thrown in Bazel. - throw new IllegalStateException(e); - } - - BazelLockFileValue oldLockfile = moduleResolutionEvent.getOnDiskLockfileValue(); // Create an updated version of the lockfile, keeping only the extension results from the old // lockfile that are still up-to-date and adding the newly resolved extension results. BazelLockFileValue newLockfile = moduleResolutionEvent.getResolutionOnlyLockfileValue().toBuilder() - .setModuleExtensions( - combineModuleExtensions( - oldLockfile.getModuleExtensions(), newExtensionInfos, depGraphValue)) + .setRegistryFileHashes(fileHashes) + .setModuleExtensions(combinedExtensionInfos) .build(); // Write the new value to the file, but only if needed. This is not just a performance @@ -115,7 +133,6 @@ public void afterCommand() throws AbruptExitException { if (!newLockfile.equals(oldLockfile)) { updateLockfile(workspaceRoot, newLockfile); } - this.moduleResolutionEvent = null; } /** diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java index 8bc8f042623a0b..cd3eff788a8962 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java @@ -20,6 +20,7 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; @@ -27,6 +28,7 @@ import com.google.devtools.build.skyframe.SkyValue; import com.ryanharter.auto.value.gson.GenerateTypeAdapter; import java.util.Map; +import java.util.Optional; /** * The result of reading the lockfile. Contains the lockfile version, module hash, definitions of @@ -37,14 +39,15 @@ @GenerateTypeAdapter public abstract class BazelLockFileValue implements SkyValue, Postable { - public static final int LOCK_FILE_VERSION = 7; + public static final int LOCK_FILE_VERSION = 8; @SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.BAZEL_LOCK_FILE; static Builder builder() { return new AutoValue_BazelLockFileValue.Builder() .setLockFileVersion(LOCK_FILE_VERSION) - .setModuleExtensions(ImmutableMap.of()); + .setModuleExtensions(ImmutableMap.of()) + .setRegistryFileHashes(ImmutableMap.of()); } /** Current version of the lock file */ @@ -62,6 +65,9 @@ static Builder builder() { /** The post-selection dep graph retrieved from the lock file. */ public abstract ImmutableMap getModuleDepGraph(); + /** Hashes of files retrieved from registries. */ + public abstract ImmutableMap> getRegistryFileHashes(); + /** Mapping the extension id to the module extension data */ public abstract ImmutableMap< ModuleExtensionId, ImmutableMap> @@ -82,6 +88,8 @@ public abstract static class Builder { public abstract Builder setModuleDepGraph(ImmutableMap value); + public abstract Builder setRegistryFileHashes(ImmutableMap> value); + public abstract Builder setModuleExtensions( ImmutableMap< ModuleExtensionId, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java index 9f121a6b8e650c..9d863d9f677596 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java @@ -31,6 +31,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.profiler.Profiler; @@ -46,9 +47,11 @@ import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.build.skyframe.SkyframeLookupResult; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.SequencedMap; import java.util.Set; import javax.annotation.Nullable; @@ -63,6 +66,14 @@ public class BazelModuleResolutionFunction implements SkyFunction { public static final Precomputed BAZEL_COMPATIBILITY_MODE = new Precomputed<>("bazel_compatibility_mode"); + private record Result( + Selection.Result selectionResult, + ImmutableMap> registryFileHashes) {} + + private static class ModuleResolutionComputeState implements Environment.SkyKeyComputeState { + Result discoverAndSelectResult; + } + @Override @Nullable public SkyValue compute(SkyKey skyKey, Environment env) @@ -91,15 +102,17 @@ public SkyValue compute(SkyKey skyKey, Environment env) } var state = env.getState(ModuleResolutionComputeState::new); - if (state.selectionResult == null) { - state.selectionResult = discoverAndSelect(env, root, allowedYankedVersions); - if (state.selectionResult == null) { + if (state.discoverAndSelectResult == null) { + state.discoverAndSelectResult = discoverAndSelect(env, root, allowedYankedVersions); + if (state.discoverAndSelectResult == null) { return null; } } + SequencedMap> registryFileHashes = + new LinkedHashMap<>(state.discoverAndSelectResult.registryFileHashes); ImmutableSet repoSpecKeys = - state.selectionResult.getResolvedDepGraph().values().stream() + state.discoverAndSelectResult.selectionResult.getResolvedDepGraph().values().stream() // Modules with a null registry have a non-registry override. We don't need to // fetch or store the repo spec in this case. .filter(module -> module.getRegistry() != null) @@ -108,11 +121,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) SkyframeLookupResult repoSpecResults = env.getValuesAndExceptions(repoSpecKeys); ImmutableMap.Builder remoteRepoSpecs = ImmutableMap.builder(); for (RepoSpecKey repoSpecKey : repoSpecKeys) { - RepoSpec repoSpec = (RepoSpec) repoSpecResults.get(repoSpecKey); - if (repoSpec == null) { + RepoSpecValue repoSpecValue = (RepoSpecValue) repoSpecResults.get(repoSpecKey); + if (repoSpecValue == null) { return null; } - remoteRepoSpecs.put(repoSpecKey.getModuleKey(), repoSpec); + remoteRepoSpecs.put(repoSpecKey.getModuleKey(), repoSpecValue.repoSpec()); + registryFileHashes.putAll(repoSpecValue.registryFileHashes()); } ImmutableMap finalDepGraph; @@ -120,36 +134,38 @@ public SkyValue compute(SkyKey skyKey, Environment env) Profiler.instance().profile(ProfilerTask.BZLMOD, "compute final dep graph")) { finalDepGraph = computeFinalDepGraph( - state.selectionResult.getResolvedDepGraph(), + state.discoverAndSelectResult.selectionResult.getResolvedDepGraph(), root.getOverrides(), remoteRepoSpecs.buildOrThrow()); } return BazelModuleResolutionValue.create( - finalDepGraph, state.selectionResult.getUnprunedDepGraph()); + finalDepGraph, + state.discoverAndSelectResult.selectionResult.getUnprunedDepGraph(), + ImmutableMap.copyOf(registryFileHashes)); } @Nullable - private static Selection.Result discoverAndSelect( + private static Result discoverAndSelect( Environment env, RootModuleFileValue root, Optional> allowedYankedVersions) throws BazelModuleResolutionFunctionException, InterruptedException { - ImmutableMap initialDepGraph; + Discovery.Result discoveryResult; try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.BZLMOD, "discovery")) { - initialDepGraph = Discovery.run(env, root); + discoveryResult = Discovery.run(env, root); } catch (ExternalDepsException e) { throw new BazelModuleResolutionFunctionException(e, Transience.PERSISTENT); } - if (initialDepGraph == null) { + if (discoveryResult == null) { return null; } - verifyAllOverridesAreOnExistentModules(initialDepGraph, root.getOverrides()); + verifyAllOverridesAreOnExistentModules(discoveryResult.depGraph(), root.getOverrides()); Selection.Result selectionResult; try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.BZLMOD, "selection")) { - selectionResult = Selection.run(initialDepGraph, root.getOverrides()); + selectionResult = Selection.run(discoveryResult.depGraph(), root.getOverrides()); } catch (ExternalDepsException e) { throw new BazelModuleResolutionFunctionException(e, Transience.PERSISTENT); } @@ -176,7 +192,7 @@ private static Selection.Result discoverAndSelect( try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.BZLMOD, "verify root module direct deps")) { verifyRootModuleDirectDepsAreAccurate( - initialDepGraph.get(ModuleKey.ROOT), + discoveryResult.depGraph().get(ModuleKey.ROOT), resolvedDepGraph.get(ModuleKey.ROOT), Objects.requireNonNull(CHECK_DIRECT_DEPENDENCIES.get(env)), env.getListener()); @@ -195,7 +211,7 @@ private static Selection.Result discoverAndSelect( checkNoYankedVersions(resolvedDepGraph, yankedVersionValues, allowedYankedVersions); } - return selectionResult; + return new Result(selectionResult, discoveryResult.registryFileHashes()); } private static void verifyAllOverridesAreOnExistentModules( @@ -337,10 +353,6 @@ private static ImmutableMap computeFinalDepGraph( return finalDepGraph.buildOrThrow(); } - private static class ModuleResolutionComputeState implements Environment.SkyKeyComputeState { - Selection.Result selectionResult; - } - static class BazelModuleResolutionFunctionException extends SkyFunctionException { BazelModuleResolutionFunctionException(ExternalDepsException e, Transience transience) { super(e, transience); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java index 19c642cfa5d7f7..60971030d549b0 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java @@ -17,10 +17,12 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; +import java.util.Optional; /** * The result of the selection process, containing both the pruned and the un-pruned dependency @@ -46,9 +48,16 @@ abstract class BazelModuleResolutionValue implements SkyValue { */ abstract ImmutableMap getUnprunedDepGraph(); + /** + * Hashes of files obtained (or known to be missing) from registries while performing resolution. + */ + abstract ImmutableMap> getRegistryFileHashes(); + static BazelModuleResolutionValue create( ImmutableMap resolvedDepGraph, - ImmutableMap unprunedDepGraph) { - return new AutoValue_BazelModuleResolutionValue(resolvedDepGraph, unprunedDepGraph); + ImmutableMap unprunedDepGraph, + ImmutableMap> registryFileHashes) { + return new AutoValue_BazelModuleResolutionValue( + resolvedDepGraph, unprunedDepGraph, registryFileHashes); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java index a69e53791f247f..d425bc4518042d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java @@ -17,6 +17,7 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.ryanharter.auto.value.gson.GenerateTypeAdapter; /** Stores the values of flags and environment variables that affect the resolution */ @@ -25,7 +26,7 @@ abstract class BzlmodFlagsAndEnvVars { public static BzlmodFlagsAndEnvVars create( - ImmutableList registries, + ImmutableSet registries, ImmutableMap moduleOverrides, ImmutableList yankedVersions, String envVarYankedVersions, @@ -43,7 +44,7 @@ public static BzlmodFlagsAndEnvVars create( } /** Registries provided via command line */ - public abstract ImmutableList cmdRegistries(); + public abstract ImmutableSet cmdRegistries(); /** ModulesOverride provided via command line */ public abstract ImmutableMap cmdModuleOverrides(); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Discovery.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Discovery.java index 5da2540e923f1a..a3e970db148a8d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Discovery.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Discovery.java @@ -17,9 +17,11 @@ import static java.util.stream.Collectors.joining; + import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.bazel.bzlmod.InterimModule.DepSpec; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.server.FailureDetails; import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyKey; @@ -28,10 +30,13 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Queue; +import java.util.SequencedMap; import java.util.Set; import javax.annotation.Nullable; @@ -43,13 +48,16 @@ final class Discovery { private Discovery() {} + public record Result( + ImmutableMap depGraph, + ImmutableMap> registryFileHashes) {} + /** * Runs module discovery. This function follows SkyFunction semantics (returns null if a Skyframe * dependency is missing and this function needs a restart). */ @Nullable - public static ImmutableMap run( - Environment env, RootModuleFileValue root) + public static Result run(Environment env, RootModuleFileValue root) throws InterruptedException, ExternalDepsException { String rootModuleName = root.getModule().getName(); ImmutableMap overrides = root.getOverrides(); @@ -60,9 +68,11 @@ public static ImmutableMap run( .withDepSpecsTransformed(InterimModule.applyOverrides(overrides, rootModuleName))); Queue unexpanded = new ArrayDeque<>(); Map predecessors = new HashMap<>(); + SequencedMap> registryFileHashes = + new LinkedHashMap<>(root.getRegistryFileHashes()); unexpanded.add(ModuleKey.ROOT); while (!unexpanded.isEmpty()) { - Set unexpandedSkyKeys = new HashSet<>(); + Set unexpandedSkyKeys = new LinkedHashSet<>(); while (!unexpanded.isEmpty()) { InterimModule module = depGraph.get(unexpanded.remove()); for (DepSpec depSpec : module.getDeps().values()) { @@ -109,6 +119,7 @@ public static ImmutableMap run( .getModule() .withDepSpecsTransformed( InterimModule.applyOverrides(overrides, rootModuleName))); + registryFileHashes.putAll(moduleFileValue.getRegistryFileHashes()); unexpanded.add(depKey); } } @@ -116,6 +127,6 @@ public static ImmutableMap run( if (env.valuesMissing()) { return null; } - return ImmutableMap.copyOf(depGraph); + return new Result(ImmutableMap.copyOf(depGraph), ImmutableMap.copyOf(registryFileHashes)); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java index a90bfb91619eee..9cb0e2331cea06 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java @@ -26,6 +26,8 @@ import com.google.common.collect.ImmutableTable; import com.google.common.collect.Table; import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; +import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.RepositoryName; @@ -475,10 +477,61 @@ public RepoRecordedInput.Dirents read(JsonReader jsonReader) throws IOException } }; + // This can't reuse the existing type adapter factory for Optional as we need to explicitly + // serialize null values but don't want to rely on GSON's serializeNulls. + private static final class OptionalChecksumTypeAdapterFactory implements TypeAdapterFactory { + + @Override + public TypeAdapter create(Gson gson, TypeToken typeToken) { + if (typeToken.getRawType() != Optional.class) { + return null; + } + Type type = typeToken.getType(); + if (!(type instanceof ParameterizedType)) { + return null; + } + Type elementType = ((ParameterizedType) type).getActualTypeArguments()[0]; + if (elementType != Checksum.class) { + return null; + } + @SuppressWarnings("unchecked") + TypeAdapter typeAdapter = (TypeAdapter) new OptionalChecksumTypeAdapter(); + return typeAdapter; + } + + private static class OptionalChecksumTypeAdapter extends TypeAdapter> { + // This value must not be a valid checksum string. + private static final String NOT_FOUND_MARKER = "not found"; + + @Override + public void write(JsonWriter jsonWriter, Optional checksum) throws IOException { + if (checksum.isPresent()) { + jsonWriter.value(checksum.get().toString()); + } else { + jsonWriter.value(NOT_FOUND_MARKER); + } + } + + @Override + public Optional read(JsonReader jsonReader) throws IOException { + String checksumString = jsonReader.nextString(); + if (NOT_FOUND_MARKER.equals(checksumString)) { + return Optional.empty(); + } + try { + return Optional.of(Checksum.fromString(RepositoryCache.KeyType.SHA256, checksumString)); + } catch (Checksum.InvalidChecksumException e) { + throw new JsonParseException(String.format("Invalid checksum: %s", checksumString), e); + } + } + } + } + public static Gson createLockFileGson(Path moduleFilePath, Path workspaceRoot) { return newGsonBuilder() .setPrettyPrinting() .registerTypeAdapterFactory(new LocationTypeAdapterFactory(moduleFilePath, workspaceRoot)) + .registerTypeAdapterFactory(new OptionalChecksumTypeAdapterFactory()) .create(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index e8725240937f6a..578e78a69d9d1d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -21,8 +21,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.events.StoredEventHandler; import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.profiler.SilentCloseable; @@ -49,6 +51,15 @@ */ public class IndexRegistry implements Registry { + /** + * How to handle the list of file hashes known from the lockfile when downloading files from the + * registry. + */ + public enum KnownFileHashesMode { + IGNORE, + USE_AND_UPDATE; + } + /** The unresolved version of the url. Ex: has %workspace% placeholder */ private final String unresolvedUri; @@ -56,7 +67,10 @@ public class IndexRegistry implements Registry { private final DownloadManager downloadManager; private final Map clientEnv; private final Gson gson; + private final ImmutableMap> knownFileHashes; + private final KnownFileHashesMode knownFileHashesMode; private volatile Optional bazelRegistryJson; + private volatile StoredEventHandler bazelRegistryJsonEvents; private static final String SOURCE_JSON_FILENAME = "source.json"; @@ -64,7 +78,9 @@ public IndexRegistry( URI uri, String unresolvedUri, DownloadManager downloadManager, - Map clientEnv) { + Map clientEnv, + ImmutableMap> knownFileHashes, + KnownFileHashesMode knownFileHashesMode) { this.uri = uri; this.unresolvedUri = unresolvedUri; this.downloadManager = downloadManager; @@ -73,6 +89,8 @@ public IndexRegistry( new GsonBuilder() .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES) .create(); + this.knownFileHashes = knownFileHashes; + this.knownFileHashesMode = knownFileHashesMode; } @Override @@ -92,14 +110,46 @@ private String constructUrl(String base, String... segments) { } /** Grabs a file from the given URL. Returns {@link Optional#empty} if the file doesn't exist. */ - private Optional grabFile(String url, ExtendedEventHandler eventHandler) + private Optional grabFile( + String url, ExtendedEventHandler eventHandler, boolean useChecksum) throws IOException, InterruptedException { + var maybeContent = doGrabFile(url, eventHandler, useChecksum); + if (knownFileHashesMode == KnownFileHashesMode.USE_AND_UPDATE && useChecksum) { + eventHandler.post(RegistryFileDownloadEvent.create(url, maybeContent)); + } + return maybeContent; + } + + private Optional doGrabFile( + String url, ExtendedEventHandler eventHandler, boolean useChecksum) + throws IOException, InterruptedException { + Optional checksum; + if (knownFileHashesMode != KnownFileHashesMode.IGNORE && useChecksum) { + Optional knownChecksum = knownFileHashes.get(url); + if (knownChecksum == null) { + // This is a new file, download without providing a checksum. + checksum = Optional.empty(); + } else if (knownChecksum.isEmpty()) { + // The file is known to not exist, so don't attempt to download it. + return Optional.empty(); + } else { + // The file is known, download with a checksum to potentially obtain a repository cache hit + // and ensure that the remote file hasn't changed. + checksum = knownChecksum; + } + } else { + checksum = Optional.empty(); + } try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + url)) { return Optional.of( - downloadManager.downloadAndReadOneUrl(new URL(url), eventHandler, clientEnv)); + downloadManager.downloadAndReadOneUrl(new URL(url), eventHandler, clientEnv, checksum)); } catch (FileNotFoundException e) { return Optional.empty(); + } catch (IOException e) { + // Include the URL in the exception message for easier debugging. + throw new IOException( + "Failed to fetch registry file %s: %s".formatted(url, e.getMessage()), e); } } @@ -109,7 +159,8 @@ public Optional getModuleFile(ModuleKey key, ExtendedEventHandler ev String url = constructUrl( uri.toString(), "modules", key.getName(), key.getVersion().toString(), "MODULE.bazel"); - return grabFile(url, eventHandler).map(content -> ModuleFile.create(content, url)); + Optional maybeContent = grabFile(url, eventHandler, /* useChecksum= */ true); + return maybeContent.map(content -> ModuleFile.create(content, url)); } /** Represents fields available in {@code bazel_registry.json} for the registry. */ @@ -153,22 +204,20 @@ private static class GitRepoSourceJson { * Grabs a JSON file from the given URL, and returns its content. Returns {@link Optional#empty} * if the file doesn't exist. */ - private Optional grabJsonFile(String url, ExtendedEventHandler eventHandler) + private Optional grabJsonFile( + String url, ExtendedEventHandler eventHandler, boolean useChecksum) throws IOException, InterruptedException { - Optional bytes = grabFile(url, eventHandler); - if (bytes.isEmpty()) { - return Optional.empty(); - } - return Optional.of(new String(bytes.get(), UTF_8)); + return grabFile(url, eventHandler, useChecksum).map(value -> new String(value, UTF_8)); } /** * Grabs a JSON file from the given URL, and returns it as a parsed object with fields in {@code * T}. Returns {@link Optional#empty} if the file doesn't exist. */ - private Optional grabJson(String url, Class klass, ExtendedEventHandler eventHandler) + private Optional grabJson( + String url, Class klass, ExtendedEventHandler eventHandler, boolean useChecksum) throws IOException, InterruptedException { - Optional jsonString = grabJsonFile(url, eventHandler); + Optional jsonString = grabJsonFile(url, eventHandler, useChecksum); if (jsonString.isEmpty() || jsonString.get().isBlank()) { return Optional.empty(); } @@ -195,7 +244,7 @@ public RepoSpec getRepoSpec(ModuleKey key, ExtendedEventHandler eventHandler) key.getName(), key.getVersion().toString(), SOURCE_JSON_FILENAME); - Optional jsonString = grabJsonFile(jsonUrl, eventHandler); + Optional jsonString = grabJsonFile(jsonUrl, eventHandler, /* useChecksum= */ true); if (jsonString.isEmpty()) { throw new FileNotFoundException( String.format("Module %s's %s not found in registry %s", key, SOURCE_JSON_FILENAME, uri)); @@ -232,14 +281,18 @@ private Optional getBazelRegistryJson(ExtendedEventHandler ev if (bazelRegistryJson == null) { synchronized (this) { if (bazelRegistryJson == null) { + var storedEventHandler = new StoredEventHandler(); bazelRegistryJson = grabJson( constructUrl(uri.toString(), "bazel_registry.json"), BazelRegistryJson.class, - eventHandler); + storedEventHandler, + /* useChecksum= */ true); + bazelRegistryJsonEvents = storedEventHandler; } } } + bazelRegistryJsonEvents.replayOn(eventHandler); return bazelRegistryJson; } @@ -348,7 +401,9 @@ public Optional> getYankedVersions( grabJson( constructUrl(uri.toString(), "modules", moduleName, "metadata.json"), MetadataJson.class, - eventHandler); + eventHandler, + // metadata.json is not immutable + /* useChecksum= */ false); if (metadataJson.isEmpty()) { return Optional.empty(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java index 542a62e92f84df..65f8d9946f49bb 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.bazel.bzlmod.CompiledModuleFile.IncludeStatement; @@ -34,6 +35,7 @@ import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.events.StoredEventHandler; import com.google.devtools.build.lib.packages.BazelStarlarkEnvironment; import com.google.devtools.build.lib.packages.StarlarkExportable; import com.google.devtools.build.lib.profiler.Profiler; @@ -80,7 +82,8 @@ */ public class ModuleFileFunction implements SkyFunction { - public static final Precomputed> REGISTRIES = new Precomputed<>("registries"); + public static final Precomputed> REGISTRIES = + new Precomputed<>("registries"); public static final Precomputed IGNORE_DEV_DEPS = new Precomputed<>("ignore_dev_dependency"); @@ -157,6 +160,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (getModuleFileResult == null) { return null; } + getModuleFileResult.downloadEventHandler.replayOn(env.getListener()); String moduleFileHash = new Fingerprint().addBytes(getModuleFileResult.moduleFile.getContent()).hexDigestAndReset(); @@ -217,8 +221,11 @@ public SkyValue compute(SkyKey skyKey, Environment env) module.getVersion()); } - - return NonRootModuleFileValue.create(module, moduleFileHash); + return NonRootModuleFileValue.create( + module, + moduleFileHash, + RegistryFileDownloadEvent.collectToMap( + getModuleFileResult.downloadEventHandler.getPosts())); } @Nullable @@ -529,7 +536,10 @@ private static ModuleThreadContext execModuleFile( * * @param registry can be null if this module has a non-registry override. */ - private record GetModuleFileResult(ModuleFile moduleFile, @Nullable Registry registry) {} + private record GetModuleFileResult( + ModuleFile moduleFile, + @Nullable Registry registry, + StoredEventHandler downloadEventHandler) {} @Nullable private GetModuleFileResult getModuleFile( @@ -560,7 +570,8 @@ private GetModuleFileResult getModuleFile( ModuleFile.create( readModuleFile(moduleFilePath.asPath()), moduleFileLabel.getUnambiguousCanonicalForm()), - /* registry= */ null); + /* registry= */ null, + new StoredEventHandler()); } // Otherwise, we should get the module file from a registry. @@ -573,13 +584,11 @@ private GetModuleFileResult getModuleFile( + " non-registry override?", key.getName()); } - // TODO(wyv): Move registry object creation to BazelRepositoryModule so we don't repeatedly - // create them, and we can better report the error (is it a flag error or override error?). - List registries = Objects.requireNonNull(REGISTRIES.get(env)); + ImmutableSet registries = Objects.requireNonNull(REGISTRIES.get(env)); if (override instanceof RegistryOverride registryOverride) { String overrideRegistry = registryOverride.getRegistry(); if (!overrideRegistry.isEmpty()) { - registries = ImmutableList.of(overrideRegistry); + registries = ImmutableSet.of(overrideRegistry); } } else if (override != null) { // This should never happen. @@ -607,13 +616,14 @@ private GetModuleFileResult getModuleFile( // Now go through the list of registries and use the first one that contains the requested // module. + StoredEventHandler downloadEventHandler = new StoredEventHandler(); for (Registry registry : registryObjects) { try { - Optional moduleFile = registry.getModuleFile(key, env.getListener()); + Optional moduleFile = registry.getModuleFile(key, downloadEventHandler); if (moduleFile.isEmpty()) { continue; } - return new GetModuleFileResult(moduleFile.get(), registry); + return new GetModuleFileResult(moduleFile.get(), registry, downloadEventHandler); } catch (IOException e) { throw errorf( Code.ERROR_ACCESSING_REGISTRY, e, "Error accessing registry %s", registry.getUrl()); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java index 7c7e38fd6c3db6..da64efb9ff5e97 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java @@ -18,12 +18,14 @@ import com.google.auto.value.AutoValue; import com.google.auto.value.extension.memoized.Memoized; import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; +import java.util.Optional; import javax.annotation.Nullable; /** The result of {@link ModuleFileFunction}. */ @@ -41,12 +43,22 @@ public abstract class ModuleFileValue implements SkyValue { /** The hash string of Module.bazel (using SHA256) */ public abstract String getModuleFileHash(); + /** + * Hashes of files obtained (or known to be missing) from registries while obtaining this module + * file. + */ + public abstract ImmutableMap> getRegistryFileHashes(); + /** The {@link ModuleFileValue} for non-root modules. */ @AutoValue public abstract static class NonRootModuleFileValue extends ModuleFileValue { - public static NonRootModuleFileValue create(InterimModule module, String moduleFileHash) { - return new AutoValue_ModuleFileValue_NonRootModuleFileValue(module, moduleFileHash); + public static NonRootModuleFileValue create( + InterimModule module, + String moduleFileHash, + ImmutableMap> registryFileHashes) { + return new AutoValue_ModuleFileValue_NonRootModuleFileValue( + module, moduleFileHash, registryFileHashes); } } @@ -77,6 +89,12 @@ public abstract static class RootModuleFileValue extends ModuleFileValue { */ public abstract ImmutableMap getIncludeLabelToCompiledModuleFile(); + @Override + public ImmutableMap> getRegistryFileHashes() { + // The root module is not obtained from a registry. + return ImmutableMap.of(); + } + public static RootModuleFileValue create( InterimModule module, String moduleFileHash, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java index 03ba85dc2dd4fc..e24b3452fe8e07 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java @@ -15,7 +15,10 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import java.net.URISyntaxException; +import java.util.Optional; /** A factory type for {@link Registry}. */ public interface RegistryFactory { @@ -25,5 +28,6 @@ public interface RegistryFactory { * *

Outside of tests, only {@link RegistryFunction} should call this method. */ - Registry createRegistry(String url) throws URISyntaxException; + Registry createRegistry(String url, ImmutableMap> fileHashes) + throws URISyntaxException; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java index d943e69ef4d44f..06efd4050cbd0d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java @@ -15,11 +15,15 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.bzlmod.IndexRegistry.KnownFileHashesMode; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.vfs.Path; import java.net.URI; import java.net.URISyntaxException; import java.util.Map; +import java.util.Optional; import java.util.function.Supplier; /** Prod implementation of {@link RegistryFactory}. */ @@ -38,7 +42,9 @@ public RegistryFactoryImpl( } @Override - public Registry createRegistry(String unresolvedUrl) throws URISyntaxException { + public Registry createRegistry( + String unresolvedUrl, ImmutableMap> knownFileHashes) + throws URISyntaxException { URI uri = new URI(unresolvedUrl.replace("%workspace%", workspacePath.getPathString())); if (uri.getScheme() == null) { throw new URISyntaxException( @@ -52,10 +58,19 @@ public Registry createRegistry(String unresolvedUrl) throws URISyntaxException { "Registry URL path is not valid -- did you mean to use file:///foo/bar " + "or file:///c:/foo/bar for Windows?"); } - return switch (uri.getScheme()) { - case "http", "https", "file" -> - new IndexRegistry(uri, unresolvedUrl, downloadManager, clientEnvironmentSupplier.get()); - default -> throw new URISyntaxException(uri.toString(), "Unrecognized registry URL protocol"); - }; + var knownFileHashesMode = + switch (uri.getScheme()) { + case "http", "https" -> KnownFileHashesMode.USE_AND_UPDATE; + case "file" -> KnownFileHashesMode.IGNORE; + default -> + throw new URISyntaxException(uri.toString(), "Unrecognized registry URL protocol"); + }; + return new IndexRegistry( + uri, + unresolvedUrl, + downloadManager, + clientEnvironmentSupplier.get(), + knownFileHashes, + knownFileHashesMode); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFileDownloadEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFileDownloadEvent.java new file mode 100644 index 00000000000000..e7482604277b15 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFileDownloadEvent.java @@ -0,0 +1,39 @@ +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.common.collect.ImmutableMap; +import com.google.common.hash.Hashing; +import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; +import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; +import java.util.Collection; +import java.util.Optional; + +/** Event that records the fact that a file has been downloaded from a remote registry. */ +public record RegistryFileDownloadEvent(String uri, Optional checksum) + implements Postable { + + public static RegistryFileDownloadEvent create(String uri, Optional content) { + return new RegistryFileDownloadEvent(uri, content.map(RegistryFileDownloadEvent::computeHash)); + } + + static ImmutableMap> collectToMap( + Collection postables) { + ImmutableMap.Builder> builder = ImmutableMap.builder(); + for (Postable postable : postables) { + if (postable instanceof RegistryFileDownloadEvent event) { + builder.put(event.uri(), event.checksum()); + } + } + return builder.buildKeepingLast(); + } + + private static Checksum computeHash(byte[] bytes) { + try { + return Checksum.fromString( + RepositoryCache.KeyType.SHA256, Hashing.sha256().hashBytes(bytes).toString()); + } catch (Checksum.InvalidChecksumException e) { + // This can't happen since HashCode.toString() always returns a valid hash. + throw new IllegalStateException(e); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java index e99fc2bbcbbbf5..0c6c12a7ac84bc 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java @@ -35,9 +35,14 @@ public RegistryFunction(RegistryFactory registryFactory) { @Nullable public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException, RegistryException { + BazelLockFileValue lockfile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); + if (lockfile == null) { + return null; + } + RegistryKey key = (RegistryKey) skyKey.argument(); try { - return registryFactory.createRegistry(key.getUrl()); + return registryFactory.createRegistry(key.getUrl(), lockfile.getRegistryFileHashes()); } catch (URISyntaxException e) { throw new RegistryException( ExternalDepsException.withCauseAndMessage( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java index 201312bc0e4a4e..d6ad0894c63f02 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java @@ -16,7 +16,6 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.Maps; -import com.google.devtools.build.skyframe.SkyValue; import com.ryanharter.auto.value.gson.GenerateTypeAdapter; import javax.annotation.Nullable; @@ -26,7 +25,7 @@ */ @AutoValue @GenerateTypeAdapter -public abstract class RepoSpec implements SkyValue { +public abstract class RepoSpec { /** * The unambiguous canonical label string for the bzl file this repository rule is defined in, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecFunction.java index b9f436999f629b..49ef25b0e6c91f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecFunction.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import com.google.devtools.build.lib.events.StoredEventHandler; import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.profiler.SilentCloseable; @@ -43,10 +44,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) return null; } + StoredEventHandler downloadEvents = new StoredEventHandler(); + RepoSpec repoSpec; try (SilentCloseable c = Profiler.instance() .profile(ProfilerTask.BZLMOD, () -> "compute repo spec: " + key.getModuleKey())) { - return registry.getRepoSpec(key.getModuleKey(), env.getListener()); + repoSpec = registry.getRepoSpec(key.getModuleKey(), downloadEvents); } catch (IOException e) { throw new RepoSpecException( ExternalDepsException.withCauseAndMessage( @@ -55,6 +58,9 @@ public SkyValue compute(SkyKey skyKey, Environment env) "Unable to get module repo spec for %s from registry", key.getModuleKey())); } + downloadEvents.replayOn(env.getListener()); + return new RepoSpecValue( + repoSpec, RegistryFileDownloadEvent.collectToMap(downloadEvents.getPosts())); } static final class RepoSpecException extends SkyFunctionException { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecValue.java new file mode 100644 index 00000000000000..1dd868fe2d4af0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecValue.java @@ -0,0 +1,25 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; +import com.google.devtools.build.skyframe.SkyValue; +import java.util.Optional; + +/** The value for {@link RepoSpecFunction}. */ +record RepoSpecValue(RepoSpec repoSpec, ImmutableMap> registryFileHashes) + implements SkyValue {} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionValue.java index b0a91017fd1d4b..2182ef0e2165eb 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionValue.java @@ -46,8 +46,8 @@ public abstract class SingleExtensionValue implements SkyValue { public abstract ImmutableBiMap getCanonicalRepoNameToInternalNames(); /** - * Returns the information stored about the extension in the lockfile. Is empty if the lockfile - * mode is not UPDATE. + * Returns the information stored about the extension in the lockfile. Non-empty if and only if + * the lockfile mode is UPDATE. */ public abstract Optional getLockFileInfo(); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java index 402f96da8b657a..81ef6f1d6ce1dc 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java @@ -162,6 +162,43 @@ boolean hasCanonicalId(String cacheKey, KeyType keyType, String canonicalId) { @Nullable public Path get(String cacheKey, Path targetPath, KeyType keyType, String canonicalId) throws IOException, InterruptedException { + Path cacheValue = findCacheValue(cacheKey, keyType, canonicalId); + if (cacheValue == null) { + return null; + } + + targetPath.getParentDirectory().createDirectoryAndParents(); + if (useHardlinks) { + FileSystemUtils.createHardLink(targetPath, cacheValue); + } else { + FileSystemUtils.copyFile(cacheValue, targetPath); + } + + return targetPath; + } + + /** + * Get the content of a cached value, if it exists. + * + * @param cacheKey The string key to cache the value by. + * @param keyType The type of key used. See: KeyType + * @return The bytes of the cache value. If cache value does not exist, returns null. + * @throws IOException + */ + @Nullable + public byte[] getBytes(String cacheKey, KeyType keyType) + throws IOException, InterruptedException { + Path cacheValue = findCacheValue(cacheKey, keyType, /* canonicalId= */ null); + if (cacheValue == null) { + return null; + } + + return FileSystemUtils.readContent(cacheValue); + } + + @Nullable + private Path findCacheValue(String cacheKey, KeyType keyType, String canonicalId) + throws IOException, InterruptedException { Preconditions.checkState(isEnabled()); assertKeyIsValid(cacheKey, keyType); @@ -186,20 +223,13 @@ public Path get(String cacheKey, Path targetPath, KeyType keyType, String canoni } } - targetPath.getParentDirectory().createDirectoryAndParents(); - if (useHardlinks) { - FileSystemUtils.createHardLink(targetPath, cacheValue); - } else { - FileSystemUtils.copyFile(cacheValue, targetPath); - } - try { FileSystemUtils.touchFile(cacheValue); } catch (IOException e) { // Ignore, because the cache might be on a read-only volume. } - return targetPath; + return cacheValue; } /** @@ -214,6 +244,54 @@ public Path get(String cacheKey, Path targetPath, KeyType keyType, String canoni */ public void put(String cacheKey, Path sourcePath, KeyType keyType, String canonicalId) throws IOException { + storeCacheValue( + cacheKey, tmpName -> FileSystemUtils.copyFile(sourcePath, tmpName), keyType, canonicalId); + } + + /** + * Adds an in-memory value to the cache. + * + * @param content The byte content of the value to be cached. + * @param keyType The type of key used. See: KeyType + * @throws IOException + */ + public void put(String cacheKey, byte[] content, KeyType keyType) throws IOException { + storeCacheValue( + cacheKey, + tmpName -> FileSystemUtils.writeContent(tmpName, content), + keyType, + /* canonicalId= */ null); + } + + /** + * Adds an in-memory value to the cache. + * + * @param content The byte content of the value to be cached. + * @param keyType The type of key used. See: KeyType + * @throws IOException + */ + public void put(byte[] content, KeyType keyType) throws IOException { + String cacheKey = keyType.newHasher().putBytes(content).hash().toString(); + put(cacheKey, content, keyType); + } + + interface FileWriter { + void writeTo(Path name) throws IOException; + } + + /** + * Copies a value from a specified path into the cache. + * + * @param cacheKey The string key to cache the value by. + * @param fileWriter A function that writes the value to a given file. + * @param keyType The type of key used. See: KeyType + * @param canonicalId If set to a non-empty String associate the file with this name, allowing + * restricted cache lookups later. + * @throws IOException + */ + private void storeCacheValue( + String cacheKey, FileWriter fileWriter, KeyType keyType, String canonicalId) + throws IOException { Preconditions.checkState(isEnabled()); assertKeyIsValid(cacheKey, keyType); @@ -223,7 +301,7 @@ public void put(String cacheKey, Path sourcePath, KeyType keyType, String canoni Path cacheValue = cacheEntry.getRelative(DEFAULT_CACHE_FILENAME); Path tmpName = cacheEntry.getRelative(TMP_PREFIX + UUID.randomUUID()); cacheEntry.createDirectoryAndParents(); - FileSystemUtils.copyFile(sourcePath, tmpName); + fileWriter.writeTo(tmpName); try { tmpName.renameTo(cacheValue); } catch (FileAccessException e) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java index a56ce634c24307..9737437cec0cf9 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java @@ -382,16 +382,37 @@ private Path downloadInExecutor( * @param originalUrl the original URL of the file * @param eventHandler CLI progress reporter * @param clientEnv environment variables in shell issuing this command + * @param checksum checksum of the file used to verify the content and obtain repository cache + * hits * @throws IllegalArgumentException on parameter badness, which should be checked beforehand * @throws IOException if download was attempted and ended up failing * @throws InterruptedException if this thread is being cast into oblivion */ public byte[] downloadAndReadOneUrl( - URL originalUrl, ExtendedEventHandler eventHandler, Map clientEnv) + URL originalUrl, + ExtendedEventHandler eventHandler, + Map clientEnv, + Optional checksum) throws IOException, InterruptedException { if (Thread.interrupted()) { throw new InterruptedException(); } + + if (repositoryCache.isEnabled() && checksum.isPresent()) { + String cacheKey = checksum.get().toString(); + try { + byte[] content = repositoryCache.getBytes(cacheKey, checksum.get().getKeyType()); + if (content != null) { + // Cache hit! + eventHandler.post( + new RepositoryCacheHitEvent("Bazel module fetching", cacheKey, originalUrl)); + return content; + } + } catch (IOException e) { + // Ignore error trying to get. We'll just download again. + } + } + Map>> authHeaders = ImmutableMap.of(); ImmutableList rewrittenUrls = ImmutableList.of(originalUrl); @@ -418,15 +439,27 @@ public byte[] downloadAndReadOneUrl( authHeaders = rewriter.updateAuthHeaders(rewrittenUrlMappings, authHeaders, netrcCreds); } + if (disableDownload) { + throw new IOException( + String.format("Failed to download %s: download is disabled.", originalUrl)); + } + if (rewrittenUrls.isEmpty()) { throw new IOException(getRewriterBlockedAllUrlsMessage(ImmutableList.of(originalUrl))); } HttpDownloader httpDownloader = new HttpDownloader(); + byte[] content = null; for (int attempt = 0; attempt <= retries; ++attempt) { try { - return httpDownloader.downloadAndReadOneUrl( - rewrittenUrls.get(0), credentialFactory.create(authHeaders), eventHandler, clientEnv); + content = + httpDownloader.downloadAndReadOneUrl( + rewrittenUrls.get(0), + credentialFactory.create(authHeaders), + checksum, + eventHandler, + clientEnv); + break; } catch (ContentLengthMismatchException e) { if (attempt == retries) { throw e; @@ -435,8 +468,18 @@ public byte[] downloadAndReadOneUrl( throw new InterruptedException(e.getMessage()); } } + if (content == null) { + throw new IllegalStateException("Unexpected error: file should have been downloaded."); + } - throw new IllegalStateException("Unexpected error: file should have been downloaded."); + if (repositoryCache.isEnabled()) { + if (checksum.isPresent()) { + repositoryCache.put(checksum.get().toString(), content, checksum.get().getKeyType()); + } else { + repositoryCache.put(content, KeyType.SHA256); + } + } + return content; } @Nullable diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java index 35e0ea2ebb8577..796ac03f631a54 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java @@ -147,6 +147,7 @@ public void download( public byte[] downloadAndReadOneUrl( URL url, Credentials credentials, + Optional checksum, ExtendedEventHandler eventHandler, Map clientEnv) throws IOException, InterruptedException { @@ -155,8 +156,7 @@ public byte[] downloadAndReadOneUrl( ByteArrayOutputStream out = new ByteArrayOutputStream(); SEMAPHORE.acquire(); try (HttpStream payload = - multiplexer.connect( - url, Optional.empty(), ImmutableMap.of(), credentials, Optional.empty())) { + multiplexer.connect(url, checksum, ImmutableMap.of(), credentials, Optional.empty())) { ByteStreams.copy(payload, out); } catch (SocketTimeoutException e) { // SocketTimeoutExceptions are InterruptedIOExceptions; however they do not signify diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java index ed1ccb52f70453..344048abe5413c 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java @@ -185,6 +185,9 @@ public BazelPackageLoader buildImpl() { ImmutableMap::of, directories, EXTERNAL_PACKAGE_HELPER)) + .put( + SkyFunctions.BAZEL_LOCK_FILE, + new BazelLockFileFunction(directories.getWorkspace())) .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction()) diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java index 526e2c4803bcfb..56c441fd966016 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java @@ -15,6 +15,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; @@ -208,7 +209,7 @@ public ImmutableList getPrecomputedValues() { RepositoryDelegatorFunction.FORCE_FETCH_DISABLED), PrecomputedValue.injected(RepositoryDelegatorFunction.VENDOR_DIRECTORY, Optional.empty()), PrecomputedValue.injected(RepositoryDelegatorFunction.DISABLE_NATIVE_REPO_RULES, false), - PrecomputedValue.injected(ModuleFileFunction.REGISTRIES, ImmutableList.of()), + PrecomputedValue.injected(ModuleFileFunction.REGISTRIES, ImmutableSet.of()), PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), PrecomputedValue.injected(ModuleFileFunction.MODULE_OVERRIDES, ImmutableMap.of()), PrecomputedValue.injected(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java index 67f03490530089..42c3eccf4f0daf 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java @@ -239,7 +239,7 @@ protected void useRuleClassProvider(ConfiguredRuleClassProvider ruleClassProvide .setExtraPrecomputeValues( ImmutableList.of( PrecomputedValue.injected( - ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())), + ModuleFileFunction.REGISTRIES, ImmutableSet.of(registry.getUrl())), PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), PrecomputedValue.injected( RepositoryDelegatorFunction.DISABLE_NATIVE_REPO_RULES, false), @@ -293,7 +293,7 @@ private void reinitializeSkyframeExecutor() { PrecomputedValue.injected( RepositoryDelegatorFunction.VENDOR_DIRECTORY, Optional.empty()), PrecomputedValue.injected( - ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())), + ModuleFileFunction.REGISTRIES, ImmutableSet.of(registry.getUrl())), PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), PrecomputedValue.injected(RepositoryDelegatorFunction.DISABLE_NATIVE_REPO_RULES, false), PrecomputedValue.injected( diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java index eef29a05e0c6e7..89b27e16855c28 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java @@ -308,7 +308,7 @@ public void initializeSkyframeExecutor( .addAll(analysisMock.getPrecomputedValues()) .add( PrecomputedValue.injected( - ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl()))) + ModuleFileFunction.REGISTRIES, ImmutableSet.of(registry.getUrl()))) .addAll(extraPrecomputedValues()) .build(); PackageFactory.BuilderForTesting pkgFactoryBuilder = diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index a5fb0a9f64aeb1..bd24f466621a39 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -116,11 +116,14 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:registry", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution_impl", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/packages", "//src/main/java/net/starlark/java/eval", "//src/main/java/net/starlark/java/syntax", + "//third_party:gson", "//third_party:guava", ], ) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java index 4c707b943f4e4e..ab49053e01d916 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java @@ -61,7 +61,6 @@ import com.google.devtools.build.skyframe.RecordingDifferencer; import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; import com.google.devtools.build.skyframe.SkyFunction; -import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; @@ -147,7 +146,7 @@ public void setup() throws Exception { differencer, StarlarkSemantics.builder().setBool(BuildLanguageOptions.ENABLE_BZLMOD, true).build()); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of()); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( differencer, CheckDirectDepsMode.OFF); @@ -364,10 +363,8 @@ public void setDepGraph(ImmutableMap depGraph) { @Override @Nullable - public SkyValue compute(SkyKey skyKey, Environment env) - throws SkyFunctionException, InterruptedException { - - return BazelModuleResolutionValue.create(depGraph, ImmutableMap.of()); + public SkyValue compute(SkyKey skyKey, Environment env) { + return BazelModuleResolutionValue.create(depGraph, ImmutableMap.of(), ImmutableMap.of()); } } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java index 0aaf96ef375b99..f4dc8815ef01cd 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; @@ -192,6 +193,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) .setFlags(flags) .setLocalOverrideHashes(localOverrideHashes) .setModuleDepGraph(key.depGraph()) + .setRegistryFileHashes(ImmutableMap.of()) .build()); return new SkyValue() {}; @@ -203,7 +205,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) PrecomputedValue.STARLARK_SEMANTICS.set( differencer, StarlarkSemantics.builder().setBool(BuildLanguageOptions.ENABLE_BZLMOD, true).build()); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of()); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); @@ -275,7 +277,7 @@ public void moduleWithFlags() throws Exception { ImmutableList yankedVersions = ImmutableList.of("2.4", "2.3"); LocalPathOverride override = LocalPathOverride.create("override_path"); - ImmutableList registries = ImmutableList.of("registry1", "registry2"); + ImmutableSet registries = ImmutableSet.of("registry1", "registry2"); ImmutableMap moduleOverride = ImmutableMap.of("my_dep_1", override.getPath()); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java index 218a36cb303bb8..e183354eb794be 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java @@ -21,6 +21,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.BlazeVersionInfo; @@ -344,7 +345,7 @@ private void setupModulesForCompatibility() throws IOException { .addModule( createModuleKey("b", "1.0"), "module(name='b', version='1.0', bazel_compatibility=['<=5.1.4', '-5.1.2']);"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); } @Test @@ -407,7 +408,7 @@ private void setupModulesForYankedVersion() throws Exception { "bazel_dep(name='b', version='1.0')") .addModule(createModuleKey("b", "1.0"), "module(name='b', version='1.0');") .addYankedVersion("b", ImmutableMap.of(Version.parse("1.0"), "1.0 is a bad version!")); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); } @Test @@ -437,7 +438,7 @@ public void overrideOnNonexistentModule() throws Exception { "module(name='b', version='1.1')", "bazel_dep(name='c', version='1.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext); @@ -483,7 +484,7 @@ public void testPrintBehavior() throws Exception { "bazel_dep(name='c', version='1.0')", "print('hello from b@1.1')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index e3c1569fef0675..8e9944f95ea59c 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; @@ -146,7 +147,7 @@ public void setup() throws Exception { differencer); PrecomputedValue.STARLARK_SEMANTICS.set(differencer, StarlarkSemantics.DEFAULT); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of()); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); @@ -170,7 +171,7 @@ public void testRepoSpec_bazelModule() throws Exception { createModuleKey("bbb", "1.0"), "module(name='bbb', version='1.0');bazel_dep(name='ccc',version='2.0')") .addModule(createModuleKey("ccc", "2.0"), "module(name='ccc', version='2.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); RepositoryName repo = RepositoryName.create("ccc~"); EvaluationResult result = @@ -201,7 +202,7 @@ public void testRepoSpec_nonRegistryOverride() throws Exception { createModuleKey("bbb", "1.0"), "module(name='bbb', version='1.0');bazel_dep(name='ccc',version='2.0')") .addModule(createModuleKey("ccc", "2.0"), "module(name='ccc', version='2.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); RepositoryName repo = RepositoryName.create("ccc~"); EvaluationResult result = @@ -234,7 +235,7 @@ public void testRepoSpec_singleVersionOverride() throws Exception { "module(name='bbb', version='1.0');bazel_dep(name='ccc',version='2.0')") .addModule(createModuleKey("ccc", "2.0"), "module(name='ccc', version='2.0')") .addModule(createModuleKey("ccc", "3.0"), "module(name='ccc', version='3.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); RepositoryName repo = RepositoryName.create("ccc~"); EvaluationResult result = @@ -270,7 +271,7 @@ public void testRepoSpec_multipleVersionOverride() throws Exception { "module(name='ccc', version='2.0');bazel_dep(name='ddd',version='2.0')") .addModule(createModuleKey("ddd", "1.0"), "module(name='ddd', version='1.0')") .addModule(createModuleKey("ddd", "2.0"), "module(name='ddd', version='2.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); RepositoryName repo = RepositoryName.create("ddd~2.0"); EvaluationResult result = diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java index 742b5d6ca72655..81e4685a404dbe 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java @@ -23,6 +23,8 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; @@ -30,6 +32,7 @@ import com.google.devtools.build.lib.analysis.util.AnalysisMock; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil.InterimModuleBuilder; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryModule; import com.google.devtools.build.lib.clock.BlazeClock; import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; @@ -91,11 +94,17 @@ abstract static class DiscoveryValue implements SkyValue { static final SkyFunctionName FUNCTION_NAME = SkyFunctionName.createHermetic("test_discovery"); static final SkyKey KEY = () -> FUNCTION_NAME; - static DiscoveryValue create(ImmutableMap depGraph) { - return new AutoValue_DiscoveryTest_DiscoveryValue(depGraph); + static DiscoveryValue create( + ImmutableMap depGraph, + ImmutableMap> registryFileHashes) { + return new AutoValue_DiscoveryTest_DiscoveryValue(depGraph, registryFileHashes); } abstract ImmutableMap getDepGraph(); + + // Uses Optional rather than Optional for easier testing (Checksum doesn't + // implement equals()). + abstract ImmutableMap> getRegistryFileHashes(); } static class DiscoveryFunction implements SkyFunction { @@ -107,14 +116,21 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (root == null) { return null; } - ImmutableMap depGraph; + Discovery.Result discoveryResult; try { - depGraph = Discovery.run(env, root); + discoveryResult = Discovery.run(env, root); } catch (ExternalDepsException e) { throw new BazelModuleResolutionFunction.BazelModuleResolutionFunctionException( e, SkyFunctionException.Transience.PERSISTENT); } - return depGraph == null ? null : DiscoveryValue.create(depGraph); + return discoveryResult == null + ? null + : DiscoveryValue.create( + discoveryResult.depGraph(), + ImmutableMap.copyOf( + Maps.transformValues( + discoveryResult.registryFileHashes(), + value -> value.map(Checksum::toString)))); } } @@ -168,6 +184,7 @@ private void setUpWithBuiltinModules(ImmutableMap b SyscallCache.NO_CACHE, externalFilesHelper)) .put(DiscoveryValue.FUNCTION_NAME, new DiscoveryFunction()) + .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put( SkyFunctions.MODULE_FILE, new ModuleFileFunction( @@ -236,7 +253,7 @@ public void testSimpleDiamond() throws Exception { createModuleKey("ddd", "3.0"), // Add a random override here; it should be ignored "module(name='ddd', version='3.0');local_path_override(module_name='ff',path='f')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext); @@ -260,6 +277,15 @@ public void testSimpleDiamond() throws Exception { .setRegistry(registry) .buildEntry(), InterimModuleBuilder.create("ddd", "3.0").setRegistry(registry).buildEntry()); + assertThat(discoveryValue.getRegistryFileHashes()) + .containsExactly( + registry.getUrl() + "/modules/bbb/1.0/MODULE.bazel", + Optional.of("3f48e6d8694e0aa0d16617fd97b7d84da0e17ee9932c18cbc71888c12563372d"), + registry.getUrl() + "/modules/ccc/2.0/MODULE.bazel", + Optional.of("e613d4192495192c3d46ee444dc9882a176a9e7a243d1b5a840ab0f01553e8d6"), + registry.getUrl() + "/modules/ddd/3.0/MODULE.bazel", + Optional.of("f80d91453520d193b0b79f1501eb902b5b01a991762cc7fb659fc580b95648fd")) + .inOrder(); } @Test @@ -278,7 +304,7 @@ public void testDevDependency() throws Exception { "bazel_dep(name='ccc',version='2.0',dev_dependency=True)") .addModule(createModuleKey("ccc", "1.0"), "module(name='ccc', version='1.0')") .addModule(createModuleKey("ccc", "2.0"), "module(name='ccc', version='2.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext); @@ -313,7 +339,7 @@ public void testIgnoreDevDependency() throws Exception { "bazel_dep(name='ccc',version='2.0',dev_dependency=True)") .addModule(createModuleKey("ccc", "1.0"), "module(name='ccc', version='1.0')") .addModule(createModuleKey("ccc", "2.0"), "module(name='ccc', version='2.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true); EvaluationResult result = @@ -346,7 +372,7 @@ public void testCircularDependency() throws Exception { .addModule( createModuleKey("ccc", "2.0"), "module(name='ccc', version='2.0');bazel_dep(name='bbb',version='1.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext); @@ -383,7 +409,7 @@ public void testCircularDependencyOnRootModule() throws Exception { createModuleKey("bbb", "1.0"), "module(name='bbb', version='1.0');bazel_dep(name='aaa',version='2.0')") .addModule(createModuleKey("aaa", "2.0"), "module(name='aaa', version='2.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext); @@ -419,7 +445,7 @@ public void testSingleVersionOverride() throws Exception { "module(name='bbb', version='0.1');bazel_dep(name='ccc',version='1.0')") .addModule(createModuleKey("ccc", "1.0"), "module(name='ccc', version='1.0');") .addModule(createModuleKey("ccc", "2.0"), "module(name='ccc', version='2.0');"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext); @@ -461,7 +487,7 @@ public void testRegistryOverride() throws Exception { "module(name='aaa',version='0.1')", "bazel_dep(name='bbb',version='0.1')", "single_version_override(module_name='ccc',registry='" + registry2.getUrl() + "')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry1.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry1.getUrl())); EvaluationResult result = evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext); @@ -503,7 +529,7 @@ public void testLocalPathOverride() throws Exception { createModuleKey("bbb", "0.1"), "module(name='bbb', version='0.1');bazel_dep(name='ccc',version='1.0')") .addModule(createModuleKey("ccc", "1.0"), "module(name='ccc', version='1.0');"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext); @@ -525,6 +551,11 @@ public void testLocalPathOverride() throws Exception { InterimModuleBuilder.create("ccc", "2.0") .setKey(createModuleKey("ccc", "")) .buildEntry()); + assertThat(discoveryValue.getRegistryFileHashes()) + .containsExactly( + registry.getUrl() + "/modules/bbb/0.1/MODULE.bazel", + Optional.of("3f9e1a600b4adeee1c1a92b92df9d086eca4bbdde656c122872f48f8f3b874a3")) + .inOrder(); } @Test @@ -553,7 +584,7 @@ public void testBuiltinModules_forRoot() throws Exception { .newFakeRegistry("/foo") .addModule(createModuleKey("foo", "1.0"), "module(name='foo', version='1.0')") .addModule(createModuleKey("foo", "2.0"), "module(name='foo', version='2.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate(ImmutableList.of(DiscoveryValue.KEY), evaluationContext); @@ -587,5 +618,13 @@ public void testBuiltinModules_forRoot() throws Exception { .addDep("local_config_platform", createModuleKey("local_config_platform", "")) .setRegistry(registry) .buildEntry()); + + assertThat(discoveryValue.getRegistryFileHashes()) + .containsExactly( + registry.getUrl() + "/modules/foo/2.0/MODULE.bazel", + Optional.of("76ecb05b455aecab4ec958c1deb17e4cbbe6e708d9c4e85fceda2317f6c86d7b"), + registry.getUrl() + "/modules/foo/1.0/MODULE.bazel", + Optional.of("4d887e8dfc1863861e3aa5601eeeebca5d8f110977895f1de4bdb2646e546fb5")) + .inOrder(); } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java index 10500fb331dfe8..6ab4818d4d2cef 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java @@ -20,6 +20,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.HashMap; @@ -63,26 +64,33 @@ public String getUrl() { @Override public Optional getModuleFile(ModuleKey key, ExtendedEventHandler eventHandler) { - return Optional.ofNullable(modules.get(key)) - .map(value -> value.getBytes(UTF_8)) - .map( - content -> - ModuleFile.create( - content, - String.format( - "%s/modules/%s/%s/MODULE.bazel", - url, key.getName(), key.getVersion().toString()))); + String uri = + String.format( + "%s/modules/%s/%s/MODULE.bazel", url, key.getName(), key.getVersion().toString()); + var maybeContent = Optional.ofNullable(modules.get(key)).map(value -> value.getBytes(UTF_8)); + eventHandler.post(RegistryFileDownloadEvent.create(uri, maybeContent)); + return maybeContent.map(content -> ModuleFile.create(content, uri)); } @Override public RepoSpec getRepoSpec(ModuleKey key, ExtendedEventHandler eventHandler) { - return RepoSpec.builder() - .setRuleClassName("local_repository") - .setAttributes( - AttributeValues.create( - ImmutableMap.of( - "path", rootPath + "/" + key.getCanonicalRepoNameWithVersion().getName()))) - .build(); + RepoSpec repoSpec = + RepoSpec.builder() + .setRuleClassName("local_repository") + .setAttributes( + AttributeValues.create( + ImmutableMap.of( + "path", rootPath + "/" + key.getCanonicalRepoNameWithVersion().getName()))) + .build(); + eventHandler.post( + RegistryFileDownloadEvent.create( + "%s/modules/%s/%s/source.json" + .formatted(url, key.getName(), key.getVersion().toString()), + Optional.of( + GsonTypeAdapterUtil.createSingleExtensionUsagesValueHashGson() + .toJson(repoSpec) + .getBytes(UTF_8)))); + return repoSpec; } @Override @@ -118,7 +126,8 @@ public FakeRegistry newFakeRegistry(String rootPath) { } @Override - public Registry createRegistry(String url) { + public Registry createRegistry( + String url, ImmutableMap> fileHashes) { return Preconditions.checkNotNull(registries.get(url), "unknown registry url: %s", url); } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java index 2401275b5c74b3..4e4012ed876d23 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java @@ -23,11 +23,15 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.common.eventbus.Subscribe; +import com.google.common.hash.Hashing; import com.google.devtools.build.lib.authandtls.BasicHttpAuthenticationEncoder; import com.google.devtools.build.lib.authandtls.Netrc; import com.google.devtools.build.lib.authandtls.NetrcCredentials; import com.google.devtools.build.lib.authandtls.NetrcParser; import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; import com.google.devtools.build.lib.bazel.repository.downloader.UnrecoverableHttpException; @@ -38,6 +42,8 @@ import java.io.IOException; import java.io.Writer; import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import org.junit.Before; import org.junit.Rule; @@ -49,18 +55,39 @@ /** Tests for {@link IndexRegistry}. */ @RunWith(JUnit4.class) public class IndexRegistryTest extends FoundationTestCase { + private static class EventRecorder { + private final List downloadEvents = new ArrayList<>(); + + @Subscribe + public void onRegistryFileDownloadEvent(RegistryFileDownloadEvent downloadEvent) { + downloadEvents.add(downloadEvent); + } + + public ImmutableMap> getRecordedHashes() { + return downloadEvents.stream() + .collect( + ImmutableMap.toImmutableMap( + RegistryFileDownloadEvent::uri, RegistryFileDownloadEvent::checksum)); + } + } + private final String authToken = BasicHttpAuthenticationEncoder.encode("rinne", "rinnepass", UTF_8); private DownloadManager downloadManager; + private EventRecorder eventRecorder; @Rule public final TestHttpServer server = new TestHttpServer(authToken); @Rule public final TemporaryFolder tempFolder = new TemporaryFolder(); private RegistryFactory registryFactory; + private RepositoryCache repositoryCache; @Before public void setUp() throws Exception { + eventRecorder = new EventRecorder(); + eventBus.register(eventRecorder); Path workspaceRoot = scratch.dir("/ws"); - downloadManager = new DownloadManager(new RepositoryCache(), new HttpDownloader()); + repositoryCache = new RepositoryCache(); + downloadManager = new DownloadManager(repositoryCache, new HttpDownloader()); registryFactory = new RegistryFactoryImpl( workspaceRoot, downloadManager, Suppliers.ofInstance(ImmutableMap.of())); @@ -71,12 +98,18 @@ public void testHttpUrl() throws Exception { server.serve("/myreg/modules/foo/1.0/MODULE.bazel", "lol"); server.start(); - Registry registry = registryFactory.createRegistry(server.getUrl() + "/myreg"); - assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) + Registry registry = + registryFactory.createRegistry(server.getUrl() + "/myreg", ImmutableMap.of()); + assertThat( + registry.getModuleFile( + createModuleKey("foo", "1.0"), reporter)) .hasValue( ModuleFile.create( "lol".getBytes(UTF_8), server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel")); - assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty(); + assertThat( + registry.getModuleFile( + createModuleKey("bar", "1.0"), reporter)) + .isEmpty(); } @Test @@ -87,20 +120,30 @@ public void testHttpUrlWithNetrcCreds() throws Exception { NetrcParser.parseAndClose( new ByteArrayInputStream( "machine [::1] login rinne password rinnepass\n".getBytes(UTF_8))); - Registry registry = registryFactory.createRegistry(server.getUrl() + "/myreg"); + Registry registry = + registryFactory.createRegistry(server.getUrl() + "/myreg", ImmutableMap.of()); - UnrecoverableHttpException e = + var e = assertThrows( - UnrecoverableHttpException.class, + IOException.class, () -> registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)); - assertThat(e).hasMessageThat().contains("GET returned 401 Unauthorized"); + assertThat(e) + .hasMessageThat() + .isEqualTo( + "Failed to fetch registry file %s: GET returned 401 Unauthorized" + .formatted(server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel")); downloadManager.setNetrcCreds(new NetrcCredentials(netrc)); - assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) + assertThat( + registry.getModuleFile( + createModuleKey("foo", "1.0"), reporter)) .hasValue( ModuleFile.create( "lol".getBytes(UTF_8), server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel")); - assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty(); + assertThat( + registry.getModuleFile( + createModuleKey("bar", "1.0"), reporter)) + .isEmpty(); } @Test @@ -113,10 +156,15 @@ public void testFileUrl() throws Exception { Registry registry = registryFactory.createRegistry( - new File(tempFolder.getRoot(), "fakereg").toURI().toString()); - assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) + new File(tempFolder.getRoot(), "fakereg").toURI().toString(), ImmutableMap.of()); + assertThat( + registry.getModuleFile( + createModuleKey("foo", "1.0"), reporter)) .hasValue(ModuleFile.create("lol".getBytes(UTF_8), file.toURI().toString())); - assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty(); + assertThat( + registry.getModuleFile( + createModuleKey("bar", "1.0"), reporter)) + .isEmpty(); } @Test @@ -149,8 +197,10 @@ public void testGetArchiveRepoSpec() throws Exception { "}"); server.start(); - Registry registry = registryFactory.createRegistry(server.getUrl()); - assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) + Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of()); + assertThat( + registry.getRepoSpec( + createModuleKey("foo", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() .setUrls( @@ -163,7 +213,9 @@ public void testGetArchiveRepoSpec() throws Exception { .setRemotePatches(ImmutableMap.of()) .setRemotePatchStrip(0) .build()); - assertThat(registry.getRepoSpec(createModuleKey("bar", "2.0"), reporter)) + assertThat( + registry.getRepoSpec( + createModuleKey("bar", "2.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() .setUrls( @@ -193,8 +245,10 @@ public void testGetLocalPathRepoSpec() throws Exception { "}"); server.start(); - Registry registry = registryFactory.createRegistry(server.getUrl()); - assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) + Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of()); + assertThat( + registry.getRepoSpec( + createModuleKey("foo", "1.0"), reporter)) .isEqualTo( RepoSpec.builder() .setRuleClassName("local_repository") @@ -215,8 +269,10 @@ public void testGetRepoInvalidRegistryJsonSpec() throws Exception { " \"strip_prefix\": \"pref\"", "}"); - Registry registry = registryFactory.createRegistry(server.getUrl()); - assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) + Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of()); + assertThat( + registry.getRepoSpec( + createModuleKey("foo", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() .setUrls(ImmutableList.of("http://mysite.com/thing.zip")) @@ -246,9 +302,12 @@ public void testGetRepoInvalidModuleJsonSpec() throws Exception { "}"); server.start(); - Registry registry = registryFactory.createRegistry(server.getUrl()); + Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of()); assertThrows( - IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); + IOException.class, + () -> + registry.getRepoSpec( + createModuleKey("foo", "1.0"), reporter)); } @Test @@ -273,7 +332,7 @@ public void testGetYankedVersion() throws Exception { + " }\n" + "}"); server.start(); - Registry registry = registryFactory.createRegistry(server.getUrl()); + Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of()); Optional> yankedVersion = registry.getYankedVersions("red-pill", reporter); assertThat(yankedVersion) @@ -294,8 +353,12 @@ public void testArchiveWithExplicitType() throws Exception { "}"); server.start(); - Registry registry = registryFactory.createRegistry(server.getUrl()); - assertThat(registry.getRepoSpec(createModuleKey("archive_type", "1.0"), reporter)) + Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of()); + assertThat( + registry.getRepoSpec( + createModuleKey("archive_type", "1.0"), + reporter + )) .isEqualTo( new ArchiveRepoSpecBuilder() .setUrls(ImmutableList.of("https://mysite.com/thing?format=zip")) @@ -306,4 +369,230 @@ public void testArchiveWithExplicitType() throws Exception { .setRemotePatchStrip(0) .build()); } + + @Test + public void testGetModuleFileChecksums() throws Exception { + repositoryCache.setRepositoryCachePath(scratch.dir("cache")); + + server.serve("/myreg/modules/foo/1.0/MODULE.bazel", "old"); + server.serve("/myreg/modules/foo/2.0/MODULE.bazel", "new"); + server.start(); + + var knownFiles = + ImmutableMap.of( + server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel", + Optional.of(sha256("old")), + server.getUrl() + "/myreg/modules/unused/1.0/MODULE.bazel", + Optional.of(sha256("unused"))); + Registry registry = registryFactory.createRegistry(server.getUrl() + "/myreg", knownFiles); + assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) + .hasValue( + ModuleFile.create( + "old".getBytes(UTF_8), server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel")); + assertThat(registry.getModuleFile(createModuleKey("foo", "2.0"), reporter)) + .hasValue( + ModuleFile.create( + "new".getBytes(UTF_8), server.getUrl() + "/myreg/modules/foo/2.0/MODULE.bazel")); + assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty(); + + var recordedChecksums = eventRecorder.getRecordedHashes(); + assertThat( + Maps.transformValues( + recordedChecksums, maybeChecksum -> maybeChecksum.map(Checksum::toString))) + .containsExactly( + server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel", + Optional.of(sha256("old").toString()), + server.getUrl() + "/myreg/modules/foo/2.0/MODULE.bazel", + Optional.of(sha256("new").toString()), + server.getUrl() + "/myreg/modules/bar/1.0/MODULE.bazel", + Optional.empty()) + .inOrder(); + + registry = registryFactory.createRegistry(server.getUrl() + "/myreg", recordedChecksums); + // Test that the recorded hashes are used for repo cache hits even when the server content + // changes. + server.unserve("/myreg/modules/foo/1.0/MODULE.bazel"); + server.unserve("/myreg/modules/foo/2.0/MODULE.bazel"); + server.serve("/myreg/modules/bar/1.0/MODULE.bazel", "no longer 404"); + assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) + .hasValue( + ModuleFile.create( + "old".getBytes(UTF_8), server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel")); + assertThat(registry.getModuleFile(createModuleKey("foo", "2.0"), reporter)) + .hasValue( + ModuleFile.create( + "new".getBytes(UTF_8), server.getUrl() + "/myreg/modules/foo/2.0/MODULE.bazel")); + assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty(); + } + + @Test + public void testGetModuleFileChecksumMismatch() throws Exception { + repositoryCache.setRepositoryCachePath(scratch.dir("cache")); + + server.serve("/myreg/modules/foo/1.0/MODULE.bazel", "fake"); + server.start(); + + var knownFiles = + ImmutableMap.of( + server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel", + Optional.of(sha256("original"))); + Registry registry = registryFactory.createRegistry(server.getUrl() + "/myreg", knownFiles); + var e = + assertThrows( + IOException.class, + () -> registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)); + assertThat(e) + .hasMessageThat() + .isEqualTo( + "Failed to fetch registry file %s: Checksum was %s but wanted %s" + .formatted( + server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel", + sha256("fake"), + sha256("original"))); + } + + @Test + public void testGetRepoSpecChecksum() throws Exception { + repositoryCache.setRepositoryCachePath(scratch.dir("cache")); + + String registryJson = + """ + { + "module_base_path": "/hello/foo" + } + """; + server.serve("/bazel_registry.json", registryJson); + String sourceJson = + """ + { + "type": "local_path", + "path": "../bar/project_x" + } + """; + server.serve("/modules/foo/1.0/source.json", sourceJson.getBytes(UTF_8)); + server.start(); + + var knownFiles = + ImmutableMap.of( + server.getUrl() + "/modules/foo/2.0/source.json", Optional.of(sha256("unused"))); + Registry registry = registryFactory.createRegistry(server.getUrl(), knownFiles); + assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) + .isEqualTo( + RepoSpec.builder() + .setRuleClassName("local_repository") + .setAttributes( + AttributeValues.create(ImmutableMap.of("path", "/hello/bar/project_x"))) + .build()); + + var recordedChecksums = eventRecorder.getRecordedHashes(); + assertThat( + Maps.transformValues(recordedChecksums, checksum -> checksum.map(Checksum::toString))) + .containsExactly( + server.getUrl() + "/bazel_registry.json", + Optional.of(sha256(registryJson).toString()), + server.getUrl() + "/modules/foo/1.0/source.json", + Optional.of(sha256(sourceJson).toString())); + + registry = registryFactory.createRegistry(server.getUrl(), recordedChecksums); + // Test that the recorded hashes are used for repo cache hits even when the server content + // changes. + server.unserve("/bazel_registry.json"); + server.unserve("/modules/foo/1.0/source.json"); + assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) + .isEqualTo( + RepoSpec.builder() + .setRuleClassName("local_repository") + .setAttributes( + AttributeValues.create(ImmutableMap.of("path", "/hello/bar/project_x"))) + .build()); + } + + @Test + public void testGetRepoSpecChecksumMismatch() throws Exception { + repositoryCache.setRepositoryCachePath(scratch.dir("cache")); + + String registryJson = + """ + { + "module_base_path": "/hello/foo" + } + """; + server.serve("/bazel_registry.json", registryJson.getBytes(UTF_8)); + String sourceJson = + """ + { + "type": "local_path", + "path": "../bar/project_x" + } + """; + String maliciousSourceJson = sourceJson.replaceAll("project_x", "malicious"); + server.serve("/modules/foo/1.0/source.json", maliciousSourceJson.getBytes(UTF_8)); + server.start(); + + var knownFiles = + ImmutableMap.of( + server.getUrl() + "/bazel_registry.json", + Optional.of(sha256(registryJson)), + server.getUrl() + "/modules/foo/1.0/source.json", + Optional.of(sha256(sourceJson))); + Registry registry = registryFactory.createRegistry(server.getUrl(), knownFiles); + var e = + assertThrows( + IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); + assertThat(e) + .hasMessageThat() + .isEqualTo( + "Failed to fetch registry file %s: Checksum was %s but wanted %s" + .formatted( + server.getUrl() + "/modules/foo/1.0/source.json", + sha256(maliciousSourceJson), + sha256(sourceJson))); + } + + @Test + public void testBazelRegistryChecksumMismatch() throws Exception { + repositoryCache.setRepositoryCachePath(scratch.dir("cache")); + + String registryJson = + """ + { + "module_base_path": "/hello/foo" + } + """; + String maliciousRegistryJson = registryJson.replaceAll("foo", "malicious"); + server.serve("/bazel_registry.json", maliciousRegistryJson.getBytes(UTF_8)); + String sourceJson = + """ + { + "type": "local_path", + "path": "../bar/project_x" + } + """; + server.serve("/modules/foo/1.0/source.json", sourceJson.getBytes(UTF_8)); + server.start(); + + var knownFiles = + ImmutableMap.of( + server.getUrl() + "/bazel_registry.json", + Optional.of(sha256(registryJson)), + server.getUrl() + "/modules/foo/1.0/source.json", + Optional.of(sha256(sourceJson))); + Registry registry = registryFactory.createRegistry(server.getUrl(), knownFiles); + var e = + assertThrows( + IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); + assertThat(e) + .hasMessageThat() + .isEqualTo( + "Failed to fetch registry file %s: Checksum was %s but wanted %s" + .formatted( + server.getUrl() + "/bazel_registry.json", + sha256(maliciousRegistryJson), + sha256(registryJson))); + } + + private static Checksum sha256(String content) throws Checksum.InvalidChecksumException { + return Checksum.fromString( + RepositoryCache.KeyType.SHA256, Hashing.sha256().hashString(content, UTF_8).toString()); + } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index 5bc28bf8601fb7..6985ab01ffc668 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -278,7 +278,7 @@ public void setup() throws Exception { ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java index cea99f30491dea..bf4d985c03e506 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java @@ -142,6 +142,7 @@ private void setUpWithBuiltinModules(ImmutableMap b new TimestampGranularityMonitor(BlazeClock.instance())), SyscallCache.NO_CACHE, externalFilesHelper)) + .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put( SkyFunctions.MODULE_FILE, new ModuleFileFunction( @@ -223,7 +224,7 @@ public void testRootModule() throws Exception { "multiple_version_override(module_name='fff',versions=['1.0','2.0'])", "archive_override(module_name='ggg',urls=['https://hello.com/world.zip'])"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate( @@ -271,7 +272,7 @@ public void testRootModule_noModuleFunctionIsOkay() throws Exception { rootDirectory.getRelative("MODULE.bazel").getPathString(), "bazel_dep(name='bbb',version='1.0')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate( @@ -297,7 +298,7 @@ public void testRootModule_badSelfOverride() throws Exception { "module(name='aaa')", "single_version_override(module_name='aaa',version='7')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate( @@ -318,7 +319,7 @@ public void testRootModule_overrideBuiltinModule() throws Exception { "module(name='aaa')", "local_path_override(module_name='bazel_tools',path='./bazel_tools_new')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate( @@ -352,7 +353,7 @@ public void testRootModule_include_good() throws Exception { rootDirectory.getRelative("python/toolchains/toolchains.MODULE.bazel").getPathString(), "register_toolchains('//:python-whatever')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate( @@ -389,7 +390,7 @@ public void testRootModule_include_bad_otherRepoLabel() throws Exception { "module(name='aaa')", "include('@haha//java:java.MODULE.bazel')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate( @@ -405,7 +406,7 @@ public void testRootModule_include_bad_relativeLabel() throws Exception { "module(name='aaa')", "include(':relative.MODULE.bazel')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate( @@ -421,7 +422,7 @@ public void testRootModule_include_bad_notEndingInModuleBazel() throws Exception "module(name='aaa')", "include('//:MODULE.bazel.segment')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate( @@ -437,7 +438,7 @@ public void testRootModule_include_bad_badLabelSyntax() throws Exception { "module(name='aaa')", "include('//haha/:::.MODULE.bazel')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); reporter.removeHandler(failFastHandler); // expect failures EvaluationResult result = @@ -458,7 +459,7 @@ public void testRootModule_include_bad_moduleAfterInclude() throws Exception { "module(name='bet-you-didnt-expect-this-didya')", "bazel_dep(name='java-foo', version='1.0', repo_name='foo')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); reporter.removeHandler(failFastHandler); // expect failures EvaluationResult result = @@ -484,7 +485,7 @@ public void testRootModule_include_bad_repoNameCollision() throws Exception { rootDirectory.getRelative("python/python.MODULE.bazel").getPathString(), "bazel_dep(name='python-foo', version='1.0', repo_name='foo')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); reporter.removeHandler(failFastHandler); // expect failures EvaluationResult result = @@ -506,7 +507,7 @@ public void testRootModule_include_bad_tryingToLeakBindings() throws Exception { rootDirectory.getRelative("java/java.MODULE.bazel").getPathString(), "bazel_dep(name=FOO_NAME, version='1.0')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); reporter.removeHandler(failFastHandler); // expect failures EvaluationResult result = @@ -519,7 +520,7 @@ public void testRootModule_include_bad_tryingToLeakBindings() throws Exception { @Test public void forgotVersion() throws Exception { FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); SkyKey skyKey = ModuleFileValue.key(createModuleKey("bbb", ""), null); EvaluationResult result = @@ -547,7 +548,7 @@ public void testRegistriesCascade() throws Exception { createModuleKey("bbb", "1.0"), "module(name='bbb',version='1.0');bazel_dep(name='ddd',version='3.0')"); ModuleFileFunction.REGISTRIES.set( - differencer, ImmutableList.of(registry1.getUrl(), registry2.getUrl(), registry3.getUrl())); + differencer, ImmutableSet.of(registry1.getUrl(), registry2.getUrl(), registry3.getUrl())); SkyKey skyKey = ModuleFileValue.key(createModuleKey("bbb", "1.0"), null); EvaluationResult result = @@ -573,7 +574,7 @@ public void testNonRootModuleCannotUseInclude() throws Exception { createModuleKey("foo", "1.0"), "module(name='foo',version='1.0')", "include('//java:MODULE.bazel.segment')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate( @@ -602,7 +603,7 @@ public void testLocalPathOverride() throws Exception { .addModule( createModuleKey("bbb", "1.0"), "module(name='bbb',version='1.0');bazel_dep(name='ccc',version='3.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); // The version is empty here due to the override. SkyKey skyKey = @@ -653,7 +654,7 @@ public void testCommandLineModuleOverrides() throws Exception { .addModule( createModuleKey("bbb", "1.0"), "module(name='bbb',version='1.0');bazel_dep(name='ccc',version='3.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); // The version is empty here due to the override. SkyKey skyKey = @@ -688,7 +689,7 @@ public void testRegistryOverride() throws Exception { createModuleKey("bbb", "1.0"), "module(name='bbb',version='1.0',compatibility_level=6)", "bazel_dep(name='ccc',version='3.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry1.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry1.getUrl())); // Override the registry for B to be registry2 (instead of the default registry1). SkyKey skyKey = @@ -731,7 +732,7 @@ public void testModuleExtensions_good() throws Exception { "maven.dep(coord='junit')", "use_repo(maven, 'junit', 'guava')", "maven.dep(coord='guava')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); ModuleKey myMod = createModuleKey("mymod", "1.0"); SkyKey skyKey = ModuleFileValue.key(myMod, null); @@ -873,7 +874,7 @@ public void testModuleExtensions_duplicateProxy_asRoot() throws Exception { "myext4 = use_extension('//:defs.bzl','myext')", "myext4.tag(name = 'tag4')", "use_repo(myext4, 'delta')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of()); SkyKey skyKey = ModuleFileValue.KEY_FOR_ROOT_MODULE; EvaluationResult result = @@ -972,7 +973,7 @@ public void testModuleExtensions_duplicateProxy_asDep() throws Exception { "myext4 = use_extension('//:defs.bzl','myext')", "myext4.tag(name = 'tag4')", "use_repo(myext4, 'delta')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); ModuleKey myMod = createModuleKey("mymod", "1.0"); SkyKey skyKey = ModuleFileValue.key(myMod, null); @@ -1039,7 +1040,7 @@ public void testModuleExtensions_repoNameCollision_localRepoName() throws Except "module(name='mymod',version='1.0')", "myext = use_extension('//:defs.bzl','myext')", "use_repo(myext, mymod='some_repo')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); SkyKey skyKey = ModuleFileValue.key(createModuleKey("mymod", "1.0"), null); reporter.removeHandler(failFastHandler); // expect failures @@ -1059,7 +1060,7 @@ public void testModuleExtensions_repoNameCollision_exportedRepoName() throws Exc "module(name='mymod',version='1.0')", "myext = use_extension('//:defs.bzl','myext')", "use_repo(myext, 'some_repo', again='some_repo')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); SkyKey skyKey = ModuleFileValue.key(createModuleKey("mymod", "1.0"), null); reporter.removeHandler(failFastHandler); // expect failures @@ -1078,7 +1079,7 @@ public void testModuleExtensions_innate() throws Exception { "http_archive = use_repo_rule('@bazel_tools//:http.bzl','http_archive')", "http_archive(name='guava',url='guava.com')", "http_archive(name='vuaga',url='vuaga.com',dev_dependency=True)"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of()); SkyKey skyKey = ModuleFileValue.KEY_FOR_ROOT_MODULE; EvaluationResult result = @@ -1272,7 +1273,7 @@ public void testBuiltinModules_forRoot() throws Exception { scratch.overwriteFile( rootDirectory.getRelative("MODULE.bazel").getPathString(), "bazel_dep(name='foo',version='1.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of()); SkyKey skyKey = ModuleFileValue.KEY_FOR_ROOT_MODULE; EvaluationResult result = @@ -1308,7 +1309,7 @@ public void testBuiltinModules_forBuiltinModules() throws Exception { rootDirectory.getRelative("tools/MODULE.bazel").getPathString(), "module(name='bazel_tools',version='1.0')", "bazel_dep(name='foo',version='2.0')"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of()); SkyKey skyKey = ModuleFileValue.key(createModuleKey("bazel_tools", ""), builtinModules.get("bazel_tools")); @@ -1333,7 +1334,7 @@ public void moduleRepoName() throws Exception { rootDirectory.getRelative("MODULE.bazel").getPathString(), "module(name='aaa',version='0.1',repo_name='bbb')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate( @@ -1357,7 +1358,7 @@ public void moduleRepoName_conflict() throws Exception { "module(name='aaa',version='0.1',repo_name='bbb')", "bazel_dep(name='bbb',version='1.0')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); reporter.removeHandler(failFastHandler); // expect failures evaluator.evaluate(ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); @@ -1372,7 +1373,7 @@ public void module_calledTwice() throws Exception { "module(name='aaa',version='0.1',repo_name='bbb')", "module(name='aaa',version='0.1',repo_name='bbb')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); reporter.removeHandler(failFastHandler); // expect failures evaluator.evaluate(ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); @@ -1387,7 +1388,7 @@ public void module_calledLate() throws Exception { "use_extension('//:extensions.bzl', 'my_ext')", "module(name='aaa',version='0.1',repo_name='bbb')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); reporter.removeHandler(failFastHandler); // expect failures evaluator.evaluate(ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); @@ -1401,7 +1402,7 @@ public void restrictedSyntax() throws Exception { rootDirectory.getRelative("MODULE.bazel").getPathString(), "if 3+5>7: module(name='aaa',version='0.1',repo_name='bbb')"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); reporter.removeHandler(failFastHandler); // expect failures evaluator.evaluate(ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); @@ -1424,7 +1425,7 @@ public void isolatedUsageWithoutImports() throws Exception { rootDirectory.getRelative("MODULE.bazel").getPathString(), "isolated_ext = use_extension('//:extensions.bzl', 'my_ext', isolate = True)"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); EvaluationResult result = evaluator.evaluate( @@ -1451,7 +1452,7 @@ public void isolatedUsageNotExported() throws Exception { rootDirectory.getRelative("MODULE.bazel").getPathString(), "use_extension('//:extensions.bzl', 'my_ext', isolate = True)"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); reporter.removeHandler(failFastHandler); // expect failures evaluator.evaluate(ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); @@ -1466,7 +1467,7 @@ public void isolatedUsage_notEnabled() throws Exception { rootDirectory.getRelative("MODULE.bazel").getPathString(), "use_extension('//:extensions.bzl', 'my_ext', isolate = True)"); FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); reporter.removeHandler(failFastHandler); // expect failures evaluator.evaluate(ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java index 4c6c63299c87a1..c56ad709836890 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java @@ -43,10 +43,14 @@ public void badSchemes() throws Exception { new DownloadManager(new RepositoryCache(), new HttpDownloader()), Suppliers.ofInstance(ImmutableMap.of())); Throwable exception = - assertThrows(URISyntaxException.class, () -> registryFactory.createRegistry("/home/www")); + assertThrows( + URISyntaxException.class, + () -> registryFactory.createRegistry("/home/www", ImmutableMap.of())); assertThat(exception).hasMessageThat().contains("Registry URL has no scheme"); exception = - assertThrows(URISyntaxException.class, () -> registryFactory.createRegistry("foo://bar")); + assertThrows( + URISyntaxException.class, + () -> registryFactory.createRegistry("foo://bar", ImmutableMap.of())); assertThat(exception).hasMessageThat().contains("Unrecognized registry URL protocol"); } @@ -61,7 +65,9 @@ public void badPath() throws Exception { Throwable exception = assertThrows( URISyntaxException.class, - () -> registryFactory.createRegistry("file:c:/path/to/workspace/registry")); + () -> + registryFactory.createRegistry( + "file:c:/path/to/workspace/registry", ImmutableMap.of())); assertThat(exception).hasMessageThat().contains("Registry URL path is not valid"); } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/TestHttpServer.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/TestHttpServer.java index 629e4be1e0508c..8a7628fb1c2b39 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/TestHttpServer.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/TestHttpServer.java @@ -78,6 +78,10 @@ public void serve(String path, String... lines) { serve(path, JOINER.join(lines).getBytes(UTF_8)); } + public void unserve(String path) { + server.removeContext(path); + } + public String getUrl() throws MalformedURLException { return new URL("http", "[::1]", server.getAddress().getPort(), "").toString(); } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloaderTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloaderTest.java index 0e59d412abb975..fb086b3eed986f 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloaderTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloaderTest.java @@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.hash.Hashing; import com.google.common.io.ByteStreams; import com.google.devtools.build.lib.authandtls.StaticCredentials; import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache; @@ -43,6 +44,7 @@ import java.net.Socket; import java.net.SocketTimeoutException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -516,6 +518,7 @@ public void downloadAndReadOneUrl_ok() throws IOException, InterruptedException httpDownloader.downloadAndReadOneUrl( new URL(String.format("http://localhost:%d/foo", server.getLocalPort())), StaticCredentials.EMPTY, + Optional.empty(), eventHandler, Collections.emptyMap()), UTF_8)) @@ -551,11 +554,90 @@ public void downloadAndReadOneUrl_notFound() throws IOException, InterruptedExce httpDownloader.downloadAndReadOneUrl( new URL(String.format("http://localhost:%d/foo", server.getLocalPort())), StaticCredentials.EMPTY, + Optional.empty(), eventHandler, Collections.emptyMap())); } } + @Test + public void downloadAndReadOneUrl_checksumProvided() + throws IOException, Checksum.InvalidChecksumException, InterruptedException { + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName(null))) { + @SuppressWarnings("unused") + Future possiblyIgnoredError = + executor.submit( + () -> { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines( + socket, + "HTTP/1.1 200 OK", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 5", + "", + "hello"); + } + return null; + }); + + assertThat( + new String( + httpDownloader.downloadAndReadOneUrl( + new URL(String.format("http://localhost:%d/foo", server.getLocalPort())), + StaticCredentials.EMPTY, + Optional.of( + Checksum.fromString( + RepositoryCache.KeyType.SHA256, + Hashing.sha256().hashString("hello", UTF_8).toString())), + eventHandler, + Collections.emptyMap()), + UTF_8)) + .isEqualTo("hello"); + } + } + + @Test + public void downloadAndReadOneUrl_checksumMismatch() throws IOException { + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName(null))) { + @SuppressWarnings("unused") + Future possiblyIgnoredError = + executor.submit( + () -> { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines( + socket, + "HTTP/1.1 200 OK", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 9", + "", + "malicious"); + } + return null; + }); + + var e = + assertThrows( + UnrecoverableHttpException.class, + () -> + httpDownloader.downloadAndReadOneUrl( + new URL(String.format("http://localhost:%d/foo", server.getLocalPort())), + StaticCredentials.EMPTY, + Optional.of( + Checksum.fromString( + RepositoryCache.KeyType.SHA256, + Hashing.sha256().hashUnencodedChars("hello").toString())), + eventHandler, + Collections.emptyMap())); + assertThat(e).hasMessageThat().contains("Checksum was"); + } + } + @Test public void download_contentLengthMismatch_propagateErrorIfNotRetry() throws Exception { Downloader downloader = mock(Downloader.class); diff --git a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java index af85d6ab0ef39f..8f60ae25a85157 100644 --- a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java +++ b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java @@ -18,6 +18,7 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.devtools.build.lib.actions.ActionKeyContext; import com.google.devtools.build.lib.analysis.BlazeDirectories; @@ -380,7 +381,7 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru PrecomputedValue.injected( ModuleFileFunction.MODULE_OVERRIDES, ImmutableMap.of()), PrecomputedValue.injected( - ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())), + ModuleFileFunction.REGISTRIES, ImmutableSet.of(registry.getUrl())), PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, @@ -417,7 +418,7 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru PrecomputedValue.injected( RepositoryDelegatorFunction.VENDOR_DIRECTORY, Optional.empty()), PrecomputedValue.injected( - ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())), + ModuleFileFunction.REGISTRIES, ImmutableSet.of(registry.getUrl())), PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), PrecomputedValue.injected(RepositoryDelegatorFunction.DISABLE_NATIVE_REPO_RULES, false), PrecomputedValue.injected( diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java index 943afea62d580d..0c88fc5db85402 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java @@ -175,7 +175,7 @@ public void setupDelegator() throws Exception { .addModule( createModuleKey("bazel_tools", "1.0"), "module(name='bazel_tools', version='1.0');"); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of(registry.getUrl())); HashFunction hashFunction = fileSystem.getDigestFunction().getHashFunction(); evaluator = diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java index 9d6c010fa9fe0c..a6410802cefe00 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java @@ -19,6 +19,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.eventbus.EventBus; import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; @@ -268,7 +269,7 @@ protected ImmutableList extraPrecomputedValues() { registry = FakeRegistry.DEFAULT_FACTORY.newFakeRegistry(moduleRoot.getPathString()); return ImmutableList.of( PrecomputedValue.injected( - ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())), + ModuleFileFunction.REGISTRIES, ImmutableSet.of(registry.getUrl())), PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), diff --git a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py index fb04b521cbb98f..8f7e07bfc1beb9 100644 --- a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py +++ b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py @@ -133,26 +133,66 @@ def testChangeModuleInRegistryWithLockfile(self): # hence find no errors self.RunBazel(['build', '--nobuild', '//:all']) - def testChangeFlagWithLockfile(self): - # Add module 'sss' to the registry with dep on 'aaa' - self.main_registry.createCcModule('sss', '1.3', {'aaa': '1.1'}) - # Create a project with deps on 'sss' self.ScratchFile( 'MODULE.bazel', [ 'bazel_dep(name = "sss", version = "1.3")', + 'bazel_dep(name = "bbb", version = "1.1")', + ], + ) + # Shutdown bazel to empty any cache of the deps tree + self.RunBazel(['shutdown']) + # Even adding a new dependency should not fail due to the registry change + self.RunBazel(['build', '--nobuild', '//:all']) + + def testAddModuleToRegistryWithLockfile(self): + # Create a project with deps on the BCR's 'platforms' module + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "platforms", version = "0.0.9")', ], ) self.ScratchFile('BUILD', ['filegroup(name = "hello")']) self.RunBazel(['build', '--nobuild', '//:all']) - # Change registry -> update 'sss' module file (corrupt it) - module_dir = self.main_registry.root.joinpath('modules', 'sss', '1.3') + # Add a broken 'platforms' module to the first registry + module_dir = self.main_registry.root.joinpath('modules', 'platforms', '0.0.9') scratchFile(module_dir.joinpath('MODULE.bazel'), ['whatever!']) # Shutdown bazel to empty any cache of the deps tree self.RunBazel(['shutdown']) - # Running with the lockfile, but adding a flag should cause resolution rerun + # Running with the lockfile, should not recognize the registry changes + # hence find no errors + self.RunBazel(['build', '--nobuild', '//:all']) + + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "platforms", version = "0.0.9")', + 'bazel_dep(name = "bbb", version = "1.1")', + ], + ) + # Shutdown bazel to empty any cache of the deps tree + self.RunBazel(['shutdown']) + # Even adding a new dependency should not fail due to the registry change + self.RunBazel(['build', '--nobuild', '//:all']) + + def testChangeFlagWithLockfile(self): + # Create a project with an outdated direct dep on aaa + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "aaa", version = "1.0")', + 'bazel_dep(name = "bbb", version = "1.1")', + ], + ) + self.ScratchFile('BUILD', ['filegroup(name = "hello")']) + self.RunBazel(['build', '--nobuild', '//:all']) + + # Shutdown bazel to empty any cache of the deps tree + self.RunBazel(['shutdown']) + # Running with the lockfile, but the changed flag value should be honored exit_code, _, stderr = self.RunBazel( [ 'build', @@ -163,9 +203,9 @@ def testChangeFlagWithLockfile(self): allow_failure=True, ) self.AssertExitCode(exit_code, 48, stderr) - self.assertRegex( + self.assertIn( + "ERROR: For repository 'aaa', the root module requires module version aaa@1.0, but got aaa@1.1", '\n'.join(stderr), - "ERROR: .*/sss/1.3/MODULE.bazel:1:9: invalid character: '!'", ) def testLockfileErrorMode(self): @@ -1104,6 +1144,62 @@ def testLockfileRecreationOnlyUsesUpToDateBuildResults(self): self.assertEqual(old_data, new_data) + def testLockfileExtensionsUpdatedIncrementally(self): + self.ScratchFile( + 'MODULE.bazel', + [ + 'lockfile_ext1 = use_extension("extension.bzl", "lockfile_ext1")', + 'use_repo(lockfile_ext1, "hello1")', + 'lockfile_ext2 = use_extension("extension.bzl", "lockfile_ext2")', + 'use_repo(lockfile_ext2, "hello2")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("BUILD", "filegroup(name=\'lala\')")', + '', + 'repo_rule = repository_rule(implementation=_repo_rule_impl)', + '', + 'def _module_ext1_impl(ctx):', + ' print("Hello from ext1!")', + ' repo_rule(name="hello1")', + '', + 'lockfile_ext1 = module_extension(', + ' implementation=_module_ext1_impl,', + ')', + '', + 'def _module_ext2_impl(ctx):', + ' print("Hello from ext2!")', + ' repo_rule(name="hello2")', + '', + 'lockfile_ext2 = module_extension(', + ' implementation=_module_ext2_impl,', + ')', + ], + ) + + _, _, stderr = self.RunBazel(['build', '@hello1//:all']) + stderr = '\n'.join(stderr) + self.assertIn('Hello from ext1!', stderr) + self.assertNotIn('Hello from ext2!', stderr) + + self.RunBazel(['shutdown']) + + _, _, stderr = self.RunBazel(['build', '@hello1//:all', "@hello2//:all"]) + stderr = '\n'.join(stderr) + self.assertNotIn('Hello from ext1!', stderr) + self.assertIn('Hello from ext2!', stderr) + + self.RunBazel(['shutdown']) + + _, _, stderr = self.RunBazel(['build', '@hello1//:all', "@hello2//:all"]) + stderr = '\n'.join(stderr) + self.assertNotIn('Hello from ext1!', stderr) + self.assertNotIn('Hello from ext2!', stderr) + def testExtensionOsAndArch(self): self.ScratchFile( 'MODULE.bazel', diff --git a/src/test/shell/bazel/starlark_repository_test.sh b/src/test/shell/bazel/starlark_repository_test.sh index a53803f14d8486..5742bb5ad35629 100755 --- a/src/test/shell/bazel/starlark_repository_test.sh +++ b/src/test/shell/bazel/starlark_repository_test.sh @@ -2327,6 +2327,9 @@ genrule( ) EOF + # Ensure that all Bzlmod-related downloads have happened before disabling + # downloads. + bazel mod deps || fail "Failed to cache Bazel modules" bazel build --repository_disable_download //:it > "${TEST_log}" 2>&1 \ && fail "Expected failure" || : expect_log "Failed to download repository @.*: download is disabled" diff --git a/src/test/tools/bzlmod/MODULE.bazel.lock b/src/test/tools/bzlmod/MODULE.bazel.lock index aba6193eda8dac..43fb80ed1b9c9a 100644 --- a/src/test/tools/bzlmod/MODULE.bazel.lock +++ b/src/test/tools/bzlmod/MODULE.bazel.lock @@ -1,5 +1,5 @@ { - "lockFileVersion": 7, + "lockFileVersion": 8, "moduleFileHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "flags": { "cmdRegistries": [ @@ -1074,6 +1074,63 @@ } } }, + "registryFileHashes": { + "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", + "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", + "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", + "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/source.json": "7e3a9adf473e9af076ae485ed649d5641ad50ec5c11718103f34de03170d94ad", + "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef", + "https://bcr.bazel.build/modules/apple_support/1.5.0/source.json": "eb98a7627c0bc486b57f598ad8da50f6625d974c8f723e9ea71bd39f709c9862", + "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", + "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", + "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", + "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", + "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/source.json": "2e0e90f6788740b72f0db3c19c46155a82ec01cfc1527c35b58f3f8f0180da29", + "https://bcr.bazel.build/modules/buildozer/7.1.1.1/MODULE.bazel": "21c6a7d08e3171d3e13b003407caefe7ebe007693e217a053cc1f49f008ce010", + "https://bcr.bazel.build/modules/buildozer/7.1.1.1/source.json": "a9ced884dedcf1c45d11052d53d854e368b05aa8fbbf0f983037fbed4d3ea4c6", + "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", + "https://bcr.bazel.build/modules/googletest/1.11.0/source.json": "c73d9ef4268c91bd0c1cd88f1f9dfa08e814b1dbe89b5f594a9f08ba0244d206", + "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", + "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37", + "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615", + "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814", + "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", + "https://bcr.bazel.build/modules/platforms/0.0.9/source.json": "cd74d854bf16a9e002fb2ca7b1a421f4403cda29f824a765acd3a8c56f8d43e6", + "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", + "https://bcr.bazel.build/modules/protobuf/21.7/source.json": "bbe500720421e582ff2d18b0802464205138c06056f443184de39fbb8187b09b", + "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", + "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858", + "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", + "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", + "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", + "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", + "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430", + "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", + "https://bcr.bazel.build/modules/rules_java/7.5.0/MODULE.bazel": "b329bf9aa07a58bd1ccb37bfdcd9528acf6f12712efb38c3a8553c2cc2494806", + "https://bcr.bazel.build/modules/rules_java/7.5.0/source.json": "72762e4e144dd1bc54e18b90be52e31a4ca9cf11d7358a06e1b87b74e839e9ad", + "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", + "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/source.json": "a075731e1b46bc8425098512d038d416e966ab19684a10a34f4741295642fc35", + "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", + "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", + "https://bcr.bazel.build/modules/rules_license/0.0.7/source.json": "355cc5737a0f294e560d52b1b7a6492d4fff2caf0bef1a315df5a298fca2d34a", + "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", + "https://bcr.bazel.build/modules/rules_pkg/0.7.0/source.json": "c2557066e0c0342223ba592510ad3d812d4963b9024831f7f66fd0584dd8c66c", + "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", + "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", + "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/source.json": "d57902c052424dfda0e71646cb12668d39c4620ee0544294d9d941e7d12bc3a9", + "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", + "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7", + "https://bcr.bazel.build/modules/rules_python/0.22.1/source.json": "57226905e783bae7c37c2dd662be078728e48fa28ee4324a7eabcafb5a43d014", + "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", + "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", + "https://bcr.bazel.build/modules/stardoc/0.5.1/source.json": "a96f95e02123320aa015b956f29c00cb818fa891ef823d55148e1a362caacf29", + "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", + "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/source.json": "f1ef7d3f9e0e26d4b23d1c39b5f5de71f584dd7d1b4ef83d9bbba6ec7a6a6459", + "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", + "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27", + "https://bcr.bazel.build/modules/zlib/1.3/MODULE.bazel": "6a9c02f19a24dcedb05572b2381446e27c272cd383aed11d41d99da9e3167a72", + "https://bcr.bazel.build/modules/zlib/1.3/source.json": "b6b43d0737af846022636e6e255fd4a96fee0d34f08f3830e6e0bac51465c37c" + }, "moduleExtensions": { "@@apple_support~//crosstool:setup.bzl%apple_cc_configure_extension": { "general": {