From 9ab1e2b9421eb99786ff3e03871dfdc37c905545 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Wed, 21 Apr 2021 16:06:37 +0200 Subject: [PATCH 1/2] Add remote_patches attribute for http_archive --- tools/build_defs/repo/http.bzl | 12 +++++++++++- tools/build_defs/repo/utils.bzl | 22 +++++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/tools/build_defs/repo/http.bzl b/tools/build_defs/repo/http.bzl index 1affd6649a0340..09446bf7ff7b6a 100644 --- a/tools/build_defs/repo/http.bzl +++ b/tools/build_defs/repo/http.bzl @@ -118,7 +118,7 @@ def _http_archive_impl(ctx): auth = auth, ) workspace_and_buildfile(ctx) - patch(ctx) + patch(ctx, auth = auth) return update_attrs(ctx.attr, _http_archive_attrs.keys(), {"sha256": download_info.sha256}) @@ -279,6 +279,16 @@ following: `"zip"`, `"jar"`, `"war"`, `"tar"`, `"tar.gz"`, `"tgz"`, "patch command line tool if `patch_tool` attribute is specified or there are " + "arguments other than `-p` in `patch_args` attribute.", ), + "remote_patches": attr.string_list( + default = [], + doc = + "A list of URLs of patch files that are to be applied as patches after " + + "extracting the archive and applied patch files from the `patches` attribute. " + + "By default, it uses the Bazel-native patch implementation " + + "which doesn't support fuzz match and binary patch, but Bazel will fall back to use " + + "patch command line tool if `patch_tool` attribute is specified or there are " + + "arguments other than `-p` in `patch_args` attribute.", + ), "patch_tool": attr.string( default = "", doc = "The patch(1) utility to use. If this is specified, Bazel will use the specifed " + diff --git a/tools/build_defs/repo/utils.bzl b/tools/build_defs/repo/utils.bzl index 5a1147e283c8dd..fdfc94e8ab0482 100644 --- a/tools/build_defs/repo/utils.bzl +++ b/tools/build_defs/repo/utils.bzl @@ -69,7 +69,18 @@ def _use_native_patch(patch_args): return False return True -def patch(ctx, patches = None, patch_cmds = None, patch_cmds_win = None, patch_tool = None, patch_args = None): +def _download_patch(ctx, patch_url, auth): + name = patch_url.split("/")[-1] + patch_path = ctx.path(".tmp_remote_patches").get_child(name) + ctx.download( + patch_url, + patch_path, + canonical_id = ctx.attr.canonical_id, + auth = auth, + ) + return patch_path + +def patch(ctx, patches = None, patch_cmds = None, patch_cmds_win = None, patch_tool = None, patch_args = None, auth = None): """Implementation of patching an already extracted repository. This rule is intended to be used in the implementation function of @@ -95,10 +106,15 @@ def patch(ctx, patches = None, patch_cmds = None, patch_cmds_win = None, patch_t bash_exe = ctx.os.environ["BAZEL_SH"] if "BAZEL_SH" in ctx.os.environ else "bash" powershell_exe = ctx.os.environ["BAZEL_POWERSHELL"] if "BAZEL_POWERSHELL" in ctx.os.environ else "powershell.exe" - if patches == None and hasattr(ctx.attr, "patches"): - patches = ctx.attr.patches if patches == None: patches = [] + if hasattr(ctx.attr, "patches") and ctx.attr.patches: + patches += ctx.attr.patches + + if hasattr(ctx.attr, "remote_patches") and ctx.attr.remote_patches: + for patch_url in ctx.attr.remote_patches: + patch_path = _download_patch(ctx, patch_url, auth) + patches.append(patch_path) if patch_cmds == None and hasattr(ctx.attr, "patch_cmds"): patch_cmds = ctx.attr.patch_cmds From f74388af77faf83b6d250916dfa389266a169ade Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Thu, 22 Apr 2021 16:18:39 +0200 Subject: [PATCH 2/2] First working version --- .../com/google/devtools/build/lib/bazel/BUILD | 3 + .../lib/bazel/BazelRepositoryModule.java | 13 +++ .../lib/bazel/bzlmod/ArchiveFetcher.java | 31 ++++++- .../lib/bazel/bzlmod/ArchiveOverride.java | 10 ++- .../lib/bazel/bzlmod/FetcherFactory.java | 6 +- .../build/lib/bazel/bzlmod/IndexRegistry.java | 14 +++- .../build/lib/bazel/bzlmod/Module.java | 12 +++ .../lib/bazel/bzlmod/ModuleFileFunction.java | 11 ++- .../lib/bazel/bzlmod/ModuleFileGlobals.java | 16 ++-- .../lib/bazel/bzlmod/RegistryFactoryImpl.java | 11 +-- .../build/lib/bazel/bzlmod/repo/BUILD | 7 +- .../repo/BazelModuleRepoInfoFunction.java | 84 +++++++++++-------- .../repo/ModuleRuleRepoInfoFunction.java | 14 ++-- .../google/devtools/build/lib/skyframe/BUILD | 2 +- .../build/lib/skyframe/SkyframeExecutor.java | 7 +- .../packages/AbstractPackageLoader.java | 11 +++ .../repository/ModuleFileGlobalsApi.java | 6 +- .../lib/bazel/bzlmod/IndexRegistryTest.java | 2 + 18 files changed, 189 insertions(+), 71 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD index 88a90f3fd78b80..9417cf15cddbd5 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD @@ -26,6 +26,9 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster", "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories", "//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:fetch", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:registry", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution", "//src/main/java/com/google/devtools/build/lib/bazel/commands", "//src/main/java/com/google/devtools/build/lib/bazel/repository", "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 2c061ec68978ea..be2cebf1e73036 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -25,6 +25,12 @@ import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.bazel.bzlmod.DiscoveryFunction; +import com.google.devtools.build.lib.bazel.bzlmod.FetcherFactory; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.RegistryFactory; +import com.google.devtools.build.lib.bazel.bzlmod.RegistryFactoryImpl; +import com.google.devtools.build.lib.bazel.bzlmod.SelectionFunction; import com.google.devtools.build.lib.bazel.commands.FetchCommand; import com.google.devtools.build.lib.bazel.commands.SyncCommand; import com.google.devtools.build.lib.bazel.repository.LocalConfigPlatformFunction; @@ -202,6 +208,13 @@ public void workspaceInit( managedDirectoriesKnowledge, BazelSkyframeExecutorConstants.EXTERNAL_PACKAGE_HELPER); builder.addSkyFunction(SkyFunctions.REPOSITORY_DIRECTORY, repositoryDelegatorFunction); + FetcherFactory fetcherFactory = new FetcherFactory(directories.getWorkspace()); + RegistryFactory registryFactory = new RegistryFactoryImpl( + new HttpDownloader(), clientEnvironmentSupplier, fetcherFactory); + builder.addSkyFunction(SkyFunctions.MODULE_FILE, new ModuleFileFunction( + fetcherFactory, registryFactory, directories.getWorkspace())); + builder.addSkyFunction(SkyFunctions.DISCOVERY, new DiscoveryFunction()); + builder.addSkyFunction(SkyFunctions.SELECTION, new SelectionFunction()); filesystem = runtime.getFileSystem(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveFetcher.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveFetcher.java index f1646972ca740e..a4c6cab2d53735 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveFetcher.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveFetcher.java @@ -1,18 +1,24 @@ package com.google.devtools.build.lib.bazel.bzlmod; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.vfs.Path; + import java.net.URL; import java.util.Objects; +import java.util.stream.Collectors; public class ArchiveFetcher implements EarlyFetcher { private final ImmutableList urls; + private final ImmutableList patches; private final String integrity; private final String stripPrefix; - ArchiveFetcher(ImmutableList urls, String integrity, String stripPrefix) { + ArchiveFetcher(ImmutableList urls, ImmutableList patches, String integrity, + String stripPrefix) { this.urls = urls; + this.patches = patches; this.integrity = integrity; this.stripPrefix = stripPrefix; } @@ -27,6 +33,25 @@ public Path fetch(String repoName, Path vendorDir) { return null; } + public ImmutableMap toHttpArchiveAttrs(String repoName) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + builder.put("name", repoName) + .put("urls", urls.stream().map(URL::toString).collect(Collectors.toList())) + .put("sha256", integrity.substring(7)); + + if (patches != null) { + builder.put("remote_patches", patches.stream().map(URL::toString).collect(Collectors.toList())); + } + + // TODO: add patch_strip to this class + builder.put("patch_args", ImmutableList.of("-p0")); + if (stripPrefix != null) { + // TODO: implement integrity attribute in http_archive + builder.put("strip_prefix", stripPrefix); + } + return builder.build(); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -37,19 +62,21 @@ public boolean equals(Object o) { } ArchiveFetcher that = (ArchiveFetcher) o; return Objects.equals(urls, that.urls) && + Objects.equals(patches, that.patches) && Objects.equals(integrity, that.integrity) && Objects.equals(stripPrefix, that.stripPrefix); } @Override public int hashCode() { - return Objects.hash(urls, integrity, stripPrefix); + return Objects.hash(urls, patches, integrity, stripPrefix); } @Override public String toString() { return "ArchiveFetcher{" + "urls=" + urls + + "patches=" + patches + ", integrity='" + integrity + '\'' + ", stripPrefix='" + stripPrefix + '\'' + '}'; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveOverride.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveOverride.java index 17df6ce9900038..8cd41703457fc6 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveOverride.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveOverride.java @@ -7,19 +7,21 @@ @AutoValue public abstract class ArchiveOverride implements NonRegistryOverride { - public static ArchiveOverride create(ImmutableList urls, String integrity, - String stripPrefix) { - return new AutoValue_ArchiveOverride(urls, integrity, stripPrefix); + public static ArchiveOverride create(ImmutableList urls, ImmutableList patches, + String integrity, String stripPrefix) { + return new AutoValue_ArchiveOverride(urls, patches, integrity, stripPrefix); } public abstract ImmutableList getUrls(); + public abstract ImmutableList getPatches(); + public abstract String getIntegrity(); public abstract String getStripPrefix(); @Override public EarlyFetcher toEarlyFetcher(FetcherFactory fetcherFactory) { - return fetcherFactory.createArchiveFetcher(getUrls(), getIntegrity(), getStripPrefix()); + return fetcherFactory.createArchiveFetcher(getUrls(), getPatches(), getIntegrity(), getStripPrefix()); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/FetcherFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/FetcherFactory.java index b3ea1ff90611fb..00fed378761389 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/FetcherFactory.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/FetcherFactory.java @@ -12,10 +12,10 @@ public FetcherFactory(Path workspaceRoot) { this.workspaceRoot = workspaceRoot; } - public EarlyFetcher createArchiveFetcher(ImmutableList urls, String integrity, - String stripPrefix) { + public EarlyFetcher createArchiveFetcher(ImmutableList urls, ImmutableList patches, + String integrity, String stripPrefix) { // TODO: add patches - return new ArchiveFetcher(urls, integrity, stripPrefix); + return new ArchiveFetcher(urls, patches, integrity, stripPrefix); } public LocalPathFetcher createLocalPathFetcher(String path) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index 8bf6070cb46269..85db370a85cb8f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -3,7 +3,6 @@ import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; import com.google.devtools.build.lib.events.ExtendedEventHandler; -import com.google.devtools.build.lib.vfs.Path; import com.google.gson.FieldNamingPolicy; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -70,7 +69,7 @@ private static class SourceJson { URL url; String integrity; String stripPrefix; - String[] patchFiles; + String[] patches; int patchStrip; } @@ -100,6 +99,7 @@ public Fetcher getFetcher(ModuleKey key, ExtendedEventHandler eventHandler) URL sourceUrl = sourceJson.get().url; ImmutableList.Builder urls = new ImmutableList.Builder<>(); if (bazelRegistryJson.isPresent()) { + // TODO: check if mirrors is null for (String mirror : bazelRegistryJson.get().mirrors) { StringBuilder url = new StringBuilder(mirror); if (url.charAt(url.length() - 1) != '/') { @@ -114,8 +114,18 @@ public Fetcher getFetcher(ModuleKey key, ExtendedEventHandler eventHandler) } } urls.add(sourceUrl); + + ImmutableList.Builder patchUrls = new ImmutableList.Builder<>(); + if (sourceJson.get().patches != null) { + for (String name : sourceJson.get().patches) { + patchUrls.add(new URL(getUrl() + String + .format("/modules/%s/%s/patches/%s", key.getName(), key.getVersion(), name))); + } + } + return fetcherFactory.createArchiveFetcher( urls.build(), + patchUrls.build(), sourceJson.get().integrity, sourceJson.get().stripPrefix); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java index 59324084815ef7..e08c691a6221f8 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java @@ -3,6 +3,8 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableMap; +import jdk.internal.jline.internal.Nullable; + @AutoValue public abstract class Module { @@ -12,6 +14,12 @@ public abstract class Module { public abstract ImmutableMap getDeps(); + @Nullable + public abstract Fetcher getFetcher(); + + @Nullable + public abstract Registry getRegistry(); + public abstract Builder toBuilder(); static Builder builder() { @@ -27,6 +35,10 @@ public abstract static class Builder { public abstract Builder setDeps(ImmutableMap value); + public abstract Builder setFetcher(Fetcher value); + + public abstract Builder setRegistry(Registry value); + abstract ImmutableMap.Builder depsBuilder(); public Builder addDep(String depRepoName, ModuleKey depKey) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java index 376cb92ff5be1f..16743b9560f43d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java @@ -38,7 +38,7 @@ public class ModuleFileFunction implements SkyFunction { // TODO: populate this with the value of a flag. - public static final Precomputed> REGISTRIES = new Precomputed<>("registries"); +// public static final Precomputed> REGISTRIES = new Precomputed<>("registries"); private final FetcherFactory fetcherFactory; private final RegistryFactory registryFactory; @@ -89,7 +89,8 @@ public SkyValue compute(SkyKey skyKey, Environment env) moduleKey, starlarkSemantics, env); // Perform some sanity checks. - Module module = moduleFileGlobals.buildModule(); + Module module = moduleFileGlobals.buildModule(getModuleFileResult.earlyFetcher, + getModuleFileResult.registry); if (!module.getName().equals(moduleKey.getName())) { throw errorf("the MODULE.bazel file of %s declares a different name (%s)", moduleKey, module.getName()); @@ -114,7 +115,8 @@ private SkyValue computeForRootModule(StarlarkSemantics starlarkSemantics, Envir byte[] moduleFile = readFile(moduleFilePath.asPath()); ModuleFileGlobals moduleFileGlobals = execModuleFile(moduleFile, ModuleFileValue.ROOT_MODULE_KEY, starlarkSemantics, env); - Module module = moduleFileGlobals.buildModule(); + // TODO: should we add a fetcher for root module? + Module module = moduleFileGlobals.buildModule(null,null); // Check that overrides don't contain the root itself (we need to set the override for the root // module to "local path" of the workspace root). @@ -180,7 +182,8 @@ private Optional getModuleFile(ModuleKey key, StarlarkOverr return Optional.of(result); } - List registries = Objects.requireNonNull(REGISTRIES.get(env)); +// List registries = Objects.requireNonNull(REGISTRIES.get(env)); + List registries = ImmutableList.of("https://storage.googleapis.com/bcr.bazel.build"); if (override instanceof RegistryOverride) { String overrideRegistry = ((RegistryOverride) override).getRegistry(); if (!overrideRegistry.isEmpty()) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java index 0f2b0c48c232b3..75c42fa516a135 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java @@ -10,10 +10,12 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; + +import net.starlark.java.eval.Dict; import net.starlark.java.eval.EvalException; import net.starlark.java.eval.Starlark; -public class ModuleFileGlobals implements ModuleFileGlobalsApi { +public class ModuleFileGlobals implements ModuleFileGlobalsApi { private boolean moduleCalled = false; private final Module.Builder module = Module.builder(); @@ -24,7 +26,7 @@ public ModuleFileGlobals() { } @Override - public void module(String name, String version) + public void module(String name, String version, Dict kwargs) throws EvalException { if (moduleCalled) { throw Starlark.errorf("the module() directive can only be called once"); @@ -73,7 +75,8 @@ public StarlarkOverrideApi archiveOverride(Object urls, String integrity, } catch (MalformedURLException e) { throw new ModuleFileFunctionException(e); } - return ArchiveOverride.create(urlList.build(), integrity, stripPrefix); + // TODO: add patch file support here as well + return ArchiveOverride.create(urlList.build(), ImmutableList.of(), integrity, stripPrefix); } @Override @@ -81,8 +84,11 @@ public StarlarkOverrideApi localPathOverride(String path) { return LocalPathOverride.create(path); } - public Module buildModule() { - return module.setDeps(ImmutableMap.copyOf(deps)).build(); + public Module buildModule(Fetcher fetcher, Registry registry) { + return module.setDeps(ImmutableMap.copyOf(deps)) + .setFetcher(fetcher) + .setRegistry(registry) + .build(); } public ImmutableMap buildOverrides() { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java index 948f84b96cee51..ae477f5e1a3481 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java @@ -1,21 +1,22 @@ package com.google.devtools.build.lib.bazel.bzlmod; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; -import com.google.devtools.build.lib.vfs.Path; + import java.net.URI; import java.net.URISyntaxException; import java.util.Map; +import java.util.function.Supplier; public class RegistryFactoryImpl implements RegistryFactory { private final HttpDownloader httpDownloader; - private final Map clientEnv; + private final Supplier> clientEnvironmentSupplier; private final FetcherFactory fetcherFactory; - public RegistryFactoryImpl(HttpDownloader httpDownloader, Map clientEnv, + public RegistryFactoryImpl(HttpDownloader httpDownloader, Supplier> clientEnvironmentSupplier, FetcherFactory fetcherFactory) { this.httpDownloader = httpDownloader; - this.clientEnv = clientEnv; + this.clientEnvironmentSupplier = clientEnvironmentSupplier; this.fetcherFactory = fetcherFactory; } @@ -26,7 +27,7 @@ public Registry getRegistryWithUrl(String url) throws URISyntaxException { case "http": case "https": case "file": - return new IndexRegistry(uri, httpDownloader, clientEnv, fetcherFactory); + return new IndexRegistry(uri, httpDownloader, clientEnvironmentSupplier.get(), fetcherFactory); } return null; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo/BUILD index 54cbc579224612..3b94a6412f0ada 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo/BUILD @@ -36,12 +36,17 @@ java_library( name = "repo_info_functions", srcs = [ "BazelModuleRepoInfoFunction.java", - "ModuleRuleRepoInfoFunction.java", "LockFileParser.java", + "ModuleRuleRepoInfoFunction.java", ], deps = [ ":repo_info_values", "//src/main/java/com/google/devtools/build/lib/actions:file_metadata", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:fetch", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:registry", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/pkgcache", "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value", "//src/main/java/com/google/devtools/build/lib/vfs", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo/BazelModuleRepoInfoFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo/BazelModuleRepoInfoFunction.java index 7455613a6bee65..4ebd8fb571dce5 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo/BazelModuleRepoInfoFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo/BazelModuleRepoInfoFunction.java @@ -2,14 +2,13 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.devtools.build.lib.actions.FileValue; -import com.google.devtools.build.lib.pkgcache.PathPackageLocator; -import com.google.devtools.build.lib.skyframe.PrecomputedValue; -import com.google.devtools.build.lib.vfs.FileSystemUtils; -import com.google.devtools.build.lib.vfs.Path; -import com.google.devtools.build.lib.vfs.PathFragment; -import com.google.devtools.build.lib.vfs.Root; -import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.bzlmod.ArchiveFetcher; +import com.google.devtools.build.lib.bazel.bzlmod.Fetcher; +import com.google.devtools.build.lib.bazel.bzlmod.Module; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.SelectionValue; +import com.google.devtools.build.lib.bazel.bzlmod.repo.RepositoryInfo.Builder; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; @@ -17,45 +16,59 @@ import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; +import java.net.URISyntaxException; import javax.annotation.Nullable; public class BazelModuleRepoInfoFunction implements SkyFunction { + private static final String HTTP_ARCHIVE = "@bazel_tools//tools/build_defs/repo:http.bzl%http_archive"; + @Nullable @Override public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, InterruptedException { Preconditions.checkArgument(skyKey == BazelModuleRepoInfoValue.key()); - PathPackageLocator packageLocator = PrecomputedValue.PATH_PACKAGE_LOCATOR.get(env); - ImmutableList packagePath = packageLocator.getPathEntries(); - - // In Bazel, there should be only one workspace root. - Root workspaceRoot = packagePath.get(0); - RootedPath lockFile = RootedPath.toRootedPath(workspaceRoot, - PathFragment.create("native_module_repos.json")); - - FileValue fileValue = (FileValue) env.getValue(FileValue.key(lockFile)); - if (env.valuesMissing()) { + SelectionValue selectionValue = (SelectionValue) env.getValue(SelectionValue.KEY); + if (selectionValue == null) { return null; } - if (!fileValue.exists() || !fileValue.isFile()) { - throw new ResolvedBazelModuleRepositoriesFunctionException( - new IOException("Expect lock file native_module_repos.json to exist at workspace root."), - Transience.TRANSIENT); - } + ImmutableMap.Builder repositories = ImmutableMap.builder(); - Path lockFilePath = lockFile.asPath(); - byte[] bytes; - try { - bytes = FileSystemUtils.readWithKnownFileSize(lockFilePath, lockFilePath.getFileSize()); - } catch (IOException ex) { - throw new ResolvedBazelModuleRepositoriesFunctionException(ex, Transience.TRANSIENT); + for (ModuleKey moduleKey : selectionValue.getDepGraph().keySet()) { + if (!moduleKey.getName().equals(selectionValue.getRootModuleName())) { + try { + Module module = selectionValue.getDepGraph().get(moduleKey); + Fetcher fetcher; + if (module.getFetcher() != null) { + fetcher = module.getFetcher(); + } else { + fetcher = module.getRegistry().getFetcher(moduleKey, env.getListener()); + } + if (fetcher instanceof ArchiveFetcher) { + ArchiveFetcher archiveFetcher = (ArchiveFetcher) fetcher; + RepositoryInfo.Builder builder = new Builder(); + String name = "@" + moduleKey.getName(); + RepositoryInfo repositoryInfo = builder.setName(name) + .setRuleClass(HTTP_ARCHIVE) + .setAttributes(archiveFetcher.toHttpArchiveAttrs(moduleKey.getName())) + // TODO: implement repo_deps + .setRepoDeps(ImmutableList.of()) + // TODO: implement repo_mappings + .setRepoMappings(ImmutableMap.of()) + // TODO: impement vendor_dir + .setVendorDir("") + .build(); + repositories.put(name, repositoryInfo); + } + // TODO: implement for other fetcher type + } catch (IOException e) { + throw new BazelModuleRepoInfoFunctionException(e, Transience.PERSISTENT); + } + } } - - return new BazelModuleRepoInfoValue( - LockFileParser.loadRepositoryInfos(new String(bytes))); + return new BazelModuleRepoInfoValue(repositories.build()); } @Nullable @@ -64,8 +77,11 @@ public String extractTag(SkyKey skyKey) { return null; } - private static final class ResolvedBazelModuleRepositoriesFunctionException extends SkyFunctionException { - ResolvedBazelModuleRepositoriesFunctionException(IOException e, Transience transience) { + private static final class BazelModuleRepoInfoFunctionException extends SkyFunctionException { + BazelModuleRepoInfoFunctionException(IOException e, Transience transience) { + super(e, transience); + } + BazelModuleRepoInfoFunctionException(URISyntaxException e, Transience transience) { super(e, transience); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo/ModuleRuleRepoInfoFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo/ModuleRuleRepoInfoFunction.java index 4f272293174ca7..e6c0e7144024c3 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo/ModuleRuleRepoInfoFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo/ModuleRuleRepoInfoFunction.java @@ -2,6 +2,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; import com.google.devtools.build.lib.skyframe.PrecomputedValue; @@ -41,9 +42,10 @@ public SkyValue compute(SkyKey skyKey, Environment env) } if (!fileValue.exists() || !fileValue.isFile()) { - throw new ResolvedModuleRuleRepositoriesFunctionException( - new IOException("Expect lock file module_rule_repos.json to exist at workspace root."), - Transience.TRANSIENT); +// throw new ResolvedModuleRuleRepositoriesFunctionException( +// new IOException("Expect lock file module_rule_repos.json to exist at workspace root."), +// Transience.TRANSIENT); + return new ModuleRuleRepoInfoValue(ImmutableMap.of()); } Path lockFilePath = lockFile.asPath(); @@ -51,7 +53,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) try { bytes = FileSystemUtils.readWithKnownFileSize(lockFilePath, lockFilePath.getFileSize()); } catch (IOException ex) { - throw new ResolvedModuleRuleRepositoriesFunctionException(ex, Transience.TRANSIENT); + throw new ModuleRuleRepoInfoFunctionException(ex, Transience.TRANSIENT); } return new ModuleRuleRepoInfoValue( @@ -64,8 +66,8 @@ public String extractTag(SkyKey skyKey) { return null; } - private static final class ResolvedModuleRuleRepositoriesFunctionException extends SkyFunctionException { - ResolvedModuleRuleRepositoriesFunctionException(IOException e, Transience transience) { + private static final class ModuleRuleRepoInfoFunctionException extends SkyFunctionException { + ModuleRuleRepoInfoFunctionException(IOException e, Transience transience) { super(e, transience); } } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD index 0f782097131114..c98a4c9a138b0a 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD @@ -266,8 +266,8 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:workspace_status_action", "//src/main/java/com/google/devtools/build/lib/analysis/platform", "//src/main/java/com/google/devtools/build/lib/analysis/platform:utils", - "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo:repo_info_values", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo:repo_info_functions", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo:repo_info_values", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo:repo_rule_creator", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo:repo_rule_value", "//src/main/java/com/google/devtools/build/lib/bugreport", diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index 8621842331ddc7..1a22688bf9443d 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -103,9 +103,9 @@ import com.google.devtools.build.lib.analysis.configuredtargets.OutputFileConfiguredTarget; import com.google.devtools.build.lib.analysis.starlark.StarlarkTransition; import com.google.devtools.build.lib.analysis.starlark.StarlarkTransition.TransitionException; -import com.google.devtools.build.lib.bazel.bzlmod.repo.BzlmodRepoRuleValue; import com.google.devtools.build.lib.bazel.bzlmod.repo.BazelModuleRepoInfoFunction; import com.google.devtools.build.lib.bazel.bzlmod.repo.BazelModuleRepoInfoValue; +import com.google.devtools.build.lib.bazel.bzlmod.repo.BzlmodRepoRuleValue; import com.google.devtools.build.lib.bazel.bzlmod.repo.ModuleRuleRepoInfoFunction; import com.google.devtools.build.lib.bazel.bzlmod.repo.ModuleRuleRepoInfoValue; import com.google.devtools.build.lib.bugreport.BugReport; @@ -217,6 +217,9 @@ import com.google.devtools.common.options.OptionsParsingException; import com.google.devtools.common.options.OptionsProvider; import com.google.errorprone.annotations.ForOverride; + +import net.starlark.java.eval.StarlarkSemantics; + import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; @@ -239,8 +242,8 @@ import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; + import javax.annotation.Nullable; -import net.starlark.java.eval.StarlarkSemantics; /** * A helper object to support Skyframe-driven execution. diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java b/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java index 0930d254b9c31c..92c3a3f5af9992 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java @@ -30,11 +30,18 @@ import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ServerDirectories; +import com.google.devtools.build.lib.bazel.bzlmod.DiscoveryFunction; +import com.google.devtools.build.lib.bazel.bzlmod.FetcherFactory; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.RegistryFactory; +import com.google.devtools.build.lib.bazel.bzlmod.RegistryFactoryImpl; +import com.google.devtools.build.lib.bazel.bzlmod.SelectionFunction; import com.google.devtools.build.lib.bazel.bzlmod.repo.BzlmodRepoRuleValue; import com.google.devtools.build.lib.bazel.bzlmod.repo.BazelModuleRepoInfoFunction; import com.google.devtools.build.lib.bazel.bzlmod.repo.BazelModuleRepoInfoValue; import com.google.devtools.build.lib.bazel.bzlmod.repo.ModuleRuleRepoInfoFunction; import com.google.devtools.build.lib.bazel.bzlmod.repo.ModuleRuleRepoInfoValue; +import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; import com.google.devtools.build.lib.clock.BlazeClock; import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor; @@ -466,6 +473,10 @@ public Path getBuildFileForPackage(PackageIdentifier packageName) { } }; ImmutableMap.Builder builder = ImmutableMap.builder(); + FetcherFactory fetcherFactory = new FetcherFactory(directories.getWorkspace()); + // TODO: clientEnv is still empty here + RegistryFactory registryFactory = new RegistryFactoryImpl( + new HttpDownloader(), ImmutableMap.of(), fetcherFactory); builder .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) .put( diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/repository/ModuleFileGlobalsApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/repository/ModuleFileGlobalsApi.java index 3ec2b3ad192416..9f4f5fccb33f54 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/repository/ModuleFileGlobalsApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/repository/ModuleFileGlobalsApi.java @@ -19,6 +19,7 @@ import net.starlark.java.annot.Param; import net.starlark.java.annot.ParamType; import net.starlark.java.annot.StarlarkMethod; +import net.starlark.java.eval.Dict; import net.starlark.java.eval.EvalException; /** @@ -45,8 +46,9 @@ public interface ModuleFileGlobalsApi kwargs) throws EvalException, InterruptedException; @StarlarkMethod( name = "bazel_dep", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java index be82d6a4e65c2d..cad3dea20a85b6 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java @@ -88,6 +88,7 @@ public void testGetFetcher() throws Exception { new URL("https://mirror.bazel.build/mysite.com/thing.zip"), new URL("file:///home/bazel/mymirror/mysite.com/thing.zip"), new URL("http://mysite.com/thing.zip")), + ImmutableList.of(), "sha256-blah", "pref")); assertThat(registry.getFetcher(ModuleKey.create("bar", "2.0"), reporter)) @@ -96,6 +97,7 @@ public void testGetFetcher() throws Exception { new URL("https://mirror.bazel.build/example.com/archive.jar?with=query"), new URL("file:///home/bazel/mymirror/example.com/archive.jar?with=query"), new URL("https://example.com/archive.jar?with=query")), + ImmutableList.of(), "sha256-bleh", null)); }