Merge pull request bazelbuild#6 from meteorcloudy/mvp-version-0

Mvp version 0

src/main/java/com/google/devtools/build/lib/bazel/BUILD

@@ -26,6 +26,9 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
         "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",
         "//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration",
+        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:fetch",
+        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:registry",
+        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution",
         "//src/main/java/com/google/devtools/build/lib/bazel/commands",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -25,6 +25,12 @@
 import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
 import com.google.devtools.build.lib.analysis.RuleDefinition;
 import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.bazel.bzlmod.DiscoveryFunction;
+import com.google.devtools.build.lib.bazel.bzlmod.FetcherFactory;
+import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction;
+import com.google.devtools.build.lib.bazel.bzlmod.RegistryFactory;
+import com.google.devtools.build.lib.bazel.bzlmod.RegistryFactoryImpl;
+import com.google.devtools.build.lib.bazel.bzlmod.SelectionFunction;
 import com.google.devtools.build.lib.bazel.commands.FetchCommand;
 import com.google.devtools.build.lib.bazel.commands.SyncCommand;
 import com.google.devtools.build.lib.bazel.repository.LocalConfigPlatformFunction;
@@ -202,6 +208,13 @@ public void workspaceInit(
             managedDirectoriesKnowledge,
             BazelSkyframeExecutorConstants.EXTERNAL_PACKAGE_HELPER);
     builder.addSkyFunction(SkyFunctions.REPOSITORY_DIRECTORY, repositoryDelegatorFunction);
+    FetcherFactory fetcherFactory = new FetcherFactory(directories.getWorkspace());
+    RegistryFactory registryFactory = new RegistryFactoryImpl(
+        new HttpDownloader(), clientEnvironmentSupplier, fetcherFactory);
+    builder.addSkyFunction(SkyFunctions.MODULE_FILE, new ModuleFileFunction(
+        fetcherFactory, registryFactory, directories.getWorkspace()));
+    builder.addSkyFunction(SkyFunctions.DISCOVERY, new DiscoveryFunction());
+    builder.addSkyFunction(SkyFunctions.SELECTION, new SelectionFunction());
     filesystem = runtime.getFileSystem();
   }
 

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveFetcher.java

@@ -1,18 +1,24 @@
 package com.google.devtools.build.lib.bazel.bzlmod;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.devtools.build.lib.vfs.Path;
+
 import java.net.URL;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 public class ArchiveFetcher implements EarlyFetcher {
 
   private final ImmutableList<URL> urls;
+  private final ImmutableList<URL> patches;
   private final String integrity;
   private final String stripPrefix;
 
-  ArchiveFetcher(ImmutableList<URL> urls, String integrity, String stripPrefix) {
+  ArchiveFetcher(ImmutableList<URL> urls, ImmutableList<URL> patches, String integrity,
+      String stripPrefix) {
     this.urls = urls;
+    this.patches = patches;
     this.integrity = integrity;
     this.stripPrefix = stripPrefix;
   }
@@ -27,6 +33,25 @@ public Path fetch(String repoName, Path vendorDir) {
     return null;
   }
 
+  public ImmutableMap<String, Object> toHttpArchiveAttrs(String repoName) {
+    ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
+    builder.put("name", repoName)
+        .put("urls", urls.stream().map(URL::toString).collect(Collectors.toList()))
+        .put("sha256", integrity.substring(7));
+
+    if (patches != null) {
+      builder.put("remote_patches", patches.stream().map(URL::toString).collect(Collectors.toList()));
+    }
+
+    // TODO: add patch_strip to this class
+    builder.put("patch_args", ImmutableList.of("-p0"));
+    if (stripPrefix != null) {
+      // TODO: implement integrity attribute in http_archive
+      builder.put("strip_prefix", stripPrefix);
+    }
+    return builder.build();
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -37,19 +62,21 @@ public boolean equals(Object o) {
     }
     ArchiveFetcher that = (ArchiveFetcher) o;
     return Objects.equals(urls, that.urls) &&
+        Objects.equals(patches, that.patches) &&
         Objects.equals(integrity, that.integrity) &&
         Objects.equals(stripPrefix, that.stripPrefix);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(urls, integrity, stripPrefix);
+    return Objects.hash(urls, patches, integrity, stripPrefix);
   }
 
   @Override
   public String toString() {
     return "ArchiveFetcher{" +
         "urls=" + urls +
+        "patches=" + patches +
         ", integrity='" + integrity + '\'' +
         ", stripPrefix='" + stripPrefix + '\'' +
         '}';

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveOverride.java

@@ -7,19 +7,21 @@
 @AutoValue
 public abstract class ArchiveOverride implements NonRegistryOverride {
 
-  public static ArchiveOverride create(ImmutableList<URL> urls, String integrity,
-      String stripPrefix) {
-    return new AutoValue_ArchiveOverride(urls, integrity, stripPrefix);
+  public static ArchiveOverride create(ImmutableList<URL> urls, ImmutableList<URL> patches,
+      String integrity, String stripPrefix) {
+    return new AutoValue_ArchiveOverride(urls, patches, integrity, stripPrefix);
   }
 
   public abstract ImmutableList<URL> getUrls();
 
+  public abstract ImmutableList<URL> getPatches();
+
   public abstract String getIntegrity();
 
   public abstract String getStripPrefix();
 
   @Override
   public EarlyFetcher toEarlyFetcher(FetcherFactory fetcherFactory) {
-    return fetcherFactory.createArchiveFetcher(getUrls(), getIntegrity(), getStripPrefix());
+    return fetcherFactory.createArchiveFetcher(getUrls(), getPatches(), getIntegrity(), getStripPrefix());
   }
 }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/FetcherFactory.java

@@ -12,10 +12,10 @@ public FetcherFactory(Path workspaceRoot) {
     this.workspaceRoot = workspaceRoot;
   }
 
-  public EarlyFetcher createArchiveFetcher(ImmutableList<URL> urls, String integrity,
-      String stripPrefix) {
+  public EarlyFetcher createArchiveFetcher(ImmutableList<URL> urls, ImmutableList<URL> patches,
+      String integrity, String stripPrefix) {
     // TODO: add patches
-    return new ArchiveFetcher(urls, integrity, stripPrefix);
+    return new ArchiveFetcher(urls, patches, integrity, stripPrefix);
   }
 
   public LocalPathFetcher createLocalPathFetcher(String path) {

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java

@@ -3,7 +3,6 @@
 import com.google.common.collect.ImmutableList;
 import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
 import com.google.devtools.build.lib.events.ExtendedEventHandler;
-import com.google.devtools.build.lib.vfs.Path;
 import com.google.gson.FieldNamingPolicy;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
@@ -70,7 +69,7 @@ private static class SourceJson {
     URL url;
     String integrity;
     String stripPrefix;
-    String[] patchFiles;
+    String[] patches;
     int patchStrip;
   }
 
@@ -100,6 +99,7 @@ public Fetcher getFetcher(ModuleKey key, ExtendedEventHandler eventHandler)
     URL sourceUrl = sourceJson.get().url;
     ImmutableList.Builder<URL> urls = new ImmutableList.Builder<>();
     if (bazelRegistryJson.isPresent()) {
+      // TODO: check if mirrors is null
       for (String mirror : bazelRegistryJson.get().mirrors) {
         StringBuilder url = new StringBuilder(mirror);
         if (url.charAt(url.length() - 1) != '/') {
@@ -114,8 +114,18 @@ public Fetcher getFetcher(ModuleKey key, ExtendedEventHandler eventHandler)
       }
     }
     urls.add(sourceUrl);
+
+    ImmutableList.Builder<URL> patchUrls = new ImmutableList.Builder<>();
+    if (sourceJson.get().patches != null) {
+      for (String name : sourceJson.get().patches) {
+        patchUrls.add(new URL(getUrl() + String
+            .format("/modules/%s/%s/patches/%s", key.getName(), key.getVersion(), name)));
+      }
+    }
+
     return fetcherFactory.createArchiveFetcher(
         urls.build(),
+        patchUrls.build(),
         sourceJson.get().integrity,
         sourceJson.get().stripPrefix);
   }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java

@@ -3,6 +3,8 @@
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableMap;
 
+import jdk.internal.jline.internal.Nullable;
+
 @AutoValue
 public abstract class Module {
 
@@ -12,6 +14,12 @@ public abstract class Module {
 
   public abstract ImmutableMap<String, ModuleKey> getDeps();
 
+  @Nullable
+  public abstract Fetcher getFetcher();
+
+  @Nullable
+  public abstract Registry getRegistry();
+
   public abstract Builder toBuilder();
 
   static Builder builder() {
@@ -27,6 +35,10 @@ public abstract static class Builder {
 
     public abstract Builder setDeps(ImmutableMap<String, ModuleKey> value);
 
+    public abstract Builder setFetcher(Fetcher value);
+
+    public abstract Builder setRegistry(Registry value);
+
     abstract ImmutableMap.Builder<String, ModuleKey> depsBuilder();
 
     public Builder addDep(String depRepoName, ModuleKey depKey) {

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java

@@ -38,7 +38,7 @@
 public class ModuleFileFunction implements SkyFunction {
 
   // TODO: populate this with the value of a flag.
-  public static final Precomputed<List<String>> REGISTRIES = new Precomputed<>("registries");
+//  public static final Precomputed<List<String>> REGISTRIES = new Precomputed<>("registries");
 
   private final FetcherFactory fetcherFactory;
   private final RegistryFactory registryFactory;
@@ -89,7 +89,8 @@ public SkyValue compute(SkyKey skyKey, Environment env)
         moduleKey, starlarkSemantics, env);
 
     // Perform some sanity checks.
-    Module module = moduleFileGlobals.buildModule();
+    Module module = moduleFileGlobals.buildModule(getModuleFileResult.earlyFetcher,
+        getModuleFileResult.registry);
     if (!module.getName().equals(moduleKey.getName())) {
       throw errorf("the MODULE.bazel file of %s declares a different name (%s)", moduleKey,
           module.getName());
@@ -114,7 +115,8 @@ private SkyValue computeForRootModule(StarlarkSemantics starlarkSemantics, Envir
     byte[] moduleFile = readFile(moduleFilePath.asPath());
     ModuleFileGlobals moduleFileGlobals = execModuleFile(moduleFile,
         ModuleFileValue.ROOT_MODULE_KEY, starlarkSemantics, env);
-    Module module = moduleFileGlobals.buildModule();
+    // TODO: should we add a fetcher for root module?
+    Module module = moduleFileGlobals.buildModule(null,null);
 
     // Check that overrides don't contain the root itself (we need to set the override for the root
     // module to "local path" of the workspace root).
@@ -180,7 +182,8 @@ private Optional<GetModuleFileResult> getModuleFile(ModuleKey key, StarlarkOverr
       return Optional.of(result);
     }
 
-    List<String> registries = Objects.requireNonNull(REGISTRIES.get(env));
+//    List<String> registries = Objects.requireNonNull(REGISTRIES.get(env));
+    List<String> registries = ImmutableList.of("https://storage.googleapis.com/bcr.bazel.build");
     if (override instanceof RegistryOverride) {
       String overrideRegistry = ((RegistryOverride) override).getRegistry();
       if (!overrideRegistry.isEmpty()) {

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java

@@ -10,10 +10,12 @@
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
+
+import net.starlark.java.eval.Dict;
 import net.starlark.java.eval.EvalException;
 import net.starlark.java.eval.Starlark;
 
-public class ModuleFileGlobals implements ModuleFileGlobalsApi {
+public class ModuleFileGlobals implements ModuleFileGlobalsApi<ModuleFileFunctionException> {
 
   private boolean moduleCalled = false;
   private final Module.Builder module = Module.builder();
@@ -24,7 +26,7 @@ public ModuleFileGlobals() {
   }
 
   @Override
-  public void module(String name, String version)
+  public void module(String name, String version, Dict<String, Object> kwargs)
       throws EvalException {
     if (moduleCalled) {
       throw Starlark.errorf("the module() directive can only be called once");
@@ -73,16 +75,20 @@ public StarlarkOverrideApi archiveOverride(Object urls, String integrity,
     } catch (MalformedURLException e) {
       throw new ModuleFileFunctionException(e);
     }
-    return ArchiveOverride.create(urlList.build(), integrity, stripPrefix);
+    // TODO: add patch file support here as well
+    return ArchiveOverride.create(urlList.build(), ImmutableList.of(), integrity, stripPrefix);
   }
 
   @Override
   public StarlarkOverrideApi localPathOverride(String path) {
     return LocalPathOverride.create(path);
   }
 
-  public Module buildModule() {
-    return module.setDeps(ImmutableMap.copyOf(deps)).build();
+  public Module buildModule(Fetcher fetcher, Registry registry) {
+    return module.setDeps(ImmutableMap.copyOf(deps))
+        .setFetcher(fetcher)
+        .setRegistry(registry)
+        .build();
   }
 
   public ImmutableMap<String, StarlarkOverrideApi> buildOverrides() {

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java

@@ -1,21 +1,22 @@
 package com.google.devtools.build.lib.bazel.bzlmod;
 
 import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
-import com.google.devtools.build.lib.vfs.Path;
+
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Map;
+import java.util.function.Supplier;
 
 public class RegistryFactoryImpl implements RegistryFactory {
 
   private final HttpDownloader httpDownloader;
-  private final Map<String, String> clientEnv;
+  private final Supplier<Map<String, String>> clientEnvironmentSupplier;
   private final FetcherFactory fetcherFactory;
 
-  public RegistryFactoryImpl(HttpDownloader httpDownloader, Map<String, String> clientEnv,
+  public RegistryFactoryImpl(HttpDownloader httpDownloader, Supplier<Map<String, String>> clientEnvironmentSupplier,
       FetcherFactory fetcherFactory) {
     this.httpDownloader = httpDownloader;
-    this.clientEnv = clientEnv;
+    this.clientEnvironmentSupplier = clientEnvironmentSupplier;
     this.fetcherFactory = fetcherFactory;
   }
 
@@ -26,7 +27,7 @@ public Registry getRegistryWithUrl(String url) throws URISyntaxException {
       case "http":
       case "https":
       case "file":
-        return new IndexRegistry(uri, httpDownloader, clientEnv, fetcherFactory);
+        return new IndexRegistry(uri, httpDownloader, clientEnvironmentSupplier.get(), fetcherFactory);
     }
     return null;
   }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/repo/BUILD

@@ -36,12 +36,17 @@ java_library(
     name = "repo_info_functions",
     srcs = [
         "BazelModuleRepoInfoFunction.java",
-        "ModuleRuleRepoInfoFunction.java",
         "LockFileParser.java",
+        "ModuleRuleRepoInfoFunction.java",
     ],
     deps = [
         ":repo_info_values",
         "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
+        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common",
+        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:fetch",
+        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:registry",
+        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
         "//src/main/java/com/google/devtools/build/lib/pkgcache",
         "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
         "//src/main/java/com/google/devtools/build/lib/vfs",