-
Notifications
You must be signed in to change notification settings - Fork 4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
A Starlark config file to allow controlling the vendored repos via:
- ignore('repo_name','repo_name', ....) function -->
Bazel will never vendor the repo or consider this directory while building in vendor mode.
- pin('repo_name','repo_name', ....) function -->
Bazel will pin the contents of this repo, will not update it while vendoring, and will always use it as is when building in vendor mode
Co-authored-by: Yun Peng <pcloudy@google.com>
PiperOrigin-RevId: 639000162
Change-Id: I278359f6e92eab290dac9cc4ef3136aa140cf8ba- Loading branch information
1 parent
683d47d
commit c1e32d2
Showing
14 changed files
with
539 additions
and
65 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
184 changes: 184 additions & 0 deletions
184
src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorFileFunction.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,184 @@ | ||
| // Copyright 2024 The Bazel Authors. All rights reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package com.google.devtools.build.lib.bazel.bzlmod; | ||
|
|
||
| import static java.nio.charset.StandardCharsets.UTF_8; | ||
|
|
||
| import com.google.common.collect.ImmutableList; | ||
| import com.google.devtools.build.lib.actions.FileValue; | ||
| import com.google.devtools.build.lib.cmdline.LabelConstants; | ||
| import com.google.devtools.build.lib.events.Event; | ||
| import com.google.devtools.build.lib.packages.BazelStarlarkEnvironment; | ||
| import com.google.devtools.build.lib.packages.DotBazelFileSyntaxChecker; | ||
| import com.google.devtools.build.lib.packages.VendorThreadContext; | ||
| import com.google.devtools.build.lib.skyframe.PrecomputedValue; | ||
| import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; | ||
| import com.google.devtools.build.lib.vfs.FileSystemUtils; | ||
| import com.google.devtools.build.lib.vfs.Path; | ||
| import com.google.devtools.build.lib.vfs.Root; | ||
| import com.google.devtools.build.lib.vfs.RootedPath; | ||
| import com.google.devtools.build.skyframe.SkyFunction; | ||
| import com.google.devtools.build.skyframe.SkyFunctionException; | ||
| import com.google.devtools.build.skyframe.SkyFunctionException.Transience; | ||
| import com.google.devtools.build.skyframe.SkyKey; | ||
| import com.google.devtools.build.skyframe.SkyValue; | ||
| import java.io.IOException; | ||
| import java.util.Optional; | ||
| import javax.annotation.Nullable; | ||
| import net.starlark.java.eval.EvalException; | ||
| import net.starlark.java.eval.Mutability; | ||
| import net.starlark.java.eval.Starlark; | ||
| import net.starlark.java.eval.StarlarkSemantics; | ||
| import net.starlark.java.eval.StarlarkThread; | ||
| import net.starlark.java.eval.SymbolGenerator; | ||
| import net.starlark.java.syntax.ParserInput; | ||
| import net.starlark.java.syntax.Program; | ||
| import net.starlark.java.syntax.StarlarkFile; | ||
| import net.starlark.java.syntax.SyntaxError; | ||
|
|
||
| /** | ||
| * The function to evaluate the VENDOR.bazel file under the vendor directory specified by the flag: | ||
| * --vendor_dir. | ||
| */ | ||
| public class VendorFileFunction implements SkyFunction { | ||
|
|
||
| public static final Precomputed<Optional<Path>> VENDOR_DIRECTORY = | ||
| new Precomputed<>("vendor_directory"); | ||
|
|
||
| private static final String VENDOR_FILE_HEADER = | ||
| """ | ||
| ############################################################################### | ||
| # This file is used to configure how external repositories are handled in vendor mode. | ||
| # ONLY the two following functions can be used: | ||
| # | ||
| # ignore('@@<canonical repo name>', ...) is used to completely ignore this repo from vendoring. | ||
| # Bazel will use the normal external cache and fetch process for this repo. | ||
| # | ||
| # pin('@@<canonical repo name>', ...) is used to pin the contents of this repo under the vendor | ||
| # directory as if there is a --override_repository flag for this repo. | ||
| # Note that Bazel will NOT update the vendored source for this repo while running vendor command | ||
| # unless it's unpinned. The user can modify and maintain the vendored source for this repo manually. | ||
| ############################################################################### | ||
| """; | ||
|
|
||
| private final BazelStarlarkEnvironment starlarkEnv; | ||
|
|
||
| public VendorFileFunction(BazelStarlarkEnvironment starlarkEnv) { | ||
| this.starlarkEnv = starlarkEnv; | ||
| } | ||
|
|
||
| @Nullable | ||
| @Override | ||
| public SkyValue compute(SkyKey skyKey, Environment env) | ||
| throws SkyFunctionException, InterruptedException { | ||
| if (VENDOR_DIRECTORY.get(env).isEmpty()) { | ||
| throw new VendorFileFunctionException( | ||
| new IllegalStateException( | ||
| "VENDOR.bazel file is not accessible with vendor mode off (without --vendor_dir" | ||
| + " flag)"), | ||
| Transience.PERSISTENT); | ||
| } | ||
|
|
||
| Path vendorPath = VENDOR_DIRECTORY.get(env).get(); | ||
| RootedPath vendorFilePath = | ||
| RootedPath.toRootedPath(Root.fromPath(vendorPath), LabelConstants.VENDOR_FILE_NAME); | ||
|
|
||
| FileValue vendorFileValue = (FileValue) env.getValue(FileValue.key(vendorFilePath)); | ||
| if (vendorFileValue == null) { | ||
| return null; | ||
| } | ||
| if (!vendorFileValue.exists()) { | ||
| createVendorFile(vendorPath, vendorFilePath.asPath()); | ||
| return VendorFileValue.create(ImmutableList.of(), ImmutableList.of()); | ||
| } | ||
|
|
||
| StarlarkSemantics starlarkSemantics = PrecomputedValue.STARLARK_SEMANTICS.get(env); | ||
| if (starlarkSemantics == null) { | ||
| return null; | ||
| } | ||
| VendorThreadContext context = | ||
| getVendorFileContext(env, skyKey, vendorFilePath.asPath(), starlarkSemantics); | ||
| return VendorFileValue.create(context.getIgnoredRepos(), context.getPinnedRepos()); | ||
| } | ||
|
|
||
| private VendorThreadContext getVendorFileContext( | ||
| Environment env, SkyKey skyKey, Path vendorFilePath, StarlarkSemantics starlarkSemantics) | ||
| throws VendorFileFunctionException, InterruptedException { | ||
| try (Mutability mu = Mutability.create("vendor file")) { | ||
| StarlarkFile vendorFile = readAndParseVendorFile(vendorFilePath, env); | ||
| new DotBazelFileSyntaxChecker("VENDOR.bazel files", /* canLoadBzl= */ false) | ||
| .check(vendorFile); | ||
| net.starlark.java.eval.Module predeclaredEnv = | ||
| net.starlark.java.eval.Module.withPredeclared( | ||
| starlarkSemantics, starlarkEnv.getStarlarkGlobals().getVendorToplevels()); | ||
| Program program = Program.compileFile(vendorFile, predeclaredEnv); | ||
| StarlarkThread thread = | ||
| StarlarkThread.create( | ||
| mu, starlarkSemantics, /* contextDescription= */ "", SymbolGenerator.create(skyKey)); | ||
| VendorThreadContext context = new VendorThreadContext(); | ||
| context.storeInThread(thread); | ||
| Starlark.execFileProgram(program, predeclaredEnv, thread); | ||
| return context; | ||
| } catch (SyntaxError.Exception | EvalException e) { | ||
| throw new VendorFileFunctionException( | ||
| new BadVendorFileException("error parsing VENDOR.bazel file: " + e.getMessage()), | ||
| Transience.PERSISTENT); | ||
| } | ||
| } | ||
|
|
||
| private void createVendorFile(Path vendorPath, Path vendorFilePath) | ||
| throws VendorFileFunctionException { | ||
| try { | ||
| vendorPath.createDirectoryAndParents(); | ||
| byte[] vendorFileContents = VENDOR_FILE_HEADER.getBytes(UTF_8); | ||
| FileSystemUtils.writeContent(vendorFilePath, vendorFileContents); | ||
| } catch (IOException e) { | ||
| throw new VendorFileFunctionException( | ||
| new IOException("error creating VENDOR.bazel file", e), Transience.TRANSIENT); | ||
| } | ||
| } | ||
|
|
||
| private static StarlarkFile readAndParseVendorFile(Path path, Environment env) | ||
| throws VendorFileFunctionException { | ||
| byte[] contents; | ||
| try { | ||
| contents = FileSystemUtils.readWithKnownFileSize(path, path.getFileSize()); | ||
| } catch (IOException e) { | ||
| throw new VendorFileFunctionException( | ||
| new IOException("error reading VENDOR.bazel file", e), Transience.TRANSIENT); | ||
| } | ||
| StarlarkFile starlarkFile = | ||
| StarlarkFile.parse(ParserInput.fromUTF8(contents, path.getPathString())); | ||
| if (!starlarkFile.ok()) { | ||
| Event.replayEventsOn(env.getListener(), starlarkFile.errors()); | ||
| throw new VendorFileFunctionException( | ||
| new BadVendorFileException("error parsing VENDOR.bazel file"), Transience.PERSISTENT); | ||
| } | ||
| return starlarkFile; | ||
| } | ||
|
|
||
| /** Thrown when something is wrong with the contents of the VENDOR.bazel file. */ | ||
| public static class BadVendorFileException extends Exception { | ||
| public BadVendorFileException(String message) { | ||
| super(message); | ||
| } | ||
| } | ||
|
|
||
| static class VendorFileFunctionException extends SkyFunctionException { | ||
| private VendorFileFunctionException(Exception e, Transience transience) { | ||
| super(e, transience); | ||
| } | ||
| } | ||
| } |
39 changes: 39 additions & 0 deletions
39
src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorFileValue.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| // Copyright 2024 The Bazel Authors. All rights reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package com.google.devtools.build.lib.bazel.bzlmod; | ||
|
|
||
| import com.google.auto.value.AutoValue; | ||
| import com.google.common.collect.ImmutableList; | ||
| import com.google.devtools.build.lib.cmdline.RepositoryName; | ||
| import com.google.devtools.build.lib.skyframe.SkyFunctions; | ||
| import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; | ||
| import com.google.devtools.build.skyframe.SkyKey; | ||
| import com.google.devtools.build.skyframe.SkyValue; | ||
|
|
||
| /** Represent the parsed VENDOR.bazel file */ | ||
| @AutoValue | ||
| public abstract class VendorFileValue implements SkyValue { | ||
|
|
||
| @SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.VENDOR_FILE; | ||
|
|
||
| public abstract ImmutableList<RepositoryName> getIgnoredRepos(); | ||
|
|
||
| public abstract ImmutableList<RepositoryName> getPinnedRepos(); | ||
|
|
||
| public static VendorFileValue create( | ||
| ImmutableList<RepositoryName> ignoredRepos, ImmutableList<RepositoryName> pinnedRepos) { | ||
| return new AutoValue_VendorFileValue(ignoredRepos, pinnedRepos); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.