From f54fe07209acc25340df8d2e02993b1add2deafa Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Tue, 2 Feb 2021 23:32:52 -0800 Subject: [PATCH] Add --experimental_repository_disable_download to allow users disable download for external repos This PR adds a flag `--experimental_repository_disable_download` which when set will prevent Bazel from downloading external repositories. However, `local_repository`, `new_local_repository` and external repositories already downloaded in the repository cache would still work. Closes #12940. PiperOrigin-RevId: 355332927 --- .../lib/bazel/BazelRepositoryModule.java | 2 + .../bazel/repository/RepositoryOptions.java | 9 ++ .../downloader/DownloadManager.java | 10 +++ .../shell/bazel/starlark_repository_test.sh | 88 +++++++++++++++++++ 4 files changed, 109 insertions(+) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 48c7f60466cb55..422cb5af577b60 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -237,6 +237,8 @@ public void beforeCommand(CommandEnvironment env) { RepositoryOptions repoOptions = env.getOptions().getOptions(RepositoryOptions.class); if (repoOptions != null) { + downloadManager.setDisableDownload(repoOptions.disableDownload); + repositoryCache.setHardlink(repoOptions.useHardlinks); if (repoOptions.experimentalScaleTimeouts > 0.0) { starlarkRepositoryFunction.setTimeoutScaling(repoOptions.experimentalScaleTimeouts); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java index 105a0c77583ce9..183f32dd610c9e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java @@ -56,6 +56,15 @@ public class RepositoryOptions extends OptionsBase { + " cache hit, rather than copying. This is inteded to save disk space.") public boolean useHardlinks; + @Option( + name = "experimental_repository_disable_download", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.BAZEL_CLIENT_OPTIONS, + effectTags = {OptionEffectTag.UNKNOWN}, + metadataTags = {OptionMetadataTag.EXPERIMENTAL}, + help = "If set, downloading external repositories is not allowed.") + public boolean disableDownload; + @Option( name = "distdir", oldName = "experimental_distdir", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java index 26243acecdbb04..a117b7d2b268c3 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java @@ -46,6 +46,7 @@ public class DownloadManager { private List distdir = ImmutableList.of(); private UrlRewriter rewriter; private final Downloader downloader; + private boolean disableDownload = false; public DownloadManager(RepositoryCache repositoryCache, Downloader downloader) { this.repositoryCache = repositoryCache; @@ -60,6 +61,10 @@ public void setUrlRewriter(UrlRewriter rewriter) { this.rewriter = rewriter; } + public void setDisableDownload(boolean disableDownload) { + this.disableDownload = disableDownload; + } + /** * Downloads file to disk and returns path. * @@ -194,6 +199,11 @@ public Path download( } } + if (disableDownload) { + throw new IOException( + String.format("Failed to download repo %s: download is disabled.", repo)); + } + try { downloader.download( urls, authHeaders, checksum, canonicalId, destination, eventHandler, clientEnv, type); diff --git a/src/test/shell/bazel/starlark_repository_test.sh b/src/test/shell/bazel/starlark_repository_test.sh index 4829da32da583e..e5ef96782993e4 100755 --- a/src/test/shell/bazel/starlark_repository_test.sh +++ b/src/test/shell/bazel/starlark_repository_test.sh @@ -2117,4 +2117,92 @@ EOF || fail "Expected success despite needing a file behind basic auth" } +function test_disable_download_should_prevent_downloading() { + mkdir x + echo 'exports_files(["file.txt"])' > x/BUILD + echo 'Hello World' > x/file.txt + tar cvf x.tar x + sha256=$(sha256sum x.tar | head -c 64) + serve_file x.tar + + mkdir main + cd main + cat > WORKSPACE < BUILD <<'EOF' +genrule( + name = "it", + srcs = ["@ext//x:file.txt"], + outs = ["it.txt"], + cmd = "cp $< $@", +) +EOF + + bazel build --experimental_repository_disable_download //:it > "${TEST_log}" 2>&1 \ + && fail "Expected failure" || : + expect_log "Failed to download repo ext: download is disabled" +} + +function test_disable_download_should_allow_distdir() { + mkdir x + echo 'exports_files(["file.txt"])' > x/BUILD + echo 'Hello World' > x/file.txt + tar cvf x.tar x + sha256=$(sha256sum x.tar | head -c 64) + + mkdir main + cp x.tar main + cd main + cat > WORKSPACE < BUILD <<'EOF' +genrule( + name = "it", + srcs = ["@ext//x:file.txt"], + outs = ["it.txt"], + cmd = "cp $< $@", +) +EOF + + bazel build --distdir="." --experimental_repository_disable_download //:it || fail "Failed to build" +} + +function test_disable_download_should_allow_local_repository() { + mkdir x + echo 'exports_files(["file.txt"])' > x/BUILD + echo 'Hello World' > x/file.txt + touch x/WORKSPACE + + mkdir main + cd main + cat > WORKSPACE < BUILD <<'EOF' +genrule( + name = "it", + srcs = ["@ext//:file.txt"], + outs = ["it.txt"], + cmd = "cp $< $@", +) +EOF + + bazel build --experimental_repository_disable_download //:it || fail "Failed to build" +} + run_suite "local repository tests"