From 9296068be5e3808eb03a3b61f3af3a2c88f7ab7d Mon Sep 17 00:00:00 2001 From: Yannic Bonenberger Date: Wed, 2 Nov 2022 06:12:59 -0700 Subject: [PATCH] [remote/downloader] Don't include headers in `FetchBlobRequest` Including the headers in the request is very inefficient as credentials should never change the content of the downloaded archive. In fact, given that Bazel verifies the checksum of the downloaded file, the credentials cannot possibly used in a way where they influence the outcome of the download (other than deciding whether or not the user is allowed to download the blob at all). Hence, the credentials should not be included in the request. Users that need to send credentials to the remote downloader should do so by passing the credentials as metadata to the gRPC call. Note that the remote downloader is behind an experimental flag, so this change does not need to go thorugh the incompatible change process. Closes #16595. PiperOrigin-RevId: 485576157 Change-Id: I8afc7c818e4eed63ac1f70c3e4c2115a1d365830 --- .../build/lib/remote/downloader/BUILD | 1 - .../downloader/GrpcRemoteDownloader.java | 65 +--------------- .../downloader/GrpcRemoteDownloaderTest.java | 78 +------------------ 3 files changed, 3 insertions(+), 141 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/downloader/BUILD b/src/main/java/com/google/devtools/build/lib/remote/downloader/BUILD index d980d2a6e0bc2c..1804656458feb5 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/downloader/BUILD +++ b/src/main/java/com/google/devtools/build/lib/remote/downloader/BUILD @@ -22,7 +22,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib/remote/options", "//src/main/java/com/google/devtools/build/lib/remote/util", "//src/main/java/com/google/devtools/build/lib/vfs", - "//third_party:gson", "//third_party:guava", "//third_party:jsr305", "//third_party/grpc-java:grpc-jar", diff --git a/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java b/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java index d94efdc0c60899..be7f5411ed8573 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java +++ b/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java @@ -23,8 +23,6 @@ import build.bazel.remote.execution.v2.RequestMetadata; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.bazel.repository.downloader.Downloader; import com.google.devtools.build.lib.bazel.repository.downloader.HashOutputStream; @@ -38,9 +36,6 @@ import com.google.devtools.build.lib.remote.util.TracingMetadataUtils; import com.google.devtools.build.lib.remote.util.Utils; import com.google.devtools.build.lib.vfs.Path; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; import io.grpc.CallCredentials; import io.grpc.Channel; import io.grpc.StatusRuntimeException; @@ -51,7 +46,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import javax.annotation.Nullable; @@ -82,7 +76,6 @@ public class GrpcRemoteDownloader implements AutoCloseable, Downloader { // supported by Bazel. private static final String QUALIFIER_CHECKSUM_SRI = "checksum.sri"; private static final String QUALIFIER_CANONICAL_ID = "bazel.canonical_id"; - private static final String QUALIFIER_AUTH_HEADERS = "bazel.auth_headers"; public GrpcRemoteDownloader( String buildRequestId, @@ -131,13 +124,7 @@ public void download( RemoteActionExecutionContext.create(metadata); final FetchBlobRequest request = - newFetchBlobRequest( - options.remoteInstanceName, - urls, - authHeaders, - checksum, - canonicalId, - options.remoteDownloaderSendAllHeaders); + newFetchBlobRequest(options.remoteInstanceName, urls, checksum, canonicalId); try { FetchBlobResponse response = retrier.execute( @@ -175,10 +162,8 @@ public void download( static FetchBlobRequest newFetchBlobRequest( String instanceName, List urls, - Map>> authHeaders, com.google.common.base.Optional checksum, - String canonicalId, - boolean includeAllHeaders) { + String canonicalId) { FetchBlobRequest.Builder requestBuilder = FetchBlobRequest.newBuilder().setInstanceName(instanceName); for (URL url : urls) { @@ -195,13 +180,6 @@ static FetchBlobRequest newFetchBlobRequest( requestBuilder.addQualifiers( Qualifier.newBuilder().setName(QUALIFIER_CANONICAL_ID).setValue(canonicalId).build()); } - if (!authHeaders.isEmpty()) { - requestBuilder.addQualifiers( - Qualifier.newBuilder() - .setName(QUALIFIER_AUTH_HEADERS) - .setValue(authHeadersJson(urls, authHeaders, includeAllHeaders)) - .build()); - } return requestBuilder.build(); } @@ -224,43 +202,4 @@ private OutputStream newOutputStream( } return out; } - - private static String authHeadersJson( - List urls, Map>> authHeaders, boolean includeAllHeaders) { - ImmutableSet hostSet = - urls.stream().map(URL::getHost).collect(ImmutableSet.toImmutableSet()); - Map subObjects = new TreeMap<>(); - for (Map.Entry>> entry : authHeaders.entrySet()) { - URI uri = entry.getKey(); - // Only add headers that are relevant to the hosts. - if (!hostSet.contains(uri.getHost())) { - continue; - } - - JsonObject subObject = new JsonObject(); - Map> orderedHeaders = new TreeMap<>(entry.getValue()); - for (Map.Entry> subEntry : orderedHeaders.entrySet()) { - if (includeAllHeaders) { - JsonArray values = new JsonArray(subEntry.getValue().size()); - for (String value : subEntry.getValue()) { - values.add(value); - } - subObject.add(subEntry.getKey(), values); - } else { - String value = Iterables.getFirst(subEntry.getValue(), null); - if (value != null) { - subObject.addProperty(subEntry.getKey(), value); - } - } - } - subObjects.put(uri.toString(), subObject); - } - - JsonObject authHeadersJson = new JsonObject(); - for (Map.Entry entry : subObjects.entrySet()) { - authHeadersJson.add(entry.getKey(), entry.getValue()); - } - - return new Gson().toJson(authHeadersJson); - } } diff --git a/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java b/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java index 9ca60e4710bbae..c08093d8b4f081 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java @@ -350,82 +350,10 @@ public void testFetchBlobRequest() throws Exception { new URL("http://example.com/a"), new URL("http://example.com/b"), new URL("file:/not/limited/to/http")), - ImmutableMap.of( - new URI("http://example.com"), - ImmutableMap.of( - "Some-Header", ImmutableList.of("some header content"), - "Another-Header", - ImmutableList.of("another header content", "even more header content")), - new URI("http://example.org"), - ImmutableMap.of( - "Org-Header", - ImmutableList.of("org header content", "and a second one", "and a third one"))), com.google.common.base.Optional.of( Checksum.fromSubresourceIntegrity( "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")), - "canonical ID", - /* includeAllHeaders= */ false); - - final String expectedAuthHeadersJson = - "{" - + "\"http://example.com\":{" - + "\"Another-Header\":\"another header content\"," - + "\"Some-Header\":\"some header content\"" - + "}" - + "}"; - - assertThat(request) - .isEqualTo( - FetchBlobRequest.newBuilder() - .setInstanceName("instance name") - .addUris("http://example.com/a") - .addUris("http://example.com/b") - .addUris("file:/not/limited/to/http") - .addQualifiers( - Qualifier.newBuilder() - .setName("checksum.sri") - .setValue("sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")) - .addQualifiers( - Qualifier.newBuilder().setName("bazel.canonical_id").setValue("canonical ID")) - .addQualifiers( - Qualifier.newBuilder() - .setName("bazel.auth_headers") - .setValue(expectedAuthHeadersJson)) - .build()); - } - - @Test - public void testFetchBlobRequestWithAllHeaders() throws Exception { - FetchBlobRequest request = - GrpcRemoteDownloader.newFetchBlobRequest( - "instance name", - ImmutableList.of( - new URL("http://example.com/a"), - new URL("http://example.com/b"), - new URL("file:/not/limited/to/http")), - ImmutableMap.of( - new URI("http://example.com"), - ImmutableMap.of( - "Some-Header", ImmutableList.of("some header content"), - "Another-Header", - ImmutableList.of("another header content", "even more header content")), - new URI("http://example.org"), - ImmutableMap.of( - "Org-Header", - ImmutableList.of("org header content", "and a second one", "and a third one"))), - com.google.common.base.Optional.of( - Checksum.fromSubresourceIntegrity( - "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")), - "canonical ID", - /* includeAllHeaders= */ true); - - final String expectedAuthHeadersJson = - "{" - + "\"http://example.com\":{" - + "\"Another-Header\":[\"another header content\",\"even more header content\"]," - + "\"Some-Header\":[\"some header content\"]" - + "}" - + "}"; + "canonical ID"); assertThat(request) .isEqualTo( @@ -440,10 +368,6 @@ public void testFetchBlobRequestWithAllHeaders() throws Exception { .setValue("sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")) .addQualifiers( Qualifier.newBuilder().setName("bazel.canonical_id").setValue("canonical ID")) - .addQualifiers( - Qualifier.newBuilder() - .setName("bazel.auth_headers") - .setValue(expectedAuthHeadersJson)) .build()); } }