[remote/downloader] Don't include headers in FetchBlobRequest

Including the headers in the request is very inefficient as credentials should never change the content of the downloaded archive. In fact, given that Bazel verifies the checksum of the downloaded file, the credentials cannot possibly used in a way where they influence the outcome of the download (other than deciding whether or not the user is allowed to download the blob at all). Hence, the credentials should not be included in the request.

Users that need to send credentials to the remote downloader should do so by passing the credentials as metadata to the gRPC call.

Note that the remote downloader is behind an experimental flag, so this change does not need to go thorugh the incompatible change process.

Closes bazelbuild#16595.

PiperOrigin-RevId: 485576157
Change-Id: I8afc7c818e4eed63ac1f70c3e4c2115a1d365830

Author: Yannic

src/main/java/com/google/devtools/build/lib/remote/downloader/BUILD

@@ -22,7 +22,6 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/remote/options",
         "//src/main/java/com/google/devtools/build/lib/remote/util",
         "//src/main/java/com/google/devtools/build/lib/vfs",
-        "//third_party:gson",
         "//third_party:guava",
         "//third_party:jsr305",
         "//third_party/grpc-java:grpc-jar",

src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java

@@ -23,8 +23,6 @@
 import build.bazel.remote.execution.v2.RequestMetadata;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
 import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
 import com.google.devtools.build.lib.bazel.repository.downloader.Downloader;
 import com.google.devtools.build.lib.bazel.repository.downloader.HashOutputStream;
@@ -38,9 +36,6 @@
 import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
 import com.google.devtools.build.lib.remote.util.Utils;
 import com.google.devtools.build.lib.vfs.Path;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonObject;
 import io.grpc.CallCredentials;
 import io.grpc.Channel;
 import io.grpc.StatusRuntimeException;
@@ -51,7 +46,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import javax.annotation.Nullable;
@@ -82,7 +76,6 @@ public class GrpcRemoteDownloader implements AutoCloseable, Downloader {
   // supported by Bazel.
   private static final String QUALIFIER_CHECKSUM_SRI = "checksum.sri";
   private static final String QUALIFIER_CANONICAL_ID = "bazel.canonical_id";
-  private static final String QUALIFIER_AUTH_HEADERS = "bazel.auth_headers";
 
   public GrpcRemoteDownloader(
       String buildRequestId,
@@ -131,13 +124,7 @@ public void download(
         RemoteActionExecutionContext.create(metadata);
 
     final FetchBlobRequest request =
-        newFetchBlobRequest(
-            options.remoteInstanceName,
-            urls,
-            authHeaders,
-            checksum,
-            canonicalId,
-            options.remoteDownloaderSendAllHeaders);
+        newFetchBlobRequest(options.remoteInstanceName, urls, checksum, canonicalId);
     try {
       FetchBlobResponse response =
           retrier.execute(
@@ -175,10 +162,8 @@ public void download(
   static FetchBlobRequest newFetchBlobRequest(
       String instanceName,
       List<URL> urls,
-      Map<URI, Map<String, List<String>>> authHeaders,
       com.google.common.base.Optional<Checksum> checksum,
-      String canonicalId,
-      boolean includeAllHeaders) {
+      String canonicalId) {
     FetchBlobRequest.Builder requestBuilder =
         FetchBlobRequest.newBuilder().setInstanceName(instanceName);
     for (URL url : urls) {
@@ -195,13 +180,6 @@ static FetchBlobRequest newFetchBlobRequest(
       requestBuilder.addQualifiers(
           Qualifier.newBuilder().setName(QUALIFIER_CANONICAL_ID).setValue(canonicalId).build());
     }
-    if (!authHeaders.isEmpty()) {
-      requestBuilder.addQualifiers(
-          Qualifier.newBuilder()
-              .setName(QUALIFIER_AUTH_HEADERS)
-              .setValue(authHeadersJson(urls, authHeaders, includeAllHeaders))
-              .build());
-    }
 
     return requestBuilder.build();
   }
@@ -224,43 +202,4 @@ private OutputStream newOutputStream(
     }
     return out;
   }
-
-  private static String authHeadersJson(
-      List<URL> urls, Map<URI, Map<String, List<String>>> authHeaders, boolean includeAllHeaders) {
-    ImmutableSet<String> hostSet =
-        urls.stream().map(URL::getHost).collect(ImmutableSet.toImmutableSet());
-    Map<String, JsonObject> subObjects = new TreeMap<>();
-    for (Map.Entry<URI, Map<String, List<String>>> entry : authHeaders.entrySet()) {
-      URI uri = entry.getKey();
-      // Only add headers that are relevant to the hosts.
-      if (!hostSet.contains(uri.getHost())) {
-        continue;
-      }
-
-      JsonObject subObject = new JsonObject();
-      Map<String, List<String>> orderedHeaders = new TreeMap<>(entry.getValue());
-      for (Map.Entry<String, List<String>> subEntry : orderedHeaders.entrySet()) {
-        if (includeAllHeaders) {
-          JsonArray values = new JsonArray(subEntry.getValue().size());
-          for (String value : subEntry.getValue()) {
-            values.add(value);
-          }
-          subObject.add(subEntry.getKey(), values);
-        } else {
-          String value = Iterables.getFirst(subEntry.getValue(), null);
-          if (value != null) {
-            subObject.addProperty(subEntry.getKey(), value);
-          }
-        }
-      }
-      subObjects.put(uri.toString(), subObject);
-    }
-
-    JsonObject authHeadersJson = new JsonObject();
-    for (Map.Entry<String, JsonObject> entry : subObjects.entrySet()) {
-      authHeadersJson.add(entry.getKey(), entry.getValue());
-    }
-
-    return new Gson().toJson(authHeadersJson);
-  }
 }

src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java

@@ -350,82 +350,10 @@ public void testFetchBlobRequest() throws Exception {
                 new URL("http://example.com/a"),
                 new URL("http://example.com/b"),
                 new URL("file:/not/limited/to/http")),
-            ImmutableMap.of(
-                new URI("http://example.com"),
-                ImmutableMap.of(
-                    "Some-Header", ImmutableList.of("some header content"),
-                    "Another-Header",
-                        ImmutableList.of("another header content", "even more header content")),
-                new URI("http://example.org"),
-                ImmutableMap.of(
-                    "Org-Header",
-                    ImmutableList.of("org header content", "and a second one", "and a third one"))),
             com.google.common.base.Optional.<Checksum>of(
                 Checksum.fromSubresourceIntegrity(
                     "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")),
-            "canonical ID",
-            /* includeAllHeaders= */ false);
-
-    final String expectedAuthHeadersJson =
-        "{"
-            + "\"http://example.com\":{"
-            + "\"Another-Header\":\"another header content\","
-            + "\"Some-Header\":\"some header content\""
-            + "}"
-            + "}";
-
-    assertThat(request)
-        .isEqualTo(
-            FetchBlobRequest.newBuilder()
-                .setInstanceName("instance name")
-                .addUris("http://example.com/a")
-                .addUris("http://example.com/b")
-                .addUris("file:/not/limited/to/http")
-                .addQualifiers(
-                    Qualifier.newBuilder()
-                        .setName("checksum.sri")
-                        .setValue("sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="))
-                .addQualifiers(
-                    Qualifier.newBuilder().setName("bazel.canonical_id").setValue("canonical ID"))
-                .addQualifiers(
-                    Qualifier.newBuilder()
-                        .setName("bazel.auth_headers")
-                        .setValue(expectedAuthHeadersJson))
-                .build());
-  }
-
-  @Test
-  public void testFetchBlobRequestWithAllHeaders() throws Exception {
-    FetchBlobRequest request =
-        GrpcRemoteDownloader.newFetchBlobRequest(
-            "instance name",
-            ImmutableList.of(
-                new URL("http://example.com/a"),
-                new URL("http://example.com/b"),
-                new URL("file:/not/limited/to/http")),
-            ImmutableMap.of(
-                new URI("http://example.com"),
-                ImmutableMap.of(
-                    "Some-Header", ImmutableList.of("some header content"),
-                    "Another-Header",
-                        ImmutableList.of("another header content", "even more header content")),
-                new URI("http://example.org"),
-                ImmutableMap.of(
-                    "Org-Header",
-                    ImmutableList.of("org header content", "and a second one", "and a third one"))),
-            com.google.common.base.Optional.<Checksum>of(
-                Checksum.fromSubresourceIntegrity(
-                    "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")),
-            "canonical ID",
-            /* includeAllHeaders= */ true);
-
-    final String expectedAuthHeadersJson =
-        "{"
-            + "\"http://example.com\":{"
-            + "\"Another-Header\":[\"another header content\",\"even more header content\"],"
-            + "\"Some-Header\":[\"some header content\"]"
-            + "}"
-            + "}";
+            "canonical ID");
 
     assertThat(request)
         .isEqualTo(
@@ -440,10 +368,6 @@ public void testFetchBlobRequestWithAllHeaders() throws Exception {
                         .setValue("sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="))
                 .addQualifiers(
                     Qualifier.newBuilder().setName("bazel.canonical_id").setValue("canonical ID"))
-                .addQualifiers(
-                    Qualifier.newBuilder()
-                        .setName("bazel.auth_headers")
-                        .setValue(expectedAuthHeadersJson))
                 .build());
   }
 }