UrlRewriter should be able to load credentials from .netrc (#14834)

Addresses #13111

Closes #14066.

PiperOrigin-RevId: 424854105
(cherry picked from commit 1e53b1f)

Co-authored-by: Denys Kurylenko <[email protected]>

Author: brentleyjones

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -325,7 +325,10 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
       try {
         UrlRewriter rewriter =
             UrlRewriter.getDownloaderUrlRewriter(
-                repoOptions == null ? null : repoOptions.downloaderConfig, env.getReporter());
+                repoOptions == null ? null : repoOptions.downloaderConfig,
+                env.getReporter(),
+                env.getClientEnv(),
+                env.getRuntime().getFileSystem());
         downloadManager.setUrlRewriter(rewriter);
       } catch (UrlRewriterParseException e) {
         // It's important that the build stops ASAP, because this config file may be required for

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD

@@ -13,16 +13,20 @@ java_library(
     srcs = glob(["*.java"]),
     deps = [
         "//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info",
+        "//src/main/java/com/google/devtools/build/lib/authandtls",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache:events",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream",
         "//src/main/java/com/google/devtools/build/lib/clock",
         "//src/main/java/com/google/devtools/build/lib/concurrent",
         "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/util",
+        "//src/main/java/com/google/devtools/build/lib/util:os",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//src/main/java/net/starlark/java/syntax",
+        "//third_party:auth",
+        "//third_party:auto_value",
         "//third_party:guava",
         "//third_party:jsr305",
     ],

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java

@@ -15,6 +15,7 @@
 package com.google.devtools.build.lib.bazel.repository.downloader;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableList.toImmutableList;
 
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Optional;
@@ -108,12 +109,14 @@ public Path download(
       throw new InterruptedException();
     }
 
-    List<URL> rewrittenUrls = originalUrls;
+    ImmutableList<URL> rewrittenUrls = ImmutableList.copyOf(originalUrls);
     Map<URI, Map<String, String>> rewrittenAuthHeaders = authHeaders;
 
     if (rewriter != null) {
-      rewrittenUrls = rewriter.amend(originalUrls);
-      rewrittenAuthHeaders = rewriter.updateAuthHeaders(rewrittenUrls, authHeaders);
+      ImmutableList<UrlRewriter.RewrittenURL> rewrittenUrlMappings = rewriter.amend(originalUrls);
+      rewrittenUrls =
+          rewrittenUrlMappings.stream().map(url -> url.url()).collect(toImmutableList());
+      rewrittenAuthHeaders = rewriter.updateAuthHeaders(rewrittenUrlMappings, authHeaders);
     }
 
     URL mainUrl; // The "main" URL for this request

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/UrlRewriter.java

@@ -16,15 +16,23 @@
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static java.nio.charset.StandardCharsets.ISO_8859_1;
 
+import com.google.auth.Credentials;
+import com.google.auto.value.AutoValue;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Ascii;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.authandtls.Netrc;
+import com.google.devtools.build.lib.authandtls.NetrcCredentials;
+import com.google.devtools.build.lib.authandtls.NetrcParser;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.Reader;
@@ -38,14 +46,17 @@
 import java.nio.file.Paths;
 import java.util.Base64;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import javax.annotation.Nullable;
+import net.starlark.java.syntax.Location;
 
 /**
  * Helper class for taking URLs and converting them according to an optional config specified by
@@ -59,13 +70,19 @@ public class UrlRewriter {
   private static final ImmutableSet<String> REWRITABLE_SCHEMES = ImmutableSet.of("http", "https");
 
   private final UrlRewriterConfig config;
-  private final Function<URL, List<URL>> rewriter;
+  private final Function<URL, List<RewrittenURL>> rewriter;
+  @Nullable private final Credentials netrcCreds;
 
   @VisibleForTesting
-  UrlRewriter(Consumer<String> log, String filePathForErrorReporting, Reader reader)
+  UrlRewriter(
+      Consumer<String> log,
+      String filePathForErrorReporting,
+      Reader reader,
+      @Nullable Credentials netrcCreds)
       throws UrlRewriterParseException {
     Preconditions.checkNotNull(reader, "UrlRewriterConfig source must be set");
     this.config = new UrlRewriterConfig(filePathForErrorReporting, reader);
+    this.netrcCreds = netrcCreds;
 
     this.rewriter = this::rewrite;
   }
@@ -75,89 +92,124 @@ public class UrlRewriter {
    *
    * @param configPath Path to the config file to use. May be null.
    * @param reporter Used for logging when URLs are rewritten.
+   * @param clientEnv a map of the current Bazel command's environment
+   * @param fileSystem the Blaze file system
    */
-  public static UrlRewriter getDownloaderUrlRewriter(String configPath, Reporter reporter)
+  public static UrlRewriter getDownloaderUrlRewriter(
+      String configPath,
+      Reporter reporter,
+      ImmutableMap<String, String> clientEnv,
+      FileSystem fileSystem)
       throws UrlRewriterParseException {
     Consumer<String> log = str -> reporter.handle(Event.info(str));
 
+    // "empty" UrlRewriter shouldn't alter auth headers
     if (Strings.isNullOrEmpty(configPath)) {
-      return new UrlRewriter(log, "", new StringReader(""));
+      return new UrlRewriter(log, "", new StringReader(""), null);
     }
 
+    Credentials creds = null;
+    try {
+      creds = newCredentialsFromNetrc(clientEnv, fileSystem);
+    } catch (UrlRewriterParseException e) {
+      // If the credentials extraction failed, we're letting bazel try without credentials.
+    }
     try (BufferedReader reader = Files.newBufferedReader(Paths.get(configPath))) {
-      return new UrlRewriter(log, configPath, reader);
+      return new UrlRewriter(log, configPath, reader, creds);
     } catch (IOException e) {
       throw new UncheckedIOException(e);
     }
   }
 
   /**
    * Rewrites {@code urls} using the configuration provided to {@link
-   * #getDownloaderUrlRewriter(String, Reporter)}. The returned list of URLs may be empty if the
-   * configuration used blocks all the input URLs.
+   * #getDownloaderUrlRewriter(String, Reporter, ImmutableMap, FileSystem)}. The returned list of
+   * URLs may be empty if the configuration used blocks all the input URLs.
    *
    * @param urls The input list of {@link URL}s. May be empty.
    * @return The amended lists of URLs.
    */
-  public List<URL> amend(List<URL> urls) {
+  public ImmutableList<RewrittenURL> amend(List<URL> urls) {
     Objects.requireNonNull(urls, "URLS to check must be set but may be empty");
 
-    ImmutableList<URL> rewritten =
-        urls.stream().map(rewriter).flatMap(Collection::stream).collect(toImmutableList());
-
-    return rewritten;
+    return urls.stream().map(rewriter).flatMap(Collection::stream).collect(toImmutableList());
   }
 
   /**
-   * Updates {@code authHeaders} using the userInfo available in the provided {@code urls}.
+   * Updates {@code authHeaders} using the userInfo available in the provided {@code urls}. Note
+   * that if the same url is present in both {@code authHeaders} and <b>download config</b> then it
+   * will be overridden with the value from <b>download config</b>.
    *
    * @param urls The input list of {@link URL}s. May be empty.
    * @param authHeaders A map of the URLs and their corresponding auth tokens.
    * @return A map of the updated authentication headers.
    */
   public Map<URI, Map<String, String>> updateAuthHeaders(
-      List<URL> urls, Map<URI, Map<String, String>> authHeaders) {
-    ImmutableMap.Builder<URI, Map<String, String>> authHeadersBuilder =
-        ImmutableMap.<URI, Map<String, String>>builder().putAll(authHeaders);
+      List<RewrittenURL> urls, Map<URI, Map<String, String>> authHeaders) {
+    Map<URI, Map<String, String>> updatedAuthHeaders = new HashMap<>(authHeaders);
 
-    for (URL url : urls) {
-      String userInfo = url.getUserInfo();
+    for (RewrittenURL url : urls) {
+      // if URL was not re-written by UrlRewriter in first place, we should not attach auth headers
+      // to it
+      if (!url.rewritten()) {
+        continue;
+      }
+
+      String userInfo = url.url().getUserInfo();
       if (userInfo != null) {
         try {
           String token =
               "Basic " + Base64.getEncoder().encodeToString(userInfo.getBytes(ISO_8859_1));
-          authHeadersBuilder.put(url.toURI(), ImmutableMap.of("Authorization", token));
+          updatedAuthHeaders.put(url.url().toURI(), ImmutableMap.of("Authorization", token));
         } catch (URISyntaxException e) {
           // If the credentials extraction failed, we're letting bazel try without credentials.
         }
+      } else if (this.netrcCreds != null) {
+        try {
+          Map<String, List<String>> urlAuthHeaders =
+              this.netrcCreds.getRequestMetadata(url.url().toURI());
+          if (urlAuthHeaders == null || urlAuthHeaders.isEmpty()) {
+            continue;
+          }
+          // there could be multiple Auth headers, take the first one
+          Map.Entry<String, List<String>> firstAuthHeader =
+              urlAuthHeaders.entrySet().stream().findFirst().get();
+          if (firstAuthHeader.getValue() != null && !firstAuthHeader.getValue().isEmpty()) {
+            updatedAuthHeaders.put(
+                url.url().toURI(),
+                ImmutableMap.of(firstAuthHeader.getKey(), firstAuthHeader.getValue().get(0)));
+          }
+        } catch (URISyntaxException | IOException e) {
+          // If the credentials extraction failed, we're letting bazel try without credentials.
+        }
       }
     }
 
-    return authHeadersBuilder.build();
+    return ImmutableMap.copyOf(updatedAuthHeaders);
   }
 
-  private ImmutableList<URL> rewrite(URL url) {
+  private ImmutableList<RewrittenURL> rewrite(URL url) {
     Preconditions.checkNotNull(url);
 
     // Cowardly refuse to rewrite non-HTTP(S) urls
     if (REWRITABLE_SCHEMES.stream()
         .noneMatch(scheme -> Ascii.equalsIgnoreCase(scheme, url.getProtocol()))) {
-      return ImmutableList.of(url);
+      return ImmutableList.of(RewrittenURL.create(url, false));
     }
 
-    List<URL> rewrittenUrls = applyRewriteRules(url);
+    ImmutableList<RewrittenURL> rewrittenUrls = applyRewriteRules(url);
 
-    ImmutableList.Builder<URL> toReturn = ImmutableList.builder();
+    ImmutableList.Builder<RewrittenURL> toReturn = ImmutableList.builder();
     // Now iterate over the URLs
-    for (URL consider : rewrittenUrls) {
+    for (RewrittenURL consider : rewrittenUrls) {
       // If there's an allow entry, add it to the set to return and continue
-      if (isAllowMatched(consider)) {
+      if (isAllowMatched(consider.url())) {
         toReturn.add(consider);
         continue;
       }
 
       // If there's no block that matches the domain, add it to the set to return and continue
-      if (!isBlockMatched(consider)) {
+      if (!isBlockMatched(consider.url())) {
         toReturn.add(consider);
       }
     }
@@ -192,7 +244,7 @@ private static boolean isMatchingHostName(URL url, String host) {
     return host.equals(url.getHost()) || url.getHost().endsWith("." + host);
   }
 
-  private ImmutableList<URL> applyRewriteRules(URL url) {
+  private ImmutableList<RewrittenURL> applyRewriteRules(URL url) {
     String withoutScheme = url.toString().substring(url.getProtocol().length() + 3);
 
     ImmutableSet.Builder<String> rewrittenUrls = ImmutableSet.builder();
@@ -210,11 +262,12 @@ private ImmutableList<URL> applyRewriteRules(URL url) {
     }
 
     if (!matchMade) {
-      return ImmutableList.of(url);
+      return ImmutableList.of(RewrittenURL.create(url, false));
     }
 
     return rewrittenUrls.build().stream()
         .map(urlString -> prefixWithProtocol(urlString, url.getProtocol()))
+        .map(plainUrl -> RewrittenURL.create(plainUrl, true))
         .collect(toImmutableList());
   }
 
@@ -232,8 +285,64 @@ private static URL prefixWithProtocol(String url, String protocol) {
     }
   }
 
+  /**
+   * Create a new {@link Credentials} object by parsing the .netrc file with following order to
+   * search it:
+   *
+   * <ol>
+   *   <li>If environment variable $NETRC exists, use it as the path to the .netrc file
+   *   <li>Fallback to $HOME/.netrc or $USERPROFILE/.netrc
+   * </ol>
+   *
+   * @return the {@link Credentials} object or {@code null} if there is no .netrc file.
+   * @throws UrlRewriterParseException in case the credentials can't be constructed.
+   */
+  // TODO : consider re-using RemoteModule.newCredentialsFromNetrc
+  @VisibleForTesting
+  static Credentials newCredentialsFromNetrc(Map<String, String> clientEnv, FileSystem fileSystem)
+      throws UrlRewriterParseException {
+    final Optional<String> homeDir;
+    if (OS.getCurrent() == OS.WINDOWS) {
+      homeDir = Optional.ofNullable(clientEnv.get("USERPROFILE"));
+    } else {
+      homeDir = Optional.ofNullable(clientEnv.get("HOME"));
+    }
+    String netrcFileString =
+        Optional.ofNullable(clientEnv.get("NETRC"))
+            .orElseGet(() -> homeDir.map(home -> home + "/.netrc").orElse(null));
+    if (netrcFileString == null) {
+      return null;
+    }
+    Location location = Location.fromFileLineColumn(netrcFileString, 0, 0);
+
+    Path netrcFile = fileSystem.getPath(netrcFileString);
+    if (netrcFile.exists()) {
+      try {
+        Netrc netrc = NetrcParser.parseAndClose(netrcFile.getInputStream());
+        return new NetrcCredentials(netrc);
+      } catch (IOException e) {
+        throw new UrlRewriterParseException(
+            "Failed to parse " + netrcFile.getPathString() + ": " + e.getMessage(), location);
+      }
+    } else {
+      return null;
+    }
+  }
+
   @Nullable
   public String getAllBlockedMessage() {
     return config.getAllBlockedMessage();
   }
+
+  /** Holds the URL along with meta-info, such as whether URL was re-written or not. */
+  @AutoValue
+  public abstract static class RewrittenURL {
+    static RewrittenURL create(URL url, boolean rewritten) {
+      return new AutoValue_UrlRewriter_RewrittenURL(url, rewritten);
+    }
+
+    abstract URL url();
+
+    abstract boolean rewritten();
+  }
 }

src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD

@@ -16,13 +16,16 @@ java_library(
     name = "DownloaderTestSuite_lib",
     srcs = glob(["*.java"]),
     deps = [
+        "//src/main/java/com/google/devtools/build/lib/authandtls",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
         "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/util",
         "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs",
         "//src/main/java/net/starlark/java/syntax",
         "//src/test/java/com/google/devtools/build/lib/testutil",
+        "//third_party:auth",
         "//third_party:guava",
         "//third_party:jsr305",
         "//third_party:junit4",