Add netrc support to --bes_backend (#15970)

Progress on #15856

Fixes #15928

Closes #15930.

PiperOrigin-RevId: 462379746
Change-Id: Ia7ae470bdcbd97c6cb42cc290c3891393ec9ce3a

Co-authored-by: Yannic Bonenberger <[email protected]>

Author: tjgq

src/main/java/com/google/devtools/build/lib/authandtls/BUILD

@@ -15,6 +15,8 @@ java_library(
     srcs = glob(["*.java"]),
     deps = [
         "//src/main/java/com/google/devtools/build/lib/concurrent",
+        "//src/main/java/com/google/devtools/build/lib/events",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/common/options",
         "//third_party:auth",
         "//third_party:auto_value",

src/main/java/com/google/devtools/build/lib/authandtls/GoogleAuthUtils.java

@@ -19,6 +19,10 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
 import io.grpc.CallCredentials;
 import io.grpc.ClientInterceptor;
 import io.grpc.ManagedChannel;
@@ -41,6 +45,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import javax.annotation.Nullable;
@@ -186,14 +192,17 @@ private static NettyChannelBuilder newNettyChannelBuilder(String targetUrl, Stri
   }
 
   /**
-   * Create a new {@link CallCredentials} object.
+   * Create a new {@link CallCredentials} object from the authentication flags, or null if no flags
+   * are set.
    *
-   * @throws IOException in case the call credentials can't be constructed.
+   * @throws IOException in case the credentials can't be constructed.
    */
-  public static CallCredentials newCallCredentials(AuthAndTLSOptions options) throws IOException {
-    Credentials creds = newCredentials(options);
-    if (creds != null) {
-      return MoreCallCredentials.from(creds);
+  @Nullable
+  public static CallCredentials newGoogleCallCredentials(AuthAndTLSOptions options)
+      throws IOException {
+    Optional<Credentials> creds = newGoogleCredentials(options);
+    if (creds.isPresent()) {
+      return MoreCallCredentials.from(creds.get());
     }
     return null;
   }
@@ -210,18 +219,52 @@ public static CallCredentialsProvider newCallCredentialsProvider(@Nullable Crede
   }
 
   /**
-   * Create a new {@link Credentials} object, or {@code null} if no options are provided.
+   * Create a new {@link Credentials} with following order:
+   *
+   * <ol>
+   *   <li>If authentication enabled by flags, use it to create credentials
+   *   <li>Use .netrc to provide credentials if exists
+   *   <li>Otherwise, return {@code null}
+   * </ol>
    *
    * @throws IOException in case the credentials can't be constructed.
    */
   @Nullable
-  public static Credentials newCredentials(@Nullable AuthAndTLSOptions options) throws IOException {
+  public static Credentials newCredentials(
+      Reporter reporter,
+      Map<String, String> clientEnv,
+      FileSystem fileSystem,
+      AuthAndTLSOptions authAndTlsOptions)
+      throws IOException {
+    Optional<Credentials> credentials = newGoogleCredentials(authAndTlsOptions);
+
+    if (credentials.isEmpty()) {
+      // Fallback to .netrc if it exists.
+      try {
+        credentials = newCredentialsFromNetrc(clientEnv, fileSystem);
+      } catch (IOException e) {
+        // TODO(yannic): Make this fail the build.
+        reporter.handle(Event.warn(e.getMessage()));
+      }
+    }
+
+    return credentials.orElse(null);
+  }
+
+  /**
+   * Create a new {@link Credentials} object from the authentication flags, or null if no flags are
+   * set.
+   *
+   * @throws IOException in case the credentials can't be constructed.
+   */
+  public static Optional<Credentials> newGoogleCredentials(@Nullable AuthAndTLSOptions options)
+      throws IOException {
     if (options == null) {
-      return null;
+      return Optional.empty();
     } else if (options.googleCredentials != null) {
       // Credentials from file
       try (InputStream authFile = new FileInputStream(options.googleCredentials)) {
-        return newCredentials(authFile, options.googleAuthScopes);
+        return Optional.of(newGoogleCredentialsFromFile(authFile, options.googleAuthScopes));
       } catch (FileNotFoundException e) {
         String message =
             String.format(
@@ -230,10 +273,11 @@ public static Credentials newCredentials(@Nullable AuthAndTLSOptions options) th
         throw new IOException(message, e);
       }
     } else if (options.useGoogleDefaultCredentials) {
-      return newCredentials(
-          null /* Google Application Default Credentials */, options.googleAuthScopes);
+      return Optional.of(
+          newGoogleCredentialsFromFile(
+              null /* Google Application Default Credentials */, options.googleAuthScopes));
     }
-    return null;
+    return Optional.empty();
   }
 
   /**
@@ -242,7 +286,7 @@ public static Credentials newCredentials(@Nullable AuthAndTLSOptions options) th
    * @throws IOException in case the credentials can't be constructed.
    */
   @VisibleForTesting
-  public static Credentials newCredentials(
+  public static Credentials newGoogleCredentialsFromFile(
       @Nullable InputStream credentialsFile, List<String> authScopes) throws IOException {
     try {
       GoogleCredentials creds =
@@ -258,4 +302,40 @@ public static Credentials newCredentials(
       throw new IOException(message, e);
     }
   }
+
+  /**
+   * Create a new {@link Credentials} object by parsing the .netrc file with following order to
+   * search it:
+   *
+   * <ol>
+   *   <li>If environment variable $NETRC exists, use it as the path to the .netrc file
+   *   <li>Fallback to $HOME/.netrc
+   * </ol>
+   *
+   * @return the {@link Credentials} object or {@code null} if there is no .netrc file.
+   * @throws IOException in case the credentials can't be constructed.
+   */
+  @VisibleForTesting
+  static Optional<Credentials> newCredentialsFromNetrc(
+      Map<String, String> clientEnv, FileSystem fileSystem) throws IOException {
+    Optional<String> netrcFileString =
+        Optional.ofNullable(clientEnv.get("NETRC"))
+            .or(() -> Optional.ofNullable(clientEnv.get("HOME")).map(home -> home + "/.netrc"));
+    if (netrcFileString.isEmpty()) {
+      return Optional.empty();
+    }
+
+    Path netrcFile = fileSystem.getPath(netrcFileString.get());
+    if (!netrcFile.exists()) {
+      return Optional.empty();
+    }
+
+    try {
+      Netrc netrc = NetrcParser.parseAndClose(netrcFile.getInputStream());
+      return Optional.of(new NetrcCredentials(netrc));
+    } catch (IOException e) {
+      throw new IOException(
+          "Failed to parse " + netrcFile.getPathString() + ": " + e.getMessage(), e);
+    }
+  }
 }

src/main/java/com/google/devtools/build/lib/buildeventservice/BUILD

@@ -57,6 +57,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/util/io:out-err",
         "//src/main/java/com/google/devtools/common/options",
         "//src/main/protobuf:failure_details_java_proto",
+        "//third_party:auth",
         "//third_party:auto_value",
         "//third_party:flogger",
         "//third_party:guava",

src/main/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java

@@ -14,8 +14,10 @@
 
 package com.google.devtools.build.lib.buildeventservice;
 
+import com.google.auth.Credentials;
 import com.google.auto.value.AutoValue;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -24,9 +26,11 @@
 import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
 import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceClient;
 import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceGrpcClient;
+import com.google.devtools.build.lib.runtime.CommandEnvironment;
 import io.grpc.ClientInterceptor;
 import io.grpc.ManagedChannel;
 import io.grpc.Metadata;
+import io.grpc.auth.MoreCallCredentials;
 import io.grpc.stub.MetadataUtils;
 import java.io.IOException;
 import java.util.Map;
@@ -70,15 +74,28 @@ protected Class<BuildEventServiceOptions> optionsClass() {
 
   @Override
   protected BuildEventServiceClient getBesClient(
-      BuildEventServiceOptions besOptions, AuthAndTLSOptions authAndTLSOptions) throws IOException {
+      CommandEnvironment env,
+      BuildEventServiceOptions besOptions,
+      AuthAndTLSOptions authAndTLSOptions)
+      throws IOException {
     BackendConfig newConfig = BackendConfig.create(besOptions, authAndTLSOptions);
     if (client == null || !Objects.equals(config, newConfig)) {
       clearBesClient();
+      Preconditions.checkState(config == null);
+      Preconditions.checkState(client == null);
+
+      Credentials credentials =
+          GoogleAuthUtils.newCredentials(
+              env.getReporter(),
+              env.getClientEnv(),
+              env.getRuntime().getFileSystem(),
+              newConfig.authAndTLSOptions());
+
       config = newConfig;
       client =
           new BuildEventServiceGrpcClient(
               newGrpcChannel(config),
-              GoogleAuthUtils.newCallCredentials(config.authAndTLSOptions()),
+              credentials != null ? MoreCallCredentials.from(credentials) : null,
               makeGrpcInterceptor(config));
     }
     return client;

src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java

@@ -677,7 +677,7 @@ private BuildEventServiceTransport createBesTransport(
 
     final BuildEventServiceClient besClient;
     try {
-      besClient = getBesClient(besOptions, authTlsOptions);
+      besClient = getBesClient(cmdEnv, besOptions, authTlsOptions);
     } catch (IOException | OptionsParsingException e) {
       reportError(
           reporter,
@@ -822,7 +822,7 @@ private static AbruptExitException createAbruptExitException(
   protected abstract Class<BESOptionsT> optionsClass();
 
   protected abstract BuildEventServiceClient getBesClient(
-      BESOptionsT besOptions, AuthAndTLSOptions authAndTLSOptions)
+      CommandEnvironment env, BESOptionsT besOptions, AuthAndTLSOptions authAndTLSOptions)
       throws IOException, OptionsParsingException;
 
   protected abstract void clearBesClient();

src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java

@@ -49,9 +49,6 @@
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions.UnresolvedScopedCredentialHelper;
 import com.google.devtools.build.lib.authandtls.CallCredentialsProvider;
 import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
-import com.google.devtools.build.lib.authandtls.Netrc;
-import com.google.devtools.build.lib.authandtls.NetrcCredentials;
-import com.google.devtools.build.lib.authandtls.NetrcParser;
 import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment;
 import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperProvider;
 import com.google.devtools.build.lib.bazel.repository.downloader.Downloader;
@@ -1052,94 +1049,6 @@ RemoteActionContextProvider getActionContextProvider() {
     return actionContextProvider;
   }
 
-  /**
-   * Create a new {@link Credentials} object by parsing the .netrc file with following order to
-   * search it:
-   *
-   * <ol>
-   *   <li>If environment variable $NETRC exists, use it as the path to the .netrc file
-   *   <li>Fallback to $HOME/.netrc
-   * </ol>
-   *
-   * @return the {@link Credentials} object or {@code null} if there is no .netrc file.
-   * @throws IOException in case the credentials can't be constructed.
-   */
-  @VisibleForTesting
-  static Credentials newCredentialsFromNetrc(Map<String, String> clientEnv, FileSystem fileSystem)
-      throws IOException {
-    String netrcFileString =
-        Optional.ofNullable(clientEnv.get("NETRC"))
-            .orElseGet(
-                () ->
-                    Optional.ofNullable(clientEnv.get("HOME"))
-                        .map(home -> home + "/.netrc")
-                        .orElse(null));
-    if (netrcFileString == null) {
-      return null;
-    }
-
-    Path netrcFile = fileSystem.getPath(netrcFileString);
-    if (netrcFile.exists()) {
-      try {
-        Netrc netrc = NetrcParser.parseAndClose(netrcFile.getInputStream());
-        return new NetrcCredentials(netrc);
-      } catch (IOException e) {
-        throw new IOException(
-            "Failed to parse " + netrcFile.getPathString() + ": " + e.getMessage(), e);
-      }
-    } else {
-      return null;
-    }
-  }
-
-  /**
-   * Create a new {@link Credentials} with following order:
-   *
-   * <ol>
-   *   <li>If authentication enabled by flags, use it to create credentials
-   *   <li>Use .netrc to provide credentials if exists
-   *   <li>Otherwise, return {@code null}
-   * </ol>
-   *
-   * @throws IOException in case the credentials can't be constructed.
-   */
-  @VisibleForTesting
-  static Credentials newCredentials(
-      Map<String, String> clientEnv,
-      FileSystem fileSystem,
-      Reporter reporter,
-      AuthAndTLSOptions authAndTlsOptions,
-      RemoteOptions remoteOptions)
-      throws IOException {
-    Credentials creds = GoogleAuthUtils.newCredentials(authAndTlsOptions);
-
-    // Fallback to .netrc if it exists
-    if (creds == null) {
-      try {
-        creds = newCredentialsFromNetrc(clientEnv, fileSystem);
-      } catch (IOException e) {
-        reporter.handle(Event.warn(e.getMessage()));
-      }
-
-      try {
-        if (creds != null
-            && remoteOptions.remoteCache != null
-            && Ascii.toLowerCase(remoteOptions.remoteCache).startsWith("http://")
-            && !creds.getRequestMetadata(new URI(remoteOptions.remoteCache)).isEmpty()) {
-          reporter.handle(
-              Event.warn(
-                  "Username and password from .netrc is transmitted in plaintext to "
-                      + remoteOptions.remoteCache
-                      + ". Please consider using an HTTPS endpoint."));
-        }
-      } catch (URISyntaxException e) {
-        throw new IOException(e.getMessage(), e);
-      }
-    }
-
-    return creds;
-  }
-
   @VisibleForTesting
   static CredentialHelperProvider newCredentialHelperProvider(
       CredentialHelperEnvironment environment,
@@ -1163,6 +1072,35 @@ static CredentialHelperProvider newCredentialHelperProvider(
     return builder.build();
   }
 
+  static Credentials newCredentials(
+      Map<String, String> clientEnv,
+      FileSystem fileSystem,
+      Reporter reporter,
+      AuthAndTLSOptions authAndTlsOptions,
+      RemoteOptions remoteOptions)
+      throws IOException {
+    Credentials credentials =
+        GoogleAuthUtils.newCredentials(reporter, clientEnv, fileSystem, authAndTlsOptions);
+
+    try {
+      if (credentials != null
+          && remoteOptions.remoteCache != null
+          && Ascii.toLowerCase(remoteOptions.remoteCache).startsWith("http://")
+          && !credentials.getRequestMetadata(new URI(remoteOptions.remoteCache)).isEmpty()) {
+        // TODO(yannic): Make this a error aborting the build.
+        reporter.handle(
+            Event.warn(
+                "Credentials are transmitted in plaintext to "
+                    + remoteOptions.remoteCache
+                    + ". Please consider using an HTTPS endpoint."));
+      }
+    } catch (URISyntaxException e) {
+      throw new IOException(e.getMessage(), e);
+    }
+
+    return credentials;
+  }
+
   @VisibleForTesting
   @AutoValue
   abstract static class ScopedCredentialHelper {