Add --output=files mode to cquery (bazelbuild#15979)

With the new output mode `--output=files`, cquery lists all files advertised by the matched targets in the currently requested output groups.

This new mode has the following advantages over `--output=starlark` combined with an appropriate handcrafted `--starlark:expr`:
* provides a canonical answer to the very common "Where are my build outputs?" question
* is more friendly to new users as it doesn't require knowing about providers and non-BUILD dialect Starlark
* takes the value of `--output_groups` into account
* stays as close to the logic for build summaries printed by `bazel build` as possible

Fixes bazelbuild#8739

RELNOTES: `cquery`'s new output mode [`--output=files`](https://bazel.build/docs/cquery#files-output) lists the output files of the targets matching the query. It takes the current value of `--output_groups` into account.

Closes bazelbuild#15552.

PiperOrigin-RevId: 462630629
Change-Id: Ic648f22aa160ee57b476180561b444f08799ebb6

Author: fmeum

site/docs/cquery.html

@@ -452,6 +452,26 @@ <h3>Graph output</h3>
 
 </p>
 
+<h3>Files output</h3>
+
+<pre>
+--output=files
+</pre>
+
+<p>
+  This option prints a list of the output files produced by each target matched
+  by the query similar to the list printed at the end of a <code>bazel build</code>
+  invocation. The output contains only the files advertised in the requested
+  output groups as determined by the
+  <a href="https://docs.bazel.build/version/main/reference/command-line-reference#flag--output_groups"><code>--output_groups</code></a>
+  flag and never contains source files.
+
+  Note: The output of <code>bazel cquery --output=files //pkg:foo</code> contains the output
+  files of <code>//pkg:foo</code> in <i>all</i> configurations that occur in the build (also
+  see the <a href="#target-pattern-evaluation">section on target pattern evaluation</a>. If that
+  is not desired, wrap you query in <a href="#config"><code>config(..., target)</code></a>.
+</p>
+
 <h3>Defining the output format using Starlark</h3>
 
 <pre>

src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactHelper.java

@@ -21,6 +21,8 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.configuredtargets.InputFileConfiguredTarget;
+import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget;
 import com.google.devtools.build.lib.analysis.test.TestProvider;
 import com.google.devtools.build.lib.collect.nestedset.NestedSet;
 import com.google.devtools.build.lib.collect.nestedset.NestedSet.Node;
@@ -236,6 +238,36 @@ static ArtifactsToBuild getAllArtifactsToBuild(
         allOutputGroups.build(), /*allOutputGroupsImportant=*/ allOutputGroupsImportant);
   }
 
+  /**
+   * Returns false if the build outputs provided by the target should never be shown to users.
+   *
+   * <p>Always returns false for hidden rules and source file targets.
+   */
+  public static boolean shouldConsiderForDisplay(ConfiguredTarget configuredTarget) {
+    // TODO(bazel-team): this is quite ugly. Add a marker provider for this check.
+    if (configuredTarget instanceof InputFileConfiguredTarget) {
+      // Suppress display of source files (because we do no work to build them).
+      return false;
+    }
+    if (configuredTarget instanceof RuleConfiguredTarget) {
+      RuleConfiguredTarget ruleCt = (RuleConfiguredTarget) configuredTarget;
+      if (ruleCt.getRuleClassString().contains("$")) {
+        // Suppress display of hidden rules
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Returns true if the given artifact should be shown to users as a build output.
+   *
+   * <p>Always returns false for middleman and source artifacts.
+   */
+  public static boolean shouldDisplay(Artifact artifact) {
+    return !artifact.isSourceArtifact() && !artifact.isMiddlemanArtifact();
+  }
+
   /**
    * Recursive procedure filtering a target/aspect's declared {@code
    * NestedSet<ArtifactsInOutputGroup>} and {@code NestedSet<Artifact>} to only include {@link

src/main/java/com/google/devtools/build/lib/buildtool/BuildResultPrinter.java

@@ -25,9 +25,7 @@
 import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
 import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper;
 import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
-import com.google.devtools.build.lib.analysis.configuredtargets.InputFileConfiguredTarget;
 import com.google.devtools.build.lib.analysis.configuredtargets.OutputFileConfiguredTarget;
-import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget;
 import com.google.devtools.build.lib.cmdline.Label;
 import com.google.devtools.build.lib.collect.nestedset.NestedSet;
 import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
@@ -140,7 +138,7 @@ void showBuildResult(
             TopLevelArtifactHelper.getAllArtifactsToBuild(target, context)
                 .getImportantArtifacts()
                 .toList()) {
-          if (shouldPrint(artifact)) {
+          if (TopLevelArtifactHelper.shouldDisplay(artifact)) {
             if (headerFlag) {
               outErr.printErr("Target " + label + " up-to-date:\n");
               headerFlag = false;
@@ -195,7 +193,7 @@ void showBuildResult(
             outErr.printErr("Aspect " + aspectName + " of " + label + " up-to-date:\n");
             headerFlag = false;
           }
-          if (shouldPrint(importantArtifact)) {
+          if (TopLevelArtifactHelper.shouldDisplay(importantArtifact)) {
             outErr.printErrLn(formatArtifactForShowResults(prettyPrinter, importantArtifact));
           }
         }
@@ -216,10 +214,6 @@ void showBuildResult(
     }
   }
 
-  private boolean shouldPrint(Artifact artifact) {
-    return !artifact.isSourceArtifact() && !artifact.isMiddlemanArtifact();
-  }
-
   private String formatArtifactForShowResults(PathPrettyPrinter prettyPrinter, Artifact artifact) {
     return "  " + prettyPrinter.getPrettyPath(artifact.getPath().asFragment());
   }
@@ -268,18 +262,9 @@ private Collection<ConfiguredTarget> filterTargetsToPrint(
       Collection<ConfiguredTarget> configuredTargets) {
     ImmutableList.Builder<ConfiguredTarget> result = ImmutableList.builder();
     for (ConfiguredTarget configuredTarget : configuredTargets) {
-      // TODO(bazel-team): this is quite ugly. Add a marker provider for this check.
-      if (configuredTarget instanceof InputFileConfiguredTarget) {
-        // Suppress display of source files (because we do no work to build them).
+      if (!TopLevelArtifactHelper.shouldConsiderForDisplay(configuredTarget)) {
         continue;
       }
-      if (configuredTarget instanceof RuleConfiguredTarget) {
-        RuleConfiguredTarget ruleCt = (RuleConfiguredTarget) configuredTarget;
-        if (ruleCt.getRuleClassString().contains("$")) {
-          // Suppress display of hidden rules
-          continue;
-        }
-      }
       if (configuredTarget instanceof OutputFileConfiguredTarget) {
         // Suppress display of generated files (because they appear underneath
         // their generating rule), EXCEPT those ones which are not part of the

src/main/java/com/google/devtools/build/lib/buildtool/CqueryBuildTool.java

@@ -57,6 +57,7 @@ protected ConfiguredTargetQueryEnvironment getQueryEnvironment(
         env.getRelativeWorkingDirectory(),
         env.getPackageManager().getPackagePath(),
         () -> walkableGraph,
-        cqueryOptions);
+        cqueryOptions,
+        request.getTopLevelArtifactContext());
   }
 }

src/main/java/com/google/devtools/build/lib/query2/BUILD

@@ -49,6 +49,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/analysis:target_and_configuration",
         "//src/main/java/com/google/devtools/build/lib/analysis:toolchain_collection",
         "//src/main/java/com/google/devtools/build/lib/analysis:toolchain_context",
+        "//src/main/java/com/google/devtools/build/lib/analysis:top_level_artifact_context",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto",
         "//src/main/java/com/google/devtools/build/lib/causes",

src/main/java/com/google/devtools/build/lib/query2/cquery/BuildOutputFormatterCallback.java

@@ -37,7 +37,7 @@ class BuildOutputFormatterCallback extends CqueryThreadsafeCallback {
       OutputStream out,
       SkyframeExecutor skyframeExecutor,
       TargetAccessor<KeyedConfiguredTarget> accessor) {
-    super(eventHandler, options, out, skyframeExecutor, accessor);
+    super(eventHandler, options, out, skyframeExecutor, accessor, /*uniquifyResults=*/ false);
   }
 
   @Override

src/main/java/com/google/devtools/build/lib/query2/cquery/ConfiguredTargetQueryEnvironment.java

@@ -25,6 +25,7 @@
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.devtools.build.lib.analysis.ConfiguredTarget;
 import com.google.devtools.build.lib.analysis.ConfiguredTargetValue;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
 import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
 import com.google.devtools.build.lib.analysis.config.transitions.TransitionFactory;
 import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget;
@@ -81,6 +82,8 @@ public class ConfiguredTargetQueryEnvironment
 
   private CqueryOptions cqueryOptions;
 
+  private final TopLevelArtifactContext topLevelArtifactContext;
+
   private final KeyExtractor<KeyedConfiguredTarget, ConfiguredTargetKey>
       configuredTargetKeyExtractor;
 
@@ -119,7 +122,8 @@ public ConfiguredTargetQueryEnvironment(
       PathFragment parserPrefix,
       PathPackageLocator pkgPath,
       Supplier<WalkableGraph> walkableGraphSupplier,
-      Set<Setting> settings)
+      Set<Setting> settings,
+      TopLevelArtifactContext topLevelArtifactContext)
       throws InterruptedException {
     super(
         keepGoing,
@@ -135,6 +139,7 @@ public ConfiguredTargetQueryEnvironment(
     this.configuredTargetKeyExtractor = KeyedConfiguredTarget::getConfiguredTargetKey;
     this.transitiveConfigurations =
         getTransitiveConfigurations(transitiveConfigurationKeys, walkableGraphSupplier.get());
+    this.topLevelArtifactContext = topLevelArtifactContext;
   }
 
   public ConfiguredTargetQueryEnvironment(
@@ -147,7 +152,8 @@ public ConfiguredTargetQueryEnvironment(
       PathFragment parserPrefix,
       PathPackageLocator pkgPath,
       Supplier<WalkableGraph> walkableGraphSupplier,
-      CqueryOptions cqueryOptions)
+      CqueryOptions cqueryOptions,
+      TopLevelArtifactContext topLevelArtifactContext)
       throws InterruptedException {
     this(
         keepGoing,
@@ -159,7 +165,8 @@ public ConfiguredTargetQueryEnvironment(
         parserPrefix,
         pkgPath,
         walkableGraphSupplier,
-        cqueryOptions.toSettings());
+        cqueryOptions.toSettings(),
+        topLevelArtifactContext);
     this.cqueryOptions = cqueryOptions;
   }
 
@@ -270,7 +277,9 @@ private static ImmutableMap<String, BuildConfiguration> getTransitiveConfigurati
             accessor,
             kct -> getFwdDeps(ImmutableList.of(kct))),
         new StarlarkOutputFormatterCallback(
-            eventHandler, cqueryOptions, out, skyframeExecutor, accessor));
+            eventHandler, cqueryOptions, out, skyframeExecutor, accessor),
+        new FilesOutputFormatterCallback(
+            eventHandler, cqueryOptions, out, skyframeExecutor, accessor, topLevelArtifactContext));
   }
 
   @Override

src/main/java/com/google/devtools/build/lib/query2/cquery/CqueryThreadsafeCallback.java

@@ -15,6 +15,7 @@
 
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
 import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
 import com.google.devtools.build.lib.events.ExtendedEventHandler;
 import com.google.devtools.build.lib.query2.NamedThreadSafeOutputFormatterCallback;
@@ -55,14 +56,16 @@ public abstract class CqueryThreadsafeCallback
   protected final ConfiguredTargetAccessor accessor;
 
   private final List<String> result = new ArrayList<>();
+  private final boolean uniquifyResults;
 
   @SuppressWarnings("DefaultCharset")
   CqueryThreadsafeCallback(
       ExtendedEventHandler eventHandler,
       CqueryOptions options,
       OutputStream out,
       SkyframeExecutor skyframeExecutor,
-      TargetAccessor<KeyedConfiguredTarget> accessor) {
+      TargetAccessor<KeyedConfiguredTarget> accessor,
+      boolean uniquifyResults) {
     this.eventHandler = eventHandler;
     this.options = options;
     if (out != null) {
@@ -72,6 +75,7 @@ public abstract class CqueryThreadsafeCallback
     }
     this.skyframeExecutor = skyframeExecutor;
     this.accessor = (ConfiguredTargetAccessor) accessor;
+    this.uniquifyResults = uniquifyResults;
   }
 
   public void addResult(String string) {
@@ -86,7 +90,8 @@ public List<String> getResult() {
   @Override
   public void close(boolean failFast) throws InterruptedException, IOException {
     if (!failFast && printStream != null) {
-      for (String s : result) {
+      List<String> resultsToPrint = uniquifyResults ? ImmutableSet.copyOf(result).asList() : result;
+      for (String s : resultsToPrint) {
         // TODO(ulfjack): We should use queryOptions.getLineTerminator() instead.
         printStream.append(s).append("\n");
       }

src/main/java/com/google/devtools/build/lib/query2/cquery/FilesOutputFormatterCallback.java

@@ -0,0 +1,68 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.cquery;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper;
+import com.google.devtools.build.lib.events.ExtendedEventHandler;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.TargetAccessor;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * Cquery output formatter that prints the set of output files advertised by the matched targets.
+ */
+public class FilesOutputFormatterCallback extends CqueryThreadsafeCallback {
+
+  private final TopLevelArtifactContext topLevelArtifactContext;
+
+  FilesOutputFormatterCallback(
+      ExtendedEventHandler eventHandler,
+      CqueryOptions options,
+      OutputStream out,
+      SkyframeExecutor skyframeExecutor,
+      TargetAccessor<KeyedConfiguredTarget> accessor,
+      TopLevelArtifactContext topLevelArtifactContext) {
+    // Different targets may provide the same artifact, so we deduplicate the collection of all
+    // results at the end.
+    super(eventHandler, options, out, skyframeExecutor, accessor, /*uniquifyResults=*/ true);
+    this.topLevelArtifactContext = topLevelArtifactContext;
+  }
+
+  @Override
+  public String getName() {
+    return "files";
+  }
+
+  @Override
+  public void processOutput(Iterable<KeyedConfiguredTarget> partialResult)
+      throws IOException, InterruptedException {
+    for (KeyedConfiguredTarget keyedTarget : partialResult) {
+      ConfiguredTarget target = keyedTarget.getConfiguredTarget();
+      if (!TopLevelArtifactHelper.shouldConsiderForDisplay(target)) {
+        continue;
+      }
+      TopLevelArtifactHelper.getAllArtifactsToBuild(target, topLevelArtifactContext)
+          .getImportantArtifacts()
+          .toList()
+          .stream()
+          .filter(TopLevelArtifactHelper::shouldDisplay)
+          .map(Artifact::getExecPathString)
+          .forEach(this::addResult);
+    }
+  }
+}

src/main/java/com/google/devtools/build/lib/query2/cquery/GraphOutputFormatterCallback.java

@@ -86,7 +86,7 @@ public Comparator<KeyedConfiguredTarget> comparator() {
       SkyframeExecutor skyframeExecutor,
       TargetAccessor<KeyedConfiguredTarget> accessor,
       DepsRetriever depsRetriever) {
-    super(eventHandler, options, out, skyframeExecutor, accessor);
+    super(eventHandler, options, out, skyframeExecutor, accessor, /*uniquifyResults=*/ false);
     this.depsRetriever = depsRetriever;
   }
 

src/main/java/com/google/devtools/build/lib/query2/cquery/LabelAndConfigurationOutputFormatterCallback.java

@@ -36,7 +36,7 @@ public class LabelAndConfigurationOutputFormatterCallback extends CqueryThreadsa
       SkyframeExecutor skyframeExecutor,
       TargetAccessor<KeyedConfiguredTarget> accessor,
       boolean showKind) {
-    super(eventHandler, options, out, skyframeExecutor, accessor);
+    super(eventHandler, options, out, skyframeExecutor, accessor, /*uniquifyResults=*/ false);
     this.showKind = showKind;
   }
 

src/main/java/com/google/devtools/build/lib/query2/cquery/ProtoOutputFormatterCallback.java

@@ -75,7 +75,7 @@ public String formatName() {
       TargetAccessor<KeyedConfiguredTarget> accessor,
       AspectResolver resolver,
       OutputType outputType) {
-    super(eventHandler, options, out, skyframeExecutor, accessor);
+    super(eventHandler, options, out, skyframeExecutor, accessor, /*uniquifyResults=*/ false);
     this.outputType = outputType;
     this.skyframeExecutor = skyframeExecutor;
     this.resolver = resolver;

src/main/java/com/google/devtools/build/lib/query2/cquery/StarlarkOutputFormatterCallback.java

@@ -142,7 +142,7 @@ public Object providers(ConfiguredTarget target) {
       SkyframeExecutor skyframeExecutor,
       TargetAccessor<KeyedConfiguredTarget> accessor)
       throws QueryException, InterruptedException {
-    super(eventHandler, options, out, skyframeExecutor, accessor);
+    super(eventHandler, options, out, skyframeExecutor, accessor, /*uniquifyResults=*/ false);
 
     ParserInput input = null;
     String exceptionMessagePrefix;

src/main/java/com/google/devtools/build/lib/query2/cquery/TransitionsOutputFormatterCallback.java

@@ -83,7 +83,7 @@ public String getName() {
       TargetAccessor<KeyedConfiguredTarget> accessor,
       BuildConfiguration hostConfiguration,
       @Nullable TransitionFactory<RuleTransitionData> trimmingTransitionFactory) {
-    super(eventHandler, options, out, skyframeExecutor, accessor);
+    super(eventHandler, options, out, skyframeExecutor, accessor, /*uniquifyResults=*/ false);
     this.hostConfiguration = hostConfiguration;
     this.trimmingTransitionFactory = trimmingTransitionFactory;
     this.partialResultMap = Maps.newHashMap();