diff --git a/site/docs/cquery.html b/site/docs/cquery.html index 6d5ad43af4744e..5355d6e796c4ff 100644 --- a/site/docs/cquery.html +++ b/site/docs/cquery.html @@ -369,7 +369,7 @@

Transitions

without the options diff.

-

Proto

+

Protocol message output

 --output=proto
@@ -401,6 +401,58 @@ 

--[no]proto:include_configurations

included as rule_input fields.

+

Formatting output with a Starlark expression

+ +
+--output=starlark
+
+ +

+ This output format evaluates and prints a given + Starlark + expression for each resulting target. The expression may be specified with the + --starlark:expr flag. +

+ The specified expression can access only the core Starlark + + built-in constants and functions, and a global target, of type + Target. + Note it does not have access to many of the + additional built-in constants and functions supported for rule/macro code, such as + glob() or rule() +

+

Examples

+ +

+Print a space-separated list of the base names of all files produced by //foo: +

+
+  bazel cquery //foo --output=starlark \
+    --starlark:expr="' '.join([f.basename for f in target.files.to_list()])"
+
+

+Print a space-separated list of the paths of all files produced by rule targets in +//bar and its subpackages: +

+
+  bazel cquery 'kind(rule, //bar/...)' --output=starlark \
+    --starlark:expr="' '.join([f.path for f in target.files.to_list()])"
+
+

+Print a list of the mnemonics of all actions registered by //foo. +

+
+  bazel cquery //foo --output=starlark \
+    --starlark:expr="[a.mnemonic for a in target.actions]"
+
+

+Print a list of compilation outputs registered by a cc_library //baz. +

+
+  bazel cquery //baz --output=starlark \
+    --starlark:expr="[f.path for f in target.output_groups.compilation_outputs.to_list()]"
+
+

cquery vs. query

diff --git a/src/main/java/com/google/devtools/build/lib/query2/PostAnalysisQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/PostAnalysisQueryEnvironment.java index 4334b595c7fafe..a0ae6d5e58de6e 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/PostAnalysisQueryEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/query2/PostAnalysisQueryEnvironment.java @@ -137,7 +137,8 @@ public PostAnalysisQueryEnvironment( SkyframeExecutor skyframeExecutor, BuildConfiguration hostConfiguration, @Nullable TransitionFactory trimmingTransitionFactory, - PackageManager packageManager); + PackageManager packageManager) + throws QueryException, InterruptedException; public abstract String getOutputFormat(); diff --git a/src/main/java/com/google/devtools/build/lib/query2/cquery/ConfiguredTargetQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/cquery/ConfiguredTargetQueryEnvironment.java index 246e712fc741b3..37a82530b7eea2 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/cquery/ConfiguredTargetQueryEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/query2/cquery/ConfiguredTargetQueryEnvironment.java @@ -206,7 +206,8 @@ private static ImmutableMap getTransitiveConfigurati SkyframeExecutor skyframeExecutor, BuildConfiguration hostConfiguration, @Nullable TransitionFactory trimmingTransitionFactory, - PackageManager packageManager) { + PackageManager packageManager) + throws QueryException, InterruptedException { AspectResolver aspectResolver = cqueryOptions.aspectDeps.createResolver(packageManager, eventHandler); return ImmutableList.of( @@ -247,6 +248,8 @@ private static ImmutableMap getTransitiveConfigurati aspectResolver, OutputType.JSON), new BuildOutputFormatterCallback( + eventHandler, cqueryOptions, out, skyframeExecutor, accessor), + new StarlarkOutputFormatterCallback( eventHandler, cqueryOptions, out, skyframeExecutor, accessor)); } diff --git a/src/main/java/com/google/devtools/build/lib/query2/cquery/CqueryOptions.java b/src/main/java/com/google/devtools/build/lib/query2/cquery/CqueryOptions.java index 14aea6cb2d2b01..c8dcfc37c5403a 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/cquery/CqueryOptions.java +++ b/src/main/java/com/google/devtools/build/lib/query2/cquery/CqueryOptions.java @@ -82,4 +82,14 @@ public enum Transitions { "if enabled, proto output will include information about configurations. When disabled," + "cquery proto output format resembles query output format.") public boolean protoIncludeConfigurations; + + @Option( + name = "starlark:expr", + defaultValue = "str(target.label)", + documentationCategory = OptionDocumentationCategory.QUERY, + effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, + help = + "A Starlark expression to format each configured target in cquery's" + + " --output=starlark mode. The configured target is bound to 'target'.") + public String expr; } diff --git a/src/main/java/com/google/devtools/build/lib/query2/cquery/StarlarkOutputFormatterCallback.java b/src/main/java/com/google/devtools/build/lib/query2/cquery/StarlarkOutputFormatterCallback.java new file mode 100644 index 00000000000000..d629aa93b1dee7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/cquery/StarlarkOutputFormatterCallback.java @@ -0,0 +1,113 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.query2.cquery; + +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.TargetAccessor; +import com.google.devtools.build.lib.query2.engine.QueryException; +import com.google.devtools.build.lib.server.FailureDetails.ConfigurableQuery.Code; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.EvalUtils; +import com.google.devtools.build.lib.syntax.Expression; +import com.google.devtools.build.lib.syntax.FileOptions; +import com.google.devtools.build.lib.syntax.Module; +import com.google.devtools.build.lib.syntax.Mutability; +import com.google.devtools.build.lib.syntax.ParserInput; +import com.google.devtools.build.lib.syntax.Starlark; +import com.google.devtools.build.lib.syntax.StarlarkFunction; +import com.google.devtools.build.lib.syntax.StarlarkSemantics; +import com.google.devtools.build.lib.syntax.StarlarkThread; +import com.google.devtools.build.lib.syntax.SyntaxError; +import java.io.OutputStream; + +/** + * Starlark output formatter for cquery results. Each configured target will result in an evaluation + * of the Starlark expression specified by {@code --expr}. + */ +public class StarlarkOutputFormatterCallback extends CqueryThreadsafeCallback { + private static final Object[] NO_ARGS = new Object[0]; + + // Starlark function with single required parameter "target", a ConfiguredTarget query result. + private final StarlarkFunction exprEvalFn; + + StarlarkOutputFormatterCallback( + ExtendedEventHandler eventHandler, + CqueryOptions options, + OutputStream out, + SkyframeExecutor skyframeExecutor, + TargetAccessor accessor) + throws QueryException, InterruptedException { + super(eventHandler, options, out, skyframeExecutor, accessor); + + // Validate that options.expr is a pure expression (for example, that it does not attempt + // to escape its scope via unbalanced parens). + ParserInput exprParserInput = ParserInput.fromString(options.expr, "--starlark:expr"); + try { + Expression.parse(exprParserInput); + } catch (SyntaxError.Exception ex) { + throw new QueryException( + "invalid --starlark:expr: " + ex.getMessage(), Code.STARLARK_SYNTAX_ERROR); + } + + // Create a synthetic file that defines a function with single parameter "target", + // whose body is provided by the user's expression. + String fileBody = "def f(target): return (" + options.expr + ")\n" + "f"; + ParserInput input = ParserInput.fromString(fileBody, "--starlark:expr"); + + try { + StarlarkThread thread = + new StarlarkThread(Mutability.create("foo"), StarlarkSemantics.DEFAULT); + this.exprEvalFn = + (StarlarkFunction) EvalUtils.exec(input, FileOptions.DEFAULT, Module.create(), thread); + } catch (SyntaxError.Exception ex) { + throw new QueryException( + "invalid --starlark:expr: " + ex.getMessage(), Code.STARLARK_SYNTAX_ERROR); + } catch (EvalException ex) { + throw new QueryException( + "invalid --starlark:expr: " + ex.getMessageWithStack(), Code.STARLARK_EVAL_ERROR); + } + } + + @Override + public String getName() { + return "starlark"; + } + + @Override + public void processOutput(Iterable partialResult) throws InterruptedException { + StarlarkThread thread = + new StarlarkThread(Mutability.create("cquery evaluation"), StarlarkSemantics.DEFAULT); + thread.setMaxExecutionSteps(500_000L); + + for (ConfiguredTarget target : partialResult) { + try { + // Invoke exprEvalFn with `target` argument. + Object result = Starlark.fastcall(thread, this.exprEvalFn, new Object[] {target}, NO_ARGS); + + addResult(Starlark.str(result)); + } catch (EvalException ex) { + eventHandler.handle( + Event.error( + String.format( + "Starlark evaluation error for %s: %s", + target.getLabel(), ex.getMessageWithStack()))); + continue; + } + } + } +} diff --git a/src/main/protobuf/failure_details.proto b/src/main/protobuf/failure_details.proto index 50e29a55c6cc84..0ffaff2ff18c7f 100644 --- a/src/main/protobuf/failure_details.proto +++ b/src/main/protobuf/failure_details.proto @@ -816,6 +816,8 @@ message ConfigurableQuery { ATTRIBUTE_MISSING = 7 [(metadata) = { exit_code: 2 }]; INCORRECT_CONFIG_ARGUMENT_ERROR = 8 [(metadata) = { exit_code: 2 }]; TARGET_MISSING = 9 [(metadata) = { exit_code: 2 }]; + STARLARK_SYNTAX_ERROR = 10 [(metadata) = { exit_code: 2 }]; + STARLARK_EVAL_ERROR = 11 [(metadata) = { exit_code: 2 }]; } Code code = 1; diff --git a/src/test/shell/integration/configured_query_test.sh b/src/test/shell/integration/configured_query_test.sh index bcb0e14eaa84f5..09f16da8b2dd44 100755 --- a/src/test/shell/integration/configured_query_test.sh +++ b/src/test/shell/integration/configured_query_test.sh @@ -810,4 +810,89 @@ EOF assert_equals "$(grep some_file output | wc -l | egrep -o '[0-9]+')" "1" } +function test_starlark_output_mode() { + local -r pkg=$FUNCNAME + mkdir -p $pkg + cat > $pkg/BUILD < output \ + 2>"$TEST_log" || fail "Expected success" + + assert_contains "//$pkg:pylib%foo" output + assert_contains "//$pkg:pylibtwo%foo" output + + bazel cquery "//$pkg:all" --output=starlark \ + --starlark:expr="str(target.label) + '%' + str(target.files.to_list()[1].is_directory)" \ + > output 2>"$TEST_log" || fail "Expected success" + + assert_contains "//$pkg:pylibtwo%False" output + # pylib evaluation will fail, as it has only one output file. + assert_contains "Starlark evaluation error for //$pkg:pylib" "$TEST_log" +} + +function test_starlark_output_invalid_expression() { + local -r pkg=$FUNCNAME + mkdir -p $pkg + touch $pkg/BUILD + + bazel cquery "//$pkg:all" --output=starlark \ + --starlark:expr="no_symbol" \ + > output 2>"$TEST_log" && fail "Expected failure" + + assert_contains "invalid --starlark:expr: name 'no_symbol' is not defined" $TEST_log + + bazel cquery "//$pkg:all" --output=starlark \ + --starlark:expr="def foo(): return 5" \ + > output 2>"$TEST_log" && fail "Expected failure" + + assert_contains "syntax error at 'def': expected expression" $TEST_log +} + +function test_starlark_output_cc_library_files() { + local -r pkg=$FUNCNAME + mkdir -p $pkg + cat > $pkg/BUILD < output 2>"$TEST_log" || fail "Expected failure" + + if "$is_windows"; then + assert_contains "cclib.lib" output + else + assert_contains "libcclib.a" output + assert_contains "libcclib.so" output + fi +} + +function test_starlark_file_output() { + local -r pkg=$FUNCNAME + mkdir -p $pkg + cat > $pkg/BUILD < output 2>"$TEST_log" || fail "Expected failure" + + assert_contains "^path=$pkg/foo$" output +} + run_suite "${PRODUCT_NAME} configured query tests"