feat: yq

.bazelci/presubmit.yml

@@ -12,7 +12,11 @@ tasks:
     test_targets:
       - "//..."
   windows:
+    build_flags:
+      - "--build_tag_filters=-no-windows-ci"
     build_targets:
       - "//..."
+    test_flags:
+      - "--test_tag_filters=-no-windows-ci"
     test_targets:
       - "//..."

.prettierignore

@@ -1,5 +1,6 @@
 docs/*.md
 lib/tests/jq/*.json
+lib/tests/yq/empty.yaml
 lib/lib/tests/write_source_files/*.js
 lib/lib/tests/write_source_files/subdir/*.js
 lib/lib/tests/write_source_files/subdir/subsubdir/*.js

docs/BUILD.bazel

@@ -83,4 +83,9 @@ stardoc_with_diff_test(
     bzl_library_target = "//lib:repo_utils",
 )
 
+stardoc_with_diff_test(
+    name = "yq",
+    bzl_library_target = "//lib:yq",
+)
+
 update_docs()

docs/yq.md

@@ -0,0 +1,114 @@
+<!-- Generated with Stardoc: http://skydoc.bazel.build -->
+
+Public API for yq
+
+<a id="#yq"></a>
+
+## yq
+
+<pre>
+yq(<a href="#yq-name">name</a>, <a href="#yq-srcs">srcs</a>, <a href="#yq-expression">expression</a>, <a href="#yq-args">args</a>, <a href="#yq-outs">outs</a>, <a href="#yq-kwargs">kwargs</a>)
+</pre>
+
+Invoke yq with an expression on a set of input files.
+
+For yq documentation, see https://mikefarah.gitbook.io/yq.
+
+To use this rule you must register the yq toolchain in your WORKSPACE:
+
+```starlark
+load("@aspect_bazel_lib//lib:repositories.bzl", "register_yq_toolchains")
+
+register_yq_toolchains(version = "4.24.4")
+```
+
+Usage examples:
+
+```starlark
+load("@aspect_bazel_lib//lib:yq.bzl", "yq")
+```
+
+```starlark
+# Remove fields
+yq(
+    name = "safe-config",
+    srcs = ["config.yaml"],
+    filter = "del(.credentials)",
+)
+```
+
+```starlark
+# Merge two yaml documents
+yq(
+    name = "merged",
+    srcs = [
+        "a.yaml",
+        "b.yaml",
+    ],
+    expression = ". as $item ireduce ({}; . * $item )",
+)
+```
+
+```starlark
+# Split a yaml file into several files
+yq(
+    name = "split",
+    srcs = ["multidoc.yaml"],
+    outs = [
+        "first.yml",
+        "second.yml",
+    ],
+    args = [
+        "-s '.a'",  # Split expression
+        "--no-doc", # Exclude document separator --
+    ],
+)
+```
+
+```starlark
+# Convert a yaml file to json
+yq(
+    name = "convert-to-json",
+    srcs = ["foo.yaml"],
+    args = ["-o=json"],
+    outs = ["foo.json"],
+)
+```
+
+```starlark
+# Convert a json file to yaml
+yq(
+    name = "convert",
+    srcs = ["bar.json"],
+    args = ["-P"],
+    outs = ["bar.yaml"],
+)
+```
+
+```starlark
+# Call yq in a genrule
+genrule(
+    name = "generate",
+    srcs = ["farm.yaml"],
+    outs = ["genrule_output.yaml"],
+    cmd = "$(YQ_BIN) '.moo = "cow"' $(location farm.yaml) > $@",
+    toolchains = ["@yq_toolchains//:resolved_toolchain"],
+)
+```
+
+yq is capable of parsing and outputting to other formats. See their [docs](https://mikefarah.gitbook.io/yq) for more examples.
+
+
+**PARAMETERS**
+
+
+| Name  | Description | Default Value |
+| :------------- | :------------- | :------------- |
+| <a id="yq-name"></a>name |  Name of the rule   |  none |
+| <a id="yq-srcs"></a>srcs |  List of input file labels   |  none |
+| <a id="yq-expression"></a>expression |  yq expression (https://mikefarah.gitbook.io/yq/commands/evaluate). Defaults to the identity expression "."   |  <code>"."</code> |
+| <a id="yq-args"></a>args |  Additional args to pass to yq. Note that you do not need to pass _eval_ or _eval-all_ as this is handled automatically based on the number <code>srcs</code>. Passing the output format or the parse format is optional as these can be guessed based on the file extensions in <code>srcs</code> and <code>outs</code>.   |  <code>[]</code> |
+| <a id="yq-outs"></a>outs |  Name of the output files. Defaults to a single output with the name plus a ".yaml" extension, or the extension corresponding to a passed output argment (e.g., "-o=json"). For split operations you must declare all outputs as the name of the output files depends on the expression.   |  <code>None</code> |
+| <a id="yq-kwargs"></a>kwargs |  Other common named parameters such as <code>tags</code> or <code>visibility</code>   |  none |
+
+

internal_deps.bzl

@@ -6,7 +6,7 @@ statement from these, that's a bug in our distribution.
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
-load("//lib:repositories.bzl", "register_jq_toolchains")
+load("//lib:repositories.bzl", "register_jq_toolchains", "register_yq_toolchains")
 
 # buildifier: disable=unnamed-macro
 def bazel_lib_internal_deps():
@@ -67,3 +67,4 @@ def bazel_lib_internal_deps():
 
     # Register toolchains for tests
     register_jq_toolchains(version = "1.6")
+    register_yq_toolchains(version = "4.24.4")

lib/BUILD.bazel

@@ -13,6 +13,11 @@ toolchain_type(
     visibility = ["//visibility:public"],
 )
 
+toolchain_type(
+    name = "yq_toolchain_type",
+    visibility = ["//visibility:public"],
+)
+
 bzl_library(
     name = "docs",
     srcs = ["docs.bzl"],
@@ -144,3 +149,10 @@ bzl_library(
         "//lib/private:repo_utils",
     ],
 )
+
+bzl_library(
+    name = "yq",
+    srcs = ["yq.bzl"],
+    visibility = ["//visibility:public"],
+    deps = ["//lib/private:yq"],
+)

lib/private/BUILD.bazel

@@ -151,3 +151,9 @@ bzl_library(
     visibility = ["//lib:__subpackages__"],
     deps = [":repo_utils"],
 )
+
+bzl_library(
+    name = "yq",
+    srcs = ["yq.bzl"],
+    visibility = ["//lib:__subpackages__"],
+)

lib/private/yq.bzl

@@ -0,0 +1,69 @@
+"""Implementation for yq rule"""
+
+_yq_attrs = {
+    "srcs": attr.label_list(
+        allow_files = [".yaml", ".json", ".xml"],
+        mandatory = True,
+        allow_empty = True,
+    ),
+    "expression": attr.string(mandatory = False),
+    "args": attr.string_list(),
+    "outs": attr.output_list(mandatory = True),
+}
+
+def is_split_operation(args):
+    for arg in args:
+        if arg.startswith("-s") or arg.startswith("--split-exp"):
+            return True
+    return False
+
+def _escape_path(path):
+    return "/".join([".." for t in path.split("/")]) + "/"
+
+def _yq_impl(ctx):
+    yq_bin = ctx.toolchains["@aspect_bazel_lib//lib:yq_toolchain_type"].yqinfo.bin
+
+    outs = ctx.outputs.outs
+    args = ctx.attr.args[:]
+    inputs = ctx.files.srcs[:]
+
+    split_operation = is_split_operation(args)
+
+    if "eval" in args or "eval-all" in args:
+        fail("Do not pass 'eval' or 'eval-all' into yq; this is already set based on the number of srcs")
+    if not split_operation and len(outs) > 1:
+        fail("Cannot specify multiple outputs when -s or --split-exp is not set")
+    if "-i" in args or "--inplace" in args:
+        fail("Cannot use arg -i or --inplace as it is not bazel-idiomatic to update the input file")
+    if len(ctx.attr.srcs) == 0 and "-n" not in args and "--null-input" not in args:
+        args = args + ["--null-input"]
+
+    # For split operations, yq outputs files in the same directory so we
+    # must cd to the correct output dir before executing it
+    bin_dir = ctx.bin_dir.path + "/" + ctx.label.package
+    escape_bin_dir = _escape_path(bin_dir)
+    cmd = "cd {bin_dir} && {yq} {args} {eval_cmd} {expression} {sources}{maybe_out}".format(
+        bin_dir = ctx.bin_dir.path + "/" + ctx.label.package,
+        yq = escape_bin_dir + yq_bin.path,
+        eval_cmd = "eval" if len(inputs) <= 1 else "eval-all",
+        args = " ".join(args),
+        expression = "'%s'" % ctx.attr.expression if ctx.attr.expression else "",
+        sources = " ".join(["'%s%s'" % (escape_bin_dir, file.path) for file in ctx.files.srcs]),
+        # In the -s/--split-exr case, the out file names are determined by the yq expression
+        maybe_out = (" > %s%s" % (escape_bin_dir, outs[0].path)) if len(outs) == 1 else "",
+    )
+
+    ctx.actions.run_shell(
+        tools = [yq_bin],
+        inputs = inputs,
+        outputs = outs,
+        command = cmd,
+        mnemonic = "yq",
+    )
+
+    return DefaultInfo(files = depset(outs), runfiles = ctx.runfiles(outs))
+
+yq_lib = struct(
+    attrs = _yq_attrs,
+    implementation = _yq_impl,
+)