From aaf02505a50e0b09ba6b8e8350f05d1c876a68d4 Mon Sep 17 00:00:00 2001 From: James Leitch Date: Thu, 11 Jan 2024 21:01:44 -0700 Subject: [PATCH] Expose Toolchains and add ripgrep/sd toolchains. --- internal_deps.bzl | 4 +- lib/BUILD.bazel | 8 ++ lib/extensions.bzl | 22 +++ lib/private/ripgrep_toolchain.bzl | 220 ++++++++++++++++++++++++++++++ lib/private/sd_toolchain.bzl | 204 +++++++++++++++++++++++++++ lib/repositories.bzl | 56 ++++++++ lib/tests/ripgrep/BUILD.bazel | 19 +++ lib/tests/sd/BUILD.bazel | 19 +++ lib/toolchains.bzl | 15 ++ tools/ripgrep_mirror_release.sh | 58 ++++++++ tools/sd_mirror_release.sh | 58 ++++++++ 11 files changed, 682 insertions(+), 1 deletion(-) create mode 100644 lib/private/ripgrep_toolchain.bzl create mode 100644 lib/private/sd_toolchain.bzl create mode 100644 lib/tests/ripgrep/BUILD.bazel create mode 100644 lib/tests/sd/BUILD.bazel create mode 100644 lib/toolchains.bzl create mode 100755 tools/ripgrep_mirror_release.sh create mode 100755 tools/sd_mirror_release.sh diff --git a/internal_deps.bzl b/internal_deps.bzl index 33fec82f0..168f8e5bb 100644 --- a/internal_deps.bzl +++ b/internal_deps.bzl @@ -4,7 +4,7 @@ Users should *not* need to install these. If users see a load() statement from these, that's a bug in our distribution. """ -load("//lib:repositories.bzl", "register_bats_toolchains", "register_coreutils_toolchains", "register_jq_toolchains", "register_tar_toolchains", "register_yq_toolchains") +load("//lib:repositories.bzl", "register_bats_toolchains", "register_coreutils_toolchains", "register_jq_toolchains", "register_ripgrep_toolchains", "register_sd_toolchains", "register_tar_toolchains", "register_yq_toolchains") load("//lib:utils.bzl", http_archive = "maybe_http_archive") # buildifier: disable=unnamed-macro @@ -66,6 +66,8 @@ def bazel_lib_internal_deps(): # Register toolchains for tests register_jq_toolchains() + register_ripgrep_toolchains() + register_sd_toolchains() register_yq_toolchains() register_coreutils_toolchains() register_tar_toolchains() diff --git a/lib/BUILD.bazel b/lib/BUILD.bazel index c1d808754..fac7d1b9c 100644 --- a/lib/BUILD.bazel +++ b/lib/BUILD.bazel @@ -40,6 +40,14 @@ toolchain_type( name = "jq_toolchain_type", ) +toolchain_type( + name = "ripgrep_toolchain_type", +) + +toolchain_type( + name = "sd_toolchain_type", +) + toolchain_type( name = "yq_toolchain_type", ) diff --git a/lib/extensions.bzl b/lib/extensions.bzl index 635c9226d..8eb68cdaa 100644 --- a/lib/extensions.bzl +++ b/lib/extensions.bzl @@ -11,6 +11,10 @@ load( "DEFAULT_EXPAND_TEMPLATE_REPOSITORY", "DEFAULT_JQ_REPOSITORY", "DEFAULT_JQ_VERSION", + "DEFAULT_RIPGREP_REPOSITORY", + "DEFAULT_RIPGREP_VERSION", + "DEFAULT_SD_REPOSITORY", + "DEFAULT_SD_VERSION", "DEFAULT_TAR_REPOSITORY", "DEFAULT_YQ_REPOSITORY", "DEFAULT_YQ_VERSION", @@ -21,6 +25,8 @@ load( "register_coreutils_toolchains", "register_expand_template_toolchains", "register_jq_toolchains", + "register_ripgrep_toolchains", + "register_sd_toolchains", "register_tar_toolchains", "register_yq_toolchains", "register_zstd_toolchains", @@ -68,6 +74,20 @@ def _toolchains_extension_impl(mctx): toolchain_repos_fn = lambda name, version: register_jq_toolchains(name = name, version = version, register = False), ) + extension_utils.toolchain_repos_bfs( + mctx = mctx, + get_tag_fn = lambda tags: tags.ripgrep, + toolchain_name = "ripgrep", + toolchain_repos_fn = lambda name, version: register_ripgrep_toolchains(name = name, version = version, register = False), + ) + + extension_utils.toolchain_repos_bfs( + mctx = mctx, + get_tag_fn = lambda tags: tags.sd, + toolchain_name = "sd", + toolchain_repos_fn = lambda name, version: register_sd_toolchains(name = name, version = version, register = False), + ) + extension_utils.toolchain_repos_bfs( mctx = mctx, get_tag_fn = lambda tags: tags.yq, @@ -123,6 +143,8 @@ toolchains = module_extension( "copy_directory": tag_class(attrs = {"name": attr.string(default = DEFAULT_COPY_DIRECTORY_REPOSITORY)}), "copy_to_directory": tag_class(attrs = {"name": attr.string(default = DEFAULT_COPY_TO_DIRECTORY_REPOSITORY)}), "jq": tag_class(attrs = {"name": attr.string(default = DEFAULT_JQ_REPOSITORY), "version": attr.string(default = DEFAULT_JQ_VERSION)}), + "ripgrep": tag_class(attrs = {"name": attr.string(default = DEFAULT_RIPGREP_REPOSITORY), "version": attr.string(default = DEFAULT_RIPGREP_VERSION)}), + "sd": tag_class(attrs = {"name": attr.string(default = DEFAULT_SD_REPOSITORY), "version": attr.string(default = DEFAULT_SD_VERSION)}), "yq": tag_class(attrs = {"name": attr.string(default = DEFAULT_YQ_REPOSITORY), "version": attr.string(default = DEFAULT_YQ_VERSION)}), "coreutils": tag_class(attrs = {"name": attr.string(default = DEFAULT_COREUTILS_REPOSITORY), "version": attr.string(default = DEFAULT_COREUTILS_VERSION)}), "tar": tag_class(attrs = {"name": attr.string(default = DEFAULT_TAR_REPOSITORY)}), diff --git a/lib/private/ripgrep_toolchain.bzl b/lib/private/ripgrep_toolchain.bzl new file mode 100644 index 000000000..fdf390ba9 --- /dev/null +++ b/lib/private/ripgrep_toolchain.bzl @@ -0,0 +1,220 @@ +"Setup ripgrep (rg) toolchain repositories and rules" + +# Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl +RIPGREP_PLATFORMS = { + "darwin_amd64": struct( + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:x86_64", + ], + ), + "darwin_arm64": struct( + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:aarch64", + ], + ), + "linux_amd64": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + ), + "linux_arm64": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], + ), + "windows_amd64": struct( + compatible_with = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], + ), +} + +DEFAULT_RIPGREP_VERSION = "14.1.0" + +# https://github.com/BurntSushi/ripgrep +# +# The integrity hashes can be automatically fetched for the ripgrep releases by running +# `tools/ripgrep_mirror_release.sh`. +RIPGREP_VERSIONS = { + "14.1.0": { + "darwin_arm64": { + "filename": "ripgrep-14.1.0-aarch64-apple-darwin.tar.gz", + "sha256": "sha256-/FnKPqpbW8+hSI7rgCkbrQ6OKELgXUQA/Hsp1e5L0ms=" + }, + "linux_arm64": { + "filename": "ripgrep-14.1.0-aarch64-unknown-linux-gnu.tar.gz", + "sha256": "sha256-yMIQuZhE+/FrejbRyWPoNRvKX/LdfHiPX7pKwYuoxg0=" + }, + "musleabi_linux_armv7": { + "filename": "ripgrep-14.1.0-armv7-unknown-linux-musleabi.tar.gz", + "sha256": "sha256-FgQVP5gg+ExAiSWJQSncCyF3DHINj8vzmsSDknqawpg=" + }, + "musleabihf_linux_armv7": { + "filename": "ripgrep-14.1.0-armv7-unknown-linux-musleabihf.tar.gz", + "sha256": "sha256-BWXgjITywVS+y/LK8JYBHY4ul7kqcbodERP0PTWbHqA=" + }, + "linux_powerpc64": { + "filename": "ripgrep-14.1.0-powerpc64-unknown-linux-gnu.tar.gz", + "sha256": "sha256-uCG3Mg2bekSYaNT/3ltc/ENp/bE9zzJUNb9PC1rkb+g=" + }, + "linux_s390x": { + "filename": "ripgrep-14.1.0-s390x-unknown-linux-gnu.tar.gz", + "sha256": "sha256-4Wb0qM1nAJkq+LqI62p8WrKDjkQG3x6PBW6cRVQxRpE=" + }, + "darwin_amd64": { + "filename": "ripgrep-14.1.0-x86_64-apple-darwin.tar.gz", + "sha256": "sha256-TYgvyUXlqbYIClwFBvf+Sq6hkcGGU1VCjdS/oJb5dLU=" + }, + "windows_amd64": { + "filename": "ripgrep-14.1.0-x86_64-pc-windows-msvc.zip", + "sha256": "sha256-/k917fqlDw1P7L9HaWt2KfNEnJwsWk2oKHUxOeWi4gM=" + }, + "linux_amd64": { + "filename": "ripgrep-14.1.0-x86_64-unknown-linux-musl.tar.gz", + "sha256": "sha256-+EdXsH9CX+XPEdh99mRGkcZEpc0jSKLGcIlCcpmdO6c=" + } + } +} + +RipgrepInfo = provider( + doc = "Provide info for executing ripgrep", + fields = { + "bin": "Executable ripgrep binary", + }, +) + +def _ripgrep_toolchain_impl(ctx): + binary = ctx.file.binary + + # Make the $(RIPGREP_BIN) variable available in places like genrules. + # See https://docs.bazel.build/versions/main/be/make-variables.html#custom_variables + template_variables = platform_common.TemplateVariableInfo({ + "RIPGREP_BIN": binary.path, + }) + default_info = DefaultInfo( + files = depset([binary]), + runfiles = ctx.runfiles(files = [binary]), + ) + ripgrep_info = RipgrepInfo( + bin = binary, + ) + + # Export all the providers inside our ToolchainInfo + # so the resolved_toolchain rule can grab and re-export them. + toolchain_info = platform_common.ToolchainInfo( + ripgrep_info = ripgrep_info, + template_variables = template_variables, + default = default_info, + ) + + return [default_info, toolchain_info, template_variables] + +ripgrep_toolchain = rule( + implementation = _ripgrep_toolchain_impl, + attrs = { + "binary": attr.label( + mandatory = True, + allow_single_file = True, + ), + }, +) + +def _ripgrep_toolchains_repo_impl(rctx): + # Expose a concrete toolchain which is the result of Bazel resolving the toolchain + # for the execution or target platform. + # Workaround for https://github.com/bazelbuild/bazel/issues/14009 + starlark_content = """# @generated by @aspect_bazel_lib//lib/private:ripgrep_toolchain.bzl + +# Forward all the providers +def _resolved_toolchain_impl(ctx): + toolchain_info = ctx.toolchains["@aspect_bazel_lib//lib:ripgrep_toolchain_type"] + return [ + toolchain_info, + toolchain_info.default, + toolchain_info.ripgrep_info, + toolchain_info.template_variables, + ] + +# Copied from java_toolchain_alias +# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl +resolved_toolchain = rule( + implementation = _resolved_toolchain_impl, + toolchains = ["@aspect_bazel_lib//lib:ripgrep_toolchain_type"], + incompatible_use_toolchain_transition = True, +) +""" + rctx.file("defs.bzl", starlark_content) + + build_content = """# @generated by @aspect_bazel_lib//lib/private:ripgrep_toolchain.bzl +# +# These can be registered in the workspace file or passed to --extra_toolchains flag. +# By default all these toolchains are registered by the ripgrep_register_toolchains macro +# so you don't normally need to interact with these targets. + +load(":defs.bzl", "resolved_toolchain") + +resolved_toolchain(name = "resolved_toolchain", visibility = ["//visibility:public"]) + +""" + + for [platform, meta] in RIPGREP_PLATFORMS.items(): + build_content += """ +toolchain( + name = "{platform}_toolchain", + exec_compatible_with = {compatible_with}, + toolchain = "@{user_repository_name}_{platform}//:ripgrep_toolchain", + toolchain_type = "@aspect_bazel_lib//lib:ripgrep_toolchain_type", +) +""".format( + platform = platform, + user_repository_name = rctx.attr.user_repository_name, + compatible_with = meta.compatible_with, + ) + + # Base BUILD file for this repository + rctx.file("BUILD.bazel", build_content) + +ripgrep_toolchains_repo = repository_rule( + _ripgrep_toolchains_repo_impl, + doc = """Creates a repository with toolchain definitions for all known platforms + which can be registered or selected.""", + attrs = { + "user_repository_name": attr.string(doc = "Base name for toolchains repository"), + }, +) + +def _ripgrep_platform_repo_impl(rctx): + is_windows = rctx.attr.platform.startswith("windows_") + platform = rctx.attr.platform + filename = RIPGREP_VERSIONS[rctx.attr.version][platform]["filename"] + url = "https://github.com/BurntSushi/ripgrep/releases/download/{}/{}".format( + rctx.attr.version, + filename, + ) + rctx.download_and_extract( + url = url, + stripPrefix = filename.replace(".zip", "").replace(".tar.gz", ""), + integrity = RIPGREP_VERSIONS[rctx.attr.version][platform]["sha256"], + ) + build_content = """# @generated by @aspect_bazel_lib//lib/private:ripgrep_toolchain.bzl +load("@aspect_bazel_lib//lib/private:ripgrep_toolchain.bzl", "ripgrep_toolchain") +exports_files(["{0}"]) +ripgrep_toolchain(name = "ripgrep_toolchain", binary = "{0}", visibility = ["//visibility:public"]) +""".format("rg.exe" if is_windows else "rg") + + # Base BUILD file for this repository + rctx.file("BUILD.bazel", build_content) + +ripgrep_platform_repo = repository_rule( + implementation = _ripgrep_platform_repo_impl, + doc = "Fetch external tools needed for ripgrep toolchain", + attrs = { + "version": attr.string(mandatory = True, values = RIPGREP_VERSIONS.keys()), + "platform": attr.string(mandatory = True, values = RIPGREP_PLATFORMS.keys()), + }, +) diff --git a/lib/private/sd_toolchain.bzl b/lib/private/sd_toolchain.bzl new file mode 100644 index 000000000..f4fd20b55 --- /dev/null +++ b/lib/private/sd_toolchain.bzl @@ -0,0 +1,204 @@ +"Setup search & displace (sd) toolchain repositories and rules" + +# Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl +SD_PLATFORMS = { + "darwin_amd64": struct( + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:x86_64", + ], + ), + "darwin_arm64": struct( + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:aarch64", + ], + ), + "linux_amd64": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + ), + "linux_arm64": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], + ), + "windows_amd64": struct( + compatible_with = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], + ), +} + +DEFAULT_SD_VERSION = "v1.0.0" + +# https://github.com/chmln/sd +# +# The integrity hashes can be automatically fetched for the sd releases by running +# `tools/sd_mirror_release.sh`. +SD_VERSIONS = { + "v1.0.0": { + "darwin_arm64": { + "filename": "sd-v1.0.0-aarch64-apple-darwin.tar.gz", + "sha256": "sha256-p2Ssm5Zsbwz+qmbyrS1JRbCPjZPZc+0aYry/vhwn6YE=" + }, + "linux_arm64": { + "filename": "sd-v1.0.0-aarch64-unknown-linux-musl.tar.gz", + "sha256": "sha256-npn46d1wxwFbjCDkknC92Z6cr5ZqQaEZgvTXhfaljJ4=" + }, + "darwin_amd64": { + "filename": "sd-v1.0.0-x86_64-apple-darwin.tar.gz", + "sha256": "sha256-hAjrg8eYjNxyr8TRcXlSAZmbpNMhx3KJIf+FVlkV510=" + }, + "windows_amd64": { + "filename": "sd-v1.0.0-x86_64-pc-windows-msvc.zip", + "sha256": "sha256-t8yZN+vB9zbIfVkpueoigAU68Q+OMg2cIMqQw69GpdU=" + }, + "linux_amd64": { + "filename": "sd-v1.0.0-x86_64-unknown-linux-musl.tar.gz", + "sha256": "sha256-n0Lk/seEj6jW7qt7EJD1ycnjdMlKmXTbb/M98FLJ4TI=" + } + } +} + +SdInfo = provider( + doc = "Provide info for executing sd", + fields = { + "bin": "Executable sd binary", + }, +) + +def _sd_toolchain_impl(ctx): + binary = ctx.file.binary + + # Make the $(SD_BIN) variable available in places like genrules. + # See https://docs.bazel.build/versions/main/be/make-variables.html#custom_variables + template_variables = platform_common.TemplateVariableInfo({ + "SD_BIN": binary.path, + }) + default_info = DefaultInfo( + files = depset([binary]), + runfiles = ctx.runfiles(files = [binary]), + ) + sd_info = SdInfo( + bin = binary, + ) + + # Export all the providers inside our ToolchainInfo + # so the resolved_toolchain rule can grab and re-export them. + toolchain_info = platform_common.ToolchainInfo( + sd_info = sd_info, + template_variables = template_variables, + default = default_info, + ) + + return [default_info, toolchain_info, template_variables] + +sd_toolchain = rule( + implementation = _sd_toolchain_impl, + attrs = { + "binary": attr.label( + mandatory = True, + allow_single_file = True, + ), + }, +) + +def _sd_toolchains_repo_impl(rctx): + # Expose a concrete toolchain which is the result of Bazel resolving the toolchain + # for the execution or target platform. + # Workaround for https://github.com/bazelbuild/bazel/issues/14009 + starlark_content = """# @generated by @aspect_bazel_lib//lib/private:sd_toolchain.bzl + +# Forward all the providers +def _resolved_toolchain_impl(ctx): + toolchain_info = ctx.toolchains["@aspect_bazel_lib//lib:sd_toolchain_type"] + return [ + toolchain_info, + toolchain_info.default, + toolchain_info.sd_info, + toolchain_info.template_variables, + ] + +# Copied from java_toolchain_alias +# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl +resolved_toolchain = rule( + implementation = _resolved_toolchain_impl, + toolchains = ["@aspect_bazel_lib//lib:sd_toolchain_type"], + incompatible_use_toolchain_transition = True, +) +""" + rctx.file("defs.bzl", starlark_content) + + build_content = """# @generated by @aspect_bazel_lib//lib/private:sd_toolchain.bzl +# +# These can be registered in the workspace file or passed to --extra_toolchains flag. +# By default all these toolchains are registered by the sd_register_toolchains macro +# so you don't normally need to interact with these targets. + +load(":defs.bzl", "resolved_toolchain") + +resolved_toolchain(name = "resolved_toolchain", visibility = ["//visibility:public"]) + +""" + + for [platform, meta] in SD_PLATFORMS.items(): + build_content += """ +toolchain( + name = "{platform}_toolchain", + exec_compatible_with = {compatible_with}, + toolchain = "@{user_repository_name}_{platform}//:sd_toolchain", + toolchain_type = "@aspect_bazel_lib//lib:sd_toolchain_type", +) +""".format( + platform = platform, + user_repository_name = rctx.attr.user_repository_name, + compatible_with = meta.compatible_with, + ) + + # Base BUILD file for this repository + rctx.file("BUILD.bazel", build_content) + +sd_toolchains_repo = repository_rule( + _sd_toolchains_repo_impl, + doc = """Creates a repository with toolchain definitions for all known platforms + which can be registered or selected.""", + attrs = { + "user_repository_name": attr.string(doc = "Base name for toolchains repository"), + }, +) + +def _sd_platform_repo_impl(rctx): + is_windows = rctx.attr.platform.startswith("windows_") + platform = rctx.attr.platform + filename = SD_VERSIONS[rctx.attr.version][platform]["filename"] + url = "https://github.com/chmln/sd/releases/download/{}/{}".format( + rctx.attr.version, + filename, + ) + rctx.download_and_extract( + url = url, + stripPrefix = filename.replace(".zip", "").replace(".tar.gz", ""), + integrity = SD_VERSIONS[rctx.attr.version][platform]["sha256"], + ) + build_content = """# @generated by @aspect_bazel_lib//lib/private:sd_toolchain.bzl +load("@aspect_bazel_lib//lib/private:sd_toolchain.bzl", "sd_toolchain") +exports_files(["{0}"]) +sd_toolchain(name = "sd_toolchain", binary = "{0}", visibility = ["//visibility:public"]) +""".format("sd.exe" if is_windows else "sd") + + # Base BUILD file for this repository + rctx.file("BUILD.bazel", build_content) + +sd_platform_repo = repository_rule( + implementation = _sd_platform_repo_impl, + doc = "Fetch external tools needed for sd toolchain", + attrs = { + "version": attr.string(mandatory = True, values = SD_VERSIONS.keys()), + "platform": attr.string(mandatory = True, values = SD_PLATFORMS.keys()), + }, +) diff --git a/lib/repositories.bzl b/lib/repositories.bzl index 1562438b0..e2de34d4c 100644 --- a/lib/repositories.bzl +++ b/lib/repositories.bzl @@ -7,6 +7,8 @@ load("//lib/private:copy_to_directory_toolchain.bzl", "COPY_TO_DIRECTORY_PLATFOR load("//lib/private:coreutils_toolchain.bzl", "COREUTILS_PLATFORMS", "coreutils_platform_repo", "coreutils_toolchains_repo", _DEFAULT_COREUTILS_VERSION = "DEFAULT_COREUTILS_VERSION") load("//lib/private:expand_template_toolchain.bzl", "EXPAND_TEMPLATE_PLATFORMS", "expand_template_platform_repo", "expand_template_toolchains_repo") load("//lib/private:jq_toolchain.bzl", "JQ_PLATFORMS", "jq_host_alias_repo", "jq_platform_repo", "jq_toolchains_repo", _DEFAULT_JQ_VERSION = "DEFAULT_JQ_VERSION") +load("//lib/private:ripgrep_toolchain.bzl", "RIPGREP_PLATFORMS", "ripgrep_platform_repo", "ripgrep_toolchains_repo", _DEFAULT_RIPGREP_VERSION = "DEFAULT_RIPGREP_VERSION") +load("//lib/private:sd_toolchain.bzl", "SD_PLATFORMS", "sd_platform_repo", "sd_toolchains_repo", _DEFAULT_SD_VERSION = "DEFAULT_SD_VERSION") load("//lib/private:source_toolchains_repo.bzl", "source_toolchains_repo") load("//lib/private:tar_toolchain.bzl", "BSDTAR_PLATFORMS", "bsdtar_binary_repo", "tar_toolchains_repo") load("//lib/private:yq_toolchain.bzl", "YQ_PLATFORMS", "yq_host_alias_repo", "yq_platform_repo", "yq_toolchains_repo", _DEFAULT_YQ_VERSION = "DEFAULT_YQ_VERSION") @@ -54,6 +56,58 @@ def register_jq_toolchains(name = DEFAULT_JQ_REPOSITORY, version = DEFAULT_JQ_VE user_repository_name = name, ) +DEFAULT_RIPGREP_REPOSITORY = "ripgrep" +DEFAULT_RIPGREP_VERSION = _DEFAULT_RIPGREP_VERSION + +def register_ripgrep_toolchains(name = DEFAULT_RIPGREP_REPOSITORY, version = DEFAULT_RIPGREP_VERSION, register = True): + """Registers ripgrep (rg) toolchain and repositories + + Args: + name: override the prefix for the generated toolchain repositories + version: the version of ripgrep to execute (see https://github.com/BurntSushi/ripgrep/releases) + register: whether to call through to native.register_toolchains. + Should be True for WORKSPACE users, but false when used under bzlmod extension + """ + for [platform, meta] in RIPGREP_PLATFORMS.items(): + ripgrep_platform_repo( + name = "%s_%s" % (name, platform), + platform = platform, + version = version, + ) + if register: + native.register_toolchains("@%s_toolchains//:%s_toolchain" % (name, platform)) + + ripgrep_toolchains_repo( + name = "%s_toolchains" % name, + user_repository_name = name, + ) + +DEFAULT_SD_REPOSITORY = "sd" +DEFAULT_SD_VERSION = _DEFAULT_SD_VERSION + +def register_sd_toolchains(name = DEFAULT_SD_REPOSITORY, version = DEFAULT_SD_VERSION, register = True): + """Registers search & displace (sd) toolchain and repositories + + Args: + name: override the prefix for the generated toolchain repositories + version: the version of sd to execute (see https://github.com/chmln/sd/releases) + register: whether to call through to native.register_toolchains. + Should be True for WORKSPACE users, but false when used under bzlmod extension + """ + for [platform, meta] in SD_PLATFORMS.items(): + sd_platform_repo( + name = "%s_%s" % (name, platform), + platform = platform, + version = version, + ) + if register: + native.register_toolchains("@%s_toolchains//:%s_toolchain" % (name, platform)) + + sd_toolchains_repo( + name = "%s_toolchains" % name, + user_repository_name = name, + ) + DEFAULT_YQ_REPOSITORY = "yq" DEFAULT_YQ_VERSION = _DEFAULT_YQ_VERSION @@ -346,6 +400,8 @@ def aspect_bazel_lib_register_toolchains(): register_expand_template_toolchains() register_coreutils_toolchains() register_jq_toolchains() + register_ripgrep_toolchains() + register_sd_toolchains() register_yq_toolchains() register_tar_toolchains() register_zstd_toolchains() diff --git a/lib/tests/ripgrep/BUILD.bazel b/lib/tests/ripgrep/BUILD.bazel new file mode 100644 index 000000000..dfbe5c630 --- /dev/null +++ b/lib/tests/ripgrep/BUILD.bazel @@ -0,0 +1,19 @@ +load("//lib:testing.bzl", "assert_contains") + +genrule( + name = "ripgrep", + srcs = [], + outs = ["ripgrep.txt"], + cmd = """echo ripgrep | $(RIPGREP_BIN) ripgrep > $@""", + toolchains = ["@ripgrep_toolchains//:resolved_toolchain"], +) + +assert_contains( + name = "test_ripgrep", + actual = "ripgrep.txt", + expected = """ripgrep""", + target_compatible_with = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), +) diff --git a/lib/tests/sd/BUILD.bazel b/lib/tests/sd/BUILD.bazel new file mode 100644 index 000000000..7354b3fc5 --- /dev/null +++ b/lib/tests/sd/BUILD.bazel @@ -0,0 +1,19 @@ +load("//lib:testing.bzl", "assert_contains") + +genrule( + name = "sd", + srcs = [], + outs = ["sd.txt"], + cmd = """echo sed | $(SD_BIN) sed sd > $@""", + toolchains = ["@sd_toolchains//:resolved_toolchain"], +) + +assert_contains( + name = "sd_ripgrep", + actual = "sd.txt", + expected = """sd""", + target_compatible_with = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), +) diff --git a/lib/toolchains.bzl b/lib/toolchains.bzl new file mode 100644 index 000000000..f019f3d53 --- /dev/null +++ b/lib/toolchains.bzl @@ -0,0 +1,15 @@ +"Public API" + +load("//lib/private:coreutils_toolchain.bzl", _coreutils_toolchain = "coreutils_toolchain") +load("//lib/private:jq_toolchain.bzl", _jq_toolchain = "jq_toolchain") +load("//lib/private:ripgrep_toolchain.bzl", _ripgrep_toolchain = "ripgrep_toolchain") +load("//lib/private:sd_toolchain.bzl", _sd_toolchain = "sd_toolchain") +load("//lib/private:tar_toolchain.bzl", _tar_toolchain = "tar_toolchain") +load("//lib/private:yq_toolchain.bzl", _yq_toolchain = "yq_toolchain") + +coreutils_toolchain = _coreutils_toolchain +jq_toolchain = _jq_toolchain +ripgrep_toolchain = _ripgrep_toolchain +sd_toolchain = _sd_toolchain +tar_toolchain = _tar_toolchain +yq_toolchain = _yq_toolchain diff --git a/tools/ripgrep_mirror_release.sh b/tools/ripgrep_mirror_release.sh new file mode 100755 index 000000000..e5c2fe0c5 --- /dev/null +++ b/tools/ripgrep_mirror_release.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -o errexit -o nounset -o pipefail + +JQ_FILTER='map({ + "key": .tag_name, + "value": .assets + | map(select( + (.name | startswith("ripgrep-")) and + ((.name | endswith(".tar.gz")) or (.name | endswith(".zip"))) and + (.name | contains("i686") | not) and + ( + ( (.name | contains("windows")) and (.name | contains("gnu") | not) ) or + ( (.name | contains("windows") | not) and ( (.name | contains("gnu")) or (.name | contains("musl")) ) and (.name | contains("gnueabihf") | not) ) or + ( .name | contains("darwin") ) + ) + )) + | map({ + key: .name | + ltrimstr("ripgrep-") | + rtrimstr(".tar.gz") | + rtrimstr(".zip") | + sub("-pc"; "") | + sub("-apple"; "") | + sub("-unknown"; "") | + sub("x86_64"; "amd64") | + sub("aarch64"; "arm64") | + gsub("\\d+.\\d+.\\d+-"; "") | + rtrimstr("-msvc") | + rtrimstr("-gnu") | + rtrimstr("-musl") | + split("-") | + reverse | + join("_"), + value: { + filename: .name, + sha256: "sha256-", + } + }) + | from_entries +}) | from_entries +' + +INFO="$(curl --silent -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/BurntSushi/ripgrep/releases?per_page=1 | jq "$JQ_FILTER")" + +for VERSION in $(jq -r 'keys | join("\n")' <<<$INFO); do + for PLATFORM in $(jq -r ".[\"$VERSION\"] | keys | join(\"\n\")" <<<$INFO); do + FILENAME=$(jq -r ".[\"$VERSION\"][\"$PLATFORM\"].filename" <<<$INFO) + SHA256=$(curl -fLs "https://github.com/BurntSushi/ripgrep/releases/download/$VERSION/$FILENAME" | sha256sum | xxd -r -p | base64) + INFO=$(jq ".[\"$VERSION\"][\"$PLATFORM\"].sha256 = \"sha256-$SHA256\"" <<<$INFO) + done +done + +echo -n "RIPGREP_VERSIONS = " +echo $INFO | jq -M + +echo "" +echo "Copy the version info into lib/private/ripgrep_toolchain.bzl" diff --git a/tools/sd_mirror_release.sh b/tools/sd_mirror_release.sh new file mode 100755 index 000000000..e04f575ec --- /dev/null +++ b/tools/sd_mirror_release.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -o errexit -o nounset -o pipefail + +JQ_FILTER='map({ + "key": .tag_name, + "value": .assets + | map(select( + (.name | startswith("sd-")) and + ((.name | endswith(".tar.gz")) or (.name | endswith(".zip"))) and + (.name | contains("i686") | not) and + ( + ( (.name | contains("windows")) and (.name | contains("gnu") | not) ) or + ( (.name | contains("windows") | not) and ( (.name | contains("gnu")) or (.name | contains("musl")) ) and (.name | contains("gnueabihf") | not) ) or + ( .name | contains("darwin") ) + ) + )) + | map({ + key: .name | + ltrimstr("sd-v") | + rtrimstr(".tar.gz") | + rtrimstr(".zip") | + sub("-pc"; "") | + sub("-apple"; "") | + sub("-unknown"; "") | + sub("x86_64"; "amd64") | + sub("aarch64"; "arm64") | + gsub("\\d+.\\d+.\\d+-"; "") | + rtrimstr("-msvc") | + rtrimstr("-gnu") | + rtrimstr("-musl") | + split("-") | + reverse | + join("_"), + value: { + filename: .name, + sha256: "sha256-", + } + }) + | from_entries +}) | from_entries +' + +INFO="$(curl --silent -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/chmln/sd/releases?per_page=1 | jq "$JQ_FILTER")" + +for VERSION in $(jq -r 'keys | join("\n")' <<<$INFO); do + for PLATFORM in $(jq -r ".[\"$VERSION\"] | keys | join(\"\n\")" <<<$INFO); do + FILENAME=$(jq -r ".[\"$VERSION\"][\"$PLATFORM\"].filename" <<<$INFO) + SHA256=$(curl -fLs "https://github.com/chmln/sd/releases/download/$VERSION/$FILENAME" | sha256sum | xxd -r -p | base64) + INFO=$(jq ".[\"$VERSION\"][\"$PLATFORM\"].sha256 = \"sha256-$SHA256\"" <<<$INFO) + done +done + +echo -n "SD_VERSIONS = " +echo $INFO | jq -M + +echo "" +echo "Copy the version info into lib/private/sd_toolchain.bzl"