bazel-contrib · thesayyn · Sep 24, 2023 · Sep 22, 2023 · Sep 22, 2023 · Sep 22, 2023
diff --git a/.bazelrc b/.bazelrc
@@ -19,10 +19,6 @@ build --embed_label=v1.2.3
 # Mock versioning command to test the --stamp behavior
 build --workspace_status_command="echo BUILD_SCM_VERSION 1.2.3"
 
-# For testing always use toolchains from source
-build --@aspect_bazel_lib//tools:use_source_toolchains
-build --extra_toolchains=//tools:all
-
 # For releasing, use --workspace_status_command and stamp
 common:release --workspace_status_command "${PWD}/workspace_status.sh"
 common:release -c opt

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -27,11 +27,6 @@ jobs:
  # Bazelisk will download bazel to here
  XDG_CACHE_HOME: ~/.cache/bazel-repo
  run: bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc test //...
- - name: Check Versions
- env:
- # Bazelisk will download bazel to here
- XDG_CACHE_HOME: ~/.cache/bazel-repo
- run: ./scripts/check_versions.sh ${{ env.GITHUB_REF_NAME }}
  - name: Build release artifacts
  run: |
  if [ -n "$(git status --porcelain)" ]; then

diff --git a/.github/workflows/release_prep.sh b/.github/workflows/release_prep.sh
@@ -7,6 +7,9 @@ set -o errexit -o nounset -o pipefail
 cat >.git/info/attributes <<EOF
 # Omit folders that users don't need, making the distribution artifact smaller
 lib/tests export-ignore
+
+# Substitution for the _VERSION_PRIVATE placeholder at the top of this file
+tools/version.bzl export-subst
 EOF
 
 # Set by GH actions, see

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -38,11 +38,3 @@ repos:
  exclude: &exclude_pattern "^(docs|e2e|lib/tests)/"
  - id: mixed-line-ending
  - id: trailing-whitespace
-
- - repo: local
- hooks:
- - id: check-versions
- name: Check Versions
- entry: ./scripts/check_versions.sh
- language: script
- stages: [pre-push]
diff --git a/MODULE.bazel b/MODULE.bazel
@@ -25,6 +25,18 @@ register_toolchains(
  "@expand_template_toolchains//:all",
 )
 
+# To allow /tools to be built from source
+bazel_dep(name = "rules_go", version = "0.41.0", repo_name = "io_bazel_rules_go")
+bazel_dep(name = "gazelle", version = "0.33.0")
+
+go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
+go_deps.from_file(go_mod = "//:go.mod")
+use_repo(
+ go_deps,
+ "com_github_bmatcuk_doublestar_v4",
+ "org_golang_x_exp",
+)
+
 # Development-only dependencies
 
 bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.4.1", dev_dependency = True)

diff --git a/WORKSPACE b/WORKSPACE
@@ -29,22 +29,27 @@ load("//lib:host_repo.bzl", "host_repo")
 
 host_repo(name = "aspect_bazel_lib_host")
 
+############################################
+# rules_go
+
 load("//:deps.bzl", "go_dependencies")
 
 # gazelle:repository_macro deps.bzl%go_dependencies
 # gazelle:repository go_repository name=org_golang_x_tools importpath=golang.org/x/tools
 # https://github.com/bazelbuild/bazel-gazelle/issues/1217#issuecomment-1152236735
 go_dependencies()
 
-############################################
-# Gazelle, for generating bzl_library targets
 load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
-load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
 
 go_rules_dependencies()
 
 go_register_toolchains(version = "1.18.3")
 
+############################################
+# Gazelle, for generating bzl_library targets
+
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+
 gazelle_dependencies()
 
 # Buildifier

diff --git a/deps.bzl b/deps.bzl
@@ -24,8 +24,8 @@ def go_dependencies():
  name = "org_golang_x_exp",
  build_file_proto_mode = "disable_global",
  importpath = "golang.org/x/exp",
- sum = "h1:m9O6OTJ627iFnN2JIWfdqlZCzneRO6EEBsHXI25P8ws=",
- version = "v0.0.0-20221230185412-738e83a70c30",
+ sum = "h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw=",
+ version = "v0.0.0-20230713183714-613f0c0eb8a1",
  )
  go_repository(
  name = "org_golang_x_mod",

diff --git a/e2e/copy_to_directory/WORKSPACE b/e2e/copy_to_directory/WORKSPACE
@@ -6,3 +6,24 @@ local_repository(
 load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies")
 
 aspect_bazel_lib_dependencies()
+
+############################################
+# rules_go is needed to consume tools from sources
+
+load("@aspect_bazel_lib//:internal_deps.bzl", "bazel_lib_internal_deps")
+
+bazel_lib_internal_deps()
+
+load("@aspect_bazel_lib//:deps.bzl", "go_dependencies")
+
+go_dependencies()
+
+load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
+
+go_rules_dependencies()
+
+go_register_toolchains(version = "1.18.3")
+
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+
+gazelle_dependencies()
diff --git a/e2e/smoke/WORKSPACE b/e2e/smoke/WORKSPACE
@@ -10,3 +10,24 @@ aspect_bazel_lib_dependencies()
 register_jq_toolchains()
 
 register_yq_toolchains()
+
+############################################
+# rules_go is needed to consume tools from sources
+
+load("@aspect_bazel_lib//:internal_deps.bzl", "bazel_lib_internal_deps")
+
+bazel_lib_internal_deps()
+
+load("@aspect_bazel_lib//:deps.bzl", "go_dependencies")
+
+go_dependencies()
+
+load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
+
+go_rules_dependencies()
+
+go_register_toolchains(version = "1.18.3")
+
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+
+gazelle_dependencies()
diff --git a/lib/BUILD.bazel b/lib/BUILD.bazel
@@ -235,6 +235,7 @@ bzl_library(
  "//lib/private/docs:expand_template_toolchain",
  "//lib/private/docs:jq_toolchain",
  "//lib/private/docs:local_config_platform",
+ "//lib/private/docs:source_toolchains_repo",
  "//lib/private/docs:yq_toolchain",
  ],
 )

diff --git a/lib/private/copy_directory_toolchain.bzl b/lib/private/copy_directory_toolchain.bzl
@@ -6,7 +6,8 @@
 # `tools/copy_directory/mirror_release.sh`. To calculate for a specific release run
 # `tools/copy_directory/mirror_release.sh <release_version>`
 
-load("//tools:versions.bzl", "COPY_DIRECTORY_INTEGRITY", "COPY_DIRECTORY_VERSION")
+load("//tools:sri.bzl", "COPY_DIRECTORY_INTEGRITY")
+load("//tools:version.bzl", "VERSION")
 
 # Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl
 COPY_DIRECTORY_PLATFORMS = {
@@ -158,7 +159,7 @@ def _copy_directory_platform_repo_impl(rctx):
 
  # https://github.com/aspect-build/bazel-lib/releases/download/v1.19.0/copy_directory-linux_amd64
  url = "https://github.com/aspect-build/bazel-lib/releases/download/v{0}/copy_directory-{1}{2}".format(
- COPY_DIRECTORY_VERSION,
+ VERSION,
  release_platform,
  ".exe" if is_windows else "",
  )

diff --git a/lib/private/copy_to_directory_toolchain.bzl b/lib/private/copy_to_directory_toolchain.bzl
@@ -6,7 +6,8 @@
 # `tools/copy_to_directory/mirror_release.sh`. To calculate for a specific release run
 # `tools/copy_to_directory/mirror_release.sh <release_version>`
 
-load("//tools:versions.bzl", "COPY_TO_DIRECTORY_INTEGRITY", "COPY_TO_DIRECTORY_VERSION")
+load("//tools:sri.bzl", "COPY_TO_DIRECTORY_INTEGRITY")
+load("//tools:version.bzl", "VERSION")
 
 # Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl
 COPY_TO_DIRECTORY_PLATFORMS = {
@@ -158,7 +159,7 @@ def _copy_to_directory_platform_repo_impl(rctx):
 
  # https://github.com/aspect-build/bazel-lib/releases/download/v1.19.0/copy_to_directory-linux_amd64
  url = "https://github.com/aspect-build/bazel-lib/releases/download/v{0}/copy_to_directory-{1}{2}".format(
- COPY_TO_DIRECTORY_VERSION,
+ VERSION,
  release_platform,
  ".exe" if is_windows else "",
  )

diff --git a/lib/private/docs/BUILD.bazel b/lib/private/docs/BUILD.bazel
@@ -244,13 +244,24 @@ bzl_library(
 bzl_library(
  name = "copy_directory_toolchain",
  srcs = ["//lib/private:copy_directory_toolchain.bzl"],
- deps = ["//tools:versions"],
+ deps = [
+ "//tools:sri",
+ "//tools:version",
+ ],
 )
 
 bzl_library(
  name = "copy_to_directory_toolchain",
  srcs = ["//lib/private:copy_to_directory_toolchain.bzl"],
- deps = ["//tools:versions"],
+ deps = [
+ "//tools:sri",
+ "//tools:version",
+ ],
+)
+
+bzl_library(
+ name = "source_toolchains_repo",
+ srcs = ["//lib/private:source_toolchains_repo.bzl"],
 )
 
 bzl_library(
@@ -263,7 +274,7 @@ bzl_library(
  srcs = ["//lib/private:expand_template_toolchain.bzl"],
  deps = [
  "//lib:stamping",
- "//tools:versions",
+ "//tools:sri",
  ],
 )
 

diff --git a/lib/private/expand_template_toolchain.bzl b/lib/private/expand_template_toolchain.bzl
@@ -6,7 +6,8 @@
 # `tools/expand_template/mirror_release.sh`. To calculate for a specific release run
 # `tools/expand_template/mirror_release.sh <release_version>`
 
-load("//tools:versions.bzl", "EXPAND_TEMPLATE_INTEGRITY", "EXPAND_TEMPLATE_VERSION")
+load("//tools:sri.bzl", "EXPAND_TEMPLATE_INTEGRITY")
+load("//tools:version.bzl", "VERSION")
 
 # Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl
 EXPAND_TEMPLATE_PLATFORMS = {
@@ -158,7 +159,7 @@ def _expand_template_platform_repo_impl(rctx):
 
  # https://github.com/aspect-build/bazel-lib/releases/download/v1.19.0/expand_template-linux_amd64
  url = "https://github.com/aspect-build/bazel-lib/releases/download/v{0}/expand_template-{1}{2}".format(
- EXPAND_TEMPLATE_VERSION,
+ VERSION,
  release_platform,
  ".exe" if is_windows else "",
  )

diff --git a/lib/private/source_toolchains_repo.bzl b/lib/private/source_toolchains_repo.bzl
@@ -0,0 +1,90 @@
+"""Create a repository to hold the toolchains
+
+This follows guidance here:
+https://docs.bazel.build/versions/main/skylark/deploying.html#registering-toolchains
+"
+Note that in order to resolve toolchains in the analysis phase
+Bazel needs to analyze all toolchain targets that are registered.
+Bazel will not need to analyze all targets referenced by toolchain.toolchain attribute.
+If in order to register toolchains you need to perform complex computation in the repository,
+consider splitting the repository with toolchain targets
+from the repository with <LANG>_toolchain targets.
+Former will be always fetched,
+and the latter will only be fetched when user actually needs to build <LANG> code.
+"
+The "complex computation" in our case is simply downloading large artifacts.
+This guidance tells us how to avoid that: we put the toolchain targets in the alias repository
+with only the toolchain attribute pointing into the platform-specific repositories.
+"""
+
+# Add more platforms as needed to mirror all the binaries
+# published by the upstream project.
+
+DEFS_TMPL = """\
+# Generated by source_toolchains_repo.bzl for {toolchain_type}
+load("@bazel_skylib//lib:structs.bzl", "structs")
+
+# Forward all the providers
+def _resolved_toolchain_impl(ctx):
+ toolchain_info = ctx.toolchains["{toolchain_type}"]
+ return [toolchain_info] + structs.to_dict(toolchain_info).values()
+
+# Copied from java_toolchain_alias
+# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl
+resolved_toolchain = rule(
+ implementation = _resolved_toolchain_impl,
+ toolchains = ["{toolchain_type}"],
+ incompatible_use_toolchain_transition = True,
+)
+"""
+
+BUILD_TMPL = """\
+# Generated by source_toolchains_repo.bzl
+#
+# These can be registered in the workspace file or passed to --extra_toolchains flag.
+# By default all of these toolchains are registered by the oci_register_toolchains macro
+# so you don't normally need to interact with these targets.
+
+load(":defs.bzl", "resolved_toolchain")
+load("{toolchain_rule_load_from}", toolchain_rule = "{toolchain_rule}")
+
+resolved_toolchain(name = "current_toolchain", visibility = ["//visibility:public"])
+
+toolchain_rule(
+ name = "source",
+ bin = "{binary}",
+ visibility = ["//visibility:public"],
+)
+
+toolchain(
+ name = "toolchain",
+ toolchain = ":source",
+ toolchain_type = "{toolchain_type}",
+)
+"""
+
+def _source_toolchains_repo_impl(rctx):
+ # Expose a concrete toolchain which is the result of Bazel resolving the toolchain
+ # for the execution or target platform.
+ # Workaround for https://github.com/bazelbuild/bazel/issues/14009
+ rctx.file("defs.bzl", DEFS_TMPL.format(
+ toolchain_type = rctx.attr.toolchain_type,
+ ))
+
+ rctx.file("BUILD.bazel", BUILD_TMPL.format(
+ toolchain_type = rctx.attr.toolchain_type,
+ toolchain_rule_load_from = rctx.attr.toolchain_rule_load_from,
+ toolchain_rule = rctx.attr.toolchain_rule,
+ binary = rctx.attr.binary,
+ ))
+
+source_toolchains_repo = repository_rule(
+ _source_toolchains_repo_impl,
+ doc = "Creates a repository with toolchain definitions for source binaries.",
+ attrs = {
+ "toolchain_type": attr.string(doc = "Label to the toolchain_type", mandatory = True),
+ "toolchain_rule_load_from": attr.string(doc = "Label to the concrete toolchain rule to load from", mandatory = True),
+ "toolchain_rule": attr.string(doc = "Name of the concerete toolchain rule", mandatory = True),
+ "binary": attr.string(doc = "Label to the binary", mandatory = True),
+ },
+)