refactor: consume tools from source if unstamped (#543)

bazel-contrib · Sep 24, 2023 · ef364b5 · ef364b5
1 parent b60dc04
commit ef364b5
Show file tree

Hide file tree

Showing 26 changed files with 325 additions and 218 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -19,11 +19,10 @@ build --embed_label=v1.2.3
 # Mock versioning command to test the --stamp behavior
 build --workspace_status_command="echo BUILD_SCM_VERSION 1.2.3"
 
-# For testing always use toolchains from source
-build --@aspect_bazel_lib//tools:use_source_toolchains
-build --extra_toolchains=//tools:all
-
 # For releasing, use --workspace_status_command and stamp
+# before adding more flags to the release config make sure it does not
+# affect the hashes of /tools. See tools/release.bzl for opt transition
+# add appropriate commandline transition there to match the configuration.
 common:release --workspace_status_command "${PWD}/workspace_status.sh"
 common:release -c opt
 common:release --stamp

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -173,6 +173,10 @@ jobs:
         if: matrix.bzlmodEnabled
         run: echo "bzlmod_flag=--enable_bzlmod" >> $GITHUB_OUTPUT
 
+      - name: Make dev dependencies available (bzlmod)
+        if: matrix.bzlmodEnabled
+        run: sed -i 's/dev_dependency = True/dev_dependency = False/g' MODULE.bazel
+
       - name: Write rbe credentials
         if: ${{ matrix.config == 'rbe' }}
         working-directory: ${{ matrix.folder }}

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -22,16 +22,12 @@ jobs:
             ~/.cache/bazel-repo
           key: bazel-cache-${{ hashFiles('**/BUILD.bazel', '**/*.bzl', 'WORKSPACE') }}
           restore-keys: bazel-cache-
-      - name: bazel test //...
+      - name: bazel test //... (release)
         env:
           # Bazelisk will download bazel to here
           XDG_CACHE_HOME: ~/.cache/bazel-repo
-        run: bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc test //...
-      - name: Check Versions
-        env:
-          # Bazelisk will download bazel to here
-          XDG_CACHE_HOME: ~/.cache/bazel-repo
-        run: ./scripts/check_versions.sh ${{ env.GITHUB_REF_NAME }}
+        # Warning: DO NOT drop --config=release as it ensures that tools/integrity.bzl is up-to-date with release config.
+        run: bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc test //... --config=release
       - name: Build release artifacts
         run: |
           if [ -n "$(git status --porcelain)" ]; then

diff --git a/.github/workflows/release_prep.sh b/.github/workflows/release_prep.sh
@@ -7,6 +7,9 @@ set -o errexit -o nounset -o pipefail
 cat >.git/info/attributes <<EOF
 # Omit folders that users don't need, making the distribution artifact smaller
 lib/tests export-ignore
+
+# Substitution for the _VERSION_PRIVATE placeholder
+tools/version.bzl export-subst
 EOF
 
 # Set by GH actions, see

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -6,8 +6,6 @@
 # but we don't want to run the other hooks on commit messages
 default_stages: [commit]
 
-default_install_hook_types: ["pre-commit", "pre-push"]
-
 repos:
   # Check formatting and lint for starlark code
   - repo: https://github.com/keith/pre-commit-buildifier
@@ -38,11 +36,3 @@ repos:
         exclude: &exclude_pattern "^(docs|e2e|lib/tests)/"
       - id: mixed-line-ending
       - id: trailing-whitespace
-
-  - repo: local
-    hooks:
-      - id: check-versions
-        name: Check Versions
-        entry: ./scripts/check_versions.sh
-        language: script
-        stages: [pre-push]
diff --git a/MODULE.bazel b/MODULE.bazel
@@ -25,7 +25,23 @@ register_toolchains(
     "@expand_template_toolchains//:all",
 )
 
+# To allow /tools to be built from source
+bazel_dep(name = "rules_go", version = "0.41.0", dev_dependency = True, repo_name = "io_bazel_rules_go")
+
+go_deps = use_extension(
+    "@gazelle//:extensions.bzl",
+    "go_deps",
+    dev_dependency = True,
+)
+go_deps.from_file(go_mod = "//:go.mod")
+use_repo(
+    go_deps,
+    "com_github_bmatcuk_doublestar_v4",
+    "org_golang_x_exp",
+)
+
 # Development-only dependencies
 
 bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.4.1", dev_dependency = True)
 bazel_dep(name = "buildifier_prebuilt", version = "6.1.0", dev_dependency = True)
+bazel_dep(name = "gazelle", version = "0.33.0", dev_dependency = True)
diff --git a/WORKSPACE b/WORKSPACE
@@ -29,22 +29,27 @@ load("//lib:host_repo.bzl", "host_repo")
 
 host_repo(name = "aspect_bazel_lib_host")
 
+############################################
+# rules_go
+
 load("//:deps.bzl", "go_dependencies")
 
 # gazelle:repository_macro deps.bzl%go_dependencies
 # gazelle:repository go_repository name=org_golang_x_tools importpath=golang.org/x/tools
 # https://github.com/bazelbuild/bazel-gazelle/issues/1217#issuecomment-1152236735
 go_dependencies()
 
-############################################
-# Gazelle, for generating bzl_library targets
 load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
-load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
 
 go_rules_dependencies()
 
 go_register_toolchains(version = "1.18.3")
 
+############################################
+# Gazelle, for generating bzl_library targets
+
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+
 gazelle_dependencies()
 
 # Buildifier

diff --git a/deps.bzl b/deps.bzl
@@ -24,8 +24,8 @@ def go_dependencies():
         name = "org_golang_x_exp",
         build_file_proto_mode = "disable_global",
         importpath = "golang.org/x/exp",
-        sum = "h1:m9O6OTJ627iFnN2JIWfdqlZCzneRO6EEBsHXI25P8ws=",
-        version = "v0.0.0-20221230185412-738e83a70c30",
+        sum = "h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw=",
+        version = "v0.0.0-20230713183714-613f0c0eb8a1",
     )
     go_repository(
         name = "org_golang_x_mod",

diff --git a/e2e/copy_to_directory/WORKSPACE b/e2e/copy_to_directory/WORKSPACE
@@ -6,3 +6,24 @@ local_repository(
 load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies")
 
 aspect_bazel_lib_dependencies()
+
+############################################
+# rules_go is needed to consume tools from sources
+
+load("@aspect_bazel_lib//:internal_deps.bzl", "bazel_lib_internal_deps")
+
+bazel_lib_internal_deps()
+
+load("@aspect_bazel_lib//:deps.bzl", "go_dependencies")
+
+go_dependencies()
+
+load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
+
+go_rules_dependencies()
+
+go_register_toolchains(version = "1.18.3")
+
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+
+gazelle_dependencies()
diff --git a/e2e/smoke/WORKSPACE b/e2e/smoke/WORKSPACE
@@ -10,3 +10,24 @@ aspect_bazel_lib_dependencies()
 register_jq_toolchains()
 
 register_yq_toolchains()
+
+############################################
+# rules_go is needed to consume tools from sources
+
+load("@aspect_bazel_lib//:internal_deps.bzl", "bazel_lib_internal_deps")
+
+bazel_lib_internal_deps()
+
+load("@aspect_bazel_lib//:deps.bzl", "go_dependencies")
+
+go_dependencies()
+
+load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
+
+go_rules_dependencies()
+
+go_register_toolchains(version = "1.18.3")
+
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+
+gazelle_dependencies()
diff --git a/lib/BUILD.bazel b/lib/BUILD.bazel
@@ -235,6 +235,7 @@ bzl_library(
         "//lib/private/docs:expand_template_toolchain",
         "//lib/private/docs:jq_toolchain",
         "//lib/private/docs:local_config_platform",
+        "//lib/private/docs:source_toolchains_repo",
         "//lib/private/docs:yq_toolchain",
     ],
 )

diff --git a/lib/private/copy_directory_toolchain.bzl b/lib/private/copy_directory_toolchain.bzl
@@ -6,7 +6,8 @@
 # `tools/copy_directory/mirror_release.sh`. To calculate for a specific release run
 # `tools/copy_directory/mirror_release.sh <release_version>`
 
-load("//tools:versions.bzl", "COPY_DIRECTORY_INTEGRITY", "COPY_DIRECTORY_VERSION")
+load("//tools:integrity.bzl", "COPY_DIRECTORY_INTEGRITY")
+load("//tools:version.bzl", "VERSION")
 
 # Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl
 COPY_DIRECTORY_PLATFORMS = {
@@ -158,7 +159,7 @@ def _copy_directory_platform_repo_impl(rctx):
 
     # https://github.com/aspect-build/bazel-lib/releases/download/v1.19.0/copy_directory-linux_amd64
     url = "https://github.com/aspect-build/bazel-lib/releases/download/v{0}/copy_directory-{1}{2}".format(
-        COPY_DIRECTORY_VERSION,
+        VERSION,
         release_platform,
         ".exe" if is_windows else "",
     )

diff --git a/lib/private/copy_to_directory_toolchain.bzl b/lib/private/copy_to_directory_toolchain.bzl
@@ -6,7 +6,8 @@
 # `tools/copy_to_directory/mirror_release.sh`. To calculate for a specific release run
 # `tools/copy_to_directory/mirror_release.sh <release_version>`
 
-load("//tools:versions.bzl", "COPY_TO_DIRECTORY_INTEGRITY", "COPY_TO_DIRECTORY_VERSION")
+load("//tools:integrity.bzl", "COPY_TO_DIRECTORY_INTEGRITY")
+load("//tools:version.bzl", "VERSION")
 
 # Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl
 COPY_TO_DIRECTORY_PLATFORMS = {
@@ -158,7 +159,7 @@ def _copy_to_directory_platform_repo_impl(rctx):
 
     # https://github.com/aspect-build/bazel-lib/releases/download/v1.19.0/copy_to_directory-linux_amd64
     url = "https://github.com/aspect-build/bazel-lib/releases/download/v{0}/copy_to_directory-{1}{2}".format(
-        COPY_TO_DIRECTORY_VERSION,
+        VERSION,
         release_platform,
         ".exe" if is_windows else "",
     )

diff --git a/lib/private/docs/BUILD.bazel b/lib/private/docs/BUILD.bazel
@@ -244,13 +244,24 @@ bzl_library(
 bzl_library(
     name = "copy_directory_toolchain",
     srcs = ["//lib/private:copy_directory_toolchain.bzl"],
-    deps = ["//tools:versions"],
+    deps = [
+        "//tools:sri",
+        "//tools:version",
+    ],
 )
 
 bzl_library(
     name = "copy_to_directory_toolchain",
     srcs = ["//lib/private:copy_to_directory_toolchain.bzl"],
-    deps = ["//tools:versions"],
+    deps = [
+        "//tools:sri",
+        "//tools:version",
+    ],
+)
+
+bzl_library(
+    name = "source_toolchains_repo",
+    srcs = ["//lib/private:source_toolchains_repo.bzl"],
 )
 
 bzl_library(
@@ -263,7 +274,7 @@ bzl_library(
     srcs = ["//lib/private:expand_template_toolchain.bzl"],
     deps = [
         "//lib:stamping",
-        "//tools:versions",
+        "//tools:sri",
     ],
 )
 

diff --git a/lib/private/expand_template_toolchain.bzl b/lib/private/expand_template_toolchain.bzl
@@ -6,7 +6,8 @@
 # `tools/expand_template/mirror_release.sh`. To calculate for a specific release run
 # `tools/expand_template/mirror_release.sh <release_version>`
 
-load("//tools:versions.bzl", "EXPAND_TEMPLATE_INTEGRITY", "EXPAND_TEMPLATE_VERSION")
+load("//tools:integrity.bzl", "EXPAND_TEMPLATE_INTEGRITY")
+load("//tools:version.bzl", "VERSION")
 
 # Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl
 EXPAND_TEMPLATE_PLATFORMS = {
@@ -158,7 +159,7 @@ def _expand_template_platform_repo_impl(rctx):
 
     # https://github.com/aspect-build/bazel-lib/releases/download/v1.19.0/expand_template-linux_amd64
     url = "https://github.com/aspect-build/bazel-lib/releases/download/v{0}/expand_template-{1}{2}".format(
-        EXPAND_TEMPLATE_VERSION,
+        VERSION,
         release_platform,
         ".exe" if is_windows else "",
     )

diff --git a/lib/private/source_toolchains_repo.bzl b/lib/private/source_toolchains_repo.bzl
@@ -0,0 +1,90 @@
+"""Create a repository to hold the toolchains
+
+This follows guidance here:
+https://docs.bazel.build/versions/main/skylark/deploying.html#registering-toolchains
+"
+Note that in order to resolve toolchains in the analysis phase
+Bazel needs to analyze all toolchain targets that are registered.
+Bazel will not need to analyze all targets referenced by toolchain.toolchain attribute.
+If in order to register toolchains you need to perform complex computation in the repository,
+consider splitting the repository with toolchain targets
+from the repository with <LANG>_toolchain targets.
+Former will be always fetched,
+and the latter will only be fetched when user actually needs to build <LANG> code.
+"
+The "complex computation" in our case is simply downloading large artifacts.
+This guidance tells us how to avoid that: we put the toolchain targets in the alias repository
+with only the toolchain attribute pointing into the platform-specific repositories.
+"""
+
+# Add more platforms as needed to mirror all the binaries
+# published by the upstream project.
+
+DEFS_TMPL = """\
+# Generated by source_toolchains_repo.bzl for {toolchain_type}
+load("@bazel_skylib//lib:structs.bzl", "structs")
+
+# Forward all the providers
+def _resolved_toolchain_impl(ctx):
+    toolchain_info = ctx.toolchains["{toolchain_type}"]
+    return [toolchain_info] + structs.to_dict(toolchain_info).values()
+
+# Copied from java_toolchain_alias
+# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl
+resolved_toolchain = rule(
+    implementation = _resolved_toolchain_impl,
+    toolchains = ["{toolchain_type}"],
+    incompatible_use_toolchain_transition = True,
+)
+"""
+
+BUILD_TMPL = """\
+# Generated by source_toolchains_repo.bzl
+#
+# These can be registered in the workspace file or passed to --extra_toolchains flag.
+# By default all of these toolchains are registered by the oci_register_toolchains macro
+# so you don't normally need to interact with these targets.
+
+load(":defs.bzl", "resolved_toolchain")
+load("{toolchain_rule_load_from}", toolchain_rule = "{toolchain_rule}")
+
+resolved_toolchain(name = "current_toolchain", visibility = ["//visibility:public"])
+
+toolchain_rule(
+    name = "source",
+    bin = "{binary}",
+    visibility = ["//visibility:public"],
+)
+
+toolchain(
+    name = "toolchain",
+    toolchain = ":source",
+    toolchain_type = "{toolchain_type}",
+)
+"""
+
+def _source_toolchains_repo_impl(rctx):
+    # Expose a concrete toolchain which is the result of Bazel resolving the toolchain
+    # for the execution or target platform.
+    # Workaround for https://github.com/bazelbuild/bazel/issues/14009
+    rctx.file("defs.bzl", DEFS_TMPL.format(
+        toolchain_type = rctx.attr.toolchain_type,
+    ))
+
+    rctx.file("BUILD.bazel", BUILD_TMPL.format(
+        toolchain_type = rctx.attr.toolchain_type,
+        toolchain_rule_load_from = rctx.attr.toolchain_rule_load_from,
+        toolchain_rule = rctx.attr.toolchain_rule,
+        binary = rctx.attr.binary,
+    ))
+
+source_toolchains_repo = repository_rule(
+    _source_toolchains_repo_impl,
+    doc = "Creates a repository with toolchain definitions for source binaries.",
+    attrs = {
+        "toolchain_type": attr.string(doc = "Label to the toolchain_type", mandatory = True),
+        "toolchain_rule_load_from": attr.string(doc = "Label to the concrete toolchain rule to load from", mandatory = True),
+        "toolchain_rule": attr.string(doc = "Name of the concerete toolchain rule", mandatory = True),
+        "binary": attr.string(doc = "Label to the binary", mandatory = True),
+    },
+)