diff --git a/docs/BUILD b/docs/BUILD index 9fedb72de4..4264bd3ed4 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -71,6 +71,14 @@ bzl_library( ], ) +bzl_library( + name = "packaging_bzl", + srcs = [ + "//python:packaging.bzl", + "//python/private:stamp.bzl", + ], +) + stardoc( name = "core-docs", out = "python.md_", @@ -103,6 +111,7 @@ stardoc( name = "packaging-docs", out = "packaging.md_", input = "//python:packaging.bzl", + deps = [":packaging_bzl"], ) [ diff --git a/docs/packaging.md b/docs/packaging.md index 975a98219e..e08874b09f 100755 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -31,7 +31,7 @@ This rule is intended to be used as data dependency to py_wheel rule 
 py_wheel(name, abi, author, author_email, classifiers, console_scripts, deps, description_file,
          distribution, entry_points, extra_requires, homepage, license, platform, python_requires,
-         python_tag, requires, strip_path_prefixes, version)
+         python_tag, requires, stamp, strip_path_prefixes, version)
@@ -101,7 +101,27 @@ py_wheel( | python_requires | A string specifying what other distributions need to be installed when this one is. See the section on [Declaring required dependency](https://setuptools.readthedocs.io/en/latest/userguide/dependency_management.html#declaring-dependencies) for details and examples of the format of this argument. | String | optional | "" | | python_tag | Supported Python version(s), eg py3, cp35.cp36, etc | String | optional | "py3" | | requires | List of requirements for this package | List of strings | optional | [] | +| stamp | Whether to encode build information into the wheel. Possible values:

- stamp = 1: Always stamp the build information into the wheel, even in [--nostamp](https://docs.bazel.build/versions/main/user-manual.html#flag--stamp) builds. This setting should be avoided, since it potentially kills remote caching for the target and any downstream actions that depend on it.

- stamp = 0: Always replace build information by constant values. This gives good build result caching.

- stamp = -1: Embedding of build information is controlled by the [--[no]stamp](https://docs.bazel.build/versions/main/user-manual.html#flag--stamp) flag.

Stamped targets are not rebuilt unless their dependencies change. | Integer | optional | -1 | | strip_path_prefixes | path prefixes to strip from files added to the generated package | List of strings | optional | [] | -| version | Version number of the package | String | required | | +| version | Version number of the package. Note that this attribute suppots stamp format strings. Eg 1.2.3-{BUILD_TIMESTAMP} | String | required | | + + + + +## PyWheelInfo + +
+PyWheelInfo(name_file, wheel)
+
+ +Information about a wheel produced by `py_wheel` + +**FIELDS** + + +| Name | Description | +| :-------------: | :-------------: | +| name_file | File: A file containing the canonical name of the wheel (after stamping, if enabled). | +| wheel | File: The wheel file itself. | diff --git a/examples/wheel/BUILD b/examples/wheel/BUILD index e60fd11733..0c24da8218 100644 --- a/examples/wheel/BUILD +++ b/examples/wheel/BUILD @@ -58,6 +58,20 @@ py_wheel( ], ) +# Package just a specific py_libraries, without their dependencies +py_wheel( + name = "minimal_with_py_library_with_stamp", + # Package data. We're building "example_minimal_library-0.0.1-py3-none-any.whl" + distribution = "example_minimal_library", + python_tag = "py3", + stamp = 1, + version = "0.1.{BUILD_TIMESTAMP}", + deps = [ + "//examples/wheel/lib:module_with_data", + "//examples/wheel/lib:simple_module", + ], +) + # Use py_package to collect all transitive dependencies of a target, # selecting just the files within a specific python package. py_package( diff --git a/python/packaging.bzl b/python/packaging.bzl index 5eac83a5ab..bc85139fad 100644 --- a/python/packaging.bzl +++ b/python/packaging.bzl @@ -14,6 +14,19 @@ """Rules for building wheels.""" +load("//python/private:stamp.bzl", "is_stamping_enabled") + +PyWheelInfo = provider( + doc = "Information about a wheel produced by `py_wheel`", + fields = { + "name_file": ( + "File: A file containing the canonical name of the wheel (after " + + "stamping, if enabled)." + ), + "wheel": "File: The wheel file itself.", + }, +) + def _path_inside_wheel(input_file): # input_file.short_path is sometimes relative ("../${repository_root}/foobar") # which is not a valid path within a zip file. Fix that. @@ -110,6 +123,8 @@ def _py_wheel_impl(ctx): _escape_filename_segment(ctx.attr.platform), ]) + ".whl") + name_file = ctx.actions.declare_file(ctx.label.name + ".name") + inputs_to_package = depset( direct = ctx.files.deps, ) @@ -133,9 +148,15 @@ def _py_wheel_impl(ctx): args.add("--python_requires", ctx.attr.python_requires) args.add("--abi", ctx.attr.abi) args.add("--platform", ctx.attr.platform) - args.add("--out", outfile.path) + args.add("--out", outfile) + args.add("--name_file", name_file) args.add_all(ctx.attr.strip_path_prefixes, format_each = "--strip_path_prefix=%s") + # Pass workspace status files if stamping is enabled + if is_stamping_enabled(ctx.attr): + args.add("--volatile_status_file", ctx.version_file) + other_inputs.append(ctx.version_file) + args.add("--input_file_list", packageinputfile) extra_headers = [] @@ -193,15 +214,21 @@ def _py_wheel_impl(ctx): ctx.actions.run( inputs = depset(direct = other_inputs, transitive = [inputs_to_package]), - outputs = [outfile], + outputs = [outfile, name_file], arguments = [args], executable = ctx.executable._wheelmaker, progress_message = "Building wheel", ) - return [DefaultInfo( - files = depset([outfile]), - data_runfiles = ctx.runfiles(files = [outfile]), - )] + return [ + DefaultInfo( + files = depset([outfile]), + runfiles = ctx.runfiles(files = [outfile]), + ), + PyWheelInfo( + wheel = outfile, + name_file = name_file, + ), + ] def _concat_dicts(*dicts): result = {} @@ -247,9 +274,35 @@ platform = select({ default = "py3", doc = "Supported Python version(s), eg `py3`, `cp35.cp36`, etc", ), + "stamp": attr.int( + doc = """\ +Whether to encode build information into the wheel. Possible values: + +- `stamp = 1`: Always stamp the build information into the wheel, even in \ +[--nostamp](https://docs.bazel.build/versions/main/user-manual.html#flag--stamp) builds. \ +This setting should be avoided, since it potentially kills remote caching for the target and \ +any downstream actions that depend on it. + +- `stamp = 0`: Always replace build information by constant values. This gives good build result caching. + +- `stamp = -1`: Embedding of build information is controlled by the \ +[--[no]stamp](https://docs.bazel.build/versions/main/user-manual.html#flag--stamp) flag. + +Stamped targets are not rebuilt unless their dependencies change. + """, + default = -1, + values = [1, 0, -1], + ), "version": attr.string( mandatory = True, - doc = "Version number of the package", + doc = ( + "Version number of the package. Note that this attribute " + + "suppots stamp format strings. Eg `1.2.3-{BUILD_TIMESTAMP}`" + ), + ), + "_stamp_flag": attr.label( + doc = "A setting used to determine whether or not the `--stamp` flag is enabled", + default = Label("//python/private:stamp"), ), } diff --git a/python/private/BUILD b/python/private/BUILD index 90fcd3bc99..53c2cbccc9 100644 --- a/python/private/BUILD +++ b/python/private/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load(":stamp.bzl", "stamp_build_setting") + licenses(["notice"]) # Apache 2.0 filegroup( @@ -39,3 +41,6 @@ exports_files( ], visibility = ["//docs:__pkg__"], ) + +# Used to determine the use of `--stamp` in Starlark rules +stamp_build_setting(name = "stamp") diff --git a/python/private/stamp.bzl b/python/private/stamp.bzl new file mode 100644 index 0000000000..86ea3fc99c --- /dev/null +++ b/python/private/stamp.bzl @@ -0,0 +1,73 @@ +"""A small utility module dedicated to detecting whether or not the `--stamp` flag is enabled + +This module can be removed likely after the following PRs ar addressed: +- https://github.com/bazelbuild/bazel/issues/11164 +""" + +StampSettingInfo = provider( + doc = "Information about the `--stamp` command line flag", + fields = { + "value": "bool: Whether or not the `--stamp` flag was enabled", + }, +) + +def _stamp_build_setting_impl(ctx): + return StampSettingInfo(value = ctx.attr.value) + +_stamp_build_setting = rule( + doc = """\ +Whether to encode build information into the binary. Possible values: + +- stamp = 1: Always stamp the build information into the binary, even in [--nostamp][stamp] builds. \ +This setting should be avoided, since it potentially kills remote caching for the binary and \ +any downstream actions that depend on it. +- stamp = 0: Always replace build information by constant values. This gives good build result caching. +- stamp = -1: Embedding of build information is controlled by the [--[no]stamp][stamp] flag. + +Stamped binaries are not rebuilt unless their dependencies change. +[stamp]: https://docs.bazel.build/versions/main/user-manual.html#flag--stamp + """, + implementation = _stamp_build_setting_impl, + attrs = { + "value": attr.bool( + doc = "The default value of the stamp build flag", + mandatory = True, + ), + }, +) + +def stamp_build_setting(name, visibility = ["//visibility:public"]): + native.config_setting( + name = "stamp_detect", + values = {"stamp": "1"}, + visibility = visibility, + ) + + _stamp_build_setting( + name = name, + value = select({ + ":stamp_detect": True, + "//conditions:default": False, + }), + visibility = visibility, + ) + +def is_stamping_enabled(attr): + """Determine whether or not build staming is enabled + + Args: + attr (struct): A rule's struct of attributes (`ctx.attr`) + + Returns: + bool: The stamp value + """ + stamp_num = getattr(attr, "stamp", -1) + if stamp_num == 1: + return True + elif stamp_num == 0: + return False + elif stamp_num == -1: + stamp_flag = getattr(attr, "_stamp_flag", None) + return stamp_flag[StampSettingInfo].value if stamp_flag else False + else: + fail("Unexpected `stamp` value: {}".format(stamp_num)) diff --git a/tools/wheelmaker.py b/tools/wheelmaker.py index 050599fa53..a2e4a16544 100644 --- a/tools/wheelmaker.py +++ b/tools/wheelmaker.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pathlib import Path import argparse import base64 import collections import hashlib import os -import os.path import re import sys import zipfile @@ -64,15 +64,18 @@ def __exit__(self, type, value, traceback): self._zipfile.close() self._zipfile = None - def filename(self): - if self._outfile: - return self._outfile + def wheelname(self) -> str: components = [self._name, self._version] if self._build_tag: components.append(self._build_tag) components += [self._python_tag, self._abi, self._platform] return '-'.join(components) + '.whl' + def filename(self) -> str: + if self._outfile: + return self._outfile + return self.wheelname() + def disttags(self): return ['-'.join([self._python_tag, self._abi, self._platform])] @@ -201,7 +204,27 @@ def get_files_to_package(input_files): return files -def main(): +def resolve_version_stamp(version: str, resolve_version_stamp: Path) -> str: + """Resolve workspace status stamps format strings found in the version string + + Args: + version (str): The raw version represenation for the wheel (may include stamp variables) + resolve_version_stamp (Path): The path to a volatile workspace status file + + Returns: + str: A resolved version string + """ + for line in resolve_version_stamp.read_text().splitlines(): + if not line: + continue + key, value = line.split(' ', maxsplit=1) + stamp = "{" + key + "}" + version = version.replace(stamp, value) + + return version + + +def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description='Builds a python wheel') metadata_group = parser.add_argument_group( "Wheel name, version and platform") @@ -222,6 +245,9 @@ def main(): output_group = parser.add_argument_group("Output file location") output_group.add_argument('--out', type=str, default=None, help="Override name of ouptut file") + output_group.add_argument('--name_file', type=Path, + help="A file where the canonical name of the " + "wheel will be written") output_group.add_argument('--strip_path_prefix', type=str, @@ -266,7 +292,18 @@ def main(): '--extra_requires', type=str, action='append', help="List of optional requirements in a 'requirement;option name'. " "Can be supplied multiple times.") - arguments = parser.parse_args(sys.argv[1:]) + + build_group = parser.add_argument_group("Building requirements") + build_group.add_argument( + '--volatile_status_file', type=Path, + help="Pass in the stamp info file for stamping" + ) + + return parser.parse_args(sys.argv[1:]) + + +def main() -> None: + arguments = parse_args() if arguments.input_file: input_files = [i.split(';') for i in arguments.input_file] @@ -286,8 +323,14 @@ def main(): strip_prefixes = [p for p in arguments.strip_path_prefix] + if arguments.volatile_status_file: + version = resolve_version_stamp(arguments.version, + arguments.volatile_status_file) + else: + version = arguments.version + with WheelMaker(name=arguments.name, - version=arguments.version, + version=version, build_tag=arguments.build_tag, python_tag=arguments.python_tag, abi=arguments.abi, @@ -333,6 +376,12 @@ def main(): maker.add_recordfile() + # Since stamping may otherwise change the target name of the + # wheel, the canonical name (with stamps resolved) is written + # to a file so consumers of the wheel can easily determine + # the correct name. + arguments.name_file.write_text(maker.wheelname()) + if __name__ == '__main__': main()