diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..fb496ed760 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +docs/*.md linguist-generated=true diff --git a/docs/BUILD b/docs/BUILD index e08b751158..94a7c8451a 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -24,6 +24,7 @@ licenses(["notice"]) # Apache 2.0 _DOCS = { "packaging": "//docs:packaging-docs", "pip": "//docs:pip-docs", + "pip_repository": "//docs:pip-repository", "python": "//docs:core-docs", } @@ -104,6 +105,18 @@ stardoc( ], ) +stardoc( + name = "pip-repository", + out = "pip_repository.md_", + input = "//python/pip_install:pip_repository.bzl", + target_compatible_with = _NOT_WINDOWS, + deps = [ + ":bazel_repo_tools", + ":pip_install_bzl", + "//third_party/github.com/bazelbuild/bazel-skylib/lib:versions", + ], +) + stardoc( name = "packaging-docs", out = "packaging.md_", diff --git a/docs/pip.md b/docs/pip.md index 73ed79d1da..13cade4eff 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -74,7 +74,8 @@ Accepts a `requirements.txt` file and installs the dependencies listed within. Those dependencies become available in a generated `requirements.bzl` file. -This macro runs a repository rule that invokes `pip`. In your WORKSPACE file: +This macro wraps the [`pip_repository`](./pip_repository.md) rule that invokes `pip`. +In your WORKSPACE file: ```python pip_install( @@ -140,9 +141,9 @@ alias( | Name | Description | Default Value | | :-------------: | :-------------: | :-------------: | -| requirements | A 'requirements.txt' pip requirements file. | none | +| requirements | A 'requirements.txt' pip requirements file. | None | | name | A unique name for the created external repository (default 'pip'). | "pip" | -| kwargs | Keyword arguments passed directly to the pip_repository repository rule. | none | +| kwargs | Additional arguments to the [pip_repository](./pip_repository.md) repository rule. | none | @@ -158,7 +159,8 @@ Accepts a locked/compiled requirements file and installs the dependencies listed Those dependencies become available in a generated `requirements.bzl` file. You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below. -This macro runs a repository rule that invokes `pip`. In your WORKSPACE file: +This macro wraps the [`pip_repository`](./pip_repository.md) rule that invokes `pip`, with `incremental` set. +In your WORKSPACE file: ```python load("@rules_python//python:pip.bzl", "pip_parse") @@ -242,9 +244,9 @@ See the example in rules_python/examples/pip_parse_vendored. | Name | Description | Default Value | | :-------------: | :-------------: | :-------------: | -| requirements_lock | A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. | none | +| requirements_lock | A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. Note that if your lockfile is platform-dependent, you can use the requirements_[platform] attributes. | none | | name | The name of the generated repository. The generated repositories containing each requirement will be of the form <name>_<requirement-name>. | "pip_parsed_deps" | -| kwargs | Additional keyword arguments for the underlying pip_repository rule. | none | +| kwargs | Additional arguments to the [pip_repository](./pip_repository.md) repository rule. | none | diff --git a/docs/pip_repository.md b/docs/pip_repository.md new file mode 100644 index 0000000000..ef7f72bcd2 --- /dev/null +++ b/docs/pip_repository.md @@ -0,0 +1,140 @@ + + + + +## pip_repository + +
+pip_repository(name, annotations, enable_implicit_namespace_pkgs, environment, extra_pip_args,
+               incremental, isolated, pip_data_exclude, python_interpreter, python_interpreter_target,
+               quiet, repo_prefix, requirements, requirements_darwin, requirements_linux,
+               requirements_lock, requirements_windows, timeout)
+
+ +A rule for importing `requirements.txt` dependencies into Bazel. + +This rule imports a `requirements.txt` file and generates a new +`requirements.bzl` file. This is used via the `WORKSPACE` pattern: + +```python +pip_repository( + name = "foo", + requirements = ":requirements.txt", +) +``` + +You can then reference imported dependencies from your `BUILD` file with: + +```python +load("@foo//:requirements.bzl", "requirement") +py_library( + name = "bar", + ... + deps = [ + "//my/other:dep", + requirement("requests"), + requirement("numpy"), + ], +) +``` + +Or alternatively: +```python +load("@foo//:requirements.bzl", "all_requirements") +py_binary( + name = "baz", + ... + deps = [ + ":foo", + ] + all_requirements, +) +``` + + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :-------------: | :-------------: | :-------------: | :-------------: | :-------------: | +| name | A unique name for this repository. | Name | required | | +| annotations | Optional annotations to apply to packages | Dictionary: String -> String | optional | {} | +| enable_implicit_namespace_pkgs | If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary and py_test targets must specify either legacy_create_init=False or the global Bazel option --incompatible_default_to_explicit_init_py to prevent __init__.py being automatically generated in every directory.

This option is required to support some packages which cannot handle the conversion to pkg-util style. | Boolean | optional | False | +| environment | Environment variables to set in the pip subprocess. Can be used to set common variables such as http_proxy, https_proxy and no_proxy Note that pip is run with "--isolated" on the CLI so PIP_<VAR>_<NAME> style env vars are ignored, but env vars that control requests and urllib3 can be passed. | Dictionary: String -> String | optional | {} | +| extra_pip_args | Extra arguments to pass on to pip. Must not contain spaces. | List of strings | optional | [] | +| incremental | Create the repository in incremental mode. | Boolean | optional | False | +| isolated | Whether or not to pass the [--isolated](https://pip.pypa.io/en/stable/cli/pip/#cmdoption-isolated) flag to the underlying pip command. Alternatively, the RULES_PYTHON_PIP_ISOLATED enviornment varaible can be used to control this flag. | Boolean | optional | True | +| pip_data_exclude | Additional data exclusion parameters to add to the pip packages BUILD file. | List of strings | optional | [] | +| python_interpreter | The python interpreter to use. This can either be an absolute path or the name of a binary found on the host's PATH environment variable. If no value is set python3 is defaulted for Unix systems and python.exe for Windows. | String | optional | "" | +| python_interpreter_target | If you are using a custom python interpreter built by another repository rule, use this attribute to specify its BUILD target. This allows pip_repository to invoke pip using the same interpreter as your toolchain. If set, takes precedence over python_interpreter. | Label | optional | None | +| quiet | If True, suppress printing stdout and stderr output to the terminal. | Boolean | optional | True | +| repo_prefix | Prefix for the generated packages. For non-incremental mode the packages will be of the form

@<name>//<prefix><sanitized-package-name>/...

For incremental mode the packages will be of the form

@<prefix><sanitized-package-name>//... | String | optional | "" | +| requirements | A 'requirements.txt' pip requirements file. | Label | optional | None | +| requirements_darwin | Override the requirements_lock attribute when the host platform is Mac OS | Label | optional | None | +| requirements_linux | Override the requirements_lock attribute when the host platform is Linux | Label | optional | None | +| requirements_lock | A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. | Label | optional | None | +| requirements_windows | Override the requirements_lock attribute when the host platform is Windows | Label | optional | None | +| timeout | Timeout (in seconds) on the rule's execution duration. | Integer | optional | 600 | + + + + +## whl_library + +
+whl_library(name, annotation, enable_implicit_namespace_pkgs, environment, extra_pip_args, isolated,
+            pip_data_exclude, python_interpreter, python_interpreter_target, quiet, repo, repo_prefix,
+            requirement, timeout)
+
+ + +Download and extracts a single wheel based into a bazel repo based on the requirement string passed in. +Instantiated from pip_repository and inherits config options from there. + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :-------------: | :-------------: | :-------------: | :-------------: | :-------------: | +| name | A unique name for this repository. | Name | required | | +| annotation | Optional json encoded file containing annotation to apply to the extracted wheel. See package_annotation | Label | optional | None | +| enable_implicit_namespace_pkgs | If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary and py_test targets must specify either legacy_create_init=False or the global Bazel option --incompatible_default_to_explicit_init_py to prevent __init__.py being automatically generated in every directory.

This option is required to support some packages which cannot handle the conversion to pkg-util style. | Boolean | optional | False | +| environment | Environment variables to set in the pip subprocess. Can be used to set common variables such as http_proxy, https_proxy and no_proxy Note that pip is run with "--isolated" on the CLI so PIP_<VAR>_<NAME> style env vars are ignored, but env vars that control requests and urllib3 can be passed. | Dictionary: String -> String | optional | {} | +| extra_pip_args | Extra arguments to pass on to pip. Must not contain spaces. | List of strings | optional | [] | +| isolated | Whether or not to pass the [--isolated](https://pip.pypa.io/en/stable/cli/pip/#cmdoption-isolated) flag to the underlying pip command. Alternatively, the RULES_PYTHON_PIP_ISOLATED enviornment varaible can be used to control this flag. | Boolean | optional | True | +| pip_data_exclude | Additional data exclusion parameters to add to the pip packages BUILD file. | List of strings | optional | [] | +| python_interpreter | The python interpreter to use. This can either be an absolute path or the name of a binary found on the host's PATH environment variable. If no value is set python3 is defaulted for Unix systems and python.exe for Windows. | String | optional | "" | +| python_interpreter_target | If you are using a custom python interpreter built by another repository rule, use this attribute to specify its BUILD target. This allows pip_repository to invoke pip using the same interpreter as your toolchain. If set, takes precedence over python_interpreter. | Label | optional | None | +| quiet | If True, suppress printing stdout and stderr output to the terminal. | Boolean | optional | True | +| repo | Pointer to parent repo name. Used to make these rules rerun if the parent repo changes. | String | required | | +| repo_prefix | Prefix for the generated packages. For non-incremental mode the packages will be of the form

@<name>//<prefix><sanitized-package-name>/...

For incremental mode the packages will be of the form

@<prefix><sanitized-package-name>//... | String | optional | "" | +| requirement | Python requirement string describing the package to make available | String | required | | +| timeout | Timeout (in seconds) on the rule's execution duration. | Integer | optional | 600 | + + + + +## package_annotation + +
+package_annotation(additive_build_content, copy_files, copy_executables, data, data_exclude_glob,
+                   srcs_exclude_glob)
+
+ +Annotations to apply to the BUILD file content from package generated from a `pip_repository` rule. + +[cf]: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/copy_file_doc.md + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :-------------: | :-------------: | :-------------: | +| additive_build_content | Raw text to add to the generated BUILD file of a package. | None | +| copy_files | A mapping of src and out files for [@bazel_skylib//rules:copy_file.bzl][cf] | {} | +| copy_executables | A mapping of src and out files for [@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as executable. | {} | +| data | A list of labels to add as data dependencies to the generated py_library target. | [] | +| data_exclude_glob | A list of exclude glob patterns to add as data to the generated py_library target. | [] | +| srcs_exclude_glob | A list of labels to add as srcs to the generated py_library target. | [] | + + diff --git a/python/pip.bzl b/python/pip.bzl index 93096a86a4..9813fd5b96 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -20,12 +20,13 @@ load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compi compile_pip_requirements = _compile_pip_requirements package_annotation = _package_annotation -def pip_install(requirements, name = "pip", **kwargs): +def pip_install(requirements = None, name = "pip", **kwargs): """Accepts a `requirements.txt` file and installs the dependencies listed within. Those dependencies become available in a generated `requirements.bzl` file. - This macro runs a repository rule that invokes `pip`. In your WORKSPACE file: + This macro wraps the [`pip_repository`](./pip_repository.md) rule that invokes `pip`. + In your WORKSPACE file: ```python pip_install( @@ -88,7 +89,7 @@ def pip_install(requirements, name = "pip", **kwargs): Args: requirements (Label): A 'requirements.txt' pip requirements file. name (str, optional): A unique name for the created external repository (default 'pip'). - **kwargs (dict): Keyword arguments passed directly to the `pip_repository` repository rule. + **kwargs (dict): Additional arguments to the [`pip_repository`](./pip_repository.md) repository rule. """ # Just in case our dependencies weren't already fetched @@ -107,7 +108,8 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs): Those dependencies become available in a generated `requirements.bzl` file. You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below. - This macro runs a repository rule that invokes `pip`. In your WORKSPACE file: + This macro wraps the [`pip_repository`](./pip_repository.md) rule that invokes `pip`, with `incremental` set. + In your WORKSPACE file: ```python load("@rules_python//python:pip.bzl", "pip_parse") @@ -191,10 +193,11 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs): of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. + Note that if your lockfile is platform-dependent, you can use the `requirements_[platform]` + attributes. name (str, optional): The name of the generated repository. The generated repositories containing each requirement will be of the form _. - **kwargs (dict): Additional keyword arguments for the underlying - `pip_repository` rule. + **kwargs (dict): Additional arguments to the [`pip_repository`](./pip_repository.md) repository rule. """ # Just in case our dependencies weren't already fetched diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 4b03892d16..d9888a2990 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -119,26 +119,40 @@ package(default_visibility = ["//visibility:public"]) exports_files(["requirements.bzl"]) """ +def _locked_requirements(rctx): + os = rctx.os.name.lower() + requirements_txt = rctx.attr.requirements_lock + if os.startswith("mac os") and rctx.attr.requirements_darwin != None: + requirements_txt = rctx.attr.requirements_darwin + elif os.startswith("linux") and rctx.attr.requirements_linux != None: + requirements_txt = rctx.attr.requirements_linux + elif "win" in os and rctx.attr.requirements_windows != None: + requirements_txt = rctx.attr.requirements_windows + if not requirements_txt: + fail("""\ +Incremental mode requires a requirements_lock attribute be specified, +or a platform-specific lockfile using one of the requirements_* attributes. +""") + return requirements_txt + def _pip_repository_impl(rctx): python_interpreter = _resolve_python_interpreter(rctx) - if rctx.attr.incremental and not rctx.attr.requirements_lock: - fail("Incremental mode requires a requirements_lock attribute be specified.") - # Write the annotations file to pass to the wheel maker annotations = {package: json.decode(data) for (package, data) in rctx.attr.annotations.items()} annotations_file = rctx.path("annotations.json") rctx.file(annotations_file, json.encode_indent(annotations, indent = " " * 4)) if rctx.attr.incremental: + requirements_txt = _locked_requirements(rctx) args = [ python_interpreter, "-m", "python.pip_install.parse_requirements_to_bzl", "--requirements_lock", - rctx.path(rctx.attr.requirements_lock), + rctx.path(requirements_txt), "--requirements_lock_label", - str(rctx.attr.requirements_lock), + str(requirements_txt), # pass quiet and timeout args through to child repos. "--quiet", str(rctx.attr.quiet), @@ -282,6 +296,14 @@ pip_repository_attrs = { allow_single_file = True, doc = "A 'requirements.txt' pip requirements file.", ), + "requirements_darwin": attr.label( + allow_single_file = True, + doc = "Override the requirements_lock attribute when the host platform is Mac OS", + ), + "requirements_linux": attr.label( + allow_single_file = True, + doc = "Override the requirements_lock attribute when the host platform is Linux", + ), "requirements_lock": attr.label( allow_single_file = True, doc = """ @@ -290,6 +312,10 @@ of 'requirements' no resolve will take place and pip_repository will create indi wheels are fetched/built only for the targets specified by 'build/run/test'. """, ), + "requirements_windows": attr.label( + allow_single_file = True, + doc = "Override the requirements_lock attribute when the host platform is Windows", + ), } pip_repository_attrs.update(**common_attrs)