Introduce compile_pip_requirements rule

This uses pip-tools to compile a requirements.in file to a requirements.txt file,
allowing transitive dependency versions to be pinned so that builds are reproducible.

Fixes bazelbuild#176

examples/pip_install/.bazelrc

@@ -0,0 +1 @@
+test --test_output=errors

examples/pip_install/BUILD

@@ -1,5 +1,6 @@
 load("@pip//:requirements.bzl", "requirement")
 load("@rules_python//python:defs.bzl", "py_binary", "py_test")
+load("@rules_python//python/pip_install:requirements.bzl", "compile_pip_requirements")
 
 # Toolchain setup, this is optional.
 # Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE).
@@ -40,3 +41,8 @@ py_test(
  srcs = ["test.py"],
  deps = [":main"],
 )
+
+# Check that our compiled requirements are up-to-date
+compile_pip_requirements(
+ name = "requirements",
+)

examples/pip_install/requirements.in

@@ -0,0 +1 @@
+boto3==1.14.51

examples/pip_install/requirements.txt

@@ -1 +1,43 @@
-boto3==1.14.51
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+# bazel run //:requirements.update
+#
+boto3==1.14.51 \
+ --hash=sha256:a6bdb808e948bd264af135af50efb76253e85732c451fa605b7a287faf022432 \
+ --hash=sha256:f9dbccbcec916051c6588adbccae86547308ac4cd154f1eb7cf6422f0e391a71
+ # via -r ./requirements.in
+botocore==1.17.63 \
+ --hash=sha256:40f13f6c9c29c307a9dc5982739e537ddce55b29787b90c3447b507e3283bcd6 \
+ --hash=sha256:aa88eafc6295132f4bc606f1df32b3248e0fa611724c0a216aceda767948ac75
+ # via
+ # boto3
+ # s3transfer
+docutils==0.15.2 \
+ --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \
+ --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \
+ --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99
+ # via botocore
+jmespath==0.10.0 \
+ --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \
+ --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f
+ # via
+ # boto3
+ # botocore
+python-dateutil==2.8.1 \
+ --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
+ --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a
+ # via botocore
+s3transfer==0.3.3 \
+ --hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \
+ --hash=sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db
+ # via boto3
+six==1.15.0 \
+ --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
+ # via python-dateutil
+urllib3==1.25.11 \
+ --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \
+ --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e
+ # via botocore

python/BUILD

@@ -24,6 +24,8 @@ In an ideal renaming, we'd move the packaging rules to a different package so
 that @rules_python//python is only concerned with the core rules.
 """
 
+load("//python/pip_install:requirements.bzl", "compile_pip_requirements")
+
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"]) # Apache 2.0
@@ -145,3 +147,8 @@ exports_files([
  "pip.bzl",
  "whl.bzl",
 ])
+
+compile_pip_requirements(
+ name = "requirements",
+ extra_args = ["--allow-unsafe"],
+)

python/pip_install/BUILD

@@ -1,7 +1,10 @@
+exports_files(["pip_compile.py"])
+
 filegroup(
  name = "distribution",
  srcs = glob(["*.bzl"]) + [
  "BUILD",
+ "pip_compile.py",
  "//python/pip_install/extract_wheels:distribution",
  "//python/pip_install/parse_requirements_to_bzl:distribution",
  ],

python/pip_install/pip_compile.py

@@ -0,0 +1,89 @@
+"Set defaults for the pip-compile command to run it under Bazel"
+
+import os
+import sys
+from shutil import copyfile
+
+from piptools.scripts.compile import cli
+
+if len(sys.argv) < 4:
+ print(
+ "Expected at least two arguments: requirements_in requirements_out",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+
+requirements_in = sys.argv.pop(1)
+requirements_txt = sys.argv.pop(1)
+update_target_name = sys.argv.pop(1)
+
+# Before loading click, set the locale for its parser.
+# If it leaks through to the system setting, it may fail:
+# RuntimeError: Click will abort further execution because Python 3 was configured to use ASCII
+# as encoding for the environment. Consult https://click.palletsprojects.com/python3/ for
+# mitigation steps.
+os.environ["LC_ALL"] = "C.UTF-8"
+os.environ["LANG"] = "C.UTF-8"
+
+UPDATE = True
+# Detect if we are running under `bazel test`
+if "TEST_TMPDIR" in os.environ:
+ UPDATE = False
+ # pip-compile wants the cache files to be writeable, but if we point
+ # to the real user cache, Bazel sandboxing makes the file read-only
+ # and we fail.
+ # In theory this makes the test more hermetic as well.
+ sys.argv.append("--cache-dir")
+ sys.argv.append(os.environ["TEST_TMPDIR"])
+ # Make a copy for pip-compile to read and mutate
+ requirements_out = os.path.join(
+ os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out"
+ )
+ copyfile(requirements_txt, requirements_out)
+
+elif "BUILD_WORKING_DIRECTORY" in os.environ:
+ os.chdir(os.environ['BUILD_WORKING_DIRECTORY'])
+else:
+ print(
+ "Expected to find BUILD_WORKING_DIRECTORY in environment",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+
+update_target_pkg = "/".join(requirements_in.split('/')[:-1])
+# $(rootpath) in the workspace root gives ./requirements.in
+if update_target_pkg == ".":
+ update_target_pkg = ""
+update_command = "bazel run //%s:%s" % (update_target_pkg, update_target_name)
+
+os.environ["CUSTOM_COMPILE_COMMAND"] = update_command
+
+sys.argv.append("--generate-hashes")
+sys.argv.append("--output-file")
+sys.argv.append(requirements_txt if UPDATE else requirements_out)
+sys.argv.append(requirements_in)
+
+if UPDATE:
+ print("Updating " + requirements_txt)
+ cli()
+else:
+ # cli will exit(0) on success
+ try:
+ print("Checking " + requirements_txt)
+ cli()
+ print("cl() should exit", file=sys.stderr)
+ sys.exit(1)
+ except SystemExit:
+ golden = open(requirements_txt).readlines()
+ out = open(requirements_out).readlines()
+ if golden != out:
+ import difflib
+
+ print(''.join(difflib.unified_diff(golden, out)), file=sys.stderr)
+ print(
+ "Lock file out of date. Run '"
+ + update_command
+ + "' to update.",
+ file=sys.stderr,
+ )
+ sys.exit(1)

python/pip_install/repositories.bzl

@@ -4,11 +4,21 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 
 _RULE_DEPS = [
+ (
+ "pypi__click",
+ "https://files.pythonhosted.org/packages/d2/3d/fa76db83bf75c4f8d338c2fd15c8d33fdd7ad23a9b5e57eb6c5de26b430e/click-7.1.2-py2.py3-none-any.whl",
+ "dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc",
+ ),
  (
  "pypi__pip",
  "https://files.pythonhosted.org/packages/fe/ef/60d7ba03b5c442309ef42e7d69959f73aacccd0d86008362a681c4698e83/pip-21.0.1-py3-none-any.whl",
  "37fd50e056e2aed635dec96594606f0286640489b0db0ce7607f7e51890372d5",
  ),
+ (
+ "pypi__pip_tools",
+ "https://files.pythonhosted.org/packages/6d/16/75d65bdccd48bb59a08e2bf167b01d8532f65604270d0a292f0f16b7b022/pip_tools-5.5.0-py2.py3-none-any.whl",
+ "10841c1e56c234d610d0466447685b9ea4ee4a2c274f858c0ef3c33d9bd0d985",
+ ),
  (
  "pypi__pkginfo",
  "https://files.pythonhosted.org/packages/4f/3c/535287349af1b117e082f8e77feca52fbe2fdf61ef1e6da6bcc2a72a3a79/pkginfo-1.6.1-py2.py3-none-any.whl",

python/pip_install/requirements.bzl

@@ -0,0 +1,94 @@
+"Rules to verify and update pip-compile locked requirements.txt"
+
+load("//python:defs.bzl", "py_binary", "py_test")
+load("//python/pip_install:repositories.bzl", "requirement")
+
+def compile_pip_requirements(
+ name,
+ extra_args = [],
+ visibility = ["//visibility:private"],
+ requirements_in = "requirements.in",
+ requirements_txt = "requirements.txt",
+ **kwargs):
+ """
+ Macro creating targets for running pip-compile
+
+ Produce a filegroup by default, named "[name]" which can be included in the data
+ of some other compile_pip_requirements rule that references these requirements
+ (e.g. with `-r ../other/requirements.txt`)
+
+ Produce two targets for checking pip-compile:
+
+ - validate with `bazel test <name>_test`
+ - update with `bazel run <name>.update`
+
+ Args:
+ name: base name for generated targets, typically "requirements"
+ extra_args: passed to pip-compile
+ visibility: passed to both the _test and .update rules
+ requirements_in: file expressing desired dependencies
+ requirements_txt: result of "compiling" the requirements.in file
+ **kwargs: other bazel attributes passed to the "_test" rule
+ """
+ requirements_in = kwargs.pop("requirements_in", name + ".in")
+ requirements_txt = kwargs.pop("requirements_locked", name + ".txt")
+
+ # "Default" target produced by this macro
+ # Allow a compile_pip_requirements rule to include another one in the data
+ # for a requirements file that does `-r ../other/requirements.txt`
+ native.filegroup(
+ name = name,
+ srcs = kwargs.pop("data", []) + [requirements_txt],
+ visibility = visibility,
+ )
+
+ data = [name, requirements_in, requirements_txt]
+
+ # Use the Label constructor so this is expanded in the context of the file
+ # where it appears, which is to say, in @rules_python
+ pip_compile = Label("//python/pip_install:pip_compile.py")
+
+ loc = "$(rootpath %s)"
+
+ args = [
+ loc % requirements_in,
+ loc % requirements_txt,
+ name + ".update",
+ ] + extra_args
+
+ deps = [
+ requirement("click"),
+ requirement("pip"),
+ requirement("pip_tools"),
+ requirement("setuptools"),
+ ]
+
+ attrs = {
+ "args": args,
+ "data": data,
+ "deps": deps,
+ "main": pip_compile,
+ "srcs": [pip_compile],
+ "visibility": visibility,
+ }
+
+ # cheap way to detect the bazel version
+ _bazel_version_4_or_greater = "propeller_optimize" in dir(native)
+
+ # Bazel 4.0 added the "env" attribute to py_test/py_binary
+ if _bazel_version_4_or_greater:
+ attrs["env"] = kwargs.pop("env", {})
+
+ py_binary(
+ name = name + ".update",
+ **attrs
+ )
+
+ timeout = kwargs.pop("timeout", "short")
+
+ py_test(
+ name = name + "_test",
+ timeout = timeout,
+ # kwargs could contain test-specific attributes like size or timeout
+ **dict(attrs, **kwargs)
+ )

python/requirements.in

@@ -0,0 +1,10 @@
+pip==9.0.3
+setuptools==44.0.0
+wheel==0.30.0a0
+
+# For tests
+mock==2.0.0
+# This is a transitive dependency of mock, which only appears on some python versions
+# Right now our repo doesn't pin our Python interpreter, so our locked requirements
+# may differ between local dev and CI, for example.
+funcsigs==1.0.2

python/requirements.txt

@@ -1,6 +1,36 @@
-pip==9.0.3
-setuptools==44.0.0
-wheel==0.30.0a0
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+# bazel run //python:requirements.update
+#
+funcsigs==1.0.2 \
+ --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
+ --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50
+ # via -r python/requirements.in
+mock==2.0.0 \
+ --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
+ --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba
+ # via -r python/requirements.in
+pbr==5.5.1 \
+ --hash=sha256:5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9 \
+ --hash=sha256:b236cde0ac9a6aedd5e3c34517b423cd4fd97ef723849da6b0d2231142d89c00
+ # via mock
+six==1.15.0 \
+ --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
+ # via mock
+wheel==0.30.0a0 \
+ --hash=sha256:98f3e09b4ad7f5649a7e3d00e0e005ec1824ddcd6ec16c5086c05b1d91ada6da \
+ --hash=sha256:cd19aa9325d3af1c641b0a23502b12696159171d2a2f4b84308df9a075c2a4a0
+ # via -r python/requirements.in
 
-# For tests
-mock==2.0.0
+# The following packages are considered to be unsafe in a requirements file:
+pip==9.0.3 \
+ --hash=sha256:7bf48f9a693be1d58f49f7af7e0ae9fe29fd671cde8a55e6edca3581c4ef5796 \
+ --hash=sha256:c3ede34530e0e0b2381e7363aded78e0c33291654937e7373032fda04e8803e5
+ # via -r python/requirements.in
+setuptools==44.0.0 \
+ --hash=sha256:180081a244d0888b0065e18206950d603f6550721bd6f8c0a10221ed467dd78e \
+ --hash=sha256:e5baf7723e5bb8382fc146e33032b241efc63314211a3a120aaa55d62d2bb008
+ # via -r python/requirements.in