From c98bc8f232ee12aa7b732473de5f6f5508125d1a Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Wed, 22 Jun 2022 09:23:41 +1000 Subject: [PATCH] Validation to ensure requirements_lock is pinned. (#732) * Light validation to ensure lockfile is pinned. * Clean up * . Co-authored-by: Alex Eagle --- examples/pip_install/requirements.in | 9 +++------ examples/pip_install/requirements.txt | 4 +--- examples/pip_parse/BUILD | 2 +- examples/pip_parse/requirements.in | 3 +++ examples/pip_parse/requirements.txt | 6 ------ examples/pip_parse/requirements_lock.txt | 10 ++++------ .../parse_requirements_to_bzl/__init__.py | 16 +++++++++++++++- 7 files changed, 27 insertions(+), 23 deletions(-) create mode 100644 examples/pip_parse/requirements.in delete mode 100644 examples/pip_parse/requirements.txt diff --git a/examples/pip_install/requirements.in b/examples/pip_install/requirements.in index 593e5ad95..2351369d1 100644 --- a/examples/pip_install/requirements.in +++ b/examples/pip_install/requirements.in @@ -1,6 +1,3 @@ -boto3==1.14.51 -s3cmd==2.1.0 -yamllint==1.26.3 - -# Last available for Python 3.6. -setuptools==59.6.0 +boto3~=1.14.51 +s3cmd~=2.1.0 +yamllint~=1.26.3 diff --git a/examples/pip_install/requirements.txt b/examples/pip_install/requirements.txt index 43bfad4e2..26de1adaa 100644 --- a/examples/pip_install/requirements.txt +++ b/examples/pip_install/requirements.txt @@ -98,6 +98,4 @@ yamllint==1.26.3 \ setuptools==59.6.0 \ --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \ --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e - # via - # -r requirements.in - # yamllint + # via yamllint diff --git a/examples/pip_parse/BUILD b/examples/pip_parse/BUILD index 92b59ae77..653f75ce2 100644 --- a/examples/pip_parse/BUILD +++ b/examples/pip_parse/BUILD @@ -59,7 +59,7 @@ alias( compile_pip_requirements( name = "requirements", extra_args = ["--allow-unsafe"], - requirements_in = "requirements.txt", + requirements_in = "requirements.in", requirements_txt = "requirements_lock.txt", ) diff --git a/examples/pip_parse/requirements.in b/examples/pip_parse/requirements.in new file mode 100644 index 000000000..ec2102fdd --- /dev/null +++ b/examples/pip_parse/requirements.in @@ -0,0 +1,3 @@ +requests~=2.25.1 +s3cmd~=2.1.0 +yamllint~=1.26.3 diff --git a/examples/pip_parse/requirements.txt b/examples/pip_parse/requirements.txt deleted file mode 100644 index e31519403..000000000 --- a/examples/pip_parse/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -requests==2.25.1 -s3cmd==2.1.0 -yamllint==1.26.3 - -# Last avialable for python3.6 -setuptools==59.6.0 diff --git a/examples/pip_parse/requirements_lock.txt b/examples/pip_parse/requirements_lock.txt index f270699e7..d3cb1f5bc 100644 --- a/examples/pip_parse/requirements_lock.txt +++ b/examples/pip_parse/requirements_lock.txt @@ -66,11 +66,11 @@ pyyaml==6.0 \ requests==2.25.1 \ --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e - # via -r requirements.txt + # via -r requirements.in s3cmd==2.1.0 \ --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 - # via -r requirements.txt + # via -r requirements.in six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 @@ -81,12 +81,10 @@ urllib3==1.26.7 \ # via requests yamllint==1.26.3 \ --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e - # via -r requirements.txt + # via -r requirements.in # The following packages are considered to be unsafe in a requirements file: setuptools==59.6.0 \ --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \ --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e - # via - # -r requirements.txt - # yamllint + # via yamllint diff --git a/python/pip_install/parse_requirements_to_bzl/__init__.py b/python/pip_install/parse_requirements_to_bzl/__init__.py index 9519d6203..83526a7a0 100644 --- a/python/pip_install/parse_requirements_to_bzl/__init__.py +++ b/python/pip_install/parse_requirements_to_bzl/__init__.py @@ -31,16 +31,30 @@ def parse_install_requirements( parser = RequirementsFileParser(ps, line_parser) install_req_and_lines: List[Tuple[InstallRequirement, str]] = [] _, content = get_file_content(requirements_lock, ps) + unpinned_reqs = [] for parsed_line, (_, line) in zip( parser.parse(requirements_lock, constraint=False), preprocess(content) ): if parsed_line.is_requirement: + install_req = constructors.install_req_from_line(parsed_line.requirement) + if not install_req.is_pinned: + unpinned_reqs.append(str(install_req)) install_req_and_lines.append( - (constructors.install_req_from_line(parsed_line.requirement), line) + (install_req, line) ) else: extra_pip_args.extend(shlex.split(line)) + + if len(unpinned_reqs) > 0: + unpinned_reqs_str = "\n".join(unpinned_reqs) + raise RuntimeError(f"""\ +The `requirements_lock` file must be fully pinned. See `compile_pip_requirements`. +Alternatively, use `pip-tools` or a similar mechanism to produce a pinned lockfile. + +The following requirements were not pinned: +{unpinned_reqs_str}""") + return install_req_and_lines