diff --git a/.github/actions/gh-action-pip-audit/.github/workflows/ci.yml b/.github/actions/gh-action-pip-audit/.github/workflows/ci.yml new file mode 100644 index 0000000000..b05f795437 --- /dev/null +++ b/.github/actions/gh-action-pip-audit/.github/workflows/ci.yml @@ -0,0 +1,18 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.7" + - name: lint + run: make lint diff --git a/.github/actions/gh-action-pip-audit/.github/workflows/selftest.yml b/.github/actions/gh-action-pip-audit/.github/workflows/selftest.yml new file mode 100644 index 0000000000..864028402a --- /dev/null +++ b/.github/actions/gh-action-pip-audit/.github/workflows/selftest.yml @@ -0,0 +1,90 @@ +name: Self-test + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +jobs: + selftest-requirements: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./ + id: pip-audit + with: + inputs: ./test/vulnerable.txt + no-deps: true + # NOTE: We intentionally allow failure here, since the self-test + # explicitly uses a vulnerable requirements file. + internal-be-careful-allow-failure: true + - name: assert expected output + env: + PIP_AUDIT_OUTPUT: "${{ steps.pip-audit.outputs.internal-be-careful-output }}" + run: | + grep -E 'pyyaml\s+\|\s+5.1' <<< $(base64 -d <<< "${PIP_AUDIT_OUTPUT}") + + selftest-environment: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: make the environment vulnerable + run: | + python -m pip install --no-deps --requirement ./test/vulnerable.txt + - uses: ./ + id: pip-audit + with: + # NOTE: We intentionally allow failure here, since the self-test + # explicitly uses a vulnerable requirements file. + internal-be-careful-allow-failure: true + - name: assert expected output + env: + PIP_AUDIT_OUTPUT: "${{ steps.pip-audit.outputs.internal-be-careful-output }}" + run: | + grep -E 'pyyaml\s+\|\s+5.1' <<< $(base64 -d <<< "${PIP_AUDIT_OUTPUT}") + + selftest-virtualenv: + strategy: + matrix: + local: [true, false] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: make a virtual environment vulnerable + run: | + python -m venv env + ./env/bin/python -m pip install --upgrade pip wheel + ./env/bin/python -m pip install --no-deps --requirement ./test/vulnerable.txt + - uses: ./ + id: pip-audit + with: + virtual-environment: env/ + local: ${{ matrix.local }} + # NOTE: We intentionally allow failure here, since the self-test + # explicitly uses a vulnerable requirements file. + internal-be-careful-allow-failure: true + - name: assert expected output + env: + PIP_AUDIT_OUTPUT: "${{ steps.pip-audit.outputs.internal-be-careful-output }}" + run: | + grep -E 'pyyaml\s+\|\s+5.1' <<< $(base64 -d <<< "${PIP_AUDIT_OUTPUT}") + + selftest-pyproject: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./ + id: pip-audit + with: + # should attempt to discover test/pyproject/pyproject.toml + inputs: test/pyproject/ + # NOTE: We intentionally allow failure here, since the self-test + # explicitly uses a vulnerable requirements file. + internal-be-careful-allow-failure: true + - name: assert expected output + env: + PIP_AUDIT_OUTPUT: "${{ steps.pip-audit.outputs.internal-be-careful-output }}" + run: | + grep -E 'pyyaml\s+\|\s+5.1' <<< $(base64 -d <<< "${PIP_AUDIT_OUTPUT}") diff --git a/.github/actions/gh-action-pip-audit/.gitignore b/.github/actions/gh-action-pip-audit/.gitignore new file mode 100644 index 0000000000..bdaab25d58 --- /dev/null +++ b/.github/actions/gh-action-pip-audit/.gitignore @@ -0,0 +1 @@ +env/ diff --git a/.github/actions/gh-action-pip-audit/LICENSE b/.github/actions/gh-action-pip-audit/LICENSE new file mode 100644 index 0000000000..f433b1a53f --- /dev/null +++ b/.github/actions/gh-action-pip-audit/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/.github/actions/gh-action-pip-audit/Makefile b/.github/actions/gh-action-pip-audit/Makefile new file mode 100644 index 0000000000..77fd0ac52c --- /dev/null +++ b/.github/actions/gh-action-pip-audit/Makefile @@ -0,0 +1,17 @@ +.PHONY: all +all: + @echo "Run my targets individually!" + +env/pyvenv.cfg: dev-requirements.txt + python -m venv env + ./env/bin/python -m pip install --upgrade pip + ./env/bin/python -m pip install --requirement dev-requirements.txt + +.PHONY: dev +dev: env/pyvenv.cfg + +.PHONY: lint +lint: env/pyvenv.cfg action.py + ./env/bin/python -m black action.py + ./env/bin/python -m isort action.py + ./env/bin/python -m flake8 --max-line-length 100 action.py diff --git a/.github/actions/gh-action-pip-audit/README.md b/.github/actions/gh-action-pip-audit/README.md new file mode 100644 index 0000000000..825721c951 --- /dev/null +++ b/.github/actions/gh-action-pip-audit/README.md @@ -0,0 +1,365 @@ +gh-action-pip-audit +=================== + +[![CI](https://github.com/trailofbits/gh-action-pip-audit/actions/workflows/ci.yml/badge.svg)](https://github.com/trailofbits/gh-action-pip-audit/actions/workflows/ci.yml) +[![Self-test](https://github.com/trailofbits/gh-action-pip-audit/actions/workflows/selftest.yml/badge.svg)](https://github.com/trailofbits/gh-action-pip-audit/actions/workflows/selftest.yml) + +A GitHub Action that uses [`pip-audit`](https://github.com/trailofbits/pip-audit) +to scan Python dependencies for known vulnerabilities. + +## Index + +* [Usage](#usage) +* [Configuration](#configuration) + * [⚠️ Internal options ⚠️](#internal-options) +* [Troubleshooting](#troubleshooting) +* [Licensing](#licensing) +* [Code of Conduct](#code-of-conduct) + +## Usage + +Simply add `trailofbits/gh-action-pip-audit` to one of your workflows: + +```yaml +jobs: + selftest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: install + run: python -m pip install . + - uses: trailofbits/gh-action-pip-audit@v1.0.0 +``` + +Or, with a virtual environment: + +```yaml +jobs: + selftest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: install + run: | + python -m venv env/ + source env/bin/activate + python -m pip install . + - uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + virtual-environment: env/ +``` + +By default, `pip-audit` will run in "`pip list` source" mode, meaning that it'll +attempt to collect dependencies from the local environment. See +the [configuration](#configuration) documentation below for more input +and behavioral options. + +## Configuration + +`gh-action-pip-audit` takes a variety of configuration inputs, all of which are +optional. + +### `inputs` + +**Default**: Empty, indicating "`pip list` source" mode + +The `inputs` setting controls what sources `pip-audit` runs on. + +To audit one or more requirements-style inputs: + +```yaml +- uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + inputs: requirements.txt dev-requirements.txt +``` + +To audit a project that uses `pyproject.toml` for its dependencies: + +```yaml +- uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + # NOTE: this can be `.`, for the current directory + inputs: path/to/project/ +``` + +### `virtual-environment` + +**Default**: Empty, indicating no virtual environment + +The `virtual-environment` setting controls the +[virtual environment](https://docs.python.org/3/tutorial/venv.html) that this +action loads to, if specified. The value is the top-level directory for the +virtual environment, which is conventionally named `env` or `venv`. + +Depending on your CI and project configuration, you may or may not need this +setting. Specifically, you only need it if you satisfy *all* of the following +conditions: + +1. You are auditing an *environment* (**not** a requirements file or other + project metadata) +2. Your environment is not already "active", i.e. `python -m pip` points to a + different `pip` than the one that your environment uses + +Example: use the virtual environment specified at `env/`, relative to the +current directory: + +```yaml +- uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + virtual-environment: env/ + # Note the absence of `input:`, since we're auditing the environment. +``` + +### `local` + +**Default**: `false` + +The `local` setting corresponds to `pip-audit`'s `--local` flag, which controls +whether non-local dependencies are included when auditing in "`pip list` source" +mode. + +By default all dependencies are included; with `local: true`, only dependencies +installed directly into the current environment are included. + +Example: + +```yaml +- uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + local: true +``` + +### `vulnerability-service` + +**Default**: `PyPI` + +**Options**: `PyPI`, `OSV` (case insensitive) + +The `vulnerability-service` setting controls which vulnerability service is used for the audit. +It's directly equivalent to `pip-audit --vulnerability-service=...`. + +To audit with OSV instead of PyPI: + +```yaml +- uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + vulnerability-service: osv +``` + +### `require-hashes` + +**Default**: `false` + +The `require-hashes` setting controls whether strict hash checking is enabled. +It's directly equivalent to `pip-audit --require-hashes ...`. + +Example: + +```yaml +- uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + # NOTE: only works with requirements-style inputs + inputs: requirements.txt + require-hashes: true +``` + +### `no-deps` + +**Default**: `false` + +The `no-deps` setting controls whether dependency resolution is performed. +It's directly equivalent to `pip-audit --no-deps ...`. + +Example: + +```yaml +- uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + # NOTE: only works with requirements-style inputs + inputs: requirements.txt + no-deps: true +``` + +### `summary` + +**Default**: `true` + +The `summary` setting controls whether a GitHub +[job summary](https://github.blog/2022-05-09-supercharging-github-actions-with-job-summaries/) +is rendered at the end of the action. + +Example: + +```yaml +- uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + summary: false + ``` + +### `index-url` + +**Default**: Empty, indicating [PyPI](https://pypi.org) + +The `index-url` setting specifies a base URL for an alternative PEP 503-compatible +package index. + +**This is probably not want you want.** If your goal is to add *complementary* +indices to search (such as a corporate index with private packages), see +[`extra-index-urls`](#extra-index-urls). + +Example: + +```yaml +- uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + index-url: https://example.corporate.local/simple +``` + +### `extra-index-urls` + +**Default**: Empty (no extra indexes are searched by default) + +The `extra-index-urls` setting specifies one or more *extra* PEP 503-compatible packages +indexes to search when resolving dependencies. Each URL is whitespace-separated. + +Example: + +```yaml +- uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + extra-index-urls: | + https://example.corporate.local/simple + https://prod.corporate.local/simple +``` + +### `ignore-vulns` + +**Default**: Empty (no vulnerabilities are ignored) + +The `ignore-vulns` setting specifies one or more vulnerability IDs to +ignore (i.e., exclude from the results) if present. Each ID is whitespace-separated. + +Example + +```yaml +- uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + ignore-vulns: | + GHSA-XXXX-YYYYYY + PYSEC-AAAA-BBBBB +``` + +### Internal options +
+ ⚠️ Internal options ⚠️ + + Everything below is considered "internal," which means that it + isn't part of the stable public settings and may be removed or changed at + any point. **You probably do not need these settings.** + + All internal options are prefixed with `internal-be-careful-`. + + #### `internal-be-careful-allow-failure` + + **Default**: `false` + + The `internal-be-careful-allow-failure` setting allows the job to pass, even + if the underlying `pip-audit` run fails (e.g. due to vulnerabilities detected). + + Be very careful with this setting! Using it unwittingly will prevent the action + from failing your CI when `pip-audit` fails, which is probably not what you want. + + Example: + + ```yaml + - uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + internal-be-careful-allow-failure: true + ``` + + #### `internal-be-careful-debug` + + **Default**: `false` + + The `internal-be-careful-debug` setting enables additional debug logs, + both within `pip-audit` itself and the action's harness code. You can + use it to debug troublesome configurations. + + Be mindful that `pip-audit`'s own debug logs contain HTTP requests, + which may or may not be sensitive in your use case. + + Example: + + ```yaml + - uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + internal-be-careful-debug: true + ``` + +
+ +## Troubleshooting + +This section is still a work in progress. Please help us improve it! + +### The action takes longer than I expect! + +If you're auditing a requirements file, consider setting `no-deps: true` or +`require-hashes: true`: + +```yaml +- uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + inputs: requirements.txt + require-hashes: true +``` + +or: + +```yaml +- uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + inputs: requirements.txt + no-deps: true +``` + +See the +["`pip-audit` takes longer than I expect!"](https://github.com/trailofbits/pip-audit#pip-audit-takes-longer-than-i-expect) +troubleshooting for more details. + +### The action shows dependencies that aren't in my environment! + +In the default ("`pip list` source") configuration, `pip-audit` collects all +dependencies that are visible in the current environment. + +Depending on the project or CI's configuration, this can include packages installed +by the host system itself, or other Python projects that happen to be installed. + +To minimize external dependencies, you can opt into a virtual environment: + +```yaml +- uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + # must be populated earlier in the CI + virtual-environment: env/ +``` + +and, more aggressively, specify that only dependencies marked as "local" +in the virtual environment should be included: + +```yaml +- uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + # must be populated earlier in the CI + virtual-environment: env/ + local: true +``` + +## Licensing + +`gh-action-pip-audit` is licensed under the Apache 2.0 License. + +## Code of Conduct + +Everyone interacting with this project is expected to follow the +[PSF Code of Conduct](https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md). diff --git a/.github/actions/gh-action-pip-audit/action.py b/.github/actions/gh-action-pip-audit/action.py new file mode 100755 index 0000000000..aac0643f1d --- /dev/null +++ b/.github/actions/gh-action-pip-audit/action.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 + +# action.py: run pip-audit +# +# most state is passed in as environment variables; the only argument +# is a whitespace-separated list of inputs + +import os +import subprocess +import sys +from base64 import b64encode +from pathlib import Path + +_OUTPUTS = [sys.stderr] +_SUMMARY = Path(os.getenv("GITHUB_STEP_SUMMARY")).open("a") +_RENDER_SUMMARY = os.getenv("GHA_PIP_AUDIT_SUMMARY", "true") == "true" +_DEBUG = os.getenv("GHA_PIP_AUDIT_INTERNAL_BE_CAREFUL_DEBUG", "false") != "false" + +if _RENDER_SUMMARY: + _OUTPUTS.append(_SUMMARY) + + +def _summary(msg): + if _RENDER_SUMMARY: + print(msg, file=_SUMMARY) + + +def _debug(msg): + if _DEBUG: + print(f"\033[93mDEBUG: {msg}\033[0m", file=sys.stderr) + + +def _log(msg): + for output in _OUTPUTS: + print(msg, file=output) + + +def _pip_audit(*args): + return ["python", "-m", "pip_audit", *args] + + +def _fatal_help(msg): + print(f"::error::❌ {msg}") + sys.exit(1) + + +inputs = [Path(p).resolve() for p in sys.argv[1].split()] +summary = Path(os.getenv("GITHUB_STEP_SUMMARY")).open("a") + +# The arguments we pass into `pip-audit` get built up in this list. +pip_audit_args = [ + # The spinner is useless in the CI. + "--progress-spinner=off", + # We intend to emit a Markdown-formatted table. + "--format=markdown", + # `pip cache dir` doesn't work in this container for some reason, and I + # haven't debugged it yet. + "--cache-dir=/tmp/pip-audit-cache", + # Include full descriptions in the output. + "--desc", + # Write the output to this logfile, which we'll turn into the step summary (if configured). + "--output=/tmp/pip-audit-output.txt", +] + +if _DEBUG: + pip_audit_args.append("--verbose") + +if os.getenv("GHA_PIP_AUDIT_NO_DEPS", "false") != "false": + pip_audit_args.append("--no-deps") + +if os.getenv("GHA_PIP_AUDIT_REQUIRE_HASHES", "false") != "false": + pip_audit_args.append("--require-hashes") + +if os.getenv("GHA_PIP_AUDIT_LOCAL", "false") != "false": + pip_audit_args.append("--local") + +index_url = os.getenv("GHA_PIP_AUDIT_INDEX_URL") +if index_url != "": + pip_audit_args.extend(["--index-url", index_url]) + + +extra_index_urls = os.getenv("GHA_PIP_AUDIT_EXTRA_INDEX_URLS", "").split() +for url in extra_index_urls: + pip_audit_args.extend(["--extra-index-url", url]) + + +ignored_vuln_ids = os.getenv("GHA_PIP_AUDIT_IGNORE_VULNS", "").split() +for vuln_id in ignored_vuln_ids: + pip_audit_args.extend(["--ignore-vuln", vuln_id]) + +pip_audit_args.extend( + [ + "--vulnerability-service", + os.getenv("GHA_PIP_AUDIT_VULNERABILITY_SERVICE", "pypi").lower(), + ] +) + +# If inputs is empty, we let `pip-audit` run in "`pip list` source" mode by not +# adding any explicit input argument(s). +# Otherwise, we handle either exactly one project path (a directory) +# or one or more requirements-style inputs (all files). +for input_ in inputs: + # Forbid things that look like flags. This isn't a security boundary; just + # a way to prevent (less motivated) users from breaking the action on themselves. + if str(input_).startswith("-"): + _fatal_help(f"input {input_} looks like a flag") + + if input_.is_dir(): + if len(inputs) != 1: + _fatal_help("pip-audit only supports one project directory at a time") + pip_audit_args.append(input_) + else: + if not input_.is_file(): + _fatal_help(f"input {input_} does not look like a file") + pip_audit_args.extend(["--requirement", input_]) + +_debug(f"running: pip-audit {[str(a) for a in pip_audit_args]}") + +status = subprocess.run( + _pip_audit(*pip_audit_args), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env={**os.environ, "PIP_NO_CACHE_DIR": "1"}, +) + +_debug(status.stdout) + +if status.returncode == 0: + _log("🎉 pip-audit exited successfully") +else: + _log("❌ pip-audit found one or more problems") + + with open("/tmp/pip-audit-output.txt", "r") as io: + output = io.read() + + # This is really nasty: our output contains multiple lines, + # so we can't naively stuff it into an output (since this is all done + # in-channel as a special command on stdout). + print(f"::set-output name=output::{b64encode(output.encode()).decode()}") + + _log(output) + + +_summary( + """ +
+ + Raw `pip-audit` output + + +``` + """ +) +_log(status.stdout) +_summary( + """ +``` +
+ """ +) + +# Normally, we exit with the same code as `pip-audit`, but the user can +# explicitly configure the CI to always pass. +# This is primarily useful for our own self-test workflows. +if os.getenv("GHA_PIP_AUDIT_INTERNAL_BE_CAREFUL_ALLOW_FAILURE", "false") != "false": + sys.exit(0) +else: + sys.exit(status.returncode) diff --git a/.github/actions/gh-action-pip-audit/action.yml b/.github/actions/gh-action-pip-audit/action.yml new file mode 100644 index 0000000000..3574e61fae --- /dev/null +++ b/.github/actions/gh-action-pip-audit/action.yml @@ -0,0 +1,87 @@ +name: "gh-action-pip-audit" +author: "William Woodruff " +description: "Use pip-audit to scan Python dependencies for known vulnerabilities" +inputs: + summary: + description: "render a Markdown summary of the audit (default true)" + required: false + default: true + no-deps: + description: "don't do any dependency resolution (requires fully pinned requirements) (default false)" + required: false + default: false + require-hashes: + description: "enforce hashes (requirements-style inputs only) (default false)" + required: false + default: false + vulnerability-service: + description: "the vulnerability service to use (PyPI or OSV, defaults to PyPI)" + required: false + default: "PyPI" + inputs: + description: "the inputs to audit, whitespace separated (defaults to current path)" + required: false + default: "" + virtual-environment: + description: "the virtual environment to audit within (default none)" + required: false + default: "" + local: + description: "for environmental audits, consider only packages marked local (default false)" + required: false + default: false + index-url: + description: "the base URL for the PEP 503-compatible package index to use" + required: false + default: "" + extra-index-urls: + description: "extra PEP 503-compatible indexes to use, whitespace separated" + required: false + default: "" + ignore-vulns: + description: "vulnerabilities to explicitly exclude, if present (whitespace separated)" + required: false + default: "" + internal-be-careful-allow-failure: + description: "don't fail the job if the audit fails (default false)" + required: false + default: false + internal-be-careful-debug: + description: "run with debug logs (default false)" + required: false + default: false +outputs: + internal-be-careful-output: + description: "the column-formatted output from pip-audit, wrapped as base64" + value: "${{ steps.pip-audit.outputs.output }}" +runs: + using: "composite" + steps: + - name: Set up pip-audit + run: | + # NOTE: Sourced, not executed as a script. + source "${{ github.action_path }}/setup/setup.bash" + env: + GHA_PIP_AUDIT_VIRTUAL_ENVIRONMENT: "${{ inputs.virtual-environment }}" + shell: bash + + - name: Run pip-audit + id: pip-audit + run: | + # NOTE: Sourced, not executed as a script. + source "${{ github.action_path }}/setup/venv.bash" + + ${{ github.action_path }}/action.py "${{ inputs.inputs }}" + env: + GHA_PIP_AUDIT_SUMMARY: "${{ inputs.summary }}" + GHA_PIP_AUDIT_NO_DEPS: "${{ inputs.no-deps }}" + GHA_PIP_AUDIT_REQUIRE_HASHES: "${{ inputs.require-hashes }}" + GHA_PIP_AUDIT_VULNERABILITY_SERVICE: "${{ inputs.vulnerability-service }}" + GHA_PIP_AUDIT_VIRTUAL_ENVIRONMENT: "${{ inputs.virtual-environment }}" + GHA_PIP_AUDIT_LOCAL: "${{ inputs.local }}" + GHA_PIP_AUDIT_INDEX_URL: "${{ inputs.index-url }}" + GHA_PIP_AUDIT_EXTRA_INDEX_URLS: "${{ inputs.extra-index-urls }}" + GHA_PIP_AUDIT_IGNORE_VULNS: "${{ inputs.ignore-vulns }}" + GHA_PIP_AUDIT_INTERNAL_BE_CAREFUL_ALLOW_FAILURE: "${{ inputs.internal-be-careful-allow-failure }}" + GHA_PIP_AUDIT_INTERNAL_BE_CAREFUL_DEBUG: "${{ inputs.internal-be-careful-debug }}" + shell: bash diff --git a/.github/actions/gh-action-pip-audit/dev-requirements.txt b/.github/actions/gh-action-pip-audit/dev-requirements.txt new file mode 100644 index 0000000000..f086aa46b8 --- /dev/null +++ b/.github/actions/gh-action-pip-audit/dev-requirements.txt @@ -0,0 +1,3 @@ +flake8 +isort +black diff --git a/.github/actions/gh-action-pip-audit/requirements.txt b/.github/actions/gh-action-pip-audit/requirements.txt new file mode 100644 index 0000000000..ca00871fd9 --- /dev/null +++ b/.github/actions/gh-action-pip-audit/requirements.txt @@ -0,0 +1 @@ +pip-audit==2.4.3 diff --git a/.github/actions/gh-action-pip-audit/setup/setup.bash b/.github/actions/gh-action-pip-audit/setup/setup.bash new file mode 100644 index 0000000000..e4d8a82395 --- /dev/null +++ b/.github/actions/gh-action-pip-audit/setup/setup.bash @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -eo pipefail + +die() { + echo "::error::${1}" + exit 1 +} + +# NOTE: This file is meant to be sourced, not executed as a script. +if [[ "${0}" == "${BASH_SOURCE[0]}" ]]; then + die "Internal error: setup harness was executed instead of being sourced?" +fi + +# Load the virtual environment, if there is one. +source "${GITHUB_ACTION_PATH}/setup/venv.bash" + +# Check the Python version, making sure it's new enough (3.7+) +# The installation step immediately below will technically catch this, +# but doing it explicitly gives us the opportunity to produce a better +# error message. +vers=$(python -V | cut -d ' ' -f2) +maj_vers=$(cut -d '.' -f1 <<< "${vers}") +min_vers=$(cut -d '.' -f2 <<< "${vers}") + +[[ "${maj_vers}" == "3" && "${min_vers}" -ge 7 ]] || die "Bad Python version: ${vers}" + +python -m pip install --requirement "${GITHUB_ACTION_PATH}/requirements.txt" diff --git a/.github/actions/gh-action-pip-audit/setup/venv.bash b/.github/actions/gh-action-pip-audit/setup/venv.bash new file mode 100644 index 0000000000..3e60645381 --- /dev/null +++ b/.github/actions/gh-action-pip-audit/setup/venv.bash @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -eo pipefail + +die() { + echo "::error::${1}" + exit 1 +} + +# NOTE: This file is meant to be sourced, not executed as a script. +if [[ "${0}" == "${BASH_SOURCE[0]}" ]]; then + die "Internal error: setup harness was executed instead of being sourced?" +fi + +# If the user has explicitly specified a virtual environment, then we install +# `pip-audit` into it rather than into whatever environment the default +# `python -m pip install ...` invocation might happen to choose. +if [[ -n "${GHA_PIP_AUDIT_VIRTUAL_ENVIRONMENT}" ]] ; then + if [[ -d "${GHA_PIP_AUDIT_VIRTUAL_ENVIRONMENT}" ]]; then + source "${GHA_PIP_AUDIT_VIRTUAL_ENVIRONMENT}/bin/activate" + else + die "Fatal: virtual environment is not a directory" + fi +fi diff --git a/.github/actions/gh-action-pip-audit/test/pyproject/pyproject.toml b/.github/actions/gh-action-pip-audit/test/pyproject/pyproject.toml new file mode 100644 index 0000000000..12ce5e7375 --- /dev/null +++ b/.github/actions/gh-action-pip-audit/test/pyproject/pyproject.toml @@ -0,0 +1,6 @@ +# this is not a real pyproject.toml; only enough to run the selftests in CI. + +[project] +dependencies = [ + "pyyaml==5.1" +] diff --git a/.github/actions/gh-action-pip-audit/test/vulnerable.txt b/.github/actions/gh-action-pip-audit/test/vulnerable.txt new file mode 100644 index 0000000000..6c05a614d2 --- /dev/null +++ b/.github/actions/gh-action-pip-audit/test/vulnerable.txt @@ -0,0 +1 @@ +pyyaml==5.1 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 10e2bc0ed1..a5ee8534ac 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -201,8 +201,20 @@ jobs: run: | pip install "tox==3.24.4" + - name: Install Library Into Virtualenv + run: | + python -m venv venv/ + source venv/bin/activate + python -m pip install . + + - name: Run Pip Audit Check + uses: ./.github/actions/gh-action-pip-audit/ # v1.0.0 + with: + virtual-environment: venv/ + - name: Run Checks run: | + rm -rf venv/ || true script -e -c "tox -e black-check,checks,import-timings,lint,pylint" micro-benchmarks: diff --git a/scripts/check_asf_license_headers.py b/scripts/check_asf_license_headers.py index 961826b479..e8df041b17 100755 --- a/scripts/check_asf_license_headers.py +++ b/scripts/check_asf_license_headers.py @@ -37,6 +37,7 @@ 'build', 'dist', 'docs', + 'venv/', 'libcloud/utils/iso8601.py' ]