_format: Show vulnerabilities removed via --fix in JSON and columna…

…r formats (#222)

* _format: Add `--fix` information to `VulnerabilityService` interface

* json: Add fixes to JSON format

* columns: Add fixes to Columns format

* columns: Place applied fix column in front of description

* test: Test columns formatting

* test: Test JSON formatting of fixes

* test: Check that CycloneDX logs a warning when fixes are provided

* README: Update example output

* test: Fix CLI test

* CHANGELOG: record changes

Co-authored-by: William Woodruff <william.woodruff@trailofbits.com>
Co-authored-by: William Woodruff <william@trailofbits.com>

CHANGELOG.md

@@ -12,7 +12,8 @@ All versions prior to 0.0.9 are untracked.
 
 * CLI: The `--fix` flag has been added, allowing users to attempt to
   automatically upgrade any vulnerable dependencies to the first safe version
-  available (#[212](https://github.com/trailofbits/pip-audit/pull/212))
+  available ([#212](https://github.com/trailofbits/pip-audit/pull/212),
+  [#222](https://github.com/trailofbits/pip-audit/pull/222))
 
 ### Changed
 

README.md

@@ -148,7 +148,7 @@ No known vulnerabilities found
 Audit dependencies when there are vulnerabilities present:
 ```
 $ pip-audit
-Found 2 known vulnerabilities in 1 packages
+Found 2 known vulnerabilities in 1 package
 Name  Version ID             Fix Versions
 ----  ------- -------------- ------------
 Flask 0.5     PYSEC-2019-179 1.0
@@ -158,7 +158,7 @@ Flask 0.5     PYSEC-2018-66  0.12.3
 Audit dependencies including descriptions:
 ```
 $ pip-audit --desc
-Found 2 known vulnerabilities in 1 packages
+Found 2 known vulnerabilities in 1 package
 Name  Version ID             Fix Versions Description
 ----  ------- -------------- ------------ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Flask 0.5     PYSEC-2019-179 1.0          The Pallets Project Flask before 1.0 is affected by: unexpected memory usage. The impact is: denial of service. The attack vector is: crafted encoded JSON data. The fixed version is: 1. NOTE: this may overlap CVE-2018-1000656.
@@ -168,7 +168,7 @@ Flask 0.5     PYSEC-2018-66  0.12.3       The Pallets Project flask version Befo
 Audit dependencies in JSON format:
 ```
 $ pip-audit -f json | jq
-Found 2 known vulnerabilities in 1 packages
+Found 2 known vulnerabilities in 1 package
 [
   {
     "name": "flask",
@@ -221,11 +221,11 @@ Found 2 known vulnerabilities in 1 packages
 Audit and attempt to automatically upgrade vulnerable dependencies:
 ```
 $ pip-audit --fix
-Found 2 known vulnerabilities in 1 packages and fixed 2 vulnerabilities in 1 packages
-Name  Version ID             Fix Versions
------ ------- -------------- ------------
-Flask 0.5     PYSEC-2019-179 1.0
-Flask 0.5     PYSEC-2018-66  0.12.3
+Found 2 known vulnerabilities in 1 package and fixed 2 vulnerabilities in 1 package
+Name  Version ID             Fix Versions Applied Fix
+----- ------- -------------- ------------ ----------------------------------------
+flask 0.5     PYSEC-2019-179 1.0          Successfully upgraded flask (0.5 => 1.0)
+flask 0.5     PYSEC-2018-66  0.12.3       Successfully upgraded flask (0.5 => 1.0)
 ```
 
 ## Security Model

pip_audit/_cli.py

@@ -329,7 +329,7 @@ def audit() -> None:
                 f"{'package' if fixed_pkg_count == 1 else 'packages'}"
             )
         print(summary_msg, file=sys.stderr)
-        print(formatter.format(result))
+        print(formatter.format(result, fixes))
         if pkg_count != fixed_pkg_count:
             sys.exit(1)
     else:

pip_audit/_format/columns.py

@@ -3,10 +3,11 @@
 """
 
 from itertools import zip_longest
-from typing import Any, Dict, Iterable, List, Tuple, cast
+from typing import Any, Dict, Iterable, List, Optional, Tuple, cast
 
 from packaging.version import Version
 
+import pip_audit._fix as fix
 import pip_audit._service as service
 
 from .interface import VulnerabilityFormat
@@ -39,7 +40,11 @@ def __init__(self, output_desc: bool):
         """
         self.output_desc = output_desc
 
-    def format(self, result: Dict[service.Dependency, List[service.VulnerabilityResult]]) -> str:
+    def format(
+        self,
+        result: Dict[service.Dependency, List[service.VulnerabilityResult]],
+        fixes: List[fix.FixVersion],
+    ) -> str:
         """
         Returns a column formatted string for a given mapping of dependencies to vulnerability
         results.
@@ -48,15 +53,18 @@ def format(self, result: Dict[service.Dependency, List[service.VulnerabilityResu
         """
         vuln_data: List[List[Any]] = []
         header = ["Name", "Version", "ID", "Fix Versions"]
+        if fixes:
+            header.append("Applied Fix")
         if self.output_desc:
             header.append("Description")
         vuln_data.append(header)
         for dep, vulns in result.items():
             if dep.is_skipped():
                 continue
             dep = cast(service.ResolvedDependency, dep)
+            applied_fix = next((f for f in fixes if f.dep == dep), None)
             for vuln in vulns:
-                vuln_data.append(self._format_vuln(dep, vuln))
+                vuln_data.append(self._format_vuln(dep, vuln, applied_fix))
 
         vuln_strings, sizes = tabulate(vuln_data)
 
@@ -96,14 +104,19 @@ def format(self, result: Dict[service.Dependency, List[service.VulnerabilityResu
         return columns_string
 
     def _format_vuln(
-        self, dep: service.ResolvedDependency, vuln: service.VulnerabilityResult
+        self,
+        dep: service.ResolvedDependency,
+        vuln: service.VulnerabilityResult,
+        applied_fix: Optional[fix.FixVersion],
     ) -> List[Any]:
         vuln_data = [
             dep.canonical_name,
             dep.version,
             vuln.id,
             self._format_fix_versions(vuln.fix_versions),
         ]
+        if applied_fix is not None:
+            vuln_data.append(self._format_applied_fix(applied_fix))
         if self.output_desc:
             vuln_data.append(vuln.description)
         return vuln_data
@@ -116,3 +129,16 @@ def _format_skipped_dep(self, dep: service.SkippedDependency) -> List[Any]:
             dep.canonical_name,
             dep.skip_reason,
         ]
+
+    def _format_applied_fix(self, applied_fix: fix.FixVersion) -> str:
+        if applied_fix.is_skipped():
+            applied_fix = cast(fix.SkippedFixVersion, applied_fix)
+            return (
+                f"Failed to fix {applied_fix.dep.canonical_name} ({applied_fix.dep.version}): "
+                f"{applied_fix.skip_reason}"
+            )
+        applied_fix = cast(fix.ResolvedFixVersion, applied_fix)
+        return (
+            f"Successfully upgraded {applied_fix.dep.canonical_name} ({applied_fix.dep.version} "
+            f"=> {applied_fix.version})"
+        )

pip_audit/_format/cyclonedx.py

@@ -3,6 +3,7 @@
 """
 
 import enum
+import logging
 from typing import Dict, List, cast
 
 from cyclonedx import output
@@ -11,10 +12,13 @@
 from cyclonedx.model.vulnerability import Vulnerability
 from cyclonedx.parser import BaseParser
 
+import pip_audit._fix as fix
 import pip_audit._service as service
 
 from .interface import VulnerabilityFormat
 
+logger = logging.getLogger(__name__)
+
 
 class _PipAuditResultParser(BaseParser):
     def __init__(self, result: Dict[service.Dependency, List[service.VulnerabilityResult]]):
@@ -64,13 +68,20 @@ def __init__(self, inner_format: "CycloneDxFormat.InnerFormat"):
 
         self._inner_format = inner_format
 
-    def format(self, result: Dict[service.Dependency, List[service.VulnerabilityResult]]) -> str:
+    def format(
+        self,
+        result: Dict[service.Dependency, List[service.VulnerabilityResult]],
+        fixes: List[fix.FixVersion],
+    ) -> str:
         """
         Returns a CycloneDX formatted string for a given mapping of dependencies to vulnerability
         results.
 
         See `VulnerabilityFormat.format`.
         """
+        if fixes:
+            logger.warning("--fix output is unsupported by CycloneDX formats")
+
         parser = _PipAuditResultParser(result)
         bom = Bom.from_parser(parser)
 

pip_audit/_format/interface.py

@@ -4,6 +4,7 @@
 from abc import ABC
 from typing import Dict, List
 
+import pip_audit._fix as fix
 import pip_audit._service as service
 
 
@@ -13,7 +14,9 @@ class VulnerabilityFormat(ABC):
     """
 
     def format(
-        self, result: Dict[service.Dependency, List[service.VulnerabilityResult]]
+        self,
+        result: Dict[service.Dependency, List[service.VulnerabilityResult]],
+        fixes: List[fix.FixVersion],
     ) -> str:  # pragma: no cover
         """
         Convert a mapping of dependencies to vulnerabilities into a string.

pip_audit/_format/json.py

@@ -5,6 +5,7 @@
 import json
 from typing import Any, Dict, List, cast
 
+import pip_audit._fix as fix
 import pip_audit._service as service
 
 from .interface import VulnerabilityFormat
@@ -25,16 +26,26 @@ def __init__(self, output_desc: bool):
         """
         self.output_desc = output_desc
 
-    def format(self, result: Dict[service.Dependency, List[service.VulnerabilityResult]]) -> str:
+    def format(
+        self,
+        result: Dict[service.Dependency, List[service.VulnerabilityResult]],
+        fixes: List[fix.FixVersion],
+    ) -> str:
         """
         Returns a JSON formatted string for a given mapping of dependencies to vulnerability
         results.
 
         See `VulnerabilityFormat.format`.
         """
-        output_json = []
+        output_json = {}
+        dep_json = []
         for dep, vulns in result.items():
-            output_json.append(self._format_dep(dep, vulns))
+            dep_json.append(self._format_dep(dep, vulns))
+        output_json["dependencies"] = dep_json
+        fix_json = []
+        for f in fixes:
+            fix_json.append(self._format_fix(f))
+        output_json["fixes"] = fix_json
         return json.dumps(output_json)
 
     def _format_dep(
@@ -62,3 +73,18 @@ def _format_vuln(self, vuln: service.VulnerabilityResult) -> Dict[str, Any]:
         if self.output_desc:
             vuln_json["description"] = vuln.description
         return vuln_json
+
+    def _format_fix(self, fix_version: fix.FixVersion) -> Dict[str, Any]:
+        if fix_version.is_skipped():
+            fix_version = cast(fix.SkippedFixVersion, fix_version)
+            return {
+                "name": fix_version.dep.canonical_name,
+                "version": str(fix_version.dep.version),
+                "skip_reason": fix_version.skip_reason,
+            }
+        fix_version = cast(fix.ResolvedFixVersion, fix_version)
+        return {
+            "name": fix_version.dep.canonical_name,
+            "old_version": str(fix_version.dep.version),
+            "new_version": str(fix_version.version),
+        }

test/format/conftest.py

@@ -3,10 +3,15 @@
 import pytest
 from packaging.version import Version
 
+import pip_audit._fix as fix
 import pip_audit._service as service
 
+_RESOLVED_DEP_FOO = service.ResolvedDependency(name="foo", version=Version("1.0"))
+_RESOLVED_DEP_BAR = service.ResolvedDependency(name="bar", version=Version("0.1"))
+_SKIPPED_DEP = service.SkippedDependency(name="bar", skip_reason="skip-reason")
+
 _TEST_VULN_DATA: Dict[service.Dependency, List[service.VulnerabilityResult]] = {
-    service.ResolvedDependency(name="foo", version=Version("1.0")): [
+    _RESOLVED_DEP_FOO: [
         service.VulnerabilityResult(
             id="VULN-0",
             description="The first vulnerability",
@@ -21,7 +26,7 @@
             fix_versions=[Version("1.0")],
         ),
     ],
-    service.ResolvedDependency(name="bar", version=Version("0.1")): [
+    _RESOLVED_DEP_BAR: [
         service.VulnerabilityResult(
             id="VULN-2",
             description="The third vulnerability",
@@ -31,7 +36,7 @@
 }
 
 _TEST_VULN_DATA_SKIPPED_DEP: Dict[service.Dependency, List[service.VulnerabilityResult]] = {
-    service.ResolvedDependency(name="foo", version=Version("1.0")): [
+    _RESOLVED_DEP_FOO: [
         service.VulnerabilityResult(
             id="VULN-0",
             description="The first vulnerability",
@@ -41,9 +46,19 @@
             ],
         ),
     ],
-    service.SkippedDependency(name="bar", skip_reason="skip-reason"): [],
+    _SKIPPED_DEP: [],
 }
 
+_TEST_FIX_DATA: List[fix.FixVersion] = [
+    fix.ResolvedFixVersion(dep=_RESOLVED_DEP_FOO, version=Version("1.8")),
+    fix.ResolvedFixVersion(dep=_RESOLVED_DEP_BAR, version=Version("0.3")),
+]
+
+_TEST_SKIPPED_FIX_DATA: List[fix.FixVersion] = [
+    fix.ResolvedFixVersion(dep=_RESOLVED_DEP_FOO, version=Version("1.8")),
+    fix.SkippedFixVersion(dep=_RESOLVED_DEP_BAR, skip_reason="skip-reason"),
+]
+
 
 @pytest.fixture(autouse=True)
 def vuln_data():
@@ -53,3 +68,13 @@ def vuln_data():
 @pytest.fixture(autouse=True)
 def vuln_data_skipped_dep():
     return _TEST_VULN_DATA_SKIPPED_DEP
+
+
+@pytest.fixture(autouse=True)
+def fix_data():
+    return _TEST_FIX_DATA
+
+
+@pytest.fixture(autouse=True)
+def skipped_fix_data():
+    return _TEST_SKIPPED_FIX_DATA

test/format/test_columns.py

@@ -8,7 +8,7 @@ def test_columns(vuln_data):
 foo  1.0     VULN-0 1.1,1.4      The first vulnerability
 foo  1.0     VULN-1 1.0          The second vulnerability
 bar  0.1     VULN-2              The third vulnerability"""
-    assert columns_format.format(vuln_data) == expected_columns
+    assert columns_format.format(vuln_data, list()) == expected_columns
 
 
 def test_columns_no_desc(vuln_data):
@@ -18,7 +18,7 @@ def test_columns_no_desc(vuln_data):
 foo  1.0     VULN-0 1.1,1.4
 foo  1.0     VULN-1 1.0
 bar  0.1     VULN-2"""
-    assert columns_format.format(vuln_data) == expected_columns
+    assert columns_format.format(vuln_data, list()) == expected_columns
 
 
 def test_columns_skipped_dep(vuln_data_skipped_dep):
@@ -29,5 +29,24 @@ def test_columns_skipped_dep(vuln_data_skipped_dep):
 Name Skip Reason
 ---- -----------
 bar  skip-reason"""
-    print(columns_format.format(vuln_data_skipped_dep))
-    assert columns_format.format(vuln_data_skipped_dep) == expected_columns
+    assert columns_format.format(vuln_data_skipped_dep, list()) == expected_columns
+
+
+def test_columns_fix(vuln_data, fix_data):
+    columns_format = format.ColumnsFormat(False)
+    expected_columns = """Name Version ID     Fix Versions Applied Fix
+---- ------- ------ ------------ --------------------------------------
+foo  1.0     VULN-0 1.1,1.4      Successfully upgraded foo (1.0 => 1.8)
+foo  1.0     VULN-1 1.0          Successfully upgraded foo (1.0 => 1.8)
+bar  0.1     VULN-2              Successfully upgraded bar (0.1 => 0.3)"""
+    assert columns_format.format(vuln_data, fix_data) == expected_columns
+
+
+def test_columns_skipped_fix(vuln_data, skipped_fix_data):
+    columns_format = format.ColumnsFormat(False)
+    expected_columns = """Name Version ID     Fix Versions Applied Fix
+---- ------- ------ ------------ --------------------------------------
+foo  1.0     VULN-0 1.1,1.4      Successfully upgraded foo (1.0 => 1.8)
+foo  1.0     VULN-1 1.0          Successfully upgraded foo (1.0 => 1.8)
+bar  0.1     VULN-2              Failed to fix bar (0.1): skip-reason"""
+    assert columns_format.format(vuln_data, skipped_fix_data) == expected_columns