From 89342e5fbd859fee2181cb43a03b8d5bf3297542 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 26 Jan 2023 20:12:06 +0530 Subject: [PATCH 1/2] Add istio improver Signed-off-by: Tushar Goel --- vulnerabilities/importers/istio.py | 86 +++++++++++++++++++++++++++ vulnerabilities/improvers/__init__.py | 1 + 2 files changed, 87 insertions(+) diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index c73f5eb31..951fa5297 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -6,13 +6,20 @@ # See https://github.com/nexB/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +import logging import re +from datetime import datetime from pathlib import Path +from typing import Iterable +from typing import List +from typing import Mapping +from typing import Optional from typing import Set import pytz import saneyaml from dateutil import parser +from django.db.models.query import QuerySet from packageurl import PackageURL from univers.version_constraint import VersionConstraint from univers.version_range import GitHubVersionRange @@ -23,10 +30,22 @@ from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference +from vulnerabilities.importer import UnMergeablePackageError +from vulnerabilities.improver import Improver +from vulnerabilities.improver import Inference +from vulnerabilities.models import Advisory +from vulnerabilities.package_managers import GitHubTagsAPI +from vulnerabilities.package_managers import VersionAPI +from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage +from vulnerabilities.utils import get_affected_packages_by_patched_package +from vulnerabilities.utils import nearest_patched_package +from vulnerabilities.utils import resolve_version_range from vulnerabilities.utils import split_markdown_front_matter is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match +logger = logging.getLogger(__name__) + class IstioImporter(Importer): spdx_license_expression = "Apache-2.0" @@ -136,3 +155,70 @@ def get_data_from_md(self, path): with open(path) as f: front_matter, _ = split_markdown_front_matter(f.read()) return saneyaml.load(front_matter) + + +class IstioImprover(Improver): + def __init__(self) -> None: + self.versions_fetcher_by_purl: Mapping[str, VersionAPI] = {} + self.vesions_by_purl = {} + + @property + def interesting_advisories(self) -> QuerySet: + return Advisory.objects.filter(created_by=IstioImporter.qualified_name) + + def get_package_versions( + self, package_url: PackageURL, until: Optional[datetime] = None + ) -> List[str]: + """ + Return a list of `valid_versions` for the `package_url` + """ + api_name = "istio/istio" + versions_fetcher = GitHubTagsAPI() + return versions_fetcher.get_until(package_name=api_name, until=until).valid_versions + + def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: + """ + Yield Inferences for the given advisory data + """ + if not advisory_data.affected_packages: + return + for affected_package in advisory_data.affected_packages: + purl = affected_package.package + affected_version_range = affected_package.affected_version_range + pkg_type = purl.type + pkg_namespace = purl.namespace + pkg_name = purl.name + if not self.vesions_by_purl.get("istio/istio"): + valid_versions = self.get_package_versions( + package_url=purl, until=advisory_data.date_published + ) + self.vesions_by_purl["istio/istio"] = valid_versions + valid_versions = self.vesions_by_purl["istio/istio"] + aff_vers, unaff_vers = resolve_version_range( + affected_version_range=affected_version_range, + package_versions=valid_versions, + ) + affected_purls = [ + PackageURL(type=pkg_type, namespace=pkg_namespace, name=pkg_name, version=version) + for version in aff_vers + ] + + unaffected_purls = [ + PackageURL(type=pkg_type, namespace=pkg_namespace, name=pkg_name, version=version) + for version in unaff_vers + ] + + affected_packages: List[LegacyAffectedPackage] = nearest_patched_package( + vulnerable_packages=affected_purls, resolved_packages=unaffected_purls + ) + + for ( + fixed_package, + affected_packages, + ) in get_affected_packages_by_patched_package(affected_packages).items(): + yield Inference.from_advisory_data( + advisory_data, + confidence=100, # We are getting all valid versions to get this inference + affected_purls=affected_packages, + fixed_purl=fixed_package, + ) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 98192e264..86a3525ef 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -17,6 +17,7 @@ importers.github.GitHubBasicImprover, importers.debian.DebianBasicImprover, importers.gitlab.GitLabBasicImprover, + importers.istio.IstioImprover, oval.DebianOvalBasicImprover, oval.UbuntuOvalBasicImprover, ] From 8b99c6dda728695cab78b14c41f6fa29e0042be8 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 26 Jan 2023 23:17:55 +0530 Subject: [PATCH 2/2] Add tests for istio improver Signed-off-by: Tushar Goel --- .../istio/istio-improver-expected.json | 302 ++++++++++++++++++ vulnerabilities/tests/test_istio.py | 31 ++ 2 files changed, 333 insertions(+) create mode 100644 vulnerabilities/tests/test_data/istio/istio-improver-expected.json diff --git a/vulnerabilities/tests/test_data/istio/istio-improver-expected.json b/vulnerabilities/tests/test_data/istio/istio-improver-expected.json new file mode 100644 index 000000000..6fe76c557 --- /dev/null +++ b/vulnerabilities/tests/test_data/istio/istio-improver-expected.json @@ -0,0 +1,302 @@ +[ + { + "vulnerability_id": null, + "aliases": [ + "CVE-2019-12243" + ], + "confidence": 100, + "summary": "Incorrect access control.", + "affected_purls": [ + { + "type": "golang", + "namespace": "istio.io", + "name": "istio", + "version": "1.1.0", + "qualifiers": null, + "subpath": null + }, + { + "type": "golang", + "namespace": "istio.io", + "name": "istio", + "version": "1.1.1", + "qualifiers": null, + "subpath": null + }, + { + "type": "golang", + "namespace": "istio.io", + "name": "istio", + "version": "1.1.2", + "qualifiers": null, + "subpath": null + }, + { + "type": "golang", + "namespace": "istio.io", + "name": "istio", + "version": "1.1.3", + "qualifiers": null, + "subpath": null + }, + { + "type": "golang", + "namespace": "istio.io", + "name": "istio", + "version": "1.1.4", + "qualifiers": null, + "subpath": null + }, + { + "type": "golang", + "namespace": "istio.io", + "name": "istio", + "version": "1.1.5", + "qualifiers": null, + "subpath": null + }, + { + "type": "golang", + "namespace": "istio.io", + "name": "istio", + "version": "1.1.6", + "qualifiers": null, + "subpath": null + }, + { + "type": "golang", + "namespace": "istio.io", + "name": "istio", + "version": "1.1.7", + "qualifiers": null, + "subpath": null + }, + { + "type": "golang", + "namespace": "istio.io", + "name": "istio", + "version": "1.1.8", + "qualifiers": null, + "subpath": null + } + ], + "fixed_purl": null, + "references": [ + { + "reference_id": "ISTIO-SECURITY-2019-001", + "url": "https://istio.io/latest/news/security/ISTIO-SECURITY-2019-001/", + "severities": [] + } + ], + "weaknesses": [] + }, + { + "vulnerability_id": null, + "aliases": [ + "CVE-2019-12243" + ], + "confidence": 100, + "summary": "Incorrect access control.", + "affected_purls": [ + { + "type": "github", + "namespace": "istio", + "name": "istio", + "version": "1.1.0", + "qualifiers": null, + "subpath": null + }, + { + "type": "github", + "namespace": "istio", + "name": "istio", + "version": "1.1.1", + "qualifiers": null, + "subpath": null + }, + { + "type": "github", + "namespace": "istio", + "name": "istio", + "version": "1.1.2", + "qualifiers": null, + "subpath": null + }, + { + "type": "github", + "namespace": "istio", + "name": "istio", + "version": "1.1.3", + "qualifiers": null, + "subpath": null + }, + { + "type": "github", + "namespace": "istio", + "name": "istio", + "version": "1.1.4", + "qualifiers": null, + "subpath": null + }, + { + "type": "github", + "namespace": "istio", + "name": "istio", + "version": "1.1.5", + "qualifiers": null, + "subpath": null + }, + { + "type": "github", + "namespace": "istio", + "name": "istio", + "version": "1.1.6", + "qualifiers": null, + "subpath": null + }, + { + "type": "github", + "namespace": "istio", + "name": "istio", + "version": "1.1.7", + "qualifiers": null, + "subpath": null + }, + { + "type": "github", + "namespace": "istio", + "name": "istio", + "version": "1.1.8", + "qualifiers": null, + "subpath": null + } + ], + "fixed_purl": null, + "references": [ + { + "reference_id": "ISTIO-SECURITY-2019-001", + "url": "https://istio.io/latest/news/security/ISTIO-SECURITY-2019-001/", + "severities": [] + } + ], + "weaknesses": [] + }, + { + "vulnerability_id": null, + "aliases": [ + "CVE-2019-12243" + ], + "confidence": 100, + "summary": "Incorrect access control.", + "affected_purls": [ + { + "type": "golang", + "namespace": "istio.io", + "name": "istio", + "version": "1.1.0", + "qualifiers": null, + "subpath": null + }, + { + "type": "golang", + "namespace": "istio.io", + "name": "istio", + "version": "1.1.15", + "qualifiers": null, + "subpath": null + }, + { + "type": "golang", + "namespace": "istio.io", + "name": "istio", + "version": "1.3.0", + "qualifiers": null, + "subpath": null + }, + { + "type": "golang", + "namespace": "istio.io", + "name": "istio", + "version": "1.3.1", + "qualifiers": null, + "subpath": null + }, + { + "type": "golang", + "namespace": "istio.io", + "name": "istio", + "version": "1.5.0", + "qualifiers": null, + "subpath": null + } + ], + "fixed_purl": null, + "references": [ + { + "reference_id": "ISTIO-SECURITY-2019-001", + "url": "https://istio.io/latest/news/security/ISTIO-SECURITY-2019-001/", + "severities": [] + } + ], + "weaknesses": [] + }, + { + "vulnerability_id": null, + "aliases": [ + "CVE-2019-12243" + ], + "confidence": 100, + "summary": "Incorrect access control.", + "affected_purls": [ + { + "type": "github", + "namespace": "istio", + "name": "istio", + "version": "1.1.0", + "qualifiers": null, + "subpath": null + }, + { + "type": "github", + "namespace": "istio", + "name": "istio", + "version": "1.1.15", + "qualifiers": null, + "subpath": null + }, + { + "type": "github", + "namespace": "istio", + "name": "istio", + "version": "1.3.0", + "qualifiers": null, + "subpath": null + }, + { + "type": "github", + "namespace": "istio", + "name": "istio", + "version": "1.3.1", + "qualifiers": null, + "subpath": null + }, + { + "type": "github", + "namespace": "istio", + "name": "istio", + "version": "1.5.0", + "qualifiers": null, + "subpath": null + } + ], + "fixed_purl": null, + "references": [ + { + "reference_id": "ISTIO-SECURITY-2019-001", + "url": "https://istio.io/latest/news/security/ISTIO-SECURITY-2019-001/", + "severities": [] + } + ], + "weaknesses": [] + } +] \ No newline at end of file diff --git a/vulnerabilities/tests/test_istio.py b/vulnerabilities/tests/test_istio.py index 60e222b8c..93a714459 100644 --- a/vulnerabilities/tests/test_istio.py +++ b/vulnerabilities/tests/test_istio.py @@ -7,9 +7,14 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import json import os +from unittest import mock +from vulnerabilities.importer import AdvisoryData from vulnerabilities.importers.istio import IstioImporter +from vulnerabilities.importers.istio import IstioImprover +from vulnerabilities.improvers.default import DefaultImprover from vulnerabilities.tests import util_tests BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -43,3 +48,29 @@ def test_istio_process_file(): expected_file = os.path.join(TEST_DIR, f"istio-expected.json") result = [data.to_dict() for data in list(IstioImporter().process_file(path))] util_tests.check_results_against_json(result, expected_file) + + +@mock.patch("vulnerabilities.importers.istio.IstioImprover.get_package_versions") +def test_istio_improver(mock_response): + advisory_file = os.path.join(TEST_DIR, f"istio-expected.json") + expected_file = os.path.join(TEST_DIR, f"istio-improver-expected.json") + with open(advisory_file) as exp: + advisories = [AdvisoryData.from_dict(adv) for adv in (json.load(exp))] + mock_response.return_value = [ + "1.1.0", + "1.1.1", + "1.1.2", + "1.1.3", + "1.1.4", + "1.1.5", + "1.1.6", + "1.1.7", + "1.1.8", + ] + improvers = [IstioImprover(), DefaultImprover()] + result = [] + for improver in improvers: + for advisory in advisories: + inference = [data.to_dict() for data in improver.get_inferences(advisory)] + result.extend(inference) + util_tests.check_results_against_json(result, expected_file)