From 0fe73ef18b5e2af63251eadf9809a03d0a803907 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Mon, 20 May 2024 00:39:59 +0300 Subject: [PATCH] Rename migration file Fix migration conflict Fix cargo test ( add ref type ) Fix merge conflict Add api test for epss Keep the EPSS score separate from the severity score range Add epss tab Add epss to severity scoring Add published_at date to the Vulnerability score model. Add EPSS importer Add EPSS model Add EPSS UI Add EPSS to api Fix api test Signed-off-by: ziadhany --- vulnerabilities/api.py | 9 +- vulnerabilities/import_runner.py | 1 + vulnerabilities/importer.py | 6 + vulnerabilities/importers/__init__.py | 2 + vulnerabilities/importers/epss.py | 67 +++++++ ...erabilityseverity_published_at_and_more.py | 43 +++++ vulnerabilities/models.py | 4 + vulnerabilities/severity_systems.py | 14 ++ .../templates/vulnerability_details.html | 178 ++++++++++++------ vulnerabilities/tests/test_api.py | 51 ++++- vulnerabilities/views.py | 12 +- 11 files changed, 325 insertions(+), 62 deletions(-) create mode 100644 vulnerabilities/importers/epss.py create mode 100644 vulnerabilities/migrations/0059_vulnerabilityseverity_published_at_and_more.py diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index 743a3dea2..64007c59c 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -38,7 +38,14 @@ class VulnerabilitySeveritySerializer(serializers.ModelSerializer): class Meta: model = VulnerabilitySeverity - fields = ["value", "scoring_system", "scoring_elements"] + fields = ["value", "scoring_system", "scoring_elements", "published_at"] + + def to_representation(self, instance): + data = super().to_representation(instance) + published_at = data.get("published_at", None) + if not published_at: + data.pop("published_at") + return data class VulnerabilityReferenceSerializer(serializers.ModelSerializer): diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index 89eca49d6..d8f3f5102 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -189,6 +189,7 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver defaults={ "value": str(severity.value), "scoring_elements": str(severity.scoring_elements), + "published_at": str(severity.published_at), }, ) if updated: diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index 972196d65..005534512 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -52,12 +52,17 @@ class VulnerabilitySeverity: system: ScoringSystem value: str scoring_elements: str = "" + published_at: Optional[datetime.datetime] = None def to_dict(self): + published_at_dict = ( + {"published_at": self.published_at.isoformat()} if self.published_at else {} + ) return { "system": self.system.identifier, "value": self.value, "scoring_elements": self.scoring_elements, + **published_at_dict, } @classmethod @@ -70,6 +75,7 @@ def from_dict(cls, severity: dict): system=SCORING_SYSTEMS[severity["system"]], value=severity["value"], scoring_elements=severity.get("scoring_elements", ""), + published_at=severity.get("published_at"), ) diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index cedd8902b..70b9190b1 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -15,6 +15,7 @@ from vulnerabilities.importers import debian from vulnerabilities.importers import debian_oval from vulnerabilities.importers import elixir_security +from vulnerabilities.importers import epss from vulnerabilities.importers import fireeye from vulnerabilities.importers import gentoo from vulnerabilities.importers import github @@ -71,6 +72,7 @@ oss_fuzz.OSSFuzzImporter, ruby.RubyImporter, github_osv.GithubOSVImporter, + epss.EPSSImporter, ] IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY} diff --git a/vulnerabilities/importers/epss.py b/vulnerabilities/importers/epss.py new file mode 100644 index 000000000..83822fa5d --- /dev/null +++ b/vulnerabilities/importers/epss.py @@ -0,0 +1,67 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import csv +import gzip +import logging +import urllib.request +from datetime import datetime +from typing import Iterable + +from vulnerabilities import severity_systems +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import Importer +from vulnerabilities.importer import Reference +from vulnerabilities.importer import VulnerabilitySeverity + +logger = logging.getLogger(__name__) + + +class EPSSImporter(Importer): + """Exploit Prediction Scoring System (EPSS) Importer""" + + advisory_url = "https://epss.cyentia.com/epss_scores-current.csv.gz" + spdx_license_expression = "unknown" + importer_name = "EPSS Importer" + + def advisory_data(self) -> Iterable[AdvisoryData]: + response = urllib.request.urlopen(self.advisory_url) + with gzip.open(response, "rb") as f: + lines = [l.decode("utf-8") for l in f.readlines()] + + epss_reader = csv.reader(lines) + model_version, score_date = next( + epss_reader + ) # score_date='score_date:2024-05-19T00:00:00+0000' + published_at = datetime.strptime(score_date[11::], "%Y-%m-%dT%H:%M:%S%z") + + next(epss_reader) # skip the header row + for epss_row in epss_reader: + cve, score, percentile = epss_row + + if not cve or not score or not percentile: + logger.error(f"Invalid epss row: {epss_row}") + continue + + severity = VulnerabilitySeverity( + system=severity_systems.EPSS, + value=score, + scoring_elements=percentile, + published_at=published_at, + ) + + references = Reference( + url=f"https://api.first.org/data/v1/epss?cve={cve}", + severities=[severity], + ) + + yield AdvisoryData( + aliases=[cve], + references=[references], + url=self.advisory_url, + ) diff --git a/vulnerabilities/migrations/0059_vulnerabilityseverity_published_at_and_more.py b/vulnerabilities/migrations/0059_vulnerabilityseverity_published_at_and_more.py new file mode 100644 index 000000000..a737b9c9f --- /dev/null +++ b/vulnerabilities/migrations/0059_vulnerabilityseverity_published_at_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 4.1.13 on 2024-08-06 09:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0058_alter_vulnerabilityreference_options_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="vulnerabilityseverity", + name="published_at", + field=models.DateTimeField( + blank=True, + help_text="UTC Date of publication of the vulnerability severity", + null=True, + ), + ), + migrations.AlterField( + model_name="vulnerabilityseverity", + name="scoring_system", + field=models.CharField( + choices=[ + ("cvssv2", "CVSSv2 Base Score"), + ("cvssv3", "CVSSv3 Base Score"), + ("cvssv3.1", "CVSSv3.1 Base Score"), + ("rhbs", "RedHat Bugzilla severity"), + ("rhas", "RedHat Aggregate severity"), + ("archlinux", "Archlinux Vulnerability Group Severity"), + ("cvssv3.1_qr", "CVSSv3.1 Qualitative Severity Rating"), + ("generic_textual", "Generic textual severity rating"), + ("apache_httpd", "Apache Httpd Severity"), + ("apache_tomcat", "Apache Tomcat Severity"), + ("epss", "Exploit Prediction Scoring System"), + ], + help_text="Identifier for the scoring system used. Available choices are: cvssv2: CVSSv2 Base Score,\ncvssv3: CVSSv3 Base Score,\ncvssv3.1: CVSSv3.1 Base Score,\nrhbs: RedHat Bugzilla severity,\nrhas: RedHat Aggregate severity,\narchlinux: Archlinux Vulnerability Group Severity,\ncvssv3.1_qr: CVSSv3.1 Qualitative Severity Rating,\ngeneric_textual: Generic textual severity rating,\napache_httpd: Apache Httpd Severity,\napache_tomcat: Apache Tomcat Severity,\nepss: Exploit Prediction Scoring System ", + max_length=50, + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index f09794565..4e7939c30 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -936,6 +936,10 @@ class VulnerabilitySeverity(models.Model): "For example a CVSS vector string as used to compute a CVSS score.", ) + published_at = models.DateTimeField( + blank=True, null=True, help_text="UTC Date of publication of the vulnerability severity" + ) + class Meta: unique_together = ["reference", "scoring_system", "value"] ordering = ["reference", "scoring_system", "value"] diff --git a/vulnerabilities/severity_systems.py b/vulnerabilities/severity_systems.py index 6260750b2..bc8d6219d 100644 --- a/vulnerabilities/severity_systems.py +++ b/vulnerabilities/severity_systems.py @@ -157,6 +157,19 @@ def get(self, scoring_elements: str) -> dict: "Low", ] + +@dataclasses.dataclass(order=True) +class EPSSScoringSystem(ScoringSystem): + def compute(self, scoring_elements: str): + return NotImplementedError + + +EPSS = EPSSScoringSystem( + identifier="epss", + name="Exploit Prediction Scoring System", + url="https://www.first.org/epss/", +) + SCORING_SYSTEMS = { system.identifier: system for system in ( @@ -170,5 +183,6 @@ def get(self, scoring_elements: str) -> dict: GENERIC, APACHE_HTTPD, APACHE_TOMCAT, + EPSS, ) } diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index f26cb61fc..4ddbad9cd 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -60,13 +60,25 @@ - {% if vulnerability.kev %}
  • + + {% if vulnerability.kev %} +
  • Known Exploited Vulnerabilities -
    • {% endif %} + + {% endif %} + +
  • + + + EPSS + + +
    • +
  • @@ -390,87 +402,141 @@ {% endfor %} {% if vulnerability.kev %} -
    -
    - Known Exploited Vulnerabilities -
    - - - - - - - - {% if vulnerability.kev.description %} - - - - - {% endif %} - {% if vulnerability.kev.required_action %} +
    +
    + Known Exploited Vulnerabilities +
    +
    - - Known Ransomware Campaign Use: - - {{ vulnerability.kev.get_known_ransomware_campaign_use_type }}
    - - Description: - - {{ vulnerability.kev.description }}
    + - + - {% endif %} - - {% if vulnerability.kev.resources_and_notes %} + + {% if vulnerability.kev.description %} + + + + + {% endif %} + {% if vulnerability.kev.required_action %} + + + + + {% endif %} + + {% if vulnerability.kev.resources_and_notes %} + + + + + {% endif %} + + {% if vulnerability.kev.due_date %} + + + + + {% endif %} + {% if vulnerability.kev.date_added %} + + + + + {% endif %} + + +
    - Required Action: + data-tooltip="'Known' if this vulnerability is known to have been leveraged as part of a ransomware campaign; 'Unknown' if CISA lacks confirmation that the vulnerability has been utilized for ransomware"> + Known Ransomware Campaign Use: {{ vulnerability.kev.required_action }}{{ vulnerability.kev.get_known_ransomware_campaign_use_type }}
    + + Description: + + {{ vulnerability.kev.description }}
    + + Required Action: + + {{ vulnerability.kev.required_action }}
    + + Notes: + + {{ vulnerability.kev.resources_and_notes }}
    + + Due Date: + + {{ vulnerability.kev.due_date }}
    + + Date Added: + + {{ vulnerability.kev.date_added }}
    +
    + {% endif %} + + {% for severity in severities %} + {% if severity.scoring_system == 'epss' %} +
    +
    + Exploit Prediction Scoring System +
    + + - + - {% endif %} - - {% if vulnerability.kev.due_date %} + - + - {% endif %} - {% if vulnerability.kev.date_added %} + + {% if severity.published_at %} - + - {% endif %} - - + {% endif %} + +
    - Notes: + data-tooltip="the percentile of the current score, the proportion of all scored vulnerabilities with the same or a lower EPSS score"> + Percentile: {{ vulnerability.kev.resources_and_notes }}{{ severity.scoring_elements }}
    - Due Date: + data-tooltip="the EPSS score representing the probability [0-1] of exploitation in the wild in the next 30 days (following score publication)"> + EPSS score: {{ vulnerability.kev.due_date }}{{ severity.value }}
    - Date Added: + data-tooltip="When was the time we fetched epss"> + Published at: {{ vulnerability.kev.date_added }}{{ severity.published_at }}
    +
    {% endif %} - - + {% empty %} +
    + + + There are no EPSS available. + + +
    + {% endfor %}
    diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index 5194d64db..93401855d 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -29,7 +29,9 @@ from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference from vulnerabilities.models import VulnerabilityRelatedReference +from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness +from vulnerabilities.severity_systems import EPSS BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data") @@ -213,6 +215,23 @@ def setUp(self): PackageRelatedVulnerability.objects.create( package=pkg, vulnerability=self.vulnerability, fix=True ) + + self.reference1 = VulnerabilityReference.objects.create( + reference_id="", + url="https://.com", + ) + + VulnerabilitySeverity.objects.create( + reference=self.reference1, + scoring_system=EPSS.identifier, + scoring_elements=".0016", + value="0.526", + ) + + VulnerabilityRelatedReference.objects.create( + reference=self.reference1, vulnerability=self.vulnerability + ) + self.weaknesses = Weakness.objects.create(cwe_id=119) self.weaknesses.vulnerabilities.add(self.vulnerability) self.invalid_weaknesses = Weakness.objects.create( @@ -256,7 +275,21 @@ def test_api_with_single_vulnerability(self): }, ], "affected_packages": [], - "references": [], + "references": [ + { + "reference_url": "https://.com", + "reference_id": "", + "reference_type": "", + "scores": [ + { + "value": "0.526", + "scoring_system": "epss", + "scoring_elements": ".0016", + } + ], + "url": "https://.com", + } + ], "weaknesses": [ { "cwe_id": 119, @@ -286,7 +319,21 @@ def test_api_with_single_vulnerability_with_filters(self): }, ], "affected_packages": [], - "references": [], + "references": [ + { + "reference_url": "https://.com", + "reference_id": "", + "reference_type": "", + "scores": [ + { + "value": "0.526", + "scoring_system": "epss", + "scoring_elements": ".0016", + } + ], + "url": "https://.com", + } + ], "weaknesses": [ { "cwe_id": 119, diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 391c165e7..68ce09faf 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -28,6 +28,7 @@ from vulnerabilities.forms import PackageSearchForm from vulnerabilities.forms import VulnerabilitySearchForm from vulnerabilities.models import VulnerabilityStatusType +from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import get_severity_range from vulnerablecode.settings import env @@ -137,7 +138,11 @@ def get_context_data(self, **kwargs): status = self.object.get_status_label severity_vectors = [] + severity_values = set() for s in self.object.severities: + if s.scoring_system == EPSS.identifier: + continue + if s.scoring_elements and s.scoring_system in SCORING_SYSTEMS: try: vector_values = SCORING_SYSTEMS[s.scoring_system].get(s.scoring_elements) @@ -145,14 +150,15 @@ def get_context_data(self, **kwargs): except (CVSS2MalformedError, CVSS3MalformedError, NotImplementedError): logging.error(f"CVSSMalformedError for {s.scoring_elements}") + if s.value: + severity_values.add(s.value) + context.update( { "vulnerability": self.object, "vulnerability_search_form": VulnerabilitySearchForm(self.request.GET), "severities": list(self.object.severities), - "severity_score_range": get_severity_range( - {s.value for s in self.object.severities} - ), + "severity_score_range": get_severity_range(severity_values), "severity_vectors": severity_vectors, "references": self.object.references.all(), "aliases": self.object.aliases.all(),