Skip to content

Commit

Permalink
Merge branch 'main' into 1287-resume-combining-affected-fixed-tabs
Browse files Browse the repository at this point in the history
Reference: #1287
- Fixed merge conflicts.
Signed-off-by: John M. Horan <johnmhoran@gmail.com>
  • Loading branch information
johnmhoran committed Aug 7, 2024
2 parents b3e8b40 + 53b84d1 commit 8333952
Show file tree
Hide file tree
Showing 13 changed files with 617 additions and 58 deletions.
9 changes: 8 additions & 1 deletion vulnerabilities/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,14 @@
class VulnerabilitySeveritySerializer(serializers.ModelSerializer):
class Meta:
model = VulnerabilitySeverity
fields = ["value", "scoring_system", "scoring_elements"]
fields = ["value", "scoring_system", "scoring_elements", "published_at"]

def to_representation(self, instance):
data = super().to_representation(instance)
published_at = data.get("published_at", None)
if not published_at:
data.pop("published_at")
return data


class VulnerabilityReferenceSerializer(serializers.ModelSerializer):
Expand Down
1 change: 1 addition & 0 deletions vulnerabilities/import_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver
defaults={
"value": str(severity.value),
"scoring_elements": str(severity.scoring_elements),
"published_at": str(severity.published_at),
},
)
if updated:
Expand Down
6 changes: 6 additions & 0 deletions vulnerabilities/importer.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,17 @@ class VulnerabilitySeverity:
system: ScoringSystem
value: str
scoring_elements: str = ""
published_at: Optional[datetime.datetime] = None

def to_dict(self):
published_at_dict = (
{"published_at": self.published_at.isoformat()} if self.published_at else {}
)
return {
"system": self.system.identifier,
"value": self.value,
"scoring_elements": self.scoring_elements,
**published_at_dict,
}

@classmethod
Expand All @@ -70,6 +75,7 @@ def from_dict(cls, severity: dict):
system=SCORING_SYSTEMS[severity["system"]],
value=severity["value"],
scoring_elements=severity.get("scoring_elements", ""),
published_at=severity.get("published_at"),
)


Expand Down
2 changes: 2 additions & 0 deletions vulnerabilities/importers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from vulnerabilities.importers import debian
from vulnerabilities.importers import debian_oval
from vulnerabilities.importers import elixir_security
from vulnerabilities.importers import epss
from vulnerabilities.importers import fireeye
from vulnerabilities.importers import gentoo
from vulnerabilities.importers import github
Expand Down Expand Up @@ -71,6 +72,7 @@
oss_fuzz.OSSFuzzImporter,
ruby.RubyImporter,
github_osv.GithubOSVImporter,
epss.EPSSImporter,
]

IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}
67 changes: 67 additions & 0 deletions vulnerabilities/importers/epss.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
#
# Copyright (c) nexB Inc. and others. All rights reserved.
# VulnerableCode is a trademark of nexB Inc.
# SPDX-License-Identifier: Apache-2.0
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
# See https://github.com/nexB/vulnerablecode for support or download.
# See https://aboutcode.org for more information about nexB OSS projects.
#
import csv
import gzip
import logging
import urllib.request
from datetime import datetime
from typing import Iterable

from vulnerabilities import severity_systems
from vulnerabilities.importer import AdvisoryData
from vulnerabilities.importer import Importer
from vulnerabilities.importer import Reference
from vulnerabilities.importer import VulnerabilitySeverity

logger = logging.getLogger(__name__)


class EPSSImporter(Importer):
"""Exploit Prediction Scoring System (EPSS) Importer"""

advisory_url = "https://epss.cyentia.com/epss_scores-current.csv.gz"
spdx_license_expression = "unknown"
importer_name = "EPSS Importer"

def advisory_data(self) -> Iterable[AdvisoryData]:
response = urllib.request.urlopen(self.advisory_url)
with gzip.open(response, "rb") as f:
lines = [l.decode("utf-8") for l in f.readlines()]

epss_reader = csv.reader(lines)
model_version, score_date = next(
epss_reader
) # score_date='score_date:2024-05-19T00:00:00+0000'
published_at = datetime.strptime(score_date[11::], "%Y-%m-%dT%H:%M:%S%z")

next(epss_reader) # skip the header row
for epss_row in epss_reader:
cve, score, percentile = epss_row

if not cve or not score or not percentile:
logger.error(f"Invalid epss row: {epss_row}")
continue

severity = VulnerabilitySeverity(
system=severity_systems.EPSS,
value=score,
scoring_elements=percentile,
published_at=published_at,
)

references = Reference(
url=f"https://api.first.org/data/v1/epss?cve={cve}",
severities=[severity],
)

yield AdvisoryData(
aliases=[cve],
references=[references],
url=self.advisory_url,
)
133 changes: 133 additions & 0 deletions vulnerabilities/management/commands/export.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
#
# Copyright (c) nexB Inc. and others. All rights reserved.
# VulnerableCode is a trademark of nexB Inc.
# SPDX-License-Identifier: Apache-2.0
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
# See https://github.com/nexB/vulnerablecode for support or download.
# See https://aboutcode.org for more information about nexB OSS projects.
#
import logging
import os
from hashlib import sha512
from pathlib import Path

import saneyaml
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from packageurl import PackageURL

from vulnerabilities.models import Package

logger = logging.getLogger(__name__)


class Command(BaseCommand):
help = "export vulnerablecode data"

def add_arguments(self, parser):
parser.add_argument("path")

def handle(self, *args, **options):
if options["path"]:
git_path = Path(options["path"])
if not git_path.is_dir():
raise CommandError("Please enter a valid path")

self.export_data(git_path)

self.stdout.write(self.style.SUCCESS("Successfully exported vulnerablecode data"))

def export_data(self, git_path):
"""
export vulnerablecode data
by running `python manage.py export /path/vulnerablecode-data`
"""
self.stdout.write("Exporting vulnerablecode data")

ecosystems = [pkg.type for pkg in Package.objects.distinct("type")]

for ecosystem in ecosystems:
package_files = {} # {"package path": "data" }
vul_files = {} # {"vulnerability path": "data" }

for purl in (
Package.objects.filter(type=ecosystem)
.prefetch_related("vulnerabilities")
.paginated()
):
purl_without_version = PackageURL(
type=purl.type,
namespace=purl.namespace,
name=purl.name,
)

# ./aboutcode-packages-ed5/maven/org.apache.log4j/log4j-core/versions/vulnerabilities.yml
pkg_filepath = (
f"./aboutcode-packages-{get_purl_hash(purl_without_version)}/{purl.type}/{purl.namespace}/{purl.name}"
f"/versions/vulnerabilities.yml"
)

package_data = {
"purl": str(purl),
"affected_by_vulnerabilities": [
vuln.vulnerability_id for vuln in purl.affected_by
],
"fixing_vulnerabilities": [vuln.vulnerability_id for vuln in purl.fixing],
}

if pkg_filepath in package_files:
package_files[pkg_filepath]["versions"].append(package_data)
else:
package_files[pkg_filepath] = {
"package": str(purl_without_version),
"versions": [package_data],
}

for vul in purl.vulnerabilities.all():
vulnerability_id = vul.vulnerability_id
# ./aboutcode-vulnerabilities-12/34/VCID-1223-3434-34343/VCID-1223-3434-34343.yml
vul_filepath = (
f"./aboutcode-vulnerabilities-{vulnerability_id[5:7]}/{vulnerability_id[10:12]}"
f"/{vulnerability_id}/{vulnerability_id}.yml"
)
vul_files[vul_filepath] = {
"vulnerability_id": vul.vulnerability_id,
"aliases": [alias.alias for alias in vul.get_aliases],
"summary": vul.summary,
"severities": [severity for severity in vul.severities.values()],
"references": [ref for ref in vul.references.values()],
"weaknesses": [
"CWE-" + str(weakness["cwe_id"]) for weakness in vul.weaknesses.values()
],
}

for items in [package_files, vul_files]:
for filepath, data in items.items():
create_file(filepath, git_path, data)

self.stdout.write(f"Successfully exported {ecosystem} data")


def create_file(filepath, git_path, data):
"""
Check if the directories exist if it doesn't exist create a new one then Create the file
./aboutcode-vulnerabilities-12/34/VCID-1223-3434-34343/VCID-1223-3434-34343.yml
./aboutcode-packages-ed5/maven/org.apache.log4j/log4j-core/versions/vulnerabilities.yml
./aboutcode-packages-ed5/maven/org.apache.log4j/log4j-core/versions/1.2.3/vulnerabilities.yml
"""
filepath = git_path.joinpath(filepath)
dirname = os.path.dirname(filepath)
os.makedirs(dirname, exist_ok=True)
data = saneyaml.dump(data)
with open(filepath, encoding="utf-8", mode="w") as f:
f.write(data)


def get_purl_hash(purl: PackageURL, length: int = 3) -> str:
"""
Return a short lower cased hash of a purl.
https://github.com/nexB/purldb/pull/235/files#diff-a1fd023bd42d73f56019d540f38be711255403547add15108540d70f9948dd40R154
"""
purl_bytes = str(purl).encode("utf-8")
short_hash = sha512(purl_bytes).hexdigest()[:length]
return short_hash.lower()
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Generated by Django 4.1.13 on 2024-08-06 09:38

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
("vulnerabilities", "0058_alter_vulnerabilityreference_options_and_more"),
]

operations = [
migrations.AddField(
model_name="vulnerabilityseverity",
name="published_at",
field=models.DateTimeField(
blank=True,
help_text="UTC Date of publication of the vulnerability severity",
null=True,
),
),
migrations.AlterField(
model_name="vulnerabilityseverity",
name="scoring_system",
field=models.CharField(
choices=[
("cvssv2", "CVSSv2 Base Score"),
("cvssv3", "CVSSv3 Base Score"),
("cvssv3.1", "CVSSv3.1 Base Score"),
("rhbs", "RedHat Bugzilla severity"),
("rhas", "RedHat Aggregate severity"),
("archlinux", "Archlinux Vulnerability Group Severity"),
("cvssv3.1_qr", "CVSSv3.1 Qualitative Severity Rating"),
("generic_textual", "Generic textual severity rating"),
("apache_httpd", "Apache Httpd Severity"),
("apache_tomcat", "Apache Tomcat Severity"),
("epss", "Exploit Prediction Scoring System"),
],
help_text="Identifier for the scoring system used. Available choices are: cvssv2: CVSSv2 Base Score,\ncvssv3: CVSSv3 Base Score,\ncvssv3.1: CVSSv3.1 Base Score,\nrhbs: RedHat Bugzilla severity,\nrhas: RedHat Aggregate severity,\narchlinux: Archlinux Vulnerability Group Severity,\ncvssv3.1_qr: CVSSv3.1 Qualitative Severity Rating,\ngeneric_textual: Generic textual severity rating,\napache_httpd: Apache Httpd Severity,\napache_tomcat: Apache Tomcat Severity,\nepss: Exploit Prediction Scoring System ",
max_length=50,
),
),
]
4 changes: 4 additions & 0 deletions vulnerabilities/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -936,6 +936,10 @@ class VulnerabilitySeverity(models.Model):
"For example a CVSS vector string as used to compute a CVSS score.",
)

published_at = models.DateTimeField(
blank=True, null=True, help_text="UTC Date of publication of the vulnerability severity"
)

class Meta:
unique_together = ["reference", "scoring_system", "value"]
ordering = ["reference", "scoring_system", "value"]
Expand Down
14 changes: 14 additions & 0 deletions vulnerabilities/severity_systems.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,19 @@ def get(self, scoring_elements: str) -> dict:
"Low",
]


@dataclasses.dataclass(order=True)
class EPSSScoringSystem(ScoringSystem):
def compute(self, scoring_elements: str):
return NotImplementedError


EPSS = EPSSScoringSystem(
identifier="epss",
name="Exploit Prediction Scoring System",
url="https://www.first.org/epss/",
)

SCORING_SYSTEMS = {
system.identifier: system
for system in (
Expand All @@ -170,5 +183,6 @@ def get(self, scoring_elements: str) -> dict:
GENERIC,
APACHE_HTTPD,
APACHE_TOMCAT,
EPSS,
)
}
Loading

0 comments on commit 8333952

Please sign in to comment.