-
Notifications
You must be signed in to change notification settings - Fork 201
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into 1287-resume-combining-affected-fixed-tabs
Reference: #1287 - Fixed merge conflicts. Signed-off-by: John M. Horan <johnmhoran@gmail.com>
- Loading branch information
Showing
13 changed files
with
617 additions
and
58 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,67 @@ | ||
# | ||
# Copyright (c) nexB Inc. and others. All rights reserved. | ||
# VulnerableCode is a trademark of nexB Inc. | ||
# SPDX-License-Identifier: Apache-2.0 | ||
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. | ||
# See https://github.com/nexB/vulnerablecode for support or download. | ||
# See https://aboutcode.org for more information about nexB OSS projects. | ||
# | ||
import csv | ||
import gzip | ||
import logging | ||
import urllib.request | ||
from datetime import datetime | ||
from typing import Iterable | ||
|
||
from vulnerabilities import severity_systems | ||
from vulnerabilities.importer import AdvisoryData | ||
from vulnerabilities.importer import Importer | ||
from vulnerabilities.importer import Reference | ||
from vulnerabilities.importer import VulnerabilitySeverity | ||
|
||
logger = logging.getLogger(__name__) | ||
|
||
|
||
class EPSSImporter(Importer): | ||
"""Exploit Prediction Scoring System (EPSS) Importer""" | ||
|
||
advisory_url = "https://epss.cyentia.com/epss_scores-current.csv.gz" | ||
spdx_license_expression = "unknown" | ||
importer_name = "EPSS Importer" | ||
|
||
def advisory_data(self) -> Iterable[AdvisoryData]: | ||
response = urllib.request.urlopen(self.advisory_url) | ||
with gzip.open(response, "rb") as f: | ||
lines = [l.decode("utf-8") for l in f.readlines()] | ||
|
||
epss_reader = csv.reader(lines) | ||
model_version, score_date = next( | ||
epss_reader | ||
) # score_date='score_date:2024-05-19T00:00:00+0000' | ||
published_at = datetime.strptime(score_date[11::], "%Y-%m-%dT%H:%M:%S%z") | ||
|
||
next(epss_reader) # skip the header row | ||
for epss_row in epss_reader: | ||
cve, score, percentile = epss_row | ||
|
||
if not cve or not score or not percentile: | ||
logger.error(f"Invalid epss row: {epss_row}") | ||
continue | ||
|
||
severity = VulnerabilitySeverity( | ||
system=severity_systems.EPSS, | ||
value=score, | ||
scoring_elements=percentile, | ||
published_at=published_at, | ||
) | ||
|
||
references = Reference( | ||
url=f"https://api.first.org/data/v1/epss?cve={cve}", | ||
severities=[severity], | ||
) | ||
|
||
yield AdvisoryData( | ||
aliases=[cve], | ||
references=[references], | ||
url=self.advisory_url, | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,133 @@ | ||
# | ||
# Copyright (c) nexB Inc. and others. All rights reserved. | ||
# VulnerableCode is a trademark of nexB Inc. | ||
# SPDX-License-Identifier: Apache-2.0 | ||
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. | ||
# See https://github.com/nexB/vulnerablecode for support or download. | ||
# See https://aboutcode.org for more information about nexB OSS projects. | ||
# | ||
import logging | ||
import os | ||
from hashlib import sha512 | ||
from pathlib import Path | ||
|
||
import saneyaml | ||
from django.core.management.base import BaseCommand | ||
from django.core.management.base import CommandError | ||
from packageurl import PackageURL | ||
|
||
from vulnerabilities.models import Package | ||
|
||
logger = logging.getLogger(__name__) | ||
|
||
|
||
class Command(BaseCommand): | ||
help = "export vulnerablecode data" | ||
|
||
def add_arguments(self, parser): | ||
parser.add_argument("path") | ||
|
||
def handle(self, *args, **options): | ||
if options["path"]: | ||
git_path = Path(options["path"]) | ||
if not git_path.is_dir(): | ||
raise CommandError("Please enter a valid path") | ||
|
||
self.export_data(git_path) | ||
|
||
self.stdout.write(self.style.SUCCESS("Successfully exported vulnerablecode data")) | ||
|
||
def export_data(self, git_path): | ||
""" | ||
export vulnerablecode data | ||
by running `python manage.py export /path/vulnerablecode-data` | ||
""" | ||
self.stdout.write("Exporting vulnerablecode data") | ||
|
||
ecosystems = [pkg.type for pkg in Package.objects.distinct("type")] | ||
|
||
for ecosystem in ecosystems: | ||
package_files = {} # {"package path": "data" } | ||
vul_files = {} # {"vulnerability path": "data" } | ||
|
||
for purl in ( | ||
Package.objects.filter(type=ecosystem) | ||
.prefetch_related("vulnerabilities") | ||
.paginated() | ||
): | ||
purl_without_version = PackageURL( | ||
type=purl.type, | ||
namespace=purl.namespace, | ||
name=purl.name, | ||
) | ||
|
||
# ./aboutcode-packages-ed5/maven/org.apache.log4j/log4j-core/versions/vulnerabilities.yml | ||
pkg_filepath = ( | ||
f"./aboutcode-packages-{get_purl_hash(purl_without_version)}/{purl.type}/{purl.namespace}/{purl.name}" | ||
f"/versions/vulnerabilities.yml" | ||
) | ||
|
||
package_data = { | ||
"purl": str(purl), | ||
"affected_by_vulnerabilities": [ | ||
vuln.vulnerability_id for vuln in purl.affected_by | ||
], | ||
"fixing_vulnerabilities": [vuln.vulnerability_id for vuln in purl.fixing], | ||
} | ||
|
||
if pkg_filepath in package_files: | ||
package_files[pkg_filepath]["versions"].append(package_data) | ||
else: | ||
package_files[pkg_filepath] = { | ||
"package": str(purl_without_version), | ||
"versions": [package_data], | ||
} | ||
|
||
for vul in purl.vulnerabilities.all(): | ||
vulnerability_id = vul.vulnerability_id | ||
# ./aboutcode-vulnerabilities-12/34/VCID-1223-3434-34343/VCID-1223-3434-34343.yml | ||
vul_filepath = ( | ||
f"./aboutcode-vulnerabilities-{vulnerability_id[5:7]}/{vulnerability_id[10:12]}" | ||
f"/{vulnerability_id}/{vulnerability_id}.yml" | ||
) | ||
vul_files[vul_filepath] = { | ||
"vulnerability_id": vul.vulnerability_id, | ||
"aliases": [alias.alias for alias in vul.get_aliases], | ||
"summary": vul.summary, | ||
"severities": [severity for severity in vul.severities.values()], | ||
"references": [ref for ref in vul.references.values()], | ||
"weaknesses": [ | ||
"CWE-" + str(weakness["cwe_id"]) for weakness in vul.weaknesses.values() | ||
], | ||
} | ||
|
||
for items in [package_files, vul_files]: | ||
for filepath, data in items.items(): | ||
create_file(filepath, git_path, data) | ||
|
||
self.stdout.write(f"Successfully exported {ecosystem} data") | ||
|
||
|
||
def create_file(filepath, git_path, data): | ||
""" | ||
Check if the directories exist if it doesn't exist create a new one then Create the file | ||
./aboutcode-vulnerabilities-12/34/VCID-1223-3434-34343/VCID-1223-3434-34343.yml | ||
./aboutcode-packages-ed5/maven/org.apache.log4j/log4j-core/versions/vulnerabilities.yml | ||
./aboutcode-packages-ed5/maven/org.apache.log4j/log4j-core/versions/1.2.3/vulnerabilities.yml | ||
""" | ||
filepath = git_path.joinpath(filepath) | ||
dirname = os.path.dirname(filepath) | ||
os.makedirs(dirname, exist_ok=True) | ||
data = saneyaml.dump(data) | ||
with open(filepath, encoding="utf-8", mode="w") as f: | ||
f.write(data) | ||
|
||
|
||
def get_purl_hash(purl: PackageURL, length: int = 3) -> str: | ||
""" | ||
Return a short lower cased hash of a purl. | ||
https://github.com/nexB/purldb/pull/235/files#diff-a1fd023bd42d73f56019d540f38be711255403547add15108540d70f9948dd40R154 | ||
""" | ||
purl_bytes = str(purl).encode("utf-8") | ||
short_hash = sha512(purl_bytes).hexdigest()[:length] | ||
return short_hash.lower() |
43 changes: 43 additions & 0 deletions
43
vulnerabilities/migrations/0059_vulnerabilityseverity_published_at_and_more.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
# Generated by Django 4.1.13 on 2024-08-06 09:38 | ||
|
||
from django.db import migrations, models | ||
|
||
|
||
class Migration(migrations.Migration): | ||
|
||
dependencies = [ | ||
("vulnerabilities", "0058_alter_vulnerabilityreference_options_and_more"), | ||
] | ||
|
||
operations = [ | ||
migrations.AddField( | ||
model_name="vulnerabilityseverity", | ||
name="published_at", | ||
field=models.DateTimeField( | ||
blank=True, | ||
help_text="UTC Date of publication of the vulnerability severity", | ||
null=True, | ||
), | ||
), | ||
migrations.AlterField( | ||
model_name="vulnerabilityseverity", | ||
name="scoring_system", | ||
field=models.CharField( | ||
choices=[ | ||
("cvssv2", "CVSSv2 Base Score"), | ||
("cvssv3", "CVSSv3 Base Score"), | ||
("cvssv3.1", "CVSSv3.1 Base Score"), | ||
("rhbs", "RedHat Bugzilla severity"), | ||
("rhas", "RedHat Aggregate severity"), | ||
("archlinux", "Archlinux Vulnerability Group Severity"), | ||
("cvssv3.1_qr", "CVSSv3.1 Qualitative Severity Rating"), | ||
("generic_textual", "Generic textual severity rating"), | ||
("apache_httpd", "Apache Httpd Severity"), | ||
("apache_tomcat", "Apache Tomcat Severity"), | ||
("epss", "Exploit Prediction Scoring System"), | ||
], | ||
help_text="Identifier for the scoring system used. Available choices are: cvssv2: CVSSv2 Base Score,\ncvssv3: CVSSv3 Base Score,\ncvssv3.1: CVSSv3.1 Base Score,\nrhbs: RedHat Bugzilla severity,\nrhas: RedHat Aggregate severity,\narchlinux: Archlinux Vulnerability Group Severity,\ncvssv3.1_qr: CVSSv3.1 Qualitative Severity Rating,\ngeneric_textual: Generic textual severity rating,\napache_httpd: Apache Httpd Severity,\napache_tomcat: Apache Tomcat Severity,\nepss: Exploit Prediction Scoring System ", | ||
max_length=50, | ||
), | ||
), | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.