diff --git a/vulnerabilities/admin.py b/vulnerabilities/admin.py index 3936d2896..01cf0b8d1 100644 --- a/vulnerabilities/admin.py +++ b/vulnerabilities/admin.py @@ -37,7 +37,7 @@ class VulnerabilityAdmin(admin.ModelAdmin): @admin.register(VulnerabilityReference) class VulnerabilityReferenceAdmin(admin.ModelAdmin): - search_fields = ["vulnerability__vulnerability_id", "reference_id", "url"] + search_fields = ["vulnerabilityrelatedreference__vulnerability__id", "reference_id", "url"] @admin.register(Package) diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index a9c6e31df..1b339070e 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -30,6 +30,7 @@ from rest_framework.decorators import action from rest_framework.response import Response +from vulnerabilities.models import Alias from vulnerabilities.models import Package from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference @@ -44,10 +45,11 @@ class Meta: class VulnerabilityReferenceSerializer(serializers.ModelSerializer): scores = VulnerabilitySeveritySerializer(many=True, source="vulnerabilityseverity_set") + reference_url = serializers.CharField(source="url") class Meta: model = VulnerabilityReference - fields = ["reference_id", "url", "scores"] + fields = ["reference_url", "reference_id", "scores"] class MinimalPackageSerializer(serializers.HyperlinkedModelSerializer): @@ -71,36 +73,64 @@ class MinimalVulnerabilitySerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Vulnerability - fields = ["url", "vulnerability_id", "references", "summary"] + fields = ["url", "vulnerability_id", "summary", "references"] + + +class AliasSerializer(serializers.HyperlinkedModelSerializer): + """ + Used for nesting inside package focused APIs. + """ + + class Meta: + model = Alias + fields = ["alias"] class VulnerabilitySerializer(serializers.HyperlinkedModelSerializer): - resolved_packages = MinimalPackageSerializer(many=True, source="resolved_to", read_only=True) - unresolved_packages = MinimalPackageSerializer( - many=True, source="vulnerable_to", read_only=True - ) + fixed_packages = MinimalPackageSerializer(many=True, source="resolved_to", read_only=True) + affected_packages = MinimalPackageSerializer(many=True, source="vulnerable_to", read_only=True) references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") + aliases = AliasSerializer(many=True, source="alias") class Meta: model = Vulnerability - fields = "__all__" + fields = [ + "url", + "vulnerability_id", + "summary", + "aliases", + "fixed_packages", + "affected_packages", + "references", + ] class PackageSerializer(serializers.HyperlinkedModelSerializer): - unresolved_vulnerabilities = MinimalVulnerabilitySerializer( + purl = serializers.CharField(source="package_url") + affected_by_vulnerabilities = MinimalVulnerabilitySerializer( many=True, source="vulnerable_to", read_only=True ) - resolved_vulnerabilities = MinimalVulnerabilitySerializer( + fixing_vulnerabilities = MinimalVulnerabilitySerializer( many=True, source="resolved_to", read_only=True ) - purl = serializers.CharField(source="package_url") class Meta: model = Package - exclude = ["vulnerabilities"] + fields = [ + "url", + "purl", + "type", + "namespace", + "name", + "version", + "qualifiers", + "subpath", + "affected_by_vulnerabilities", + "fixing_vulnerabilities", + ] class PackageFilterSet(filters.FilterSet): diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py index 069392d1a..66d87599d 100644 --- a/vulnerabilities/importers/alpine_linux.py +++ b/vulnerabilities/importers/alpine_linux.py @@ -54,7 +54,6 @@ class AlpineImporter(Importer): license_url = "https://secdb.alpinelinux.org/license.txt" def advisory_data(self) -> Iterable[AdvisoryData]: - advisories = [] page_response_content = fetch_response(BASE_URL).content advisory_directory_links = fetch_advisory_directory_links(page_response_content) advisory_links = [] @@ -68,8 +67,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]: if not record["packages"]: LOGGER.error(f'"packages" not found in {link!r}') continue - advisories.extend(process_record(record)) - return advisories + yield from process_record(record) def fetch_response(url): @@ -127,7 +125,7 @@ def check_for_attributes(record) -> bool: return True -def process_record(record: dict) -> List[AdvisoryData]: +def process_record(record: dict) -> Iterable[AdvisoryData]: """ Return a list of AdvisoryData objects by processing data present in that `record` @@ -136,22 +134,18 @@ def process_record(record: dict) -> List[AdvisoryData]: LOGGER.error(f'"packages" not found in this record {record!r}') return [] - advisories: List[AdvisoryData] = [] - for package in record["packages"]: if not package["pkg"]: LOGGER.error(f'"pkg" not found in this package {package!r}') continue if not check_for_attributes(record): continue - loaded_advisories = load_advisories( + yield from load_advisories( package["pkg"], record["distroversion"], record["reponame"], record["archs"], ) - advisories.extend(loaded_advisories) - return advisories def load_advisories( diff --git a/vulnerabilities/migrations/0011_vulnerability_packages_alter_package_vulnerabilities_and_more.py b/vulnerabilities/migrations/0011_vulnerability_packages_alter_package_vulnerabilities_and_more.py new file mode 100644 index 000000000..a33e5b599 --- /dev/null +++ b/vulnerabilities/migrations/0011_vulnerability_packages_alter_package_vulnerabilities_and_more.py @@ -0,0 +1,29 @@ +# Generated by Django 4.0.3 on 2022-04-26 08:42 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('vulnerabilities', '0010_vulnerabilityrelatedreference_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='vulnerability', + name='packages', + field=models.ManyToManyField(through='vulnerabilities.PackageRelatedVulnerability', to='vulnerabilities.package'), + ), + migrations.AlterField( + model_name='package', + name='vulnerabilities', + field=models.ManyToManyField(through='vulnerabilities.PackageRelatedVulnerability', to='vulnerabilities.vulnerability'), + ), + migrations.AlterField( + model_name='packagerelatedvulnerability', + name='package', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.package'), + ), + ] diff --git a/vulnerabilities/migrations/0012_alter_vulnerability_vulnerability_id.py b/vulnerabilities/migrations/0012_alter_vulnerability_vulnerability_id.py new file mode 100644 index 000000000..244fc6ac8 --- /dev/null +++ b/vulnerabilities/migrations/0012_alter_vulnerability_vulnerability_id.py @@ -0,0 +1,18 @@ +# Generated by Django 4.0.4 on 2022-05-03 09:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('vulnerabilities', '0011_vulnerability_packages_alter_package_vulnerabilities_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='vulnerability', + name='vulnerability_id', + field=models.CharField(blank=True, help_text='Unique identifier for a vulnerability in the external representation. It is prefixed with VULCOID-', max_length=20, unique=True), + ), + ] diff --git a/vulnerabilities/migrations/0013_auto_20220503_0941.py b/vulnerabilities/migrations/0013_auto_20220503_0941.py new file mode 100644 index 000000000..72f780676 --- /dev/null +++ b/vulnerabilities/migrations/0013_auto_20220503_0941.py @@ -0,0 +1,22 @@ +# Generated by Django 4.0.4 on 2022-05-03 09:41 + +from django.db import migrations + +from django.utils.http import int_to_base36 + +class Migration(migrations.Migration): + + dependencies = [ + ('vulnerabilities', '0012_alter_vulnerability_vulnerability_id'), + ] + + def save_vulnerability_id(apps, schema_editor): + Vulnerabilities = apps.get_model("vulnerabilities", "Vulnerability") + for vulnerability in Vulnerabilities.objects.all(): + if not vulnerability.vulnerability_id: + vulnerability.vulnerability_id = f"VULCOID-{int_to_base36(vulnerability.id).upper()}" + vulnerability.save() + + operations = [ + migrations.RunPython(save_vulnerbaility_id) + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 170bb7edc..6d8982763 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -29,6 +29,7 @@ from django.core.validators import MaxValueValidator from django.core.validators import MinValueValidator from django.db import models +from django.utils.http import int_to_base36 from packageurl import PackageURL from packageurl.contrib.django.models import PackageURLMixin @@ -47,12 +48,12 @@ class Vulnerability(models.Model): stored as ``Alias``. """ - vulnerability_id = models.UUIDField( - default=uuid.uuid4, - editable=False, + vulnerability_id = models.CharField( unique=True, - help_text="Unique identifier for a vulnerability in this database, assigned automatically. " - "In the external representation it is prefixed with VULCOID-", + blank=True, + max_length=20, + help_text="Unique identifier for a vulnerability in the external representation. " + "It is prefixed with VULCOID-", ) summary = models.TextField( @@ -63,17 +64,23 @@ class Vulnerability(models.Model): references = models.ManyToManyField( to="VulnerabilityReference", through="VulnerabilityRelatedReference" ) + packages = models.ManyToManyField( + to="Package", + through="PackageRelatedVulnerability", + ) - @property - def vulcoid(self): - return f"VULCOID-{self.vulnerability_id}" + def save(self, *args, **kwargs): + super().save(*args, **kwargs) + if not self.vulnerability_id: + self.vulnerability_id = f"VULCOID-{int_to_base36(self.id).upper()}" + super().save(update_fields=["vulnerability_id"]) @property def vulnerable_to(self): """ Return packages that are vulnerable to this vulnerability. """ - return self.packages.filter(vulnerabilities__packagerelatedvulnerability__fix=False) + return self.packages.filter(packagerelatedvulnerability__fix=False) @property def resolved_to(self): @@ -81,10 +88,18 @@ def resolved_to(self): Returns packages that first received patch against this vulnerability in their particular version history. """ - return self.packages.filter(vulnerabilities__packagerelatedvulnerability__fix=True) + return self.packages.filter(packagerelatedvulnerability__fix=True) + + @property + def alias(self): + """ + Returns packages that first received patch against this vulnerability + in their particular version history. + """ + return self.aliases.all() def __str__(self): - return self.vulcoid + return self.vulnerability_id class Meta: verbose_name_plural = "Vulnerabilities" @@ -150,10 +165,7 @@ class Package(PackageURLMixin): """ vulnerabilities = models.ManyToManyField( - to="Vulnerability", - through="PackageRelatedVulnerability", - through_fields=("package", "vulnerability"), - related_name="packages", + to="Vulnerability", through="PackageRelatedVulnerability" ) # Remove the `qualifers` and `set_package_url` overrides after @@ -218,8 +230,14 @@ def __str__(self): class PackageRelatedVulnerability(models.Model): # TODO: Fix related_name - package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name="package") - vulnerability = models.ForeignKey(Vulnerability, on_delete=models.CASCADE) + package = models.ForeignKey( + Package, + on_delete=models.CASCADE, + ) + vulnerability = models.ForeignKey( + Vulnerability, + on_delete=models.CASCADE, + ) created_by = models.CharField( max_length=100, blank=True, diff --git a/vulnerabilities/templates/package_update.html b/vulnerabilities/templates/package_update.html index 881b3ef84..b45afec74 100644 --- a/vulnerabilities/templates/package_update.html +++ b/vulnerabilities/templates/package_update.html @@ -30,7 +30,7 @@

-

Vulnerable To

+

Affected By

{% for vulnerability in impacted_vuln %} @@ -49,7 +49,7 @@

-

Safe To

+

Fixing

{% for vulnerability in resolved_vuln %} diff --git a/vulnerabilities/templates/packages.html b/vulnerabilities/templates/packages.html index 9cebe4264..f9632bbe5 100644 --- a/vulnerabilities/templates/packages.html +++ b/vulnerabilities/templates/packages.html @@ -36,8 +36,8 @@

- - + + {% for package in packages %} diff --git a/vulnerabilities/templates/vulnerabilities.html b/vulnerabilities/templates/vulnerabilities.html index 06d885cff..91e529215 100644 --- a/vulnerabilities/templates/vulnerabilities.html +++ b/vulnerabilities/templates/vulnerabilities.html @@ -31,12 +31,12 @@

Package URLVulnerabilitiesPatched VulnerabilitiesAffected By VulnerabilitiesFixing Vulnerabilities
- - + + {% for vulnerability in vulnerabilities %} - + diff --git a/vulnerabilities/templates/vulnerability.html b/vulnerabilities/templates/vulnerability.html index 28563016d..2b197278d 100644 --- a/vulnerabilities/templates/vulnerability.html +++ b/vulnerabilities/templates/vulnerability.html @@ -2,7 +2,7 @@ {% block title %}

-{{vulnerability.vulnerability_id}} +Vulnerability ID: {{vulnerability.vulnerability_id}}

{% endblock %} @@ -16,6 +16,16 @@

Summary:

{% endif %} +
+ {% if aliases %} +

Aliases:

+ {% for alias in aliases %} +

+ {{alias}} +

+ {% endfor %} + {% endif %} +
{% if object_list %} @@ -77,29 +87,53 @@

Severity

{% endif %} {% if vulnerability.vulnerable_to.all %} -

Vulnerable Packages

-
- {% for package in vulnerability.vulnerable_to.all %} - - {{package.package_url}} - - {% endfor %} -
+

Affected Packages

+
Vulnerability IDVulnerable packagesPatched packagesAffected packagesFixed packages
{{vulnerability.vulnerability_id}}{{vulnerability.vulcoid}} {{vulnerability.vulnerable_package_count}} {{vulnerability.patched_package_count}}
+ + + + {% for package in vulnerability.vulnerable_to.all %} + + {% if package.package_url %} + + + {% else %} + + + {% endif %} + + + {% endfor %} +
Packages
+ {{package.package_url}} + -
{% else %} -

No available vulnerable packages

+

No available Affected packages

{% endif %} {% if vulnerability.resolved_to.all %} -

Patched Packages

-
+

Fixed Packages

+ + + + {% for package in vulnerability.resolved_to.all %} - - {{package.package_url}} - + + {% if package.package_url %} + + + {% else %} + + + {% endif %} + + {% endfor %} - +
Packages
+ {{package.package_url}} + -
{% else %} -

No available patched packages

+

No available fixed packages

{% endif %}
diff --git a/vulnerabilities/tests/test_alpine.py b/vulnerabilities/tests/test_alpine.py index 26538f4cf..595835405 100644 --- a/vulnerabilities/tests/test_alpine.py +++ b/vulnerabilities/tests/test_alpine.py @@ -455,7 +455,7 @@ def test_process_record(caplog): ), ] with open(os.path.join(TEST_DATA, os.path.join(TEST_DATA, "v3.11", "main.json"))) as f: - found_advisories = process_record(json.loads(f.read())) + found_advisories = list(process_record(json.loads(f.read()))) assert found_advisories == expected_advisories assert ( "'4.10-1-r1' is not a valid AlpineVersion InvalidVersion(\"'4.10-1-r1' is not a valid \")" @@ -510,7 +510,7 @@ def test_fetch_advisory_links_failure(caplog): def test_process_record_without_packages(caplog): with open(os.path.join(TEST_DATA, os.path.join(TEST_DATA, "v3.3", "community.json"))) as f: - assert process_record(json.loads(f.read())) == [] + assert list(process_record(json.loads(f.read()))) == [] assert ( "\"packages\" not found in this record {'apkurl': '{{urlprefix}}/{{distroversion}}/{{reponame}}/{{arch}}/{{pkg.name}}-{{pkg.ver}}.apk', 'archs': ['armhf', 'x86', 'x86_64'], 'reponame': 'community', 'urlprefix': 'https://dl-cdn.alpinelinux.org/alpine', 'distroversion': 'v3.3', 'packages': []}" in caplog.text diff --git a/vulnerabilities/tests/test_fix_api.py b/vulnerabilities/tests/test_fix_api.py index 72d270e6a..a00e1a9e9 100644 --- a/vulnerabilities/tests/test_fix_api.py +++ b/vulnerabilities/tests/test_fix_api.py @@ -49,7 +49,15 @@ def test_api_with_single_vulnerability(self): response = self.client.get( f"/api/vulnerabilities/{self.vulnerability.id}", format="json" ).data - assert response["summary"] == "test" + assert response == { + "url": f"http://testserver/api/vulnerabilities/{self.vulnerability.id}", + "vulnerability_id": "VULCOID-Y", + "summary": "test", + "aliases": [], + "fixed_packages": [], + "affected_packages": [], + "references": [], + } class APITestCasePackage(TestCase): @@ -84,9 +92,18 @@ def test_api_response(self): def test_api_with_single_vulnerability(self): response = self.client.get(f"/api/packages/{self.package.id}", format="json").data - assert response["name"] == "test-vulnDB" - assert response["version"] == "1.0" - assert response["type"] == "generic" + assert response == { + "url": f"http://testserver/api/packages/{self.package.id}", + "purl": "pkg:generic/nginx/test-vulnDB@1.0", + "type": "generic", + "namespace": "nginx", + "name": "test-vulnDB", + "version": "1.0", + "qualifiers": {}, + "subpath": "", + "affected_by_vulnerabilities": [], + "fixing_vulnerabilities": [], + } class CPEApi(TestCase): diff --git a/vulnerabilities/tests/test_view.py b/vulnerabilities/tests/test_view.py index fc044ffd2..650f6ffe5 100644 --- a/vulnerabilities/tests/test_view.py +++ b/vulnerabilities/tests/test_view.py @@ -1,6 +1,8 @@ from django.test import Client from django.test import TestCase +from vulnerabilities.models import Vulnerability + class PackageSearchTestCase(TestCase): def setUp(self): @@ -18,3 +20,25 @@ def test_paginator(self): self.assertEqual(response.status_code, 200) response = self.client.get("/packages/search?type=&name=&page=") self.assertEqual(response.status_code, 200) + + +class VulnerabilitySearchTestCase(TestCase): + def setUp(self): + vulnerability = Vulnerability(summary="test") + vulnerability.save() + self.id = vulnerability.id + self.client = Client() + + def test_vulnerabilties(self): + """ + Test Vulnerability View + """ + response = self.client.get(f"/vulnerabilities/{self.id}") + self.assertEqual(response.status_code, 200) + + def test_vulnerabilties(self): + """ + Test Vulnerability Search View + """ + response = self.client.get(f"/vulnerabilities/search") + self.assertEqual(response.status_code, 200) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index b9329978e..54325074a 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -78,12 +78,12 @@ def request_to_queryset(request): .annotate( vulnerability_count=Count( "vulnerabilities", - filter=Q(vulnerabilities__packagerelatedvulnerability__fix=False), + filter=Q(packagerelatedvulnerability__fix=False), ), # TODO: consider renaming to fixed in the future patched_vulnerability_count=Count( "vulnerabilities", - filter=Q(vulnerabilities__packagerelatedvulnerability__fix=True), + filter=Q(packagerelatedvulnerability__fix=True), ), ) .prefetch_related() @@ -110,7 +110,7 @@ def get(self, request): def request_to_vulnerabilities(request): vuln_id = request.GET["vuln_id"] return list( - models.Vulnerability.objects.filter(vulnerability_id__icontains=vuln_id).annotate( + models.Vulnerability.objects.filter(vulnerability_id=vuln_id).annotate( vulnerable_package_count=Count( "packages", filter=Q(packagerelatedvulnerability__fix=False) ), @@ -129,18 +129,18 @@ class PackageUpdate(UpdateView): def get_context_data(self, **kwargs): context = super(PackageUpdate, self).get_context_data(**kwargs) - resolved_vuln, unresolved_vuln = self._package_vulnerabilities(self.kwargs["pk"]) + resolved_vuln, unresolved_vuln = self._package_vulnerabilities() context["resolved_vuln"] = resolved_vuln context["impacted_vuln"] = unresolved_vuln return context - def _package_vulnerabilities(self, package_pk): + def _package_vulnerabilities(self): # This can be further optimised by caching get_object result first time it # is called package = self.get_object() - resolved_vuln = [i for i in package.resolved_to.values("vulnerability_id", "pk")] - unresolved_vuln = [i for i in package.vulnerable_to.values("vulnerability_id", "pk")] + resolved_vuln = [i for i in package.resolved_to] + unresolved_vuln = [i for i in package.vulnerable_to] return resolved_vuln, unresolved_vuln @@ -154,11 +154,15 @@ class VulnerabilityDetails(ListView): def get_context_data(self, **kwargs): context = super(VulnerabilityDetails, self).get_context_data(**kwargs) - context["vulnerability"] = models.Vulnerability.objects.get(id=self.kwargs["pk"]) + vulnerability = models.Vulnerability.objects.get(id=self.kwargs["pk"]) + context["vulnerability"] = vulnerability + context["aliases"] = vulnerability.aliases.alias() return context def get_queryset(self): - return models.VulnerabilityReference.objects.filter(vulnerability_id=self.kwargs["pk"]) + return models.VulnerabilityReference.objects.filter( + vulnerabilityrelatedreference__vulnerability__id=self.kwargs["pk"] + ) class HomePage(View):